#include "Python.h"
#include "pycore_bitutils.h" // _Py_popcount32()
#include "pycore_call.h" // _PyObject_VectorcallTstate()
#include "pycore_ceval.h" // _PY_EVAL_EVENTS_BITS
#include "pycore_code.h" // _PyCode_Clear_Executors()
#include "pycore_critical_section.h" // _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED()
#include "pycore_frame.h" // PyFrameObject
#include "pycore_interpframe.h" // _PyFrame_GetBytecode()
#include "pycore_long.h" // _PyLong_GetZero()
#include "pycore_modsupport.h" // _PyModule_CreateInitialized()
#include "pycore_namespace.h" // _PyNamespace_New()
#include "pycore_opcode_metadata.h" // IS_VALID_OPCODE()
#include "pycore_opcode_utils.h" // IS_CONDITIONAL_JUMP_OPCODE()
#include "pycore_optimizer.h" // _PyExecutorObject
#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_UINTPTR_RELEASE()
#include "pycore_pystate.h" // _PyInterpreterState_GET()
#include "pycore_runtime_structs.h" // _PyCoMonitoringData
#include "pycore_tuple.h" // _PyTuple_FromArraySteal()
#include "opcode_ids.h"
/* Uncomment this to dump debugging output when assertions fail */
// #define INSTRUMENT_DEBUG 1
#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED)
#define ASSERT_WORLD_STOPPED_OR_LOCKED(obj) \
if (!_PyInterpreterState_GET()->stoptheworld.world_stopped) { \
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj); \
}
#define ASSERT_WORLD_STOPPED() assert(_PyInterpreterState_GET()->stoptheworld.world_stopped);
#else
#define ASSERT_WORLD_STOPPED_OR_LOCKED(obj)
#define ASSERT_WORLD_STOPPED()
#endif
#ifdef Py_GIL_DISABLED
#define LOCK_CODE(code) \
assert(!_PyInterpreterState_GET()->stoptheworld.world_stopped); \
Py_BEGIN_CRITICAL_SECTION(code)
#define UNLOCK_CODE() Py_END_CRITICAL_SECTION()
#define MODIFY_BYTECODE(code, func, ...) \
do { \
PyCodeObject *co = (code); \
for (Py_ssize_t i = 0; i < code->co_tlbc->size; i++) { \
char *bc = co->co_tlbc->entries[i]; \
if (bc == NULL) { \
continue; \
} \
(func)(code, (_Py_CODEUNIT *)bc, __VA_ARGS__); \
} \
} while (0)
#else
#define LOCK_CODE(code)
#define UNLOCK_CODE()
#define MODIFY_BYTECODE(code, func, ...) \
(func)(code, _PyCode_CODE(code), __VA_ARGS__)
#endif
PyObject _PyInstrumentation_DISABLE = _PyObject_HEAD_INIT(&PyBaseObject_Type);
PyObject _PyInstrumentation_MISSING = _PyObject_HEAD_INIT(&PyBaseObject_Type);
static const int8_t EVENT_FOR_OPCODE[256] = {
[RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN,
[INSTRUMENTED_RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN,
[CALL] = PY_MONITORING_EVENT_CALL,
[INSTRUMENTED_CALL] = PY_MONITORING_EVENT_CALL,
[CALL_KW] = PY_MONITORING_EVENT_CALL,
[INSTRUMENTED_CALL_KW] = PY_MONITORING_EVENT_CALL,
[CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL,
[INSTRUMENTED_CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL,
[LOAD_SUPER_ATTR] = PY_MONITORING_EVENT_CALL,
[INSTRUMENTED_LOAD_SUPER_ATTR] = PY_MONITORING_EVENT_CALL,
[RESUME] = -1,
[YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD,
[INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD,
[JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP,
[JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP,
[POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH_RIGHT,
[POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH_RIGHT,
[POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT,
[POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT,
[INSTRUMENTED_JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP,
[INSTRUMENTED_JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP,
[INSTRUMENTED_POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH_RIGHT,
[INSTRUMENTED_POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH_RIGHT,
[INSTRUMENTED_POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT,
[INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT,
[FOR_ITER] = PY_MONITORING_EVENT_BRANCH_LEFT,
[INSTRUMENTED_FOR_ITER] = PY_MONITORING_EVENT_BRANCH_LEFT,
[POP_ITER] = PY_MONITORING_EVENT_BRANCH_RIGHT,
[INSTRUMENTED_POP_ITER] = PY_MONITORING_EVENT_BRANCH_RIGHT,
[END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION,
[INSTRUMENTED_END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION,
[END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION,
[INSTRUMENTED_END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION,
[NOT_TAKEN] = PY_MONITORING_EVENT_BRANCH_LEFT,
[INSTRUMENTED_NOT_TAKEN] = PY_MONITORING_EVENT_BRANCH_LEFT,
[END_ASYNC_FOR] = PY_MONITORING_EVENT_BRANCH_RIGHT,
};
static const uint8_t DE_INSTRUMENT[256] = {
[INSTRUMENTED_RESUME] = RESUME,
[INSTRUMENTED_RETURN_VALUE] = RETURN_VALUE,
[INSTRUMENTED_CALL] = CALL,
[INSTRUMENTED_CALL_KW] = CALL_KW,
[INSTRUMENTED_CALL_FUNCTION_EX] = CALL_FUNCTION_EX,
[INSTRUMENTED_YIELD_VALUE] = YIELD_VALUE,
[INSTRUMENTED_JUMP_FORWARD] = JUMP_FORWARD,
[INSTRUMENTED_JUMP_BACKWARD] = JUMP_BACKWARD,
[INSTRUMENTED_POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE,
[INSTRUMENTED_POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE,
[INSTRUMENTED_POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE,
[INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE,
[INSTRUMENTED_FOR_ITER] = FOR_ITER,
[INSTRUMENTED_POP_ITER] = POP_ITER,
[INSTRUMENTED_END_FOR] = END_FOR,
[INSTRUMENTED_END_SEND] = END_SEND,
[INSTRUMENTED_LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR,
[INSTRUMENTED_NOT_TAKEN] = NOT_TAKEN,
[INSTRUMENTED_END_ASYNC_FOR] = END_ASYNC_FOR,
};
static const uint8_t INSTRUMENTED_OPCODES[256] = {
[RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE,
[INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE,
[CALL] = INSTRUMENTED_CALL,
[INSTRUMENTED_CALL] = INSTRUMENTED_CALL,
[CALL_KW] = INSTRUMENTED_CALL_KW,
[INSTRUMENTED_CALL_KW] = INSTRUMENTED_CALL_KW,
[CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX,
[INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX,
[YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE,
[INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE,
[RESUME] = INSTRUMENTED_RESUME,
[INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME,
[JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD,
[INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD,
[JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD,
[INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD,
[POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE,
[INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE,
[POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE,
[INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE,
[POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE,
[INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE,
[POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE,
[INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE,
[END_FOR] = INSTRUMENTED_END_FOR,
[INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR,
[END_SEND] = INSTRUMENTED_END_SEND,
[INSTRUMENTED_END_SEND] = INSTRUMENTED_END_SEND,
[FOR_ITER] = INSTRUMENTED_FOR_ITER,
[INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER,
[POP_ITER] = INSTRUMENTED_POP_ITER,
[INSTRUMENTED_POP_ITER] = INSTRUMENTED_POP_ITER,
[LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR,
[INSTRUMENTED_LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR,
[NOT_TAKEN] = INSTRUMENTED_NOT_TAKEN,
[INSTRUMENTED_NOT_TAKEN] = INSTRUMENTED_NOT_TAKEN,
[END_ASYNC_FOR] = INSTRUMENTED_END_ASYNC_FOR,
[INSTRUMENTED_END_ASYNC_FOR] = INSTRUMENTED_END_ASYNC_FOR,
[INSTRUMENTED_LINE] = INSTRUMENTED_LINE,
[INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION,
};
static inline bool
opcode_has_event(int opcode)
{
return (
opcode != INSTRUMENTED_LINE &&
INSTRUMENTED_OPCODES[opcode] > 0
);
}
uint8_t
_PyCode_Deinstrument(uint8_t opcode)
{
return DE_INSTRUMENT[opcode];
}
static inline bool
is_instrumented(int opcode)
{
assert(opcode != 0);
assert(opcode != RESERVED);
return opcode < ENTER_EXECUTOR && opcode >= MIN_INSTRUMENTED_OPCODE;
}
#ifndef NDEBUG
static inline bool
monitors_equals(_Py_LocalMonitors a, _Py_LocalMonitors b)
{
for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) {
if (a.tools[i] != b.tools[i]) {
return false;
}
}
return true;
}
#endif
static inline _Py_LocalMonitors
monitors_sub(_Py_LocalMonitors a, _Py_LocalMonitors b)
{
_Py_LocalMonitors res;
for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) {
res.tools[i] = a.tools[i] & ~b.tools[i];
}
return res;
}
#ifndef NDEBUG
static inline _Py_LocalMonitors
monitors_and(_Py_LocalMonitors a, _Py_LocalMonitors b)
{
_Py_LocalMonitors res;
for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) {
res.tools[i] = a.tools[i] & b.tools[i];
}
return res;
}
#endif
/* The union of the *local* events in a and b.
* Global events like RAISE are ignored.
* Used for instrumentation, as only local
* events get instrumented.
*/
static inline _Py_LocalMonitors
local_union(_Py_GlobalMonitors a, _Py_LocalMonitors b)
{
_Py_LocalMonitors res;
for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) {
res.tools[i] = a.tools[i] | b.tools[i];
}
return res;
}
static inline bool
monitors_are_empty(_Py_LocalMonitors m)
{
for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) {
if (m.tools[i]) {
return false;
}
}
return true;
}
static inline bool
multiple_tools(_Py_LocalMonitors *m)
{
for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) {
if (_Py_popcount32(m->tools[i]) > 1) {
return true;
}
}
return false;
}
static inline _PyMonitoringEventSet
get_local_events(_Py_LocalMonitors *m, int tool_id)
{
_PyMonitoringEventSet result = 0;
for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) {
if ((m->tools[e] >> tool_id) & 1) {
result |= (1 tools[e] >> tool_id) & 1) {
result |= (1 co_firstlineno;
assert(delta > NO_LINE);
return delta;
}
static int
compute_line(PyCodeObject *code, int line_delta)
{
if (line_delta == NO_LINE) {
return -1;
}
assert(line_delta > NO_LINE);
return code->co_firstlineno + line_delta;
}
int
_PyInstruction_GetLength(PyCodeObject *code, int offset)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
_Py_CODEUNIT inst = _Py_GetBaseCodeUnit(code, offset);
return 1 + _PyOpcode_Caches[inst.op.code];
}
static inline uint8_t *
get_original_opcode_ptr(_PyCoLineInstrumentationData *line_data, int index)
{
return &line_data->data[index*line_data->bytes_per_entry];
}
static inline void
set_original_opcode(_PyCoLineInstrumentationData *line_data, int index, uint8_t opcode)
{
line_data->data[index*line_data->bytes_per_entry] = opcode;
}
static inline int
get_line_delta(_PyCoLineInstrumentationData *line_data, int index)
{
uint8_t *ptr = &line_data->data[index*line_data->bytes_per_entry+1];
assert(line_data->bytes_per_entry >= 2);
uint32_t value = *ptr;
for (int idx = 2; idx < line_data->bytes_per_entry; idx++) {
ptr++;
int shift = (idx-1)*8;
value |= ((uint32_t)(*ptr)) = NO_LINE);
uint32_t adjusted = line_delta - NO_LINE;
uint8_t *ptr = &line_data->data[index*line_data->bytes_per_entry+1];
assert(adjusted < (1ULL bytes_per_entry-1)*8)));
assert(line_data->bytes_per_entry >= 2);
*ptr = adjusted & 0xff;
for (int idx = 2; idx < line_data->bytes_per_entry; idx++) {
ptr++;
adjusted >>= 8;
*ptr = adjusted & 0xff;
}
}
#ifdef INSTRUMENT_DEBUG
static void
dump_instrumentation_data_tools(PyCodeObject *code, uint8_t *tools, int i, FILE*out)
{
if (tools == NULL) {
fprintf(out, "tools = NULL");
}
else {
fprintf(out, "tools = %d", tools[i]);
}
}
static void
dump_instrumentation_data_lines(PyCodeObject *code, _PyCoLineInstrumentationData *lines, int i, FILE*out)
{
if (lines == NULL) {
fprintf(out, ", lines = NULL");
}
else {
int opcode = _PyCode_GetOriginalOpcode(lines, i);
int line_delta = get_line_delta(lines, i);
if (opcode == 0) {
fprintf(out, ", lines = {original_opcode = No LINE (0), line_delta = %d)", line_delta);
}
else {
fprintf(out, ", lines = {original_opcode = %s, line_delta = %d)", _PyOpcode_OpName[opcode], line_delta);
}
}
}
static void
dump_instrumentation_data_line_tools(PyCodeObject *code, uint8_t *line_tools, int i, FILE*out)
{
if (line_tools == NULL) {
fprintf(out, ", line_tools = NULL");
}
else {
fprintf(out, ", line_tools = %d", line_tools[i]);
}
}
static void
dump_instrumentation_data_per_instruction(PyCodeObject *code, _PyCoMonitoringData *data, int i, FILE*out)
{
if (data->per_instruction_opcodes == NULL) {
fprintf(out, ", per-inst opcode = NULL");
}
else {
fprintf(out, ", per-inst opcode = %s", _PyOpcode_OpName[data->per_instruction_opcodes[i]]);
}
if (data->per_instruction_tools == NULL) {
fprintf(out, ", per-inst tools = NULL");
}
else {
fprintf(out, ", per-inst tools = %d", data->per_instruction_tools[i]);
}
}
static void
dump_global_monitors(const char *prefix, _Py_GlobalMonitors monitors, FILE*out)
{
fprintf(out, "%s monitors:\n", prefix);
for (int event = 0; event < _PY_MONITORING_UNGROUPED_EVENTS; event++) {
fprintf(out, " Event %d: Tools %x\n", event, monitors.tools[event]);
}
}
static void
dump_local_monitors(const char *prefix, _Py_LocalMonitors monitors, FILE*out)
{
fprintf(out, "%s monitors:\n", prefix);
for (int event = 0; event < _PY_MONITORING_UNGROUPED_EVENTS; event++) {
fprintf(out, " Event %d: Tools %x\n", event, monitors.tools[event]);
}
}
/** NOTE:
* Do not use PyCode_Addr2Line to determine the line number in instrumentation,
* as `PyCode_Addr2Line` uses the monitoring data if it is available.
*/
/* No error checking -- Don't use this for anything but experimental debugging */
static void
dump_instrumentation_data(PyCodeObject *code, int star, FILE*out)
{
_PyCoMonitoringData *data = code->_co_monitoring;
fprintf(out, "\n");
PyObject_Print(code->co_name, out, Py_PRINT_RAW);
fprintf(out, "\n");
if (data == NULL) {
fprintf(out, "NULL\n");
return;
}
dump_global_monitors("Global", _PyInterpreterState_GET()->monitors, out);
dump_local_monitors("Code", data->local_monitors, out);
dump_local_monitors("Active", data->active_monitors, out);
int code_len = (int)Py_SIZE(code);
bool starred = false;
PyCodeAddressRange range;
_PyCode_InitAddressRange(code, &range);
for (int i = 0; i < code_len; i += _PyInstruction_GetLength(code, i)) {
_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];
int opcode = instr->op.code;
if (i == star) {
fprintf(out, "** ");
starred = true;
}
fprintf(out, "Offset: %d, line: %d %s: ", i, _PyCode_CheckLineNumber(i*2, &range), _PyOpcode_OpName[opcode]);
dump_instrumentation_data_tools(code, data->tools, i, out);
dump_instrumentation_data_lines(code, data->lines, i, out);
dump_instrumentation_data_line_tools(code, data->line_tools, i, out);
dump_instrumentation_data_per_instruction(code, data, i, out);
fprintf(out, "\n");
;
}
if (!starred && star >= 0) {
fprintf(out, "Error offset not at valid instruction offset: %d\n", star);
fprintf(out, " ");
dump_instrumentation_data_tools(code, data->tools, star, out);
dump_instrumentation_data_lines(code, data->lines, star, out);
dump_instrumentation_data_line_tools(code, data->line_tools, star, out);
dump_instrumentation_data_per_instruction(code, data, star, out);
fprintf(out, "\n");
}
}
#define CHECK(test) do { \
if (!(test)) { \
dump_instrumentation_data(code, i, stderr); \
} \
assert(test); \
} while (0)
static bool
valid_opcode(int opcode)
{
if (opcode == INSTRUMENTED_LINE) {
return true;
}
if (IS_VALID_OPCODE(opcode) &&
opcode != CACHE &&
opcode != RESERVED &&
opcode < 254)
{
return true;
}
return false;
}
static void
sanity_check_instrumentation(PyCodeObject *code)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
_PyCoMonitoringData *data = code->_co_monitoring;
if (data == NULL) {
return;
}
_Py_GlobalMonitors global_monitors = _PyInterpreterState_GET()->monitors;
_Py_LocalMonitors active_monitors;
if (code->_co_monitoring) {
_Py_LocalMonitors local_monitors = code->_co_monitoring->local_monitors;
active_monitors = local_union(global_monitors, local_monitors);
}
else {
_Py_LocalMonitors empty = (_Py_LocalMonitors) { 0 };
active_monitors = local_union(global_monitors, empty);
}
assert(monitors_equals(
code->_co_monitoring->active_monitors,
active_monitors));
int code_len = (int)Py_SIZE(code);
PyCodeAddressRange range;
_PyCode_InitAddressRange(co, &range);
for (int i = 0; i < code_len;) {
_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];
int opcode = instr->op.code;
int base_opcode = _Py_GetBaseCodeUnit(code, i).op.code;
CHECK(valid_opcode(opcode));
CHECK(valid_opcode(base_opcode));
if (opcode == INSTRUMENTED_INSTRUCTION) {
opcode = data->per_instruction_opcodes[i];
if (!is_instrumented(opcode)) {
CHECK(_PyOpcode_Deopt[opcode] == opcode);
}
}
if (opcode == INSTRUMENTED_LINE) {
CHECK(data->lines);
opcode = _PyCode_GetOriginalOpcode(data->lines, i);
CHECK(valid_opcode(opcode));
CHECK(opcode != END_FOR);
CHECK(opcode != RESUME);
CHECK(opcode != RESUME_CHECK);
CHECK(opcode != RESUME_CHECK_JIT);
CHECK(opcode != INSTRUMENTED_RESUME);
if (!is_instrumented(opcode)) {
CHECK(_PyOpcode_Deopt[opcode] == opcode);
}
CHECK(opcode != INSTRUMENTED_LINE);
}
else if (data->lines) {
/* If original_opcode is INSTRUMENTED_INSTRUCTION
* *and* we are executing a INSTRUMENTED_LINE instruction
* that has de-instrumented itself, then we will execute
* an invalid INSTRUMENTED_INSTRUCTION */
CHECK(_PyCode_GetOriginalOpcode(data->lines, i) != INSTRUMENTED_INSTRUCTION);
}
if (opcode == INSTRUMENTED_INSTRUCTION) {
CHECK(data->per_instruction_opcodes[i] != 0);
opcode = data->per_instruction_opcodes[i];
}
if (is_instrumented(opcode)) {
CHECK(DE_INSTRUMENT[opcode] == base_opcode);
int event = EVENT_FOR_OPCODE[DE_INSTRUMENT[opcode]];
if (event < 0) {
/* RESUME fixup */
event = instr->op.arg ? 1: 0;
}
CHECK(active_monitors.tools[event] != 0);
}
if (data->lines && _PyCode_GetOriginalOpcode(data->lines, i)) {
int line1 = compute_line(code, get_line_delta(data->lines, i));
int line2 = _PyCode_CheckLineNumber(i*sizeof(_Py_CODEUNIT), &range);
CHECK(line1 == line2);
}
CHECK(valid_opcode(opcode));
if (data->tools) {
uint8_t local_tools = data->tools[i];
if (opcode_has_event(base_opcode)) {
int event = EVENT_FOR_OPCODE[base_opcode];
if (event == -1) {
/* RESUME fixup */
event = _PyCode_CODE(code)[i].op.arg;
}
CHECK((active_monitors.tools[event] & local_tools) == local_tools);
}
else {
CHECK(local_tools == 0xff);
}
}
i += _PyInstruction_GetLength(code, i);
assert(i co_executors->executors[inst.op.arg];
opcode = _PyOpcode_Deopt[exec->vm_data.opcode];
inst.op.code = opcode;
inst.op.arg = exec->vm_data.oparg;
assert(inst.op.code < MIN_SPECIALIZED_OPCODE);
return inst;
}
if (opcode == INSTRUMENTED_LINE) {
opcode = _PyCode_GetOriginalOpcode(code->_co_monitoring->lines, i);
}
if (opcode == INSTRUMENTED_INSTRUCTION) {
opcode = code->_co_monitoring->per_instruction_opcodes[i];
}
CHECK(opcode != INSTRUMENTED_INSTRUCTION);
CHECK(opcode != INSTRUMENTED_LINE);
int deinstrumented = DE_INSTRUMENT[opcode];
if (deinstrumented) {
inst.op.code = deinstrumented;
}
else {
inst.op.code = _PyOpcode_Deopt[opcode];
}
assert(inst.op.code < MIN_SPECIALIZED_OPCODE);
return inst;
}
static void
de_instrument(PyCodeObject *code, _Py_CODEUNIT *bytecode, _PyCoMonitoringData *monitoring, int i,
int event)
{
assert(event != PY_MONITORING_EVENT_INSTRUCTION);
assert(event != PY_MONITORING_EVENT_LINE);
_Py_CODEUNIT *instr = &bytecode[i];
uint8_t *opcode_ptr = &instr->op.code;
int opcode = *opcode_ptr;
assert(opcode != ENTER_EXECUTOR);
if (opcode == INSTRUMENTED_LINE) {
opcode_ptr = get_original_opcode_ptr(monitoring->lines, i);
opcode = *opcode_ptr;
}
if (opcode == INSTRUMENTED_INSTRUCTION) {
opcode_ptr = &monitoring->per_instruction_opcodes[i];
opcode = *opcode_ptr;
}
int deinstrumented = DE_INSTRUMENT[opcode];
if (deinstrumented == 0) {
return;
}
CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented);
FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, deinstrumented);
if (_PyOpcode_Caches[deinstrumented]) {
FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff,
adaptive_counter_warmup().value_and_backoff);
}
}
static void
de_instrument_line(PyCodeObject *code, _Py_CODEUNIT *bytecode, _PyCoMonitoringData *monitoring,
int i)
{
_Py_CODEUNIT *instr = &bytecode[i];
int opcode = instr->op.code;
if (opcode != INSTRUMENTED_LINE) {
return;
}
_PyCoLineInstrumentationData *lines = monitoring->lines;
int original_opcode = _PyCode_GetOriginalOpcode(lines, i);
if (original_opcode == INSTRUMENTED_INSTRUCTION) {
set_original_opcode(lines, i, monitoring->per_instruction_opcodes[i]);
}
CHECK(original_opcode != 0);
CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]);
FT_ATOMIC_STORE_UINT8(instr->op.code, original_opcode);
if (_PyOpcode_Caches[original_opcode]) {
FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff,
adaptive_counter_warmup().value_and_backoff);
}
assert(instr->op.code != INSTRUMENTED_LINE);
}
static void
de_instrument_per_instruction(PyCodeObject *code, _Py_CODEUNIT *bytecode,
_PyCoMonitoringData *monitoring, int i)
{
_Py_CODEUNIT *instr = &bytecode[i];
uint8_t *opcode_ptr = &instr->op.code;
int opcode = *opcode_ptr;
if (opcode == INSTRUMENTED_LINE) {
opcode_ptr = get_original_opcode_ptr(monitoring->lines, i);
opcode = *opcode_ptr;
}
if (opcode != INSTRUMENTED_INSTRUCTION) {
return;
}
int original_opcode = monitoring->per_instruction_opcodes[i];
CHECK(original_opcode != 0);
CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]);
FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, original_opcode);
if (_PyOpcode_Caches[original_opcode]) {
FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff,
adaptive_counter_warmup().value_and_backoff);
}
assert(*opcode_ptr != INSTRUMENTED_INSTRUCTION);
assert(instr->op.code != INSTRUMENTED_INSTRUCTION);
}
static void
instrument(PyCodeObject *code, _Py_CODEUNIT *bytecode, _PyCoMonitoringData *monitoring, int i)
{
_Py_CODEUNIT *instr = &bytecode[i];
uint8_t *opcode_ptr = &instr->op.code;
int opcode =*opcode_ptr;
if (opcode == INSTRUMENTED_LINE) {
opcode_ptr = get_original_opcode_ptr(monitoring->lines, i);
opcode = *opcode_ptr;
}
if (opcode == INSTRUMENTED_INSTRUCTION) {
opcode_ptr = &monitoring->per_instruction_opcodes[i];
opcode = *opcode_ptr;
CHECK(opcode != INSTRUMENTED_INSTRUCTION && opcode != INSTRUMENTED_LINE);
CHECK(opcode == _PyOpcode_Deopt[opcode]);
}
CHECK(opcode != 0);
if (!is_instrumented(opcode)) {
int deopt = _PyOpcode_Deopt[opcode];
int instrumented = INSTRUMENTED_OPCODES[deopt];
assert(instrumented);
FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, instrumented);
if (_PyOpcode_Caches[deopt]) {
FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff,
adaptive_counter_warmup().value_and_backoff);
}
}
}
static void
instrument_line(PyCodeObject *code, _Py_CODEUNIT *bytecode, _PyCoMonitoringData *monitoring, int i)
{
uint8_t *opcode_ptr = &bytecode[i].op.code;
int opcode = *opcode_ptr;
if (opcode == INSTRUMENTED_LINE) {
return;
}
set_original_opcode(monitoring->lines, i, _PyOpcode_Deopt[opcode]);
CHECK(get_line_delta(monitoring->lines, i) > NO_LINE);
FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, INSTRUMENTED_LINE);
}
static void
instrument_per_instruction(PyCodeObject *code, _Py_CODEUNIT *bytecode,
_PyCoMonitoringData *monitoring, int i)
{
_Py_CODEUNIT *instr = &bytecode[i];
uint8_t *opcode_ptr = &instr->op.code;
int opcode = *opcode_ptr;
if (opcode == INSTRUMENTED_LINE) {
opcode_ptr = get_original_opcode_ptr(monitoring->lines, i);
opcode = *opcode_ptr;
}
if (opcode == INSTRUMENTED_INSTRUCTION) {
assert(monitoring->per_instruction_opcodes[i] > 0);
return;
}
CHECK(opcode != 0);
if (is_instrumented(opcode)) {
monitoring->per_instruction_opcodes[i] = opcode;
}
else {
assert(opcode != 0);
assert(_PyOpcode_Deopt[opcode] != 0);
assert(_PyOpcode_Deopt[opcode] != RESUME);
monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode];
}
assert(monitoring->per_instruction_opcodes[i] > 0);
FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, INSTRUMENTED_INSTRUCTION);
}
static void
remove_tools(PyCodeObject * code, int offset, int event, int tools)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
assert(event != PY_MONITORING_EVENT_LINE);
assert(event != PY_MONITORING_EVENT_INSTRUCTION);
assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event));
assert(opcode_has_event(_Py_GetBaseCodeUnit(code, offset).op.code));
_PyCoMonitoringData *monitoring = code->_co_monitoring;
assert(monitoring);
bool should_de_instrument;
if (monitoring->tools) {
monitoring->tools[offset] &= ~tools;
should_de_instrument = (monitoring->tools[offset] == 0);
}
else {
/* Single tool */
uint8_t single_tool = monitoring->active_monitors.tools[event];
assert(_Py_popcount32(single_tool) monitors.tools[event];
int local_tools = code->_co_monitoring->local_monitors.tools[event];
return tools == ((global_tools | local_tools) & tools);
}
#endif
static void
remove_line_tools(PyCodeObject * code, int offset, int tools)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
_PyCoMonitoringData *monitoring = code->_co_monitoring;
assert(monitoring);
bool should_de_instrument;
if (monitoring->line_tools)
{
uint8_t *toolsptr = &monitoring->line_tools[offset];
*toolsptr &= ~tools;
should_de_instrument = (*toolsptr == 0);
}
else {
/* Single tool */
uint8_t single_tool = monitoring->active_monitors.tools[PY_MONITORING_EVENT_LINE];
assert(_Py_popcount32(single_tool) _co_monitoring);
if (code->_co_monitoring &&
code->_co_monitoring->tools
) {
code->_co_monitoring->tools[offset] |= tools;
}
else {
/* Single tool */
assert(_Py_popcount32(tools) == 1);
assert(tools_is_subset_for_event(code, event, tools));
}
MODIFY_BYTECODE(code, instrument, code->_co_monitoring, offset);
}
static void
add_line_tools(PyCodeObject * code, int offset, int tools)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_LINE, tools));
assert(code->_co_monitoring);
if (code->_co_monitoring->line_tools) {
code->_co_monitoring->line_tools[offset] |= tools;
}
else {
/* Single tool */
assert(_Py_popcount32(tools) == 1);
}
MODIFY_BYTECODE(code, instrument_line, code->_co_monitoring, offset);
}
static void
add_per_instruction_tools(PyCodeObject * code, int offset, int tools)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_INSTRUCTION, tools));
assert(code->_co_monitoring);
if (code->_co_monitoring->per_instruction_tools) {
code->_co_monitoring->per_instruction_tools[offset] |= tools;
}
else {
/* Single tool */
assert(_Py_popcount32(tools) == 1);
}
MODIFY_BYTECODE(code, instrument_per_instruction, code->_co_monitoring, offset);
}
static void
remove_per_instruction_tools(PyCodeObject * code, int offset, int tools)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
_PyCoMonitoringData *monitoring = code->_co_monitoring;
assert(code->_co_monitoring);
bool should_de_instrument;
if (code->_co_monitoring->per_instruction_tools) {
uint8_t *toolsptr = &code->_co_monitoring->per_instruction_tools[offset];
*toolsptr &= ~tools;
should_de_instrument = (*toolsptr == 0);
}
else {
/* Single tool */
uint8_t single_tool = code->_co_monitoring->active_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION];
assert(_Py_popcount32(single_tool) monitoring_callables[tool][event];
if (instrument == NULL) {
return 0;
}
int old_what = tstate->what_event;
tstate->what_event = event;
tstate->tracing++;
PyObject *res = _PyObject_VectorcallTstate(tstate, instrument, args, nargsf, NULL);
tstate->tracing--;
tstate->what_event = old_what;
if (res == NULL) {
return -1;
}
if (res == &_PyInstrumentation_DISABLE) {
assert(_Py_IsImmortal(res));
return 1;
}
Py_DECREF(res);
return 0;
}
static const int8_t MOST_SIGNIFICANT_BITS[16] = {
-1, 0, 1, 1,
2, 2, 2, 2,
3, 3, 3, 3,
3, 3, 3, 3,
};
/* We could use _Py_bit_length here, but that is designed for larger (32/64)
* bit ints, and can perform relatively poorly on platforms without the
* necessary intrinsics. */
static inline int most_significant_bit(uint8_t bits) {
assert(bits != 0);
if (bits > 15) {
return MOST_SIGNIFICANT_BITS[bits>>4]+4;
}
return MOST_SIGNIFICANT_BITS[bits];
}
static uint32_t
global_version(PyInterpreterState *interp)
{
uint32_t version = (uint32_t)_Py_atomic_load_uintptr_relaxed(
&interp->ceval.instrumentation_version);
#ifdef Py_DEBUG
PyThreadState *tstate = _PyThreadState_GET();
uint32_t thread_version =
(uint32_t)(_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) &
~_PY_EVAL_EVENTS_MASK);
assert(thread_version == version);
#endif
return version;
}
/* Atomically set the given version in the given location, without touching
anything in _PY_EVAL_EVENTS_MASK. */
static void
set_version_raw(uintptr_t *ptr, uint32_t version)
{
uintptr_t old = _Py_atomic_load_uintptr_relaxed(ptr);
uintptr_t new;
do {
new = (old & _PY_EVAL_EVENTS_MASK) | version;
} while (!_Py_atomic_compare_exchange_uintptr(ptr, &old, new));
}
static void
set_global_version(PyThreadState *tstate, uint32_t version)
{
ASSERT_WORLD_STOPPED();
assert((version & _PY_EVAL_EVENTS_MASK) == 0);
PyInterpreterState *interp = tstate->interp;
set_version_raw(&interp->ceval.instrumentation_version, version);
#ifdef Py_GIL_DISABLED
// Set the version on all threads in free-threaded builds.
_Py_FOR_EACH_TSTATE_BEGIN(interp, tstate) {
set_version_raw(&tstate->eval_breaker, version);
};
_Py_FOR_EACH_TSTATE_END(interp);
#else
// Normal builds take the current version from instrumentation_version when
// attaching a thread, so we only have to set the current thread's version.
set_version_raw(&tstate->eval_breaker, version);
#endif
}
static bool
is_version_up_to_date(PyCodeObject *code, PyInterpreterState *interp)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
return global_version(interp) == code->_co_instrumentation_version;
}
#ifndef NDEBUG
static bool
instrumentation_cross_checks(PyInterpreterState *interp, PyCodeObject *code)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
_Py_LocalMonitors expected = local_union(
interp->monitors,
code->_co_monitoring->local_monitors);
return monitors_equals(code->_co_monitoring->active_monitors, expected);
}
static int
debug_check_sanity(PyInterpreterState *interp, PyCodeObject *code)
{
int res;
LOCK_CODE(code);
res = is_version_up_to_date(code, interp) &&
instrumentation_cross_checks(interp, code);
UNLOCK_CODE();
return res;
}
#endif
static inline uint8_t
get_tools_for_instruction(PyCodeObject *code, PyInterpreterState *interp, int i, int event)
{
uint8_t tools;
assert(event != PY_MONITORING_EVENT_LINE);
assert(event != PY_MONITORING_EVENT_INSTRUCTION);
if (event >= _PY_MONITORING_UNGROUPED_EVENTS) {
assert(event == PY_MONITORING_EVENT_C_RAISE ||
event == PY_MONITORING_EVENT_C_RETURN);
event = PY_MONITORING_EVENT_CALL;
}
assert(_PY_MONITORING_IS_UNGROUPED_EVENT(event));
CHECK(debug_check_sanity(interp, code));
if (PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) {
/* Instrumented events use per-instruction tool bitmaps. */
if (code->_co_monitoring->tools) {
tools = code->_co_monitoring->tools[i];
}
else {
tools = code->_co_monitoring->active_monitors.tools[event];
}
}
else {
/* Other (non-instrumented) events are not tied to specific instructions;
* use the code-object-level active_monitors bitmap instead. */
tools = code->_co_monitoring->active_monitors.tools[event];
}
return tools;
}
static const char *const event_names [] = {
[PY_MONITORING_EVENT_PY_START] = "PY_START",
[PY_MONITORING_EVENT_PY_RESUME] = "PY_RESUME",
[PY_MONITORING_EVENT_PY_RETURN] = "PY_RETURN",
[PY_MONITORING_EVENT_PY_YIELD] = "PY_YIELD",
[PY_MONITORING_EVENT_CALL] = "CALL",
[PY_MONITORING_EVENT_LINE] = "LINE",
[PY_MONITORING_EVENT_INSTRUCTION] = "INSTRUCTION",
[PY_MONITORING_EVENT_JUMP] = "JUMP",
[PY_MONITORING_EVENT_BRANCH] = "BRANCH",
[PY_MONITORING_EVENT_BRANCH_LEFT] = "BRANCH_LEFT",
[PY_MONITORING_EVENT_BRANCH_RIGHT] = "BRANCH_RIGHT",
[PY_MONITORING_EVENT_C_RETURN] = "C_RETURN",
[PY_MONITORING_EVENT_PY_THROW] = "PY_THROW",
[PY_MONITORING_EVENT_RAISE] = "RAISE",
[PY_MONITORING_EVENT_RERAISE] = "RERAISE",
[PY_MONITORING_EVENT_EXCEPTION_HANDLED] = "EXCEPTION_HANDLED",
[PY_MONITORING_EVENT_C_RAISE] = "C_RAISE",
[PY_MONITORING_EVENT_PY_UNWIND] = "PY_UNWIND",
[PY_MONITORING_EVENT_STOP_ITERATION] = "STOP_ITERATION",
};
/* Disable an "other" (non-instrumented) event (e.g. PY_UNWIND) for a single
* tool on this code object. Must be called with the world stopped or the
* code lock held. */
static void
remove_local_tool(PyCodeObject *code, PyInterpreterState *interp,
int event, int tool)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
assert(_PY_MONITORING_IS_UNGROUPED_EVENT(event));
assert(!PY_MONITORING_IS_INSTRUMENTED_EVENT(event));
assert(code->_co_monitoring);
code->_co_monitoring->local_monitors.tools[event] &= ~(1 _co_monitoring->active_monitors.tools[event] =
interp->monitors.tools[event] |
code->_co_monitoring->local_monitors.tools[event];
}
static int
call_instrumentation_vector(
_Py_CODEUNIT *instr, PyThreadState *tstate, int event,
_PyInterpreterFrame *frame, _Py_CODEUNIT *arg2, Py_ssize_t nargs, PyObject *args[])
{
if (tstate->tracing) {
return 0;
}
assert(!_PyErr_Occurred(tstate));
assert(args[0] == NULL);
PyCodeObject *code = _PyFrame_GetCode(frame);
assert(args[1] == NULL);
args[1] = (PyObject *)code;
int offset = (int)(instr - _PyFrame_GetBytecode(frame));
/* Offset visible to user should be the offset in bytes, as that is the
* convention for APIs involving code offsets. */
int bytes_arg2 = (int)(arg2 - _PyFrame_GetBytecode(frame)) * (int)sizeof(_Py_CODEUNIT);
PyObject *arg2_obj = PyLong_FromLong(bytes_arg2);
if (arg2_obj == NULL) {
return -1;
}
assert(args[2] == NULL);
args[2] = arg2_obj;
PyInterpreterState *interp = tstate->interp;
uint8_t tools = get_tools_for_instruction(code, interp, offset, event);
size_t nargsf = (size_t) nargs | PY_VECTORCALL_ARGUMENTS_OFFSET;
PyObject **callargs = &args[1];
int err = 0;
while (tools) {
int tool = most_significant_bit(tools);
assert(tool >= 0 && tool < 8);
assert(tools & (1 instr_ptr != instr_ptr) {
/* The callback has caused a jump (by setting the line number) */
return frame->instr_ptr;
}
return dest;
}
static void
call_instrumentation_vector_protected(
PyThreadState *tstate, int event,
_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, Py_ssize_t nargs, PyObject *args[])
{
assert(_PyErr_Occurred(tstate));
PyObject *exc = _PyErr_GetRaisedException(tstate);
int err = call_instrumentation_vector(instr, tstate, event, frame, instr, nargs, args);
if (err) {
Py_XDECREF(exc);
}
else {
_PyErr_SetRaisedException(tstate, exc);
}
assert(_PyErr_Occurred(tstate));
}
Py_NO_INLINE void
_Py_call_instrumentation_exc2(
PyThreadState *tstate, int event,
_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1)
{
assert(_PyErr_Occurred(tstate));
PyObject *args[5] = { NULL, NULL, NULL, arg0, arg1 };
call_instrumentation_vector_protected(tstate, event, frame, instr, 4, args);
}
int
_Py_Instrumentation_GetLine(PyCodeObject *code, _PyCoLineInstrumentationData *line_data, int index)
{
assert(line_data != NULL);
assert(index < Py_SIZE(code));
int line_delta = get_line_delta(line_data, index);
int line = compute_line(code, line_delta);
return line;
}
Py_NO_INLINE int
_Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *prev)
{
PyCodeObject *code = _PyFrame_GetCode(frame);
assert(tstate->tracing == 0);
assert(debug_check_sanity(tstate->interp, code));
_Py_CODEUNIT *bytecode = _PyFrame_GetBytecode(frame);
int i = (int)(instr - bytecode);
_PyCoMonitoringData *monitoring = code->_co_monitoring;
_PyCoLineInstrumentationData *line_data = monitoring->lines;
PyInterpreterState *interp = tstate->interp;
int line = _Py_Instrumentation_GetLine(code, line_data, i);
assert(line >= 0);
assert(prev != NULL);
int prev_index = (int)(prev - bytecode);
int prev_line = _Py_Instrumentation_GetLine(code, line_data, prev_index);
if (prev_line == line) {
int prev_opcode = bytecode[prev_index].op.code;
/* RESUME and INSTRUMENTED_RESUME are needed for the operation of
* instrumentation, so must never be hidden by an INSTRUMENTED_LINE.
*/
if (prev_opcode != RESUME && prev_opcode != INSTRUMENTED_RESUME) {
goto done;
}
}
uint8_t tools = code->_co_monitoring->line_tools != NULL ?
code->_co_monitoring->line_tools[i] :
(interp->monitors.tools[PY_MONITORING_EVENT_LINE] |
code->_co_monitoring->local_monitors.tools[PY_MONITORING_EVENT_LINE]
);
/* Special case sys.settrace to avoid boxing the line number,
* only to immediately unbox it. */
if (tools & (1 c_tracefunc != NULL) {
PyFrameObject *frame_obj = _PyFrame_GetFrameObject(frame);
if (frame_obj == NULL) {
return -1;
}
if (frame_obj->f_trace_lines) {
/* Need to set tracing and what_event as if using
* the instrumentation call. */
int old_what = tstate->what_event;
tstate->what_event = PY_MONITORING_EVENT_LINE;
tstate->tracing++;
/* Call c_tracefunc directly, having set the line number. */
Py_INCREF(frame_obj);
frame_obj->f_lineno = line;
int err = tstate->c_tracefunc(tstate->c_traceobj, frame_obj, PyTrace_LINE, Py_None);
frame_obj->f_lineno = 0;
tstate->tracing--;
tstate->what_event = old_what;
Py_DECREF(frame_obj);
if (err) {
return -1;
}
}
}
tools &= (255 - (1 = 0 && tool < PY_MONITORING_SYS_PROFILE_ID);
assert(tools & (1 per_instruction_opcodes);
int next_opcode = instrumentation_data->per_instruction_opcodes[offset];
if (tstate->tracing) {
return next_opcode;
}
assert(debug_check_sanity(tstate->interp, code));
PyInterpreterState *interp = tstate->interp;
uint8_t tools = instrumentation_data->per_instruction_tools != NULL ?
instrumentation_data->per_instruction_tools[offset] :
(interp->monitors.tools[PY_MONITORING_EVENT_INSTRUCTION] |
code->_co_monitoring->local_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION]
);
int bytes_offset = offset * (int)sizeof(_Py_CODEUNIT);
PyObject *offset_obj = PyLong_FromLong(bytes_offset);
if (offset_obj == NULL) {
return -1;
}
PyObject *args[3] = { NULL, (PyObject *)code, offset_obj };
while (tools) {
int tool = most_significant_bit(tools);
assert(tool >= 0 && tool < 8);
assert(tools & (1 tools;
assert(tools != NULL);
int code_len = (int)Py_SIZE(code);
for (int i = 0; i < code_len; i++) {
_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];
int opcode = instr->op.code;
assert(opcode != ENTER_EXECUTOR);
if (opcode == INSTRUMENTED_LINE) {
opcode = _PyCode_GetOriginalOpcode(code->_co_monitoring->lines, i);
}
if (opcode == INSTRUMENTED_INSTRUCTION) {
opcode = code->_co_monitoring->per_instruction_opcodes[i];
}
bool instrumented = is_instrumented(opcode);
if (instrumented) {
opcode = DE_INSTRUMENT[opcode];
assert(opcode != 0);
}
opcode = _PyOpcode_Deopt[opcode];
if (opcode_has_event(opcode)) {
if (instrumented) {
int8_t event;
if (opcode == RESUME) {
event = instr->op.arg != 0;
}
else {
event = EVENT_FOR_OPCODE[opcode];
assert(event > 0);
}
assert(event >= 0);
assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event));
tools[i] = code->_co_monitoring->active_monitors.tools[event];
CHECK(tools[i] != 0);
}
else {
tools[i] = 0;
}
}
#ifdef Py_DEBUG
/* Initialize tools for invalid locations to all ones to try to catch errors */
else {
tools[i] = 0xff;
}
for (int j = 1; j bytes_per_entry = bytes_per_entry;
int code_len = (int)Py_SIZE(code);
PyCodeAddressRange range;
_PyCode_InitAddressRange(code, &range);
int current_line = -1;
for (int i = 0; i < code_len; ) {
int opcode = _Py_GetBaseCodeUnit(code, i).op.code;
int line = _PyCode_CheckLineNumber(i*(int)sizeof(_Py_CODEUNIT), &range);
set_line_delta(line_data, i, compute_line_delta(code, line));
int length = _PyInstruction_GetLength(code, i);
if (i < code->_co_firsttraceable) {
set_original_opcode(line_data, i, 0);
}
else {
switch (opcode) {
case END_ASYNC_FOR:
case END_FOR:
case END_SEND:
case RESUME:
case POP_ITER:
/* END_FOR cannot start a line, as it is skipped by FOR_ITER
* END_SEND cannot start a line, as it is skipped by SEND
* RESUME and POP_ITER must not be instrumented with INSTRUMENTED_LINE */
set_original_opcode(line_data, i, 0);
break;
default:
/* Set original_opcode to the opcode iff the instruction
* starts a line, and thus should be instrumented.
* This saves having to perform this check every time the
* we turn instrumentation on or off, and serves as a sanity
* check when debugging.
*/
if (line != current_line && line >= 0) {
set_original_opcode(line_data, i, opcode);
CHECK(get_line_delta(line_data, i) != NO_LINE);
}
else {
set_original_opcode(line_data, i, 0);
}
current_line = line;
}
}
for (int j = 1; j < length; j++) {
set_original_opcode(line_data, i+j, 0);
set_line_delta(line_data, i+j, NO_LINE);
}
i += length;
}
for (int i = code->_co_firsttraceable; i < code_len; ) {
_Py_CODEUNIT inst =_Py_GetBaseCodeUnit(code, i);
int opcode = inst.op.code;
int oparg = 0;
while (opcode == EXTENDED_ARG) {
oparg = (oparg co_exceptiontable);
unsigned char *end = start + PyBytes_GET_SIZE(code->co_exceptiontable);
unsigned char *scan = start;
while (scan < end) {
int start_offset, size, handler;
scan = parse_varint(scan, &start_offset);
assert(start_offset >= 0 && start_offset < code_len);
scan = parse_varint(scan, &size);
assert(size >= 0 && start_offset+size = 0 && handler < code_len);
int depth_and_lasti;
scan = parse_varint(scan, &depth_and_lasti);
int original_opcode = _Py_GetBaseCodeUnit(code, handler).op.code;
/* Skip if not the start of a line.
* END_ASYNC_FOR is a bit special as it marks the end of
* an `async for` loop, which should not generate its own
* line event. */
if (get_line_delta(line_data, handler) != NO_LINE && original_opcode != END_ASYNC_FOR) {
set_original_opcode(line_data, handler, original_opcode);
}
}
}
static void
initialize_line_tools(PyCodeObject *code, _Py_LocalMonitors *all_events)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
uint8_t *line_tools = code->_co_monitoring->line_tools;
assert(line_tools != NULL);
int code_len = (int)Py_SIZE(code);
for (int i = 0; i < code_len; i++) {
line_tools[i] = all_events->tools[PY_MONITORING_EVENT_LINE];
}
}
static int
allocate_instrumentation_data(PyCodeObject *code)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
if (code->_co_monitoring == NULL) {
_PyCoMonitoringData *monitoring = PyMem_Malloc(sizeof(_PyCoMonitoringData));
if (monitoring == NULL) {
PyErr_NoMemory();
return -1;
}
monitoring->local_monitors = (_Py_LocalMonitors){ 0 };
monitoring->active_monitors = (_Py_LocalMonitors){ 0 };
monitoring->tools = NULL;
monitoring->lines = NULL;
monitoring->line_tools = NULL;
monitoring->per_instruction_opcodes = NULL;
monitoring->per_instruction_tools = NULL;
_Py_atomic_store_ptr_release(&code->_co_monitoring, monitoring);
}
return 0;
}
static int
update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
int code_len = (int)Py_SIZE(code);
if (allocate_instrumentation_data(code)) {
return -1;
}
// If the local monitors are out of date, clear them up
_Py_LocalMonitors *local_monitors = &code->_co_monitoring->local_monitors;
for (int i = 0; i < PY_MONITORING_TOOL_IDS; i++) {
if (code->_co_monitoring->tool_versions[i] != interp->monitoring_tool_versions[i]) {
for (int j = 0; j < _PY_MONITORING_UNGROUPED_EVENTS; j++) {
local_monitors->tools[j] &= ~(1 monitors,
code->_co_monitoring->local_monitors);
bool multitools = multiple_tools(&all_events);
if (code->_co_monitoring->tools == NULL && multitools) {
code->_co_monitoring->tools = PyMem_Malloc(code_len);
if (code->_co_monitoring->tools == NULL) {
PyErr_NoMemory();
return -1;
}
initialize_tools(code);
}
if (all_events.tools[PY_MONITORING_EVENT_LINE]) {
if (code->_co_monitoring->lines == NULL) {
PyCodeAddressRange range;
_PyCode_InitAddressRange(code, &range);
int max_line = code->co_firstlineno + 1;
_PyCode_InitAddressRange(code, &range);
for (int i = code->_co_firsttraceable; i < code_len; ) {
int line = _PyCode_CheckLineNumber(i*(int)sizeof(_Py_CODEUNIT), &range);
if (line > max_line) {
max_line = line;
}
int length = _PyInstruction_GetLength(code, i);
i += length;
}
int bytes_per_entry;
int max_delta = max_line - code->co_firstlineno;
/* We store delta+2 in the table, so 253 is max for one byte */
if (max_delta < 256+NO_LINE) {
bytes_per_entry = 2;
}
else if (max_delta < (1 lines, lines);
}
if (multitools && code->_co_monitoring->line_tools == NULL) {
code->_co_monitoring->line_tools = PyMem_Malloc(code_len);
if (code->_co_monitoring->line_tools == NULL) {
PyErr_NoMemory();
return -1;
}
initialize_line_tools(code, &all_events);
}
}
if (all_events.tools[PY_MONITORING_EVENT_INSTRUCTION]) {
if (code->_co_monitoring->per_instruction_opcodes == NULL) {
code->_co_monitoring->per_instruction_opcodes = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData));
if (code->_co_monitoring->per_instruction_opcodes == NULL) {
PyErr_NoMemory();
return -1;
}
// Initialize all of the instructions so if local events change while another thread is executing
// we know what the original opcode was.
for (int i = 0; i < code_len; i++) {
int opcode = _PyCode_CODE(code)[i].op.code;
code->_co_monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode];
}
}
if (multitools && code->_co_monitoring->per_instruction_tools == NULL) {
code->_co_monitoring->per_instruction_tools = PyMem_Malloc(code_len);
if (code->_co_monitoring->per_instruction_tools == NULL) {
PyErr_NoMemory();
return -1;
}
for (int i = 0; i < code_len; i++) {
code->_co_monitoring->per_instruction_tools[i] = 0;
}
}
}
return 0;
}
static int
force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
#ifdef _Py_TIER2
if (code->co_executors != NULL) {
_PyCode_Clear_Executors(code);
}
_Py_Executors_InvalidateDependency(interp, code, 1);
#endif
int code_len = (int)Py_SIZE(code);
/* Exit early to avoid creating instrumentation
* data for potential statically allocated code
* objects.
* See https://github.com/python/cpython/issues/108390 */
if (code->co_flags & CO_NO_MONITORING_EVENTS) {
return 0;
}
if (update_instrumentation_data(code, interp)) {
return -1;
}
_Py_LocalMonitors active_events = local_union(
interp->monitors,
code->_co_monitoring->local_monitors);
_Py_LocalMonitors new_events;
_Py_LocalMonitors removed_events;
bool restarted = interp->last_restart_version > code->_co_instrumentation_version;
if (restarted) {
removed_events = code->_co_monitoring->active_monitors;
new_events = active_events;
}
else {
removed_events = monitors_sub(code->_co_monitoring->active_monitors, active_events);
new_events = monitors_sub(active_events, code->_co_monitoring->active_monitors);
assert(monitors_are_empty(monitors_and(new_events, removed_events)));
}
code->_co_monitoring->active_monitors = active_events;
if (monitors_are_empty(new_events) && monitors_are_empty(removed_events)) {
goto done;
}
/* Insert instrumentation */
for (int i = code->_co_firsttraceable; i < code_len; i+= _PyInstruction_GetLength(code, i)) {
assert(_PyCode_CODE(code)[i].op.code != ENTER_EXECUTOR);
_Py_CODEUNIT instr = _Py_GetBaseCodeUnit(code, i);
CHECK(instr.op.code != 0);
int base_opcode = instr.op.code;
if (opcode_has_event(base_opcode)) {
int8_t event;
if (base_opcode == RESUME) {
event = instr.op.arg > 0;
}
else {
event = EVENT_FOR_OPCODE[base_opcode];
assert(event > 0);
}
uint8_t removed_tools = removed_events.tools[event];
if (removed_tools) {
remove_tools(code, i, event, removed_tools);
}
uint8_t new_tools = new_events.tools[event];
if (new_tools) {
add_tools(code, i, event, new_tools);
}
}
}
// GH-103845: We need to remove both the line and instruction instrumentation before
// adding new ones, otherwise we may remove the newly added instrumentation.
uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE];
uint8_t removed_per_instruction_tools = removed_events.tools[PY_MONITORING_EVENT_INSTRUCTION];
if (removed_line_tools) {
_PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines;
for (int i = code->_co_firsttraceable; i < code_len;) {
if (_PyCode_GetOriginalOpcode(line_data, i)) {
remove_line_tools(code, i, removed_line_tools);
}
i += _PyInstruction_GetLength(code, i);
}
}
if (removed_per_instruction_tools) {
for (int i = code->_co_firsttraceable; i < code_len;) {
int opcode = _Py_GetBaseCodeUnit(code, i).op.code;
if (opcode == RESUME || opcode == END_FOR) {
i += _PyInstruction_GetLength(code, i);
continue;
}
remove_per_instruction_tools(code, i, removed_per_instruction_tools);
i += _PyInstruction_GetLength(code, i);
}
}
#ifdef INSTRUMENT_DEBUG
sanity_check_instrumentation(code);
#endif
uint8_t new_line_tools = new_events.tools[PY_MONITORING_EVENT_LINE];
uint8_t new_per_instruction_tools = new_events.tools[PY_MONITORING_EVENT_INSTRUCTION];
if (new_line_tools) {
_PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines;
for (int i = code->_co_firsttraceable; i < code_len;) {
if (_PyCode_GetOriginalOpcode(line_data, i)) {
add_line_tools(code, i, new_line_tools);
}
i += _PyInstruction_GetLength(code, i);
}
}
if (new_per_instruction_tools) {
for (int i = code->_co_firsttraceable; i < code_len;) {
int opcode = _Py_GetBaseCodeUnit(code, i).op.code;
if (opcode == RESUME || opcode == END_FOR) {
i += _PyInstruction_GetLength(code, i);
continue;
}
add_per_instruction_tools(code, i, new_per_instruction_tools);
i += _PyInstruction_GetLength(code, i);
}
}
done:
FT_ATOMIC_STORE_UINTPTR_RELEASE(code->_co_instrumentation_version,
global_version(interp));
#ifdef INSTRUMENT_DEBUG
sanity_check_instrumentation(code);
#endif
return 0;
}
static int
instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
if (is_version_up_to_date(code, interp)) {
assert(
interp->ceval.instrumentation_version == 0 ||
instrumentation_cross_checks(interp, code)
);
return 0;
}
return force_instrument_lock_held(code, interp);
}
int
_Py_Instrument(PyCodeObject *code, PyInterpreterState *interp)
{
int res;
LOCK_CODE(code);
res = instrument_lock_held(code, interp);
UNLOCK_CODE();
return res;
}
#define C_RETURN_EVENTS \
((1 previous;
}
}
done:
_Py_FOR_EACH_TSTATE_END(interp);
return err;
}
static void
set_events(_Py_GlobalMonitors *m, int tool_id, _PyMonitoringEventSet events)
{
assert(0 tools[e];
int active = (events >> e) & 1;
*tools &= ~(1 e) & 1;
*tools &= ~(1 co_name);
return -1;
}
if (check_tool(interp, tool_id)) {
return -1;
}
if (allocate_instrumentation_data(code)) {
return -1;
}
code->_co_monitoring->tool_versions[tool_id] = interp->monitoring_tool_versions[tool_id];
_Py_LocalMonitors *local = &code->_co_monitoring->local_monitors;
uint32_t existing_events = get_local_events(local, tool_id);
if (existing_events == events) {
return 0;
}
set_local_events(local, tool_id, events);
return force_instrument_lock_held(code, interp);
}
int
_PyMonitoring_GetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet *events)
{
assert(0 _co_monitoring == NULL) {
*events = 0;
return 0;
}
_Py_LocalMonitors *local = &code->_co_monitoring->local_monitors;
*events = get_local_events(local, tool_id);
return 0;
}
int _PyMonitoring_ClearToolId(int tool_id)
{
assert(0 monitoring_tool_versions[tool_id] = version;
// Set the new global version so all the code objects can refresh the
// instrumentation.
set_global_version(_PyThreadState_GET(), version);
#ifdef _Py_TIER2
_Py_Executors_InvalidateAll(interp, 1);
#endif
int res = instrument_all_executing_code_objects(interp);
_PyEval_StartTheWorld(interp);
return res;
}
/*[clinic input]
module monitoring
[clinic start generated code]*/
/*[clinic end generated code: output=da39a3ee5e6b4b0d input=37257f5987a360cf]*/
/*[clinic end generated code]*/
#include "clinic/instrumentation.c.h"
static int
check_valid_tool(int tool_id)
{
if (tool_id < 0 || tool_id >= PY_MONITORING_SYS_PROFILE_ID) {
PyErr_Format(PyExc_ValueError, "invalid tool %d (must be between 0 and 5)", tool_id);
return -1;
}
return 0;
}
/*[clinic input]
monitoring.use_tool_id
tool_id: int
name: object
/
[clinic start generated code]*/
static PyObject *
monitoring_use_tool_id_impl(PyObject *module, int tool_id, PyObject *name)
/*[clinic end generated code: output=30d76dc92b7cd653 input=ebc453761c621be1]*/
{
if (check_valid_tool(tool_id)) {
return NULL;
}
if (!PyUnicode_Check(name)) {
PyErr_SetString(PyExc_ValueError, "tool name must be a str");
return NULL;
}
PyInterpreterState *interp = _PyInterpreterState_GET();
if (interp->monitoring_tool_names[tool_id] != NULL) {
PyErr_Format(PyExc_ValueError, "tool %d is already in use", tool_id);
return NULL;
}
interp->monitoring_tool_names[tool_id] = Py_NewRef(name);
Py_RETURN_NONE;
}
/*[clinic input]
monitoring.clear_tool_id
tool_id: int
/
[clinic start generated code]*/
static PyObject *
monitoring_clear_tool_id_impl(PyObject *module, int tool_id)
/*[clinic end generated code: output=04defc23470b1be7 input=af643d6648a66163]*/
{
if (check_valid_tool(tool_id)) {
return NULL;
}
PyInterpreterState *interp = _PyInterpreterState_GET();
if (interp->monitoring_tool_names[tool_id] != NULL) {
if (_PyMonitoring_ClearToolId(tool_id) < 0) {
return NULL;
}
}
Py_RETURN_NONE;
}
/*[clinic input]
monitoring.free_tool_id
tool_id: int
/
[clinic start generated code]*/
static PyObject *
monitoring_free_tool_id_impl(PyObject *module, int tool_id)
/*[clinic end generated code: output=86c2d2a1219a8591 input=a23fb6be3a8618e9]*/
{
if (check_valid_tool(tool_id)) {
return NULL;
}
PyInterpreterState *interp = _PyInterpreterState_GET();
if (interp->monitoring_tool_names[tool_id] != NULL) {
if (_PyMonitoring_ClearToolId(tool_id) < 0) {
return NULL;
}
}
Py_CLEAR(interp->monitoring_tool_names[tool_id]);
Py_RETURN_NONE;
}
/*[clinic input]
monitoring.get_tool
tool_id: int
/
[clinic start generated code]*/
static PyObject *
monitoring_get_tool_impl(PyObject *module, int tool_id)
/*[clinic end generated code: output=1c05a98b404a9a16 input=eeee9bebd0bcae9d]*/
/*[clinic end generated code]*/
{
if (check_valid_tool(tool_id)) {
return NULL;
}
PyInterpreterState *interp = _PyInterpreterState_GET();
PyObject *name = interp->monitoring_tool_names[tool_id];
if (name == NULL) {
Py_RETURN_NONE;
}
return Py_NewRef(name);
}
/*[clinic input]
monitoring.register_callback
tool_id: int
event: int
func: object
/
[clinic start generated code]*/
static PyObject *
monitoring_register_callback_impl(PyObject *module, int tool_id, int event,
PyObject *func)
/*[clinic end generated code: output=e64daa363004030c input=df6d70ea4cf81007]*/
{
if (check_valid_tool(tool_id)) {
return NULL;
}
if (_Py_popcount32(event) != 1) {
PyErr_SetString(PyExc_ValueError, "The callback can only be set for one event at a time");
return NULL;
}
int event_id = _Py_bit_length(event)-1;
if (event_id < 0 || event_id >= _PY_MONITORING_EVENTS) {
PyErr_Format(PyExc_ValueError, "invalid event %d", event);
return NULL;
}
if (PySys_Audit("sys.monitoring.register_callback", "O", func) < 0) {
return NULL;
}
if (func == Py_None) {
func = NULL;
}
func = _PyMonitoring_RegisterCallback(tool_id, event_id, func);
if (func == NULL) {
Py_RETURN_NONE;
}
return func;
}
/*[clinic input]
monitoring.get_events -> int
tool_id: int
/
[clinic start generated code]*/
static int
monitoring_get_events_impl(PyObject *module, int tool_id)
/*[clinic end generated code: output=4450cc13f826c8c0 input=a64b238f76c4b2f7]*/
{
if (check_valid_tool(tool_id)) {
return -1;
}
_Py_GlobalMonitors *m = &_PyInterpreterState_GET()->monitors;
_PyMonitoringEventSet event_set = get_events(m, tool_id);
return event_set;
}
/*[clinic input]
monitoring.set_events
tool_id: int
event_set: int
/
[clinic start generated code]*/
static PyObject *
monitoring_set_events_impl(PyObject *module, int tool_id, int event_set)
/*[clinic end generated code: output=1916c1e49cfb5bdb input=a77ba729a242142b]*/
{
if (check_valid_tool(tool_id)) {
return NULL;
}
if (event_set < 0 || event_set >= (1 tool_id) & 1) {
event_set |= (1 right) {
offset = src_offset/2 + 2;
other_event = PY_MONITORING_EVENT_BRANCH_LEFT;
assert(_Py_GetBaseCodeUnit(code, offset).op.code == NOT_TAKEN);
}
else {
offset = src_offset/2;
other_event = PY_MONITORING_EVENT_BRANCH_RIGHT;
}
}
else {
// Orphaned NOT_TAKEN -- Jump removed by the compiler
return res;
}
PyInterpreterState *interp = _PyInterpreterState_GET();
_PyEval_StopTheWorld(interp);
remove_tools(code, offset, other_event, 1 tool_id);
_PyEval_StartTheWorld(interp);
}
return res;
}
static PyObject *make_branch_handler(int tool_id, PyObject *handler, bool right)
{
_PyLegacyBranchEventHandler *callback =
PyObject_NEW(_PyLegacyBranchEventHandler, &_PyLegacyBranchEventHandler_Type);
if (callback == NULL) {
return NULL;
}
callback->vectorcall = branch_handler_vectorcall;
callback->handler = Py_NewRef(handler);
callback->right = right;
callback->tool_id = tool_id;
return (PyObject *)callback;
}
PyObject *
_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj)
{
assert(0 monitoring_callables[tool_id][PY_MONITORING_EVENT_BRANCH_RIGHT] = right;
res = interp->monitoring_callables[tool_id][PY_MONITORING_EVENT_BRANCH_LEFT];
interp->monitoring_callables[tool_id][PY_MONITORING_EVENT_BRANCH_LEFT] = left;
_PyEval_StartTheWorld(interp);
Py_XDECREF(old_right);
}
else {
PyInterpreterState *interp = _PyInterpreterState_GET();
_PyEval_StopTheWorld(interp);
res = interp->monitoring_callables[tool_id][event_id];
interp->monitoring_callables[tool_id][event_id] = Py_XNewRef(obj);
_PyEval_StartTheWorld(interp);
}
if (res != NULL && Py_TYPE(res) == &_PyLegacyBranchEventHandler_Type) {
_PyLegacyBranchEventHandler *wrapper = (_PyLegacyBranchEventHandler *)res;
res = Py_NewRef(wrapper->handler);
Py_DECREF(wrapper);
}
return res;
}
/* Branch Iterator */
typedef struct {
PyObject_HEAD
PyCodeObject *bi_code;
int bi_offset;
} branchesiterator;
#define branchesiterator_CAST(op) ((branchesiterator *)(op))
static PyObject *
int_triple(int a, int b, int c) {
PyObject *obja = PyLong_FromLong(a);
PyObject *objb = NULL;
PyObject *objc = NULL;
if (obja == NULL) {
goto error;
}
objb = PyLong_FromLong(b);
if (objb == NULL) {
goto error;
}
objc = PyLong_FromLong(c);
if (objc == NULL) {
goto error;
}
PyObject *array[3] = { obja, objb, objc };
return _PyTuple_FromArraySteal(array, 3);
error:
Py_XDECREF(obja);
Py_XDECREF(objb);
Py_XDECREF(objc);
return NULL;
}
static PyObject *
branchesiter_next(PyObject *op)
{
branchesiterator *bi = branchesiterator_CAST(op);
int offset = bi->bi_offset;
int oparg = 0;
while (offset < Py_SIZE(bi->bi_code)) {
_Py_CODEUNIT inst = _Py_GetBaseCodeUnit(bi->bi_code, offset);
int next_offset = offset + 1 + _PyOpcode_Caches[inst.op.code];
switch(inst.op.code) {
case EXTENDED_ARG:
oparg = (oparg bi_code, src_offset).op.code == END_SEND);
assert(_Py_GetBaseCodeUnit(bi->bi_code, src_offset+1).op.code == NOT_TAKEN);
not_taken = src_offset + 2;
return int_triple(src_offset *2, not_taken*2, next_offset*2);
default:
oparg = 0;
}
offset = next_offset;
}
return NULL;
}
static void
branchesiter_dealloc(PyObject *op)
{
branchesiterator *bi = branchesiterator_CAST(op);
Py_DECREF(bi->bi_code);
PyObject_Free(bi);
}
static PyTypeObject _PyBranchesIterator = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
"line_iterator", /* tp_name */
sizeof(branchesiterator), /* tp_basicsize */
0, /* tp_itemsize */
/* methods */
.tp_dealloc = branchesiter_dealloc,
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
.tp_iter = PyObject_SelfIter,
.tp_iternext = branchesiter_next,
.tp_free = PyObject_Del,
};
PyObject *
_PyInstrumentation_BranchesIterator(PyCodeObject *code)
{
branchesiterator *bi = (branchesiterator *)PyType_GenericAlloc(&_PyBranchesIterator, 0);
if (bi == NULL) {
return NULL;
}
bi->bi_code = (PyCodeObject*)Py_NewRef(code);
bi->bi_offset = 0;
return (PyObject *)bi;
}