#ifdef _Py_JIT
#include "Python.h"
#include "pycore_abstract.h"
#include "pycore_bitutils.h"
#include "pycore_call.h"
#include "pycore_ceval.h"
#include "pycore_critical_section.h"
#include "pycore_dict.h"
#include "pycore_floatobject.h"
#include "pycore_frame.h"
#include "pycore_function.h"
#include "pycore_genobject.h"
#include "pycore_import.h"
#include "pycore_interpframe.h"
#include "pycore_interpolation.h"
#include "pycore_intrinsics.h"
#include "pycore_jit_publish.h"
#include "pycore_lazyimportobject.h"
#include "pycore_list.h"
#include "pycore_long.h"
#include "pycore_mmap.h"
#include "pycore_opcode_metadata.h"
#include "pycore_opcode_utils.h"
#include "pycore_optimizer.h"
#include "pycore_pyerrors.h"
#include "pycore_setobject.h"
#include "pycore_sliceobject.h"
#include "pycore_template.h"
#include "pycore_tuple.h"
#include "pycore_unicodeobject.h"
#include "pycore_jit.h"
// Memory management stuff: ///////////////////////////////////////////////////
#ifndef MS_WINDOWS
#include
#endif
static size_t
get_page_size(void)
{
#ifdef MS_WINDOWS
SYSTEM_INFO si;
GetSystemInfo(&si);
return si.dwPageSize;
#else
return sysconf(_SC_PAGESIZE);
#endif
}
static void
jit_error(const char *message)
{
#ifdef MS_WINDOWS
int hint = GetLastError();
#else
int hint = errno;
#endif
PyErr_Format(PyExc_RuntimeWarning, "JIT %s (%d)", message, hint);
}
static int
address_in_executor_array(_PyExecutorObject **ptrs, size_t count, uintptr_t addr)
{
for (size_t i = 0; i < count; i++) {
_PyExecutorObject *exec = ptrs[i];
if (exec->jit_code == NULL || exec->jit_size == 0) {
continue;
}
uintptr_t start = (uintptr_t)exec->jit_code;
uintptr_t end = start + exec->jit_size;
if (addr >= start && addr < end) {
return 1;
}
}
return 0;
}
static int
address_in_executor_list(_PyExecutorObject *head, uintptr_t addr)
{
for (_PyExecutorObject *exec = head;
exec != NULL;
exec = exec->vm_data.links.next)
{
if (exec->jit_code == NULL || exec->jit_size == 0) {
continue;
}
uintptr_t start = (uintptr_t)exec->jit_code;
uintptr_t end = start + exec->jit_size;
if (addr >= start && addr < end) {
return 1;
}
}
return 0;
}
PyAPI_FUNC(int)
_PyJIT_AddressInJitCode(PyInterpreterState *interp, uintptr_t addr)
{
if (interp == NULL) {
return 0;
}
if (address_in_executor_array(interp->executor_ptrs, interp->executor_count, addr)) {
return 1;
}
if (address_in_executor_list(interp->executor_deletion_list_head, addr)) {
return 1;
}
return 0;
}
static unsigned char *
jit_alloc(size_t size)
{
if (size > PY_MAX_JIT_CODE_SIZE) {
jit_error("code too big; refactor bytecodes.c to keep uop size down, or reduce maximum trace length.");
return NULL;
}
assert(size);
assert(size % get_page_size() == 0);
#ifdef MS_WINDOWS
int flags = MEM_COMMIT | MEM_RESERVE;
unsigned char *memory = VirtualAlloc(NULL, size, flags, PAGE_READWRITE);
int failed = memory == NULL;
#else
int flags = MAP_ANONYMOUS | MAP_PRIVATE;
int prot = PROT_READ | PROT_WRITE;
unsigned char *memory = mmap(NULL, size, prot, flags, -1, 0);
int failed = memory == MAP_FAILED;
if (!failed) {
(void)_PyAnnotateMemoryMap(memory, size, "cpython:jit");
}
#endif
if (failed) {
jit_error("unable to allocate memory");
return NULL;
}
return memory;
}
static int
jit_free(unsigned char *memory, size_t size)
{
assert(size);
assert(size % get_page_size() == 0);
#ifdef MS_WINDOWS
int failed = !VirtualFree(memory, 0, MEM_RELEASE);
#else
int failed = munmap(memory, size);
#endif
if (failed) {
jit_error("unable to free memory");
return -1;
}
OPT_STAT_ADD(jit_freed_memory_size, size);
return 0;
}
static int
mark_executable(unsigned char *memory, size_t size)
{
if (size == 0) {
return 0;
}
assert(size % get_page_size() == 0);
// Do NOT ever leave the memory writable! Also, don't forget to flush the
// i-cache (I cannot begin to tell you how horrible that is to debug):
#ifdef MS_WINDOWS
if (!FlushInstructionCache(GetCurrentProcess(), memory, size)) {
jit_error("unable to flush instruction cache");
return -1;
}
DWORD old;
int failed = !VirtualProtect(memory, size, PAGE_EXECUTE_READ, &old);
#else
__builtin___clear_cache((char *)memory, (char *)memory + size);
int failed = mprotect(memory, size, PROT_EXEC | PROT_READ);
#endif
if (failed) {
jit_error("unable to protect executable memory");
return -1;
}
return 0;
}
// JIT compiler stuff: /////////////////////////////////////////////////////////
#define GOT_SLOT_SIZE sizeof(uintptr_t)
#define SYMBOL_MASK_WORDS 8
typedef uint32_t symbol_mask[SYMBOL_MASK_WORDS];
typedef struct {
unsigned char *mem;
symbol_mask mask;
size_t size;
} symbol_state;
typedef struct {
symbol_state trampolines;
symbol_state got_symbols;
uintptr_t instruction_starts[UOP_MAX_TRACE_LENGTH];
} jit_state;
// Warning! AArch64 requires you to get your hands dirty. These are your gloves:
// value[value_start : value_start + len]
static uint32_t
get_bits(uint64_t value, uint8_t value_start, uint8_t width)
{
assert(width > value_start) & ((1ULL mem + index * size;
assert((size_t)(index + 1) * size size);
return slot;
}
// Return the address of the GOT slot for the requested symbol ordinal.
static uintptr_t
got_symbol_address(int ordinal, jit_state *state)
{
return (uintptr_t)get_symbol_slot(ordinal, &state->got_symbols, GOT_SLOT_SIZE);
}
// Many of these patches are "relaxing", meaning that they can rewrite the
// code they're patching to be more efficient (like turning a 64-bit memory
// load into a 32-bit immediate load). These patches have an "x" in their name.
// Relative patches have an "r" in their name.
// 32-bit absolute address.
void
patch_32(unsigned char *location, uint64_t value)
{
// Check that we're not out of range of 32 unsigned bits:
assert(value < (1ULL = -(1 = -(1LL nop; call XXX
loc8[-2] = 0x90;
loc8[-1] = 0xE8;
value = relaxed;
}
else if (loc8[-2] == 0xFF && loc8[-1] == 0x25) {
// jmp qword ptr [rip + AAA] -> nop; jmp XXX
loc8[-2] = 0x90;
loc8[-1] = 0xE9;
value = relaxed;
}
}
patch_32r(location, value);
}
void patch_got_symbol(jit_state *state, int ordinal);
void patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *state);
void patch_x86_64_trampoline(unsigned char *location, int ordinal, jit_state *state);
#include "jit_stencils.h"
#if defined(__aarch64__) || defined(_M_ARM64)
#define TRAMPOLINE_SIZE 16
#define DATA_ALIGN 8
#elif defined(__x86_64__) && defined(__APPLE__)
// LLVM 20 on macOS x86_64 debug builds: GOT entries may exceed 2GB PC-relative
// range.
#define TRAMPOLINE_SIZE 16 // 14 bytes + 2 bytes padding for alignment
#define DATA_ALIGN 8
#else
#define TRAMPOLINE_SIZE 0
#define DATA_ALIGN 1
#endif
// Populate the GOT entry for the given symbol ordinal with its resolved address.
void
patch_got_symbol(jit_state *state, int ordinal)
{
uint64_t value = (uintptr_t)symbols_map[ordinal];
unsigned char *location = (unsigned char *)get_symbol_slot(ordinal, &state->got_symbols, GOT_SLOT_SIZE);
patch_64(location, value);
}
// Generate and patch AArch64 trampolines. The symbols to jump to are stored
// in the jit_stencils.h in the symbols_map.
void
patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *state)
{
uint64_t value = (uintptr_t)symbols_map[ordinal];
int64_t range = value - (uintptr_t)location;
// If we are in range of 28 signed bits, we patch the instruction with
// the address of the symbol.
if (range >= -(1 > 32;
patch_aarch64_26r(location, (uintptr_t)p);
}
// Generate and patch x86_64 trampolines.
void
patch_x86_64_trampoline(unsigned char *location, int ordinal, jit_state *state)
{
uint64_t value = (uintptr_t)symbols_map[ordinal];
int64_t range = (int64_t)value - 4 - (int64_t)location;
// If we are in range of 32 signed bits, we can patch directly
if (range >= -(1LL jit_code;
size_t size = executor->jit_size;
if (memory) {
executor->jit_code = NULL;
executor->jit_size = 0;
_PyJit_UnregisterCode(executor->jit_registration);
executor->jit_registration = NULL;
if (jit_free(memory, size)) {
PyErr_FormatUnraisable("Exception ignored while "
"freeing JIT memory");
}
}
}
// Avoid excessive bloat due to asserts in stencils
int
_Py_jit_assertion_failure(int line)
{
printf("Assertion failure at line %d of executor_cases.c.h", line);
fflush(stdout);
abort();
return 0;
}
#endif // _Py_JIT