#!/usr/bin/env uv run
# /// script
# dependencies = ["pycparser"]
# ///
"""
TypeOffset is a C# class that mirrors the in-memory layout of heap
allocated Python objects.
This script parses the Python C headers and outputs the TypeOffset
C# class.
Requirements:
- pycparser
- clang
"""
import os
import shutil
import sys
import sysconfig
import subprocess
from io import StringIO
from pathlib import Path
from pycparser import c_ast, c_parser
# rename some members from their C name when generating the C#
_typeoffset_member_renames = {
"ht_name": "name",
"ht_qualname": "qualname",
"getitem": "spec_cache_getitem",
}
def _check_output(*args, **kwargs):
return subprocess.check_output(*args, **kwargs, encoding="utf8")
class AstParser(object):
"""Walk an AST and determine the members of all structs"""
def __init__(self):
self._typedefs = {}
self._typedecls = {}
self._structs = {}
self._struct_stack = []
self._struct_members_stack = []
self._ptr_decl_depth = 0
self._struct_members = {}
self._decl_names = {}
def get_struct_members(self, name):
"""return a list of (name, type) of struct members"""
defs = self._typedefs.get(name)
if defs is None:
return None
node = self._get_leaf_node(defs)
name = node.name
if name is None:
name = defs.declname
return self._struct_members.get(name)
def visit(self, node):
if isinstance(node, c_ast.FileAST):
self.visit_ast(node)
elif isinstance(node, c_ast.Typedef):
self.visit_typedef(node)
elif isinstance(node, c_ast.TypeDecl):
self.visit_typedecl(node)
elif isinstance(node, c_ast.Struct):
self.visit_struct(node)
elif isinstance(node, c_ast.Decl):
self.visit_decl(node)
elif isinstance(node, c_ast.FuncDecl):
self.visit_funcdecl(node)
elif isinstance(node, c_ast.PtrDecl):
self.visit_ptrdecl(node)
elif isinstance(node, c_ast.IdentifierType):
self.visit_identifier(node)
elif isinstance(node, c_ast.Union):
self.visit_union(node)
def visit_ast(self, ast):
for _name, node in ast.children():
self.visit(node)
def visit_typedef(self, typedef):
self._typedefs[typedef.name] = typedef.type
self.visit(typedef.type)
def visit_typedecl(self, typedecl):
self._decl_names[typedecl.type] = typedecl.declname
self.visit(typedecl.type)
def visit_struct(self, struct):
if struct.decls:
self._structs[self._get_struct_name(struct)] = struct
# recurse into the struct
self._struct_stack.insert(0, struct)
for decl in struct.decls:
self._struct_members_stack.insert(0, decl.name)
self.visit(decl)
self._struct_members_stack.pop(0)
self._struct_stack.pop(0)
elif self._ptr_decl_depth or self._struct_members_stack:
# the struct is empty, but add it as a member to the current
# struct as the current member maybe a pointer to it.
self._add_struct_member(struct.name)
def visit_decl(self, decl):
self.visit(decl.type)
def visit_funcdecl(self, funcdecl):
self.visit(funcdecl.type)
def visit_ptrdecl(self, ptrdecl):
self._ptr_decl_depth += 1
self.visit(ptrdecl.type)
self._ptr_decl_depth -= 1
def visit_identifier(self, identifier):
type_name = " ".join(identifier.names)
self._add_struct_member(type_name)
def visit_union(self, union):
# Treat the field as if it was just the first declaration for now. This
# is not really correct, but handles the one case that is relevant for
# us right now (ob_refcnt being "split" in Python 3.12)
if self._struct_members_stack and union.decls:
decl = union.decls[0]
self._struct_members_stack.pop(0)
self._struct_members_stack.insert(0, decl.name)
self.visit(decl)
def _add_struct_member(self, type_name):
if not (self._struct_stack and self._struct_members_stack):
return
# add member to current struct
current_struct = self._struct_stack[0]
member_name = self._struct_members_stack[0]
struct_members = self._struct_members.setdefault(
self._get_struct_name(current_struct), []
)
# get the node associated with this type
node = None
if type_name in self._typedefs:
node = self._get_leaf_node(self._typedefs[type_name])
# If the struct was only declared when the typedef was created, its member
# information will not have been recorded and we have to look it up in the
# structs
if isinstance(node, c_ast.Struct) and node.decls is None:
if node.name in self._structs:
node = self._structs[node.name]
elif type_name in self._structs:
node = self._structs[type_name]
# If it's a struct (and not a pointer to a struct) expand
# it into the current struct definition
if not self._ptr_decl_depth and isinstance(node, c_ast.Struct):
for decl in node.decls or []:
self._struct_members_stack.insert(0, decl.name)
self.visit(decl)
self._struct_members_stack.pop(0)
else:
# otherwise add it as a single member
struct_members.append((member_name, type_name))
def _get_leaf_node(self, node):
if isinstance(node, c_ast.Typedef):
return self._get_leaf_node(node.type)
if isinstance(node, c_ast.TypeDecl):
return self._get_leaf_node(node.type)
return node
def _get_struct_name(self, node):
return node.name or self._decl_names.get(node) or "_struct_%d" % id(node)
class Writer(object):
def __init__(self):
self._stream = StringIO()
def append(self, indent=0, code=""):
self._stream.write("%s%s\n" % (indent * " ", code))
def extend(self, s):
self._stream.write(s)
def to_string(self):
return self._stream.getvalue()
def preprocess_python_headers(*, cc=None, include_py=None):
"""Return Python.h pre-processed, ready for parsing.
Requires clang.
"""
this_path = Path(__file__).parent
fake_libc_include = this_path / "fake_libc_include"
include_dirs = [fake_libc_include]
if cc is None:
cc = shutil.which("clang")
if cc is None:
cc = shutil.which("gcc")
if cc is None:
raise RuntimeError("No suitable C compiler found, need clang or gcc")
if include_py is None:
include_py = sysconfig.get_config_var("INCLUDEPY")
include_py = Path(include_py)
include_dirs.append(include_py)
include_args = [c for p in include_dirs for c in ["-I", str(p)]]
# fmt: off
defines = [
"-D", "__attribute__(x)=",
"-D", "__inline__=inline",
"-D", "__asm__=;#pragma asm",
"-D", "__int64=long long",
"-D", "_POSIX_THREADS",
]
if sys.platform == "win32":
defines.extend([
"-D", "__inline=inline",
"-D", "__ptr32=",
"-D", "__ptr64=",
"-D", "__declspec(x)=",
])
#fmt: on
if hasattr(sys, "abiflags"):
if "d" in sys.abiflags:
defines.extend(("-D", "PYTHON_WITH_PYDEBUG"))
if "u" in sys.abiflags:
defines.extend(("-D", "PYTHON_WITH_WIDE_UNICODE"))
python_h = include_py / "Python.h"
cmd = [cc, "-pthread"] + include_args + defines + ["-E", str(python_h)]
# normalize as the parser doesn't like windows line endings.
lines = []
for line in _check_output(cmd).splitlines():
if line.startswith("#"):
line = line.replace("\\", "/")
lines.append(line)
return "\n".join(lines)
def gen_interop_head(writer, version, abi_flags):
filename = os.path.basename(__file__)
class_definition = f"""
// Auto-generated by {filename}.
// DO NOT MODIFY BY HAND.
// Python {".".join(map(str, version[:2]))}: ABI flags: '{abi_flags}'
// ReSharper disable InconsistentNaming
// ReSharper disable IdentifierTypo
using System;
using System.Diagnostics.CodeAnalysis;
using System.Runtime.InteropServices;
using Python.Runtime.Native;
namespace Python.Runtime
{{"""
writer.extend(class_definition)
def gen_interop_tail(writer):
tail = """}
"""
writer.extend(tail)
def gen_heap_type_members(parser, writer, type_name):
"""Generate the TypeOffset C# class"""
members = parser.get_struct_members("PyHeapTypeObject")
class_definition = f"""
[SuppressMessage("Style", "IDE1006:Naming Styles",
Justification = "Following CPython",
Scope = "type")]
[StructLayout(LayoutKind.Sequential)]
internal class {type_name} : GeneratedTypeOffsets, ITypeOffsets
{{
public {type_name}() {{ }}
// Auto-generated from PyHeapTypeObject in Python.h
"""
# All the members are sizeof(void*) so we don't need to do any
# extra work to determine the size based on the type.
for name, _type in members:
name = _typeoffset_member_renames.get(name, name)
class_definition += " public int %s { get; private set; }\n" % name
class_definition += """ }
"""
writer.extend(class_definition)
def gen_structure_code(parser, writer, type_name, indent):
members = parser.get_struct_members(type_name)
if members is None:
return False
out = writer.append
out(indent, "[StructLayout(LayoutKind.Sequential)]")
out(indent, f"internal struct {type_name}")
out(indent, "{")
for name, _type in members:
out(indent + 1, f"public IntPtr {name};")
out(indent, "}")
out()
return True
def main(*, cc=None, include_py=None, version=None, out=None):
# preprocess Python.h and build the AST
python_h = preprocess_python_headers(cc=cc, include_py=include_py)
parser = c_parser.CParser()
ast = parser.parse(python_h)
# extract struct members from the AST
ast_parser = AstParser()
ast_parser.visit(ast)
writer = Writer()
if include_py and not version:
raise RuntimeError("If the include path is overridden, version must be "
"defined"
)
if version:
version = version.split('.')
else:
version = sys.version_info
# generate the C# code
abi_flags = getattr(sys, "abiflags", "").replace("m", "")
gen_interop_head(writer, version, abi_flags)
type_name = f"TypeOffset{version[0]}{version[1]}{abi_flags}"
gen_heap_type_members(ast_parser, writer, type_name)
gen_interop_tail(writer)
interop_cs = writer.to_string()
if not out or out == "-":
print(interop_cs)
else:
with open(out, "w") as fh:
fh.write(interop_cs)
if __name__ == "__main__":
import argparse
a = argparse.ArgumentParser("Interop file generator for Python.NET")
a.add_argument("--cc", help="C compiler to use, either clang or gcc")
a.add_argument("--include-py", help="Include path of Python")
a.add_argument("--version", help="Python version")
a.add_argument("--out", help="Output path", default="-")
args = a.parse_args()
sys.exit(main(
cc=args.cc,
include_py=args.include_py,
out=args.out,
version=args.version
))