GitHub Viewer
import glob
import os.path
import re
import subprocess
EXCLUDE_HEADERS = frozenset((
# Don't parse pthread_stubs.h: special header file used by WASM
'pthread_stubs.h',
# Don't parse dynamic_annotations.h: not included by Python.h.
'dynamic_annotations.h',
# Skip Include/pystats.h: the code is skipped unless if Python
# is built with --enable-pystats (if the Py_STATS macro is defined)
'pystats.h',
))
# Checkout of Python Git repository
CPYTHON_URL = 'https://github.com/python/cpython'
GIT_DIR = os.path.normpath(os.path.join(os.path.dirname(__file__), '..', 'cpython_git'))
PATH_LIMITED_API = 'Include'
PATH_CPYTHON_API = os.path.join('Include', 'cpython')
PATH_INTERNAL_API = os.path.join('Include', 'internal')
POSIXMODULE_H = 'Modules/posixmodule.h'
RE_IDENTIFIER = r'[A-Za-z_][A-Za-z0-9_]*'
RE_STRUCT_START = re.compile(r'^(?:typedef +)?struct(?: +([A-Za-z0-9_]+))? *{', re.MULTILINE)
RE_STRUCT_END = re.compile(r'^}(?: +([A-Za-z0-9_]+))? *;', re.MULTILINE)
TYPEDEFS = {
'_object': 'PyObject',
'_longobject': 'PyLongObject',
'_typeobject': 'PyTypeObject',
'PyCodeObject': 'PyCodeObject',
'_frame': 'PyFrameObject',
'_ts': 'PyThreadState',
'_is': 'PyInterpreterState',
'_xid': '_PyCrossInterpreterData',
'_traceback': 'PyTracebackObject',
}
PUBLIC_NAME_PREFIX = ("Py", "PY")
def run_command(cmd, cwd):
subprocess.run(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
check=True,
cwd=cwd)
def git_clone():
if os.path.exists(GIT_DIR):
return
print(f"Clone CPython Git repository: {CPYTHON_URL}")
dst_name = os.path.basename(GIT_DIR)
cmd = ['git', 'clone', CPYTHON_URL, dst_name]
run_command(cmd, cwd=os.path.dirname(GIT_DIR))
_CLEANED = False
_FETCHED = False
def git_switch_branch(branch):
git_clone()
global _CLEANED
if not _CLEANED:
cmd = ['git', 'clean', '-fdx']
run_command(cmd, cwd=GIT_DIR)
cmd = ['git', 'checkout', '.']
run_command(cmd, cwd=GIT_DIR)
_CLEANED = True
if branch == 'main':
cmd = ['git', 'switch', branch]
run_command(cmd, cwd=GIT_DIR)
global _FETCHED
if not _FETCHED:
print(f"Update the CPython Git repository (git fetch)")
cmd = ['git', 'fetch']
run_command(cmd, cwd=GIT_DIR)
_FETCHED = True
cmd = ['git', 'merge', '--ff']
run_command(cmd, cwd=GIT_DIR)
else:
cmd = ['git', 'checkout', branch]
run_command(cmd, cwd=GIT_DIR)
def list_files(path):
if not os.path.exists(path):
return []
files = glob.glob(os.path.join(path, '*.h'))
if path == PATH_INTERNAL_API:
files.append(POSIXMODULE_H)
files = [name for name in files
if os.path.basename(name) not in EXCLUDE_HEADERS]
return files
def _get_types(filename, names):
with open(filename, encoding="utf-8") as fp:
content = fp.read()
for match in RE_STRUCT_START.finditer(content):
struct_name = match.group(1)
match2 = RE_STRUCT_END.search(content, match.end())
if not match2:
raise Exception(f"{filename}: cannot find end of: {match.group()}")
name = match2.group(1)
if not name:
name = struct_name
if not name:
raise Exception(f"{filename}: structure has no name: {match.group()})")
if name in TYPEDEFS:
name = TYPEDEFS[name]
names.add(name)
if 'pthread_mutex_t' in names:
raise Exception('pthread_stubs.h was parsed')
def get_types_path(directory):
names = set()
for filename in list_files(directory):
_get_types(filename, names)
return sorted(names)
def get_types():
limited = get_types_path(PATH_LIMITED_API)
cpython = get_types_path(PATH_CPYTHON_API)
internal = get_types_path(PATH_INTERNAL_API)
return (limited, cpython, internal)
def grep(regex, filenames, group=0):
for filename in filenames:
with open(filename, encoding='utf-8') as fp:
content = fp.read()
for match in regex.finditer(content):
yield match.group(group)
def is_function_public(name):
return name.startswith(PUBLIC_NAME_PREFIX)
def get_macros_static_inline_funcs():
files = list_files(PATH_LIMITED_API) + list_files(PATH_CPYTHON_API)
# Match '#define func('
# Don't match '#define constant (&obj)': space before '('
regex = re.compile(fr'^ *# *define (_?P[Yy][A-Za-z_]+)\(', re.MULTILINE)
macros = set(grep(regex, files, group=1))
regex = re.compile(fr'^static inline [^(\n]+ ({RE_IDENTIFIER}) *\(', re.MULTILINE)
funcs = set(grep(regex, files, group=1))
# Remove macros only used to cast arguments types. Like:
# "static inline void Py_INCREF(...) { ...}"
# "#define Py_INCREF(obj) Py_INCREF(_PyObject_CAST(obj))"
# Only count the static inline function, ignore the macro.
macros = macros - funcs
# In Python 3.10, the Py_INCREF() was wrapping the _Py_INCREF() static
# inline function.
# In Python 3.11, Py_NewRef() macro just calls _Py_NewRef() static inline
# function.
for name in list(macros):
if f"_{name}" in funcs:
macros.discard(name)
# Remove PyDTrace_xxx functions
for name in list(funcs):
if name.startswith("PyDTrace_"):
funcs.discard(name)
# Remove private static inline functions
private_macros = set()
private_funcs = set()
for name in list(macros):
if not is_function_public(name):
macros.discard(name)
private_macros.add(name)
for name in list(funcs):
if not is_function_public(name):
funcs.discard(name)
private_funcs.add(name)
return (macros, funcs, private_macros, private_funcs)
def get_functions():
regex = re.compile(
# Ignore "#define PyAPI_FUNC(RTYPE) ..." (pyport.h)
fr'(?