FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

gh-91719: Add pycore_opcode.h internal header file by vstinner · Pull Request #91906 · python/cpython · GitHub

  1. python
  2. cpython

Repository navigation

gh-91719: Add pycore_opcode.h internal header file #91906

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vstinner merged 2 commits into from
Apr 25, 2022
Conversation 13 Commits 2 Checks 0 Files changed
Merged

gh-91719: Add pycore_opcode.h internal header file #91906

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d0af188
gh-91719: Add pycore_opcode.h internal header file
vstinner Apr 25, 2022
75a9bcf
Update PCbuild/regen.targets
vstinner Apr 25, 2022
File filter

Filter by extension

Filter by extension .c  (6) .h  (2) .in  (1) .py  (1) .targets  (1) All 5 file types selected
Viewed files
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Unified
Split
Hide whitespace
Diff view
Unified
Split
Hide whitespace
581 changes: 581 additions & 0 deletions Include/internal/pycore_opcode.h
Show comments
View file
Open in desktop

Large diffs are not rendered by default.

566 changes: 2 additions & 564 deletions Include/opcode.h
Show comments
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 3 additions & 1 deletion Makefile.pre.in
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -1311,8 +1311,10 @@ regen-opcode:
# using Tools/scripts/generate_opcode_h.py
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_opcode_h.py \
$(srcdir)/Lib/opcode.py \
$(srcdir)/Include/opcode.h.new
$(srcdir)/Include/opcode.h.new \
$(srcdir)/Include/internal/pycore_opcode.h.new
$(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new
$(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode.h $(srcdir)/Include/internal/pycore_opcode.h.new

.PHONY: regen-token
regen-token:
Expand Down
1 change: 1 addition & 0 deletions Objects/codeobject.c
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
#include "structmember.h" // PyMemberDef
#include "pycore_code.h" // _PyCodeConstructor
#include "pycore_interp.h" // PyInterpreterState.co_extra_freefuncs
#include "pycore_opcode.h" // _PyOpcode_Deopt
#include "pycore_pystate.h" // _PyInterpreterState_GET()
#include "pycore_tuple.h" // _PyTuple_ITEMS()
#include "clinic/codeobject.c.h"
Expand Down
5 changes: 3 additions & 2 deletions Objects/frameobject.c
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,11 @@

#include "Python.h"
#include "pycore_ceval.h" // _PyEval_BuiltinsFromGlobals()
#include "pycore_moduleobject.h" // _PyModule_GetDict()
#include "pycore_object.h" // _PyObject_GC_UNTRACK()
#include "pycore_code.h" // CO_FAST_LOCAL, etc.
#include "pycore_function.h" // _PyFunction_FromConstructor()
#include "pycore_moduleobject.h" // _PyModule_GetDict()
#include "pycore_object.h" // _PyObject_GC_UNTRACK()
#include "pycore_opcode.h" // _PyOpcode_Caches

#include "frameobject.h" // PyFrameObject
#include "pycore_frame.h"
Expand Down
4 changes: 2 additions & 2 deletions Objects/genobject.c
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,12 @@
#include "Python.h"
#include "pycore_call.h" // _PyObject_CallNoArgs()
#include "pycore_ceval.h" // _PyEval_EvalFrame()
#include "pycore_frame.h" // _PyInterpreterFrame
#include "pycore_genobject.h" // struct _Py_async_gen_state
#include "pycore_object.h" // _PyObject_GC_UNTRACK()
#include "pycore_opcode.h" // _PyOpcode_Deopt
#include "pycore_pyerrors.h" // _PyErr_ClearExcState()
#include "pycore_pystate.h" // _PyThreadState_GET()
#include "pycore_frame.h" // _PyInterpreterFrame
#include "frameobject.h" // PyFrameObject
#include "structmember.h" // PyMemberDef
#include "opcode.h" // SEND

Expand Down
4 changes: 2 additions & 2 deletions PCbuild/regen.targets
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
<Argument>-C</Argument>
</_ASTOutputs>
<_OpcodeSources Include="$(PySourcePath)Tools\scripts\generate_opcode_h.py;$(PySourcePath)Lib\opcode.py" />
<_OpcodeOutputs Include="$(PySourcePath)Include\opcode.h;$(PySourcePath)Python\opcode_targets.h" />
<_OpcodeOutputs Include="$(PySourcePath)Include\opcode.h;$(PySourcePath)Include\internal\pycore_opcode.h;$(PySourcePath)Python\opcode_targets.h" />
<_TokenSources Include="$(PySourcePath)Grammar\Tokens" />
<_TokenOutputs Include="$(PySourcePath)Doc\library\token-list.inc">
<Format>rst</Format>
Expand Down Expand Up @@ -59,7 +59,7 @@
Inputs="@(_OpcodeSources)" Outputs="@(_OpcodeOutputs)"
DependsOnTargets="FindPythonForBuild">
<Message Text="Regenerate @(_OpcodeOutputs->'%(Filename)%(Extension)',' ')" Importance="high" />
<Exec Command="$(PythonForBuild) Tools\scripts\generate_opcode_h.py Lib\opcode.py Include\opcode.h"
<Exec Command="$(PythonForBuild) Tools\scripts\generate_opcode_h.py Lib\opcode.py Include\opcode.h Include\internal\pycore_opcode.h"
WorkingDirectory="$(PySourcePath)" />
<Exec Command="$(PythonForBuild) Python\makeopcodetargets.py Python\opcode_targets.h"
WorkingDirectory="$(PySourcePath)" />
Expand Down
1 change: 1 addition & 0 deletions Python/ceval.c
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
#include "pycore_long.h" // _PyLong_GetZero()
#include "pycore_object.h" // _PyObject_GC_TRACK()
#include "pycore_moduleobject.h" // PyModuleObject
#include "pycore_opcode.h" // EXTRA_CASES

gvanrossum Apr 26, 2022

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Choose a reason Spam Abuse Off Topic Outdated Duplicate Resolved Low Quality

I don't know why you call out EXTRA_CASES in the comment -- this file uses several other tables defined there as well.

vstinner Apr 27, 2022

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Choose a reason Spam Abuse Off Topic Outdated Duplicate Resolved Low Quality

When I add an #include, I put the same of a single symbol get from this #include, even if many other symbols are used. I only do that to check time to time if the include is still used: here I just have check if EXTRA_CASES is still used in the file. Otherwise, for each #include, I have to remove the line, try to build, and check if the C file still builds successfully.

gvanrossum Apr 27, 2022

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Choose a reason Spam Abuse Off Topic Outdated Duplicate Resolved Low Quality

Would be nice if there was tooling that did this instead. Without the comment.
#include "pycore_pyerrors.h" // _PyErr_Fetch()
#include "pycore_pylifecycle.h" // _PyErr_Print()
#include "pycore_pymem.h" // _PyMem_IsPtrFreed()
Expand Down
11 changes: 6 additions & 5 deletions Python/compile.c
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,18 @@

#include <stdbool.h>

// Need _PyOpcode_RelativeJump of pycore_opcode.h
#define NEED_OPCODE_TABLES

#include "Python.h"
#include "pycore_ast.h" // _PyAST_GetDocString()
#include "pycore_compile.h" // _PyFuture_FromAST()
#include "pycore_code.h" // _PyCode_New()
#include "pycore_pymem.h" // _PyMem_IsPtrFreed()
#include "pycore_compile.h" // _PyFuture_FromAST()
#include "pycore_long.h" // _PyLong_GetZero()
#include "pycore_opcode.h" // _PyOpcode_Caches
#include "pycore_pymem.h" // _PyMem_IsPtrFreed()
#include "pycore_symtable.h" // PySTEntryObject

#define NEED_OPCODE_TABLES
#include "opcode.h" // EXTENDED_ARG


#define DEFAULT_BLOCK_SIZE 16
#define DEFAULT_CODE_SIZE 128
Expand Down
2 changes: 1 addition & 1 deletion Python/specialize.c
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
#include "pycore_long.h"
#include "pycore_moduleobject.h"
#include "pycore_object.h"
#include "opcode.h"
#include "pycore_opcode.h" // _PyOpcode_Caches
#include "structmember.h" // struct PyMemberDef, T_OFFSET_EX

#include <stdlib.h> // rand()
Expand Down
87 changes: 59 additions & 28 deletions Tools/scripts/generate_opcode_h.py
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,16 @@
import sys
import tokenize

header = """
/* Auto-generated by Tools/scripts/generate_opcode_h.py from Lib/opcode.py */
SCRIPT_NAME = "Tools/scripts/generate_opcode_h.py"
PYTHON_OPCODE = "Lib/opcode.py"

header = f"""
// Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE}

#ifndef Py_OPCODE_H
#define Py_OPCODE_H
#ifdef __cplusplus
extern "C" {
extern "C" {{
#endif


Expand All @@ -28,6 +32,29 @@
#endif /* !Py_OPCODE_H */
"""

internal_header = f"""
// Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE}

#ifndef Py_INTERNAL_OPCODE_H
#define Py_INTERNAL_OPCODE_H
#ifdef __cplusplus
extern "C" {{
#endif

#ifndef Py_BUILD_CORE
# error "this header requires Py_BUILD_CORE define"
#endif

#include "opcode.h"
""".lstrip()

internal_footer = """
#ifdef __cplusplus
}
#endif
#endif // !Py_INTERNAL_OPCODE_H
"""

DEFINE = "#define {:<38} {:>3}\n"

UINT32_MASK = (1<<32)-1
Expand All @@ -43,7 +70,7 @@ def write_int_array_from_ops(name, ops, out):
assert bits == 0
out.write(f"}};\n")

def main(opcode_py, outfile='Include/opcode.h'):
def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/internal/pycore_opcode.h'):
opcode = {}
if hasattr(tokenize, 'open'):
fp = tokenize.open(opcode_py) # Python 3.2+
Expand Down Expand Up @@ -75,8 +102,10 @@ def main(opcode_py, outfile='Include/opcode.h'):
opname_including_specialized[255] = 'DO_TRACING'
used[255] = True

with open(outfile, 'w') as fobj:
with (open(outfile, 'w') as fobj, open(internaloutfile, 'w') as iobj):
fobj.write(header)
iobj.write(internal_header)

for name in opname:
if name in opmap:
fobj.write(DEFINE.format(name, opmap[name]))
Expand All @@ -86,28 +115,29 @@ def main(opcode_py, outfile='Include/opcode.h'):
for name, op in specialized_opmap.items():
fobj.write(DEFINE.format(name, op))

fobj.write("\nextern const uint8_t _PyOpcode_Caches[256];\n")
fobj.write("\nextern const uint8_t _PyOpcode_Deopt[256];\n")
fobj.write("\n#ifdef NEED_OPCODE_TABLES\n")
write_int_array_from_ops("_PyOpcode_RelativeJump", opcode['hasjrel'], fobj)
write_int_array_from_ops("_PyOpcode_Jump", opcode['hasjrel'] + opcode['hasjabs'], fobj)
iobj.write("\nextern const uint8_t _PyOpcode_Caches[256];\n")
iobj.write("\nextern const uint8_t _PyOpcode_Deopt[256];\n")
iobj.write("\n#ifdef NEED_OPCODE_TABLES\n")
write_int_array_from_ops("_PyOpcode_RelativeJump", opcode['hasjrel'], iobj)
write_int_array_from_ops("_PyOpcode_Jump", opcode['hasjrel'] + opcode['hasjabs'], iobj)

fobj.write("\nconst uint8_t _PyOpcode_Caches[256] = {\n")
iobj.write("\nconst uint8_t _PyOpcode_Caches[256] = {\n")
for i, entries in enumerate(opcode["_inline_cache_entries"]):
if entries:
fobj.write(f" [{opname[i]}] = {entries},\n")
fobj.write("};\n")
iobj.write(f" [{opname[i]}] = {entries},\n")
iobj.write("};\n")

deoptcodes = {}
for basic in opmap:
deoptcodes[basic] = basic
for basic, family in opcode["_specializations"].items():
for specialized in family:
deoptcodes[specialized] = basic
fobj.write("\nconst uint8_t _PyOpcode_Deopt[256] = {\n")
iobj.write("\nconst uint8_t _PyOpcode_Deopt[256] = {\n")
for opt, deopt in sorted(deoptcodes.items()):
fobj.write(f" [{opt}] = {deopt},\n")
fobj.write("};\n")
fobj.write("#endif /* OPCODE_TABLES */\n")
iobj.write(f" [{opt}] = {deopt},\n")
iobj.write("};\n")
iobj.write("#endif // NEED_OPCODE_TABLES\n")

fobj.write("\n")
fobj.write("#define HAS_CONST(op) (false\\")
Expand All @@ -119,28 +149,29 @@ def main(opcode_py, outfile='Include/opcode.h'):
for i, (op, _) in enumerate(opcode["_nb_ops"]):
fobj.write(DEFINE.format(op, i))

fobj.write("\n")
fobj.write("#ifdef Py_DEBUG\n")
fobj.write("static const char *const _PyOpcode_OpName[256] = {\n")
iobj.write("\n")
iobj.write("#ifdef Py_DEBUG\n")
iobj.write("static const char *const _PyOpcode_OpName[256] = {\n")
for op, name in enumerate(opname_including_specialized):
if name[0] != "<":
op = name
fobj.write(f''' [{op}] = "{name}",\n''')
fobj.write("};\n")
fobj.write("#endif\n")
iobj.write(f''' [{op}] = "{name}",\n''')
iobj.write("};\n")
iobj.write("#endif\n")

fobj.write("\n")
fobj.write("#define EXTRA_CASES \\\n")
iobj.write("\n")
iobj.write("#define EXTRA_CASES \\\n")
for i, flag in enumerate(used):
if not flag:
fobj.write(f" case {i}: \\\n")
fobj.write(" ;\n")
iobj.write(f" case {i}: \\\n")
iobj.write(" ;\n")

fobj.write(footer)
iobj.write(internal_footer)


print(f"{outfile} regenerated from {opcode_py}")


if __name__ == '__main__':
main(sys.argv[1], sys.argv[2])
main(sys.argv[1], sys.argv[2], sys.argv[3])
Back | FazBrowse Home | New Git URL