| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
1 parent 47022a0 commit 10a91d7
10 files changed
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -2122,10 +2122,12 @@ Async and await | |||
| 2122 | 2122 | Apart from the node classes, the :mod:`ast` module defines these utility functions | |
| 2123 | 2123 | and classes for traversing abstract syntax trees: | |
| 2124 | 2124 | ||
| 2125 | - .. function:: parse(source, filename='<unknown>', mode='exec', *, type_comments=False, feature_version=None) | ||
| 2125 | + .. function:: parse(source, filename='<unknown>', mode='exec', *, type_comments=False, feature_version=None, optimize=-1) | ||
| 2126 | 2126 | ||
| 2127 | 2127 | Parse the source into an AST node. Equivalent to ``compile(source, | |
| 2128 | - filename, mode, ast.PyCF_ONLY_AST)``. | ||
| 2128 | + filename, mode, flags=FLAGS_VALUE, optimize=optimize)``, | ||
| 2129 | + where ``FLAGS_VALUE`` is ``ast.PyCF_ONLY_AST`` if ``optimize <= 0`` | ||
| 2130 | + and ``ast.PyCF_OPTIMIZED_AST`` otherwise. | ||
| 2129 | 2131 | ||
| 2130 | 2132 | If ``type_comments=True`` is given, the parser is modified to check | |
| 2131 | 2133 | and return type comments as specified by :pep:`484` and :pep:`526`. | |
@@ -2171,6 +2173,7 @@ and classes for traversing abstract syntax trees: | |||
| 2171 | 2173 | ||
| 2172 | 2174 | .. versionchanged:: 3.13 | |
| 2173 | 2175 | The minimum supported version for feature_version is now (3,7) | |
| 2176 | + The ``optimize`` argument was added. | ||
| 2174 | 2177 | ||
| 2175 | 2178 | ||
| 2176 | 2179 | .. function:: unparse(ast_obj) | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -85,6 +85,12 @@ Other Language Changes | |||
| 85 | 85 | This change will affect tools using docstrings, like :mod:`doctest`. | |
| 86 | 86 | (Contributed by Inada Naoki in :gh:`81283`.) | |
| 87 | 87 | ||
| 88 | + * The :func:`compile` built-in can now accept a new flag, | ||
| 89 | + ``ast.PyCF_OPTIMIZED_AST``, which is similar to ``ast.PyCF_ONLY_AST`` | ||
| 90 | + except that the returned ``AST`` is optimized according to the value | ||
| 91 | + of the ``optimize`` argument. | ||
| 92 | + (Contributed by Irit Katriel in :gh:`108113`). | ||
| 93 | + | ||
| 88 | 94 | New Modules | |
| 89 | 95 | =========== | |
| 90 | 96 | ||
@@ -94,6 +100,14 @@ New Modules | |||
| 94 | 100 | Improved Modules | |
| 95 | 101 | ================ | |
| 96 | 102 | ||
| 103 | + ast | ||
| 104 | + --- | ||
| 105 | + | ||
| 106 | + * :func:`ast.parse` now accepts an optional argument ``optimize`` | ||
| 107 | + which is passed on to the :func:`compile` built-in. This makes it | ||
| 108 | + possible to obtain an optimized ``AST``. | ||
| 109 | + (Contributed by Irit Katriel in :gh:`108113`). | ||
| 110 | + | ||
| 97 | 111 | array | |
| 98 | 112 | ----- | |
| 99 | 113 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -19,9 +19,10 @@ | |||
| 19 | 19 | #define PyCF_TYPE_COMMENTS 0x1000 | |
| 20 | 20 | #define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000 | |
| 21 | 21 | #define PyCF_ALLOW_INCOMPLETE_INPUT 0x4000 | |
| 22 | + #define PyCF_OPTIMIZED_AST (0x8000 | PyCF_ONLY_AST) | ||
| 22 | 23 | #define PyCF_COMPILE_MASK (PyCF_ONLY_AST | PyCF_ALLOW_TOP_LEVEL_AWAIT | \ | |
| 23 | 24 | PyCF_TYPE_COMMENTS | PyCF_DONT_IMPLY_DEDENT | \ | |
| 24 | - PyCF_ALLOW_INCOMPLETE_INPUT) | ||
| 25 | + PyCF_ALLOW_INCOMPLETE_INPUT | PyCF_OPTIMIZED_AST) | ||
| 25 | 26 | ||
| 26 | 27 | typedef struct { | |
| 27 | 28 | int cf_flags; /* bitmask of CO_xxx flags relevant to future */ | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -32,13 +32,15 @@ | |||
| 32 | 32 | ||
| 33 | 33 | ||
| 34 | 34 | def parse(source, filename='<unknown>', mode='exec', *, | |
| 35 | - type_comments=False, feature_version=None): | ||
| 35 | + type_comments=False, feature_version=None, optimize=-1): | ||
| 36 | 36 | """ | |
| 37 | 37 | Parse the source into an AST node. | |
| 38 | 38 | Equivalent to compile(source, filename, mode, PyCF_ONLY_AST). | |
| 39 | 39 | Pass type_comments=True to get back type comments where the syntax allows. | |
| 40 | 40 | """ | |
| 41 | 41 | flags = PyCF_ONLY_AST | |
| 42 | + if optimize > 0: | ||
| 43 | + flags |= PyCF_OPTIMIZED_AST | ||
| 42 | 44 | if type_comments: | |
| 43 | 45 | flags |= PyCF_TYPE_COMMENTS | |
| 44 | 46 | if feature_version is None: | |
@@ -50,7 +52,7 @@ def parse(source, filename='<unknown>', mode='exec', *, | |||
| 50 | 52 | feature_version = minor | |
| 51 | 53 | # Else it should be an int giving the minor version for 3.x. | |
| 52 | 54 | return compile(source, filename, mode, flags, | |
| 53 | - _feature_version=feature_version) | ||
| 55 | + _feature_version=feature_version, optimize=optimize) | ||
| 54 | 56 | ||
| 55 | 57 | ||
| 56 | 58 | def literal_eval(node_or_string): | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -357,6 +357,34 @@ def test_ast_validation(self): | |||
| 357 | 357 | tree = ast.parse(snippet) | |
| 358 | 358 | compile(tree, '<string>', 'exec') | |
| 359 | 359 | ||
| 360 | + def test_optimization_levels__debug__(self): | ||
| 361 | + cases = [(-1, '__debug__'), (0, '__debug__'), (1, False), (2, False)] | ||
| 362 | + for (optval, expected) in cases: | ||
| 363 | + with self.subTest(optval=optval, expected=expected): | ||
| 364 | + res = ast.parse("__debug__", optimize=optval) | ||
| 365 | + self.assertIsInstance(res.body[0], ast.Expr) | ||
| 366 | + if isinstance(expected, bool): | ||
| 367 | + self.assertIsInstance(res.body[0].value, ast.Constant) | ||
| 368 | + self.assertEqual(res.body[0].value.value, expected) | ||
| 369 | + else: | ||
| 370 | + self.assertIsInstance(res.body[0].value, ast.Name) | ||
| 371 | + self.assertEqual(res.body[0].value.id, expected) | ||
| 372 | + | ||
| 373 | + def test_optimization_levels_const_folding(self): | ||
| 374 | + folded = ('Expr', (1, 0, 1, 5), ('Constant', (1, 0, 1, 5), 3, None)) | ||
| 375 | + not_folded = ('Expr', (1, 0, 1, 5), | ||
| 376 | + ('BinOp', (1, 0, 1, 5), | ||
| 377 | + ('Constant', (1, 0, 1, 1), 1, None), | ||
| 378 | + ('Add',), | ||
| 379 | + ('Constant', (1, 4, 1, 5), 2, None))) | ||
| 380 | + | ||
| 381 | + cases = [(-1, not_folded), (0, not_folded), (1, folded), (2, folded)] | ||
| 382 | + for (optval, expected) in cases: | ||
| 383 | + with self.subTest(optval=optval): | ||
| 384 | + tree = ast.parse("1 + 2", optimize=optval) | ||
| 385 | + res = to_tuple(tree.body[0]) | ||
| 386 | + self.assertEqual(res, expected) | ||
| 387 | + | ||
| 360 | 388 | def test_invalid_position_information(self): | |
| 361 | 389 | invalid_linenos = [ | |
| 362 | 390 | (10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1) | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -369,16 +369,17 @@ def f(): """doc""" | |||
| 369 | 369 | (1, False, 'doc', False, False), | |
| 370 | 370 | (2, False, None, False, False)] | |
| 371 | 371 | for optval, *expected in values: | |
| 372 | + with self.subTest(optval=optval): | ||
| 372 | 373 | # test both direct compilation and compilation via AST | |
| 373 | - codeobjs = [] | ||
| 374 | - codeobjs.append(compile(codestr, "<test>", "exec", optimize=optval)) | ||
| 375 | - tree = ast.parse(codestr) | ||
| 376 | - codeobjs.append(compile(tree, "<test>", "exec", optimize=optval)) | ||
| 377 | - for code in codeobjs: | ||
| 378 | - ns = {} | ||
| 379 | - exec(code, ns) | ||
| 380 | - rv = ns['f']() | ||
| 381 | - self.assertEqual(rv, tuple(expected)) | ||
| 374 | + codeobjs = [] | ||
| 375 | + codeobjs.append(compile(codestr, "<test>", "exec", optimize=optval)) | ||
| 376 | + tree = ast.parse(codestr) | ||
| 377 | + codeobjs.append(compile(tree, "<test>", "exec", optimize=optval)) | ||
| 378 | + for code in codeobjs: | ||
| 379 | + ns = {} | ||
| 380 | + exec(code, ns) | ||
| 381 | + rv = ns['f']() | ||
| 382 | + self.assertEqual(rv, tuple(expected)) | ||
| 382 | 383 | ||
| 383 | 384 | def test_compile_top_level_await_no_coro(self): | |
| 384 | 385 | """Make sure top level non-await codes get the correct coroutine flags""" | |
@@ -517,6 +518,28 @@ def test_compile_async_generator(self): | |||
| 517 | 518 | exec(co, glob) | |
| 518 | 519 | self.assertEqual(type(glob['ticker']()), AsyncGeneratorType) | |
| 519 | 520 | ||
| 521 | + def test_compile_ast(self): | ||
| 522 | + args = ("a*(1+2)", "f.py", "exec") | ||
| 523 | + raw = compile(*args, flags = ast.PyCF_ONLY_AST).body[0] | ||
| 524 | + opt = compile(*args, flags = ast.PyCF_OPTIMIZED_AST).body[0] | ||
| 525 | + | ||
| 526 | + for tree in (raw, opt): | ||
| 527 | + self.assertIsInstance(tree.value, ast.BinOp) | ||
| 528 | + self.assertIsInstance(tree.value.op, ast.Mult) | ||
| 529 | + self.assertIsInstance(tree.value.left, ast.Name) | ||
| 530 | + self.assertEqual(tree.value.left.id, 'a') | ||
| 531 | + | ||
| 532 | + raw_right = raw.value.right # expect BinOp(1, '+', 2) | ||
| 533 | + self.assertIsInstance(raw_right, ast.BinOp) | ||
| 534 | + self.assertIsInstance(raw_right.left, ast.Constant) | ||
| 535 | + self.assertEqual(raw_right.left.value, 1) | ||
| 536 | + self.assertIsInstance(raw_right.right, ast.Constant) | ||
| 537 | + self.assertEqual(raw_right.right.value, 2) | ||
| 538 | + | ||
| 539 | + opt_right = opt.value.right # expect Constant(3) | ||
| 540 | + self.assertIsInstance(opt_right, ast.Constant) | ||
| 541 | + self.assertEqual(opt_right.value, 3) | ||
| 542 | + | ||
| 520 | 543 | def test_delattr(self): | |
| 521 | 544 | sys.spam = 1 | |
| 522 | 545 | delattr(sys, 'spam') | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -0,0 +1,8 @@ | |||
| 1 | + The :func:`compile` built-in can now accept a new flag, | ||
| 2 | + ``ast.PyCF_OPTIMIZED_AST``, which is similar to ``ast.PyCF_ONLY_AST`` | ||
| 3 | + except that the returned ``AST`` is optimized according to the value | ||
| 4 | + of the ``optimize`` argument. | ||
| 5 | + | ||
| 6 | + :func:`ast.parse` now accepts an optional argument ``optimize`` | ||
| 7 | + which is passed on to the :func:`compile` built-in. This makes it | ||
| 8 | + possible to obtain an optimized ``AST``. | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1208,6 +1208,9 @@ def visitModule(self, mod): | |||
| 1208 | 1208 | self.emit('if (PyModule_AddIntMacro(m, PyCF_TYPE_COMMENTS) < 0) {', 1) | |
| 1209 | 1209 | self.emit("return -1;", 2) | |
| 1210 | 1210 | self.emit('}', 1) | |
| 1211 | + self.emit('if (PyModule_AddIntMacro(m, PyCF_OPTIMIZED_AST) < 0) {', 1) | ||
| 1212 | + self.emit("return -1;", 2) | ||
| 1213 | + self.emit('}', 1) | ||
| 1211 | 1214 | for dfn in mod.dfns: | |
| 1212 | 1215 | self.visit(dfn) | |
| 1213 | 1216 | self.emit("return 0;", 1) | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -21,6 +21,7 @@ | |||
| 21 | 21 | #include "pycore_pyerrors.h" // _PyErr_GetRaisedException, _Py_Offer_Suggestions | |
| 22 | 22 | #include "pycore_pylifecycle.h" // _Py_UnhandledKeyboardInterrupt | |
| 23 | 23 | #include "pycore_pystate.h" // _PyInterpreterState_GET() | |
| 24 | + #include "pycore_symtable.h" // _PyFuture_FromAST() | ||
| 24 | 25 | #include "pycore_sysmodule.h" // _PySys_Audit() | |
| 25 | 26 | #include "pycore_traceback.h" // _PyTraceBack_Print_Indented() | |
| 26 | 27 | ||
@@ -1790,6 +1791,24 @@ run_pyc_file(FILE *fp, PyObject *globals, PyObject *locals, | |||
| 1790 | 1791 | return NULL; | |
| 1791 | 1792 | } | |
| 1792 | 1793 | ||
| 1794 | + static int | ||
| 1795 | + ast_optimize(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, | ||
| 1796 | + int optimize, PyArena *arena) | ||
| 1797 | + { | ||
| 1798 | + PyFutureFeatures future; | ||
| 1799 | + if (!_PyFuture_FromAST(mod, filename, &future)) { | ||
| 1800 | + return -1; | ||
| 1801 | + } | ||
| 1802 | + int flags = future.ff_features | cf->cf_flags; | ||
| 1803 | + if (optimize == -1) { | ||
| 1804 | + optimize = _Py_GetConfig()->optimization_level; | ||
| 1805 | + } | ||
| 1806 | + if (!_PyAST_Optimize(mod, arena, optimize, flags)) { | ||
| 1807 | + return -1; | ||
| 1808 | + } | ||
| 1809 | + return 0; | ||
| 1810 | + } | ||
| 1811 | + | ||
| 1793 | 1812 | PyObject * | |
| 1794 | 1813 | Py_CompileStringObject(const char *str, PyObject *filename, int start, | |
| 1795 | 1814 | PyCompilerFlags *flags, int optimize) | |
@@ -1806,6 +1825,12 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, | |||
| 1806 | 1825 | return NULL; | |
| 1807 | 1826 | } | |
| 1808 | 1827 | if (flags && (flags->cf_flags & PyCF_ONLY_AST)) { | |
| 1828 | + if ((flags->cf_flags & PyCF_OPTIMIZED_AST) == PyCF_OPTIMIZED_AST) { | ||
| 1829 | + if (ast_optimize(mod, filename, flags, optimize, arena) < 0) { | ||
| 1830 | + _PyArena_Free(arena); | ||
| 1831 | + return NULL; | ||
| 1832 | + } | ||
| 1833 | + } | ||
| 1809 | 1834 | PyObject *result = PyAST_mod2obj(mod); | |
| 1810 | 1835 | _PyArena_Free(arena); | |
| 1811 | 1836 | return result; | |
| Back | FazBrowse Home | New Git URL |
0 commit comments