FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

bpo-36287: Make ast.dump() not output optional fields and attributes. by serhiy-storchaka · Pull Request #18843 · python/cpython · GitHub

  1. python
  2. cpython

Repository navigation

bpo-36287: Make ast.dump() not output optional fields and attributes. #18843

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
serhiy-storchaka merged 9 commits into from
Mar 9, 2020
Conversation 14 Commits 9 Checks 0 Files changed
Merged

bpo-36287: Make ast.dump() not output optional fields and attributes. #18843

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
380ea25
bpo-36287: Make ast.dump() not output optional fields and attributes.
serhiy-storchaka Mar 8, 2020
61c2fbd
Update examples in docs.
serhiy-storchaka Mar 8, 2020
049ab20
Remove commented out code.
serhiy-storchaka Mar 8, 2020
cb613cb
Add an empty line.
serhiy-storchaka Mar 8, 2020
cd96837
Remove commented out lines.
serhiy-storchaka Mar 8, 2020
0fa4d0e
Merge branch 'master' into ast-optional
serhiy-storchaka Mar 9, 2020
3a58208
Apply review suggestions.
serhiy-storchaka Mar 9, 2020
d17cfd7
Refactor.
serhiy-storchaka Mar 9, 2020
2b456bd
Merge branch 'master' into ast-optional
serhiy-storchaka Mar 9, 2020
File filter

Filter by extension

Filter by extension .c  (1) .py  (3) .rst  (1) All 3 file types selected
Viewed files
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Unified
Split
Hide whitespace
Diff view
Unified
Split
Hide whitespace
Next Next commit
bpo-36287: Make ast.dump() not output optional fields and attributes. 
Values for optional fields and attributes of AST nodes are now set as
class attributes (e.g. Constant.kind is set to None).
  • Loading branch information
serhiy-storchaka committed Mar 8, 2020
commit 380ea25e2419580c856cacedf6bcb11142575543
37 changes: 21 additions & 16 deletions Lib/ast.py
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -131,23 +131,27 @@ def _format(node, level=0):
value = getattr(node, field)
except AttributeError:
keywords = True
continue
if value is None and getattr(type(node), field, ...) is None:
keywords = True
continue
value, simple = _format(value, level)
allsimple = allsimple and simple
if keywords:
args.append('%s=%s' % (field, value))
else:
value, simple = _format(value, level)
allsimple = allsimple and simple
if keywords:
args.append('%s=%s' % (field, value))
else:
args.append(value)
args.append(value)
if include_attributes and node._attributes:
for attr in node._attributes:
try:
value = getattr(node, attr)
except AttributeError:
pass
else:
value, simple = _format(value, level)
allsimple = allsimple and simple
args.append('%s=%s' % (attr, value))
continue
if value is None and getattr(type(node), attr, ...) is None:
continue
value, simple = _format(value, level)
allsimple = allsimple and simple
args.append('%s=%s' % (attr, value))
if allsimple and len(args) <= 3:
return '%s(%s)' % (node.__class__.__name__, ', '.join(args)), not args
return '%s(%s%s)' % (node.__class__.__name__, prefix, sep.join(args)), False
Expand All @@ -170,9 +174,10 @@ def copy_location(new_node, old_node):
attributes) from *old_node* to *new_node* if possible, and return *new_node*.
"""
for attr in 'lineno', 'col_offset', 'end_lineno', 'end_col_offset':
if attr in old_node._attributes and attr in new_node._attributes \
and hasattr(old_node, attr):
setattr(new_node, attr, getattr(old_node, attr))
if attr in old_node._attributes and attr in new_node._attributes:
value = getattr(old_node, attr, None)
if value is not None:
setattr(new_node, attr, value)
return new_node


Expand All @@ -191,7 +196,7 @@ def _fix(node, lineno, col_offset, end_lineno, end_col_offset):
else:
lineno = node.lineno
if 'end_lineno' in node._attributes:
if not hasattr(node, 'end_lineno'):
if getattr(node, 'end_lineno', None) is None:
node.end_lineno = end_lineno
else:
end_lineno = node.end_lineno
Expand All @@ -201,7 +206,7 @@ def _fix(node, lineno, col_offset, end_lineno, end_col_offset):
else:
col_offset = node.col_offset
if 'end_col_offset' in node._attributes:
if not hasattr(node, 'end_col_offset'):
if getattr(node, 'end_col_offset', None) is None:
node.end_col_offset = end_col_offset
else:
end_col_offset = node.end_col_offset
Expand Down
35 changes: 21 additions & 14 deletions Lib/test/test_ast.py
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -353,9 +353,11 @@ def test_arguments(self):
'kw_defaults', 'kwarg', 'defaults'))

with self.assertRaises(AttributeError):
x.vararg
x.args
self.assertIsNone(x.vararg)

x = ast.arguments(*range(1, 8))
self.assertEqual(x.args, 2)
self.assertEqual(x.vararg, 3)

def test_field_attr_writable(self):
Expand Down Expand Up @@ -650,18 +652,18 @@ def test_dump(self):
node = ast.parse('spam(eggs, "and cheese")')
self.assertEqual(ast.dump(node),
"Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), "
"args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese', kind=None)], "
"args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')], "
"keywords=[]))], type_ignores=[])"
)
self.assertEqual(ast.dump(node, annotate_fields=False),
"Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), "
"Constant('and cheese', None)], []))], [])"
"Constant('and cheese')], []))], [])"
)
self.assertEqual(ast.dump(node, include_attributes=True),
"Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), "
"lineno=1, col_offset=0, end_lineno=1, end_col_offset=4), "
"args=[Name(id='eggs', ctx=Load(), lineno=1, col_offset=5, "
"end_lineno=1, end_col_offset=9), Constant(value='and cheese', kind=None, "
"end_lineno=1, end_col_offset=9), Constant(value='and cheese', "
"lineno=1, col_offset=11, end_lineno=1, end_col_offset=23)], keywords=[], "
"lineno=1, col_offset=0, end_lineno=1, end_col_offset=24), "
"lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)], type_ignores=[])"
Expand All @@ -677,7 +679,7 @@ def test_dump_indent(self):
func=Name(id='spam', ctx=Load()),
args=[
Name(id='eggs', ctx=Load()),
Constant(value='and cheese', kind=None)],
Constant(value='and cheese')],
keywords=[]))],
type_ignores=[])""")
self.assertEqual(ast.dump(node, annotate_fields=False, indent='\t'), """\
Expand All @@ -688,7 +690,7 @@ def test_dump_indent(self):
\t\t\t\tName('spam', Load()),
\t\t\t\t[
\t\t\t\t\tName('eggs', Load()),
\t\t\t\t\tConstant('and cheese', None)],
\t\t\t\t\tConstant('and cheese')],
\t\t\t\t[]))],
\t[])""")
self.assertEqual(ast.dump(node, include_attributes=True, indent=3), """\
Expand All @@ -713,7 +715,6 @@ def test_dump_indent(self):
end_col_offset=9),
Constant(
value='and cheese',
kind=None,
lineno=1,
col_offset=11,
end_lineno=1,
Expand Down Expand Up @@ -762,7 +763,7 @@ def test_copy_location(self):
src = ast.parse('1 + 1', mode='eval')
src.body.right = ast.copy_location(ast.Num(2), src.body.right)
self.assertEqual(ast.dump(src, include_attributes=True),
'Expression(body=BinOp(left=Constant(value=1, kind=None, lineno=1, col_offset=0, '
'Expression(body=BinOp(left=Constant(value=1, lineno=1, col_offset=0, '
'end_lineno=1, end_col_offset=1), op=Add(), right=Constant(value=2, '
'lineno=1, col_offset=4, end_lineno=1, end_col_offset=5), lineno=1, '
'col_offset=0, end_lineno=1, end_col_offset=5))'
Expand All @@ -777,7 +778,7 @@ def test_fix_missing_locations(self):
self.assertEqual(ast.dump(src, include_attributes=True),
"Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), "
"lineno=1, col_offset=0, end_lineno=1, end_col_offset=5), "
"args=[Constant(value='spam', kind=None, lineno=1, col_offset=6, end_lineno=1, "
"args=[Constant(value='spam', lineno=1, col_offset=6, end_lineno=1, "
"end_col_offset=12)], keywords=[], lineno=1, col_offset=0, end_lineno=1, "
"end_col_offset=13), lineno=1, col_offset=0, end_lineno=1, "
"end_col_offset=13), Expr(value=Call(func=Name(id='spam', ctx=Load(), "
Expand All @@ -787,22 +788,28 @@ def test_fix_missing_locations(self):
"end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)], "
"type_ignores=[])"
)
#Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), lineno=1, col_offset=0, end_lineno=1, end_col_offset=5), args=[Constant(value='spam', lineno=1, col_offset=6, end_lineno=1, end_col_offset=12)], keywords=[], lineno=1, col_offset=0, end_lineno=1, end_col_offset=13), lineno=1, col_offset=0, end_lineno=1, end_col_offset=13), Expr(value=Call(
#func=Name(id='spam', ctx=Load(), lineno=1, col_offset=0),
#args=[Constant(value='eggs', lineno=1, col_offset=0)], keywords=[], lineno=1, col_offset=0), lineno=1, col_offset=0)], type_ignores=[])
#Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), lineno=1, col_offset=0, end_lineno=1, end_col_offset=5), args=[Constant(value='spam', lineno=1, col_offset=6, end_lineno=1, end_col_offset=12)], keywords=[], lineno=1, col_offset=0, end_lineno=1, end_col_offset=13), lineno=1, col_offset=0, end_lineno=1, end_col_offset=13), Expr(value=Call(
#func=Name(id='spam', ctx=Load(), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0),
#args=[Constant(value='eggs', lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)], keywords=[], lineno=1, col_offset=0, end_lineno=1, end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)], type_ignores=[])

def test_increment_lineno(self):
src = ast.parse('1 + 1', mode='eval')
self.assertEqual(ast.increment_lineno(src, n=3), src)
self.assertEqual(ast.dump(src, include_attributes=True),
'Expression(body=BinOp(left=Constant(value=1, kind=None, lineno=4, col_offset=0, '
'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, kind=None, '
'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, '
'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, '
'lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, '
'col_offset=0, end_lineno=4, end_col_offset=5))'
)
# issue10869: do not increment lineno of root twice
src = ast.parse('1 + 1', mode='eval')
self.assertEqual(ast.increment_lineno(src.body, n=3), src.body)
self.assertEqual(ast.dump(src, include_attributes=True),
'Expression(body=BinOp(left=Constant(value=1, kind=None, lineno=4, col_offset=0, '
'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, kind=None, '
'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, '
'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, '
'lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, '
'col_offset=0, end_lineno=4, end_col_offset=5))'
)
Expand All @@ -821,7 +828,7 @@ def test_iter_child_nodes(self):
self.assertEqual(next(iterator).value, 23)
self.assertEqual(next(iterator).value, 42)
self.assertEqual(ast.dump(next(iterator)),
"keyword(arg='eggs', value=Constant(value='leek', kind=None))"
"keyword(arg='eggs', value=Constant(value='leek'))"
)

def test_get_docstring(self):
Expand Down
3 changes: 3 additions & 0 deletions Misc/NEWS.d/next/Library/2020-03-08-09-53-55.bpo-36287.mxr5m8.rst
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
:func:`ast.dump()` no longer output optional fields and attributes. Values
Comment thread
serhiy-storchaka marked this conversation as resolved.
Outdated
for optional fields and attributes of AST nodes are now set as class
attributes (e.g. ``Constant.kind`` is set to ``None``).
13 changes: 13 additions & 0 deletions Parser/asdl_c.py
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -957,6 +957,8 @@ def visitProduct(self, prod, name):
(name, name, len(prod.attributes)), 1)
else:
self.emit("if (!add_attributes(state->%s_type, NULL, 0)) return 0;" % name, 1)
self.emit_defaults(name, prod.fields, 1)
self.emit_defaults(name, prod.attributes, 1)

def visitSum(self, sum, name):
self.emit('state->%s_type = make_type("%s", state->AST_type, NULL, 0);' %
Expand All @@ -968,6 +970,7 @@ def visitSum(self, sum, name):
(name, name, len(sum.attributes)), 1)
else:
self.emit("if (!add_attributes(state->%s_type, NULL, 0)) return 0;" % name, 1)
self.emit_defaults(name, sum.attributes, 1)
simple = is_simple(sum)
for t in sum.types:
self.visitConstructor(t, name, simple)
Expand All @@ -981,12 +984,22 @@ def visitConstructor(self, cons, name, simple):
(cons.name, cons.name, name, fields, len(cons.fields)), 1)
self.emit("if (!state->%s_type) return 0;" % cons.name, 1)
self.emit_type("%s_type" % cons.name)
self.emit_defaults(cons.name, cons.fields, 1)
if simple:
self.emit("state->%s_singleton = PyType_GenericNew((PyTypeObject *)"
"state->%s_type, NULL, NULL);" %
(cons.name, cons.name), 1)
self.emit("if (!state->%s_singleton) return 0;" % cons.name, 1)

def emit_defaults(self, name, fields, depth):
for field in fields:
if field.opt:
self.emit('if (PyObject_SetAttr(state->%s_type, state->%s, Py_None) == -1)' %
(name, field.name), depth)
#self.emit('if (PyObject_SetAttrString(state->%s_type, "%s", Py_None) == -1)' %
#(name, field.name), depth)
self.emit("return 0;", depth+1)


class ASTModuleVisitor(PickleVisitor):

Expand Down
Loading
Back | FazBrowse Home | New Git URL