FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

Make ELLIPSIS a separate token. This makes it a syntax error to write… · python/cpython@dde0028 · GitHub

  1. python
  2. cpython

Repository navigation

Commit dde0028

Browse files
birkenfeld
committed
Make ELLIPSIS a separate token. This makes it a syntax error to write ". . ." for Ellipsis.
1 parent 428f064 commit dde0028

9 files changed

Lines changed: 111 additions & 95 deletions

File tree

‎Grammar/Grammar‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ power: atom trailer* ['**' factor]
107107
atom: ('(' [yield_expr|testlist_gexp] ')' |
108108
'[' [listmaker] ']' |
109109
'{' [dictsetmaker] '}' |
110-
NAME | NUMBER | STRING+ | '.' '.' '.')
110+
NAME | NUMBER | STRING+ | '...')
111111
listmaker: test ( list_for | (',' test)* [','] )
112112
testlist_gexp: test ( gen_for | (',' test)* [','] )
113113
lambdef: 'lambda' [varargslist] ':' test

‎Include/token.h‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,10 +59,11 @@ extern "C" {
5959
#define DOUBLESLASHEQUAL 49
6060
#define AT 50
6161
#define RARROW 51
62+
#define ELLIPSIS 52
6263
/* Don't forget to update the table _PyParser_TokenNames in tokenizer.c! */
63-
#define OP 52
64-
#define ERRORTOKEN 53
65-
#define N_TOKENS 54
64+
#define OP 53
65+
#define ERRORTOKEN 54
66+
#define N_TOKENS 55
6667

6768
/* Special definitions for cooperation with parser */
6869

‎Lib/compiler/transformer.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ def __init__(self):
113113
token.LBRACE: self.atom_lbrace,
114114
token.NUMBER: self.atom_number,
115115
token.STRING: self.atom_string,
116-
token.DOT: self.atom_ellipsis,
116+
token.ELLIPSIS: self.atom_ellipsis,
117117
token.NAME: self.atom_name,
118118
}
119119
self.encoding = None

‎Lib/test/test_grammar.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,7 @@ def testStringLiterals(self):
121121
def testEllipsis(self):
122122
x = ...
123123
self.assert_(x is Ellipsis)
124+
self.assertRaises(SyntaxError, eval, ".. .")
124125

125126
class GrammarTests(unittest.TestCase):
126127

‎Lib/token.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,9 +61,10 @@
6161
DOUBLESLASHEQUAL = 49
6262
AT = 50
6363
RARROW = 51
64-
OP = 52
65-
ERRORTOKEN = 53
66-
N_TOKENS = 54
64+
ELLIPSIS = 52
65+
OP = 53
66+
ERRORTOKEN = 54
67+
N_TOKENS = 55
6768
NT_OFFSET = 256
6869
#--end constants--
6970

‎Lib/tokenize.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ def maybe(*choices): return group(*choices) + '?'
8383
r"~")
8484

8585
Bracket = '[][(){}]'
86-
Special = group(r'\r?\n', r'[:;.,@]')
86+
Special = group(r'\r?\n', r'\.\.\.', r'[:;.,@]')
8787
Funny = group(Operator, Bracket, Special)
8888

8989
PlainToken = group(Number, Funny, String, Name)
@@ -334,8 +334,8 @@ def generate_tokens(readline):
334334
spos, epos, pos = (lnum, start), (lnum, end), end
335335
token, initial = line[start:end], line[start]
336336

337-
if initial in numchars or \
338-
(initial == '.' and token != '.'): # ordinary number
337+
if (initial in numchars or # ordinary number
338+
(initial == '.' and token != '.' and token != '...')):
339339
yield (NUMBER, token, spos, epos, line)
340340
elif initial in '\r\n':
341341
yield (NL if parenlev > 0 else NEWLINE,

‎Parser/tokenizer.c‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ char *_PyParser_TokenNames[] = {
9393
"DOUBLESLASHEQUAL",
9494
"AT",
9595
"RARROW",
96+
"ELLIPSIS",
9697
/* This table must match the #defines in token.h! */
9798
"OP",
9899
"<ERRORTOKEN>",
@@ -1082,6 +1083,16 @@ PyToken_ThreeChars(int c1, int c2, int c3)
10821083
break;
10831084
}
10841085
break;
1086+
case '.':
1087+
switch (c2) {
1088+
case '.':
1089+
switch (c3) {
1090+
case '.':
1091+
return ELLIPSIS;
1092+
}
1093+
break;
1094+
}
1095+
break;
10851096
}
10861097
return OP;
10871098
}
@@ -1278,13 +1289,22 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
12781289
c = tok_nextc(tok);
12791290
if (isdigit(c)) {
12801291
goto fraction;
1281-
}
1282-
else {
1292+
} else if (c == '.') {
1293+
c = tok_nextc(tok);
1294+
if (c == '.') {
1295+
*p_start = tok->start;
1296+
*p_end = tok->cur;
1297+
return ELLIPSIS;
1298+
} else {
1299+
tok_backup(tok, c);
1300+
}
1301+
tok_backup(tok, '.');
1302+
} else {
12831303
tok_backup(tok, c);
1284-
*p_start = tok->start;
1285-
*p_end = tok->cur;
1286-
return DOT;
12871304
}
1305+
*p_start = tok->start;
1306+
*p_end = tok->cur;
1307+
return DOT;
12881308
}
12891309

12901310
/* Number */

‎Python/ast.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1410,7 +1410,7 @@ ast_for_atom(struct compiling *c, const node *n)
14101410
PyArena_AddPyObject(c->c_arena, pynum);
14111411
return Num(pynum, LINENO(n), n->n_col_offset, c->c_arena);
14121412
}
1413-
case DOT: /* Ellipsis */
1413+
case ELLIPSIS: /* Ellipsis */
14141414
return Ellipsis(LINENO(n), n->n_col_offset, c->c_arena);
14151415
case LPAR: /* some parenthesized expressions */
14161416
ch = CHILD(n, 1);

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL