bpo-34683: Make SyntaxError column offsets consistently 1-indexed (gh… · python-emscripten/cpython@025eb98 · GitHub

1. python-emscripten
2. cpython

Repository navigation

• Code
• Pull requests
• Actions
• Projects
• Security and quality
• Insights

Commit 025eb98

Browse filesBrowse the repository at this point in the historyBrowse files

ammaraskar

authored and

gvanrossum

committed

bpo-34683: Make SyntaxError column offsets consistently 1-indexed (pythongh-9338)

Also point to start of tokens in parsing errors. Fixes bpo-34683

1 parent 223e501 commit 025eb98
Copy full SHA for 025eb98

11 files changed

+65-21Lines changed: 65 additions & 21 deletions

File tree

Expand file treeCollapse file tree

Open diff view settings

Filter options

• Lib/test
  • test_exceptions.py
  • test_future.py
  • test_global.py
  • test_support.py
  • test_symtable.py
• Misc/NEWS.d/next/Core and Builtins
  • 2018-09-15-19-32-34.bpo-34683.msCiQE.rst
• Parser
  • parsetok.c
• Python
  • ast.c
  • compile.c
  • future.c
  • symtable.c

Expand file treeCollapse file tree

Open diff view settings

Collapse file

‎Lib/test/test_exceptions.py‎

Copy file name to clipboardExpand all lines: Lib/test/test_exceptions.py

+39Lines changed: 39 additions & 0 deletions

Original file line number	Diff line number	Diff line change
@@ -187,6 +187,45 @@ def check(src, lineno, offset):
187	187	check('def spam():\n print(1)\n print(2)', 3, 10)
188	188	check('Python = "Python" +', 1, 20)
189	189	check('Python = "\u1e54\xfd\u0163\u0125\xf2\xf1" +', 1, 20)
	190	+ check('x = "a', 1, 7)
	191	+ check('lambda x: x = 2', 1, 1)
	192	+
	193	+ # Errors thrown by compile.c
	194	+ check('class foo:return 1', 1, 11)
	195	+ check('def f():\n continue', 2, 3)
	196	+ check('def f():\n break', 2, 3)
	197	+ check('try:\n pass\nexcept:\n pass\nexcept ValueError:\n pass', 2, 3)
	198	+
	199	+ # Errors thrown by tokenizer.c
	200	+ check('(0x+1)', 1, 3)
	201	+ check('x = 0xI', 1, 6)
	202	+ check('0010 + 2', 1, 4)
	203	+ check('x = 32e-+4', 1, 8)
	204	+ check('x = 0o9', 1, 6)
	205	+
	206	+ # Errors thrown by symtable.c
	207	+ check('x = [(yield i) for i in range(3)]', 1, 6)
	208	+ check('def f():\n from _ import *', 1, 1)
	209	+ check('def f(x, x):\n pass', 1, 1)
	210	+ check('def f(x):\n nonlocal x', 2, 3)
	211	+ check('def f(x):\n x = 1\n global x', 3, 3)
	212	+ check('nonlocal x', 1, 1)
	213	+ check('def f():\n global x\n nonlocal x', 2, 3)
	214	+
	215	+ # Errors thrown by ast.c
	216	+ check('for 1 in []: pass', 1, 5)
	217	+ check('def f(*):\n pass', 1, 7)
	218	+ check('[*x for x in xs]', 1, 2)
	219	+ check('def f():\n x, y: int', 2, 3)
	220	+ check('(yield i) = 2', 1, 1)
	221	+ check('foo(x for x in range(10), 100)', 1, 5)
	222	+ check('foo(1=2)', 1, 5)
	223	+
	224	+ # Errors thrown by future.c
	225	+ check('from __future__ import doesnt_exist', 1, 1)
	226	+ check('from __future__ import braces', 1, 1)
	227	+ check('x=1\nfrom __future__ import division', 2, 1)
	228	+
190	229
191	230	@cpython_only
192	231	def testSettingException(self):

Collapse file

‎Lib/test/test_future.py‎

Copy file name to clipboardExpand all lines: Lib/test/test_future.py

+3-3Lines changed: 3 additions & 3 deletions

Original file line number	Diff line number	Diff line change
@@ -15,7 +15,7 @@ def get_error_location(msg):
15	15
16	16	class FutureTest(unittest.TestCase):
17	17
18		- def check_syntax_error(self, err, basename, lineno, offset=0):
	18	+ def check_syntax_error(self, err, basename, lineno, offset=1):
19	19	self.assertIn('%s.py, line %d' % (basename, lineno), str(err))
20	20	self.assertEqual(os.path.basename(err.filename), basename + '.py')
21	21	self.assertEqual(err.lineno, lineno)
@@ -68,12 +68,12 @@ def test_badfuture8(self):
68	68	def test_badfuture9(self):
69	69	with self.assertRaises(SyntaxError) as cm:
70	70	from test import badsyntax_future9
71		- self.check_syntax_error(cm.exception, "badsyntax_future9", 3, 0)
	71	+ self.check_syntax_error(cm.exception, "badsyntax_future9", 3)
72	72
73	73	def test_badfuture10(self):
74	74	with self.assertRaises(SyntaxError) as cm:
75	75	from test import badsyntax_future10
76		- self.check_syntax_error(cm.exception, "badsyntax_future10", 3, 0)
	76	+ self.check_syntax_error(cm.exception, "badsyntax_future10", 3)
77	77
78	78	def test_parserhack(self):
79	79	# test that the parser.c::future_hack function works as expected

Collapse file

‎Lib/test/test_global.py‎

Copy file name to clipboardExpand all lines: Lib/test/test_global.py

+3-3Lines changed: 3 additions & 3 deletions

Original file line number	Diff line number	Diff line change
@@ -24,15 +24,15 @@ def wrong1():
24	24	global a
25	25	global b
26	26	"""
27		- check_syntax_error(self, prog_text_1, lineno=4, offset=4)
	27	+ check_syntax_error(self, prog_text_1, lineno=4, offset=5)
28	28
29	29	def test2(self):
30	30	prog_text_2 = """\
31	31	def wrong2():
32	32	print(x)
33	33	global x
34	34	"""
35		- check_syntax_error(self, prog_text_2, lineno=3, offset=4)
	35	+ check_syntax_error(self, prog_text_2, lineno=3, offset=5)
36	36
37	37	def test3(self):
38	38	prog_text_3 = """\
@@ -41,7 +41,7 @@ def wrong3():
41	41	x = 2
42	42	global x
43	43	"""
44		- check_syntax_error(self, prog_text_3, lineno=4, offset=4)
	44	+ check_syntax_error(self, prog_text_3, lineno=4, offset=5)
45	45
46	46	def test4(self):
47	47	prog_text_4 = """\

Collapse file

‎Lib/test/test_support.py‎

Copy file name to clipboardExpand all lines: Lib/test/test_support.py

+1-1Lines changed: 1 addition & 1 deletion

Original file line number	Diff line number	Diff line change
@@ -286,7 +286,7 @@ def test_make_bad_fd(self):
286	286	self.assertEqual(cm.exception.errno, errno.EBADF)
287	287
288	288	def test_check_syntax_error(self):
289		- support.check_syntax_error(self, "def class", lineno=1, offset=9)
	289	+ support.check_syntax_error(self, "def class", lineno=1, offset=5)
290	290	with self.assertRaises(AssertionError):
291	291	support.check_syntax_error(self, "x=1")
292	292

Collapse file

‎Lib/test/test_symtable.py‎

Copy file name to clipboardExpand all lines: Lib/test/test_symtable.py

+1-1Lines changed: 1 addition & 1 deletion

Original file line number	Diff line number	Diff line change
@@ -169,7 +169,7 @@ def checkfilename(brokencode, offset):
169	169	else:
170	170	self.fail("no SyntaxError for %r" % (brokencode,))
171	171	checkfilename("def f(x): foo)(", 14) # parse-time
172		- checkfilename("def f(x): global x", 10) # symtable-build-time
	172	+ checkfilename("def f(x): global x", 11) # symtable-build-time
173	173	symtable.symtable("pass", b"spam", "exec")
174	174	with self.assertWarns(DeprecationWarning), \
175	175	self.assertRaises(TypeError):

Collapse file

‎Misc/NEWS.d/next/Core and Builtins/2018-09-15-19-32-34.bpo-34683.msCiQE.rst‎

Copy file name to clipboard

+1Lines changed: 1 addition & 0 deletions

• Display the source diff
• Display the rich diff

Original file line number	Diff line number	Diff line change
@@ -0,0 +1 @@
	1	+ Fixed a bug where some SyntaxError error pointed to locations that were off-by-one.

Collapse file

‎Parser/parsetok.c‎

Copy file name to clipboardExpand all lines: Parser/parsetok.c

+6-2Lines changed: 6 additions & 2 deletions

Original file line number	Diff line number	Diff line change
@@ -187,6 +187,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
187	187	parser_state *ps;
188	188	node *n;
189	189	int started = 0;
	190	+ int col_offset;
190	191
191	192	if ((ps = PyParser_New(g, start)) == NULL) {
192	193	err_ret->error = E_NOMEM;
@@ -203,7 +204,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
203	204	int type;
204	205	size_t len;
205	206	char *str;
206		- int col_offset;
	207	+ col_offset = -1;
207	208
208	209	type = PyTokenizer_Get(tok, &a, &b);
209	210	if (type == ERRORTOKEN) {
@@ -321,7 +322,10 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
321	322	if (tok->buf != NULL) {
322	323	size_t len;
323	324	assert(tok->cur - tok->buf < INT_MAX);
324		- err_ret->offset = (int)(tok->cur - tok->buf);
	325	+ /* if we've managed to parse a token, point the offset to its start,
	326	+ * else use the current reading position of the tokenizer
	327	+ */
	328	+ err_ret->offset = col_offset != -1 ? col_offset + 1 : ((int)(tok->cur - tok->buf));
325	329	len = tok->inp - tok->buf;
326	330	err_ret->text = (char *) PyObject_MALLOC(len + 1);
327	331	if (err_ret->text != NULL) {

Collapse file

‎Python/ast.c‎

Copy file name to clipboardExpand all lines: Python/ast.c

+1-1Lines changed: 1 addition & 1 deletion

Original file line number	Diff line number	Diff line change
@@ -683,7 +683,7 @@ ast_error(struct compiling *c, const node *n, const char *errmsg)
683	683	Py_INCREF(Py_None);
684	684	loc = Py_None;
685	685	}
686		- tmp = Py_BuildValue("(OiiN)", c->c_filename, LINENO(n), n->n_col_offset, loc);
	686	+ tmp = Py_BuildValue("(OiiN)", c->c_filename, LINENO(n), n->n_col_offset + 1, loc);
687	687	if (!tmp)
688	688	return 0;
689	689	errstr = PyUnicode_FromString(errmsg);

Collapse file

‎Python/compile.c‎

Copy file name to clipboardExpand all lines: Python/compile.c

+1-1Lines changed: 1 addition & 1 deletion

Original file line number	Diff line number	Diff line change
@@ -4830,7 +4830,7 @@ compiler_error(struct compiler *c, const char *errstr)
4830	4830	loc = Py_None;
4831	4831	}
4832	4832	u = Py_BuildValue("(OiiO)", c->c_filename, c->u->u_lineno,
4833		- c->u->u_col_offset, loc);
	4833	+ c->u->u_col_offset + 1, loc);
4834	4834	if (!u)
4835	4835	goto exit;
4836	4836	v = Py_BuildValue("(zO)", errstr, u);

Collapse file

‎Python/future.c‎

Copy file name to clipboardExpand all lines: Python/future.c

+2-2Lines changed: 2 additions & 2 deletions

Original file line number	Diff line number	Diff line change
@@ -48,12 +48,12 @@ future_check_features(PyFutureFeatures *ff, stmt_ty s, PyObject *filename)
48	48	} else if (strcmp(feature, "braces") == 0) {
49	49	PyErr_SetString(PyExc_SyntaxError,
50	50	"not a chance");
51		- PyErr_SyntaxLocationObject(filename, s->lineno, s->col_offset);
	51	+ PyErr_SyntaxLocationObject(filename, s->lineno, s->col_offset + 1);
52	52	return 0;
53	53	} else {
54	54	PyErr_Format(PyExc_SyntaxError,
55	55	UNDEFINED_FUTURE_FEATURE, feature);
56		- PyErr_SyntaxLocationObject(filename, s->lineno, s->col_offset);
	56	+ PyErr_SyntaxLocationObject(filename, s->lineno, s->col_offset + 1);
57	57	return 0;
58	58	}
59	59	}

0 commit comments

Comments

0 (0)