FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

gh-99612: Fix PyUnicode_DecodeUTF8Stateful() for ASCII-only data (GH-… · python/cpython@f08e52c · GitHub

/ cpython Public

Commit f08e52c

Browse files
serhiy-storchaka
authored
gh-99612: Fix PyUnicode_DecodeUTF8Stateful() for ASCII-only data (GH-99613)
Previously *consumed was not set in this case.
1 parent d460c8e commit f08e52c

4 files changed

Lines changed: 95 additions & 0 deletions

File tree

‎Lib/test/test_capi/test_codecs.py‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
import unittest
2+
from test.support import import_helper
3+
4+
_testcapi = import_helper.import_module('_testcapi')
5+
6+
7+
class CAPITest(unittest.TestCase):
8+
9+
def test_decodeutf8(self):
10+
"""Test PyUnicode_DecodeUTF8()"""
11+
decodeutf8 = _testcapi.unicode_decodeutf8
12+
13+
for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
14+
b = s.encode('utf-8')
15+
self.assertEqual(decodeutf8(b), s)
16+
self.assertEqual(decodeutf8(b, 'strict'), s)
17+
18+
self.assertRaises(UnicodeDecodeError, decodeutf8, b'\x80')
19+
self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xc0')
20+
self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xff')
21+
self.assertRaises(UnicodeDecodeError, decodeutf8, b'a\xf0\x9f')
22+
self.assertEqual(decodeutf8(b'a\xf0\x9f', 'replace'), 'a\ufffd')
23+
self.assertEqual(decodeutf8(b'a\xf0\x9fb', 'replace'), 'a\ufffdb')
24+
25+
self.assertRaises(LookupError, decodeutf8, b'a\x80', 'foo')
26+
# TODO: Test PyUnicode_DecodeUTF8() with NULL as data and
27+
# negative size.
28+
29+
def test_decodeutf8stateful(self):
30+
"""Test PyUnicode_DecodeUTF8Stateful()"""
31+
decodeutf8stateful = _testcapi.unicode_decodeutf8stateful
32+
33+
for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
34+
b = s.encode('utf-8')
35+
self.assertEqual(decodeutf8stateful(b), (s, len(b)))
36+
self.assertEqual(decodeutf8stateful(b, 'strict'), (s, len(b)))
37+
38+
self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\x80')
39+
self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xc0')
40+
self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xff')
41+
self.assertEqual(decodeutf8stateful(b'a\xf0\x9f'), ('a', 1))
42+
self.assertEqual(decodeutf8stateful(b'a\xf0\x9f', 'replace'), ('a', 1))
43+
self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'a\xf0\x9fb')
44+
self.assertEqual(decodeutf8stateful(b'a\xf0\x9fb', 'replace'), ('a\ufffdb', 4))
45+
46+
self.assertRaises(LookupError, decodeutf8stateful, b'a\x80', 'foo')
47+
# TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as data and
48+
# negative size.
49+
# TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as the address of
50+
# "consumed".
51+
52+
53+
if __name__ == "__main__":
54+
unittest.main()

‎Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fix :c:func:`PyUnicode_DecodeUTF8Stateful` for ASCII-only data:
2+
``*consumed`` was not set.

‎Modules/_testcapi/unicode.c‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,40 @@ unicode_asutf8andsize(PyObject *self, PyObject *args)
239239
return Py_BuildValue("(Nn)", result, utf8_len);
240240
}
241241

242+
/* Test PyUnicode_DecodeUTF8() */
243+
static PyObject *
244+
unicode_decodeutf8(PyObject *self, PyObject *args)
245+
{
246+
const char *data;
247+
Py_ssize_t size;
248+
const char *errors = NULL;
249+
250+
if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors))
251+
return NULL;
252+
253+
return PyUnicode_DecodeUTF8(data, size, errors);
254+
}
255+
256+
/* Test PyUnicode_DecodeUTF8Stateful() */
257+
static PyObject *
258+
unicode_decodeutf8stateful(PyObject *self, PyObject *args)
259+
{
260+
const char *data;
261+
Py_ssize_t size;
262+
const char *errors = NULL;
263+
Py_ssize_t consumed = 123456789;
264+
PyObject *result;
265+
266+
if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors))
267+
return NULL;
268+
269+
result = PyUnicode_DecodeUTF8Stateful(data, size, errors, &consumed);
270+
if (!result) {
271+
return NULL;
272+
}
273+
return Py_BuildValue("(Nn)", result, consumed);
274+
}
275+
242276
/* Test PyUnicode_Concat() */
243277
static PyObject *
244278
unicode_concat(PyObject *self, PyObject *args)
@@ -1025,6 +1059,8 @@ static PyMethodDef TestMethods[] = {
10251059
{"unicode_asucs4", unicode_asucs4, METH_VARARGS},
10261060
{"unicode_asutf8", unicode_asutf8, METH_VARARGS},
10271061
{"unicode_asutf8andsize", unicode_asutf8andsize, METH_VARARGS},
1062+
{"unicode_decodeutf8", unicode_decodeutf8, METH_VARARGS},
1063+
{"unicode_decodeutf8stateful",unicode_decodeutf8stateful, METH_VARARGS},
10281064
{"unicode_concat", unicode_concat, METH_VARARGS},
10291065
{"unicode_splitlines", unicode_splitlines, METH_VARARGS},
10301066
{"unicode_split", unicode_split, METH_VARARGS},

‎Objects/unicodeobject.c‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4530,6 +4530,9 @@ unicode_decode_utf8(const char *s, Py_ssize_t size,
45304530
}
45314531
s += ascii_decode(s, end, PyUnicode_1BYTE_DATA(u));
45324532
if (s == end) {
4533+
if (consumed) {
4534+
*consumed = size;
4535+
}
45334536
return u;
45344537
}
45354538

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL