FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

gh-150638: Improve performance of json.loads and json.load for numeric data by eendebakpt · Pull Request #150639 · python/cpython · GitHub

Repository navigation

Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension .c  (1) .py  (1) .rst  (1) All 3 file types selected
Viewed files
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Unified
Split
Hide whitespace
Diff view
Unified
Split
Hide whitespace
22 changes: 22 additions & 0 deletions Lib/test/test_json/test_decode.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,28 @@ def test_limit_int(self):
with self.assertRaises(ValueError):
self.loads('1' * (maxdigits + 1))

def test_int_boundaries(self):
# Values around the signed/unsigned 64-bit limits and the
# 19-vs-20 digit fast-path threshold of the C accelerator.
for s in ['0', '-0',
'9223372036854775807', # LLONG_MAX
'9223372036854775808', # LLONG_MAX + 1
'-9223372036854775808', # LLONG_MIN
'-9223372036854775809', # LLONG_MIN - 1
'9999999999999999999', # largest 19-digit
'-9999999999999999999',
'18446744073709551615', # ULLONG_MAX (20 digits)
'18446744073709551616', # ULLONG_MAX + 1
'10000000000000000000', # smallest 20-digit
'-10000000000000000000']:
with self.subTest(s=s):
self.assertEqual(self.loads(s), int(s))

def test_long_float(self):
# A float longer than the C accelerator's stack buffer.
s = '0.' + '1' * 200
self.assertEqual(self.loads(s), float(s))

def test_explicit_cls_skips_json_decoder_default(self):
class CustomDecoder:
pass
Expand Down
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Speed up :func:`json.loads` and :func:`json.load` parsing of numbers.
64 changes: 49 additions & 15 deletions Modules/_json.c
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -997,7 +997,6 @@ _match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_
Py_ssize_t idx = start;
int is_float = 0;
PyObject *rval;
PyObject *numstr = NULL;
PyObject *custom_func;

str = PyUnicode_DATA(pystr);
Expand Down Expand Up @@ -1064,32 +1063,67 @@ _match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_

if (custom_func) {
/* copy the section we determined to be a number */
numstr = PyUnicode_FromKindAndData(kind,
(char*)str + kind * start,
idx - start);
if (numstr == NULL)
PyObject *numstr = PyUnicode_FromKindAndData(kind,
(char*)str + kind * start,
idx - start);
if (numstr == NULL) {
return NULL;
}
rval = PyObject_CallOneArg(custom_func, numstr);
Py_DECREF(numstr);
}
else {
Py_ssize_t i, n;
char *buf;

n = idx - start;
if (!is_float && n <= 19) {
*next_idx_ptr = idx;
if (PyUnicode_READ(kind, str, start) == '-') {
/* integer with at most 18 digits (excluding sign) fits in
long long (10 ** 18 - 1 <= 2 ** 63 < 10 ** 19 - 1) */
long long value = 0;
for (i = start + 1; i < idx; i++) {
value = value * 10 - (PyUnicode_READ(kind, str, i) - '0');
}
return PyLong_FromLongLong(value);
}
else {
/* integer with at most 19 digits fits in unsigned long
long (10 ** 19 - 1 <= 2 ** 64 - 1 < 10 ** 20 - 1) */
unsigned long long value = 0;
for (i = start; i < idx; i++) {
value = value * 10 + (PyUnicode_READ(kind, str, i) - '0');
}
return PyLong_FromUnsignedLongLong(value);
}
}

/* Straight conversion to ASCII, to avoid costly conversion of
decimal unicode digits (which cannot appear here) */
n = idx - start;
numstr = PyBytes_FromStringAndSize(NULL, n);
if (numstr == NULL)
return NULL;
buf = PyBytes_AS_STRING(numstr);
char stackbuf[64];
char *buf = stackbuf;
if (n >= (Py_ssize_t)sizeof(stackbuf)) {
buf = PyMem_Malloc(n + 1);
if (buf == NULL) {
PyErr_NoMemory();
return NULL;
}
}
for (i = 0; i < n; i++) {
buf[i] = (char) PyUnicode_READ(kind, str, i + start);
}
if (is_float)
rval = PyFloat_FromString(numstr);
else
buf[n] = '\0';
if (is_float) {
double d = PyOS_string_to_double(buf, NULL, NULL);
rval = (d == -1.0 && PyErr_Occurred()) ? NULL : PyFloat_FromDouble(d);
}
else {
rval = PyLong_FromString(buf, NULL, 10);
}
if (buf != stackbuf) {
PyMem_Free(buf);
}
}
Py_DECREF(numstr);
*next_idx_ptr = idx;
return rval;
}
Expand Down
Loading

Back | FazBrowse Home | New Git URL