FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

gh-127936: convert marshal module to use import/export API for ints (PEP 757) by skirpichev · Pull Request #128530 · python/cpython · GitHub

  1. python
  2. cpython

Repository navigation

gh-127936: convert marshal module to use import/export API for ints (PEP 757) #128530

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vstinner merged 14 commits into from
Jan 23, 2025
Conversation 62 Commits 14 Checks 0 Files changed
Merged

gh-127936: convert marshal module to use import/export API for ints (PEP 757) #128530

Changes from 5 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
4bd6b0c
gh-127936: convert marshal module to use import/export API for ints (…
skirpichev Dec 14, 2024
44c7c6a
Apply suggestions from code review
skirpichev Jan 7, 2025
e960c17
Merge branch 'master' into port-marshal-to-pep757/127936
skirpichev Jan 7, 2025
95d89a7
address review: local decl
skirpichev Jan 7, 2025
77be1bd
Merge branch 'main' into port-marshal-to-pep757/127936
skirpichev Jan 8, 2025
1e0db95
Apply suggestions from code review
skirpichev Jan 14, 2025
d66af47
Apply suggestions from code review
skirpichev Jan 14, 2025
f152274
Update Python/marshal.c
skirpichev Jan 14, 2025
fb505db
Merge branch 'master' into port-marshal-to-pep757/127936
skirpichev Jan 14, 2025
035c772
address review: bitsize (naming)
skirpichev Jan 14, 2025
9d95c35
Apply suggestions from code review
skirpichev Jan 15, 2025
47eaacf
Update Python/marshal.c
vstinner Jan 23, 2025
9353013
Update Python/marshal.c
vstinner Jan 23, 2025
82cb784
Merge branch 'main' into port-marshal-to-pep757/127936
skirpichev Jan 23, 2025
File filter

Filter by extension

Filter by extension .c  (1) All 1 file type selected
Viewed files
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Unified
Split
Hide whitespace
Diff view
Unified
Split
Hide whitespace
233 changes: 158 additions & 75 deletions Python/marshal.c
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -240,10 +240,6 @@ w_short_pstring(const void *s, Py_ssize_t n, WFILE *p)
#define PyLong_MARSHAL_SHIFT 15
#define PyLong_MARSHAL_BASE ((short)1 << PyLong_MARSHAL_SHIFT)
#define PyLong_MARSHAL_MASK (PyLong_MARSHAL_BASE - 1)
#if PyLong_SHIFT % PyLong_MARSHAL_SHIFT != 0
#error "PyLong_SHIFT must be a multiple of PyLong_MARSHAL_SHIFT"
#endif
#define PyLong_MARSHAL_RATIO (PyLong_SHIFT / PyLong_MARSHAL_SHIFT)

#define W_TYPE(t, p) do { \
w_byte((t) | flag, (p)); \
Expand All @@ -252,47 +248,102 @@ w_short_pstring(const void *s, Py_ssize_t n, WFILE *p)
static PyObject *
_PyMarshal_WriteObjectToString(PyObject *x, int version, int allow_code);

#define _r_digits(bs) \
Comment thread
skirpichev marked this conversation as resolved.
Resolved
static void \
_r_digits##bs(const uint##bs##_t *digits, Py_ssize_t n, uint8_t negative, \
Py_ssize_t marshal_ratio, WFILE *p) \
{ \
/* set l to number of base PyLong_MARSHAL_BASE digits */ \
Py_ssize_t l = (n - 1)*marshal_ratio; \
Comment thread
picnixz marked this conversation as resolved.
Resolved
uint##bs##_t d = digits[n - 1]; \
\
assert(d != 0); /* a PyLong is always normalized */ \
Comment thread
vstinner marked this conversation as resolved.
Resolved
do { \
d >>= PyLong_MARSHAL_SHIFT; \
l++; \
} while (d != 0); \
if (l > SIZE32_MAX) { \
p->depth--; \
p->error = WFERR_UNMARSHALLABLE; \
return; \
} \
w_long((long)(negative ? -l : l), p); \
\
for (Py_ssize_t i = 0; i < n - 1; i++) { \
d = digits[i]; \
for (Py_ssize_t j = 0; j < marshal_ratio; j++) { \
w_short(d & PyLong_MARSHAL_MASK, p); \
d >>= PyLong_MARSHAL_SHIFT; \
} \
assert (d == 0); \
Comment thread
skirpichev marked this conversation as resolved.
Resolved
} \
d = digits[n - 1]; \
do { \
w_short(d & PyLong_MARSHAL_MASK, p); \
d >>= PyLong_MARSHAL_SHIFT; \
} while (d != 0); \
}
_r_digits(16)
_r_digits(32)

Comment thread
skirpichev marked this conversation as resolved.
Resolved
static void
w_PyLong(const PyLongObject *ob, char flag, WFILE *p)
{
Py_ssize_t i, j, n, l;
digit d;

W_TYPE(TYPE_LONG, p);
if (_PyLong_IsZero(ob)) {
w_long((long)0, p);
return;
}

/* set l to number of base PyLong_MARSHAL_BASE digits */
n = _PyLong_DigitCount(ob);
l = (n-1) * PyLong_MARSHAL_RATIO;
d = ob->long_value.ob_digit[n-1];
assert(d != 0); /* a PyLong is always normalized */
do {
d >>= PyLong_MARSHAL_SHIFT;
l++;
} while (d != 0);
if (l > SIZE32_MAX) {
PyLongExport long_export;

if (PyLong_Export((PyObject *)ob, &long_export) < 0) {
p->depth--;
p->error = WFERR_UNMARSHALLABLE;
return;
}
w_long((long)(_PyLong_IsNegative(ob) ? -l : l), p);
if (!long_export.digits) {
int8_t sign = long_export.value < 0 ? -1 : 1;
uint64_t abs_value = Py_ABS(long_export.value);
uint64_t d = abs_value;
long l = 0;

/* set l to number of base PyLong_MARSHAL_BASE digits */
do {
d >>= PyLong_MARSHAL_SHIFT;
l += sign;
} while (d);
w_long(l, p);

for (i=0; i < n-1; i++) {
d = ob->long_value.ob_digit[i];
for (j=0; j < PyLong_MARSHAL_RATIO; j++) {
d = abs_value;
Comment thread
skirpichev marked this conversation as resolved.
Resolved
do {
w_short(d & PyLong_MARSHAL_MASK, p);
d >>= PyLong_MARSHAL_SHIFT;
}
assert (d == 0);
} while (d);
return;
}

const PyLongLayout *layout = PyLong_GetNativeLayout();
Py_ssize_t marshal_ratio = layout->bits_per_digit/PyLong_MARSHAL_SHIFT;

/* must be a multiple of PyLong_MARSHAL_SHIFT */
assert(layout->bits_per_digit % PyLong_MARSHAL_SHIFT == 0);
Comment thread
skirpichev marked this conversation as resolved.
Resolved
Comment thread
skirpichev marked this conversation as resolved.
Resolved

/* other assumptions on PyLongObject internals */
assert(layout->bits_per_digit <= 32);
assert(layout->digits_order == -1);
assert(layout->digit_endianness == (PY_LITTLE_ENDIAN ? -1 : 1));
assert(layout->digit_size == 2 || layout->digit_size == 4);
Comment thread
skirpichev marked this conversation as resolved.
Resolved

if (layout->digit_size == 4) {
Comment thread
skirpichev marked this conversation as resolved.
Resolved
_r_digits32(long_export.digits, long_export.ndigits,
long_export.negative, marshal_ratio, p);
}
else {
_r_digits16(long_export.digits, long_export.ndigits,
long_export.negative, marshal_ratio, p);
}
d = ob->long_value.ob_digit[n-1];
do {
w_short(d & PyLong_MARSHAL_MASK, p);
d >>= PyLong_MARSHAL_SHIFT;
} while (d != 0);
PyLong_FreeExport(&long_export);
}

static void
Expand Down Expand Up @@ -875,17 +926,60 @@ r_long64(RFILE *p)
1 /* signed */);
}

#define _w_digits(bs) \
Comment thread
skirpichev marked this conversation as resolved.
Resolved
static int \
_w_digits##bs(uint##bs##_t *digits, Py_ssize_t size, \
Py_ssize_t marshal_ratio, \
int shorts_in_top_digit, RFILE *p) \
{ \
uint##bs##_t d; \
\
for (Py_ssize_t i = 0; i < size - 1; i++) { \
Comment thread
vstinner marked this conversation as resolved.
Resolved
d = 0; \
for (Py_ssize_t j = 0; j < marshal_ratio; j++) { \
int md = r_short(p); \
if (md < 0 || md > PyLong_MARSHAL_BASE) { \
goto bad_digit; \
} \
d += (uint##bs##_t)md << j*PyLong_MARSHAL_SHIFT; \
} \
digits[i] = d; \
} \
\
d = 0; \
for (Py_ssize_t j = 0; j < shorts_in_top_digit; j++) { \
int md = r_short(p); \
if (md < 0 || md > PyLong_MARSHAL_BASE) { \
goto bad_digit; \
} \
/* topmost marshal digit should be nonzero */ \
if (md == 0 && j == shorts_in_top_digit - 1) { \
PyErr_SetString(PyExc_ValueError, \
"bad marshal data (unnormalized long data)"); \
return -1; \
} \
d += (uint##bs##_t)md << j*PyLong_MARSHAL_SHIFT; \
} \
assert(!PyErr_Occurred()); \
/* top digit should be nonzero, else the resulting PyLong won't be \
normalized */ \
digits[size - 1] = d; \
return 0; \
\
bad_digit: \
Comment thread
skirpichev marked this conversation as resolved.
Resolved
if (!PyErr_Occurred()) { \
PyErr_SetString(PyExc_ValueError, \
"bad marshal data (digit out of range in long)"); \
} \
return -1; \
}
_w_digits(32)
_w_digits(16)

Comment thread
skirpichev marked this conversation as resolved.
Resolved
Comment thread
skirpichev marked this conversation as resolved.
Resolved
static PyObject *
r_PyLong(RFILE *p)
{
PyLongObject *ob;
long n, size, i;
int j, md, shorts_in_top_digit;
digit d;

n = r_long(p);
if (n == 0)
return (PyObject *)_PyLong_New(0);
long n = r_long(p);
if (n == -1 && PyErr_Occurred()) {
return NULL;
}
Expand All @@ -895,51 +989,40 @@ r_PyLong(RFILE *p)
return NULL;
}

size = 1 + (Py_ABS(n) - 1) / PyLong_MARSHAL_RATIO;
shorts_in_top_digit = 1 + (Py_ABS(n) - 1) % PyLong_MARSHAL_RATIO;
ob = _PyLong_New(size);
if (ob == NULL)
return NULL;
const PyLongLayout *layout = PyLong_GetNativeLayout();
Py_ssize_t marshal_ratio = layout->bits_per_digit/PyLong_MARSHAL_SHIFT;

_PyLong_SetSignAndDigitCount(ob, n < 0 ? -1 : 1, size);
/* must be a multiple of PyLong_MARSHAL_SHIFT */
assert(layout->bits_per_digit % PyLong_MARSHAL_SHIFT == 0);
Comment thread
skirpichev marked this conversation as resolved.
Resolved
Comment thread
skirpichev marked this conversation as resolved.
Resolved

for (i = 0; i < size-1; i++) {
d = 0;
for (j=0; j < PyLong_MARSHAL_RATIO; j++) {
md = r_short(p);
if (md < 0 || md > PyLong_MARSHAL_BASE)
goto bad_digit;
d += (digit)md << j*PyLong_MARSHAL_SHIFT;
}
ob->long_value.ob_digit[i] = d;
/* other assumptions on PyLongObject internals */
assert(layout->bits_per_digit <= 32);
Comment thread
skirpichev marked this conversation as resolved.
Resolved
assert(layout->digits_order == -1);
assert(layout->digit_endianness == (PY_LITTLE_ENDIAN ? -1 : 1));
assert(layout->digit_size == 2 || layout->digit_size == 4);

Py_ssize_t size = 1 + (Py_ABS(n) - 1) / marshal_ratio;
Comment thread
skirpichev marked this conversation as resolved.
Resolved
Comment thread
skirpichev marked this conversation as resolved.
Resolved
int shorts_in_top_digit = 1 + (Py_ABS(n) - 1) % marshal_ratio;
Comment thread
skirpichev marked this conversation as resolved.
Resolved
void *digits;
PyLongWriter *writer = PyLongWriter_Create(n < 0, size, &digits);

if (writer == NULL) {
Comment thread
skirpichev marked this conversation as resolved.
Resolved
return NULL;
}

d = 0;
for (j=0; j < shorts_in_top_digit; j++) {
md = r_short(p);
if (md < 0 || md > PyLong_MARSHAL_BASE)
goto bad_digit;
/* topmost marshal digit should be nonzero */
if (md == 0 && j == shorts_in_top_digit - 1) {
Py_DECREF(ob);
PyErr_SetString(PyExc_ValueError,
"bad marshal data (unnormalized long data)");
return NULL;
}
d += (digit)md << j*PyLong_MARSHAL_SHIFT;
int ret;

if (layout->digit_size == 4) {
ret = _w_digits32(digits, size, marshal_ratio, shorts_in_top_digit, p);
}
assert(!PyErr_Occurred());
/* top digit should be nonzero, else the resulting PyLong won't be
normalized */
ob->long_value.ob_digit[size-1] = d;
return (PyObject *)ob;
bad_digit:
Py_DECREF(ob);
if (!PyErr_Occurred()) {
PyErr_SetString(PyExc_ValueError,
"bad marshal data (digit out of range in long)");
else {
ret = _w_digits16(digits, size, marshal_ratio, shorts_in_top_digit, p);
}
if (ret < 0) {
PyLongWriter_Discard(writer);
return NULL;
}
return NULL;
return PyLongWriter_Finish(writer);
}

static double
Expand Down
Back | FazBrowse Home | New Git URL