FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

bpo-22005: Fixed unpickling instances of datetime classes pickled by … · python/cpython@8452ca1 · GitHub

  1. python
  2. cpython

Repository navigation

Commit 8452ca1

Browse files
serhiy-storchaka
authored
bpo-22005: Fixed unpickling instances of datetime classes pickled by Python 2. (GH-11017)
encoding='latin1' should be used for successful decoding.
1 parent 4c49da0 commit 8452ca1

5 files changed

Lines changed: 333 additions & 95 deletions

File tree

‎Doc/library/pickle.rst‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,9 @@ process more convenient:
243243
*errors* tell pickle how to decode 8-bit string instances pickled by Python
244244
2; these default to 'ASCII' and 'strict', respectively. The *encoding* can
245245
be 'bytes' to read these 8-bit string instances as bytes objects.
246+
Using ``encoding='latin1'`` is required for unpickling NumPy arrays and
247+
instances of :class:`~datetime.datetime`, :class:`~datetime.date` and
248+
:class:`~datetime.time` pickled by Python 2.
246249

247250
.. function:: loads(bytes_object, \*, fix_imports=True, encoding="ASCII", errors="strict")
248251

@@ -260,6 +263,9 @@ process more convenient:
260263
*errors* tell pickle how to decode 8-bit string instances pickled by Python
261264
2; these default to 'ASCII' and 'strict', respectively. The *encoding* can
262265
be 'bytes' to read these 8-bit string instances as bytes objects.
266+
Using ``encoding='latin1'`` is required for unpickling NumPy arrays and
267+
instances of :class:`~datetime.datetime`, :class:`~datetime.date` and
268+
:class:`~datetime.time` pickled by Python 2.
263269

264270

265271
The :mod:`pickle` module defines three exceptions:

‎Lib/datetime.py‎

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -808,9 +808,19 @@ def __new__(cls, year, month=None, day=None):
808808
809809
year, month, day (required, base 1)
810810
"""
811-
if month is None and isinstance(year, bytes) and len(year) == 4 and \
812-
1 <= year[2] <= 12:
811+
if (month is None and
812+
isinstance(year, (bytes, str)) and len(year) == 4 and
813+
1 <= ord(year[2:3]) <= 12):
813814
# Pickle support
815+
if isinstance(year, str):
816+
try:
817+
year = year.encode('latin1')
818+
except UnicodeEncodeError:
819+
# More informative error message.
820+
raise ValueError(
821+
"Failed to encode latin1 string when unpickling "
822+
"a date object. "
823+
"pickle.load(data, encoding='latin1') is assumed.")
814824
self = object.__new__(cls)
815825
self.__setstate(year)
816826
self._hashcode = -1
@@ -1184,8 +1194,18 @@ def __new__(cls, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold
11841194
tzinfo (default to None)
11851195
fold (keyword only, default to zero)
11861196
"""
1187-
if isinstance(hour, bytes) and len(hour) == 6 and hour[0]&0x7F < 24:
1197+
if (isinstance(hour, (bytes, str)) and len(hour) == 6 and
1198+
ord(hour[0:1])&0x7F < 24):
11881199
# Pickle support
1200+
if isinstance(hour, str):
1201+
try:
1202+
hour = hour.encode('latin1')
1203+
except UnicodeEncodeError:
1204+
# More informative error message.
1205+
raise ValueError(
1206+
"Failed to encode latin1 string when unpickling "
1207+
"a time object. "
1208+
"pickle.load(data, encoding='latin1') is assumed.")
11891209
self = object.__new__(cls)
11901210
self.__setstate(hour, minute or None)
11911211
self._hashcode = -1
@@ -1496,8 +1516,18 @@ class datetime(date):
14961516

14971517
def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0,
14981518
microsecond=0, tzinfo=None, *, fold=0):
1499-
if isinstance(year, bytes) and len(year) == 10 and 1 <= year[2]&0x7F <= 12:
1519+
if (isinstance(year, (bytes, str)) and len(year) == 10 and
1520+
1 <= ord(year[2:3])&0x7F <= 12):
15001521
# Pickle support
1522+
if isinstance(year, str):
1523+
try:
1524+
year = bytes(year, 'latin1')
1525+
except UnicodeEncodeError:
1526+
# More informative error message.
1527+
raise ValueError(
1528+
"Failed to encode latin1 string when unpickling "
1529+
"a datetime object. "
1530+
"pickle.load(data, encoding='latin1') is assumed.")
15011531
self = object.__new__(cls)
15021532
self.__setstate(year, month)
15031533
self._hashcode = -1

‎Lib/test/datetimetester.py‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import _strptime
3939
#
4040

41+
pickle_loads = {pickle.loads, pickle._loads}
4142

4243
pickle_choices = [(pickle, pickle, proto)
4344
for proto in range(pickle.HIGHEST_PROTOCOL + 1)]
@@ -1434,6 +1435,19 @@ def test_pickling(self):
14341435
self.assertEqual(orig, derived)
14351436
self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2))
14361437

1438+
def test_compat_unpickle(self):
1439+
tests = [
1440+
b"cdatetime\ndate\n(S'\\x07\\xdf\\x0b\\x1b'\ntR.",
1441+
b'cdatetime\ndate\n(U\x04\x07\xdf\x0b\x1btR.',
1442+
b'\x80\x02cdatetime\ndate\nU\x04\x07\xdf\x0b\x1b\x85R.',
1443+
]
1444+
args = 2015, 11, 27
1445+
expected = self.theclass(*args)
1446+
for data in tests:
1447+
for loads in pickle_loads:
1448+
derived = loads(data, encoding='latin1')
1449+
self.assertEqual(derived, expected)
1450+
14371451
def test_compare(self):
14381452
t1 = self.theclass(2, 3, 4)
14391453
t2 = self.theclass(2, 3, 4)
@@ -2098,6 +2112,24 @@ def test_pickling_subclass_datetime(self):
20982112
derived = unpickler.loads(green)
20992113
self.assertEqual(orig, derived)
21002114

2115+
def test_compat_unpickle(self):
2116+
tests = [
2117+
b'cdatetime\ndatetime\n('
2118+
b"S'\\x07\\xdf\\x0b\\x1b\\x14;\\x01\\x00\\x10\\x00'\ntR.",
2119+
2120+
b'cdatetime\ndatetime\n('
2121+
b'U\n\x07\xdf\x0b\x1b\x14;\x01\x00\x10\x00tR.',
2122+
2123+
b'\x80\x02cdatetime\ndatetime\n'
2124+
b'U\n\x07\xdf\x0b\x1b\x14;\x01\x00\x10\x00\x85R.',
2125+
]
2126+
args = 2015, 11, 27, 20, 59, 1, 64**2
2127+
expected = self.theclass(*args)
2128+
for data in tests:
2129+
for loads in pickle_loads:
2130+
derived = loads(data, encoding='latin1')
2131+
self.assertEqual(derived, expected)
2132+
21012133
def test_more_compare(self):
21022134
# The test_compare() inherited from TestDate covers the error cases.
21032135
# We just want to test lexicographic ordering on the members datetime
@@ -3069,6 +3101,19 @@ def test_pickling_subclass_time(self):
30693101
derived = unpickler.loads(green)
30703102
self.assertEqual(orig, derived)
30713103

3104+
def test_compat_unpickle(self):
3105+
tests = [
3106+
b"cdatetime\ntime\n(S'\\x14;\\x10\\x00\\x10\\x00'\ntR.",
3107+
b'cdatetime\ntime\n(U\x06\x14;\x10\x00\x10\x00tR.',
3108+
b'\x80\x02cdatetime\ntime\nU\x06\x14;\x10\x00\x10\x00\x85R.',
3109+
]
3110+
args = 20, 59, 16, 64**2
3111+
expected = self.theclass(*args)
3112+
for data in tests:
3113+
for loads in pickle_loads:
3114+
derived = loads(data, encoding='latin1')
3115+
self.assertEqual(derived, expected)
3116+
30723117
def test_bool(self):
30733118
# time is always True.
30743119
cls = self.theclass
@@ -3441,6 +3486,40 @@ def test_pickling(self):
34413486
self.assertEqual(derived.tzname(), 'cookie')
34423487
self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2))
34433488

3489+
def test_compat_unpickle(self):
3490+
tests = [
3491+
b"cdatetime\ntime\n(S'\\x05\\x06\\x07\\x01\\xe2@'\n"
3492+
b"ctest.datetimetester\nPicklableFixedOffset\n(tR"
3493+
b"(dS'_FixedOffset__offset'\ncdatetime\ntimedelta\n"
3494+
b"(I-1\nI68400\nI0\ntRs"
3495+
b"S'_FixedOffset__dstoffset'\nNs"
3496+
b"S'_FixedOffset__name'\nS'cookie'\nsbtR.",
3497+
3498+
b'cdatetime\ntime\n(U\x06\x05\x06\x07\x01\xe2@'
3499+
b'ctest.datetimetester\nPicklableFixedOffset\n)R'
3500+
b'}(U\x14_FixedOffset__offsetcdatetime\ntimedelta\n'
3501+
b'(J\xff\xff\xff\xffJ0\x0b\x01\x00K\x00tR'
3502+
b'U\x17_FixedOffset__dstoffsetN'
3503+
b'U\x12_FixedOffset__nameU\x06cookieubtR.',
3504+
3505+
b'\x80\x02cdatetime\ntime\nU\x06\x05\x06\x07\x01\xe2@'
3506+
b'ctest.datetimetester\nPicklableFixedOffset\n)R'
3507+
b'}(U\x14_FixedOffset__offsetcdatetime\ntimedelta\n'
3508+
b'J\xff\xff\xff\xffJ0\x0b\x01\x00K\x00\x87R'
3509+
b'U\x17_FixedOffset__dstoffsetN'
3510+
b'U\x12_FixedOffset__nameU\x06cookieub\x86R.',
3511+
]
3512+
3513+
tinfo = PicklableFixedOffset(-300, 'cookie')
3514+
expected = self.theclass(5, 6, 7, 123456, tzinfo=tinfo)
3515+
for data in tests:
3516+
for loads in pickle_loads:
3517+
derived = loads(data, encoding='latin1')
3518+
self.assertEqual(derived, expected, repr(data))
3519+
self.assertIsInstance(derived.tzinfo, PicklableFixedOffset)
3520+
self.assertEqual(derived.utcoffset(), timedelta(minutes=-300))
3521+
self.assertEqual(derived.tzname(), 'cookie')
3522+
34443523
def test_more_bool(self):
34453524
# time is always True.
34463525
cls = self.theclass
@@ -3789,6 +3868,43 @@ def test_pickling(self):
37893868
self.assertEqual(derived.tzname(), 'cookie')
37903869
self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2))
37913870

3871+
def test_compat_unpickle(self):
3872+
tests = [
3873+
b'cdatetime\ndatetime\n'
3874+
b"(S'\\x07\\xdf\\x0b\\x1b\\x14;\\x01\\x01\\xe2@'\n"
3875+
b'ctest.datetimetester\nPicklableFixedOffset\n(tR'
3876+
b"(dS'_FixedOffset__offset'\ncdatetime\ntimedelta\n"
3877+
b'(I-1\nI68400\nI0\ntRs'
3878+
b"S'_FixedOffset__dstoffset'\nNs"
3879+
b"S'_FixedOffset__name'\nS'cookie'\nsbtR.",
3880+
3881+
b'cdatetime\ndatetime\n'
3882+
b'(U\n\x07\xdf\x0b\x1b\x14;\x01\x01\xe2@'
3883+
b'ctest.datetimetester\nPicklableFixedOffset\n)R'
3884+
b'}(U\x14_FixedOffset__offsetcdatetime\ntimedelta\n'
3885+
b'(J\xff\xff\xff\xffJ0\x0b\x01\x00K\x00tR'
3886+
b'U\x17_FixedOffset__dstoffsetN'
3887+
b'U\x12_FixedOffset__nameU\x06cookieubtR.',
3888+
3889+
b'\x80\x02cdatetime\ndatetime\n'
3890+
b'U\n\x07\xdf\x0b\x1b\x14;\x01\x01\xe2@'
3891+
b'ctest.datetimetester\nPicklableFixedOffset\n)R'
3892+
b'}(U\x14_FixedOffset__offsetcdatetime\ntimedelta\n'
3893+
b'J\xff\xff\xff\xffJ0\x0b\x01\x00K\x00\x87R'
3894+
b'U\x17_FixedOffset__dstoffsetN'
3895+
b'U\x12_FixedOffset__nameU\x06cookieub\x86R.',
3896+
]
3897+
args = 2015, 11, 27, 20, 59, 1, 123456
3898+
tinfo = PicklableFixedOffset(-300, 'cookie')
3899+
expected = self.theclass(*args, **{'tzinfo': tinfo})
3900+
for data in tests:
3901+
for loads in pickle_loads:
3902+
derived = loads(data, encoding='latin1')
3903+
self.assertEqual(derived, expected)
3904+
self.assertIsInstance(derived.tzinfo, PicklableFixedOffset)
3905+
self.assertEqual(derived.utcoffset(), timedelta(minutes=-300))
3906+
self.assertEqual(derived.tzname(), 'cookie')
3907+
37923908
def test_extreme_hashes(self):
37933909
# If an attempt is made to hash these via subtracting the offset
37943910
# then hashing a datetime object, OverflowError results. The

‎Misc/NEWS.d/next/Library/2017-10-12-22-39-55.bpo-22005.lGP-sc.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Implemented unpickling instances of :class:`~datetime.datetime`,
2+
:class:`~datetime.date` and :class:`~datetime.time` pickled by Python 2.
3+
``encoding='latin1'`` should be used for successful decoding.

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL