FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

fallback: count buffer bytes independently of memoryview shape by zhechen-cs · Pull Request #753 · msgpack/msgpack-python · GitHub

Repository navigation

Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension .py  (2) All 1 file type selected
Viewed files
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Unified
Split
Hide whitespace
Diff view
Unified
Split
Hide whitespace
7 changes: 5 additions & 2 deletions msgpack/fallback.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,9 @@ def _get_data_from_buffer(obj):
view = memoryview(obj)
if view.itemsize != 1:
raise ValueError("cannot unpack from multi-byte object")
return view
if not view.c_contiguous:
return memoryview(view.tobytes())
return view.cast("B")


def unpackb(packed, **kwargs):
Expand All @@ -92,6 +94,7 @@ def unpackb(packed, **kwargs):

See :class:`Unpacker` for options.
"""
packed = _get_data_from_buffer(packed)
unpacker = Unpacker(None, max_buffer_size=len(packed), **kwargs)
unpacker.feed(packed)
try:
Expand Down Expand Up @@ -339,7 +342,7 @@ def feed(self, next_bytes):
self._buf_checkpoint = 0

# Use extend here: INPLACE_ADD += doesn't reliably typecast memoryview in jython
self._buffer.extend(view if view.contiguous else view.tobytes())
self._buffer.extend(view)
view.release()

def _consume(self):
Expand Down
45 changes: 43 additions & 2 deletions test/test_buffer.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from pytest import raises
from pytest import mark, raises

from msgpack import Packer, packb, unpackb
from msgpack import BufferFull, Packer, Unpacker, packb, unpackb


def test_unpack_buffer():
Expand Down Expand Up @@ -47,3 +47,44 @@ def test_packer_getbuffer():
buffer.release()
packer.pack(42)
assert bytes(packer) == b"\x92*\xa5hello*"


@mark.parametrize("shape", [(2, 3), (3, 2), (1, 6)])
def test_unpack_multidimensional_buffer(shape):
packed = packb(b"abcd")
view = memoryview(packed).cast("B", shape)
assert unpackb(view) == b"abcd"

unpacker = Unpacker(max_buffer_size=len(packed))
unpacker.feed(view)
assert unpacker.unpack() == b"abcd"


def test_feed_multidimensional_buffer_limit():
view = memoryview(packb(b"abcd")).cast("B", (2, 3))
unpacker = Unpacker(max_buffer_size=5)
unpacker.feed(packb(42))
with raises(BufferFull):
unpacker.feed(view)
assert unpacker.unpack() == 42
unpacker.feed(packb(43))
assert unpacker.unpack() == 43


def test_unpack_scalar_buffer():
view = memoryview(packb(42)).cast("B", ())
assert unpackb(view) == 42
unpacker = Unpacker()
unpacker.feed(view)
assert unpacker.unpack() == 42


def test_unpack_strided_multidimensional_buffer():
packed = packb(b"abcd")
padded = packed[:3] + b"xxx" + packed[3:] + b"xxx"
view = memoryview(padded).cast("B", (4, 3))[::2]
assert not view.contiguous
assert unpackb(view) == b"abcd"
unpacker = Unpacker(max_buffer_size=len(packed))
unpacker.feed(view)
assert unpacker.unpack() == b"abcd"
Loading

Back | FazBrowse Home | New Git URL