zhangyiss/python-by-example-150-challenges
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8th day of python challenges 111-117

Browse Source
This commit is contained in:
abd.shallal
2019-08-04 15:26:35 +03:00
parent b04c1b055f
commit 627802c383
3215 changed files with 760227 additions and 491 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/__init__.py Normal file
View File

2
venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/common.py Normal file
View File

@@ -0,0 +1,2 @@
frombytes = lambda obj, data: obj.frombytes(data)
tobytes = lambda obj: obj.tobytes()

22
venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_buffer.py Normal file
View File

@@ -0,0 +1,22 @@
# coding: utf-8
from pandas.io.msgpack import packb, unpackb
from .common import frombytes
def test_unpack_buffer():
from array import array
buf = array("b")
frombytes(buf, packb((b"foo", b"bar")))
obj = unpackb(buf, use_list=1)
assert [b"foo", b"bar"] == obj
def test_unpack_bytearray():
buf = bytearray(packb(("foo", "bar")))
obj = unpackb(buf, use_list=1)
assert [b"foo", b"bar"] == obj
expected_type = bytes
assert all(type(s) == expected_type for s in obj)

151
venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_case.py Normal file
View File

@@ -0,0 +1,151 @@
# coding: utf-8
from pandas.io.msgpack import packb, unpackb
def check(length, obj):
v = packb(obj)
assert (
len(v) == length
), "{obj!r} length should be {length!r} but get {got:!r}".format(
obj=obj, length=length, got=len(v)
)
assert unpackb(v, use_list=0) == obj
def test_1():
for o in [
None,
True,
False,
0,
1,
(1 << 6),
(1 << 7) - 1,
-1,
-((1 << 5) - 1),
-(1 << 5),
]:
check(1, o)
def test_2():
for o in [1 << 7, (1 << 8) - 1, -((1 << 5) + 1), -(1 << 7)]:
check(2, o)
def test_3():
for o in [1 << 8, (1 << 16) - 1, -((1 << 7) + 1), -(1 << 15)]:
check(3, o)
def test_5():
for o in [1 << 16, (1 << 32) - 1, -((1 << 15) + 1), -(1 << 31)]:
check(5, o)
def test_9():
for o in [
1 << 32,
(1 << 64) - 1,
-((1 << 31) + 1),
-(1 << 63),
1.0,
0.1,
-0.1,
-1.0,
]:
check(9, o)
def check_raw(overhead, num):
check(num + overhead, b" " * num)
def test_fixraw():
check_raw(1, 0)
check_raw(1, (1 << 5) - 1)
def test_raw16():
check_raw(3, 1 << 5)
check_raw(3, (1 << 16) - 1)
def test_raw32():
check_raw(5, 1 << 16)
def check_array(overhead, num):
check(num + overhead, (None,) * num)
def test_fixarray():
check_array(1, 0)
check_array(1, (1 << 4) - 1)
def test_array16():
check_array(3, 1 << 4)
check_array(3, (1 << 16) - 1)
def test_array32():
check_array(5, (1 << 16))
def match(obj, buf):
assert packb(obj) == buf
assert unpackb(buf, use_list=0) == obj
def test_match():
cases = [
(None, b"\xc0"),
(False, b"\xc2"),
(True, b"\xc3"),
(0, b"\x00"),
(127, b"\x7f"),
(128, b"\xcc\x80"),
(256, b"\xcd\x01\x00"),
(-1, b"\xff"),
(-33, b"\xd0\xdf"),
(-129, b"\xd1\xff\x7f"),
({1: 1}, b"\x81\x01\x01"),
(1.0, b"\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"),
((), b"\x90"),
(
tuple(range(15)),
(b"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09" b"\x0a\x0b\x0c\x0d\x0e"),
),
(
tuple(range(16)),
(
b"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07"
b"\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
),
),
({}, b"\x80"),
(
{x: x for x in range(15)},
(
b"\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07"
b"\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e"
),
),
(
{x: x for x in range(16)},
(
b"\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06"
b"\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e"
b"\x0f\x0f"
),
),
]
for v, p in cases:
match(v, p)
def test_unicode():
assert unpackb(packb("foobar"), use_list=1) == b"foobar"

38
venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_except.py Normal file
View File

@@ -0,0 +1,38 @@
# coding: utf-8
from datetime import datetime
import pytest
from pandas.io.msgpack import packb, unpackb
class DummyException(Exception):
pass
class TestExceptions:
def test_raise_on_find_unsupported_value(self):
msg = "can't serialize datetime"
with pytest.raises(TypeError, match=msg):
packb(datetime.now())
def test_raise_from_object_hook(self):
def hook(_):
raise DummyException()
with pytest.raises(DummyException):
unpackb(packb({}), object_hook=hook)
with pytest.raises(DummyException):
unpackb(packb({"fizz": "buzz"}), object_hook=hook)
with pytest.raises(DummyException):
unpackb(packb({"fizz": "buzz"}), object_pairs_hook=hook)
with pytest.raises(DummyException):
unpackb(packb({"fizz": {"buzz": "spam"}}), object_hook=hook)
with pytest.raises(DummyException):
unpackb(packb({"fizz": {"buzz": "spam"}}), object_pairs_hook=hook)
def test_invalid_value(self):
msg = "Unpack failed: error"
with pytest.raises(ValueError, match=msg):
unpackb(b"\xd9\x97#DL_")

63
venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_extension.py Normal file
View File

@@ -0,0 +1,63 @@
import array
import pandas.io.msgpack as msgpack
from pandas.io.msgpack import ExtType
from .common import frombytes, tobytes
def test_pack_ext_type():
def p(s):
packer = msgpack.Packer()
packer.pack_ext_type(0x42, s)
return packer.bytes()
assert p(b"A") == b"\xd4\x42A" # fixext 1
assert p(b"AB") == b"\xd5\x42AB" # fixext 2
assert p(b"ABCD") == b"\xd6\x42ABCD" # fixext 4
assert p(b"ABCDEFGH") == b"\xd7\x42ABCDEFGH" # fixext 8
assert p(b"A" * 16) == b"\xd8\x42" + b"A" * 16 # fixext 16
assert p(b"ABC") == b"\xc7\x03\x42ABC" # ext 8
assert p(b"A" * 0x0123) == b"\xc8\x01\x23\x42" + b"A" * 0x0123 # ext 16
assert (
p(b"A" * 0x00012345) == b"\xc9\x00\x01\x23\x45\x42" + b"A" * 0x00012345
) # ext 32
def test_unpack_ext_type():
def check(b, expected):
assert msgpack.unpackb(b) == expected
check(b"\xd4\x42A", ExtType(0x42, b"A")) # fixext 1
check(b"\xd5\x42AB", ExtType(0x42, b"AB")) # fixext 2
check(b"\xd6\x42ABCD", ExtType(0x42, b"ABCD")) # fixext 4
check(b"\xd7\x42ABCDEFGH", ExtType(0x42, b"ABCDEFGH")) # fixext 8
check(b"\xd8\x42" + b"A" * 16, ExtType(0x42, b"A" * 16)) # fixext 16
check(b"\xc7\x03\x42ABC", ExtType(0x42, b"ABC")) # ext 8
check(b"\xc8\x01\x23\x42" + b"A" * 0x0123, ExtType(0x42, b"A" * 0x0123)) # ext 16
check(
b"\xc9\x00\x01\x23\x45\x42" + b"A" * 0x00012345,
ExtType(0x42, b"A" * 0x00012345),
) # ext 32
def test_extension_type():
def default(obj):
print("default called", obj)
if isinstance(obj, array.array):
typecode = 123 # application specific typecode
data = tobytes(obj)
return ExtType(typecode, data)
raise TypeError("Unknown type object {obj!r}".format(obj))
def ext_hook(code, data):
print("ext_hook called", code, data)
assert code == 123
obj = array.array("d")
frombytes(obj, data)
return obj
obj = [42, b"hello", array.array("d", [1.1, 2.2, 3.3])]
s = msgpack.packb(obj, default=default)
obj2 = msgpack.unpackb(s, ext_hook=ext_hook)
assert obj == obj2

84
venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_format.py Normal file
View File

@@ -0,0 +1,84 @@
# coding: utf-8
from pandas.io.msgpack import unpackb
def check(src, should, use_list=0):
assert unpackb(src, use_list=use_list) == should
def testSimpleValue():
check(b"\x93\xc0\xc2\xc3", (None, False, True))
def testFixnum():
check(b"\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff", ((0, 64, 127), (-32, -16, -1)))
def testFixArray():
check(b"\x92\x90\x91\x91\xc0", ((), ((None,),)))
def testFixRaw():
check(b"\x94\xa0\xa1a\xa2bc\xa3def", (b"", b"a", b"bc", b"def"))
def testFixMap():
check(
b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80", {False: {None: None}, True: {None: {}}}
)
def testUnsignedInt():
check(
b"\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00"
b"\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00"
b"\xce\xff\xff\xff\xff",
(0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295),
)
def testSignedInt():
check(
b"\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00"
b"\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00"
b"\xd2\xff\xff\xff\xff",
(0, -128, -1, 0, -32768, -1, 0, -2147483648, -1),
)
def testRaw():
check(
b"\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00"
b"\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab",
(b"", b"a", b"ab", b"", b"a", b"ab"),
)
def testArray():
check(
b"\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00"
b"\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02"
b"\xc2\xc3",
((), (None,), (False, True), (), (None,), (False, True)),
)
def testMap():
check(
b"\x96"
b"\xde\x00\x00"
b"\xde\x00\x01\xc0\xc2"
b"\xde\x00\x02\xc0\xc2\xc3\xc2"
b"\xdf\x00\x00\x00\x00"
b"\xdf\x00\x00\x00\x01\xc0\xc2"
b"\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2",
(
{},
{None: False},
{True: False, None: False},
{},
{None: False},
{True: False, None: False},
),
)

107
venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_limits.py Normal file
View File

@@ -0,0 +1,107 @@
# coding: utf-8
import pytest
from pandas.io.msgpack import ExtType, Packer, Unpacker, packb, unpackb
class TestLimits:
def test_integer(self):
x = -(2 ** 63)
assert unpackb(packb(x)) == x
msg = (
r"((long |Python )?(int )?too (big|large) to convert"
r"( to C (unsigned )?long))?"
)
with pytest.raises((OverflowError, ValueError), match=msg):
packb(x - 1)
x = 2 ** 64 - 1
assert unpackb(packb(x)) == x
with pytest.raises((OverflowError, ValueError), match=msg):
packb(x + 1)
def test_array_header(self):
packer = Packer()
packer.pack_array_header(2 ** 32 - 1)
with pytest.raises((OverflowError, ValueError)):
packer.pack_array_header(2 ** 32)
def test_map_header(self):
packer = Packer()
packer.pack_map_header(2 ** 32 - 1)
with pytest.raises((OverflowError, ValueError)):
packer.pack_array_header(2 ** 32)
def test_max_str_len(self):
d = "x" * 3
packed = packb(d)
unpacker = Unpacker(max_str_len=3, encoding="utf-8")
unpacker.feed(packed)
assert unpacker.unpack() == d
unpacker = Unpacker(max_str_len=2, encoding="utf-8")
unpacker.feed(packed)
msg = "3 exceeds max_str_len"
with pytest.raises(ValueError, match=msg):
unpacker.unpack()
def test_max_bin_len(self):
d = b"x" * 3
packed = packb(d, use_bin_type=True)
unpacker = Unpacker(max_bin_len=3)
unpacker.feed(packed)
assert unpacker.unpack() == d
unpacker = Unpacker(max_bin_len=2)
unpacker.feed(packed)
msg = "3 exceeds max_bin_len"
with pytest.raises(ValueError, match=msg):
unpacker.unpack()
def test_max_array_len(self):
d = [1, 2, 3]
packed = packb(d)
unpacker = Unpacker(max_array_len=3)
unpacker.feed(packed)
assert unpacker.unpack() == d
unpacker = Unpacker(max_array_len=2)
unpacker.feed(packed)
msg = "3 exceeds max_array_len"
with pytest.raises(ValueError, match=msg):
unpacker.unpack()
def test_max_map_len(self):
d = {1: 2, 3: 4, 5: 6}
packed = packb(d)
unpacker = Unpacker(max_map_len=3)
unpacker.feed(packed)
assert unpacker.unpack() == d
unpacker = Unpacker(max_map_len=2)
unpacker.feed(packed)
msg = "3 exceeds max_map_len"
with pytest.raises(ValueError, match=msg):
unpacker.unpack()
def test_max_ext_len(self):
d = ExtType(42, b"abc")
packed = packb(d)
unpacker = Unpacker(max_ext_len=3)
unpacker.feed(packed)
assert unpacker.unpack() == d
unpacker = Unpacker(max_ext_len=2)
unpacker.feed(packed)
msg = "4 exceeds max_ext_len"
with pytest.raises(ValueError, match=msg):
unpacker.unpack()

90
venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_newspec.py Normal file
View File

@@ -0,0 +1,90 @@
# coding: utf-8
from pandas.io.msgpack import ExtType, packb, unpackb
def test_str8():
header = b"\xd9"
data = b"x" * 32
b = packb(data.decode(), use_bin_type=True)
assert len(b) == len(data) + 2
assert b[0:2] == header + b"\x20"
assert b[2:] == data
assert unpackb(b) == data
data = b"x" * 255
b = packb(data.decode(), use_bin_type=True)
assert len(b) == len(data) + 2
assert b[0:2] == header + b"\xff"
assert b[2:] == data
assert unpackb(b) == data
def test_bin8():
header = b"\xc4"
data = b""
b = packb(data, use_bin_type=True)
assert len(b) == len(data) + 2
assert b[0:2] == header + b"\x00"
assert b[2:] == data
assert unpackb(b) == data
data = b"x" * 255
b = packb(data, use_bin_type=True)
assert len(b) == len(data) + 2
assert b[0:2] == header + b"\xff"
assert b[2:] == data
assert unpackb(b) == data
def test_bin16():
header = b"\xc5"
data = b"x" * 256
b = packb(data, use_bin_type=True)
assert len(b) == len(data) + 3
assert b[0:1] == header
assert b[1:3] == b"\x01\x00"
assert b[3:] == data
assert unpackb(b) == data
data = b"x" * 65535
b = packb(data, use_bin_type=True)
assert len(b) == len(data) + 3
assert b[0:1] == header
assert b[1:3] == b"\xff\xff"
assert b[3:] == data
assert unpackb(b) == data
def test_bin32():
header = b"\xc6"
data = b"x" * 65536
b = packb(data, use_bin_type=True)
assert len(b) == len(data) + 5
assert b[0:1] == header
assert b[1:5] == b"\x00\x01\x00\x00"
assert b[5:] == data
assert unpackb(b) == data
def test_ext():
def check(ext, packed):
assert packb(ext) == packed
assert unpackb(packed) == ext
check(ExtType(0x42, b"Z"), b"\xd4\x42Z") # fixext 1
check(ExtType(0x42, b"ZZ"), b"\xd5\x42ZZ") # fixext 2
check(ExtType(0x42, b"Z" * 4), b"\xd6\x42" + b"Z" * 4) # fixext 4
check(ExtType(0x42, b"Z" * 8), b"\xd7\x42" + b"Z" * 8) # fixext 8
check(ExtType(0x42, b"Z" * 16), b"\xd8\x42" + b"Z" * 16) # fixext 16
# ext 8
check(ExtType(0x42, b""), b"\xc7\x00\x42")
check(ExtType(0x42, b"Z" * 255), b"\xc7\xff\x42" + b"Z" * 255)
# ext 16
check(ExtType(0x42, b"Z" * 256), b"\xc8\x01\x00\x42" + b"Z" * 256)
check(ExtType(0x42, b"Z" * 0xFFFF), b"\xc8\xff\xff\x42" + b"Z" * 0xFFFF)
# ext 32
check(ExtType(0x42, b"Z" * 0x10000), b"\xc9\x00\x01\x00\x00\x42" + b"Z" * 0x10000)
# needs large memory
# check(ExtType(0x42, b'Z'*0xffffffff),
# b'\xc9\xff\xff\xff\xff\x42' + b'Z'*0xffffffff)

71
venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_obj.py Normal file
View File

@@ -0,0 +1,71 @@
# coding: utf-8
import pytest
from pandas.io.msgpack import packb, unpackb
class DecodeError(Exception):
pass
class TestObj:
def _arr_to_str(self, arr):
return "".join(str(c) for c in arr)
def bad_complex_decoder(self, o):
raise DecodeError("Ooops!")
def _decode_complex(self, obj):
if b"__complex__" in obj:
return complex(obj[b"real"], obj[b"imag"])
return obj
def _encode_complex(self, obj):
if isinstance(obj, complex):
return {b"__complex__": True, b"real": 1, b"imag": 2}
return obj
def test_encode_hook(self):
packed = packb([3, 1 + 2j], default=self._encode_complex)
unpacked = unpackb(packed, use_list=1)
assert unpacked[1] == {b"__complex__": True, b"real": 1, b"imag": 2}
def test_decode_hook(self):
packed = packb([3, {b"__complex__": True, b"real": 1, b"imag": 2}])
unpacked = unpackb(packed, object_hook=self._decode_complex, use_list=1)
assert unpacked[1] == 1 + 2j
def test_decode_pairs_hook(self):
packed = packb([3, {1: 2, 3: 4}])
prod_sum = 1 * 2 + 3 * 4
unpacked = unpackb(
packed, object_pairs_hook=lambda l: sum(k * v for k, v in l), use_list=1
)
assert unpacked[1] == prod_sum
def test_only_one_obj_hook(self):
msg = "object_pairs_hook and object_hook are mutually exclusive"
with pytest.raises(TypeError, match=msg):
unpackb(b"", object_hook=lambda x: x, object_pairs_hook=lambda x: x)
def test_bad_hook(self):
msg = r"can't serialize \(1\+2j\)"
with pytest.raises(TypeError, match=msg):
packed = packb([3, 1 + 2j], default=lambda o: o)
unpacked = unpackb(packed, use_list=1) # noqa
def test_array_hook(self):
packed = packb([1, 2, 3])
unpacked = unpackb(packed, list_hook=self._arr_to_str, use_list=1)
assert unpacked == "123"
def test_an_exception_in_objecthook1(self):
with pytest.raises(DecodeError, match="Ooops!"):
packed = packb({1: {"__complex__": True, "real": 1, "imag": 2}})
unpackb(packed, object_hook=self.bad_complex_decoder)
def test_an_exception_in_objecthook2(self):
with pytest.raises(DecodeError, match="Ooops!"):
packed = packb({1: [{"__complex__": True, "real": 1, "imag": 2}]})
unpackb(packed, list_hook=self.bad_complex_decoder, use_list=1)

171
venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_pack.py Normal file
View File

@@ -0,0 +1,171 @@
# coding: utf-8
from collections import OrderedDict
from io import BytesIO
import struct
import pytest
from pandas.io.msgpack import Packer, Unpacker, packb, unpackb
class TestPack:
def check(self, data, use_list=False):
re = unpackb(packb(data), use_list=use_list)
assert re == data
def testPack(self):
test_data = [
0,
1,
127,
128,
255,
256,
65535,
65536,
-1,
-32,
-33,
-128,
-129,
-32768,
-32769,
1.0,
b"",
b"a",
b"a" * 31,
b"a" * 32,
None,
True,
False,
(),
((),),
((), None),
{None: 0},
(1 << 23),
]
for td in test_data:
self.check(td)
def testPackUnicode(self):
test_data = ["", "abcd", ["defgh"], "Русский текст"]
for td in test_data:
re = unpackb(packb(td, encoding="utf-8"), use_list=1, encoding="utf-8")
assert re == td
packer = Packer(encoding="utf-8")
data = packer.pack(td)
re = Unpacker(BytesIO(data), encoding="utf-8", use_list=1).unpack()
assert re == td
def testPackUTF32(self):
test_data = ["", "abcd", ["defgh"], "Русский текст"]
for td in test_data:
re = unpackb(packb(td, encoding="utf-32"), use_list=1, encoding="utf-32")
assert re == td
def testPackBytes(self):
test_data = [b"", b"abcd", (b"defgh",)]
for td in test_data:
self.check(td)
def testIgnoreUnicodeErrors(self):
re = unpackb(
packb(b"abc\xeddef"), encoding="utf-8", unicode_errors="ignore", use_list=1
)
assert re == "abcdef"
def testStrictUnicodeUnpack(self):
msg = (
r"'utf-*8' codec can't decode byte 0xed in position 3:"
" invalid continuation byte"
)
with pytest.raises(UnicodeDecodeError, match=msg):
unpackb(packb(b"abc\xeddef"), encoding="utf-8", use_list=1)
def testStrictUnicodePack(self):
msg = (
r"'ascii' codec can't encode character '\\xed' in position 3:"
r" ordinal not in range\(128\)"
)
with pytest.raises(UnicodeEncodeError, match=msg):
packb("abc\xeddef", encoding="ascii", unicode_errors="strict")
def testIgnoreErrorsPack(self):
re = unpackb(
packb("abcФФФdef", encoding="ascii", unicode_errors="ignore"),
encoding="utf-8",
use_list=1,
)
assert re == "abcdef"
def testNoEncoding(self):
msg = "Can't encode unicode string: no encoding is specified"
with pytest.raises(TypeError, match=msg):
packb("abc", encoding=None)
def testDecodeBinary(self):
re = unpackb(packb("abc"), encoding=None, use_list=1)
assert re == b"abc"
def testPackFloat(self):
assert packb(1.0, use_single_float=True) == b"\xca" + struct.pack(">f", 1.0)
assert packb(1.0, use_single_float=False) == b"\xcb" + struct.pack(">d", 1.0)
def testArraySize(self, sizes=[0, 5, 50, 1000]):
bio = BytesIO()
packer = Packer()
for size in sizes:
bio.write(packer.pack_array_header(size))
for i in range(size):
bio.write(packer.pack(i))
bio.seek(0)
unpacker = Unpacker(bio, use_list=1)
for size in sizes:
assert unpacker.unpack() == list(range(size))
def test_manualreset(self, sizes=[0, 5, 50, 1000]):
packer = Packer(autoreset=False)
for size in sizes:
packer.pack_array_header(size)
for i in range(size):
packer.pack(i)
bio = BytesIO(packer.bytes())
unpacker = Unpacker(bio, use_list=1)
for size in sizes:
assert unpacker.unpack() == list(range(size))
packer.reset()
assert packer.bytes() == b""
def testMapSize(self, sizes=[0, 5, 50, 1000]):
bio = BytesIO()
packer = Packer()
for size in sizes:
bio.write(packer.pack_map_header(size))
for i in range(size):
bio.write(packer.pack(i)) # key
bio.write(packer.pack(i * 2)) # value
bio.seek(0)
unpacker = Unpacker(bio)
for size in sizes:
assert unpacker.unpack() == {i: i * 2 for i in range(size)}
def test_odict(self):
seq = [(b"one", 1), (b"two", 2), (b"three", 3), (b"four", 4)]
od = OrderedDict(seq)
assert unpackb(packb(od), use_list=1) == dict(seq)
def pair_hook(seq):
return list(seq)
assert unpackb(packb(od), object_pairs_hook=pair_hook, use_list=1) == seq
def test_pairlist(self):
pairlist = [(b"a", 1), (2, b"b"), (b"foo", b"bar")]
packer = Packer()
packed = packer.pack_map_pairs(pairlist)
unpacked = unpackb(packed, object_pairs_hook=list)
assert pairlist == unpacked

71
venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_read_size.py Normal file
View File

@@ -0,0 +1,71 @@
"""Test Unpacker's read_array_header and read_map_header methods"""
from pandas.io.msgpack import OutOfData, Unpacker, packb
UnexpectedTypeException = ValueError
def test_read_array_header():
unpacker = Unpacker()
unpacker.feed(packb(["a", "b", "c"]))
assert unpacker.read_array_header() == 3
assert unpacker.unpack() == b"a"
assert unpacker.unpack() == b"b"
assert unpacker.unpack() == b"c"
try:
unpacker.unpack()
assert 0, "should raise exception"
except OutOfData:
assert 1, "okay"
def test_read_map_header():
unpacker = Unpacker()
unpacker.feed(packb({"a": "A"}))
assert unpacker.read_map_header() == 1
assert unpacker.unpack() == b"a"
assert unpacker.unpack() == b"A"
try:
unpacker.unpack()
assert 0, "should raise exception"
except OutOfData:
assert 1, "okay"
def test_incorrect_type_array():
unpacker = Unpacker()
unpacker.feed(packb(1))
try:
unpacker.read_array_header()
assert 0, "should raise exception"
except UnexpectedTypeException:
assert 1, "okay"
def test_incorrect_type_map():
unpacker = Unpacker()
unpacker.feed(packb(1))
try:
unpacker.read_map_header()
assert 0, "should raise exception"
except UnexpectedTypeException:
assert 1, "okay"
def test_correct_type_nested_array():
unpacker = Unpacker()
unpacker.feed(packb({"a": ["b", "c", "d"]}))
try:
unpacker.read_array_header()
assert 0, "should raise exception"
except UnexpectedTypeException:
assert 1, "okay"
def test_incorrect_type_nested_map():
unpacker = Unpacker()
unpacker.feed(packb([{"a": "b"}]))
try:
unpacker.read_map_header()
assert 0, "should raise exception"
except UnexpectedTypeException:
assert 1, "okay"

47
venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_seq.py Normal file
View File

@@ -0,0 +1,47 @@
# coding: utf-8
import io
import pandas.io.msgpack as msgpack
binarydata = bytes(bytearray(range(256)))
def gen_binary_data(idx):
return binarydata[: idx % 300]
def test_exceeding_unpacker_read_size():
dumpf = io.BytesIO()
packer = msgpack.Packer()
NUMBER_OF_STRINGS = 6
read_size = 16
# 5 ok for read_size=16, while 6 glibc detected *** python: double free or
# corruption (fasttop):
# 20 ok for read_size=256, while 25 segfaults / glibc detected *** python:
# double free or corruption (!prev)
# 40 ok for read_size=1024, while 50 introduces errors
# 7000 ok for read_size=1024*1024, while 8000 leads to glibc detected ***
# python: double free or corruption (!prev):
for idx in range(NUMBER_OF_STRINGS):
data = gen_binary_data(idx)
dumpf.write(packer.pack(data))
f = io.BytesIO(dumpf.getvalue())
dumpf.close()
unpacker = msgpack.Unpacker(f, read_size=read_size, use_list=1)
read_count = 0
for idx, o in enumerate(unpacker):
assert type(o) == bytes
assert o == gen_binary_data(idx)
read_count += 1
assert read_count == NUMBER_OF_STRINGS

102
venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_sequnpack.py Normal file
View File

@@ -0,0 +1,102 @@
# coding: utf-8
from io import BytesIO
import pytest
from pandas.io.msgpack import BufferFull, OutOfData, Unpacker
class TestPack:
def test_partial_data(self):
unpacker = Unpacker()
msg = "No more data to unpack"
for data in [b"\xa5", b"h", b"a", b"l", b"l"]:
unpacker.feed(data)
with pytest.raises(StopIteration, match=msg):
next(iter(unpacker))
unpacker.feed(b"o")
assert next(iter(unpacker)) == b"hallo"
def test_foobar(self):
unpacker = Unpacker(read_size=3, use_list=1)
unpacker.feed(b"foobar")
assert unpacker.unpack() == ord(b"f")
assert unpacker.unpack() == ord(b"o")
assert unpacker.unpack() == ord(b"o")
assert unpacker.unpack() == ord(b"b")
assert unpacker.unpack() == ord(b"a")
assert unpacker.unpack() == ord(b"r")
msg = "No more data to unpack"
with pytest.raises(OutOfData, match=msg):
unpacker.unpack()
unpacker.feed(b"foo")
unpacker.feed(b"bar")
k = 0
for o, e in zip(unpacker, "foobarbaz"):
assert o == ord(e)
k += 1
assert k == len(b"foobar")
def test_foobar_skip(self):
unpacker = Unpacker(read_size=3, use_list=1)
unpacker.feed(b"foobar")
assert unpacker.unpack() == ord(b"f")
unpacker.skip()
assert unpacker.unpack() == ord(b"o")
unpacker.skip()
assert unpacker.unpack() == ord(b"a")
unpacker.skip()
msg = "No more data to unpack"
with pytest.raises(OutOfData, match=msg):
unpacker.unpack()
def test_maxbuffersize_read_size_exceeds_max_buffer_size(self):
msg = "read_size should be less or equal to max_buffer_size"
with pytest.raises(ValueError, match=msg):
Unpacker(read_size=5, max_buffer_size=3)
def test_maxbuffersize_bufferfull(self):
unpacker = Unpacker(read_size=3, max_buffer_size=3, use_list=1)
unpacker.feed(b"foo")
with pytest.raises(BufferFull, match=r"^$"):
unpacker.feed(b"b")
def test_maxbuffersize(self):
unpacker = Unpacker(read_size=3, max_buffer_size=3, use_list=1)
unpacker.feed(b"foo")
assert ord("f") == next(unpacker)
unpacker.feed(b"b")
assert ord("o") == next(unpacker)
assert ord("o") == next(unpacker)
assert ord("b") == next(unpacker)
def test_readbytes(self):
unpacker = Unpacker(read_size=3)
unpacker.feed(b"foobar")
assert unpacker.unpack() == ord(b"f")
assert unpacker.read_bytes(3) == b"oob"
assert unpacker.unpack() == ord(b"a")
assert unpacker.unpack() == ord(b"r")
# Test buffer refill
unpacker = Unpacker(BytesIO(b"foobar"), read_size=3)
assert unpacker.unpack() == ord(b"f")
assert unpacker.read_bytes(3) == b"oob"
assert unpacker.unpack() == ord(b"a")
assert unpacker.unpack() == ord(b"r")
def test_issue124(self):
unpacker = Unpacker()
unpacker.feed(b"\xa1?\xa1!")
assert tuple(unpacker) == (b"?", b"!")
assert tuple(unpacker) == ()
unpacker.feed(b"\xa1?\xa1")
assert tuple(unpacker) == (b"?",)
assert tuple(unpacker) == ()
unpacker.feed(b"!")
assert tuple(unpacker) == (b"!",)
assert tuple(unpacker) == ()

26
venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_subtype.py Normal file
View File

@@ -0,0 +1,26 @@
# coding: utf-8
from collections import namedtuple
from pandas.io.msgpack import packb
class MyList(list):
pass
class MyDict(dict):
pass
class MyTuple(tuple):
pass
MyNamedTuple = namedtuple("MyNamedTuple", "x y")
def test_types():
assert packb(MyDict()) == packb(dict())
assert packb(MyList()) == packb(list())
assert packb(MyNamedTuple(1, 2)) == packb((1, 2))

64
venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_unpack.py Normal file
View File

@@ -0,0 +1,64 @@
from io import BytesIO
import sys
import pytest
from pandas.io.msgpack import ExtType, OutOfData, Unpacker, packb
class TestUnpack:
def test_unpack_array_header_from_file(self):
f = BytesIO(packb([1, 2, 3, 4]))
unpacker = Unpacker(f)
assert unpacker.read_array_header() == 4
assert unpacker.unpack() == 1
assert unpacker.unpack() == 2
assert unpacker.unpack() == 3
assert unpacker.unpack() == 4
msg = "No more data to unpack"
with pytest.raises(OutOfData, match=msg):
unpacker.unpack()
def test_unpacker_hook_refcnt(self):
if not hasattr(sys, "getrefcount"):
pytest.skip("no sys.getrefcount()")
result = []
def hook(x):
result.append(x)
return x
basecnt = sys.getrefcount(hook)
up = Unpacker(object_hook=hook, list_hook=hook)
assert sys.getrefcount(hook) >= basecnt + 2
up.feed(packb([{}]))
up.feed(packb([{}]))
assert up.unpack() == [{}]
assert up.unpack() == [{}]
assert result == [{}, [{}], {}, [{}]]
del up
assert sys.getrefcount(hook) == basecnt
def test_unpacker_ext_hook(self):
class MyUnpacker(Unpacker):
def __init__(self):
super().__init__(ext_hook=self._hook, encoding="utf-8")
def _hook(self, code, data):
if code == 1:
return int(data)
else:
return ExtType(code, data)
unpacker = MyUnpacker()
unpacker.feed(packb({"a": 1}, encoding="utf-8"))
assert unpacker.unpack() == {"a": 1}
unpacker.feed(packb({"a": ExtType(1, b"123")}, encoding="utf-8"))
assert unpacker.unpack() == {"a": 123}
unpacker.feed(packb({"a": ExtType(2, b"321")}, encoding="utf-8"))
assert unpacker.unpack() == {"a": ExtType(2, b"321")}

30
venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_unpack_raw.py Normal file
View File

@@ -0,0 +1,30 @@
"""Tests for cases where the user seeks to obtain packed msgpack objects"""
import io
from pandas.io.msgpack import Unpacker, packb
def test_write_bytes():
unpacker = Unpacker()
unpacker.feed(b"abc")
f = io.BytesIO()
assert unpacker.unpack(f.write) == ord("a")
assert f.getvalue() == b"a"
f = io.BytesIO()
assert unpacker.skip(f.write) is None
assert f.getvalue() == b"b"
f = io.BytesIO()
assert unpacker.skip() is None
assert f.getvalue() == b""
def test_write_bytes_multi_buffer():
long_val = (5) * 100
expected = packb(long_val)
unpacker = Unpacker(io.BytesIO(expected), read_size=3, max_buffer_size=3)
f = io.BytesIO()
unpacked = unpacker.unpack(f.write)
assert unpacked == long_val
assert f.getvalue() == expected