From 6422acdcdd80afc3ffbf88fb6cdafecbb7c1eea7 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Wed, 3 Jul 2019 13:31:03 +0100 Subject: [PATCH 1/4] Cosmetic change: Define Bitlist/Bitvector serialization using bytes, not bigints --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index d5855a755..a4abef966 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -41,11 +41,16 @@ def serialize(obj: SSZValue): if isinstance(obj, BasicValue): return serialize_basic(obj) elif isinstance(obj, Bitvector): - as_integer = sum([obj[i] << i for i in range(len(obj))]) - return as_integer.to_bytes((len(obj) + 7) // 8, "little") + as_bytearray = [0] * ((len(obj) + 7) // 8) + for i in range(len(obj)): + as_bytearray[i // 8] |= obj[i] << (i % 8) + return bytes(as_bytearray) elif isinstance(obj, Bitlist): - as_integer = (1 << len(obj)) + sum([obj[i] << i for i in range(len(obj))]) - return as_integer.to_bytes((as_integer.bit_length() + 7) // 8, "little") + as_bytearray = [0] * (len(obj) // 8 + 1) + for i in range(len(obj)): + as_bytearray[i // 8] |= obj[i] << (i % 8) + as_bytearray[len(obj) // 8] |= 1 << (len(obj) % 8) + return bytes(as_bytearray) elif isinstance(obj, Series): return encode_series(obj) else: @@ -92,12 +97,11 @@ def encode_series(values: Series): def pack(values: Series): if isinstance(values, bytes): # Bytes and BytesN are already packed return values - elif isinstance(values, Bitvector): - as_integer = sum([values[i] << i for i in range(len(values))]) - return as_integer.to_bytes((values.length + 7) // 8, "little") - elif isinstance(values, Bitlist): - as_integer = sum([values[i] << i for i in range(len(values))]) - return as_integer.to_bytes((values.length + 7) // 8, "little") + elif isinstance(values, Bitvector) or isinstance(values, Bitlist): + as_bytearray = [0] * ((len(values) + 7) // 8) + for i in range(len(values)): + as_bytearray[i // 8] |= values[i] << (i % 8) + return bytes(as_bytearray) return b''.join([serialize_basic(value) for value in values]) From 619b2a3573b3a0c83cc50e13adb7d97a3816539a Mon Sep 17 00:00:00 2001 From: dankrad Date: Wed, 3 Jul 2019 15:10:37 +0100 Subject: [PATCH 2/4] Update test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py Co-Authored-By: Diederik Loerakker --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index a4abef966..1e0c806d9 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -97,7 +97,7 @@ def encode_series(values: Series): def pack(values: Series): if isinstance(values, bytes): # Bytes and BytesN are already packed return values - elif isinstance(values, Bitvector) or isinstance(values, Bitlist): + elif isinstance(values, (Bitvector, Bitlist)): as_bytearray = [0] * ((len(values) + 7) // 8) for i in range(len(values)): as_bytearray[i // 8] |= values[i] << (i % 8) From b98679957b5e056e852c4a5f57b1eab4d29da118 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 12 Jul 2019 22:11:33 +0200 Subject: [PATCH 3/4] use as_bytes function to reduce code duplication, and for later usage --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 23 ++++++++----------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 7 +++++- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 1e0c806d9..2a7d92314 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -41,15 +41,13 @@ def serialize(obj: SSZValue): if isinstance(obj, BasicValue): return serialize_basic(obj) elif isinstance(obj, Bitvector): - as_bytearray = [0] * ((len(obj) + 7) // 8) - for i in range(len(obj)): - as_bytearray[i // 8] |= obj[i] << (i % 8) - return bytes(as_bytearray) + return obj.as_bytes() elif isinstance(obj, Bitlist): - as_bytearray = [0] * (len(obj) // 8 + 1) - for i in range(len(obj)): - as_bytearray[i // 8] |= obj[i] << (i % 8) - as_bytearray[len(obj) // 8] |= 1 << (len(obj) % 8) + as_bytearray = list(obj.as_bytes()) + if len(obj) % 8 == 0: + as_bytearray.append(1) + else: + as_bytearray[len(obj) // 8] |= 1 << (len(obj) % 8) return bytes(as_bytearray) elif isinstance(obj, Series): return encode_series(obj) @@ -97,11 +95,10 @@ def encode_series(values: Series): def pack(values: Series): if isinstance(values, bytes): # Bytes and BytesN are already packed return values - elif isinstance(values, (Bitvector, Bitlist)): - as_bytearray = [0] * ((len(values) + 7) // 8) - for i in range(len(values)): - as_bytearray[i // 8] |= values[i] << (i % 8) - return bytes(as_bytearray) + elif isinstance(values, Bits): + # packs the bits in bytes, left-aligned. + # Exclusive length delimiting bits for bitlists. + return values.as_bytes() return b''.join([serialize_basic(value) for value in values]) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 2ec4b5ce2..1f199e6e1 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -354,7 +354,12 @@ class BitElementsType(ElementsType): class Bits(BaseList, metaclass=BitElementsType): - pass + + def as_bytes(self): + as_bytearray = [0] * ((len(self) + 7) // 8) + for i in range(len(self)): + as_bytearray[i // 8] |= int(self[i]) << (i % 8) + return bytes(as_bytearray) class Bitlist(Bits): From ac6d019870b8a79ad57a9043f30ec5cc4eafe82e Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 12 Jul 2019 22:20:07 +0200 Subject: [PATCH 4/4] bits serialization clear now, directly to bytes --- specs/simple-serialize.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 8d9c33103..915cb772a 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -120,8 +120,10 @@ return b"" ### `Bitvector[N]` ```python -as_integer = sum([value[i] << i for i in range(len(value))]) -return as_integer.to_bytes((N + 7) // 8, "little") +array = [0] * ((N + 7) // 8) +for i in range(N): + array[i // 8] |= value[i] << (i % 8) +return bytes(array) ``` ### `Bitlist[N]` @@ -129,8 +131,11 @@ return as_integer.to_bytes((N + 7) // 8, "little") Note that from the offset coding, the length (in bytes) of the bitlist is known. An additional leading `1` bit is added so that the length in bits will also be known. ```python -as_integer = (1 << len(value)) + sum([value[i] << i for i in range(len(value))]) -return as_integer.to_bytes((as_integer.bit_length() + 7) // 8, "little") +array = [0] * ((len(value) // 8) + 1) +for i in range(len(value)): + array[i // 8] |= value[i] << (i % 8) +array[len(value) // 8] |= 1 << (len(value) % 8) +return bytes(array) ``` ### Vectors, containers, lists, unions