From 761c9e55fe879a4e19b907e1a2d9673e93aebc20 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 12 May 2019 23:16:17 +0200 Subject: [PATCH 01/48] SSZ impl. rework started, see issue 1064 --- .../pyspec/eth2spec/utils/merkle_minimal.py | 32 +- .../pyspec/eth2spec/utils/minimal_ssz.py | 331 ------------------ .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 117 +++++++ .../pyspec/eth2spec/utils/ssz/ssz_switch.py | 105 ++++++ .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 115 ++++++ 5 files changed, 368 insertions(+), 332 deletions(-) delete mode 100644 test_libs/pyspec/eth2spec/utils/minimal_ssz.py create mode 100644 test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py create mode 100644 test_libs/pyspec/eth2spec/utils/ssz/ssz_switch.py create mode 100644 test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py index 7c5483de3..e3e5d35d8 100644 --- a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py @@ -1,7 +1,9 @@ from .hash_function import hash -zerohashes = [b'\x00' * 32] +ZERO_BYTES32 = b'\x00' * 32 + +zerohashes = [ZERO_BYTES32] for layer in range(1, 32): zerohashes.append(hash(zerohashes[layer - 1] + zerohashes[layer - 1])) @@ -28,3 +30,31 @@ def get_merkle_proof(tree, item_index): subindex = (item_index // 2**i) ^ 1 proof.append(tree[i][subindex] if subindex < len(tree[i]) else zerohashes[i]) return proof + + +def next_power_of_two(v: int) -> int: + """ + Get the next power of 2. (for 64 bit range ints) + Examples: + 0 -> 0, 1 -> 1, 2 -> 2, 3 -> 4, 32 -> 32, 33 -> 64 + """ + # effectively fill the bitstring (1 less, do not want to with ones, then increment for next power of 2. + v -= 1 + v |= v >> (1 << 0) + v |= v >> (1 << 1) + v |= v >> (1 << 2) + v |= v >> (1 << 3) + v |= v >> (1 << 4) + v |= v >> (1 << 5) + v += 1 + return v + + +def merkleize_chunks(chunks): + tree = chunks[::] + margin = next_power_of_two(len(chunks)) - len(chunks) + tree.extend([ZERO_BYTES32] * margin) + tree = [ZERO_BYTES32] * len(tree) + tree + for i in range(len(tree) // 2 - 1, 0, -1): + tree[i] = hash(tree[i * 2] + tree[i * 2 + 1]) + return tree[1] diff --git a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py deleted file mode 100644 index 9cc2baebb..000000000 --- a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py +++ /dev/null @@ -1,331 +0,0 @@ -from typing import Any - -from .hash_function import hash - -BYTES_PER_CHUNK = 32 -BYTES_PER_LENGTH_OFFSET = 4 -ZERO_CHUNK = b'\x00' * BYTES_PER_CHUNK - - -def SSZType(fields): - class SSZObject(): - def __init__(self, **kwargs): - for f, t in fields.items(): - if f not in kwargs: - setattr(self, f, get_zero_value(t)) - else: - setattr(self, f, kwargs[f]) - - def __eq__(self, other): - return self.fields == other.fields and self.serialize() == other.serialize() - - def __hash__(self): - return int.from_bytes(self.hash_tree_root(), byteorder="little") - - def __str__(self): - output = [] - for field in self.fields: - output.append(f'{field}: {getattr(self, field)}') - return "\n".join(output) - - def serialize(self): - return serialize_value(self, self.__class__) - - def hash_tree_root(self): - return hash_tree_root(self, self.__class__) - - SSZObject.fields = fields - return SSZObject - - -class Vector(): - def __init__(self, items): - self.items = items - self.length = len(items) - - def __getitem__(self, key): - return self.items[key] - - def __setitem__(self, key, value): - self.items[key] = value - - def __iter__(self): - return iter(self.items) - - def __len__(self): - return self.length - - -def is_basic(typ): - # if not a string, it is a complex, and cannot be basic - if not isinstance(typ, str): - return False - # "uintN": N-bit unsigned integer (where N in [8, 16, 32, 64, 128, 256]) - elif typ[:4] == 'uint' and typ[4:] in ['8', '16', '32', '64', '128', '256']: - return True - # "bool": True or False - elif typ == 'bool': - return True - # alias: "byte" -> "uint8" - elif typ == 'byte': - return True - # default - else: - return False - - -def is_constant_sized(typ): - # basic objects are fixed size by definition - if is_basic(typ): - return True - # dynamic size array type, "list": [elem_type]. - # Not constant size by definition. - elif isinstance(typ, list) and len(typ) == 1: - return False - # fixed size array type, "vector": [elem_type, length] - # Constant size, but only if the elements are. - elif isinstance(typ, list) and len(typ) == 2: - return is_constant_sized(typ[0]) - # bytes array (fixed or dynamic size) - elif isinstance(typ, str) and typ[:5] == 'bytes': - # if no length suffix, it has a dynamic size - return typ != 'bytes' - # containers are only constant-size if all of the fields are constant size. - elif hasattr(typ, 'fields'): - for subtype in typ.fields.values(): - if not is_constant_sized(subtype): - return False - return True - else: - raise Exception("Type not recognized") - - -def coerce_to_bytes(x): - if isinstance(x, str): - o = x.encode('utf-8') - assert len(o) == len(x) - return o - elif isinstance(x, bytes): - return x - else: - raise Exception("Expecting bytes") - - -def encode_series(values, types): - # Recursively serialize - parts = [(is_constant_sized(types[i]), serialize_value(values[i], types[i])) for i in range(len(values))] - - # Compute and check lengths - fixed_lengths = [len(serialized) if constant_size else BYTES_PER_LENGTH_OFFSET - for (constant_size, serialized) in parts] - variable_lengths = [len(serialized) if not constant_size else 0 - for (constant_size, serialized) in parts] - - # Check if integer is not out of bounds (Python) - assert sum(fixed_lengths + variable_lengths) < 2 ** (BYTES_PER_LENGTH_OFFSET * 8) - - # Interleave offsets of variable-size parts with fixed-size parts. - # Avoid quadratic complexity in calculation of offsets. - offset = sum(fixed_lengths) - variable_parts = [] - fixed_parts = [] - for (constant_size, serialized) in parts: - if constant_size: - fixed_parts.append(serialized) - else: - fixed_parts.append(offset.to_bytes(BYTES_PER_LENGTH_OFFSET, 'little')) - variable_parts.append(serialized) - offset += len(serialized) - - # Return the concatenation of the fixed-size parts (offsets interleaved) with the variable-size parts - return b"".join(fixed_parts + variable_parts) - - -def serialize_value(value, typ=None): - if typ is None: - typ = infer_type(value) - # "uintN" - if isinstance(typ, str) and typ[:4] == 'uint': - length = int(typ[4:]) - assert length in (8, 16, 32, 64, 128, 256) - return value.to_bytes(length // 8, 'little') - # "bool" - elif isinstance(typ, str) and typ == 'bool': - assert value in (True, False) - return b'\x01' if value is True else b'\x00' - # Vector - elif isinstance(typ, list) and len(typ) == 2: - # (regardless of element type, sanity-check if the length reported in the vector type matches the value length) - assert len(value) == typ[1] - return encode_series(value, [typ[0]] * len(value)) - # List - elif isinstance(typ, list) and len(typ) == 1: - return encode_series(value, [typ[0]] * len(value)) - # "bytes" (variable size) - elif isinstance(typ, str) and typ == 'bytes': - return coerce_to_bytes(value) - # "bytesN" (fixed size) - elif isinstance(typ, str) and len(typ) > 5 and typ[:5] == 'bytes': - assert len(value) == int(typ[5:]), (value, int(typ[5:])) - return coerce_to_bytes(value) - # containers - elif hasattr(typ, 'fields'): - values = [getattr(value, field) for field in typ.fields.keys()] - types = list(typ.fields.values()) - return encode_series(values, types) - else: - print(value, typ) - raise Exception("Type not recognized") - - -def get_zero_value(typ: Any) -> Any: - if isinstance(typ, str): - # Bytes array - if typ == 'bytes': - return b'' - # bytesN - elif typ[:5] == 'bytes' and len(typ) > 5: - length = int(typ[5:]) - return b'\x00' * length - # Basic types - elif typ == 'bool': - return False - elif typ[:4] == 'uint': - return 0 - elif typ == 'byte': - return 0x00 - else: - raise ValueError("Type not recognized") - # Vector: - elif isinstance(typ, list) and len(typ) == 2: - return [get_zero_value(typ[0]) for _ in range(typ[1])] - # List: - elif isinstance(typ, list) and len(typ) == 1: - return [] - # Container: - elif hasattr(typ, 'fields'): - return typ(**{field: get_zero_value(subtype) for field, subtype in typ.fields.items()}) - else: - print(typ) - raise Exception("Type not recognized") - - -def chunkify(bytez): - bytez += b'\x00' * (-len(bytez) % BYTES_PER_CHUNK) - return [bytez[i:i + 32] for i in range(0, len(bytez), 32)] - - -def pack(values, subtype): - return chunkify(b''.join([serialize_value(value, subtype) for value in values])) - - -def is_power_of_two(x): - return x > 0 and x & (x - 1) == 0 - - -def merkleize(chunks): - tree = chunks[::] - while not is_power_of_two(len(tree)): - tree.append(ZERO_CHUNK) - tree = [ZERO_CHUNK] * len(tree) + tree - for i in range(len(tree) // 2 - 1, 0, -1): - tree[i] = hash(tree[i * 2] + tree[i * 2 + 1]) - return tree[1] - - -def mix_in_length(root, length): - return hash(root + length.to_bytes(32, 'little')) - - -def infer_type(value): - """ - Note: defaults to uint64 for integer type inference due to lack of information. - Other integer sizes are still supported, see spec. - :param value: The value to infer a SSZ type for. - :return: The SSZ type. - """ - if hasattr(value.__class__, 'fields'): - return value.__class__ - elif isinstance(value, Vector): - if len(value) > 0: - return [infer_type(value[0]), len(value)] - else: - # Element type does not matter too much, - # assumed to be a basic type for size-encoding purposes, vector is empty. - return ['uint64'] - elif isinstance(value, list): - if len(value) > 0: - return [infer_type(value[0])] - else: - # Element type does not matter, list-content size will be encoded regardless, list is empty. - return ['uint64'] - elif isinstance(value, (bytes, str)): - return 'bytes' - elif isinstance(value, int): - return 'uint64' - else: - raise Exception("Failed to infer type") - - -def hash_tree_root(value, typ=None): - if typ is None: - typ = infer_type(value) - # ------------------------------------- - # merkleize(pack(value)) - # basic object: merkleize packed version (merkleization pads it to 32 bytes if it is not already) - if is_basic(typ): - return merkleize(pack([value], typ)) - # or a vector of basic objects - elif isinstance(typ, list) and len(typ) == 2 and is_basic(typ[0]): - assert len(value) == typ[1] - return merkleize(pack(value, typ[0])) - # ------------------------------------- - # mix_in_length(merkleize(pack(value)), len(value)) - # if value is a list of basic objects - elif isinstance(typ, list) and len(typ) == 1 and is_basic(typ[0]): - return mix_in_length(merkleize(pack(value, typ[0])), len(value)) - # (needs some extra work for non-fixed-sized bytes array) - elif typ == 'bytes': - return mix_in_length(merkleize(chunkify(coerce_to_bytes(value))), len(value)) - # ------------------------------------- - # merkleize([hash_tree_root(element) for element in value]) - # if value is a vector of composite objects - elif isinstance(typ, list) and len(typ) == 2 and not is_basic(typ[0]): - return merkleize([hash_tree_root(element, typ[0]) for element in value]) - # (needs some extra work for fixed-sized bytes array) - elif isinstance(typ, str) and typ[:5] == 'bytes' and len(typ) > 5: - assert len(value) == int(typ[5:]) - return merkleize(chunkify(coerce_to_bytes(value))) - # or a container - elif hasattr(typ, 'fields'): - return merkleize([hash_tree_root(getattr(value, field), subtype) for field, subtype in typ.fields.items()]) - # ------------------------------------- - # mix_in_length(merkleize([hash_tree_root(element) for element in value]), len(value)) - # if value is a list of composite objects - elif isinstance(typ, list) and len(typ) == 1 and not is_basic(typ[0]): - return mix_in_length(merkleize([hash_tree_root(element, typ[0]) for element in value]), len(value)) - # ------------------------------------- - else: - raise Exception("Type not recognized") - - -def truncate(container): - field_keys = list(container.fields.keys()) - truncated_fields = { - key: container.fields[key] - for key in field_keys[:-1] - } - truncated_class = SSZType(truncated_fields) - kwargs = { - field: getattr(container, field) - for field in field_keys[:-1] - } - return truncated_class(**kwargs) - - -def signing_root(container): - return hash_tree_root(truncate(container)) - - -def serialize(ssz_object): - return getattr(ssz_object, 'serialize')() diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py new file mode 100644 index 000000000..74bc1bf99 --- /dev/null +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -0,0 +1,117 @@ +from eth2spec.utils.merkle_minimal import merkleize_chunks +from .ssz_switch import * + +# SSZ Helpers +# ----------------------------- + + +def pack(values, subtype): + return b''.join([serialize(value, subtype) for value in values]) + + +def chunkify(byte_string): + byte_string += b'\x00' * (-len(byte_string) % 32) + return [byte_string[i:i + 32] for i in range(0, len(byte_string), 32)] + + +BYTES_PER_LENGTH_OFFSET = 4 + + +# SSZ Implementation +# ----------------------------- + +get_zero_value = ssz_type_switch({ + ssz_bool: lambda: False, + ssz_uint: lambda: 0, + ssz_list: lambda byte_form: b'' if byte_form else [], + ssz_vector: lambda length, elem_typ, byte_form: + (b'\x00' * length if length > 0 else b'') if byte_form else + [get_zero_value(elem_typ) for _ in range(length)], + ssz_container: lambda typ, field_names, field_types: + typ(**{f_name: get_zero_value(f_typ) for f_name, f_typ in zip(field_names, field_types)}), +}) + + +serialize = ssz_switch({ + ssz_bool: lambda value: b'\x01' if value else b'\x00', + ssz_uint: lambda value, byte_len: value.to_bytes(byte_len, 'little'), + ssz_list: lambda value, elem_typ: encode_series(value, [elem_typ] * len(value)), + ssz_vector: lambda value, elem_typ, length: encode_series(value, [elem_typ] * length), + ssz_container: lambda value, get_field_values, field_types: encode_series(get_field_values(value), field_types), +}) + +ssz_basic_type = (ssz_bool, ssz_uint) + +is_basic_type = ssz_type_switch({ + ssz_basic_type: lambda: True, + ssz_default: lambda: False, +}) + +is_fixed_size = ssz_type_switch({ + ssz_basic_type: lambda: True, + ssz_vector: lambda elem_typ: is_fixed_size(elem_typ), + ssz_container: lambda field_types: all(is_fixed_size(f_typ) for f_typ in field_types), + ssz_list: lambda: False, +}) + + +def hash_tree_root_list(value, elem_typ): + if is_basic_type(elem_typ): + return merkleize_chunks(chunkify(pack(value, elem_typ))) + else: + return merkleize_chunks([hash_tree_root(element, elem_typ) for element in value]) + + +def mix_in_length(root, length): + return hash(root + length.to_bytes(32, 'little')) + + +def hash_tree_root_container(fields): + return merkleize_chunks([hash_tree_root(field, subtype) for field, subtype in fields]) + + +hash_tree_root = ssz_switch({ + ssz_basic_type: lambda value, typ: merkleize_chunks(chunkify(pack([value], typ))), + ssz_list: lambda value, elem_typ: mix_in_length(hash_tree_root_list(value, elem_typ), len(value)), + ssz_vector: lambda value, elem_typ: hash_tree_root_list(value, elem_typ), + ssz_container: lambda value, get_field_values, field_types: hash_tree_root_container(zip(get_field_values(value), field_types)), +}) + +signing_root = ssz_switch({ + ssz_container: lambda value, get_field_values, field_types: hash_tree_root_container(zip(get_field_values(value), field_types)[:-1]), + ssz_default: lambda value, typ: hash_tree_root(value, typ), +}) + + +def encode_series(values, types): + # bytes and bytesN are already in the right format. + if isinstance(values, bytes): + return values + + # Recursively serialize + parts = [(is_fixed_size(types[i]), serialize(values[i], types[i])) for i in range(len(values))] + + # Compute and check lengths + fixed_lengths = [len(serialized) if constant_size else BYTES_PER_LENGTH_OFFSET + for (constant_size, serialized) in parts] + variable_lengths = [len(serialized) if not constant_size else 0 + for (constant_size, serialized) in parts] + + # Check if integer is not out of bounds (Python) + assert sum(fixed_lengths + variable_lengths) < 2 ** (BYTES_PER_LENGTH_OFFSET * 8) + + # Interleave offsets of variable-size parts with fixed-size parts. + # Avoid quadratic complexity in calculation of offsets. + offset = sum(fixed_lengths) + variable_parts = [] + fixed_parts = [] + for (constant_size, serialized) in parts: + if constant_size: + fixed_parts.append(serialized) + else: + fixed_parts.append(offset.to_bytes(BYTES_PER_LENGTH_OFFSET, 'little')) + variable_parts.append(serialized) + offset += len(serialized) + + # Return the concatenation of the fixed-size parts (offsets interleaved) with the variable-size parts + return b''.join(fixed_parts + variable_parts) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_switch.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_switch.py new file mode 100644 index 000000000..3da1e7cb1 --- /dev/null +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_switch.py @@ -0,0 +1,105 @@ +from typing import Dict, Any + +from .ssz_typing import * + +# SSZ Switch statement runner factory +# ----------------------------- + + +def ssz_switch(sw: Dict[Any, Any], arg_names=None): + """ + Creates an SSZ switch statement: a function, that when executed, checks every switch-statement + """ + if arg_names is None: + arg_names = ["value", "typ"] + + # Runner, the function that executes the switch when called. + # Accepts a arguments based on the arg_names declared in the ssz_switch. + def run_switch(*args): + # value may be None + value = None + try: + value = args[arg_names.index("value")] + except ValueError: + pass # no value argument + + # typ may be None when value is not None + typ = None + try: + typ = args[arg_names.index("typ")] + except ValueError: + # no typ argument expected + pass + except IndexError: + # typ argument expected, but not passed. Try to get it from the class info + typ = value.__class__ + if hasattr(typ, '__forward_arg__'): + typ = typ.__forward_arg__ + + # Now, go over all switch cases + for matchers, worker in sw.items(): + if not isinstance(matchers, tuple): + matchers = (matchers,) + # for each matcher of the case key + for m in matchers: + data = m(typ) + # if we have data, the matcher matched, and we can return the result + if data is not None: + # Supply value and type by default, and any data presented by the matcher. + kwargs = {"value": value, "typ": typ, **data} + # Filter out unwanted arguments + filtered_kwargs = {k: kwargs[k] for k in worker.__code__.co_varnames} + # run the switch case and return result + return worker(**filtered_kwargs) + raise Exception("cannot find matcher for type: %s (value: %s)" % (typ, value)) + return run_switch + + +def ssz_type_switch(sw: Dict[Any, Any]): + return ssz_switch(sw, ["typ"]) + + +# SSZ Switch matchers +# ----------------------------- + +def ssz_bool(typ): + if typ == bool: + return {} + + +def ssz_uint(typ): + # Note: only the type reference exists, + # but it really resolves to 'int' during run-time for zero computational/memory overhead. + # Hence, we check equality to the type references (which are really just 'NewType' instances), + # and don't use any sub-classing like we normally would. + if typ == uint8 or typ == uint16 or typ == uint32 or typ == uint64\ + or typ == uint128 or typ == uint256 or typ == byte: + return {"byte_len": typ.byte_len} + + +def ssz_list(typ): + if hasattr(typ, '__bases__') and List in typ.__bases__: + return {"elem_typ": read_list_elem_typ(typ), "byte_form": False} + if typ == bytes: + return {"elem_typ": uint8, "byte_form": True} + + +def ssz_vector(typ): + if hasattr(typ, '__bases__'): + if Vector in typ.__bases__: + return {"elem_typ": read_vec_elem_typ(typ), "length": read_vec_len(typ), "byte_form": False} + if BytesN in typ.__bases__: + return {"elem_typ": uint8, "length": read_bytesN_len(typ), "byte_form": True} + + +def ssz_container(typ): + if hasattr(typ, '__bases__') and SSZContainer in typ.__bases__: + def get_field_values(value): + return [getattr(value, field) for field in typ.__annotations__.keys()] + field_names = list(typ.__annotations__.keys()) + field_types = list(typ.__annotations__.values()) + return {"get_field_values": get_field_values, "field_names": field_names, "field_types": field_types} + + +def ssz_default(typ): + return {} diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py new file mode 100644 index 000000000..6a8a22586 --- /dev/null +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -0,0 +1,115 @@ +from typing import Generic, List, TypeVar, Type, Iterable, NewType + +# SSZ base length, to limit length generic type param of vector/bytesN +SSZLenAny = type('SSZLenAny', (), {}) + + +def SSZLen(length: int): + """ + SSZ length factory. Creates a type corresponding to a given length. To be used as parameter in type generics. + """ + assert length >= 0 + typ = type('SSZLen_%d' % length, (SSZLenAny,), {}) + typ.length = length + return typ + + +# SSZ element type +T = TypeVar('T') +# SSZ vector/bytesN length +L = TypeVar('L', bound=SSZLenAny) + + +# SSZ vector +# ----------------------------- + +class Vector(Generic[T, L]): + def __init__(self, *args: Iterable[T]): + self.items = list(args) + + def __getitem__(self, key): + return self.items[key] + + def __setitem__(self, key, value): + self.items[key] = value + + def __iter__(self): + return iter(self.items) + + def __len__(self): + return len(self.items) + + +def read_vec_elem_typ(vec_typ: Type[Vector[T,L]]) -> T: + assert vec_typ.__args__ is not None + return vec_typ.__args__[0] + + +def read_vec_len(vec_typ: Type[Vector[T,L]]) -> int: + assert vec_typ.__args__ is not None + return vec_typ.__args__[1].length + + +# SSZ list +# ----------------------------- +def read_list_elem_typ(list_typ: Type[List[T]]) -> T: + assert list_typ.__args__ is not None + return list_typ.__args__[0] + + +# SSZ bytesN +# ----------------------------- +class BytesN(Generic[L]): + pass + + +def read_bytesN_len(bytesN_typ: Type[BytesN[L]]) -> int: + assert bytesN_typ.__args__ is not None + return bytesN_typ.__args__[0].length + + +# SSZ integer types, with 0 computational overhead (NewType) +# ----------------------------- + +uint8 = NewType('uint8', int) +uint8.byte_len = 1 +uint16 = NewType('uint16', int) +uint16.byte_len = 2 +uint32 = NewType('uint32', int) +uint32.byte_len = 4 +uint64 = NewType('uint64', int) +uint64.byte_len = 8 +uint128 = NewType('uint128', int) +uint128.byte_len = 16 +uint256 = NewType('uint256', int) +uint256.byte_len = 32 +byte = NewType('byte', uint8) + + +# SSZ Container base class +# ----------------------------- + +# Note: importing ssz functionality locally, to avoid import loop + +class SSZContainer(object): + + def __init__(self, **kwargs): + from .ssz_impl import get_zero_value + for f, t in self.__annotations__.items(): + if f not in kwargs: + setattr(self, f, get_zero_value(t)) + else: + setattr(self, f, kwargs[f]) + + def serialize(self): + from .ssz_impl import serialize + return serialize(self, self.__class__) + + def hash_tree_root(self): + from .ssz_impl import hash_tree_root + return hash_tree_root(self, self.__class__) + + def signing_root(self): + from .ssz_impl import signing_root + return signing_root(self, self.__class__) + From 08faa86d706c31cd9690cb483d0df32f66696396 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 12 May 2019 23:56:53 +0200 Subject: [PATCH 02/48] POC ssz types spec build + update spec defs, typing still needs work --- scripts/phase0/build_spec.py | 4 +- scripts/phase0/function_puller.py | 41 ++-- specs/core/0_beacon-chain.md | 230 +++++++++--------- .../pyspec/eth2spec/utils/ssz/__init__.py | 0 4 files changed, 141 insertions(+), 134 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/utils/ssz/__init__.py diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index c8cd7348b..b226194d6 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -12,11 +12,11 @@ from typing import ( NewType, Tuple, ) -from eth2spec.utils.minimal_ssz import ( - SSZType, +from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, signing_root, ) +from eth2spec.utils.ssz.ssz_typing import * from eth2spec.utils.bls_stub import ( bls_aggregate_pubkeys, bls_verify, diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index 750f19590..f94687344 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -2,6 +2,22 @@ import sys from typing import List +def translate_ssz_type_line(line: str) -> str: + if ':' not in line: + return line + start = line[:line.index(':')] + field_type = line[line.index(':')+2:] + if field_type.startswith('['): + if ',' in line: + # TODO: translate [Foobar, SOME_THING] to Vector[Foobar, SSZLen(SOME_THING)] cleanly. + # just brute it here + field_type = 'Vector[%s, SSLen(%s)]' % (field_type[1:field_type.index(',')], field_type[field_type.index(',')+2:len(field_type)-1]) + else: + field_type = 'List[%s]' % field_type[1:len(field_type)-1] + line = start + ': ' + field_type + return line + + def get_spec(file_name: str) -> List[str]: code_lines = [] pulling_from = None @@ -21,24 +37,23 @@ def get_spec(file_name: str) -> List[str]: else: if current_typedef is not None: assert code_lines[-1] == '}' - code_lines[-1] = '})' - current_typedef[-1] = '})' - type_defs.append((current_name, current_typedef)) + code_lines[-1] = '' + code_lines.append('') pulling_from = None current_typedef = None else: if pulling_from == linenum and line == '{': - code_lines.append('%s = SSZType({' % current_name) - current_typedef = ['global_vars["%s"] = SSZType({' % current_name] + code_lines.append('class %s(SSZContainer):' % current_name) + current_typedef = current_name + type_defs.append(current_name) elif pulling_from is not None: # Add some whitespace between functions if line[:3] == 'def': code_lines.append('') code_lines.append('') - code_lines.append(line) - # Remember type def lines if current_typedef is not None: - current_typedef.append(line) + line = translate_ssz_type_line(line) + code_lines.append(line) elif pulling_from is None and len(line) > 0 and line[0] == '|': row = line[1:].split('|') if len(row) >= 2: @@ -56,16 +71,8 @@ def get_spec(file_name: str) -> List[str]: code_lines.append(row[0] + ' = ' + (row[1].replace('**TBD**', '0x1234567890123456789012345678901234567890'))) # Build type-def re-initialization code_lines.append('\n') - code_lines.append('def init_SSZ_types():') - code_lines.append(' global_vars = globals()') - for ssz_type_name, ssz_type in type_defs: - code_lines.append('') - for type_line in ssz_type: - if len(type_line) > 0: - code_lines.append(' ' + type_line) - code_lines.append('\n') code_lines.append('ssz_types = [\n') - for (ssz_type_name, _) in type_defs: + for ssz_type_name in type_defs: code_lines.append(f' {ssz_type_name},\n') code_lines.append(']') code_lines.append('\n') diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index bbca333cd..ff5a082ef 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -270,11 +270,11 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Previous fork version - 'previous_version': 'bytes4', + previous_version: bytes4 # Current fork version - 'current_version': 'bytes4', + current_version: bytes4 # Fork epoch number - 'epoch': 'uint64', + epoch: uint64 } ``` @@ -283,13 +283,13 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Shard number - 'shard': 'uint64', + shard: uint64 # Epoch number - 'epoch': 'uint64', + epoch: uint64 # Root of the previous crosslink - 'parent_root': 'bytes32', + parent_root: bytes32 # Root of the crosslinked shard data since the previous crosslink - 'data_root': 'bytes32', + data_root: bytes32 } ``` @@ -298,11 +298,11 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Root of the deposit tree - 'deposit_root': 'bytes32', + deposit_root: bytes32 # Total number of deposits - 'deposit_count': 'uint64', + deposit_count: uint64 # Block hash - 'block_hash': 'bytes32', + block_hash: bytes32 } ``` @@ -311,16 +311,16 @@ The types are defined topologically to aid in facilitating an executable version ```python { # LMD GHOST vote - 'beacon_block_root': 'bytes32', - + beacon_block_root: bytes32 + # FFG vote - 'source_epoch': 'uint64', - 'source_root': 'bytes32', - 'target_epoch': 'uint64', - 'target_root': 'bytes32', - + source_epoch: uint64 + source_root: bytes32 + target_epoch: uint64 + target_root: bytes32 + # Crosslink vote - 'crosslink': Crosslink, + crosslink: Crosslink } ``` @@ -329,9 +329,9 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Attestation data - 'data': AttestationData, + data: AttestationData # Custody bit - 'custody_bit': 'bool', + custody_bit: bool } ``` @@ -340,12 +340,12 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Validator indices - 'custody_bit_0_indices': ['uint64'], - 'custody_bit_1_indices': ['uint64'], + custody_bit_0_indices: [uint64] + custody_bit_1_indices: [uint64] # Attestation data - 'data': AttestationData, + data: AttestationData # Aggregate signature - 'signature': 'bytes96', + signature: bytes96 } ``` @@ -354,13 +354,13 @@ The types are defined topologically to aid in facilitating an executable version ```python { # BLS pubkey - 'pubkey': 'bytes48', + pubkey: bytes48 # Withdrawal credentials - 'withdrawal_credentials': 'bytes32', + withdrawal_credentials: bytes32 # Amount in Gwei - 'amount': 'uint64', + amount: uint64 # Container self-signature - 'signature': 'bytes96', + signature: bytes96 } ``` @@ -368,11 +368,11 @@ The types are defined topologically to aid in facilitating an executable version ```python { - 'slot': 'uint64', - 'parent_root': 'bytes32', - 'state_root': 'bytes32', - 'body_root': 'bytes32', - 'signature': 'bytes96', + slot: uint64 + parent_root: bytes32 + state_root: bytes32 + body_root: bytes32 + signature: bytes96 } ``` #### `Validator` @@ -380,21 +380,21 @@ The types are defined topologically to aid in facilitating an executable version ```python { # BLS public key - 'pubkey': 'bytes48', + pubkey: bytes48 # Withdrawal credentials - 'withdrawal_credentials': 'bytes32', + withdrawal_credentials: bytes32 # Epoch when became eligible for activation - 'activation_eligibility_epoch': 'uint64', + activation_eligibility_epoch: uint64 # Epoch when validator activated - 'activation_epoch': 'uint64', + activation_epoch: uint64 # Epoch when validator exited - 'exit_epoch': 'uint64', + exit_epoch: uint64 # Epoch when validator is eligible to withdraw - 'withdrawable_epoch': 'uint64', + withdrawable_epoch: uint64 # Was the validator slashed - 'slashed': 'bool', + slashed: bool # Effective balance - 'effective_balance': 'uint64', + effective_balance: uint64 } ``` @@ -403,13 +403,13 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Attester aggregation bitfield - 'aggregation_bitfield': 'bytes', + aggregation_bitfield: bytes # Attestation data - 'data': AttestationData, + data: AttestationData # Inclusion delay - 'inclusion_delay': 'uint64', + inclusion_delay: uint64 # Proposer index - 'proposer_index': 'uint64', + proposer_index: uint64 } ``` @@ -418,9 +418,9 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Block roots - 'block_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], + block_roots: [bytes32, SLOTS_PER_HISTORICAL_ROOT] # State roots - 'state_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], + state_roots: [bytes32, SLOTS_PER_HISTORICAL_ROOT] } ``` @@ -431,11 +431,11 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Proposer index - 'proposer_index': 'uint64', + proposer_index: uint64 # First block header - 'header_1': BeaconBlockHeader, + header_1: BeaconBlockHeader # Second block header - 'header_2': BeaconBlockHeader, + header_2: BeaconBlockHeader } ``` @@ -444,9 +444,9 @@ The types are defined topologically to aid in facilitating an executable version ```python { # First attestation - 'attestation_1': IndexedAttestation, + attestation_1: IndexedAttestation # Second attestation - 'attestation_2': IndexedAttestation, + attestation_2: IndexedAttestation } ``` @@ -455,13 +455,13 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Attester aggregation bitfield - 'aggregation_bitfield': 'bytes', + aggregation_bitfield: bytes # Attestation data - 'data': AttestationData, + data: AttestationData # Custody bitfield - 'custody_bitfield': 'bytes', + custody_bitfield: bytes # BLS aggregate signature - 'signature': 'bytes96', + signature: bytes96 } ``` @@ -470,11 +470,11 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Branch in the deposit tree - 'proof': ['bytes32', DEPOSIT_CONTRACT_TREE_DEPTH], + proof: [bytes32, DEPOSIT_CONTRACT_TREE_DEPTH] # Index in the deposit tree - 'index': 'uint64', + index: uint64 # Data - 'data': DepositData, + data: DepositData } ``` @@ -483,11 +483,11 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Minimum epoch for processing exit - 'epoch': 'uint64', + epoch: uint64 # Index of the exiting validator - 'validator_index': 'uint64', + validator_index: uint64 # Validator signature - 'signature': 'bytes96', + signature: bytes96 } ``` @@ -496,19 +496,19 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Sender index - 'sender': 'uint64', + sender: uint64 # Recipient index - 'recipient': 'uint64', + recipient: uint64 # Amount in Gwei - 'amount': 'uint64', + amount: uint64 # Fee in Gwei for block proposer - 'fee': 'uint64', + fee: uint64 # Inclusion slot - 'slot': 'uint64', + slot: uint64 # Sender withdrawal pubkey - 'pubkey': 'bytes48', + pubkey: bytes48 # Sender signature - 'signature': 'bytes96', + signature: bytes96 } ``` @@ -518,15 +518,15 @@ The types are defined topologically to aid in facilitating an executable version ```python { - 'randao_reveal': 'bytes96', - 'eth1_data': Eth1Data, - 'graffiti': 'bytes32', - 'proposer_slashings': [ProposerSlashing], - 'attester_slashings': [AttesterSlashing], - 'attestations': [Attestation], - 'deposits': [Deposit], - 'voluntary_exits': [VoluntaryExit], - 'transfers': [Transfer], + randao_reveal: bytes96 + eth1_data: Eth1Data + graffiti: bytes32 + proposer_slashings: [ProposerSlashing] + attester_slashings: [AttesterSlashing] + attestations: [Attestation] + deposits: [Deposit] + voluntary_exits: [VoluntaryExit] + transfers: [Transfer] } ``` @@ -535,11 +535,11 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Header - 'slot': 'uint64', - 'parent_root': 'bytes32', - 'state_root': 'bytes32', - 'body': BeaconBlockBody, - 'signature': 'bytes96', + slot: uint64 + parent_root: bytes32 + state_root: bytes32 + body: BeaconBlockBody + signature: bytes96 } ``` @@ -550,45 +550,45 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Misc - 'slot': 'uint64', - 'genesis_time': 'uint64', - 'fork': Fork, # For versioning hard forks - + slot: uint64 + genesis_time: uint64 + fork: Fork # For versioning hard forks + # Validator registry - 'validator_registry': [Validator], - 'balances': ['uint64'], - + validator_registry: [Validator] + balances: [uint64] + # Randomness and committees - 'latest_randao_mixes': ['bytes32', LATEST_RANDAO_MIXES_LENGTH], - 'latest_start_shard': 'uint64', - + latest_randao_mixes: [bytes32, LATEST_RANDAO_MIXES_LENGTH] + latest_start_shard: uint64 + # Finality - 'previous_epoch_attestations': [PendingAttestation], - 'current_epoch_attestations': [PendingAttestation], - 'previous_justified_epoch': 'uint64', - 'current_justified_epoch': 'uint64', - 'previous_justified_root': 'bytes32', - 'current_justified_root': 'bytes32', - 'justification_bitfield': 'uint64', - 'finalized_epoch': 'uint64', - 'finalized_root': 'bytes32', - + previous_epoch_attestations: [PendingAttestation] + current_epoch_attestations: [PendingAttestation] + previous_justified_epoch: uint64 + current_justified_epoch: uint64 + previous_justified_root: bytes32 + current_justified_root: bytes32 + justification_bitfield: uint64 + finalized_epoch: uint64 + finalized_root: bytes32 + # Recent state - 'current_crosslinks': [Crosslink, SHARD_COUNT], - 'previous_crosslinks': [Crosslink, SHARD_COUNT], - 'latest_block_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], - 'latest_state_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], - 'latest_active_index_roots': ['bytes32', LATEST_ACTIVE_INDEX_ROOTS_LENGTH], + current_crosslinks: [Crosslink, SHARD_COUNT] + previous_crosslinks: [Crosslink, SHARD_COUNT] + latest_block_roots: [bytes32, SLOTS_PER_HISTORICAL_ROOT] + latest_state_roots: [bytes32, SLOTS_PER_HISTORICAL_ROOT] + latest_active_index_roots: [bytes32, LATEST_ACTIVE_INDEX_ROOTS_LENGTH] # Balances slashed at every withdrawal period - 'latest_slashed_balances': ['uint64', LATEST_SLASHED_EXIT_LENGTH], + latest_slashed_balances: [uint64, LATEST_SLASHED_EXIT_LENGTH] # `latest_block_header.state_root == ZERO_HASH` temporarily - 'latest_block_header': BeaconBlockHeader, - 'historical_roots': ['bytes32'], - + latest_block_header: BeaconBlockHeader + historical_roots: [bytes32] + # Ethereum 1.0 chain data - 'latest_eth1_data': Eth1Data, - 'eth1_data_votes': [Eth1Data], - 'deposit_index': 'uint64', + latest_eth1_data: Eth1Data + eth1_data_votes: [Eth1Data] + deposit_index: uint64 } ``` diff --git a/test_libs/pyspec/eth2spec/utils/ssz/__init__.py b/test_libs/pyspec/eth2spec/utils/ssz/__init__.py new file mode 100644 index 000000000..e69de29bb From ed4416ba348cae92b1326c357b1c6e6f946e095e Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 25 May 2019 00:05:03 +0200 Subject: [PATCH 03/48] update SSZ implementation --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 94 ++++-- .../pyspec/eth2spec/utils/ssz/ssz_switch.py | 105 ------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 284 ++++++++++++++---- 3 files changed, 291 insertions(+), 192 deletions(-) delete mode 100644 test_libs/pyspec/eth2spec/utils/ssz/ssz_switch.py diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 74bc1bf99..2b0328bfa 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,5 +1,27 @@ from eth2spec.utils.merkle_minimal import merkleize_chunks -from .ssz_switch import * +from .ssz_typing import * + + +# SSZ Defaults +# ----------------------------- + + +def get_zero_value(typ): + if is_uint(typ): + return 0 + if issubclass(typ, bool): + return False + if issubclass(typ, list): + return [] + if issubclass(typ, Vector): + return typ() + if issubclass(typ, BytesN): + return typ() + if issubclass(typ, bytes): + return b'' + if issubclass(typ, SSZContainer): + return typ(**{f: get_zero_value(t) for f, t in typ.get_fields().items()}), + # SSZ Helpers # ----------------------------- @@ -14,23 +36,10 @@ def chunkify(byte_string): return [byte_string[i:i + 32] for i in range(0, len(byte_string), 32)] -BYTES_PER_LENGTH_OFFSET = 4 - - -# SSZ Implementation +# SSZ Serialization # ----------------------------- -get_zero_value = ssz_type_switch({ - ssz_bool: lambda: False, - ssz_uint: lambda: 0, - ssz_list: lambda byte_form: b'' if byte_form else [], - ssz_vector: lambda length, elem_typ, byte_form: - (b'\x00' * length if length > 0 else b'') if byte_form else - [get_zero_value(elem_typ) for _ in range(length)], - ssz_container: lambda typ, field_names, field_types: - typ(**{f_name: get_zero_value(f_typ) for f_name, f_typ in zip(field_names, field_types)}), -}) - +BYTES_PER_LENGTH_OFFSET = 4 serialize = ssz_switch({ ssz_bool: lambda value: b'\x01' if value else b'\x00', @@ -40,13 +49,6 @@ serialize = ssz_switch({ ssz_container: lambda value, get_field_values, field_types: encode_series(get_field_values(value), field_types), }) -ssz_basic_type = (ssz_bool, ssz_uint) - -is_basic_type = ssz_type_switch({ - ssz_basic_type: lambda: True, - ssz_default: lambda: False, -}) - is_fixed_size = ssz_type_switch({ ssz_basic_type: lambda: True, ssz_vector: lambda elem_typ: is_fixed_size(elem_typ), @@ -55,6 +57,27 @@ is_fixed_size = ssz_type_switch({ }) +# SSZ Hash-tree-root +# ----------------------------- + +def serialize_basic(value, typ): + if is_uint(typ): + return value.to_bytes(typ.byte_len, 'little') + if issubclass(typ, bool): + if value: + return b'\x01' + else: + return b'\x00' + + +def pack(values, subtype): + return b''.join([serialize_basic(value, subtype) for value in values]) + + +def is_basic_type(typ): + return is_uint(typ) or issubclass(typ, bool) + + def hash_tree_root_list(value, elem_typ): if is_basic_type(elem_typ): return merkleize_chunks(chunkify(pack(value, elem_typ))) @@ -77,10 +100,9 @@ hash_tree_root = ssz_switch({ ssz_container: lambda value, get_field_values, field_types: hash_tree_root_container(zip(get_field_values(value), field_types)), }) -signing_root = ssz_switch({ - ssz_container: lambda value, get_field_values, field_types: hash_tree_root_container(zip(get_field_values(value), field_types)[:-1]), - ssz_default: lambda value, typ: hash_tree_root(value, typ), -}) +# todo: signing root +def signing_root(value, typ): + pass def encode_series(values, types): @@ -115,3 +137,21 @@ def encode_series(values, types): # Return the concatenation of the fixed-size parts (offsets interleaved) with the variable-size parts return b''.join(fixed_parts + variable_parts) + + +# Implementation notes: +# - SSZContainer,Vector/BytesN.hash_tree_root/serialize functions are for ease, implementation here +# - uint types have a 'byte_len' attribute +# - uint types are not classes. They use NewType(), for performance. +# This forces us to check type equivalence by exact reference. +# There's no class. The type data comes from an annotation/argument from the context of the value. +# - Vector is not valid to create instances with. Give it a elem-type and length: Vector[FooBar, 123] +# - *The class of* a Vector instance has a `elem_type` (type, may not be a class, see uint) and `length` (int) +# - BytesN is not valid to create instances with. Give it a length: BytesN[123] +# - *The class of* a BytesN instance has a `length` (int) +# Where possible, it is preferable to create helpers that just act on the type, and don't unnecessarily use a value +# E.g. is_basic_type(). This way, we can use them in type-only contexts and have no duplicate logic. +# For every class-instance, you can get the type with my_object.__class__ +# For uints, and other NewType related, you have to rely on type information. It cannot be retrieved from the value. +# Note: we may just want to box integers instead. And then we can do bounds checking too. But it is SLOW and MEMORY INTENSIVE. +# diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_switch.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_switch.py deleted file mode 100644 index 3da1e7cb1..000000000 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_switch.py +++ /dev/null @@ -1,105 +0,0 @@ -from typing import Dict, Any - -from .ssz_typing import * - -# SSZ Switch statement runner factory -# ----------------------------- - - -def ssz_switch(sw: Dict[Any, Any], arg_names=None): - """ - Creates an SSZ switch statement: a function, that when executed, checks every switch-statement - """ - if arg_names is None: - arg_names = ["value", "typ"] - - # Runner, the function that executes the switch when called. - # Accepts a arguments based on the arg_names declared in the ssz_switch. - def run_switch(*args): - # value may be None - value = None - try: - value = args[arg_names.index("value")] - except ValueError: - pass # no value argument - - # typ may be None when value is not None - typ = None - try: - typ = args[arg_names.index("typ")] - except ValueError: - # no typ argument expected - pass - except IndexError: - # typ argument expected, but not passed. Try to get it from the class info - typ = value.__class__ - if hasattr(typ, '__forward_arg__'): - typ = typ.__forward_arg__ - - # Now, go over all switch cases - for matchers, worker in sw.items(): - if not isinstance(matchers, tuple): - matchers = (matchers,) - # for each matcher of the case key - for m in matchers: - data = m(typ) - # if we have data, the matcher matched, and we can return the result - if data is not None: - # Supply value and type by default, and any data presented by the matcher. - kwargs = {"value": value, "typ": typ, **data} - # Filter out unwanted arguments - filtered_kwargs = {k: kwargs[k] for k in worker.__code__.co_varnames} - # run the switch case and return result - return worker(**filtered_kwargs) - raise Exception("cannot find matcher for type: %s (value: %s)" % (typ, value)) - return run_switch - - -def ssz_type_switch(sw: Dict[Any, Any]): - return ssz_switch(sw, ["typ"]) - - -# SSZ Switch matchers -# ----------------------------- - -def ssz_bool(typ): - if typ == bool: - return {} - - -def ssz_uint(typ): - # Note: only the type reference exists, - # but it really resolves to 'int' during run-time for zero computational/memory overhead. - # Hence, we check equality to the type references (which are really just 'NewType' instances), - # and don't use any sub-classing like we normally would. - if typ == uint8 or typ == uint16 or typ == uint32 or typ == uint64\ - or typ == uint128 or typ == uint256 or typ == byte: - return {"byte_len": typ.byte_len} - - -def ssz_list(typ): - if hasattr(typ, '__bases__') and List in typ.__bases__: - return {"elem_typ": read_list_elem_typ(typ), "byte_form": False} - if typ == bytes: - return {"elem_typ": uint8, "byte_form": True} - - -def ssz_vector(typ): - if hasattr(typ, '__bases__'): - if Vector in typ.__bases__: - return {"elem_typ": read_vec_elem_typ(typ), "length": read_vec_len(typ), "byte_form": False} - if BytesN in typ.__bases__: - return {"elem_typ": uint8, "length": read_bytesN_len(typ), "byte_form": True} - - -def ssz_container(typ): - if hasattr(typ, '__bases__') and SSZContainer in typ.__bases__: - def get_field_values(value): - return [getattr(value, field) for field in typ.__annotations__.keys()] - field_names = list(typ.__annotations__.keys()) - field_types = list(typ.__annotations__.values()) - return {"get_field_values": get_field_values, "field_names": field_names, "field_types": field_types} - - -def ssz_default(typ): - return {} diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 6a8a22586..dc23427b3 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,72 +1,18 @@ -from typing import Generic, List, TypeVar, Type, Iterable, NewType +from typing import List, Iterable, TypeVar, Type, NewType +from typing import Union +from inspect import isclass -# SSZ base length, to limit length generic type param of vector/bytesN -SSZLenAny = type('SSZLenAny', (), {}) - - -def SSZLen(length: int): - """ - SSZ length factory. Creates a type corresponding to a given length. To be used as parameter in type generics. - """ - assert length >= 0 - typ = type('SSZLen_%d' % length, (SSZLenAny,), {}) - typ.length = length - return typ - - -# SSZ element type T = TypeVar('T') -# SSZ vector/bytesN length -L = TypeVar('L', bound=SSZLenAny) - - -# SSZ vector -# ----------------------------- - -class Vector(Generic[T, L]): - def __init__(self, *args: Iterable[T]): - self.items = list(args) - - def __getitem__(self, key): - return self.items[key] - - def __setitem__(self, key, value): - self.items[key] = value - - def __iter__(self): - return iter(self.items) - - def __len__(self): - return len(self.items) - - -def read_vec_elem_typ(vec_typ: Type[Vector[T,L]]) -> T: - assert vec_typ.__args__ is not None - return vec_typ.__args__[0] - - -def read_vec_len(vec_typ: Type[Vector[T,L]]) -> int: - assert vec_typ.__args__ is not None - return vec_typ.__args__[1].length - # SSZ list # ----------------------------- + + def read_list_elem_typ(list_typ: Type[List[T]]) -> T: assert list_typ.__args__ is not None return list_typ.__args__[0] -# SSZ bytesN -# ----------------------------- -class BytesN(Generic[L]): - pass - - -def read_bytesN_len(bytesN_typ: Type[BytesN[L]]) -> int: - assert bytesN_typ.__args__ is not None - return bytesN_typ.__args__[0].length - # SSZ integer types, with 0 computational overhead (NewType) # ----------------------------- @@ -94,8 +40,9 @@ byte = NewType('byte', uint8) class SSZContainer(object): def __init__(self, **kwargs): + cls = self.__class__ from .ssz_impl import get_zero_value - for f, t in self.__annotations__.items(): + for f, t in cls.get_fields().items(): if f not in kwargs: setattr(self, f, get_zero_value(t)) else: @@ -113,3 +60,220 @@ class SSZContainer(object): from .ssz_impl import signing_root return signing_root(self, self.__class__) + def get_field_values(self): + cls = self.__class__ + return [getattr(self, field) for field in cls.get_field_names()] + + @classmethod + def get_fields(cls): + return dict(cls.__annotations__) + + @classmethod + def get_field_names(cls): + return list(cls.__annotations__.keys()) + + @classmethod + def get_field_types(cls): + # values of annotations are the types corresponding to the fields, not instance values. + return list(cls.__annotations__.values()) + + +def is_uint(typ): + # Note: only the type reference exists, + # but it really resolves to 'int' during run-time for zero computational/memory overhead. + # Hence, we check equality to the type references (which are really just 'NewType' instances), + # and don't use any sub-classing like we normally would. + return typ == uint8 or typ == uint16 or typ == uint32 or typ == uint64 \ + or typ == uint128 or typ == uint256 or typ == byte + +# SSZ vector +# ----------------------------- + + +def _is_vector_instance_of(a, b): + if not hasattr(b, 'elem_type') or not hasattr(b, 'length'): + # Vector (b) is not an instance of Vector[X, Y] (a) + return False + if not hasattr(a, 'elem_type') or not hasattr(a, 'length'): + # Vector[X, Y] (b) is an instance of Vector (a) + return True + + # Vector[X, Y] (a) is an instance of Vector[X, Y] (b) + return a.elem_type == b.elem_type and a.length == b.length + + +def _is_equal_vector_type(a, b): + if not hasattr(a, 'elem_type') or not hasattr(a, 'length'): + if not hasattr(b, 'elem_type') or not hasattr(b, 'length'): + # Vector == Vector + return True + # Vector != Vector[X, Y] + return False + if not hasattr(b, 'elem_type') or not hasattr(b, 'length'): + # Vector[X, Y] != Vector + return False + # Vector[X, Y] == Vector[X, Y] + return a.elem_type == b.elem_type and a.length == b.length + + +class VectorMeta(type): + def __new__(cls, class_name, parents, attrs): + out = type.__new__(cls, class_name, parents, attrs) + if 'elem_type' in attrs and 'length' in attrs: + setattr(out, 'elem_type', attrs['elem_type']) + setattr(out, 'length', attrs['length']) + return out + + def __getitem__(self, params): + return self.__class__(self.__name__, (Vector,), {'elem_type': params[0], 'length': params[1]}) + + def __subclasscheck__(self, sub): + return _is_vector_instance_of(self, sub) + + def __instancecheck__(self, other): + return _is_vector_instance_of(self, other.__class__) + + def __eq__(self, other): + return _is_equal_vector_type(self, other) + + def __ne__(self, other): + return not _is_equal_vector_type(self, other) + + +class Vector(metaclass=VectorMeta): + + def __init__(self, *args: Iterable[T]): + + cls = self.__class__ + if not hasattr(cls, 'elem_type'): + raise TypeError("Type Vector without elem_type data cannot be instantiated") + if not hasattr(cls, 'length'): + raise TypeError("Type Vector without length data cannot be instantiated") + + if len(args) != cls.length: + if len(args) == 0: + from .ssz_impl import get_zero_value + args = [get_zero_value(cls.elem_type) for _ in range(cls.length)] + else: + raise TypeError("Typed vector with length %d cannot hold %d items" % (cls.length, len(args))) + + self.items = list(args) + + # cannot check non-class objects + if isclass(cls.elem_type): + for i, item in enumerate(self.items): + if not isinstance(item, cls.elem_type): + raise TypeError("Typed vector cannot hold differently typed value" + " at index %d. Got type: %s, expected type: %s" % (i, type(item), cls.elem_type)) + + def serialize(self): + from .ssz_impl import serialize + return serialize(self, self.__class__) + + def hash_tree_root(self): + from .ssz_impl import hash_tree_root + return hash_tree_root(self, self.__class__) + + def __getitem__(self, key): + return self.items[key] + + def __setitem__(self, key, value): + self.items[key] = value + + def __iter__(self): + return iter(self.items) + + def __len__(self): + return len(self.items) + + +def _is_bytes_n_instance_of(a, b): + if not hasattr(b, 'length'): + # BytesN (b) is not an instance of BytesN[X] (a) + return False + if not hasattr(a, 'length'): + # BytesN[X] (b) is an instance of BytesN (a) + return True + + # BytesN[X] (a) is an instance of BytesN[X] (b) + return a.length == b.length + + +def _is_equal_bytes_n_type(a, b): + if not hasattr(a, 'length'): + if not hasattr(b, 'length'): + # BytesN == BytesN + return True + # BytesN != BytesN[X] + return False + if not hasattr(b, 'length'): + # BytesN[X] != BytesN + return False + # BytesN[X] == BytesN[X] + return a.length == b.length + + +class BytesNMeta(type): + def __new__(cls, class_name, parents, attrs): + out = type.__new__(cls, class_name, parents, attrs) + if 'length' in attrs: + setattr(out, 'length', attrs['length']) + return out + + def __getitem__(self, n): + return self.__class__(self.__name__, (BytesN,), {'length': n}) + + def __subclasscheck__(self, sub): + return _is_bytes_n_instance_of(self, sub) + + def __instancecheck__(self, other): + return _is_bytes_n_instance_of(self, other.__class__) + + def __eq__(self, other): + return _is_equal_bytes_n_type(self, other) + + def __ne__(self, other): + return not _is_equal_bytes_n_type(self, other) + + +def parse_bytes(val): + if val is None: + return None + if isinstance(val, str): + # TODO: import from eth-utils instead, and do: hexstr_if_str(to_bytes, val) + return None + if isinstance(val, bytes): + return val + if isinstance(val, int): + return bytes([val]) + return None + + +class BytesN(bytes, metaclass=BytesNMeta): + def __new__(cls, *args): + if not hasattr(cls, 'length'): + return + bytesval = None + if len(args) == 1: + val: Union[bytes, int, str] = args[0] + bytesval = parse_bytes(val) + elif len(args) > 1: + # TODO: each int is 1 byte, check size, create bytesval + bytesval = bytes(args) + + if bytesval is None: + if cls.length == 0: + bytesval = b'' + else: + bytesval = b'\x00' * cls.length + if len(bytesval) != cls.length: + raise TypeError("bytesN[%d] cannot be initialized with value of %d bytes" % (cls.length, len(bytesval))) + return super().__new__(cls, bytesval) + + def serialize(self): + from .ssz_impl import serialize + return serialize(self, self.__class__) + + def hash_tree_root(self): + from .ssz_impl import hash_tree_root + return hash_tree_root(self, self.__class__) From 81cb4a23b31ef11ae2b17d7b4f19d92608466cfe Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 25 May 2019 00:10:06 +0200 Subject: [PATCH 04/48] update some common usage of SSZ types, as a start --- scripts/phase0/build_spec.py | 6 ++-- scripts/phase0/function_puller.py | 18 ------------ specs/core/0_beacon-chain.md | 48 +++++++++++++++---------------- 3 files changed, 27 insertions(+), 45 deletions(-) diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index b226194d6..b188dada2 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -33,9 +33,9 @@ Epoch = NewType('Epoch', int) # uint64 Shard = NewType('Shard', int) # uint64 ValidatorIndex = NewType('ValidatorIndex', int) # uint64 Gwei = NewType('Gwei', int) # uint64 -Bytes32 = NewType('Bytes32', bytes) # bytes32 -BLSPubkey = NewType('BLSPubkey', bytes) # bytes48 -BLSSignature = NewType('BLSSignature', bytes) # bytes96 +Bytes32 = BytesN[32] +BLSPubkey = NewType('BLSPubkey', BytesN[48]) +BLSSignature = NewType('BLSSignature', BytesN[96]) Store = None """) diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index f94687344..5c1cf859b 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -2,22 +2,6 @@ import sys from typing import List -def translate_ssz_type_line(line: str) -> str: - if ':' not in line: - return line - start = line[:line.index(':')] - field_type = line[line.index(':')+2:] - if field_type.startswith('['): - if ',' in line: - # TODO: translate [Foobar, SOME_THING] to Vector[Foobar, SSZLen(SOME_THING)] cleanly. - # just brute it here - field_type = 'Vector[%s, SSLen(%s)]' % (field_type[1:field_type.index(',')], field_type[field_type.index(',')+2:len(field_type)-1]) - else: - field_type = 'List[%s]' % field_type[1:len(field_type)-1] - line = start + ': ' + field_type - return line - - def get_spec(file_name: str) -> List[str]: code_lines = [] pulling_from = None @@ -51,8 +35,6 @@ def get_spec(file_name: str) -> List[str]: if line[:3] == 'def': code_lines.append('') code_lines.append('') - if current_typedef is not None: - line = translate_ssz_type_line(line) code_lines.append(line) elif pulling_from is None and len(line) > 0 and line[0] == '|': row = line[1:].split('|') diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ff5a082ef..35d273a2e 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -340,8 +340,8 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Validator indices - custody_bit_0_indices: [uint64] - custody_bit_1_indices: [uint64] + custody_bit_0_indices: List[uint64] + custody_bit_1_indices: List[uint64] # Attestation data data: AttestationData # Aggregate signature @@ -418,9 +418,9 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Block roots - block_roots: [bytes32, SLOTS_PER_HISTORICAL_ROOT] + block_roots: Vector[bytes32, SLOTS_PER_HISTORICAL_ROOT] # State roots - state_roots: [bytes32, SLOTS_PER_HISTORICAL_ROOT] + state_roots: Vector[bytes32, SLOTS_PER_HISTORICAL_ROOT] } ``` @@ -470,7 +470,7 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Branch in the deposit tree - proof: [bytes32, DEPOSIT_CONTRACT_TREE_DEPTH] + proof: Vector[bytes32, DEPOSIT_CONTRACT_TREE_DEPTH] # Index in the deposit tree index: uint64 # Data @@ -521,12 +521,12 @@ The types are defined topologically to aid in facilitating an executable version randao_reveal: bytes96 eth1_data: Eth1Data graffiti: bytes32 - proposer_slashings: [ProposerSlashing] - attester_slashings: [AttesterSlashing] - attestations: [Attestation] - deposits: [Deposit] - voluntary_exits: [VoluntaryExit] - transfers: [Transfer] + proposer_slashings: List[ProposerSlashing] + attester_slashings: List[AttesterSlashing] + attestations: List[Attestation] + deposits: List[Deposit] + voluntary_exits: List[VoluntaryExit] + transfers: List[Transfer] } ``` @@ -555,16 +555,16 @@ The types are defined topologically to aid in facilitating an executable version fork: Fork # For versioning hard forks # Validator registry - validator_registry: [Validator] - balances: [uint64] + validator_registry: List[Validator] + balances: List[uint64] # Randomness and committees - latest_randao_mixes: [bytes32, LATEST_RANDAO_MIXES_LENGTH] + latest_randao_mixes: Vector[bytes32, LATEST_RANDAO_MIXES_LENGTH] latest_start_shard: uint64 # Finality - previous_epoch_attestations: [PendingAttestation] - current_epoch_attestations: [PendingAttestation] + previous_epoch_attestations: List[PendingAttestation] + current_epoch_attestations: List[PendingAttestation] previous_justified_epoch: uint64 current_justified_epoch: uint64 previous_justified_root: bytes32 @@ -574,20 +574,20 @@ The types are defined topologically to aid in facilitating an executable version finalized_root: bytes32 # Recent state - current_crosslinks: [Crosslink, SHARD_COUNT] - previous_crosslinks: [Crosslink, SHARD_COUNT] - latest_block_roots: [bytes32, SLOTS_PER_HISTORICAL_ROOT] - latest_state_roots: [bytes32, SLOTS_PER_HISTORICAL_ROOT] - latest_active_index_roots: [bytes32, LATEST_ACTIVE_INDEX_ROOTS_LENGTH] + current_crosslinks: Vector[Crosslink, SHARD_COUNT] + previous_crosslinks: Vector[Crosslink, SHARD_COUNT] + latest_block_roots: Vector[bytes32, SLOTS_PER_HISTORICAL_ROOT] + latest_state_roots: Vector[bytes32, SLOTS_PER_HISTORICAL_ROOT] + latest_active_index_roots: Vector[bytes32, LATEST_ACTIVE_INDEX_ROOTS_LENGTH] # Balances slashed at every withdrawal period - latest_slashed_balances: [uint64, LATEST_SLASHED_EXIT_LENGTH] + latest_slashed_balances: Vector[uint64, LATEST_SLASHED_EXIT_LENGTH] # `latest_block_header.state_root == ZERO_HASH` temporarily latest_block_header: BeaconBlockHeader - historical_roots: [bytes32] + historical_roots: List[bytes32] # Ethereum 1.0 chain data latest_eth1_data: Eth1Data - eth1_data_votes: [Eth1Data] + eth1_data_votes: List[Eth1Data] deposit_index: uint64 } ``` From 5b6a98b1075f0f0c24505d065e83817eca76edf0 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Sat, 25 May 2019 14:06:42 -0400 Subject: [PATCH 05/48] Some updates --- test_libs/pyspec/eth2spec/utils/__init__.py | 4 + .../pyspec/eth2spec/utils/ssz/__init__.py | 1 + .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 151 ++++++++---------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 79 +++++++-- 4 files changed, 137 insertions(+), 98 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/__init__.py b/test_libs/pyspec/eth2spec/utils/__init__.py index e69de29bb..9f8b36428 100644 --- a/test_libs/pyspec/eth2spec/utils/__init__.py +++ b/test_libs/pyspec/eth2spec/utils/__init__.py @@ -0,0 +1,4 @@ +from .merkle_minimal import * +from .hash_function import * +from .bls_stub import * +from .ssz import * diff --git a/test_libs/pyspec/eth2spec/utils/ssz/__init__.py b/test_libs/pyspec/eth2spec/utils/ssz/__init__.py index e69de29bb..752d77c43 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/__init__.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/__init__.py @@ -0,0 +1 @@ +from .ssz_impl import * diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 2b0328bfa..8b6c20a17 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,64 +1,13 @@ -from eth2spec.utils.merkle_minimal import merkleize_chunks +from ..merkle_minimal import merkleize_chunks, hash from .ssz_typing import * - -# SSZ Defaults -# ----------------------------- - - -def get_zero_value(typ): - if is_uint(typ): - return 0 - if issubclass(typ, bool): - return False - if issubclass(typ, list): - return [] - if issubclass(typ, Vector): - return typ() - if issubclass(typ, BytesN): - return typ() - if issubclass(typ, bytes): - return b'' - if issubclass(typ, SSZContainer): - return typ(**{f: get_zero_value(t) for f, t in typ.get_fields().items()}), - - -# SSZ Helpers -# ----------------------------- - - -def pack(values, subtype): - return b''.join([serialize(value, subtype) for value in values]) - - -def chunkify(byte_string): - byte_string += b'\x00' * (-len(byte_string) % 32) - return [byte_string[i:i + 32] for i in range(0, len(byte_string), 32)] - - # SSZ Serialization # ----------------------------- BYTES_PER_LENGTH_OFFSET = 4 -serialize = ssz_switch({ - ssz_bool: lambda value: b'\x01' if value else b'\x00', - ssz_uint: lambda value, byte_len: value.to_bytes(byte_len, 'little'), - ssz_list: lambda value, elem_typ: encode_series(value, [elem_typ] * len(value)), - ssz_vector: lambda value, elem_typ, length: encode_series(value, [elem_typ] * length), - ssz_container: lambda value, get_field_values, field_types: encode_series(get_field_values(value), field_types), -}) - -is_fixed_size = ssz_type_switch({ - ssz_basic_type: lambda: True, - ssz_vector: lambda elem_typ: is_fixed_size(elem_typ), - ssz_container: lambda field_types: all(is_fixed_size(f_typ) for f_typ in field_types), - ssz_list: lambda: False, -}) - - -# SSZ Hash-tree-root -# ----------------------------- +def is_basic_type(typ): + return is_uint(typ) or typ == bool def serialize_basic(value, typ): if is_uint(typ): @@ -69,41 +18,29 @@ def serialize_basic(value, typ): else: return b'\x00' - -def pack(values, subtype): - return b''.join([serialize_basic(value, subtype) for value in values]) - - -def is_basic_type(typ): - return is_uint(typ) or issubclass(typ, bool) - - -def hash_tree_root_list(value, elem_typ): - if is_basic_type(elem_typ): - return merkleize_chunks(chunkify(pack(value, elem_typ))) +def is_fixed_size(typ): + if is_basic_type(typ): + return True + elif is_list_type(typ): + return False + elif is_vector_type(typ): + return is_fixed_size(read_vector_elem_typ(typ)) + elif is_container_typ(typ): + return all([is_fixed_size(t) for t in typ.get_field_types()]) else: - return merkleize_chunks([hash_tree_root(element, elem_typ) for element in value]) - - -def mix_in_length(root, length): - return hash(root + length.to_bytes(32, 'little')) - - -def hash_tree_root_container(fields): - return merkleize_chunks([hash_tree_root(field, subtype) for field, subtype in fields]) - - -hash_tree_root = ssz_switch({ - ssz_basic_type: lambda value, typ: merkleize_chunks(chunkify(pack([value], typ))), - ssz_list: lambda value, elem_typ: mix_in_length(hash_tree_root_list(value, elem_typ), len(value)), - ssz_vector: lambda value, elem_typ: hash_tree_root_list(value, elem_typ), - ssz_container: lambda value, get_field_values, field_types: hash_tree_root_container(zip(get_field_values(value), field_types)), -}) - -# todo: signing root -def signing_root(value, typ): - pass + raise Exception("Type not supported: {}".format(typ)) +def serialize(obj, typ=None): + if typ is None: + typ = infer_type(obj) + if is_basic_type(typ): + return serialize_basic(obj, typ) + elif is_list_type(typ) or is_vector_type(typ): + return encode_series(list(obj), [read_elem_typ(typ)]*len(obj)) + elif is_container_typ(typ): + return encode_series(obj.get_field_values(), typ.get_field_types()) + else: + raise Exception("Type not supported: {}".format(typ)) def encode_series(values, types): # bytes and bytesN are already in the right format. @@ -138,6 +75,46 @@ def encode_series(values, types): # Return the concatenation of the fixed-size parts (offsets interleaved) with the variable-size parts return b''.join(fixed_parts + variable_parts) +# SSZ Hash-tree-root +# ----------------------------- + +def pack(values, subtype): + if isinstance(values, bytes): + return values + return b''.join([serialize_basic(value, subtype) for value in values]) + +def chunkify(bytez): + bytez += b'\x00' * (-len(bytez) % 32) + return [bytez[i:i + 32] for i in range(0, len(bytez), 32)] + +def mix_in_length(root, length): + return hash(root + length.to_bytes(32, 'little')) + +def hash_tree_root(obj, typ=None): + if typ is None: + typ = infer_type(obj) + if is_basic_type(typ): + return merkleize_chunks(chunkify(serialize_basic(obj, typ))) + elif is_list_type(typ) or is_vector_type(typ): + subtype = read_elem_typ(typ) + if is_basic_type(subtype): + leaves = chunkify(pack(obj, subtype)) + else: + leaves = [hash_tree_root(elem, subtype) for elem in obj] + leaf_root = merkleize_chunks(leaves) + return mix_in_length(leaf_root, len(obj)) if is_list_type(typ) else leaf_root + elif is_container_typ(typ): + leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in zip(obj.get_field_values(), typ.get_field_types())] + return merkleize_chunks(chunkify(b''.join(leaves))) + else: + raise Exception("Type not supported: obj {} type {}".format(obj, typ)) + +def signing_root(value, typ): + if typ is None: + typ = infer_type(obj) + assert is_container_typ(typ) + leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in zip(obj.get_field_values(), typ.get_field_types())[:-1]] + return merkleize_chunks(chunkify(b''.join(leaves))) # Implementation notes: # - SSZContainer,Vector/BytesN.hash_tree_root/serialize functions are for ease, implementation here diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index dc23427b3..78c82be45 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -3,14 +3,7 @@ from typing import Union from inspect import isclass T = TypeVar('T') - -# SSZ list -# ----------------------------- - - -def read_list_elem_typ(list_typ: Type[List[T]]) -> T: - assert list_typ.__args__ is not None - return list_typ.__args__[0] +L = TypeVar('L') @@ -41,7 +34,6 @@ class SSZContainer(object): def __init__(self, **kwargs): cls = self.__class__ - from .ssz_impl import get_zero_value for f, t in cls.get_fields().items(): if f not in kwargs: setattr(self, f, get_zero_value(t)) @@ -125,7 +117,11 @@ class VectorMeta(type): return out def __getitem__(self, params): - return self.__class__(self.__name__, (Vector,), {'elem_type': params[0], 'length': params[1]}) + if not isinstance(params, tuple) or len(params) != 2: + raise Exception("Vector must be instantiated with two args: elem type and length") + o = self.__class__(self.__name__, (Vector,), {'elem_type': params[0], 'length': params[1]}) + o._name = 'Vector' + return o def __subclasscheck__(self, sub): return _is_vector_instance_of(self, sub) @@ -152,7 +148,6 @@ class Vector(metaclass=VectorMeta): if len(args) != cls.length: if len(args) == 0: - from .ssz_impl import get_zero_value args = [get_zero_value(cls.elem_type) for _ in range(cls.length)] else: raise TypeError("Typed vector with length %d cannot hold %d items" % (cls.length, len(args))) @@ -218,6 +213,8 @@ class BytesNMeta(type): out = type.__new__(cls, class_name, parents, attrs) if 'length' in attrs: setattr(out, 'length', attrs['length']) + out._name = 'Vector' + out.elem_type = byte return out def __getitem__(self, n): @@ -277,3 +274,63 @@ class BytesN(bytes, metaclass=BytesNMeta): def hash_tree_root(self): from .ssz_impl import hash_tree_root return hash_tree_root(self, self.__class__) + +# SSZ Defaults +# ----------------------------- + +def get_zero_value(typ): + if is_uint(typ): + return 0 + if issubclass(typ, bool): + return False + if issubclass(typ, list): + return [] + if issubclass(typ, Vector): + return typ() + if issubclass(typ, BytesN): + return typ() + if issubclass(typ, bytes): + return b'' + if issubclass(typ, SSZContainer): + return typ(**{f: get_zero_value(t) for f, t in typ.get_fields().items()}), + +# Type helpers +# ----------------------------- + +def infer_type(obj): + if is_uint(obj.__class__): + return obj.__class__ + elif isinstance(obj, int): + return uint64 + elif isinstance(obj, list): + return List[infer_type(obj[0])] + elif isinstance(obj, (Vector, SSZContainer, bool, BytesN, bytes)): + return obj.__class__ + else: + raise Exception("Unknown type for {}".format(obj)) + +def is_list_type(typ): + return (hasattr(typ, '_name') and typ._name == 'List') or typ == bytes + +def is_vector_type(typ): + return hasattr(typ, '_name') and typ._name == 'Vector' + +def is_container_typ(typ): + return hasattr(typ, 'get_fields') + +def read_list_elem_typ(list_typ: Type[List[T]]) -> T: + assert list_typ.__args__ is not None + return list_typ.__args__[0] + +def read_vector_elem_typ(vector_typ: Type[Vector[T, L]]) -> T: + return vector_typ.elem_type + +def read_elem_typ(typ): + if typ == bytes: + return byte + elif is_list_type(typ): + return read_list_elem_typ(typ) + elif is_vector_type(typ): + return read_vector_elem_typ(typ) + else: + raise Exception("Unexpected type: {}".format(typ)) From 3b4d9b7a859777a3e228e4348e2ca7075f87d78a Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sat, 25 May 2019 16:14:52 -0400 Subject: [PATCH 06/48] Class-ified the type definitions --- specs/core/0_beacon-chain.md | 60 ++++++++++++------------------------ 1 file changed, 20 insertions(+), 40 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 35d273a2e..8442819c4 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -268,20 +268,19 @@ The types are defined topologically to aid in facilitating an executable version #### `Fork` ```python -{ +class Fork(SSZContainer): # Previous fork version previous_version: bytes4 # Current fork version current_version: bytes4 # Fork epoch number epoch: uint64 -} ``` #### `Crosslink` ```python -{ +class Crosslink(SSZContainer): # Shard number shard: uint64 # Epoch number @@ -290,26 +289,24 @@ The types are defined topologically to aid in facilitating an executable version parent_root: bytes32 # Root of the crosslinked shard data since the previous crosslink data_root: bytes32 -} ``` #### `Eth1Data` ```python -{ +class Eth1Data(SSZContainer): # Root of the deposit tree deposit_root: bytes32 # Total number of deposits deposit_count: uint64 # Block hash block_hash: bytes32 -} ``` #### `AttestationData` ```python -{ +class AttestationData(SSZContainer): # LMD GHOST vote beacon_block_root: bytes32 @@ -321,24 +318,22 @@ The types are defined topologically to aid in facilitating an executable version # Crosslink vote crosslink: Crosslink -} ``` #### `AttestationDataAndCustodyBit` ```python -{ +class AttestationDataAndCustodyBit(SSZContainer): # Attestation data data: AttestationData # Custody bit custody_bit: bool -} ``` #### `IndexedAttestation` ```python -{ +class IndexedAttestation(SSZContainer): # Validator indices custody_bit_0_indices: List[uint64] custody_bit_1_indices: List[uint64] @@ -346,13 +341,12 @@ The types are defined topologically to aid in facilitating an executable version data: AttestationData # Aggregate signature signature: bytes96 -} ``` #### `DepositData` ```python -{ +class DepositData(SSZContainer): # BLS pubkey pubkey: bytes48 # Withdrawal credentials @@ -361,24 +355,22 @@ The types are defined topologically to aid in facilitating an executable version amount: uint64 # Container self-signature signature: bytes96 -} ``` #### `BeaconBlockHeader` ```python -{ +class BeaconBlockHeader(SSZContainer): slot: uint64 parent_root: bytes32 state_root: bytes32 body_root: bytes32 signature: bytes96 -} ``` #### `Validator` ```python -{ +class Validator(SSZContainer): # BLS public key pubkey: bytes48 # Withdrawal credentials @@ -395,13 +387,12 @@ The types are defined topologically to aid in facilitating an executable version slashed: bool # Effective balance effective_balance: uint64 -} ``` #### `PendingAttestation` ```python -{ +class PendingAttestation(SSZContainer): # Attester aggregation bitfield aggregation_bitfield: bytes # Attestation data @@ -410,18 +401,16 @@ The types are defined topologically to aid in facilitating an executable version inclusion_delay: uint64 # Proposer index proposer_index: uint64 -} ``` #### `HistoricalBatch` ```python -{ +class HistoricalBatch(SSZContainer): # Block roots block_roots: Vector[bytes32, SLOTS_PER_HISTORICAL_ROOT] # State roots state_roots: Vector[bytes32, SLOTS_PER_HISTORICAL_ROOT] -} ``` ### Beacon operations @@ -429,31 +418,29 @@ The types are defined topologically to aid in facilitating an executable version #### `ProposerSlashing` ```python -{ +class ProposerSlashing(SSZContainer): # Proposer index proposer_index: uint64 # First block header header_1: BeaconBlockHeader # Second block header header_2: BeaconBlockHeader -} ``` #### `AttesterSlashing` ```python -{ +class AttesterSlashing(SSZContainer): # First attestation attestation_1: IndexedAttestation # Second attestation attestation_2: IndexedAttestation -} ``` #### `Attestation` ```python -{ +class Attestation(SSZContainer): # Attester aggregation bitfield aggregation_bitfield: bytes # Attestation data @@ -462,39 +449,36 @@ The types are defined topologically to aid in facilitating an executable version custody_bitfield: bytes # BLS aggregate signature signature: bytes96 -} ``` #### `Deposit` ```python -{ +class Deposit(SSZContainer): # Branch in the deposit tree proof: Vector[bytes32, DEPOSIT_CONTRACT_TREE_DEPTH] # Index in the deposit tree index: uint64 # Data data: DepositData -} ``` #### `VoluntaryExit` ```python -{ +class VoluntaryExit(SSZContainer): # Minimum epoch for processing exit epoch: uint64 # Index of the exiting validator validator_index: uint64 # Validator signature signature: bytes96 -} ``` #### `Transfer` ```python -{ +class Transfer(SSZContainer): # Sender index sender: uint64 # Recipient index @@ -509,7 +493,6 @@ The types are defined topologically to aid in facilitating an executable version pubkey: bytes48 # Sender signature signature: bytes96 -} ``` ### Beacon blocks @@ -517,7 +500,7 @@ The types are defined topologically to aid in facilitating an executable version #### `BeaconBlockBody` ```python -{ +class BeaconBlockBody(SSZContainer): randao_reveal: bytes96 eth1_data: Eth1Data graffiti: bytes32 @@ -527,20 +510,18 @@ The types are defined topologically to aid in facilitating an executable version deposits: List[Deposit] voluntary_exits: List[VoluntaryExit] transfers: List[Transfer] -} ``` #### `BeaconBlock` ```python -{ +class BeaconBlock(SSZContainer): # Header slot: uint64 parent_root: bytes32 state_root: bytes32 body: BeaconBlockBody signature: bytes96 -} ``` ### Beacon state @@ -548,7 +529,7 @@ The types are defined topologically to aid in facilitating an executable version #### `BeaconState` ```python -{ +class BeaconState(SSZContainer): # Misc slot: uint64 genesis_time: uint64 @@ -589,7 +570,6 @@ The types are defined topologically to aid in facilitating an executable version latest_eth1_data: Eth1Data eth1_data_votes: List[Eth1Data] deposit_index: uint64 -} ``` ## Custom types From e14f789779ad88903f56acb459a03c3b71bfb79c Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Mon, 27 May 2019 08:07:00 -0400 Subject: [PATCH 07/48] Presentation edits --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 6 +++--- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 8b6c20a17..79675fb4a 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -104,7 +104,7 @@ def hash_tree_root(obj, typ=None): leaf_root = merkleize_chunks(leaves) return mix_in_length(leaf_root, len(obj)) if is_list_type(typ) else leaf_root elif is_container_typ(typ): - leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in zip(obj.get_field_values(), typ.get_field_types())] + leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields().items()] return merkleize_chunks(chunkify(b''.join(leaves))) else: raise Exception("Type not supported: obj {} type {}".format(obj, typ)) @@ -113,11 +113,11 @@ def signing_root(value, typ): if typ is None: typ = infer_type(obj) assert is_container_typ(typ) - leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in zip(obj.get_field_values(), typ.get_field_types())[:-1]] + leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields().items()] return merkleize_chunks(chunkify(b''.join(leaves))) # Implementation notes: -# - SSZContainer,Vector/BytesN.hash_tree_root/serialize functions are for ease, implementation here +# - Container,Vector/BytesN.hash_tree_root/serialize functions are for ease, implementation here # - uint types have a 'byte_len' attribute # - uint types are not classes. They use NewType(), for performance. # This forces us to check type equivalence by exact reference. diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 78c82be45..76e518e64 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -30,7 +30,7 @@ byte = NewType('byte', uint8) # Note: importing ssz functionality locally, to avoid import loop -class SSZContainer(object): +class Container(object): def __init__(self, **kwargs): cls = self.__class__ @@ -291,7 +291,7 @@ def get_zero_value(typ): return typ() if issubclass(typ, bytes): return b'' - if issubclass(typ, SSZContainer): + if issubclass(typ, Container): return typ(**{f: get_zero_value(t) for f, t in typ.get_fields().items()}), # Type helpers @@ -304,7 +304,7 @@ def infer_type(obj): return uint64 elif isinstance(obj, list): return List[infer_type(obj[0])] - elif isinstance(obj, (Vector, SSZContainer, bool, BytesN, bytes)): + elif isinstance(obj, (Vector, Container, bool, BytesN, bytes)): return obj.__class__ else: raise Exception("Unknown type for {}".format(obj)) From d5eab257d0a45ba9dd25be8968fe10766751d5b6 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 18:01:46 +0200 Subject: [PATCH 08/48] improve impl, box less used integer types, use uint64 as default --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 48 +++---- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 129 +++++++++++++----- 2 files changed, 116 insertions(+), 61 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 79675fb4a..4b8545b17 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -6,18 +6,21 @@ from .ssz_typing import * BYTES_PER_LENGTH_OFFSET = 4 + def is_basic_type(typ): return is_uint(typ) or typ == bool + def serialize_basic(value, typ): if is_uint(typ): - return value.to_bytes(typ.byte_len, 'little') + return value.to_bytes(uint_byte_size(typ), 'little') if issubclass(typ, bool): if value: return b'\x01' else: return b'\x00' + def is_fixed_size(typ): if is_basic_type(typ): return True @@ -30,9 +33,9 @@ def is_fixed_size(typ): else: raise Exception("Type not supported: {}".format(typ)) -def serialize(obj, typ=None): - if typ is None: - typ = infer_type(obj) + +@infer_input_type +def serialize(obj, typ): if is_basic_type(typ): return serialize_basic(obj, typ) elif is_list_type(typ) or is_vector_type(typ): @@ -42,6 +45,7 @@ def serialize(obj, typ=None): else: raise Exception("Type not supported: {}".format(typ)) + def encode_series(values, types): # bytes and bytesN are already in the right format. if isinstance(values, bytes): @@ -75,24 +79,28 @@ def encode_series(values, types): # Return the concatenation of the fixed-size parts (offsets interleaved) with the variable-size parts return b''.join(fixed_parts + variable_parts) + # SSZ Hash-tree-root # ----------------------------- + def pack(values, subtype): if isinstance(values, bytes): return values return b''.join([serialize_basic(value, subtype) for value in values]) + def chunkify(bytez): bytez += b'\x00' * (-len(bytez) % 32) return [bytez[i:i + 32] for i in range(0, len(bytez), 32)] + def mix_in_length(root, length): return hash(root + length.to_bytes(32, 'little')) -def hash_tree_root(obj, typ=None): - if typ is None: - typ = infer_type(obj) + +@infer_input_type +def hash_tree_root(obj, typ): if is_basic_type(typ): return merkleize_chunks(chunkify(serialize_basic(obj, typ))) elif is_list_type(typ) or is_vector_type(typ): @@ -104,31 +112,15 @@ def hash_tree_root(obj, typ=None): leaf_root = merkleize_chunks(leaves) return mix_in_length(leaf_root, len(obj)) if is_list_type(typ) else leaf_root elif is_container_typ(typ): - leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields().items()] + leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields()] return merkleize_chunks(chunkify(b''.join(leaves))) else: raise Exception("Type not supported: obj {} type {}".format(obj, typ)) -def signing_root(value, typ): - if typ is None: - typ = infer_type(obj) + +@infer_input_type +def signing_root(obj, typ): assert is_container_typ(typ) - leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields().items()] + leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields()[:-1]] return merkleize_chunks(chunkify(b''.join(leaves))) -# Implementation notes: -# - Container,Vector/BytesN.hash_tree_root/serialize functions are for ease, implementation here -# - uint types have a 'byte_len' attribute -# - uint types are not classes. They use NewType(), for performance. -# This forces us to check type equivalence by exact reference. -# There's no class. The type data comes from an annotation/argument from the context of the value. -# - Vector is not valid to create instances with. Give it a elem-type and length: Vector[FooBar, 123] -# - *The class of* a Vector instance has a `elem_type` (type, may not be a class, see uint) and `length` (int) -# - BytesN is not valid to create instances with. Give it a length: BytesN[123] -# - *The class of* a BytesN instance has a `length` (int) -# Where possible, it is preferable to create helpers that just act on the type, and don't unnecessarily use a value -# E.g. is_basic_type(). This way, we can use them in type-only contexts and have no duplicate logic. -# For every class-instance, you can get the type with my_object.__class__ -# For uints, and other NewType related, you have to rely on type information. It cannot be retrieved from the value. -# Note: we may just want to box integers instead. And then we can do bounds checking too. But it is SLOW and MEMORY INTENSIVE. -# diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 76e518e64..f117d9400 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,30 +1,82 @@ -from typing import List, Iterable, TypeVar, Type, NewType +from typing import List, Iterable, Type, NewType from typing import Union from inspect import isclass -T = TypeVar('T') -L = TypeVar('L') - - -# SSZ integer types, with 0 computational overhead (NewType) +# SSZ integers # ----------------------------- -uint8 = NewType('uint8', int) -uint8.byte_len = 1 -uint16 = NewType('uint16', int) -uint16.byte_len = 2 -uint32 = NewType('uint32', int) -uint32.byte_len = 4 -uint64 = NewType('uint64', int) -uint64.byte_len = 8 -uint128 = NewType('uint128', int) -uint128.byte_len = 16 -uint256 = NewType('uint256', int) -uint256.byte_len = 32 +class uint(int): + byte_len = 0 + def __new__(cls, value, *args, **kwargs): + if value < 0: + raise ValueError("unsigned types must not be negative") + return super().__new__(cls, value) + + +class uint8(uint): + byte_len = 1 + def __new__(cls, value, *args, **kwargs): + if value.bit_length() > 8: + raise ValueError("value out of bounds for uint8") + return super().__new__(cls, value) + +# Alias for uint8 byte = NewType('byte', uint8) +class uint16(uint): + byte_len = 2 + def __new__(cls, value, *args, **kwargs): + if value.bit_length() > 16: + raise ValueError("value out of bounds for uint16") + return super().__new__(cls, value) + + +class uint32(uint): + byte_len = 4 + def __new__(cls, value, *args, **kwargs): + if value.bit_length() > 32: + raise ValueError("value out of bounds for uint16") + return super().__new__(cls, value) + + +# We simply default to uint64. But do give it a name, for readability +uint64 = NewType('uint64', int) + + +class uint128(uint): + byte_len = 16 + def __new__(cls, value, *args, **kwargs): + if value.bit_length() > 128: + raise ValueError("value out of bounds for uint128") + return super().__new__(cls, value) + + +class uint256(uint): + byte_len = 32 + def __new__(cls, value, *args, **kwargs): + if value.bit_length() > 256: + raise ValueError("value out of bounds for uint256") + return super().__new__(cls, value) + + +def is_uint(typ): + # All integers are uint in the scope of the spec here. + # Since we default to uint64. Bounds can be checked elsewhere. + return issubclass(typ, int) + + +def uint_byte_size(typ): + if issubclass(typ, uint): + return typ.byte_len + elif issubclass(typ, int): + # Default to uint64 + return 8 + else: + raise TypeError("Type %s is not an uint (or int-default uint64) type" % typ) + + # SSZ Container base class # ----------------------------- @@ -57,9 +109,13 @@ class Container(object): return [getattr(self, field) for field in cls.get_field_names()] @classmethod - def get_fields(cls): + def get_fields_dict(cls): return dict(cls.__annotations__) + @classmethod + def get_fields(cls): + return dict(cls.__annotations__).items() + @classmethod def get_field_names(cls): return list(cls.__annotations__.keys()) @@ -70,14 +126,6 @@ class Container(object): return list(cls.__annotations__.values()) -def is_uint(typ): - # Note: only the type reference exists, - # but it really resolves to 'int' during run-time for zero computational/memory overhead. - # Hence, we check equality to the type references (which are really just 'NewType' instances), - # and don't use any sub-classing like we normally would. - return typ == uint8 or typ == uint16 or typ == uint32 or typ == uint64 \ - or typ == uint128 or typ == uint256 or typ == byte - # SSZ vector # ----------------------------- @@ -138,7 +186,7 @@ class VectorMeta(type): class Vector(metaclass=VectorMeta): - def __init__(self, *args: Iterable[T]): + def __init__(self, *args: Iterable): cls = self.__class__ if not hasattr(cls, 'elem_type'): @@ -275,6 +323,7 @@ class BytesN(bytes, metaclass=BytesNMeta): from .ssz_impl import hash_tree_root return hash_tree_root(self, self.__class__) + # SSZ Defaults # ----------------------------- @@ -292,7 +341,8 @@ def get_zero_value(typ): if issubclass(typ, bytes): return b'' if issubclass(typ, Container): - return typ(**{f: get_zero_value(t) for f, t in typ.get_fields().items()}), + return typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}), + # Type helpers # ----------------------------- @@ -309,17 +359,30 @@ def infer_type(obj): else: raise Exception("Unknown type for {}".format(obj)) + +def infer_input_type(fn): + """ + Decorator to run infer_type on the obj if typ argument is None + """ + def infer_helper(obj, typ=None): + if typ is None: + typ = infer_type(obj) + return fn(obj, typ) + return infer_helper + + def is_list_type(typ): return (hasattr(typ, '_name') and typ._name == 'List') or typ == bytes def is_vector_type(typ): - return hasattr(typ, '_name') and typ._name == 'Vector' + return issubclass(typ, Vector) def is_container_typ(typ): - return hasattr(typ, 'get_fields') + return issubclass(typ, Container) def read_list_elem_typ(list_typ: Type[List[T]]) -> T: - assert list_typ.__args__ is not None + if list_typ.__args__ is None or len(list_typ.__args__) != 1: + raise TypeError("Supplied list-type is invalid, no element type found.") return list_typ.__args__[0] def read_vector_elem_typ(vector_typ: Type[Vector[T, L]]) -> T: @@ -333,4 +396,4 @@ def read_elem_typ(typ): elif is_vector_type(typ): return read_vector_elem_typ(typ) else: - raise Exception("Unexpected type: {}".format(typ)) + raise TypeError("Unexpected type: {}".format(typ)) From f3088884b3c6441271245982fd9c6050d3a3d581 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 20:29:53 +0200 Subject: [PATCH 09/48] Minor adjustments + getting the beacon spec doc ready --- specs/core/0_beacon-chain.md | 128 +++++++++--------- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 6 + 3 files changed, 71 insertions(+), 65 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8442819c4..cae9dc55d 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -268,11 +268,11 @@ The types are defined topologically to aid in facilitating an executable version #### `Fork` ```python -class Fork(SSZContainer): +class Fork(Container): # Previous fork version - previous_version: bytes4 + previous_version: Bytes4 # Current fork version - current_version: bytes4 + current_version: Bytes4 # Fork epoch number epoch: uint64 ``` @@ -280,41 +280,41 @@ class Fork(SSZContainer): #### `Crosslink` ```python -class Crosslink(SSZContainer): +class Crosslink(Container): # Shard number shard: uint64 # Epoch number epoch: uint64 # Root of the previous crosslink - parent_root: bytes32 + parent_root: Bytes32 # Root of the crosslinked shard data since the previous crosslink - data_root: bytes32 + data_root: Bytes32 ``` #### `Eth1Data` ```python -class Eth1Data(SSZContainer): +class Eth1Data(Container): # Root of the deposit tree - deposit_root: bytes32 + deposit_root: Bytes32 # Total number of deposits deposit_count: uint64 # Block hash - block_hash: bytes32 + block_hash: Bytes32 ``` #### `AttestationData` ```python -class AttestationData(SSZContainer): +class AttestationData(Container): # LMD GHOST vote - beacon_block_root: bytes32 + beacon_block_root: Bytes32 # FFG vote source_epoch: uint64 - source_root: bytes32 + source_root: Bytes32 target_epoch: uint64 - target_root: bytes32 + target_root: Bytes32 # Crosslink vote crosslink: Crosslink @@ -323,7 +323,7 @@ class AttestationData(SSZContainer): #### `AttestationDataAndCustodyBit` ```python -class AttestationDataAndCustodyBit(SSZContainer): +class AttestationDataAndCustodyBit(Container): # Attestation data data: AttestationData # Custody bit @@ -333,48 +333,48 @@ class AttestationDataAndCustodyBit(SSZContainer): #### `IndexedAttestation` ```python -class IndexedAttestation(SSZContainer): +class IndexedAttestation(Container): # Validator indices custody_bit_0_indices: List[uint64] custody_bit_1_indices: List[uint64] # Attestation data data: AttestationData # Aggregate signature - signature: bytes96 + signature: Bytes96 ``` #### `DepositData` ```python -class DepositData(SSZContainer): +class DepositData(Container): # BLS pubkey - pubkey: bytes48 + pubkey: Bytes48 # Withdrawal credentials - withdrawal_credentials: bytes32 + withdrawal_credentials: Bytes32 # Amount in Gwei amount: uint64 # Container self-signature - signature: bytes96 + signature: Bytes96 ``` #### `BeaconBlockHeader` ```python -class BeaconBlockHeader(SSZContainer): +class BeaconBlockHeader(Container): slot: uint64 - parent_root: bytes32 - state_root: bytes32 - body_root: bytes32 - signature: bytes96 + parent_root: Bytes32 + state_root: Bytes32 + body_root: Bytes32 + signature: Bytes96 ``` #### `Validator` ```python -class Validator(SSZContainer): +class Validator(Container): # BLS public key - pubkey: bytes48 + pubkey: Bytes48 # Withdrawal credentials - withdrawal_credentials: bytes32 + withdrawal_credentials: Bytes32 # Epoch when became eligible for activation activation_eligibility_epoch: uint64 # Epoch when validator activated @@ -392,7 +392,7 @@ class Validator(SSZContainer): #### `PendingAttestation` ```python -class PendingAttestation(SSZContainer): +class PendingAttestation(Container): # Attester aggregation bitfield aggregation_bitfield: bytes # Attestation data @@ -406,11 +406,11 @@ class PendingAttestation(SSZContainer): #### `HistoricalBatch` ```python -class HistoricalBatch(SSZContainer): +class HistoricalBatch(Container): # Block roots - block_roots: Vector[bytes32, SLOTS_PER_HISTORICAL_ROOT] + block_roots: Vector[Bytes32, SLOTS_PER_HISTORICAL_ROOT] # State roots - state_roots: Vector[bytes32, SLOTS_PER_HISTORICAL_ROOT] + state_roots: Vector[Bytes32, SLOTS_PER_HISTORICAL_ROOT] ``` ### Beacon operations @@ -418,7 +418,7 @@ class HistoricalBatch(SSZContainer): #### `ProposerSlashing` ```python -class ProposerSlashing(SSZContainer): +class ProposerSlashing(Container): # Proposer index proposer_index: uint64 # First block header @@ -430,7 +430,7 @@ class ProposerSlashing(SSZContainer): #### `AttesterSlashing` ```python -class AttesterSlashing(SSZContainer): +class AttesterSlashing(Container): # First attestation attestation_1: IndexedAttestation # Second attestation @@ -440,7 +440,7 @@ class AttesterSlashing(SSZContainer): #### `Attestation` ```python -class Attestation(SSZContainer): +class Attestation(Container): # Attester aggregation bitfield aggregation_bitfield: bytes # Attestation data @@ -448,15 +448,15 @@ class Attestation(SSZContainer): # Custody bitfield custody_bitfield: bytes # BLS aggregate signature - signature: bytes96 + signature: Bytes96 ``` #### `Deposit` ```python -class Deposit(SSZContainer): +class Deposit(Container): # Branch in the deposit tree - proof: Vector[bytes32, DEPOSIT_CONTRACT_TREE_DEPTH] + proof: Vector[Bytes32, DEPOSIT_CONTRACT_TREE_DEPTH] # Index in the deposit tree index: uint64 # Data @@ -466,19 +466,19 @@ class Deposit(SSZContainer): #### `VoluntaryExit` ```python -class VoluntaryExit(SSZContainer): +class VoluntaryExit(Container): # Minimum epoch for processing exit epoch: uint64 # Index of the exiting validator validator_index: uint64 # Validator signature - signature: bytes96 + signature: Bytes96 ``` #### `Transfer` ```python -class Transfer(SSZContainer): +class Transfer(Container): # Sender index sender: uint64 # Recipient index @@ -490,9 +490,9 @@ class Transfer(SSZContainer): # Inclusion slot slot: uint64 # Sender withdrawal pubkey - pubkey: bytes48 + pubkey: Bytes48 # Sender signature - signature: bytes96 + signature: Bytes96 ``` ### Beacon blocks @@ -500,10 +500,10 @@ class Transfer(SSZContainer): #### `BeaconBlockBody` ```python -class BeaconBlockBody(SSZContainer): - randao_reveal: bytes96 +class BeaconBlockBody(Container): + randao_reveal: Bytes96 eth1_data: Eth1Data - graffiti: bytes32 + graffiti: Bytes32 proposer_slashings: List[ProposerSlashing] attester_slashings: List[AttesterSlashing] attestations: List[Attestation] @@ -515,13 +515,13 @@ class BeaconBlockBody(SSZContainer): #### `BeaconBlock` ```python -class BeaconBlock(SSZContainer): +class BeaconBlock(Container): # Header slot: uint64 - parent_root: bytes32 - state_root: bytes32 + parent_root: Bytes32 + state_root: Bytes32 body: BeaconBlockBody - signature: bytes96 + signature: Bytes96 ``` ### Beacon state @@ -529,7 +529,7 @@ class BeaconBlock(SSZContainer): #### `BeaconState` ```python -class BeaconState(SSZContainer): +class BeaconState(Container): # Misc slot: uint64 genesis_time: uint64 @@ -540,7 +540,7 @@ class BeaconState(SSZContainer): balances: List[uint64] # Randomness and committees - latest_randao_mixes: Vector[bytes32, LATEST_RANDAO_MIXES_LENGTH] + latest_randao_mixes: Vector[Bytes32, LATEST_RANDAO_MIXES_LENGTH] latest_start_shard: uint64 # Finality @@ -548,23 +548,23 @@ class BeaconState(SSZContainer): current_epoch_attestations: List[PendingAttestation] previous_justified_epoch: uint64 current_justified_epoch: uint64 - previous_justified_root: bytes32 - current_justified_root: bytes32 + previous_justified_root: Bytes32 + current_justified_root: Bytes32 justification_bitfield: uint64 finalized_epoch: uint64 - finalized_root: bytes32 + finalized_root: Bytes32 # Recent state current_crosslinks: Vector[Crosslink, SHARD_COUNT] previous_crosslinks: Vector[Crosslink, SHARD_COUNT] - latest_block_roots: Vector[bytes32, SLOTS_PER_HISTORICAL_ROOT] - latest_state_roots: Vector[bytes32, SLOTS_PER_HISTORICAL_ROOT] - latest_active_index_roots: Vector[bytes32, LATEST_ACTIVE_INDEX_ROOTS_LENGTH] + latest_block_roots: Vector[Bytes32, SLOTS_PER_HISTORICAL_ROOT] + latest_state_roots: Vector[Bytes32, SLOTS_PER_HISTORICAL_ROOT] + latest_active_index_roots: Vector[Bytes32, LATEST_ACTIVE_INDEX_ROOTS_LENGTH] # Balances slashed at every withdrawal period latest_slashed_balances: Vector[uint64, LATEST_SLASHED_EXIT_LENGTH] # `latest_block_header.state_root == ZERO_HASH` temporarily latest_block_header: BeaconBlockHeader - historical_roots: List[bytes32] + historical_roots: List[Bytes32] # Ethereum 1.0 chain data latest_eth1_data: Eth1Data @@ -583,9 +583,9 @@ We define the following Python custom types for type hinting and readability: | `Shard` | `uint64` | a shard number | | `ValidatorIndex` | `uint64` | a validator registry index | | `Gwei` | `uint64` | an amount in Gwei | -| `Bytes32` | `bytes32` | 32 bytes of binary data | -| `BLSPubkey` | `bytes48` | a BLS12-381 public key | -| `BLSSignature` | `bytes96` | a BLS12-381 signature | +| `Bytes32` | `Bytes32` | 32 bytes of binary data | +| `BLSPubkey` | `Bytes48` | a BLS12-381 public key | +| `BLSSignature` | `Bytes96` | a BLS12-381 signature | ## Helper functions @@ -595,7 +595,7 @@ We define the following Python custom types for type hinting and readability: ```python def xor(bytes1: Bytes32, bytes2: Bytes32) -> Bytes32: - return bytes(a ^ b for a, b in zip(bytes1, bytes2)) + return Bytes32(a ^ b for a, b in zip(bytes1, bytes2)) ``` ### `hash` @@ -610,7 +610,7 @@ The `hash` function is SHA256. ### `signing_root` -`def signing_root(object: SSZContainer) -> Bytes32` is a function defined in the [SimpleSerialize spec](../simple-serialize.md#self-signed-containers) to compute signing messages. +`def signing_root(object: Container) -> Bytes32` is a function defined in the [SimpleSerialize spec](../simple-serialize.md#self-signed-containers) to compute signing messages. ### `slot_to_epoch` diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 4b8545b17..655cd9da4 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -29,7 +29,7 @@ def is_fixed_size(typ): elif is_vector_type(typ): return is_fixed_size(read_vector_elem_typ(typ)) elif is_container_typ(typ): - return all([is_fixed_size(t) for t in typ.get_field_types()]) + return all(is_fixed_size(t) for t in typ.get_field_types()) else: raise Exception("Type not supported: {}".format(typ)) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index f117d9400..1d8aed0de 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -108,6 +108,9 @@ class Container(object): cls = self.__class__ return [getattr(self, field) for field in cls.get_field_names()] + def __repr__(self): + return {field: getattr(self, field) for field in self.get_field_names()} + @classmethod def get_fields_dict(cls): return dict(cls.__annotations__) @@ -217,6 +220,9 @@ class Vector(metaclass=VectorMeta): from .ssz_impl import hash_tree_root return hash_tree_root(self, self.__class__) + def __repr__(self): + return {'length': self.__class__.length, 'items': self.items} + def __getitem__(self, key): return self.items[key] From 132d3c976a0529d358363d0106960b05da7285dd Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 21:14:59 +0200 Subject: [PATCH 10/48] fix spec builder --- scripts/phase0/build_spec.py | 18 ++++++----- scripts/phase0/function_puller.py | 50 +++++++++++++++++++++---------- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index b188dada2..59d9e9f9d 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -16,7 +16,10 @@ from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, signing_root, ) -from eth2spec.utils.ssz.ssz_typing import * +from eth2spec.utils.ssz.ssz_typing import ( + uint8, uint16, uint32, uint64, uint128, uint256, + Container, Vector, BytesN +) from eth2spec.utils.bls_stub import ( bls_aggregate_pubkeys, bls_verify, @@ -24,19 +27,18 @@ from eth2spec.utils.bls_stub import ( ) from eth2spec.utils.hash_function import hash - -# stub, will get overwritten by real var -SLOTS_PER_EPOCH = 64 - +# Note: 'int' type defaults to being interpreted as a uint64 by SSZ implementation. Slot = NewType('Slot', int) # uint64 Epoch = NewType('Epoch', int) # uint64 Shard = NewType('Shard', int) # uint64 ValidatorIndex = NewType('ValidatorIndex', int) # uint64 Gwei = NewType('Gwei', int) # uint64 + +Bytes4 = BytesN[4] Bytes32 = BytesN[32] -BLSPubkey = NewType('BLSPubkey', BytesN[48]) -BLSSignature = NewType('BLSSignature', BytesN[96]) -Store = None +Bytes48 = BytesN[48] +Bytes96 = BytesN[96] + """) code_lines += function_puller.get_spec(sourcefile) diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index 5c1cf859b..0047d5321 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -1,4 +1,3 @@ -import sys from typing import List @@ -6,9 +5,11 @@ def get_spec(file_name: str) -> List[str]: code_lines = [] pulling_from = None current_name = None + # list of current type definition being parsed, or None otherwise current_typedef = None + # list of (name, definition lines list) tuples. type_defs = [] - for linenum, line in enumerate(open(sys.argv[1]).readlines()): + for linenum, line in enumerate(open(file_name).readlines()): line = line.rstrip() if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`': current_name = line[line[:-1].rfind('`') + 1: -1] @@ -19,22 +20,24 @@ def get_spec(file_name: str) -> List[str]: if pulling_from is None: pulling_from = linenum else: - if current_typedef is not None: - assert code_lines[-1] == '}' - code_lines[-1] = '' - code_lines.append('') pulling_from = None - current_typedef = None + if current_typedef is not None: + type_defs.append((current_name, current_typedef)) + current_typedef = None else: - if pulling_from == linenum and line == '{': - code_lines.append('class %s(SSZContainer):' % current_name) - current_typedef = current_name - type_defs.append(current_name) - elif pulling_from is not None: + if pulling_from is not None: # Add some whitespace between functions - if line[:3] == 'def': + if line[:3] == 'def' or line[:5] == 'class': code_lines.append('') code_lines.append('') + # Check for SSZ type definitions + if len(line) > 18 and line[:6] == 'class ' and line[-12:] == '(Container):': + name = line[6:-12] + # Check consistency with markdown header + assert name == current_name + current_typedef = [] + if current_typedef is not None: + current_typedef.append(line) code_lines.append(line) elif pulling_from is None and len(line) > 0 and line[0] == '|': row = line[1:].split('|') @@ -54,11 +57,26 @@ def get_spec(file_name: str) -> List[str]: # Build type-def re-initialization code_lines.append('\n') code_lines.append('ssz_types = [\n') - for ssz_type_name in type_defs: - code_lines.append(f' {ssz_type_name},\n') + for (ssz_type_name, _) in type_defs: + code_lines.append(f' {ssz_type_name},') code_lines.append(']') code_lines.append('\n') - code_lines.append('def get_ssz_type_by_name(name: str) -> SSZType:') + code_lines.append('def init_SSZ_types():') + code_lines.append(' global_vars = globals()') + for ssz_type_name, ssz_type in type_defs: + code_lines.append('') + for type_line in ssz_type: + if len(type_line) > 0: + code_lines.append(' ' + type_line) + code_lines.append('\n') + for (ssz_type_name, _) in type_defs: + code_lines.append(f' global_vars["{ssz_type_name}"] = {ssz_type_name},') + code_lines.append(' global_vars["ssz_types"] = [') + for (ssz_type_name, _) in type_defs: + code_lines.append(f' {ssz_type_name},') + code_lines.append(' ]') + code_lines.append('\n') + code_lines.append('def get_ssz_type_by_name(name: str) -> Container:') code_lines.append(' return globals()[name]') code_lines.append('') return code_lines From d63b553a2da3f5293d7202183c18cfdf28816b7f Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 21:45:47 +0200 Subject: [PATCH 11/48] efficiency bugfix in bytes encoding, improve typing doc + bugfix --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 4 ++-- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 655cd9da4..46d842e16 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -14,7 +14,7 @@ def is_basic_type(typ): def serialize_basic(value, typ): if is_uint(typ): return value.to_bytes(uint_byte_size(typ), 'little') - if issubclass(typ, bool): + if is_bool_type(typ): if value: return b'\x01' else: @@ -39,7 +39,7 @@ def serialize(obj, typ): if is_basic_type(typ): return serialize_basic(obj, typ) elif is_list_type(typ) or is_vector_type(typ): - return encode_series(list(obj), [read_elem_typ(typ)]*len(obj)) + return encode_series(obj, [read_elem_typ(typ)]*len(obj)) elif is_container_typ(typ): return encode_series(obj.get_field_values(), typ.get_field_types()) else: diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 1d8aed0de..93a272c8c 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -376,12 +376,20 @@ def infer_input_type(fn): return fn(obj, typ) return infer_helper +def is_bool_type(typ): + return issubclass(typ, bool) def is_list_type(typ): + """ + Checks if the given type is a kind of list. Can be bytes. + """ return (hasattr(typ, '_name') and typ._name == 'List') or typ == bytes def is_vector_type(typ): - return issubclass(typ, Vector) + """ + Checks if the given type is a kind of vector. Can be BytesN. + """ + return issubclass(typ, Vector) or issubclass(typ, BytesN) def is_container_typ(typ): return issubclass(typ, Container) From 87b3466eae90ebe557899083ca50e3babc29bb6f Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 21:46:14 +0200 Subject: [PATCH 12/48] update encoder and decoder for reading from parsed data --- test_libs/pyspec/eth2spec/debug/decode.py | 45 +++++++++++++---------- test_libs/pyspec/eth2spec/debug/encode.py | 25 +++++++------ 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/decode.py b/test_libs/pyspec/eth2spec/debug/decode.py index e9aa8bc2b..261692bed 100644 --- a/test_libs/pyspec/eth2spec/debug/decode.py +++ b/test_libs/pyspec/eth2spec/debug/decode.py @@ -1,28 +1,35 @@ -from eth2spec.utils.minimal_ssz import hash_tree_root +from eth2spec.utils.ssz.ssz_impl import hash_tree_root +from eth2spec.utils.ssz.ssz_typing import * -def decode(json, typ): - if isinstance(typ, str) and typ[:4] == 'uint': - return json - elif typ == 'bool': - assert json in (True, False) - return json - elif isinstance(typ, list): - return [decode(element, typ[0]) for element in json] - elif isinstance(typ, str) and typ[:4] == 'byte': - return bytes.fromhex(json[2:]) - elif hasattr(typ, 'fields'): +def decode(data, typ): + if is_uint(typ): + return data + elif is_bool_type(typ): + assert data in (True, False) + return data + elif issubclass(typ, list): + elem_typ = read_list_elem_typ(typ) + return [decode(element, elem_typ) for element in data] + elif issubclass(typ, Vector): + elem_typ = read_vector_elem_typ(typ) + return Vector(decode(element, elem_typ) for element in data) + elif issubclass(typ, bytes): + return bytes.fromhex(data[2:]) + elif issubclass(typ, BytesN): + return BytesN(bytes.fromhex(data[2:])) + elif is_container_typ(typ): temp = {} - for field, subtype in typ.fields.items(): - temp[field] = decode(json[field], subtype) - if field + "_hash_tree_root" in json: - assert(json[field + "_hash_tree_root"][2:] == + for field, subtype in typ.get_fields(): + temp[field] = decode(data[field], subtype) + if field + "_hash_tree_root" in data: + assert(data[field + "_hash_tree_root"][2:] == hash_tree_root(temp[field], subtype).hex()) ret = typ(**temp) - if "hash_tree_root" in json: - assert(json["hash_tree_root"][2:] == + if "hash_tree_root" in data: + assert(data["hash_tree_root"][2:] == hash_tree_root(ret, typ).hex()) return ret else: - print(json, typ) + print(data, typ) raise Exception("Type not recognized") diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index b38e5fe98..3c0658c8f 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -1,24 +1,27 @@ -from eth2spec.utils.minimal_ssz import hash_tree_root +from eth2spec.utils.ssz.ssz_impl import hash_tree_root +from eth2spec.utils.ssz.ssz_typing import * def encode(value, typ, include_hash_tree_roots=False): - if isinstance(typ, str) and typ[:4] == 'uint': - if typ[4:] == '128' or typ[4:] == '256': + if is_uint(typ): + if issubclass(typ, uint) and typ.byte_len > 8: return str(value) return value - elif typ == 'bool': + elif is_bool_type(typ): assert value in (True, False) return value - elif isinstance(typ, list): - return [encode(element, typ[0], include_hash_tree_roots) for element in value] - elif isinstance(typ, str) and typ[:4] == 'byte': + elif issubclass(typ, list) or issubclass(typ, Vector): + elem_typ = read_elem_typ(typ) + return [encode(element, elem_typ, include_hash_tree_roots) for element in value] + elif issubclass(typ, bytes): return '0x' + value.hex() - elif hasattr(typ, 'fields'): + elif is_container_typ(typ): ret = {} - for field, subtype in typ.fields.items(): - ret[field] = encode(getattr(value, field), subtype, include_hash_tree_roots) + for field, subtype in typ.get_fields(): + field_value = getattr(value, field) + ret[field] = encode(field_value, subtype, include_hash_tree_roots) if include_hash_tree_roots: - ret[field + "_hash_tree_root"] = '0x' + hash_tree_root(getattr(value, field), subtype).hex() + ret[field + "_hash_tree_root"] = '0x' + hash_tree_root(field_value, subtype).hex() if include_hash_tree_roots: ret["hash_tree_root"] = '0x' + hash_tree_root(value, typ).hex() return ret From c68944bd53cbe8759c233a188f3e58943012e84c Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 22:18:34 +0200 Subject: [PATCH 13/48] separate type (direct) from kinds (alias incl) --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 26 +++++++-------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 33 +++++++++++++++---- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 46d842e16..c3cc579bd 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -8,11 +8,11 @@ BYTES_PER_LENGTH_OFFSET = 4 def is_basic_type(typ): - return is_uint(typ) or typ == bool + return is_uint_type(typ) or is_bool_type(typ) def serialize_basic(value, typ): - if is_uint(typ): + if is_uint_type(typ): return value.to_bytes(uint_byte_size(typ), 'little') if is_bool_type(typ): if value: @@ -24,11 +24,11 @@ def serialize_basic(value, typ): def is_fixed_size(typ): if is_basic_type(typ): return True - elif is_list_type(typ): + elif is_list_kind(typ): return False - elif is_vector_type(typ): - return is_fixed_size(read_vector_elem_typ(typ)) - elif is_container_typ(typ): + elif is_vector_kind(typ): + return is_fixed_size(read_vector_elem_type(typ)) + elif is_container_type(typ): return all(is_fixed_size(t) for t in typ.get_field_types()) else: raise Exception("Type not supported: {}".format(typ)) @@ -38,9 +38,9 @@ def is_fixed_size(typ): def serialize(obj, typ): if is_basic_type(typ): return serialize_basic(obj, typ) - elif is_list_type(typ) or is_vector_type(typ): - return encode_series(obj, [read_elem_typ(typ)]*len(obj)) - elif is_container_typ(typ): + elif is_list_kind(typ) or is_vector_kind(typ): + return encode_series(obj, [read_elem_type(typ)]*len(obj)) + elif is_container_type(typ): return encode_series(obj.get_field_values(), typ.get_field_types()) else: raise Exception("Type not supported: {}".format(typ)) @@ -103,15 +103,15 @@ def mix_in_length(root, length): def hash_tree_root(obj, typ): if is_basic_type(typ): return merkleize_chunks(chunkify(serialize_basic(obj, typ))) - elif is_list_type(typ) or is_vector_type(typ): - subtype = read_elem_typ(typ) + elif is_list_kind(typ) or is_vector_kind(typ): + subtype = read_elem_type(typ) if is_basic_type(subtype): leaves = chunkify(pack(obj, subtype)) else: leaves = [hash_tree_root(elem, subtype) for elem in obj] leaf_root = merkleize_chunks(leaves) return mix_in_length(leaf_root, len(obj)) if is_list_type(typ) else leaf_root - elif is_container_typ(typ): + elif is_container_type(typ): leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields()] return merkleize_chunks(chunkify(b''.join(leaves))) else: @@ -120,7 +120,7 @@ def hash_tree_root(obj, typ): @infer_input_type def signing_root(obj, typ): - assert is_container_typ(typ) + assert is_container_type(typ) leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields()[:-1]] return merkleize_chunks(chunkify(b''.join(leaves))) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 93a272c8c..d16c66abb 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -61,7 +61,7 @@ class uint256(uint): return super().__new__(cls, value) -def is_uint(typ): +def is_uint_type(typ): # All integers are uint in the scope of the spec here. # Since we default to uint64. Bounds can be checked elsewhere. return issubclass(typ, int) @@ -380,29 +380,48 @@ def is_bool_type(typ): return issubclass(typ, bool) def is_list_type(typ): + """ + Checks if the given type is a list. + """ + return (hasattr(typ, '_name') and typ._name == 'List') + +def is_bytes_type(typ): + # Do not accept subclasses of bytes here, to avoid confusion with BytesN + return typ == bytes + +def is_list_kind(typ): """ Checks if the given type is a kind of list. Can be bytes. """ - return (hasattr(typ, '_name') and typ._name == 'List') or typ == bytes + return is_list_type(typ) or is_bytes_type(typ) def is_vector_type(typ): + """ + Checks if the given type is a vector. + """ + return issubclass(typ, Vector) + +def is_bytesn_type(typ): + return issubclass(typ, BytesN) + +def is_vector_kind(typ): """ Checks if the given type is a kind of vector. Can be BytesN. """ - return issubclass(typ, Vector) or issubclass(typ, BytesN) + return is_vector_type(typ) or is_bytesn_type(typ) -def is_container_typ(typ): +def is_container_type(typ): return issubclass(typ, Container) -def read_list_elem_typ(list_typ: Type[List[T]]) -> T: +def read_list_elem_type(list_typ: Type[List[T]]) -> T: if list_typ.__args__ is None or len(list_typ.__args__) != 1: raise TypeError("Supplied list-type is invalid, no element type found.") return list_typ.__args__[0] -def read_vector_elem_typ(vector_typ: Type[Vector[T, L]]) -> T: +def read_vector_elem_type(vector_typ: Type[Vector[T, L]]) -> T: return vector_typ.elem_type -def read_elem_typ(typ): +def read_elem_type(typ): if typ == bytes: return byte elif is_list_type(typ): From 0f79ed709bd4846299fb4379ed22d1de47f94e02 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 22:19:18 +0200 Subject: [PATCH 14/48] update yaml encoder/decoder and obj randomizer for typed ssz usage --- test_libs/pyspec/eth2spec/debug/decode.py | 16 +-- test_libs/pyspec/eth2spec/debug/encode.py | 11 +- .../pyspec/eth2spec/debug/random_value.py | 112 +++++++++--------- 3 files changed, 68 insertions(+), 71 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/decode.py b/test_libs/pyspec/eth2spec/debug/decode.py index 261692bed..c9657dc28 100644 --- a/test_libs/pyspec/eth2spec/debug/decode.py +++ b/test_libs/pyspec/eth2spec/debug/decode.py @@ -3,22 +3,22 @@ from eth2spec.utils.ssz.ssz_typing import * def decode(data, typ): - if is_uint(typ): + if is_uint_type(typ): return data elif is_bool_type(typ): assert data in (True, False) return data - elif issubclass(typ, list): - elem_typ = read_list_elem_typ(typ) + elif is_list_type(typ): + elem_typ = read_list_elem_type(typ) return [decode(element, elem_typ) for element in data] - elif issubclass(typ, Vector): - elem_typ = read_vector_elem_typ(typ) + elif is_vector_type(typ): + elem_typ = read_vector_elem_type(typ) return Vector(decode(element, elem_typ) for element in data) - elif issubclass(typ, bytes): + elif is_bytes_type(typ): return bytes.fromhex(data[2:]) - elif issubclass(typ, BytesN): + elif is_bytesn_type(typ): return BytesN(bytes.fromhex(data[2:])) - elif is_container_typ(typ): + elif is_container_type(typ): temp = {} for field, subtype in typ.get_fields(): temp[field] = decode(data[field], subtype) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index 3c0658c8f..832203e35 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -3,19 +3,20 @@ from eth2spec.utils.ssz.ssz_typing import * def encode(value, typ, include_hash_tree_roots=False): - if is_uint(typ): + if is_uint_type(typ): + # Larger uints are boxed and the class declares their byte length if issubclass(typ, uint) and typ.byte_len > 8: return str(value) return value elif is_bool_type(typ): assert value in (True, False) return value - elif issubclass(typ, list) or issubclass(typ, Vector): - elem_typ = read_elem_typ(typ) + elif is_list_type(typ) or is_vector_type(typ): + elem_typ = read_elem_type(typ) return [encode(element, elem_typ, include_hash_tree_roots) for element in value] - elif issubclass(typ, bytes): + elif issubclass(typ, bytes): # both bytes and BytesN return '0x' + value.hex() - elif is_container_typ(typ): + elif is_container_type(typ): ret = {} for field, subtype in typ.get_fields(): field_value = getattr(value, field) diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index f28181943..5abd73086 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -2,10 +2,11 @@ from random import Random from typing import Any from enum import Enum +from eth2spec.utils.ssz.ssz_typing import * +from eth2spec.utils.ssz.ssz_impl import is_basic_type -UINT_SIZES = [8, 16, 32, 64, 128, 256] - -basic_types = ["uint%d" % v for v in UINT_SIZES] + ['bool', 'byte'] +# in bytes +UINT_SIZES = [1, 2, 4, 8, 16, 32] random_mode_names = ["random", "zero", "max", "nil", "one", "lengthy"] @@ -49,60 +50,61 @@ def get_random_ssz_object(rng: Random, """ if chaos: mode = rng.choice(list(RandomizationMode)) - if isinstance(typ, str): - # Bytes array - if typ == 'bytes': - if mode == RandomizationMode.mode_nil_count: - return b'' - if mode == RandomizationMode.mode_max_count: - return get_random_bytes_list(rng, max_bytes_length) - if mode == RandomizationMode.mode_one_count: - return get_random_bytes_list(rng, 1) - if mode == RandomizationMode.mode_zero: - return b'\x00' - if mode == RandomizationMode.mode_max: - return b'\xff' - return get_random_bytes_list(rng, rng.randint(0, max_bytes_length)) - elif typ[:5] == 'bytes' and len(typ) > 5: - length = int(typ[5:]) - # Sanity, don't generate absurdly big random values - # If a client is aiming to performance-test, they should create a benchmark suite. - assert length <= max_bytes_length - if mode == RandomizationMode.mode_zero: - return b'\x00' * length - if mode == RandomizationMode.mode_max: - return b'\xff' * length - return get_random_bytes_list(rng, length) - # Basic types - else: - if mode == RandomizationMode.mode_zero: - return get_min_basic_value(typ) - if mode == RandomizationMode.mode_max: - return get_max_basic_value(typ) - return get_random_basic_value(rng, typ) + # Bytes array + if is_bytes_type(typ): + if mode == RandomizationMode.mode_nil_count: + return b'' + if mode == RandomizationMode.mode_max_count: + return get_random_bytes_list(rng, max_bytes_length) + if mode == RandomizationMode.mode_one_count: + return get_random_bytes_list(rng, 1) + if mode == RandomizationMode.mode_zero: + return b'\x00' + if mode == RandomizationMode.mode_max: + return b'\xff' + return get_random_bytes_list(rng, rng.randint(0, max_bytes_length)) + elif is_bytesn_type(typ): + length = typ.length + # Sanity, don't generate absurdly big random values + # If a client is aiming to performance-test, they should create a benchmark suite. + assert length <= max_bytes_length + if mode == RandomizationMode.mode_zero: + return b'\x00' * length + if mode == RandomizationMode.mode_max: + return b'\xff' * length + return get_random_bytes_list(rng, length) + # Basic types + elif is_basic_type(typ): + if mode == RandomizationMode.mode_zero: + return get_min_basic_value(typ) + if mode == RandomizationMode.mode_max: + return get_max_basic_value(typ) + return get_random_basic_value(rng, typ) # Vector: - elif isinstance(typ, list) and len(typ) == 2: + elif is_vector_type(typ): + elem_typ = read_vector_elem_type(typ) return [ - get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode, chaos) - for _ in range(typ[1]) + get_random_ssz_object(rng, elem_typ, max_bytes_length, max_list_length, mode, chaos) + for _ in range(typ.length) ] # List: - elif isinstance(typ, list) and len(typ) == 1: + elif is_list_type(typ): + elem_typ = read_list_elem_type(typ) length = rng.randint(0, max_list_length) if mode == RandomizationMode.mode_one_count: length = 1 if mode == RandomizationMode.mode_max_count: length = max_list_length return [ - get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode, chaos) + get_random_ssz_object(rng, elem_typ, max_bytes_length, max_list_length, mode, chaos) for _ in range(length) ] # Container: - elif hasattr(typ, 'fields'): + elif is_container_type(typ): return typ(**{ field: get_random_ssz_object(rng, subtype, max_bytes_length, max_list_length, mode, chaos) - for field, subtype in typ.fields.items() + for field, subtype in typ.get_fields() }) else: print(typ) @@ -114,39 +116,33 @@ def get_random_bytes_list(rng: Random, length: int) -> bytes: def get_random_basic_value(rng: Random, typ: str) -> Any: - if typ == 'bool': + if is_bool_type(typ): return rng.choice((True, False)) - if typ[:4] == 'uint': - size = int(typ[4:]) + if is_uint_type(typ): + size = uint_byte_size(typ) assert size in UINT_SIZES - return rng.randint(0, 2**size - 1) - if typ == 'byte': - return rng.randint(0, 8) + return rng.randint(0, 256**size - 1) else: raise ValueError("Not a basic type") def get_min_basic_value(typ: str) -> Any: - if typ == 'bool': + if is_bool_type(typ): return False - if typ[:4] == 'uint': - size = int(typ[4:]) + if is_uint_type(typ): + size = uint_byte_size(typ) assert size in UINT_SIZES return 0 - if typ == 'byte': - return 0x00 else: raise ValueError("Not a basic type") def get_max_basic_value(typ: str) -> Any: - if typ == 'bool': + if is_bool_type(typ): return True - if typ[:4] == 'uint': - size = int(typ[4:]) + if is_uint_type(typ): + size = uint_byte_size(typ) assert size in UINT_SIZES - return 2**size - 1 - if typ == 'byte': - return 0xff + return 256**size - 1 else: raise ValueError("Not a basic type") From d023d2d20f06474f7322d113049e3365a733a74a Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 23:40:05 +0200 Subject: [PATCH 15/48] lots of bugfixes --- test_libs/pyspec/eth2spec/debug/encode.py | 4 +- .../pyspec/eth2spec/debug/random_value.py | 6 +- .../pyspec/eth2spec/utils/merkle_minimal.py | 7 +- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 5 +- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 90 ++++++++++++++----- 5 files changed, 81 insertions(+), 31 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index 832203e35..a3c3c1189 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -4,6 +4,8 @@ from eth2spec.utils.ssz.ssz_typing import * def encode(value, typ, include_hash_tree_roots=False): if is_uint_type(typ): + if hasattr(typ, '__supertype__'): + typ = typ.__supertype__ # Larger uints are boxed and the class declares their byte length if issubclass(typ, uint) and typ.byte_len > 8: return str(value) @@ -14,7 +16,7 @@ def encode(value, typ, include_hash_tree_roots=False): elif is_list_type(typ) or is_vector_type(typ): elem_typ = read_elem_type(typ) return [encode(element, elem_typ, include_hash_tree_roots) for element in value] - elif issubclass(typ, bytes): # both bytes and BytesN + elif isinstance(typ, type) and issubclass(typ, bytes): # both bytes and BytesN return '0x' + value.hex() elif is_container_type(typ): ret = {} diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index 5abd73086..ab9c4c885 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -115,7 +115,7 @@ def get_random_bytes_list(rng: Random, length: int) -> bytes: return bytes(rng.getrandbits(8) for _ in range(length)) -def get_random_basic_value(rng: Random, typ: str) -> Any: +def get_random_basic_value(rng: Random, typ) -> Any: if is_bool_type(typ): return rng.choice((True, False)) if is_uint_type(typ): @@ -126,7 +126,7 @@ def get_random_basic_value(rng: Random, typ: str) -> Any: raise ValueError("Not a basic type") -def get_min_basic_value(typ: str) -> Any: +def get_min_basic_value(typ) -> Any: if is_bool_type(typ): return False if is_uint_type(typ): @@ -137,7 +137,7 @@ def get_min_basic_value(typ: str) -> Any: raise ValueError("Not a basic type") -def get_max_basic_value(typ: str) -> Any: +def get_max_basic_value(typ) -> Any: if is_bool_type(typ): return True if is_uint_type(typ): diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py index e3e5d35d8..420f0b5f1 100644 --- a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py @@ -34,10 +34,13 @@ def get_merkle_proof(tree, item_index): def next_power_of_two(v: int) -> int: """ - Get the next power of 2. (for 64 bit range ints) + Get the next power of 2. (for 64 bit range ints). + 0 is a special case, to have non-empty defaults. Examples: - 0 -> 0, 1 -> 1, 2 -> 2, 3 -> 4, 32 -> 32, 33 -> 64 + 0 -> 1, 1 -> 1, 2 -> 2, 3 -> 4, 32 -> 32, 33 -> 64 """ + if v == 0: + return 1 # effectively fill the bitstring (1 less, do not want to with ones, then increment for next power of 2. v -= 1 v |= v >> (1 << 0) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index c3cc579bd..826714c96 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -112,7 +112,7 @@ def hash_tree_root(obj, typ): leaf_root = merkleize_chunks(leaves) return mix_in_length(leaf_root, len(obj)) if is_list_type(typ) else leaf_root elif is_container_type(typ): - leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields()] + leaves = [hash_tree_root(field_value, field_typ) for field_value, field_typ in obj.get_typed_values()] return merkleize_chunks(chunkify(b''.join(leaves))) else: raise Exception("Type not supported: obj {} type {}".format(obj, typ)) @@ -121,6 +121,7 @@ def hash_tree_root(obj, typ): @infer_input_type def signing_root(obj, typ): assert is_container_type(typ) - leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields()[:-1]] + # ignore last field + leaves = [hash_tree_root(field_value, field_typ) for field_value, field_typ in obj.get_typed_values()[:-1]] return merkleize_chunks(chunkify(b''.join(leaves))) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index d16c66abb..0121ded9e 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,6 +1,6 @@ -from typing import List, Iterable, Type, NewType -from typing import Union from inspect import isclass +from typing import List, Iterable, TypeVar, Type, NewType +from typing import Union # SSZ integers @@ -64,17 +64,25 @@ class uint256(uint): def is_uint_type(typ): # All integers are uint in the scope of the spec here. # Since we default to uint64. Bounds can be checked elsewhere. - return issubclass(typ, int) + + # However, some are wrapped in a NewType + if hasattr(typ, '__supertype__'): + # get the type that the NewType is wrapping + typ = typ.__supertype__ + + return isinstance(typ, type) and issubclass(typ, int) def uint_byte_size(typ): - if issubclass(typ, uint): - return typ.byte_len - elif issubclass(typ, int): - # Default to uint64 - return 8 - else: - raise TypeError("Type %s is not an uint (or int-default uint64) type" % typ) + if hasattr(typ, '__supertype__'): + typ = typ.__supertype__ + if isinstance(typ, type): + if issubclass(typ, uint): + return typ.byte_len + elif issubclass(typ, int): + # Default to uint64 + return 8 + raise TypeError("Type %s is not an uint (or int-default uint64) type" % typ) # SSZ Container base class @@ -86,7 +94,7 @@ class Container(object): def __init__(self, **kwargs): cls = self.__class__ - for f, t in cls.get_fields().items(): + for f, t in cls.get_fields(): if f not in kwargs: setattr(self, f, get_zero_value(t)) else: @@ -117,7 +125,10 @@ class Container(object): @classmethod def get_fields(cls): - return dict(cls.__annotations__).items() + return list(dict(cls.__annotations__).items()) + + def get_typed_values(self): + return list(zip(self.get_field_values(), self.get_field_types())) @classmethod def get_field_names(cls): @@ -134,6 +145,9 @@ class Container(object): def _is_vector_instance_of(a, b): + # Other must not be a BytesN + if issubclass(b, bytes): + return False if not hasattr(b, 'elem_type') or not hasattr(b, 'length'): # Vector (b) is not an instance of Vector[X, Y] (a) return False @@ -146,6 +160,9 @@ def _is_vector_instance_of(a, b): def _is_equal_vector_type(a, b): + # Other must not be a BytesN + if issubclass(b, bytes): + return False if not hasattr(a, 'elem_type') or not hasattr(a, 'length'): if not hasattr(b, 'elem_type') or not hasattr(b, 'length'): # Vector == Vector @@ -237,6 +254,9 @@ class Vector(metaclass=VectorMeta): def _is_bytes_n_instance_of(a, b): + # Other has to be a Bytes derivative class to be a BytesN + if not issubclass(b, bytes): + return False if not hasattr(b, 'length'): # BytesN (b) is not an instance of BytesN[X] (a) return False @@ -249,6 +269,9 @@ def _is_bytes_n_instance_of(a, b): def _is_equal_bytes_n_type(a, b): + # Other has to be a Bytes derivative class to be a BytesN + if not issubclass(b, bytes): + return False if not hasattr(a, 'length'): if not hasattr(b, 'length'): # BytesN == BytesN @@ -267,7 +290,7 @@ class BytesNMeta(type): out = type.__new__(cls, class_name, parents, attrs) if 'length' in attrs: setattr(out, 'length', attrs['length']) - out._name = 'Vector' + out._name = 'BytesN' out.elem_type = byte return out @@ -318,7 +341,7 @@ class BytesN(bytes, metaclass=BytesNMeta): else: bytesval = b'\x00' * cls.length if len(bytesval) != cls.length: - raise TypeError("bytesN[%d] cannot be initialized with value of %d bytes" % (cls.length, len(bytesval))) + raise TypeError("BytesN[%d] cannot be initialized with value of %d bytes" % (cls.length, len(bytesval))) return super().__new__(cls, bytesval) def serialize(self): @@ -334,7 +357,7 @@ class BytesN(bytes, metaclass=BytesNMeta): # ----------------------------- def get_zero_value(typ): - if is_uint(typ): + if is_uint_type(typ): return 0 if issubclass(typ, bool): return False @@ -354,7 +377,7 @@ def get_zero_value(typ): # ----------------------------- def infer_type(obj): - if is_uint(obj.__class__): + if is_uint_type(obj.__class__): return obj.__class__ elif isinstance(obj, int): return uint64 @@ -370,39 +393,50 @@ def infer_input_type(fn): """ Decorator to run infer_type on the obj if typ argument is None """ + def infer_helper(obj, typ=None): if typ is None: typ = infer_type(obj) return fn(obj, typ) + return infer_helper + def is_bool_type(typ): - return issubclass(typ, bool) + if hasattr(typ, '__supertype__'): + typ = typ.__supertype__ + return isinstance(typ, type) and issubclass(typ, bool) + def is_list_type(typ): """ Checks if the given type is a list. """ - return (hasattr(typ, '_name') and typ._name == 'List') + return hasattr(typ, '_name') and typ._name == 'List' + def is_bytes_type(typ): # Do not accept subclasses of bytes here, to avoid confusion with BytesN return typ == bytes + def is_list_kind(typ): """ Checks if the given type is a kind of list. Can be bytes. """ return is_list_type(typ) or is_bytes_type(typ) + def is_vector_type(typ): """ Checks if the given type is a vector. """ - return issubclass(typ, Vector) + return isinstance(typ, type) and issubclass(typ, Vector) + def is_bytesn_type(typ): - return issubclass(typ, BytesN) + return isinstance(typ, type) and issubclass(typ, BytesN) + def is_vector_kind(typ): """ @@ -410,23 +444,33 @@ def is_vector_kind(typ): """ return is_vector_type(typ) or is_bytesn_type(typ) + def is_container_type(typ): - return issubclass(typ, Container) + return isinstance(typ, type) and issubclass(typ, Container) + + +T = TypeVar('T') +L = TypeVar('L') + def read_list_elem_type(list_typ: Type[List[T]]) -> T: if list_typ.__args__ is None or len(list_typ.__args__) != 1: raise TypeError("Supplied list-type is invalid, no element type found.") return list_typ.__args__[0] + def read_vector_elem_type(vector_typ: Type[Vector[T, L]]) -> T: return vector_typ.elem_type + def read_elem_type(typ): if typ == bytes: return byte elif is_list_type(typ): - return read_list_elem_typ(typ) + return read_list_elem_type(typ) elif is_vector_type(typ): - return read_vector_elem_typ(typ) + return read_vector_elem_type(typ) + elif issubclass(typ, bytes): + return byte else: raise TypeError("Unexpected type: {}".format(typ)) From b4c4df6a09e6feff9f1bdc1595f9703e836bf233 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 23:41:29 +0200 Subject: [PATCH 16/48] bugfix in sss_types global building in script --- scripts/phase0/function_puller.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index 0047d5321..d7765a4a4 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -70,10 +70,10 @@ def get_spec(file_name: str) -> List[str]: code_lines.append(' ' + type_line) code_lines.append('\n') for (ssz_type_name, _) in type_defs: - code_lines.append(f' global_vars["{ssz_type_name}"] = {ssz_type_name},') + code_lines.append(f' global_vars["{ssz_type_name}"] = {ssz_type_name}') code_lines.append(' global_vars["ssz_types"] = [') for (ssz_type_name, _) in type_defs: - code_lines.append(f' {ssz_type_name},') + code_lines.append(f' "{ssz_type_name}",') code_lines.append(' ]') code_lines.append('\n') code_lines.append('def get_ssz_type_by_name(name: str) -> Container:') From 54b14b5ac305ecb3fcc25086b6a63c81dafc61a9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 23:41:48 +0200 Subject: [PATCH 17/48] update ssz-static generator --- test_generators/ssz_static/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/ssz_static/main.py b/test_generators/ssz_static/main.py index 1234294db..abb167613 100644 --- a/test_generators/ssz_static/main.py +++ b/test_generators/ssz_static/main.py @@ -2,7 +2,7 @@ from random import Random from eth2spec.debug import random_value, encode from eth2spec.phase0 import spec -from eth2spec.utils.minimal_ssz import ( +from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, signing_root, serialize, From 9e61cc2ed3124c2118b820543f5424d680153854 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 28 May 2019 00:45:13 +0200 Subject: [PATCH 18/48] bugfix: typing edge-case: python bool is subclass of int. --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 0121ded9e..326dcb748 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -70,7 +70,7 @@ def is_uint_type(typ): # get the type that the NewType is wrapping typ = typ.__supertype__ - return isinstance(typ, type) and issubclass(typ, int) + return isinstance(typ, type) and issubclass(typ, int) and not issubclass(typ, bool) def uint_byte_size(typ): From 5e28adf5561172cde56b5af09e8de481ff9fbd0c Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 28 May 2019 00:51:27 +0200 Subject: [PATCH 19/48] bugfix: don't forget about var-length bytes getting a length mixin --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 826714c96..96eaff481 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -110,7 +110,7 @@ def hash_tree_root(obj, typ): else: leaves = [hash_tree_root(elem, subtype) for elem in obj] leaf_root = merkleize_chunks(leaves) - return mix_in_length(leaf_root, len(obj)) if is_list_type(typ) else leaf_root + return mix_in_length(leaf_root, len(obj)) if is_list_kind(typ) else leaf_root elif is_container_type(typ): leaves = [hash_tree_root(field_value, field_typ) for field_value, field_typ in obj.get_typed_values()] return merkleize_chunks(chunkify(b''.join(leaves))) From 00ffaf4d71e6f26667f37616418618cef7fc0559 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 28 May 2019 12:37:41 +0800 Subject: [PATCH 20/48] Fix ssz path --- test_generators/operations/deposits.py | 2 +- test_libs/pyspec/tests/helpers.py | 2 +- test_libs/pyspec/tests/test_sanity.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test_generators/operations/deposits.py b/test_generators/operations/deposits.py index 075ccbd5b..297a27859 100644 --- a/test_generators/operations/deposits.py +++ b/test_generators/operations/deposits.py @@ -5,7 +5,7 @@ from eth_utils import ( from gen_base import gen_suite, gen_typing from preset_loader import loader from eth2spec.debug.encode import encode -from eth2spec.utils.minimal_ssz import signing_root +from eth2spec.utils.ssz.ssz_impl import signing_root from eth2spec.utils.merkle_minimal import get_merkle_root, calc_merkle_tree_from_leaves, get_merkle_proof from typing import List, Tuple diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index a4849bfbb..ca40bf1d8 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -3,7 +3,7 @@ from copy import deepcopy from py_ecc import bls import eth2spec.phase0.spec as spec -from eth2spec.utils.minimal_ssz import signing_root +from eth2spec.utils.ssz.ssz_impl import signing_root from eth2spec.phase0.spec import ( # constants ZERO_HASH, diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index 1c05e6b53..b4d7a8e8b 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -5,7 +5,7 @@ import pytest from py_ecc import bls import eth2spec.phase0.spec as spec -from eth2spec.utils.minimal_ssz import signing_root +from eth2spec.utils.ssz.ssz_impl import signing_root from eth2spec.phase0.spec import ( # constants ZERO_HASH, From f0ceefc36d8d54670be1b8e48a0b5128d0d6208b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 28 May 2019 13:57:42 +0800 Subject: [PATCH 21/48] Make `test_libs/pyspec/tests` pass 1. Use `typing_inspect` library to check if it's `typing.List[Any]`. Use it in `is_list_type`. 2. Add `__hash__`, `__eq__`, `__repr__` methods to some classes. --- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 56 +++++++++++++------ test_libs/pyspec/requirements.txt | 1 + 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 326dcb748..3662c52b5 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,6 +1,7 @@ from inspect import isclass from typing import List, Iterable, TypeVar, Type, NewType from typing import Union +from typing_inspect import get_origin # SSZ integers @@ -119,6 +120,18 @@ class Container(object): def __repr__(self): return {field: getattr(self, field) for field in self.get_field_names()} + def __str__(self): + output = [] + for field in self.get_field_names(): + output.append(f'{field}: {getattr(self, field)}') + return "\n".join(output) + + def __eq__(self, other): + return self.hash_tree_root() == other.hash_tree_root() + + def __hash__(self): + return hash(self.hash_tree_root()) + @classmethod def get_fields_dict(cls): return dict(cls.__annotations__) @@ -203,6 +216,9 @@ class VectorMeta(type): def __ne__(self, other): return not _is_equal_vector_type(self, other) + def __hash__(self): + return hash(self.__class__) + class Vector(metaclass=VectorMeta): @@ -252,6 +268,9 @@ class Vector(metaclass=VectorMeta): def __len__(self): return len(self.items) + def __eq__(self, other): + return self.hash_tree_root() == other.hash_tree_root() + def _is_bytes_n_instance_of(a, b): # Other has to be a Bytes derivative class to be a BytesN @@ -309,6 +328,9 @@ class BytesNMeta(type): def __ne__(self, other): return not _is_equal_bytes_n_type(self, other) + def __hash__(self): + return hash(self.__class__) + def parse_bytes(val): if val is None: @@ -355,23 +377,25 @@ class BytesN(bytes, metaclass=BytesNMeta): # SSZ Defaults # ----------------------------- - def get_zero_value(typ): + result = None if is_uint_type(typ): - return 0 - if issubclass(typ, bool): - return False - if issubclass(typ, list): - return [] - if issubclass(typ, Vector): - return typ() - if issubclass(typ, BytesN): - return typ() - if issubclass(typ, bytes): - return b'' - if issubclass(typ, Container): - return typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}), - + result = 0 + elif is_list_type(typ): + result = [] + elif issubclass(typ, bool): + result = False + elif issubclass(typ, Vector): + result = typ() + elif issubclass(typ, BytesN): + result = typ() + elif issubclass(typ, bytes): + result = b'' + elif issubclass(typ, Container): + result = typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}) + else: + return Exception("Type not supported: {}".format(typ)) + return result # Type helpers # ----------------------------- @@ -412,7 +436,7 @@ def is_list_type(typ): """ Checks if the given type is a list. """ - return hasattr(typ, '_name') and typ._name == 'List' + return get_origin(typ) is List or get_origin(typ) is list def is_bytes_type(typ): diff --git a/test_libs/pyspec/requirements.txt b/test_libs/pyspec/requirements.txt index 78d41708d..3b38930bd 100644 --- a/test_libs/pyspec/requirements.txt +++ b/test_libs/pyspec/requirements.txt @@ -2,3 +2,4 @@ eth-utils>=1.3.0,<2 eth-typing>=2.1.0,<3.0.0 pycryptodome==3.7.3 py_ecc>=1.6.0 +typing_inspect==0.4.0 From 19601df57237384f30123b45bde0ec8becbe7f63 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Tue, 28 May 2019 09:30:35 -0400 Subject: [PATCH 22/48] Starting work on partials --- .../pyspec/eth2spec/utils/ssz/__init__.py | 1 + .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 6 +- .../pyspec/eth2spec/utils/ssz/ssz_partials.py | 81 +++++++++++++++++++ .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 23 +++--- 4 files changed, 98 insertions(+), 13 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py diff --git a/test_libs/pyspec/eth2spec/utils/ssz/__init__.py b/test_libs/pyspec/eth2spec/utils/ssz/__init__.py index 752d77c43..6543bd9e5 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/__init__.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/__init__.py @@ -1 +1,2 @@ from .ssz_impl import * +from .ssz_partials import * diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 655cd9da4..3bca0b820 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -35,7 +35,7 @@ def is_fixed_size(typ): @infer_input_type -def serialize(obj, typ): +def serialize(obj, typ=None): if is_basic_type(typ): return serialize_basic(obj, typ) elif is_list_type(typ) or is_vector_type(typ): @@ -100,7 +100,7 @@ def mix_in_length(root, length): @infer_input_type -def hash_tree_root(obj, typ): +def hash_tree_root(obj, typ=None): if is_basic_type(typ): return merkleize_chunks(chunkify(serialize_basic(obj, typ))) elif is_list_type(typ) or is_vector_type(typ): @@ -119,7 +119,7 @@ def hash_tree_root(obj, typ): @infer_input_type -def signing_root(obj, typ): +def signing_root(obj, typ=None): assert is_container_typ(typ) leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields()[:-1]] return merkleize_chunks(chunkify(b''.join(leaves))) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py new file mode 100644 index 000000000..00a22279c --- /dev/null +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py @@ -0,0 +1,81 @@ +from ..merkle_minimal import hash, next_power_of_two +from .ssz_typing import * +from .ssz_impl import * + +ZERO_CHUNK = b'\x00' * 32 + +def last_power_of_two(x): + return next_power_of_two(x+1) // 2 + +def concat_generalized_indices(x, y): + return x * last_power_of_two(y) + y - last_power_of_two(y) + +def rebase(objs, new_root): + return {concat_generalized_indices(new_root, k): v for k,v in objs.items()} + +def constrict_generalized_index(x, q): + depth = last_power_of_two(x // q) + o = depth + x - q * depth + if concat_generalized_indices(q, o) != x: + return None + return o + +def unrebase(objs, q): + o = {} + for k,v in objs.items(): + new_k = constrict_generalized_index(k, q) + if new_k is not None: + o[new_k] = v + return o + +def filler(starting_position, chunk_count): + at, skip, end = chunk_count, 1, next_power_of_two(chunk_count) + value = ZERO_CHUNK + o = {} + while at < end: + while at % (skip*2) == 0: + skip *= 2 + value = hash(value + value) + o[starting_position + at] = value + at += skip + return o + +def merkle_tree_of_chunks(chunks, root): + starting_index = root * next_power_of_two(len(chunks)) + o = {starting_index+i: chunk for i,chunk in enumerate(chunks)} + o = {**o, **filler(starting_index, len(chunks))} + return o + +def is_bottom_layer_type(typ): + return ( + is_basic_type(typ) or + (is_list_type(typ) or is_vector_type(typ)) and is_basic_type(read_elem_typ(typ)) + ) + +@infer_input_type +def get_fields(obj, typ=None): + if is_container_typ(typ): + return obj.get_fields() + elif is_list_type(typ) or is_vector_type(typ): + subtype = read_elem_typ(typ) + return zip([subtype] * len(obj), obj) + else: + raise Exception("Invalid type") + +@infer_input_type +def ssz_all(obj, typ=None, root=1): + if is_list_type(typ): + o = {root * 2 + 1: len(obj).to_bytes(32, 'little')} + base = root * 2 + else: + o = {} + base = root + if is_bottom_layer_type(typ): + data = serialize_basic(obj, typ) if is_basic_type(typ) else pack(obj, read_elem_typ(typ)) + return {**o, **merkle_tree_of_chunks(chunkify(data), base)} + else: + fields = get_fields(obj, typ=typ) + sub_base = base * next_power_of_two(len(fields)) + for i, (elem, subtype) in enumerate(fields): + o = {**o, **ssz_all(elem, typ=subtype, root=sub_base+i)} + return {**o, **filter(sub_base, len(fields))} diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 1d8aed0de..411c21976 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,7 +1,10 @@ -from typing import List, Iterable, Type, NewType +from typing import List, Iterable, Type, NewType, TypeVar from typing import Union from inspect import isclass +T = TypeVar('T') +L = TypeVar('L') + # SSZ integers # ----------------------------- @@ -64,13 +67,13 @@ class uint256(uint): def is_uint(typ): # All integers are uint in the scope of the spec here. # Since we default to uint64. Bounds can be checked elsewhere. - return issubclass(typ, int) + return (isinstance(typ, int.__class__) and issubclass(typ, int)) or typ == uint64 def uint_byte_size(typ): - if issubclass(typ, uint): + if isinstance(typ, int.__class__) and issubclass(typ, uint): return typ.byte_len - elif issubclass(typ, int): + elif typ in (int, uint64): # Default to uint64 return 8 else: @@ -109,7 +112,7 @@ class Container(object): return [getattr(self, field) for field in cls.get_field_names()] def __repr__(self): - return {field: getattr(self, field) for field in self.get_field_names()} + return repr({field: getattr(self, field) for field in self.get_field_names()}) @classmethod def get_fields_dict(cls): @@ -221,7 +224,7 @@ class Vector(metaclass=VectorMeta): return hash_tree_root(self, self.__class__) def __repr__(self): - return {'length': self.__class__.length, 'items': self.items} + return repr({'length': self.__class__.length, 'items': self.items}) def __getitem__(self, key): return self.items[key] @@ -370,10 +373,10 @@ def infer_input_type(fn): """ Decorator to run infer_type on the obj if typ argument is None """ - def infer_helper(obj, typ=None): + def infer_helper(obj, *args, typ=None, **kwargs): if typ is None: typ = infer_type(obj) - return fn(obj, typ) + return fn(obj, *args, typ=typ, **kwargs) return infer_helper @@ -381,10 +384,10 @@ def is_list_type(typ): return (hasattr(typ, '_name') and typ._name == 'List') or typ == bytes def is_vector_type(typ): - return issubclass(typ, Vector) + return isinstance(typ, int.__class__) and issubclass(typ, Vector) def is_container_typ(typ): - return issubclass(typ, Container) + return isinstance(typ, int.__class__) and issubclass(typ, Container) def read_list_elem_typ(list_typ: Type[List[T]]) -> T: if list_typ.__args__ is None or len(list_typ.__args__) != 1: From 053d6e7805a61698022bdae397af4450e52519fc Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Tue, 28 May 2019 14:33:12 -0400 Subject: [PATCH 23/48] Simplified hash tree root --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 45 ++++++++++++------- .../pyspec/eth2spec/utils/ssz/ssz_partials.py | 16 ------- 2 files changed, 29 insertions(+), 32 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 56789337e..21dba1034 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -52,7 +52,7 @@ def encode_series(values, types): return values # Recursively serialize - parts = [(is_fixed_size(types[i]), serialize(values[i], types[i])) for i in range(len(values))] + parts = [(is_fixed_size(types[i]), serialize(values[i], typ=types[i])) for i in range(len(values))] # Compute and check lengths fixed_lengths = [len(serialized) if constant_size else BYTES_PER_LENGTH_OFFSET @@ -99,29 +99,42 @@ def mix_in_length(root, length): return hash(root + length.to_bytes(32, 'little')) +def is_bottom_layer_kind(typ): + return ( + is_basic_type(typ) or + (is_list_kind(typ) or is_vector_kind(typ)) and is_basic_type(read_elem_type(typ)) + ) + + +@infer_input_type +def get_typed_values(obj, typ=None): + if is_container_type(typ): + return obj.get_typed_values() + elif is_list_kind(typ) or is_vector_kind(typ): + elem_type = read_elem_type(typ) + return zip(obj, [elem_type] * len(obj)) + else: + raise Exception("Invalid type") + + @infer_input_type def hash_tree_root(obj, typ=None): - if is_basic_type(typ): - return merkleize_chunks(chunkify(serialize_basic(obj, typ))) - elif is_list_kind(typ) or is_vector_kind(typ): - subtype = read_elem_type(typ) - if is_basic_type(subtype): - leaves = chunkify(pack(obj, subtype)) - else: - leaves = [hash_tree_root(elem, subtype) for elem in obj] - leaf_root = merkleize_chunks(leaves) - return mix_in_length(leaf_root, len(obj)) if is_list_kind(typ) else leaf_root - elif is_container_type(typ): - leaves = [hash_tree_root(field_value, field_typ) for field_value, field_typ in obj.get_typed_values()] - return merkleize_chunks(chunkify(b''.join(leaves))) + if is_bottom_layer_kind(typ): + data = serialize_basic(obj, typ) if is_basic_type(typ) else pack(obj, read_elem_type(typ)) + leaves = chunkify(data) else: - raise Exception("Type not supported: obj {} type {}".format(obj, typ)) + fields = get_typed_values(obj, typ=typ) + leaves = [hash_tree_root(field_value, typ=field_typ) for field_value, field_typ in fields] + if is_list_kind(typ): + return mix_in_length(merkleize_chunks(leaves), len(obj)) + else: + return merkleize_chunks(leaves) @infer_input_type def signing_root(obj, typ): assert is_container_type(typ) # ignore last field - leaves = [hash_tree_root(field_value, field_typ) for field_value, field_typ in obj.get_typed_values()[:-1]] + leaves = [hash_tree_root(field_value, typ=field_typ) for field_value, field_typ in obj.get_typed_values()[:-1]] return merkleize_chunks(chunkify(b''.join(leaves))) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py index d8b9bae6f..b30ebfb4c 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py @@ -46,22 +46,6 @@ def merkle_tree_of_chunks(chunks, root): o = {**o, **filler(starting_index, len(chunks))} return o -def is_bottom_layer_type(typ): - return ( - is_basic_type(typ) or - (is_list_type(typ) or is_vector_type(typ)) and is_basic_type(read_elem_typ(typ)) - ) - -@infer_input_type -def get_typed_values(obj, typ=None): - if is_container_typ(typ): - return obj.get_typed_values() - elif is_list_type(typ) or is_vector_type(typ): - elem_type = read_elem_typ(typ) - return zip(obj, [elem_type] * len(obj)) - else: - raise Exception("Invalid type") - @infer_input_type def ssz_all(obj, typ=None, root=1): if is_list_type(typ): From ebdf74c6b808fb5583ace4a03a013eee35c33324 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 30 May 2019 15:57:39 +0800 Subject: [PATCH 24/48] kick the CI --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 694c7bc92..ed3e16f63 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -12,6 +12,7 @@ L = TypeVar('L') class uint(int): byte_len = 0 + def __new__(cls, value, *args, **kwargs): if value < 0: raise ValueError("unsigned types must not be negative") From 8fae0f8c78deb30a10be45457d0f8b668349e62e Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Thu, 30 May 2019 09:42:42 -0400 Subject: [PATCH 25/48] Added support for SSZ partials --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 15 +- .../pyspec/eth2spec/utils/ssz/ssz_partials.py | 208 +++++++++++++++++- 2 files changed, 213 insertions(+), 10 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 21dba1034..0ef89f97f 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -14,11 +14,22 @@ def is_basic_type(typ): def serialize_basic(value, typ): if is_uint_type(typ): return value.to_bytes(uint_byte_size(typ), 'little') - if is_bool_type(typ): + elif is_bool_type(typ): if value: return b'\x01' else: return b'\x00' + else: + raise Exception("Type not supported: {}".format(typ)) + +def deserialize_basic(value, typ): + if is_uint_type(typ): + return typ(int.from_bytes(value, 'little')) + elif is_bool_type(typ): + assert value in (b'\x00', b'\x01') + return True if value == b'\x01' else False + else: + raise Exception("Type not supported: {}".format(typ)) def is_fixed_size(typ): @@ -112,7 +123,7 @@ def get_typed_values(obj, typ=None): return obj.get_typed_values() elif is_list_kind(typ) or is_vector_kind(typ): elem_type = read_elem_type(typ) - return zip(obj, [elem_type] * len(obj)) + return list(zip(obj, [elem_type] * len(obj))) else: raise Exception("Invalid type") diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py index b30ebfb4c..6a243f4f3 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py @@ -36,7 +36,7 @@ def filler(starting_position, chunk_count): while at % (skip*2) == 0: skip *= 2 value = hash(value + value) - o[starting_position + at] = value + o[(starting_position + at) // skip] = value at += skip return o @@ -47,19 +47,211 @@ def merkle_tree_of_chunks(chunks, root): return o @infer_input_type -def ssz_all(obj, typ=None, root=1): - if is_list_type(typ): +def ssz_leaves(obj, typ=None, root=1): + if is_list_kind(typ): o = {root * 2 + 1: len(obj).to_bytes(32, 'little')} base = root * 2 else: o = {} base = root - if is_bottom_layer_type(typ): - data = serialize_basic(obj, typ) if is_basic_type(typ) else pack(obj, read_elem_typ(typ)) - return {**o, **merkle_tree_of_chunks(chunkify(data), base)} + if is_bottom_layer_kind(typ): + data = serialize_basic(obj, typ) if is_basic_type(typ) else pack(obj, read_elem_type(typ)) + q = {**o, **merkle_tree_of_chunks(chunkify(data), base)} + #print(obj, root, typ, base, list(q.keys())) + return(q) else: fields = get_typed_values(obj, typ=typ) sub_base = base * next_power_of_two(len(fields)) for i, (elem, elem_type) in enumerate(fields): - o = {**o, **ssz_all(elem, typ=elem_type, root=sub_base+i)} - return {**o, **filter(sub_base, len(fields))} + o = {**o, **ssz_leaves(elem, typ=elem_type, root=sub_base+i)} + q = {**o, **filler(sub_base, len(fields))} + #print(obj, root, typ, base, list(q.keys())) + return(q) + +def fill(objects): + objects = {k:v for k,v in objects.items()} + keys = sorted(objects.keys())[::-1] + pos = 0 + while pos < len(keys): + k = keys[pos] + if k in objects and k^1 in objects and k//2 not in objects: + objects[k//2] = hash(objects[k&-2] + objects[k|1]) + keys.append(k // 2) + pos += 1 + return objects + +@infer_input_type +def ssz_full(obj, typ=None): + return fill(ssz_leaves(obj, typ=typ)) + +def get_basic_type_size(typ): + if is_uint_type(typ): + return uint_byte_size(typ) + elif is_bool_type(typ): + return 1 + else: + raise Exception("Type not basic: {}".format(typ)) + +def get_bottom_layer_element_position(typ, base, length, index): + """ + Returns the generalized index and the byte range of the index'th value + in the list with the given base generalized index and given length + """ + assert index < (1 if is_basic_type(typ) else length) + elem_typ = typ if is_basic_type(typ) else read_elem_type(typ) + elem_size = get_basic_type_size(elem_typ) + chunk_index = index * elem_size // 32 + chunk_count = (1 if is_basic_type(typ) else length) * elem_size // 32 + generalized_index = base * next_power_of_two(chunk_count) + chunk_index + start = elem_size * index % 32 + return generalized_index, start, start+elem_size + +@infer_input_type +def get_generalized_indices(obj, path, typ=None, root=1): + if len(path) == 0: + return [root] if is_basic_type(typ) else list(ssz_leaves(obj, typ=typ, root=root).keys()) + if path[0] == '__len__': + return [root * 2 + 1] if is_list_type(typ) else [] + base = root * 2 if is_list_kind(typ) else root + if is_bottom_layer_kind(typ): + length = 1 if is_basic_type(typ) else len(obj) + index, _, _ = get_bottom_layer_element_position(typ, base, length, path[0]) + return [index] + else: + if is_container_type(typ): + fields = typ.get_field_names() + field_count, index = len(fields), fields.index(path[0]) + elem_type = typ.get_field_types()[index] + child = obj.get_field_values()[index] + else: + field_count, index, elem_type, child = len(obj), path[0], read_elem_type(typ), obj[path[0]] + return get_generalized_indices( + child, + path[1:], + typ=elem_type, + root=base * next_power_of_two(field_count) + index + ) + +def get_branch_indices(tree_index): + o = [tree_index, tree_index ^ 1] + while o[-1] > 1: + o.append((o[-1] // 2) ^ 1) + return o[:-1] + +def remove_redundant_indices(obj): + return {k:v for k,v in obj.items() if not (k*2 in obj and k*2+1 in obj)} + +def merge(*args): + o = {} + for arg in args: + o = {**o, **arg} + return fill(o) + +@infer_input_type +def get_nodes_along_path(obj, path, typ=None): + indices = get_generalized_indices(obj, path, typ=typ) + return remove_redundant_indices(merge(*({i:obj.objects[i] for i in get_branch_indices(index)} for index in indices))) + +class OutOfRangeException(Exception): + pass + +class SSZPartial(): + def __init__(self, typ, objects, root=1): + assert not is_basic_type(typ) + self.objects = objects + self.typ = typ + self.root = root + if is_container_type(self.typ): + for field in self.typ.get_field_names(): + try: + setattr(self, field, self.getter(field)) + except OutOfRangeException: + pass + + def getter(self, index): + base = self.root * 2 if is_list_kind(self.typ) else self.root + if is_bottom_layer_kind(self.typ): + tree_index, start, end = get_bottom_layer_element_position( + self.typ, base, len(self), index + ) + if tree_index not in self.objects: + raise OutOfRangeException("Do not have required data") + else: + return deserialize_basic( + self.objects[tree_index][start:end], + self.typ if is_basic_type(self.typ) else read_elem_type(self.typ) + ) + else: + if is_container_type(self.typ): + fields = self.typ.get_field_names() + field_count, index = len(fields), fields.index(index) + elem_type = self.typ.get_field_types()[index] + else: + field_count, index, elem_type = len(self), index, read_elem_type(self.typ) + tree_index = base * next_power_of_two(field_count) + index + if tree_index not in self.objects: + raise OutOfRangeException("Do not have required data") + if is_basic_type(elem_type): + return deserialize_basic(self.objects[tree_index][:get_basic_type_size(elem_type)], elem_type) + else: + return ssz_partial(elem_type, self.objects, root=tree_index) + + def __getitem__(self, index): + return self.getter(index) + + def __iter__(self): + return (self[i] for i in range(len(self))) + + def __len__(self): + if is_list_kind(self.typ): + if self.root*2+1 not in self.objects: + raise OutOfRangeException("Do not have required data: {}".format(self.root*2+1)) + return int.from_bytes(self.objects[self.root*2+1], 'little') + elif is_vector_kind(self.typ): + return self.typ.length + elif is_container_type(self.typ): + return len(self.typ.get_fields()) + else: + raise Exception("Unsupported type: {}".format(self.typ)) + + def full_value(self): + if is_bytes_type(self.typ) or is_bytesn_type(self.typ): + return bytes([self.getter(i) for i in range(len(self))]) + elif is_list_kind(self.typ): + return [self[i] for i in range(len(self))] + elif is_vector_kind(self.typ): + return self.typ(*(self[i] for i in range(len(self)))) + elif is_container_type(self.typ): + full_value = lambda x: x.full_value() if hasattr(x, 'full_value') else x + return self.typ(**{field: full_value(self.getter(field)) for field in self.typ.get_field_names()}) + elif is_basic_type(self.typ): + return self.getter(0) + else: + raise Exception("Unsupported type: {}".format(self.typ)) + + def hash_tree_root(self): + o = {**self.objects} + keys = sorted(o.keys())[::-1] + pos = 0 + while pos < len(keys): + k = keys[pos] + if k in o and k^1 in o and k//2 not in o: + o[k//2] = hash(o[k&-2] + o[k|1]) + keys.append(k // 2) + pos += 1 + return o[self.root] + + def __str__(self): + return str(self.full_value()) + +def ssz_partial(typ, objects, root=1): + ssz_type = ( + Container if is_container_type(typ) else + typ if (is_vector_type(typ) or is_bytesn_type(typ)) else object + ) + class Partial(SSZPartial, ssz_type): + pass + if is_container_type(typ): + Partial.__annotations__ = typ.__annotations__ + o = Partial(typ, objects, root=root) + return o From 3cb43fcc2742f75cd203050e8ecff3207d297c17 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 1 Jun 2019 01:49:52 +0200 Subject: [PATCH 26/48] fix ssz infer type decorator, one None argument too many --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index ed3e16f63..47b4b5c9a 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -420,10 +420,10 @@ def infer_input_type(fn): """ Decorator to run infer_type on the obj if typ argument is None """ - def infer_helper(obj, *args, typ=None, **kwargs): + def infer_helper(obj, typ=None, **kwargs): if typ is None: typ = infer_type(obj) - return fn(obj, *args, typ=typ, **kwargs) + return fn(obj, typ=typ, **kwargs) return infer_helper From 1c734d30a0d44849bb42687e4b3719ee5d3432f3 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 1 Jun 2019 01:50:28 +0200 Subject: [PATCH 27/48] fix utils init --- test_libs/pyspec/eth2spec/utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/__init__.py b/test_libs/pyspec/eth2spec/utils/__init__.py index 9f8b36428..482044798 100644 --- a/test_libs/pyspec/eth2spec/utils/__init__.py +++ b/test_libs/pyspec/eth2spec/utils/__init__.py @@ -1,4 +1,4 @@ from .merkle_minimal import * from .hash_function import * -from .bls_stub import * +from .bls import * from .ssz import * From 33233c98ff4ef033098a0efcdef075f0e56bdedf Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 1 Jun 2019 01:51:09 +0200 Subject: [PATCH 28/48] fix ssz imports --- test_libs/pyspec/eth2spec/test/helpers/attestations.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/block.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/block_header.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/deposits.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/genesis.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/transfers.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py | 2 +- test_libs/pyspec/eth2spec/test/sanity/test_blocks.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 6ac0b994e..45ee10c3e 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -16,7 +16,7 @@ from eth2spec.test.helpers.bitfields import set_bitfield_bit from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block from eth2spec.test.helpers.keys import privkeys from eth2spec.utils.bls import bls_sign, bls_aggregate_signatures -from eth2spec.utils.minimal_ssz import hash_tree_root +from eth2spec.utils.ssz.ssz_impl import hash_tree_root def build_attestation_data(state, slot, shard): diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index 715cf82db..28fd1229f 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -8,7 +8,7 @@ from eth2spec.phase0.spec import ( ) from eth2spec.test.helpers.keys import privkeys from eth2spec.utils.bls import bls_sign, only_with_bls -from eth2spec.utils.minimal_ssz import signing_root, hash_tree_root +from eth2spec.utils.ssz.ssz_impl import signing_root, hash_tree_root # Fully ignore the function if BLS is off, beacon-proposer index calculation is slow. diff --git a/test_libs/pyspec/eth2spec/test/helpers/block_header.py b/test_libs/pyspec/eth2spec/test/helpers/block_header.py index 9aba62d37..ad3fbc166 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block_header.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block_header.py @@ -3,7 +3,7 @@ import eth2spec.phase0.spec as spec from eth2spec.phase0.spec import get_domain from eth2spec.utils.bls import bls_sign -from eth2spec.utils.minimal_ssz import signing_root +from eth2spec.utils.ssz.ssz_impl import signing_root def sign_block_header(state, header, privkey): diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index 2db3ae03c..3080eba7f 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -5,7 +5,7 @@ from eth2spec.phase0.spec import get_domain, DepositData, verify_merkle_branch, from eth2spec.test.helpers.keys import pubkeys, privkeys from eth2spec.utils.bls import bls_sign from eth2spec.utils.merkle_minimal import calc_merkle_tree_from_leaves, get_merkle_root, get_merkle_proof -from eth2spec.utils.minimal_ssz import signing_root +from eth2spec.utils.ssz.ssz_impl import signing_root def build_deposit_data(state, pubkey, privkey, amount, withdrawal_credentials, signed=False): diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index 01011cacd..489791907 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -3,7 +3,7 @@ import eth2spec.phase0.spec as spec from eth2spec.phase0.spec import Eth1Data, ZERO_HASH, get_active_validator_indices from eth2spec.test.helpers.keys import pubkeys -from eth2spec.utils.minimal_ssz import hash_tree_root +from eth2spec.utils.ssz.ssz_impl import hash_tree_root def build_mock_validator(i: int, balance: int): diff --git a/test_libs/pyspec/eth2spec/test/helpers/transfers.py b/test_libs/pyspec/eth2spec/test/helpers/transfers.py index 2045f48ad..9913ef88a 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/transfers.py +++ b/test_libs/pyspec/eth2spec/test/helpers/transfers.py @@ -5,7 +5,7 @@ from eth2spec.phase0.spec import get_current_epoch, get_active_validator_indices from eth2spec.test.helpers.keys import pubkeys, privkeys from eth2spec.test.helpers.state import get_balance from eth2spec.utils.bls import bls_sign -from eth2spec.utils.minimal_ssz import signing_root +from eth2spec.utils.ssz.ssz_impl import signing_root def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=None, signed=False): diff --git a/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py b/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py index 54376d694..e4a815877 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py @@ -3,7 +3,7 @@ import eth2spec.phase0.spec as spec from eth2spec.phase0.spec import VoluntaryExit, get_domain from eth2spec.utils.bls import bls_sign -from eth2spec.utils.minimal_ssz import signing_root +from eth2spec.utils.ssz.ssz_impl import signing_root def build_voluntary_exit(state, epoch, validator_index, privkey, signed=False): diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 654a41d62..3d2234795 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -3,7 +3,7 @@ from copy import deepcopy import eth2spec.phase0.spec as spec from eth2spec.utils.bls import bls_sign -from eth2spec.utils.minimal_ssz import signing_root +from eth2spec.utils.ssz.ssz_impl import signing_root from eth2spec.phase0.spec import ( # SSZ VoluntaryExit, From 77d4f0b407d05db6fae645d629878afeefe9aeef Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 1 Jun 2019 01:53:50 +0200 Subject: [PATCH 29/48] update generator type hinting to use new type syntax --- .../eth2spec/test/sanity/test_blocks.py | 29 ++++++++++--------- .../pyspec/eth2spec/test/test_finality.py | 9 +++--- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 3d2234795..11ae4918e 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -1,4 +1,5 @@ from copy import deepcopy +from typing import List import eth2spec.phase0.spec as spec from eth2spec.utils.bls import bls_sign @@ -36,7 +37,7 @@ def test_empty_block_transition(state): yield 'pre', state block = build_empty_block_for_next_slot(state, signed=True) - yield 'blocks', [block], [spec.BeaconBlock] + yield 'blocks', [block], List[spec.BeaconBlock] state_transition(state, block) yield 'post', state @@ -54,7 +55,7 @@ def test_skipped_slots(state): block = build_empty_block_for_next_slot(state) block.slot += 3 sign_block(state, block) - yield 'blocks', [block], [spec.BeaconBlock] + yield 'blocks', [block], List[spec.BeaconBlock] state_transition(state, block) yield 'post', state @@ -72,7 +73,7 @@ def test_empty_epoch_transition(state): block = build_empty_block_for_next_slot(state) block.slot += spec.SLOTS_PER_EPOCH sign_block(state, block) - yield 'blocks', [block], [spec.BeaconBlock] + yield 'blocks', [block], List[spec.BeaconBlock] state_transition(state, block) yield 'post', state @@ -91,7 +92,7 @@ def test_empty_epoch_transition(state): # block = build_empty_block_for_next_slot(state) # block.slot += spec.SLOTS_PER_EPOCH * 5 # sign_block(state, block, proposer_index=0) -# yield 'blocks', [block], [spec.BeaconBlock] +# yield 'blocks', [block], List[spec.BeaconBlock] # # state_transition(state, block) # yield 'post', state @@ -119,7 +120,7 @@ def test_proposer_slashing(state): block = build_empty_block_for_next_slot(state) block.body.proposer_slashings.append(proposer_slashing) sign_block(state, block) - yield 'blocks', [block], [spec.BeaconBlock] + yield 'blocks', [block], List[spec.BeaconBlock] state_transition(state, block) yield 'post', state @@ -152,7 +153,7 @@ def test_attester_slashing(state): block = build_empty_block_for_next_slot(state) block.body.attester_slashings.append(attester_slashing) sign_block(state, block) - yield 'blocks', [block], [spec.BeaconBlock] + yield 'blocks', [block], List[spec.BeaconBlock] state_transition(state, block) yield 'post', state @@ -189,7 +190,7 @@ def test_deposit_in_block(state): block.body.deposits.append(deposit) sign_block(state, block) - yield 'blocks', [block], [spec.BeaconBlock] + yield 'blocks', [block], List[spec.BeaconBlock] state_transition(state, block) yield 'post', state @@ -216,7 +217,7 @@ def test_deposit_top_up(state): block.body.deposits.append(deposit) sign_block(state, block) - yield 'blocks', [block], [spec.BeaconBlock] + yield 'blocks', [block], List[spec.BeaconBlock] state_transition(state, block) yield 'post', state @@ -252,7 +253,7 @@ def test_attestation(state): sign_block(state, epoch_block) state_transition(state, epoch_block) - yield 'blocks', [attestation_block, epoch_block], [spec.BeaconBlock] + yield 'blocks', [attestation_block, epoch_block], List[spec.BeaconBlock] yield 'post', state assert len(state.current_epoch_attestations) == 0 @@ -298,7 +299,7 @@ def test_voluntary_exit(state): sign_block(state, exit_block) state_transition(state, exit_block) - yield 'blocks', [initiate_exit_block, exit_block], [spec.BeaconBlock] + yield 'blocks', [initiate_exit_block, exit_block], List[spec.BeaconBlock] yield 'post', state assert state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH @@ -326,7 +327,7 @@ def test_transfer(state): block.body.transfers.append(transfer) sign_block(state, block) - yield 'blocks', [block], [spec.BeaconBlock] + yield 'blocks', [block], List[spec.BeaconBlock] state_transition(state, block) yield 'post', state @@ -355,7 +356,7 @@ def test_balance_driven_status_transitions(state): sign_block(state, block) state_transition(state, block) - yield 'blocks', [block], [spec.BeaconBlock] + yield 'blocks', [block], List[spec.BeaconBlock] yield 'post', state assert state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH @@ -371,7 +372,7 @@ def test_historical_batch(state): block = build_empty_block_for_next_slot(state, signed=True) state_transition(state, block) - yield 'blocks', [block], [spec.BeaconBlock] + yield 'blocks', [block], List[spec.BeaconBlock] yield 'post', state assert state.slot == block.slot @@ -399,7 +400,7 @@ def test_historical_batch(state): # # state_transition(state, block) # -# yield 'blocks', [block], [spec.BeaconBlock] +# yield 'blocks', [block], List[spec.BeaconBlock] # yield 'post', state # # assert state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index cdd09bf23..b1948ad8d 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -1,4 +1,5 @@ from copy import deepcopy +from typing import List import eth2spec.phase0.spec as spec from eth2spec.phase0.spec import ( @@ -86,7 +87,7 @@ def test_finality_rule_4(state): assert state.finalized_epoch == prev_state.current_justified_epoch assert state.finalized_root == prev_state.current_justified_root - yield 'blocks', blocks, [spec.BeaconBlock] + yield 'blocks', blocks, List[spec.BeaconBlock] yield 'post', state @@ -116,7 +117,7 @@ def test_finality_rule_1(state): assert state.finalized_epoch == prev_state.previous_justified_epoch assert state.finalized_root == prev_state.previous_justified_root - yield 'blocks', blocks, [spec.BeaconBlock] + yield 'blocks', blocks, List[spec.BeaconBlock] yield 'post', state @@ -148,7 +149,7 @@ def test_finality_rule_2(state): blocks += new_blocks - yield 'blocks', blocks, [spec.BeaconBlock] + yield 'blocks', blocks, List[spec.BeaconBlock] yield 'post', state @@ -197,5 +198,5 @@ def test_finality_rule_3(state): assert state.finalized_epoch == prev_state.current_justified_epoch assert state.finalized_root == prev_state.current_justified_root - yield 'blocks', blocks, [spec.BeaconBlock] + yield 'blocks', blocks, List[spec.BeaconBlock] yield 'post', state From df25f22f012d606daaf2264489e75e2a9f040a54 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 1 Jun 2019 01:55:20 +0200 Subject: [PATCH 30/48] get rid of scope-problem inducing init trick --- test_libs/pyspec/eth2spec/utils/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/__init__.py b/test_libs/pyspec/eth2spec/utils/__init__.py index 482044798..e69de29bb 100644 --- a/test_libs/pyspec/eth2spec/utils/__init__.py +++ b/test_libs/pyspec/eth2spec/utils/__init__.py @@ -1,4 +0,0 @@ -from .merkle_minimal import * -from .hash_function import * -from .bls import * -from .ssz import * From 7df788c7d5ba3e070b391cebf6713654d7999fc8 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 1 Jun 2019 02:22:14 +0200 Subject: [PATCH 31/48] resolve linting problems, except ssz-partials --- scripts/phase0/build_spec.py | 4 ++-- scripts/phase0/function_puller.py | 2 +- test_libs/pyspec/eth2spec/debug/decode.py | 7 ++++++- test_libs/pyspec/eth2spec/debug/encode.py | 6 +++++- test_libs/pyspec/eth2spec/debug/random_value.py | 8 +++++++- test_libs/pyspec/eth2spec/utils/ssz/__init__.py | 2 -- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 10 ++++++++-- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 13 ++++++++----- 8 files changed, 37 insertions(+), 15 deletions(-) diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index c7a377297..0d5f64094 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -18,8 +18,8 @@ from eth2spec.utils.ssz.ssz_impl import ( signing_root, ) from eth2spec.utils.ssz.ssz_typing import ( - uint8, uint16, uint32, uint64, uint128, uint256, - Container, Vector, BytesN + # unused: uint8, uint16, uint32, uint128, uint256, + uint64, Container, Vector, BytesN ) from eth2spec.utils.hash_function import hash from eth2spec.utils.bls import ( diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index d7765a4a4..ea8688e86 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -68,7 +68,7 @@ def get_spec(file_name: str) -> List[str]: for type_line in ssz_type: if len(type_line) > 0: code_lines.append(' ' + type_line) - code_lines.append('\n') + code_lines.append('') for (ssz_type_name, _) in type_defs: code_lines.append(f' global_vars["{ssz_type_name}"] = {ssz_type_name}') code_lines.append(' global_vars["ssz_types"] = [') diff --git a/test_libs/pyspec/eth2spec/debug/decode.py b/test_libs/pyspec/eth2spec/debug/decode.py index c9657dc28..86716c5f4 100644 --- a/test_libs/pyspec/eth2spec/debug/decode.py +++ b/test_libs/pyspec/eth2spec/debug/decode.py @@ -1,5 +1,10 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root -from eth2spec.utils.ssz.ssz_typing import * +from eth2spec.utils.ssz.ssz_typing import ( + is_uint_type, is_bool_type, is_list_type, + is_vector_type, is_bytes_type, is_bytesn_type, is_container_type, + read_vector_elem_type, read_list_elem_type, + Vector, BytesN +) def decode(data, typ): diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index a3c3c1189..8be567f17 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -1,5 +1,9 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root -from eth2spec.utils.ssz.ssz_typing import * +from eth2spec.utils.ssz.ssz_typing import ( + is_uint_type, is_bool_type, is_list_type, is_vector_type, is_container_type, + read_elem_type, + uint +) def encode(value, typ, include_hash_tree_roots=False): diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index ab9c4c885..43c3de349 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -2,9 +2,15 @@ from random import Random from typing import Any from enum import Enum -from eth2spec.utils.ssz.ssz_typing import * from eth2spec.utils.ssz.ssz_impl import is_basic_type +from eth2spec.utils.ssz.ssz_typing import ( + is_uint_type, is_bool_type, is_list_type, + is_vector_type, is_bytes_type, is_bytesn_type, is_container_type, + read_vector_elem_type, read_list_elem_type, + uint_byte_size +) + # in bytes UINT_SIZES = [1, 2, 4, 8, 16, 32] diff --git a/test_libs/pyspec/eth2spec/utils/ssz/__init__.py b/test_libs/pyspec/eth2spec/utils/ssz/__init__.py index 6543bd9e5..e69de29bb 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/__init__.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/__init__.py @@ -1,2 +0,0 @@ -from .ssz_impl import * -from .ssz_partials import * diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 0ef89f97f..fac8d8051 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,5 +1,11 @@ from ..merkle_minimal import merkleize_chunks, hash -from .ssz_typing import * +from eth2spec.utils.ssz.ssz_typing import ( + is_uint_type, is_bool_type, is_container_type, + is_list_kind, is_vector_kind, + read_vector_elem_type, read_elem_type, + uint_byte_size, + infer_input_type +) # SSZ Serialization # ----------------------------- @@ -22,6 +28,7 @@ def serialize_basic(value, typ): else: raise Exception("Type not supported: {}".format(typ)) + def deserialize_basic(value, typ): if is_uint_type(typ): return typ(int.from_bytes(value, 'little')) @@ -148,4 +155,3 @@ def signing_root(obj, typ): # ignore last field leaves = [hash_tree_root(field_value, typ=field_typ) for field_value, field_typ in obj.get_typed_values()[:-1]] return merkleize_chunks(chunkify(b''.join(leaves))) - diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 47b4b5c9a..9424e9bd3 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -3,10 +3,6 @@ from typing import List, Iterable, TypeVar, Type, NewType from typing import Union from typing_inspect import get_origin -T = TypeVar('T') -L = TypeVar('L') - - # SSZ integers # ----------------------------- @@ -21,6 +17,7 @@ class uint(int): class uint8(uint): byte_len = 1 + def __new__(cls, value, *args, **kwargs): if value.bit_length() > 8: raise ValueError("value out of bounds for uint8") @@ -32,6 +29,7 @@ byte = NewType('byte', uint8) class uint16(uint): byte_len = 2 + def __new__(cls, value, *args, **kwargs): if value.bit_length() > 16: raise ValueError("value out of bounds for uint16") @@ -40,6 +38,7 @@ class uint16(uint): class uint32(uint): byte_len = 4 + def __new__(cls, value, *args, **kwargs): if value.bit_length() > 32: raise ValueError("value out of bounds for uint16") @@ -52,6 +51,7 @@ uint64 = NewType('uint64', int) class uint128(uint): byte_len = 16 + def __new__(cls, value, *args, **kwargs): if value.bit_length() > 128: raise ValueError("value out of bounds for uint128") @@ -60,6 +60,7 @@ class uint128(uint): class uint256(uint): byte_len = 32 + def __new__(cls, value, *args, **kwargs): if value.bit_length() > 256: raise ValueError("value out of bounds for uint256") @@ -397,12 +398,14 @@ def get_zero_value(typ): elif issubclass(typ, Container): result = typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}) else: - return Exception("Type not supported: {}".format(typ)) + return Exception("Type not supported: {}".format(typ)) return result + # Type helpers # ----------------------------- + def infer_type(obj): if is_uint_type(obj.__class__): return obj.__class__ From f3a517b6f62c1dc24291ece031addc7cfe6758fe Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 1 Jun 2019 02:32:11 +0200 Subject: [PATCH 32/48] fix minor lint problems --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 9424e9bd3..7f20284a8 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -6,6 +6,7 @@ from typing_inspect import get_origin # SSZ integers # ----------------------------- + class uint(int): byte_len = 0 @@ -23,6 +24,7 @@ class uint8(uint): raise ValueError("value out of bounds for uint8") return super().__new__(cls, value) + # Alias for uint8 byte = NewType('byte', uint8) From b79f01e2fa3aaae32c57015df67159e3eefa7a06 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 1 Jun 2019 02:39:41 +0200 Subject: [PATCH 33/48] update flake8 to support type annotation in linting, ignore operator format and default param equals without spaces warnings --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 401e4bdc9..7d7c75052 100644 --- a/Makefile +++ b/Makefile @@ -40,11 +40,11 @@ citest: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; python -m pytest --junitxml=test-reports/eth2spec/test_results.xml . install_lint: - cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install flake8==3.5.0 + cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install flake8==3.7.7 lint: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; \ - flake8 --max-line-length=120 ./eth2spec; + flake8 --ignore=E252,W504 --max-line-length=120 ./eth2spec; # "make pyspec" to create the pyspec for all phases. pyspec: $(PY_SPEC_ALL_TARGETS) From fae1e9285d59767ee6749389a04bfc8cd085b359 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 1 Jun 2019 02:40:29 +0200 Subject: [PATCH 34/48] resolve some new lint issues detected by newer version, and a few looked over ones --- .../eth2spec/test/block_processing/test_process_deposit.py | 4 ++-- test_libs/pyspec/eth2spec/test/helpers/bitfields.py | 6 +++--- test_libs/pyspec/eth2spec/test/helpers/transfers.py | 2 +- test_libs/pyspec/eth2spec/utils/hash_function.py | 3 ++- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py index 0430dd12f..b12ec6bd9 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py @@ -138,7 +138,7 @@ def test_wrong_deposit_for_deposit_count(state): pubkey_1, privkey_1, spec.MAX_EFFECTIVE_BALANCE, - withdrawal_credentials=b'\x00'*32, + withdrawal_credentials=b'\x00' * 32, signed=True, ) deposit_count_1 = len(deposit_data_leaves) @@ -153,7 +153,7 @@ def test_wrong_deposit_for_deposit_count(state): pubkey_2, privkey_2, spec.MAX_EFFECTIVE_BALANCE, - withdrawal_credentials=b'\x00'*32, + withdrawal_credentials=b'\x00' * 32, signed=True, ) diff --git a/test_libs/pyspec/eth2spec/test/helpers/bitfields.py b/test_libs/pyspec/eth2spec/test/helpers/bitfields.py index 7c25d073a..50e5b6cba 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/bitfields.py +++ b/test_libs/pyspec/eth2spec/test/helpers/bitfields.py @@ -5,7 +5,7 @@ def set_bitfield_bit(bitfield, i): byte_index = i // 8 bit_index = i % 8 return ( - bitfield[:byte_index] + - bytes([bitfield[byte_index] | (1 << bit_index)]) + - bitfield[byte_index + 1:] + bitfield[:byte_index] + + bytes([bitfield[byte_index] | (1 << bit_index)]) + + bitfield[byte_index + 1:] ) diff --git a/test_libs/pyspec/eth2spec/test/helpers/transfers.py b/test_libs/pyspec/eth2spec/test/helpers/transfers.py index 9913ef88a..91f8ea60e 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/transfers.py +++ b/test_libs/pyspec/eth2spec/test/helpers/transfers.py @@ -36,7 +36,7 @@ def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=Non # ensure withdrawal_credentials reproducible state.validator_registry[transfer.sender].withdrawal_credentials = ( - spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(transfer.pubkey)[1:] + spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(transfer.pubkey)[1:] ) return transfer diff --git a/test_libs/pyspec/eth2spec/utils/hash_function.py b/test_libs/pyspec/eth2spec/utils/hash_function.py index acd13edc4..f965827d0 100644 --- a/test_libs/pyspec/eth2spec/utils/hash_function.py +++ b/test_libs/pyspec/eth2spec/utils/hash_function.py @@ -1,4 +1,5 @@ from hashlib import sha256 -def hash(x): return sha256(x).digest() +def hash(x): + return sha256(x).digest() diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index fac8d8051..9c1fcdc61 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -57,7 +57,7 @@ def serialize(obj, typ=None): if is_basic_type(typ): return serialize_basic(obj, typ) elif is_list_kind(typ) or is_vector_kind(typ): - return encode_series(obj, [read_elem_type(typ)]*len(obj)) + return encode_series(obj, [read_elem_type(typ)] * len(obj)) elif is_container_type(typ): return encode_series(obj.get_field_values(), typ.get_field_types()) else: From 8e8ef2d0a40e1f37468b3c0622249f85e99d18cc Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 1 Jun 2019 03:19:29 +0200 Subject: [PATCH 35/48] Fix forgotten setup.py change, make generators using ssz work again --- test_libs/pyspec/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test_libs/pyspec/setup.py b/test_libs/pyspec/setup.py index 1a131a417..e99b911ee 100644 --- a/test_libs/pyspec/setup.py +++ b/test_libs/pyspec/setup.py @@ -9,5 +9,6 @@ setup( "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.7.3", "py_ecc>=1.6.0", + "typing_inspect==0.4.0" ] ) From a5576059f8e5504e065dffb428129695390bb4b1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 3 Jun 2019 17:14:29 +0800 Subject: [PATCH 36/48] Fix `ssz_partials.py` linter errors --- .../pyspec/eth2spec/utils/ssz/ssz_partials.py | 93 ++++++++++++++----- 1 file changed, 71 insertions(+), 22 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py index 6a243f4f3..6723674eb 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py @@ -1,17 +1,44 @@ from ..merkle_minimal import hash, next_power_of_two -from .ssz_typing import * -from .ssz_impl import * +from .ssz_typing import ( + Container, + infer_input_type, + is_bool_type, + is_bytes_type, + is_bytesn_type, + is_container_type, + is_list_kind, + is_list_type, + is_uint_type, + is_vector_kind, + is_vector_type, + read_elem_type, + uint_byte_size, +) +from .ssz_impl import ( + chunkify, + deserialize_basic, + get_typed_values, + is_basic_type, + is_bottom_layer_kind, + pack, + serialize_basic, +) + ZERO_CHUNK = b'\x00' * 32 + def last_power_of_two(x): - return next_power_of_two(x+1) // 2 + return next_power_of_two(x + 1) // 2 + def concat_generalized_indices(x, y): return x * last_power_of_two(y) + y - last_power_of_two(y) + def rebase(objs, new_root): - return {concat_generalized_indices(new_root, k): v for k,v in objs.items()} + return {concat_generalized_indices(new_root, k): v for k, v in objs.items()} + def constrict_generalized_index(x, q): depth = last_power_of_two(x // q) @@ -20,32 +47,36 @@ def constrict_generalized_index(x, q): return None return o + def unrebase(objs, q): o = {} - for k,v in objs.items(): + for k, v in objs.items(): new_k = constrict_generalized_index(k, q) if new_k is not None: o[new_k] = v return o + def filler(starting_position, chunk_count): at, skip, end = chunk_count, 1, next_power_of_two(chunk_count) value = ZERO_CHUNK o = {} while at < end: - while at % (skip*2) == 0: + while at % (skip * 2) == 0: skip *= 2 value = hash(value + value) o[(starting_position + at) // skip] = value at += skip return o + def merkle_tree_of_chunks(chunks, root): starting_index = root * next_power_of_two(len(chunks)) - o = {starting_index+i: chunk for i,chunk in enumerate(chunks)} + o = {starting_index + i: chunk for i, chunk in enumerate(chunks)} o = {**o, **filler(starting_index, len(chunks))} return o + @infer_input_type def ssz_leaves(obj, typ=None, root=1): if is_list_kind(typ): @@ -57,33 +88,36 @@ def ssz_leaves(obj, typ=None, root=1): if is_bottom_layer_kind(typ): data = serialize_basic(obj, typ) if is_basic_type(typ) else pack(obj, read_elem_type(typ)) q = {**o, **merkle_tree_of_chunks(chunkify(data), base)} - #print(obj, root, typ, base, list(q.keys())) + # print(obj, root, typ, base, list(q.keys())) return(q) else: fields = get_typed_values(obj, typ=typ) sub_base = base * next_power_of_two(len(fields)) for i, (elem, elem_type) in enumerate(fields): - o = {**o, **ssz_leaves(elem, typ=elem_type, root=sub_base+i)} + o = {**o, **ssz_leaves(elem, typ=elem_type, root=sub_base + i)} q = {**o, **filler(sub_base, len(fields))} - #print(obj, root, typ, base, list(q.keys())) + # print(obj, root, typ, base, list(q.keys())) return(q) + def fill(objects): - objects = {k:v for k,v in objects.items()} + objects = {k: v for k, v in objects.items()} keys = sorted(objects.keys())[::-1] pos = 0 while pos < len(keys): k = keys[pos] - if k in objects and k^1 in objects and k//2 not in objects: - objects[k//2] = hash(objects[k&-2] + objects[k|1]) + if k in objects and k ^ 1 in objects and k // 2 not in objects: + objects[k // 2] = hash(objects[k & - 2] + objects[k | 1]) keys.append(k // 2) pos += 1 return objects + @infer_input_type def ssz_full(obj, typ=None): return fill(ssz_leaves(obj, typ=typ)) + def get_basic_type_size(typ): if is_uint_type(typ): return uint_byte_size(typ) @@ -92,6 +126,7 @@ def get_basic_type_size(typ): else: raise Exception("Type not basic: {}".format(typ)) + def get_bottom_layer_element_position(typ, base, length, index): """ Returns the generalized index and the byte range of the index'th value @@ -104,7 +139,8 @@ def get_bottom_layer_element_position(typ, base, length, index): chunk_count = (1 if is_basic_type(typ) else length) * elem_size // 32 generalized_index = base * next_power_of_two(chunk_count) + chunk_index start = elem_size * index % 32 - return generalized_index, start, start+elem_size + return generalized_index, start, start + elem_size + @infer_input_type def get_generalized_indices(obj, path, typ=None, root=1): @@ -132,14 +168,17 @@ def get_generalized_indices(obj, path, typ=None, root=1): root=base * next_power_of_two(field_count) + index ) + def get_branch_indices(tree_index): o = [tree_index, tree_index ^ 1] while o[-1] > 1: o.append((o[-1] // 2) ^ 1) return o[:-1] + def remove_redundant_indices(obj): - return {k:v for k,v in obj.items() if not (k*2 in obj and k*2+1 in obj)} + return {k: v for k, v in obj.items() if not (k * 2 in obj and k * 2 + 1 in obj)} + def merge(*args): o = {} @@ -147,14 +186,19 @@ def merge(*args): o = {**o, **arg} return fill(o) + @infer_input_type def get_nodes_along_path(obj, path, typ=None): indices = get_generalized_indices(obj, path, typ=typ) - return remove_redundant_indices(merge(*({i:obj.objects[i] for i in get_branch_indices(index)} for index in indices))) + return remove_redundant_indices(merge( + *({i: obj.objects[i] for i in get_branch_indices(index)} for index in indices) + )) + class OutOfRangeException(Exception): pass + class SSZPartial(): def __init__(self, typ, objects, root=1): assert not is_basic_type(typ) @@ -204,9 +248,9 @@ class SSZPartial(): def __len__(self): if is_list_kind(self.typ): - if self.root*2+1 not in self.objects: - raise OutOfRangeException("Do not have required data: {}".format(self.root*2+1)) - return int.from_bytes(self.objects[self.root*2+1], 'little') + if self.root * 2 + 1 not in self.objects: + raise OutOfRangeException("Do not have required data: {}".format(self.root * 2 + 1)) + return int.from_bytes(self.objects[self.root * 2 + 1], 'little') elif is_vector_kind(self.typ): return self.typ.length elif is_container_type(self.typ): @@ -222,7 +266,9 @@ class SSZPartial(): elif is_vector_kind(self.typ): return self.typ(*(self[i] for i in range(len(self)))) elif is_container_type(self.typ): - full_value = lambda x: x.full_value() if hasattr(x, 'full_value') else x + def full_value(x): + return x.full_value() if hasattr(x, 'full_value') else x + return self.typ(**{field: full_value(self.getter(field)) for field in self.typ.get_field_names()}) elif is_basic_type(self.typ): return self.getter(0) @@ -235,8 +281,8 @@ class SSZPartial(): pos = 0 while pos < len(keys): k = keys[pos] - if k in o and k^1 in o and k//2 not in o: - o[k//2] = hash(o[k&-2] + o[k|1]) + if k in o and k ^ 1 in o and k // 2 not in o: + o[k // 2] = hash(o[k & - 2] + o[k | 1]) keys.append(k // 2) pos += 1 return o[self.root] @@ -244,13 +290,16 @@ class SSZPartial(): def __str__(self): return str(self.full_value()) + def ssz_partial(typ, objects, root=1): ssz_type = ( Container if is_container_type(typ) else typ if (is_vector_type(typ) or is_bytesn_type(typ)) else object ) + class Partial(SSZPartial, ssz_type): pass + if is_container_type(typ): Partial.__annotations__ = typ.__annotations__ o = Partial(typ, objects, root=root) From 6cd981128db1d4b4f215f4749a323ac26ce3433e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 3 Jun 2019 17:21:38 +0800 Subject: [PATCH 37/48] Delete extra newline --- scripts/phase0/function_puller.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index ea8688e86..0f4b21c0e 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -68,7 +68,6 @@ def get_spec(file_name: str) -> List[str]: for type_line in ssz_type: if len(type_line) > 0: code_lines.append(' ' + type_line) - code_lines.append('') for (ssz_type_name, _) in type_defs: code_lines.append(f' global_vars["{ssz_type_name}"] = {ssz_type_name}') code_lines.append(' global_vars["ssz_types"] = [') From fb584bc067e4aa474e4db04f27435cdeb36e15b8 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 3 Jun 2019 17:23:30 +0200 Subject: [PATCH 38/48] fix linting dependency + caching issue --- .circleci/config.yml | 10 +++++----- Makefile | 3 --- test_libs/pyspec/requirements-testing.txt | 1 + 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f3c5f6a81..244d20c55 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,13 +60,13 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec-03 + venv_name: v2-pyspec reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Install pyspec requirements - command: make install_test && make install_lint + command: make install_test - save_cached_venv: - venv_name: v1-pyspec-03 + venv_name: v2-pyspec reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' venv_path: ./test_libs/pyspec/venv test: @@ -77,7 +77,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec-03 + venv_name: v2-pyspec reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run py-tests @@ -92,7 +92,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec-03 + venv_name: v2-pyspec reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run linter diff --git a/Makefile b/Makefile index 7d7c75052..51cb1aaa3 100644 --- a/Makefile +++ b/Makefile @@ -39,9 +39,6 @@ test: $(PY_SPEC_ALL_TARGETS) citest: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; python -m pytest --junitxml=test-reports/eth2spec/test_results.xml . -install_lint: - cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install flake8==3.7.7 - lint: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; \ flake8 --ignore=E252,W504 --max-line-length=120 ./eth2spec; diff --git a/test_libs/pyspec/requirements-testing.txt b/test_libs/pyspec/requirements-testing.txt index 388a878a9..331d0fa28 100644 --- a/test_libs/pyspec/requirements-testing.txt +++ b/test_libs/pyspec/requirements-testing.txt @@ -1,3 +1,4 @@ -r requirements.txt pytest>=3.6,<3.7 ../config_helpers +flake8==3.7.7 From a7ee6f108ef5dbde7efc05d7b807ee067946f986 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 4 Jun 2019 17:42:21 +0800 Subject: [PATCH 39/48] Refactor --- test_libs/pyspec/eth2spec/debug/decode.py | 3 +- test_libs/pyspec/eth2spec/debug/encode.py | 3 +- .../pyspec/eth2spec/debug/random_value.py | 44 ++++----- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 89 +++++++++++-------- 4 files changed, 80 insertions(+), 59 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/decode.py b/test_libs/pyspec/eth2spec/debug/decode.py index 86716c5f4..5ce116025 100644 --- a/test_libs/pyspec/eth2spec/debug/decode.py +++ b/test_libs/pyspec/eth2spec/debug/decode.py @@ -36,5 +36,4 @@ def decode(data, typ): hash_tree_root(ret, typ).hex()) return ret else: - print(data, typ) - raise Exception("Type not recognized") + raise Exception(f"Type not recognized: data={data}, typ={typ}") diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index 8be567f17..61dd87928 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -33,5 +33,4 @@ def encode(value, typ, include_hash_tree_roots=False): ret["hash_tree_root"] = '0x' + hash_tree_root(value, typ).hex() return ret else: - print(value, typ) - raise Exception("Type not recognized") + raise Exception(f"Type not recognized: value={value}, typ={typ}") diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index 43c3de349..189c1268a 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -56,65 +56,69 @@ def get_random_ssz_object(rng: Random, """ if chaos: mode = rng.choice(list(RandomizationMode)) - # Bytes array if is_bytes_type(typ): + # Bytes array if mode == RandomizationMode.mode_nil_count: return b'' - if mode == RandomizationMode.mode_max_count: + elif mode == RandomizationMode.mode_max_count: return get_random_bytes_list(rng, max_bytes_length) - if mode == RandomizationMode.mode_one_count: + elif mode == RandomizationMode.mode_one_count: return get_random_bytes_list(rng, 1) - if mode == RandomizationMode.mode_zero: + elif mode == RandomizationMode.mode_zero: return b'\x00' - if mode == RandomizationMode.mode_max: + elif mode == RandomizationMode.mode_max: return b'\xff' - return get_random_bytes_list(rng, rng.randint(0, max_bytes_length)) + else: + return get_random_bytes_list(rng, rng.randint(0, max_bytes_length)) elif is_bytesn_type(typ): + # BytesN length = typ.length # Sanity, don't generate absurdly big random values # If a client is aiming to performance-test, they should create a benchmark suite. assert length <= max_bytes_length if mode == RandomizationMode.mode_zero: return b'\x00' * length - if mode == RandomizationMode.mode_max: + elif mode == RandomizationMode.mode_max: return b'\xff' * length - return get_random_bytes_list(rng, length) - # Basic types + else: + return get_random_bytes_list(rng, length) elif is_basic_type(typ): + # Basic types if mode == RandomizationMode.mode_zero: return get_min_basic_value(typ) - if mode == RandomizationMode.mode_max: + elif mode == RandomizationMode.mode_max: return get_max_basic_value(typ) - return get_random_basic_value(rng, typ) - # Vector: + else: + return get_random_basic_value(rng, typ) elif is_vector_type(typ): + # Vector elem_typ = read_vector_elem_type(typ) return [ get_random_ssz_object(rng, elem_typ, max_bytes_length, max_list_length, mode, chaos) for _ in range(typ.length) ] - # List: elif is_list_type(typ): + # List elem_typ = read_list_elem_type(typ) length = rng.randint(0, max_list_length) if mode == RandomizationMode.mode_one_count: length = 1 - if mode == RandomizationMode.mode_max_count: + elif mode == RandomizationMode.mode_max_count: length = max_list_length + return [ get_random_ssz_object(rng, elem_typ, max_bytes_length, max_list_length, mode, chaos) for _ in range(length) ] - # Container: elif is_container_type(typ): + # Container return typ(**{ field: get_random_ssz_object(rng, subtype, max_bytes_length, max_list_length, mode, chaos) for field, subtype in typ.get_fields() }) else: - print(typ) - raise Exception("Type not recognized") + raise Exception(f"Type not recognized: typ={typ}") def get_random_bytes_list(rng: Random, length: int) -> bytes: @@ -129,7 +133,7 @@ def get_random_basic_value(rng: Random, typ) -> Any: assert size in UINT_SIZES return rng.randint(0, 256**size - 1) else: - raise ValueError("Not a basic type") + raise ValueError(f"Not a basic type: typ={typ}") def get_min_basic_value(typ) -> Any: @@ -140,7 +144,7 @@ def get_min_basic_value(typ) -> Any: assert size in UINT_SIZES return 0 else: - raise ValueError("Not a basic type") + raise ValueError(f"Not a basic type: typ={typ}") def get_max_basic_value(typ) -> Any: @@ -151,4 +155,4 @@ def get_max_basic_value(typ) -> Any: assert size in UINT_SIZES return 256**size - 1 else: - raise ValueError("Not a basic type") + raise ValueError(f"Not a basic type: typ={typ}") diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 7f20284a8..fcc3fab82 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -83,13 +83,15 @@ def is_uint_type(typ): def uint_byte_size(typ): if hasattr(typ, '__supertype__'): typ = typ.__supertype__ + if isinstance(typ, type): if issubclass(typ, uint): return typ.byte_len elif issubclass(typ, int): # Default to uint64 return 8 - raise TypeError("Type %s is not an uint (or int-default uint64) type" % typ) + else: + raise TypeError("Type %s is not an uint (or int-default uint64) type" % typ) # SSZ Container base class @@ -167,32 +169,34 @@ def _is_vector_instance_of(a, b): # Other must not be a BytesN if issubclass(b, bytes): return False - if not hasattr(b, 'elem_type') or not hasattr(b, 'length'): + elif not hasattr(b, 'elem_type') or not hasattr(b, 'length'): # Vector (b) is not an instance of Vector[X, Y] (a) return False - if not hasattr(a, 'elem_type') or not hasattr(a, 'length'): + elif not hasattr(a, 'elem_type') or not hasattr(a, 'length'): # Vector[X, Y] (b) is an instance of Vector (a) return True - - # Vector[X, Y] (a) is an instance of Vector[X, Y] (b) - return a.elem_type == b.elem_type and a.length == b.length + else: + # Vector[X, Y] (a) is an instance of Vector[X, Y] (b) + return a.elem_type == b.elem_type and a.length == b.length def _is_equal_vector_type(a, b): # Other must not be a BytesN if issubclass(b, bytes): return False - if not hasattr(a, 'elem_type') or not hasattr(a, 'length'): + elif not hasattr(a, 'elem_type') or not hasattr(a, 'length'): if not hasattr(b, 'elem_type') or not hasattr(b, 'length'): # Vector == Vector return True - # Vector != Vector[X, Y] - return False - if not hasattr(b, 'elem_type') or not hasattr(b, 'length'): + else: + # Vector != Vector[X, Y] + return False + elif not hasattr(b, 'elem_type') or not hasattr(b, 'length'): # Vector[X, Y] != Vector return False - # Vector[X, Y] == Vector[X, Y] - return a.elem_type == b.elem_type and a.length == b.length + else: + # Vector[X, Y] == Vector[X, Y] + return a.elem_type == b.elem_type and a.length == b.length class VectorMeta(type): @@ -233,7 +237,7 @@ class Vector(metaclass=VectorMeta): cls = self.__class__ if not hasattr(cls, 'elem_type'): raise TypeError("Type Vector without elem_type data cannot be instantiated") - if not hasattr(cls, 'length'): + elif not hasattr(cls, 'length'): raise TypeError("Type Vector without length data cannot be instantiated") if len(args) != cls.length: @@ -282,32 +286,34 @@ def _is_bytes_n_instance_of(a, b): # Other has to be a Bytes derivative class to be a BytesN if not issubclass(b, bytes): return False - if not hasattr(b, 'length'): + elif not hasattr(b, 'length'): # BytesN (b) is not an instance of BytesN[X] (a) return False - if not hasattr(a, 'length'): + elif not hasattr(a, 'length'): # BytesN[X] (b) is an instance of BytesN (a) return True - - # BytesN[X] (a) is an instance of BytesN[X] (b) - return a.length == b.length + else: + # BytesN[X] (a) is an instance of BytesN[X] (b) + return a.length == b.length def _is_equal_bytes_n_type(a, b): # Other has to be a Bytes derivative class to be a BytesN if not issubclass(b, bytes): return False - if not hasattr(a, 'length'): + elif not hasattr(a, 'length'): if not hasattr(b, 'length'): # BytesN == BytesN return True - # BytesN != BytesN[X] - return False - if not hasattr(b, 'length'): + else: + # BytesN != BytesN[X] + return False + elif not hasattr(b, 'length'): # BytesN[X] != BytesN return False - # BytesN[X] == BytesN[X] - return a.length == b.length + else: + # BytesN[X] == BytesN[X] + return a.length == b.length class BytesNMeta(type): @@ -341,14 +347,15 @@ class BytesNMeta(type): def parse_bytes(val): if val is None: return None - if isinstance(val, str): + elif isinstance(val, str): # TODO: import from eth-utils instead, and do: hexstr_if_str(to_bytes, val) return None - if isinstance(val, bytes): + elif isinstance(val, bytes): return val - if isinstance(val, int): + elif isinstance(val, int): return bytes([val]) - return None + else: + return None class BytesN(bytes, metaclass=BytesNMeta): @@ -433,6 +440,9 @@ def infer_input_type(fn): def is_bool_type(typ): + """ + Checks if the given type is a bool. + """ if hasattr(typ, '__supertype__'): typ = typ.__supertype__ return isinstance(typ, type) and issubclass(typ, bool) @@ -446,36 +456,45 @@ def is_list_type(typ): def is_bytes_type(typ): + """ + Check if the given type is a ``bytes``. + """ # Do not accept subclasses of bytes here, to avoid confusion with BytesN return typ == bytes +def is_bytesn_type(typ): + """ + Check if the given type is a BytesN. + """ + return isinstance(typ, type) and issubclass(typ, BytesN) + + def is_list_kind(typ): """ - Checks if the given type is a kind of list. Can be bytes. + Check if the given type is a kind of list. Can be bytes. """ return is_list_type(typ) or is_bytes_type(typ) def is_vector_type(typ): """ - Checks if the given type is a vector. + Check if the given type is a vector. """ return isinstance(typ, type) and issubclass(typ, Vector) -def is_bytesn_type(typ): - return isinstance(typ, type) and issubclass(typ, BytesN) - - def is_vector_kind(typ): """ - Checks if the given type is a kind of vector. Can be BytesN. + Check if the given type is a kind of vector. Can be BytesN. """ return is_vector_type(typ) or is_bytesn_type(typ) def is_container_type(typ): + """ + Check if the given type is a container. + """ return isinstance(typ, type) and issubclass(typ, Container) From e2eab66a9e1fb0a1cfa8afbc9ff9aaf0a06489d5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 4 Jun 2019 17:55:05 +0800 Subject: [PATCH 40/48] Refactor --- test_libs/pyspec/eth2spec/debug/random_value.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index 189c1268a..3edcc8808 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -12,9 +12,9 @@ from eth2spec.utils.ssz.ssz_typing import ( ) # in bytes -UINT_SIZES = [1, 2, 4, 8, 16, 32] +UINT_SIZES = (1, 2, 4, 8, 16, 32) -random_mode_names = ["random", "zero", "max", "nil", "one", "lengthy"] +random_mode_names = ("random", "zero", "max", "nil", "one", "lengthy") class RandomizationMode(Enum): @@ -128,7 +128,7 @@ def get_random_bytes_list(rng: Random, length: int) -> bytes: def get_random_basic_value(rng: Random, typ) -> Any: if is_bool_type(typ): return rng.choice((True, False)) - if is_uint_type(typ): + elif is_uint_type(typ): size = uint_byte_size(typ) assert size in UINT_SIZES return rng.randint(0, 256**size - 1) @@ -139,7 +139,7 @@ def get_random_basic_value(rng: Random, typ) -> Any: def get_min_basic_value(typ) -> Any: if is_bool_type(typ): return False - if is_uint_type(typ): + elif is_uint_type(typ): size = uint_byte_size(typ) assert size in UINT_SIZES return 0 @@ -150,7 +150,7 @@ def get_min_basic_value(typ) -> Any: def get_max_basic_value(typ) -> Any: if is_bool_type(typ): return True - if is_uint_type(typ): + elif is_uint_type(typ): size = uint_byte_size(typ) assert size in UINT_SIZES return 256**size - 1 From 6d55ba9c8c6e2b497722f6388f4037fea0f127eb Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 4 Jun 2019 18:59:52 +0800 Subject: [PATCH 41/48] minor refactor --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index fcc3fab82..5bd6c0a24 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -233,7 +233,6 @@ class VectorMeta(type): class Vector(metaclass=VectorMeta): def __init__(self, *args: Iterable): - cls = self.__class__ if not hasattr(cls, 'elem_type'): raise TypeError("Type Vector without elem_type data cannot be instantiated") @@ -282,6 +281,10 @@ class Vector(metaclass=VectorMeta): return self.hash_tree_root() == other.hash_tree_root() +# SSZ BytesN +# ----------------------------- + + def _is_bytes_n_instance_of(a, b): # Other has to be a Bytes derivative class to be a BytesN if not issubclass(b, bytes): @@ -441,7 +444,7 @@ def infer_input_type(fn): def is_bool_type(typ): """ - Checks if the given type is a bool. + Check if the given type is a bool. """ if hasattr(typ, '__supertype__'): typ = typ.__supertype__ @@ -450,7 +453,7 @@ def is_bool_type(typ): def is_list_type(typ): """ - Checks if the given type is a list. + Check if the given type is a list. """ return get_origin(typ) is List or get_origin(typ) is list From 8631cad251e0ce84e400a737c937a48c914228fd Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Tue, 4 Jun 2019 15:22:34 +0200 Subject: [PATCH 42/48] Apply suggestions from code review Co-Authored-By: Danny Ryan --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 9c1fcdc61..ea4313c67 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -109,6 +109,7 @@ def pack(values, subtype): def chunkify(bytez): + # pad `bytez` to nearest 32-byte multiple bytez += b'\x00' * (-len(bytez) % 32) return [bytez[i:i + 32] for i in range(0, len(bytez), 32)] From fe9c708d8345611b495d61552f6095e168d88952 Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Tue, 4 Jun 2019 15:31:20 +0200 Subject: [PATCH 43/48] Fix whitespace Co-Authored-By: Danny Ryan --- specs/core/0_beacon-chain.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index bc0b79f47..06c1c302d 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -532,15 +532,12 @@ class BeaconState(Container): slot: uint64 genesis_time: uint64 fork: Fork # For versioning hard forks - # Validator registry validator_registry: List[Validator] balances: List[uint64] - # Randomness and committees latest_randao_mixes: Vector[Bytes32, LATEST_RANDAO_MIXES_LENGTH] latest_start_shard: uint64 - # Finality previous_epoch_attestations: List[PendingAttestation] current_epoch_attestations: List[PendingAttestation] @@ -551,7 +548,6 @@ class BeaconState(Container): justification_bitfield: uint64 finalized_epoch: uint64 finalized_root: Bytes32 - # Recent state current_crosslinks: Vector[Crosslink, SHARD_COUNT] previous_crosslinks: Vector[Crosslink, SHARD_COUNT] @@ -561,7 +557,6 @@ class BeaconState(Container): latest_slashed_balances: Vector[uint64, LATEST_SLASHED_EXIT_LENGTH] latest_block_header: BeaconBlockHeader historical_roots: List[Bytes32] - # Ethereum 1.0 chain data latest_eth1_data: Eth1Data eth1_data_votes: List[Eth1Data] From 59137fd5a6cc629733be6ae0b59e962418fd124c Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 4 Jun 2019 15:40:15 +0200 Subject: [PATCH 44/48] fix get_zero_value exception raise + fix up type checks --- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 5bd6c0a24..1e05ccd5d 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -394,24 +394,22 @@ class BytesN(bytes, metaclass=BytesNMeta): # SSZ Defaults # ----------------------------- def get_zero_value(typ): - result = None if is_uint_type(typ): - result = 0 + return 0 elif is_list_type(typ): - result = [] - elif issubclass(typ, bool): - result = False - elif issubclass(typ, Vector): - result = typ() - elif issubclass(typ, BytesN): - result = typ() - elif issubclass(typ, bytes): - result = b'' - elif issubclass(typ, Container): - result = typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}) + return [] + elif is_bool_type(typ): + return False + elif is_vector_type(typ): + return typ() + elif is_bytesn_type(typ): + return typ() + elif is_bytes_type(typ): + return b'' + elif is_container_type(typ): + return typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}) else: - return Exception("Type not supported: {}".format(typ)) - return result + raise Exception("Type not supported: {}".format(typ)) # Type helpers @@ -522,7 +520,7 @@ def read_elem_type(typ): return read_list_elem_type(typ) elif is_vector_type(typ): return read_vector_elem_type(typ) - elif issubclass(typ, bytes): + elif issubclass(typ, bytes): # bytes or bytesN return byte else: raise TypeError("Unexpected type: {}".format(typ)) From 578328bb54f0ccd410e763d29a9a45f4294b2c2a Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 4 Jun 2019 18:11:50 +0200 Subject: [PATCH 45/48] call comments: remove partials from ssz rework PR, review that seperately --- .../pyspec/eth2spec/utils/ssz/ssz_partials.py | 306 ------------------ 1 file changed, 306 deletions(-) delete mode 100644 test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py deleted file mode 100644 index 6723674eb..000000000 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py +++ /dev/null @@ -1,306 +0,0 @@ -from ..merkle_minimal import hash, next_power_of_two -from .ssz_typing import ( - Container, - infer_input_type, - is_bool_type, - is_bytes_type, - is_bytesn_type, - is_container_type, - is_list_kind, - is_list_type, - is_uint_type, - is_vector_kind, - is_vector_type, - read_elem_type, - uint_byte_size, -) -from .ssz_impl import ( - chunkify, - deserialize_basic, - get_typed_values, - is_basic_type, - is_bottom_layer_kind, - pack, - serialize_basic, -) - - -ZERO_CHUNK = b'\x00' * 32 - - -def last_power_of_two(x): - return next_power_of_two(x + 1) // 2 - - -def concat_generalized_indices(x, y): - return x * last_power_of_two(y) + y - last_power_of_two(y) - - -def rebase(objs, new_root): - return {concat_generalized_indices(new_root, k): v for k, v in objs.items()} - - -def constrict_generalized_index(x, q): - depth = last_power_of_two(x // q) - o = depth + x - q * depth - if concat_generalized_indices(q, o) != x: - return None - return o - - -def unrebase(objs, q): - o = {} - for k, v in objs.items(): - new_k = constrict_generalized_index(k, q) - if new_k is not None: - o[new_k] = v - return o - - -def filler(starting_position, chunk_count): - at, skip, end = chunk_count, 1, next_power_of_two(chunk_count) - value = ZERO_CHUNK - o = {} - while at < end: - while at % (skip * 2) == 0: - skip *= 2 - value = hash(value + value) - o[(starting_position + at) // skip] = value - at += skip - return o - - -def merkle_tree_of_chunks(chunks, root): - starting_index = root * next_power_of_two(len(chunks)) - o = {starting_index + i: chunk for i, chunk in enumerate(chunks)} - o = {**o, **filler(starting_index, len(chunks))} - return o - - -@infer_input_type -def ssz_leaves(obj, typ=None, root=1): - if is_list_kind(typ): - o = {root * 2 + 1: len(obj).to_bytes(32, 'little')} - base = root * 2 - else: - o = {} - base = root - if is_bottom_layer_kind(typ): - data = serialize_basic(obj, typ) if is_basic_type(typ) else pack(obj, read_elem_type(typ)) - q = {**o, **merkle_tree_of_chunks(chunkify(data), base)} - # print(obj, root, typ, base, list(q.keys())) - return(q) - else: - fields = get_typed_values(obj, typ=typ) - sub_base = base * next_power_of_two(len(fields)) - for i, (elem, elem_type) in enumerate(fields): - o = {**o, **ssz_leaves(elem, typ=elem_type, root=sub_base + i)} - q = {**o, **filler(sub_base, len(fields))} - # print(obj, root, typ, base, list(q.keys())) - return(q) - - -def fill(objects): - objects = {k: v for k, v in objects.items()} - keys = sorted(objects.keys())[::-1] - pos = 0 - while pos < len(keys): - k = keys[pos] - if k in objects and k ^ 1 in objects and k // 2 not in objects: - objects[k // 2] = hash(objects[k & - 2] + objects[k | 1]) - keys.append(k // 2) - pos += 1 - return objects - - -@infer_input_type -def ssz_full(obj, typ=None): - return fill(ssz_leaves(obj, typ=typ)) - - -def get_basic_type_size(typ): - if is_uint_type(typ): - return uint_byte_size(typ) - elif is_bool_type(typ): - return 1 - else: - raise Exception("Type not basic: {}".format(typ)) - - -def get_bottom_layer_element_position(typ, base, length, index): - """ - Returns the generalized index and the byte range of the index'th value - in the list with the given base generalized index and given length - """ - assert index < (1 if is_basic_type(typ) else length) - elem_typ = typ if is_basic_type(typ) else read_elem_type(typ) - elem_size = get_basic_type_size(elem_typ) - chunk_index = index * elem_size // 32 - chunk_count = (1 if is_basic_type(typ) else length) * elem_size // 32 - generalized_index = base * next_power_of_two(chunk_count) + chunk_index - start = elem_size * index % 32 - return generalized_index, start, start + elem_size - - -@infer_input_type -def get_generalized_indices(obj, path, typ=None, root=1): - if len(path) == 0: - return [root] if is_basic_type(typ) else list(ssz_leaves(obj, typ=typ, root=root).keys()) - if path[0] == '__len__': - return [root * 2 + 1] if is_list_type(typ) else [] - base = root * 2 if is_list_kind(typ) else root - if is_bottom_layer_kind(typ): - length = 1 if is_basic_type(typ) else len(obj) - index, _, _ = get_bottom_layer_element_position(typ, base, length, path[0]) - return [index] - else: - if is_container_type(typ): - fields = typ.get_field_names() - field_count, index = len(fields), fields.index(path[0]) - elem_type = typ.get_field_types()[index] - child = obj.get_field_values()[index] - else: - field_count, index, elem_type, child = len(obj), path[0], read_elem_type(typ), obj[path[0]] - return get_generalized_indices( - child, - path[1:], - typ=elem_type, - root=base * next_power_of_two(field_count) + index - ) - - -def get_branch_indices(tree_index): - o = [tree_index, tree_index ^ 1] - while o[-1] > 1: - o.append((o[-1] // 2) ^ 1) - return o[:-1] - - -def remove_redundant_indices(obj): - return {k: v for k, v in obj.items() if not (k * 2 in obj and k * 2 + 1 in obj)} - - -def merge(*args): - o = {} - for arg in args: - o = {**o, **arg} - return fill(o) - - -@infer_input_type -def get_nodes_along_path(obj, path, typ=None): - indices = get_generalized_indices(obj, path, typ=typ) - return remove_redundant_indices(merge( - *({i: obj.objects[i] for i in get_branch_indices(index)} for index in indices) - )) - - -class OutOfRangeException(Exception): - pass - - -class SSZPartial(): - def __init__(self, typ, objects, root=1): - assert not is_basic_type(typ) - self.objects = objects - self.typ = typ - self.root = root - if is_container_type(self.typ): - for field in self.typ.get_field_names(): - try: - setattr(self, field, self.getter(field)) - except OutOfRangeException: - pass - - def getter(self, index): - base = self.root * 2 if is_list_kind(self.typ) else self.root - if is_bottom_layer_kind(self.typ): - tree_index, start, end = get_bottom_layer_element_position( - self.typ, base, len(self), index - ) - if tree_index not in self.objects: - raise OutOfRangeException("Do not have required data") - else: - return deserialize_basic( - self.objects[tree_index][start:end], - self.typ if is_basic_type(self.typ) else read_elem_type(self.typ) - ) - else: - if is_container_type(self.typ): - fields = self.typ.get_field_names() - field_count, index = len(fields), fields.index(index) - elem_type = self.typ.get_field_types()[index] - else: - field_count, index, elem_type = len(self), index, read_elem_type(self.typ) - tree_index = base * next_power_of_two(field_count) + index - if tree_index not in self.objects: - raise OutOfRangeException("Do not have required data") - if is_basic_type(elem_type): - return deserialize_basic(self.objects[tree_index][:get_basic_type_size(elem_type)], elem_type) - else: - return ssz_partial(elem_type, self.objects, root=tree_index) - - def __getitem__(self, index): - return self.getter(index) - - def __iter__(self): - return (self[i] for i in range(len(self))) - - def __len__(self): - if is_list_kind(self.typ): - if self.root * 2 + 1 not in self.objects: - raise OutOfRangeException("Do not have required data: {}".format(self.root * 2 + 1)) - return int.from_bytes(self.objects[self.root * 2 + 1], 'little') - elif is_vector_kind(self.typ): - return self.typ.length - elif is_container_type(self.typ): - return len(self.typ.get_fields()) - else: - raise Exception("Unsupported type: {}".format(self.typ)) - - def full_value(self): - if is_bytes_type(self.typ) or is_bytesn_type(self.typ): - return bytes([self.getter(i) for i in range(len(self))]) - elif is_list_kind(self.typ): - return [self[i] for i in range(len(self))] - elif is_vector_kind(self.typ): - return self.typ(*(self[i] for i in range(len(self)))) - elif is_container_type(self.typ): - def full_value(x): - return x.full_value() if hasattr(x, 'full_value') else x - - return self.typ(**{field: full_value(self.getter(field)) for field in self.typ.get_field_names()}) - elif is_basic_type(self.typ): - return self.getter(0) - else: - raise Exception("Unsupported type: {}".format(self.typ)) - - def hash_tree_root(self): - o = {**self.objects} - keys = sorted(o.keys())[::-1] - pos = 0 - while pos < len(keys): - k = keys[pos] - if k in o and k ^ 1 in o and k // 2 not in o: - o[k // 2] = hash(o[k & - 2] + o[k | 1]) - keys.append(k // 2) - pos += 1 - return o[self.root] - - def __str__(self): - return str(self.full_value()) - - -def ssz_partial(typ, objects, root=1): - ssz_type = ( - Container if is_container_type(typ) else - typ if (is_vector_type(typ) or is_bytesn_type(typ)) else object - ) - - class Partial(SSZPartial, ssz_type): - pass - - if is_container_type(typ): - Partial.__annotations__ = typ.__annotations__ - o = Partial(typ, objects, root=root) - return o From 6168a90a20dcd567f27642fcb54af4b5485e1f1b Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 4 Jun 2019 18:12:23 +0200 Subject: [PATCH 46/48] speed and simplicity improvement for next_power_of_two function --- test_libs/pyspec/eth2spec/utils/merkle_minimal.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py index 420f0b5f1..3f1f130a4 100644 --- a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py @@ -41,16 +41,7 @@ def next_power_of_two(v: int) -> int: """ if v == 0: return 1 - # effectively fill the bitstring (1 less, do not want to with ones, then increment for next power of 2. - v -= 1 - v |= v >> (1 << 0) - v |= v >> (1 << 1) - v |= v >> (1 << 2) - v |= v >> (1 << 3) - v |= v >> (1 << 4) - v |= v >> (1 << 5) - v += 1 - return v + return 1 << (v-1).bit_length() def merkleize_chunks(chunks): From 4bf3a26afc3e35910cb530c33914e8140773b7af Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 4 Jun 2019 18:18:18 +0200 Subject: [PATCH 47/48] fix formatting --- test_libs/pyspec/eth2spec/utils/merkle_minimal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py index 3f1f130a4..c508f0df2 100644 --- a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py @@ -41,7 +41,7 @@ def next_power_of_two(v: int) -> int: """ if v == 0: return 1 - return 1 << (v-1).bit_length() + return 1 << (v - 1).bit_length() def merkleize_chunks(chunks): From 9bdb18245e23ea6af565c3cde6bf12b826adb144 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 4 Jun 2019 18:22:42 +0200 Subject: [PATCH 48/48] remove tautological type definition --- specs/core/0_beacon-chain.md | 1 - 1 file changed, 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 06c1c302d..4940d35f4 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -574,7 +574,6 @@ We define the following Python custom types for type hinting and readability: | `Shard` | `uint64` | a shard number | | `ValidatorIndex` | `uint64` | a validator registry index | | `Gwei` | `uint64` | an amount in Gwei | -| `Bytes32` | `Bytes32` | 32 bytes of binary data | | `BLSPubkey` | `Bytes48` | a BLS12-381 public key | | `BLSSignature` | `Bytes96` | a BLS12-381 signature |