From 828dd455ba158aaf187c2805a9561d90bbd0b20e Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 Mar 2019 10:18:57 -0600 Subject: [PATCH 01/11] add basic dependencies and build script for phase0 testing --- .gitignore | 5 + Makefile | 15 +++ requirements.txt | 6 + scripts/__init__.py | 0 scripts/phase0/__init__.py | 0 scripts/phase0/bls_stub.py | 12 ++ scripts/phase0/build_spec.py | 43 +++++++ scripts/phase0/function_puller.py | 46 +++++++ scripts/phase0/minimal_ssz.py | 190 +++++++++++++++++++++++++++++ scripts/phase0/monkey_patches.py | 29 +++++ scripts/phase0/state_transition.py | 84 +++++++++++++ tests/phase0/conftest.py | 6 + 12 files changed, 436 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 requirements.txt create mode 100644 scripts/__init__.py create mode 100644 scripts/phase0/__init__.py create mode 100644 scripts/phase0/bls_stub.py create mode 100644 scripts/phase0/build_spec.py create mode 100644 scripts/phase0/function_puller.py create mode 100644 scripts/phase0/minimal_ssz.py create mode 100644 scripts/phase0/monkey_patches.py create mode 100644 scripts/phase0/state_transition.py create mode 100644 tests/phase0/conftest.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..36c14f343 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*.pyc +/__pycache__ +/venv + +/build \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..724a0392e --- /dev/null +++ b/Makefile @@ -0,0 +1,15 @@ +SPEC_DIR = ./specs +SCRIPT_DIR = ./scripts +BUILD_DIR = ./build + +.PHONY: clean all + + +clean: + rm -rf $(BUILD_DIR) + + +$(BUILD_DIR)/phase0: + mkdir -p $@ + python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $(SCRIPT_DIR)/phase0/minimal_ssz.py \ + $(SCRIPT_DIR)/phase0/bls_stub.py $(SCRIPT_DIR)/phase0/state_transition.py $(SCRIPT_DIR)/phase0/monkey_patches.py > $@/spec.py diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..9145e951e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +eth-utils>=1.3.0,<2 +eth-typing>=2.1.0,<3.0.0 +oyaml==0.7 +pycryptodome==3.7.3 +py_ecc>=1.6.0 +pytest>=3.6,<3.7 diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/phase0/__init__.py b/scripts/phase0/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/phase0/bls_stub.py b/scripts/phase0/bls_stub.py new file mode 100644 index 000000000..7e3a6a308 --- /dev/null +++ b/scripts/phase0/bls_stub.py @@ -0,0 +1,12 @@ + + +def bls_verify(pubkey, message_hash, signature, domain): + return True + + +def bls_verify_multiple(pubkeys, message_hashes, signature, domain): + return True + + +def bls_aggregate_pubkeys(pubkeys): + return b'\x42'*96 diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py new file mode 100644 index 000000000..c4f8ab38c --- /dev/null +++ b/scripts/phase0/build_spec.py @@ -0,0 +1,43 @@ +import sys +import function_puller + +code_lines = [] + +for i in (1, 2, 3, 4, 8, 32, 48, 96): + code_lines.append("def int_to_bytes%d(x): return x.to_bytes(%d, 'little')" % (i, i)) +code_lines.append("SLOTS_PER_EPOCH = 64") # stub, will get overwritten by real var +code_lines.append("def slot_to_epoch(x): return x // SLOTS_PER_EPOCH") + +code_lines.append(""" +from typing import ( + Any, + Callable, + List, + NewType, + Tuple, +) + + +Slot = NewType('Slot', int) # uint64 +Epoch = NewType('Epoch', int) # uint64 +Shard = NewType('Shard', int) # uint64 +ValidatorIndex = NewType('ValidatorIndex', int) # uint64 +Gwei = NewType('Gwei', int) # uint64 +Bytes32 = NewType('Bytes32', bytes) # bytes32 +BLSPubkey = NewType('BLSPubkey', bytes) # bytes48 +BLSSignature = NewType('BLSSignature', bytes) # bytes96 +Any = None +Store = None +""") + + +code_lines += function_puller.get_lines(sys.argv[1]) + +print(open(sys.argv[2]).read()) +print(open(sys.argv[3]).read()) + +for line in code_lines: + print(line) + +print(open(sys.argv[4]).read()) +print(open(sys.argv[5]).read()) diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py new file mode 100644 index 000000000..8d1c1a0cc --- /dev/null +++ b/scripts/phase0/function_puller.py @@ -0,0 +1,46 @@ +import sys + + +def get_lines(file_name): + code_lines = [] + pulling_from = None + current_name = None + processing_typedef = False + for linenum, line in enumerate(open(sys.argv[1]).readlines()): + line = line.rstrip() + if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`': + current_name = line[line[:-1].rfind('`')+1: -1] + if line[:9] == '```python': + assert pulling_from is None + pulling_from = linenum + 1 + elif line[:3] == '```': + if pulling_from is None: + pulling_from = linenum + else: + if processing_typedef: + assert code_lines[-1] == '}' + code_lines[-1] = '})' + pulling_from = None + processing_typedef = False + else: + if pulling_from == linenum and line == '{': + code_lines.append('%s = SSZType({' % current_name) + processing_typedef = True + elif pulling_from is not None: + code_lines.append(line) + elif pulling_from is None and len(line) > 0 and line[0] == '|': + row = line[1:].split('|') + if len(row) >= 2: + for i in range(2): + row[i] = row[i].strip().strip('`') + if '`' in row[i]: + row[i] = row[i][:row[i].find('`')] + eligible = True + if row[0][0] not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_': + eligible = False + for c in row[0]: + if c not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789': + eligible = False + if eligible: + code_lines.append(row[0] + ' = ' + (row[1].replace('**TBD**', '0x1234567890123567890123456789012357890'))) + return code_lines diff --git a/scripts/phase0/minimal_ssz.py b/scripts/phase0/minimal_ssz.py new file mode 100644 index 000000000..5caaf8f09 --- /dev/null +++ b/scripts/phase0/minimal_ssz.py @@ -0,0 +1,190 @@ +from utils.hash import hash + + +BYTES_PER_CHUNK = 32 +BYTES_PER_LENGTH_PREFIX = 4 +ZERO_CHUNK = b'\x00' * BYTES_PER_CHUNK + +def SSZType(fields): + class SSZObject(): + def __init__(self, **kwargs): + for f in fields: + if f not in kwargs: + raise Exception("Missing constructor argument: %s" % f) + setattr(self, f, kwargs[f]) + + def __eq__(self, other): + return ( + self.fields == other.fields and + self.serialize() == other.serialize() + ) + + def __hash__(self): + return int.from_bytes(self.hash_tree_root(), byteorder="little") + + def __str__(self): + output = [] + for field in self.fields: + output.append(f'{field}: {getattr(self, field)}') + return "\n".join(output) + + def serialize(self): + return serialize_value(self, self.__class__) + + def hash_tree_root(self): + return hash_tree_root(self, self.__class__) + + SSZObject.fields = fields + return SSZObject + +class Vector(list): + def __init__(self, x): + list.__init__(self, x) + self.length = len(x) + + def append(*args): + raise Exception("Cannot change the length of a vector") + + remove = clear = extend = pop = insert = append + +def is_basic(typ): + return isinstance(typ, str) and (typ[:4] in ('uint', 'bool') or typ == 'byte') + +def is_constant_sized(typ): + if is_basic(typ): + return True + elif isinstance(typ, list) and len(typ) == 1: + return is_constant_sized(typ[0]) + elif isinstance(typ, list) and len(typ) == 2: + return False + elif isinstance(typ, str) and typ[:5] == 'bytes': + return len(typ) > 5 + elif hasattr(typ, 'fields'): + for subtype in typ.fields.values(): + if not is_constant_sized(subtype): + return False + return True + else: + raise Exception("Type not recognized") + +def coerce_to_bytes(x): + if isinstance(x, str): + o = x.encode('utf-8') + assert len(o) == len(x) + return o + elif isinstance(x, bytes): + return x + else: + raise Exception("Expecting bytes") + +def serialize_value(value, typ=None): + if typ is None: + typ = infer_type(value) + if isinstance(typ, str) and typ[:4] == 'uint': + length = int(typ[4:]) + assert length in (8, 16, 32, 64, 128, 256) + return value.to_bytes(length // 8, 'little') + elif typ == 'bool': + assert value in (True, False) + return b'\x01' if value is True else b'\x00' + elif (isinstance(typ, list) and len(typ) == 1) or typ == 'bytes': + serialized_bytes = coerce_to_bytes(value) if typ == 'bytes' else b''.join([serialize_value(element, typ[0]) for element in value]) + assert len(serialized_bytes) < 2**(8 * BYTES_PER_LENGTH_PREFIX) + serialized_length = len(serialized_bytes).to_bytes(BYTES_PER_LENGTH_PREFIX, 'little') + return serialized_length + serialized_bytes + elif isinstance(typ, list) and len(typ) == 2: + assert len(value) == typ[1] + return b''.join([serialize_value(element, typ[0]) for element in value]) + elif isinstance(typ, str) and len(typ) > 5 and typ[:5] == 'bytes': + assert len(value) == int(typ[5:]), (value, int(typ[5:])) + return coerce_to_bytes(value) + elif hasattr(typ, 'fields'): + serialized_bytes = b''.join([serialize_value(getattr(value, field), subtype) for field, subtype in typ.fields.items()]) + if is_constant_sized(typ): + return serialized_bytes + else: + assert len(serialized_bytes) < 2**(8 * BYTES_PER_LENGTH_PREFIX) + serialized_length = len(serialized_bytes).to_bytes(BYTES_PER_LENGTH_PREFIX, 'little') + return serialized_length + serialized_bytes + else: + print(value, typ) + raise Exception("Type not recognized") + +def chunkify(bytez): + bytez += b'\x00' * (-len(bytez) % BYTES_PER_CHUNK) + return [bytez[i:i+32] for i in range(0, len(bytez), 32)] + +def pack(values, subtype): + return chunkify(b''.join([serialize_value(value, subtype) for value in values])) + +def is_power_of_two(x): + return x > 0 and x & (x-1) == 0 + +def merkleize(chunks): + tree = chunks[::] + while not is_power_of_two(len(tree)): + tree.append(ZERO_CHUNK) + tree = [ZERO_CHUNK] * len(tree) + tree + for i in range(len(tree)//2-1, 0, -1): + tree[i] = hash(tree[i*2] + tree[i*2+1]) + return tree[1] + +def mix_in_length(root, length): + return hash(root + length.to_bytes(32, 'little')) + +def infer_type(value): + if hasattr(value.__class__, 'fields'): + return value.__class__ + elif isinstance(value, Vector): + return [infer_type(value[0]) if len(value) > 0 else 'uint64', len(value)] + elif isinstance(value, list): + return [infer_type(value[0])] if len(value) > 0 else ['uint64'] + elif isinstance(value, (bytes, str)): + return 'bytes' + elif isinstance(value, int): + return 'uint64' + else: + raise Exception("Failed to infer type") + +def hash_tree_root(value, typ=None): + if typ is None: + typ = infer_type(value) + if is_basic(typ): + return merkleize(pack([value], typ)) + elif isinstance(typ, list) and len(typ) == 1 and is_basic(typ[0]): + return mix_in_length(merkleize(pack(value, typ[0])), len(value)) + elif isinstance(typ, list) and len(typ) == 1 and not is_basic(typ[0]): + return mix_in_length(merkleize([hash_tree_root(element, typ[0]) for element in value]), len(value)) + elif isinstance(typ, list) and len(typ) == 2 and is_basic(typ[0]): + assert len(value) == typ[1] + return merkleize(pack(value, typ[0])) + elif typ == 'bytes': + return mix_in_length(merkleize(chunkify(coerce_to_bytes(value))), len(value)) + elif isinstance(typ, str) and typ[:5] == 'bytes' and len(typ) > 5: + assert len(value) == int(typ[5:]) + return merkleize(chunkify(coerce_to_bytes(value))) + elif isinstance(typ, list) and len(typ) == 2 and not is_basic(typ[0]): + return merkleize([hash_tree_root(element, typ[0]) for element in value]) + elif hasattr(typ, 'fields'): + return merkleize([hash_tree_root(getattr(value, field), subtype) for field, subtype in typ.fields.items()]) + else: + raise Exception("Type not recognized") + +def truncate(container): + field_keys = list(container.fields.keys()) + truncated_fields = { + key: container.fields[key] + for key in field_keys[:-1] + } + truncated_class = SSZType(truncated_fields) + kwargs = { + field: getattr(container, field) + for field in field_keys[:-1] + } + return truncated_class(**kwargs) + +def signed_root(container): + return hash_tree_root(truncate(container)) + +def serialize(ssz_object): + return getattr(ssz_object, 'serialize')() diff --git a/scripts/phase0/monkey_patches.py b/scripts/phase0/monkey_patches.py new file mode 100644 index 000000000..8a35b8f27 --- /dev/null +++ b/scripts/phase0/monkey_patches.py @@ -0,0 +1,29 @@ +# Monkey patch validator shuffling cache +_get_shuffling = get_shuffling +shuffling_cache = {} +def get_shuffling(seed: Bytes32, + validators: List[Validator], + epoch: Epoch) -> List[List[ValidatorIndex]]: + + param_hash = (seed, hash_tree_root(validators, [Validator]), epoch) + + if param_hash in shuffling_cache: + # print("Cache hit, epoch={0}".format(epoch)) + return shuffling_cache[param_hash] + else: + # print("Cache miss, epoch={0}".format(epoch)) + ret = _get_shuffling(seed, validators, epoch) + shuffling_cache[param_hash] = ret + return ret + + +# Monkey patch hash cache +_hash = hash +hash_cache = {} +def hash(x): + if x in hash_cache: + return hash_cache[x] + else: + ret = _hash(x) + hash_cache[x] = ret + return ret diff --git a/scripts/phase0/state_transition.py b/scripts/phase0/state_transition.py new file mode 100644 index 000000000..f78119cf2 --- /dev/null +++ b/scripts/phase0/state_transition.py @@ -0,0 +1,84 @@ + + +def process_transaction_type(state: BeaconState, + transactions: List[Any], + max_transactions: int, + tx_fn: Callable[[BeaconState, Any], None]) -> None: + assert len(transactions) <= max_transactions + for transaction in transactions: + tx_fn(state, transaction) + + +def process_transactions(state: BeaconState, block: BeaconBlock) -> None: + process_transaction_type( + state, + block.body.proposer_slashings, + MAX_PROPOSER_SLASHINGS, + process_proposer_slashing, + ) + process_transaction_type( + state, + block.body.attester_slashings, + MAX_ATTESTER_SLASHINGS, + process_attester_slashing, + ) + process_transaction_type( + state, + block.body.attestations, + MAX_ATTESTATIONS, + process_attestation, + ) + process_transaction_type( + state, + block.body.deposits, + MAX_DEPOSITS, + process_deposit, + ) + process_transaction_type( + state, + block.body.voluntary_exits, + MAX_VOLUNTARY_EXITS, + process_voluntary_exit, + ) + assert len(block.body.transfers) == len(set(block.body.transfers)) + process_transaction_type( + state, + block.body.transfers, + MAX_TRANSFERS, + process_transfer, + ) + + +def process_block(state: BeaconState, + block: BeaconBlock, + verify_state_root: bool=False) -> None: + process_block_header(state, block) + process_randao(state, block) + process_eth1_data(state, block) + process_transactions(state, block) + if verify_state_root: + verify_block_state_root(state, block) + + +def process_epoch_transition(state: BeaconState) -> None: + update_justification_and_finalization(state) + process_crosslinks(state) + maybe_reset_eth1_period(state) + apply_rewards(state) + process_ejections(state) + update_registry_and_shuffling_data(state) + process_slashings(state) + process_exit_queue(state) + finish_epoch_update(state) + + +def state_transition(state: BeaconState, + block: BeaconBlock, + verify_state_root: bool=False) -> BeaconState: + while state.slot < block.slot: + cache_state(state) + if (state.slot + 1) % SLOTS_PER_EPOCH == 0: + process_epoch_transition(state) + advance_slot(state) + if block.slot == state.slot: + process_block(state, block) diff --git a/tests/phase0/conftest.py b/tests/phase0/conftest.py new file mode 100644 index 000000000..d3ebabaa2 --- /dev/null +++ b/tests/phase0/conftest.py @@ -0,0 +1,6 @@ +import pytest +from build.phase0 import spec + + +# @pytest.fixture(autouse=True) +# def build_clean(): \ No newline at end of file From 839590b5f41f4b07a755ef00f7a96563858b3e7a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 Mar 2019 12:51:52 -0600 Subject: [PATCH 02/11] initial pytests passing --- .gitignore | 3 +- Makefile | 8 +- build/__init__.py | 0 build/phase0/__init__.py | 0 build/phase0/spec.py | 1620 +++++++++++++++++ {scripts => build}/phase0/state_transition.py | 72 +- build/utils/__init__.py | 0 {scripts/phase0 => build/utils}/bls_stub.py | 0 build/utils/hash_function.py | 6 + build/utils/merkle_minimal.py | 28 + .../phase0 => build/utils}/minimal_ssz.py | 2 +- .../phase0 => build/utils}/monkey_patches.py | 0 scripts/phase0/build_spec.py | 64 +- tests/__init__.py | 0 tests/conftest.py | 0 tests/phase0/conftest.py | 132 +- tests/phase0/test_sanity.py | 632 +++++++ 17 files changed, 2514 insertions(+), 53 deletions(-) create mode 100644 build/__init__.py create mode 100644 build/phase0/__init__.py create mode 100644 build/phase0/spec.py rename {scripts => build}/phase0/state_transition.py (56%) create mode 100644 build/utils/__init__.py rename {scripts/phase0 => build/utils}/bls_stub.py (100%) create mode 100644 build/utils/hash_function.py create mode 100644 build/utils/merkle_minimal.py rename {scripts/phase0 => build/utils}/minimal_ssz.py (99%) rename {scripts/phase0 => build/utils}/monkey_patches.py (100%) create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/phase0/test_sanity.py diff --git a/.gitignore b/.gitignore index 36c14f343..5e19cd2a5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ *.pyc /__pycache__ /venv - -/build \ No newline at end of file +/.pytest_cache diff --git a/Makefile b/Makefile index 724a0392e..745f8f901 100644 --- a/Makefile +++ b/Makefile @@ -5,11 +5,7 @@ BUILD_DIR = ./build .PHONY: clean all -clean: - rm -rf $(BUILD_DIR) - - $(BUILD_DIR)/phase0: mkdir -p $@ - python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $(SCRIPT_DIR)/phase0/minimal_ssz.py \ - $(SCRIPT_DIR)/phase0/bls_stub.py $(SCRIPT_DIR)/phase0/state_transition.py $(SCRIPT_DIR)/phase0/monkey_patches.py > $@/spec.py + python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $@/spec.py + touch $(BUILD_DIR)/__init__.py $(BUILD_DIR)/phase0/__init__.py diff --git a/build/__init__.py b/build/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/build/phase0/__init__.py b/build/phase0/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/build/phase0/spec.py b/build/phase0/spec.py new file mode 100644 index 000000000..8c05b1208 --- /dev/null +++ b/build/phase0/spec.py @@ -0,0 +1,1620 @@ +from build.utils.minimal_ssz import * +from build.utils.bls_stub import * +def int_to_bytes1(x): return x.to_bytes(1, 'little') +def int_to_bytes2(x): return x.to_bytes(2, 'little') +def int_to_bytes3(x): return x.to_bytes(3, 'little') +def int_to_bytes4(x): return x.to_bytes(4, 'little') +def int_to_bytes8(x): return x.to_bytes(8, 'little') +def int_to_bytes32(x): return x.to_bytes(32, 'little') +def int_to_bytes48(x): return x.to_bytes(48, 'little') +def int_to_bytes96(x): return x.to_bytes(96, 'little') +SLOTS_PER_EPOCH = 64 +def slot_to_epoch(x): return x // SLOTS_PER_EPOCH + +from typing import ( + Any, + Callable, + List, + NewType, + Tuple, +) + + +Slot = NewType('Slot', int) # uint64 +Epoch = NewType('Epoch', int) # uint64 +Shard = NewType('Shard', int) # uint64 +ValidatorIndex = NewType('ValidatorIndex', int) # uint64 +Gwei = NewType('Gwei', int) # uint64 +Bytes32 = NewType('Bytes32', bytes) # bytes32 +BLSPubkey = NewType('BLSPubkey', bytes) # bytes48 +BLSSignature = NewType('BLSSignature', bytes) # bytes96 +Any = None +Store = None + +SHARD_COUNT = 2**10 +TARGET_COMMITTEE_SIZE = 2**7 +MAX_BALANCE_CHURN_QUOTIENT = 2**5 +MAX_INDICES_PER_SLASHABLE_VOTE = 2**12 +MAX_EXIT_DEQUEUES_PER_EPOCH = 2**2 +SHUFFLE_ROUND_COUNT = 90 +DEPOSIT_CONTRACT_ADDRESS = 0x1234567890123567890123456789012357890 +DEPOSIT_CONTRACT_TREE_DEPTH = 2**5 +MIN_DEPOSIT_AMOUNT = 2**0 * 10**9 +MAX_DEPOSIT_AMOUNT = 2**5 * 10**9 +FORK_CHOICE_BALANCE_INCREMENT = 2**0 * 10**9 +EJECTION_BALANCE = 2**4 * 10**9 +GENESIS_FORK_VERSION = 0 +GENESIS_SLOT = 2**32 +GENESIS_EPOCH = slot_to_epoch(GENESIS_SLOT) +GENESIS_START_SHARD = 0 +FAR_FUTURE_EPOCH = 2**64 - 1 +ZERO_HASH = int_to_bytes32(0) +EMPTY_SIGNATURE = int_to_bytes96(0) +BLS_WITHDRAWAL_PREFIX_BYTE = int_to_bytes1(0) +SECONDS_PER_SLOT = 6 +MIN_ATTESTATION_INCLUSION_DELAY = 2**2 +SLOTS_PER_EPOCH = 2**6 +MIN_SEED_LOOKAHEAD = 2**0 +ACTIVATION_EXIT_DELAY = 2**2 +EPOCHS_PER_ETH1_VOTING_PERIOD = 2**4 +SLOTS_PER_HISTORICAL_ROOT = 2**13 +MIN_VALIDATOR_WITHDRAWABILITY_DELAY = 2**8 +PERSISTENT_COMMITTEE_PERIOD = 2**11 +LATEST_RANDAO_MIXES_LENGTH = 2**13 +LATEST_ACTIVE_INDEX_ROOTS_LENGTH = 2**13 +LATEST_SLASHED_EXIT_LENGTH = 2**13 +BASE_REWARD_QUOTIENT = 2**5 +WHISTLEBLOWER_REWARD_QUOTIENT = 2**9 +ATTESTATION_INCLUSION_REWARD_QUOTIENT = 2**3 +INACTIVITY_PENALTY_QUOTIENT = 2**24 +MIN_PENALTY_QUOTIENT = 2**5 +MAX_PROPOSER_SLASHINGS = 2**4 +MAX_ATTESTER_SLASHINGS = 2**0 +MAX_ATTESTATIONS = 2**7 +MAX_DEPOSITS = 2**4 +MAX_VOLUNTARY_EXITS = 2**4 +MAX_TRANSFERS = 2**4 +DOMAIN_BEACON_BLOCK = 0 +DOMAIN_RANDAO = 1 +DOMAIN_ATTESTATION = 2 +DOMAIN_DEPOSIT = 3 +DOMAIN_VOLUNTARY_EXIT = 4 +DOMAIN_TRANSFER = 5 +Fork = SSZType({ + # Previous fork version + 'previous_version': 'bytes4', + # Current fork version + 'current_version': 'bytes4', + # Fork epoch number + 'epoch': 'uint64', +}) +Crosslink = SSZType({ + # Epoch number + 'epoch': 'uint64', + # Shard data since the previous crosslink + 'crosslink_data_root': 'bytes32', +}) +Eth1Data = SSZType({ + # Root of the deposit tree + 'deposit_root': 'bytes32', + # Block hash + 'block_hash': 'bytes32', +}) +Eth1DataVote = SSZType({ + # Data being voted for + 'eth1_data': Eth1Data, + # Vote count + 'vote_count': 'uint64', +}) +AttestationData = SSZType({ + # LMD GHOST vote + 'slot': 'uint64', + 'beacon_block_root': 'bytes32', + + # FFG vote + 'source_epoch': 'uint64', + 'source_root': 'bytes32', + 'target_root': 'bytes32', + + # Crosslink vote + 'shard': 'uint64', + 'previous_crosslink': Crosslink, + 'crosslink_data_root': 'bytes32', +}) +AttestationDataAndCustodyBit = SSZType({ + # Attestation data + 'data': AttestationData, + # Custody bit + 'custody_bit': 'bool', +}) +SlashableAttestation = SSZType({ + # Validator indices + 'validator_indices': ['uint64'], + # Attestation data + 'data': AttestationData, + # Custody bitfield + 'custody_bitfield': 'bytes', + # Aggregate signature + 'aggregate_signature': 'bytes96', +}) +DepositInput = SSZType({ + # BLS pubkey + 'pubkey': 'bytes48', + # Withdrawal credentials + 'withdrawal_credentials': 'bytes32', + # A BLS signature of this `DepositInput` + 'proof_of_possession': 'bytes96', +}) +DepositData = SSZType({ + # Amount in Gwei + 'amount': 'uint64', + # Timestamp from deposit contract + 'timestamp': 'uint64', + # Deposit input + 'deposit_input': DepositInput, +}) +BeaconBlockHeader = SSZType({ + 'slot': 'uint64', + 'previous_block_root': 'bytes32', + 'state_root': 'bytes32', + 'block_body_root': 'bytes32', + 'signature': 'bytes96', +}) +Validator = SSZType({ + # BLS public key + 'pubkey': 'bytes48', + # Withdrawal credentials + 'withdrawal_credentials': 'bytes32', + # Epoch when validator activated + 'activation_epoch': 'uint64', + # Epoch when validator exited + 'exit_epoch': 'uint64', + # Epoch when validator is eligible to withdraw + 'withdrawable_epoch': 'uint64', + # Did the validator initiate an exit + 'initiated_exit': 'bool', + # Was the validator slashed + 'slashed': 'bool', +}) +PendingAttestation = SSZType({ + # Attester aggregation bitfield + 'aggregation_bitfield': 'bytes', + # Attestation data + 'data': AttestationData, + # Custody bitfield + 'custody_bitfield': 'bytes', + # Inclusion slot + 'inclusion_slot': 'uint64', +}) +HistoricalBatch = SSZType({ + # Block roots + 'block_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], + # State roots + 'state_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], +}) +ProposerSlashing = SSZType({ + # Proposer index + 'proposer_index': 'uint64', + # First block header + 'header_1': BeaconBlockHeader, + # Second block header + 'header_2': BeaconBlockHeader, +}) +AttesterSlashing = SSZType({ + # First slashable attestation + 'slashable_attestation_1': SlashableAttestation, + # Second slashable attestation + 'slashable_attestation_2': SlashableAttestation, +}) +Attestation = SSZType({ + # Attester aggregation bitfield + 'aggregation_bitfield': 'bytes', + # Attestation data + 'data': AttestationData, + # Custody bitfield + 'custody_bitfield': 'bytes', + # BLS aggregate signature + 'aggregate_signature': 'bytes96', +}) +Deposit = SSZType({ + # Branch in the deposit tree + 'proof': ['bytes32', DEPOSIT_CONTRACT_TREE_DEPTH], + # Index in the deposit tree + 'index': 'uint64', + # Data + 'deposit_data': DepositData, +}) +VoluntaryExit = SSZType({ + # Minimum epoch for processing exit + 'epoch': 'uint64', + # Index of the exiting validator + 'validator_index': 'uint64', + # Validator signature + 'signature': 'bytes96', +}) +Transfer = SSZType({ + # Sender index + 'sender': 'uint64', + # Recipient index + 'recipient': 'uint64', + # Amount in Gwei + 'amount': 'uint64', + # Fee in Gwei for block proposer + 'fee': 'uint64', + # Inclusion slot + 'slot': 'uint64', + # Sender withdrawal pubkey + 'pubkey': 'bytes48', + # Sender signature + 'signature': 'bytes96', +}) +BeaconBlockBody = SSZType({ + 'randao_reveal': 'bytes96', + 'eth1_data': Eth1Data, + 'proposer_slashings': [ProposerSlashing], + 'attester_slashings': [AttesterSlashing], + 'attestations': [Attestation], + 'deposits': [Deposit], + 'voluntary_exits': [VoluntaryExit], + 'transfers': [Transfer], +}) +BeaconBlock = SSZType({ + # Header + 'slot': 'uint64', + 'previous_block_root': 'bytes32', + 'state_root': 'bytes32', + 'body': BeaconBlockBody, + 'signature': 'bytes96', +}) +BeaconState = SSZType({ + # Misc + 'slot': 'uint64', + 'genesis_time': 'uint64', + 'fork': Fork, # For versioning hard forks + + # Validator registry + 'validator_registry': [Validator], + 'validator_balances': ['uint64'], + 'validator_registry_update_epoch': 'uint64', + + # Randomness and committees + 'latest_randao_mixes': ['bytes32', LATEST_RANDAO_MIXES_LENGTH], + 'previous_shuffling_start_shard': 'uint64', + 'current_shuffling_start_shard': 'uint64', + 'previous_shuffling_epoch': 'uint64', + 'current_shuffling_epoch': 'uint64', + 'previous_shuffling_seed': 'bytes32', + 'current_shuffling_seed': 'bytes32', + + # Finality + 'previous_epoch_attestations': [PendingAttestation], + 'current_epoch_attestations': [PendingAttestation], + 'previous_justified_epoch': 'uint64', + 'current_justified_epoch': 'uint64', + 'previous_justified_root': 'bytes32', + 'current_justified_root': 'bytes32', + 'justification_bitfield': 'uint64', + 'finalized_epoch': 'uint64', + 'finalized_root': 'bytes32', + + # Recent state + 'latest_crosslinks': [Crosslink, SHARD_COUNT], + 'latest_block_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], + 'latest_state_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], + 'latest_active_index_roots': ['bytes32', LATEST_ACTIVE_INDEX_ROOTS_LENGTH], + 'latest_slashed_balances': ['uint64', LATEST_SLASHED_EXIT_LENGTH], # Balances slashed at every withdrawal period + 'latest_block_header': BeaconBlockHeader, # `latest_block_header.state_root == ZERO_HASH` temporarily + 'historical_roots': ['bytes32'], + + # Ethereum 1.0 chain data + 'latest_eth1_data': Eth1Data, + 'eth1_data_votes': [Eth1DataVote], + 'deposit_index': 'uint64' +}) +def xor(bytes1: Bytes32, bytes2: Bytes32) -> Bytes32: + return bytes(a ^ b for a, b in zip(bytes1, bytes2)) +def get_temporary_block_header(block: BeaconBlock) -> BeaconBlockHeader: + """ + Return the block header corresponding to a block with ``state_root`` set to ``ZERO_HASH``. + """ + return BeaconBlockHeader( + slot=block.slot, + previous_block_root=block.previous_block_root, + state_root=ZERO_HASH, + block_body_root=hash_tree_root(block.body), + signature=block.signature, + ) +def slot_to_epoch(slot: Slot) -> Epoch: + """ + Return the epoch number of the given ``slot``. + """ + return slot // SLOTS_PER_EPOCH +def get_previous_epoch(state: BeaconState) -> Epoch: + """` + Return the previous epoch of the given ``state``. + """ + return get_current_epoch(state) - 1 +def get_current_epoch(state: BeaconState) -> Epoch: + """ + Return the current epoch of the given ``state``. + """ + return slot_to_epoch(state.slot) +def get_epoch_start_slot(epoch: Epoch) -> Slot: + """ + Return the starting slot of the given ``epoch``. + """ + return epoch * SLOTS_PER_EPOCH +def is_active_validator(validator: Validator, epoch: Epoch) -> bool: + """ + Check if ``validator`` is active. + """ + return validator.activation_epoch <= epoch < validator.exit_epoch +def get_active_validator_indices(validators: List[Validator], epoch: Epoch) -> List[ValidatorIndex]: + """ + Get indices of active validators from ``validators``. + """ + return [i for i, v in enumerate(validators) if is_active_validator(v, epoch)] +def get_permuted_index(index: int, list_size: int, seed: Bytes32) -> int: + """ + Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy. + + Utilizes 'swap or not' shuffling found in + https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf + See the 'generalized domain' algorithm on page 3. + """ + assert index < list_size + assert list_size <= 2**40 + + for round in range(SHUFFLE_ROUND_COUNT): + pivot = bytes_to_int(hash(seed + int_to_bytes1(round))[0:8]) % list_size + flip = (pivot - index) % list_size + position = max(index, flip) + source = hash(seed + int_to_bytes1(round) + int_to_bytes4(position // 256)) + byte = source[(position % 256) // 8] + bit = (byte >> (position % 8)) % 2 + index = flip if bit else index + + return index +def split(values: List[Any], split_count: int) -> List[List[Any]]: + """ + Splits ``values`` into ``split_count`` pieces. + """ + list_length = len(values) + return [ + values[(list_length * i // split_count): (list_length * (i + 1) // split_count)] + for i in range(split_count) + ] +def get_epoch_committee_count(active_validator_count: int) -> int: + """ + Return the number of committees in one epoch. + """ + return max( + 1, + min( + SHARD_COUNT // SLOTS_PER_EPOCH, + active_validator_count // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE, + ) + ) * SLOTS_PER_EPOCH +def get_shuffling(seed: Bytes32, + validators: List[Validator], + epoch: Epoch) -> List[List[ValidatorIndex]]: + """ + Shuffle active validators and split into crosslink committees. + Return a list of committees (each a list of validator indices). + """ + # Shuffle active validator indices + active_validator_indices = get_active_validator_indices(validators, epoch) + length = len(active_validator_indices) + shuffled_indices = [active_validator_indices[get_permuted_index(i, length, seed)] for i in range(length)] + + # Split the shuffled active validator indices + return split(shuffled_indices, get_epoch_committee_count(length)) +def get_previous_epoch_committee_count(state: BeaconState) -> int: + """ + Return the number of committees in the previous epoch of the given ``state``. + """ + previous_active_validators = get_active_validator_indices( + state.validator_registry, + state.previous_shuffling_epoch, + ) + return get_epoch_committee_count(len(previous_active_validators)) +def get_current_epoch_committee_count(state: BeaconState) -> int: + """ + Return the number of committees in the current epoch of the given ``state``. + """ + current_active_validators = get_active_validator_indices( + state.validator_registry, + state.current_shuffling_epoch, + ) + return get_epoch_committee_count(len(current_active_validators)) +def get_next_epoch_committee_count(state: BeaconState) -> int: + """ + Return the number of committees in the next epoch of the given ``state``. + """ + next_active_validators = get_active_validator_indices( + state.validator_registry, + get_current_epoch(state) + 1, + ) + return get_epoch_committee_count(len(next_active_validators)) +def get_crosslink_committees_at_slot(state: BeaconState, + slot: Slot, + registry_change: bool=False) -> List[Tuple[List[ValidatorIndex], Shard]]: + """ + Return the list of ``(committee, shard)`` tuples for the ``slot``. + + Note: There are two possible shufflings for crosslink committees for a + ``slot`` in the next epoch -- with and without a `registry_change` + """ + epoch = slot_to_epoch(slot) + current_epoch = get_current_epoch(state) + previous_epoch = get_previous_epoch(state) + next_epoch = current_epoch + 1 + + assert previous_epoch <= epoch <= next_epoch + + if epoch == current_epoch: + committees_per_epoch = get_current_epoch_committee_count(state) + seed = state.current_shuffling_seed + shuffling_epoch = state.current_shuffling_epoch + shuffling_start_shard = state.current_shuffling_start_shard + elif epoch == previous_epoch: + committees_per_epoch = get_previous_epoch_committee_count(state) + seed = state.previous_shuffling_seed + shuffling_epoch = state.previous_shuffling_epoch + shuffling_start_shard = state.previous_shuffling_start_shard + elif epoch == next_epoch: + epochs_since_last_registry_update = current_epoch - state.validator_registry_update_epoch + if registry_change: + committees_per_epoch = get_next_epoch_committee_count(state) + seed = generate_seed(state, next_epoch) + shuffling_epoch = next_epoch + current_committees_per_epoch = get_current_epoch_committee_count(state) + shuffling_start_shard = (state.current_shuffling_start_shard + current_committees_per_epoch) % SHARD_COUNT + elif epochs_since_last_registry_update > 1 and is_power_of_two(epochs_since_last_registry_update): + committees_per_epoch = get_next_epoch_committee_count(state) + seed = generate_seed(state, next_epoch) + shuffling_epoch = next_epoch + shuffling_start_shard = state.current_shuffling_start_shard + else: + committees_per_epoch = get_current_epoch_committee_count(state) + seed = state.current_shuffling_seed + shuffling_epoch = state.current_shuffling_epoch + shuffling_start_shard = state.current_shuffling_start_shard + + shuffling = get_shuffling( + seed, + state.validator_registry, + shuffling_epoch, + ) + offset = slot % SLOTS_PER_EPOCH + committees_per_slot = committees_per_epoch // SLOTS_PER_EPOCH + slot_start_shard = (shuffling_start_shard + committees_per_slot * offset) % SHARD_COUNT + + return [ + ( + shuffling[committees_per_slot * offset + i], + (slot_start_shard + i) % SHARD_COUNT, + ) + for i in range(committees_per_slot) + ] +def get_block_root(state: BeaconState, + slot: Slot) -> Bytes32: + """ + Return the block root at a recent ``slot``. + """ + assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT + return state.latest_block_roots[slot % SLOTS_PER_HISTORICAL_ROOT] +def get_state_root(state: BeaconState, + slot: Slot) -> Bytes32: + """ + Return the state root at a recent ``slot``. + """ + assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT + return state.latest_state_roots[slot % SLOTS_PER_HISTORICAL_ROOT] +def get_randao_mix(state: BeaconState, + epoch: Epoch) -> Bytes32: + """ + Return the randao mix at a recent ``epoch``. + """ + assert get_current_epoch(state) - LATEST_RANDAO_MIXES_LENGTH < epoch <= get_current_epoch(state) + return state.latest_randao_mixes[epoch % LATEST_RANDAO_MIXES_LENGTH] +def get_active_index_root(state: BeaconState, + epoch: Epoch) -> Bytes32: + """ + Return the index root at a recent ``epoch``. + """ + assert get_current_epoch(state) - LATEST_ACTIVE_INDEX_ROOTS_LENGTH + ACTIVATION_EXIT_DELAY < epoch <= get_current_epoch(state) + ACTIVATION_EXIT_DELAY + return state.latest_active_index_roots[epoch % LATEST_ACTIVE_INDEX_ROOTS_LENGTH] +def generate_seed(state: BeaconState, + epoch: Epoch) -> Bytes32: + """ + Generate a seed for the given ``epoch``. + """ + return hash( + get_randao_mix(state, epoch - MIN_SEED_LOOKAHEAD) + + get_active_index_root(state, epoch) + + int_to_bytes32(epoch) + ) +def get_beacon_proposer_index(state: BeaconState, + slot: Slot, + registry_change: bool=False) -> ValidatorIndex: + """ + Return the beacon proposer index for the ``slot``. + """ + epoch = slot_to_epoch(slot) + current_epoch = get_current_epoch(state) + previous_epoch = get_previous_epoch(state) + next_epoch = current_epoch + 1 + + assert previous_epoch <= epoch <= next_epoch + + first_committee, _ = get_crosslink_committees_at_slot(state, slot, registry_change)[0] + return first_committee[epoch % len(first_committee)] +def verify_merkle_branch(leaf: Bytes32, proof: List[Bytes32], depth: int, index: int, root: Bytes32) -> bool: + """ + Verify that the given ``leaf`` is on the merkle branch ``proof`` + starting with the given ``root``. + """ + value = leaf + for i in range(depth): + if index // (2**i) % 2: + value = hash(proof[i] + value) + else: + value = hash(value + proof[i]) + return value == root +def get_attestation_participants(state: BeaconState, + attestation_data: AttestationData, + bitfield: bytes) -> List[ValidatorIndex]: + """ + Return the participant indices at for the ``attestation_data`` and ``bitfield``. + """ + # Find the committee in the list with the desired shard + crosslink_committees = get_crosslink_committees_at_slot(state, attestation_data.slot) + + assert attestation_data.shard in [shard for _, shard in crosslink_committees] + crosslink_committee = [committee for committee, shard in crosslink_committees if shard == attestation_data.shard][0] + + assert verify_bitfield(bitfield, len(crosslink_committee)) + + # Find the participating attesters in the committee + participants = [] + for i, validator_index in enumerate(crosslink_committee): + aggregation_bit = get_bitfield_bit(bitfield, i) + if aggregation_bit == 0b1: + participants.append(validator_index) + return participants +def is_power_of_two(value: int) -> bool: + """ + Check if ``value`` is a power of two integer. + """ + return (value > 0) and (value & (value - 1) == 0) +def bytes_to_int(data: bytes) -> int: + return int.from_bytes(data, 'little') +def get_effective_balance(state: BeaconState, index: ValidatorIndex) -> Gwei: + """ + Return the effective balance (also known as "balance at stake") for a validator with the given ``index``. + """ + return min(state.validator_balances[index], MAX_DEPOSIT_AMOUNT) +def get_total_balance(state: BeaconState, validators: List[ValidatorIndex]) -> Gwei: + """ + Return the combined effective balance of an array of ``validators``. + """ + return sum([get_effective_balance(state, i) for i in validators]) +def get_fork_version(fork: Fork, + epoch: Epoch) -> bytes: + """ + Return the fork version of the given ``epoch``. + """ + if epoch < fork.epoch: + return fork.previous_version + else: + return fork.current_version +def get_domain(fork: Fork, + epoch: Epoch, + domain_type: int) -> int: + """ + Get the domain number that represents the fork meta and signature domain. + """ + return bytes_to_int(get_fork_version(fork, epoch) + int_to_bytes4(domain_type)) +def get_bitfield_bit(bitfield: bytes, i: int) -> int: + """ + Extract the bit in ``bitfield`` at position ``i``. + """ + return (bitfield[i // 8] >> (i % 8)) % 2 +def verify_bitfield(bitfield: bytes, committee_size: int) -> bool: + """ + Verify ``bitfield`` against the ``committee_size``. + """ + if len(bitfield) != (committee_size + 7) // 8: + return False + + # Check `bitfield` is padded with zero bits only + for i in range(committee_size, len(bitfield) * 8): + if get_bitfield_bit(bitfield, i) == 0b1: + return False + + return True +def verify_slashable_attestation(state: BeaconState, slashable_attestation: SlashableAttestation) -> bool: + """ + Verify validity of ``slashable_attestation`` fields. + """ + if slashable_attestation.custody_bitfield != b'\x00' * len(slashable_attestation.custody_bitfield): # [TO BE REMOVED IN PHASE 1] + return False + + if len(slashable_attestation.validator_indices) == 0: + return False + + for i in range(len(slashable_attestation.validator_indices) - 1): + if slashable_attestation.validator_indices[i] >= slashable_attestation.validator_indices[i + 1]: + return False + + if not verify_bitfield(slashable_attestation.custody_bitfield, len(slashable_attestation.validator_indices)): + return False + + if len(slashable_attestation.validator_indices) > MAX_INDICES_PER_SLASHABLE_VOTE: + return False + + custody_bit_0_indices = [] + custody_bit_1_indices = [] + for i, validator_index in enumerate(slashable_attestation.validator_indices): + if get_bitfield_bit(slashable_attestation.custody_bitfield, i) == 0b0: + custody_bit_0_indices.append(validator_index) + else: + custody_bit_1_indices.append(validator_index) + + return bls_verify_multiple( + pubkeys=[ + bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_0_indices]), + bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_1_indices]), + ], + message_hashes=[ + hash_tree_root(AttestationDataAndCustodyBit(data=slashable_attestation.data, custody_bit=0b0)), + hash_tree_root(AttestationDataAndCustodyBit(data=slashable_attestation.data, custody_bit=0b1)), + ], + signature=slashable_attestation.aggregate_signature, + domain=get_domain(state.fork, slot_to_epoch(slashable_attestation.data.slot), DOMAIN_ATTESTATION), + ) +def is_double_vote(attestation_data_1: AttestationData, + attestation_data_2: AttestationData) -> bool: + """ + Check if ``attestation_data_1`` and ``attestation_data_2`` have the same target. + """ + target_epoch_1 = slot_to_epoch(attestation_data_1.slot) + target_epoch_2 = slot_to_epoch(attestation_data_2.slot) + return target_epoch_1 == target_epoch_2 +def is_surround_vote(attestation_data_1: AttestationData, + attestation_data_2: AttestationData) -> bool: + """ + Check if ``attestation_data_1`` surrounds ``attestation_data_2``. + """ + source_epoch_1 = attestation_data_1.source_epoch + source_epoch_2 = attestation_data_2.source_epoch + target_epoch_1 = slot_to_epoch(attestation_data_1.slot) + target_epoch_2 = slot_to_epoch(attestation_data_2.slot) + + return source_epoch_1 < source_epoch_2 and target_epoch_2 < target_epoch_1 +def integer_squareroot(n: int) -> int: + """ + The largest integer ``x`` such that ``x**2`` is less than or equal to ``n``. + """ + assert n >= 0 + x = n + y = (x + 1) // 2 + while y < x: + x = y + y = (x + n // x) // 2 + return x +def get_delayed_activation_exit_epoch(epoch: Epoch) -> Epoch: + """ + Return the epoch at which an activation or exit triggered in ``epoch`` takes effect. + """ + return epoch + 1 + ACTIVATION_EXIT_DELAY +def process_deposit(state: BeaconState, deposit: Deposit) -> None: + """ + Process a deposit from Ethereum 1.0. + Note that this function mutates ``state``. + """ + deposit_input = deposit.deposit_data.deposit_input + + # Should equal 8 bytes for deposit_data.amount + + # 8 bytes for deposit_data.timestamp + + # 176 bytes for deposit_data.deposit_input + # It should match the deposit_data in the eth1.0 deposit contract + serialized_deposit_data = serialize(deposit.deposit_data) + # Deposits must be processed in order + assert deposit.index == state.deposit_index + + # Verify the Merkle branch + merkle_branch_is_valid = verify_merkle_branch( + leaf=hash(serialized_deposit_data), + proof=deposit.proof, + depth=DEPOSIT_CONTRACT_TREE_DEPTH, + index=deposit.index, + root=state.latest_eth1_data.deposit_root, + ) + assert merkle_branch_is_valid + + # Increment the next deposit index we are expecting. Note that this + # needs to be done here because while the deposit contract will never + # create an invalid Merkle branch, it may admit an invalid deposit + # object, and we need to be able to skip over it + state.deposit_index += 1 + + validator_pubkeys = [v.pubkey for v in state.validator_registry] + pubkey = deposit_input.pubkey + amount = deposit.deposit_data.amount + withdrawal_credentials = deposit_input.withdrawal_credentials + + if pubkey not in validator_pubkeys: + # Verify the proof of possession + proof_is_valid = bls_verify( + pubkey=deposit_input.pubkey, + message_hash=signed_root(deposit_input), + signature=deposit_input.proof_of_possession, + domain=get_domain( + state.fork, + get_current_epoch(state), + DOMAIN_DEPOSIT, + ) + ) + if not proof_is_valid: + return + + # Add new validator + validator = Validator( + pubkey=pubkey, + withdrawal_credentials=withdrawal_credentials, + activation_epoch=FAR_FUTURE_EPOCH, + exit_epoch=FAR_FUTURE_EPOCH, + withdrawable_epoch=FAR_FUTURE_EPOCH, + initiated_exit=False, + slashed=False, + ) + + # Note: In phase 2 registry indices that have been withdrawn for a long time will be recycled. + state.validator_registry.append(validator) + state.validator_balances.append(amount) + else: + # Increase balance by deposit amount + state.validator_balances[validator_pubkeys.index(pubkey)] += amount +def activate_validator(state: BeaconState, index: ValidatorIndex, is_genesis: bool) -> None: + """ + Activate the validator of the given ``index``. + Note that this function mutates ``state``. + """ + validator = state.validator_registry[index] + + validator.activation_epoch = GENESIS_EPOCH if is_genesis else get_delayed_activation_exit_epoch(get_current_epoch(state)) +def initiate_validator_exit(state: BeaconState, index: ValidatorIndex) -> None: + """ + Initiate the validator of the given ``index``. + Note that this function mutates ``state``. + """ + validator = state.validator_registry[index] + validator.initiated_exit = True +def exit_validator(state: BeaconState, index: ValidatorIndex) -> None: + """ + Exit the validator of the given ``index``. + Note that this function mutates ``state``. + """ + validator = state.validator_registry[index] + delayed_activation_exit_epoch = get_delayed_activation_exit_epoch(get_current_epoch(state)) + + # The following updates only occur if not previous exited + if validator.exit_epoch <= delayed_activation_exit_epoch: + return + else: + validator.exit_epoch = delayed_activation_exit_epoch +def slash_validator(state: BeaconState, index: ValidatorIndex) -> None: + """ + Slash the validator with index ``index``. + Note that this function mutates ``state``. + """ + validator = state.validator_registry[index] + assert state.slot < get_epoch_start_slot(validator.withdrawable_epoch) # [TO BE REMOVED IN PHASE 2] + exit_validator(state, index) + state.latest_slashed_balances[get_current_epoch(state) % LATEST_SLASHED_EXIT_LENGTH] += get_effective_balance(state, index) + + whistleblower_index = get_beacon_proposer_index(state, state.slot) + whistleblower_reward = get_effective_balance(state, index) // WHISTLEBLOWER_REWARD_QUOTIENT + state.validator_balances[whistleblower_index] += whistleblower_reward + state.validator_balances[index] -= whistleblower_reward + validator.slashed = True + validator.withdrawable_epoch = get_current_epoch(state) + LATEST_SLASHED_EXIT_LENGTH +def prepare_validator_for_withdrawal(state: BeaconState, index: ValidatorIndex) -> None: + """ + Set the validator with the given ``index`` as withdrawable + ``MIN_VALIDATOR_WITHDRAWABILITY_DELAY`` after the current epoch. + Note that this function mutates ``state``. + """ + validator = state.validator_registry[index] + validator.withdrawable_epoch = get_current_epoch(state) + MIN_VALIDATOR_WITHDRAWABILITY_DELAY +def get_empty_block() -> BeaconBlock: + """ + Get an empty ``BeaconBlock``. + """ + return BeaconBlock( + slot=GENESIS_SLOT, + previous_block_root=ZERO_HASH, + state_root=ZERO_HASH, + body=BeaconBlockBody( + randao_reveal=EMPTY_SIGNATURE, + eth1_data=Eth1Data( + deposit_root=ZERO_HASH, + block_hash=ZERO_HASH, + ), + proposer_slashings=[], + attester_slashings=[], + attestations=[], + deposits=[], + voluntary_exits=[], + transfers=[], + ), + signature=EMPTY_SIGNATURE, + ) +def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], + genesis_time: int, + genesis_eth1_data: Eth1Data) -> BeaconState: + """ + Get the genesis ``BeaconState``. + """ + state = BeaconState( + # Misc + slot=GENESIS_SLOT, + genesis_time=genesis_time, + fork=Fork( + previous_version=int_to_bytes4(GENESIS_FORK_VERSION), + current_version=int_to_bytes4(GENESIS_FORK_VERSION), + epoch=GENESIS_EPOCH, + ), + + # Validator registry + validator_registry=[], + validator_balances=[], + validator_registry_update_epoch=GENESIS_EPOCH, + + # Randomness and committees + latest_randao_mixes=[ZERO_HASH for _ in range(LATEST_RANDAO_MIXES_LENGTH)], + previous_shuffling_start_shard=GENESIS_START_SHARD, + current_shuffling_start_shard=GENESIS_START_SHARD, + previous_shuffling_epoch=GENESIS_EPOCH, + current_shuffling_epoch=GENESIS_EPOCH, + previous_shuffling_seed=ZERO_HASH, + current_shuffling_seed=ZERO_HASH, + + # Finality + previous_epoch_attestations=[], + current_epoch_attestations=[], + previous_justified_epoch=GENESIS_EPOCH, + current_justified_epoch=GENESIS_EPOCH, + previous_justified_root=ZERO_HASH, + current_justified_root=ZERO_HASH, + justification_bitfield=0, + finalized_epoch=GENESIS_EPOCH, + finalized_root=ZERO_HASH, + + # Recent state + latest_crosslinks=[Crosslink(epoch=GENESIS_EPOCH, crosslink_data_root=ZERO_HASH) for _ in range(SHARD_COUNT)], + latest_block_roots=[ZERO_HASH for _ in range(SLOTS_PER_HISTORICAL_ROOT)], + latest_state_roots=[ZERO_HASH for _ in range(SLOTS_PER_HISTORICAL_ROOT)], + latest_active_index_roots=[ZERO_HASH for _ in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH)], + latest_slashed_balances=[0 for _ in range(LATEST_SLASHED_EXIT_LENGTH)], + latest_block_header=get_temporary_block_header(get_empty_block()), + historical_roots=[], + + # Ethereum 1.0 chain data + latest_eth1_data=genesis_eth1_data, + eth1_data_votes=[], + deposit_index=0, + ) + + # Process genesis deposits + for deposit in genesis_validator_deposits: + process_deposit(state, deposit) + + # Process genesis activations + for validator_index, _ in enumerate(state.validator_registry): + if get_effective_balance(state, validator_index) >= MAX_DEPOSIT_AMOUNT: + activate_validator(state, validator_index, is_genesis=True) + + genesis_active_index_root = hash_tree_root(get_active_validator_indices(state.validator_registry, GENESIS_EPOCH)) + for index in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH): + state.latest_active_index_roots[index] = genesis_active_index_root + state.current_shuffling_seed = generate_seed(state, GENESIS_EPOCH) + + return state +def get_ancestor(store: Store, block: BeaconBlock, slot: Slot) -> BeaconBlock: + """ + Get the ancestor of ``block`` with slot number ``slot``; return ``None`` if not found. + """ + if block.slot == slot: + return block + elif block.slot < slot: + return None + else: + return get_ancestor(store, store.get_parent(block), slot) +def lmd_ghost(store: Store, start_state: BeaconState, start_block: BeaconBlock) -> BeaconBlock: + """ + Execute the LMD-GHOST algorithm to find the head ``BeaconBlock``. + """ + validators = start_state.validator_registry + active_validator_indices = get_active_validator_indices(validators, slot_to_epoch(start_state.slot)) + attestation_targets = [ + (validator_index, get_latest_attestation_target(store, validator_index)) + for validator_index in active_validator_indices + ] + + def get_vote_count(block: BeaconBlock) -> int: + return sum( + get_effective_balance(start_state.validator_balances[validator_index]) // FORK_CHOICE_BALANCE_INCREMENT + for validator_index, target in attestation_targets + if get_ancestor(store, target, block.slot) == block + ) + + head = start_block + while 1: + children = get_children(store, head) + if len(children) == 0: + return head + head = max(children, key=lambda x: (get_vote_count(x), hash_tree_root(x))) +def cache_state(state: BeaconState) -> None: + previous_slot_state_root = hash_tree_root(state) + + # store the previous slot's post state transition root + state.latest_state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_slot_state_root + + # cache state root in stored latest_block_header if empty + if state.latest_block_header.state_root == ZERO_HASH: + state.latest_block_header.state_root = previous_slot_state_root + + # store latest known block for previous slot + state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = hash_tree_root(state.latest_block_header) +def get_current_total_balance(state: BeaconState) -> Gwei: + return get_total_balance(state, get_active_validator_indices(state.validator_registry, get_current_epoch(state))) +def get_previous_total_balance(state: BeaconState) -> Gwei: + return get_total_balance(state, get_active_validator_indices(state.validator_registry, get_previous_epoch(state))) +def get_attesting_indices(state: BeaconState, attestations: List[PendingAttestation]) -> List[ValidatorIndex]: + output = set() + for a in attestations: + output = output.union(get_attestation_participants(state, a.data, a.aggregation_bitfield)) + return sorted(list(output)) +def get_attesting_balance(state: BeaconState, attestations: List[PendingAttestation]) -> Gwei: + return get_total_balance(state, get_attesting_indices(state, attestations)) +def get_current_epoch_boundary_attestations(state: BeaconState) -> List[PendingAttestation]: + return [ + a for a in state.current_epoch_attestations + if a.data.target_root == get_block_root(state, get_epoch_start_slot(get_current_epoch(state))) + ] +def get_previous_epoch_boundary_attestations(state: BeaconState) -> List[PendingAttestation]: + return [ + a for a in state.previous_epoch_attestations + if a.data.target_root == get_block_root(state, get_epoch_start_slot(get_previous_epoch(state))) + ] +def get_previous_epoch_matching_head_attestations(state: BeaconState) -> List[PendingAttestation]: + return [ + a for a in state.previous_epoch_attestations + if a.data.beacon_block_root == get_block_root(state, a.data.slot) + ] +def get_winning_root_and_participants(state: BeaconState, shard: Shard) -> Tuple[Bytes32, List[ValidatorIndex]]: + all_attestations = state.current_epoch_attestations + state.previous_epoch_attestations + valid_attestations = [ + a for a in all_attestations if a.data.previous_crosslink == state.latest_crosslinks[shard] + ] + all_roots = [a.data.crosslink_data_root for a in valid_attestations] + + # handle when no attestations for shard available + if len(all_roots) == 0: + return ZERO_HASH, [] + + def get_attestations_for(root) -> List[PendingAttestation]: + return [a for a in valid_attestations if a.data.crosslink_data_root == root] + + # Winning crosslink root is the root with the most votes for it, ties broken in favor of + # lexicographically higher hash + winning_root = max(all_roots, key=lambda r: (get_attesting_balance(state, get_attestations_for(r)), r)) + + return winning_root, get_attesting_indices(state, get_attestations_for(winning_root)) +def earliest_attestation(state: BeaconState, validator_index: ValidatorIndex) -> PendingAttestation: + return min([ + a for a in state.previous_epoch_attestations if + validator_index in get_attestation_participants(state, a.data, a.aggregation_bitfield) + ], key=lambda a: a.inclusion_slot) +def inclusion_slot(state: BeaconState, validator_index: ValidatorIndex) -> Slot: + return earliest_attestation(state, validator_index).inclusion_slot +def inclusion_distance(state: BeaconState, validator_index: ValidatorIndex) -> int: + attestation = earliest_attestation(state, validator_index) + return attestation.inclusion_slot - attestation.data.slot +def update_justification_and_finalization(state: BeaconState) -> None: + new_justified_epoch = state.current_justified_epoch + new_finalized_epoch = state.finalized_epoch + + # Rotate the justification bitfield up one epoch to make room for the current epoch + state.justification_bitfield <<= 1 + # If the previous epoch gets justified, fill the second last bit + previous_boundary_attesting_balance = get_attesting_balance(state, get_previous_epoch_boundary_attestations(state)) + if previous_boundary_attesting_balance * 3 >= get_previous_total_balance(state) * 2: + new_justified_epoch = get_current_epoch(state) - 1 + state.justification_bitfield |= 2 + # If the current epoch gets justified, fill the last bit + current_boundary_attesting_balance = get_attesting_balance(state, get_current_epoch_boundary_attestations(state)) + if current_boundary_attesting_balance * 3 >= get_current_total_balance(state) * 2: + new_justified_epoch = get_current_epoch(state) + state.justification_bitfield |= 1 + + # Process finalizations + bitfield = state.justification_bitfield + current_epoch = get_current_epoch(state) + # The 2nd/3rd/4th most recent epochs are all justified, the 2nd using the 4th as source + if (bitfield >> 1) % 8 == 0b111 and state.previous_justified_epoch == current_epoch - 3: + new_finalized_epoch = state.previous_justified_epoch + # The 2nd/3rd most recent epochs are both justified, the 2nd using the 3rd as source + if (bitfield >> 1) % 4 == 0b11 and state.previous_justified_epoch == current_epoch - 2: + new_finalized_epoch = state.previous_justified_epoch + # The 1st/2nd/3rd most recent epochs are all justified, the 1st using the 3rd as source + if (bitfield >> 0) % 8 == 0b111 and state.current_justified_epoch == current_epoch - 2: + new_finalized_epoch = state.current_justified_epoch + # The 1st/2nd most recent epochs are both justified, the 1st using the 2nd as source + if (bitfield >> 0) % 4 == 0b11 and state.current_justified_epoch == current_epoch - 1: + new_finalized_epoch = state.current_justified_epoch + + # Update state jusification/finality fields + state.previous_justified_epoch = state.current_justified_epoch + state.previous_justified_root = state.current_justified_root + if new_justified_epoch != state.current_justified_epoch: + state.current_justified_epoch = new_justified_epoch + state.current_justified_root = get_block_root(state, get_epoch_start_slot(new_justified_epoch)) + if new_finalized_epoch != state.finalized_epoch: + state.finalized_epoch = new_finalized_epoch + state.finalized_root = get_block_root(state, get_epoch_start_slot(new_finalized_epoch)) +def process_crosslinks(state: BeaconState) -> None: + current_epoch = get_current_epoch(state) + previous_epoch = current_epoch - 1 + next_epoch = current_epoch + 1 + for slot in range(get_epoch_start_slot(previous_epoch), get_epoch_start_slot(next_epoch)): + for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot): + winning_root, participants = get_winning_root_and_participants(state, shard) + participating_balance = get_total_balance(state, participants) + total_balance = get_total_balance(state, crosslink_committee) + if 3 * participating_balance >= 2 * total_balance: + state.latest_crosslinks[shard] = Crosslink( + epoch=slot_to_epoch(slot), + crosslink_data_root=winning_root + ) +def maybe_reset_eth1_period(state: BeaconState) -> None: + if (get_current_epoch(state) + 1) % EPOCHS_PER_ETH1_VOTING_PERIOD == 0: + for eth1_data_vote in state.eth1_data_votes: + # If a majority of all votes were for a particular eth1_data value, + # then set that as the new canonical value + if eth1_data_vote.vote_count * 2 > EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH: + state.latest_eth1_data = eth1_data_vote.eth1_data + state.eth1_data_votes = [] +def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: + if get_previous_total_balance(state) == 0: + return 0 + + adjusted_quotient = integer_squareroot(get_previous_total_balance(state)) // BASE_REWARD_QUOTIENT + return get_effective_balance(state, index) // adjusted_quotient // 5 +def get_inactivity_penalty(state: BeaconState, index: ValidatorIndex, epochs_since_finality: int) -> Gwei: + return ( + get_base_reward(state, index) + + get_effective_balance(state, index) * epochs_since_finality // INACTIVITY_PENALTY_QUOTIENT // 2 + ) +def get_justification_and_finalization_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: + epochs_since_finality = get_current_epoch(state) + 1 - state.finalized_epoch + if epochs_since_finality <= 4: + return compute_normal_justification_and_finalization_deltas(state) + else: + return compute_inactivity_leak_deltas(state) +def compute_normal_justification_and_finalization_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: + # deltas[0] for rewards + # deltas[1] for penalties + deltas = [ + [0 for index in range(len(state.validator_registry))], + [0 for index in range(len(state.validator_registry))] + ] + # Some helper variables + boundary_attestations = get_previous_epoch_boundary_attestations(state) + boundary_attesting_balance = get_attesting_balance(state, boundary_attestations) + total_balance = get_previous_total_balance(state) + total_attesting_balance = get_attesting_balance(state, state.previous_epoch_attestations) + matching_head_attestations = get_previous_epoch_matching_head_attestations(state) + matching_head_balance = get_attesting_balance(state, matching_head_attestations) + # Process rewards or penalties for all validators + for index in get_active_validator_indices(state.validator_registry, get_previous_epoch(state)): + # Expected FFG source + if index in get_attesting_indices(state, state.previous_epoch_attestations): + deltas[0][index] += get_base_reward(state, index) * total_attesting_balance // total_balance + # Inclusion speed bonus + deltas[0][index] += ( + get_base_reward(state, index) * MIN_ATTESTATION_INCLUSION_DELAY // + inclusion_distance(state, index) + ) + else: + deltas[1][index] += get_base_reward(state, index) + # Expected FFG target + if index in get_attesting_indices(state, boundary_attestations): + deltas[0][index] += get_base_reward(state, index) * boundary_attesting_balance // total_balance + else: + deltas[1][index] += get_base_reward(state, index) + # Expected head + if index in get_attesting_indices(state, matching_head_attestations): + deltas[0][index] += get_base_reward(state, index) * matching_head_balance // total_balance + else: + deltas[1][index] += get_base_reward(state, index) + # Proposer bonus + if index in get_attesting_indices(state, state.previous_epoch_attestations): + proposer_index = get_beacon_proposer_index(state, inclusion_slot(state, index)) + deltas[0][proposer_index] += get_base_reward(state, index) // ATTESTATION_INCLUSION_REWARD_QUOTIENT + return deltas +def compute_inactivity_leak_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: + # deltas[0] for rewards + # deltas[1] for penalties + deltas = [ + [0 for index in range(len(state.validator_registry))], + [0 for index in range(len(state.validator_registry))] + ] + boundary_attestations = get_previous_epoch_boundary_attestations(state) + matching_head_attestations = get_previous_epoch_matching_head_attestations(state) + active_validator_indices = get_active_validator_indices(state.validator_registry, get_previous_epoch(state)) + epochs_since_finality = get_current_epoch(state) + 1 - state.finalized_epoch + for index in active_validator_indices: + if index not in get_attesting_indices(state, state.previous_epoch_attestations): + deltas[1][index] += get_inactivity_penalty(state, index, epochs_since_finality) + else: + # If a validator did attest, apply a small penalty for getting attestations included late + deltas[0][index] += ( + get_base_reward(state, index) * MIN_ATTESTATION_INCLUSION_DELAY // + inclusion_distance(state, index) + ) + deltas[1][index] += get_base_reward(state, index) + if index not in get_attesting_indices(state, boundary_attestations): + deltas[1][index] += get_inactivity_penalty(state, index, epochs_since_finality) + if index not in get_attesting_indices(state, matching_head_attestations): + deltas[1][index] += get_base_reward(state, index) + # Penalize slashed-but-inactive validators as though they were active but offline + for index in range(len(state.validator_registry)): + eligible = ( + index not in active_validator_indices and + state.validator_registry[index].slashed and + get_current_epoch(state) < state.validator_registry[index].withdrawable_epoch + ) + if eligible: + deltas[1][index] += ( + 2 * get_inactivity_penalty(state, index, epochs_since_finality) + + get_base_reward(state, index) + ) + return deltas +def get_crosslink_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: + # deltas[0] for rewards + # deltas[1] for penalties + deltas = [ + [0 for index in range(len(state.validator_registry))], + [0 for index in range(len(state.validator_registry))] + ] + previous_epoch_start_slot = get_epoch_start_slot(get_previous_epoch(state)) + current_epoch_start_slot = get_epoch_start_slot(get_current_epoch(state)) + for slot in range(previous_epoch_start_slot, current_epoch_start_slot): + for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot): + winning_root, participants = get_winning_root_and_participants(state, shard) + participating_balance = get_total_balance(state, participants) + total_balance = get_total_balance(state, crosslink_committee) + for index in crosslink_committee: + if index in participants: + deltas[0][index] += get_base_reward(state, index) * participating_balance // total_balance + else: + deltas[1][index] += get_base_reward(state, index) + return deltas +def apply_rewards(state: BeaconState) -> None: + deltas1 = get_justification_and_finalization_deltas(state) + deltas2 = get_crosslink_deltas(state) + for i in range(len(state.validator_registry)): + state.validator_balances[i] = max( + 0, + state.validator_balances[i] + deltas1[0][i] + deltas2[0][i] - deltas1[1][i] - deltas2[1][i] + ) +def process_ejections(state: BeaconState) -> None: + """ + Iterate through the validator registry + and eject active validators with balance below ``EJECTION_BALANCE``. + """ + for index in get_active_validator_indices(state.validator_registry, get_current_epoch(state)): + if state.validator_balances[index] < EJECTION_BALANCE: + exit_validator(state, index) +def should_update_validator_registry(state: BeaconState) -> bool: + # Must have finalized a new block + if state.finalized_epoch <= state.validator_registry_update_epoch: + return False + # Must have processed new crosslinks on all shards of the current epoch + shards_to_check = [ + (state.current_shuffling_start_shard + i) % SHARD_COUNT + for i in range(get_current_epoch_committee_count(state)) + ] + for shard in shards_to_check: + if state.latest_crosslinks[shard].epoch <= state.validator_registry_update_epoch: + return False + return True +def update_validator_registry(state: BeaconState) -> None: + """ + Update validator registry. + Note that this function mutates ``state``. + """ + current_epoch = get_current_epoch(state) + # The active validators + active_validator_indices = get_active_validator_indices(state.validator_registry, current_epoch) + # The total effective balance of active validators + total_balance = get_total_balance(state, active_validator_indices) + + # The maximum balance churn in Gwei (for deposits and exits separately) + max_balance_churn = max( + MAX_DEPOSIT_AMOUNT, + total_balance // (2 * MAX_BALANCE_CHURN_QUOTIENT) + ) + + # Activate validators within the allowable balance churn + balance_churn = 0 + for index, validator in enumerate(state.validator_registry): + if validator.activation_epoch == FAR_FUTURE_EPOCH and state.validator_balances[index] >= MAX_DEPOSIT_AMOUNT: + # Check the balance churn would be within the allowance + balance_churn += get_effective_balance(state, index) + if balance_churn > max_balance_churn: + break + + # Activate validator + activate_validator(state, index, is_genesis=False) + + # Exit validators within the allowable balance churn + balance_churn = 0 + for index, validator in enumerate(state.validator_registry): + if validator.exit_epoch == FAR_FUTURE_EPOCH and validator.initiated_exit: + # Check the balance churn would be within the allowance + balance_churn += get_effective_balance(state, index) + if balance_churn > max_balance_churn: + break + + # Exit validator + exit_validator(state, index) + + state.validator_registry_update_epoch = current_epoch +def update_registry_and_shuffling_data(state: BeaconState) -> None: + # First set previous shuffling data to current shuffling data + state.previous_shuffling_epoch = state.current_shuffling_epoch + state.previous_shuffling_start_shard = state.current_shuffling_start_shard + state.previous_shuffling_seed = state.current_shuffling_seed + current_epoch = get_current_epoch(state) + next_epoch = current_epoch + 1 + # Check if we should update, and if so, update + if should_update_validator_registry(state): + update_validator_registry(state) + # If we update the registry, update the shuffling data and shards as well + state.current_shuffling_epoch = next_epoch + state.current_shuffling_start_shard = ( + state.current_shuffling_start_shard + + get_current_epoch_committee_count(state) % SHARD_COUNT + ) + state.current_shuffling_seed = generate_seed(state, state.current_shuffling_epoch) + else: + # If processing at least one crosslink keeps failing, then reshuffle every power of two, + # but don't update the current_shuffling_start_shard + epochs_since_last_registry_update = current_epoch - state.validator_registry_update_epoch + if epochs_since_last_registry_update > 1 and is_power_of_two(epochs_since_last_registry_update): + state.current_shuffling_epoch = next_epoch + state.current_shuffling_seed = generate_seed(state, state.current_shuffling_epoch) +def process_slashings(state: BeaconState) -> None: + """ + Process the slashings. + Note that this function mutates ``state``. + """ + current_epoch = get_current_epoch(state) + active_validator_indices = get_active_validator_indices(state.validator_registry, current_epoch) + total_balance = get_total_balance(state, active_validator_indices) + + # Compute `total_penalties` + total_at_start = state.latest_slashed_balances[(current_epoch + 1) % LATEST_SLASHED_EXIT_LENGTH] + total_at_end = state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH] + total_penalties = total_at_end - total_at_start + + for index, validator in enumerate(state.validator_registry): + if validator.slashed and current_epoch == validator.withdrawable_epoch - LATEST_SLASHED_EXIT_LENGTH // 2: + penalty = max( + get_effective_balance(state, index) * min(total_penalties * 3, total_balance) // total_balance, + get_effective_balance(state, index) // MIN_PENALTY_QUOTIENT + ) + state.validator_balances[index] -= penalty +def process_exit_queue(state: BeaconState) -> None: + """ + Process the exit queue. + Note that this function mutates ``state``. + """ + def eligible(index): + validator = state.validator_registry[index] + # Filter out dequeued validators + if validator.withdrawable_epoch != FAR_FUTURE_EPOCH: + return False + # Dequeue if the minimum amount of time has passed + else: + return get_current_epoch(state) >= validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY + + eligible_indices = filter(eligible, list(range(len(state.validator_registry)))) + # Sort in order of exit epoch, and validators that exit within the same epoch exit in order of validator index + sorted_indices = sorted(eligible_indices, key=lambda index: state.validator_registry[index].exit_epoch) + for dequeues, index in enumerate(sorted_indices): + if dequeues >= MAX_EXIT_DEQUEUES_PER_EPOCH: + break + prepare_validator_for_withdrawal(state, index) +def finish_epoch_update(state: BeaconState) -> None: + current_epoch = get_current_epoch(state) + next_epoch = current_epoch + 1 + # Set active index root + index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % LATEST_ACTIVE_INDEX_ROOTS_LENGTH + state.latest_active_index_roots[index_root_position] = hash_tree_root( + get_active_validator_indices(state.validator_registry, next_epoch + ACTIVATION_EXIT_DELAY) + ) + # Set total slashed balances + state.latest_slashed_balances[next_epoch % LATEST_SLASHED_EXIT_LENGTH] = ( + state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH] + ) + # Set randao mix + state.latest_randao_mixes[next_epoch % LATEST_RANDAO_MIXES_LENGTH] = get_randao_mix(state, current_epoch) + # Set historical root accumulator + if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0: + historical_batch = HistoricalBatch( + block_roots=state.latest_block_roots, + state_roots=state.latest_state_roots, + ) + state.historical_roots.append(hash_tree_root(historical_batch)) + # Rotate current/previous epoch attestations + state.previous_epoch_attestations = state.current_epoch_attestations + state.current_epoch_attestations = [] +def advance_slot(state: BeaconState) -> None: + state.slot += 1 +def process_block_header(state: BeaconState, block: BeaconBlock) -> None: + # Verify that the slots match + assert block.slot == state.slot + # Verify that the parent matches + assert block.previous_block_root == hash_tree_root(state.latest_block_header) + # Save current block as the new latest block + state.latest_block_header = get_temporary_block_header(block) + # Verify proposer signature + proposer = state.validator_registry[get_beacon_proposer_index(state, state.slot)] + assert bls_verify( + pubkey=proposer.pubkey, + message_hash=signed_root(block), + signature=block.signature, + domain=get_domain(state.fork, get_current_epoch(state), DOMAIN_BEACON_BLOCK) + ) +def process_randao(state: BeaconState, block: BeaconBlock) -> None: + proposer = state.validator_registry[get_beacon_proposer_index(state, state.slot)] + # Verify that the provided randao value is valid + assert bls_verify( + pubkey=proposer.pubkey, + message_hash=hash_tree_root(get_current_epoch(state)), + signature=block.body.randao_reveal, + domain=get_domain(state.fork, get_current_epoch(state), DOMAIN_RANDAO) + ) + # Mix it in + state.latest_randao_mixes[get_current_epoch(state) % LATEST_RANDAO_MIXES_LENGTH] = ( + xor(get_randao_mix(state, get_current_epoch(state)), + hash(block.body.randao_reveal)) + ) +def process_eth1_data(state: BeaconState, block: BeaconBlock) -> None: + for eth1_data_vote in state.eth1_data_votes: + # If someone else has already voted for the same hash, add to its counter + if eth1_data_vote.eth1_data == block.body.eth1_data: + eth1_data_vote.vote_count += 1 + return + # If we're seeing this hash for the first time, make a new counter + state.eth1_data_votes.append(Eth1DataVote(eth1_data=block.body.eth1_data, vote_count=1)) +def process_proposer_slashing(state: BeaconState, + proposer_slashing: ProposerSlashing) -> None: + """ + Process ``ProposerSlashing`` transaction. + Note that this function mutates ``state``. + """ + proposer = state.validator_registry[proposer_slashing.proposer_index] + # Verify that the epoch is the same + assert slot_to_epoch(proposer_slashing.header_1.slot) == slot_to_epoch(proposer_slashing.header_2.slot) + # But the headers are different + assert proposer_slashing.header_1 != proposer_slashing.header_2 + # Proposer is not yet slashed + assert proposer.slashed is False + # Signatures are valid + for header in (proposer_slashing.header_1, proposer_slashing.header_2): + assert bls_verify( + pubkey=proposer.pubkey, + message_hash=signed_root(header), + signature=header.signature, + domain=get_domain(state.fork, slot_to_epoch(header.slot), DOMAIN_BEACON_BLOCK) + ) + slash_validator(state, proposer_slashing.proposer_index) +def process_attester_slashing(state: BeaconState, + attester_slashing: AttesterSlashing) -> None: + """ + Process ``AttesterSlashing`` transaction. + Note that this function mutates ``state``. + """ + attestation1 = attester_slashing.slashable_attestation_1 + attestation2 = attester_slashing.slashable_attestation_2 + # Check that the attestations are conflicting + assert attestation1.data != attestation2.data + assert ( + is_double_vote(attestation1.data, attestation2.data) or + is_surround_vote(attestation1.data, attestation2.data) + ) + assert verify_slashable_attestation(state, attestation1) + assert verify_slashable_attestation(state, attestation2) + slashable_indices = [ + index for index in attestation1.validator_indices + if ( + index in attestation2.validator_indices and + state.validator_registry[index].slashed is False + ) + ] + assert len(slashable_indices) >= 1 + for index in slashable_indices: + slash_validator(state, index) +def process_attestation(state: BeaconState, attestation: Attestation) -> None: + """ + Process ``Attestation`` transaction. + Note that this function mutates ``state``. + """ + # Can't submit attestations that are too far in history (or in prehistory) + assert attestation.data.slot >= GENESIS_SLOT + assert state.slot <= attestation.data.slot + SLOTS_PER_EPOCH + # Can't submit attestations too quickly + assert attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot + # Verify that the justified epoch and root is correct + if slot_to_epoch(attestation.data.slot) >= get_current_epoch(state): + # Case 1: current epoch attestations + assert attestation.data.source_epoch == state.current_justified_epoch + assert attestation.data.source_root == state.current_justified_root + else: + # Case 2: previous epoch attestations + assert attestation.data.source_epoch == state.previous_justified_epoch + assert attestation.data.source_root == state.previous_justified_root + # Check that the crosslink data is valid + acceptable_crosslink_data = { + # Case 1: Latest crosslink matches the one in the state + attestation.data.previous_crosslink, + # Case 2: State has already been updated, state's latest crosslink matches the crosslink + # the attestation is trying to create + Crosslink( + crosslink_data_root=attestation.data.crosslink_data_root, + epoch=slot_to_epoch(attestation.data.slot) + ) + } + assert state.latest_crosslinks[attestation.data.shard] in acceptable_crosslink_data + # Attestation must be nonempty! + assert attestation.aggregation_bitfield != b'\x00' * len(attestation.aggregation_bitfield) + # Custody must be empty (to be removed in phase 1) + assert attestation.custody_bitfield == b'\x00' * len(attestation.custody_bitfield) + # Get the committee for the specific shard that this attestation is for + crosslink_committee = [ + committee for committee, shard in get_crosslink_committees_at_slot(state, attestation.data.slot) + if shard == attestation.data.shard + ][0] + # Custody bitfield must be a subset of the attestation bitfield + for i in range(len(crosslink_committee)): + if get_bitfield_bit(attestation.aggregation_bitfield, i) == 0b0: + assert get_bitfield_bit(attestation.custody_bitfield, i) == 0b0 + # Verify aggregate signature + participants = get_attestation_participants(state, attestation.data, attestation.aggregation_bitfield) + custody_bit_1_participants = get_attestation_participants(state, attestation.data, attestation.custody_bitfield) + custody_bit_0_participants = [i for i in participants if i not in custody_bit_1_participants] + + assert bls_verify_multiple( + pubkeys=[ + bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_0_participants]), + bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_1_participants]), + ], + message_hashes=[ + hash_tree_root(AttestationDataAndCustodyBit(data=attestation.data, custody_bit=0b0)), + hash_tree_root(AttestationDataAndCustodyBit(data=attestation.data, custody_bit=0b1)), + ], + signature=attestation.aggregate_signature, + domain=get_domain(state.fork, slot_to_epoch(attestation.data.slot), DOMAIN_ATTESTATION), + ) + # Crosslink data root is zero (to be removed in phase 1) + assert attestation.data.crosslink_data_root == ZERO_HASH + # Apply the attestation + pending_attestation = PendingAttestation( + data=attestation.data, + aggregation_bitfield=attestation.aggregation_bitfield, + custody_bitfield=attestation.custody_bitfield, + inclusion_slot=state.slot + ) + if slot_to_epoch(attestation.data.slot) == get_current_epoch(state): + state.current_epoch_attestations.append(pending_attestation) + elif slot_to_epoch(attestation.data.slot) == get_previous_epoch(state): + state.previous_epoch_attestations.append(pending_attestation) +def process_voluntary_exit(state: BeaconState, exit: VoluntaryExit) -> None: + """ + Process ``VoluntaryExit`` transaction. + Note that this function mutates ``state``. + """ + validator = state.validator_registry[exit.validator_index] + # Verify the validator has not yet exited + assert validator.exit_epoch == FAR_FUTURE_EPOCH + # Verify the validator has not initiated an exit + assert validator.initiated_exit is False + # Exits must specify an epoch when they become valid; they are not valid before then + assert get_current_epoch(state) >= exit.epoch + # Must have been in the validator set long enough + assert get_current_epoch(state) - validator.activation_epoch >= PERSISTENT_COMMITTEE_PERIOD + # Verify signature + assert bls_verify( + pubkey=validator.pubkey, + message_hash=signed_root(exit), + signature=exit.signature, + domain=get_domain(state.fork, exit.epoch, DOMAIN_VOLUNTARY_EXIT) + ) + # Run the exit + initiate_validator_exit(state, exit.validator_index) +def process_transfer(state: BeaconState, transfer: Transfer) -> None: + """ + Process ``Transfer`` transaction. + Note that this function mutates ``state``. + """ + # Verify the amount and fee aren't individually too big (for anti-overflow purposes) + assert state.validator_balances[transfer.sender] >= max(transfer.amount, transfer.fee) + # Verify that we have enough ETH to send, and that after the transfer the balance will be either + # exactly zero or at least MIN_DEPOSIT_AMOUNT + assert ( + state.validator_balances[transfer.sender] == transfer.amount + transfer.fee or + state.validator_balances[transfer.sender] >= transfer.amount + transfer.fee + MIN_DEPOSIT_AMOUNT + ) + # A transfer is valid in only one slot + assert state.slot == transfer.slot + # Only withdrawn or not-yet-deposited accounts can transfer + assert ( + get_current_epoch(state) >= state.validator_registry[transfer.sender].withdrawable_epoch or + state.validator_registry[transfer.sender].activation_epoch == FAR_FUTURE_EPOCH + ) + # Verify that the pubkey is valid + assert ( + state.validator_registry[transfer.sender].withdrawal_credentials == + BLS_WITHDRAWAL_PREFIX_BYTE + hash(transfer.pubkey)[1:] + ) + # Verify that the signature is valid + assert bls_verify( + pubkey=transfer.pubkey, + message_hash=signed_root(transfer), + signature=transfer.signature, + domain=get_domain(state.fork, slot_to_epoch(transfer.slot), DOMAIN_TRANSFER) + ) + # Process the transfer + state.validator_balances[transfer.sender] -= transfer.amount + transfer.fee + state.validator_balances[transfer.recipient] += transfer.amount + state.validator_balances[get_beacon_proposer_index(state, state.slot)] += transfer.fee +def verify_block_state_root(state: BeaconState, block: BeaconBlock) -> None: + assert block.state_root == hash_tree_root(state) + +# Monkey patch validator shuffling cache +_get_shuffling = get_shuffling +shuffling_cache = {} +def get_shuffling(seed: Bytes32, + validators: List[Validator], + epoch: Epoch) -> List[List[ValidatorIndex]]: + + param_hash = (seed, hash_tree_root(validators, [Validator]), epoch) + + if param_hash in shuffling_cache: + # print("Cache hit, epoch={0}".format(epoch)) + return shuffling_cache[param_hash] + else: + # print("Cache miss, epoch={0}".format(epoch)) + ret = _get_shuffling(seed, validators, epoch) + shuffling_cache[param_hash] = ret + return ret + + +# Monkey patch hash cache +_hash = hash +hash_cache = {} +def hash(x): + if x in hash_cache: + return hash_cache[x] + else: + ret = _hash(x) + hash_cache[x] = ret + return ret + \ No newline at end of file diff --git a/scripts/phase0/state_transition.py b/build/phase0/state_transition.py similarity index 56% rename from scripts/phase0/state_transition.py rename to build/phase0/state_transition.py index f78119cf2..2bd33f6d6 100644 --- a/scripts/phase0/state_transition.py +++ b/build/phase0/state_transition.py @@ -1,3 +1,18 @@ +import build.phase0.spec as spec + + +from typing import ( + Any, + Callable, + List, + NewType, + Tuple, +) + +from build.phase0.spec import ( + BeaconState, + BeaconBlock, +) def process_transaction_type(state: BeaconState, @@ -13,72 +28,73 @@ def process_transactions(state: BeaconState, block: BeaconBlock) -> None: process_transaction_type( state, block.body.proposer_slashings, - MAX_PROPOSER_SLASHINGS, - process_proposer_slashing, + spec.MAX_PROPOSER_SLASHINGS, + spec.process_proposer_slashing, ) process_transaction_type( state, block.body.attester_slashings, - MAX_ATTESTER_SLASHINGS, - process_attester_slashing, + spec.MAX_ATTESTER_SLASHINGS, + spec.process_attester_slashing, ) process_transaction_type( state, block.body.attestations, - MAX_ATTESTATIONS, - process_attestation, + spec.MAX_ATTESTATIONS, + spec.process_attestation, ) process_transaction_type( state, block.body.deposits, - MAX_DEPOSITS, - process_deposit, + spec.MAX_DEPOSITS, + spec.process_deposit, ) process_transaction_type( state, block.body.voluntary_exits, - MAX_VOLUNTARY_EXITS, - process_voluntary_exit, + spec.MAX_VOLUNTARY_EXITS, + spec.process_voluntary_exit, ) assert len(block.body.transfers) == len(set(block.body.transfers)) process_transaction_type( state, block.body.transfers, - MAX_TRANSFERS, - process_transfer, + spec.MAX_TRANSFERS, + spec.process_transfer, ) def process_block(state: BeaconState, block: BeaconBlock, verify_state_root: bool=False) -> None: - process_block_header(state, block) - process_randao(state, block) - process_eth1_data(state, block) + spec.process_block_header(state, block) + spec.process_randao(state, block) + spec.process_eth1_data(state, block) + process_transactions(state, block) if verify_state_root: - verify_block_state_root(state, block) + spec.verify_block_state_root(state, block) def process_epoch_transition(state: BeaconState) -> None: - update_justification_and_finalization(state) - process_crosslinks(state) - maybe_reset_eth1_period(state) - apply_rewards(state) - process_ejections(state) - update_registry_and_shuffling_data(state) - process_slashings(state) - process_exit_queue(state) - finish_epoch_update(state) + spec.update_justification_and_finalization(state) + spec.process_crosslinks(state) + spec.maybe_reset_eth1_period(state) + spec.apply_rewards(state) + spec.process_ejections(state) + spec.update_registry_and_shuffling_data(state) + spec.process_slashings(state) + spec.process_exit_queue(state) + spec.finish_epoch_update(state) def state_transition(state: BeaconState, block: BeaconBlock, verify_state_root: bool=False) -> BeaconState: while state.slot < block.slot: - cache_state(state) - if (state.slot + 1) % SLOTS_PER_EPOCH == 0: + spec.cache_state(state) + if (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0: process_epoch_transition(state) - advance_slot(state) + spec.advance_slot(state) if block.slot == state.slot: process_block(state, block) diff --git a/build/utils/__init__.py b/build/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/phase0/bls_stub.py b/build/utils/bls_stub.py similarity index 100% rename from scripts/phase0/bls_stub.py rename to build/utils/bls_stub.py diff --git a/build/utils/hash_function.py b/build/utils/hash_function.py new file mode 100644 index 000000000..da5b4d979 --- /dev/null +++ b/build/utils/hash_function.py @@ -0,0 +1,6 @@ +from hashlib import sha256 +from eth_utils import keccak + + +# def hash(x): return sha256(x).digest() +def hash(x): return keccak(x) diff --git a/build/utils/merkle_minimal.py b/build/utils/merkle_minimal.py new file mode 100644 index 000000000..a811350ce --- /dev/null +++ b/build/utils/merkle_minimal.py @@ -0,0 +1,28 @@ +from .hash_function import hash + + +zerohashes = [b'\x00' * 32] +for i in range(1, 32): + zerohashes.append(hash(zerohashes[i-1] + zerohashes[i-1])) + +# Compute a Merkle root of a right-zerobyte-padded 2**32 sized tree +def calc_merkle_tree_from_leaves(values): + values = list(values) + tree = [values[::]] + for h in range(32): + if len(values) % 2 == 1: + values.append(zerohashes[h]) + # print(values) + values = [hash(values[i] + values[i+1]) for i in range(0, len(values), 2)] + tree.append(values[::]) + return tree + +def get_merkle_root(values): + return calc_merkle_tree_from_leaves(values)[-1][0] + +def get_merkle_proof(tree, item_index): + proof = [] + for i in range(32): + subindex = (item_index//2**i)^1 + proof.append(tree[i][subindex] if subindex < len(tree[i]) else zerohashes[i]) + return proof diff --git a/scripts/phase0/minimal_ssz.py b/build/utils/minimal_ssz.py similarity index 99% rename from scripts/phase0/minimal_ssz.py rename to build/utils/minimal_ssz.py index 5caaf8f09..845de18c3 100644 --- a/scripts/phase0/minimal_ssz.py +++ b/build/utils/minimal_ssz.py @@ -1,4 +1,4 @@ -from utils.hash import hash +from .hash_function import hash BYTES_PER_CHUNK = 32 diff --git a/scripts/phase0/monkey_patches.py b/build/utils/monkey_patches.py similarity index 100% rename from scripts/phase0/monkey_patches.py rename to build/utils/monkey_patches.py diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index c4f8ab38c..8b5941b62 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -1,14 +1,18 @@ import sys import function_puller -code_lines = [] -for i in (1, 2, 3, 4, 8, 32, 48, 96): - code_lines.append("def int_to_bytes%d(x): return x.to_bytes(%d, 'little')" % (i, i)) -code_lines.append("SLOTS_PER_EPOCH = 64") # stub, will get overwritten by real var -code_lines.append("def slot_to_epoch(x): return x // SLOTS_PER_EPOCH") +def build_spec(sourcefile, outfile): + code_lines = [] -code_lines.append(""" + code_lines.append("from build.utils.minimal_ssz import *") + code_lines.append("from build.utils.bls_stub import *") + for i in (1, 2, 3, 4, 8, 32, 48, 96): + code_lines.append("def int_to_bytes%d(x): return x.to_bytes(%d, 'little')" % (i, i)) + code_lines.append("SLOTS_PER_EPOCH = 64") # stub, will get overwritten by real var + code_lines.append("def slot_to_epoch(x): return x // SLOTS_PER_EPOCH") + + code_lines.append(""" from typing import ( Any, Callable, @@ -28,16 +32,48 @@ BLSPubkey = NewType('BLSPubkey', bytes) # bytes48 BLSSignature = NewType('BLSSignature', bytes) # bytes96 Any = None Store = None -""") + """) -code_lines += function_puller.get_lines(sys.argv[1]) + code_lines += function_puller.get_lines(sourcefile) -print(open(sys.argv[2]).read()) -print(open(sys.argv[3]).read()) + code_lines.append(""" +# Monkey patch validator shuffling cache +_get_shuffling = get_shuffling +shuffling_cache = {} +def get_shuffling(seed: Bytes32, + validators: List[Validator], + epoch: Epoch) -> List[List[ValidatorIndex]]: -for line in code_lines: - print(line) + param_hash = (seed, hash_tree_root(validators, [Validator]), epoch) -print(open(sys.argv[4]).read()) -print(open(sys.argv[5]).read()) + if param_hash in shuffling_cache: + # print("Cache hit, epoch={0}".format(epoch)) + return shuffling_cache[param_hash] + else: + # print("Cache miss, epoch={0}".format(epoch)) + ret = _get_shuffling(seed, validators, epoch) + shuffling_cache[param_hash] = ret + return ret + + +# Monkey patch hash cache +_hash = hash +hash_cache = {} +def hash(x): + if x in hash_cache: + return hash_cache[x] + else: + ret = _hash(x) + hash_cache[x] = ret + return ret + """) + + with open(outfile, 'w') as out: + out.write("\n".join(code_lines)) + + +if __name__ == '__main__': + if len(sys.argv) < 3: + print("Error: spec source and outfile must defined") + build_spec(sys.argv[1], sys.argv[2]) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/phase0/conftest.py b/tests/phase0/conftest.py index d3ebabaa2..7e2800afd 100644 --- a/tests/phase0/conftest.py +++ b/tests/phase0/conftest.py @@ -1,6 +1,134 @@ import pytest + +from py_ecc import bls + from build.phase0 import spec +from build.utils.merkle_minimal import ( + calc_merkle_tree_from_leaves, + get_merkle_proof, + get_merkle_root, +) +from build.phase0.spec import ( + Deposit, + DepositData, + DepositInput, + Eth1Data, + get_genesis_beacon_state, + verify_merkle_branch, + hash, +) -# @pytest.fixture(autouse=True) -# def build_clean(): \ No newline at end of file + +privkeys_list = [i+1 for i in range(1000)] +pubkeys_list = [bls.privtopub(privkey) for privkey in privkeys_list] +pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys_list, pubkeys_list)} + + +@pytest.fixture +def privkeys(): + return privkeys_list + + +@pytest.fixture +def pubkeys(): + return pubkeys_list + + +def overwrite_spec_config(config): + for field in config: + setattr(spec, field, config[field]) + if field == "LATEST_RANDAO_MIXES_LENGTH": + spec.BeaconState.fields['latest_randao_mixes'][1] = config[field] + elif field == "SHARD_COUNT": + spec.BeaconState.fields['latest_crosslinks'][1] = config[field] + elif field == "SLOTS_PER_HISTORICAL_ROOT": + spec.BeaconState.fields['latest_block_roots'][1] = config[field] + spec.BeaconState.fields['latest_state_roots'][1] = config[field] + spec.HistoricalBatch.fields['block_roots'][1] = config[field] + spec.HistoricalBatch.fields['state_roots'][1] = config[field] + elif field == "LATEST_ACTIVE_INDEX_ROOTS_LENGTH": + spec.BeaconState.fields['latest_active_index_roots'][1] = config[field] + elif field == "LATEST_SLASHED_EXIT_LENGTH": + spec.BeaconState.fields['latest_slashed_balances'][1] = config[field] + + +@pytest.fixture +def config(): + return { + "SHARD_COUNT": 8, + "MIN_ATTESTATION_INCLUSION_DELAY": 2, + "TARGET_COMMITTEE_SIZE": 4, + "SLOTS_PER_EPOCH": 8, + "GENESIS_EPOCH": spec.GENESIS_SLOT // 8, + "SLOTS_PER_HISTORICAL_ROOT": 64, + "LATEST_RANDAO_MIXES_LENGTH": 64, + "LATEST_ACTIVE_INDEX_ROOTS_LENGTH": 64, + "LATEST_SLASHED_EXIT_LENGTH": 64, + } + + +@pytest.fixture(autouse=True) +def overwrite_config(config): + overwrite_spec_config(config) + + +def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves): + deposit_timestamp = 0 + proof_of_possession = b'\x33' * 96 + + deposit_data_list = [] + for i in range(num_validators): + pubkey = pubkeys_list[i] + privkey = pubkey_to_privkey[pubkey] + deposit_data = DepositData( + amount=spec.MAX_DEPOSIT_AMOUNT, + timestamp=deposit_timestamp, + deposit_input=DepositInput( + pubkey=pubkey, + withdrawal_credentials=privkey.to_bytes(32, byteorder='big'), + proof_of_possession=proof_of_possession, + ), + ) + item = hash(deposit_data.serialize()) + deposit_data_leaves.append(item) + tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) + root = get_merkle_root((tuple(deposit_data_leaves))) + proof = list(get_merkle_proof(tree, item_index=i)) + assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, i, root) + deposit_data_list.append(deposit_data) + + genesis_validator_deposits = [] + for i in range(num_validators): + genesis_validator_deposits.append(Deposit( + proof=list(get_merkle_proof(tree, item_index=i)), + index=i, + deposit_data=deposit_data_list[i] + )) + return genesis_validator_deposits, root + + +def create_genesis_state(num_validators, deposit_data_leaves): + initial_deposits, deposit_root = create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves) + return get_genesis_beacon_state( + initial_deposits, + genesis_time=0, + genesis_eth1_data=Eth1Data( + deposit_root=deposit_root, + block_hash=spec.ZERO_HASH, + ), + ) + +@pytest.fixture +def num_validators(): + return 100 + + +@pytest.fixture +def deposit_data_leaves(): + return list() + + +@pytest.fixture +def state(num_validators, deposit_data_leaves): + return create_genesis_state(num_validators, deposit_data_leaves) diff --git a/tests/phase0/test_sanity.py b/tests/phase0/test_sanity.py new file mode 100644 index 000000000..573c3ba21 --- /dev/null +++ b/tests/phase0/test_sanity.py @@ -0,0 +1,632 @@ +import os +import sys +import time + +from copy import deepcopy +from py_ecc import bls +import build.phase0.spec as spec + +from build.utils.minimal_ssz import signed_root +from build.phase0.spec import ( + # SSZ + Attestation, + AttestationData, + AttestationDataAndCustodyBit, + BeaconBlockHeader, + Deposit, + DepositData, + DepositInput, + Eth1Data, + Transfer, + ProposerSlashing, + Validator, + VoluntaryExit, + # functions + int_to_bytes32, + int_to_bytes48, + get_active_validator_indices, + get_attestation_participants, + get_block_root, + get_crosslink_committees_at_slot, + get_current_epoch, + get_domain, + get_empty_block, + get_epoch_start_slot, + get_genesis_beacon_state, + get_state_root, + advance_slot, + slot_to_epoch, + cache_state, + verify_merkle_branch, + hash, +) +from build.phase0.state_transition import ( + state_transition, +) +from build.utils.merkle_minimal import ( + calc_merkle_tree_from_leaves, + get_merkle_proof, + get_merkle_root, +) +# from state_test_gen import ( + # generate_from_test, + # dump_json, + # dump_yaml, +# ) + + +def get_empty_root(): + return get_merkle_root((spec.ZERO_HASH,)) + + +def construct_empty_block_for_next_slot(state): + empty_block = get_empty_block() + empty_block.slot = state.slot + 1 + previous_block_header = deepcopy(state.latest_block_header) + if previous_block_header.state_root == spec.ZERO_HASH: + previous_block_header.state_root = state.hash_tree_root() + empty_block.previous_block_root = previous_block_header.hash_tree_root() + return empty_block + + +def create_deposit_data(state, pubkey, privkey, amount): + deposit_input = DepositInput( + pubkey=pubkey, + withdrawal_credentials=privkey.to_bytes(32, byteorder='big'), + proof_of_possession=b'00'*96, + ) + proof_of_possession = bls.sign( + message_hash=signed_root(deposit_input), + privkey=privkey, + domain=get_domain( + state.fork, + get_current_epoch(state), + spec.DOMAIN_DEPOSIT, + ) + ) + deposit_input.proof_of_possession = proof_of_possession + deposit_data = DepositData( + amount=amount, + timestamp=0, + deposit_input=deposit_input, + ) + return deposit_data + + +def build_attestation_data(state, slot, shard): + assert state.slot >= slot + + block_root = construct_empty_block_for_next_slot(state).previous_block_root + + epoch_start_slot = get_epoch_start_slot(get_current_epoch(state)) + if epoch_start_slot == slot: + epoch_boundary_root = block_root + else: + get_block_root(state, epoch_start_slot) + + if slot < epoch_start_slot: + justified_block_root = state.previous_justified_root + else: + justified_block_root = state.current_justified_root + + return AttestationData( + slot=slot, + shard=shard, + beacon_block_root=block_root, + source_epoch=state.current_justified_epoch, + source_root=justified_block_root, + target_root=epoch_boundary_root, + crosslink_data_root=spec.ZERO_HASH, + previous_crosslink=deepcopy(state.latest_crosslinks[shard]), + ) + + +def test_slot_transition(state): + test_state = deepcopy(state) + cache_state(test_state) + advance_slot(test_state) + assert test_state.slot == state.slot + 1 + assert get_state_root(test_state, state.slot) == state.hash_tree_root() + return test_state + + +def test_empty_block_transition(state): + test_state = deepcopy(state) + + block = construct_empty_block_for_next_slot(test_state) + state_transition(test_state, block) + + assert len(test_state.eth1_data_votes) == len(state.eth1_data_votes) + 1 + assert get_block_root(test_state, state.slot) == block.previous_block_root + + return state, [block], test_state + + +def test_skipped_slots(state): + test_state = deepcopy(state) + block = construct_empty_block_for_next_slot(test_state) + block.slot += 3 + + state_transition(test_state, block) + + assert test_state.slot == block.slot + for slot in range(state.slot, test_state.slot): + assert get_block_root(test_state, slot) == block.previous_block_root + + return state, [block], test_state + + +def test_empty_epoch_transition(state): + test_state = deepcopy(state) + block = construct_empty_block_for_next_slot(test_state) + block.slot += spec.SLOTS_PER_EPOCH + + state_transition(test_state, block) + + assert test_state.slot == block.slot + for slot in range(state.slot, test_state.slot): + assert get_block_root(test_state, slot) == block.previous_block_root + + return state, [block], test_state + + +def test_empty_epoch_transition_not_finalizing(state): + test_state = deepcopy(state) + block = construct_empty_block_for_next_slot(test_state) + block.slot += spec.SLOTS_PER_EPOCH * 5 + + state_transition(test_state, block) + + assert test_state.slot == block.slot + assert test_state.finalized_epoch < get_current_epoch(test_state) - 4 + + return state, [block], test_state + + +def test_proposer_slashing(state, pubkeys, privkeys): + test_state = deepcopy(state) + current_epoch = get_current_epoch(test_state) + validator_index = get_active_validator_indices(test_state.validator_registry, current_epoch)[-1] + pubkey = pubkeys[validator_index] + privkey = privkeys[validator_index] + slot = spec.GENESIS_SLOT + header_1 = BeaconBlockHeader( + slot=slot, + previous_block_root=b'\x00'*32, + state_root=b'\x00'*32, + block_body_root=b'\x00'*32, + signature=b'\x00'*96 + ) + header_2 = deepcopy(header_1) + header_2.previous_block_root = b'\x02'*32 + header_2.slot = slot + 1 + + domain = get_domain( + fork=test_state.fork, + epoch=get_current_epoch(test_state), + domain_type=spec.DOMAIN_BEACON_BLOCK, + ) + header_1.signature = bls.sign( + message_hash=signed_root(header_1), + privkey=privkey, + domain=domain, + ) + header_2.signature = bls.sign( + message_hash=signed_root(header_2), + privkey=privkey, + domain=domain, + ) + + proposer_slashing = ProposerSlashing( + proposer_index=validator_index, + header_1=header_1, + header_2=header_2, + ) + + # + # Add to state via block transition + # + block = construct_empty_block_for_next_slot(test_state) + block.body.proposer_slashings.append(proposer_slashing) + state_transition(test_state, block) + + assert not state.validator_registry[validator_index].initiated_exit + assert not state.validator_registry[validator_index].slashed + + slashed_validator = test_state.validator_registry[validator_index] + assert not slashed_validator.initiated_exit + assert slashed_validator.slashed + assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH + assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH + # lost whistleblower reward + assert test_state.validator_balances[validator_index] < state.validator_balances[validator_index] + + return state, [block], test_state + + +def test_deposit_in_block(state, deposit_data_leaves, pubkeys, privkeys): + pre_state = deepcopy(state) + test_deposit_data_leaves = deepcopy(deposit_data_leaves) + + index = len(test_deposit_data_leaves) + pubkey = pubkeys[index] + privkey = privkeys[index] + deposit_data = create_deposit_data(pre_state, pubkey, privkey, spec.MAX_DEPOSIT_AMOUNT) + + item = hash(deposit_data.serialize()) + test_deposit_data_leaves.append(item) + tree = calc_merkle_tree_from_leaves(tuple(test_deposit_data_leaves)) + root = get_merkle_root((tuple(test_deposit_data_leaves))) + proof = list(get_merkle_proof(tree, item_index=index)) + assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, root) + + deposit = Deposit( + proof=list(proof), + index=index, + deposit_data=deposit_data, + ) + + pre_state.latest_eth1_data.deposit_root = root + post_state = deepcopy(pre_state) + block = construct_empty_block_for_next_slot(post_state) + block.body.deposits.append(deposit) + + state_transition(post_state, block) + assert len(post_state.validator_registry) == len(state.validator_registry) + 1 + assert len(post_state.validator_balances) == len(state.validator_balances) + 1 + assert post_state.validator_registry[index].pubkey == pubkeys[index] + + return pre_state, [block], post_state + + +def test_deposit_top_up(state, pubkeys, privkeys, deposit_data_leaves): + pre_state = deepcopy(state) + test_deposit_data_leaves = deepcopy(deposit_data_leaves) + + validator_index = 0 + amount = spec.MAX_DEPOSIT_AMOUNT // 4 + pubkey = pubkeys[validator_index] + privkey = privkeys[validator_index] + deposit_data = create_deposit_data(pre_state, pubkey, privkey, amount) + + merkle_index = len(test_deposit_data_leaves) + item = hash(deposit_data.serialize()) + test_deposit_data_leaves.append(item) + tree = calc_merkle_tree_from_leaves(tuple(test_deposit_data_leaves)) + root = get_merkle_root((tuple(test_deposit_data_leaves))) + proof = list(get_merkle_proof(tree, item_index=merkle_index)) + assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, merkle_index, root) + + deposit = Deposit( + proof=list(proof), + index=merkle_index, + deposit_data=deposit_data, + ) + + pre_state.latest_eth1_data.deposit_root = root + block = construct_empty_block_for_next_slot(pre_state) + block.body.deposits.append(deposit) + + pre_balance = pre_state.validator_balances[validator_index] + post_state = deepcopy(pre_state) + state_transition(post_state, block) + assert len(post_state.validator_registry) == len(pre_state.validator_registry) + assert len(post_state.validator_balances) == len(pre_state.validator_balances) + assert post_state.validator_balances[validator_index] == pre_balance + amount + + return pre_state, [block], post_state + + +def test_attestation(state, pubkeys, privkeys): + test_state = deepcopy(state) + slot = state.slot + shard = state.current_shuffling_start_shard + attestation_data = build_attestation_data(state, slot, shard) + + crosslink_committees = get_crosslink_committees_at_slot(state, slot) + crosslink_committee = [committee for committee, _shard in crosslink_committees if _shard == attestation_data.shard][0] + + committee_size = len(crosslink_committee) + bitfield_length = (committee_size + 7) // 8 + aggregation_bitfield = b'\x01' + b'\x00' * (bitfield_length - 1) + custody_bitfield = b'\x00' * bitfield_length + attestation = Attestation( + aggregation_bitfield=aggregation_bitfield, + data=attestation_data, + custody_bitfield=custody_bitfield, + aggregate_signature=b'\x00'*96, + ) + participants = get_attestation_participants( + test_state, + attestation.data, + attestation.aggregation_bitfield, + ) + assert len(participants) == 1 + + validator_index = participants[0] + pubkey = pubkeys[validator_index] + privkey = privkeys[validator_index] + + message_hash = AttestationDataAndCustodyBit( + data=attestation.data, + custody_bit=0b0, + ).hash_tree_root() + + attestation.aggregation_signature = bls.sign( + message_hash=message_hash, + privkey=privkey, + domain=get_domain( + fork=test_state.fork, + epoch=get_current_epoch(test_state), + domain_type=spec.DOMAIN_ATTESTATION, + ) + ) + + # + # Add to state via block transition + # + attestation_block = construct_empty_block_for_next_slot(test_state) + attestation_block.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + attestation_block.body.attestations.append(attestation) + state_transition(test_state, attestation_block) + + assert len(test_state.current_epoch_attestations) == len(state.current_epoch_attestations) + 1 + + # + # Epoch transition should move to previous_epoch_attestations + # + pre_current_epoch_attestations = deepcopy(test_state.current_epoch_attestations) + + epoch_block = construct_empty_block_for_next_slot(test_state) + epoch_block.slot += spec.SLOTS_PER_EPOCH + state_transition(test_state, epoch_block) + + assert len(test_state.current_epoch_attestations) == 0 + assert test_state.previous_epoch_attestations == pre_current_epoch_attestations + + return state, [attestation_block, epoch_block], test_state + + +def test_voluntary_exit(state, pubkeys, privkeys): + pre_state = deepcopy(state) + validator_index = get_active_validator_indices(pre_state.validator_registry, get_current_epoch(pre_state))[-1] + pubkey = pubkeys[validator_index] + + # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit + pre_state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # artificially trigger registry update at next epoch transition + pre_state.validator_registry_update_epoch -= 1 + + post_state = deepcopy(pre_state) + + voluntary_exit = VoluntaryExit( + epoch=get_current_epoch(pre_state), + validator_index=validator_index, + signature=b'\x00'*96, + ) + voluntary_exit.signature = bls.sign( + message_hash=signed_root(voluntary_exit), + privkey=privkeys[validator_index], + domain=get_domain( + fork=pre_state.fork, + epoch=get_current_epoch(pre_state), + domain_type=spec.DOMAIN_VOLUNTARY_EXIT, + ) + ) + + # + # Add to state via block transition + # + initiate_exit_block = construct_empty_block_for_next_slot(post_state) + initiate_exit_block.body.voluntary_exits.append(voluntary_exit) + state_transition(post_state, initiate_exit_block) + + assert not pre_state.validator_registry[validator_index].initiated_exit + assert post_state.validator_registry[validator_index].initiated_exit + assert post_state.validator_registry[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH + + # + # Process within epoch transition + # + exit_block = construct_empty_block_for_next_slot(post_state) + exit_block.slot += spec.SLOTS_PER_EPOCH + state_transition(post_state, exit_block) + + assert post_state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH + + return pre_state, [initiate_exit_block, exit_block], post_state + + +def test_transfer(state, pubkeys, privkeys): + pre_state = deepcopy(state) + current_epoch = get_current_epoch(pre_state) + sender_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[-1] + recipient_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0] + transfer_pubkey = pubkeys[-1] + transfer_privkey = privkeys[-1] + amount = pre_state.validator_balances[sender_index] + pre_transfer_recipient_balance = pre_state.validator_balances[recipient_index] + transfer = Transfer( + sender=sender_index, + recipient=recipient_index, + amount=amount, + fee=0, + slot=pre_state.slot + 1, + pubkey=transfer_pubkey, + signature=b'\x00'*96, + ) + transfer.signature = bls.sign( + message_hash=signed_root(transfer), + privkey=transfer_privkey, + domain=get_domain( + fork=pre_state.fork, + epoch=get_current_epoch(pre_state), + domain_type=spec.DOMAIN_TRANSFER, + ) + ) + + # ensure withdrawal_credentials reproducable + pre_state.validator_registry[sender_index].withdrawal_credentials = ( + spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(transfer_pubkey)[1:] + ) + # un-activate so validator can transfer + pre_state.validator_registry[sender_index].activation_epoch = spec.FAR_FUTURE_EPOCH + + post_state = deepcopy(pre_state) + # + # Add to state via block transition + # + block = construct_empty_block_for_next_slot(post_state) + block.body.transfers.append(transfer) + state_transition(post_state, block) + + sender_balance = post_state.validator_balances[sender_index] + recipient_balance = post_state.validator_balances[recipient_index] + assert sender_balance == 0 + assert recipient_balance == pre_transfer_recipient_balance + amount + + return pre_state, [block], post_state + + +def test_ejection(state): + pre_state = deepcopy(state) + + current_epoch = get_current_epoch(pre_state) + validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[-1] + + assert pre_state.validator_registry[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH + + # set validator balance to below ejection threshold + pre_state.validator_balances[validator_index] = spec.EJECTION_BALANCE - 1 + + post_state = deepcopy(pre_state) + # + # trigger epoch transition + # + block = construct_empty_block_for_next_slot(post_state) + block.slot += spec.SLOTS_PER_EPOCH + state_transition(post_state, block) + + assert post_state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH + + return pre_state, [block], post_state + + +def test_historical_batch(state): + pre_state = deepcopy(state) + pre_state.slot += spec.SLOTS_PER_HISTORICAL_ROOT - (pre_state.slot % spec.SLOTS_PER_HISTORICAL_ROOT) - 1 + + post_state = deepcopy(pre_state) + + block = construct_empty_block_for_next_slot(post_state) + + state_transition(post_state, block) + + assert post_state.slot == block.slot + assert get_current_epoch(post_state) % (spec.SLOTS_PER_HISTORICAL_ROOT // spec.SLOTS_PER_EPOCH) == 0 + assert len(post_state.historical_roots) == len(pre_state.historical_roots) + 1 + + return pre_state, [block], post_state + + +def sanity_tests(num_validators=100, config=None): + print(f"Buidling state with {num_validators} validators...") + if config: + overwrite_spec_config(config) + genesis_state = create_genesis_state(num_validators=num_validators) + print("done!") + print() + + test_cases = [] + + print("Running some sanity check tests...\n") + test_slot_transition(genesis_state) + print("Passed slot transition test\n") + test_cases.append( + generate_from_test(test_empty_block_transition, genesis_state, config=config, fields=['slot']) + ) + print("Passed empty block transition test\n") + test_cases.append( + generate_from_test(test_skipped_slots, genesis_state, config=config, fields=['slot', 'latest_block_roots']) + ) + print("Passed skipped slot test\n") + test_cases.append( + generate_from_test(test_empty_epoch_transition, genesis_state, config=config, fields=['slot', 'latest_block_roots']) + ) + print("Passed empty epoch transition test\n") + test_cases.append( + generate_from_test(test_empty_epoch_transition_not_finalizing, genesis_state, config=config, fields=['slot', 'finalized_epoch']) + ) + print("Passed non-finalizing epoch test\n") + test_cases.append( + generate_from_test(test_proposer_slashing, genesis_state, config=config, fields=['validator_registry', 'validator_balances']) + ) + print("Passed proposer slashing test\n") + test_cases.append( + generate_from_test(test_attestation, genesis_state, config=config, fields=['previous_epoch_attestations', 'current_epoch_attestations']) + ) + print("Passed attestation test\n") + test_cases.append( + generate_from_test(test_deposit_in_block, genesis_state, config=config, fields=['validator_registry', 'validator_balances']) + ) + print("Passed deposit test\n") + test_cases.append( + generate_from_test(test_deposit_top_up, genesis_state, config=config, fields=['validator_registry', 'validator_balances']) + ) + print("Passed deposit top up test\n") + test_cases.append( + generate_from_test(test_voluntary_exit, genesis_state, config=config, fields=['validator_registry']) + ) + print("Passed voluntary exit test\n") + test_cases.append( + generate_from_test(test_transfer, genesis_state, config=config, fields=['validator_balances']) + ) + print("Passed transfer test\n") + test_cases.append( + generate_from_test(test_ejection, genesis_state, config=config, fields=['validator_registry']) + ) + print("Passed ejection test\n") + test_cases.append( + generate_from_test(test_historical_batch, genesis_state, config=config, fields=['historical_roots']) + ) + print("Passed historical batch test\n") + print("done!") + + return test_cases + + +if __name__ == "__main__": + config = { + "SHARD_COUNT": 8, + "MIN_ATTESTATION_INCLUSION_DELAY": 2, + "TARGET_COMMITTEE_SIZE": 4, + "SLOTS_PER_EPOCH": 8, + "GENESIS_EPOCH": spec.GENESIS_SLOT // 8, + "SLOTS_PER_HISTORICAL_ROOT": 64, + "LATEST_RANDAO_MIXES_LENGTH": 64, + "LATEST_ACTIVE_INDEX_ROOTS_LENGTH": 64, + "LATEST_SLASHED_EXIT_LENGTH": 64, + } + + test_cases = sanity_tests(32, config) + # uncomment below to run/generate against the default config + # test_cases = sanity_tests(100) + + test = {} + metadata = {} + metadata['title'] = "Sanity tests" + metadata['summary'] = "Basic sanity checks from phase 0 spec pythonization. All tests are run with `verify_signatures` as set to False." + metadata['test_suite'] = "beacon_state" + metadata['fork'] = "tchaikovsky" + metadata['version'] = "v0.5.0" + test['metadata'] = metadata + test['test_cases'] = test_cases + + if '--output-json' in sys.argv: + os.makedirs('output', exist_ok=True) + with open("output/sanity_check_tests.json", "w+") as outfile: + dump_json(test, outfile) + if '--output-yaml' in sys.argv: + os.makedirs('output', exist_ok=True) + with open("output/sanity_check_tests.yaml", "w+") as outfile: + dump_yaml(test, outfile) From f41caa713b167d986acfdbeb61fb9f1bb8d1ce81 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 Mar 2019 13:06:45 -0600 Subject: [PATCH 03/11] add circleci config --- .circleci/config.yml | 46 ++++++++++++++++++++++++++++++++++++++++++++ .gitignore | 2 ++ Makefile | 4 +--- 3 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..98d2367c9 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,46 @@ +# Python CircleCI 2.0 configuration file +version: 2 +jobs: + build: + docker: + - image: circleci/python:3.6 + working_directory: ~/repo + + steps: + - checkout + # Download and cache dependencies + - restore_cache: + keys: + - v1-dependencies-{{ checksum "requirements.txt" }} + # fallback to using the latest cache if no exact match is found + - v1-dependencies- + + - run: + name: install dependencies + command: | + python3 -m venv venv + . venv/bin/activate + pip install -r requirements.txt + - run: + name: build phase0 spec + command: make build/phase0 + + - save_cache: + paths: + - ./venv + key: v1-dependencies-{{ checksum "requirements.txt" }} + + # run tests! + # this example uses Django's built-in test-runner + # other common Python testing frameworks include pytest and nose + # https://pytest.org + # https://nose.readthedocs.io + - run: + name: run tests + command: | + . venv/bin/activate + pytest tests + + - store_artifacts: + path: test-reports + destination: test-reports \ No newline at end of file diff --git a/.gitignore b/.gitignore index 5e19cd2a5..f9a966cea 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ /__pycache__ /venv /.pytest_cache + +build/phase0/spec.py diff --git a/Makefile b/Makefile index 745f8f901..8be65fd44 100644 --- a/Makefile +++ b/Makefile @@ -2,10 +2,8 @@ SPEC_DIR = ./specs SCRIPT_DIR = ./scripts BUILD_DIR = ./build -.PHONY: clean all +.PHONY: clean all $(BUILD_DIR)/phase0 $(BUILD_DIR)/phase0: - mkdir -p $@ python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $@/spec.py - touch $(BUILD_DIR)/__init__.py $(BUILD_DIR)/phase0/__init__.py From 6715a0d4cc0c7b4842dac7cd88264de0f46ccde6 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 Mar 2019 14:14:26 -0600 Subject: [PATCH 04/11] reconfigure build a bit --- .circleci/config.yml | 7 +- .gitignore | 2 +- Makefile | 6 ++ build/phase0/spec.py | 4 +- build/phase0/state_transition.py | 4 +- build/utils/monkey_patches.py | 29 ----- scripts/phase0/build_spec.py | 4 +- tests/phase0/conftest.py | 2 +- tests/phase0/test_sanity.py | 4 +- {build => utils}/__init__.py | 0 {build/utils => utils/phase0}/__init__.py | 0 {build/utils => utils/phase0}/bls_stub.py | 0 .../utils => utils/phase0}/hash_function.py | 0 .../utils => utils/phase0}/merkle_minimal.py | 0 {build/utils => utils/phase0}/minimal_ssz.py | 0 utils/phase0/state_transition.py | 100 ++++++++++++++++++ 16 files changed, 117 insertions(+), 45 deletions(-) delete mode 100644 build/utils/monkey_patches.py rename {build => utils}/__init__.py (100%) rename {build/utils => utils/phase0}/__init__.py (100%) rename {build/utils => utils/phase0}/bls_stub.py (100%) rename {build/utils => utils/phase0}/hash_function.py (100%) rename {build/utils => utils/phase0}/merkle_minimal.py (100%) rename {build/utils => utils/phase0}/minimal_ssz.py (100%) create mode 100644 utils/phase0/state_transition.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 98d2367c9..02871530e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,11 +30,6 @@ jobs: - ./venv key: v1-dependencies-{{ checksum "requirements.txt" }} - # run tests! - # this example uses Django's built-in test-runner - # other common Python testing frameworks include pytest and nose - # https://pytest.org - # https://nose.readthedocs.io - run: name: run tests command: | @@ -43,4 +38,4 @@ jobs: - store_artifacts: path: test-reports - destination: test-reports \ No newline at end of file + destination: test-reports diff --git a/.gitignore b/.gitignore index f9a966cea..dfb38d170 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ /venv /.pytest_cache -build/phase0/spec.py +build/ diff --git a/Makefile b/Makefile index 8be65fd44..593ea8bf4 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,15 @@ SPEC_DIR = ./specs SCRIPT_DIR = ./scripts BUILD_DIR = ./build +UTILS_DIR = ./utils .PHONY: clean all $(BUILD_DIR)/phase0 $(BUILD_DIR)/phase0: + mkdir -p $@ python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $@/spec.py + mkdir -p $@/utils + cp $(UTILS_DIR)/phase0/* $@/utils + cp $(UTILS_DIR)/phase0/state_transition.py $@ + touch $@/__init__.py $@/utils/__init__.py diff --git a/build/phase0/spec.py b/build/phase0/spec.py index 8c05b1208..3fd82c33f 100644 --- a/build/phase0/spec.py +++ b/build/phase0/spec.py @@ -1,5 +1,5 @@ -from build.utils.minimal_ssz import * -from build.utils.bls_stub import * +from build.phase0.utils.minimal_ssz import * +from build.phase0.utils.bls_stub import * def int_to_bytes1(x): return x.to_bytes(1, 'little') def int_to_bytes2(x): return x.to_bytes(2, 'little') def int_to_bytes3(x): return x.to_bytes(3, 'little') diff --git a/build/phase0/state_transition.py b/build/phase0/state_transition.py index 2bd33f6d6..170f647ab 100644 --- a/build/phase0/state_transition.py +++ b/build/phase0/state_transition.py @@ -1,4 +1,4 @@ -import build.phase0.spec as spec +from . import spec from typing import ( @@ -9,7 +9,7 @@ from typing import ( Tuple, ) -from build.phase0.spec import ( +from .spec import ( BeaconState, BeaconBlock, ) diff --git a/build/utils/monkey_patches.py b/build/utils/monkey_patches.py deleted file mode 100644 index 8a35b8f27..000000000 --- a/build/utils/monkey_patches.py +++ /dev/null @@ -1,29 +0,0 @@ -# Monkey patch validator shuffling cache -_get_shuffling = get_shuffling -shuffling_cache = {} -def get_shuffling(seed: Bytes32, - validators: List[Validator], - epoch: Epoch) -> List[List[ValidatorIndex]]: - - param_hash = (seed, hash_tree_root(validators, [Validator]), epoch) - - if param_hash in shuffling_cache: - # print("Cache hit, epoch={0}".format(epoch)) - return shuffling_cache[param_hash] - else: - # print("Cache miss, epoch={0}".format(epoch)) - ret = _get_shuffling(seed, validators, epoch) - shuffling_cache[param_hash] = ret - return ret - - -# Monkey patch hash cache -_hash = hash -hash_cache = {} -def hash(x): - if x in hash_cache: - return hash_cache[x] - else: - ret = _hash(x) - hash_cache[x] = ret - return ret diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index 8b5941b62..eb4f580bd 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -5,8 +5,8 @@ import function_puller def build_spec(sourcefile, outfile): code_lines = [] - code_lines.append("from build.utils.minimal_ssz import *") - code_lines.append("from build.utils.bls_stub import *") + code_lines.append("from build.phase0.utils.minimal_ssz import *") + code_lines.append("from build.phase0.utils.bls_stub import *") for i in (1, 2, 3, 4, 8, 32, 48, 96): code_lines.append("def int_to_bytes%d(x): return x.to_bytes(%d, 'little')" % (i, i)) code_lines.append("SLOTS_PER_EPOCH = 64") # stub, will get overwritten by real var diff --git a/tests/phase0/conftest.py b/tests/phase0/conftest.py index 7e2800afd..7d372f164 100644 --- a/tests/phase0/conftest.py +++ b/tests/phase0/conftest.py @@ -4,7 +4,7 @@ from py_ecc import bls from build.phase0 import spec -from build.utils.merkle_minimal import ( +from build.phase0.utils.merkle_minimal import ( calc_merkle_tree_from_leaves, get_merkle_proof, get_merkle_root, diff --git a/tests/phase0/test_sanity.py b/tests/phase0/test_sanity.py index 573c3ba21..0e04df5dd 100644 --- a/tests/phase0/test_sanity.py +++ b/tests/phase0/test_sanity.py @@ -6,7 +6,7 @@ from copy import deepcopy from py_ecc import bls import build.phase0.spec as spec -from build.utils.minimal_ssz import signed_root +from build.phase0.utils.minimal_ssz import signed_root from build.phase0.spec import ( # SSZ Attestation, @@ -43,7 +43,7 @@ from build.phase0.spec import ( from build.phase0.state_transition import ( state_transition, ) -from build.utils.merkle_minimal import ( +from build.phase0.utils.merkle_minimal import ( calc_merkle_tree_from_leaves, get_merkle_proof, get_merkle_root, diff --git a/build/__init__.py b/utils/__init__.py similarity index 100% rename from build/__init__.py rename to utils/__init__.py diff --git a/build/utils/__init__.py b/utils/phase0/__init__.py similarity index 100% rename from build/utils/__init__.py rename to utils/phase0/__init__.py diff --git a/build/utils/bls_stub.py b/utils/phase0/bls_stub.py similarity index 100% rename from build/utils/bls_stub.py rename to utils/phase0/bls_stub.py diff --git a/build/utils/hash_function.py b/utils/phase0/hash_function.py similarity index 100% rename from build/utils/hash_function.py rename to utils/phase0/hash_function.py diff --git a/build/utils/merkle_minimal.py b/utils/phase0/merkle_minimal.py similarity index 100% rename from build/utils/merkle_minimal.py rename to utils/phase0/merkle_minimal.py diff --git a/build/utils/minimal_ssz.py b/utils/phase0/minimal_ssz.py similarity index 100% rename from build/utils/minimal_ssz.py rename to utils/phase0/minimal_ssz.py diff --git a/utils/phase0/state_transition.py b/utils/phase0/state_transition.py new file mode 100644 index 000000000..170f647ab --- /dev/null +++ b/utils/phase0/state_transition.py @@ -0,0 +1,100 @@ +from . import spec + + +from typing import ( + Any, + Callable, + List, + NewType, + Tuple, +) + +from .spec import ( + BeaconState, + BeaconBlock, +) + + +def process_transaction_type(state: BeaconState, + transactions: List[Any], + max_transactions: int, + tx_fn: Callable[[BeaconState, Any], None]) -> None: + assert len(transactions) <= max_transactions + for transaction in transactions: + tx_fn(state, transaction) + + +def process_transactions(state: BeaconState, block: BeaconBlock) -> None: + process_transaction_type( + state, + block.body.proposer_slashings, + spec.MAX_PROPOSER_SLASHINGS, + spec.process_proposer_slashing, + ) + process_transaction_type( + state, + block.body.attester_slashings, + spec.MAX_ATTESTER_SLASHINGS, + spec.process_attester_slashing, + ) + process_transaction_type( + state, + block.body.attestations, + spec.MAX_ATTESTATIONS, + spec.process_attestation, + ) + process_transaction_type( + state, + block.body.deposits, + spec.MAX_DEPOSITS, + spec.process_deposit, + ) + process_transaction_type( + state, + block.body.voluntary_exits, + spec.MAX_VOLUNTARY_EXITS, + spec.process_voluntary_exit, + ) + assert len(block.body.transfers) == len(set(block.body.transfers)) + process_transaction_type( + state, + block.body.transfers, + spec.MAX_TRANSFERS, + spec.process_transfer, + ) + + +def process_block(state: BeaconState, + block: BeaconBlock, + verify_state_root: bool=False) -> None: + spec.process_block_header(state, block) + spec.process_randao(state, block) + spec.process_eth1_data(state, block) + + process_transactions(state, block) + if verify_state_root: + spec.verify_block_state_root(state, block) + + +def process_epoch_transition(state: BeaconState) -> None: + spec.update_justification_and_finalization(state) + spec.process_crosslinks(state) + spec.maybe_reset_eth1_period(state) + spec.apply_rewards(state) + spec.process_ejections(state) + spec.update_registry_and_shuffling_data(state) + spec.process_slashings(state) + spec.process_exit_queue(state) + spec.finish_epoch_update(state) + + +def state_transition(state: BeaconState, + block: BeaconBlock, + verify_state_root: bool=False) -> BeaconState: + while state.slot < block.slot: + spec.cache_state(state) + if (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0: + process_epoch_transition(state) + spec.advance_slot(state) + if block.slot == state.slot: + process_block(state, block) From d9ac06e8edb814ba12128ae1fe0164496320a77c Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 Mar 2019 14:16:04 -0600 Subject: [PATCH 05/11] remove build from git --- build/phase0/__init__.py | 0 build/phase0/spec.py | 1620 ------------------------------ build/phase0/state_transition.py | 100 -- 3 files changed, 1720 deletions(-) delete mode 100644 build/phase0/__init__.py delete mode 100644 build/phase0/spec.py delete mode 100644 build/phase0/state_transition.py diff --git a/build/phase0/__init__.py b/build/phase0/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/build/phase0/spec.py b/build/phase0/spec.py deleted file mode 100644 index 3fd82c33f..000000000 --- a/build/phase0/spec.py +++ /dev/null @@ -1,1620 +0,0 @@ -from build.phase0.utils.minimal_ssz import * -from build.phase0.utils.bls_stub import * -def int_to_bytes1(x): return x.to_bytes(1, 'little') -def int_to_bytes2(x): return x.to_bytes(2, 'little') -def int_to_bytes3(x): return x.to_bytes(3, 'little') -def int_to_bytes4(x): return x.to_bytes(4, 'little') -def int_to_bytes8(x): return x.to_bytes(8, 'little') -def int_to_bytes32(x): return x.to_bytes(32, 'little') -def int_to_bytes48(x): return x.to_bytes(48, 'little') -def int_to_bytes96(x): return x.to_bytes(96, 'little') -SLOTS_PER_EPOCH = 64 -def slot_to_epoch(x): return x // SLOTS_PER_EPOCH - -from typing import ( - Any, - Callable, - List, - NewType, - Tuple, -) - - -Slot = NewType('Slot', int) # uint64 -Epoch = NewType('Epoch', int) # uint64 -Shard = NewType('Shard', int) # uint64 -ValidatorIndex = NewType('ValidatorIndex', int) # uint64 -Gwei = NewType('Gwei', int) # uint64 -Bytes32 = NewType('Bytes32', bytes) # bytes32 -BLSPubkey = NewType('BLSPubkey', bytes) # bytes48 -BLSSignature = NewType('BLSSignature', bytes) # bytes96 -Any = None -Store = None - -SHARD_COUNT = 2**10 -TARGET_COMMITTEE_SIZE = 2**7 -MAX_BALANCE_CHURN_QUOTIENT = 2**5 -MAX_INDICES_PER_SLASHABLE_VOTE = 2**12 -MAX_EXIT_DEQUEUES_PER_EPOCH = 2**2 -SHUFFLE_ROUND_COUNT = 90 -DEPOSIT_CONTRACT_ADDRESS = 0x1234567890123567890123456789012357890 -DEPOSIT_CONTRACT_TREE_DEPTH = 2**5 -MIN_DEPOSIT_AMOUNT = 2**0 * 10**9 -MAX_DEPOSIT_AMOUNT = 2**5 * 10**9 -FORK_CHOICE_BALANCE_INCREMENT = 2**0 * 10**9 -EJECTION_BALANCE = 2**4 * 10**9 -GENESIS_FORK_VERSION = 0 -GENESIS_SLOT = 2**32 -GENESIS_EPOCH = slot_to_epoch(GENESIS_SLOT) -GENESIS_START_SHARD = 0 -FAR_FUTURE_EPOCH = 2**64 - 1 -ZERO_HASH = int_to_bytes32(0) -EMPTY_SIGNATURE = int_to_bytes96(0) -BLS_WITHDRAWAL_PREFIX_BYTE = int_to_bytes1(0) -SECONDS_PER_SLOT = 6 -MIN_ATTESTATION_INCLUSION_DELAY = 2**2 -SLOTS_PER_EPOCH = 2**6 -MIN_SEED_LOOKAHEAD = 2**0 -ACTIVATION_EXIT_DELAY = 2**2 -EPOCHS_PER_ETH1_VOTING_PERIOD = 2**4 -SLOTS_PER_HISTORICAL_ROOT = 2**13 -MIN_VALIDATOR_WITHDRAWABILITY_DELAY = 2**8 -PERSISTENT_COMMITTEE_PERIOD = 2**11 -LATEST_RANDAO_MIXES_LENGTH = 2**13 -LATEST_ACTIVE_INDEX_ROOTS_LENGTH = 2**13 -LATEST_SLASHED_EXIT_LENGTH = 2**13 -BASE_REWARD_QUOTIENT = 2**5 -WHISTLEBLOWER_REWARD_QUOTIENT = 2**9 -ATTESTATION_INCLUSION_REWARD_QUOTIENT = 2**3 -INACTIVITY_PENALTY_QUOTIENT = 2**24 -MIN_PENALTY_QUOTIENT = 2**5 -MAX_PROPOSER_SLASHINGS = 2**4 -MAX_ATTESTER_SLASHINGS = 2**0 -MAX_ATTESTATIONS = 2**7 -MAX_DEPOSITS = 2**4 -MAX_VOLUNTARY_EXITS = 2**4 -MAX_TRANSFERS = 2**4 -DOMAIN_BEACON_BLOCK = 0 -DOMAIN_RANDAO = 1 -DOMAIN_ATTESTATION = 2 -DOMAIN_DEPOSIT = 3 -DOMAIN_VOLUNTARY_EXIT = 4 -DOMAIN_TRANSFER = 5 -Fork = SSZType({ - # Previous fork version - 'previous_version': 'bytes4', - # Current fork version - 'current_version': 'bytes4', - # Fork epoch number - 'epoch': 'uint64', -}) -Crosslink = SSZType({ - # Epoch number - 'epoch': 'uint64', - # Shard data since the previous crosslink - 'crosslink_data_root': 'bytes32', -}) -Eth1Data = SSZType({ - # Root of the deposit tree - 'deposit_root': 'bytes32', - # Block hash - 'block_hash': 'bytes32', -}) -Eth1DataVote = SSZType({ - # Data being voted for - 'eth1_data': Eth1Data, - # Vote count - 'vote_count': 'uint64', -}) -AttestationData = SSZType({ - # LMD GHOST vote - 'slot': 'uint64', - 'beacon_block_root': 'bytes32', - - # FFG vote - 'source_epoch': 'uint64', - 'source_root': 'bytes32', - 'target_root': 'bytes32', - - # Crosslink vote - 'shard': 'uint64', - 'previous_crosslink': Crosslink, - 'crosslink_data_root': 'bytes32', -}) -AttestationDataAndCustodyBit = SSZType({ - # Attestation data - 'data': AttestationData, - # Custody bit - 'custody_bit': 'bool', -}) -SlashableAttestation = SSZType({ - # Validator indices - 'validator_indices': ['uint64'], - # Attestation data - 'data': AttestationData, - # Custody bitfield - 'custody_bitfield': 'bytes', - # Aggregate signature - 'aggregate_signature': 'bytes96', -}) -DepositInput = SSZType({ - # BLS pubkey - 'pubkey': 'bytes48', - # Withdrawal credentials - 'withdrawal_credentials': 'bytes32', - # A BLS signature of this `DepositInput` - 'proof_of_possession': 'bytes96', -}) -DepositData = SSZType({ - # Amount in Gwei - 'amount': 'uint64', - # Timestamp from deposit contract - 'timestamp': 'uint64', - # Deposit input - 'deposit_input': DepositInput, -}) -BeaconBlockHeader = SSZType({ - 'slot': 'uint64', - 'previous_block_root': 'bytes32', - 'state_root': 'bytes32', - 'block_body_root': 'bytes32', - 'signature': 'bytes96', -}) -Validator = SSZType({ - # BLS public key - 'pubkey': 'bytes48', - # Withdrawal credentials - 'withdrawal_credentials': 'bytes32', - # Epoch when validator activated - 'activation_epoch': 'uint64', - # Epoch when validator exited - 'exit_epoch': 'uint64', - # Epoch when validator is eligible to withdraw - 'withdrawable_epoch': 'uint64', - # Did the validator initiate an exit - 'initiated_exit': 'bool', - # Was the validator slashed - 'slashed': 'bool', -}) -PendingAttestation = SSZType({ - # Attester aggregation bitfield - 'aggregation_bitfield': 'bytes', - # Attestation data - 'data': AttestationData, - # Custody bitfield - 'custody_bitfield': 'bytes', - # Inclusion slot - 'inclusion_slot': 'uint64', -}) -HistoricalBatch = SSZType({ - # Block roots - 'block_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], - # State roots - 'state_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], -}) -ProposerSlashing = SSZType({ - # Proposer index - 'proposer_index': 'uint64', - # First block header - 'header_1': BeaconBlockHeader, - # Second block header - 'header_2': BeaconBlockHeader, -}) -AttesterSlashing = SSZType({ - # First slashable attestation - 'slashable_attestation_1': SlashableAttestation, - # Second slashable attestation - 'slashable_attestation_2': SlashableAttestation, -}) -Attestation = SSZType({ - # Attester aggregation bitfield - 'aggregation_bitfield': 'bytes', - # Attestation data - 'data': AttestationData, - # Custody bitfield - 'custody_bitfield': 'bytes', - # BLS aggregate signature - 'aggregate_signature': 'bytes96', -}) -Deposit = SSZType({ - # Branch in the deposit tree - 'proof': ['bytes32', DEPOSIT_CONTRACT_TREE_DEPTH], - # Index in the deposit tree - 'index': 'uint64', - # Data - 'deposit_data': DepositData, -}) -VoluntaryExit = SSZType({ - # Minimum epoch for processing exit - 'epoch': 'uint64', - # Index of the exiting validator - 'validator_index': 'uint64', - # Validator signature - 'signature': 'bytes96', -}) -Transfer = SSZType({ - # Sender index - 'sender': 'uint64', - # Recipient index - 'recipient': 'uint64', - # Amount in Gwei - 'amount': 'uint64', - # Fee in Gwei for block proposer - 'fee': 'uint64', - # Inclusion slot - 'slot': 'uint64', - # Sender withdrawal pubkey - 'pubkey': 'bytes48', - # Sender signature - 'signature': 'bytes96', -}) -BeaconBlockBody = SSZType({ - 'randao_reveal': 'bytes96', - 'eth1_data': Eth1Data, - 'proposer_slashings': [ProposerSlashing], - 'attester_slashings': [AttesterSlashing], - 'attestations': [Attestation], - 'deposits': [Deposit], - 'voluntary_exits': [VoluntaryExit], - 'transfers': [Transfer], -}) -BeaconBlock = SSZType({ - # Header - 'slot': 'uint64', - 'previous_block_root': 'bytes32', - 'state_root': 'bytes32', - 'body': BeaconBlockBody, - 'signature': 'bytes96', -}) -BeaconState = SSZType({ - # Misc - 'slot': 'uint64', - 'genesis_time': 'uint64', - 'fork': Fork, # For versioning hard forks - - # Validator registry - 'validator_registry': [Validator], - 'validator_balances': ['uint64'], - 'validator_registry_update_epoch': 'uint64', - - # Randomness and committees - 'latest_randao_mixes': ['bytes32', LATEST_RANDAO_MIXES_LENGTH], - 'previous_shuffling_start_shard': 'uint64', - 'current_shuffling_start_shard': 'uint64', - 'previous_shuffling_epoch': 'uint64', - 'current_shuffling_epoch': 'uint64', - 'previous_shuffling_seed': 'bytes32', - 'current_shuffling_seed': 'bytes32', - - # Finality - 'previous_epoch_attestations': [PendingAttestation], - 'current_epoch_attestations': [PendingAttestation], - 'previous_justified_epoch': 'uint64', - 'current_justified_epoch': 'uint64', - 'previous_justified_root': 'bytes32', - 'current_justified_root': 'bytes32', - 'justification_bitfield': 'uint64', - 'finalized_epoch': 'uint64', - 'finalized_root': 'bytes32', - - # Recent state - 'latest_crosslinks': [Crosslink, SHARD_COUNT], - 'latest_block_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], - 'latest_state_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], - 'latest_active_index_roots': ['bytes32', LATEST_ACTIVE_INDEX_ROOTS_LENGTH], - 'latest_slashed_balances': ['uint64', LATEST_SLASHED_EXIT_LENGTH], # Balances slashed at every withdrawal period - 'latest_block_header': BeaconBlockHeader, # `latest_block_header.state_root == ZERO_HASH` temporarily - 'historical_roots': ['bytes32'], - - # Ethereum 1.0 chain data - 'latest_eth1_data': Eth1Data, - 'eth1_data_votes': [Eth1DataVote], - 'deposit_index': 'uint64' -}) -def xor(bytes1: Bytes32, bytes2: Bytes32) -> Bytes32: - return bytes(a ^ b for a, b in zip(bytes1, bytes2)) -def get_temporary_block_header(block: BeaconBlock) -> BeaconBlockHeader: - """ - Return the block header corresponding to a block with ``state_root`` set to ``ZERO_HASH``. - """ - return BeaconBlockHeader( - slot=block.slot, - previous_block_root=block.previous_block_root, - state_root=ZERO_HASH, - block_body_root=hash_tree_root(block.body), - signature=block.signature, - ) -def slot_to_epoch(slot: Slot) -> Epoch: - """ - Return the epoch number of the given ``slot``. - """ - return slot // SLOTS_PER_EPOCH -def get_previous_epoch(state: BeaconState) -> Epoch: - """` - Return the previous epoch of the given ``state``. - """ - return get_current_epoch(state) - 1 -def get_current_epoch(state: BeaconState) -> Epoch: - """ - Return the current epoch of the given ``state``. - """ - return slot_to_epoch(state.slot) -def get_epoch_start_slot(epoch: Epoch) -> Slot: - """ - Return the starting slot of the given ``epoch``. - """ - return epoch * SLOTS_PER_EPOCH -def is_active_validator(validator: Validator, epoch: Epoch) -> bool: - """ - Check if ``validator`` is active. - """ - return validator.activation_epoch <= epoch < validator.exit_epoch -def get_active_validator_indices(validators: List[Validator], epoch: Epoch) -> List[ValidatorIndex]: - """ - Get indices of active validators from ``validators``. - """ - return [i for i, v in enumerate(validators) if is_active_validator(v, epoch)] -def get_permuted_index(index: int, list_size: int, seed: Bytes32) -> int: - """ - Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy. - - Utilizes 'swap or not' shuffling found in - https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf - See the 'generalized domain' algorithm on page 3. - """ - assert index < list_size - assert list_size <= 2**40 - - for round in range(SHUFFLE_ROUND_COUNT): - pivot = bytes_to_int(hash(seed + int_to_bytes1(round))[0:8]) % list_size - flip = (pivot - index) % list_size - position = max(index, flip) - source = hash(seed + int_to_bytes1(round) + int_to_bytes4(position // 256)) - byte = source[(position % 256) // 8] - bit = (byte >> (position % 8)) % 2 - index = flip if bit else index - - return index -def split(values: List[Any], split_count: int) -> List[List[Any]]: - """ - Splits ``values`` into ``split_count`` pieces. - """ - list_length = len(values) - return [ - values[(list_length * i // split_count): (list_length * (i + 1) // split_count)] - for i in range(split_count) - ] -def get_epoch_committee_count(active_validator_count: int) -> int: - """ - Return the number of committees in one epoch. - """ - return max( - 1, - min( - SHARD_COUNT // SLOTS_PER_EPOCH, - active_validator_count // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE, - ) - ) * SLOTS_PER_EPOCH -def get_shuffling(seed: Bytes32, - validators: List[Validator], - epoch: Epoch) -> List[List[ValidatorIndex]]: - """ - Shuffle active validators and split into crosslink committees. - Return a list of committees (each a list of validator indices). - """ - # Shuffle active validator indices - active_validator_indices = get_active_validator_indices(validators, epoch) - length = len(active_validator_indices) - shuffled_indices = [active_validator_indices[get_permuted_index(i, length, seed)] for i in range(length)] - - # Split the shuffled active validator indices - return split(shuffled_indices, get_epoch_committee_count(length)) -def get_previous_epoch_committee_count(state: BeaconState) -> int: - """ - Return the number of committees in the previous epoch of the given ``state``. - """ - previous_active_validators = get_active_validator_indices( - state.validator_registry, - state.previous_shuffling_epoch, - ) - return get_epoch_committee_count(len(previous_active_validators)) -def get_current_epoch_committee_count(state: BeaconState) -> int: - """ - Return the number of committees in the current epoch of the given ``state``. - """ - current_active_validators = get_active_validator_indices( - state.validator_registry, - state.current_shuffling_epoch, - ) - return get_epoch_committee_count(len(current_active_validators)) -def get_next_epoch_committee_count(state: BeaconState) -> int: - """ - Return the number of committees in the next epoch of the given ``state``. - """ - next_active_validators = get_active_validator_indices( - state.validator_registry, - get_current_epoch(state) + 1, - ) - return get_epoch_committee_count(len(next_active_validators)) -def get_crosslink_committees_at_slot(state: BeaconState, - slot: Slot, - registry_change: bool=False) -> List[Tuple[List[ValidatorIndex], Shard]]: - """ - Return the list of ``(committee, shard)`` tuples for the ``slot``. - - Note: There are two possible shufflings for crosslink committees for a - ``slot`` in the next epoch -- with and without a `registry_change` - """ - epoch = slot_to_epoch(slot) - current_epoch = get_current_epoch(state) - previous_epoch = get_previous_epoch(state) - next_epoch = current_epoch + 1 - - assert previous_epoch <= epoch <= next_epoch - - if epoch == current_epoch: - committees_per_epoch = get_current_epoch_committee_count(state) - seed = state.current_shuffling_seed - shuffling_epoch = state.current_shuffling_epoch - shuffling_start_shard = state.current_shuffling_start_shard - elif epoch == previous_epoch: - committees_per_epoch = get_previous_epoch_committee_count(state) - seed = state.previous_shuffling_seed - shuffling_epoch = state.previous_shuffling_epoch - shuffling_start_shard = state.previous_shuffling_start_shard - elif epoch == next_epoch: - epochs_since_last_registry_update = current_epoch - state.validator_registry_update_epoch - if registry_change: - committees_per_epoch = get_next_epoch_committee_count(state) - seed = generate_seed(state, next_epoch) - shuffling_epoch = next_epoch - current_committees_per_epoch = get_current_epoch_committee_count(state) - shuffling_start_shard = (state.current_shuffling_start_shard + current_committees_per_epoch) % SHARD_COUNT - elif epochs_since_last_registry_update > 1 and is_power_of_two(epochs_since_last_registry_update): - committees_per_epoch = get_next_epoch_committee_count(state) - seed = generate_seed(state, next_epoch) - shuffling_epoch = next_epoch - shuffling_start_shard = state.current_shuffling_start_shard - else: - committees_per_epoch = get_current_epoch_committee_count(state) - seed = state.current_shuffling_seed - shuffling_epoch = state.current_shuffling_epoch - shuffling_start_shard = state.current_shuffling_start_shard - - shuffling = get_shuffling( - seed, - state.validator_registry, - shuffling_epoch, - ) - offset = slot % SLOTS_PER_EPOCH - committees_per_slot = committees_per_epoch // SLOTS_PER_EPOCH - slot_start_shard = (shuffling_start_shard + committees_per_slot * offset) % SHARD_COUNT - - return [ - ( - shuffling[committees_per_slot * offset + i], - (slot_start_shard + i) % SHARD_COUNT, - ) - for i in range(committees_per_slot) - ] -def get_block_root(state: BeaconState, - slot: Slot) -> Bytes32: - """ - Return the block root at a recent ``slot``. - """ - assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT - return state.latest_block_roots[slot % SLOTS_PER_HISTORICAL_ROOT] -def get_state_root(state: BeaconState, - slot: Slot) -> Bytes32: - """ - Return the state root at a recent ``slot``. - """ - assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT - return state.latest_state_roots[slot % SLOTS_PER_HISTORICAL_ROOT] -def get_randao_mix(state: BeaconState, - epoch: Epoch) -> Bytes32: - """ - Return the randao mix at a recent ``epoch``. - """ - assert get_current_epoch(state) - LATEST_RANDAO_MIXES_LENGTH < epoch <= get_current_epoch(state) - return state.latest_randao_mixes[epoch % LATEST_RANDAO_MIXES_LENGTH] -def get_active_index_root(state: BeaconState, - epoch: Epoch) -> Bytes32: - """ - Return the index root at a recent ``epoch``. - """ - assert get_current_epoch(state) - LATEST_ACTIVE_INDEX_ROOTS_LENGTH + ACTIVATION_EXIT_DELAY < epoch <= get_current_epoch(state) + ACTIVATION_EXIT_DELAY - return state.latest_active_index_roots[epoch % LATEST_ACTIVE_INDEX_ROOTS_LENGTH] -def generate_seed(state: BeaconState, - epoch: Epoch) -> Bytes32: - """ - Generate a seed for the given ``epoch``. - """ - return hash( - get_randao_mix(state, epoch - MIN_SEED_LOOKAHEAD) + - get_active_index_root(state, epoch) + - int_to_bytes32(epoch) - ) -def get_beacon_proposer_index(state: BeaconState, - slot: Slot, - registry_change: bool=False) -> ValidatorIndex: - """ - Return the beacon proposer index for the ``slot``. - """ - epoch = slot_to_epoch(slot) - current_epoch = get_current_epoch(state) - previous_epoch = get_previous_epoch(state) - next_epoch = current_epoch + 1 - - assert previous_epoch <= epoch <= next_epoch - - first_committee, _ = get_crosslink_committees_at_slot(state, slot, registry_change)[0] - return first_committee[epoch % len(first_committee)] -def verify_merkle_branch(leaf: Bytes32, proof: List[Bytes32], depth: int, index: int, root: Bytes32) -> bool: - """ - Verify that the given ``leaf`` is on the merkle branch ``proof`` - starting with the given ``root``. - """ - value = leaf - for i in range(depth): - if index // (2**i) % 2: - value = hash(proof[i] + value) - else: - value = hash(value + proof[i]) - return value == root -def get_attestation_participants(state: BeaconState, - attestation_data: AttestationData, - bitfield: bytes) -> List[ValidatorIndex]: - """ - Return the participant indices at for the ``attestation_data`` and ``bitfield``. - """ - # Find the committee in the list with the desired shard - crosslink_committees = get_crosslink_committees_at_slot(state, attestation_data.slot) - - assert attestation_data.shard in [shard for _, shard in crosslink_committees] - crosslink_committee = [committee for committee, shard in crosslink_committees if shard == attestation_data.shard][0] - - assert verify_bitfield(bitfield, len(crosslink_committee)) - - # Find the participating attesters in the committee - participants = [] - for i, validator_index in enumerate(crosslink_committee): - aggregation_bit = get_bitfield_bit(bitfield, i) - if aggregation_bit == 0b1: - participants.append(validator_index) - return participants -def is_power_of_two(value: int) -> bool: - """ - Check if ``value`` is a power of two integer. - """ - return (value > 0) and (value & (value - 1) == 0) -def bytes_to_int(data: bytes) -> int: - return int.from_bytes(data, 'little') -def get_effective_balance(state: BeaconState, index: ValidatorIndex) -> Gwei: - """ - Return the effective balance (also known as "balance at stake") for a validator with the given ``index``. - """ - return min(state.validator_balances[index], MAX_DEPOSIT_AMOUNT) -def get_total_balance(state: BeaconState, validators: List[ValidatorIndex]) -> Gwei: - """ - Return the combined effective balance of an array of ``validators``. - """ - return sum([get_effective_balance(state, i) for i in validators]) -def get_fork_version(fork: Fork, - epoch: Epoch) -> bytes: - """ - Return the fork version of the given ``epoch``. - """ - if epoch < fork.epoch: - return fork.previous_version - else: - return fork.current_version -def get_domain(fork: Fork, - epoch: Epoch, - domain_type: int) -> int: - """ - Get the domain number that represents the fork meta and signature domain. - """ - return bytes_to_int(get_fork_version(fork, epoch) + int_to_bytes4(domain_type)) -def get_bitfield_bit(bitfield: bytes, i: int) -> int: - """ - Extract the bit in ``bitfield`` at position ``i``. - """ - return (bitfield[i // 8] >> (i % 8)) % 2 -def verify_bitfield(bitfield: bytes, committee_size: int) -> bool: - """ - Verify ``bitfield`` against the ``committee_size``. - """ - if len(bitfield) != (committee_size + 7) // 8: - return False - - # Check `bitfield` is padded with zero bits only - for i in range(committee_size, len(bitfield) * 8): - if get_bitfield_bit(bitfield, i) == 0b1: - return False - - return True -def verify_slashable_attestation(state: BeaconState, slashable_attestation: SlashableAttestation) -> bool: - """ - Verify validity of ``slashable_attestation`` fields. - """ - if slashable_attestation.custody_bitfield != b'\x00' * len(slashable_attestation.custody_bitfield): # [TO BE REMOVED IN PHASE 1] - return False - - if len(slashable_attestation.validator_indices) == 0: - return False - - for i in range(len(slashable_attestation.validator_indices) - 1): - if slashable_attestation.validator_indices[i] >= slashable_attestation.validator_indices[i + 1]: - return False - - if not verify_bitfield(slashable_attestation.custody_bitfield, len(slashable_attestation.validator_indices)): - return False - - if len(slashable_attestation.validator_indices) > MAX_INDICES_PER_SLASHABLE_VOTE: - return False - - custody_bit_0_indices = [] - custody_bit_1_indices = [] - for i, validator_index in enumerate(slashable_attestation.validator_indices): - if get_bitfield_bit(slashable_attestation.custody_bitfield, i) == 0b0: - custody_bit_0_indices.append(validator_index) - else: - custody_bit_1_indices.append(validator_index) - - return bls_verify_multiple( - pubkeys=[ - bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_0_indices]), - bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_1_indices]), - ], - message_hashes=[ - hash_tree_root(AttestationDataAndCustodyBit(data=slashable_attestation.data, custody_bit=0b0)), - hash_tree_root(AttestationDataAndCustodyBit(data=slashable_attestation.data, custody_bit=0b1)), - ], - signature=slashable_attestation.aggregate_signature, - domain=get_domain(state.fork, slot_to_epoch(slashable_attestation.data.slot), DOMAIN_ATTESTATION), - ) -def is_double_vote(attestation_data_1: AttestationData, - attestation_data_2: AttestationData) -> bool: - """ - Check if ``attestation_data_1`` and ``attestation_data_2`` have the same target. - """ - target_epoch_1 = slot_to_epoch(attestation_data_1.slot) - target_epoch_2 = slot_to_epoch(attestation_data_2.slot) - return target_epoch_1 == target_epoch_2 -def is_surround_vote(attestation_data_1: AttestationData, - attestation_data_2: AttestationData) -> bool: - """ - Check if ``attestation_data_1`` surrounds ``attestation_data_2``. - """ - source_epoch_1 = attestation_data_1.source_epoch - source_epoch_2 = attestation_data_2.source_epoch - target_epoch_1 = slot_to_epoch(attestation_data_1.slot) - target_epoch_2 = slot_to_epoch(attestation_data_2.slot) - - return source_epoch_1 < source_epoch_2 and target_epoch_2 < target_epoch_1 -def integer_squareroot(n: int) -> int: - """ - The largest integer ``x`` such that ``x**2`` is less than or equal to ``n``. - """ - assert n >= 0 - x = n - y = (x + 1) // 2 - while y < x: - x = y - y = (x + n // x) // 2 - return x -def get_delayed_activation_exit_epoch(epoch: Epoch) -> Epoch: - """ - Return the epoch at which an activation or exit triggered in ``epoch`` takes effect. - """ - return epoch + 1 + ACTIVATION_EXIT_DELAY -def process_deposit(state: BeaconState, deposit: Deposit) -> None: - """ - Process a deposit from Ethereum 1.0. - Note that this function mutates ``state``. - """ - deposit_input = deposit.deposit_data.deposit_input - - # Should equal 8 bytes for deposit_data.amount + - # 8 bytes for deposit_data.timestamp + - # 176 bytes for deposit_data.deposit_input - # It should match the deposit_data in the eth1.0 deposit contract - serialized_deposit_data = serialize(deposit.deposit_data) - # Deposits must be processed in order - assert deposit.index == state.deposit_index - - # Verify the Merkle branch - merkle_branch_is_valid = verify_merkle_branch( - leaf=hash(serialized_deposit_data), - proof=deposit.proof, - depth=DEPOSIT_CONTRACT_TREE_DEPTH, - index=deposit.index, - root=state.latest_eth1_data.deposit_root, - ) - assert merkle_branch_is_valid - - # Increment the next deposit index we are expecting. Note that this - # needs to be done here because while the deposit contract will never - # create an invalid Merkle branch, it may admit an invalid deposit - # object, and we need to be able to skip over it - state.deposit_index += 1 - - validator_pubkeys = [v.pubkey for v in state.validator_registry] - pubkey = deposit_input.pubkey - amount = deposit.deposit_data.amount - withdrawal_credentials = deposit_input.withdrawal_credentials - - if pubkey not in validator_pubkeys: - # Verify the proof of possession - proof_is_valid = bls_verify( - pubkey=deposit_input.pubkey, - message_hash=signed_root(deposit_input), - signature=deposit_input.proof_of_possession, - domain=get_domain( - state.fork, - get_current_epoch(state), - DOMAIN_DEPOSIT, - ) - ) - if not proof_is_valid: - return - - # Add new validator - validator = Validator( - pubkey=pubkey, - withdrawal_credentials=withdrawal_credentials, - activation_epoch=FAR_FUTURE_EPOCH, - exit_epoch=FAR_FUTURE_EPOCH, - withdrawable_epoch=FAR_FUTURE_EPOCH, - initiated_exit=False, - slashed=False, - ) - - # Note: In phase 2 registry indices that have been withdrawn for a long time will be recycled. - state.validator_registry.append(validator) - state.validator_balances.append(amount) - else: - # Increase balance by deposit amount - state.validator_balances[validator_pubkeys.index(pubkey)] += amount -def activate_validator(state: BeaconState, index: ValidatorIndex, is_genesis: bool) -> None: - """ - Activate the validator of the given ``index``. - Note that this function mutates ``state``. - """ - validator = state.validator_registry[index] - - validator.activation_epoch = GENESIS_EPOCH if is_genesis else get_delayed_activation_exit_epoch(get_current_epoch(state)) -def initiate_validator_exit(state: BeaconState, index: ValidatorIndex) -> None: - """ - Initiate the validator of the given ``index``. - Note that this function mutates ``state``. - """ - validator = state.validator_registry[index] - validator.initiated_exit = True -def exit_validator(state: BeaconState, index: ValidatorIndex) -> None: - """ - Exit the validator of the given ``index``. - Note that this function mutates ``state``. - """ - validator = state.validator_registry[index] - delayed_activation_exit_epoch = get_delayed_activation_exit_epoch(get_current_epoch(state)) - - # The following updates only occur if not previous exited - if validator.exit_epoch <= delayed_activation_exit_epoch: - return - else: - validator.exit_epoch = delayed_activation_exit_epoch -def slash_validator(state: BeaconState, index: ValidatorIndex) -> None: - """ - Slash the validator with index ``index``. - Note that this function mutates ``state``. - """ - validator = state.validator_registry[index] - assert state.slot < get_epoch_start_slot(validator.withdrawable_epoch) # [TO BE REMOVED IN PHASE 2] - exit_validator(state, index) - state.latest_slashed_balances[get_current_epoch(state) % LATEST_SLASHED_EXIT_LENGTH] += get_effective_balance(state, index) - - whistleblower_index = get_beacon_proposer_index(state, state.slot) - whistleblower_reward = get_effective_balance(state, index) // WHISTLEBLOWER_REWARD_QUOTIENT - state.validator_balances[whistleblower_index] += whistleblower_reward - state.validator_balances[index] -= whistleblower_reward - validator.slashed = True - validator.withdrawable_epoch = get_current_epoch(state) + LATEST_SLASHED_EXIT_LENGTH -def prepare_validator_for_withdrawal(state: BeaconState, index: ValidatorIndex) -> None: - """ - Set the validator with the given ``index`` as withdrawable - ``MIN_VALIDATOR_WITHDRAWABILITY_DELAY`` after the current epoch. - Note that this function mutates ``state``. - """ - validator = state.validator_registry[index] - validator.withdrawable_epoch = get_current_epoch(state) + MIN_VALIDATOR_WITHDRAWABILITY_DELAY -def get_empty_block() -> BeaconBlock: - """ - Get an empty ``BeaconBlock``. - """ - return BeaconBlock( - slot=GENESIS_SLOT, - previous_block_root=ZERO_HASH, - state_root=ZERO_HASH, - body=BeaconBlockBody( - randao_reveal=EMPTY_SIGNATURE, - eth1_data=Eth1Data( - deposit_root=ZERO_HASH, - block_hash=ZERO_HASH, - ), - proposer_slashings=[], - attester_slashings=[], - attestations=[], - deposits=[], - voluntary_exits=[], - transfers=[], - ), - signature=EMPTY_SIGNATURE, - ) -def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], - genesis_time: int, - genesis_eth1_data: Eth1Data) -> BeaconState: - """ - Get the genesis ``BeaconState``. - """ - state = BeaconState( - # Misc - slot=GENESIS_SLOT, - genesis_time=genesis_time, - fork=Fork( - previous_version=int_to_bytes4(GENESIS_FORK_VERSION), - current_version=int_to_bytes4(GENESIS_FORK_VERSION), - epoch=GENESIS_EPOCH, - ), - - # Validator registry - validator_registry=[], - validator_balances=[], - validator_registry_update_epoch=GENESIS_EPOCH, - - # Randomness and committees - latest_randao_mixes=[ZERO_HASH for _ in range(LATEST_RANDAO_MIXES_LENGTH)], - previous_shuffling_start_shard=GENESIS_START_SHARD, - current_shuffling_start_shard=GENESIS_START_SHARD, - previous_shuffling_epoch=GENESIS_EPOCH, - current_shuffling_epoch=GENESIS_EPOCH, - previous_shuffling_seed=ZERO_HASH, - current_shuffling_seed=ZERO_HASH, - - # Finality - previous_epoch_attestations=[], - current_epoch_attestations=[], - previous_justified_epoch=GENESIS_EPOCH, - current_justified_epoch=GENESIS_EPOCH, - previous_justified_root=ZERO_HASH, - current_justified_root=ZERO_HASH, - justification_bitfield=0, - finalized_epoch=GENESIS_EPOCH, - finalized_root=ZERO_HASH, - - # Recent state - latest_crosslinks=[Crosslink(epoch=GENESIS_EPOCH, crosslink_data_root=ZERO_HASH) for _ in range(SHARD_COUNT)], - latest_block_roots=[ZERO_HASH for _ in range(SLOTS_PER_HISTORICAL_ROOT)], - latest_state_roots=[ZERO_HASH for _ in range(SLOTS_PER_HISTORICAL_ROOT)], - latest_active_index_roots=[ZERO_HASH for _ in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH)], - latest_slashed_balances=[0 for _ in range(LATEST_SLASHED_EXIT_LENGTH)], - latest_block_header=get_temporary_block_header(get_empty_block()), - historical_roots=[], - - # Ethereum 1.0 chain data - latest_eth1_data=genesis_eth1_data, - eth1_data_votes=[], - deposit_index=0, - ) - - # Process genesis deposits - for deposit in genesis_validator_deposits: - process_deposit(state, deposit) - - # Process genesis activations - for validator_index, _ in enumerate(state.validator_registry): - if get_effective_balance(state, validator_index) >= MAX_DEPOSIT_AMOUNT: - activate_validator(state, validator_index, is_genesis=True) - - genesis_active_index_root = hash_tree_root(get_active_validator_indices(state.validator_registry, GENESIS_EPOCH)) - for index in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH): - state.latest_active_index_roots[index] = genesis_active_index_root - state.current_shuffling_seed = generate_seed(state, GENESIS_EPOCH) - - return state -def get_ancestor(store: Store, block: BeaconBlock, slot: Slot) -> BeaconBlock: - """ - Get the ancestor of ``block`` with slot number ``slot``; return ``None`` if not found. - """ - if block.slot == slot: - return block - elif block.slot < slot: - return None - else: - return get_ancestor(store, store.get_parent(block), slot) -def lmd_ghost(store: Store, start_state: BeaconState, start_block: BeaconBlock) -> BeaconBlock: - """ - Execute the LMD-GHOST algorithm to find the head ``BeaconBlock``. - """ - validators = start_state.validator_registry - active_validator_indices = get_active_validator_indices(validators, slot_to_epoch(start_state.slot)) - attestation_targets = [ - (validator_index, get_latest_attestation_target(store, validator_index)) - for validator_index in active_validator_indices - ] - - def get_vote_count(block: BeaconBlock) -> int: - return sum( - get_effective_balance(start_state.validator_balances[validator_index]) // FORK_CHOICE_BALANCE_INCREMENT - for validator_index, target in attestation_targets - if get_ancestor(store, target, block.slot) == block - ) - - head = start_block - while 1: - children = get_children(store, head) - if len(children) == 0: - return head - head = max(children, key=lambda x: (get_vote_count(x), hash_tree_root(x))) -def cache_state(state: BeaconState) -> None: - previous_slot_state_root = hash_tree_root(state) - - # store the previous slot's post state transition root - state.latest_state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_slot_state_root - - # cache state root in stored latest_block_header if empty - if state.latest_block_header.state_root == ZERO_HASH: - state.latest_block_header.state_root = previous_slot_state_root - - # store latest known block for previous slot - state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = hash_tree_root(state.latest_block_header) -def get_current_total_balance(state: BeaconState) -> Gwei: - return get_total_balance(state, get_active_validator_indices(state.validator_registry, get_current_epoch(state))) -def get_previous_total_balance(state: BeaconState) -> Gwei: - return get_total_balance(state, get_active_validator_indices(state.validator_registry, get_previous_epoch(state))) -def get_attesting_indices(state: BeaconState, attestations: List[PendingAttestation]) -> List[ValidatorIndex]: - output = set() - for a in attestations: - output = output.union(get_attestation_participants(state, a.data, a.aggregation_bitfield)) - return sorted(list(output)) -def get_attesting_balance(state: BeaconState, attestations: List[PendingAttestation]) -> Gwei: - return get_total_balance(state, get_attesting_indices(state, attestations)) -def get_current_epoch_boundary_attestations(state: BeaconState) -> List[PendingAttestation]: - return [ - a for a in state.current_epoch_attestations - if a.data.target_root == get_block_root(state, get_epoch_start_slot(get_current_epoch(state))) - ] -def get_previous_epoch_boundary_attestations(state: BeaconState) -> List[PendingAttestation]: - return [ - a for a in state.previous_epoch_attestations - if a.data.target_root == get_block_root(state, get_epoch_start_slot(get_previous_epoch(state))) - ] -def get_previous_epoch_matching_head_attestations(state: BeaconState) -> List[PendingAttestation]: - return [ - a for a in state.previous_epoch_attestations - if a.data.beacon_block_root == get_block_root(state, a.data.slot) - ] -def get_winning_root_and_participants(state: BeaconState, shard: Shard) -> Tuple[Bytes32, List[ValidatorIndex]]: - all_attestations = state.current_epoch_attestations + state.previous_epoch_attestations - valid_attestations = [ - a for a in all_attestations if a.data.previous_crosslink == state.latest_crosslinks[shard] - ] - all_roots = [a.data.crosslink_data_root for a in valid_attestations] - - # handle when no attestations for shard available - if len(all_roots) == 0: - return ZERO_HASH, [] - - def get_attestations_for(root) -> List[PendingAttestation]: - return [a for a in valid_attestations if a.data.crosslink_data_root == root] - - # Winning crosslink root is the root with the most votes for it, ties broken in favor of - # lexicographically higher hash - winning_root = max(all_roots, key=lambda r: (get_attesting_balance(state, get_attestations_for(r)), r)) - - return winning_root, get_attesting_indices(state, get_attestations_for(winning_root)) -def earliest_attestation(state: BeaconState, validator_index: ValidatorIndex) -> PendingAttestation: - return min([ - a for a in state.previous_epoch_attestations if - validator_index in get_attestation_participants(state, a.data, a.aggregation_bitfield) - ], key=lambda a: a.inclusion_slot) -def inclusion_slot(state: BeaconState, validator_index: ValidatorIndex) -> Slot: - return earliest_attestation(state, validator_index).inclusion_slot -def inclusion_distance(state: BeaconState, validator_index: ValidatorIndex) -> int: - attestation = earliest_attestation(state, validator_index) - return attestation.inclusion_slot - attestation.data.slot -def update_justification_and_finalization(state: BeaconState) -> None: - new_justified_epoch = state.current_justified_epoch - new_finalized_epoch = state.finalized_epoch - - # Rotate the justification bitfield up one epoch to make room for the current epoch - state.justification_bitfield <<= 1 - # If the previous epoch gets justified, fill the second last bit - previous_boundary_attesting_balance = get_attesting_balance(state, get_previous_epoch_boundary_attestations(state)) - if previous_boundary_attesting_balance * 3 >= get_previous_total_balance(state) * 2: - new_justified_epoch = get_current_epoch(state) - 1 - state.justification_bitfield |= 2 - # If the current epoch gets justified, fill the last bit - current_boundary_attesting_balance = get_attesting_balance(state, get_current_epoch_boundary_attestations(state)) - if current_boundary_attesting_balance * 3 >= get_current_total_balance(state) * 2: - new_justified_epoch = get_current_epoch(state) - state.justification_bitfield |= 1 - - # Process finalizations - bitfield = state.justification_bitfield - current_epoch = get_current_epoch(state) - # The 2nd/3rd/4th most recent epochs are all justified, the 2nd using the 4th as source - if (bitfield >> 1) % 8 == 0b111 and state.previous_justified_epoch == current_epoch - 3: - new_finalized_epoch = state.previous_justified_epoch - # The 2nd/3rd most recent epochs are both justified, the 2nd using the 3rd as source - if (bitfield >> 1) % 4 == 0b11 and state.previous_justified_epoch == current_epoch - 2: - new_finalized_epoch = state.previous_justified_epoch - # The 1st/2nd/3rd most recent epochs are all justified, the 1st using the 3rd as source - if (bitfield >> 0) % 8 == 0b111 and state.current_justified_epoch == current_epoch - 2: - new_finalized_epoch = state.current_justified_epoch - # The 1st/2nd most recent epochs are both justified, the 1st using the 2nd as source - if (bitfield >> 0) % 4 == 0b11 and state.current_justified_epoch == current_epoch - 1: - new_finalized_epoch = state.current_justified_epoch - - # Update state jusification/finality fields - state.previous_justified_epoch = state.current_justified_epoch - state.previous_justified_root = state.current_justified_root - if new_justified_epoch != state.current_justified_epoch: - state.current_justified_epoch = new_justified_epoch - state.current_justified_root = get_block_root(state, get_epoch_start_slot(new_justified_epoch)) - if new_finalized_epoch != state.finalized_epoch: - state.finalized_epoch = new_finalized_epoch - state.finalized_root = get_block_root(state, get_epoch_start_slot(new_finalized_epoch)) -def process_crosslinks(state: BeaconState) -> None: - current_epoch = get_current_epoch(state) - previous_epoch = current_epoch - 1 - next_epoch = current_epoch + 1 - for slot in range(get_epoch_start_slot(previous_epoch), get_epoch_start_slot(next_epoch)): - for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot): - winning_root, participants = get_winning_root_and_participants(state, shard) - participating_balance = get_total_balance(state, participants) - total_balance = get_total_balance(state, crosslink_committee) - if 3 * participating_balance >= 2 * total_balance: - state.latest_crosslinks[shard] = Crosslink( - epoch=slot_to_epoch(slot), - crosslink_data_root=winning_root - ) -def maybe_reset_eth1_period(state: BeaconState) -> None: - if (get_current_epoch(state) + 1) % EPOCHS_PER_ETH1_VOTING_PERIOD == 0: - for eth1_data_vote in state.eth1_data_votes: - # If a majority of all votes were for a particular eth1_data value, - # then set that as the new canonical value - if eth1_data_vote.vote_count * 2 > EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH: - state.latest_eth1_data = eth1_data_vote.eth1_data - state.eth1_data_votes = [] -def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: - if get_previous_total_balance(state) == 0: - return 0 - - adjusted_quotient = integer_squareroot(get_previous_total_balance(state)) // BASE_REWARD_QUOTIENT - return get_effective_balance(state, index) // adjusted_quotient // 5 -def get_inactivity_penalty(state: BeaconState, index: ValidatorIndex, epochs_since_finality: int) -> Gwei: - return ( - get_base_reward(state, index) + - get_effective_balance(state, index) * epochs_since_finality // INACTIVITY_PENALTY_QUOTIENT // 2 - ) -def get_justification_and_finalization_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: - epochs_since_finality = get_current_epoch(state) + 1 - state.finalized_epoch - if epochs_since_finality <= 4: - return compute_normal_justification_and_finalization_deltas(state) - else: - return compute_inactivity_leak_deltas(state) -def compute_normal_justification_and_finalization_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: - # deltas[0] for rewards - # deltas[1] for penalties - deltas = [ - [0 for index in range(len(state.validator_registry))], - [0 for index in range(len(state.validator_registry))] - ] - # Some helper variables - boundary_attestations = get_previous_epoch_boundary_attestations(state) - boundary_attesting_balance = get_attesting_balance(state, boundary_attestations) - total_balance = get_previous_total_balance(state) - total_attesting_balance = get_attesting_balance(state, state.previous_epoch_attestations) - matching_head_attestations = get_previous_epoch_matching_head_attestations(state) - matching_head_balance = get_attesting_balance(state, matching_head_attestations) - # Process rewards or penalties for all validators - for index in get_active_validator_indices(state.validator_registry, get_previous_epoch(state)): - # Expected FFG source - if index in get_attesting_indices(state, state.previous_epoch_attestations): - deltas[0][index] += get_base_reward(state, index) * total_attesting_balance // total_balance - # Inclusion speed bonus - deltas[0][index] += ( - get_base_reward(state, index) * MIN_ATTESTATION_INCLUSION_DELAY // - inclusion_distance(state, index) - ) - else: - deltas[1][index] += get_base_reward(state, index) - # Expected FFG target - if index in get_attesting_indices(state, boundary_attestations): - deltas[0][index] += get_base_reward(state, index) * boundary_attesting_balance // total_balance - else: - deltas[1][index] += get_base_reward(state, index) - # Expected head - if index in get_attesting_indices(state, matching_head_attestations): - deltas[0][index] += get_base_reward(state, index) * matching_head_balance // total_balance - else: - deltas[1][index] += get_base_reward(state, index) - # Proposer bonus - if index in get_attesting_indices(state, state.previous_epoch_attestations): - proposer_index = get_beacon_proposer_index(state, inclusion_slot(state, index)) - deltas[0][proposer_index] += get_base_reward(state, index) // ATTESTATION_INCLUSION_REWARD_QUOTIENT - return deltas -def compute_inactivity_leak_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: - # deltas[0] for rewards - # deltas[1] for penalties - deltas = [ - [0 for index in range(len(state.validator_registry))], - [0 for index in range(len(state.validator_registry))] - ] - boundary_attestations = get_previous_epoch_boundary_attestations(state) - matching_head_attestations = get_previous_epoch_matching_head_attestations(state) - active_validator_indices = get_active_validator_indices(state.validator_registry, get_previous_epoch(state)) - epochs_since_finality = get_current_epoch(state) + 1 - state.finalized_epoch - for index in active_validator_indices: - if index not in get_attesting_indices(state, state.previous_epoch_attestations): - deltas[1][index] += get_inactivity_penalty(state, index, epochs_since_finality) - else: - # If a validator did attest, apply a small penalty for getting attestations included late - deltas[0][index] += ( - get_base_reward(state, index) * MIN_ATTESTATION_INCLUSION_DELAY // - inclusion_distance(state, index) - ) - deltas[1][index] += get_base_reward(state, index) - if index not in get_attesting_indices(state, boundary_attestations): - deltas[1][index] += get_inactivity_penalty(state, index, epochs_since_finality) - if index not in get_attesting_indices(state, matching_head_attestations): - deltas[1][index] += get_base_reward(state, index) - # Penalize slashed-but-inactive validators as though they were active but offline - for index in range(len(state.validator_registry)): - eligible = ( - index not in active_validator_indices and - state.validator_registry[index].slashed and - get_current_epoch(state) < state.validator_registry[index].withdrawable_epoch - ) - if eligible: - deltas[1][index] += ( - 2 * get_inactivity_penalty(state, index, epochs_since_finality) + - get_base_reward(state, index) - ) - return deltas -def get_crosslink_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: - # deltas[0] for rewards - # deltas[1] for penalties - deltas = [ - [0 for index in range(len(state.validator_registry))], - [0 for index in range(len(state.validator_registry))] - ] - previous_epoch_start_slot = get_epoch_start_slot(get_previous_epoch(state)) - current_epoch_start_slot = get_epoch_start_slot(get_current_epoch(state)) - for slot in range(previous_epoch_start_slot, current_epoch_start_slot): - for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot): - winning_root, participants = get_winning_root_and_participants(state, shard) - participating_balance = get_total_balance(state, participants) - total_balance = get_total_balance(state, crosslink_committee) - for index in crosslink_committee: - if index in participants: - deltas[0][index] += get_base_reward(state, index) * participating_balance // total_balance - else: - deltas[1][index] += get_base_reward(state, index) - return deltas -def apply_rewards(state: BeaconState) -> None: - deltas1 = get_justification_and_finalization_deltas(state) - deltas2 = get_crosslink_deltas(state) - for i in range(len(state.validator_registry)): - state.validator_balances[i] = max( - 0, - state.validator_balances[i] + deltas1[0][i] + deltas2[0][i] - deltas1[1][i] - deltas2[1][i] - ) -def process_ejections(state: BeaconState) -> None: - """ - Iterate through the validator registry - and eject active validators with balance below ``EJECTION_BALANCE``. - """ - for index in get_active_validator_indices(state.validator_registry, get_current_epoch(state)): - if state.validator_balances[index] < EJECTION_BALANCE: - exit_validator(state, index) -def should_update_validator_registry(state: BeaconState) -> bool: - # Must have finalized a new block - if state.finalized_epoch <= state.validator_registry_update_epoch: - return False - # Must have processed new crosslinks on all shards of the current epoch - shards_to_check = [ - (state.current_shuffling_start_shard + i) % SHARD_COUNT - for i in range(get_current_epoch_committee_count(state)) - ] - for shard in shards_to_check: - if state.latest_crosslinks[shard].epoch <= state.validator_registry_update_epoch: - return False - return True -def update_validator_registry(state: BeaconState) -> None: - """ - Update validator registry. - Note that this function mutates ``state``. - """ - current_epoch = get_current_epoch(state) - # The active validators - active_validator_indices = get_active_validator_indices(state.validator_registry, current_epoch) - # The total effective balance of active validators - total_balance = get_total_balance(state, active_validator_indices) - - # The maximum balance churn in Gwei (for deposits and exits separately) - max_balance_churn = max( - MAX_DEPOSIT_AMOUNT, - total_balance // (2 * MAX_BALANCE_CHURN_QUOTIENT) - ) - - # Activate validators within the allowable balance churn - balance_churn = 0 - for index, validator in enumerate(state.validator_registry): - if validator.activation_epoch == FAR_FUTURE_EPOCH and state.validator_balances[index] >= MAX_DEPOSIT_AMOUNT: - # Check the balance churn would be within the allowance - balance_churn += get_effective_balance(state, index) - if balance_churn > max_balance_churn: - break - - # Activate validator - activate_validator(state, index, is_genesis=False) - - # Exit validators within the allowable balance churn - balance_churn = 0 - for index, validator in enumerate(state.validator_registry): - if validator.exit_epoch == FAR_FUTURE_EPOCH and validator.initiated_exit: - # Check the balance churn would be within the allowance - balance_churn += get_effective_balance(state, index) - if balance_churn > max_balance_churn: - break - - # Exit validator - exit_validator(state, index) - - state.validator_registry_update_epoch = current_epoch -def update_registry_and_shuffling_data(state: BeaconState) -> None: - # First set previous shuffling data to current shuffling data - state.previous_shuffling_epoch = state.current_shuffling_epoch - state.previous_shuffling_start_shard = state.current_shuffling_start_shard - state.previous_shuffling_seed = state.current_shuffling_seed - current_epoch = get_current_epoch(state) - next_epoch = current_epoch + 1 - # Check if we should update, and if so, update - if should_update_validator_registry(state): - update_validator_registry(state) - # If we update the registry, update the shuffling data and shards as well - state.current_shuffling_epoch = next_epoch - state.current_shuffling_start_shard = ( - state.current_shuffling_start_shard + - get_current_epoch_committee_count(state) % SHARD_COUNT - ) - state.current_shuffling_seed = generate_seed(state, state.current_shuffling_epoch) - else: - # If processing at least one crosslink keeps failing, then reshuffle every power of two, - # but don't update the current_shuffling_start_shard - epochs_since_last_registry_update = current_epoch - state.validator_registry_update_epoch - if epochs_since_last_registry_update > 1 and is_power_of_two(epochs_since_last_registry_update): - state.current_shuffling_epoch = next_epoch - state.current_shuffling_seed = generate_seed(state, state.current_shuffling_epoch) -def process_slashings(state: BeaconState) -> None: - """ - Process the slashings. - Note that this function mutates ``state``. - """ - current_epoch = get_current_epoch(state) - active_validator_indices = get_active_validator_indices(state.validator_registry, current_epoch) - total_balance = get_total_balance(state, active_validator_indices) - - # Compute `total_penalties` - total_at_start = state.latest_slashed_balances[(current_epoch + 1) % LATEST_SLASHED_EXIT_LENGTH] - total_at_end = state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH] - total_penalties = total_at_end - total_at_start - - for index, validator in enumerate(state.validator_registry): - if validator.slashed and current_epoch == validator.withdrawable_epoch - LATEST_SLASHED_EXIT_LENGTH // 2: - penalty = max( - get_effective_balance(state, index) * min(total_penalties * 3, total_balance) // total_balance, - get_effective_balance(state, index) // MIN_PENALTY_QUOTIENT - ) - state.validator_balances[index] -= penalty -def process_exit_queue(state: BeaconState) -> None: - """ - Process the exit queue. - Note that this function mutates ``state``. - """ - def eligible(index): - validator = state.validator_registry[index] - # Filter out dequeued validators - if validator.withdrawable_epoch != FAR_FUTURE_EPOCH: - return False - # Dequeue if the minimum amount of time has passed - else: - return get_current_epoch(state) >= validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY - - eligible_indices = filter(eligible, list(range(len(state.validator_registry)))) - # Sort in order of exit epoch, and validators that exit within the same epoch exit in order of validator index - sorted_indices = sorted(eligible_indices, key=lambda index: state.validator_registry[index].exit_epoch) - for dequeues, index in enumerate(sorted_indices): - if dequeues >= MAX_EXIT_DEQUEUES_PER_EPOCH: - break - prepare_validator_for_withdrawal(state, index) -def finish_epoch_update(state: BeaconState) -> None: - current_epoch = get_current_epoch(state) - next_epoch = current_epoch + 1 - # Set active index root - index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % LATEST_ACTIVE_INDEX_ROOTS_LENGTH - state.latest_active_index_roots[index_root_position] = hash_tree_root( - get_active_validator_indices(state.validator_registry, next_epoch + ACTIVATION_EXIT_DELAY) - ) - # Set total slashed balances - state.latest_slashed_balances[next_epoch % LATEST_SLASHED_EXIT_LENGTH] = ( - state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH] - ) - # Set randao mix - state.latest_randao_mixes[next_epoch % LATEST_RANDAO_MIXES_LENGTH] = get_randao_mix(state, current_epoch) - # Set historical root accumulator - if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0: - historical_batch = HistoricalBatch( - block_roots=state.latest_block_roots, - state_roots=state.latest_state_roots, - ) - state.historical_roots.append(hash_tree_root(historical_batch)) - # Rotate current/previous epoch attestations - state.previous_epoch_attestations = state.current_epoch_attestations - state.current_epoch_attestations = [] -def advance_slot(state: BeaconState) -> None: - state.slot += 1 -def process_block_header(state: BeaconState, block: BeaconBlock) -> None: - # Verify that the slots match - assert block.slot == state.slot - # Verify that the parent matches - assert block.previous_block_root == hash_tree_root(state.latest_block_header) - # Save current block as the new latest block - state.latest_block_header = get_temporary_block_header(block) - # Verify proposer signature - proposer = state.validator_registry[get_beacon_proposer_index(state, state.slot)] - assert bls_verify( - pubkey=proposer.pubkey, - message_hash=signed_root(block), - signature=block.signature, - domain=get_domain(state.fork, get_current_epoch(state), DOMAIN_BEACON_BLOCK) - ) -def process_randao(state: BeaconState, block: BeaconBlock) -> None: - proposer = state.validator_registry[get_beacon_proposer_index(state, state.slot)] - # Verify that the provided randao value is valid - assert bls_verify( - pubkey=proposer.pubkey, - message_hash=hash_tree_root(get_current_epoch(state)), - signature=block.body.randao_reveal, - domain=get_domain(state.fork, get_current_epoch(state), DOMAIN_RANDAO) - ) - # Mix it in - state.latest_randao_mixes[get_current_epoch(state) % LATEST_RANDAO_MIXES_LENGTH] = ( - xor(get_randao_mix(state, get_current_epoch(state)), - hash(block.body.randao_reveal)) - ) -def process_eth1_data(state: BeaconState, block: BeaconBlock) -> None: - for eth1_data_vote in state.eth1_data_votes: - # If someone else has already voted for the same hash, add to its counter - if eth1_data_vote.eth1_data == block.body.eth1_data: - eth1_data_vote.vote_count += 1 - return - # If we're seeing this hash for the first time, make a new counter - state.eth1_data_votes.append(Eth1DataVote(eth1_data=block.body.eth1_data, vote_count=1)) -def process_proposer_slashing(state: BeaconState, - proposer_slashing: ProposerSlashing) -> None: - """ - Process ``ProposerSlashing`` transaction. - Note that this function mutates ``state``. - """ - proposer = state.validator_registry[proposer_slashing.proposer_index] - # Verify that the epoch is the same - assert slot_to_epoch(proposer_slashing.header_1.slot) == slot_to_epoch(proposer_slashing.header_2.slot) - # But the headers are different - assert proposer_slashing.header_1 != proposer_slashing.header_2 - # Proposer is not yet slashed - assert proposer.slashed is False - # Signatures are valid - for header in (proposer_slashing.header_1, proposer_slashing.header_2): - assert bls_verify( - pubkey=proposer.pubkey, - message_hash=signed_root(header), - signature=header.signature, - domain=get_domain(state.fork, slot_to_epoch(header.slot), DOMAIN_BEACON_BLOCK) - ) - slash_validator(state, proposer_slashing.proposer_index) -def process_attester_slashing(state: BeaconState, - attester_slashing: AttesterSlashing) -> None: - """ - Process ``AttesterSlashing`` transaction. - Note that this function mutates ``state``. - """ - attestation1 = attester_slashing.slashable_attestation_1 - attestation2 = attester_slashing.slashable_attestation_2 - # Check that the attestations are conflicting - assert attestation1.data != attestation2.data - assert ( - is_double_vote(attestation1.data, attestation2.data) or - is_surround_vote(attestation1.data, attestation2.data) - ) - assert verify_slashable_attestation(state, attestation1) - assert verify_slashable_attestation(state, attestation2) - slashable_indices = [ - index for index in attestation1.validator_indices - if ( - index in attestation2.validator_indices and - state.validator_registry[index].slashed is False - ) - ] - assert len(slashable_indices) >= 1 - for index in slashable_indices: - slash_validator(state, index) -def process_attestation(state: BeaconState, attestation: Attestation) -> None: - """ - Process ``Attestation`` transaction. - Note that this function mutates ``state``. - """ - # Can't submit attestations that are too far in history (or in prehistory) - assert attestation.data.slot >= GENESIS_SLOT - assert state.slot <= attestation.data.slot + SLOTS_PER_EPOCH - # Can't submit attestations too quickly - assert attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot - # Verify that the justified epoch and root is correct - if slot_to_epoch(attestation.data.slot) >= get_current_epoch(state): - # Case 1: current epoch attestations - assert attestation.data.source_epoch == state.current_justified_epoch - assert attestation.data.source_root == state.current_justified_root - else: - # Case 2: previous epoch attestations - assert attestation.data.source_epoch == state.previous_justified_epoch - assert attestation.data.source_root == state.previous_justified_root - # Check that the crosslink data is valid - acceptable_crosslink_data = { - # Case 1: Latest crosslink matches the one in the state - attestation.data.previous_crosslink, - # Case 2: State has already been updated, state's latest crosslink matches the crosslink - # the attestation is trying to create - Crosslink( - crosslink_data_root=attestation.data.crosslink_data_root, - epoch=slot_to_epoch(attestation.data.slot) - ) - } - assert state.latest_crosslinks[attestation.data.shard] in acceptable_crosslink_data - # Attestation must be nonempty! - assert attestation.aggregation_bitfield != b'\x00' * len(attestation.aggregation_bitfield) - # Custody must be empty (to be removed in phase 1) - assert attestation.custody_bitfield == b'\x00' * len(attestation.custody_bitfield) - # Get the committee for the specific shard that this attestation is for - crosslink_committee = [ - committee for committee, shard in get_crosslink_committees_at_slot(state, attestation.data.slot) - if shard == attestation.data.shard - ][0] - # Custody bitfield must be a subset of the attestation bitfield - for i in range(len(crosslink_committee)): - if get_bitfield_bit(attestation.aggregation_bitfield, i) == 0b0: - assert get_bitfield_bit(attestation.custody_bitfield, i) == 0b0 - # Verify aggregate signature - participants = get_attestation_participants(state, attestation.data, attestation.aggregation_bitfield) - custody_bit_1_participants = get_attestation_participants(state, attestation.data, attestation.custody_bitfield) - custody_bit_0_participants = [i for i in participants if i not in custody_bit_1_participants] - - assert bls_verify_multiple( - pubkeys=[ - bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_0_participants]), - bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_1_participants]), - ], - message_hashes=[ - hash_tree_root(AttestationDataAndCustodyBit(data=attestation.data, custody_bit=0b0)), - hash_tree_root(AttestationDataAndCustodyBit(data=attestation.data, custody_bit=0b1)), - ], - signature=attestation.aggregate_signature, - domain=get_domain(state.fork, slot_to_epoch(attestation.data.slot), DOMAIN_ATTESTATION), - ) - # Crosslink data root is zero (to be removed in phase 1) - assert attestation.data.crosslink_data_root == ZERO_HASH - # Apply the attestation - pending_attestation = PendingAttestation( - data=attestation.data, - aggregation_bitfield=attestation.aggregation_bitfield, - custody_bitfield=attestation.custody_bitfield, - inclusion_slot=state.slot - ) - if slot_to_epoch(attestation.data.slot) == get_current_epoch(state): - state.current_epoch_attestations.append(pending_attestation) - elif slot_to_epoch(attestation.data.slot) == get_previous_epoch(state): - state.previous_epoch_attestations.append(pending_attestation) -def process_voluntary_exit(state: BeaconState, exit: VoluntaryExit) -> None: - """ - Process ``VoluntaryExit`` transaction. - Note that this function mutates ``state``. - """ - validator = state.validator_registry[exit.validator_index] - # Verify the validator has not yet exited - assert validator.exit_epoch == FAR_FUTURE_EPOCH - # Verify the validator has not initiated an exit - assert validator.initiated_exit is False - # Exits must specify an epoch when they become valid; they are not valid before then - assert get_current_epoch(state) >= exit.epoch - # Must have been in the validator set long enough - assert get_current_epoch(state) - validator.activation_epoch >= PERSISTENT_COMMITTEE_PERIOD - # Verify signature - assert bls_verify( - pubkey=validator.pubkey, - message_hash=signed_root(exit), - signature=exit.signature, - domain=get_domain(state.fork, exit.epoch, DOMAIN_VOLUNTARY_EXIT) - ) - # Run the exit - initiate_validator_exit(state, exit.validator_index) -def process_transfer(state: BeaconState, transfer: Transfer) -> None: - """ - Process ``Transfer`` transaction. - Note that this function mutates ``state``. - """ - # Verify the amount and fee aren't individually too big (for anti-overflow purposes) - assert state.validator_balances[transfer.sender] >= max(transfer.amount, transfer.fee) - # Verify that we have enough ETH to send, and that after the transfer the balance will be either - # exactly zero or at least MIN_DEPOSIT_AMOUNT - assert ( - state.validator_balances[transfer.sender] == transfer.amount + transfer.fee or - state.validator_balances[transfer.sender] >= transfer.amount + transfer.fee + MIN_DEPOSIT_AMOUNT - ) - # A transfer is valid in only one slot - assert state.slot == transfer.slot - # Only withdrawn or not-yet-deposited accounts can transfer - assert ( - get_current_epoch(state) >= state.validator_registry[transfer.sender].withdrawable_epoch or - state.validator_registry[transfer.sender].activation_epoch == FAR_FUTURE_EPOCH - ) - # Verify that the pubkey is valid - assert ( - state.validator_registry[transfer.sender].withdrawal_credentials == - BLS_WITHDRAWAL_PREFIX_BYTE + hash(transfer.pubkey)[1:] - ) - # Verify that the signature is valid - assert bls_verify( - pubkey=transfer.pubkey, - message_hash=signed_root(transfer), - signature=transfer.signature, - domain=get_domain(state.fork, slot_to_epoch(transfer.slot), DOMAIN_TRANSFER) - ) - # Process the transfer - state.validator_balances[transfer.sender] -= transfer.amount + transfer.fee - state.validator_balances[transfer.recipient] += transfer.amount - state.validator_balances[get_beacon_proposer_index(state, state.slot)] += transfer.fee -def verify_block_state_root(state: BeaconState, block: BeaconBlock) -> None: - assert block.state_root == hash_tree_root(state) - -# Monkey patch validator shuffling cache -_get_shuffling = get_shuffling -shuffling_cache = {} -def get_shuffling(seed: Bytes32, - validators: List[Validator], - epoch: Epoch) -> List[List[ValidatorIndex]]: - - param_hash = (seed, hash_tree_root(validators, [Validator]), epoch) - - if param_hash in shuffling_cache: - # print("Cache hit, epoch={0}".format(epoch)) - return shuffling_cache[param_hash] - else: - # print("Cache miss, epoch={0}".format(epoch)) - ret = _get_shuffling(seed, validators, epoch) - shuffling_cache[param_hash] = ret - return ret - - -# Monkey patch hash cache -_hash = hash -hash_cache = {} -def hash(x): - if x in hash_cache: - return hash_cache[x] - else: - ret = _hash(x) - hash_cache[x] = ret - return ret - \ No newline at end of file diff --git a/build/phase0/state_transition.py b/build/phase0/state_transition.py deleted file mode 100644 index 170f647ab..000000000 --- a/build/phase0/state_transition.py +++ /dev/null @@ -1,100 +0,0 @@ -from . import spec - - -from typing import ( - Any, - Callable, - List, - NewType, - Tuple, -) - -from .spec import ( - BeaconState, - BeaconBlock, -) - - -def process_transaction_type(state: BeaconState, - transactions: List[Any], - max_transactions: int, - tx_fn: Callable[[BeaconState, Any], None]) -> None: - assert len(transactions) <= max_transactions - for transaction in transactions: - tx_fn(state, transaction) - - -def process_transactions(state: BeaconState, block: BeaconBlock) -> None: - process_transaction_type( - state, - block.body.proposer_slashings, - spec.MAX_PROPOSER_SLASHINGS, - spec.process_proposer_slashing, - ) - process_transaction_type( - state, - block.body.attester_slashings, - spec.MAX_ATTESTER_SLASHINGS, - spec.process_attester_slashing, - ) - process_transaction_type( - state, - block.body.attestations, - spec.MAX_ATTESTATIONS, - spec.process_attestation, - ) - process_transaction_type( - state, - block.body.deposits, - spec.MAX_DEPOSITS, - spec.process_deposit, - ) - process_transaction_type( - state, - block.body.voluntary_exits, - spec.MAX_VOLUNTARY_EXITS, - spec.process_voluntary_exit, - ) - assert len(block.body.transfers) == len(set(block.body.transfers)) - process_transaction_type( - state, - block.body.transfers, - spec.MAX_TRANSFERS, - spec.process_transfer, - ) - - -def process_block(state: BeaconState, - block: BeaconBlock, - verify_state_root: bool=False) -> None: - spec.process_block_header(state, block) - spec.process_randao(state, block) - spec.process_eth1_data(state, block) - - process_transactions(state, block) - if verify_state_root: - spec.verify_block_state_root(state, block) - - -def process_epoch_transition(state: BeaconState) -> None: - spec.update_justification_and_finalization(state) - spec.process_crosslinks(state) - spec.maybe_reset_eth1_period(state) - spec.apply_rewards(state) - spec.process_ejections(state) - spec.update_registry_and_shuffling_data(state) - spec.process_slashings(state) - spec.process_exit_queue(state) - spec.finish_epoch_update(state) - - -def state_transition(state: BeaconState, - block: BeaconBlock, - verify_state_root: bool=False) -> BeaconState: - while state.slot < block.slot: - spec.cache_state(state) - if (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0: - process_epoch_transition(state) - spec.advance_slot(state) - if block.slot == state.slot: - process_block(state, block) From 55c337a35c7cb6caff0d1fe6f3179d2b3161a579 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 Mar 2019 16:20:24 -0600 Subject: [PATCH 06/11] seperate tests 'sanity' and 'minimal-config' vs all --- Makefile | 13 ++- tests/phase0/conftest.py | 102 +++++------------ tests/phase0/helpers.py | 139 +++++++++++++++++++++++ tests/phase0/test_sanity.py | 216 +++++------------------------------- 4 files changed, 202 insertions(+), 268 deletions(-) create mode 100644 tests/phase0/helpers.py diff --git a/Makefile b/Makefile index 593ea8bf4..f0f7557e9 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,19 @@ SCRIPT_DIR = ./scripts BUILD_DIR = ./build UTILS_DIR = ./utils -.PHONY: clean all $(BUILD_DIR)/phase0 +.PHONY: clean all test + + +all: $(BUILD_DIR)/phase0 + + +clean: + rm -rf $(BUILD_DIR) + + +test: + pytest -m "sanity and minimal_config" tests/ $(BUILD_DIR)/phase0: mkdir -p $@ diff --git a/tests/phase0/conftest.py b/tests/phase0/conftest.py index 7d372f164..e92896e92 100644 --- a/tests/phase0/conftest.py +++ b/tests/phase0/conftest.py @@ -1,28 +1,27 @@ import pytest -from py_ecc import bls - from build.phase0 import spec -from build.phase0.utils.merkle_minimal import ( - calc_merkle_tree_from_leaves, - get_merkle_proof, - get_merkle_root, -) -from build.phase0.spec import ( - Deposit, - DepositData, - DepositInput, - Eth1Data, - get_genesis_beacon_state, - verify_merkle_branch, - hash, +from tests.phase0.helpers import ( + privkeys_list, + pubkeys_list, + create_genesis_state, ) -privkeys_list = [i+1 for i in range(1000)] -pubkeys_list = [bls.privtopub(privkey) for privkey in privkeys_list] -pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys_list, pubkeys_list)} +DEFAULT_CONFIG = {} # no change + +MINIMAL_CONFIG = { + "SHARD_COUNT": 8, + "MIN_ATTESTATION_INCLUSION_DELAY": 2, + "TARGET_COMMITTEE_SIZE": 4, + "SLOTS_PER_EPOCH": 8, + "GENESIS_EPOCH": spec.GENESIS_SLOT // 8, + "SLOTS_PER_HISTORICAL_ROOT": 64, + "LATEST_RANDAO_MIXES_LENGTH": 64, + "LATEST_ACTIVE_INDEX_ROOTS_LENGTH": 64, + "LATEST_SLASHED_EXIT_LENGTH": 64, +} @pytest.fixture @@ -53,19 +52,14 @@ def overwrite_spec_config(config): spec.BeaconState.fields['latest_slashed_balances'][1] = config[field] -@pytest.fixture -def config(): - return { - "SHARD_COUNT": 8, - "MIN_ATTESTATION_INCLUSION_DELAY": 2, - "TARGET_COMMITTEE_SIZE": 4, - "SLOTS_PER_EPOCH": 8, - "GENESIS_EPOCH": spec.GENESIS_SLOT // 8, - "SLOTS_PER_HISTORICAL_ROOT": 64, - "LATEST_RANDAO_MIXES_LENGTH": 64, - "LATEST_ACTIVE_INDEX_ROOTS_LENGTH": 64, - "LATEST_SLASHED_EXIT_LENGTH": 64, - } +@pytest.fixture( + params=[ + pytest.param(MINIMAL_CONFIG, marks=pytest.mark.minimal_config), + DEFAULT_CONFIG, + ] +) +def config(request): + return request.param @pytest.fixture(autouse=True) @@ -73,52 +67,6 @@ def overwrite_config(config): overwrite_spec_config(config) -def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves): - deposit_timestamp = 0 - proof_of_possession = b'\x33' * 96 - - deposit_data_list = [] - for i in range(num_validators): - pubkey = pubkeys_list[i] - privkey = pubkey_to_privkey[pubkey] - deposit_data = DepositData( - amount=spec.MAX_DEPOSIT_AMOUNT, - timestamp=deposit_timestamp, - deposit_input=DepositInput( - pubkey=pubkey, - withdrawal_credentials=privkey.to_bytes(32, byteorder='big'), - proof_of_possession=proof_of_possession, - ), - ) - item = hash(deposit_data.serialize()) - deposit_data_leaves.append(item) - tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) - root = get_merkle_root((tuple(deposit_data_leaves))) - proof = list(get_merkle_proof(tree, item_index=i)) - assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, i, root) - deposit_data_list.append(deposit_data) - - genesis_validator_deposits = [] - for i in range(num_validators): - genesis_validator_deposits.append(Deposit( - proof=list(get_merkle_proof(tree, item_index=i)), - index=i, - deposit_data=deposit_data_list[i] - )) - return genesis_validator_deposits, root - - -def create_genesis_state(num_validators, deposit_data_leaves): - initial_deposits, deposit_root = create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves) - return get_genesis_beacon_state( - initial_deposits, - genesis_time=0, - genesis_eth1_data=Eth1Data( - deposit_root=deposit_root, - block_hash=spec.ZERO_HASH, - ), - ) - @pytest.fixture def num_validators(): return 100 diff --git a/tests/phase0/helpers.py b/tests/phase0/helpers.py new file mode 100644 index 000000000..fa0ba61b5 --- /dev/null +++ b/tests/phase0/helpers.py @@ -0,0 +1,139 @@ +from copy import deepcopy + +from py_ecc import bls + +import build.phase0.spec as spec +from build.phase0.utils.minimal_ssz import signed_root +from build.phase0.spec import ( + AttestationData, + Deposit, + DepositInput, + DepositData, + Eth1Data, + get_block_root, + get_current_epoch, + get_domain, + get_empty_block, + get_epoch_start_slot, + get_genesis_beacon_state, + verify_merkle_branch, + hash, +) +from build.phase0.utils.merkle_minimal import ( + calc_merkle_tree_from_leaves, + get_merkle_proof, + get_merkle_root, +) + + +privkeys_list = [i+1 for i in range(1000)] +pubkeys_list = [bls.privtopub(privkey) for privkey in privkeys_list] +pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys_list, pubkeys_list)} + + +def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves): + deposit_timestamp = 0 + proof_of_possession = b'\x33' * 96 + + deposit_data_list = [] + for i in range(num_validators): + pubkey = pubkeys_list[i] + privkey = pubkey_to_privkey[pubkey] + deposit_data = DepositData( + amount=spec.MAX_DEPOSIT_AMOUNT, + timestamp=deposit_timestamp, + deposit_input=DepositInput( + pubkey=pubkey, + withdrawal_credentials=privkey.to_bytes(32, byteorder='big'), + proof_of_possession=proof_of_possession, + ), + ) + item = hash(deposit_data.serialize()) + deposit_data_leaves.append(item) + tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) + root = get_merkle_root((tuple(deposit_data_leaves))) + proof = list(get_merkle_proof(tree, item_index=i)) + assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, i, root) + deposit_data_list.append(deposit_data) + + genesis_validator_deposits = [] + for i in range(num_validators): + genesis_validator_deposits.append(Deposit( + proof=list(get_merkle_proof(tree, item_index=i)), + index=i, + deposit_data=deposit_data_list[i] + )) + return genesis_validator_deposits, root + + +def create_genesis_state(num_validators, deposit_data_leaves): + initial_deposits, deposit_root = create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves) + return get_genesis_beacon_state( + initial_deposits, + genesis_time=0, + genesis_eth1_data=Eth1Data( + deposit_root=deposit_root, + block_hash=spec.ZERO_HASH, + ), + ) + +def build_empty_block_for_next_slot(state): + empty_block = get_empty_block() + empty_block.slot = state.slot + 1 + previous_block_header = deepcopy(state.latest_block_header) + if previous_block_header.state_root == spec.ZERO_HASH: + previous_block_header.state_root = state.hash_tree_root() + empty_block.previous_block_root = previous_block_header.hash_tree_root() + return empty_block + + +def build_deposit_data(state, pubkey, privkey, amount): + deposit_input = DepositInput( + pubkey=pubkey, + withdrawal_credentials=privkey.to_bytes(32, byteorder='big'), + proof_of_possession=b'00'*96, + ) + proof_of_possession = bls.sign( + message_hash=signed_root(deposit_input), + privkey=privkey, + domain=get_domain( + state.fork, + get_current_epoch(state), + spec.DOMAIN_DEPOSIT, + ) + ) + deposit_input.proof_of_possession = proof_of_possession + deposit_data = DepositData( + amount=amount, + timestamp=0, + deposit_input=deposit_input, + ) + return deposit_data + + +def build_attestation_data(state, slot, shard): + assert state.slot >= slot + + block_root = build_empty_block_for_next_slot(state).previous_block_root + + epoch_start_slot = get_epoch_start_slot(get_current_epoch(state)) + if epoch_start_slot == slot: + epoch_boundary_root = block_root + else: + get_block_root(state, epoch_start_slot) + + if slot < epoch_start_slot: + justified_block_root = state.previous_justified_root + else: + justified_block_root = state.current_justified_root + + return AttestationData( + slot=slot, + shard=shard, + beacon_block_root=block_root, + source_epoch=state.current_justified_epoch, + source_root=justified_block_root, + target_root=epoch_boundary_root, + crosslink_data_root=spec.ZERO_HASH, + previous_crosslink=deepcopy(state.latest_crosslinks[shard]), + ) diff --git a/tests/phase0/test_sanity.py b/tests/phase0/test_sanity.py index 0e04df5dd..bfbb2de94 100644 --- a/tests/phase0/test_sanity.py +++ b/tests/phase0/test_sanity.py @@ -1,8 +1,10 @@ import os import sys import time - from copy import deepcopy + +import pytest + from py_ecc import bls import build.phase0.spec as spec @@ -48,78 +50,15 @@ from build.phase0.utils.merkle_minimal import ( get_merkle_proof, get_merkle_root, ) -# from state_test_gen import ( - # generate_from_test, - # dump_json, - # dump_yaml, -# ) +from tests.phase0.helpers import ( + build_attestation_data, + build_deposit_data, + build_empty_block_for_next_slot, +) -def get_empty_root(): - return get_merkle_root((spec.ZERO_HASH,)) - - -def construct_empty_block_for_next_slot(state): - empty_block = get_empty_block() - empty_block.slot = state.slot + 1 - previous_block_header = deepcopy(state.latest_block_header) - if previous_block_header.state_root == spec.ZERO_HASH: - previous_block_header.state_root = state.hash_tree_root() - empty_block.previous_block_root = previous_block_header.hash_tree_root() - return empty_block - - -def create_deposit_data(state, pubkey, privkey, amount): - deposit_input = DepositInput( - pubkey=pubkey, - withdrawal_credentials=privkey.to_bytes(32, byteorder='big'), - proof_of_possession=b'00'*96, - ) - proof_of_possession = bls.sign( - message_hash=signed_root(deposit_input), - privkey=privkey, - domain=get_domain( - state.fork, - get_current_epoch(state), - spec.DOMAIN_DEPOSIT, - ) - ) - deposit_input.proof_of_possession = proof_of_possession - deposit_data = DepositData( - amount=amount, - timestamp=0, - deposit_input=deposit_input, - ) - return deposit_data - - -def build_attestation_data(state, slot, shard): - assert state.slot >= slot - - block_root = construct_empty_block_for_next_slot(state).previous_block_root - - epoch_start_slot = get_epoch_start_slot(get_current_epoch(state)) - if epoch_start_slot == slot: - epoch_boundary_root = block_root - else: - get_block_root(state, epoch_start_slot) - - if slot < epoch_start_slot: - justified_block_root = state.previous_justified_root - else: - justified_block_root = state.current_justified_root - - return AttestationData( - slot=slot, - shard=shard, - beacon_block_root=block_root, - source_epoch=state.current_justified_epoch, - source_root=justified_block_root, - target_root=epoch_boundary_root, - crosslink_data_root=spec.ZERO_HASH, - previous_crosslink=deepcopy(state.latest_crosslinks[shard]), - ) - +# mark entire file as 'sanity' +pytestmark = pytest.mark.sanity def test_slot_transition(state): test_state = deepcopy(state) @@ -133,7 +72,7 @@ def test_slot_transition(state): def test_empty_block_transition(state): test_state = deepcopy(state) - block = construct_empty_block_for_next_slot(test_state) + block = build_empty_block_for_next_slot(test_state) state_transition(test_state, block) assert len(test_state.eth1_data_votes) == len(state.eth1_data_votes) + 1 @@ -144,7 +83,7 @@ def test_empty_block_transition(state): def test_skipped_slots(state): test_state = deepcopy(state) - block = construct_empty_block_for_next_slot(test_state) + block = build_empty_block_for_next_slot(test_state) block.slot += 3 state_transition(test_state, block) @@ -158,7 +97,7 @@ def test_skipped_slots(state): def test_empty_epoch_transition(state): test_state = deepcopy(state) - block = construct_empty_block_for_next_slot(test_state) + block = build_empty_block_for_next_slot(test_state) block.slot += spec.SLOTS_PER_EPOCH state_transition(test_state, block) @@ -172,7 +111,7 @@ def test_empty_epoch_transition(state): def test_empty_epoch_transition_not_finalizing(state): test_state = deepcopy(state) - block = construct_empty_block_for_next_slot(test_state) + block = build_empty_block_for_next_slot(test_state) block.slot += spec.SLOTS_PER_EPOCH * 5 state_transition(test_state, block) @@ -226,7 +165,7 @@ def test_proposer_slashing(state, pubkeys, privkeys): # # Add to state via block transition # - block = construct_empty_block_for_next_slot(test_state) + block = build_empty_block_for_next_slot(test_state) block.body.proposer_slashings.append(proposer_slashing) state_transition(test_state, block) @@ -251,7 +190,7 @@ def test_deposit_in_block(state, deposit_data_leaves, pubkeys, privkeys): index = len(test_deposit_data_leaves) pubkey = pubkeys[index] privkey = privkeys[index] - deposit_data = create_deposit_data(pre_state, pubkey, privkey, spec.MAX_DEPOSIT_AMOUNT) + deposit_data = build_deposit_data(pre_state, pubkey, privkey, spec.MAX_DEPOSIT_AMOUNT) item = hash(deposit_data.serialize()) test_deposit_data_leaves.append(item) @@ -268,7 +207,7 @@ def test_deposit_in_block(state, deposit_data_leaves, pubkeys, privkeys): pre_state.latest_eth1_data.deposit_root = root post_state = deepcopy(pre_state) - block = construct_empty_block_for_next_slot(post_state) + block = build_empty_block_for_next_slot(post_state) block.body.deposits.append(deposit) state_transition(post_state, block) @@ -287,7 +226,7 @@ def test_deposit_top_up(state, pubkeys, privkeys, deposit_data_leaves): amount = spec.MAX_DEPOSIT_AMOUNT // 4 pubkey = pubkeys[validator_index] privkey = privkeys[validator_index] - deposit_data = create_deposit_data(pre_state, pubkey, privkey, amount) + deposit_data = build_deposit_data(pre_state, pubkey, privkey, amount) merkle_index = len(test_deposit_data_leaves) item = hash(deposit_data.serialize()) @@ -304,7 +243,7 @@ def test_deposit_top_up(state, pubkeys, privkeys, deposit_data_leaves): ) pre_state.latest_eth1_data.deposit_root = root - block = construct_empty_block_for_next_slot(pre_state) + block = build_empty_block_for_next_slot(pre_state) block.body.deposits.append(deposit) pre_balance = pre_state.validator_balances[validator_index] @@ -365,7 +304,7 @@ def test_attestation(state, pubkeys, privkeys): # # Add to state via block transition # - attestation_block = construct_empty_block_for_next_slot(test_state) + attestation_block = build_empty_block_for_next_slot(test_state) attestation_block.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation_block.body.attestations.append(attestation) state_transition(test_state, attestation_block) @@ -377,7 +316,7 @@ def test_attestation(state, pubkeys, privkeys): # pre_current_epoch_attestations = deepcopy(test_state.current_epoch_attestations) - epoch_block = construct_empty_block_for_next_slot(test_state) + epoch_block = build_empty_block_for_next_slot(test_state) epoch_block.slot += spec.SLOTS_PER_EPOCH state_transition(test_state, epoch_block) @@ -417,7 +356,7 @@ def test_voluntary_exit(state, pubkeys, privkeys): # # Add to state via block transition # - initiate_exit_block = construct_empty_block_for_next_slot(post_state) + initiate_exit_block = build_empty_block_for_next_slot(post_state) initiate_exit_block.body.voluntary_exits.append(voluntary_exit) state_transition(post_state, initiate_exit_block) @@ -428,7 +367,7 @@ def test_voluntary_exit(state, pubkeys, privkeys): # # Process within epoch transition # - exit_block = construct_empty_block_for_next_slot(post_state) + exit_block = build_empty_block_for_next_slot(post_state) exit_block.slot += spec.SLOTS_PER_EPOCH state_transition(post_state, exit_block) @@ -476,7 +415,7 @@ def test_transfer(state, pubkeys, privkeys): # # Add to state via block transition # - block = construct_empty_block_for_next_slot(post_state) + block = build_empty_block_for_next_slot(post_state) block.body.transfers.append(transfer) state_transition(post_state, block) @@ -503,7 +442,7 @@ def test_ejection(state): # # trigger epoch transition # - block = construct_empty_block_for_next_slot(post_state) + block = build_empty_block_for_next_slot(post_state) block.slot += spec.SLOTS_PER_EPOCH state_transition(post_state, block) @@ -518,7 +457,7 @@ def test_historical_batch(state): post_state = deepcopy(pre_state) - block = construct_empty_block_for_next_slot(post_state) + block = build_empty_block_for_next_slot(post_state) state_transition(post_state, block) @@ -527,106 +466,3 @@ def test_historical_batch(state): assert len(post_state.historical_roots) == len(pre_state.historical_roots) + 1 return pre_state, [block], post_state - - -def sanity_tests(num_validators=100, config=None): - print(f"Buidling state with {num_validators} validators...") - if config: - overwrite_spec_config(config) - genesis_state = create_genesis_state(num_validators=num_validators) - print("done!") - print() - - test_cases = [] - - print("Running some sanity check tests...\n") - test_slot_transition(genesis_state) - print("Passed slot transition test\n") - test_cases.append( - generate_from_test(test_empty_block_transition, genesis_state, config=config, fields=['slot']) - ) - print("Passed empty block transition test\n") - test_cases.append( - generate_from_test(test_skipped_slots, genesis_state, config=config, fields=['slot', 'latest_block_roots']) - ) - print("Passed skipped slot test\n") - test_cases.append( - generate_from_test(test_empty_epoch_transition, genesis_state, config=config, fields=['slot', 'latest_block_roots']) - ) - print("Passed empty epoch transition test\n") - test_cases.append( - generate_from_test(test_empty_epoch_transition_not_finalizing, genesis_state, config=config, fields=['slot', 'finalized_epoch']) - ) - print("Passed non-finalizing epoch test\n") - test_cases.append( - generate_from_test(test_proposer_slashing, genesis_state, config=config, fields=['validator_registry', 'validator_balances']) - ) - print("Passed proposer slashing test\n") - test_cases.append( - generate_from_test(test_attestation, genesis_state, config=config, fields=['previous_epoch_attestations', 'current_epoch_attestations']) - ) - print("Passed attestation test\n") - test_cases.append( - generate_from_test(test_deposit_in_block, genesis_state, config=config, fields=['validator_registry', 'validator_balances']) - ) - print("Passed deposit test\n") - test_cases.append( - generate_from_test(test_deposit_top_up, genesis_state, config=config, fields=['validator_registry', 'validator_balances']) - ) - print("Passed deposit top up test\n") - test_cases.append( - generate_from_test(test_voluntary_exit, genesis_state, config=config, fields=['validator_registry']) - ) - print("Passed voluntary exit test\n") - test_cases.append( - generate_from_test(test_transfer, genesis_state, config=config, fields=['validator_balances']) - ) - print("Passed transfer test\n") - test_cases.append( - generate_from_test(test_ejection, genesis_state, config=config, fields=['validator_registry']) - ) - print("Passed ejection test\n") - test_cases.append( - generate_from_test(test_historical_batch, genesis_state, config=config, fields=['historical_roots']) - ) - print("Passed historical batch test\n") - print("done!") - - return test_cases - - -if __name__ == "__main__": - config = { - "SHARD_COUNT": 8, - "MIN_ATTESTATION_INCLUSION_DELAY": 2, - "TARGET_COMMITTEE_SIZE": 4, - "SLOTS_PER_EPOCH": 8, - "GENESIS_EPOCH": spec.GENESIS_SLOT // 8, - "SLOTS_PER_HISTORICAL_ROOT": 64, - "LATEST_RANDAO_MIXES_LENGTH": 64, - "LATEST_ACTIVE_INDEX_ROOTS_LENGTH": 64, - "LATEST_SLASHED_EXIT_LENGTH": 64, - } - - test_cases = sanity_tests(32, config) - # uncomment below to run/generate against the default config - # test_cases = sanity_tests(100) - - test = {} - metadata = {} - metadata['title'] = "Sanity tests" - metadata['summary'] = "Basic sanity checks from phase 0 spec pythonization. All tests are run with `verify_signatures` as set to False." - metadata['test_suite'] = "beacon_state" - metadata['fork'] = "tchaikovsky" - metadata['version'] = "v0.5.0" - test['metadata'] = metadata - test['test_cases'] = test_cases - - if '--output-json' in sys.argv: - os.makedirs('output', exist_ok=True) - with open("output/sanity_check_tests.json", "w+") as outfile: - dump_json(test, outfile) - if '--output-yaml' in sys.argv: - os.makedirs('output', exist_ok=True) - with open("output/sanity_check_tests.yaml", "w+") as outfile: - dump_yaml(test, outfile) From 4440be4e1f92154242960eeaa12320c3b0f5e404 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 Mar 2019 16:30:16 -0600 Subject: [PATCH 07/11] add comment to Makefile --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index f0f7557e9..b45cec410 100644 --- a/Makefile +++ b/Makefile @@ -14,9 +14,12 @@ clean: rm -rf $(BUILD_DIR) +# runs a limited set of tests against a minimal config +# run pytest with `-m` option to full suite test: pytest -m "sanity and minimal_config" tests/ + $(BUILD_DIR)/phase0: mkdir -p $@ python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $@/spec.py From a07219c57045a22d13304148dbae8bb9121a0181 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 19 Mar 2019 11:39:19 +0800 Subject: [PATCH 08/11] Fix linter --- scripts/phase0/build_spec.py | 1 - scripts/phase0/function_puller.py | 2 +- tests/phase0/helpers.py | 5 +++-- tests/phase0/test_sanity.py | 34 ++++++++----------------------- utils/phase0/bls_stub.py | 2 +- utils/phase0/hash_function.py | 5 +++-- utils/phase0/merkle_minimal.py | 12 ++++++----- utils/phase0/minimal_ssz.py | 24 ++++++++++++++++++---- utils/phase0/state_transition.py | 2 +- 9 files changed, 45 insertions(+), 42 deletions(-) diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index eb4f580bd..ae5a5a4f2 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -34,7 +34,6 @@ Any = None Store = None """) - code_lines += function_puller.get_lines(sourcefile) code_lines.append(""" diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index 8d1c1a0cc..7d5796fc7 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -9,7 +9,7 @@ def get_lines(file_name): for linenum, line in enumerate(open(sys.argv[1]).readlines()): line = line.rstrip() if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`': - current_name = line[line[:-1].rfind('`')+1: -1] + current_name = line[line[:-1].rfind('`') + 1: -1] if line[:9] == '```python': assert pulling_from is None pulling_from = linenum + 1 diff --git a/tests/phase0/helpers.py b/tests/phase0/helpers.py index fa0ba61b5..f7c39ffec 100644 --- a/tests/phase0/helpers.py +++ b/tests/phase0/helpers.py @@ -26,7 +26,7 @@ from build.phase0.utils.merkle_minimal import ( ) -privkeys_list = [i+1 for i in range(1000)] +privkeys_list = [i + 1 for i in range(1000)] pubkeys_list = [bls.privtopub(privkey) for privkey in privkeys_list] pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys_list, pubkeys_list)} @@ -77,6 +77,7 @@ def create_genesis_state(num_validators, deposit_data_leaves): ), ) + def build_empty_block_for_next_slot(state): empty_block = get_empty_block() empty_block.slot = state.slot + 1 @@ -91,7 +92,7 @@ def build_deposit_data(state, pubkey, privkey, amount): deposit_input = DepositInput( pubkey=pubkey, withdrawal_credentials=privkey.to_bytes(32, byteorder='big'), - proof_of_possession=b'00'*96, + proof_of_possession=b'\x00' * 96, ) proof_of_possession = bls.sign( message_hash=signed_root(deposit_input), diff --git a/tests/phase0/test_sanity.py b/tests/phase0/test_sanity.py index bfbb2de94..8799c1ffb 100644 --- a/tests/phase0/test_sanity.py +++ b/tests/phase0/test_sanity.py @@ -1,6 +1,3 @@ -import os -import sys -import time from copy import deepcopy import pytest @@ -12,32 +9,21 @@ from build.phase0.utils.minimal_ssz import signed_root from build.phase0.spec import ( # SSZ Attestation, - AttestationData, AttestationDataAndCustodyBit, BeaconBlockHeader, Deposit, - DepositData, - DepositInput, - Eth1Data, Transfer, ProposerSlashing, - Validator, VoluntaryExit, # functions - int_to_bytes32, - int_to_bytes48, get_active_validator_indices, get_attestation_participants, get_block_root, get_crosslink_committees_at_slot, get_current_epoch, get_domain, - get_empty_block, - get_epoch_start_slot, - get_genesis_beacon_state, get_state_root, advance_slot, - slot_to_epoch, cache_state, verify_merkle_branch, hash, @@ -60,6 +46,7 @@ from tests.phase0.helpers import ( # mark entire file as 'sanity' pytestmark = pytest.mark.sanity + def test_slot_transition(state): test_state = deepcopy(state) cache_state(test_state) @@ -126,18 +113,17 @@ def test_proposer_slashing(state, pubkeys, privkeys): test_state = deepcopy(state) current_epoch = get_current_epoch(test_state) validator_index = get_active_validator_indices(test_state.validator_registry, current_epoch)[-1] - pubkey = pubkeys[validator_index] privkey = privkeys[validator_index] slot = spec.GENESIS_SLOT header_1 = BeaconBlockHeader( slot=slot, - previous_block_root=b'\x00'*32, - state_root=b'\x00'*32, - block_body_root=b'\x00'*32, - signature=b'\x00'*96 + previous_block_root=b'\x00' * 32, + state_root=b'\x00' * 32, + block_body_root=b'\x00' * 32, + signature=b'\x00' * 96 ) header_2 = deepcopy(header_1) - header_2.previous_block_root = b'\x02'*32 + header_2.previous_block_root = b'\x02' * 32 header_2.slot = slot + 1 domain = get_domain( @@ -273,7 +259,7 @@ def test_attestation(state, pubkeys, privkeys): aggregation_bitfield=aggregation_bitfield, data=attestation_data, custody_bitfield=custody_bitfield, - aggregate_signature=b'\x00'*96, + aggregate_signature=b'\x00' * 96, ) participants = get_attestation_participants( test_state, @@ -283,7 +269,6 @@ def test_attestation(state, pubkeys, privkeys): assert len(participants) == 1 validator_index = participants[0] - pubkey = pubkeys[validator_index] privkey = privkeys[validator_index] message_hash = AttestationDataAndCustodyBit( @@ -329,7 +314,6 @@ def test_attestation(state, pubkeys, privkeys): def test_voluntary_exit(state, pubkeys, privkeys): pre_state = deepcopy(state) validator_index = get_active_validator_indices(pre_state.validator_registry, get_current_epoch(pre_state))[-1] - pubkey = pubkeys[validator_index] # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit pre_state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH @@ -341,7 +325,7 @@ def test_voluntary_exit(state, pubkeys, privkeys): voluntary_exit = VoluntaryExit( epoch=get_current_epoch(pre_state), validator_index=validator_index, - signature=b'\x00'*96, + signature=b'\x00' * 96, ) voluntary_exit.signature = bls.sign( message_hash=signed_root(voluntary_exit), @@ -392,7 +376,7 @@ def test_transfer(state, pubkeys, privkeys): fee=0, slot=pre_state.slot + 1, pubkey=transfer_pubkey, - signature=b'\x00'*96, + signature=b'\x00' * 96, ) transfer.signature = bls.sign( message_hash=signed_root(transfer), diff --git a/utils/phase0/bls_stub.py b/utils/phase0/bls_stub.py index 7e3a6a308..108c4ef71 100644 --- a/utils/phase0/bls_stub.py +++ b/utils/phase0/bls_stub.py @@ -9,4 +9,4 @@ def bls_verify_multiple(pubkeys, message_hashes, signature, domain): def bls_aggregate_pubkeys(pubkeys): - return b'\x42'*96 + return b'\x42' * 96 diff --git a/utils/phase0/hash_function.py b/utils/phase0/hash_function.py index da5b4d979..21e6555bf 100644 --- a/utils/phase0/hash_function.py +++ b/utils/phase0/hash_function.py @@ -1,6 +1,7 @@ -from hashlib import sha256 +# from hashlib import sha256 from eth_utils import keccak # def hash(x): return sha256(x).digest() -def hash(x): return keccak(x) +def hash(x): + return keccak(x) diff --git a/utils/phase0/merkle_minimal.py b/utils/phase0/merkle_minimal.py index a811350ce..7c5483de3 100644 --- a/utils/phase0/merkle_minimal.py +++ b/utils/phase0/merkle_minimal.py @@ -2,8 +2,9 @@ from .hash_function import hash zerohashes = [b'\x00' * 32] -for i in range(1, 32): - zerohashes.append(hash(zerohashes[i-1] + zerohashes[i-1])) +for layer in range(1, 32): + zerohashes.append(hash(zerohashes[layer - 1] + zerohashes[layer - 1])) + # Compute a Merkle root of a right-zerobyte-padded 2**32 sized tree def calc_merkle_tree_from_leaves(values): @@ -12,17 +13,18 @@ def calc_merkle_tree_from_leaves(values): for h in range(32): if len(values) % 2 == 1: values.append(zerohashes[h]) - # print(values) - values = [hash(values[i] + values[i+1]) for i in range(0, len(values), 2)] + values = [hash(values[i] + values[i + 1]) for i in range(0, len(values), 2)] tree.append(values[::]) return tree + def get_merkle_root(values): return calc_merkle_tree_from_leaves(values)[-1][0] + def get_merkle_proof(tree, item_index): proof = [] for i in range(32): - subindex = (item_index//2**i)^1 + subindex = (item_index // 2**i) ^ 1 proof.append(tree[i][subindex] if subindex < len(tree[i]) else zerohashes[i]) return proof diff --git a/utils/phase0/minimal_ssz.py b/utils/phase0/minimal_ssz.py index 845de18c3..08bd68357 100644 --- a/utils/phase0/minimal_ssz.py +++ b/utils/phase0/minimal_ssz.py @@ -5,6 +5,7 @@ BYTES_PER_CHUNK = 32 BYTES_PER_LENGTH_PREFIX = 4 ZERO_CHUNK = b'\x00' * BYTES_PER_CHUNK + def SSZType(fields): class SSZObject(): def __init__(self, **kwargs): @@ -37,6 +38,7 @@ def SSZType(fields): SSZObject.fields = fields return SSZObject + class Vector(list): def __init__(self, x): list.__init__(self, x) @@ -47,9 +49,11 @@ class Vector(list): remove = clear = extend = pop = insert = append + def is_basic(typ): return isinstance(typ, str) and (typ[:4] in ('uint', 'bool') or typ == 'byte') + def is_constant_sized(typ): if is_basic(typ): return True @@ -67,6 +71,7 @@ def is_constant_sized(typ): else: raise Exception("Type not recognized") + def coerce_to_bytes(x): if isinstance(x, str): o = x.encode('utf-8') @@ -77,6 +82,7 @@ def coerce_to_bytes(x): else: raise Exception("Expecting bytes") + def serialize_value(value, typ=None): if typ is None: typ = infer_type(value) @@ -110,28 +116,34 @@ def serialize_value(value, typ=None): print(value, typ) raise Exception("Type not recognized") + def chunkify(bytez): bytez += b'\x00' * (-len(bytez) % BYTES_PER_CHUNK) - return [bytez[i:i+32] for i in range(0, len(bytez), 32)] + return [bytez[i:i + 32] for i in range(0, len(bytez), 32)] + def pack(values, subtype): return chunkify(b''.join([serialize_value(value, subtype) for value in values])) + def is_power_of_two(x): - return x > 0 and x & (x-1) == 0 + return x > 0 and x & (x - 1) == 0 + def merkleize(chunks): tree = chunks[::] while not is_power_of_two(len(tree)): tree.append(ZERO_CHUNK) tree = [ZERO_CHUNK] * len(tree) + tree - for i in range(len(tree)//2-1, 0, -1): - tree[i] = hash(tree[i*2] + tree[i*2+1]) + for i in range(len(tree) // 2 - 1, 0, -1): + tree[i] = hash(tree[i * 2] + tree[i * 2 + 1]) return tree[1] + def mix_in_length(root, length): return hash(root + length.to_bytes(32, 'little')) + def infer_type(value): if hasattr(value.__class__, 'fields'): return value.__class__ @@ -146,6 +158,7 @@ def infer_type(value): else: raise Exception("Failed to infer type") + def hash_tree_root(value, typ=None): if typ is None: typ = infer_type(value) @@ -170,6 +183,7 @@ def hash_tree_root(value, typ=None): else: raise Exception("Type not recognized") + def truncate(container): field_keys = list(container.fields.keys()) truncated_fields = { @@ -183,8 +197,10 @@ def truncate(container): } return truncated_class(**kwargs) + def signed_root(container): return hash_tree_root(truncate(container)) + def serialize(ssz_object): return getattr(ssz_object, 'serialize')() diff --git a/utils/phase0/state_transition.py b/utils/phase0/state_transition.py index 170f647ab..92d67c45a 100644 --- a/utils/phase0/state_transition.py +++ b/utils/phase0/state_transition.py @@ -1,7 +1,7 @@ from . import spec -from typing import ( +from typing import ( # noqa: F401 Any, Callable, List, From 8fc1fe5f2075d290d55c16e3cbe18aa782fff6c2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 19 Mar 2019 11:46:32 +0800 Subject: [PATCH 09/11] Use `EMPTY_SIGNATURE` and `ZERO_HASH` in the tests --- tests/phase0/helpers.py | 6 +++++- tests/phase0/test_sanity.py | 17 ++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/phase0/helpers.py b/tests/phase0/helpers.py index f7c39ffec..76206b00d 100644 --- a/tests/phase0/helpers.py +++ b/tests/phase0/helpers.py @@ -5,11 +5,15 @@ from py_ecc import bls import build.phase0.spec as spec from build.phase0.utils.minimal_ssz import signed_root from build.phase0.spec import ( + # constants + EMPTY_SIGNATURE, + # SSZ AttestationData, Deposit, DepositInput, DepositData, Eth1Data, + # functions get_block_root, get_current_epoch, get_domain, @@ -92,7 +96,7 @@ def build_deposit_data(state, pubkey, privkey, amount): deposit_input = DepositInput( pubkey=pubkey, withdrawal_credentials=privkey.to_bytes(32, byteorder='big'), - proof_of_possession=b'\x00' * 96, + proof_of_possession=EMPTY_SIGNATURE, ) proof_of_possession = bls.sign( message_hash=signed_root(deposit_input), diff --git a/tests/phase0/test_sanity.py b/tests/phase0/test_sanity.py index 8799c1ffb..8f04f316c 100644 --- a/tests/phase0/test_sanity.py +++ b/tests/phase0/test_sanity.py @@ -7,6 +7,9 @@ import build.phase0.spec as spec from build.phase0.utils.minimal_ssz import signed_root from build.phase0.spec import ( + # constants + EMPTY_SIGNATURE, + ZERO_HASH, # SSZ Attestation, AttestationDataAndCustodyBit, @@ -117,10 +120,10 @@ def test_proposer_slashing(state, pubkeys, privkeys): slot = spec.GENESIS_SLOT header_1 = BeaconBlockHeader( slot=slot, - previous_block_root=b'\x00' * 32, - state_root=b'\x00' * 32, - block_body_root=b'\x00' * 32, - signature=b'\x00' * 96 + previous_block_root=ZERO_HASH, + state_root=ZERO_HASH, + block_body_root=ZERO_HASH, + signature=EMPTY_SIGNATURE, ) header_2 = deepcopy(header_1) header_2.previous_block_root = b'\x02' * 32 @@ -259,7 +262,7 @@ def test_attestation(state, pubkeys, privkeys): aggregation_bitfield=aggregation_bitfield, data=attestation_data, custody_bitfield=custody_bitfield, - aggregate_signature=b'\x00' * 96, + aggregate_signature=EMPTY_SIGNATURE, ) participants = get_attestation_participants( test_state, @@ -325,7 +328,7 @@ def test_voluntary_exit(state, pubkeys, privkeys): voluntary_exit = VoluntaryExit( epoch=get_current_epoch(pre_state), validator_index=validator_index, - signature=b'\x00' * 96, + signature=EMPTY_SIGNATURE, ) voluntary_exit.signature = bls.sign( message_hash=signed_root(voluntary_exit), @@ -376,7 +379,7 @@ def test_transfer(state, pubkeys, privkeys): fee=0, slot=pre_state.slot + 1, pubkey=transfer_pubkey, - signature=b'\x00' * 96, + signature=EMPTY_SIGNATURE, ) transfer.signature = bls.sign( message_hash=signed_root(transfer), From d645a9a0389ea571f39e901ce74274ee9d3e704f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 19 Mar 2019 09:49:32 -0600 Subject: [PATCH 10/11] ensure run verify-state root with block Co-Authored-By: djrtwo --- utils/phase0/state_transition.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/phase0/state_transition.py b/utils/phase0/state_transition.py index 92d67c45a..eefc3d409 100644 --- a/utils/phase0/state_transition.py +++ b/utils/phase0/state_transition.py @@ -97,4 +97,4 @@ def state_transition(state: BeaconState, process_epoch_transition(state) spec.advance_slot(state) if block.slot == state.slot: - process_block(state, block) + process_block(state, block, verify_state_root) From fbd0bb9226102ef4dbde606c0853d8dd74e0b60b Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 Mar 2019 09:56:04 -0600 Subject: [PATCH 11/11] withdrwaal_key uses pubkey --- tests/phase0/helpers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/phase0/helpers.py b/tests/phase0/helpers.py index 76206b00d..510361e9c 100644 --- a/tests/phase0/helpers.py +++ b/tests/phase0/helpers.py @@ -42,13 +42,13 @@ def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves): deposit_data_list = [] for i in range(num_validators): pubkey = pubkeys_list[i] - privkey = pubkey_to_privkey[pubkey] deposit_data = DepositData( amount=spec.MAX_DEPOSIT_AMOUNT, timestamp=deposit_timestamp, deposit_input=DepositInput( pubkey=pubkey, - withdrawal_credentials=privkey.to_bytes(32, byteorder='big'), + # insecurely use pubkey as withdrawal key as well + withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:], proof_of_possession=proof_of_possession, ), ) @@ -95,7 +95,8 @@ def build_empty_block_for_next_slot(state): def build_deposit_data(state, pubkey, privkey, amount): deposit_input = DepositInput( pubkey=pubkey, - withdrawal_credentials=privkey.to_bytes(32, byteorder='big'), + # insecurely use pubkey as withdrawal key as well + withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:], proof_of_possession=EMPTY_SIGNATURE, ) proof_of_possession = bls.sign(