From 8d19e7a5703ae9976b87e0e3b5e2f8ef99d9c76a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 20 Mar 2019 08:22:47 -0600 Subject: [PATCH 001/121] port tests from dev to master --- Makefile | 2 +- .../test_process_block_header.py | 26 +++ .../block_processing/test_process_deposit.py | 132 +++++++++++++ .../block_processing/test_voluntary_exit.py | 174 ++++++++++++++++++ tests/phase0/conftest.py | 6 + tests/phase0/helpers.py | 58 +++++- utils/phase0/minimal_ssz.py | 21 ++- 7 files changed, 410 insertions(+), 9 deletions(-) create mode 100644 tests/phase0/block_processing/test_process_block_header.py create mode 100644 tests/phase0/block_processing/test_process_deposit.py create mode 100644 tests/phase0/block_processing/test_voluntary_exit.py diff --git a/Makefile b/Makefile index b45cec410..88f17dcf9 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ clean: # runs a limited set of tests against a minimal config # run pytest with `-m` option to full suite test: - pytest -m "sanity and minimal_config" tests/ + pytest -m minimal_config tests/ $(BUILD_DIR)/phase0: diff --git a/tests/phase0/block_processing/test_process_block_header.py b/tests/phase0/block_processing/test_process_block_header.py new file mode 100644 index 000000000..4ec7e336f --- /dev/null +++ b/tests/phase0/block_processing/test_process_block_header.py @@ -0,0 +1,26 @@ +from copy import deepcopy +import pytest + + +from build.phase0.spec import ( + get_beacon_proposer_index, + process_block_header, +) +from tests.phase0.helpers import ( + build_empty_block_for_next_slot, +) + +# mark entire file as 'header' +pytestmark = pytest.mark.header + + +def test_proposer_slashed(state): + pre_state = deepcopy(state) + + block = build_empty_block_for_next_slot(pre_state) + proposer_index = get_beacon_proposer_index(pre_state, block.slot) + pre_state.validator_registry[proposer_index].slashed = True + with pytest.raises(AssertionError): + process_block_header(pre_state, block) + + return state, [block], None diff --git a/tests/phase0/block_processing/test_process_deposit.py b/tests/phase0/block_processing/test_process_deposit.py new file mode 100644 index 000000000..297ad37f1 --- /dev/null +++ b/tests/phase0/block_processing/test_process_deposit.py @@ -0,0 +1,132 @@ +from copy import deepcopy +import pytest + +import build.phase0.spec as spec + +from build.phase0.spec import ( + Deposit, + process_deposit, +) +from tests.phase0.helpers import ( + build_deposit, +) + + +# mark entire file as 'voluntary_exits' +pytestmark = pytest.mark.voluntary_exits + + +def test_success(state, deposit_data_leaves, pubkeys, privkeys): + pre_state = deepcopy(state) + + index = len(deposit_data_leaves) + pubkey = pubkeys[index] + privkey = privkeys[index] + deposit, root, deposit_data_leaves = build_deposit( + pre_state, + deposit_data_leaves, + pubkey, + privkey, + spec.MAX_DEPOSIT_AMOUNT, + ) + + pre_state.latest_eth1_data.deposit_root = root + pre_state.latest_eth1_data.deposit_count = len(deposit_data_leaves) + + post_state = deepcopy(pre_state) + + process_deposit(post_state, deposit) + + assert len(post_state.validator_registry) == len(state.validator_registry) + 1 + assert len(post_state.validator_balances) == len(state.validator_balances) + 1 + assert post_state.validator_registry[index].pubkey == pubkeys[index] + assert post_state.deposit_index == post_state.latest_eth1_data.deposit_count + + return pre_state, deposit, post_state + + +def test_success_top_up(state, deposit_data_leaves, pubkeys, privkeys): + pre_state = deepcopy(state) + + validator_index = 0 + amount = spec.MAX_DEPOSIT_AMOUNT // 4 + pubkey = pubkeys[validator_index] + privkey = privkeys[validator_index] + deposit, root, deposit_data_leaves = build_deposit( + pre_state, + deposit_data_leaves, + pubkey, + privkey, + amount, + ) + + pre_state.latest_eth1_data.deposit_root = root + pre_state.latest_eth1_data.deposit_count = len(deposit_data_leaves) + pre_balance = pre_state.validator_balances[validator_index] + + post_state = deepcopy(pre_state) + + process_deposit(post_state, deposit) + + assert len(post_state.validator_registry) == len(state.validator_registry) + assert len(post_state.validator_balances) == len(state.validator_balances) + assert post_state.deposit_index == post_state.latest_eth1_data.deposit_count + assert post_state.validator_balances[validator_index] == pre_balance + amount + + return pre_state, deposit, post_state + + +def test_wrong_index(state, deposit_data_leaves, pubkeys, privkeys): + pre_state = deepcopy(state) + + index = len(deposit_data_leaves) + pubkey = pubkeys[index] + privkey = privkeys[index] + deposit, root, deposit_data_leaves = build_deposit( + pre_state, + deposit_data_leaves, + pubkey, + privkey, + spec.MAX_DEPOSIT_AMOUNT, + ) + + # mess up deposit_index + deposit.index = pre_state.deposit_index + 1 + + pre_state.latest_eth1_data.deposit_root = root + pre_state.latest_eth1_data.deposit_count = len(deposit_data_leaves) + + post_state = deepcopy(pre_state) + + with pytest.raises(AssertionError): + process_deposit(post_state, deposit) + + return pre_state, deposit, None + + +def test_bad_merkle_proof(state, deposit_data_leaves, pubkeys, privkeys): + pre_state = deepcopy(state) + + index = len(deposit_data_leaves) + pubkey = pubkeys[index] + privkey = privkeys[index] + deposit, root, deposit_data_leaves = build_deposit( + pre_state, + deposit_data_leaves, + pubkey, + privkey, + spec.MAX_DEPOSIT_AMOUNT, + ) + + # mess up merkle branch + deposit.proof[-1] = spec.ZERO_HASH + + pre_state.latest_eth1_data.deposit_root = root + pre_state.latest_eth1_data.deposit_count = len(deposit_data_leaves) + + post_state = deepcopy(pre_state) + + with pytest.raises(AssertionError): + process_deposit(post_state, deposit) + + return pre_state, deposit, None diff --git a/tests/phase0/block_processing/test_voluntary_exit.py b/tests/phase0/block_processing/test_voluntary_exit.py new file mode 100644 index 000000000..0801e4292 --- /dev/null +++ b/tests/phase0/block_processing/test_voluntary_exit.py @@ -0,0 +1,174 @@ +from copy import deepcopy +import pytest + +import build.phase0.spec as spec + +from build.phase0.spec import ( + get_active_validator_indices, + get_current_epoch, + process_voluntary_exit, +) +from tests.phase0.helpers import ( + build_voluntary_exit, +) + + +# mark entire file as 'voluntary_exits' +pytestmark = pytest.mark.voluntary_exits + + +def test_success(state, pub_to_priv): + pre_state = deepcopy(state) + # + # setup pre_state + # + # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit + pre_state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + + # + # build voluntary exit + # + current_epoch = get_current_epoch(pre_state) + validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0] + privkey = pub_to_priv[pre_state.validator_registry[validator_index].pubkey] + + voluntary_exit = build_voluntary_exit( + pre_state, + current_epoch, + validator_index, + privkey, + ) + + post_state = deepcopy(pre_state) + + # + # test valid exit + # + process_voluntary_exit(post_state, voluntary_exit) + + assert not pre_state.validator_registry[validator_index].initiated_exit + assert post_state.validator_registry[validator_index].initiated_exit + + return pre_state, voluntary_exit, post_state + + +def test_validator_not_active(state, pub_to_priv): + pre_state = deepcopy(state) + current_epoch = get_current_epoch(pre_state) + validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0] + privkey = pub_to_priv[pre_state.validator_registry[validator_index].pubkey] + + # + # setup pre_state + # + pre_state.validator_registry[validator_index].activation_epoch = spec.FAR_FUTURE_EPOCH + + # + # build and test voluntary exit + # + voluntary_exit = build_voluntary_exit( + pre_state, + current_epoch, + validator_index, + privkey, + ) + + with pytest.raises(AssertionError): + process_voluntary_exit(pre_state, voluntary_exit) + + return pre_state, voluntary_exit, None + + +def test_validator_already_exited(state, pub_to_priv): + pre_state = deepcopy(state) + # + # setup pre_state + # + # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow validator able to exit + pre_state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + + current_epoch = get_current_epoch(pre_state) + validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0] + privkey = pub_to_priv[pre_state.validator_registry[validator_index].pubkey] + + # but validator already has exited + pre_state.validator_registry[validator_index].exit_epoch = current_epoch + 2 + + # + # build voluntary exit + # + voluntary_exit = build_voluntary_exit( + pre_state, + current_epoch, + validator_index, + privkey, + ) + + with pytest.raises(AssertionError): + process_voluntary_exit(pre_state, voluntary_exit) + + return pre_state, voluntary_exit, None + + +def test_validator_already_initiated_exit(state, pub_to_priv): + pre_state = deepcopy(state) + # + # setup pre_state + # + # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow validator able to exit + pre_state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + + current_epoch = get_current_epoch(pre_state) + validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0] + privkey = pub_to_priv[pre_state.validator_registry[validator_index].pubkey] + + # but validator already has initiated exit + pre_state.validator_registry[validator_index].initiated_exit = True + + # + # build voluntary exit + # + voluntary_exit = build_voluntary_exit( + pre_state, + current_epoch, + validator_index, + privkey, + ) + + with pytest.raises(AssertionError): + process_voluntary_exit(pre_state, voluntary_exit) + + return pre_state, voluntary_exit, None + + +def test_validator_not_active_long_enough(state, pub_to_priv): + pre_state = deepcopy(state) + # + # setup pre_state + # + current_epoch = get_current_epoch(pre_state) + validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0] + privkey = pub_to_priv[pre_state.validator_registry[validator_index].pubkey] + + # but validator already has initiated exit + pre_state.validator_registry[validator_index].initiated_exit = True + + # + # build voluntary exit + # + voluntary_exit = build_voluntary_exit( + pre_state, + current_epoch, + validator_index, + privkey, + ) + + assert ( + current_epoch - pre_state.validator_registry[validator_index].activation_epoch < + spec.PERSISTENT_COMMITTEE_PERIOD + ) + + with pytest.raises(AssertionError): + process_voluntary_exit(pre_state, voluntary_exit) + + return pre_state, voluntary_exit, None diff --git a/tests/phase0/conftest.py b/tests/phase0/conftest.py index e92896e92..395929028 100644 --- a/tests/phase0/conftest.py +++ b/tests/phase0/conftest.py @@ -5,6 +5,7 @@ from build.phase0 import spec from tests.phase0.helpers import ( privkeys_list, pubkeys_list, + pubkey_to_privkey, create_genesis_state, ) @@ -34,6 +35,11 @@ def pubkeys(): return pubkeys_list +@pytest.fixture +def pub_to_priv(): + return pubkey_to_privkey + + def overwrite_spec_config(config): for field in config: setattr(spec, field, config[field]) diff --git a/tests/phase0/helpers.py b/tests/phase0/helpers.py index 510361e9c..5c61685a6 100644 --- a/tests/phase0/helpers.py +++ b/tests/phase0/helpers.py @@ -13,6 +13,7 @@ from build.phase0.spec import ( DepositInput, DepositData, Eth1Data, + VoluntaryExit, # functions get_block_root, get_current_epoch, @@ -71,17 +72,29 @@ def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves): def create_genesis_state(num_validators, deposit_data_leaves): - initial_deposits, deposit_root = create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves) + initial_deposits, deposit_root = create_mock_genesis_validator_deposits( + num_validators, + deposit_data_leaves, + ) return get_genesis_beacon_state( initial_deposits, genesis_time=0, genesis_eth1_data=Eth1Data( deposit_root=deposit_root, + deposit_count=len(initial_deposits), block_hash=spec.ZERO_HASH, ), ) +def force_registry_change_at_next_epoch(state): + # artificially trigger registry update at next epoch transition + state.finalized_epoch = get_current_epoch(state) - 1 + for crosslink in state.latest_crosslinks: + crosslink.epoch = state.finalized_epoch + state.validator_registry_update_epoch = state.finalized_epoch - 1 + + def build_empty_block_for_next_slot(state): empty_block = get_empty_block() empty_block.slot = state.slot + 1 @@ -143,3 +156,46 @@ def build_attestation_data(state, slot, shard): crosslink_data_root=spec.ZERO_HASH, previous_crosslink=deepcopy(state.latest_crosslinks[shard]), ) + + +def build_voluntary_exit(state, epoch, validator_index, privkey): + voluntary_exit = VoluntaryExit( + epoch=epoch, + validator_index=validator_index, + signature=EMPTY_SIGNATURE, + ) + voluntary_exit.signature = bls.sign( + message_hash=signed_root(voluntary_exit), + privkey=privkey, + domain=get_domain( + fork=state.fork, + epoch=epoch, + domain_type=spec.DOMAIN_VOLUNTARY_EXIT, + ) + ) + + return voluntary_exit + + +def build_deposit(state, + deposit_data_leaves, + pubkey, + privkey, + amount): + deposit_data = build_deposit_data(state, pubkey, privkey, amount) + + item = hash(deposit_data.serialize()) + index = len(deposit_data_leaves) + deposit_data_leaves.append(item) + tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) + root = get_merkle_root((tuple(deposit_data_leaves))) + proof = list(get_merkle_proof(tree, item_index=index)) + assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, root) + + deposit = Deposit( + proof=list(proof), + index=index, + deposit_data=deposit_data, + ) + + return deposit, root, deposit_data_leaves diff --git a/utils/phase0/minimal_ssz.py b/utils/phase0/minimal_ssz.py index 08bd68357..c4828d08f 100644 --- a/utils/phase0/minimal_ssz.py +++ b/utils/phase0/minimal_ssz.py @@ -39,15 +39,22 @@ def SSZType(fields): return SSZObject -class Vector(list): - def __init__(self, x): - list.__init__(self, x) - self.length = len(x) +class Vector(): + def __init__(self, items): + self.items = items + self.length = len(items) - def append(*args): - raise Exception("Cannot change the length of a vector") + def __getitem__(self, key): + return self.items[key] - remove = clear = extend = pop = insert = append + def __setitem__(self, key, value): + self.items[key] = value + + def __iter__(self): + return iter(self.items) + + def __len__(self): + return self.length def is_basic(typ): From c26d09540df8ccdeb66d37af9bf87435f4dbd6de Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 20 Mar 2019 08:51:40 -0600 Subject: [PATCH 002/121] port header tests --- .../test_process_block_header.py | 68 +++++++++++++++++-- 1 file changed, 63 insertions(+), 5 deletions(-) diff --git a/tests/phase0/block_processing/test_process_block_header.py b/tests/phase0/block_processing/test_process_block_header.py index 4ec7e336f..8998a6e8f 100644 --- a/tests/phase0/block_processing/test_process_block_header.py +++ b/tests/phase0/block_processing/test_process_block_header.py @@ -3,7 +3,8 @@ import pytest from build.phase0.spec import ( - get_beacon_proposer_index, + cache_state, + advance_slot, process_block_header, ) from tests.phase0.helpers import ( @@ -14,13 +15,70 @@ from tests.phase0.helpers import ( pytestmark = pytest.mark.header -def test_proposer_slashed(state): +def test_sucess(state): + pre_state = deepcopy(state) + block = build_empty_block_for_next_slot(pre_state) + + # + # setup pre_state to be ready for block transition + # + cache_state(pre_state) + advance_slot(pre_state) + + post_state = deepcopy(pre_state) + + # + # test block header + # + process_block_header(post_state, block) + + return state, [block], post_state + + +def test_invalid_slot(state): pre_state = deepcopy(state) + # mess up previous block root block = build_empty_block_for_next_slot(pre_state) - proposer_index = get_beacon_proposer_index(pre_state, block.slot) - pre_state.validator_registry[proposer_index].slashed = True + block.previous_block_root = b'\12'*32 + + # + # setup pre_state advancing two slots to induce error + # + cache_state(pre_state) + advance_slot(pre_state) + advance_slot(pre_state) + + post_state = deepcopy(pre_state) + + # + # test block header + # with pytest.raises(AssertionError): - process_block_header(pre_state, block) + process_block_header(post_state, block) + + return state, [block], None + + +def test_invalid_previous_block_root(state): + pre_state = deepcopy(state) + + # mess up previous block root + block = build_empty_block_for_next_slot(pre_state) + block.previous_block_root = b'\12'*32 + + # + # setup pre_state to be ready for block transition + # + cache_state(pre_state) + advance_slot(pre_state) + + post_state = deepcopy(pre_state) + + # + # test block header + # + with pytest.raises(AssertionError): + process_block_header(post_state, block) return state, [block], None From c10384d65fa653cc395490a84bb1abe227bc74ad Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 20 Mar 2019 09:19:58 -0600 Subject: [PATCH 003/121] use signed_root for block id purposes in blocks/state --- specs/core/0_beacon-chain.md | 7 ++++--- tests/phase0/helpers.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index daa1bc108..57ad5e110 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -684,7 +684,8 @@ def get_temporary_block_header(block: BeaconBlock) -> BeaconBlockHeader: previous_block_root=block.previous_block_root, state_root=ZERO_HASH, block_body_root=hash_tree_root(block.body), - signature=block.signature, + # signed_root(block) is used for block id purposes so signature is a stub + signature=EMPTY_SIGNATURE, ) ``` @@ -1689,7 +1690,7 @@ def cache_state(state: BeaconState) -> None: state.latest_block_header.state_root = previous_slot_state_root # store latest known block for previous slot - state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = hash_tree_root(state.latest_block_header) + state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = signed_root(state.latest_block_header) ``` ### Per-epoch processing @@ -2250,7 +2251,7 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: # Verify that the slots match assert block.slot == state.slot # Verify that the parent matches - assert block.previous_block_root == hash_tree_root(state.latest_block_header) + assert block.previous_block_root == signed_root(state.latest_block_header) # Save current block as the new latest block state.latest_block_header = get_temporary_block_header(block) # Verify proposer signature diff --git a/tests/phase0/helpers.py b/tests/phase0/helpers.py index 510361e9c..16f5a5ea4 100644 --- a/tests/phase0/helpers.py +++ b/tests/phase0/helpers.py @@ -88,7 +88,7 @@ def build_empty_block_for_next_slot(state): previous_block_header = deepcopy(state.latest_block_header) if previous_block_header.state_root == spec.ZERO_HASH: previous_block_header.state_root = state.hash_tree_root() - empty_block.previous_block_root = previous_block_header.hash_tree_root() + empty_block.previous_block_root = signed_root(previous_block_header) return empty_block From b65ff4988d4129bce535843d1e25b1290b72fb4d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 20 Mar 2019 10:20:08 -0600 Subject: [PATCH 004/121] fix updated tests --- .../test_process_block_header.py | 84 +++++++------------ 1 file changed, 30 insertions(+), 54 deletions(-) diff --git a/tests/phase0/block_processing/test_process_block_header.py b/tests/phase0/block_processing/test_process_block_header.py index 8998a6e8f..241466437 100644 --- a/tests/phase0/block_processing/test_process_block_header.py +++ b/tests/phase0/block_processing/test_process_block_header.py @@ -3,6 +3,7 @@ import pytest from build.phase0.spec import ( + get_beacon_proposer_index, cache_state, advance_slot, process_block_header, @@ -15,70 +16,45 @@ from tests.phase0.helpers import ( pytestmark = pytest.mark.header -def test_sucess(state): - pre_state = deepcopy(state) - block = build_empty_block_for_next_slot(pre_state) +def prepare_state_for_header_processing(state): + cache_state(state) + advance_slot(state) - # - # setup pre_state to be ready for block transition - # - cache_state(pre_state) - advance_slot(pre_state) - post_state = deepcopy(pre_state) +def run_block_header_processing(state, block, valid=True): + """ + Run ``process_block_header`` returning the pre and post state. + If ``valid == False``, run expecting ``AssertionError`` + """ + prepare_state_for_header_processing(state) + post_state = deepcopy(state) + + if not valid: + with pytest.raises(AssertionError): + process_block_header(post_state, block) + return state, None - # - # test block header - # process_block_header(post_state, block) + return state, post_state - return state, [block], post_state + +def test_success(state): + block = build_empty_block_for_next_slot(state) + pre_state, post_state = run_block_header_processing(state, block) + return state, block, post_state def test_invalid_slot(state): - pre_state = deepcopy(state) + block = build_empty_block_for_next_slot(state) + block.slot = state.slot + 2 # invalid slot - # mess up previous block root - block = build_empty_block_for_next_slot(pre_state) - block.previous_block_root = b'\12'*32 - - # - # setup pre_state advancing two slots to induce error - # - cache_state(pre_state) - advance_slot(pre_state) - advance_slot(pre_state) - - post_state = deepcopy(pre_state) - - # - # test block header - # - with pytest.raises(AssertionError): - process_block_header(post_state, block) - - return state, [block], None + pre_state, post_state = run_block_header_processing(state, block, valid=False) + return pre_state, block, None def test_invalid_previous_block_root(state): - pre_state = deepcopy(state) + block = build_empty_block_for_next_slot(state) + block.previous_block_root = b'\12'*32 # invalid prev root - # mess up previous block root - block = build_empty_block_for_next_slot(pre_state) - block.previous_block_root = b'\12'*32 - - # - # setup pre_state to be ready for block transition - # - cache_state(pre_state) - advance_slot(pre_state) - - post_state = deepcopy(pre_state) - - # - # test block header - # - with pytest.raises(AssertionError): - process_block_header(post_state, block) - - return state, [block], None + pre_state, post_state = run_block_header_processing(state, block, valid=False) + return pre_state, block, None From 3e70478078608db8bd250981cdf9a1546f9ca143 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 21 Mar 2019 08:36:45 -0600 Subject: [PATCH 005/121] #805 direct to master --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 57ad5e110..fead1fad9 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -2134,8 +2134,8 @@ def update_registry_and_shuffling_data(state: BeaconState) -> None: state.current_shuffling_epoch = next_epoch state.current_shuffling_start_shard = ( state.current_shuffling_start_shard + - get_current_epoch_committee_count(state) % SHARD_COUNT - ) + get_current_epoch_committee_count(state) + ) % SHARD_COUNT state.current_shuffling_seed = generate_seed(state, state.current_shuffling_epoch) else: # If processing at least one crosslink keeps failing, then reshuffle every power of two, From 65b1b8555cbf1bd87ff59ee0d72e7f01bad3a2de Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 25 Mar 2019 10:37:34 -0600 Subject: [PATCH 006/121] cleanup tests to more easily dump output --- .gitignore | 1 + tests/phase0/__init__.py | 0 .../block_processing/test_process_deposit.py | 18 +++++-- .../block_processing/test_voluntary_exit.py | 21 ++++---- tests/phase0/conftest.py | 18 ------- tests/phase0/helpers.py | 14 ++--- tests/phase0/test_sanity.py | 18 ++++--- utils/phase0/jsonize.py | 52 +++++++++++++++++++ 8 files changed, 95 insertions(+), 47 deletions(-) create mode 100644 tests/phase0/__init__.py create mode 100644 utils/phase0/jsonize.py diff --git a/.gitignore b/.gitignore index dfb38d170..f33dd5256 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ /.pytest_cache build/ +output/ diff --git a/tests/phase0/__init__.py b/tests/phase0/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/phase0/block_processing/test_process_deposit.py b/tests/phase0/block_processing/test_process_deposit.py index 297ad37f1..0c5770195 100644 --- a/tests/phase0/block_processing/test_process_deposit.py +++ b/tests/phase0/block_processing/test_process_deposit.py @@ -4,11 +4,13 @@ import pytest import build.phase0.spec as spec from build.phase0.spec import ( - Deposit, + ZERO_HASH, process_deposit, ) from tests.phase0.helpers import ( build_deposit, + privkeys, + pubkeys, ) @@ -16,8 +18,10 @@ from tests.phase0.helpers import ( pytestmark = pytest.mark.voluntary_exits -def test_success(state, deposit_data_leaves, pubkeys, privkeys): +def test_success(state): pre_state = deepcopy(state) + # fill previous deposits with zero-hash + deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) index = len(deposit_data_leaves) pubkey = pubkeys[index] @@ -45,8 +49,9 @@ def test_success(state, deposit_data_leaves, pubkeys, privkeys): return pre_state, deposit, post_state -def test_success_top_up(state, deposit_data_leaves, pubkeys, privkeys): +def test_success_top_up(state): pre_state = deepcopy(state) + deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) validator_index = 0 amount = spec.MAX_DEPOSIT_AMOUNT // 4 @@ -76,8 +81,10 @@ def test_success_top_up(state, deposit_data_leaves, pubkeys, privkeys): return pre_state, deposit, post_state -def test_wrong_index(state, deposit_data_leaves, pubkeys, privkeys): +def test_wrong_index(state): pre_state = deepcopy(state) + deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) + index = len(deposit_data_leaves) pubkey = pubkeys[index] @@ -104,8 +111,9 @@ def test_wrong_index(state, deposit_data_leaves, pubkeys, privkeys): return pre_state, deposit, None -def test_bad_merkle_proof(state, deposit_data_leaves, pubkeys, privkeys): +def test_bad_merkle_proof(state): pre_state = deepcopy(state) + deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) index = len(deposit_data_leaves) pubkey = pubkeys[index] diff --git a/tests/phase0/block_processing/test_voluntary_exit.py b/tests/phase0/block_processing/test_voluntary_exit.py index 0801e4292..6adc81464 100644 --- a/tests/phase0/block_processing/test_voluntary_exit.py +++ b/tests/phase0/block_processing/test_voluntary_exit.py @@ -10,6 +10,7 @@ from build.phase0.spec import ( ) from tests.phase0.helpers import ( build_voluntary_exit, + pubkey_to_privkey, ) @@ -17,7 +18,7 @@ from tests.phase0.helpers import ( pytestmark = pytest.mark.voluntary_exits -def test_success(state, pub_to_priv): +def test_success(state): pre_state = deepcopy(state) # # setup pre_state @@ -30,7 +31,7 @@ def test_success(state, pub_to_priv): # current_epoch = get_current_epoch(pre_state) validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0] - privkey = pub_to_priv[pre_state.validator_registry[validator_index].pubkey] + privkey = pubkey_to_privkey[pre_state.validator_registry[validator_index].pubkey] voluntary_exit = build_voluntary_exit( pre_state, @@ -52,11 +53,11 @@ def test_success(state, pub_to_priv): return pre_state, voluntary_exit, post_state -def test_validator_not_active(state, pub_to_priv): +def test_validator_not_active(state): pre_state = deepcopy(state) current_epoch = get_current_epoch(pre_state) validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0] - privkey = pub_to_priv[pre_state.validator_registry[validator_index].pubkey] + privkey = pubkey_to_privkey[pre_state.validator_registry[validator_index].pubkey] # # setup pre_state @@ -79,7 +80,7 @@ def test_validator_not_active(state, pub_to_priv): return pre_state, voluntary_exit, None -def test_validator_already_exited(state, pub_to_priv): +def test_validator_already_exited(state): pre_state = deepcopy(state) # # setup pre_state @@ -89,7 +90,7 @@ def test_validator_already_exited(state, pub_to_priv): current_epoch = get_current_epoch(pre_state) validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0] - privkey = pub_to_priv[pre_state.validator_registry[validator_index].pubkey] + privkey = pubkey_to_privkey[pre_state.validator_registry[validator_index].pubkey] # but validator already has exited pre_state.validator_registry[validator_index].exit_epoch = current_epoch + 2 @@ -110,7 +111,7 @@ def test_validator_already_exited(state, pub_to_priv): return pre_state, voluntary_exit, None -def test_validator_already_initiated_exit(state, pub_to_priv): +def test_validator_already_initiated_exit(state): pre_state = deepcopy(state) # # setup pre_state @@ -120,7 +121,7 @@ def test_validator_already_initiated_exit(state, pub_to_priv): current_epoch = get_current_epoch(pre_state) validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0] - privkey = pub_to_priv[pre_state.validator_registry[validator_index].pubkey] + privkey = pubkey_to_privkey[pre_state.validator_registry[validator_index].pubkey] # but validator already has initiated exit pre_state.validator_registry[validator_index].initiated_exit = True @@ -141,14 +142,14 @@ def test_validator_already_initiated_exit(state, pub_to_priv): return pre_state, voluntary_exit, None -def test_validator_not_active_long_enough(state, pub_to_priv): +def test_validator_not_active_long_enough(state): pre_state = deepcopy(state) # # setup pre_state # current_epoch = get_current_epoch(pre_state) validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0] - privkey = pub_to_priv[pre_state.validator_registry[validator_index].pubkey] + privkey = pubkey_to_privkey[pre_state.validator_registry[validator_index].pubkey] # but validator already has initiated exit pre_state.validator_registry[validator_index].initiated_exit = True diff --git a/tests/phase0/conftest.py b/tests/phase0/conftest.py index 395929028..36a087941 100644 --- a/tests/phase0/conftest.py +++ b/tests/phase0/conftest.py @@ -3,9 +3,6 @@ import pytest from build.phase0 import spec from tests.phase0.helpers import ( - privkeys_list, - pubkeys_list, - pubkey_to_privkey, create_genesis_state, ) @@ -25,21 +22,6 @@ MINIMAL_CONFIG = { } -@pytest.fixture -def privkeys(): - return privkeys_list - - -@pytest.fixture -def pubkeys(): - return pubkeys_list - - -@pytest.fixture -def pub_to_priv(): - return pubkey_to_privkey - - def overwrite_spec_config(config): for field in config: setattr(spec, field, config[field]) diff --git a/tests/phase0/helpers.py b/tests/phase0/helpers.py index 134fa797d..20054b821 100644 --- a/tests/phase0/helpers.py +++ b/tests/phase0/helpers.py @@ -31,18 +31,20 @@ from build.phase0.utils.merkle_minimal import ( ) -privkeys_list = [i + 1 for i in range(1000)] -pubkeys_list = [bls.privtopub(privkey) for privkey in privkeys_list] -pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys_list, pubkeys_list)} +privkeys = [i + 1 for i in range(1000)] +pubkeys = [bls.privtopub(privkey) for privkey in privkeys] +pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} -def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves): +def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves=None): + if not deposit_data_leaves: + deposit_data_leaves = [] deposit_timestamp = 0 proof_of_possession = b'\x33' * 96 deposit_data_list = [] for i in range(num_validators): - pubkey = pubkeys_list[i] + pubkey = pubkeys[i] deposit_data = DepositData( amount=spec.MAX_DEPOSIT_AMOUNT, timestamp=deposit_timestamp, @@ -71,7 +73,7 @@ def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves): return genesis_validator_deposits, root -def create_genesis_state(num_validators, deposit_data_leaves): +def create_genesis_state(num_validators, deposit_data_leaves=None): initial_deposits, deposit_root = create_mock_genesis_validator_deposits( num_validators, deposit_data_leaves, diff --git a/tests/phase0/test_sanity.py b/tests/phase0/test_sanity.py index 8f04f316c..6de6dfb45 100644 --- a/tests/phase0/test_sanity.py +++ b/tests/phase0/test_sanity.py @@ -43,6 +43,8 @@ from tests.phase0.helpers import ( build_attestation_data, build_deposit_data, build_empty_block_for_next_slot, + privkeys, + pubkeys, ) @@ -112,7 +114,7 @@ def test_empty_epoch_transition_not_finalizing(state): return state, [block], test_state -def test_proposer_slashing(state, pubkeys, privkeys): +def test_proposer_slashing(state): test_state = deepcopy(state) current_epoch = get_current_epoch(test_state) validator_index = get_active_validator_indices(test_state.validator_registry, current_epoch)[-1] @@ -172,9 +174,9 @@ def test_proposer_slashing(state, pubkeys, privkeys): return state, [block], test_state -def test_deposit_in_block(state, deposit_data_leaves, pubkeys, privkeys): +def test_deposit_in_block(state): pre_state = deepcopy(state) - test_deposit_data_leaves = deepcopy(deposit_data_leaves) + test_deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) index = len(test_deposit_data_leaves) pubkey = pubkeys[index] @@ -207,9 +209,9 @@ def test_deposit_in_block(state, deposit_data_leaves, pubkeys, privkeys): return pre_state, [block], post_state -def test_deposit_top_up(state, pubkeys, privkeys, deposit_data_leaves): +def test_deposit_top_up(state): pre_state = deepcopy(state) - test_deposit_data_leaves = deepcopy(deposit_data_leaves) + test_deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) validator_index = 0 amount = spec.MAX_DEPOSIT_AMOUNT // 4 @@ -245,7 +247,7 @@ def test_deposit_top_up(state, pubkeys, privkeys, deposit_data_leaves): return pre_state, [block], post_state -def test_attestation(state, pubkeys, privkeys): +def test_attestation(state): test_state = deepcopy(state) slot = state.slot shard = state.current_shuffling_start_shard @@ -314,7 +316,7 @@ def test_attestation(state, pubkeys, privkeys): return state, [attestation_block, epoch_block], test_state -def test_voluntary_exit(state, pubkeys, privkeys): +def test_voluntary_exit(state): pre_state = deepcopy(state) validator_index = get_active_validator_indices(pre_state.validator_registry, get_current_epoch(pre_state))[-1] @@ -363,7 +365,7 @@ def test_voluntary_exit(state, pubkeys, privkeys): return pre_state, [initiate_exit_block, exit_block], post_state -def test_transfer(state, pubkeys, privkeys): +def test_transfer(state): pre_state = deepcopy(state) current_epoch = get_current_epoch(pre_state) sender_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[-1] diff --git a/utils/phase0/jsonize.py b/utils/phase0/jsonize.py new file mode 100644 index 000000000..816192ec6 --- /dev/null +++ b/utils/phase0/jsonize.py @@ -0,0 +1,52 @@ +from .minimal_ssz import hash_tree_root + + +def jsonize(value, typ, include_hash_tree_roots=False): + if isinstance(typ, str) and typ[:4] == 'uint': + return value + elif typ == 'bool': + assert value in (True, False) + return value + elif isinstance(typ, list): + return [jsonize(element, typ[0], include_hash_tree_roots) for element in value] + elif isinstance(typ, str) and typ[:4] == 'byte': + return '0x' + value.hex() + elif hasattr(typ, 'fields'): + ret = {} + for field, subtype in typ.fields.items(): + ret[field] = jsonize(getattr(value, field), subtype, include_hash_tree_roots) + if include_hash_tree_roots: + ret[field + "_hash_tree_root"] = '0x' + hash_tree_root(getattr(value, field), subtype).hex() + if include_hash_tree_roots: + ret["hash_tree_root"] = '0x' + hash_tree_root(value, typ).hex() + return ret + else: + print(value, typ) + raise Exception("Type not recognized") + + +def dejsonize(json, typ): + if isinstance(typ, str) and typ[:4] == 'uint': + return json + elif typ == 'bool': + assert json in (True, False) + return json + elif isinstance(typ, list): + return [dejsonize(element, typ[0]) for element in json] + elif isinstance(typ, str) and typ[:4] == 'byte': + return bytes.fromhex(json[2:]) + elif hasattr(typ, 'fields'): + temp = {} + for field, subtype in typ.fields.items(): + temp[field] = dejsonize(json[field], subtype) + if field + "_hash_tree_root" in json: + assert(json[field + "_hash_tree_root"][2:] == + hash_tree_root(temp[field], subtype).hex()) + ret = typ(**temp) + if "hash_tree_root" in json: + assert(json["hash_tree_root"][2:] == + hash_tree_root(ret, typ).hex()) + return ret + else: + print(json, typ) + raise Exception("Type not recognized") From 3fc24f3d415280484ff1c99813d060d39b242983 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sun, 31 Mar 2019 21:20:43 -0500 Subject: [PATCH 007/121] Replace with empty instead of popping finished challenges --- specs/core/1_custody-game.md | 44 +++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index fd754634e..41ba9d953 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -31,6 +31,7 @@ - [`get_crosslink_chunk_count`](#get_crosslink_chunk_count) - [`get_custody_chunk_bit`](#get_custody_chunk_bit) - [`epoch_to_custody_period`](#epoch_to_custody_period) + - [`replace_empty_or_append`](#replace_empty_or_append) - [`verify_custody_key`](#verify_custody_key) - [Per-block processing](#per-block-processing) - [Transactions](#transactions) @@ -229,6 +230,18 @@ def epoch_to_custody_period(epoch: Epoch) -> int: return epoch // EPOCHS_PER_CUSTODY_PERIOD ``` +### `replace_empty_or_append` + +```python +def replace_empty_or_append(list: List[Any], empty_element: Any, new_element: Any) -> int: + for i in range(len(list)): + if list[i] == empty_element: + list[i] = new_element + return i + list.append(new_element) + return len(list) - 1 +``` + ### `verify_custody_key` ```python @@ -321,7 +334,7 @@ def process_chunk_challenge(state: BeaconState, depth = math.log2(next_power_of_two(get_custody_chunk_count(challenge.attestation))) assert challenge.chunk_index < 2**depth # Add new chunk challenge record - state.custody_chunk_challenge_records.append(CustodyChunkChallengeRecord( + new_record = CustodyChunkChallengeRecord( challenge_index=state.custody_challenge_index, challenger_index=get_beacon_proposer_index(state, state.slot), responder_index=challenge.responder_index @@ -329,7 +342,13 @@ def process_chunk_challenge(state: BeaconState, crosslink_data_root=challenge.attestation.data.crosslink_data_root, depth=depth, chunk_index=challenge.chunk_index, - )) + ) + replace_empty_or_append( + list=state.custody_chunk_challenge_records, + empty_element=CustodyChunkChallengeRecord(), + new_element=new_record + ) + state.custody_challenge_index += 1 # Postpone responder withdrawability responder.withdrawable_epoch = FAR_FUTURE_EPOCH @@ -385,7 +404,7 @@ def process_bit_challenge(state: BeaconState, custody_bit = get_bitfield_bit(attestation.custody_bitfield, attesters.index(responder_index)) assert custody_bit != chunk_bits_xor # Add new bit challenge record - state.custody_bit_challenge_records.append(CustodyBitChallengeRecord( + new_record = CustodyBitChallengeRecord( challenge_index=state.custody_challenge_index, challenger_index=challenge.challenger_index, responder_index=challenge.responder_index, @@ -393,7 +412,12 @@ def process_bit_challenge(state: BeaconState, crosslink_data_root=challenge.attestation.crosslink_data_root, chunk_bits=challenge.chunk_bits, responder_key=challenge.responder_key, - )) + ) + replace_empty_or_append( + list=state.custody_bit_challenge_records, + empty_element=CustodyBitChallengeRecord(), + new_element=new_record + ) state.custody_challenge_index += 1 # Postpone responder withdrawability responder.withdrawable_epoch = FAR_FUTURE_EPOCH @@ -434,7 +458,8 @@ def process_chunk_challenge_response(state: BeaconState, root=challenge.crosslink_data_root, ) # Clear the challenge - state.custody_chunk_challenge_records.remove(challenge) + records = state.custody_chunk_challenge_records + records[records.index(challenge)] = CustodyChunkChallengeRecord() # Reward the proposer proposer_index = get_beacon_proposer_index(state, state.slot) increase_balance(state, proposer_index, base_reward(state, index) // MINOR_REWARD_QUOTIENT) @@ -457,7 +482,8 @@ def process_bit_challenge_response(state: BeaconState, # Verify the chunk bit does not match the challenge chunk bit assert get_custody_chunk_bit(challenge.responder_key, response.chunk) != get_bitfield_bit(challenge.chunk_bits, response.chunk_index) # Clear the challenge - state.custody_bit_challenge_records.remove(challenge) + records = state.custody_bit_challenge_records + records[records.index(challenge)] = CustodyBitChallengeRecord() # Slash challenger slash_validator(state, challenge.challenger_index, challenge.responder_index) ``` @@ -471,12 +497,14 @@ def process_challenge_deadlines(state: BeaconState) -> None: for challenge in state.custody_chunk_challenge_records: if get_current_epoch(state) > challenge.deadline: slash_validator(state, challenge.responder_index, challenge.challenger_index) - state.custody_chunk_challenge_records.remove(challenge) + records = state.custody_chunk_challenge_records + records[records.index(challenge)] = CustodyChunkChallengeRecord() for challenge in state.custody_bit_challenge_records: if get_current_epoch(state) > challenge.deadline: slash_validator(state, challenge.responder_index, challenge.challenger_index) - state.custody_bit_challenge_records.remove(challenge) + records = state.custody_bit_challenge_records + records[records.index(challenge)] = CustodyBitChallengeRecord() ``` In `process_penalties_and_exits`, change the definition of `eligible` to the following (note that it is not a pure function because `state` is declared in the surrounding scope): From ad07649ec3aa56737191052385aca1c2a482747c Mon Sep 17 00:00:00 2001 From: Piper Merriam Date: Mon, 18 Mar 2019 08:57:02 -0600 Subject: [PATCH 008/121] Update spec to use SOS style offset layout for variable size values. --- specs/simple-serialize.md | 61 ++++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 14 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 378a1a7cb..ef442042b 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -33,6 +33,8 @@ This is a **work in progress** describing typing, serialization and Merkleizatio ### Composite types +Composite types are limited to a maximum of `2**32 - 1` elements. + * **container**: ordered heterogenous collection of values * key-pair curly bracket notation `{}`, e.g. `{"foo": "uint64", "bar": "bool"}` * **vector**: ordered fixed-length homogeneous collection of values @@ -54,43 +56,74 @@ For convenience we alias: We recursively define the `serialize` function which consumes an object `value` (of the type specified) and returns a bytestring of type `"bytes"`. -*Note*: In the function definitions below (`serialize`, `hash_tree_root`, `signed_root`, etc.) objects implicitly carry their type. +> *Note*: In the function definitions below (`serialize`, `hash_tree_root`, `signed_root`, etc.) objects implicitly carry their type. -### `"uintN"` +### Basic Types + +For basic types the `serialize` function is defined as follows. + +#### `"uintN"` ```python assert N in [8, 16, 32, 64, 128, 256] return value.to_bytes(N // 8, "little") ``` -### `"bool"` +#### `"bool"` ```python assert value in (True, False) return b"\x01" if value is True else b"\x00" ``` -### Vectors, containers, lists +### Composite Types (Vectors, Containers and Lists) + +The serialized representation of composite types is comprised of three binary segments. + +* The first segment contains the concatenation of the serialized representation of **only** the *fixed size* types: + - This section is empty in the case of a purely *variable-size* type. +* The second segment contains the concatenation of the `uint32` serialized offsets where the serialized representation of the *variable sized* types can be found in the third section. + - This section is empty in the case of a purely *fixed size* type. + - The first offset will always be the length of the serialized *fixed size* elements + the length of all of the serialized offsets. + - Subsequent offsets are the first offset + the combined lengths of the serialized representations for all of the previous *variable size* elements. +* The third segment contains the concatenation of the serialized representations of **only** the *variable size* types. + - This section is empty in the case of a purely *fixed size* type. + + +#### `"vector"`, `"container"` and `"list"` + +For conmposite types the `serialize` function is defined as follows. -If `value` is fixed-size: ```python -return "".join([serialize(element) for element in value]) +# section 1 +section_1 = ''.join([serialize(element) for element in value if is_fixed_size(element)]) + +# section 3 +section_3_parts = [serialize(element) for element in value if is_variable_size(element)] +section_3 ''.join(section_3_parts) + +# section 2 +section_1_length = len(section_1) +section_2_length = 4 * len(section_3_parts) +section_3_base_offset = section_1_length + section_2_length +section_3_lengths = [len(part) for part in section_3_parts] +section_3_offsets = [ + (section_3_base_offset + sum(section_3_lengths[:index])) + for index in range(len(section_3_parts)) +] +assert all(offset < 2**32 for offset in section_3_offsets) +section_2 = ''.join([serialize(offset) for offset in section_3_offsets]) + +return ''.join([section_1, section_2, section_3]) ``` -If `value` is variable-size: - -```python -serialized_bytes = "".join([serialize(element) for element in value]) -assert len(serialized_bytes) < 2**(8 * BYTES_PER_LENGTH_PREFIX) -serialized_length = len(serialized_bytes).to_bytes(BYTES_PER_LENGTH_PREFIX, "little") -return serialized_length + serialized_bytes -``` ## Deserialization Because serialization is an injective function (i.e. two distinct objects of the same type will serialize to different values) any bytestring has at most one object it could deserialize to. Efficient algorithms for computing this object can be found in [the implementations](#implementations). + ## Merkleization We first define helper functions: From 4a0459a087ecd3cef8ba819a82bcb2c86188ca76 Mon Sep 17 00:00:00 2001 From: Piper Merriam Date: Mon, 18 Mar 2019 10:20:58 -0600 Subject: [PATCH 009/121] PR feedback --- specs/simple-serialize.md | 55 ++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index ef442042b..b695956d2 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -12,7 +12,7 @@ This is a **work in progress** describing typing, serialization and Merkleizatio - [Serialization](#serialization) - [`"uintN"`](#uintn) - [`"bool"`](#bool) - - [Vectors, containers, lists](#vectors-containers-lists) + - [Vectors, containers, lists](#composite-types-vectors-containers-and-lists) - [Deserialization](#deserialization) - [Merkleization](#merkleization) - [Self-signed containers](#self-signed-containers) @@ -33,8 +33,6 @@ This is a **work in progress** describing typing, serialization and Merkleizatio ### Composite types -Composite types are limited to a maximum of `2**32 - 1` elements. - * **container**: ordered heterogenous collection of values * key-pair curly bracket notation `{}`, e.g. `{"foo": "uint64", "bar": "bool"}` * **vector**: ordered fixed-length homogeneous collection of values @@ -78,15 +76,12 @@ return b"\x01" if value is True else b"\x00" ### Composite Types (Vectors, Containers and Lists) -The serialized representation of composite types is comprised of three binary segments. +The serialized representation of composite types is comprised of two binary segments. -* The first segment contains the concatenation of the serialized representation of **only** the *fixed size* types: - - This section is empty in the case of a purely *variable-size* type. -* The second segment contains the concatenation of the `uint32` serialized offsets where the serialized representation of the *variable sized* types can be found in the third section. - - This section is empty in the case of a purely *fixed size* type. - - The first offset will always be the length of the serialized *fixed size* elements + the length of all of the serialized offsets. - - Subsequent offsets are the first offset + the combined lengths of the serialized representations for all of the previous *variable size* elements. -* The third segment contains the concatenation of the serialized representations of **only** the *variable size* types. +* The first segment is *fixed size* for all types, containing the concatenation of *either* + - The serialized representation of value for each of the *fixed size* types + - The `"uint32"` serialized offset where the serialized representation of the *variable sized* type is located in the second section. +* The second segment contains the concatenation of the serialized representations of **only** the *variable size* types. - This section is empty in the case of a purely *fixed size* type. @@ -94,28 +89,30 @@ The serialized representation of composite types is comprised of three binary se For conmposite types the `serialize` function is defined as follows. +> *Note*: The `collate` function combines the serialized *fixed size* values +> and the serialized offsets into a single array correctly ordered with respect +> to the original element types. The implementation of this logic is not +> included in this example for simplicity. + ```python -# section 1 -section_1 = ''.join([serialize(element) for element in value if is_fixed_size(element)]) - -# section 3 -section_3_parts = [serialize(element) for element in value if is_variable_size(element)] -section_3 ''.join(section_3_parts) - # section 2 -section_1_length = len(section_1) -section_2_length = 4 * len(section_3_parts) -section_3_base_offset = section_1_length + section_2_length -section_3_lengths = [len(part) for part in section_3_parts] -section_3_offsets = [ - (section_3_base_offset + sum(section_3_lengths[:index])) - for index in range(len(section_3_parts)) -] -assert all(offset < 2**32 for offset in section_3_offsets) -section_2 = ''.join([serialize(offset) for offset in section_3_offsets]) +section_2_parts = [serialize(element) for element in value if is_variable_size(element)] +section_2_lengths = [len(part) for part in section_2_parts] +section_2 ''.join(section_2_parts) -return ''.join([section_1, section_2, section_3]) +# section 1 +section_1_fixed_parts = [serialize(element) for element in value if is_fixed_size(element)] +section_1_length = sum(len(part) for part in section_1_fixed_parts) + 4 * len(section_2_parts) +section_1_offsets = [ + section_1_length + sum(section_2_lengths[:index]) + for index in range(len(section_2_parts)) +] +assert all(offset < 2**32 for offset in section_1_offsets) +section_1_parts = collate(section_1_fixed_parts, section_1_offsets) +section_1 = ''.join(section_1_parts) + +return ''.join([section_1, section_2]) ``` From 605028bbda38b7af7191cad1018ba121f240cbd7 Mon Sep 17 00:00:00 2001 From: Piper Merriam Date: Mon, 18 Mar 2019 16:28:33 -0600 Subject: [PATCH 010/121] more precise definitions for and and expand code example for how sections are created --- specs/simple-serialize.md | 42 +++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index b695956d2..b85a20656 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -62,6 +62,8 @@ For basic types the `serialize` function is defined as follows. #### `"uintN"` +A byte string of width `N // 8` containing the little-endian encode integer. + ```python assert N in [8, 16, 32, 64, 128, 256] return value.to_bytes(N // 8, "little") @@ -69,6 +71,9 @@ return value.to_bytes(N // 8, "little") #### `"bool"` +* The byte `\x00` **if** the value is `False` +* The byte `\x01` **if** the value is `True` + ```python assert value in (True, False) return b"\x01" if value is True else b"\x00" @@ -87,29 +92,46 @@ The serialized representation of composite types is comprised of two binary segm #### `"vector"`, `"container"` and `"list"` -For conmposite types the `serialize` function is defined as follows. - -> *Note*: The `collate` function combines the serialized *fixed size* values -> and the serialized offsets into a single array correctly ordered with respect -> to the original element types. The implementation of this logic is not -> included in this example for simplicity. - +An implementation of the `serialize` function for `"Vector"`, `"Container"` and +`"List"` types would take the following form. ```python -# section 2 +# The second section is just the concatenation of the serialized *variable size* elements section_2_parts = [serialize(element) for element in value if is_variable_size(element)] section_2_lengths = [len(part) for part in section_2_parts] section_2 ''.join(section_2_parts) -# section 1 +# Serialize the *fixed size* elements section_1_fixed_parts = [serialize(element) for element in value if is_fixed_size(element)] + +# Compute the length of the first section section_1_length = sum(len(part) for part in section_1_fixed_parts) + 4 * len(section_2_parts) + +# Compute the offset values for each part of the second section section_1_offsets = [ section_1_length + sum(section_2_lengths[:index]) for index in range(len(section_2_parts)) ] assert all(offset < 2**32 for offset in section_1_offsets) -section_1_parts = collate(section_1_fixed_parts, section_1_offsets) + +# compute the appropriate indices for *fixed size* elements for the first section +fixed_size_element_indices = [index for index, element in enumerate(value) if is_fixed_size(element)] + +# compute the appropriate indices for the offsets of the *variable size* elements +variable_size_element_indices = [index for index, element in enumerate(value) if is_variable_size(element)] + +# create a list with placeholders for all values +section_1_parts = [None] * len(value) + +# populate all of the serialized *fixed size* elements +for index, data in zip(fixed_size_element_indices, section_1_fixed_parts): + section_1_parts[index] = data + +# populate all of the serialized offsets for the *variable size* elements +for index, offset in zip(variable_size_element_indices, section_1_offsets): + section_1_parts[index] = serialize(offset) + +assert not any(part is None for part in section_1_parts) section_1 = ''.join(section_1_parts) return ''.join([section_1, section_2]) From fa66640a00aed4fae5f8878d00ff02b6e50b6705 Mon Sep 17 00:00:00 2001 From: jannikluhn Date: Wed, 20 Mar 2019 09:01:27 -0600 Subject: [PATCH 011/121] Update specs/simple-serialize.md Co-Authored-By: pipermerriam --- specs/simple-serialize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index b85a20656..f2ca7c0cd 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -86,7 +86,7 @@ The serialized representation of composite types is comprised of two binary segm * The first segment is *fixed size* for all types, containing the concatenation of *either* - The serialized representation of value for each of the *fixed size* types - The `"uint32"` serialized offset where the serialized representation of the *variable sized* type is located in the second section. -* The second segment contains the concatenation of the serialized representations of **only** the *variable size* types. +* The second section contains the concatenation of the serialized representations of **only** the *variable size* types. - This section is empty in the case of a purely *fixed size* type. From 32684d582a14f59ea3d02490316ae84496a099ea Mon Sep 17 00:00:00 2001 From: jannikluhn Date: Wed, 20 Mar 2019 09:01:51 -0600 Subject: [PATCH 012/121] Update specs/simple-serialize.md Co-Authored-By: pipermerriam --- specs/simple-serialize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index f2ca7c0cd..8577dcd2b 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -85,7 +85,7 @@ The serialized representation of composite types is comprised of two binary segm * The first segment is *fixed size* for all types, containing the concatenation of *either* - The serialized representation of value for each of the *fixed size* types - - The `"uint32"` serialized offset where the serialized representation of the *variable sized* type is located in the second section. + - The `"uint32"` serialized offset where the serialized representation of the *variable sized* type is located in the second section relative to the beginning of the first section. * The second section contains the concatenation of the serialized representations of **only** the *variable size* types. - This section is empty in the case of a purely *fixed size* type. From 3741b7517b2facfd4beb50a8fd8f1d021d2cb7b1 Mon Sep 17 00:00:00 2001 From: jannikluhn Date: Wed, 20 Mar 2019 09:02:08 -0600 Subject: [PATCH 013/121] Update specs/simple-serialize.md Co-Authored-By: pipermerriam --- specs/simple-serialize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 8577dcd2b..cf4dd55ba 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -83,7 +83,7 @@ return b"\x01" if value is True else b"\x00" The serialized representation of composite types is comprised of two binary segments. -* The first segment is *fixed size* for all types, containing the concatenation of *either* +* The first section is *fixed size* for all types, containing the concatenation of *either* - The serialized representation of value for each of the *fixed size* types - The `"uint32"` serialized offset where the serialized representation of the *variable sized* type is located in the second section relative to the beginning of the first section. * The second section contains the concatenation of the serialized representations of **only** the *variable size* types. From 1ab501975cc9f615fec6d4770b60319b68672c52 Mon Sep 17 00:00:00 2001 From: jannikluhn Date: Wed, 20 Mar 2019 09:02:20 -0600 Subject: [PATCH 014/121] Update specs/simple-serialize.md Co-Authored-By: pipermerriam --- specs/simple-serialize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index cf4dd55ba..9d30f4f3d 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -84,7 +84,7 @@ return b"\x01" if value is True else b"\x00" The serialized representation of composite types is comprised of two binary segments. * The first section is *fixed size* for all types, containing the concatenation of *either* - - The serialized representation of value for each of the *fixed size* types + - The serialized representation for each of the *fixed size* elements of value - The `"uint32"` serialized offset where the serialized representation of the *variable sized* type is located in the second section relative to the beginning of the first section. * The second section contains the concatenation of the serialized representations of **only** the *variable size* types. - This section is empty in the case of a purely *fixed size* type. From ca98d752d2586968aee22cd73402dc5cff3d2b38 Mon Sep 17 00:00:00 2001 From: Piper Merriam Date: Wed, 20 Mar 2019 09:04:55 -0600 Subject: [PATCH 015/121] d --- specs/simple-serialize.md | 52 +++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 29 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 9d30f4f3d..1669c00ea 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -81,7 +81,7 @@ return b"\x01" if value is True else b"\x00" ### Composite Types (Vectors, Containers and Lists) -The serialized representation of composite types is comprised of two binary segments. +The serialized representation of composite types is comprised of two binary sections. * The first section is *fixed size* for all types, containing the concatenation of *either* - The serialized representation for each of the *fixed size* elements of value @@ -97,41 +97,35 @@ An implementation of the `serialize` function for `"Vector"`, `"Container"` and ```python # The second section is just the concatenation of the serialized *variable size* elements -section_2_parts = [serialize(element) for element in value if is_variable_size(element)] +section_2_parts = [ + serialize(element) if is_variable_size(element) + else '' + for element in value +] section_2_lengths = [len(part) for part in section_2_parts] -section_2 ''.join(section_2_parts) +section_2 = ''.join(section_2_parts) -# Serialize the *fixed size* elements -section_1_fixed_parts = [serialize(element) for element in value if is_fixed_size(element)] - -# Compute the length of the first section -section_1_length = sum(len(part) for part in section_1_fixed_parts) + 4 * len(section_2_parts) +# Compute the length of the first section (can also be extracted from the type directly) +section_1_length = sum( + len(serialize(element)) if is_fixed_size(element) + else 4 + for element in value +) # Compute the offset values for each part of the second section section_1_offsets = [ - section_1_length + sum(section_2_lengths[:index]) - for index in range(len(section_2_parts)) + section_1_length + sum(section_2_lengths[:element_index]) if is_variable_size(element) + else None + for element_index, element in enumerate(value) ] -assert all(offset < 2**32 for offset in section_1_offsets) +assert all(offset is None or offset < 2**32 for offset in section_1_offsets) -# compute the appropriate indices for *fixed size* elements for the first section -fixed_size_element_indices = [index for index, element in enumerate(value) if is_fixed_size(element)] - -# compute the appropriate indices for the offsets of the *variable size* elements -variable_size_element_indices = [index for index, element in enumerate(value) if is_variable_size(element)] - -# create a list with placeholders for all values -section_1_parts = [None] * len(value) - -# populate all of the serialized *fixed size* elements -for index, data in zip(fixed_size_element_indices, section_1_fixed_parts): - section_1_parts[index] = data - -# populate all of the serialized offsets for the *variable size* elements -for index, offset in zip(variable_size_element_indices, section_1_offsets): - section_1_parts[index] = serialize(offset) - -assert not any(part is None for part in section_1_parts) +# The first section is the concatenation of the serialized static size elements and offsets +section_1_parts = [ + serialize(element) if is_fixed_size(element) + else serialize(section_1_offsets[element_index]) + for element_index, element in enumerate(value) +] section_1 = ''.join(section_1_parts) return ''.join([section_1, section_2]) From 5f465842a4173bb50693d67cdae69ed5c2cfc5ce Mon Sep 17 00:00:00 2001 From: Piper Merriam Date: Wed, 20 Mar 2019 09:12:49 -0600 Subject: [PATCH 016/121] more language updates --- specs/simple-serialize.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 1669c00ea..4fcefb86f 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -54,7 +54,7 @@ For convenience we alias: We recursively define the `serialize` function which consumes an object `value` (of the type specified) and returns a bytestring of type `"bytes"`. -> *Note*: In the function definitions below (`serialize`, `hash_tree_root`, `signed_root`, etc.) objects implicitly carry their type. +> *Note*: In the function definitions below (`serialize`, `hash_tree_root`, `signed_root`, `is_fixed_size`, `is_variable_size` etc.) objects implicitly carry their type. ### Basic Types @@ -83,17 +83,19 @@ return b"\x01" if value is True else b"\x00" The serialized representation of composite types is comprised of two binary sections. -* The first section is *fixed size* for all types, containing the concatenation of *either* - - The serialized representation for each of the *fixed size* elements of value - - The `"uint32"` serialized offset where the serialized representation of the *variable sized* type is located in the second section relative to the beginning of the first section. +* The first section is *fixed size* for all types, containing the concatenation of + - The serialized representation for each of the *fixed size* elements from the value + - The `"uint32"` serialized offset where the serialized representation of the *variable sized* elements from the value are located in the second section. * The second section contains the concatenation of the serialized representations of **only** the *variable size* types. - This section is empty in the case of a purely *fixed size* type. +> **NOTE**: Offsets are relative to the beginning of the beginning of the entire serialized representation (the start of the first section) + #### `"vector"`, `"container"` and `"list"` -An implementation of the `serialize` function for `"Vector"`, `"Container"` and -`"List"` types would take the following form. +Below is an illustrative implementation of the `serialize` function for `"Vector"`, +`"Container"` and `"List"` types. ```python # The second section is just the concatenation of the serialized *variable size* elements From 66173b8ba3869b9c130627a3d45e1d9199a8dfe5 Mon Sep 17 00:00:00 2001 From: Piper Merriam Date: Wed, 20 Mar 2019 09:14:02 -0600 Subject: [PATCH 017/121] static > fixed --- specs/simple-serialize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 4fcefb86f..52bd52587 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -122,7 +122,7 @@ section_1_offsets = [ ] assert all(offset is None or offset < 2**32 for offset in section_1_offsets) -# The first section is the concatenation of the serialized static size elements and offsets +# The first section is the concatenation of the serialized fixed size elements and offsets section_1_parts = [ serialize(element) if is_fixed_size(element) else serialize(section_1_offsets[element_index]) From 92f002c501dac0ff6bd0da8122dc99322450e33e Mon Sep 17 00:00:00 2001 From: Piper Merriam Date: Thu, 28 Mar 2019 08:26:18 -0600 Subject: [PATCH 018/121] specify offsets better --- specs/simple-serialize.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 52bd52587..6bd9e3dd7 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -83,13 +83,21 @@ return b"\x01" if value is True else b"\x00" The serialized representation of composite types is comprised of two binary sections. -* The first section is *fixed size* for all types, containing the concatenation of - - The serialized representation for each of the *fixed size* elements from the value - - The `"uint32"` serialized offset where the serialized representation of the *variable sized* elements from the value are located in the second section. -* The second section contains the concatenation of the serialized representations of **only** the *variable size* types. +* The first section is the concatenation of a mixture of either the serialized representation for *fixed size* elements **or** a serialized offset value for *variable size* elements. + - All *fixed size* elements are represented in this section as their serialized representation. + - All *variable size* elements are represented in this section with a `"uint32"` serialized offset where the serialized representation of the element is located in the second section. + - offsets are relative to the beginning of the beginning of the entire serialized representation (the start of the first section) +* The second section is the concatenation of the serialized representations of **only** the *variable size* types. - This section is empty in the case of a purely *fixed size* type. -> **NOTE**: Offsets are relative to the beginning of the beginning of the entire serialized representation (the start of the first section) + +Offset values are subject to the following validity rules: + +- For Vector and Container types: + - The first offset **must** be equal to the length of the first section. +- For all types: + - Offsets **MAY NOT** be less than any previous offset. + - Offsets **MUST** be less than `2**32` #### `"vector"`, `"container"` and `"list"` From 9dde3a26612cbe1636529ec0d85f8462c8271f9b Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 9 Apr 2019 05:59:00 -0500 Subject: [PATCH 019/121] Update replace_empty_or_append Requires adding definitions of `empty` and `typeof` to the function puller. --- specs/core/1_custody-game.md | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 41ba9d953..88341ae98 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -233,9 +233,9 @@ def epoch_to_custody_period(epoch: Epoch) -> int: ### `replace_empty_or_append` ```python -def replace_empty_or_append(list: List[Any], empty_element: Any, new_element: Any) -> int: +def replace_empty_or_append(list: List[Any], new_element: Any) -> int: for i in range(len(list)): - if list[i] == empty_element: + if list[i] == empty(typeof(new_element)): list[i] = new_element return i list.append(new_element) @@ -343,11 +343,7 @@ def process_chunk_challenge(state: BeaconState, depth=depth, chunk_index=challenge.chunk_index, ) - replace_empty_or_append( - list=state.custody_chunk_challenge_records, - empty_element=CustodyChunkChallengeRecord(), - new_element=new_record - ) + replace_empty_or_append(state.custody_chunk_challenge_records, new_record) state.custody_challenge_index += 1 # Postpone responder withdrawability @@ -413,11 +409,7 @@ def process_bit_challenge(state: BeaconState, chunk_bits=challenge.chunk_bits, responder_key=challenge.responder_key, ) - replace_empty_or_append( - list=state.custody_bit_challenge_records, - empty_element=CustodyBitChallengeRecord(), - new_element=new_record - ) + replace_empty_or_append(state.custody_bit_challenge_records, new_record) state.custody_challenge_index += 1 # Postpone responder withdrawability responder.withdrawable_epoch = FAR_FUTURE_EPOCH From 4d2bdf8628eb430c301043e9e4d91b8f883b0ca4 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 13 Apr 2019 23:44:14 +1000 Subject: [PATCH 020/121] Cleanup spec --- specs/simple-serialize.md | 88 +++++++++------------------------------ 1 file changed, 19 insertions(+), 69 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 6bd9e3dd7..ff2e207f0 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -12,7 +12,7 @@ This is a **work in progress** describing typing, serialization and Merkleizatio - [Serialization](#serialization) - [`"uintN"`](#uintn) - [`"bool"`](#bool) - - [Vectors, containers, lists](#composite-types-vectors-containers-and-lists) + - [Vectors, containers, lists](#vectors-containers-lists) - [Deserialization](#deserialization) - [Merkleization](#merkleization) - [Self-signed containers](#self-signed-containers) @@ -22,8 +22,9 @@ This is a **work in progress** describing typing, serialization and Merkleizatio | Name | Value | Description | |-|-|-| -| `BYTES_PER_CHUNK` | `32` | Number of bytes per chunk. +| `BYTES_PER_CHUNK` | `32` | Number of bytes per chunk. | | `BYTES_PER_LENGTH_PREFIX` | `4` | Number of bytes per serialized length prefix. | +| `BITS_PER_BYTE` | `8` | Number of bits per byte. | ## Typing ### Basic types @@ -56,97 +57,46 @@ We recursively define the `serialize` function which consumes an object `value` > *Note*: In the function definitions below (`serialize`, `hash_tree_root`, `signed_root`, `is_fixed_size`, `is_variable_size` etc.) objects implicitly carry their type. -### Basic Types - -For basic types the `serialize` function is defined as follows. - -#### `"uintN"` - -A byte string of width `N // 8` containing the little-endian encode integer. +### `"uintN"` ```python assert N in [8, 16, 32, 64, 128, 256] return value.to_bytes(N // 8, "little") ``` -#### `"bool"` - -* The byte `\x00` **if** the value is `False` -* The byte `\x01` **if** the value is `True` +### `"bool"` ```python assert value in (True, False) return b"\x01" if value is True else b"\x00" ``` -### Composite Types (Vectors, Containers and Lists) - -The serialized representation of composite types is comprised of two binary sections. - -* The first section is the concatenation of a mixture of either the serialized representation for *fixed size* elements **or** a serialized offset value for *variable size* elements. - - All *fixed size* elements are represented in this section as their serialized representation. - - All *variable size* elements are represented in this section with a `"uint32"` serialized offset where the serialized representation of the element is located in the second section. - - offsets are relative to the beginning of the beginning of the entire serialized representation (the start of the first section) -* The second section is the concatenation of the serialized representations of **only** the *variable size* types. - - This section is empty in the case of a purely *fixed size* type. - - -Offset values are subject to the following validity rules: - -- For Vector and Container types: - - The first offset **must** be equal to the length of the first section. -- For all types: - - Offsets **MAY NOT** be less than any previous offset. - - Offsets **MUST** be less than `2**32` - - -#### `"vector"`, `"container"` and `"list"` - -Below is an illustrative implementation of the `serialize` function for `"Vector"`, -`"Container"` and `"List"` types. +### Vectors, containers, lists ```python -# The second section is just the concatenation of the serialized *variable size* elements -section_2_parts = [ - serialize(element) if is_variable_size(element) - else '' - for element in value -] -section_2_lengths = [len(part) for part in section_2_parts] -section_2 = ''.join(section_2_parts) +# Reccursively serialize fixed-size elements +fixed_parts = [serialize(element) if is_fixed_size(element) else None for element in value] +fixed_lengths = [len(part) if part != None else BYTES_PER_LENGTH_PREFIX for part in fixed_parts] -# Compute the length of the first section (can also be extracted from the type directly) -section_1_length = sum( - len(serialize(element)) if is_fixed_size(element) - else 4 - for element in value -) +# Reccursively serialize variable-size elements +variable_parts = [serialize(element) if is_variable_size(element) else None for element in value] +variable_lengths = [len(part) if part != None else 0 for part in variable_parts] -# Compute the offset values for each part of the second section -section_1_offsets = [ - section_1_length + sum(section_2_lengths[:element_index]) if is_variable_size(element) - else None - for element_index, element in enumerate(value) -] -assert all(offset is None or offset < 2**32 for offset in section_1_offsets) +# Compute offsets of variable-size elements +assert sum(fixed_lengths + variable_lengths) < 2**(BYTES_PER_LENGTH_PREFIX * BITS_PER_BYTE) +offsets = [sum(fixed_lengths) + sum(variable_lengths[:i]) for i in range(len(value))] -# The first section is the concatenation of the serialized fixed size elements and offsets -section_1_parts = [ - serialize(element) if is_fixed_size(element) - else serialize(section_1_offsets[element_index]) - for element_index, element in enumerate(value) -] -section_1 = ''.join(section_1_parts) +# Interleave offsets in fixed parts +fixed_parts = [part if part != None else offsets[i] for i, part in enumerate(fixed_parts)] -return ''.join([section_1, section_2]) +# Return the of fixed parts (with interleaved offsets) followed by variable parts +return "".join(fixed_parts + variable_parts) ``` - ## Deserialization Because serialization is an injective function (i.e. two distinct objects of the same type will serialize to different values) any bytestring has at most one object it could deserialize to. Efficient algorithms for computing this object can be found in [the implementations](#implementations). - ## Merkleization We first define helper functions: From aaa5a1676572819befe3ca937449e261474e51bb Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 13 Apr 2019 23:45:18 +1000 Subject: [PATCH 021/121] Update simple-serialize.md --- specs/simple-serialize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index ff2e207f0..150cf2d54 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -55,7 +55,7 @@ For convenience we alias: We recursively define the `serialize` function which consumes an object `value` (of the type specified) and returns a bytestring of type `"bytes"`. -> *Note*: In the function definitions below (`serialize`, `hash_tree_root`, `signed_root`, `is_fixed_size`, `is_variable_size` etc.) objects implicitly carry their type. +> *Note*: In the function definitions below (`serialize`, `hash_tree_root`, `signed_root`, `is_fixed_size`, `is_variable_size`, etc.) objects implicitly carry their type. ### `"uintN"` From 0695d0ad1c4725733ae0916a56340eb5ccf7c722 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 13 Apr 2019 23:48:47 +1000 Subject: [PATCH 022/121] Update simple-serialize.md --- specs/simple-serialize.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 150cf2d54..bc1fa6ccb 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -86,10 +86,10 @@ variable_lengths = [len(part) if part != None else 0 for part in variable_parts] assert sum(fixed_lengths + variable_lengths) < 2**(BYTES_PER_LENGTH_PREFIX * BITS_PER_BYTE) offsets = [sum(fixed_lengths) + sum(variable_lengths[:i]) for i in range(len(value))] -# Interleave offsets in fixed parts +# Interleave offsets with fixed parts fixed_parts = [part if part != None else offsets[i] for i, part in enumerate(fixed_parts)] -# Return the of fixed parts (with interleaved offsets) followed by variable parts +# Return the fixed parts (with offsets interleaved) followed by variable parts return "".join(fixed_parts + variable_parts) ``` From 35a6311208a61bf2b09850bb7731aa0c968066e4 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 13 Apr 2019 23:50:23 +1000 Subject: [PATCH 023/121] Update simple-serialize.md --- specs/simple-serialize.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index bc1fa6ccb..5aba9aff5 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -74,19 +74,17 @@ return b"\x01" if value is True else b"\x00" ### Vectors, containers, lists ```python -# Reccursively serialize fixed-size elements +# Reccursively serialize fixed_parts = [serialize(element) if is_fixed_size(element) else None for element in value] -fixed_lengths = [len(part) if part != None else BYTES_PER_LENGTH_PREFIX for part in fixed_parts] - -# Reccursively serialize variable-size elements variable_parts = [serialize(element) if is_variable_size(element) else None for element in value] + +# Compute and check lengths +fixed_lengths = [len(part) if part != None else BYTES_PER_LENGTH_PREFIX for part in fixed_parts] variable_lengths = [len(part) if part != None else 0 for part in variable_parts] - -# Compute offsets of variable-size elements assert sum(fixed_lengths + variable_lengths) < 2**(BYTES_PER_LENGTH_PREFIX * BITS_PER_BYTE) -offsets = [sum(fixed_lengths) + sum(variable_lengths[:i]) for i in range(len(value))] -# Interleave offsets with fixed parts +# Compute offsets of variable-size elements, and interleave with fixed parts +offsets = [sum(fixed_lengths) + sum(variable_lengths[:i]) for i in range(len(value))] fixed_parts = [part if part != None else offsets[i] for i, part in enumerate(fixed_parts)] # Return the fixed parts (with offsets interleaved) followed by variable parts From 80bd4a381b02dd7b226cab59742402990c7a7479 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 13 Apr 2019 23:55:08 +1000 Subject: [PATCH 024/121] Update simple-serialize.md --- specs/simple-serialize.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 5aba9aff5..417d70b95 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -23,7 +23,7 @@ This is a **work in progress** describing typing, serialization and Merkleizatio | Name | Value | Description | |-|-|-| | `BYTES_PER_CHUNK` | `32` | Number of bytes per chunk. | -| `BYTES_PER_LENGTH_PREFIX` | `4` | Number of bytes per serialized length prefix. | +| `BYTES_PER_LENGTH_OFFSET` | `4` | Number of bytes per serialized length prefix. | | `BITS_PER_BYTE` | `8` | Number of bits per byte. | ## Typing @@ -79,9 +79,9 @@ fixed_parts = [serialize(element) if is_fixed_size(element) else None for elemen variable_parts = [serialize(element) if is_variable_size(element) else None for element in value] # Compute and check lengths -fixed_lengths = [len(part) if part != None else BYTES_PER_LENGTH_PREFIX for part in fixed_parts] +fixed_lengths = [len(part) if part != None else BYTES_PER_LENGTH_OFFSET for part in fixed_parts] variable_lengths = [len(part) if part != None else 0 for part in variable_parts] -assert sum(fixed_lengths + variable_lengths) < 2**(BYTES_PER_LENGTH_PREFIX * BITS_PER_BYTE) +assert sum(fixed_lengths + variable_lengths) < 2**(BYTES_PER_LENGTH_OFFSET * BITS_PER_BYTE) # Compute offsets of variable-size elements, and interleave with fixed parts offsets = [sum(fixed_lengths) + sum(variable_lengths[:i]) for i in range(len(value))] From 10f3db977dc4f5e9e2ae7a1eb92034b1e4547f3e Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 13 Apr 2019 23:56:06 +1000 Subject: [PATCH 025/121] Update simple-serialize.md --- specs/simple-serialize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 417d70b95..a8f0d9add 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -23,7 +23,7 @@ This is a **work in progress** describing typing, serialization and Merkleizatio | Name | Value | Description | |-|-|-| | `BYTES_PER_CHUNK` | `32` | Number of bytes per chunk. | -| `BYTES_PER_LENGTH_OFFSET` | `4` | Number of bytes per serialized length prefix. | +| `BYTES_PER_LENGTH_OFFSET` | `4` | Number of bytes per serialized length offset. | | `BITS_PER_BYTE` | `8` | Number of bits per byte. | ## Typing From 27cf02a9b0a1914a92a8b5aac47c44ebe1699996 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 13 Apr 2019 23:59:03 +1000 Subject: [PATCH 026/121] Update simple-serialize.md --- specs/simple-serialize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index a8f0d9add..f57945749 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -84,7 +84,7 @@ variable_lengths = [len(part) if part != None else 0 for part in variable_parts] assert sum(fixed_lengths + variable_lengths) < 2**(BYTES_PER_LENGTH_OFFSET * BITS_PER_BYTE) # Compute offsets of variable-size elements, and interleave with fixed parts -offsets = [sum(fixed_lengths) + sum(variable_lengths[:i]) for i in range(len(value))] +offsets = [sum(fixed_lengths + variable_lengths[:i]) for i in range(len(value))] fixed_parts = [part if part != None else offsets[i] for i, part in enumerate(fixed_parts)] # Return the fixed parts (with offsets interleaved) followed by variable parts From a90bcc0cd4fb186beb0ef166ce7de87f526dad6a Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 14 Apr 2019 00:00:46 +1000 Subject: [PATCH 027/121] Update simple-serialize.md --- specs/simple-serialize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index f57945749..a7d80e264 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -87,7 +87,7 @@ assert sum(fixed_lengths + variable_lengths) < 2**(BYTES_PER_LENGTH_OFFSET * BIT offsets = [sum(fixed_lengths + variable_lengths[:i]) for i in range(len(value))] fixed_parts = [part if part != None else offsets[i] for i, part in enumerate(fixed_parts)] -# Return the fixed parts (with offsets interleaved) followed by variable parts +# Return the fixed parts (with offsets interleaved) followed by the variable parts return "".join(fixed_parts + variable_parts) ``` From 23c09541e2a44fe4a56641ef354d24eb24c3e461 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 14 Apr 2019 00:05:43 +1000 Subject: [PATCH 028/121] Update simple-serialize.md --- specs/simple-serialize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index a7d80e264..41999d1dc 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -83,7 +83,7 @@ fixed_lengths = [len(part) if part != None else BYTES_PER_LENGTH_OFFSET for part variable_lengths = [len(part) if part != None else 0 for part in variable_parts] assert sum(fixed_lengths + variable_lengths) < 2**(BYTES_PER_LENGTH_OFFSET * BITS_PER_BYTE) -# Compute offsets of variable-size elements, and interleave with fixed parts +# Compute offsets of variable-size parts and interleave offsets with fixed parts offsets = [sum(fixed_lengths + variable_lengths[:i]) for i in range(len(value))] fixed_parts = [part if part != None else offsets[i] for i, part in enumerate(fixed_parts)] From f6ed1df62bd09758aebbaaeadf51041df1d62936 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 14 Apr 2019 00:10:02 +1000 Subject: [PATCH 029/121] Update simple-serialize.md --- specs/simple-serialize.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 41999d1dc..ceb3652f3 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -76,11 +76,11 @@ return b"\x01" if value is True else b"\x00" ```python # Reccursively serialize fixed_parts = [serialize(element) if is_fixed_size(element) else None for element in value] -variable_parts = [serialize(element) if is_variable_size(element) else None for element in value] +variable_parts = [serialize(element) if is_variable_size(element) else "" for element in value] # Compute and check lengths fixed_lengths = [len(part) if part != None else BYTES_PER_LENGTH_OFFSET for part in fixed_parts] -variable_lengths = [len(part) if part != None else 0 for part in variable_parts] +variable_lengths = [len(part) for part in variable_parts] assert sum(fixed_lengths + variable_lengths) < 2**(BYTES_PER_LENGTH_OFFSET * BITS_PER_BYTE) # Compute offsets of variable-size parts and interleave offsets with fixed parts From 9adbaba96e01c756b474f48b13630ec0c339f396 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 14 Apr 2019 00:14:30 +1000 Subject: [PATCH 030/121] Update simple-serialize.md --- specs/simple-serialize.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index ceb3652f3..6626b9cf4 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -83,11 +83,11 @@ fixed_lengths = [len(part) if part != None else BYTES_PER_LENGTH_OFFSET for part variable_lengths = [len(part) for part in variable_parts] assert sum(fixed_lengths + variable_lengths) < 2**(BYTES_PER_LENGTH_OFFSET * BITS_PER_BYTE) -# Compute offsets of variable-size parts and interleave offsets with fixed parts +# Compute offsets of variable-size parts and interleave offsets with fixed-size parts offsets = [sum(fixed_lengths + variable_lengths[:i]) for i in range(len(value))] fixed_parts = [part if part != None else offsets[i] for i, part in enumerate(fixed_parts)] -# Return the fixed parts (with offsets interleaved) followed by the variable parts +# Return the fixed-size parts (with offsets interleaved) followed by the variable-size parts return "".join(fixed_parts + variable_parts) ``` From 97ca6721056723193d8b8fc69aa8dcadb6c38098 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 14 Apr 2019 00:18:44 +1000 Subject: [PATCH 031/121] Update simple-serialize.md --- specs/simple-serialize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 6626b9cf4..b956df4d1 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -85,7 +85,7 @@ assert sum(fixed_lengths + variable_lengths) < 2**(BYTES_PER_LENGTH_OFFSET * BIT # Compute offsets of variable-size parts and interleave offsets with fixed-size parts offsets = [sum(fixed_lengths + variable_lengths[:i]) for i in range(len(value))] -fixed_parts = [part if part != None else offsets[i] for i, part in enumerate(fixed_parts)] +fixed_parts = [part if part != None else serialize(offsets[i]) for i, part in enumerate(fixed_parts)] # Return the fixed-size parts (with offsets interleaved) followed by the variable-size parts return "".join(fixed_parts + variable_parts) From 09d927405c9020da2ed466d8e2782929589b9ab1 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 14 Apr 2019 00:22:41 +1000 Subject: [PATCH 032/121] Update simple-serialize.md --- specs/simple-serialize.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index b956df4d1..1af409c58 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -83,11 +83,11 @@ fixed_lengths = [len(part) if part != None else BYTES_PER_LENGTH_OFFSET for part variable_lengths = [len(part) for part in variable_parts] assert sum(fixed_lengths + variable_lengths) < 2**(BYTES_PER_LENGTH_OFFSET * BITS_PER_BYTE) -# Compute offsets of variable-size parts and interleave offsets with fixed-size parts +# Interleave offsets of variable-size parts with fixed-size parts offsets = [sum(fixed_lengths + variable_lengths[:i]) for i in range(len(value))] fixed_parts = [part if part != None else serialize(offsets[i]) for i, part in enumerate(fixed_parts)] -# Return the fixed-size parts (with offsets interleaved) followed by the variable-size parts +# Return the concatenation of the fixed-size parts (offsets interleaved) with the variable-size parts return "".join(fixed_parts + variable_parts) ``` From 7255b0fc0d63ae891910228cfaf635ea66258f7d Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 14 Apr 2019 00:25:47 +1000 Subject: [PATCH 033/121] Update simple-serialize.md --- specs/simple-serialize.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 1af409c58..cf482c39d 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -84,8 +84,8 @@ variable_lengths = [len(part) for part in variable_parts] assert sum(fixed_lengths + variable_lengths) < 2**(BYTES_PER_LENGTH_OFFSET * BITS_PER_BYTE) # Interleave offsets of variable-size parts with fixed-size parts -offsets = [sum(fixed_lengths + variable_lengths[:i]) for i in range(len(value))] -fixed_parts = [part if part != None else serialize(offsets[i]) for i, part in enumerate(fixed_parts)] +offsets = [serialize(sum(fixed_lengths + variable_lengths[:i])) for i in range(len(value))] +fixed_parts = [part if part != None else offsets[i] for i, part in enumerate(fixed_parts)] # Return the concatenation of the fixed-size parts (offsets interleaved) with the variable-size parts return "".join(fixed_parts + variable_parts) From 59f568073afeb7723ba33fc8fd601375adc5d205 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 14 Apr 2019 00:26:44 +1000 Subject: [PATCH 034/121] Update simple-serialize.md --- specs/simple-serialize.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index cf482c39d..03e47deb2 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -84,8 +84,8 @@ variable_lengths = [len(part) for part in variable_parts] assert sum(fixed_lengths + variable_lengths) < 2**(BYTES_PER_LENGTH_OFFSET * BITS_PER_BYTE) # Interleave offsets of variable-size parts with fixed-size parts -offsets = [serialize(sum(fixed_lengths + variable_lengths[:i])) for i in range(len(value))] -fixed_parts = [part if part != None else offsets[i] for i, part in enumerate(fixed_parts)] +variable_offsets = [serialize(sum(fixed_lengths + variable_lengths[:i])) for i in range(len(value))] +fixed_parts = [part if part != None else variable_offsets[i] for i, part in enumerate(fixed_parts)] # Return the concatenation of the fixed-size parts (offsets interleaved) with the variable-size parts return "".join(fixed_parts + variable_parts) From 62ffb897ae6949a0acb372be0e31e36dd3f16dd0 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 14 Apr 2019 00:41:48 +1000 Subject: [PATCH 035/121] Update simple-serialize.md --- specs/simple-serialize.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 03e47deb2..3f02335db 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -12,7 +12,7 @@ This is a **work in progress** describing typing, serialization and Merkleizatio - [Serialization](#serialization) - [`"uintN"`](#uintn) - [`"bool"`](#bool) - - [Vectors, containers, lists](#vectors-containers-lists) + - [Containers, vectors, lists](#containers-vectors-lists) - [Deserialization](#deserialization) - [Merkleization](#merkleization) - [Self-signed containers](#self-signed-containers) @@ -71,7 +71,7 @@ assert value in (True, False) return b"\x01" if value is True else b"\x00" ``` -### Vectors, containers, lists +### Containers, vectors, lists ```python # Reccursively serialize From 9eba123e2e2e559c16b1b34335b7cd9b8ddc3c55 Mon Sep 17 00:00:00 2001 From: Justin Date: Mon, 15 Apr 2019 07:54:08 +1000 Subject: [PATCH 036/121] Remove serialization from consensus Consensus now only cares about Merkleisation (i.e. `hash_tree_root`), not about serialization (i.e. `serialize`). This simplifies consensus code by a few tens of lines, is conceptually cleaner, and is more future proof. A corresponding change is required in the deposit contract. --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 2d150837e..cb57c0a45 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -2230,7 +2230,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: # Verify the Merkle branch merkle_branch_is_valid = verify_merkle_branch( - leaf=hash(serialize(deposit.data)), # 48 + 32 + 8 + 96 = 184 bytes serialization + leaf=hash_tree_root(deposit.data), proof=deposit.proof, depth=DEPOSIT_CONTRACT_TREE_DEPTH, index=deposit.index, From a25c436b78983b68ee40b40c79df3583b8c14159 Mon Sep 17 00:00:00 2001 From: Justin Date: Mon, 15 Apr 2019 08:14:33 +1000 Subject: [PATCH 037/121] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index cb57c0a45..dbf399a8c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1360,7 +1360,7 @@ The initial deployment phases of Ethereum 2.0 are implemented without consensus ### Deposit arguments -The deposit contract has a single `deposit` function which takes as argument a SimpleSerialize'd `DepositData`. +The deposit contract has a single `deposit` function which takes as argument the `DepositData` fields. ### Withdrawal credentials From b6b82ae494edca81d92ad9ca5872c641f7fc3a7e Mon Sep 17 00:00:00 2001 From: Justin Date: Mon, 15 Apr 2019 08:15:20 +1000 Subject: [PATCH 038/121] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index dbf399a8c..dbc1c9b5b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1360,7 +1360,7 @@ The initial deployment phases of Ethereum 2.0 are implemented without consensus ### Deposit arguments -The deposit contract has a single `deposit` function which takes as argument the `DepositData` fields. +The deposit contract has a single `deposit` function which takes as argument the `DepositData` elements. ### Withdrawal credentials From 084919e06383d1959d15484368b5f67e1ae61792 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 Apr 2019 08:29:08 +1000 Subject: [PATCH 039/121] Adjust tests --- tests/phase0/helpers.py | 4 ++-- tests/phase0/test_sanity.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/phase0/helpers.py b/tests/phase0/helpers.py index 8c8064fc1..e20bb9484 100644 --- a/tests/phase0/helpers.py +++ b/tests/phase0/helpers.py @@ -61,7 +61,7 @@ def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves=N amount=spec.MAX_DEPOSIT_AMOUNT, proof_of_possession=proof_of_possession, ) - item = hash(deposit_data.serialize()) + item = deposit_data.hash_tree_root() deposit_data_leaves.append(item) tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) root = get_merkle_root((tuple(deposit_data_leaves))) @@ -180,7 +180,7 @@ def build_deposit(state, amount): deposit_data = build_deposit_data(state, pubkey, privkey, amount) - item = hash(deposit_data.serialize()) + item = deposit_data.hash_tree_root() index = len(deposit_data_leaves) deposit_data_leaves.append(item) tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) diff --git a/tests/phase0/test_sanity.py b/tests/phase0/test_sanity.py index 08c7610c0..2c13c2415 100644 --- a/tests/phase0/test_sanity.py +++ b/tests/phase0/test_sanity.py @@ -179,7 +179,7 @@ def test_deposit_in_block(state): privkey = privkeys[index] deposit_data = build_deposit_data(pre_state, pubkey, privkey, spec.MAX_DEPOSIT_AMOUNT) - item = hash(deposit_data.serialize()) + item = deposit_data.hash_tree_root() test_deposit_data_leaves.append(item) tree = calc_merkle_tree_from_leaves(tuple(test_deposit_data_leaves)) root = get_merkle_root((tuple(test_deposit_data_leaves))) @@ -218,7 +218,7 @@ def test_deposit_top_up(state): deposit_data = build_deposit_data(pre_state, pubkey, privkey, amount) merkle_index = len(test_deposit_data_leaves) - item = hash(deposit_data.serialize()) + item = deposit_data.hash_tree_root() test_deposit_data_leaves.append(item) tree = calc_merkle_tree_from_leaves(tuple(test_deposit_data_leaves)) root = get_merkle_root((tuple(test_deposit_data_leaves))) From ed28515a9570bfb69f440e93488691d36df2e0e3 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 16 Apr 2019 16:16:13 +1000 Subject: [PATCH 040/121] Enables transferes of BAL > 32 ETH --- .../block_processing/test_process_deposit.py | 10 +++++----- py_tests/phase0/helpers.py | 2 +- py_tests/phase0/test_sanity.py | 6 +++--- specs/core/0_beacon-chain.md | 19 ++++++++++--------- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/py_tests/phase0/block_processing/test_process_deposit.py b/py_tests/phase0/block_processing/test_process_deposit.py index cd682a4d4..31862c75d 100644 --- a/py_tests/phase0/block_processing/test_process_deposit.py +++ b/py_tests/phase0/block_processing/test_process_deposit.py @@ -32,7 +32,7 @@ def test_success(state): deposit_data_leaves, pubkey, privkey, - spec.MAX_DEPOSIT_AMOUNT, + spec.MAX_EFFECTIVE_BALANCE, ) pre_state.latest_eth1_data.deposit_root = root @@ -45,7 +45,7 @@ def test_success(state): assert len(post_state.validator_registry) == len(state.validator_registry) + 1 assert len(post_state.balances) == len(state.balances) + 1 assert post_state.validator_registry[index].pubkey == pubkeys[index] - assert get_balance(post_state, index) == spec.MAX_DEPOSIT_AMOUNT + assert get_balance(post_state, index) == spec.MAX_EFFECTIVE_BALANCE assert post_state.deposit_index == post_state.latest_eth1_data.deposit_count return pre_state, deposit, post_state @@ -56,7 +56,7 @@ def test_success_top_up(state): deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) validator_index = 0 - amount = spec.MAX_DEPOSIT_AMOUNT // 4 + amount = spec.MAX_EFFECTIVE_BALANCE // 4 pubkey = pubkeys[validator_index] privkey = privkeys[validator_index] deposit, root, deposit_data_leaves = build_deposit( @@ -95,7 +95,7 @@ def test_wrong_index(state): deposit_data_leaves, pubkey, privkey, - spec.MAX_DEPOSIT_AMOUNT, + spec.MAX_EFFECTIVE_BALANCE, ) # mess up deposit_index @@ -124,7 +124,7 @@ def test_bad_merkle_proof(state): deposit_data_leaves, pubkey, privkey, - spec.MAX_DEPOSIT_AMOUNT, + spec.MAX_EFFECTIVE_BALANCE, ) # mess up merkle branch diff --git a/py_tests/phase0/helpers.py b/py_tests/phase0/helpers.py index b28f480b2..cef4f4827 100644 --- a/py_tests/phase0/helpers.py +++ b/py_tests/phase0/helpers.py @@ -58,7 +58,7 @@ def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves=N pubkey=pubkey, # insecurely use pubkey as withdrawal key as well withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:], - amount=spec.MAX_DEPOSIT_AMOUNT, + amount=spec.MAX_EFFECTIVE_BALANCE, proof_of_possession=proof_of_possession, ) item = hash(deposit_data.serialize()) diff --git a/py_tests/phase0/test_sanity.py b/py_tests/phase0/test_sanity.py index 95bf9089c..cc987002e 100644 --- a/py_tests/phase0/test_sanity.py +++ b/py_tests/phase0/test_sanity.py @@ -177,7 +177,7 @@ def test_deposit_in_block(state): index = len(test_deposit_data_leaves) pubkey = pubkeys[index] privkey = privkeys[index] - deposit_data = build_deposit_data(pre_state, pubkey, privkey, spec.MAX_DEPOSIT_AMOUNT) + deposit_data = build_deposit_data(pre_state, pubkey, privkey, spec.MAX_EFFECTIVE_BALANCE) item = hash(deposit_data.serialize()) test_deposit_data_leaves.append(item) @@ -201,7 +201,7 @@ def test_deposit_in_block(state): state_transition(post_state, block) assert len(post_state.validator_registry) == len(state.validator_registry) + 1 assert len(post_state.balances) == len(state.balances) + 1 - assert get_balance(post_state, index) == spec.MAX_DEPOSIT_AMOUNT + assert get_balance(post_state, index) == spec.MAX_EFFECTIVE_BALANCE assert post_state.validator_registry[index].pubkey == pubkeys[index] return pre_state, [block], post_state @@ -212,7 +212,7 @@ def test_deposit_top_up(state): test_deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) validator_index = 0 - amount = spec.MAX_DEPOSIT_AMOUNT // 4 + amount = spec.MAX_EFFECTIVE_BALANCE // 4 pubkey = pubkeys[validator_index] privkey = privkeys[validator_index] deposit_data = build_deposit_data(pre_state, pubkey, privkey, amount) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 2d150837e..2e84b958f 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -201,7 +201,7 @@ These configurations are updated for releases, but may be out of sync during `de | Name | Value | Unit | | - | - | :-: | | `MIN_DEPOSIT_AMOUNT` | `2**0 * 10**9` (= 1,000,000,000) | Gwei | -| `MAX_DEPOSIT_AMOUNT` | `2**5 * 10**9` (= 32,000,000,000) | Gwei | +| `MAX_EFFECTIVE_BALANCE` | `2**5 * 10**9` (= 32,000,000,000) | Gwei | | `EJECTION_BALANCE` | `2**4 * 10**9` (= 16,000,000,000) | Gwei | | `HIGH_BALANCE_INCREMENT` | `2**0 * 10**9` (= 1,000,000,000) | Gwei | @@ -1002,7 +1002,7 @@ def get_beacon_proposer_index(state: BeaconState, int_to_bytes8(i // 32) )[i % 32] candidate = first_committee[(current_epoch + i) % len(first_committee)] - if get_effective_balance(state, candidate) * 256 > MAX_DEPOSIT_AMOUNT * rand_byte: + if get_effective_balance(state, candidate) * 256 > MAX_EFFECTIVE_BALANCE * rand_byte: return candidate i += 1 ``` @@ -1081,7 +1081,7 @@ def get_effective_balance(state: BeaconState, index: ValidatorIndex) -> Gwei: """ Return the effective balance (also known as "balance at stake") for a validator with the given ``index``. """ - return min(get_balance(state, index), MAX_DEPOSIT_AMOUNT) + return min(get_balance(state, index), MAX_EFFECTIVE_BALANCE) ``` ### `get_total_balance` @@ -1373,7 +1373,7 @@ The private key corresponding to `withdrawal_pubkey` will be required to initiat ### `Deposit` logs -Every Ethereum 1.0 deposit, of size between `MIN_DEPOSIT_AMOUNT` and `MAX_DEPOSIT_AMOUNT`, emits a `Deposit` log for consumption by the beacon chain. The deposit contract does little validation, pushing most of the validator onboarding logic to the beacon chain. In particular, the proof of possession (a BLS12 signature) is not verified by the deposit contract. +Every Ethereum 1.0 deposit, of size greater than `MIN_DEPOSIT_AMOUNT`, emits a `Deposit` log for consumption by the beacon chain. The deposit contract does little validation, pushing most of the validator onboarding logic to the beacon chain. In particular, the proof of possession (a BLS12 signature) is not verified by the deposit contract. ### `Eth2Genesis` log @@ -1395,7 +1395,7 @@ For convenience, we provide the interface to the contract here: * `__init__()`: initializes the contract * `get_deposit_root() -> bytes32`: returns the current root of the deposit tree -* `deposit(bytes[512])`: adds a deposit instance to the deposit tree, incorporating the input argument and the value transferred in the given call. Note: the amount of value transferred *must* be within `MIN_DEPOSIT_AMOUNT` and `MAX_DEPOSIT_AMOUNT`, inclusive. Each of these constants are specified in units of Gwei. +* `deposit(bytes[512])`: adds a deposit instance to the deposit tree, incorporating the input argument and the value transferred in the given call. Note: the amount of value transferred *must* be greater than `MIN_DEPOSIT_AMOUNT` inclusive. Each of these constants are specified in units of Gwei. ## On genesis @@ -1495,7 +1495,7 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], # Process genesis activations for validator_index in range(len(state.validator_registry)): - if get_effective_balance(state, validator_index) >= MAX_DEPOSIT_AMOUNT: + if get_effective_balance(state, validator_index) >= MAX_EFFECTIVE_BALANCE: activate_validator(state, validator_index, is_genesis=True) genesis_active_index_root = hash_tree_root(get_active_validator_indices(state, GENESIS_EPOCH)) @@ -1936,7 +1936,7 @@ def process_balance_driven_status_transitions(state: BeaconState) -> None: """ for index, validator in enumerate(state.validator_registry): balance = get_balance(state, index) - if validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH and balance >= MAX_DEPOSIT_AMOUNT: + if validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH and balance >= MAX_EFFECTIVE_BALANCE: validator.activation_eligibility_epoch = get_current_epoch(state) if is_active_validator(validator, get_current_epoch(state)) and balance < EJECTION_BALANCE: @@ -2335,10 +2335,11 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: assert get_balance(state, transfer.sender) >= max(transfer.amount, transfer.fee) # A transfer is valid in only one slot assert state.slot == transfer.slot - # Only withdrawn or not-yet-deposited accounts can transfer + # Only withdrawn, not-yet-deposited accounts, or the balance over MAX_EFFECTIVE_BALANCE can be transfered assert ( get_current_epoch(state) >= state.validator_registry[transfer.sender].withdrawable_epoch or - state.validator_registry[transfer.sender].activation_epoch == FAR_FUTURE_EPOCH + state.validator_registry[transfer.sender].activation_epoch == FAR_FUTURE_EPOCH or + transfer.amount + transfer.fee >= get_balance(statetransfer.sender) - get_effective_balance(transfer.sender) ) # Verify that the pubkey is valid assert ( From ae0afe389fc52ed5554b80fdae7960fd61a99299 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 16 Apr 2019 20:11:51 +1000 Subject: [PATCH 041/121] Cleaner assertion --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 2e84b958f..974131027 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -2339,7 +2339,7 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: assert ( get_current_epoch(state) >= state.validator_registry[transfer.sender].withdrawable_epoch or state.validator_registry[transfer.sender].activation_epoch == FAR_FUTURE_EPOCH or - transfer.amount + transfer.fee >= get_balance(statetransfer.sender) - get_effective_balance(transfer.sender) + transfer.amount + transfer.fee + MAX_EFFECTIVE_BALANCE <= get_balance(transfer.sender) ) # Verify that the pubkey is valid assert ( From 6f56c379d6cca55c9181eb825b2c4dccacd89fca Mon Sep 17 00:00:00 2001 From: Justin Date: Wed, 17 Apr 2019 14:06:28 +1000 Subject: [PATCH 042/121] Simplify get_justification_and_finalization_deltas Cosmetic changes related to `get_justification_and_finalization_deltas`: * Review naming of misc helper functions and variables * Abstract away common logic and rework for readability * Add `MAX_FINALITY_LOOKBACK` and `BASE_REWARDS_PER_EPOCH` constants * Rescale `INACTIVITY_PENALTY_QUOTIENT` Substantive changes: * Make logic relative to `previous_epoch` throughout (as opposed to mixing `current_epoch` and `previous_epoch`) * Replace inclusion delay bonus by an inclusion delay penalty --- specs/core/0_beacon-chain.md | 144 +++++++++++++++-------------------- 1 file changed, 63 insertions(+), 81 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ddb0eb6db..706f9154f 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -185,6 +185,7 @@ These configurations are updated for releases, but may be out of sync during `de | `MAX_ATTESTATION_PARTICIPANTS` | `2**12` (= 4,096) | | `MIN_PER_EPOCH_CHURN_LIMIT` | `2**2` (= 4) | | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) | +| `BASE_REWARDS_PER_EPOCH` | `5` | | `SHUFFLE_ROUND_COUNT` | 90 | * For the safety of crosslinks `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) @@ -234,6 +235,7 @@ These configurations are updated for releases, but may be out of sync during `de | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours | | `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | 9 days | | `MAX_CROSSLINK_EPOCHS` | `2**6` (= 64) | epochs | ~7 hours | +| `MAX_FINALITY_LOOKBACK` | `2**2` (= 4) | epochs | 25.6 minutes | * `MAX_CROSSLINK_EPOCHS` should be a small constant times `SHARD_COUNT // SLOTS_PER_EPOCH` @@ -252,7 +254,7 @@ These configurations are updated for releases, but may be out of sync during `de | `BASE_REWARD_QUOTIENT` | `2**5` (= 32) | | `WHISTLEBLOWING_REWARD_QUOTIENT` | `2**9` (= 512) | | `PROPOSER_REWARD_QUOTIENT` | `2**3` (= 8) | -| `INACTIVITY_PENALTY_QUOTIENT` | `2**24` (= 16,777,216) | +| `INACTIVITY_PENALTY_QUOTIENT` | `2**25` (= 33,554,432) | | `MIN_PENALTY_QUOTIENT` | `2**5` (= 32) | * The `BASE_REWARD_QUOTIENT` parameter dictates the per-epoch reward. It corresponds to ~2.54% annual interest assuming 10 million participating ETH in every epoch. @@ -875,7 +877,7 @@ def compute_committee(validator_indices: List[ValidatorIndex], ] ``` -**Note**: this definition and the next few definitions are highly inefficient as algorithms, as they re-calculate many sub-expressions. Production implementations are expected to appropriately use caching/memoization to avoid redoing work. +Note: this definition and the next few definitions are highly inefficient as algorithms, as they re-calculate many sub-expressions. Production implementations are expected to appropriately use caching/memoization to avoid redoing work. ### `get_crosslink_committees_at_slot` @@ -1582,7 +1584,7 @@ Transition section notes: Beacon blocks that trigger unhandled Python exceptions (e.g. out-of-range list accesses) and failed `assert`s during the state transition are considered invalid. -_Note_: If there are skipped slots between a block and its parent block, run the steps in the [state-root](#state-caching), [per-epoch](#per-epoch-processing), and [per-slot](#per-slot-processing) sections once for each skipped slot and then once for the slot containing the new block. +Note: If there are skipped slots between a block and its parent block, run the steps in the [state-root](#state-caching), [per-epoch](#per-epoch-processing), and [per-slot](#per-slot-processing) sections once for each skipped slot and then once for the slot containing the new block. ### State caching @@ -1612,30 +1614,20 @@ The steps below happen when `state.slot > GENESIS_SLOT and (state.slot + 1) % SL We define epoch transition helper functions: ```python -def get_current_total_balance(state: BeaconState) -> Gwei: - return get_total_balance(state, get_active_validator_indices(state, get_current_epoch(state))) -``` - -```python -def get_previous_total_balance(state: BeaconState) -> Gwei: +def get_previous_epoch_total_balance(state: BeaconState) -> Gwei: return get_total_balance(state, get_active_validator_indices(state, get_previous_epoch(state))) ``` -```python -def get_unslashed_attesting_indices(state: BeaconState, attestations: List[PendingAttestation]) -> List[ValidatorIndex]: - output = set() - for a in attestations: - output = output.union(get_attestation_participants(state, a.data, a.aggregation_bitfield)) - return sorted(filter(lambda index: not state.validator_registry[index].slashed, list(output))) -``` +Note: The balance computed by `get_previous_epoch_total_balance` may be different to the actual total balance during the previous epoch transition. Due to the bounds on per-epoch validator churn and per-epoch rewards/penalties, the maximum balance difference is low and only marginally affects consensus safety. ```python -def get_attesting_balance(state: BeaconState, attestations: List[PendingAttestation]) -> Gwei: - return get_total_balance(state, get_unslashed_attesting_indices(state, attestations)) +def get_current_epoch_total_balance(state: BeaconState) -> Gwei: + return get_total_balance(state, get_active_validator_indices(state, get_current_epoch(state))) ``` + ```python -def get_current_epoch_boundary_attestations(state: BeaconState) -> List[PendingAttestation]: +def get_current_epoch_matching_target_attestations(state: BeaconState) -> List[PendingAttestation]: return [ a for a in state.current_epoch_attestations if a.data.target_root == get_block_root(state, get_epoch_start_slot(get_current_epoch(state))) @@ -1643,7 +1635,7 @@ def get_current_epoch_boundary_attestations(state: BeaconState) -> List[PendingA ``` ```python -def get_previous_epoch_boundary_attestations(state: BeaconState) -> List[PendingAttestation]: +def get_previous_epoch_matching_target_attestations(state: BeaconState) -> List[PendingAttestation]: return [ a for a in state.previous_epoch_attestations if a.data.target_root == get_block_root(state, get_epoch_start_slot(get_previous_epoch(state))) @@ -1658,7 +1650,18 @@ def get_previous_epoch_matching_head_attestations(state: BeaconState) -> List[Pe ] ``` -**Note**: Total balances computed for the previous epoch might be marginally different than the actual total balances during the previous epoch transition. Due to the tight bound on validator churn each epoch and small per-epoch rewards/penalties, the potential balance difference is very low and only marginally affects consensus safety. +```python +def get_unslashed_attesting_indices(state: BeaconState, attestations: List[PendingAttestation]) -> List[ValidatorIndex]: + output = set() + for a in attestations: + output = output.union(get_attestation_participants(state, a.data, a.aggregation_bitfield)) + return sorted(filter(lambda index: not state.validator_registry[index].slashed, list(output))) +``` + +```python +def get_attesting_balance(state: BeaconState, attestations: List[PendingAttestation]) -> Gwei: + return get_total_balance(state, get_unslashed_attesting_indices(state, attestations)) +``` ```python def get_winning_root_and_participants(state: BeaconState, shard: Shard) -> Tuple[Bytes32, List[ValidatorIndex]]: @@ -1701,13 +1704,13 @@ def update_justification_and_finalization(state: BeaconState) -> None: state.previous_justified_epoch = state.current_justified_epoch state.previous_justified_root = state.current_justified_root state.justification_bitfield = (state.justification_bitfield << 1) % 2**64 - previous_boundary_attesting_balance = get_attesting_balance(state, get_previous_epoch_boundary_attestations(state)) - if previous_boundary_attesting_balance * 3 >= get_previous_total_balance(state) * 2: + previous_epoch_matching_target_balance = get_attesting_balance(state, get_previous_epoch_matching_target_attestations(state)) + if previous_epoch_matching_target_balance * 3 >= get_previous_epoch_total_balance(state) * 2: state.current_justified_epoch = get_previous_epoch(state) state.current_justified_root = get_block_root(state, get_epoch_start_slot(state.current_justified_epoch)) state.justification_bitfield |= (1 << 1) - current_boundary_attesting_balance = get_attesting_balance(state, get_current_epoch_boundary_attestations(state)) - if current_boundary_attesting_balance * 3 >= get_current_total_balance(state) * 2: + current_epoch_matching_target_balance = get_attesting_balance(state, get_current_epoch_matching_target_attestations(state)) + if current_epoch_matching_target_balance * 3 >= get_current_epoch_total_balance(state) * 2: state.current_justified_epoch = get_current_epoch(state) state.current_justified_root = get_block_root(state, get_epoch_start_slot(state.current_justified_epoch)) state.justification_bitfield |= (1 << 0) @@ -1771,80 +1774,58 @@ def maybe_reset_eth1_period(state: BeaconState) -> None: #### Rewards and penalties -First, we define some additional helpers: +We first define a helper: ```python -def get_base_reward_from_total_balance(state: BeaconState, total_balance: Gwei, index: ValidatorIndex) -> Gwei: +def get_base_reward(state: BeaconState, total_balance: Gwei, index: ValidatorIndex) -> Gwei: if total_balance == 0: return 0 adjusted_quotient = integer_squareroot(total_balance) // BASE_REWARD_QUOTIENT - return get_effective_balance(state, index) // adjusted_quotient // 5 + return get_effective_balance(state, index) // adjusted_quotient // BASE_REWARDS_PER_EPOCH ``` -```python -def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: - return get_base_reward_from_total_balance(state, get_previous_total_balance(state), index) -``` - -```python -def get_inactivity_penalty(state: BeaconState, index: ValidatorIndex, epochs_since_finality: int) -> Gwei: - if epochs_since_finality <= 4: - extra_penalty = 0 - else: - extra_penalty = get_effective_balance(state, index) * epochs_since_finality // INACTIVITY_PENALTY_QUOTIENT // 2 - return get_base_reward(state, index) + extra_penalty -``` - -Note: When applying penalties in the following balance recalculations, implementers should make sure the `uint64` does not underflow. - ##### Justification and finalization ```python def get_justification_and_finalization_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: - current_epoch = get_current_epoch(state) - epochs_since_finality = current_epoch + 1 - state.finalized_epoch - rewards = [0 for index in range(len(state.validator_registry))] - penalties = [0 for index in range(len(state.validator_registry))] - # Some helper variables - boundary_attestations = get_previous_epoch_boundary_attestations(state) - boundary_attesting_balance = get_attesting_balance(state, boundary_attestations) - total_balance = get_previous_total_balance(state) - total_attesting_balance = get_attesting_balance(state, state.previous_epoch_attestations) - matching_head_attestations = get_previous_epoch_matching_head_attestations(state) - matching_head_balance = get_attesting_balance(state, matching_head_attestations) + previous_epoch = get_previous_epoch(state) eligible_validators = [ index for index, validator in enumerate(state.validator_registry) if ( - is_active_validator(validator, current_epoch) or - (validator.slashed and current_epoch < validator.withdrawable_epoch) + is_active_validator(validator, previous_epoch) or + (validator.slashed and previous_epoch < validator.withdrawable_epoch) ) ] - # Process rewards or penalties for all validators + rewards = [0 for index in range(len(state.validator_registry))] + penalties = [0 for index in range(len(state.validator_registry))] for index in eligible_validators: - base_reward = get_base_reward(state, index) - # Expected FFG source + base_reward = get_base_reward(state, get_previous_epoch_total_balance(state), index) + + # Micro-incentives for matching FFG source, matching FFG target, and matching head + for attestations in ( + state.previous_epoch_attestations, # Matching FFG source + get_previous_epoch_matching_target_attestations(state), # Matching FFG target + get_previous_epoch_matching_head_attestations(state), # Matching head + ): + if index in get_unslashed_attesting_indices(state, attestations): + rewards[index] += base_reward * get_attesting_balance(state, attestations) // get_previous_epoch_total_balance(state) + else: + penalties[index] += base_reward + + # Inclusion delay micro-penalty if index in get_unslashed_attesting_indices(state, state.previous_epoch_attestations): - rewards[index] += base_reward * total_attesting_balance // total_balance - # Inclusion speed bonus earliest_attestation = get_earliest_attestation(state, state.previous_epoch_attestations, index) inclusion_delay = earliest_attestation.inclusion_slot - earliest_attestation.data.slot - rewards[index] += base_reward * MIN_ATTESTATION_INCLUSION_DELAY // inclusion_delay - else: - penalties[index] += base_reward - # Expected FFG target - if index in get_unslashed_attesting_indices(state, boundary_attestations): - rewards[index] += base_reward * boundary_attesting_balance // total_balance - else: - penalties[index] += get_inactivity_penalty(state, index, epochs_since_finality) - # Expected head - if index in get_unslashed_attesting_indices(state, matching_head_attestations): - rewards[index] += base_reward * matching_head_balance // total_balance - else: - penalties[index] += base_reward - # Take away max rewards if we're not finalizing - if epochs_since_finality > 4: - penalties[index] += base_reward * 4 + penalties[index] += base_reward * (1 - MIN_ATTESTATION_INCLUSION_DELAY // inclusion_delay) + + # Inactivity penalty + epochs_since_finality = previous_epoch + 1 - state.finalized_epoch + if epochs_since_finality > MAX_FINALITY_LOOKBACK: + penalties[index] += BASE_REWARDS_PER_EPOCH * base_reward + if index not in get_unslashed_attesting_indices(state, get_previous_epoch_matching_target_attestations(state)): + penalties[index] += get_effective_balance(state, index) * epochs_since_finality // INACTIVITY_PENALTY_QUOTIENT + return [rewards, penalties] ``` @@ -1862,10 +1843,11 @@ def get_crosslink_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: participating_balance = get_total_balance(state, participants) total_balance = get_total_balance(state, crosslink_committee) for index in crosslink_committee: + base_reward = get_base_reward(state, get_previous_epoch_total_balance(state), index) if index in participants: - rewards[index] += get_base_reward(state, index) * participating_balance // total_balance + rewards[index] += base_reward * participating_balance // total_balance else: - penalties[index] += get_base_reward(state, index) + penalties[index] += base_reward return [rewards, penalties] ``` @@ -2176,7 +2158,7 @@ def process_proposer_attestation_rewards(state: BeaconState) -> None: for pending_attestations in (state.previous_epoch_attestations, state.current_epoch_attestations): for index in get_unslashed_attesting_indices(state, pending_attestations): if get_earliest_attestation(state, pending_attestations, index).inclusion_slot == state.slot: - base_reward = get_base_reward_from_total_balance(state, get_current_total_balance(state), index) + base_reward = get_base_reward(state, get_current_epoch_total_balance(state), index) increase_balance(state, proposer_index, base_reward // PROPOSER_REWARD_QUOTIENT) ``` From cc92ee9f67877e4c286c0e0df12dab26b0356717 Mon Sep 17 00:00:00 2001 From: Justin Date: Wed, 17 Apr 2019 15:53:24 +1000 Subject: [PATCH 043/121] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 706f9154f..d7f7095e4 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -185,7 +185,7 @@ These configurations are updated for releases, but may be out of sync during `de | `MAX_ATTESTATION_PARTICIPANTS` | `2**12` (= 4,096) | | `MIN_PER_EPOCH_CHURN_LIMIT` | `2**2` (= 4) | | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) | -| `BASE_REWARDS_PER_EPOCH` | `5` | +| `BASE_REWARDS_PER_EPOCH` | `4` | | `SHUFFLE_ROUND_COUNT` | 90 | * For the safety of crosslinks `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) From c783cdb2f4833d1de4767ab75fb5f5c49ae567e9 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 17 Apr 2019 12:31:00 -0600 Subject: [PATCH 044/121] fix bug and add transfer tests --- specs/core/0_beacon-chain.md | 2 +- .../block_processing/test_process_transfer.py | 143 ++++++++++++++++++ test_libs/pyspec/tests/helpers.py | 50 ++++++ 3 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 test_libs/pyspec/tests/block_processing/test_process_transfer.py diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 6aa562ca4..86a017508 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -2280,7 +2280,7 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: assert ( get_current_epoch(state) >= state.validator_registry[transfer.sender].withdrawable_epoch or state.validator_registry[transfer.sender].activation_epoch == FAR_FUTURE_EPOCH or - transfer.amount + transfer.fee + MAX_EFFECTIVE_BALANCE <= get_balance(transfer.sender) + transfer.amount + transfer.fee + MAX_EFFECTIVE_BALANCE <= get_balance(state, transfer.sender) ) # Verify that the pubkey is valid assert ( diff --git a/test_libs/pyspec/tests/block_processing/test_process_transfer.py b/test_libs/pyspec/tests/block_processing/test_process_transfer.py new file mode 100644 index 000000000..df49aff98 --- /dev/null +++ b/test_libs/pyspec/tests/block_processing/test_process_transfer.py @@ -0,0 +1,143 @@ +from copy import deepcopy +import pytest + +import eth2spec.phase0.spec as spec + +from eth2spec.phase0.spec import ( + get_active_validator_indices, + get_balance, + get_beacon_proposer_index, + get_current_epoch, + process_transfer, + set_balance, +) +from tests.helpers import ( + get_valid_transfer, + next_epoch, +) + + +# mark entire file as 'transfers' +pytestmark = pytest.mark.transfers + + +def run_transfer_processing(state, transfer, valid=True): + """ + Run ``process_transfer`` returning the pre and post state. + If ``valid == False``, run expecting ``AssertionError`` + """ + post_state = deepcopy(state) + + if not valid: + with pytest.raises(AssertionError): + process_transfer(post_state, transfer) + return state, None + + + process_transfer(post_state, transfer) + + proposer_index = get_beacon_proposer_index(state) + pre_transfer_sender_balance = state.balances[transfer.sender] + pre_transfer_recipient_balance = state.balances[transfer.recipient] + pre_transfer_proposer_balance = state.balances[proposer_index] + sender_balance = post_state.balances[transfer.sender] + recipient_balance = post_state.balances[transfer.recipient] + assert sender_balance == pre_transfer_sender_balance - transfer.amount - transfer.fee + assert recipient_balance == pre_transfer_recipient_balance + transfer.amount + assert post_state.balances[proposer_index] == pre_transfer_proposer_balance + transfer.fee + + return state, post_state + + +def test_success_non_activated(state): + transfer = get_valid_transfer(state) + # un-activate so validator can transfer + state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + + pre_state, post_state = run_transfer_processing(state, transfer) + + return pre_state, transfer, post_state + + +def test_success_withdrawable(state): + next_epoch(state) + + transfer = get_valid_transfer(state) + + # withdrawable_epoch in past so can transfer + state.validator_registry[transfer.sender].withdrawable_epoch = get_current_epoch(state) - 1 + + pre_state, post_state = run_transfer_processing(state, transfer) + + return pre_state, transfer, post_state + + +def test_success_active_above_max_effective(state): + sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] + amount = spec.MAX_EFFECTIVE_BALANCE // 32 + set_balance(state, sender_index, spec.MAX_EFFECTIVE_BALANCE + amount) + transfer = get_valid_transfer(state, sender_index=sender_index, amount=amount, fee=0) + + pre_state, post_state = run_transfer_processing(state, transfer) + + return pre_state, transfer, post_state + + +def test_active_but_transfer_past_effective_balance(state): + sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] + amount = spec.MAX_EFFECTIVE_BALANCE // 32 + set_balance(state, sender_index, spec.MAX_EFFECTIVE_BALANCE) + transfer = get_valid_transfer(state, sender_index=sender_index, amount=amount, fee=0) + + pre_state, post_state = run_transfer_processing(state, transfer, False) + + return pre_state, transfer, post_state + + +def test_incorrect_slot(state): + transfer = get_valid_transfer(state, slot=state.slot+1) + # un-activate so validator can transfer + state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + + pre_state, post_state = run_transfer_processing(state, transfer, False) + + return pre_state, transfer, post_state + + +def test_insufficient_balance(state): + sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] + amount = spec.MAX_EFFECTIVE_BALANCE + set_balance(state, sender_index, spec.MAX_EFFECTIVE_BALANCE) + transfer = get_valid_transfer(state, sender_index=sender_index, amount=amount + 1, fee=0) + + # un-activate so validator can transfer + state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + + pre_state, post_state = run_transfer_processing(state, transfer, False) + + return pre_state, transfer, post_state + + +def test_no_dust(state): + sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] + balance = state.balances[sender_index] + transfer = get_valid_transfer(state, sender_index=sender_index, amount=balance - spec.MIN_DEPOSIT_AMOUNT + 1, fee=0) + + # un-activate so validator can transfer + state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + + pre_state, post_state = run_transfer_processing(state, transfer, False) + + return pre_state, transfer, post_state + + +def test_invalid_pubkey(state): + transfer = get_valid_transfer(state) + state.validator_registry[transfer.sender].withdrawal_credentials = spec.ZERO_HASH + + # un-activate so validator can transfer + state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + + pre_state, post_state = run_transfer_processing(state, transfer, False) + + return pre_state, transfer, post_state \ No newline at end of file diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index 9ef891219..650790b5a 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -21,11 +21,13 @@ from eth2spec.phase0.spec import ( DepositData, Eth1Data, ProposerSlashing, + Transfer, VoluntaryExit, # functions convert_to_indexed, get_active_validator_indices, get_attestation_participants, + get_balance, get_block_root, get_crosslink_committee_for_attestation, get_current_epoch, @@ -291,6 +293,48 @@ def get_valid_attestation(state, slot=None): return attestation +def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=None): + if slot is None: + slot = state.slot + current_epoch = get_current_epoch(state) + if sender_index is None: + sender_index = get_active_validator_indices(state, current_epoch)[-1] + recipient_index = get_active_validator_indices(state, current_epoch)[0] + transfer_pubkey = pubkeys[-1] + transfer_privkey = privkeys[-1] + + if fee is None: + fee = get_balance(state, sender_index) // 32 + if amount is None: + amount = get_balance(state, sender_index) - fee + + transfer = Transfer( + sender=sender_index, + recipient=recipient_index, + amount=amount, + fee=fee, + slot=slot, + pubkey=transfer_pubkey, + signature=EMPTY_SIGNATURE, + ) + transfer.signature = bls.sign( + message_hash=signing_root(transfer), + privkey=transfer_privkey, + domain=get_domain( + fork=state.fork, + epoch=get_current_epoch(state), + domain_type=spec.DOMAIN_TRANSFER, + ) + ) + + # ensure withdrawal_credentials reproducable + state.validator_registry[transfer.sender].withdrawal_credentials = ( + spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(transfer.pubkey)[1:] + ) + + return transfer + + def get_attestation_signature(state, attestation_data, privkey, custody_bit=0b0): message_hash = AttestationDataAndCustodyBit( data=attestation_data, @@ -311,3 +355,9 @@ def get_attestation_signature(state, attestation_data, privkey, custody_bit=0b0) def next_slot(state): block = build_empty_block_for_next_slot(state) state_transition(state, block) + + +def next_epoch(state): + block = build_empty_block_for_next_slot(state) + block.slot += spec.SLOTS_PER_EPOCH - (state.slot % spec.SLOTS_PER_EPOCH) + state_transition(state, block) From 9c14900c772774d72ec03aa79d894d2954ae1cef Mon Sep 17 00:00:00 2001 From: Justin Date: Thu, 18 Apr 2019 10:45:22 +1000 Subject: [PATCH 045/121] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 86a017508..d58a8b316 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -2260,8 +2260,6 @@ def process_voluntary_exit(state: BeaconState, exit: VoluntaryExit) -> None: ##### Transfers -Note: Transfers are a temporary functionality for phases 0 and 1, to be removed in phase 2. - Verify that `len(block.body.transfers) <= MAX_TRANSFERS` and that all transfers are distinct. For each `transfer` in `block.body.transfers`, run the following function: From 0f8b1c5f3a768989853f3a8928745ca25ac11f00 Mon Sep 17 00:00:00 2001 From: Justin Date: Thu, 18 Apr 2019 10:56:15 +1000 Subject: [PATCH 046/121] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index d7f7095e4..05dca84eb 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1817,11 +1817,11 @@ def get_justification_and_finalization_deltas(state: BeaconState) -> Tuple[List[ if index in get_unslashed_attesting_indices(state, state.previous_epoch_attestations): earliest_attestation = get_earliest_attestation(state, state.previous_epoch_attestations, index) inclusion_delay = earliest_attestation.inclusion_slot - earliest_attestation.data.slot - penalties[index] += base_reward * (1 - MIN_ATTESTATION_INCLUSION_DELAY // inclusion_delay) + penalties[index] += base_reward * (inclusion_delay - MIN_ATTESTATION_INCLUSION_DELAY) // (SLOTS_PER_EPOCH - MIN_ATTESTATION_INCLUSION_DELAY) # Inactivity penalty epochs_since_finality = previous_epoch + 1 - state.finalized_epoch - if epochs_since_finality > MAX_FINALITY_LOOKBACK: + if epochs_since_finality >= MIN_EPOCHS_TO_LEAK: penalties[index] += BASE_REWARDS_PER_EPOCH * base_reward if index not in get_unslashed_attesting_indices(state, get_previous_epoch_matching_target_attestations(state)): penalties[index] += get_effective_balance(state, index) * epochs_since_finality // INACTIVITY_PENALTY_QUOTIENT From 3f9a65f1c8942c5c41dfc3ba00952465bafc6797 Mon Sep 17 00:00:00 2001 From: Justin Date: Thu, 18 Apr 2019 10:59:15 +1000 Subject: [PATCH 047/121] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 05dca84eb..482e4da49 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -235,7 +235,7 @@ These configurations are updated for releases, but may be out of sync during `de | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours | | `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | 9 days | | `MAX_CROSSLINK_EPOCHS` | `2**6` (= 64) | epochs | ~7 hours | -| `MAX_FINALITY_LOOKBACK` | `2**2` (= 4) | epochs | 25.6 minutes | +| `MIN_EPOCHS_TO_LEAK` | `2**2` (= 4) | epochs | 25.6 minutes | * `MAX_CROSSLINK_EPOCHS` should be a small constant times `SHARD_COUNT // SLOTS_PER_EPOCH` @@ -1820,8 +1820,8 @@ def get_justification_and_finalization_deltas(state: BeaconState) -> Tuple[List[ penalties[index] += base_reward * (inclusion_delay - MIN_ATTESTATION_INCLUSION_DELAY) // (SLOTS_PER_EPOCH - MIN_ATTESTATION_INCLUSION_DELAY) # Inactivity penalty - epochs_since_finality = previous_epoch + 1 - state.finalized_epoch - if epochs_since_finality >= MIN_EPOCHS_TO_LEAK: + epochs_since_finality = previous_epoch - state.finalized_epoch + if epochs_since_finality > MIN_EPOCHS_TO_LEAK: penalties[index] += BASE_REWARDS_PER_EPOCH * base_reward if index not in get_unslashed_attesting_indices(state, get_previous_epoch_matching_target_attestations(state)): penalties[index] += get_effective_balance(state, index) * epochs_since_finality // INACTIVITY_PENALTY_QUOTIENT From 91921d8e86cb74de763be1079806d740a0b8a8a1 Mon Sep 17 00:00:00 2001 From: Justin Date: Thu, 18 Apr 2019 11:52:14 +1000 Subject: [PATCH 048/121] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 482e4da49..864411c9f 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -235,7 +235,7 @@ These configurations are updated for releases, but may be out of sync during `de | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours | | `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | 9 days | | `MAX_CROSSLINK_EPOCHS` | `2**6` (= 64) | epochs | ~7 hours | -| `MIN_EPOCHS_TO_LEAK` | `2**2` (= 4) | epochs | 25.6 minutes | +| `MIN_EPOCHS_TO_INACTIVITY_PENALTY` | `2**2` (= 4) | epochs | 25.6 minutes | * `MAX_CROSSLINK_EPOCHS` should be a small constant times `SHARD_COUNT // SLOTS_PER_EPOCH` @@ -1821,7 +1821,7 @@ def get_justification_and_finalization_deltas(state: BeaconState) -> Tuple[List[ # Inactivity penalty epochs_since_finality = previous_epoch - state.finalized_epoch - if epochs_since_finality > MIN_EPOCHS_TO_LEAK: + if epochs_since_finality > MIN_EPOCHS_TO_INACTIVITY_PENALTY: penalties[index] += BASE_REWARDS_PER_EPOCH * base_reward if index not in get_unslashed_attesting_indices(state, get_previous_epoch_matching_target_attestations(state)): penalties[index] += get_effective_balance(state, index) * epochs_since_finality // INACTIVITY_PENALTY_QUOTIENT From d9afb67e29bd9558d2e7d2aff1d110b8d057775a Mon Sep 17 00:00:00 2001 From: Justin Date: Thu, 18 Apr 2019 17:45:28 +1000 Subject: [PATCH 049/121] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index d58a8b316..bab26ff77 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1366,7 +1366,7 @@ For convenience, we provide the interface to the contract here: * `__init__()`: initializes the contract * `get_deposit_root() -> bytes32`: returns the current root of the deposit tree -* `deposit(bytes[512])`: adds a deposit instance to the deposit tree, incorporating the input argument and the value transferred in the given call. Note: the amount of value transferred *must* be greater than `MIN_DEPOSIT_AMOUNT` inclusive. Each of these constants are specified in units of Gwei. +* `deposit(bytes[512])`: adds a deposit instance to the deposit tree, incorporating the input argument and the value transferred in the given call. Note: the amount of value transferred *must* be at least `MIN_DEPOSIT_AMOUNT`. Each of these constants are specified in units of Gwei. ## On genesis From 40a898f1253960e7d42c21e9163343e5b14570dd Mon Sep 17 00:00:00 2001 From: Justin Date: Thu, 18 Apr 2019 17:46:31 +1000 Subject: [PATCH 050/121] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index bab26ff77..6a01cf029 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1344,7 +1344,7 @@ The private key corresponding to `withdrawal_pubkey` will be required to initiat ### `Deposit` logs -Every Ethereum 1.0 deposit, of size greater than `MIN_DEPOSIT_AMOUNT`, emits a `Deposit` log for consumption by the beacon chain. The deposit contract does little validation, pushing most of the validator onboarding logic to the beacon chain. In particular, the proof of possession (a BLS12 signature) is not verified by the deposit contract. +Every Ethereum 1.0 deposit, of size at least `MIN_DEPOSIT_AMOUNT`, emits a `Deposit` log for consumption by the beacon chain. The deposit contract does little validation, pushing most of the validator onboarding logic to the beacon chain. In particular, the proof of possession (a BLS12-381 signature) is not verified by the deposit contract. ### `Eth2Genesis` log From 72f4e2d3b613dacf6b0ef7e29268906156d1bbba Mon Sep 17 00:00:00 2001 From: Justin Date: Thu, 18 Apr 2019 17:51:50 +1000 Subject: [PATCH 051/121] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 6a01cf029..4f4158e42 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -2274,10 +2274,10 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: assert get_balance(state, transfer.sender) >= max(transfer.amount, transfer.fee) # A transfer is valid in only one slot assert state.slot == transfer.slot - # Only withdrawn, not-yet-deposited accounts, or the balance over MAX_EFFECTIVE_BALANCE can be transfered + # Sender must be not yet eligible for activation, withdrawn, or transfer balance over MAX_EFFECTIVE_BALANCE assert ( + state.validator_registry[transfer.sender].activation_eligibility_epoch == FAR_FUTURE_EPOCH or get_current_epoch(state) >= state.validator_registry[transfer.sender].withdrawable_epoch or - state.validator_registry[transfer.sender].activation_epoch == FAR_FUTURE_EPOCH or transfer.amount + transfer.fee + MAX_EFFECTIVE_BALANCE <= get_balance(state, transfer.sender) ) # Verify that the pubkey is valid From cb5c95b84e6e2e8970fe3cf6ce68df277eb82b29 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Thu, 18 Apr 2019 12:35:22 +0200 Subject: [PATCH 052/121] Fixes tests --- .../pyspec/tests/block_processing/test_process_transfer.py | 2 +- test_libs/pyspec/tests/test_sanity.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/tests/block_processing/test_process_transfer.py b/test_libs/pyspec/tests/block_processing/test_process_transfer.py index df49aff98..71d0894bd 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/tests/block_processing/test_process_transfer.py @@ -52,7 +52,7 @@ def run_transfer_processing(state, transfer, valid=True): def test_success_non_activated(state): transfer = get_valid_transfer(state) # un-activate so validator can transfer - state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validator_registry[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH pre_state, post_state = run_transfer_processing(state, transfer) diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index ee4ab405d..2004a2f25 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -442,7 +442,7 @@ def test_transfer(state): spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(transfer_pubkey)[1:] ) # un-activate so validator can transfer - pre_state.validator_registry[sender_index].activation_epoch = spec.FAR_FUTURE_EPOCH + pre_state.validator_registry[sender_index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH post_state = deepcopy(pre_state) # From 9f4e59b0bc5c9552e79313ab3dfdeee870b6b4a0 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 18 Apr 2019 18:33:06 -0600 Subject: [PATCH 053/121] enhance finality testing -- case 1, 2, 4 --- specs/core/0_beacon-chain.md | 4 + test_libs/pyspec/tests/helpers.py | 1 - test_libs/pyspec/tests/test_finality.py | 156 ++++++++++++++++++++++++ test_libs/pyspec/tests/test_sanity.py | 27 ---- 4 files changed, 160 insertions(+), 28 deletions(-) create mode 100644 test_libs/pyspec/tests/test_finality.py diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index eaceec6ac..4cdfa9c4f 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1697,18 +1697,22 @@ def process_justification_and_finalization(state: BeaconState) -> None: current_epoch = get_current_epoch(state) # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_epoch == current_epoch - 3: + print("rule 1") state.finalized_epoch = old_previous_justified_epoch state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_epoch == current_epoch - 2: + print("rule 2") state.finalized_epoch = old_previous_justified_epoch state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source if (bitfield >> 0) % 8 == 0b111 and old_current_justified_epoch == current_epoch - 2: + print("rule 3") state.finalized_epoch = old_current_justified_epoch state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source if (bitfield >> 0) % 4 == 0b11 and old_current_justified_epoch == current_epoch - 1: + print("rule 4") state.finalized_epoch = old_current_justified_epoch state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) ``` diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index 44d2dcb4d..387c434a0 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -155,7 +155,6 @@ def build_attestation_data(state, slot, shard): current_epoch_start_slot = get_epoch_start_slot(get_current_epoch(state)) if slot < current_epoch_start_slot: - print(slot) epoch_boundary_root = get_block_root(state, get_epoch_start_slot(get_previous_epoch(state))) elif slot == current_epoch_start_slot: epoch_boundary_root = block_root diff --git a/test_libs/pyspec/tests/test_finality.py b/test_libs/pyspec/tests/test_finality.py new file mode 100644 index 000000000..8a429cb6e --- /dev/null +++ b/test_libs/pyspec/tests/test_finality.py @@ -0,0 +1,156 @@ +from copy import deepcopy + +import pytest + +import eth2spec.phase0.spec as spec + +from eth2spec.phase0.state_transition import ( + state_transition, +) +from .helpers import ( + build_empty_block_for_next_slot, + fill_aggregate_attestation, + get_current_epoch, + get_epoch_start_slot, + get_valid_attestation, + next_epoch, +) + +# mark entire file as 'state' +pytestmark = pytest.mark.state + + +def check_finality(state, + prev_state, + current_justified_changed, + previous_justified_changed, + finalized_changed): + if current_justified_changed: + assert state.current_justified_epoch > prev_state.current_justified_epoch + assert state.current_justified_root != prev_state.current_justified_root + else: + assert state.current_justified_epoch == prev_state.current_justified_epoch + assert state.current_justified_root == prev_state.current_justified_root + + if previous_justified_changed: + assert state.previous_justified_epoch > prev_state.previous_justified_epoch + assert state.previous_justified_root != prev_state.previous_justified_root + else: + assert state.previous_justified_epoch == prev_state.previous_justified_epoch + assert state.previous_justified_root == prev_state.previous_justified_root + + if finalized_changed: + assert state.finalized_epoch > prev_state.finalized_epoch + assert state.finalized_root != prev_state.finalized_root + else: + assert state.finalized_epoch == prev_state.finalized_epoch + assert state.finalized_root == prev_state.finalized_root + + +def test_finality_from_genesis_rule_4(state): + test_state = deepcopy(state) + + blocks = [] + for epoch in range(6): + old_current_justified_epoch = test_state.current_justified_epoch + old_current_justified_root = test_state.current_justified_root + for slot in range(spec.SLOTS_PER_EPOCH): + attestation = None + slot_to_attest = test_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 + if slot_to_attest >= spec.GENESIS_SLOT: + attestation = get_valid_attestation(test_state, slot_to_attest) + fill_aggregate_attestation(test_state, attestation) + block = build_empty_block_for_next_slot(test_state) + if attestation: + block.body.attestations.append(attestation) + state_transition(test_state, block) + blocks.append(block) + + if epoch == 0: + check_finality(test_state, state, False, False, False) + elif epoch == 1: + check_finality(test_state, state, False, False, False) + elif epoch == 2: + check_finality(test_state, state, True, False, False) + elif epoch >= 3: + # rule 4 of finaliy + check_finality(test_state, state, True, True, True) + assert test_state.finalized_epoch == old_current_justified_epoch + assert test_state.finalized_root == old_current_justified_root + + return state, blocks, test_state + + +def test_finality_rule_1(state): + # get past first two epochs that finality does not run on + next_epoch(state) + next_epoch(state) + + test_state = deepcopy(state) + + blocks = [] + for epoch in range(3): + old_previous_justified_epoch = test_state.previous_justified_epoch + old_previous_justified_root = test_state.previous_justified_root + for slot in range(spec.SLOTS_PER_EPOCH): + slot_to_attest = test_state.slot - spec.SLOTS_PER_EPOCH + 1 + attestation = get_valid_attestation(test_state, slot_to_attest) + fill_aggregate_attestation(test_state, attestation) + block = build_empty_block_for_next_slot(test_state) + block.body.attestations.append(attestation) + state_transition(test_state, block) + + assert len(test_state.previous_epoch_attestations) >= 0 + assert len(test_state.current_epoch_attestations) == 0 + + blocks.append(block) + + if epoch == 0: + check_finality(test_state, state, True, False, False) + elif epoch == 1: + check_finality(test_state, state, True, True, False) + elif epoch == 2: + # finalized by rule 1 + check_finality(test_state, state, True, True, True) + assert test_state.finalized_epoch == old_previous_justified_epoch + assert test_state.finalized_root == old_previous_justified_root + + +def test_finality_rule_2(state): + # get past first two epochs that finality does not run on + next_epoch(state) + next_epoch(state) + + test_state = deepcopy(state) + + blocks = [] + for epoch in range(3): + old_previous_justified_epoch = test_state.previous_justified_epoch + old_previous_justified_root = test_state.previous_justified_root + for slot in range(spec.SLOTS_PER_EPOCH): + attestation = None + if epoch == 0: + slot_to_attest = test_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 + if slot_to_attest >= get_epoch_start_slot(get_current_epoch(state)): + attestation = get_valid_attestation(test_state, slot_to_attest) + fill_aggregate_attestation(test_state, attestation) + if epoch == 2: + slot_to_attest = test_state.slot - spec.SLOTS_PER_EPOCH + 1 + attestation = get_valid_attestation(test_state, slot_to_attest) + fill_aggregate_attestation(test_state, attestation) + + block = build_empty_block_for_next_slot(test_state) + if attestation: + block.body.attestations.append(attestation) + state_transition(test_state, block) + blocks.append(block) + + if epoch == 0: + check_finality(test_state, state, True, False, False) + elif epoch == 1: + check_finality(test_state, state, True, True, False) + elif epoch == 2: + # finalized by rule 2 + check_finality(test_state, state, True, True, True) + assert test_state.finalized_epoch == old_previous_justified_epoch + assert test_state.finalized_root == old_previous_justified_root diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index 29333a7ad..e48a6b774 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -145,33 +145,6 @@ def test_empty_epoch_transition_not_finalizing(state): return state, [block], test_state -def test_full_attestations_finalizing(state): - test_state = deepcopy(state) - - for slot in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): - next_slot(test_state) - - for epoch in range(5): - for slot in range(spec.SLOTS_PER_EPOCH): - print(test_state.slot) - attestation = get_valid_attestation(test_state, test_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY) - fill_aggregate_attestation(test_state, attestation) - block = build_empty_block_for_next_slot(test_state) - block.body.attestations.append(attestation) - state_transition(test_state, block) - - if epoch == 0: - check_finality(test_state, state, False, False, False) - elif epoch == 1: - check_finality(test_state, state, False, False, False) - elif epoch == 2: - check_finality(test_state, state, True, False, False) - elif epoch == 3: - check_finality(test_state, state, True, True, False) - elif epoch == 4: - check_finality(test_state, state, True, True, True) - - def test_proposer_slashing(state): test_state = deepcopy(state) proposer_slashing = get_valid_proposer_slashing(state) From fad9b4672aa1340ead82c227c7c4d72b46dbc399 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 19 Apr 2019 18:09:29 +1000 Subject: [PATCH 054/121] Disallow transfers As discussed in yesterday's call, temporarily disable transfers until the network is deemed stable enough. We can consider doing a "test-run hard fork" changing this constant prior to the phase 1 hard fork. --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f04a04877..a133fcf42 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -257,7 +257,7 @@ These configurations are updated for releases, but may be out of sync during `de | `MAX_ATTESTATIONS` | `2**7` (= 128) | | `MAX_DEPOSITS` | `2**4` (= 16) | | `MAX_VOLUNTARY_EXITS` | `2**4` (= 16) | -| `MAX_TRANSFERS` | `2**4` (= 16) | +| `MAX_TRANSFERS` | `0` | ### Signature domains From 2b171b19c410d4242b8d750708e45111c1935d09 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 20 Apr 2019 12:18:56 +1000 Subject: [PATCH 055/121] fix generator --- test_generators/operations/deposits.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/operations/deposits.py b/test_generators/operations/deposits.py index 454c6f22d..bd9c392a5 100644 --- a/test_generators/operations/deposits.py +++ b/test_generators/operations/deposits.py @@ -29,7 +29,7 @@ def build_deposit_data(state, message_hash=signing_root(deposit_data), privkey=privkey, domain=spec.get_domain( - state.fork, + state, spec.get_current_epoch(state), spec.DOMAIN_DEPOSIT, ) From 55aa12d7bd5bb4a9d8324b4f26fd3c9d3df8b173 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 20 Apr 2019 12:23:10 +1000 Subject: [PATCH 056/121] parallelism support for make gen_yaml_tests --- Makefile | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index b39538791..0a3e03e33 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ clean: rm -rf $(PY_SPEC_ALL_TARGETS) # "make gen_yaml_tests" to run generators -gen_yaml_tests: $(YAML_TEST_DIR) $(YAML_TEST_TARGETS) +gen_yaml_tests: $(YAML_TEST_TARGETS) # runs a limited set of tests against a minimal config test: $(PY_SPEC_ALL_TARGETS) @@ -48,24 +48,30 @@ CURRENT_DIR = ${CURDIR} # The function that builds a set of suite files, by calling a generator for the given type (param 1) define build_yaml_tests - $(info running generator $(1)) - # Create the output - mkdir -p $(YAML_TEST_DIR)$(1) - - # 1) Create a virtual environment - # 2) Activate the venv, this is where dependencies are installed for the generator - # 3) Install all the necessary requirements - # 4) Run the generator. The generator is assumed to have an "main.py" file. - # 5) We output to the tests dir (generator program should accept a "-o " argument. - cd $(GENERATOR_DIR)$(1); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt; python3 main.py -o $(CURRENT_DIR)/$(YAML_TEST_DIR)$(1) -c $(CURRENT_DIR)/$(CONFIGS_DIR) - - $(info generator $(1) finished) + # Started! + # Create output directory + # Navigate to the generator + # Create a virtual environment, if it does not exist already + # Activate the venv, this is where dependencies are installed for the generator + # Install all the necessary requirements + # Run the generator. The generator is assumed to have an "main.py" file. + # We output to the tests dir (generator program should accept a "-o " argument. + echo "generator $(1) started"; \ + mkdir -p $(YAML_TEST_DIR)$(1); \ + cd $(GENERATOR_DIR)$(1); \ + if test -d venv; then python3 -m venv venv; fi; \ + . venv/bin/activate; \ + pip3 install -r requirements.txt; \ + python3 main.py -o $(CURRENT_DIR)/$(YAML_TEST_DIR)$(1) -c $(CURRENT_DIR)/$(CONFIGS_DIR); \ + echo "generator $(1) finished" endef # The tests dir itself is simply build by creating the directory (recursively creating deeper directories if necessary) $(YAML_TEST_DIR): $(info creating directory, to output yaml targets to: ${YAML_TEST_TARGETS}) mkdir -p $@ +$(YAML_TEST_DIR)/: + $(info ignoring duplicate yaml tests dir) # For any target within the tests dir, build it using the build_yaml_tests function. # (creation of output dir is a dependency) From 69ab4140decb7f98a1143dcd4582a2b872de18c9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 20 Apr 2019 12:25:24 +1000 Subject: [PATCH 057/121] Add note on parallelism --- test_generators/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test_generators/README.md b/test_generators/README.md index 743157aae..66534e5a8 100644 --- a/test_generators/README.md +++ b/test_generators/README.md @@ -28,9 +28,12 @@ make clean This runs all the generators. ```bash -make gen_yaml_tests +make -j 4 gen_yaml_tests ``` +The `-j N` flag makes the generators run in parallel, with `N` being the amount of cores. + + ### Running a single generator The make file auto-detects generators in the `test_generators/` directory, From 14ff452314d566a595ccbdaf2f4cca9fc161c69e Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 20 Apr 2019 12:28:50 +1000 Subject: [PATCH 058/121] move yaml output target --- .gitignore | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index ce047240a..3dd86fc80 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ venv build/ output/ -yaml_tests/ +eth2.0-spec-tests/ .pytest_cache # Dynamically built from Markdown spec diff --git a/Makefile b/Makefile index 0a3e03e33..b93c7ea95 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ SPEC_DIR = ./specs SCRIPT_DIR = ./scripts TEST_LIBS_DIR = ./test_libs PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec -YAML_TEST_DIR = ./yaml_tests +YAML_TEST_DIR = ./eth2.0-spec-tests/tests GENERATOR_DIR = ./test_generators CONFIGS_DIR = ./configs From f908c8d3e085432e8ec1dee8858cb9acff6eccaf Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 20 Apr 2019 15:17:33 +1000 Subject: [PATCH 059/121] Revamped balances and incentivisation --- specs/core/0_beacon-chain.md | 224 ++++++++---------- specs/light_client/sync_protocol.md | 4 +- .../eth2spec/phase0/state_transition.py | 2 - .../test_process_attester_slashing.py | 2 +- .../block_processing/test_process_deposit.py | 2 +- .../test_process_proposer_slashing.py | 2 +- test_libs/pyspec/tests/helpers.py | 2 + test_libs/pyspec/tests/test_sanity.py | 8 +- 8 files changed, 106 insertions(+), 140 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 184811b52..84e9e4230 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -59,8 +59,6 @@ - [`is_active_validator`](#is_active_validator) - [`is_slashable_validator`](#is_slashable_validator) - [`get_active_validator_indices`](#get_active_validator_indices) - - [`get_balance`](#get_balance) - - [`set_balance`](#set_balance) - [`increase_balance`](#increase_balance) - [`decrease_balance`](#decrease_balance) - [`get_permuted_index`](#get_permuted_index) @@ -178,7 +176,7 @@ These configurations are updated for releases, but may be out of sync during `de | `MAX_INDICES_PER_ATTESTATION` | `2**12` (= 4,096) | | `MIN_PER_EPOCH_CHURN_LIMIT` | `2**2` (= 4) | | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) | -| `BASE_REWARDS_PER_EPOCH` | `4` | +| `BASE_REWARDS_PER_EPOCH` | `5` | | `SHUFFLE_ROUND_COUNT` | 90 | * For the safety of crosslinks `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) @@ -410,7 +408,7 @@ The types are defined topologically to aid in facilitating an executable version # Was the validator slashed 'slashed': 'bool', # Rounded balance - 'high_balance': 'uint64' + 'effective_balance': 'uint64', } ``` @@ -733,29 +731,19 @@ def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> List[Valid return [i for i, v in enumerate(state.validator_registry) if is_active_validator(v, epoch)] ``` -### `get_balance` +### `get_next_epoch_effective_balance` ```python -def get_balance(state: BeaconState, index: ValidatorIndex) -> Gwei: +def get_next_epoch_effective_balance(state: BeaconState, index: ValidatorIndex) -> None: """ - Return the balance for a validator with the given ``index``. - """ - return state.balances[index] -``` - -### `set_balance` - -```python -def set_balance(state: BeaconState, index: ValidatorIndex, balance: Gwei) -> None: - """ - Set the balance for a validator with the given ``index`` in both ``BeaconState`` - and validator's rounded balance ``high_balance``. + Get validator effective balance for the next epoch """ + balance = min(state.balances[index], MAX_DEPOSIT_AMOUNT) validator = state.validator_registry[index] HALF_INCREMENT = HIGH_BALANCE_INCREMENT // 2 - if validator.high_balance > balance or validator.high_balance + 3 * HALF_INCREMENT < balance: - validator.high_balance = balance - balance % HIGH_BALANCE_INCREMENT - state.balances[index] = balance + if validator.effective_balance > balance or validator.effective_balance + 3 * HALF_INCREMENT < balance: + return balance - balance % HIGH_BALANCE_INCREMENT + return validator.effective_balance ``` ### `increase_balance` @@ -763,9 +751,9 @@ def set_balance(state: BeaconState, index: ValidatorIndex, balance: Gwei) -> Non ```python def increase_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: """ - Increase the balance for a validator with the given ``index`` by ``delta``. + Increase validator balance by ``delta``. """ - set_balance(state, index, get_balance(state, index) + delta) + state.balances[index] += delta ``` ### `decrease_balance` @@ -773,11 +761,9 @@ def increase_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> ```python def decrease_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: """ - Decrease the balance for a validator with the given ``index`` by ``delta``. - Set to ``0`` when underflow. + Decrease validator balance by ``delta`` with underflow protection. """ - current_balance = get_balance(state, index) - set_balance(state, index, current_balance - delta if current_balance >= delta else 0) + state.balances[index] = state.balances[index] - delta if state.balances[index] >= delta else 0 ``` ### `get_permuted_index` @@ -969,18 +955,17 @@ def generate_seed(state: BeaconState, ### `get_beacon_proposer_index` ```python -def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: +def get_beacon_proposer_index(state: BeaconState, slot: Slot=None) -> ValidatorIndex: """ - Return the beacon proposer index at ``state.slot``. + Return the beacon proposer index at ``slot``. """ current_epoch = get_current_epoch(state) - - first_committee, _ = get_crosslink_committees_at_slot(state, state.slot)[0] + first_committee, _ = get_crosslink_committees_at_slot(state, slot if slot != None else state.slot)[0] i = 0 while True: candidate = first_committee[(current_epoch + i) % len(first_committee)] random_byte = hash(generate_seed(state, current_epoch) + int_to_bytes8(i // 32))[i % 32] - if get_effective_balance(state, candidate) * 256 > MAX_DEPOSIT_AMOUNT * random_byte: + if get_effective_balance(state, candidate, current_epoch) * 256 > MAX_DEPOSIT_AMOUNT * random_byte: return candidate i += 1 ``` @@ -1031,21 +1016,21 @@ def bytes_to_int(data: bytes) -> int: ### `get_effective_balance` ```python -def get_effective_balance(state: BeaconState, index: ValidatorIndex) -> Gwei: +def get_effective_balance(state: BeaconState, index: ValidatorIndex, epoch: Epoch) -> Gwei: """ Return the effective balance (also known as "balance at stake") for a validator with the given ``index``. """ - return min(get_balance(state, index), MAX_DEPOSIT_AMOUNT) + return state.validator_registry[index].effective_balance if epoch == get_current_epoch(state) else get_next_epoch_effective_balance(state, index) ``` ### `get_total_balance` ```python -def get_total_balance(state: BeaconState, validators: List[ValidatorIndex]) -> Gwei: +def get_total_balance(state: BeaconState, validators: List[ValidatorIndex], epoch: Epoch) -> Gwei: """ Return the combined effective balance of an array of ``validators``. """ - return sum([get_effective_balance(state, i) for i in validators]) + return sum([get_effective_balance(state, i, epoch) for i in validators]) ``` ### `get_fork_version` @@ -1290,11 +1275,12 @@ def slash_validator(state: BeaconState, slashed_index: ValidatorIndex, whistlebl Slash the validator with index ``slashed_index``. Note that this function mutates ``state``. """ + current_epoch = get_current_epoch(state) initiate_validator_exit(state, slashed_index) state.validator_registry[slashed_index].slashed = True - state.validator_registry[slashed_index].withdrawable_epoch = get_current_epoch(state) + LATEST_SLASHED_EXIT_LENGTH - slashed_balance = get_effective_balance(state, slashed_index) - state.latest_slashed_balances[get_current_epoch(state) % LATEST_SLASHED_EXIT_LENGTH] += slashed_balance + state.validator_registry[slashed_index].withdrawable_epoch = current_epoch + LATEST_SLASHED_EXIT_LENGTH + slashed_balance = get_effective_balance(state, slashed_index, current_epoch) + state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH] += slashed_balance proposer_index = get_beacon_proposer_index(state) if whistleblower_index is None: @@ -1447,8 +1433,8 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], process_deposit(state, deposit) # Process genesis activations - for index in range(len(state.validator_registry)): - if get_effective_balance(state, index) >= MAX_DEPOSIT_AMOUNT: + for index, validator in enumerate(state.validator_registry): + if validator.effective_balance >= MAX_DEPOSIT_AMOUNT: activate_validator(state, index) genesis_active_index_root = hash_tree_root(get_active_validator_indices(state, GENESIS_EPOCH)) @@ -1520,7 +1506,7 @@ def lmd_ghost(store: Store, start_state: BeaconState, start_block: BeaconBlock) # made for optimized implementations that precompute and save data def get_vote_count(block: BeaconBlock) -> int: return sum( - start_state.validator_registry[validator_index].high_balance + start_state.validator_registry[validator_index].effective_balance for validator_index, target in attestation_targets if get_ancestor(store, target, block.slot) == block ) @@ -1581,38 +1567,27 @@ The steps below happen when `state.slot > GENESIS_SLOT and (state.slot + 1) % SL We define epoch transition helper functions: ```python -def get_previous_epoch_total_balance(state: BeaconState) -> Gwei: - return get_total_balance(state, get_active_validator_indices(state, get_previous_epoch(state))) +def get_total_active_balance(state: BeaconState, epoch: Epoch) -> Gwei: + return get_total_balance(state, get_active_validator_indices(state, epoch), epoch) ``` -Note: The balance computed by `get_previous_epoch_total_balance` may be different to the actual total balance during the previous epoch transition. Due to the bounds on per-epoch validator churn and per-epoch rewards/penalties, the maximum balance difference is low and only marginally affects consensus safety. - ```python -def get_current_epoch_total_balance(state: BeaconState) -> Gwei: - return get_total_balance(state, get_active_validator_indices(state, get_current_epoch(state))) +def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: + return state.current_epoch_attestations if epoch == get_current_epoch(state) else state.previous_epoch_attestations ``` - ```python -def get_current_epoch_matching_target_attestations(state: BeaconState) -> List[PendingAttestation]: +def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: return [ - a for a in state.current_epoch_attestations - if a.data.target_root == get_block_root(state, get_epoch_start_slot(get_current_epoch(state))) + a for a in get_matching_source_attestations(state, epoch) + if a.data.target_root == get_block_root(state, get_epoch_start_slot(epoch)) ] ``` ```python -def get_previous_epoch_matching_target_attestations(state: BeaconState) -> List[PendingAttestation]: +def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: return [ - a for a in state.previous_epoch_attestations - if a.data.target_root == get_block_root(state, get_epoch_start_slot(get_previous_epoch(state))) - ] -``` - -```python -def get_previous_epoch_matching_head_attestations(state: BeaconState) -> List[PendingAttestation]: - return [ - a for a in state.previous_epoch_attestations + a for a in get_matching_source_attestations(state, epoch) if a.data.beacon_block_root == get_block_root(state, a.data.slot) ] ``` @@ -1626,8 +1601,8 @@ def get_unslashed_attesting_indices(state: BeaconState, attestations: List[Pendi ``` ```python -def get_attesting_balance(state: BeaconState, attestations: List[PendingAttestation]) -> Gwei: - return get_total_balance(state, get_unslashed_attesting_indices(state, attestations)) +def get_attesting_balance(state: BeaconState, attestations: List[PendingAttestation], epoch: Epoch) -> Gwei: + return get_total_balance(state, get_unslashed_attesting_indices(state, attestations), epoch) ``` ```python @@ -1641,8 +1616,8 @@ def get_crosslink_from_attestation_data(state: BeaconState, data: AttestationDat ```python def get_winning_crosslink_and_attesting_indices(state: BeaconState, epoch: Epoch, shard: Shard) -> Tuple[Crosslink, List[ValidatorIndex]]: - pending_attestations = state.current_epoch_attestations if epoch == get_current_epoch(state) else state.previous_epoch_attestations - shard_attestations = [a for a in pending_attestations if a.data.shard == shard] + attestations = get_matching_source_attestations(state, epoch) + shard_attestations = [a for a in attestations if a.data.shard == shard] shard_crosslinks = [get_crosslink_from_attestation_data(state, a.data) for a in shard_attestations] candidate_crosslinks = [ c for c in shard_crosslinks @@ -1655,7 +1630,7 @@ def get_winning_crosslink_and_attesting_indices(state: BeaconState, epoch: Epoch return [a for a in shard_attestations if get_crosslink_from_attestation_data(state, a.data) == crosslink] # Winning crosslink has the crosslink data root with the most balance voting for it (ties broken lexicographically) winning_crosslink = max(candidate_crosslinks, key=lambda crosslink: ( - get_attesting_balance(state, get_attestations_for(crosslink)), crosslink.crosslink_data_root + get_attesting_balance(state, get_attestations_for(crosslink), epoch), crosslink.crosslink_data_root )) return winning_crosslink, get_unslashed_attesting_indices(state, get_attestations_for(winning_crosslink)) @@ -1684,13 +1659,15 @@ def process_justification_and_finalization(state: BeaconState) -> None: state.previous_justified_epoch = state.current_justified_epoch state.previous_justified_root = state.current_justified_root state.justification_bitfield = (state.justification_bitfield << 1) % 2**64 - previous_epoch_matching_target_balance = get_attesting_balance(state, get_previous_epoch_matching_target_attestations(state)) - if previous_epoch_matching_target_balance * 3 >= get_previous_epoch_total_balance(state) * 2: + epoch = get_previous_epoch(state) + previous_epoch_matching_target_balance = get_attesting_balance(state, get_matching_target_attestations(state, epoch), epoch) + if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state, epoch) * 2: state.current_justified_epoch = get_previous_epoch(state) state.current_justified_root = get_block_root(state, get_epoch_start_slot(state.current_justified_epoch)) state.justification_bitfield |= (1 << 1) - current_epoch_matching_target_balance = get_attesting_balance(state, get_current_epoch_matching_target_attestations(state)) - if current_epoch_matching_target_balance * 3 >= get_current_epoch_total_balance(state) * 2: + epoch = get_current_epoch(state) + current_epoch_matching_target_balance = get_attesting_balance(state, get_matching_target_attestations(state, epoch), epoch) + if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state, epoch) * 2: state.current_justified_epoch = get_current_epoch(state) state.current_justified_root = get_block_root(state, get_epoch_start_slot(state.current_justified_epoch)) state.justification_bitfield |= (1 << 0) @@ -1726,9 +1703,10 @@ def process_crosslinks(state: BeaconState) -> None: previous_epoch = get_previous_epoch(state) next_epoch = get_current_epoch(state) + 1 for slot in range(get_epoch_start_slot(previous_epoch), get_epoch_start_slot(next_epoch)): + epoch = slot_to_epoch(slot) for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot): - winning_crosslink, attesting_indices = get_winning_crosslink_and_attesting_indices(state, slot_to_epoch(slot), shard) - if 3 * get_total_balance(state, attesting_indices) >= 2 * get_total_balance(state, crosslink_committee): + winning_crosslink, attesting_indices = get_winning_crosslink_and_attesting_indices(state, epoch, shard) + if 3 * get_total_balance(state, attesting_indices, epoch) >= 2 * get_total_balance(state, crosslink_committee, epoch): state.current_crosslinks[shard] = winning_crosslink ``` @@ -1737,52 +1715,53 @@ def process_crosslinks(state: BeaconState) -> None: First, we define additional helpers: ```python -def get_base_reward(state: BeaconState, total_balance: Gwei, index: ValidatorIndex) -> Gwei: +def get_base_reward(state: BeaconState, index: ValidatorIndex, epoch: Epoch) -> Gwei: + total_balance = get_total_active_balance(state, epoch) if total_balance == 0: return 0 adjusted_quotient = integer_squareroot(total_balance) // BASE_REWARD_QUOTIENT - return get_effective_balance(state, index) // adjusted_quotient // BASE_REWARDS_PER_EPOCH + return get_effective_balance(state, index, epoch) // adjusted_quotient // BASE_REWARDS_PER_EPOCH ``` ```python -def get_justification_and_finalization_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: - previous_epoch = get_previous_epoch(state) - eligible_validators = [ +def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: + epoch = get_previous_epoch(state) + eligible_validator_indices = [ index for index, validator in enumerate(state.validator_registry) - if ( - is_active_validator(validator, previous_epoch) or - (validator.slashed and previous_epoch < validator.withdrawable_epoch) - ) + if is_active_validator(validator, epoch) or (validator.slashed and epoch < validator.withdrawable_epoch) ] rewards = [0 for index in range(len(state.validator_registry))] penalties = [0 for index in range(len(state.validator_registry))] - for index in eligible_validators: - base_reward = get_base_reward(state, get_previous_epoch_total_balance(state), index) + for index in eligible_validator_indices: + base_reward = get_base_reward(state, index, epoch) - # Micro-incentives for matching FFG source, matching FFG target, and matching head + # Micro-incentives for attestations matching FFG source, FFG target, and head for attestations in ( - state.previous_epoch_attestations, # Matching FFG source - get_previous_epoch_matching_target_attestations(state), # Matching FFG target - get_previous_epoch_matching_head_attestations(state), # Matching head + get_matching_source_attestations(state, epoch), + get_matching_target_attestations(state, epoch), + get_matching_source_attestations(state, epoch), ): if index in get_unslashed_attesting_indices(state, attestations): - rewards[index] += base_reward * get_attesting_balance(state, attestations) // get_previous_epoch_total_balance(state) + rewards[index] += base_reward * get_attesting_balance(state, attestations, epoch) // get_total_active_balance(state, epoch) else: penalties[index] += base_reward - # Inclusion delay micro-penalty - if index in get_unslashed_attesting_indices(state, state.previous_epoch_attestations): - earliest_attestation = get_earliest_attestation(state, state.previous_epoch_attestations, index) + if index in get_unslashed_attesting_indices(state, get_matching_source_attestations(state, epoch)): + earliest_attestation = get_earliest_attestation(state, get_matching_source_attestations(state, epoch), index) + # Proposer micro-rewards + proposer_index = get_beacon_proposer_index(state, earliest_attestation.inclusion_slot) + rewards[proposer_index] += base_reward // PROPOSER_REWARD_QUOTIENT + # Inclusion delay micro-rewards inclusion_delay = earliest_attestation.inclusion_slot - earliest_attestation.data.slot - penalties[index] += base_reward * (inclusion_delay - MIN_ATTESTATION_INCLUSION_DELAY) // (SLOTS_PER_EPOCH - MIN_ATTESTATION_INCLUSION_DELAY) + rewards[index] += base_reward * MIN_ATTESTATION_INCLUSION_DELAY // inclusion_delay # Inactivity penalty - epochs_since_finality = previous_epoch - state.finalized_epoch - if epochs_since_finality > MIN_EPOCHS_TO_INACTIVITY_PENALTY: + finality_delay = epoch - state.finalized_epoch + if finality_delay > MIN_EPOCHS_TO_INACTIVITY_PENALTY: penalties[index] += BASE_REWARDS_PER_EPOCH * base_reward - if index not in get_unslashed_attesting_indices(state, get_previous_epoch_matching_target_attestations(state)): - penalties[index] += get_effective_balance(state, index) * epochs_since_finality // INACTIVITY_PENALTY_QUOTIENT + if index not in get_unslashed_attesting_indices(state, get_matching_target_attestations(state, epoch)): + penalties[index] += get_effective_balance(state, index, epoch) * finality_delay // INACTIVITY_PENALTY_QUOTIENT return [rewards, penalties] ``` @@ -1792,12 +1771,13 @@ def get_crosslink_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: rewards = [0 for index in range(len(state.validator_registry))] penalties = [0 for index in range(len(state.validator_registry))] for slot in range(get_epoch_start_slot(get_previous_epoch(state)), get_epoch_start_slot(get_current_epoch(state))): + epoch = slot_to_epoch(slot) for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot): - winning_crosslink, attesting_indices = get_winning_crosslink_and_attesting_indices(state, slot_to_epoch(slot), shard) - attesting_balance = get_total_balance(state, attesting_indices) - committee_balance = get_total_balance(state, crosslink_committee) + winning_crosslink, attesting_indices = get_winning_crosslink_and_attesting_indices(state, epoch, shard) + attesting_balance = get_total_balance(state, attesting_indices, epoch) + committee_balance = get_total_balance(state, crosslink_committee, epoch) for index in crosslink_committee: - base_reward = get_base_reward(state, get_previous_epoch_total_balance(state), index) + base_reward = get_base_reward(state, index, epoch) if index in attesting_indices: rewards[index] += base_reward * attesting_balance // committee_balance else: @@ -1812,7 +1792,7 @@ def process_rewards_and_penalties(state: BeaconState) -> None: if get_current_epoch(state) == GENESIS_EPOCH: return - rewards1, penalties1 = get_justification_and_finalization_deltas(state) + rewards1, penalties1 = get_attestation_deltas(state) rewards2, penalties2 = get_crosslink_deltas(state) for i in range(len(state.validator_registry)): increase_balance(state, i, rewards1[i] + rewards2[i]) @@ -1827,11 +1807,10 @@ Run the following function: def process_registry_updates(state: BeaconState) -> None: # Process activation eligibility and ejections for index, validator in enumerate(state.validator_registry): - balance = get_balance(state, index) - if validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH and balance >= MAX_DEPOSIT_AMOUNT: + if validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH and validator.effective_balance >= MAX_DEPOSIT_AMOUNT: validator.activation_eligibility_epoch = get_current_epoch(state) - if is_active_validator(validator, get_current_epoch(state)) and balance < EJECTION_BALANCE: + if is_active_validator(validator, get_current_epoch(state)) and validator.effective_balance < EJECTION_BALANCE: initiate_validator_exit(state, index) # Process activations @@ -1852,7 +1831,7 @@ Run the following function: def process_slashings(state: BeaconState) -> None: current_epoch = get_current_epoch(state) active_validator_indices = get_active_validator_indices(state, current_epoch) - total_balance = get_total_balance(state, active_validator_indices) + total_balance = get_total_balance(state, active_validator_indices, current_epoch) # Compute `total_penalties` total_at_start = state.latest_slashed_balances[(current_epoch + 1) % LATEST_SLASHED_EXIT_LENGTH] @@ -1862,8 +1841,8 @@ def process_slashings(state: BeaconState) -> None: for index, validator in enumerate(state.validator_registry): if validator.slashed and current_epoch == validator.withdrawable_epoch - LATEST_SLASHED_EXIT_LENGTH // 2: penalty = max( - get_effective_balance(state, index) * min(total_penalties * 3, total_balance) // total_balance, - get_effective_balance(state, index) // MIN_PENALTY_QUOTIENT + get_effective_balance(state, index, current_epoch) * min(total_penalties * 3, total_balance) // total_balance, + get_effective_balance(state, index, current_epoch) // MIN_PENALTY_QUOTIENT ) decrease_balance(state, index, penalty) ``` @@ -1879,6 +1858,9 @@ def process_final_updates(state: BeaconState) -> None: # Reset eth1 data votes if state.slot % SLOTS_PER_ETH1_VOTING_PERIOD == 0: state.eth1_data_votes = [] + # Update effective balances + for index, validator in enumerate(state.validator_registry): + validator.effective_balance = get_next_epoch_effective_balance(state, index) # Update start shard state.latest_start_shard = (state.latest_start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT # Set active index root @@ -2079,18 +2061,6 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: state.previous_epoch_attestations.append(pending_attestation) ``` -Run `process_proposer_attestation_rewards(state)`. - -```python -def process_proposer_attestation_rewards(state: BeaconState) -> None: - proposer_index = get_beacon_proposer_index(state) - for pending_attestations in (state.previous_epoch_attestations, state.current_epoch_attestations): - for index in get_unslashed_attesting_indices(state, pending_attestations): - if get_earliest_attestation(state, pending_attestations, index).inclusion_slot == state.slot: - base_reward = get_base_reward(state, get_current_epoch_total_balance(state), index) - increase_balance(state, proposer_index, base_reward // PROPOSER_REWARD_QUOTIENT) -``` - ##### Deposits Verify that `len(block.body.deposits) == min(MAX_DEPOSITS, state.latest_eth1_data.deposit_count - state.deposit_index)`. @@ -2153,13 +2123,11 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: exit_epoch=FAR_FUTURE_EPOCH, withdrawable_epoch=FAR_FUTURE_EPOCH, slashed=False, - high_balance=0 + effective_balance=amount - amount % HIGH_BALANCE_INCREMENT, ) - # Note: In phase 2 registry indices that have been withdrawn for a long time will be recycled. state.validator_registry.append(validator) - state.balances.append(0) - set_balance(state, len(state.validator_registry) - 1, amount) + state.balances.append(amount) else: # Increase balance by deposit amount index = validator_pubkeys.index(pubkey) @@ -2212,8 +2180,8 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: Process ``Transfer`` operation. Note that this function mutates ``state``. """ - # Verify the amount and fee aren't individually too big (for anti-overflow purposes) - assert get_balance(state, transfer.sender) >= max(transfer.amount, transfer.fee) + # Verify the amount and fee are not individually too big (for anti-overflow purposes) + assert state.balances[transfer.sender] >= max(transfer.amount, transfer.fee) # A transfer is valid in only one slot assert state.slot == transfer.slot # Only withdrawn or not-yet-deposited accounts can transfer @@ -2238,8 +2206,8 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: increase_balance(state, transfer.recipient, transfer.amount) increase_balance(state, get_beacon_proposer_index(state), transfer.fee) # Verify balances are not dust - assert not (0 < get_balance(state, transfer.sender) < MIN_DEPOSIT_AMOUNT) - assert not (0 < get_balance(state, transfer.recipient) < MIN_DEPOSIT_AMOUNT) + assert not (0 < state.balances[transfer.sender] < MIN_DEPOSIT_AMOUNT) + assert not (0 < state.balances[transfer.recipient] < MIN_DEPOSIT_AMOUNT) ``` #### State root verification diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index 900b2e64f..7b8388583 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -180,8 +180,8 @@ def verify_block_validity_proof(proof: BlockValidityProof, validator_memory: Val assert proof.shard_parent_block.beacon_chain_root == hash_tree_root(proof.header) committee = compute_committee(proof.header, validator_memory) # Verify that we have >=50% support - support_balance = sum([v.high_balance for i, v in enumerate(committee) if get_bitfield_bit(proof.shard_bitfield, i) is True]) - total_balance = sum([v.high_balance for i, v in enumerate(committee)]) + support_balance = sum([v.effective_balance for i, v in enumerate(committee) if get_bitfield_bit(proof.shard_bitfield, i) is True]) + total_balance = sum([v.effective_balance for i, v in enumerate(committee)]) assert support_balance * 2 > total_balance # Verify shard attestations group_public_key = bls_aggregate_pubkeys([ diff --git a/test_libs/pyspec/eth2spec/phase0/state_transition.py b/test_libs/pyspec/eth2spec/phase0/state_transition.py index 38ecd2a02..1bef358d4 100644 --- a/test_libs/pyspec/eth2spec/phase0/state_transition.py +++ b/test_libs/pyspec/eth2spec/phase0/state_transition.py @@ -11,7 +11,6 @@ from .spec import ( BeaconState, BeaconBlock, Slot, - process_proposer_attestation_rewards, ) @@ -52,7 +51,6 @@ def process_operations(state: BeaconState, block: BeaconBlock) -> None: spec.MAX_ATTESTATIONS, spec.process_attestation, ) - process_proposer_attestation_rewards(state) assert len(block.body.deposits) == expected_deposit_count(state) process_operation_type( diff --git a/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py index 84c19145a..bcaf6fb7a 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py @@ -3,11 +3,11 @@ import pytest import eth2spec.phase0.spec as spec from eth2spec.phase0.spec import ( - get_balance, get_beacon_proposer_index, process_attester_slashing, ) from tests.helpers import ( + get_balance, get_valid_attester_slashing, next_epoch, ) diff --git a/test_libs/pyspec/tests/block_processing/test_process_deposit.py b/test_libs/pyspec/tests/block_processing/test_process_deposit.py index 4031e650d..4fb8b3a1e 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/tests/block_processing/test_process_deposit.py @@ -4,11 +4,11 @@ import pytest import eth2spec.phase0.spec as spec from eth2spec.phase0.spec import ( - get_balance, ZERO_HASH, process_deposit, ) from tests.helpers import ( + get_balance, build_deposit, privkeys, pubkeys, diff --git a/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py index 6d5f3045d..475221036 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py @@ -3,11 +3,11 @@ import pytest import eth2spec.phase0.spec as spec from eth2spec.phase0.spec import ( - get_balance, get_current_epoch, process_proposer_slashing, ) from tests.helpers import ( + get_balance, get_valid_proposer_slashing, ) diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index 44d2dcb4d..616f3b797 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -51,6 +51,8 @@ privkeys = [i + 1 for i in range(1000)] pubkeys = [bls.privtopub(privkey) for privkey in privkeys] pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} +def get_balance(state, index): + return state.balances[index] def set_bitfield_bit(bitfield, i): """ diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index 29333a7ad..508b07905 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -16,7 +16,6 @@ from eth2spec.phase0.spec import ( VoluntaryExit, # functions get_active_validator_indices, - get_balance, get_beacon_proposer_index, get_block_root, get_current_epoch, @@ -24,7 +23,6 @@ from eth2spec.phase0.spec import ( get_state_root, advance_slot, cache_state, - set_balance, slot_to_epoch, verify_merkle_branch, hash, @@ -38,6 +36,7 @@ from eth2spec.utils.merkle_minimal import ( get_merkle_root, ) from .helpers import ( + get_balance, build_deposit_data, build_empty_block_for_next_slot, fill_aggregate_attestation, @@ -53,7 +52,6 @@ from .helpers import ( # mark entire file as 'sanity' pytestmark = pytest.mark.sanity - def check_finality(state, prev_state, current_justified_changed, @@ -304,6 +302,7 @@ def test_deposit_top_up(state): def test_attestation(state): + state.slot = spec.SLOTS_PER_EPOCH test_state = deepcopy(state) attestation = get_valid_attestation(state) @@ -318,7 +317,6 @@ def test_attestation(state): assert len(test_state.current_epoch_attestations) == len(state.current_epoch_attestations) + 1 proposer_index = get_beacon_proposer_index(test_state) - assert test_state.balances[proposer_index] > state.balances[proposer_index] # # Epoch transition should move to previous_epoch_attestations @@ -443,7 +441,7 @@ def test_balance_driven_status_transitions(state): assert pre_state.validator_registry[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH # set validator balance to below ejection threshold - set_balance(pre_state, validator_index, spec.EJECTION_BALANCE - 1) + pre_state.validator_registry[validator_index].effective_balance = spec.EJECTION_BALANCE - 1 post_state = deepcopy(pre_state) # From 4d26ae255a1c9127fe4cc9dc3ed2ac830b1bf1f3 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 20 Apr 2019 15:31:15 +1000 Subject: [PATCH 060/121] Bug fix --- specs/core/0_beacon-chain.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 84e9e4230..ba17691bc 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -731,10 +731,10 @@ def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> List[Valid return [i for i, v in enumerate(state.validator_registry) if is_active_validator(v, epoch)] ``` -### `get_next_epoch_effective_balance` +### `get_current_epoch_effective_balance` ```python -def get_next_epoch_effective_balance(state: BeaconState, index: ValidatorIndex) -> None: +def get_current_epoch_effective_balance(state: BeaconState, index: ValidatorIndex) -> None: """ Get validator effective balance for the next epoch """ @@ -1020,7 +1020,7 @@ def get_effective_balance(state: BeaconState, index: ValidatorIndex, epoch: Epoc """ Return the effective balance (also known as "balance at stake") for a validator with the given ``index``. """ - return state.validator_registry[index].effective_balance if epoch == get_current_epoch(state) else get_next_epoch_effective_balance(state, index) + return get_current_epoch_effective_balance(state, index) if epoch == get_current_epoch(state) else state.validator_registry[index].effective_balance ``` ### `get_total_balance` @@ -1860,7 +1860,7 @@ def process_final_updates(state: BeaconState) -> None: state.eth1_data_votes = [] # Update effective balances for index, validator in enumerate(state.validator_registry): - validator.effective_balance = get_next_epoch_effective_balance(state, index) + validator.effective_balance = get_current_epoch_effective_balance(state, index) # Update start shard state.latest_start_shard = (state.latest_start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT # Set active index root From f07b94e77c0c2453050905ea9cf8e51ada34ff2c Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 20 Apr 2019 15:37:12 +1000 Subject: [PATCH 061/121] Fixes --- specs/core/0_beacon-chain.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 27e3b8663..e5ef8af1b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -937,13 +937,13 @@ def get_beacon_proposer_index(state: BeaconState, slot: Slot=None) -> ValidatorI """ Return the beacon proposer index at ``slot``. """ - current_epoch = get_current_epoch(state) + epoch = slot_to_epoch(slot if slot != None else state.slot) first_committee, _ = get_crosslink_committees_at_slot(state, slot if slot != None else state.slot)[0] i = 0 while True: - candidate = first_committee[(current_epoch + i) % len(first_committee)] - random_byte = hash(generate_seed(state, current_epoch) + int_to_bytes8(i // 32))[i % 32] - if get_effective_balance(state, candidate, current_epoch) * 256 > MAX_DEPOSIT_AMOUNT * random_byte: + candidate = first_committee[(epoch + i) % len(first_committee)] + random_byte = hash(generate_seed(state, epoch) + int_to_bytes8(i // 32))[i % 32] + if get_effective_balance(state, candidate, epoch) * 256 > MAX_DEPOSIT_AMOUNT * random_byte: return candidate i += 1 ``` From d700ea44066ce3efb47f58e158bc184510f266ab Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 20 Apr 2019 16:10:25 +1000 Subject: [PATCH 062/121] Fixes --- specs/core/0_beacon-chain.md | 50 ++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index e5ef8af1b..c1dd75fb0 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -58,6 +58,7 @@ - [`is_active_validator`](#is_active_validator) - [`is_slashable_validator`](#is_slashable_validator) - [`get_active_validator_indices`](#get_active_validator_indices) + - [`get_current_epoch_effective_balance`](#get_current_epoch_effective_balance) - [`increase_balance`](#increase_balance) - [`decrease_balance`](#decrease_balance) - [`get_permuted_index`](#get_permuted_index) @@ -712,7 +713,7 @@ def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> List[Valid ### `get_current_epoch_effective_balance` ```python -def get_current_epoch_effective_balance(state: BeaconState, index: ValidatorIndex) -> None: +def get_current_epoch_effective_balance(state: BeaconState, index: ValidatorIndex) -> Gwei: """ Get validator effective balance for the next epoch """ @@ -1008,7 +1009,7 @@ def get_total_balance(state: BeaconState, validators: List[ValidatorIndex], epoc """ Return the combined effective balance of an array of ``validators``. """ - return sum([get_effective_balance(state, i, epoch) for i in validators]) + return sum([get_effective_balance(state, index, epoch) for index in validators]) ``` ### `get_domain` @@ -1547,6 +1548,8 @@ def process_justification_and_finalization(state: BeaconState) -> None: if get_current_epoch(state) <= GENESIS_EPOCH + 1: return + previous_epoch = get_previous_epoch(state) + current_epoch = get_current_epoch(state) old_previous_justified_epoch = state.previous_justified_epoch old_current_justified_epoch = state.current_justified_epoch @@ -1554,22 +1557,19 @@ def process_justification_and_finalization(state: BeaconState) -> None: state.previous_justified_epoch = state.current_justified_epoch state.previous_justified_root = state.current_justified_root state.justification_bitfield = (state.justification_bitfield << 1) % 2**64 - epoch = get_previous_epoch(state) - previous_epoch_matching_target_balance = get_attesting_balance(state, get_matching_target_attestations(state, epoch), epoch) - if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state, epoch) * 2: + previous_epoch_matching_target_balance = get_attesting_balance(state, get_matching_target_attestations(state, previous_epoch), previous_epoch) + if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state, previous_epoch) * 2: state.current_justified_epoch = get_previous_epoch(state) state.current_justified_root = get_block_root(state, get_epoch_start_slot(state.current_justified_epoch)) state.justification_bitfield |= (1 << 1) - epoch = get_current_epoch(state) - current_epoch_matching_target_balance = get_attesting_balance(state, get_matching_target_attestations(state, epoch), epoch) - if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state, epoch) * 2: + current_epoch_matching_target_balance = get_attesting_balance(state, get_matching_target_attestations(state, current_epoch), current_epoch) + if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state, current_epoch) * 2: state.current_justified_epoch = get_current_epoch(state) state.current_justified_root = get_block_root(state, get_epoch_start_slot(state.current_justified_epoch)) state.justification_bitfield |= (1 << 0) # Process finalizations bitfield = state.justification_bitfield - current_epoch = get_current_epoch(state) # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_epoch == current_epoch - 3: state.finalized_epoch = old_previous_justified_epoch @@ -1611,39 +1611,39 @@ First, we define additional helpers: ```python def get_base_reward(state: BeaconState, index: ValidatorIndex, epoch: Epoch) -> Gwei: - total_balance = get_total_active_balance(state, epoch) - if total_balance == 0: + adjusted_quotient = integer_squareroot(get_total_active_balance(state, epoch)) // BASE_REWARD_QUOTIENT + if adjusted_quotient == 0: return 0 - - adjusted_quotient = integer_squareroot(total_balance) // BASE_REWARD_QUOTIENT return get_effective_balance(state, index, epoch) // adjusted_quotient // BASE_REWARDS_PER_EPOCH + ``` ```python def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: - epoch = get_previous_epoch(state) + previous_epoch = get_previous_epoch(state) + total_balance = get_total_active_balance(state, previous_epoch) eligible_validator_indices = [ index for index, validator in enumerate(state.validator_registry) - if is_active_validator(validator, epoch) or (validator.slashed and epoch < validator.withdrawable_epoch) + if is_active_validator(validator, previous_epoch) or (validator.slashed and previous_epoch < validator.withdrawable_epoch) ] rewards = [0 for index in range(len(state.validator_registry))] penalties = [0 for index in range(len(state.validator_registry))] for index in eligible_validator_indices: - base_reward = get_base_reward(state, index, epoch) + base_reward = get_base_reward(state, index, previous_epoch) # Micro-incentives for attestations matching FFG source, FFG target, and head for attestations in ( - get_matching_source_attestations(state, epoch), - get_matching_target_attestations(state, epoch), - get_matching_source_attestations(state, epoch), + get_matching_source_attestations(state, previous_epoch), + get_matching_target_attestations(state, previous_epoch), + get_matching_source_attestations(state, previous_epoch), ): if index in get_unslashed_attesting_indices(state, attestations): - rewards[index] += base_reward * get_attesting_balance(state, attestations, epoch) // get_total_active_balance(state, epoch) + rewards[index] += base_reward * get_attesting_balance(state, attestations, previous_epoch) // total_balance else: penalties[index] += base_reward - if index in get_unslashed_attesting_indices(state, get_matching_source_attestations(state, epoch)): - earliest_attestation = get_earliest_attestation(state, get_matching_source_attestations(state, epoch), index) + if index in get_unslashed_attesting_indices(state, get_matching_source_attestations(state, previous_epoch)): + earliest_attestation = get_earliest_attestation(state, get_matching_source_attestations(state, previous_epoch), index) # Proposer micro-rewards proposer_index = get_beacon_proposer_index(state, earliest_attestation.inclusion_slot) rewards[proposer_index] += base_reward // PROPOSER_REWARD_QUOTIENT @@ -1652,11 +1652,11 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: rewards[index] += base_reward * MIN_ATTESTATION_INCLUSION_DELAY // inclusion_delay # Inactivity penalty - finality_delay = epoch - state.finalized_epoch + finality_delay = previous_epoch - state.finalized_epoch if finality_delay > MIN_EPOCHS_TO_INACTIVITY_PENALTY: penalties[index] += BASE_REWARDS_PER_EPOCH * base_reward - if index not in get_unslashed_attesting_indices(state, get_matching_target_attestations(state, epoch)): - penalties[index] += get_effective_balance(state, index, epoch) * finality_delay // INACTIVITY_PENALTY_QUOTIENT + if index not in get_unslashed_attesting_indices(state, get_matching_target_attestations(state, previous_epoch)): + penalties[index] += get_effective_balance(state, index, previous_epoch) * finality_delay // INACTIVITY_PENALTY_QUOTIENT return [rewards, penalties] ``` From 06f475a844e33d4ef13b1e4097c62b20b752bc88 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 20 Apr 2019 16:32:41 +1000 Subject: [PATCH 063/121] Fixes --- specs/core/0_beacon-chain.md | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index c1dd75fb0..b6b4bb13e 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -715,12 +715,12 @@ def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> List[Valid ```python def get_current_epoch_effective_balance(state: BeaconState, index: ValidatorIndex) -> Gwei: """ - Get validator effective balance for the next epoch + Get validator effective balance for the current epoch """ balance = min(state.balances[index], MAX_DEPOSIT_AMOUNT) validator = state.validator_registry[index] HALF_INCREMENT = HIGH_BALANCE_INCREMENT // 2 - if validator.effective_balance > balance or validator.effective_balance + 3 * HALF_INCREMENT < balance: + if state.slot == GENESIS_SLOT or (validator.effective_balance > balance or validator.effective_balance + 3 * HALF_INCREMENT < balance): return balance - balance % HIGH_BALANCE_INCREMENT return validator.effective_balance ``` @@ -1705,7 +1705,7 @@ def process_registry_updates(state: BeaconState) -> None: if validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH and validator.effective_balance >= MAX_DEPOSIT_AMOUNT: validator.activation_eligibility_epoch = get_current_epoch(state) - if is_active_validator(validator, get_current_epoch(state)) and validator.effective_balance < EJECTION_BALANCE: + if is_active_validator(validator, get_current_epoch(state)) and validator.effective_balance <= EJECTION_BALANCE: initiate_validator_exit(state, index) # Process activations @@ -1956,30 +1956,21 @@ For each `deposit` in `block.body.deposits`, run the following function: ```python def process_deposit(state: BeaconState, deposit: Deposit) -> None: """ - Process a deposit from Ethereum 1.0. - Used to add a validator or top up an existing validator's - balance by some ``deposit`` amount. - + Process an Eth1 deposit, registering a validator or increasing its balance. Note that this function mutates ``state``. """ # Deposits must be processed in order assert deposit.index == state.deposit_index + state.deposit_index += 1 # Verify the Merkle branch - merkle_branch_is_valid = verify_merkle_branch( + assert verify_merkle_branch( leaf=hash(serialize(deposit.data)), # 48 + 32 + 8 + 96 = 184 bytes serialization proof=deposit.proof, depth=DEPOSIT_CONTRACT_TREE_DEPTH, index=deposit.index, root=state.latest_eth1_data.deposit_root, ) - assert merkle_branch_is_valid - - # Increment the next deposit index we are expecting. Note that this - # needs to be done here because while the deposit contract will never - # create an invalid Merkle branch, it may admit an invalid deposit - # object, and we need to be able to skip over it - state.deposit_index += 1 validator_pubkeys = [v.pubkey for v in state.validator_registry] pubkey = deposit.data.pubkey @@ -1998,11 +1989,11 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: activation_epoch=FAR_FUTURE_EPOCH, exit_epoch=FAR_FUTURE_EPOCH, withdrawable_epoch=FAR_FUTURE_EPOCH, - effective_balance=amount - amount % HIGH_BALANCE_INCREMENT, ) state.validator_registry.append(validator) state.balances.append(amount) + validator.effective_balance = get_current_epoch_effective_balance(state, len(state.validator_registry) - 1) else: # Increase balance by deposit amount index = validator_pubkeys.index(pubkey) From e184f0b3fe3a660e9a31a37eed146e73aa65f040 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 20 Apr 2019 16:35:02 +1000 Subject: [PATCH 064/121] Fix --- specs/core/0_beacon-chain.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b6b4bb13e..be0fb5d5a 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1959,10 +1959,6 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: Process an Eth1 deposit, registering a validator or increasing its balance. Note that this function mutates ``state``. """ - # Deposits must be processed in order - assert deposit.index == state.deposit_index - state.deposit_index += 1 - # Verify the Merkle branch assert verify_merkle_branch( leaf=hash(serialize(deposit.data)), # 48 + 32 + 8 + 96 = 184 bytes serialization @@ -1972,6 +1968,10 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: root=state.latest_eth1_data.deposit_root, ) + # Deposits must be processed in order + assert deposit.index == state.deposit_index + state.deposit_index += 1 + validator_pubkeys = [v.pubkey for v in state.validator_registry] pubkey = deposit.data.pubkey amount = deposit.data.amount From 7642abf1143a3be3cd74d9532a13d41440ca7f31 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 20 Apr 2019 16:36:34 +1000 Subject: [PATCH 065/121] Fix| --- specs/core/0_beacon-chain.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index be0fb5d5a..a3a9aff9d 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1972,26 +1972,25 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: assert deposit.index == state.deposit_index state.deposit_index += 1 - validator_pubkeys = [v.pubkey for v in state.validator_registry] pubkey = deposit.data.pubkey amount = deposit.data.amount - + validator_pubkeys = [v.pubkey for v in state.validator_registry] if pubkey not in validator_pubkeys: # Verify the deposit signature (proof of possession) if not bls_verify(pubkey, signing_root(deposit.data), deposit.data.signature, get_domain(state, DOMAIN_DEPOSIT)): return # Add new validator - validator = Validator( + state.validator_registry.append(Validator( pubkey=pubkey, withdrawal_credentials=deposit.data.withdrawal_credentials, activation_eligibility_epoch=FAR_FUTURE_EPOCH, activation_epoch=FAR_FUTURE_EPOCH, exit_epoch=FAR_FUTURE_EPOCH, withdrawable_epoch=FAR_FUTURE_EPOCH, - ) + )) - state.validator_registry.append(validator) + # Add initial balance state.balances.append(amount) validator.effective_balance = get_current_epoch_effective_balance(state, len(state.validator_registry) - 1) else: From a2a737b7289ea4bcbd9f77ca723cb197f1387828 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sat, 20 Apr 2019 01:45:18 -0500 Subject: [PATCH 066/121] Signal non-final status of base reward and desired issuance goal --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 624413879..c0f03e4ed 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -241,7 +241,7 @@ These configurations are updated for releases, but may be out of sync during `de | `INACTIVITY_PENALTY_QUOTIENT` | `2**24` (= 16,777,216) | | `MIN_PENALTY_QUOTIENT` | `2**5` (= 32) | -* The `BASE_REWARD_QUOTIENT` parameter dictates the per-epoch reward. It corresponds to ~2.54% annual interest assuming 10 million participating ETH in every epoch. +* **The `BASE_REWARD_QUOTIENT` is NOT final. Once all other protocol details are finalized it will be adjusted, to target a theoretical maximum total issuance of `2**21` ETH per year if `2**27` ETH is validating (and therefore `2**20` per year if `2**25` ETH is validating, etc etc)** * The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**12 epochs` (~18 days) is the time it takes the inactivity penalty to reduce the balance of non-participating [validators](#dfn-validator) to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline [validators](#dfn-validator) after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)` so after `INVERSE_SQRT_E_DROP_TIME` epochs it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. ### Max operations per block From d6644edcc9e5f25962ada04f120fbe0d63698e3d Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 20 Apr 2019 17:12:40 +1000 Subject: [PATCH 067/121] Fix test --- specs/core/0_beacon-chain.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a3a9aff9d..32f24b2e3 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1981,14 +1981,15 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: return # Add new validator - state.validator_registry.append(Validator( + validator = Validator( pubkey=pubkey, withdrawal_credentials=deposit.data.withdrawal_credentials, activation_eligibility_epoch=FAR_FUTURE_EPOCH, activation_epoch=FAR_FUTURE_EPOCH, exit_epoch=FAR_FUTURE_EPOCH, withdrawable_epoch=FAR_FUTURE_EPOCH, - )) + ) + state.validator_registry.append(validator) # Add initial balance state.balances.append(amount) From 0da60ba90d09a60ea926ccd046b3edf267fc05f3 Mon Sep 17 00:00:00 2001 From: Justin Date: Mon, 22 Apr 2019 15:12:30 +1000 Subject: [PATCH 068/121] Fix activation queue bug Fix bug [flagged by @NIC619 and @hwwhww](https://github.com/ethereum/eth2.0-specs/pull/850#issuecomment-485275575) whereby the `activation_epoch` of validators dequeued since the finalized epoch was overwritten. Cosmetic changes: 1) Remove `activate_validator` (there is no overlap between genesis and non-genesis activations) 2) Improve comments related to activation queue --- specs/core/0_beacon-chain.md | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 0ae4c1160..ea2225ffd 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -94,7 +94,6 @@ - [`bls_verify_multiple`](#bls_verify_multiple) - [`bls_aggregate_pubkeys`](#bls_aggregate_pubkeys) - [Routines for updating validator status](#routines-for-updating-validator-status) - - [`activate_validator`](#activate_validator) - [`initiate_validator_exit`](#initiate_validator_exit) - [`slash_validator`](#slash_validator) - [Ethereum 1.0 deposit contract](#ethereum-10-deposit-contract) @@ -1205,22 +1204,6 @@ def get_churn_limit(state: BeaconState) -> int: Note: All functions in this section mutate `state`. -#### `activate_validator` - -```python -def activate_validator(state: BeaconState, index: ValidatorIndex) -> None: - """ - Activate the validator of the given ``index``. - Note that this function mutates ``state``. - """ - validator = state.validator_registry[index] - if state.slot == GENESIS_SLOT: - validator.activation_eligibility_epoch = GENESIS_EPOCH - validator.activation_epoch = GENESIS_EPOCH - else: - validator.activation_epoch = get_delayed_activation_exit_epoch(get_current_epoch(state)) -``` - #### `initiate_validator_exit` ```python @@ -1340,9 +1323,10 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], process_deposit(state, deposit) # Process genesis activations - for index in range(len(state.validator_registry)): + for index, validator in enumerate(state.validator_registry): if get_effective_balance(state, index) >= MAX_DEPOSIT_AMOUNT: - activate_validator(state, index) + validator.activation_eligibility_epoch = GENESIS_EPOCH + validator.activation_epoch = GENESIS_EPOCH genesis_active_index_root = hash_tree_root(get_active_validator_indices(state, GENESIS_EPOCH)) for index in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH): @@ -1745,14 +1729,16 @@ def process_registry_updates(state: BeaconState) -> None: if is_active_validator(validator, get_current_epoch(state)) and balance < EJECTION_BALANCE: initiate_validator_exit(state, index) - # Process activations + # Queue validators are eligible for activation and not dequeued prior to finalized epoch activation_queue = sorted([ index for index, validator in enumerate(state.validator_registry) if validator.activation_eligibility_epoch != FAR_FUTURE_EPOCH and validator.activation_epoch >= get_delayed_activation_exit_epoch(state.finalized_epoch) ], key=lambda index: state.validator_registry[index].activation_eligibility_epoch) + # Dequeued validators for activation up to churn limit (without resetting activation epoch) for index in activation_queue[:get_churn_limit(state)]: - activate_validator(state, index) + if validator.activation_epoch != FAR_FUTURE_EPOCH: + validator.activation_epoch = get_delayed_activation_exit_epoch(get_current_epoch(state)) ``` #### Slashings From dc275f024d04265deaad09fc28c1e7289db43573 Mon Sep 17 00:00:00 2001 From: Justin Date: Mon, 22 Apr 2019 15:16:34 +1000 Subject: [PATCH 069/121] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ea2225ffd..d04f12a6d 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1729,7 +1729,7 @@ def process_registry_updates(state: BeaconState) -> None: if is_active_validator(validator, get_current_epoch(state)) and balance < EJECTION_BALANCE: initiate_validator_exit(state, index) - # Queue validators are eligible for activation and not dequeued prior to finalized epoch + # Queue validators eligible for activation and not dequeued for activation prior to finalized epoch activation_queue = sorted([ index for index, validator in enumerate(state.validator_registry) if validator.activation_eligibility_epoch != FAR_FUTURE_EPOCH and From c123fb1b975ba2f2ba41e0637f70f5a693cdf2a3 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Mon, 22 Apr 2019 16:13:46 +1000 Subject: [PATCH 070/121] =?UTF-8?q?Single=20effective=20balance=20per=20re?= =?UTF-8?q?view=20by=20Vitalik=E2=80=94significant=20simplification?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- specs/core/0_beacon-chain.md | 106 +++++++++++++---------------------- 1 file changed, 40 insertions(+), 66 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 32f24b2e3..8739aead9 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -58,7 +58,6 @@ - [`is_active_validator`](#is_active_validator) - [`is_slashable_validator`](#is_slashable_validator) - [`get_active_validator_indices`](#get_active_validator_indices) - - [`get_current_epoch_effective_balance`](#get_current_epoch_effective_balance) - [`increase_balance`](#increase_balance) - [`decrease_balance`](#decrease_balance) - [`get_permuted_index`](#get_permuted_index) @@ -77,7 +76,6 @@ - [`get_attesting_indices`](#get_attesting_indices) - [`int_to_bytes1`, `int_to_bytes2`, ...](#int_to_bytes1-int_to_bytes2-) - [`bytes_to_int`](#bytes_to_int) - - [`get_effective_balance`](#get_effective_balance) - [`get_total_balance`](#get_total_balance) - [`get_domain`](#get_domain) - [`get_bitfield_bit`](#get_bitfield_bit) @@ -194,7 +192,7 @@ These configurations are updated for releases, but may be out of sync during `de | `MIN_DEPOSIT_AMOUNT` | `2**0 * 10**9` (= 1,000,000,000) | Gwei | | `MAX_DEPOSIT_AMOUNT` | `2**5 * 10**9` (= 32,000,000,000) | Gwei | | `EJECTION_BALANCE` | `2**4 * 10**9` (= 16,000,000,000) | Gwei | -| `HIGH_BALANCE_INCREMENT` | `2**0 * 10**9` (= 1,000,000,000) | Gwei | +| `EFFECTIVE_BALANCE_INCREMENT` | `2**0 * 10**9` (= 1,000,000,000) | Gwei | ### Initial values @@ -710,21 +708,6 @@ def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> List[Valid return [i for i, v in enumerate(state.validator_registry) if is_active_validator(v, epoch)] ``` -### `get_current_epoch_effective_balance` - -```python -def get_current_epoch_effective_balance(state: BeaconState, index: ValidatorIndex) -> Gwei: - """ - Get validator effective balance for the current epoch - """ - balance = min(state.balances[index], MAX_DEPOSIT_AMOUNT) - validator = state.validator_registry[index] - HALF_INCREMENT = HIGH_BALANCE_INCREMENT // 2 - if state.slot == GENESIS_SLOT or (validator.effective_balance > balance or validator.effective_balance + 3 * HALF_INCREMENT < balance): - return balance - balance % HIGH_BALANCE_INCREMENT - return validator.effective_balance -``` - ### `increase_balance` ```python @@ -944,7 +927,7 @@ def get_beacon_proposer_index(state: BeaconState, slot: Slot=None) -> ValidatorI while True: candidate = first_committee[(epoch + i) % len(first_committee)] random_byte = hash(generate_seed(state, epoch) + int_to_bytes8(i // 32))[i % 32] - if get_effective_balance(state, candidate, epoch) * 256 > MAX_DEPOSIT_AMOUNT * random_byte: + if state.validator_registry[candidate].effective_balance * 256 > MAX_DEPOSIT_AMOUNT * random_byte: return candidate i += 1 ``` @@ -992,24 +975,14 @@ def bytes_to_int(data: bytes) -> int: return int.from_bytes(data, 'little') ``` -### `get_effective_balance` - -```python -def get_effective_balance(state: BeaconState, index: ValidatorIndex, epoch: Epoch) -> Gwei: - """ - Return the effective balance (also known as "balance at stake") for a validator with the given ``index``. - """ - return get_current_epoch_effective_balance(state, index) if epoch == get_current_epoch(state) else state.validator_registry[index].effective_balance -``` - ### `get_total_balance` ```python -def get_total_balance(state: BeaconState, validators: List[ValidatorIndex], epoch: Epoch) -> Gwei: +def get_total_balance(state: BeaconState, indices: List[ValidatorIndex]) -> Gwei: """ Return the combined effective balance of an array of ``validators``. """ - return sum([get_effective_balance(state, index, epoch) for index in validators]) + return sum([state.validator_registry[index].effective_balance for index in indices]) ``` ### `get_domain` @@ -1244,9 +1217,10 @@ def slash_validator(state: BeaconState, slashed_index: ValidatorIndex, whistlebl """ current_epoch = get_current_epoch(state) initiate_validator_exit(state, slashed_index) - state.validator_registry[slashed_index].slashed = True - state.validator_registry[slashed_index].withdrawable_epoch = current_epoch + LATEST_SLASHED_EXIT_LENGTH - slashed_balance = get_effective_balance(state, slashed_index, current_epoch) + slashed_validator = state.validator_registry[slashed_index] + slashed_validator.slashed = True + slashed_validator.withdrawable_epoch = current_epoch + LATEST_SLASHED_EXIT_LENGTH + slashed_balance = slashed_validator.effective_balance state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH] += slashed_balance proposer_index = get_beacon_proposer_index(state) @@ -1463,8 +1437,8 @@ The steps below happen when `state.slot > GENESIS_SLOT and (state.slot + 1) % SL We define epoch transition helper functions: ```python -def get_total_active_balance(state: BeaconState, epoch: Epoch) -> Gwei: - return get_total_balance(state, get_active_validator_indices(state, epoch), epoch) +def get_total_active_balance(state: BeaconState) -> Gwei: + return get_total_balance(state, get_active_validator_indices(state, get_current_epoch(state))) ``` ```python @@ -1497,8 +1471,8 @@ def get_unslashed_attesting_indices(state: BeaconState, attestations: List[Pendi ``` ```python -def get_attesting_balance(state: BeaconState, attestations: List[PendingAttestation], epoch: Epoch) -> Gwei: - return get_total_balance(state, get_unslashed_attesting_indices(state, attestations), epoch) +def get_attesting_balance(state: BeaconState, attestations: List[PendingAttestation]) -> Gwei: + return get_total_balance(state, get_unslashed_attesting_indices(state, attestations)) ``` ```python @@ -1526,7 +1500,7 @@ def get_winning_crosslink_and_attesting_indices(state: BeaconState, epoch: Epoch return [a for a in shard_attestations if get_crosslink_from_attestation_data(state, a.data) == crosslink] # Winning crosslink has the crosslink data root with the most balance voting for it (ties broken lexicographically) winning_crosslink = max(candidate_crosslinks, key=lambda crosslink: ( - get_attesting_balance(state, get_attestations_for(crosslink), epoch), crosslink.crosslink_data_root + get_attesting_balance(state, get_attestations_for(crosslink)), crosslink.crosslink_data_root )) return winning_crosslink, get_unslashed_attesting_indices(state, get_attestations_for(winning_crosslink)) @@ -1557,13 +1531,13 @@ def process_justification_and_finalization(state: BeaconState) -> None: state.previous_justified_epoch = state.current_justified_epoch state.previous_justified_root = state.current_justified_root state.justification_bitfield = (state.justification_bitfield << 1) % 2**64 - previous_epoch_matching_target_balance = get_attesting_balance(state, get_matching_target_attestations(state, previous_epoch), previous_epoch) - if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state, previous_epoch) * 2: + previous_epoch_matching_target_balance = get_attesting_balance(state, get_matching_target_attestations(state, previous_epoch)) + if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_epoch = get_previous_epoch(state) state.current_justified_root = get_block_root(state, get_epoch_start_slot(state.current_justified_epoch)) state.justification_bitfield |= (1 << 1) - current_epoch_matching_target_balance = get_attesting_balance(state, get_matching_target_attestations(state, current_epoch), current_epoch) - if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state, current_epoch) * 2: + current_epoch_matching_target_balance = get_attesting_balance(state, get_matching_target_attestations(state, current_epoch)) + if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_epoch = get_current_epoch(state) state.current_justified_root = get_block_root(state, get_epoch_start_slot(state.current_justified_epoch)) state.justification_bitfield |= (1 << 0) @@ -1601,7 +1575,7 @@ def process_crosslinks(state: BeaconState) -> None: epoch = slot_to_epoch(slot) for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot): winning_crosslink, attesting_indices = get_winning_crosslink_and_attesting_indices(state, epoch, shard) - if 3 * get_total_balance(state, attesting_indices, epoch) >= 2 * get_total_balance(state, crosslink_committee, epoch): + if 3 * get_total_balance(state, attesting_indices) >= 2 * get_total_balance(state, crosslink_committee): state.current_crosslinks[shard] = winning_crosslink ``` @@ -1610,18 +1584,18 @@ def process_crosslinks(state: BeaconState) -> None: First, we define additional helpers: ```python -def get_base_reward(state: BeaconState, index: ValidatorIndex, epoch: Epoch) -> Gwei: - adjusted_quotient = integer_squareroot(get_total_active_balance(state, epoch)) // BASE_REWARD_QUOTIENT +def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: + adjusted_quotient = integer_squareroot(get_total_active_balance(state)) // BASE_REWARD_QUOTIENT if adjusted_quotient == 0: return 0 - return get_effective_balance(state, index, epoch) // adjusted_quotient // BASE_REWARDS_PER_EPOCH + return state.validator_registry[index].effective_balance // adjusted_quotient // BASE_REWARDS_PER_EPOCH ``` ```python def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: previous_epoch = get_previous_epoch(state) - total_balance = get_total_active_balance(state, previous_epoch) + total_balance = get_total_active_balance(state) eligible_validator_indices = [ index for index, validator in enumerate(state.validator_registry) if is_active_validator(validator, previous_epoch) or (validator.slashed and previous_epoch < validator.withdrawable_epoch) @@ -1629,7 +1603,7 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: rewards = [0 for index in range(len(state.validator_registry))] penalties = [0 for index in range(len(state.validator_registry))] for index in eligible_validator_indices: - base_reward = get_base_reward(state, index, previous_epoch) + base_reward = get_base_reward(state, index) # Micro-incentives for attestations matching FFG source, FFG target, and head for attestations in ( @@ -1638,7 +1612,7 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: get_matching_source_attestations(state, previous_epoch), ): if index in get_unslashed_attesting_indices(state, attestations): - rewards[index] += base_reward * get_attesting_balance(state, attestations, previous_epoch) // total_balance + rewards[index] += base_reward * get_attesting_balance(state, attestations) // total_balance else: penalties[index] += base_reward @@ -1656,7 +1630,7 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: if finality_delay > MIN_EPOCHS_TO_INACTIVITY_PENALTY: penalties[index] += BASE_REWARDS_PER_EPOCH * base_reward if index not in get_unslashed_attesting_indices(state, get_matching_target_attestations(state, previous_epoch)): - penalties[index] += get_effective_balance(state, index, previous_epoch) * finality_delay // INACTIVITY_PENALTY_QUOTIENT + penalties[index] += state.validator_registry[index].effective_balance * finality_delay // INACTIVITY_PENALTY_QUOTIENT return [rewards, penalties] ``` @@ -1669,10 +1643,10 @@ def get_crosslink_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: epoch = slot_to_epoch(slot) for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot): winning_crosslink, attesting_indices = get_winning_crosslink_and_attesting_indices(state, epoch, shard) - attesting_balance = get_total_balance(state, attesting_indices, epoch) - committee_balance = get_total_balance(state, crosslink_committee, epoch) + attesting_balance = get_total_balance(state, attesting_indices) + committee_balance = get_total_balance(state, crosslink_committee) for index in crosslink_committee: - base_reward = get_base_reward(state, index, epoch) + base_reward = get_base_reward(state, index) if index in attesting_indices: rewards[index] += base_reward * attesting_balance // committee_balance else: @@ -1726,7 +1700,7 @@ Run the following function: def process_slashings(state: BeaconState) -> None: current_epoch = get_current_epoch(state) active_validator_indices = get_active_validator_indices(state, current_epoch) - total_balance = get_total_balance(state, active_validator_indices, current_epoch) + total_balance = get_total_balance(state, active_validator_indices) # Compute `total_penalties` total_at_start = state.latest_slashed_balances[(current_epoch + 1) % LATEST_SLASHED_EXIT_LENGTH] @@ -1736,8 +1710,8 @@ def process_slashings(state: BeaconState) -> None: for index, validator in enumerate(state.validator_registry): if validator.slashed and current_epoch == validator.withdrawable_epoch - LATEST_SLASHED_EXIT_LENGTH // 2: penalty = max( - get_effective_balance(state, index, current_epoch) * min(total_penalties * 3, total_balance) // total_balance, - get_effective_balance(state, index, current_epoch) // MIN_PENALTY_QUOTIENT + validator.effective_balance * min(total_penalties * 3, total_balance) // total_balance, + validator.effective_balance // MIN_PENALTY_QUOTIENT ) decrease_balance(state, index, penalty) ``` @@ -1753,9 +1727,12 @@ def process_final_updates(state: BeaconState) -> None: # Reset eth1 data votes if state.slot % SLOTS_PER_ETH1_VOTING_PERIOD == 0: state.eth1_data_votes = [] - # Update effective balances + # Update effective balances with hysteresis for index, validator in enumerate(state.validator_registry): - validator.effective_balance = get_current_epoch_effective_balance(state, index) + balance = min(state.balances[index], MAX_DEPOSIT_AMOUNT) + HALF_INCREMENT = EFFECTIVE_BALANCE_INCREMENT // 2 + if balance < validator.effective_balance or validator.effective_balance + 3 * HALF_INCREMENT < balance: + validator.effective_balance = balance - balance % EFFECTIVE_BALANCE_INCREMENT # Update start shard state.latest_start_shard = (state.latest_start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT # Set active index root @@ -1980,20 +1957,17 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: if not bls_verify(pubkey, signing_root(deposit.data), deposit.data.signature, get_domain(state, DOMAIN_DEPOSIT)): return - # Add new validator - validator = Validator( + # Add validator and balance entries + state.validator_registry.append(Validator( pubkey=pubkey, withdrawal_credentials=deposit.data.withdrawal_credentials, activation_eligibility_epoch=FAR_FUTURE_EPOCH, activation_epoch=FAR_FUTURE_EPOCH, exit_epoch=FAR_FUTURE_EPOCH, withdrawable_epoch=FAR_FUTURE_EPOCH, - ) - state.validator_registry.append(validator) - - # Add initial balance + effective_balance=amount - amount % EFFECTIVE_BALANCE_INCREMENT + )) state.balances.append(amount) - validator.effective_balance = get_current_epoch_effective_balance(state, len(state.validator_registry) - 1) else: # Increase balance by deposit amount index = validator_pubkeys.index(pubkey) From 6903f2eec738a3f486e9704e96ed62708271c0e1 Mon Sep 17 00:00:00 2001 From: Justin Date: Mon, 22 Apr 2019 16:17:14 +1000 Subject: [PATCH 071/121] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8739aead9..0adee0f95 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1965,7 +1965,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: activation_epoch=FAR_FUTURE_EPOCH, exit_epoch=FAR_FUTURE_EPOCH, withdrawable_epoch=FAR_FUTURE_EPOCH, - effective_balance=amount - amount % EFFECTIVE_BALANCE_INCREMENT + effective_balance=amount - amount % EFFECTIVE_BALANCE_INCREMENT, )) state.balances.append(amount) else: From 81ee59bca85350fd479a7833045b857284c56834 Mon Sep 17 00:00:00 2001 From: Justin Date: Mon, 22 Apr 2019 16:34:50 +1000 Subject: [PATCH 072/121] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 0adee0f95..931511b36 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -238,7 +238,7 @@ These configurations are updated for releases, but may be out of sync during `de | `WHISTLEBLOWING_REWARD_QUOTIENT` | `2**9` (= 512) | | `PROPOSER_REWARD_QUOTIENT` | `2**3` (= 8) | | `INACTIVITY_PENALTY_QUOTIENT` | `2**25` (= 33,554,432) | -| `MIN_PENALTY_QUOTIENT` | `2**5` (= 32) | +| `MIN_SLASHING_PENALTY_QUOTIENT` | `2**5` (= 32) | * The `BASE_REWARD_QUOTIENT` parameter dictates the per-epoch reward. It corresponds to ~2.54% annual interest assuming 10 million participating ETH in every epoch. * The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**12 epochs` (~18 days) is the time it takes the inactivity penalty to reduce the balance of non-participating [validators](#dfn-validator) to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline [validators](#dfn-validator) after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)` so after `INVERSE_SQRT_E_DROP_TIME` epochs it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. @@ -401,7 +401,7 @@ The types are defined topologically to aid in facilitating an executable version 'withdrawable_epoch': 'uint64', # Was the validator slashed 'slashed': 'bool', - # Rounded balance + # Effective balance 'effective_balance': 'uint64', } ``` @@ -725,7 +725,7 @@ def decrease_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> """ Decrease validator balance by ``delta`` with underflow protection. """ - state.balances[index] = state.balances[index] - delta if state.balances[index] >= delta else 0 + state.balances[index] = 0 if delta > state.balances[index] else state.balances[index] - delta ``` ### `get_permuted_index` @@ -925,10 +925,12 @@ def get_beacon_proposer_index(state: BeaconState, slot: Slot=None) -> ValidatorI first_committee, _ = get_crosslink_committees_at_slot(state, slot if slot != None else state.slot)[0] i = 0 while True: - candidate = first_committee[(epoch + i) % len(first_committee)] + candidate_index = first_committee[(epoch + i) % len(first_committee)] random_byte = hash(generate_seed(state, epoch) + int_to_bytes8(i // 32))[i % 32] - if state.validator_registry[candidate].effective_balance * 256 > MAX_DEPOSIT_AMOUNT * random_byte: - return candidate + MAX_RANDOM_BYTE = 2**8 - 1 + effective_balance = state.validator_registry[candidate_index].effective_balance + if effective_balance * MAX_RANDOM_BYTE >= MAX_DEPOSIT_AMOUNT * random_byte: + return candidate_index i += 1 ``` @@ -1217,10 +1219,9 @@ def slash_validator(state: BeaconState, slashed_index: ValidatorIndex, whistlebl """ current_epoch = get_current_epoch(state) initiate_validator_exit(state, slashed_index) - slashed_validator = state.validator_registry[slashed_index] - slashed_validator.slashed = True - slashed_validator.withdrawable_epoch = current_epoch + LATEST_SLASHED_EXIT_LENGTH - slashed_balance = slashed_validator.effective_balance + state.validator_registry[slashed_index].slashed = True + state.validator_registry[slashed_index].withdrawable_epoch = current_epoch + LATEST_SLASHED_EXIT_LENGTH + slashed_balance = state.validator_registry[slashed_index].effective_balance state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH] += slashed_balance proposer_index = get_beacon_proposer_index(state) @@ -1371,9 +1372,7 @@ def lmd_ghost(store: Store, start_state: BeaconState, start_block: BeaconBlock) active_validator_indices = get_active_validator_indices(validators, slot_to_epoch(start_state.slot)) attestation_targets = [(i, get_latest_attestation_target(store, i)) for i in active_validator_indices] - # Use the rounded-balance-with-hysteresis supplied by the protocol for fork - # choice voting. This reduces the number of recomputations that need to be - # made for optimized implementations that precompute and save data + # Use the effective balance for fork choice voting to reduce recomputations and save bandwidth def get_vote_count(block: BeaconBlock) -> int: return sum( start_state.validator_registry[validator_index].effective_balance @@ -1711,7 +1710,7 @@ def process_slashings(state: BeaconState) -> None: if validator.slashed and current_epoch == validator.withdrawable_epoch - LATEST_SLASHED_EXIT_LENGTH // 2: penalty = max( validator.effective_balance * min(total_penalties * 3, total_balance) // total_balance, - validator.effective_balance // MIN_PENALTY_QUOTIENT + validator.effective_balance // MIN_SLASHING_PENALTY_QUOTIENT ) decrease_balance(state, index, penalty) ``` @@ -1965,7 +1964,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: activation_epoch=FAR_FUTURE_EPOCH, exit_epoch=FAR_FUTURE_EPOCH, withdrawable_epoch=FAR_FUTURE_EPOCH, - effective_balance=amount - amount % EFFECTIVE_BALANCE_INCREMENT, + effective_balance=amount - amount % EFFECTIVE_BALANCE_INCREMENT )) state.balances.append(amount) else: From 1c5cc1299a7c94e8f9dc123ef103c4d84c8971ff Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 22 Apr 2019 20:49:07 +1000 Subject: [PATCH 073/121] Update specs/core/0_beacon-chain.md Co-Authored-By: JustinDrake --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index d04f12a6d..b2796a588 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1737,7 +1737,7 @@ def process_registry_updates(state: BeaconState) -> None: ], key=lambda index: state.validator_registry[index].activation_eligibility_epoch) # Dequeued validators for activation up to churn limit (without resetting activation epoch) for index in activation_queue[:get_churn_limit(state)]: - if validator.activation_epoch != FAR_FUTURE_EPOCH: + if validator.activation_epoch == FAR_FUTURE_EPOCH: validator.activation_epoch = get_delayed_activation_exit_epoch(get_current_epoch(state)) ``` From edb24ce9dda0c54c5cc86520f7ef99091ce7af10 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 22 Apr 2019 09:00:01 -0600 Subject: [PATCH 074/121] test rule 3 --- specs/core/0_beacon-chain.md | 4 + test_libs/pyspec/tests/test_finality.py | 158 +++++++++++++++++++----- 2 files changed, 132 insertions(+), 30 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 4cdfa9c4f..1df72179d 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1682,12 +1682,16 @@ def process_justification_and_finalization(state: BeaconState) -> None: state.previous_justified_root = state.current_justified_root state.justification_bitfield = (state.justification_bitfield << 1) % 2**64 previous_boundary_attesting_balance = get_attesting_balance(state, get_previous_epoch_boundary_attestations(state)) + print(previous_boundary_attesting_balance) if previous_boundary_attesting_balance * 3 >= get_previous_total_balance(state) * 2: + print("prev success") state.current_justified_epoch = get_previous_epoch(state) state.current_justified_root = get_block_root(state, get_epoch_start_slot(state.current_justified_epoch)) state.justification_bitfield |= (1 << 1) current_boundary_attesting_balance = get_attesting_balance(state, get_current_epoch_boundary_attestations(state)) + print(current_boundary_attesting_balance) if current_boundary_attesting_balance * 3 >= get_current_total_balance(state) * 2: + print("cur success") state.current_justified_epoch = get_current_epoch(state) state.current_justified_root = get_block_root(state, get_epoch_start_slot(state.current_justified_epoch)) state.justification_bitfield |= (1 << 0) diff --git a/test_libs/pyspec/tests/test_finality.py b/test_libs/pyspec/tests/test_finality.py index 8a429cb6e..ae8ba13ff 100644 --- a/test_libs/pyspec/tests/test_finality.py +++ b/test_libs/pyspec/tests/test_finality.py @@ -47,11 +47,50 @@ def check_finality(state, assert state.finalized_root == prev_state.finalized_root +def next_epoch_with_attestations(state, + fill_cur_epoch, + fill_prev_epoch): + post_state = deepcopy(state) + blocks = [] + for slot in range(spec.SLOTS_PER_EPOCH): + print("slot: %s", post_state.slot) + block = build_empty_block_for_next_slot(post_state) + if fill_prev_epoch: + print("prev") + slot_to_attest = post_state.slot - spec.SLOTS_PER_EPOCH + 1 + prev_attestation = get_valid_attestation(post_state, slot_to_attest) + block.body.attestations.append(prev_attestation) + + if fill_cur_epoch: + print("cur") + slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 + if slot_to_attest >= get_epoch_start_slot(get_current_epoch(post_state)): + cur_attestation = get_valid_attestation(post_state, slot_to_attest) + fill_aggregate_attestation(post_state, cur_attestation) + block.body.attestations.append(cur_attestation) + + state_transition(post_state, block) + blocks.append(block) + + # if fill_prev_epoch: + # assert len(post_state.previous_epoch_attestations) >= 0 + # else: + # assert len(post_state.previous_epoch_attestations) == 0 + + # if fill_cur_epoch: + # assert len(post_state.current_epoch_attestations) >= 0 + # else: + # assert len(post_state.current_epoch_attestations) == 0 + + return state, blocks, post_state + + def test_finality_from_genesis_rule_4(state): test_state = deepcopy(state) blocks = [] for epoch in range(6): + prev_state = deepcopy(test_state) old_current_justified_epoch = test_state.current_justified_epoch old_current_justified_root = test_state.current_justified_root for slot in range(spec.SLOTS_PER_EPOCH): @@ -67,14 +106,14 @@ def test_finality_from_genesis_rule_4(state): blocks.append(block) if epoch == 0: - check_finality(test_state, state, False, False, False) + check_finality(test_state, prev_state, False, False, False) elif epoch == 1: - check_finality(test_state, state, False, False, False) + check_finality(test_state, prev_state, False, False, False) elif epoch == 2: - check_finality(test_state, state, True, False, False) + check_finality(test_state, prev_state, True, False, False) elif epoch >= 3: # rule 4 of finaliy - check_finality(test_state, state, True, True, True) + check_finality(test_state, prev_state, True, True, True) assert test_state.finalized_epoch == old_current_justified_epoch assert test_state.finalized_root == old_current_justified_root @@ -90,6 +129,7 @@ def test_finality_rule_1(state): blocks = [] for epoch in range(3): + prev_state = deepcopy(test_state) old_previous_justified_epoch = test_state.previous_justified_epoch old_previous_justified_root = test_state.previous_justified_root for slot in range(spec.SLOTS_PER_EPOCH): @@ -106,15 +146,17 @@ def test_finality_rule_1(state): blocks.append(block) if epoch == 0: - check_finality(test_state, state, True, False, False) + check_finality(test_state, prev_state, True, False, False) elif epoch == 1: - check_finality(test_state, state, True, True, False) + check_finality(test_state, prev_state, True, True, False) elif epoch == 2: # finalized by rule 1 - check_finality(test_state, state, True, True, True) + check_finality(test_state, prev_state, True, True, True) assert test_state.finalized_epoch == old_previous_justified_epoch assert test_state.finalized_root == old_previous_justified_root + return state, blocks, test_state + def test_finality_rule_2(state): # get past first two epochs that finality does not run on @@ -127,30 +169,86 @@ def test_finality_rule_2(state): for epoch in range(3): old_previous_justified_epoch = test_state.previous_justified_epoch old_previous_justified_root = test_state.previous_justified_root - for slot in range(spec.SLOTS_PER_EPOCH): - attestation = None - if epoch == 0: - slot_to_attest = test_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 - if slot_to_attest >= get_epoch_start_slot(get_current_epoch(state)): - attestation = get_valid_attestation(test_state, slot_to_attest) - fill_aggregate_attestation(test_state, attestation) - if epoch == 2: - slot_to_attest = test_state.slot - spec.SLOTS_PER_EPOCH + 1 - attestation = get_valid_attestation(test_state, slot_to_attest) - fill_aggregate_attestation(test_state, attestation) - - block = build_empty_block_for_next_slot(test_state) - if attestation: - block.body.attestations.append(attestation) - state_transition(test_state, block) - blocks.append(block) - if epoch == 0: - check_finality(test_state, state, True, False, False) - elif epoch == 1: - check_finality(test_state, state, True, True, False) - elif epoch == 2: + prev_state, blocks, test_state = next_epoch_with_attestations(test_state, True, False) + check_finality(test_state, prev_state, True, False, False) + if epoch == 1: + prev_state, blocks, test_state = next_epoch_with_attestations(test_state, False, False) + check_finality(test_state, prev_state, False, True, False) + if epoch == 2: + prev_state, blocks, test_state = next_epoch_with_attestations(test_state, False, True) # finalized by rule 2 - check_finality(test_state, state, True, True, True) + check_finality(test_state, prev_state, True, False, True) assert test_state.finalized_epoch == old_previous_justified_epoch assert test_state.finalized_root == old_previous_justified_root + return state, blocks, test_state + + +def test_finality_rule_3(state): + # get past first two epochs that finality does not run on + next_epoch(state) + next_epoch(state) + + test_state = deepcopy(state) + + blocks = [] + for epoch in range(2): + prev_state = deepcopy(test_state) + for slot in range(spec.SLOTS_PER_EPOCH): + slot_to_attest = test_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 + attestation = get_valid_attestation(test_state, slot_to_attest) + fill_aggregate_attestation(test_state, attestation) + block = build_empty_block_for_next_slot(test_state) + block.body.attestations.append(attestation) + state_transition(test_state, block) + + blocks.append(block) + if epoch == 0: + check_finality(test_state, prev_state, True, False, False) + if epoch == 1: + check_finality(test_state, prev_state, True, True, True) + + prev_state = deepcopy(test_state) + next_epoch(test_state) + check_finality(test_state, prev_state, False, True, False) + + + prev_state = deepcopy(test_state) + for slot in range(spec.SLOTS_PER_EPOCH): + slot_to_attest = test_state.slot - spec.SLOTS_PER_EPOCH + 1 + attestation = get_valid_attestation(test_state, slot_to_attest) + fill_aggregate_attestation(test_state, attestation) + block = build_empty_block_for_next_slot(test_state) + block.body.attestations.append(attestation) + state_transition(test_state, block) + + assert len(test_state.previous_epoch_attestations) >= 0 + assert len(test_state.current_epoch_attestations) == 0 + + blocks.append(block) + check_finality(test_state, prev_state, True, False, True) + + + prev_state = deepcopy(test_state) + for slot in range(spec.SLOTS_PER_EPOCH): + prev_slot_to_attest = test_state.slot - spec.SLOTS_PER_EPOCH + 1 + prev_attestation = get_valid_attestation(test_state, prev_slot_to_attest) + fill_aggregate_attestation(test_state, prev_attestation) + + cur_slot_to_attest = test_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 + cur_attestation = get_valid_attestation(test_state, cur_slot_to_attest) + fill_aggregate_attestation(test_state, cur_attestation) + + block = build_empty_block_for_next_slot(test_state) + block.body.attestations.append(prev_attestation) + block.body.attestations.append(cur_attestation) + + state_transition(test_state, block) + + assert len(test_state.previous_epoch_attestations) >= 0 + assert len(test_state.current_epoch_attestations) >= 0 + + blocks.append(block) + check_finality(test_state, prev_state, True, True, True) + + return state, blocks, test_state From 5744fef808e5668ad616ca64685d1c85a5e598b3 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 22 Apr 2019 09:18:20 -0600 Subject: [PATCH 075/121] clean up some notes on deposits --- specs/core/0_beacon-chain.md | 2 +- specs/validator/0_beacon-chain-validator.md | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 20d7f5d43..615c66048 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1345,7 +1345,7 @@ For convenience, we provide the interface to the contract here: * `__init__()`: initializes the contract * `get_deposit_root() -> bytes32`: returns the current root of the deposit tree -* `deposit(bytes[512])`: adds a deposit instance to the deposit tree, incorporating the input argument and the value transferred in the given call. Note: the amount of value transferred *must* be within `MIN_DEPOSIT_AMOUNT` and `MAX_DEPOSIT_AMOUNT`, inclusive. Each of these constants are specified in units of Gwei. +* `def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96])`: adds a deposit instance to the deposit tree, incorporating the input arguments and the value transferred in the given call. Note: the amount of value transferred *must* be within `MIN_DEPOSIT_AMOUNT` and `MAX_DEPOSIT_AMOUNT`, inclusive. Each of these constants are specified in units of Gwei. ## On genesis diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 632bf2b62..966e238ab 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -101,11 +101,10 @@ In phase 0, all incoming validator deposits originate from the Ethereum 1.0 PoW To submit a deposit: * Pack the validator's [initialization parameters](#initialization) into `deposit_data`, a [`DepositData`](../core/0_beacon-chain.md#depositdata) SSZ object. -* Let `proof_of_possession` be the result of `bls_sign` of the `signing_root(deposit_data)` with `domain=DOMAIN_DEPOSIT`. -* Set `deposit_data.proof_of_possession = proof_of_possession`. * Let `amount` be the amount in Gwei to be deposited by the validator where `MIN_DEPOSIT_AMOUNT <= amount <= MAX_DEPOSIT_AMOUNT`. * Set `deposit_data.amount = amount`. -* Send a transaction on the Ethereum 1.0 chain to `DEPOSIT_CONTRACT_ADDRESS` executing `deposit(deposit_input: bytes[512])` along with `serialize(deposit_data)` as the singular `bytes` input along with a deposit of `amount` Gwei. +* Let `signature` be the result of `bls_sign` of the `signing_root(deposit_data)` with `domain=DOMAIN_DEPOSIT`. +* Send a transaction on the Ethereum 1.0 chain to `DEPOSIT_CONTRACT_ADDRESS` executing `def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96])` along with a deposit of `amount` Gwei. _Note_: Deposits made for the same `pubkey` are treated as for the same validator. A singular `Validator` will be added to `state.validator_registry` with each additional deposit amount added to the validator's balance. A validator can only be activated when total deposits for the validator pubkey meet or exceed `MAX_DEPOSIT_AMOUNT`. From d648b091b5bd35fce9653ad0e2167edeeb81d45a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 22 Apr 2019 09:33:46 -0600 Subject: [PATCH 076/121] lint --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 615c66048..0b2ccf028 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1345,7 +1345,7 @@ For convenience, we provide the interface to the contract here: * `__init__()`: initializes the contract * `get_deposit_root() -> bytes32`: returns the current root of the deposit tree -* `def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96])`: adds a deposit instance to the deposit tree, incorporating the input arguments and the value transferred in the given call. Note: the amount of value transferred *must* be within `MIN_DEPOSIT_AMOUNT` and `MAX_DEPOSIT_AMOUNT`, inclusive. Each of these constants are specified in units of Gwei. +* `deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96])`: adds a deposit instance to the deposit tree, incorporating the input arguments and the value transferred in the given call. Note: the amount of value transferred *must* be within `MIN_DEPOSIT_AMOUNT` and `MAX_DEPOSIT_AMOUNT`, inclusive. Each of these constants are specified in units of Gwei. ## On genesis From e13cec146661f8f8a47e2de03efb95ecceaeb01f Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 22 Apr 2019 10:02:31 -0600 Subject: [PATCH 077/121] increase MAX_TRANSFERS for transfer test --- test_libs/pyspec/tests/test_sanity.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index 3442e8182..3201ba936 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -380,7 +380,10 @@ def test_voluntary_exit(state): return pre_state, [initiate_exit_block, exit_block], post_state -def test_transfer(state): +def test_transfer(state, config): + # overwrite default 0 to test + spec.MAX_TRANSFERS = 1 + pre_state = deepcopy(state) current_epoch = get_current_epoch(pre_state) sender_index = get_active_validator_indices(pre_state, current_epoch)[-1] From d4a33dbcaa093588c9cca1479bb1487ccd630c63 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 22 Apr 2019 15:29:47 -0600 Subject: [PATCH 078/121] add descriptions of typeof and default functions --- specs/core/1_custody-game.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 88341ae98..c8625af5b 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -28,6 +28,8 @@ - [`BeaconState`](#beaconstate) - [`BeaconBlockBody`](#beaconblockbody) - [Helpers](#helpers) + - [`typeof`](#typeof) + - [`empty`](#empty) - [`get_crosslink_chunk_count`](#get_crosslink_chunk_count) - [`get_custody_chunk_bit`](#get_custody_chunk_bit) - [`epoch_to_custody_period`](#epoch_to_custody_period) @@ -204,6 +206,14 @@ Add the following fields to the end of the specified container objects. Fields w ## Helpers +### `typeof` + +The `typeof` function accepts and SSZ object as a single input and returns the corresponding SSZ type. + +### `empty` + +The `empty` function accepts and SSZ type as input and returns an object of that type with all fields initialized to default values. + ### `get_crosslink_chunk_count` ```python From 2650a2c0616661f8a5bc2e018df5ac34cd2d3027 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Tue, 23 Apr 2019 07:16:52 -0700 Subject: [PATCH 079/121] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 6d4f396ae..ccab159f4 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -797,7 +797,7 @@ def get_split_offset(list_size: int, chunks: int, index: int) -> int: ```python def get_epoch_committee_count(state: BeaconState, epoch: Epoch) -> int: """ - Return the number of committees in one epoch. + Return the number of committees at ``epoch``. """ active_validators = get_active_validator_indices(state, epoch) return max( @@ -813,6 +813,9 @@ def get_epoch_committee_count(state: BeaconState, epoch: Epoch) -> int: ```python def get_shard_delta(state: BeaconState, epoch: Epoch) -> int: + """ + Return the minimum number of shards that get processed at ``epoch``. + """ return min(get_epoch_committee_count(state, epoch), SHARD_COUNT - SHARD_COUNT // SLOTS_PER_EPOCH) ``` From 5619e7df9c04cfa74bdcd60cb5a43fb8215a5d64 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 23 Apr 2019 09:21:30 -0600 Subject: [PATCH 080/121] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ccab159f4..ed2b1403a 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -814,7 +814,7 @@ def get_epoch_committee_count(state: BeaconState, epoch: Epoch) -> int: ```python def get_shard_delta(state: BeaconState, epoch: Epoch) -> int: """ - Return the minimum number of shards that get processed at ``epoch``. + Return the number of shards to increment ``state.shard_shard`` during ``epoch``. """ return min(get_epoch_committee_count(state, epoch), SHARD_COUNT - SHARD_COUNT // SLOTS_PER_EPOCH) ``` From e26112af37efba0c837a986d06c3a35afb3a54ed Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 23 Apr 2019 08:36:40 -0700 Subject: [PATCH 081/121] Update 0_beacon-chain.md typo fix --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ed2b1403a..184411925 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -814,7 +814,7 @@ def get_epoch_committee_count(state: BeaconState, epoch: Epoch) -> int: ```python def get_shard_delta(state: BeaconState, epoch: Epoch) -> int: """ - Return the number of shards to increment ``state.shard_shard`` during ``epoch``. + Return the number of shards to increment ``state.latest_start_shard`` during ``epoch``. """ return min(get_epoch_committee_count(state, epoch), SHARD_COUNT - SHARD_COUNT // SLOTS_PER_EPOCH) ``` From 5e4afc2dd0cb34af647bad88620395c04e5bbaab Mon Sep 17 00:00:00 2001 From: JSON <49416440+JSON@users.noreply.github.com> Date: Tue, 23 Apr 2019 12:49:59 -0500 Subject: [PATCH 082/121] Update rpc-interface.md --- specs/networking/rpc-interface.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/networking/rpc-interface.md b/specs/networking/rpc-interface.md index 5d408b5a0..f1da8f7e3 100644 --- a/specs/networking/rpc-interface.md +++ b/specs/networking/rpc-interface.md @@ -247,7 +247,7 @@ Requests a list of block roots and slots from the peer. The `count` parameter MU Requests beacon block headers from the peer starting from `(start_root, start_slot)`. The response MUST contain no more than `max_headers` headers. `skip_slots` defines the maximum number of slots to skip between blocks. For example, requesting blocks starting at slots `2` a `skip_slots` value of `1` would return the blocks at `[2, 4, 6, 8, 10]`. In cases where a slot is empty for a given slot number, the closest previous block MUST be returned. For example, if slot `4` were empty in the previous example, the returned array would contain `[2, 3, 6, 8, 10]`. If slot three were further empty, the array would contain `[2, 6, 8, 10]` - i.e., duplicate blocks MUST be collapsed. A `skip_slots` value of `0` returns all blocks. -The function of the `skip_slots` parameter helps facilitate light client sync - for example, in [#459](https://github.com/ethereum/eth2.0-specs/issues/459) - and allows clients to balance the peers from whom they request headers. Clients could, for instance, request every 10th block from a set of peers where each per has a different starting block in order to populate block data. +The function of the `skip_slots` parameter helps facilitate light client sync - for example, in [#459](https://github.com/ethereum/eth2.0-specs/issues/459) - and allows clients to balance the peers from whom they request headers. Clients could, for instance, request every 10th block from a set of peers where each peer has a different starting block in order to populate block data. ### Beacon Block Bodies @@ -287,6 +287,6 @@ Requests the `block_bodies` associated with the provided `block_roots` from the **Response Body:** TBD -Requests contain the hashes of Merkle tree nodes that when merkelized yield the block's `state_root`. +Requests contain the hashes of Merkle tree nodes that when merkleized yield the block's `state_root`. The response will contain the values that, when hashed, yield the hashes inside the request body. From d64d97eee78645a67c436887bc5c69cf03b9d821 Mon Sep 17 00:00:00 2001 From: JSON <49416440+JSON@users.noreply.github.com> Date: Tue, 23 Apr 2019 12:52:06 -0500 Subject: [PATCH 083/121] Update core.md --- specs/test_formats/ssz_static/core.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/test_formats/ssz_static/core.md b/specs/test_formats/ssz_static/core.md index 059f11027..1d470c338 100644 --- a/specs/test_formats/ssz_static/core.md +++ b/specs/test_formats/ssz_static/core.md @@ -1,6 +1,6 @@ # Test format: SSZ static types -The goal of this type is to provide clients with a solid reference how the known SSZ objects should be encoded. +The goal of this type is to provide clients with a solid reference for how the known SSZ objects should be encoded. Each object described in the Phase-0 spec is covered. This is important, as many of the clients aiming to serialize/deserialize objects directly into structs/classes do not support (or have alternatives for) generic SSZ encoding/decoding. @@ -27,6 +27,6 @@ A test-runner can implement the following assertions: ## References -**`serialized`**: [SSZ serialization](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/simple-serialize.md#serialization) -**`root`** - [hash_tree_root](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/simple-serialize.md#merkleization) function -**`signing_root`** - [signing_root](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/simple-serialize.md#self-signed-containers) function +**`serialized`**: [SSZ serialization](../../simple-serialize.md#serialization) +**`root`** - [hash_tree_root](../../simple-serialize.md#merkleization) function +**`signing_root`** - [signing_root](../../simple-serialize.md#self-signed-containers) function From 47773f0da562bbcf15d533c1b61404c89162005c Mon Sep 17 00:00:00 2001 From: JSON <49416440+JSON@users.noreply.github.com> Date: Tue, 23 Apr 2019 12:53:09 -0500 Subject: [PATCH 084/121] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index aa5b7e302..b2b369e11 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Join the chat at https://gitter.im/ethereum/sharding](https://badges.gitter.im/ethereum/sharding.svg)](https://gitter.im/ethereum/sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -To learn more about sharding and eth2.0/Serenity, see the [sharding FAQ](https://github.com/ethereum/wiki/wiki/Sharding-FAQs) and the [research compendium](https://notes.ethereum.org/s/H1PGqDhpm). +To learn more about sharding and eth2.0/Serenity, see the [sharding FAQ](https://github.com/ethereum/wiki/wiki/Sharding-FAQ) and the [research compendium](https://notes.ethereum.org/s/H1PGqDhpm). This repo hosts the current eth2.0 specifications. Discussions about design rationale and proposed changes can be brought up and discussed as issues. Solidified, agreed upon changes to spec can be made through pull requests. @@ -11,10 +11,10 @@ This repo hosts the current eth2.0 specifications. Discussions about design rati Core specifications for eth2.0 client validation can be found in [specs/core](specs/core). These are divided into phases. Each subsequent phase depends upon the prior. The current phases specified are: * [Phase 0 -- The Beacon Chain](specs/core/0_beacon-chain.md) -* [Phase 1 -- Custody game](specs/core/1_custody-game.md) +* [Phase 1 -- Custody Game](specs/core/1_custody-game.md) * [Phase 1 -- Shard Data Chains](specs/core/1_shard-data-chains.md) -Accompanying documents can be found in [specs](specs) and include +Accompanying documents can be found in [specs](specs) and include: * [SimpleSerialize (SSZ) spec](specs/simple-serialize.md) * [BLS signature verification](specs/bls_signature.md) * [General test format](specs/test_formats/README.md) From cf1c78b2410a1982bdffaf375987c533c22ab047 Mon Sep 17 00:00:00 2001 From: JSON <49416440+JSON@users.noreply.github.com> Date: Tue, 23 Apr 2019 12:55:15 -0500 Subject: [PATCH 085/121] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 184411925..83698a771 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1,6 +1,6 @@ # Ethereum 2.0 Phase 0 -- The Beacon Chain -**NOTICE**: This document is a work in progress for researchers and implementers. It reflects recent spec changes and takes precedence over the Python proof-of-concept implementation [[python-poc]](#ref-python-poc). +**NOTICE**: This document is a work in progress for researchers and implementers. It reflects recent spec changes and takes precedence over the Python proof-of-concept implementation [[python-poc]](https://github.com/ethereum/beacon_chain). ## Table of contents From afbec6851261561086bad312fbc41b2d7f2b58a3 Mon Sep 17 00:00:00 2001 From: JSON <49416440+JSON@users.noreply.github.com> Date: Tue, 23 Apr 2019 12:57:15 -0500 Subject: [PATCH 086/121] Update README.md --- test_libs/pyspec/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/README.md b/test_libs/pyspec/README.md index 20c01bde4..b9bf86220 100644 --- a/test_libs/pyspec/README.md +++ b/test_libs/pyspec/README.md @@ -58,4 +58,4 @@ The pyspec is not a replacement. ## License -Same as the spec itself, see LICENSE file in spec repository root. +Same as the spec itself, see [LICENSE](../../LICENSE) file in spec repository root. From f164702b704a296c29b05e45251a5749f6cd101f Mon Sep 17 00:00:00 2001 From: JSON <49416440+JSON@users.noreply.github.com> Date: Tue, 23 Apr 2019 12:59:15 -0500 Subject: [PATCH 087/121] Update README.md --- specs/test_formats/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/test_formats/README.md b/specs/test_formats/README.md index da2e38c01..273659ce9 100644 --- a/specs/test_formats/README.md +++ b/specs/test_formats/README.md @@ -118,7 +118,7 @@ Separation of configuration and tests aims to: Note: Some clients prefer compile-time constants and optimizations. They should compile for each configuration once, and run the corresponding tests per build target. -The format is described in `configs/constant_presets`. +The format is described in [`configs/constant_presets`](../../configs/constant_presets/README.md#format). ## Fork-timeline @@ -129,7 +129,7 @@ A fork timeline is (preferably) loaded in as a configuration object into a clien - we may decide on an epoch number for a fork based on external events (e.g. Eth1 log event), a client should be able to activate a fork dynamically. -The format is described in `configs/fork_timelines`. +The format is described in [`configs/fork_timelines`](../../configs/fork_timelines/README.md#format). ## Config sourcing From 1d59b335904a3b0f0c8952d129a2c98bec26d8ce Mon Sep 17 00:00:00 2001 From: JSON <49416440+JSON@users.noreply.github.com> Date: Tue, 23 Apr 2019 12:59:49 -0500 Subject: [PATCH 088/121] Update README.md --- test_generators/ssz_static/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/ssz_static/README.md b/test_generators/ssz_static/README.md index d73556e1b..453d6d0e5 100644 --- a/test_generators/ssz_static/README.md +++ b/test_generators/ssz_static/README.md @@ -1,6 +1,6 @@ # SSZ-static The purpose of this test-generator is to provide test-vectors for the most important applications of SSZ: - the serialization and hashing of ETH 2.0 data types + the serialization and hashing of ETH 2.0 data types. Test-format documentation can be found [here](../../specs/test_formats/ssz_static/README.md). From 2048b657b62b568d881b562671bc314152b13b7c Mon Sep 17 00:00:00 2001 From: JSON <49416440+JSON@users.noreply.github.com> Date: Tue, 23 Apr 2019 12:59:58 -0500 Subject: [PATCH 089/121] Update sign_msg.md --- specs/test_formats/bls/sign_msg.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/test_formats/bls/sign_msg.md b/specs/test_formats/bls/sign_msg.md index dd93174f2..9916f2cc2 100644 --- a/specs/test_formats/bls/sign_msg.md +++ b/specs/test_formats/bls/sign_msg.md @@ -12,7 +12,7 @@ input: output: bytes96 -- expected signature ``` -All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x` +All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. ## Condition From dbcac289c80e719b67e9e885feef756cd184d05a Mon Sep 17 00:00:00 2001 From: JSON <49416440+JSON@users.noreply.github.com> Date: Tue, 23 Apr 2019 13:00:05 -0500 Subject: [PATCH 090/121] Update priv_to_pub.md --- specs/test_formats/bls/priv_to_pub.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/test_formats/bls/priv_to_pub.md b/specs/test_formats/bls/priv_to_pub.md index 7af148d0f..ef62241ae 100644 --- a/specs/test_formats/bls/priv_to_pub.md +++ b/specs/test_formats/bls/priv_to_pub.md @@ -9,7 +9,7 @@ input: bytes32 -- the private key output: bytes48 -- the public key ``` -All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x` +All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. ## Condition From babf2721c7ef517673e1f3497899ed36aa8d6259 Mon Sep 17 00:00:00 2001 From: JSON <49416440+JSON@users.noreply.github.com> Date: Tue, 23 Apr 2019 13:00:15 -0500 Subject: [PATCH 091/121] Update msg_hash_g2_uncompressed.md --- specs/test_formats/bls/msg_hash_g2_uncompressed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/test_formats/bls/msg_hash_g2_uncompressed.md b/specs/test_formats/bls/msg_hash_g2_uncompressed.md index f42ea9998..792fe1f03 100644 --- a/specs/test_formats/bls/msg_hash_g2_uncompressed.md +++ b/specs/test_formats/bls/msg_hash_g2_uncompressed.md @@ -11,7 +11,7 @@ input: output: List[List[bytes48]] -- 3 lists, each a length of two ``` -All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x` +All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. ## Condition From 58c50c2f08af4ca83619d6ff21270df471984c5a Mon Sep 17 00:00:00 2001 From: JSON <49416440+JSON@users.noreply.github.com> Date: Tue, 23 Apr 2019 13:00:25 -0500 Subject: [PATCH 092/121] Update msg_hash_g2_compressed.md --- specs/test_formats/bls/msg_hash_g2_compressed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/test_formats/bls/msg_hash_g2_compressed.md b/specs/test_formats/bls/msg_hash_g2_compressed.md index 4e194e90b..2feeb92ba 100644 --- a/specs/test_formats/bls/msg_hash_g2_compressed.md +++ b/specs/test_formats/bls/msg_hash_g2_compressed.md @@ -11,7 +11,7 @@ input: output: List[bytes48] -- length of two ``` -All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x` +All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. ## Condition From b6a085d0d7555c7ec3818b557930cd6087361f9d Mon Sep 17 00:00:00 2001 From: JSON <49416440+JSON@users.noreply.github.com> Date: Tue, 23 Apr 2019 13:01:21 -0500 Subject: [PATCH 093/121] Update bls_signature.md --- specs/bls_signature.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/bls_signature.md b/specs/bls_signature.md index beef19df5..18e2d8c9a 100644 --- a/specs/bls_signature.md +++ b/specs/bls_signature.md @@ -86,7 +86,7 @@ def hash_to_G2(message_hash: Bytes32, domain: uint64) -> [uint384]: ### `modular_squareroot` -`modular_squareroot(x)` returns a solution `y` to `y**2 % q == x`, and `None` if none exists. If there are two solutions the one with higher imaginary component is favored; if both solutions have equal imaginary component the one with higher real component is favored (note that this is equivalent to saying that the single solution with either imaginary component > p/2 or imaginary component zero and real component > p/2 is favored). +`modular_squareroot(x)` returns a solution `y` to `y**2 % q == x`, and `None` if none exists. If there are two solutions, the one with higher imaginary component is favored; if both solutions have equal imaginary component, the one with higher real component is favored (note that this is equivalent to saying that the single solution with either imaginary component > p/2 or imaginary component zero and real component > p/2 is favored). The following is a sample implementation; implementers are free to implement modular square roots as they wish. Note that `x2 = -x1` is an _additive modular inverse_ so real and imaginary coefficients remain in `[0 .. q-1]`. `coerce_to_int(element: Fq) -> int` is a function that takes Fq element `element` (i.e. integers `mod q`) and converts it to a regular integer. From bd0552c796b903c18c7690415c3391898528e5ae Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 23 Apr 2019 13:32:41 -0600 Subject: [PATCH 094/121] fix finality tests for each rule --- specs/core/0_beacon-chain.md | 8 - test_libs/pyspec/tests/test_finality.py | 165 +++++--------- test_libs/pyspec/tests/test_sanity.py | 27 --- tests/phase0/__init__.py | 0 .../test_process_block_header.py | 60 ------ .../block_processing/test_process_deposit.py | 140 ------------ .../block_processing/test_voluntary_exit.py | 175 --------------- tests/phase0/helpers.py | 203 ------------------ 8 files changed, 52 insertions(+), 726 deletions(-) delete mode 100644 tests/phase0/__init__.py delete mode 100644 tests/phase0/block_processing/test_process_block_header.py delete mode 100644 tests/phase0/block_processing/test_process_deposit.py delete mode 100644 tests/phase0/block_processing/test_voluntary_exit.py delete mode 100644 tests/phase0/helpers.py diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 171a120d0..084bec844 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1564,16 +1564,12 @@ def process_justification_and_finalization(state: BeaconState) -> None: state.previous_justified_root = state.current_justified_root state.justification_bitfield = (state.justification_bitfield << 1) % 2**64 previous_boundary_attesting_balance = get_attesting_balance(state, get_previous_epoch_boundary_attestations(state)) - print(previous_boundary_attesting_balance) if previous_boundary_attesting_balance * 3 >= get_previous_total_balance(state) * 2: - print("prev success") state.current_justified_epoch = get_previous_epoch(state) state.current_justified_root = get_block_root(state, get_epoch_start_slot(state.current_justified_epoch)) state.justification_bitfield |= (1 << 1) current_boundary_attesting_balance = get_attesting_balance(state, get_current_epoch_boundary_attestations(state)) - print(current_boundary_attesting_balance) if current_boundary_attesting_balance * 3 >= get_current_total_balance(state) * 2: - print("cur success") state.current_justified_epoch = get_current_epoch(state) state.current_justified_root = get_block_root(state, get_epoch_start_slot(state.current_justified_epoch)) state.justification_bitfield |= (1 << 0) @@ -1583,22 +1579,18 @@ def process_justification_and_finalization(state: BeaconState) -> None: current_epoch = get_current_epoch(state) # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_epoch == current_epoch - 3: - print("rule 1") state.finalized_epoch = old_previous_justified_epoch state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_epoch == current_epoch - 2: - print("rule 2") state.finalized_epoch = old_previous_justified_epoch state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source if (bitfield >> 0) % 8 == 0b111 and old_current_justified_epoch == current_epoch - 2: - print("rule 3") state.finalized_epoch = old_current_justified_epoch state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source if (bitfield >> 0) % 4 == 0b11 and old_current_justified_epoch == current_epoch - 1: - print("rule 4") state.finalized_epoch = old_current_justified_epoch state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) ``` diff --git a/test_libs/pyspec/tests/test_finality.py b/test_libs/pyspec/tests/test_finality.py index ae8ba13ff..8c1b4e871 100644 --- a/test_libs/pyspec/tests/test_finality.py +++ b/test_libs/pyspec/tests/test_finality.py @@ -53,35 +53,23 @@ def next_epoch_with_attestations(state, post_state = deepcopy(state) blocks = [] for slot in range(spec.SLOTS_PER_EPOCH): - print("slot: %s", post_state.slot) block = build_empty_block_for_next_slot(post_state) - if fill_prev_epoch: - print("prev") - slot_to_attest = post_state.slot - spec.SLOTS_PER_EPOCH + 1 - prev_attestation = get_valid_attestation(post_state, slot_to_attest) - block.body.attestations.append(prev_attestation) - if fill_cur_epoch: - print("cur") slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 if slot_to_attest >= get_epoch_start_slot(get_current_epoch(post_state)): cur_attestation = get_valid_attestation(post_state, slot_to_attest) fill_aggregate_attestation(post_state, cur_attestation) block.body.attestations.append(cur_attestation) + if fill_prev_epoch: + slot_to_attest = post_state.slot - spec.SLOTS_PER_EPOCH + 1 + prev_attestation = get_valid_attestation(post_state, slot_to_attest) + fill_aggregate_attestation(post_state, prev_attestation) + block.body.attestations.append(prev_attestation) + state_transition(post_state, block) blocks.append(block) - # if fill_prev_epoch: - # assert len(post_state.previous_epoch_attestations) >= 0 - # else: - # assert len(post_state.previous_epoch_attestations) == 0 - - # if fill_cur_epoch: - # assert len(post_state.current_epoch_attestations) >= 0 - # else: - # assert len(post_state.current_epoch_attestations) == 0 - return state, blocks, post_state @@ -90,20 +78,8 @@ def test_finality_from_genesis_rule_4(state): blocks = [] for epoch in range(6): - prev_state = deepcopy(test_state) - old_current_justified_epoch = test_state.current_justified_epoch - old_current_justified_root = test_state.current_justified_root - for slot in range(spec.SLOTS_PER_EPOCH): - attestation = None - slot_to_attest = test_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 - if slot_to_attest >= spec.GENESIS_SLOT: - attestation = get_valid_attestation(test_state, slot_to_attest) - fill_aggregate_attestation(test_state, attestation) - block = build_empty_block_for_next_slot(test_state) - if attestation: - block.body.attestations.append(attestation) - state_transition(test_state, block) - blocks.append(block) + prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, False) + blocks += new_blocks if epoch == 0: check_finality(test_state, prev_state, False, False, False) @@ -114,8 +90,8 @@ def test_finality_from_genesis_rule_4(state): elif epoch >= 3: # rule 4 of finaliy check_finality(test_state, prev_state, True, True, True) - assert test_state.finalized_epoch == old_current_justified_epoch - assert test_state.finalized_root == old_current_justified_root + assert test_state.finalized_epoch == prev_state.current_justified_epoch + assert test_state.finalized_root == prev_state.current_justified_root return state, blocks, test_state @@ -125,25 +101,13 @@ def test_finality_rule_1(state): next_epoch(state) next_epoch(state) + pre_state = deepcopy(state) test_state = deepcopy(state) blocks = [] for epoch in range(3): - prev_state = deepcopy(test_state) - old_previous_justified_epoch = test_state.previous_justified_epoch - old_previous_justified_root = test_state.previous_justified_root - for slot in range(spec.SLOTS_PER_EPOCH): - slot_to_attest = test_state.slot - spec.SLOTS_PER_EPOCH + 1 - attestation = get_valid_attestation(test_state, slot_to_attest) - fill_aggregate_attestation(test_state, attestation) - block = build_empty_block_for_next_slot(test_state) - block.body.attestations.append(attestation) - state_transition(test_state, block) - - assert len(test_state.previous_epoch_attestations) >= 0 - assert len(test_state.current_epoch_attestations) == 0 - - blocks.append(block) + prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, True) + blocks += new_blocks if epoch == 0: check_finality(test_state, prev_state, True, False, False) @@ -152,10 +116,10 @@ def test_finality_rule_1(state): elif epoch == 2: # finalized by rule 1 check_finality(test_state, prev_state, True, True, True) - assert test_state.finalized_epoch == old_previous_justified_epoch - assert test_state.finalized_root == old_previous_justified_root + assert test_state.finalized_epoch == prev_state.previous_justified_epoch + assert test_state.finalized_root == prev_state.previous_justified_root - return state, blocks, test_state + return pre_state, blocks, test_state def test_finality_rule_2(state): @@ -163,6 +127,7 @@ def test_finality_rule_2(state): next_epoch(state) next_epoch(state) + pre_state = deepcopy(state) test_state = deepcopy(state) blocks = [] @@ -170,85 +135,59 @@ def test_finality_rule_2(state): old_previous_justified_epoch = test_state.previous_justified_epoch old_previous_justified_root = test_state.previous_justified_root if epoch == 0: - prev_state, blocks, test_state = next_epoch_with_attestations(test_state, True, False) + prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, False) check_finality(test_state, prev_state, True, False, False) if epoch == 1: - prev_state, blocks, test_state = next_epoch_with_attestations(test_state, False, False) + prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, False) check_finality(test_state, prev_state, False, True, False) if epoch == 2: - prev_state, blocks, test_state = next_epoch_with_attestations(test_state, False, True) + prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, True) # finalized by rule 2 check_finality(test_state, prev_state, True, False, True) assert test_state.finalized_epoch == old_previous_justified_epoch assert test_state.finalized_root == old_previous_justified_root - return state, blocks, test_state + + blocks += new_blocks + + return pre_state, blocks, test_state def test_finality_rule_3(state): + """ + Test scenario described here + https://github.com/ethereum/eth2.0-specs/issues/611#issuecomment-463612892 + """ + # get past first two epochs that finality does not run on next_epoch(state) next_epoch(state) + pre_state = deepcopy(state) test_state = deepcopy(state) blocks = [] - for epoch in range(2): - prev_state = deepcopy(test_state) - for slot in range(spec.SLOTS_PER_EPOCH): - slot_to_attest = test_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 - attestation = get_valid_attestation(test_state, slot_to_attest) - fill_aggregate_attestation(test_state, attestation) - block = build_empty_block_for_next_slot(test_state) - block.body.attestations.append(attestation) - state_transition(test_state, block) + prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, False) + blocks += new_blocks + check_finality(test_state, prev_state, True, False, False) - blocks.append(block) - if epoch == 0: - check_finality(test_state, prev_state, True, False, False) - if epoch == 1: - check_finality(test_state, prev_state, True, True, True) - - prev_state = deepcopy(test_state) - next_epoch(test_state) - check_finality(test_state, prev_state, False, True, False) - - - prev_state = deepcopy(test_state) - for slot in range(spec.SLOTS_PER_EPOCH): - slot_to_attest = test_state.slot - spec.SLOTS_PER_EPOCH + 1 - attestation = get_valid_attestation(test_state, slot_to_attest) - fill_aggregate_attestation(test_state, attestation) - block = build_empty_block_for_next_slot(test_state) - block.body.attestations.append(attestation) - state_transition(test_state, block) - - assert len(test_state.previous_epoch_attestations) >= 0 - assert len(test_state.current_epoch_attestations) == 0 - - blocks.append(block) - check_finality(test_state, prev_state, True, False, True) - - - prev_state = deepcopy(test_state) - for slot in range(spec.SLOTS_PER_EPOCH): - prev_slot_to_attest = test_state.slot - spec.SLOTS_PER_EPOCH + 1 - prev_attestation = get_valid_attestation(test_state, prev_slot_to_attest) - fill_aggregate_attestation(test_state, prev_attestation) - - cur_slot_to_attest = test_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 - cur_attestation = get_valid_attestation(test_state, cur_slot_to_attest) - fill_aggregate_attestation(test_state, cur_attestation) - - block = build_empty_block_for_next_slot(test_state) - block.body.attestations.append(prev_attestation) - block.body.attestations.append(cur_attestation) - - state_transition(test_state, block) - - assert len(test_state.previous_epoch_attestations) >= 0 - assert len(test_state.current_epoch_attestations) >= 0 - - blocks.append(block) + prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, False) + blocks += new_blocks check_finality(test_state, prev_state, True, True, True) - return state, blocks, test_state + prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, False) + blocks += new_blocks + check_finality(test_state, prev_state, False, True, False) + + prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, True) + blocks += new_blocks + # rule 2 + check_finality(test_state, prev_state, True, False, True) + + prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, True) + blocks += new_blocks + # rule 3 + check_finality(test_state, prev_state, True, True, True) + assert test_state.finalized_epoch == prev_state.current_justified_epoch + assert test_state.finalized_root == prev_state.current_justified_root + + return pre_state, blocks, test_state diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index f0476f1c6..f330c2b19 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -53,33 +53,6 @@ from .helpers import ( pytestmark = pytest.mark.sanity -def check_finality(state, - prev_state, - current_justified_changed, - previous_justified_changed, - finalized_changed): - if current_justified_changed: - assert state.current_justified_epoch > prev_state.current_justified_epoch - assert state.current_justified_root != prev_state.current_justified_root - else: - assert state.current_justified_epoch == prev_state.current_justified_epoch - assert state.current_justified_root == prev_state.current_justified_root - - if previous_justified_changed: - assert state.previous_justified_epoch > prev_state.previous_justified_epoch - assert state.previous_justified_root != prev_state.previous_justified_root - else: - assert state.previous_justified_epoch == prev_state.previous_justified_epoch - assert state.previous_justified_root == prev_state.previous_justified_root - - if finalized_changed: - assert state.finalized_epoch > prev_state.finalized_epoch - assert state.finalized_root != prev_state.finalized_root - else: - assert state.finalized_epoch == prev_state.finalized_epoch - assert state.finalized_root == prev_state.finalized_root - - def test_slot_transition(state): test_state = deepcopy(state) cache_state(test_state) diff --git a/tests/phase0/__init__.py b/tests/phase0/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/phase0/block_processing/test_process_block_header.py b/tests/phase0/block_processing/test_process_block_header.py deleted file mode 100644 index 241466437..000000000 --- a/tests/phase0/block_processing/test_process_block_header.py +++ /dev/null @@ -1,60 +0,0 @@ -from copy import deepcopy -import pytest - - -from build.phase0.spec import ( - get_beacon_proposer_index, - cache_state, - advance_slot, - process_block_header, -) -from tests.phase0.helpers import ( - build_empty_block_for_next_slot, -) - -# mark entire file as 'header' -pytestmark = pytest.mark.header - - -def prepare_state_for_header_processing(state): - cache_state(state) - advance_slot(state) - - -def run_block_header_processing(state, block, valid=True): - """ - Run ``process_block_header`` returning the pre and post state. - If ``valid == False``, run expecting ``AssertionError`` - """ - prepare_state_for_header_processing(state) - post_state = deepcopy(state) - - if not valid: - with pytest.raises(AssertionError): - process_block_header(post_state, block) - return state, None - - process_block_header(post_state, block) - return state, post_state - - -def test_success(state): - block = build_empty_block_for_next_slot(state) - pre_state, post_state = run_block_header_processing(state, block) - return state, block, post_state - - -def test_invalid_slot(state): - block = build_empty_block_for_next_slot(state) - block.slot = state.slot + 2 # invalid slot - - pre_state, post_state = run_block_header_processing(state, block, valid=False) - return pre_state, block, None - - -def test_invalid_previous_block_root(state): - block = build_empty_block_for_next_slot(state) - block.previous_block_root = b'\12'*32 # invalid prev root - - pre_state, post_state = run_block_header_processing(state, block, valid=False) - return pre_state, block, None diff --git a/tests/phase0/block_processing/test_process_deposit.py b/tests/phase0/block_processing/test_process_deposit.py deleted file mode 100644 index 0c5770195..000000000 --- a/tests/phase0/block_processing/test_process_deposit.py +++ /dev/null @@ -1,140 +0,0 @@ -from copy import deepcopy -import pytest - -import build.phase0.spec as spec - -from build.phase0.spec import ( - ZERO_HASH, - process_deposit, -) -from tests.phase0.helpers import ( - build_deposit, - privkeys, - pubkeys, -) - - -# mark entire file as 'voluntary_exits' -pytestmark = pytest.mark.voluntary_exits - - -def test_success(state): - pre_state = deepcopy(state) - # fill previous deposits with zero-hash - deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) - - index = len(deposit_data_leaves) - pubkey = pubkeys[index] - privkey = privkeys[index] - deposit, root, deposit_data_leaves = build_deposit( - pre_state, - deposit_data_leaves, - pubkey, - privkey, - spec.MAX_DEPOSIT_AMOUNT, - ) - - pre_state.latest_eth1_data.deposit_root = root - pre_state.latest_eth1_data.deposit_count = len(deposit_data_leaves) - - post_state = deepcopy(pre_state) - - process_deposit(post_state, deposit) - - assert len(post_state.validator_registry) == len(state.validator_registry) + 1 - assert len(post_state.validator_balances) == len(state.validator_balances) + 1 - assert post_state.validator_registry[index].pubkey == pubkeys[index] - assert post_state.deposit_index == post_state.latest_eth1_data.deposit_count - - return pre_state, deposit, post_state - - -def test_success_top_up(state): - pre_state = deepcopy(state) - deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) - - validator_index = 0 - amount = spec.MAX_DEPOSIT_AMOUNT // 4 - pubkey = pubkeys[validator_index] - privkey = privkeys[validator_index] - deposit, root, deposit_data_leaves = build_deposit( - pre_state, - deposit_data_leaves, - pubkey, - privkey, - amount, - ) - - pre_state.latest_eth1_data.deposit_root = root - pre_state.latest_eth1_data.deposit_count = len(deposit_data_leaves) - pre_balance = pre_state.validator_balances[validator_index] - - post_state = deepcopy(pre_state) - - process_deposit(post_state, deposit) - - assert len(post_state.validator_registry) == len(state.validator_registry) - assert len(post_state.validator_balances) == len(state.validator_balances) - assert post_state.deposit_index == post_state.latest_eth1_data.deposit_count - assert post_state.validator_balances[validator_index] == pre_balance + amount - - return pre_state, deposit, post_state - - -def test_wrong_index(state): - pre_state = deepcopy(state) - deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) - - - index = len(deposit_data_leaves) - pubkey = pubkeys[index] - privkey = privkeys[index] - deposit, root, deposit_data_leaves = build_deposit( - pre_state, - deposit_data_leaves, - pubkey, - privkey, - spec.MAX_DEPOSIT_AMOUNT, - ) - - # mess up deposit_index - deposit.index = pre_state.deposit_index + 1 - - pre_state.latest_eth1_data.deposit_root = root - pre_state.latest_eth1_data.deposit_count = len(deposit_data_leaves) - - post_state = deepcopy(pre_state) - - with pytest.raises(AssertionError): - process_deposit(post_state, deposit) - - return pre_state, deposit, None - - -def test_bad_merkle_proof(state): - pre_state = deepcopy(state) - deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) - - index = len(deposit_data_leaves) - pubkey = pubkeys[index] - privkey = privkeys[index] - deposit, root, deposit_data_leaves = build_deposit( - pre_state, - deposit_data_leaves, - pubkey, - privkey, - spec.MAX_DEPOSIT_AMOUNT, - ) - - # mess up merkle branch - deposit.proof[-1] = spec.ZERO_HASH - - pre_state.latest_eth1_data.deposit_root = root - pre_state.latest_eth1_data.deposit_count = len(deposit_data_leaves) - - post_state = deepcopy(pre_state) - - with pytest.raises(AssertionError): - process_deposit(post_state, deposit) - - return pre_state, deposit, None diff --git a/tests/phase0/block_processing/test_voluntary_exit.py b/tests/phase0/block_processing/test_voluntary_exit.py deleted file mode 100644 index 6adc81464..000000000 --- a/tests/phase0/block_processing/test_voluntary_exit.py +++ /dev/null @@ -1,175 +0,0 @@ -from copy import deepcopy -import pytest - -import build.phase0.spec as spec - -from build.phase0.spec import ( - get_active_validator_indices, - get_current_epoch, - process_voluntary_exit, -) -from tests.phase0.helpers import ( - build_voluntary_exit, - pubkey_to_privkey, -) - - -# mark entire file as 'voluntary_exits' -pytestmark = pytest.mark.voluntary_exits - - -def test_success(state): - pre_state = deepcopy(state) - # - # setup pre_state - # - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - pre_state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH - - # - # build voluntary exit - # - current_epoch = get_current_epoch(pre_state) - validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0] - privkey = pubkey_to_privkey[pre_state.validator_registry[validator_index].pubkey] - - voluntary_exit = build_voluntary_exit( - pre_state, - current_epoch, - validator_index, - privkey, - ) - - post_state = deepcopy(pre_state) - - # - # test valid exit - # - process_voluntary_exit(post_state, voluntary_exit) - - assert not pre_state.validator_registry[validator_index].initiated_exit - assert post_state.validator_registry[validator_index].initiated_exit - - return pre_state, voluntary_exit, post_state - - -def test_validator_not_active(state): - pre_state = deepcopy(state) - current_epoch = get_current_epoch(pre_state) - validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0] - privkey = pubkey_to_privkey[pre_state.validator_registry[validator_index].pubkey] - - # - # setup pre_state - # - pre_state.validator_registry[validator_index].activation_epoch = spec.FAR_FUTURE_EPOCH - - # - # build and test voluntary exit - # - voluntary_exit = build_voluntary_exit( - pre_state, - current_epoch, - validator_index, - privkey, - ) - - with pytest.raises(AssertionError): - process_voluntary_exit(pre_state, voluntary_exit) - - return pre_state, voluntary_exit, None - - -def test_validator_already_exited(state): - pre_state = deepcopy(state) - # - # setup pre_state - # - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow validator able to exit - pre_state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH - - current_epoch = get_current_epoch(pre_state) - validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0] - privkey = pubkey_to_privkey[pre_state.validator_registry[validator_index].pubkey] - - # but validator already has exited - pre_state.validator_registry[validator_index].exit_epoch = current_epoch + 2 - - # - # build voluntary exit - # - voluntary_exit = build_voluntary_exit( - pre_state, - current_epoch, - validator_index, - privkey, - ) - - with pytest.raises(AssertionError): - process_voluntary_exit(pre_state, voluntary_exit) - - return pre_state, voluntary_exit, None - - -def test_validator_already_initiated_exit(state): - pre_state = deepcopy(state) - # - # setup pre_state - # - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow validator able to exit - pre_state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH - - current_epoch = get_current_epoch(pre_state) - validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0] - privkey = pubkey_to_privkey[pre_state.validator_registry[validator_index].pubkey] - - # but validator already has initiated exit - pre_state.validator_registry[validator_index].initiated_exit = True - - # - # build voluntary exit - # - voluntary_exit = build_voluntary_exit( - pre_state, - current_epoch, - validator_index, - privkey, - ) - - with pytest.raises(AssertionError): - process_voluntary_exit(pre_state, voluntary_exit) - - return pre_state, voluntary_exit, None - - -def test_validator_not_active_long_enough(state): - pre_state = deepcopy(state) - # - # setup pre_state - # - current_epoch = get_current_epoch(pre_state) - validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0] - privkey = pubkey_to_privkey[pre_state.validator_registry[validator_index].pubkey] - - # but validator already has initiated exit - pre_state.validator_registry[validator_index].initiated_exit = True - - # - # build voluntary exit - # - voluntary_exit = build_voluntary_exit( - pre_state, - current_epoch, - validator_index, - privkey, - ) - - assert ( - current_epoch - pre_state.validator_registry[validator_index].activation_epoch < - spec.PERSISTENT_COMMITTEE_PERIOD - ) - - with pytest.raises(AssertionError): - process_voluntary_exit(pre_state, voluntary_exit) - - return pre_state, voluntary_exit, None diff --git a/tests/phase0/helpers.py b/tests/phase0/helpers.py deleted file mode 100644 index 20054b821..000000000 --- a/tests/phase0/helpers.py +++ /dev/null @@ -1,203 +0,0 @@ -from copy import deepcopy - -from py_ecc import bls - -import build.phase0.spec as spec -from build.phase0.utils.minimal_ssz import signed_root -from build.phase0.spec import ( - # constants - EMPTY_SIGNATURE, - # SSZ - AttestationData, - Deposit, - DepositInput, - DepositData, - Eth1Data, - VoluntaryExit, - # functions - get_block_root, - get_current_epoch, - get_domain, - get_empty_block, - get_epoch_start_slot, - get_genesis_beacon_state, - verify_merkle_branch, - hash, -) -from build.phase0.utils.merkle_minimal import ( - calc_merkle_tree_from_leaves, - get_merkle_proof, - get_merkle_root, -) - - -privkeys = [i + 1 for i in range(1000)] -pubkeys = [bls.privtopub(privkey) for privkey in privkeys] -pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} - - -def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves=None): - if not deposit_data_leaves: - deposit_data_leaves = [] - deposit_timestamp = 0 - proof_of_possession = b'\x33' * 96 - - deposit_data_list = [] - for i in range(num_validators): - pubkey = pubkeys[i] - deposit_data = DepositData( - amount=spec.MAX_DEPOSIT_AMOUNT, - timestamp=deposit_timestamp, - deposit_input=DepositInput( - pubkey=pubkey, - # insecurely use pubkey as withdrawal key as well - withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:], - proof_of_possession=proof_of_possession, - ), - ) - item = hash(deposit_data.serialize()) - deposit_data_leaves.append(item) - tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) - root = get_merkle_root((tuple(deposit_data_leaves))) - proof = list(get_merkle_proof(tree, item_index=i)) - assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, i, root) - deposit_data_list.append(deposit_data) - - genesis_validator_deposits = [] - for i in range(num_validators): - genesis_validator_deposits.append(Deposit( - proof=list(get_merkle_proof(tree, item_index=i)), - index=i, - deposit_data=deposit_data_list[i] - )) - return genesis_validator_deposits, root - - -def create_genesis_state(num_validators, deposit_data_leaves=None): - initial_deposits, deposit_root = create_mock_genesis_validator_deposits( - num_validators, - deposit_data_leaves, - ) - return get_genesis_beacon_state( - initial_deposits, - genesis_time=0, - genesis_eth1_data=Eth1Data( - deposit_root=deposit_root, - deposit_count=len(initial_deposits), - block_hash=spec.ZERO_HASH, - ), - ) - - -def force_registry_change_at_next_epoch(state): - # artificially trigger registry update at next epoch transition - state.finalized_epoch = get_current_epoch(state) - 1 - for crosslink in state.latest_crosslinks: - crosslink.epoch = state.finalized_epoch - state.validator_registry_update_epoch = state.finalized_epoch - 1 - - -def build_empty_block_for_next_slot(state): - empty_block = get_empty_block() - empty_block.slot = state.slot + 1 - previous_block_header = deepcopy(state.latest_block_header) - if previous_block_header.state_root == spec.ZERO_HASH: - previous_block_header.state_root = state.hash_tree_root() - empty_block.previous_block_root = signed_root(previous_block_header) - return empty_block - - -def build_deposit_data(state, pubkey, privkey, amount): - deposit_input = DepositInput( - pubkey=pubkey, - # insecurely use pubkey as withdrawal key as well - withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:], - proof_of_possession=EMPTY_SIGNATURE, - ) - proof_of_possession = bls.sign( - message_hash=signed_root(deposit_input), - privkey=privkey, - domain=get_domain( - state.fork, - get_current_epoch(state), - spec.DOMAIN_DEPOSIT, - ) - ) - deposit_input.proof_of_possession = proof_of_possession - deposit_data = DepositData( - amount=amount, - timestamp=0, - deposit_input=deposit_input, - ) - return deposit_data - - -def build_attestation_data(state, slot, shard): - assert state.slot >= slot - - block_root = build_empty_block_for_next_slot(state).previous_block_root - - epoch_start_slot = get_epoch_start_slot(get_current_epoch(state)) - if epoch_start_slot == slot: - epoch_boundary_root = block_root - else: - get_block_root(state, epoch_start_slot) - - if slot < epoch_start_slot: - justified_block_root = state.previous_justified_root - else: - justified_block_root = state.current_justified_root - - return AttestationData( - slot=slot, - shard=shard, - beacon_block_root=block_root, - source_epoch=state.current_justified_epoch, - source_root=justified_block_root, - target_root=epoch_boundary_root, - crosslink_data_root=spec.ZERO_HASH, - previous_crosslink=deepcopy(state.latest_crosslinks[shard]), - ) - - -def build_voluntary_exit(state, epoch, validator_index, privkey): - voluntary_exit = VoluntaryExit( - epoch=epoch, - validator_index=validator_index, - signature=EMPTY_SIGNATURE, - ) - voluntary_exit.signature = bls.sign( - message_hash=signed_root(voluntary_exit), - privkey=privkey, - domain=get_domain( - fork=state.fork, - epoch=epoch, - domain_type=spec.DOMAIN_VOLUNTARY_EXIT, - ) - ) - - return voluntary_exit - - -def build_deposit(state, - deposit_data_leaves, - pubkey, - privkey, - amount): - deposit_data = build_deposit_data(state, pubkey, privkey, amount) - - item = hash(deposit_data.serialize()) - index = len(deposit_data_leaves) - deposit_data_leaves.append(item) - tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) - root = get_merkle_root((tuple(deposit_data_leaves))) - proof = list(get_merkle_proof(tree, item_index=index)) - assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, root) - - deposit = Deposit( - proof=list(proof), - index=index, - deposit_data=deposit_data, - ) - - return deposit, root, deposit_data_leaves From 53eb2d0368cd4df7692250bce88376655e956d43 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 23 Apr 2019 13:38:35 -0600 Subject: [PATCH 095/121] remove jsonize --- utils/phase0/jsonize.py | 52 ----------------------------------------- 1 file changed, 52 deletions(-) delete mode 100644 utils/phase0/jsonize.py diff --git a/utils/phase0/jsonize.py b/utils/phase0/jsonize.py deleted file mode 100644 index 816192ec6..000000000 --- a/utils/phase0/jsonize.py +++ /dev/null @@ -1,52 +0,0 @@ -from .minimal_ssz import hash_tree_root - - -def jsonize(value, typ, include_hash_tree_roots=False): - if isinstance(typ, str) and typ[:4] == 'uint': - return value - elif typ == 'bool': - assert value in (True, False) - return value - elif isinstance(typ, list): - return [jsonize(element, typ[0], include_hash_tree_roots) for element in value] - elif isinstance(typ, str) and typ[:4] == 'byte': - return '0x' + value.hex() - elif hasattr(typ, 'fields'): - ret = {} - for field, subtype in typ.fields.items(): - ret[field] = jsonize(getattr(value, field), subtype, include_hash_tree_roots) - if include_hash_tree_roots: - ret[field + "_hash_tree_root"] = '0x' + hash_tree_root(getattr(value, field), subtype).hex() - if include_hash_tree_roots: - ret["hash_tree_root"] = '0x' + hash_tree_root(value, typ).hex() - return ret - else: - print(value, typ) - raise Exception("Type not recognized") - - -def dejsonize(json, typ): - if isinstance(typ, str) and typ[:4] == 'uint': - return json - elif typ == 'bool': - assert json in (True, False) - return json - elif isinstance(typ, list): - return [dejsonize(element, typ[0]) for element in json] - elif isinstance(typ, str) and typ[:4] == 'byte': - return bytes.fromhex(json[2:]) - elif hasattr(typ, 'fields'): - temp = {} - for field, subtype in typ.fields.items(): - temp[field] = dejsonize(json[field], subtype) - if field + "_hash_tree_root" in json: - assert(json[field + "_hash_tree_root"][2:] == - hash_tree_root(temp[field], subtype).hex()) - ret = typ(**temp) - if "hash_tree_root" in json: - assert(json["hash_tree_root"][2:] == - hash_tree_root(ret, typ).hex()) - return ret - else: - print(json, typ) - raise Exception("Type not recognized") From f2d885f0d8e652ab48aa9dfb91305f7e15d65e3c Mon Sep 17 00:00:00 2001 From: Justin Date: Wed, 24 Apr 2019 14:23:51 +1000 Subject: [PATCH 096/121] Address Danny's comments --- specs/core/0_beacon-chain.md | 125 ++++++++++++++++++----------------- 1 file changed, 63 insertions(+), 62 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 931511b36..f61eb2ec5 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -66,8 +66,8 @@ - [`get_shard_delta`](#get_shard_delta) - [`compute_committee`](#compute_committee) - [`get_crosslink_committees_at_slot`](#get_crosslink_committees_at_slot) + - [`get_block_root_at_slot`](#get_block_root_at_slot) - [`get_block_root`](#get_block_root) - - [`get_state_root`](#get_state_root) - [`get_randao_mix`](#get_randao_mix) - [`get_active_index_root`](#get_active_index_root) - [`generate_seed`](#generate_seed) @@ -416,6 +416,8 @@ The types are defined topologically to aid in facilitating an executable version 'data': AttestationData, # Inclusion slot 'inclusion_slot': 'uint64', + # Proposer index + 'proposer_index': 'uint64', } ``` @@ -678,6 +680,7 @@ def get_epoch_start_slot(epoch: Epoch) -> Slot: ``` ### `is_active_validator` + ```python def is_active_validator(validator: Validator, epoch: Epoch) -> bool: """ @@ -687,15 +690,14 @@ def is_active_validator(validator: Validator, epoch: Epoch) -> bool: ``` ### `is_slashable_validator` + ```python def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: """ Check if ``validator`` is slashable. """ - return ( - validator.activation_epoch <= epoch < validator.withdrawable_epoch and - validator.slashed is False - ) + return validator.slashed is False and (validator.activation_epoch <= epoch < validator.withdrawable_epoch) + ``` ### `get_active_validator_indices` @@ -772,12 +774,12 @@ def get_epoch_committee_count(state: BeaconState, epoch: Epoch) -> int: """ Return the number of committees in one epoch. """ - active_validators = get_active_validator_indices(state, epoch) + active_validator_indices = get_active_validator_indices(state, epoch) return max( 1, min( SHARD_COUNT // SLOTS_PER_EPOCH, - len(active_validators) // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE, + len(active_validator_indices) // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE, ) ) * SLOTS_PER_EPOCH ``` @@ -850,11 +852,11 @@ def get_crosslink_committees_at_slot(state: BeaconState, ] ``` -### `get_block_root` +### `get_block_root_at_slot` ```python -def get_block_root(state: BeaconState, - slot: Slot) -> Bytes32: +def get_block_root_at_slot(state: BeaconState, + slot: Slot) -> Bytes32: """ Return the block root at a recent ``slot``. """ @@ -862,19 +864,17 @@ def get_block_root(state: BeaconState, return state.latest_block_roots[slot % SLOTS_PER_HISTORICAL_ROOT] ``` -`get_block_root(_, s)` should always return `signing_root` of the block in the beacon chain at slot `s`, and `get_crosslink_committees_at_slot(_, s)` should not change unless the [validator](#dfn-validator) registry changes. - -### `get_state_root` +### `get_block_root` ```python -def get_state_root(state: BeaconState, - slot: Slot) -> Bytes32: +def get_block_root(state: BeaconState, + epoch: Epoch) -> Bytes32: """ - Return the state root at a recent ``slot``. + Return the block root at a recent ``epoch``. """ - assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT - return state.latest_state_roots[slot % SLOTS_PER_HISTORICAL_ROOT] + return get_block_root_at_slot(state, get_epoch_start_slot(epoch)) ``` + ### `get_randao_mix` ```python @@ -917,17 +917,17 @@ def generate_seed(state: BeaconState, ### `get_beacon_proposer_index` ```python -def get_beacon_proposer_index(state: BeaconState, slot: Slot=None) -> ValidatorIndex: +def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: """ Return the beacon proposer index at ``slot``. """ - epoch = slot_to_epoch(slot if slot != None else state.slot) - first_committee, _ = get_crosslink_committees_at_slot(state, slot if slot != None else state.slot)[0] + epoch = slot_to_epoch(state.slot) + first_committee, _ = get_crosslink_committees_at_slot(state, state.slot)[0] + MAX_RANDOM_BYTE = 2**8 - 1 i = 0 while True: candidate_index = first_committee[(epoch + i) % len(first_committee)] random_byte = hash(generate_seed(state, epoch) + int_to_bytes8(i // 32))[i % 32] - MAX_RANDOM_BYTE = 2**8 - 1 effective_balance = state.validator_registry[candidate_index].effective_balance if effective_balance * MAX_RANDOM_BYTE >= MAX_DEPOSIT_AMOUNT * random_byte: return candidate_index @@ -1449,7 +1449,7 @@ def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> List[P def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: return [ a for a in get_matching_source_attestations(state, epoch) - if a.data.target_root == get_block_root(state, get_epoch_start_slot(epoch)) + if a.data.target_root == get_block_root(state, epoch) ] ``` @@ -1457,7 +1457,7 @@ def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> List[P def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: return [ a for a in get_matching_source_attestations(state, epoch) - if a.data.beacon_block_root == get_block_root(state, a.data.slot) + if a.data.beacon_block_root == get_block_root_at_slot(state, a.data.slot) ] ``` @@ -1484,7 +1484,7 @@ def get_crosslink_from_attestation_data(state: BeaconState, data: AttestationDat ``` ```python -def get_winning_crosslink_and_attesting_indices(state: BeaconState, epoch: Epoch, shard: Shard) -> Tuple[Crosslink, List[ValidatorIndex]]: +def get_winning_crosslink_and_attesting_indices(state: BeaconState, shard: Shard, epoch: Epoch) -> Tuple[Crosslink, List[ValidatorIndex]]: attestations = get_matching_source_attestations(state, epoch) shard_attestations = [a for a in attestations if a.data.shard == shard] shard_crosslinks = [get_crosslink_from_attestation_data(state, a.data) for a in shard_attestations] @@ -1533,12 +1533,12 @@ def process_justification_and_finalization(state: BeaconState) -> None: previous_epoch_matching_target_balance = get_attesting_balance(state, get_matching_target_attestations(state, previous_epoch)) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_epoch = get_previous_epoch(state) - state.current_justified_root = get_block_root(state, get_epoch_start_slot(state.current_justified_epoch)) + state.current_justified_root = get_block_root(state, state.current_justified_epoch) state.justification_bitfield |= (1 << 1) current_epoch_matching_target_balance = get_attesting_balance(state, get_matching_target_attestations(state, current_epoch)) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_epoch = get_current_epoch(state) - state.current_justified_root = get_block_root(state, get_epoch_start_slot(state.current_justified_epoch)) + state.current_justified_root = get_block_root(state, state.current_justified_epoch) state.justification_bitfield |= (1 << 0) # Process finalizations @@ -1546,19 +1546,19 @@ def process_justification_and_finalization(state: BeaconState) -> None: # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_epoch == current_epoch - 3: state.finalized_epoch = old_previous_justified_epoch - state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) + state.finalized_root = get_block_root(state, state.finalized_epoch) # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_epoch == current_epoch - 2: state.finalized_epoch = old_previous_justified_epoch - state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) + state.finalized_root = get_block_root(state, state.finalized_epoch) # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source if (bitfield >> 0) % 8 == 0b111 and old_current_justified_epoch == current_epoch - 2: state.finalized_epoch = old_current_justified_epoch - state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) + state.finalized_root = get_block_root(state, state.finalized_epoch) # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source if (bitfield >> 0) % 4 == 0b11 and old_current_justified_epoch == current_epoch - 1: state.finalized_epoch = old_current_justified_epoch - state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) + state.finalized_root = get_block_root(state, state.finalized_epoch) ``` #### Crosslinks @@ -1573,7 +1573,7 @@ def process_crosslinks(state: BeaconState) -> None: for slot in range(get_epoch_start_slot(previous_epoch), get_epoch_start_slot(next_epoch)): epoch = slot_to_epoch(slot) for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot): - winning_crosslink, attesting_indices = get_winning_crosslink_and_attesting_indices(state, epoch, shard) + winning_crosslink, attesting_indices = get_winning_crosslink_and_attesting_indices(state, shard, epoch) if 3 * get_total_balance(state, attesting_indices) >= 2 * get_total_balance(state, crosslink_committee): state.current_crosslinks[shard] = winning_crosslink ``` @@ -1595,40 +1595,40 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: previous_epoch = get_previous_epoch(state) total_balance = get_total_active_balance(state) - eligible_validator_indices = [ - index for index, validator in enumerate(state.validator_registry) - if is_active_validator(validator, previous_epoch) or (validator.slashed and previous_epoch < validator.withdrawable_epoch) - ] rewards = [0 for index in range(len(state.validator_registry))] penalties = [0 for index in range(len(state.validator_registry))] - for index in eligible_validator_indices: - base_reward = get_base_reward(state, index) + eligible_validator_indices = [ + index for index, v in enumerate(state.validator_registry) + if is_active_validator(v, previous_epoch) or (v.slashed and previous_epoch + 1 < v.withdrawable_epoch) + ] - # Micro-incentives for attestations matching FFG source, FFG target, and head - for attestations in ( - get_matching_source_attestations(state, previous_epoch), - get_matching_target_attestations(state, previous_epoch), - get_matching_source_attestations(state, previous_epoch), - ): - if index in get_unslashed_attesting_indices(state, attestations): - rewards[index] += base_reward * get_attesting_balance(state, attestations) // total_balance + # Micro-incentives for matching FFG source, FFG target, and head + matching_source_attestations = get_matching_source_attestations(state, previous_epoch) + matching_target_attestations = get_matching_target_attestations(state, previous_epoch) + matching_head_attestations = get_matching_head_attestations(state, previous_epoch) + for attestations in (matching_source_attestations, matching_target_attestations, matching_head_attestations): + unslashed_attesting_indices = get_unslashed_attesting_indices(state, attestations) + attesting_balance = get_attesting_balance(state, attestations) + for index in eligible_validator_indices: + if index in unslashed_attesting_indices: + rewards[index] += get_base_reward(state, index) * attesting_balance // total_balance else: - penalties[index] += base_reward + penalties[index] += get_base_reward(state, index) - if index in get_unslashed_attesting_indices(state, get_matching_source_attestations(state, previous_epoch)): - earliest_attestation = get_earliest_attestation(state, get_matching_source_attestations(state, previous_epoch), index) - # Proposer micro-rewards - proposer_index = get_beacon_proposer_index(state, earliest_attestation.inclusion_slot) - rewards[proposer_index] += base_reward // PROPOSER_REWARD_QUOTIENT - # Inclusion delay micro-rewards - inclusion_delay = earliest_attestation.inclusion_slot - earliest_attestation.data.slot - rewards[index] += base_reward * MIN_ATTESTATION_INCLUSION_DELAY // inclusion_delay + # Proposer and inclusion delay micro-rewards + if index in get_unslashed_attesting_indices(state, matching_source_attestations): + earliest_attestation = get_earliest_attestation(state, matching_source_attestations, index) + rewards[earliest_attestation.proposer_index] += get_base_reward(state, index) // PROPOSER_REWARD_QUOTIENT + inclusion_delay = earliest_attestation.inclusion_slot - earliest_attestation.data.slot + rewards[index] += get_base_reward(state, index) * MIN_ATTESTATION_INCLUSION_DELAY // inclusion_delay - # Inactivity penalty - finality_delay = previous_epoch - state.finalized_epoch - if finality_delay > MIN_EPOCHS_TO_INACTIVITY_PENALTY: - penalties[index] += BASE_REWARDS_PER_EPOCH * base_reward - if index not in get_unslashed_attesting_indices(state, get_matching_target_attestations(state, previous_epoch)): + # Inactivity penalty + finality_delay = previous_epoch - state.finalized_epoch + if finality_delay > MIN_EPOCHS_TO_INACTIVITY_PENALTY: + matching_target_attesting_indices = get_unslashed_attesting_indices(state, matching_target_attestations) + for index in eligible_validator_indices: + penalties[index] += BASE_REWARDS_PER_EPOCH * get_base_reward(state, index) + if index not in matching_target_attesting_indices: penalties[index] += state.validator_registry[index].effective_balance * finality_delay // INACTIVITY_PENALTY_QUOTIENT return [rewards, penalties] @@ -1641,7 +1641,7 @@ def get_crosslink_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: for slot in range(get_epoch_start_slot(get_previous_epoch(state)), get_epoch_start_slot(get_current_epoch(state))): epoch = slot_to_epoch(slot) for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot): - winning_crosslink, attesting_indices = get_winning_crosslink_and_attesting_indices(state, epoch, shard) + winning_crosslink, attesting_indices = get_winning_crosslink_and_attesting_indices(state, shard, epoch) attesting_balance = get_total_balance(state, attesting_indices) committee_balance = get_total_balance(state, crosslink_committee) for index in crosslink_committee: @@ -1915,7 +1915,8 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: pending_attestation = PendingAttestation( data=data, aggregation_bitfield=attestation.aggregation_bitfield, - inclusion_slot=state.slot + inclusion_slot=state.slot, + proposer_index=get_beacon_proposer_index(state), ) if target_epoch == get_current_epoch(state): state.current_epoch_attestations.append(pending_attestation) From 5587c44abe725afccd08fdad1667a03c0ada87f2 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 24 Apr 2019 14:29:35 +1000 Subject: [PATCH 097/121] Update test_libs/pyspec/tests/test_sanity.py Co-Authored-By: JustinDrake --- test_libs/pyspec/tests/test_sanity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index f87a7c808..101bcc89c 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -436,7 +436,7 @@ def test_balance_driven_status_transitions(state): assert pre_state.validator_registry[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH # set validator balance to below ejection threshold - pre_state.validator_registry[validator_index].effective_balance = spec.EJECTION_BALANCE - 1 + pre_state.validator_registry[validator_index].effective_balance = spec.EJECTION_BALANCE post_state = deepcopy(pre_state) # From df64eeefa07eb58364fe1eb86eb351f63bf32e86 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Wed, 24 Apr 2019 14:46:28 +1000 Subject: [PATCH 098/121] Start fixing tests --- specs/core/0_beacon-chain.md | 8 ++++---- .../tests/block_processing/test_process_transfer.py | 10 ++++------ test_libs/pyspec/tests/helpers.py | 1 - 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 033e3b2cb..159d2d839 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -931,7 +931,7 @@ def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: candidate_index = first_committee[(current_epoch + i) % len(first_committee)] random_byte = hash(generate_seed(state, epoch) + int_to_bytes8(i // 32))[i % 32] effective_balance = state.validator_registry[candidate_index].effective_balance - if effective_balance * MAX_RANDOM_BYTE >= MAX_DEPOSIT_AMOUNT * random_byte: + if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: return candidate_index i += 1 ``` @@ -1291,7 +1291,7 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], # Process genesis activations for index, validator in enumerate(state.validator_registry): - if validator.effective_balance >= MAX_DEPOSIT_AMOUNT: + if validator.effective_balance >= MAX_EFFECTIVE_BALANCE: validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH @@ -1662,7 +1662,7 @@ Run the following function: def process_registry_updates(state: BeaconState) -> None: # Process activation eligibility and ejections for index, validator in enumerate(state.validator_registry): - if validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH and validator.effective_balance >= MAX_DEPOSIT_AMOUNT: + if validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH and validator.effective_balance >= MAX_EFFECTIVE_BALANCE: validator.activation_eligibility_epoch = get_current_epoch(state) if is_active_validator(validator, get_current_epoch(state)) and validator.effective_balance <= EJECTION_BALANCE: @@ -1717,7 +1717,7 @@ def process_final_updates(state: BeaconState) -> None: state.eth1_data_votes = [] # Update effective balances with hysteresis for index, validator in enumerate(state.validator_registry): - balance = min(state.balances[index], MAX_DEPOSIT_AMOUNT) + balance = min(state.balances[index], MAX_EFFECTIVE_BALANCE) HALF_INCREMENT = EFFECTIVE_BALANCE_INCREMENT // 2 if balance < validator.effective_balance or validator.effective_balance + 3 * HALF_INCREMENT < balance: validator.effective_balance = balance - balance % EFFECTIVE_BALANCE_INCREMENT diff --git a/test_libs/pyspec/tests/block_processing/test_process_transfer.py b/test_libs/pyspec/tests/block_processing/test_process_transfer.py index 71d0894bd..65df822de 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/tests/block_processing/test_process_transfer.py @@ -5,11 +5,9 @@ import eth2spec.phase0.spec as spec from eth2spec.phase0.spec import ( get_active_validator_indices, - get_balance, get_beacon_proposer_index, get_current_epoch, process_transfer, - set_balance, ) from tests.helpers import ( get_valid_transfer, @@ -75,7 +73,7 @@ def test_success_withdrawable(state): def test_success_active_above_max_effective(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] amount = spec.MAX_EFFECTIVE_BALANCE // 32 - set_balance(state, sender_index, spec.MAX_EFFECTIVE_BALANCE + amount) + state.validator_registry[sender_index] = spec.MAX_EFFECTIVE_BALANCE + amount transfer = get_valid_transfer(state, sender_index=sender_index, amount=amount, fee=0) pre_state, post_state = run_transfer_processing(state, transfer) @@ -86,7 +84,7 @@ def test_success_active_above_max_effective(state): def test_active_but_transfer_past_effective_balance(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] amount = spec.MAX_EFFECTIVE_BALANCE // 32 - set_balance(state, sender_index, spec.MAX_EFFECTIVE_BALANCE) + state.validator_registry[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(state, sender_index=sender_index, amount=amount, fee=0) pre_state, post_state = run_transfer_processing(state, transfer, False) @@ -107,7 +105,7 @@ def test_incorrect_slot(state): def test_insufficient_balance(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] amount = spec.MAX_EFFECTIVE_BALANCE - set_balance(state, sender_index, spec.MAX_EFFECTIVE_BALANCE) + state.validator_registry[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(state, sender_index=sender_index, amount=amount + 1, fee=0) # un-activate so validator can transfer @@ -140,4 +138,4 @@ def test_invalid_pubkey(state): pre_state, post_state = run_transfer_processing(state, transfer, False) - return pre_state, transfer, post_state \ No newline at end of file + return pre_state, transfer, post_state diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index 162c6fa62..465825c35 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -26,7 +26,6 @@ from eth2spec.phase0.spec import ( # functions convert_to_indexed, get_active_validator_indices, - get_balance, get_attesting_indices, get_block_root, get_crosslink_committees_at_slot, From 55f042aa71c8a6a3f62e803077ecb77f33a3b432 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Wed, 24 Apr 2019 15:17:25 +1000 Subject: [PATCH 099/121] More fixes --- specs/core/0_beacon-chain.md | 4 ++-- test_libs/pyspec/tests/helpers.py | 9 ++++++--- test_libs/pyspec/tests/test_sanity.py | 16 ++++++++++++---- tests/phase0/helpers.py | 3 ++- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 159d2d839..3dd4c944c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -929,7 +929,7 @@ def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: i = 0 while True: candidate_index = first_committee[(current_epoch + i) % len(first_committee)] - random_byte = hash(generate_seed(state, epoch) + int_to_bytes8(i // 32))[i % 32] + random_byte = hash(generate_seed(state, current_epoch) + int_to_bytes8(i // 32))[i % 32] effective_balance = state.validator_registry[candidate_index].effective_balance if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: return candidate_index @@ -2011,7 +2011,7 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: assert ( state.validator_registry[transfer.sender].activation_eligibility_epoch == FAR_FUTURE_EPOCH or get_current_epoch(state) >= state.validator_registry[transfer.sender].withdrawable_epoch or - transfer.amount + transfer.fee + MAX_EFFECTIVE_BALANCE <= get_balance(state, transfer.sender) + transfer.amount + transfer.fee + MAX_EFFECTIVE_BALANCE <= state.balances[transfer.sender] ) # Verify that the pubkey is valid assert ( diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index 465825c35..cf3d2624a 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -28,6 +28,7 @@ from eth2spec.phase0.spec import ( get_active_validator_indices, get_attesting_indices, get_block_root, + get_block_root_at_slot, get_crosslink_committees_at_slot, get_current_epoch, get_domain, @@ -51,9 +52,11 @@ privkeys = [i + 1 for i in range(1000)] pubkeys = [bls.privtopub(privkey) for privkey in privkeys] pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} + def get_balance(state, index): return state.balances[index] + def set_bitfield_bit(bitfield, i): """ Set the bit in ``bitfield`` at position ``i`` to ``1``. @@ -151,16 +154,16 @@ def build_attestation_data(state, slot, shard): if slot == state.slot: block_root = build_empty_block_for_next_slot(state).previous_block_root else: - block_root = get_block_root(state, slot) + block_root = get_block_root_at_slot(state, slot) current_epoch_start_slot = get_epoch_start_slot(get_current_epoch(state)) if slot < current_epoch_start_slot: print(slot) - epoch_boundary_root = get_block_root(state, get_epoch_start_slot(get_previous_epoch(state))) + epoch_boundary_root = get_block_root(state, get_previous_epoch(state)) elif slot == current_epoch_start_slot: epoch_boundary_root = block_root else: - epoch_boundary_root = get_block_root(state, current_epoch_start_slot) + epoch_boundary_root = get_block_root(state, get_current_epoch(state)) if slot < current_epoch_start_slot: justified_epoch = state.previous_justified_epoch diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index 182c9cfe0..b54b5f75a 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -9,6 +9,7 @@ from eth2spec.utils.minimal_ssz import signing_root from eth2spec.phase0.spec import ( # constants ZERO_HASH, + SLOTS_PER_HISTORICAL_ROOT, # SSZ Deposit, Transfer, @@ -17,9 +18,9 @@ from eth2spec.phase0.spec import ( get_active_validator_indices, get_beacon_proposer_index, get_block_root, + get_block_root_at_slot, get_current_epoch, get_domain, - get_state_root, advance_slot, cache_state, slot_to_epoch, @@ -48,6 +49,13 @@ from .helpers import ( ) +def get_state_root(state, slot) -> bytes: + """ + Return the state root at a recent ``slot``. + """ + assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT + return state.latest_state_roots[slot % SLOTS_PER_HISTORICAL_ROOT] + # mark entire file as 'sanity' pytestmark = pytest.mark.sanity @@ -94,7 +102,7 @@ def test_empty_block_transition(state): state_transition(test_state, block) assert len(test_state.eth1_data_votes) == len(state.eth1_data_votes) + 1 - assert get_block_root(test_state, state.slot) == block.previous_block_root + assert get_block_root_at_slot(test_state, state.slot) == block.previous_block_root return state, [block], test_state @@ -108,7 +116,7 @@ def test_skipped_slots(state): assert test_state.slot == block.slot for slot in range(state.slot, test_state.slot): - assert get_block_root(test_state, slot) == block.previous_block_root + assert get_block_root_at_slot(test_state, slot) == block.previous_block_root return state, [block], test_state @@ -122,7 +130,7 @@ def test_empty_epoch_transition(state): assert test_state.slot == block.slot for slot in range(state.slot, test_state.slot): - assert get_block_root(test_state, slot) == block.previous_block_root + assert get_block_root_at_slot(test_state, slot) == block.previous_block_root return state, [block], test_state diff --git a/tests/phase0/helpers.py b/tests/phase0/helpers.py index 20054b821..c042dec91 100644 --- a/tests/phase0/helpers.py +++ b/tests/phase0/helpers.py @@ -16,6 +16,7 @@ from build.phase0.spec import ( VoluntaryExit, # functions get_block_root, + get_block_root_at_slot, get_current_epoch, get_domain, get_empty_block, @@ -141,7 +142,7 @@ def build_attestation_data(state, slot, shard): if epoch_start_slot == slot: epoch_boundary_root = block_root else: - get_block_root(state, epoch_start_slot) + get_block_root(state, get_current_epoch(state)) if slot < epoch_start_slot: justified_block_root = state.previous_justified_root From c37789dc5d5085b535d8d71ae2dc608fdb2fdd7d Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Wed, 24 Apr 2019 15:27:47 +1000 Subject: [PATCH 100/121] Tests fixed --- specs/core/0_beacon-chain.md | 4 ++-- .../pyspec/tests/block_processing/test_process_transfer.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 3dd4c944c..295064f77 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -921,9 +921,9 @@ def generate_seed(state: BeaconState, ```python def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: """ - Return the beacon proposer index at ``slot``. + Return the beacon proposer index at ``state.slot``. """ - current_epoch = slot_to_epoch(state.slot) + current_epoch = get_current_epoch(state) first_committee, _ = get_crosslink_committees_at_slot(state, state.slot)[0] MAX_RANDOM_BYTE = 2**8 - 1 i = 0 diff --git a/test_libs/pyspec/tests/block_processing/test_process_transfer.py b/test_libs/pyspec/tests/block_processing/test_process_transfer.py index 65df822de..0eeaa7792 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/tests/block_processing/test_process_transfer.py @@ -73,7 +73,7 @@ def test_success_withdrawable(state): def test_success_active_above_max_effective(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] amount = spec.MAX_EFFECTIVE_BALANCE // 32 - state.validator_registry[sender_index] = spec.MAX_EFFECTIVE_BALANCE + amount + state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + amount transfer = get_valid_transfer(state, sender_index=sender_index, amount=amount, fee=0) pre_state, post_state = run_transfer_processing(state, transfer) @@ -84,7 +84,7 @@ def test_success_active_above_max_effective(state): def test_active_but_transfer_past_effective_balance(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] amount = spec.MAX_EFFECTIVE_BALANCE // 32 - state.validator_registry[sender_index] = spec.MAX_EFFECTIVE_BALANCE + state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(state, sender_index=sender_index, amount=amount, fee=0) pre_state, post_state = run_transfer_processing(state, transfer, False) @@ -105,7 +105,7 @@ def test_incorrect_slot(state): def test_insufficient_balance(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] amount = spec.MAX_EFFECTIVE_BALANCE - state.validator_registry[sender_index] = spec.MAX_EFFECTIVE_BALANCE + state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(state, sender_index=sender_index, amount=amount + 1, fee=0) # un-activate so validator can transfer From b361fdb385e1fd4d025f1e342e2f9d303f4bf642 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Wed, 24 Apr 2019 15:29:46 +1000 Subject: [PATCH 101/121] bug --- specs/core/0_beacon-chain.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 295064f77..c5f13710b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1603,11 +1603,12 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: penalties[index] += get_base_reward(state, index) # Proposer and inclusion delay micro-rewards - if index in get_unslashed_attesting_indices(state, matching_source_attestations): - earliest_attestation = get_earliest_attestation(state, matching_source_attestations, index) - rewards[earliest_attestation.proposer_index] += get_base_reward(state, index) // PROPOSER_REWARD_QUOTIENT - inclusion_delay = earliest_attestation.inclusion_slot - earliest_attestation.data.slot - rewards[index] += get_base_reward(state, index) * MIN_ATTESTATION_INCLUSION_DELAY // inclusion_delay + for index in eligible_validator_indices: + if index in get_unslashed_attesting_indices(state, matching_source_attestations): + earliest_attestation = get_earliest_attestation(state, matching_source_attestations, index) + rewards[earliest_attestation.proposer_index] += get_base_reward(state, index) // PROPOSER_REWARD_QUOTIENT + inclusion_delay = earliest_attestation.inclusion_slot - earliest_attestation.data.slot + rewards[index] += get_base_reward(state, index) * MIN_ATTESTATION_INCLUSION_DELAY // inclusion_delay # Inactivity penalty finality_delay = previous_epoch - state.finalized_epoch From 4734b2288348fb9e9a2b1cd803872f4e61a8be42 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Wed, 24 Apr 2019 15:32:43 +1000 Subject: [PATCH 102/121] simplify --- specs/core/0_beacon-chain.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index c5f13710b..4a9dcc26c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1603,12 +1603,11 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: penalties[index] += get_base_reward(state, index) # Proposer and inclusion delay micro-rewards - for index in eligible_validator_indices: - if index in get_unslashed_attesting_indices(state, matching_source_attestations): - earliest_attestation = get_earliest_attestation(state, matching_source_attestations, index) - rewards[earliest_attestation.proposer_index] += get_base_reward(state, index) // PROPOSER_REWARD_QUOTIENT - inclusion_delay = earliest_attestation.inclusion_slot - earliest_attestation.data.slot - rewards[index] += get_base_reward(state, index) * MIN_ATTESTATION_INCLUSION_DELAY // inclusion_delay + for index in get_unslashed_attesting_indices(state, matching_source_attestations): + earliest_attestation = get_earliest_attestation(state, matching_source_attestations, index) + rewards[earliest_attestation.proposer_index] += get_base_reward(state, index) // PROPOSER_REWARD_QUOTIENT + inclusion_delay = earliest_attestation.inclusion_slot - earliest_attestation.data.slot + rewards[index] += get_base_reward(state, index) * MIN_ATTESTATION_INCLUSION_DELAY // inclusion_delay # Inactivity penalty finality_delay = previous_epoch - state.finalized_epoch From 1284b93416cebb0fd4b58059ed72aa7332a78814 Mon Sep 17 00:00:00 2001 From: Justin Date: Wed, 24 Apr 2019 15:53:28 +1000 Subject: [PATCH 103/121] Update simple-serialize.md --- specs/simple-serialize.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 3f02335db..882147f0f 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -55,7 +55,7 @@ For convenience we alias: We recursively define the `serialize` function which consumes an object `value` (of the type specified) and returns a bytestring of type `"bytes"`. -> *Note*: In the function definitions below (`serialize`, `hash_tree_root`, `signed_root`, `is_fixed_size`, `is_variable_size`, etc.) objects implicitly carry their type. +> *Note*: In the function definitions below (`serialize`, `hash_tree_root`, `signing_root`, `is_variable_size`, etc.) objects implicitly carry their type. ### `"uintN"` @@ -75,8 +75,8 @@ return b"\x01" if value is True else b"\x00" ```python # Reccursively serialize -fixed_parts = [serialize(element) if is_fixed_size(element) else None for element in value] -variable_parts = [serialize(element) if is_variable_size(element) else "" for element in value] +fixed_parts = [serialize(element) if not is_variable_size(element) else None for element in value] +variable_parts = [serialize(element) if is_variable_size(element) else b"" for element in value] # Compute and check lengths fixed_lengths = [len(part) if part != None else BYTES_PER_LENGTH_OFFSET for part in fixed_parts] @@ -88,7 +88,7 @@ variable_offsets = [serialize(sum(fixed_lengths + variable_lengths[:i])) for i i fixed_parts = [part if part != None else variable_offsets[i] for i, part in enumerate(fixed_parts)] # Return the concatenation of the fixed-size parts (offsets interleaved) with the variable-size parts -return "".join(fixed_parts + variable_parts) +return b"".join(fixed_parts + variable_parts) ``` ## Deserialization @@ -112,7 +112,7 @@ We now define Merkleization `hash_tree_root(value)` of an object `value` recursi ## Self-signed containers -Let `value` be a self-signed container object. The convention is that the signature (e.g. a `"bytes96"` BLS12-381 signature) be the last field of `value`. Further, the signed message for `value` is `signed_root(value) = hash_tree_root(truncate_last(value))` where `truncate_last` truncates the last element of `value`. +Let `value` be a self-signed container object. The convention is that the signature (e.g. a `"bytes96"` BLS12-381 signature) be the last field of `value`. Further, the signed message for `value` is `signing_root(value) = hash_tree_root(truncate_last(value))` where `truncate_last` truncates the last element of `value`. ## Implementations From 7f5cffb2863eba1cae815e59f00d87e5b14c062d Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 22 Apr 2019 17:46:13 +1000 Subject: [PATCH 104/121] pytests use configuration system now, add command option to conftest to switch, also fix minor testing bug --- Makefile | 4 +- test_libs/pyspec/README.md | 4 +- test_libs/pyspec/requirements-testing.txt | 3 ++ test_libs/pyspec/requirements.txt | 1 - test_libs/pyspec/tests/README.md | 0 test_libs/pyspec/tests/conftest.py | 52 ++++------------------- test_libs/pyspec/tests/helpers.py | 1 + 7 files changed, 17 insertions(+), 48 deletions(-) create mode 100644 test_libs/pyspec/requirements-testing.txt delete mode 100644 test_libs/pyspec/tests/README.md diff --git a/Makefile b/Makefile index 6586646fd..d00817daf 100644 --- a/Makefile +++ b/Makefile @@ -34,10 +34,10 @@ install_test: cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt; test: $(PY_SPEC_ALL_TARGETS) - cd $(PY_SPEC_DIR); . venv/bin/activate; python -m pytest -m minimal_config . + cd $(PY_SPEC_DIR); . venv/bin/activate; python -m pytest . citest: $(PY_SPEC_ALL_TARGETS) - cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; python -m pytest --junitxml=test-reports/eth2spec/test_results.xml -m minimal_config . + cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; python -m pytest --junitxml=test-reports/eth2spec/test_results.xml . # "make pyspec" to create the pyspec for all phases. pyspec: $(PY_SPEC_ALL_TARGETS) diff --git a/test_libs/pyspec/README.md b/test_libs/pyspec/README.md index b9bf86220..df1834210 100644 --- a/test_libs/pyspec/README.md +++ b/test_libs/pyspec/README.md @@ -38,7 +38,7 @@ Install dependencies: ```bash python3 -m venv venv . venv/bin/activate -pip3 install -r requirements.txt +pip3 install -r requirements-testing.txt ``` Note: make sure to run `make -B pyspec` from the root of the specs repository, to build the parts of the pyspec module derived from the markdown specs. @@ -46,7 +46,7 @@ The `-B` flag may be helpful to force-overwrite the `pyspec` output after you ma Run the tests: ``` -pytest -m minimal_config . +pytest --config=minimal ``` diff --git a/test_libs/pyspec/requirements-testing.txt b/test_libs/pyspec/requirements-testing.txt new file mode 100644 index 000000000..388a878a9 --- /dev/null +++ b/test_libs/pyspec/requirements-testing.txt @@ -0,0 +1,3 @@ +-r requirements.txt +pytest>=3.6,<3.7 +../config_helpers diff --git a/test_libs/pyspec/requirements.txt b/test_libs/pyspec/requirements.txt index 3296ef807..78d41708d 100644 --- a/test_libs/pyspec/requirements.txt +++ b/test_libs/pyspec/requirements.txt @@ -2,4 +2,3 @@ eth-utils>=1.3.0,<2 eth-typing>=2.1.0,<3.0.0 pycryptodome==3.7.3 py_ecc>=1.6.0 -pytest>=3.6,<3.7 diff --git a/test_libs/pyspec/tests/README.md b/test_libs/pyspec/tests/README.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/test_libs/pyspec/tests/conftest.py b/test_libs/pyspec/tests/conftest.py index bf9b1009b..f6c70d280 100644 --- a/test_libs/pyspec/tests/conftest.py +++ b/test_libs/pyspec/tests/conftest.py @@ -1,58 +1,24 @@ import pytest from eth2spec.phase0 import spec +from preset_loader import loader from .helpers import ( create_genesis_state, ) -DEFAULT_CONFIG = {} # no change - -MINIMAL_CONFIG = { - "SHARD_COUNT": 8, - "MIN_ATTESTATION_INCLUSION_DELAY": 2, - "TARGET_COMMITTEE_SIZE": 4, - "SLOTS_PER_EPOCH": 8, - "SLOTS_PER_HISTORICAL_ROOT": 64, - "LATEST_RANDAO_MIXES_LENGTH": 64, - "LATEST_ACTIVE_INDEX_ROOTS_LENGTH": 64, - "LATEST_SLASHED_EXIT_LENGTH": 64, -} - - -def overwrite_spec_config(config): - for field in config: - setattr(spec, field, config[field]) - if field == "LATEST_RANDAO_MIXES_LENGTH": - spec.BeaconState.fields['latest_randao_mixes'][1] = config[field] - elif field == "SHARD_COUNT": - spec.BeaconState.fields['current_crosslinks'][1] = config[field] - spec.BeaconState.fields['previous_crosslinks'][1] = config[field] - elif field == "SLOTS_PER_HISTORICAL_ROOT": - spec.BeaconState.fields['latest_block_roots'][1] = config[field] - spec.BeaconState.fields['latest_state_roots'][1] = config[field] - spec.HistoricalBatch.fields['block_roots'][1] = config[field] - spec.HistoricalBatch.fields['state_roots'][1] = config[field] - elif field == "LATEST_ACTIVE_INDEX_ROOTS_LENGTH": - spec.BeaconState.fields['latest_active_index_roots'][1] = config[field] - elif field == "LATEST_SLASHED_EXIT_LENGTH": - spec.BeaconState.fields['latest_slashed_balances'][1] = config[field] - - -@pytest.fixture( - params=[ - pytest.param(MINIMAL_CONFIG, marks=pytest.mark.minimal_config), - DEFAULT_CONFIG, - ] -) -def config(request): - return request.param +def pytest_addoption(parser): + parser.addoption( + "--config", action="store", default="minimal", help="config: make the pyspec use the specified configuration" + ) @pytest.fixture(autouse=True) -def overwrite_config(config): - overwrite_spec_config(config) +def config(request): + config_name = request.config.getoption("--config") + presets = loader.load_presets('../../configs/', config_name) + spec.apply_constants_preset(presets) @pytest.fixture diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index d181dadfe..a728a6acc 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -118,6 +118,7 @@ def create_genesis_state(num_validators, deposit_data_leaves=None): def build_empty_block_for_next_slot(state): empty_block = BeaconBlock() empty_block.slot = state.slot + 1 + empty_block.body.eth1_data.deposit_count = state.deposit_index previous_block_header = deepcopy(state.latest_block_header) if previous_block_header.state_root == spec.ZERO_HASH: previous_block_header.state_root = state.hash_tree_root() From aaafe92c5f3a59fd4b902b6ef89138daac760430 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 22 Apr 2019 17:54:58 +1000 Subject: [PATCH 105/121] update makefile to install requirements for tests correctly --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d00817daf..73d8adea8 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ gen_yaml_tests: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_TARGETS) # installs the packages to run pyspec tests install_test: - cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt; + cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements-testing.txt; test: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; python -m pytest . From 02e6d5b46ca74678567bb1f43cb4d25def9c8371 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 22 Apr 2019 17:59:01 +1000 Subject: [PATCH 106/121] make circle CI aware of the testing requirments file --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a7172866..4f806b00f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -61,13 +61,13 @@ jobs: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: venv_name: v1-pyspec - reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}' + reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Install pyspec requirements command: make install_test - save_cached_venv: venv_name: v1-pyspec - reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}' + reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' venv_path: ./test_libs/pyspec/venv test: docker: @@ -78,7 +78,7 @@ jobs: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: venv_name: v1-pyspec - reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}' + reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run py-tests command: make citest From 97906a633910b8989576dc7371f4ca438d435cdf Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 22 Apr 2019 18:07:55 +0800 Subject: [PATCH 107/121] Only use `setup.py` --- .circleci/config.yml | 6 ++--- Makefile | 2 +- test_generators/README.md | 2 +- test_libs/config_helpers/requirements.txt | 1 - test_libs/config_helpers/setup.py | 15 +++++++++--- test_libs/gen_helpers/requirements.txt | 2 -- test_libs/gen_helpers/setup.py | 18 ++++++++++---- test_libs/pyspec/README.md | 2 +- test_libs/pyspec/requirements-testing.txt | 3 --- test_libs/pyspec/requirements.txt | 4 ---- test_libs/pyspec/setup.py | 29 ++++++++++++++++++----- 11 files changed, 55 insertions(+), 29 deletions(-) delete mode 100644 test_libs/config_helpers/requirements.txt delete mode 100644 test_libs/gen_helpers/requirements.txt delete mode 100644 test_libs/pyspec/requirements-testing.txt delete mode 100644 test_libs/pyspec/requirements.txt diff --git a/.circleci/config.yml b/.circleci/config.yml index 4f806b00f..28d949de4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -61,13 +61,13 @@ jobs: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: venv_name: v1-pyspec - reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' + reqs_checksum: '{{ checksum "test_libs/pyspec/setup.py" }}' - run: name: Install pyspec requirements command: make install_test - save_cached_venv: venv_name: v1-pyspec - reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' + reqs_checksum: '{{ checksum "test_libs/pyspec/setup.py" }}' venv_path: ./test_libs/pyspec/venv test: docker: @@ -78,7 +78,7 @@ jobs: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: venv_name: v1-pyspec - reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' + reqs_checksum: '{{ checksum "test_libs/pyspec/setup.py" }}' - run: name: Run py-tests command: make citest diff --git a/Makefile b/Makefile index 73d8adea8..10550750a 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ gen_yaml_tests: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_TARGETS) # installs the packages to run pyspec tests install_test: - cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements-testing.txt; + cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -e .[dev]; test: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; python -m pytest . diff --git a/test_generators/README.md b/test_generators/README.md index 66534e5a8..94db105c0 100644 --- a/test_generators/README.md +++ b/test_generators/README.md @@ -72,7 +72,7 @@ Note: make sure to run `make pyspec` from the root of the specs repository, to b Install all the necessary requirements (re-run when you add more): ```bash -pip3 install -r requirements.txt +pip3 install -e .[pyspec] ``` And write your initial test generator, extending the base generator: diff --git a/test_libs/config_helpers/requirements.txt b/test_libs/config_helpers/requirements.txt deleted file mode 100644 index e441a474b..000000000 --- a/test_libs/config_helpers/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -ruamel.yaml==0.15.87 diff --git a/test_libs/config_helpers/setup.py b/test_libs/config_helpers/setup.py index 90ad94ee4..b69d19cdb 100644 --- a/test_libs/config_helpers/setup.py +++ b/test_libs/config_helpers/setup.py @@ -1,9 +1,18 @@ from distutils.core import setup + +deps = { + 'config_helpers': [ + "ruamel.yaml==0.15.87", + ], +} + +deps['dev'] = ( + deps['config_helpers'] +) + setup( name='config_helpers', packages=['preset_loader'], - install_requires=[ - "ruamel.yaml==0.15.87" - ] + install_requires=deps['config_helpers'] ) diff --git a/test_libs/gen_helpers/requirements.txt b/test_libs/gen_helpers/requirements.txt deleted file mode 100644 index 3d6a39458..000000000 --- a/test_libs/gen_helpers/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -ruamel.yaml==0.15.87 -eth-utils==1.4.1 diff --git a/test_libs/gen_helpers/setup.py b/test_libs/gen_helpers/setup.py index 5de27a6db..d4d331c06 100644 --- a/test_libs/gen_helpers/setup.py +++ b/test_libs/gen_helpers/setup.py @@ -1,10 +1,20 @@ from distutils.core import setup + +deps = { + 'gen_helpers': [ + "ruamel.yaml==0.15.87", + "eth-utils==1.4.1", + ], +} + +deps['dev'] = ( + deps['gen_helpers'] +) + + setup( name='gen_helpers', packages=['gen_base'], - install_requires=[ - "ruamel.yaml==0.15.87", - "eth-utils==1.4.1" - ] + install_requires=deps['gen_helpers'], ) diff --git a/test_libs/pyspec/README.md b/test_libs/pyspec/README.md index df1834210..ab2967024 100644 --- a/test_libs/pyspec/README.md +++ b/test_libs/pyspec/README.md @@ -38,7 +38,7 @@ Install dependencies: ```bash python3 -m venv venv . venv/bin/activate -pip3 install -r requirements-testing.txt +pip3 install -e .[dev] ``` Note: make sure to run `make -B pyspec` from the root of the specs repository, to build the parts of the pyspec module derived from the markdown specs. diff --git a/test_libs/pyspec/requirements-testing.txt b/test_libs/pyspec/requirements-testing.txt deleted file mode 100644 index 388a878a9..000000000 --- a/test_libs/pyspec/requirements-testing.txt +++ /dev/null @@ -1,3 +0,0 @@ --r requirements.txt -pytest>=3.6,<3.7 -../config_helpers diff --git a/test_libs/pyspec/requirements.txt b/test_libs/pyspec/requirements.txt deleted file mode 100644 index 78d41708d..000000000 --- a/test_libs/pyspec/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -eth-utils>=1.3.0,<2 -eth-typing>=2.1.0,<3.0.0 -pycryptodome==3.7.3 -py_ecc>=1.6.0 diff --git a/test_libs/pyspec/setup.py b/test_libs/pyspec/setup.py index 1a131a417..13ea4d6e5 100644 --- a/test_libs/pyspec/setup.py +++ b/test_libs/pyspec/setup.py @@ -1,13 +1,30 @@ from setuptools import setup, find_packages -setup( - name='pyspec', - packages=find_packages(), - tests_require=["pytest"], - install_requires=[ + + + +deps = { + 'pyspec': [ "eth-utils>=1.3.0,<2", "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.7.3", "py_ecc>=1.6.0", - ] + ], + 'test': [ + "pytest>=3.6,<3.7", + ], +} + +deps['dev'] = ( + deps['pyspec'] + + deps['test'] +) + +install_requires = deps['pyspec'] + +setup( + name='pyspec', + packages=find_packages(), + install_requires=install_requires, + extras_require=deps, ) From 0fc31e3e7d2404567fd1d6e617ebcfa8e89e6dec Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 23 Apr 2019 09:36:18 +0800 Subject: [PATCH 108/121] Update .gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 3dd86fc80..0419f51b4 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,10 @@ venv .venvs .venv /.pytest_cache +*.egg +*.egg-info +eggs +.eggs build/ output/ From ba99f8a284709a7bb648dbde59c22d4a9eb5d6e5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 23 Apr 2019 10:12:13 +0800 Subject: [PATCH 109/121] Update `install_requires` --- test_libs/config_helpers/setup.py | 8 +++++--- test_libs/gen_helpers/setup.py | 7 ++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/test_libs/config_helpers/setup.py b/test_libs/config_helpers/setup.py index b69d19cdb..836993d36 100644 --- a/test_libs/config_helpers/setup.py +++ b/test_libs/config_helpers/setup.py @@ -2,17 +2,19 @@ from distutils.core import setup deps = { - 'config_helpers': [ + 'preset_loader': [ "ruamel.yaml==0.15.87", ], } deps['dev'] = ( - deps['config_helpers'] + deps['preset_loader'] ) +install_requires = deps['preset_loader'] + setup( name='config_helpers', packages=['preset_loader'], - install_requires=deps['config_helpers'] + install_requires=install_requires, ) diff --git a/test_libs/gen_helpers/setup.py b/test_libs/gen_helpers/setup.py index d4d331c06..6d4003573 100644 --- a/test_libs/gen_helpers/setup.py +++ b/test_libs/gen_helpers/setup.py @@ -2,19 +2,20 @@ from distutils.core import setup deps = { - 'gen_helpers': [ + 'gen_base': [ "ruamel.yaml==0.15.87", "eth-utils==1.4.1", ], } deps['dev'] = ( - deps['gen_helpers'] + deps['gen_base'] ) +install_requires = deps['gen_base'] setup( name='gen_helpers', packages=['gen_base'], - install_requires=deps['gen_helpers'], + install_requires=install_requires, ) From 5437273e23c4380b33cf94aaae065fbb5686d2a7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 23 Apr 2019 10:15:01 +0800 Subject: [PATCH 110/121] Update `packages` --- .gitignore | 3 +++ test_libs/config_helpers/setup.py | 4 ++-- test_libs/gen_helpers/setup.py | 4 ++-- test_libs/pyspec/setup.py | 4 +--- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 0419f51b4..84938d298 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ eth2.0-spec-tests/ # Dynamically built from Markdown spec test_libs/pyspec/eth2spec/phase0/spec.py + +# vscode +.vscode/** diff --git a/test_libs/config_helpers/setup.py b/test_libs/config_helpers/setup.py index 836993d36..88669c092 100644 --- a/test_libs/config_helpers/setup.py +++ b/test_libs/config_helpers/setup.py @@ -1,4 +1,4 @@ -from distutils.core import setup +from setuptools import setup, find_packages deps = { @@ -15,6 +15,6 @@ install_requires = deps['preset_loader'] setup( name='config_helpers', - packages=['preset_loader'], + packages=find_packages(exclude=["tests", "tests.*"]), install_requires=install_requires, ) diff --git a/test_libs/gen_helpers/setup.py b/test_libs/gen_helpers/setup.py index 6d4003573..0c84ff2e2 100644 --- a/test_libs/gen_helpers/setup.py +++ b/test_libs/gen_helpers/setup.py @@ -1,4 +1,4 @@ -from distutils.core import setup +from setuptools import setup, find_packages deps = { @@ -16,6 +16,6 @@ install_requires = deps['gen_base'] setup( name='gen_helpers', - packages=['gen_base'], + packages=find_packages(exclude=["tests", "tests.*"]), install_requires=install_requires, ) diff --git a/test_libs/pyspec/setup.py b/test_libs/pyspec/setup.py index 13ea4d6e5..3fd9d4c0f 100644 --- a/test_libs/pyspec/setup.py +++ b/test_libs/pyspec/setup.py @@ -1,8 +1,6 @@ from setuptools import setup, find_packages - - deps = { 'pyspec': [ "eth-utils>=1.3.0,<2", @@ -24,7 +22,7 @@ install_requires = deps['pyspec'] setup( name='pyspec', - packages=find_packages(), + packages=find_packages(exclude=["tests", "tests.*"]), install_requires=install_requires, extras_require=deps, ) From b1874dc18b6cd7cc65dd52ce4f9b9c8e9ad14f3c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 23 Apr 2019 10:39:52 +0800 Subject: [PATCH 111/121] Update Makefile and CI setting 1. Move .venv to TEST_LIBS_DIR/ 2. Install `config_helpers` separately --- .circleci/config.yml | 8 ++++---- Makefile | 14 ++++++++++---- test_libs/setup.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 8 deletions(-) create mode 100644 test_libs/setup.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 28d949de4..41e809207 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,15 +60,15 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec + venv_name: v1-test_libs reqs_checksum: '{{ checksum "test_libs/pyspec/setup.py" }}' - run: name: Install pyspec requirements command: make install_test - save_cached_venv: - venv_name: v1-pyspec + venv_name: v1-test_libs reqs_checksum: '{{ checksum "test_libs/pyspec/setup.py" }}' - venv_path: ./test_libs/pyspec/venv + venv_path: ./test_libs/venv test: docker: - image: circleci/python:3.6 @@ -77,7 +77,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec + venv_name: v1-test_libs reqs_checksum: '{{ checksum "test_libs/pyspec/setup.py" }}' - run: name: Run py-tests diff --git a/Makefile b/Makefile index 10550750a..135124898 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ SPEC_DIR = ./specs SCRIPT_DIR = ./scripts TEST_LIBS_DIR = ./test_libs PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec +CONFIG_HELPERS_DIR = $(TEST_LIBS_DIR)/config_helpers YAML_TEST_DIR = ./eth2.0-spec-tests/tests GENERATOR_DIR = ./test_generators CONFIGS_DIR = ./configs @@ -23,7 +24,8 @@ all: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR) $(YAML_TEST_TARGETS) clean: rm -rf $(YAML_TEST_DIR) rm -rf $(GENERATOR_VENVS) - rm -rf $(PY_SPEC_DIR)/venv $(PY_SPEC_DIR)/.pytest_cache + rm -rf $(TEST_LIBS_DIR)/venv + rm -rf $(PY_SPEC_DIR)/.pytest_cache rm -rf $(PY_SPEC_ALL_TARGETS) # "make gen_yaml_tests" to run generators @@ -31,13 +33,17 @@ gen_yaml_tests: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_TARGETS) # installs the packages to run pyspec tests install_test: - cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -e .[dev]; + cd $(TEST_LIBS_DIR); python3 -m venv venv; . venv/bin/activate; \ + cd ..; cd $(CONFIG_HELPERS_DIR); pip3 install -e .; \ + cd ../..; cd $(PY_SPEC_DIR); pip3 install -e .[dev]; test: $(PY_SPEC_ALL_TARGETS) - cd $(PY_SPEC_DIR); . venv/bin/activate; python -m pytest . + cd $(TEST_LIBS_DIR); . venv/bin/activate; \ + cd ..; cd $(PY_SPEC_DIR); python -m pytest .; citest: $(PY_SPEC_ALL_TARGETS) - cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; python -m pytest --junitxml=test-reports/eth2spec/test_results.xml . + cd $(TEST_LIBS_DIR); . venv/bin/activate; \ + cd ..; cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; python -m pytest --junitxml=test-reports/eth2spec/test_results.xml . # "make pyspec" to create the pyspec for all phases. pyspec: $(PY_SPEC_ALL_TARGETS) diff --git a/test_libs/setup.py b/test_libs/setup.py new file mode 100644 index 000000000..b82fc369c --- /dev/null +++ b/test_libs/setup.py @@ -0,0 +1,29 @@ +from setuptools import setup, find_packages + + +deps = { + 'pyspec': [ + "eth-utils>=1.3.0,<2", + "eth-typing>=2.1.0,<3.0.0", + "pycryptodome==3.7.3", + "py_ecc>=1.6.0", + ], + 'test': [ + "pytest>=3.6,<3.7", + ], +} + +deps['dev'] = ( + deps['pyspec'] + + deps['test'] +) + +install_requires = deps['pyspec'] + + +setup( + name='pyspec', + packages=find_packages(exclude=["tests", "tests.*"]), + install_requires=install_requires, + extras_require=deps, +) From 6888d9f36b1e537684fc5cda4cfae35b7c1eacb0 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 24 Apr 2019 19:30:13 +1000 Subject: [PATCH 112/121] update config constants --- configs/constant_presets/mainnet.yaml | 9 ++++----- configs/constant_presets/minimal.yaml | 9 ++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index 8b9dade73..d63c617b7 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -10,7 +10,7 @@ SHARD_COUNT: 1024 # 2**7 (= 128) TARGET_COMMITTEE_SIZE: 128 # 2**12 (= 4,096) -MAX_ATTESTATION_PARTICIPANTS: 4096 +MAX_INDICES_PER_ATTESTATION: 4096 # 2**2 (= 4) MIN_PER_EPOCH_CHURN_LIMIT: 4 # 2**16 (= 65,536) @@ -32,7 +32,7 @@ DEPOSIT_CONTRACT_TREE_DEPTH: 32 # 2**0 * 10**9 (= 1,000,000,000) Gwei MIN_DEPOSIT_AMOUNT: 1000000000 # 2**5 * 10**9 (= 32,000,000,000) Gwei -MAX_DEPOSIT_AMOUNT: 32000000000 +MAX_EFFECTIVE_BALANCE: 32000000000 # 2**4 * 10**9 (= 16,000,000,000) Gwei EJECTION_BALANCE: 16000000000 # 2**0 * 10**9 (= 1,000,000,000) Gwei @@ -44,7 +44,6 @@ HIGH_BALANCE_INCREMENT: 1000000000 GENESIS_FORK_VERSION: 0x00000000 # 0, GENESIS_EPOCH is derived from this constant GENESIS_SLOT: 0 -GENESIS_START_SHARD: 0 # 2**64 - 1 FAR_FUTURE_EPOCH: 18446744073709551615 BLS_WITHDRAWAL_PREFIX_BYTE: 0x00 @@ -110,8 +109,8 @@ MAX_ATTESTATIONS: 128 MAX_DEPOSITS: 16 # 2**4 (= 16) MAX_VOLUNTARY_EXITS: 16 -# 2**4 (= 16) -MAX_TRANSFERS: 16 +# Originally 2**4 (= 16), disabled for now. +MAX_TRANSFERS: 0 # Signature domains diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index edc447c45..711f6737d 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -10,7 +10,7 @@ SHARD_COUNT: 8 # [customized] unsecure, but fast TARGET_COMMITTEE_SIZE: 4 # 2**12 (= 4,096) -MAX_ATTESTATION_PARTICIPANTS: 4096 +MAX_INDICES_PER_ATTESTATION: 4096 # 2**2 (= 4) MIN_PER_EPOCH_CHURN_LIMIT: 4 # 2**16 (= 65,536) @@ -32,7 +32,7 @@ DEPOSIT_CONTRACT_TREE_DEPTH: 32 # 2**0 * 10**9 (= 1,000,000,000) Gwei MIN_DEPOSIT_AMOUNT: 1000000000 # 2**5 * 10**9 (= 32,000,000,000) Gwei -MAX_DEPOSIT_AMOUNT: 32000000000 +MAX_EFFECTIVE_BALANCE: 32000000000 # 2**4 * 10**9 (= 16,000,000,000) Gwei EJECTION_BALANCE: 16000000000 # 2**0 * 10**9 (= 1,000,000,000) Gwei @@ -44,7 +44,6 @@ HIGH_BALANCE_INCREMENT: 1000000000 GENESIS_FORK_VERSION: 0x00000000 # 0, GENESIS_EPOCH is derived from this constant GENESIS_SLOT: 0 -GENESIS_START_SHARD: 0 # 2**64 - 1 FAR_FUTURE_EPOCH: 18446744073709551615 BLS_WITHDRAWAL_PREFIX_BYTE: 0x00 @@ -110,8 +109,8 @@ MAX_ATTESTATIONS: 128 MAX_DEPOSITS: 16 # 2**4 (= 16) MAX_VOLUNTARY_EXITS: 16 -# 2**4 (= 16) -MAX_TRANSFERS: 16 +# Originally 2**4 (= 16), disabled for now. +MAX_TRANSFERS: 0 # Signature domains From 9c2fa02658d7020b7d858770d70070378f3856d2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 Apr 2019 20:54:39 +1000 Subject: [PATCH 113/121] Update test_libs/pyspec/tests/test_sanity.py Co-Authored-By: JustinDrake --- test_libs/pyspec/tests/test_sanity.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index b54b5f75a..071f3f8b3 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -323,7 +323,6 @@ def test_attestation(state): assert len(test_state.current_epoch_attestations) == len(state.current_epoch_attestations) + 1 - proposer_index = get_beacon_proposer_index(test_state) # # Epoch transition should move to previous_epoch_attestations From b1e1510e213cabe4cec2322d7bf4f5334919d75a Mon Sep 17 00:00:00 2001 From: Justin Date: Wed, 24 Apr 2019 20:57:31 +1000 Subject: [PATCH 114/121] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 4a9dcc26c..cf051b5e3 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1429,6 +1429,7 @@ def get_total_active_balance(state: BeaconState) -> Gwei: ```python def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: + assert epoch in (get_current_epoch(state), get_previous_epoch(state)) return state.current_epoch_attestations if epoch == get_current_epoch(state) else state.previous_epoch_attestations ``` @@ -1472,8 +1473,7 @@ def get_crosslink_from_attestation_data(state: BeaconState, data: AttestationDat ```python def get_winning_crosslink_and_attesting_indices(state: BeaconState, shard: Shard, epoch: Epoch) -> Tuple[Crosslink, List[ValidatorIndex]]: - attestations = get_matching_source_attestations(state, epoch) - shard_attestations = [a for a in attestations if a.data.shard == shard] + shard_attestations = [a for a in get_matching_source_attestations(state, epoch) if a.data.shard == shard] shard_crosslinks = [get_crosslink_from_attestation_data(state, a.data) for a in shard_attestations] candidate_crosslinks = [ c for c in shard_crosslinks From 20d65e040b4fc140da5441aba4540a8618a6a482 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 24 Apr 2019 11:31:24 -0600 Subject: [PATCH 115/121] pr feedback --- specs/core/0_beacon-chain.md | 17 +++++++++++++++-- specs/validator/0_beacon-chain-validator.md | 4 ++-- test_libs/pyspec/tests/test_sanity.py | 11 +---------- tests/phase0/helpers.py | 2 +- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index cf051b5e3..8eebc1618 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -68,6 +68,7 @@ - [`get_crosslink_committees_at_slot`](#get_crosslink_committees_at_slot) - [`get_block_root_at_slot`](#get_block_root_at_slot) - [`get_block_root`](#get_block_root) + - [`get_state_root`](#get_state_root) - [`get_randao_mix`](#get_randao_mix) - [`get_active_index_root`](#get_active_index_root) - [`generate_seed`](#generate_seed) @@ -877,6 +878,18 @@ def get_block_root(state: BeaconState, return get_block_root_at_slot(state, get_epoch_start_slot(epoch)) ``` +### `get_state_root` + +```python +def get_state_root(state: BeaconState, + slot: Slot) -> Bytes32: + """ + Return the state root at a recent ``slot``. + """ + assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT + return state.latest_state_roots[slot % SLOTS_PER_HISTORICAL_ROOT] +``` + ### `get_randao_mix` ```python @@ -1519,12 +1532,12 @@ def process_justification_and_finalization(state: BeaconState) -> None: state.justification_bitfield = (state.justification_bitfield << 1) % 2**64 previous_epoch_matching_target_balance = get_attesting_balance(state, get_matching_target_attestations(state, previous_epoch)) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_epoch = get_previous_epoch(state) + state.current_justified_epoch = previous_epoch state.current_justified_root = get_block_root(state, state.current_justified_epoch) state.justification_bitfield |= (1 << 1) current_epoch_matching_target_balance = get_attesting_balance(state, get_matching_target_attestations(state, current_epoch)) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_epoch = get_current_epoch(state) + state.current_justified_epoch = current_epoch state.current_justified_root = get_block_root(state, state.current_justified_epoch) state.justification_bitfield |= (1 << 0) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 03d5b5f5b..8cc38eebf 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -138,7 +138,7 @@ A validator has two primary responsibilities to the beacon chain -- [proposing b ### Block proposal -A validator is expected to propose a [`BeaconBlock`](../core/0_beacon-chain.md#beaconblock) at the beginning of any slot during which `get_beacon_proposer_index(state, slot)` returns the validator's `validator_index`. To propose, the validator selects the `BeaconBlock`, `parent`, that in their view of the fork choice is the head of the chain during `slot - 1`. The validator is to create, sign, and broadcast a `block` that is a child of `parent` and that executes a valid [beacon chain state transition](../core/0_beacon-chain.md#beacon-chain-state-transition-function). +A validator is expected to propose a [`BeaconBlock`](../core/0_beacon-chain.md#beaconblock) at the beginning of any slot during which `get_beacon_proposer_index(state)` returns the validator's `validator_index`. To propose, the validator selects the `BeaconBlock`, `parent`, that in their view of the fork choice is the head of the chain during `slot - 1`. The validator is to create, sign, and broadcast a `block` that is a child of `parent` and that executes a valid [beacon chain state transition](../core/0_beacon-chain.md#beacon-chain-state-transition-function). There is one proposer per slot, so if there are N active validators any individual validator will on average be assigned to propose once per N slots (e.g. at 312500 validators = 10 million ETH, that's once per ~3 weeks). @@ -368,7 +368,7 @@ def get_committee_assignment( return assignment ``` -A validator can use the following function to see if they are supposed to propose during their assigned committee slot. This function can only be run during the slot in question and can not reliably be used to predict in advance. +A validator can use the following function to see if they are supposed to propose during their assigned committee slot. This function can only be run during the slot in question. Proposer selection is only stable within the context of the current epoch. ```python def is_proposer_at_slot(state: BeaconState, diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index 071f3f8b3..d3035576b 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -9,7 +9,6 @@ from eth2spec.utils.minimal_ssz import signing_root from eth2spec.phase0.spec import ( # constants ZERO_HASH, - SLOTS_PER_HISTORICAL_ROOT, # SSZ Deposit, Transfer, @@ -17,13 +16,12 @@ from eth2spec.phase0.spec import ( # functions get_active_validator_indices, get_beacon_proposer_index, - get_block_root, get_block_root_at_slot, + get_state_root, get_current_epoch, get_domain, advance_slot, cache_state, - slot_to_epoch, verify_merkle_branch, hash, ) @@ -49,13 +47,6 @@ from .helpers import ( ) -def get_state_root(state, slot) -> bytes: - """ - Return the state root at a recent ``slot``. - """ - assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT - return state.latest_state_roots[slot % SLOTS_PER_HISTORICAL_ROOT] - # mark entire file as 'sanity' pytestmark = pytest.mark.sanity diff --git a/tests/phase0/helpers.py b/tests/phase0/helpers.py index c042dec91..18898dd3c 100644 --- a/tests/phase0/helpers.py +++ b/tests/phase0/helpers.py @@ -142,7 +142,7 @@ def build_attestation_data(state, slot, shard): if epoch_start_slot == slot: epoch_boundary_root = block_root else: - get_block_root(state, get_current_epoch(state)) + epoch_boundary_root = get_block_root(state, get_current_epoch(state)) if slot < epoch_start_slot: justified_block_root = state.previous_justified_root From 7f720133fab5248623333f08864deb1ac4b432b7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 Apr 2019 11:38:26 -0600 Subject: [PATCH 116/121] Apply suggestions from code review Co-Authored-By: djrtwo --- test_libs/pyspec/tests/test_finality.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test_libs/pyspec/tests/test_finality.py b/test_libs/pyspec/tests/test_finality.py index 8c1b4e871..4aee1c538 100644 --- a/test_libs/pyspec/tests/test_finality.py +++ b/test_libs/pyspec/tests/test_finality.py @@ -77,7 +77,7 @@ def test_finality_from_genesis_rule_4(state): test_state = deepcopy(state) blocks = [] - for epoch in range(6): + for epoch in range(4): prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, False) blocks += new_blocks @@ -88,7 +88,7 @@ def test_finality_from_genesis_rule_4(state): elif epoch == 2: check_finality(test_state, prev_state, True, False, False) elif epoch >= 3: - # rule 4 of finaliy + # rule 4 of finality check_finality(test_state, prev_state, True, True, True) assert test_state.finalized_epoch == prev_state.current_justified_epoch assert test_state.finalized_root == prev_state.current_justified_root @@ -137,10 +137,10 @@ def test_finality_rule_2(state): if epoch == 0: prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, False) check_finality(test_state, prev_state, True, False, False) - if epoch == 1: + elif epoch == 1: prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, False) check_finality(test_state, prev_state, False, True, False) - if epoch == 2: + elif epoch == 2: prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, True) # finalized by rule 2 check_finality(test_state, prev_state, True, False, True) From 9e8a9a26fd787550710531a3b9045e2f18cd5860 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 24 Apr 2019 11:44:03 -0600 Subject: [PATCH 117/121] PR feedback --- test_libs/pyspec/tests/test_finality.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/test_libs/pyspec/tests/test_finality.py b/test_libs/pyspec/tests/test_finality.py index 4aee1c538..9992d2a78 100644 --- a/test_libs/pyspec/tests/test_finality.py +++ b/test_libs/pyspec/tests/test_finality.py @@ -73,7 +73,7 @@ def next_epoch_with_attestations(state, return state, blocks, post_state -def test_finality_from_genesis_rule_4(state): +def test_finality_rule_4(state): test_state = deepcopy(state) blocks = [] @@ -81,8 +81,10 @@ def test_finality_from_genesis_rule_4(state): prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, False) blocks += new_blocks + # justification/finalization skipped at GENESIS_EPOCH if epoch == 0: check_finality(test_state, prev_state, False, False, False) + # justification/finalization skipped at GENESIS_EPOCH + 1 elif epoch == 1: check_finality(test_state, prev_state, False, False, False) elif epoch == 2: @@ -132,8 +134,6 @@ def test_finality_rule_2(state): blocks = [] for epoch in range(3): - old_previous_justified_epoch = test_state.previous_justified_epoch - old_previous_justified_root = test_state.previous_justified_root if epoch == 0: prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, False) check_finality(test_state, prev_state, True, False, False) @@ -144,8 +144,8 @@ def test_finality_rule_2(state): prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, True) # finalized by rule 2 check_finality(test_state, prev_state, True, False, True) - assert test_state.finalized_epoch == old_previous_justified_epoch - assert test_state.finalized_root == old_previous_justified_root + assert test_state.finalized_epoch == prev_state.previous_justified_epoch + assert test_state.finalized_root == prev_state.previous_justified_root blocks += new_blocks @@ -170,19 +170,24 @@ def test_finality_rule_3(state): blocks += new_blocks check_finality(test_state, prev_state, True, False, False) + # In epoch N, JE is set to N, prev JE is set to N-1 prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, False) blocks += new_blocks check_finality(test_state, prev_state, True, True, True) + # In epoch N+1, JE is N, prev JE is N-1, and not enough messages get in to do anything prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, False) blocks += new_blocks check_finality(test_state, prev_state, False, True, False) + # In epoch N+2, JE is N, prev JE is N, and enough messages from the previous epoch get in to justify N+1. + # N+1 now becomes the JE. Not enough messages from epoch N+2 itself get in to justify N+2 prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, True) blocks += new_blocks # rule 2 check_finality(test_state, prev_state, True, False, True) + # In epoch N+3, LJE is N+1, prev LJE is N, and enough messages get in to justify epochs N+2 and N+3. prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, True) blocks += new_blocks # rule 3 From 53b06745330e1b44586008bd9e5cfbe4f51ccea1 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 24 Apr 2019 11:45:41 -0600 Subject: [PATCH 118/121] remove unnecessary var --- test_libs/pyspec/tests/test_finality.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/tests/test_finality.py b/test_libs/pyspec/tests/test_finality.py index 9992d2a78..ca048c2b2 100644 --- a/test_libs/pyspec/tests/test_finality.py +++ b/test_libs/pyspec/tests/test_finality.py @@ -52,7 +52,7 @@ def next_epoch_with_attestations(state, fill_prev_epoch): post_state = deepcopy(state) blocks = [] - for slot in range(spec.SLOTS_PER_EPOCH): + for _ in range(spec.SLOTS_PER_EPOCH): block = build_empty_block_for_next_slot(post_state) if fill_cur_epoch: slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 From 7e6a69dfaa1d03b3f260cbc42bfbdb26973ac904 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 24 Apr 2019 12:31:27 -0600 Subject: [PATCH 119/121] scale number of validators in tests based on number of slots --- test_libs/pyspec/tests/conftest.py | 4 ++-- test_libs/pyspec/tests/helpers.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/tests/conftest.py b/test_libs/pyspec/tests/conftest.py index f6c70d280..9840dc7b2 100644 --- a/test_libs/pyspec/tests/conftest.py +++ b/test_libs/pyspec/tests/conftest.py @@ -22,8 +22,8 @@ def config(request): @pytest.fixture -def num_validators(): - return 100 +def num_validators(config): + return spec.SLOTS_PER_EPOCH * 8 @pytest.fixture diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index a728a6acc..b30aaf35e 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -48,7 +48,7 @@ from eth2spec.utils.merkle_minimal import ( ) -privkeys = [i + 1 for i in range(1000)] +privkeys = [i + 1 for i in range(1024)] pubkeys = [bls.privtopub(privkey) for privkey in privkeys] pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} From 2b999a6c7e5578471f82b1442ff8633506a858ee Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 24 Apr 2019 13:15:06 -0600 Subject: [PATCH 120/121] fix off by one error for eth1 data voting --- specs/core/0_beacon-chain.md | 2 +- test_libs/pyspec/tests/test_sanity.py | 46 +++++++++++++++++++-------- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8eebc1618..c09182ffe 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1726,7 +1726,7 @@ def process_final_updates(state: BeaconState) -> None: current_epoch = get_current_epoch(state) next_epoch = current_epoch + 1 # Reset eth1 data votes - if state.slot % SLOTS_PER_ETH1_VOTING_PERIOD == 0: + if (state.slot + 1) % SLOTS_PER_ETH1_VOTING_PERIOD == 0: state.eth1_data_votes = [] # Update effective balances with hysteresis for index, validator in enumerate(state.validator_registry): diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index 9258e851d..d018fc25f 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -376,17 +376,15 @@ def test_transfer(state): def test_balance_driven_status_transitions(state): - pre_state = deepcopy(state) + current_epoch = get_current_epoch(state) + validator_index = get_active_validator_indices(state, current_epoch)[-1] - current_epoch = get_current_epoch(pre_state) - validator_index = get_active_validator_indices(pre_state, current_epoch)[-1] - - assert pre_state.validator_registry[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH + assert state.validator_registry[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH # set validator balance to below ejection threshold - pre_state.validator_registry[validator_index].effective_balance = spec.EJECTION_BALANCE + state.validator_registry[validator_index].effective_balance = spec.EJECTION_BALANCE - post_state = deepcopy(pre_state) + post_state = deepcopy(state) # # trigger epoch transition # @@ -396,14 +394,13 @@ def test_balance_driven_status_transitions(state): assert post_state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH - return pre_state, [block], post_state + return state, [block], post_state def test_historical_batch(state): - pre_state = deepcopy(state) - pre_state.slot += spec.SLOTS_PER_HISTORICAL_ROOT - (pre_state.slot % spec.SLOTS_PER_HISTORICAL_ROOT) - 1 + state.slot += spec.SLOTS_PER_HISTORICAL_ROOT - (state.slot % spec.SLOTS_PER_HISTORICAL_ROOT) - 1 - post_state = deepcopy(pre_state) + post_state = deepcopy(state) block = build_empty_block_for_next_slot(post_state) @@ -411,6 +408,29 @@ def test_historical_batch(state): assert post_state.slot == block.slot assert get_current_epoch(post_state) % (spec.SLOTS_PER_HISTORICAL_ROOT // spec.SLOTS_PER_EPOCH) == 0 - assert len(post_state.historical_roots) == len(pre_state.historical_roots) + 1 + assert len(post_state.historical_roots) == len(state.historical_roots) + 1 - return pre_state, [block], post_state + return state, [block], post_state + + +def test_eth1_data_votes(state): + post_state = deepcopy(state) + + expected_votes = 0 + assert len(state.eth1_data_votes) == expected_votes + + blocks = [] + for _ in range(spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1): + block = build_empty_block_for_next_slot(post_state) + state_transition(post_state, block) + expected_votes += 1 + assert len(post_state.eth1_data_votes) == expected_votes + blocks.append(block) + + block = build_empty_block_for_next_slot(post_state) + state_transition(post_state, block) + + assert post_state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 + assert len(post_state.eth1_data_votes) == 1 + + return state, blocks, post_state From bd5096074000efe7cf354ec53c51d1787bd2efd6 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 24 Apr 2019 13:20:55 -0600 Subject: [PATCH 121/121] tiny fix to test ouptut --- test_libs/pyspec/tests/test_sanity.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index d018fc25f..b7d31f122 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -429,6 +429,7 @@ def test_eth1_data_votes(state): block = build_empty_block_for_next_slot(post_state) state_transition(post_state, block) + blocks.append(block) assert post_state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 assert len(post_state.eth1_data_votes) == 1