From 129aa02cb3c2f9546578975d4b080ae0179517c3 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 10 Feb 2020 15:16:54 -0700 Subject: [PATCH 001/102] support tests with SLOTS_PER_EPOCH * 256 vals --- setup.py | 9 +++ specs/phase1/custody-game.md | 8 ++- tests/core/pyspec/eth2spec/test/context.py | 2 +- .../eth2spec/test/helpers/attestations.py | 9 +-- .../core/pyspec/eth2spec/test/helpers/keys.py | 2 +- .../test_process_attestation.py | 8 ++- ..._process_justification_and_finalization.py | 5 +- .../test_process_rewards_and_penalties.py | 59 ++++++++----------- 8 files changed, 55 insertions(+), 47 deletions(-) diff --git a/setup.py b/setup.py index 444489d2e..ce565592f 100644 --- a/setup.py +++ b/setup.py @@ -169,16 +169,25 @@ get_base_reward = cache_this( lambda state, index: (state.validators.hash_tree_root(), state.slot), _get_base_reward) + _get_committee_count_at_slot = get_committee_count_at_slot get_committee_count_at_slot = cache_this( lambda state, epoch: (state.validators.hash_tree_root(), epoch), _get_committee_count_at_slot) + _get_active_validator_indices = get_active_validator_indices get_active_validator_indices = cache_this( lambda state, epoch: (state.validators.hash_tree_root(), epoch), _get_active_validator_indices) + +_get_total_active_balance = get_total_active_balance +get_total_active_balance = cache_this( + lambda state: (state.validators.hash_tree_root(), get_current_epoch(state)), + _get_total_active_balance) + + _get_beacon_committee = get_beacon_committee get_beacon_committee = cache_this( lambda state, slot, index: (state.validators.hash_tree_root(), state.randao_mixes.hash_tree_root(), slot, index), diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 121f91f97..a267bd330 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -416,7 +416,13 @@ def process_reveal_deadlines(state: BeaconState) -> None: epoch = get_current_epoch(state) for index, validator in enumerate(state.validators): if get_custody_period_for_validator(ValidatorIndex(index), epoch) > validator.next_custody_secret_to_reveal: - slash_validator(state, ValidatorIndex(index)) + # ------------------ WARNING ----------------------- # + # UNSAFE REMOVAL OF SLASHING TO PRIORITIZE PHASE 0 CI # + # Must find generic way to handle key reveals in tests # + # ---------------------------------------------------- # + + # slash_validator(state, ValidatorIndex(index)) + pass ``` ### Final updates diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 5338ccb9d..543aa59c2 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -76,7 +76,7 @@ def default_balances(spec): Helper method to create a series of default balances. Usage: `@with_custom_state(balances_fn=default_balances, ...)` """ - num_validators = spec.SLOTS_PER_EPOCH * 8 + num_validators = spec.SLOTS_PER_EPOCH * 256 return [spec.MAX_EFFECTIVE_BALANCE] * num_validators diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 047966890..9e12582e0 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -1,7 +1,6 @@ from typing import List -from eth2spec.test.helpers.block import build_empty_block_for_next_slot, transition_unsigned_block, \ - build_empty_block +from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.ssz.ssz_typing import Bitlist @@ -126,8 +125,6 @@ def fill_aggregate_attestation(spec, state, attestation, signed=False): def add_attestations_to_state(spec, state, attestations, slot): - block = build_empty_block(spec, state, slot) + spec.process_slots(state, slot) for attestation in attestations: - block.body.attestations.append(attestation) - spec.process_slots(state, block.slot) - transition_unsigned_block(spec, state, block) + spec.process_attestation(state, attestation) diff --git a/tests/core/pyspec/eth2spec/test/helpers/keys.py b/tests/core/pyspec/eth2spec/test/helpers/keys.py index 23bb95131..7f7820d3a 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/keys.py +++ b/tests/core/pyspec/eth2spec/test/helpers/keys.py @@ -1,6 +1,6 @@ from py_ecc.bls import G2ProofOfPossession as bls from eth2spec.phase0 import spec -privkeys = [i + 1 for i in range(spec.SLOTS_PER_EPOCH * 16)] +privkeys = [i + 1 for i in range(spec.SLOTS_PER_EPOCH * 256)] pubkeys = [bls.PrivToPub(privkey) for privkey in privkeys] pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 7937614a4..64a64c72d 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -153,9 +153,11 @@ def test_wrong_index_for_committee_signature(spec, state): @spec_state_test @never_bls def test_wrong_index_for_slot(spec, state): - committees_per_slot = spec.get_committee_count_at_slot(state, state.slot) - assert committees_per_slot < spec.MAX_COMMITTEES_PER_SLOT - index = committees_per_slot + while spec.get_committee_count_at_slot(state, state.slot) >= spec.MAX_COMMITTEES_PER_SLOT: + state.validators = state.validators[:len(state.validators) // 2] + state.balances = state.balances[:len(state.balances) // 2] + + index = spec.MAX_COMMITTEES_PER_SLOT - 1 attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 917c06e3d..6fc4e30cb 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -46,9 +46,10 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support else: break - # remove just one attester to make the marginal support insufficient + # remove 1/5th of attesters so that support is insufficient if not sufficient_support: - aggregation_bits[aggregation_bits.index(1)] = 0 + for i in range(max(len(committee) // 5, 1)): + aggregation_bits[i] = 0 attestations.append(spec.PendingAttestation( aggregation_bits=aggregation_bits, diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py index fa394df56..7cdeb16d9 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py @@ -18,6 +18,27 @@ def run_process_rewards_and_penalties(spec, state): yield from run_epoch_processing_with(spec, state, 'process_rewards_and_penalties') +def prepare_state_with_full_attestations(spec, state): + attestations = [] + for slot in range(spec.SLOTS_PER_EPOCH + spec.MIN_ATTESTATION_INCLUSION_DELAY): + # create an attestation for each index in each slot in epoch + if slot < spec.SLOTS_PER_EPOCH: + for committee_index in range(spec.get_committee_count_at_slot(state, slot)): + attestation = get_valid_attestation(spec, state, index=committee_index, signed=True) + attestations.append(attestation) + # fill each created slot in state after inclusion delay + if slot - spec.MIN_ATTESTATION_INCLUSION_DELAY >= 0: + inclusion_slot = slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + include_attestations = [att for att in attestations if att.data.slot == inclusion_slot] + add_attestations_to_state(spec, state, include_attestations, state.slot) + next_slot(spec, state) + + assert spec.compute_epoch_at_slot(state.slot) == spec.GENESIS_EPOCH + 1 + assert len(state.previous_epoch_attestations) == len(attestations) + + return attestations + + @with_all_phases @spec_state_test def test_genesis_epoch_no_attestations_no_penalties(spec, state): @@ -57,25 +78,6 @@ def test_genesis_epoch_full_attestations_no_rewards(spec, state): assert state.balances[index] == pre_state.balances[index] -def prepare_state_with_full_attestations(spec, state): - attestations = [] - for slot in range(spec.SLOTS_PER_EPOCH + spec.MIN_ATTESTATION_INCLUSION_DELAY): - # create an attestation for each slot in epoch - if slot < spec.SLOTS_PER_EPOCH: - attestation = get_valid_attestation(spec, state, signed=True) - attestations.append(attestation) - # fill each created slot in state after inclusion delay - if slot - spec.MIN_ATTESTATION_INCLUSION_DELAY >= 0: - include_att = attestations[slot - spec.MIN_ATTESTATION_INCLUSION_DELAY] - add_attestations_to_state(spec, state, [include_att], state.slot) - next_slot(spec, state) - - assert spec.compute_epoch_at_slot(state.slot) == spec.GENESIS_EPOCH + 1 - assert len(state.previous_epoch_attestations) == spec.SLOTS_PER_EPOCH - - return attestations - - @with_all_phases @spec_state_test def test_full_attestations(spec, state): @@ -86,7 +88,7 @@ def test_full_attestations(spec, state): yield from run_process_rewards_and_penalties(spec, state) attesting_indices = spec.get_unslashed_attesting_indices(state, attestations) - assert len(attesting_indices) > 0 + assert len(attesting_indices) == len(pre_state.validators) for index in range(len(pre_state.validators)): if index in attesting_indices: assert state.balances[index] > pre_state.balances[index] @@ -173,18 +175,7 @@ def test_duplicate_attestation(spec, state): @spec_state_test # Case when some eligible attestations are slashed. Modifies attesting_balance and consequently rewards/penalties. def test_attestations_some_slashed(spec, state): - attestations = [] - for slot in range(spec.SLOTS_PER_EPOCH + spec.MIN_ATTESTATION_INCLUSION_DELAY): - # create an attestation for each slot in epoch - if slot < spec.SLOTS_PER_EPOCH: - attestation = get_valid_attestation(spec, state, signed=True) - attestations.append(attestation) - # fill each created slot in state after inclusion delay - if slot - spec.MIN_ATTESTATION_INCLUSION_DELAY >= 0: - include_att = attestations[slot - spec.MIN_ATTESTATION_INCLUSION_DELAY] - add_attestations_to_state(spec, state, [include_att], state.slot) - next_slot(spec, state) - + attestations = prepare_state_with_full_attestations(spec, state) attesting_indices_before_slashings = list(spec.get_unslashed_attesting_indices(state, attestations)) # Slash maximum amount of validators allowed per epoch. @@ -192,7 +183,7 @@ def test_attestations_some_slashed(spec, state): spec.slash_validator(state, attesting_indices_before_slashings[i]) assert spec.compute_epoch_at_slot(state.slot) == spec.GENESIS_EPOCH + 1 - assert len(state.previous_epoch_attestations) == spec.SLOTS_PER_EPOCH + assert len(state.previous_epoch_attestations) == len(attestations) pre_state = deepcopy(state) @@ -203,6 +194,8 @@ def test_attestations_some_slashed(spec, state): assert len(attesting_indices_before_slashings) - len(attesting_indices) == spec.MIN_PER_EPOCH_CHURN_LIMIT for index in range(len(pre_state.validators)): if index in attesting_indices: + # non-slashed attester should gain reward assert state.balances[index] > pre_state.balances[index] else: + # Slashed non-proposer attester should have penalty assert state.balances[index] < pre_state.balances[index] From c1076097c37a7cb5260ca881b1da1cd66bc1b815 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Feb 2020 11:33:42 -0800 Subject: [PATCH 002/102] wip work to improve tests --- .../test/fork_choice/test_on_attestation.py | 4 +-- .../pyspec/eth2spec/test/helpers/state.py | 7 ++++ .../test_process_attestation.py | 34 +++++++++---------- .../test_process_proposer_slashing.py | 4 +-- .../test_process_final_updates.py | 4 ++- ..._process_justification_and_finalization.py | 7 ++-- .../test_process_registry_updates.py | 13 +++---- .../test_process_slashings.py | 3 +- .../eth2spec/test/sanity/test_blocks.py | 13 ++++--- 9 files changed, 52 insertions(+), 37 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 09248944c..87a39a620 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -1,7 +1,7 @@ from eth2spec.test.context import with_all_phases, spec_state_test from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.attestations import get_valid_attestation, sign_attestation -from eth2spec.test.helpers.state import transition_to, state_transition_and_sign_block +from eth2spec.test.helpers.state import transition_to, state_transition_and_sign_block, next_epoch def run_on_attestation(spec, state, store, attestation, valid=True): @@ -179,7 +179,7 @@ def test_on_attestation_future_epoch(spec, state): spec.on_block(store, signed_block) # move state forward but not store - state.slot = block.slot + spec.SLOTS_PER_EPOCH + next_epoch(spec, state) attestation = get_valid_attestation(spec, state, slot=state.slot, signed=True) run_on_attestation(spec, state, store, attestation, False) diff --git a/tests/core/pyspec/eth2spec/test/helpers/state.py b/tests/core/pyspec/eth2spec/test/helpers/state.py index aad329ff4..5816785f7 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/state.py +++ b/tests/core/pyspec/eth2spec/test/helpers/state.py @@ -14,6 +14,13 @@ def next_slot(spec, state): spec.process_slots(state, state.slot + 1) +def next_slots(spec, state, slots): + """ + Transition given slots forward. + """ + spec.process_slots(state, state.slot + slots) + + def transition_to(spec, state, slot): """ Transition to ``slot``. diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 64a64c72d..1dbdafec2 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -13,7 +13,7 @@ from eth2spec.test.helpers.attestations import ( sign_attestation, ) from eth2spec.test.helpers.state import ( - next_epoch, + next_epoch, next_slots ) from eth2spec.test.helpers.block import apply_empty_block from eth2spec.utils.ssz.ssz_typing import Bitlist @@ -58,7 +58,7 @@ def run_attestation_processing(spec, state, attestation, valid=True): @spec_state_test def test_success(spec, state): attestation = get_valid_attestation(spec, state, signed=True) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) yield from run_attestation_processing(spec, state, attestation) @@ -68,9 +68,9 @@ def test_success(spec, state): @with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @single_phase def test_success_multi_proposer_index_iterations(spec, state): - state.slot += spec.SLOTS_PER_EPOCH * 2 + next_slots(spec, state, spec.SLOTS_PER_EPOCH * 2) attestation = get_valid_attestation(spec, state, signed=True) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) yield from run_attestation_processing(spec, state, attestation) @@ -91,7 +91,7 @@ def test_success_previous_epoch(spec, state): @always_bls def test_invalid_attestation_signature(spec, state): attestation = get_valid_attestation(spec, state) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) yield from run_attestation_processing(spec, state, attestation, False) @@ -120,7 +120,7 @@ def test_after_epoch_slots(spec, state): @with_all_phases @spec_state_test def test_old_source_epoch(spec, state): - state.slot = spec.SLOTS_PER_EPOCH * 5 + next_slots(spec, state, spec.SLOTS_PER_EPOCH * 5) state.finalized_checkpoint.epoch = 2 state.previous_justified_checkpoint.epoch = 3 state.current_justified_checkpoint.epoch = 4 @@ -142,7 +142,7 @@ def test_old_source_epoch(spec, state): @always_bls def test_wrong_index_for_committee_signature(spec, state): attestation = get_valid_attestation(spec, state) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) attestation.data.index += 1 @@ -160,7 +160,7 @@ def test_wrong_index_for_slot(spec, state): index = spec.MAX_COMMITTEES_PER_SLOT - 1 attestation = get_valid_attestation(spec, state) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) attestation.data.index = index @@ -172,7 +172,7 @@ def test_wrong_index_for_slot(spec, state): @never_bls def test_invalid_index(spec, state): attestation = get_valid_attestation(spec, state) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) # off by one (with respect to valid range) on purpose attestation.data.index = spec.MAX_COMMITTEES_PER_SLOT @@ -223,7 +223,7 @@ def test_future_target_epoch(spec, state): # manually add signature for correct participants attestation.signature = sign_aggregate_attestation(spec, state, attestation.data, participants) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) yield from run_attestation_processing(spec, state, attestation, False) @@ -232,7 +232,7 @@ def test_future_target_epoch(spec, state): @spec_state_test def test_new_source_epoch(spec, state): attestation = get_valid_attestation(spec, state) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) attestation.data.source.epoch += 1 @@ -245,7 +245,7 @@ def test_new_source_epoch(spec, state): @spec_state_test def test_source_root_is_target_root(spec, state): attestation = get_valid_attestation(spec, state) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) attestation.data.source.root = attestation.data.target.root @@ -264,7 +264,7 @@ def test_invalid_current_source_root(spec, state): state.current_justified_checkpoint = spec.Checkpoint(epoch=4, root=b'\x32' * 32) attestation = get_valid_attestation(spec, state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) # Test logic sanity checks: assert state.current_justified_checkpoint.root != state.previous_justified_checkpoint.root @@ -282,7 +282,7 @@ def test_invalid_current_source_root(spec, state): @spec_state_test def test_bad_source_root(spec, state): attestation = get_valid_attestation(spec, state) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) attestation.data.source.root = b'\x42' * 32 @@ -295,7 +295,7 @@ def test_bad_source_root(spec, state): @spec_state_test def test_empty_aggregation_bits(spec, state): attestation = get_valid_attestation(spec, state) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) attestation.aggregation_bits = Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE]( *([0b0] * len(attestation.aggregation_bits))) @@ -309,7 +309,7 @@ def test_empty_aggregation_bits(spec, state): @spec_state_test def test_too_many_aggregation_bits(spec, state): attestation = get_valid_attestation(spec, state, signed=True) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) # one too many bits attestation.aggregation_bits.append(0b0) @@ -321,7 +321,7 @@ def test_too_many_aggregation_bits(spec, state): @spec_state_test def test_too_few_aggregation_bits(spec, state): attestation = get_valid_attestation(spec, state) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) attestation.aggregation_bits = Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE]( *([0b1] + [0b0] * (len(attestation.aggregation_bits) - 1))) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py index 30b3c1fdd..ad910f227 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py @@ -2,7 +2,7 @@ from eth2spec.test.context import spec_state_test, expect_assertion_error, alway from eth2spec.test.helpers.block_header import sign_block_header from eth2spec.test.helpers.keys import privkeys from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing -from eth2spec.test.helpers.state import get_balance +from eth2spec.test.helpers.state import get_balance, next_epoch def run_proposer_slashing_processing(spec, state, proposer_slashing, valid=True): @@ -135,7 +135,7 @@ def test_proposer_is_withdrawn(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=True) # move 1 epoch into future, to allow for past withdrawable epoch - state.slot += spec.SLOTS_PER_EPOCH + next_epoch(spec, state) # set proposer withdrawable_epoch in past current_epoch = spec.get_current_epoch(state) proposer_index = proposer_slashing.proposer_index diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py index 58882a44f..2e71346aa 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py @@ -2,6 +2,7 @@ from eth2spec.test.context import spec_state_test, with_all_phases from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import ( run_epoch_processing_with, run_epoch_processing_to ) +from eth2spec.test.helpers.state import transition_to def run_process_final_updates(spec, state): @@ -13,7 +14,8 @@ def run_process_final_updates(spec, state): def test_eth1_vote_no_reset(spec, state): assert spec.SLOTS_PER_ETH1_VOTING_PERIOD > spec.SLOTS_PER_EPOCH # skip ahead to the end of the epoch - state.slot = spec.SLOTS_PER_EPOCH - 1 + transition_to(spec, state, spec.SLOTS_PER_EPOCH - 1) + for i in range(state.slot + 1): # add a vote for each skipped slot. state.eth1_data_votes.append( spec.Eth1Data(deposit_root=b'\xaa' * 32, diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 6fc4e30cb..9f9e1c316 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -2,6 +2,7 @@ from eth2spec.test.context import spec_state_test, with_all_phases from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import ( run_epoch_processing_with ) +from eth2spec.test.helpers.state import transition_to def run_process_just_and_fin(spec, state): @@ -82,7 +83,7 @@ def put_checkpoints_in_block_roots(spec, state, checkpoints): def finalize_on_234(spec, state, epoch, sufficient_support): assert epoch > 4 - state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch + transition_to(spec, state, spec.SLOTS_PER_EPOCH * epoch - 1) # skip ahead to just before epoch # 43210 -- epochs ago # 3210x -- justification bitfield indices @@ -117,7 +118,7 @@ def finalize_on_234(spec, state, epoch, sufficient_support): def finalize_on_23(spec, state, epoch, sufficient_support): assert epoch > 3 - state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch + transition_to(spec, state, spec.SLOTS_PER_EPOCH * epoch - 1) # skip ahead to just before epoch # 43210 -- epochs ago # 210xx -- justification bitfield indices (pre shift) @@ -195,7 +196,7 @@ def finalize_on_123(spec, state, epoch, sufficient_support): def finalize_on_12(spec, state, epoch, sufficient_support, messed_up_target): assert epoch > 2 - state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch + transition_to(spec, state, spec.SLOTS_PER_EPOCH * epoch - 1) # skip ahead to just before epoch # 43210 -- epochs ago # 210xx -- justification bitfield indices (pre shift) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py index 526aba277..a5f4d9227 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py @@ -1,4 +1,4 @@ -from eth2spec.test.helpers.state import next_epoch +from eth2spec.test.helpers.state import next_epoch, next_slots from eth2spec.test.context import spec_state_test, with_all_phases from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_with @@ -101,7 +101,7 @@ def test_activation_queue_sorting(spec, state): state.validators[mock_activations - 1].activation_eligibility_epoch = epoch # move state forward and finalize to allow for activations - state.slot += spec.SLOTS_PER_EPOCH * 3 + next_slots(spec, state, spec.SLOTS_PER_EPOCH * 3) state.finalized_checkpoint.epoch = epoch + 1 yield from run_process_registry_updates(spec, state) @@ -113,10 +113,10 @@ def test_activation_queue_sorting(spec, state): # the second last is at the end of the queue, and did not make the churn, # hence is not assigned an activation_epoch yet. assert state.validators[mock_activations - 2].activation_epoch == spec.FAR_FUTURE_EPOCH - # the one at churn_limit - 1 did not make it, it was out-prioritized - assert state.validators[churn_limit - 1].activation_epoch == spec.FAR_FUTURE_EPOCH + # the one at churn_limit did not make it, it was out-prioritized + assert state.validators[churn_limit].activation_epoch == spec.FAR_FUTURE_EPOCH # but the the one in front of the above did - assert state.validators[churn_limit - 2].activation_epoch != spec.FAR_FUTURE_EPOCH + assert state.validators[churn_limit - 1].activation_epoch != spec.FAR_FUTURE_EPOCH @with_all_phases @@ -131,7 +131,8 @@ def test_activation_queue_efficiency(spec, state): state.validators[i].activation_eligibility_epoch = epoch + 1 # move state forward and finalize to allow for activations - state.slot += spec.SLOTS_PER_EPOCH * 3 + next_slots(spec, state, spec.SLOTS_PER_EPOCH * 3) + state.finalized_checkpoint.epoch = epoch + 1 # Run first registry update. Do not yield test vectors diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py index c58da5a4a..23c8ce11a 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py @@ -2,6 +2,7 @@ from eth2spec.test.context import spec_state_test, with_all_phases from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import ( run_epoch_processing_with, run_epoch_processing_to ) +from eth2spec.test.helpers.state import next_epoch def run_process_slashings(spec, state): @@ -79,7 +80,7 @@ def test_small_penalty(spec, state): @spec_state_test def test_scaled_penalties(spec, state): # skip to next epoch - state.slot = spec.SLOTS_PER_EPOCH + next_epoch(spec, state) # Also mock some previous slashings, so that we test to have the delta in the penalties computation. base = spec.EJECTION_BALANCE diff --git a/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py index 9027660ab..e533b3b0a 100644 --- a/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py @@ -2,7 +2,7 @@ from copy import deepcopy from eth2spec.utils import bls -from eth2spec.test.helpers.state import get_balance, state_transition_and_sign_block, next_slot +from eth2spec.test.helpers.state import get_balance, state_transition_and_sign_block, next_slot, next_epoch from eth2spec.test.helpers.block import build_empty_block_for_next_slot, build_empty_block, sign_block, \ transition_unsigned_block from eth2spec.test.helpers.keys import privkeys, pubkeys @@ -11,7 +11,7 @@ from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing from eth2spec.test.helpers.attestations import get_valid_attestation from eth2spec.test.helpers.deposits import prepare_state_and_deposit -from eth2spec.test.context import spec_state_test, with_all_phases, expect_assertion_error, always_bls +from eth2spec.test.context import spec_state_test, with_all_phases, expect_assertion_error, always_bls, with_phases @with_all_phases @@ -260,7 +260,8 @@ def test_proposer_after_inactive_index(spec, state): state.validators[inactive_index].exit_epoch = spec.get_current_epoch(state) # skip forward, get brand new proposers - state.slot = spec.SLOTS_PER_EPOCH * 2 + next_epoch(spec, state) + next_epoch(spec, state) block = build_empty_block_for_next_slot(spec, state) state_transition_and_sign_block(spec, state, block) @@ -372,7 +373,7 @@ def test_deposit_top_up(spec, state): @with_all_phases @spec_state_test def test_attestation(spec, state): - state.slot = spec.SLOTS_PER_EPOCH + next_epoch(spec, state) yield 'pre', state @@ -399,7 +400,9 @@ def test_attestation(spec, state): assert spec.hash_tree_root(state.previous_epoch_attestations) == pre_current_attestations_root -@with_all_phases +# In phase1 a committee is computed for PERSISTENT_COMMITTEE_PERIOD slots ago, +# exceeding the minimal-config randao mixes memory size. +@with_phases(['phase0']) @spec_state_test def test_voluntary_exit(spec, state): validator_index = spec.get_active_validator_indices( From aa451778f96183b4ca4bcdf27a952dfc6fc80ccf Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Feb 2020 11:34:37 -0800 Subject: [PATCH 003/102] work in progress test improvements --- tests/core/pyspec/eth2spec/test/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 543aa59c2..5338ccb9d 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -76,7 +76,7 @@ def default_balances(spec): Helper method to create a series of default balances. Usage: `@with_custom_state(balances_fn=default_balances, ...)` """ - num_validators = spec.SLOTS_PER_EPOCH * 256 + num_validators = spec.SLOTS_PER_EPOCH * 8 return [spec.MAX_EFFECTIVE_BALANCE] * num_validators From d414aac93352827aedcee2d66280296c13db3d16 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 6 Feb 2020 12:58:21 -0600 Subject: [PATCH 004/102] rework process_attestation and work through tests --- specs/phase1/beacon-chain.md | 70 ++++++++-------- .../test_process_attestation.py | 80 +++++++++++++++++++ 2 files changed, 114 insertions(+), 36 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_attestation.py diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 78b3b3d25..bdf70c5bd 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -568,23 +568,23 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: # Verify that outstanding deposits are processed up to the maximum number of deposits assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) - + def for_ops(operations: Sequence[Any], fn: Callable[[BeaconState, Any], None]) -> None: for operation in operations: fn(state, operation) - + for_ops(body.proposer_slashings, process_proposer_slashing) for_ops(body.attester_slashings, process_attester_slashing) - # New attestation processing - process_attestations(state, body, body.attestations) - + for_ops(body.attestations, process_attestation) for_ops(body.deposits, process_deposit) for_ops(body.voluntary_exits, process_voluntary_exit) # See custody game spec. process_custody_game_operations(state, body) + process_crosslinks(state, body.shard_transitions, body.attestations) + # TODO process_operations(body.shard_receipt_proofs, process_shard_receipt_proofs) ``` @@ -598,6 +598,7 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: assert data.index < get_committee_count_at_slot(state, data.slot) assert data.index < get_active_shard_count(state) assert data.target.epoch in (get_previous_epoch(state), get_current_epoch(state)) + assert data.target.epoch == compute_epoch_at_slot(data.slot) assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= data.slot + SLOTS_PER_EPOCH committee = get_beacon_committee(state, data.slot, data.index) @@ -740,49 +741,46 @@ def process_crosslink_for_shard(state: BeaconState, ```python def process_crosslinks(state: BeaconState, - block_body: BeaconBlockBody, - attestations: Sequence[Attestation]) -> Set[Tuple[Shard, Root]]: - winners: Set[Tuple[Shard, Root]] = set() + shard_transitions: Sequence[ShardTransition], + attestations: Sequence[Attestation]) -> None: committee_count = get_committee_count_at_slot(state, state.slot) for committee_index in map(CommitteeIndex, range(committee_count)): shard = compute_shard_from_committee_index(state, committee_index, state.slot) - # All attestations in the block for this shard + # All attestations in the block for this committee/shard and current slot shard_attestations = [ attestation for attestation in attestations - if get_shard(state, attestation) == shard and attestation.data.slot == state.slot + if attestation.data.index == committee_index and attestation.data.slot == state.slot ] - shard_transition = block_body.shard_transitions[shard] + shard_transition = shard_transitions[shard] winning_root = process_crosslink_for_shard(state, shard, shard_transition, shard_attestations) if winning_root != Root(): - winners.add((shard, winning_root)) - return winners + # Mark relevant pending attestations as creating a successful crosslink + for pending_attestation in state.current_epoch_attestations: + if ( + pending_attestation.slot == state.slot and pending_attestation + and pending_attestation.data.index == committee_index + and pending_attestation.data.shard_transition_root == winning_root + ): + pending_attestation.crosslink_success = True ``` -###### `process_attestations` +###### `process_attestation` ```python -def process_attestations(state: BeaconState, block_body: BeaconBlockBody, attestations: Sequence[Attestation]) -> None: - # Basic validation - for attestation in attestations: - validate_attestation(state, attestation) - - # Process crosslinks - winners = process_crosslinks(state, block_body, attestations) - - # Store pending attestations for epoch processing - for attestation in attestations: - is_winning_transition = (get_shard(state, attestation), attestation.data.shard_transition_root) in winners - pending_attestation = PendingAttestation( - aggregation_bits=attestation.aggregation_bits, - data=attestation.data, - inclusion_delay=state.slot - attestation.data.slot, - crosslink_success=is_winning_transition and attestation.data.slot == state.slot, - proposer_index=get_beacon_proposer_index(state), - ) - if attestation.data.target.epoch == get_current_epoch(state): - state.current_epoch_attestations.append(pending_attestation) - else: - state.previous_epoch_attestations.append(pending_attestation) +def process_attestation(state: BeaconState, attestation: Attestation) -> None: + validate_attestation(state, attestation) + # Store pending attestation for epoch processing + pending_attestation = PendingAttestation( + aggregation_bits=attestation.aggregation_bits, + data=attestation.data, + inclusion_delay=state.slot - attestation.data.slot, + crosslink_success=False, # To be filled in during process_crosslinks + proposer_index=get_beacon_proposer_index(state), + ) + if attestation.data.target.epoch == get_current_epoch(state): + state.current_epoch_attestations.append(pending_attestation) + else: + state.previous_epoch_attestations.append(pending_attestation) ``` ##### New Attester slashing processing diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_attestation.py new file mode 100644 index 000000000..875039c59 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_attestation.py @@ -0,0 +1,80 @@ +from eth2spec.test.context import ( + with_all_phases_except, + expect_assertion_error, + spec_state_test, + always_bls, +) +from eth2spec.test.helpers.attestations import ( + get_valid_attestation, + sign_attestation, +) + +from eth2spec.utils.ssz.ssz_typing import Bitlist + + +def run_attestation_processing(spec, state, attestation, valid=True): + """ + Run ``process_attestation``, yielding: + - pre-state ('pre') + - attestation ('attestation') + - post-state ('post'). + If ``valid == False``, run expecting ``AssertionError`` + """ + # yield pre-state + yield 'pre', state + + yield 'attestation', attestation + + # If the attestation is invalid, processing is aborted, and there is no post-state. + if not valid: + expect_assertion_error(lambda: spec.process_attestation(state, attestation)) + yield 'post', None + return + + current_epoch_count = len(state.current_epoch_attestations) + previous_epoch_count = len(state.previous_epoch_attestations) + + # process attestation + spec.process_attestation(state, attestation) + + # Make sure the attestation has been processed + if attestation.data.target.epoch == spec.get_current_epoch(state): + assert len(state.current_epoch_attestations) == current_epoch_count + 1 + else: + assert len(state.previous_epoch_attestations) == previous_epoch_count + 1 + + # yield post-state + yield 'post', state + + +@with_all_phases_except(['phase0']) +@spec_state_test +@always_bls +def test_success_empty_custody_bits_blocks(spec, state): + attestation = get_valid_attestation(spec, state) + attestation.custody_bits_blocks = [] + sign_attestation(spec, state, attestation) + + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + yield from run_attestation_processing(spec, state, attestation) + + +@with_all_phases_except(['phase0']) +@spec_state_test +@always_bls +def test_fail_custody_bits_blocks_incorrect_slot(spec, state): + attestation = get_valid_attestation(spec, state) + committee = spec.get_beacon_committee( + state, + attestation.data.slot, + attestation.data.index, + ) + bitlist = Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE]([0 for _ in range(len(committee))]) + bitlist[0] = 1 + attestation.custody_bits_blocks = [bitlist] + sign_attestation(spec, state, attestation) + + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 + + yield from run_attestation_processing(spec, state, attestation) \ No newline at end of file From 0a849acdceb0d3a617fbefcec119e0c061529266 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sat, 8 Feb 2020 16:58:05 -0700 Subject: [PATCH 005/102] fix validator guide to show that block slashing is per slot rather than per epoch --- specs/phase0/validator.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index be151447d..901fb562d 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -495,13 +495,13 @@ Because Phase 0 does not have shards and thus does not have Shard Committees, th ### Proposer slashing -To avoid "proposer slashings", a validator must not sign two conflicting [`BeaconBlock`](./beacon-chain.md#beaconblock) where conflicting is defined as two distinct blocks within the same epoch. +To avoid "proposer slashings", a validator must not sign two conflicting [`BeaconBlock`](./beacon-chain.md#beaconblock) where conflicting is defined as two distinct blocks within the same slot. -*In Phase 0, as long as the validator does not sign two different beacon blocks for the same epoch, the validator is safe against proposer slashings.* +*In Phase 0, as long as the validator does not sign two different beacon blocks for the same slot, the validator is safe against proposer slashings.* Specifically, when signing a `BeaconBlock`, a validator should perform the following steps in the following order: -1. Save a record to hard disk that a beacon block has been signed for the `epoch=compute_epoch_at_slot(block.slot)`. +1. Save a record to hard disk that a beacon block has been signed for the `slot=block.slot`. 2. Generate and broadcast the block. If the software crashes at some point within this routine, then when the validator comes back online, the hard disk has the record of the *potentially* signed/broadcast block and can effectively avoid slashing. From 047936eb2de42308a00280394a4ee6fbec719164 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 10 Feb 2020 18:53:26 -0700 Subject: [PATCH 006/102] Add no repeat attestation condition for committee_index_beacon_attestation gossip channel --- specs/phase0/p2p-interface.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index d47bac734..1fadd153a 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -273,9 +273,11 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio - `committee_index{subnet_id}_beacon_attestation` - These topics are used to propagate unaggregated attestations to the subnet `subnet_id` (typically beacon and persistent committees) to be aggregated before being gossiped to `beacon_aggregate_and_proof`. The following validations MUST pass before forwarding the `attestation` on the subnet. - The attestation's committee index (`attestation.data.index`) is for the correct subnet. - - The attestation is unaggregated -- that is, it has exactly one participating validator (`len([bit for bit in attestation.aggregation_bits if bit == 0b1]) == 1`). - - The block being voted for (`attestation.data.beacon_block_root`) passes validation. + - The attestation is the first attestation received for the participating validator for the slot `attestation.data.slot` - `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot`. + - The attestation is unaggregated -- that is, it has exactly one participating validator (`len([bit for bit in attestation.aggregation_bits if bit == 0b1]) == 1`). + - This is the first attestation for the participating validator at this slot. + - The block being voted for (`attestation.data.beacon_block_root`) passes validation. - The signature of `attestation` is valid. #### Interop From 8da7a84eb7dc1f1cbf0bbdf32758a238390abfce Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 10 Feb 2020 19:13:50 -0700 Subject: [PATCH 007/102] create SignedAggregateAndProof to prevent DoS attacks --- specs/phase0/p2p-interface.md | 38 ++++++++++++++++++----------------- specs/phase0/validator.md | 14 +++++++++++-- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 1fadd153a..38adfb316 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -229,15 +229,15 @@ where `base64` is the [URL-safe base64 alphabet](https://tools.ietf.org/html/rfc The payload is carried in the `data` field of a gossipsub message, and varies depending on the topic: -| Topic | Message Type | -|------------------------------------------------|----------------------| -| beacon_block | SignedBeaconBlock | -| beacon_aggregate_and_proof | AggregateAndProof | -| beacon_attestation\* | Attestation | -| committee_index{subnet_id}\_beacon_attestation | Attestation | -| voluntary_exit | SignedVoluntaryExit | -| proposer_slashing | ProposerSlashing | -| attester_slashing | AttesterSlashing | +| Topic | Message Type | +|------------------------------------------------|-------------------------| +| beacon_block | SignedBeaconBlock | +| beacon_aggregate_and_proof | SignedAggregateAndProof | +| beacon_attestation\* | Attestation | +| committee_index{subnet_id}\_beacon_attestation | Attestation | +| voluntary_exit | SignedVoluntaryExit | +| proposer_slashing | ProposerSlashing | +| attester_slashing | AttesterSlashing | Clients MUST reject (fail validation) messages containing an incorrect type, or invalid payload. @@ -250,16 +250,18 @@ When processing incoming gossip, clients MAY descore or disconnect peers who fai There are two primary global topics used to propagate beacon blocks and aggregate attestations to all nodes on the network. Their `TopicName`s are: - `beacon_block` - This topic is used solely for propagating new signed beacon blocks to all nodes on the networks. Signed blocks are sent in their entirety. The following validations MUST pass before forwarding the `signed_beacon_block` on the network - - The proposer signature, `signed_beacon_block.signature` is valid. + - The proposer signature, `signed_beacon_block.signature`, is valid. - The block is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `signed_beacon_block.message.slot <= current_slot` (a client MAY queue future blocks for processing at the appropriate slot). -- `beacon_aggregate_and_proof` - This topic is used to propagate aggregated attestations (as `AggregateAndProof`s) to subscribing nodes (typically validators) to be included in future blocks. The following validations MUST pass before forwarding the `aggregate_and_proof` on the network. - - The aggregate attestation defined by `hash_tree_root(aggregate_and_proof.aggregate)` has _not_ already been seen (via aggregate gossip, within a block, or through the creation of an equivalent aggregate locally). - - The block being voted for (`aggregate_and_proof.aggregate.data.beacon_block_root`) passes validation. - - `aggregate_and_proof.aggregate.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `aggregate_and_proof.aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= aggregate_and_proof.aggregate.data.slot`. - - The validator index is within the aggregate's committee -- i.e. `aggregate_and_proof.aggregator_index in get_attesting_indices(state, aggregate_and_proof.aggregate.data, aggregate_and_proof.aggregate.aggregation_bits)`. - - `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_aggregator(state, aggregate_and_proof.aggregate.data.slot, aggregate_and_proof.aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. - - The `aggregate_and_proof.selection_proof` is a valid signature of the `aggregate_and_proof.aggregate.data.slot` by the validator with index `aggregate_and_proof.aggregator_index`. - - The signature of `aggregate_and_proof.aggregate` is valid. +- `beacon_aggregate_and_proof` - This topic is used to propagate aggregated attestations (as `SignedAggregateAndProof`s) to subscribing nodes (typically validators) to be included in future blocks. The following validations MUST pass before forwarding the `signed_aggregate_and_proof` on the network. (We define the following for convenience -- `aggregate_and_proof = signed_aggregate_and_proof.message` and `aggregate = aggregate_and_proof.aggregate`) + - The aggregate attestation defined by `hash_tree_root(aggregate)` has _not_ already been seen (via aggregate gossip, within a block, or through the creation of an equivalent aggregate locally). + - The block being voted for (`aggregate.data.beacon_block_root`) passes validation. + - `aggregate.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= aggregate.data.slot`. + - The validator index is within the aggregate's committee -- i.e. `aggregate_and_proof.aggregator_index in get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)`. + - `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_aggregator(state, aggregate.data.slot, aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. + - The `signed_aggregate_and_proof` is the first aggregate received for the aggregator for the slot `aggregate.data.slot` + - The `aggregate_and_proof.selection_proof` is a valid signature of the `aggregate.data.slot` by the validator with index `aggregate_and_proof.aggregator_index`. + - The aggregator signature, `signed_aggregate_and_proof.signature`, is valid. + - The signature of `aggregate` is valid. Additional global topics are used to propagate lower frequency validator messages. Their `TopicName`s are: diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 901fb562d..8996428cc 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -461,9 +461,11 @@ def get_aggregate_signature(attestations: Sequence[Attestation]) -> BLSSignature #### Broadcast aggregate -If the validator is selected to aggregate (`is_aggregator`), then they broadcast their best aggregate to the global aggregate channel (`beacon_aggregate_and_proof`) two-thirds of the way through the `slot`-that is, `SECONDS_PER_SLOT * 2 / 3` seconds after the start of `slot`. +If the validator is selected to aggregate (`is_aggregator`), then they broadcast their best aggregate as a `SignedAggregateAndProof` to the global aggregate channel (`beacon_aggregate_and_proof`) two-thirds of the way through the `slot`-that is, `SECONDS_PER_SLOT * 2 / 3` seconds after the start of `slot`. -Aggregate attestations are broadcast as `AggregateAndProof` objects to prove to the gossip channel that the validator has been selected as an aggregator. +Selection proofs are provided in `AggregateAndProof` to prove to the gossip channel that the validator has been selected as an aggregator. + +`AggregateAndProof` messages are signed and broadcast inside of `SignedAggregateAndProof` objects to prevent a class of DoS attacks and message forgeries. ##### `AggregateAndProof` @@ -479,6 +481,14 @@ Where * `aggregate` is the `aggregate_attestation` constructed in the previous section. * `selection_proof` is the signature of the slot (`get_slot_signature()`). +##### `SignedAggregateAndProof` + +```python +class SignedAggregateAndProof(Container): + message: AggregateAndProof + signature: BLSSignature +``` + ## Phase 0 attestation subnet stability Because Phase 0 does not have shards and thus does not have Shard Committees, there is no stable backbone to the attestation subnets (`committee_index{subnet_id}_beacon_attestation`). To provide this stability, each validator must: From 343168908cab7700cd28318536dc2d608500d44c Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 10 Feb 2020 19:28:37 -0700 Subject: [PATCH 008/102] prevent multiple beacon blocks from same proposer in a given slot --- specs/phase0/p2p-interface.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 38adfb316..9979cbb14 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -250,6 +250,7 @@ When processing incoming gossip, clients MAY descore or disconnect peers who fai There are two primary global topics used to propagate beacon blocks and aggregate attestations to all nodes on the network. Their `TopicName`s are: - `beacon_block` - This topic is used solely for propagating new signed beacon blocks to all nodes on the networks. Signed blocks are sent in their entirety. The following validations MUST pass before forwarding the `signed_beacon_block` on the network + - The block is the first block received for the proposer for the slot, `signed_beacon_block.message.slot`. - The proposer signature, `signed_beacon_block.signature`, is valid. - The block is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `signed_beacon_block.message.slot <= current_slot` (a client MAY queue future blocks for processing at the appropriate slot). - `beacon_aggregate_and_proof` - This topic is used to propagate aggregated attestations (as `SignedAggregateAndProof`s) to subscribing nodes (typically validators) to be included in future blocks. The following validations MUST pass before forwarding the `signed_aggregate_and_proof` on the network. (We define the following for convenience -- `aggregate_and_proof = signed_aggregate_and_proof.message` and `aggregate = aggregate_and_proof.aggregate`) @@ -258,7 +259,7 @@ There are two primary global topics used to propagate beacon blocks and aggregat - `aggregate.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= aggregate.data.slot`. - The validator index is within the aggregate's committee -- i.e. `aggregate_and_proof.aggregator_index in get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)`. - `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_aggregator(state, aggregate.data.slot, aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. - - The `signed_aggregate_and_proof` is the first aggregate received for the aggregator for the slot `aggregate.data.slot` + - The `signed_aggregate_and_proof` is the first aggregate received for the aggregator for the slot, `aggregate.data.slot` - The `aggregate_and_proof.selection_proof` is a valid signature of the `aggregate.data.slot` by the validator with index `aggregate_and_proof.aggregator_index`. - The aggregator signature, `signed_aggregate_and_proof.signature`, is valid. - The signature of `aggregate` is valid. @@ -275,7 +276,7 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio - `committee_index{subnet_id}_beacon_attestation` - These topics are used to propagate unaggregated attestations to the subnet `subnet_id` (typically beacon and persistent committees) to be aggregated before being gossiped to `beacon_aggregate_and_proof`. The following validations MUST pass before forwarding the `attestation` on the subnet. - The attestation's committee index (`attestation.data.index`) is for the correct subnet. - - The attestation is the first attestation received for the participating validator for the slot `attestation.data.slot` + - The attestation is the first attestation received for the participating validator for the slot, `attestation.data.slot` - `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot`. - The attestation is unaggregated -- that is, it has exactly one participating validator (`len([bit for bit in attestation.aggregation_bits if bit == 0b1]) == 1`). - This is the first attestation for the participating validator at this slot. From fd633d246781a27e74d354476e29c0d09ec49fc8 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 10 Feb 2020 19:35:36 -0700 Subject: [PATCH 009/102] add clarifying note to signedaggregateandproof in vaidator guide and fix tocs --- specs/phase0/validator.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 8996428cc..528bcf751 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -59,6 +59,7 @@ - [Aggregate signature](#aggregate-signature-1) - [Broadcast aggregate](#broadcast-aggregate) - [`AggregateAndProof`](#aggregateandproof) + - [`SignedAggregateAndProof`](#signedaggregateandproof) - [Phase 0 attestation subnet stability](#phase-0-attestation-subnet-stability) - [How to avoid slashing](#how-to-avoid-slashing) - [Proposer slashing](#proposer-slashing) @@ -465,7 +466,7 @@ If the validator is selected to aggregate (`is_aggregator`), then they broadcast Selection proofs are provided in `AggregateAndProof` to prove to the gossip channel that the validator has been selected as an aggregator. -`AggregateAndProof` messages are signed and broadcast inside of `SignedAggregateAndProof` objects to prevent a class of DoS attacks and message forgeries. +`AggregateAndProof` messages are signed by the aggregator and broadcast inside of `SignedAggregateAndProof` objects to prevent a class of DoS attacks and message forgeries. ##### `AggregateAndProof` From 5ee1f9b54543210afffbddb1320513649934be50 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 10 Feb 2020 21:35:39 -0700 Subject: [PATCH 010/102] fix up some p2p validation conditions based on PR feedback --- specs/phase0/p2p-interface.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 9979cbb14..c60ba560c 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -276,10 +276,9 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio - `committee_index{subnet_id}_beacon_attestation` - These topics are used to propagate unaggregated attestations to the subnet `subnet_id` (typically beacon and persistent committees) to be aggregated before being gossiped to `beacon_aggregate_and_proof`. The following validations MUST pass before forwarding the `attestation` on the subnet. - The attestation's committee index (`attestation.data.index`) is for the correct subnet. - - The attestation is the first attestation received for the participating validator for the slot, `attestation.data.slot` - `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot`. - The attestation is unaggregated -- that is, it has exactly one participating validator (`len([bit for bit in attestation.aggregation_bits if bit == 0b1]) == 1`). - - This is the first attestation for the participating validator at this slot. + - The attestation is the first attestation received for the participating validator for the slot, `attestation.data.slot`. - The block being voted for (`attestation.data.beacon_block_root`) passes validation. - The signature of `attestation` is valid. From dde69cb8e2c7182accdab3d19434b66c2df80156 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 11 Feb 2020 15:45:51 -0700 Subject: [PATCH 011/102] add explicit instrucutions for construction of signed_aggregate_and_proof. add DOMAIN_SELECTION_PROOF and DOMAIN_AGGREGATE_AND_PROOF --- configs/mainnet.yaml | 2 ++ configs/minimal.yaml | 2 ++ specs/phase0/beacon-chain.md | 13 ++++++++----- specs/phase0/validator.md | 28 +++++++++++++++++++++++++++- 4 files changed, 39 insertions(+), 6 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 6eb5641d0..74f062d9b 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -143,6 +143,8 @@ DOMAIN_BEACON_ATTESTER: 0x01000000 DOMAIN_RANDAO: 0x02000000 DOMAIN_DEPOSIT: 0x03000000 DOMAIN_VOLUNTARY_EXIT: 0x04000000 +DOMAIN_SELECTION_PROOF: 0x05000000 +DOMAIN_AGGREGATE_AND_PROOF: 0x06000000 # Phase 1 DOMAIN_SHARD_PROPOSAL: 0x80000000 DOMAIN_SHARD_COMMITTEE: 0x81000000 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 03ffa90e3..42c63e301 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -142,6 +142,8 @@ DOMAIN_BEACON_ATTESTER: 0x01000000 DOMAIN_RANDAO: 0x02000000 DOMAIN_DEPOSIT: 0x03000000 DOMAIN_VOLUNTARY_EXIT: 0x04000000 +DOMAIN_SELECTION_PROOF: 0x05000000 +DOMAIN_AGGREGATE_AND_PROOF: 0x06000000 # Phase 1 DOMAIN_SHARD_PROPOSAL: 0x80000000 DOMAIN_SHARD_COMMITTEE: 0x81000000 diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 96ea351c8..2ab51079d 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -253,11 +253,14 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | | - | - | -| `DOMAIN_BEACON_PROPOSER` | `DomainType('0x00000000')` | -| `DOMAIN_BEACON_ATTESTER` | `DomainType('0x01000000')` | -| `DOMAIN_RANDAO` | `DomainType('0x02000000')` | -| `DOMAIN_DEPOSIT` | `DomainType('0x03000000')` | -| `DOMAIN_VOLUNTARY_EXIT` | `DomainType('0x04000000')` | +| `DOMAIN_BEACON_PROPOSER` | `DomainType('0x00000000')` | +| `DOMAIN_BEACON_ATTESTER` | `DomainType('0x01000000')` | +| `DOMAIN_RANDAO` | `DomainType('0x02000000')` | +| `DOMAIN_DEPOSIT` | `DomainType('0x03000000')` | +| `DOMAIN_VOLUNTARY_EXIT` | `DomainType('0x04000000')` | +| `DOMAIN_SELECTION_PROOF` | `DomainType('0x05000000')` | +| `DOMAIN_AGGREGATE_AND_PROOF` | `DomainType('0x06000000')` | + ## Containers diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 528bcf751..ad9ef9f37 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -424,7 +424,7 @@ A validator is selected to aggregate based upon the return value of `is_aggregat ```python def get_slot_signature(state: BeaconState, slot: Slot, privkey: int) -> BLSSignature: - domain = get_domain(state, DOMAIN_BEACON_ATTESTER, compute_epoch_at_slot(slot)) + domain = get_domain(state, DOMAIN_SELECTION_PROOF, compute_epoch_at_slot(slot)) signing_root = compute_signing_root(slot, domain) return bls.Sign(privkey, signing_root) ``` @@ -468,6 +468,32 @@ Selection proofs are provided in `AggregateAndProof` to prove to the gossip chan `AggregateAndProof` messages are signed by the aggregator and broadcast inside of `SignedAggregateAndProof` objects to prevent a class of DoS attacks and message forgeries. +First, `aggregate_and_proof = get_aggregate_and_proof(state, aggregate_attestation, validator_index, privkey)` is contructed. + +```python +def get_aggregate_and_proof(state: BeaconState, + aggregate: Attestation, + aggregator_index: ValidatorIndex, + privkey: int) -> AggregateAndProof: + return AggregateAndProof( + aggregator_index=aggregator_index, + aggregate=aggregate, + selection_proof=get_slot_signature(state, aggregate.data.slot, privkey), + ) +``` + +Then `signed_aggregate_and_proof = SignedAggregateAndProof(message=aggregate_and_proof, signature=signature)` is constructed and broadast. Where `signature` is obtained from: + +```python +def get_aggregate_and_proof_signature(state: BeaconState, + aggregate_and_proof: AggregateAndProof, + privkey: int) -> BLSSignature: + aggregate = aggregate_and_proof.aggregate + domain = get_domain(state, DOMAIN_AGGREGATE_AND_PROOF, compute_epoch_at_slot(aggregate.data.slot)) + signing_root = compute_signing_root(aggregate_and_proof, domain) + return bls.Sign(privkey, signing_root) +``` + ##### `AggregateAndProof` ```python From d2e08c0cdf7d56cebf7f64180586bd19cb54b61a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 11 Feb 2020 15:46:30 -0700 Subject: [PATCH 012/102] subnet validation PR feedback --- specs/phase0/p2p-interface.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index c60ba560c..c0dd88643 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -257,9 +257,9 @@ There are two primary global topics used to propagate beacon blocks and aggregat - The aggregate attestation defined by `hash_tree_root(aggregate)` has _not_ already been seen (via aggregate gossip, within a block, or through the creation of an equivalent aggregate locally). - The block being voted for (`aggregate.data.beacon_block_root`) passes validation. - `aggregate.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= aggregate.data.slot`. - - The validator index is within the aggregate's committee -- i.e. `aggregate_and_proof.aggregator_index in get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)`. + - The aggregator's validator index is within the aggregate's committee -- i.e. `aggregate_and_proof.aggregator_index in get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)`. - `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_aggregator(state, aggregate.data.slot, aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. - - The `signed_aggregate_and_proof` is the first aggregate received for the aggregator for the slot, `aggregate.data.slot` + - The `aggregate` is the first aggregate received for the aggregator with index `aggregate_and_proof.aggregator_index` for the slot `aggregate.data.slot`. - The `aggregate_and_proof.selection_proof` is a valid signature of the `aggregate.data.slot` by the validator with index `aggregate_and_proof.aggregator_index`. - The aggregator signature, `signed_aggregate_and_proof.signature`, is valid. - The signature of `aggregate` is valid. From f7181adece04e9440e128d37fe3d11c38d7d26ff Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 11 Feb 2020 15:49:45 -0700 Subject: [PATCH 013/102] reverse params in get_aggregate_and_proof to match ssz type --- specs/phase0/validator.md | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index ad9ef9f37..0bde81e60 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -468,12 +468,12 @@ Selection proofs are provided in `AggregateAndProof` to prove to the gossip chan `AggregateAndProof` messages are signed by the aggregator and broadcast inside of `SignedAggregateAndProof` objects to prevent a class of DoS attacks and message forgeries. -First, `aggregate_and_proof = get_aggregate_and_proof(state, aggregate_attestation, validator_index, privkey)` is contructed. +First, `aggregate_and_proof = get_aggregate_and_proof(state, validator_index, aggregate_attestation, privkey)` is constructed. ```python def get_aggregate_and_proof(state: BeaconState, - aggregate: Attestation, aggregator_index: ValidatorIndex, + aggregate: Attestation, privkey: int) -> AggregateAndProof: return AggregateAndProof( aggregator_index=aggregator_index, @@ -503,11 +503,6 @@ class AggregateAndProof(Container): selection_proof: BLSSignature ``` -Where -* `aggregator_index` is the validator's `ValidatorIndex`. -* `aggregate` is the `aggregate_attestation` constructed in the previous section. -* `selection_proof` is the signature of the slot (`get_slot_signature()`). - ##### `SignedAggregateAndProof` ```python From 581257e269275ce17b8c4a84d77532de5e93081c Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 12 Feb 2020 11:59:00 -0700 Subject: [PATCH 014/102] reorder gossip conditions to put cheap checks before signature verifications --- specs/phase0/p2p-interface.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index c0dd88643..f5a82bad4 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -250,16 +250,16 @@ When processing incoming gossip, clients MAY descore or disconnect peers who fai There are two primary global topics used to propagate beacon blocks and aggregate attestations to all nodes on the network. Their `TopicName`s are: - `beacon_block` - This topic is used solely for propagating new signed beacon blocks to all nodes on the networks. Signed blocks are sent in their entirety. The following validations MUST pass before forwarding the `signed_beacon_block` on the network + - The block is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `signed_beacon_block.message.slot <= current_slot` (a client MAY queue future blocks for processing at the appropriate slot). - The block is the first block received for the proposer for the slot, `signed_beacon_block.message.slot`. - The proposer signature, `signed_beacon_block.signature`, is valid. - - The block is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `signed_beacon_block.message.slot <= current_slot` (a client MAY queue future blocks for processing at the appropriate slot). - `beacon_aggregate_and_proof` - This topic is used to propagate aggregated attestations (as `SignedAggregateAndProof`s) to subscribing nodes (typically validators) to be included in future blocks. The following validations MUST pass before forwarding the `signed_aggregate_and_proof` on the network. (We define the following for convenience -- `aggregate_and_proof = signed_aggregate_and_proof.message` and `aggregate = aggregate_and_proof.aggregate`) + - `aggregate.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= aggregate.data.slot` (a client MAY queue future aggregates for processing at the appropriate slot). - The aggregate attestation defined by `hash_tree_root(aggregate)` has _not_ already been seen (via aggregate gossip, within a block, or through the creation of an equivalent aggregate locally). - - The block being voted for (`aggregate.data.beacon_block_root`) passes validation. - - `aggregate.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= aggregate.data.slot`. - - The aggregator's validator index is within the aggregate's committee -- i.e. `aggregate_and_proof.aggregator_index in get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)`. - - `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_aggregator(state, aggregate.data.slot, aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. - The `aggregate` is the first aggregate received for the aggregator with index `aggregate_and_proof.aggregator_index` for the slot `aggregate.data.slot`. + - The block being voted for (`aggregate.data.beacon_block_root`) passes validation. + - `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_aggregator(state, aggregate.data.slot, aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. + - The aggregator's validator index is within the aggregate's committee -- i.e. `aggregate_and_proof.aggregator_index in get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)`. - The `aggregate_and_proof.selection_proof` is a valid signature of the `aggregate.data.slot` by the validator with index `aggregate_and_proof.aggregator_index`. - The aggregator signature, `signed_aggregate_and_proof.signature`, is valid. - The signature of `aggregate` is valid. @@ -276,7 +276,7 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio - `committee_index{subnet_id}_beacon_attestation` - These topics are used to propagate unaggregated attestations to the subnet `subnet_id` (typically beacon and persistent committees) to be aggregated before being gossiped to `beacon_aggregate_and_proof`. The following validations MUST pass before forwarding the `attestation` on the subnet. - The attestation's committee index (`attestation.data.index`) is for the correct subnet. - - `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot`. + - `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` (a client MAY queue future attestations for processing at the appropriate slot). - The attestation is unaggregated -- that is, it has exactly one participating validator (`len([bit for bit in attestation.aggregation_bits if bit == 0b1]) == 1`). - The attestation is the first attestation received for the participating validator for the slot, `attestation.data.slot`. - The block being voted for (`attestation.data.beacon_block_root`) passes validation. From 3404f1e07805ec98038e2849e57c05d7f06b3de8 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 12 Feb 2020 12:07:57 -0700 Subject: [PATCH 015/102] add lower bound condition on block gossip --- specs/phase0/p2p-interface.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index f5a82bad4..81536a314 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -251,6 +251,7 @@ There are two primary global topics used to propagate beacon blocks and aggregat - `beacon_block` - This topic is used solely for propagating new signed beacon blocks to all nodes on the networks. Signed blocks are sent in their entirety. The following validations MUST pass before forwarding the `signed_beacon_block` on the network - The block is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `signed_beacon_block.message.slot <= current_slot` (a client MAY queue future blocks for processing at the appropriate slot). + - The block is not from later than the latest finalized slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `signed_beacon_block.message.slot > compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)` (a client MAY choose to validate and store such blocks for additional purposes -- e.g. slashing detection, archive nodes, etc). - The block is the first block received for the proposer for the slot, `signed_beacon_block.message.slot`. - The proposer signature, `signed_beacon_block.signature`, is valid. - `beacon_aggregate_and_proof` - This topic is used to propagate aggregated attestations (as `SignedAggregateAndProof`s) to subscribing nodes (typically validators) to be included in future blocks. The following validations MUST pass before forwarding the `signed_aggregate_and_proof` on the network. (We define the following for convenience -- `aggregate_and_proof = signed_aggregate_and_proof.message` and `aggregate = aggregate_and_proof.aggregate`) From 7329cc09337e1b8cdff24c272824186cc129631d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 12 Feb 2020 15:46:48 -0700 Subject: [PATCH 016/102] pr feedback Co-Authored-By: Diederik Loerakker --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 81536a314..d7fbbf8a7 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -251,7 +251,7 @@ There are two primary global topics used to propagate beacon blocks and aggregat - `beacon_block` - This topic is used solely for propagating new signed beacon blocks to all nodes on the networks. Signed blocks are sent in their entirety. The following validations MUST pass before forwarding the `signed_beacon_block` on the network - The block is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `signed_beacon_block.message.slot <= current_slot` (a client MAY queue future blocks for processing at the appropriate slot). - - The block is not from later than the latest finalized slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `signed_beacon_block.message.slot > compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)` (a client MAY choose to validate and store such blocks for additional purposes -- e.g. slashing detection, archive nodes, etc). + - The block is from a slot greater than the latest finalized slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `signed_beacon_block.message.slot > compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)` (a client MAY choose to validate and store such blocks for additional purposes -- e.g. slashing detection, archive nodes, etc). - The block is the first block received for the proposer for the slot, `signed_beacon_block.message.slot`. - The proposer signature, `signed_beacon_block.signature`, is valid. - `beacon_aggregate_and_proof` - This topic is used to propagate aggregated attestations (as `SignedAggregateAndProof`s) to subscribing nodes (typically validators) to be included in future blocks. The following validations MUST pass before forwarding the `signed_aggregate_and_proof` on the network. (We define the following for convenience -- `aggregate_and_proof = signed_aggregate_and_proof.message` and `aggregate = aggregate_and_proof.aggregate`) From f671b86776c52cea8d530c036583bf9fa15a1df0 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 12 Feb 2020 15:48:49 -0700 Subject: [PATCH 017/102] add DoS prevention validation conditions to voluntary_exit, proposer_slashing, and attester_slashing gossipsub channels --- specs/phase0/p2p-interface.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index d7fbbf8a7..54db92539 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -252,12 +252,12 @@ There are two primary global topics used to propagate beacon blocks and aggregat - `beacon_block` - This topic is used solely for propagating new signed beacon blocks to all nodes on the networks. Signed blocks are sent in their entirety. The following validations MUST pass before forwarding the `signed_beacon_block` on the network - The block is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `signed_beacon_block.message.slot <= current_slot` (a client MAY queue future blocks for processing at the appropriate slot). - The block is from a slot greater than the latest finalized slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `signed_beacon_block.message.slot > compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)` (a client MAY choose to validate and store such blocks for additional purposes -- e.g. slashing detection, archive nodes, etc). - - The block is the first block received for the proposer for the slot, `signed_beacon_block.message.slot`. + - The block is the first block with valid signature received for the proposer for the slot, `signed_beacon_block.message.slot`. - The proposer signature, `signed_beacon_block.signature`, is valid. - `beacon_aggregate_and_proof` - This topic is used to propagate aggregated attestations (as `SignedAggregateAndProof`s) to subscribing nodes (typically validators) to be included in future blocks. The following validations MUST pass before forwarding the `signed_aggregate_and_proof` on the network. (We define the following for convenience -- `aggregate_and_proof = signed_aggregate_and_proof.message` and `aggregate = aggregate_and_proof.aggregate`) - `aggregate.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= aggregate.data.slot` (a client MAY queue future aggregates for processing at the appropriate slot). - The aggregate attestation defined by `hash_tree_root(aggregate)` has _not_ already been seen (via aggregate gossip, within a block, or through the creation of an equivalent aggregate locally). - - The `aggregate` is the first aggregate received for the aggregator with index `aggregate_and_proof.aggregator_index` for the slot `aggregate.data.slot`. + - The `aggregate` is the first valid aggregate received for the aggregator with index `aggregate_and_proof.aggregator_index` for the slot `aggregate.data.slot`. - The block being voted for (`aggregate.data.beacon_block_root`) passes validation. - `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_aggregator(state, aggregate.data.slot, aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. - The aggregator's validator index is within the aggregate's committee -- i.e. `aggregate_and_proof.aggregator_index in get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)`. @@ -267,9 +267,16 @@ There are two primary global topics used to propagate beacon blocks and aggregat Additional global topics are used to propagate lower frequency validator messages. Their `TopicName`s are: -- `voluntary_exit` - This topic is used solely for propagating signed voluntary validator exits to proposers on the network. Signed voluntary exits are sent in their entirety. Clients who receive a signed voluntary exit on this topic MUST validate the conditions within `process_voluntary_exit` before forwarding it across the network. -- `proposer_slashing` - This topic is used solely for propagating proposer slashings to proposers on the network. Proposer slashings are sent in their entirety. Clients who receive a proposer slashing on this topic MUST validate the conditions within `process_proposer_slashing` before forwarding it across the network. +- `voluntary_exit` - This topic is used solely for propagating signed voluntary validator exits to proposers on the network. Signed voluntary exits are sent in their entirety. The following validations MUST pass before forwarding the `signed_voluntary_exit` on to the network + - The voluntary exit is the first valid voluntary exit received for the validator with index `signed_voluntary_exit.message.validator_index`. + - All of the conditions within `process_voluntary_exit` pass validation. +- `proposer_slashing` - This topic is used solely for propagating proposer slashings to proposers on the network. Proposer slashings are sent in their entirety. The following validations MUST pass before forwarding the `proposer_slashing` on to the network + - The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.index`. + - All of the conditions within `process_proposer_slashing` pass validation. - `attester_slashing` - This topic is used solely for propagating attester slashings to proposers on the network. Attester slashings are sent in their entirety. Clients who receive an attester slashing on this topic MUST validate the conditions within `process_attester_slashing` before forwarding it across the network. + - At least one index in the intersection of the attesting indices of each attestation has not yet been seen in any prior `attester_slashing` (i.e. `any((set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices)).difference(prior_seen_attester_slashed_indices))`). + - All of the conditions within `process_attester_slashing` pass validation. + #### Attestation subnets @@ -279,7 +286,7 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio - The attestation's committee index (`attestation.data.index`) is for the correct subnet. - `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` (a client MAY queue future attestations for processing at the appropriate slot). - The attestation is unaggregated -- that is, it has exactly one participating validator (`len([bit for bit in attestation.aggregation_bits if bit == 0b1]) == 1`). - - The attestation is the first attestation received for the participating validator for the slot, `attestation.data.slot`. + - The attestation is the first valid attestation received for the participating validator for the slot, `attestation.data.slot`. - The block being voted for (`attestation.data.beacon_block_root`) passes validation. - The signature of `attestation` is valid. From 3bb8e0d962b83823e24c395c6ebbe1c858559a73 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 13 Feb 2020 11:17:19 -0700 Subject: [PATCH 018/102] cleanup attester slashing conditon code snippet Co-Authored-By: Diederik Loerakker --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 54db92539..7ad8759d3 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -274,7 +274,7 @@ Additional global topics are used to propagate lower frequency validator message - The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.index`. - All of the conditions within `process_proposer_slashing` pass validation. - `attester_slashing` - This topic is used solely for propagating attester slashings to proposers on the network. Attester slashings are sent in their entirety. Clients who receive an attester slashing on this topic MUST validate the conditions within `process_attester_slashing` before forwarding it across the network. - - At least one index in the intersection of the attesting indices of each attestation has not yet been seen in any prior `attester_slashing` (i.e. `any((set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices)).difference(prior_seen_attester_slashed_indices))`). + - At least one index in the intersection of the attesting indices of each attestation has not yet been seen in any prior `attester_slashing` (i.e. `attester_slashed_indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices)`, verify if `any(attester_slashed_indices.difference(prior_seen_attester_slashed_indices))`). - All of the conditions within `process_attester_slashing` pass validation. From 934c037a8a9959209ff978882f5b142f619341da Mon Sep 17 00:00:00 2001 From: Ben Edgington Date: Fri, 14 Feb 2020 14:29:45 +0000 Subject: [PATCH 019/102] Correct the duration of HISTORICAL_ROOTS_LIMIT The duration of HISTORICAL_ROOTS_LIMIT is: SECONDS_PER_SLOT * SLOTS_PER_HISTORICAL_ROOT * HISTORICAL_ROOTS_LIMIT which is 12s * 2^13 * 2^24 = 1.65e12s = 52,262 years --- specs/phase0/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 2ab51079d..acf098663 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -224,7 +224,7 @@ The following values are (non-configurable) constants used throughout the specif | - | - | :-: | :-: | | `EPOCHS_PER_HISTORICAL_VECTOR` | `2**16` (= 65,536) | epochs | ~0.8 years | | `EPOCHS_PER_SLASHINGS_VECTOR` | `2**13` (= 8,192) | epochs | ~36 days | -| `HISTORICAL_ROOTS_LIMIT` | `2**24` (= 16,777,216) | historical roots | ~26,131 years | +| `HISTORICAL_ROOTS_LIMIT` | `2**24` (= 16,777,216) | historical roots | ~52,262 years | | `VALIDATOR_REGISTRY_LIMIT` | `2**40` (= 1,099,511,627,776) | validators | ### Rewards and penalties From 09266cf6e82a56f20501021bcfbbee3172cf6f80 Mon Sep 17 00:00:00 2001 From: ethers <6937903+ethers@users.noreply.github.com> Date: Mon, 17 Feb 2020 00:14:07 +0000 Subject: [PATCH 020/102] add Gasper paper --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 49d851197..3f58b3dc4 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ The following are the broad design goals for Ethereum 2.0: * [Design Rationale](https://notes.ethereum.org/s/rkhCgQteN#) * [Phase 0 Onboarding Document](https://notes.ethereum.org/s/Bkn3zpwxB) +* [Gasper paper](https://github.com/ethereum/research/blob/master/papers/ffg%2Bghost/paper.pdf) ## For spec contributors From ceb6633eb9ef6ad0226a9119e35b99e57a58b527 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 21 Feb 2020 21:44:54 -0600 Subject: [PATCH 021/102] working through phase 1 attestation testing --- configs/mainnet.yaml | 2 +- configs/minimal.yaml | 2 +- specs/phase1/beacon-chain.md | 19 ++-- .../eth2spec/test/helpers/attestations.py | 37 +++++++- .../test/helpers/phase1/attestations.py | 82 ++++++++++++----- .../test_process_attestation.py | 37 +------- .../test_process_attestation.py | 91 +++++++------------ 7 files changed, 142 insertions(+), 128 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 74f062d9b..c7940bceb 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -179,7 +179,7 @@ MAX_SHARD_BLOCK_CHUNKS: 4 # 3 * 2**16` (= 196,608) TARGET_SHARD_BLOCK_SIZE: 196608 # Note: MAX_SHARD_BLOCKS_PER_ATTESTATION is derived from the list length. -SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] +# SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] # len(SHARD_BLOCK_OFFSETS) MAX_SHARD_BLOCKS_PER_ATTESTATION: 12 # 2**14 (= 16,384) Gwei diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 42c63e301..48fab475c 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -180,7 +180,7 @@ MAX_SHARD_BLOCK_CHUNKS: 4 # 3 * 2**16` (= 196,608) TARGET_SHARD_BLOCK_SIZE: 196608 # Note: MAX_SHARD_BLOCKS_PER_ATTESTATION is derived from the list length. -SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] +# SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] # len(SHARD_BLOCK_OFFSETS) MAX_SHARD_BLOCKS_PER_ATTESTATION: 12 # 2**14 (= 16,384) Gwei diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index bdf70c5bd..897dae369 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -57,7 +57,7 @@ - [`apply_shard_transition`](#apply_shard_transition) - [`process_crosslink_for_shard`](#process_crosslink_for_shard) - [`process_crosslinks`](#process_crosslinks) - - [`process_attestations`](#process_attestations) + - [`process_attestation`](#process_attestation) - [New Attester slashing processing](#new-attester-slashing-processing) - [Shard transition false positives](#shard-transition-false-positives) - [Light client processing](#light-client-processing) @@ -98,7 +98,7 @@ Configuration is not namespaced. Instead it is strictly an extension; | `SHARD_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | | `MAX_SHARD_BLOCK_SIZE` | `2**20` (= 1,048,576) | | | `TARGET_SHARD_BLOCK_SIZE` | `2**18` (= 262,144) | | -| `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | | +| `SHARD_BLOCK_OFFSETS` | `[0, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | | | `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | | | `MAX_GASPRICE` | `Gwei(2**14)` (= 16,384) | Gwei | | | `MIN_GASPRICE` | `Gwei(2**5)` (= 32) | Gwei | | @@ -500,7 +500,6 @@ def get_next_slot_for_shard(state: BeaconState, shard: Shard) -> Slot: return Slot(state.shard_states[shard].slot + 1) ``` - #### `get_offset_slots` ```python @@ -526,7 +525,7 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch) aggregation_bits = attestation.aggregation_bits assert len(aggregation_bits) == len(indexed_attestation.committee) - + if len(attestation.custody_bits_blocks) == 0: # fall back on phase0 behavior if there is no shard data. for participant, abit in zip(indexed_attestation.committee, aggregation_bits): @@ -612,11 +611,9 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: shard = get_shard(state, attestation) shard_start_slot = get_next_slot_for_shard(state, shard) - # Signature check - assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) # Type 1: on-time attestations if attestation.custody_bits_blocks != []: - # Correct slot + # Ensure on-time attestation assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY == state.slot # Correct data root count assert len(attestation.custody_bits_blocks) == len(get_offset_slots(state, shard_start_slot)) @@ -624,8 +621,14 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: assert data.beacon_block_root == get_block_root_at_slot(state, get_previous_slot(state.slot)) # Type 2: no shard transition, no custody bits # TODO: could only allow for older attestations. else: - # assert state.slot - compute_start_slot_at_epoch(compute_epoch_at_slot(data.slot)) < SLOTS_PER_EPOCH + # Ensure delayed attestation + # Currently commented out because breaks a ton of phase 0 tests + # assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY < state.slot + # Late attestations cannot have a shard transition root assert data.shard_transition_root == Root() + + # Signature check + assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) ``` ###### `apply_shard_transition` diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 9e12582e0..0a6e3294d 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -1,11 +1,47 @@ from typing import List from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.context import expect_assertion_error from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.ssz.ssz_typing import Bitlist +def run_attestation_processing(spec, state, attestation, valid=True): + """ + Run ``process_attestation``, yielding: + - pre-state ('pre') + - attestation ('attestation') + - post-state ('post'). + If ``valid == False``, run expecting ``AssertionError`` + """ + # yield pre-state + yield 'pre', state + + yield 'attestation', attestation + + # If the attestation is invalid, processing is aborted, and there is no post-state. + if not valid: + expect_assertion_error(lambda: spec.process_attestation(state, attestation)) + yield 'post', None + return + + current_epoch_count = len(state.current_epoch_attestations) + previous_epoch_count = len(state.previous_epoch_attestations) + + # process attestation + spec.process_attestation(state, attestation) + + # Make sure the attestation has been processed + if attestation.data.target.epoch == spec.get_current_epoch(state): + assert len(state.current_epoch_attestations) == current_epoch_count + 1 + else: + assert len(state.previous_epoch_attestations) == previous_epoch_count + 1 + + # yield post-state + yield 'post', state + + def build_attestation_data(spec, state, slot, index): assert state.slot >= slot @@ -111,7 +147,6 @@ def get_attestation_signature(spec, state, attestation_data, privkey): def fill_aggregate_attestation(spec, state, attestation, signed=False): - beacon_committee = spec.get_beacon_committee( state, attestation.data.slot, diff --git a/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py index 622183fe9..204458342 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py @@ -1,30 +1,66 @@ -from eth2spec.test.helpers.keys import privkeys +from eth2spec.utils.ssz.ssz_typing import Bitlist from eth2spec.utils import bls +from eth2spec.test.helpers.keys import privkeys +import eth2spec.test.helpers.attestations as phase0_attestations +from eth2spec.test.helpers.state import next_slot -def sign_shard_attestation(spec, beacon_state, shard_state, block, participants): - signatures = [] - message_hash = spec.ShardAttestationData( - slot=block.slot, - parent_root=block.parent_root, - ).hash_tree_root() - block_epoch = spec.compute_epoch_of_shard_slot(block.slot) - for validator_index in participants: - privkey = privkeys[validator_index] - signatures.append( - get_attestation_signature( - spec, - beacon_state, - shard_state, - message_hash, - block_epoch, - privkey, - ) + +def get_valid_on_time_attestation(spec, state, index=None, signed=False): + ''' + Construct on-time attestation for next slot + ''' + if index is None: + index = 0 + + attestation = phase0_attestations.get_valid_attestation(spec, state, state.slot, index, False) + shard = spec.get_shard(state, attestation) + + next_state = state.copy() + next_slot(spec, next_state) + offset_slots = spec.get_offset_slots(next_state, spec.get_next_slot_for_shard(next_state, shard)) + for offset_slot in offset_slots: + attestation.custody_bits_blocks.append( + Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE]([0 for _ in attestation.aggregation_bits]) ) - return bls.Aggregate(signatures) + + if signed: + sign_attestation(spec, state, attestation) + + return attestation -def get_attestation_signature(spec, beacon_state, shard_state, message_hash, block_epoch, privkey): - domain = spec.get_domain(beacon_state, spec.DOMAIN_SHARD_ATTESTER, block_epoch) - signing_root = spec.compute_signing_root(message_hash, domain) +def sign_attestation(spec, state, attestation): + if not any(attestation.custody_bits_blocks): + phase0_attestations.sign_attestation(spec, state, attestation) + return + + committee = spec.get_beacon_committee(state, attestation.data.slot, attestation.data.index) + signatures = [] + for block_index, custody_bits in enumerate(attestation.custody_bits_blocks): + for participant, abit, cbit in zip(committee, attestation.aggregation_bits, custody_bits): + if not abit: + continue + signatures.append(get_attestation_custody_signature( + spec, + state, + attestation.data, + block_index, + cbit, + privkeys[participant] + )) + + attestation.signature = bls.Aggregate(signatures) + + +def get_attestation_custody_signature(spec, state, attestation_data, block_index, bit, privkey): + domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) + signing_root = spec.compute_signing_root( + spec.AttestationCustodyBitWrapper( + attestation_data.hash_tree_root(), + block_index, + bit, + ), + domain, + ) return bls.Sign(privkey, signing_root) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 1dbdafec2..236fceef1 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -1,6 +1,5 @@ from eth2spec.test.context import ( spec_state_test, - expect_assertion_error, always_bls, never_bls, with_all_phases, spec_test, @@ -8,6 +7,7 @@ from eth2spec.test.context import ( with_custom_state, single_phase) from eth2spec.test.helpers.attestations import ( + run_attestation_processing, get_valid_attestation, sign_aggregate_attestation, sign_attestation, @@ -19,41 +19,6 @@ from eth2spec.test.helpers.block import apply_empty_block from eth2spec.utils.ssz.ssz_typing import Bitlist -def run_attestation_processing(spec, state, attestation, valid=True): - """ - Run ``process_attestation``, yielding: - - pre-state ('pre') - - attestation ('attestation') - - post-state ('post'). - If ``valid == False``, run expecting ``AssertionError`` - """ - # yield pre-state - yield 'pre', state - - yield 'attestation', attestation - - # If the attestation is invalid, processing is aborted, and there is no post-state. - if not valid: - expect_assertion_error(lambda: spec.process_attestation(state, attestation)) - yield 'post', None - return - - current_epoch_count = len(state.current_epoch_attestations) - previous_epoch_count = len(state.previous_epoch_attestations) - - # process attestation - spec.process_attestation(state, attestation) - - # Make sure the attestation has been processed - if attestation.data.target.epoch == spec.get_current_epoch(state): - assert len(state.current_epoch_attestations) == current_epoch_count + 1 - else: - assert len(state.previous_epoch_attestations) == previous_epoch_count + 1 - - # yield post-state - yield 'post', state - - @with_all_phases @spec_state_test def test_success(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_attestation.py index 875039c59..430ad014b 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_attestation.py @@ -1,61 +1,26 @@ from eth2spec.test.context import ( with_all_phases_except, - expect_assertion_error, spec_state_test, always_bls, ) +from eth2spec.test.helpers.state import next_slot, transition_to from eth2spec.test.helpers.attestations import ( - get_valid_attestation, - sign_attestation, + run_attestation_processing, + # get_valid_attestation as get_valid_late_attestation, +) +from eth2spec.test.helpers.phase1.attestations import ( + get_valid_on_time_attestation, ) - -from eth2spec.utils.ssz.ssz_typing import Bitlist - - -def run_attestation_processing(spec, state, attestation, valid=True): - """ - Run ``process_attestation``, yielding: - - pre-state ('pre') - - attestation ('attestation') - - post-state ('post'). - If ``valid == False``, run expecting ``AssertionError`` - """ - # yield pre-state - yield 'pre', state - - yield 'attestation', attestation - - # If the attestation is invalid, processing is aborted, and there is no post-state. - if not valid: - expect_assertion_error(lambda: spec.process_attestation(state, attestation)) - yield 'post', None - return - - current_epoch_count = len(state.current_epoch_attestations) - previous_epoch_count = len(state.previous_epoch_attestations) - - # process attestation - spec.process_attestation(state, attestation) - - # Make sure the attestation has been processed - if attestation.data.target.epoch == spec.get_current_epoch(state): - assert len(state.current_epoch_attestations) == current_epoch_count + 1 - else: - assert len(state.previous_epoch_attestations) == previous_epoch_count + 1 - - # yield post-state - yield 'post', state @with_all_phases_except(['phase0']) @spec_state_test @always_bls -def test_success_empty_custody_bits_blocks(spec, state): - attestation = get_valid_attestation(spec, state) - attestation.custody_bits_blocks = [] - sign_attestation(spec, state, attestation) +def test_on_time_success(spec, state): + next_slot(spec, state) + attestation = get_valid_on_time_attestation(spec, state, signed=True) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) yield from run_attestation_processing(spec, state, attestation) @@ -63,18 +28,28 @@ def test_success_empty_custody_bits_blocks(spec, state): @with_all_phases_except(['phase0']) @spec_state_test @always_bls -def test_fail_custody_bits_blocks_incorrect_slot(spec, state): - attestation = get_valid_attestation(spec, state) - committee = spec.get_beacon_committee( - state, - attestation.data.slot, - attestation.data.index, - ) - bitlist = Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE]([0 for _ in range(len(committee))]) - bitlist[0] = 1 - attestation.custody_bits_blocks = [bitlist] - sign_attestation(spec, state, attestation) +def test_on_time_empty_custody_bits_blocks(spec, state): + # Causing this test to pass causes many phase0 tests to fail + pass + """ + next_slot(spec, state) + attestation = get_valid_late_attestation(spec, state, signed=True) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 + assert not any(attestation.custody_bits_blocks) - yield from run_attestation_processing(spec, state, attestation) \ No newline at end of file + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + yield from run_attestation_processing(spec, state, attestation, False) + """ + + +@with_all_phases_except(['phase0']) +@spec_state_test +@always_bls +def test_late_with_custody_bits_blocks(spec, state): + next_slot(spec, state) + attestation = get_valid_on_time_attestation(spec, state, signed=True) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY + 1) + + yield from run_attestation_processing(spec, state, attestation, False) From 97fa3741af0ae919437532eba318b4326f33b9b9 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sat, 22 Feb 2020 09:21:39 -0600 Subject: [PATCH 022/102] working through test issues --- specs/phase1/beacon-chain.md | 3 +- .../test/fork_choice/test_get_head.py | 3 +- .../test/fork_choice/test_on_block.py | 3 +- .../eth2spec/test/helpers/attestations.py | 123 +++++++++++++++++- .../pyspec/eth2spec/test/helpers/state.py | 32 ----- .../test_process_attestation.py | 11 +- .../test_process_attestation.py | 8 +- .../pyspec/eth2spec/test/test_finality.py | 3 +- 8 files changed, 135 insertions(+), 51 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 897dae369..fe5c0d9a7 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -622,8 +622,7 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: # Type 2: no shard transition, no custody bits # TODO: could only allow for older attestations. else: # Ensure delayed attestation - # Currently commented out because breaks a ton of phase 0 tests - # assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY < state.slot + assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY < state.slot # Late attestations cannot have a shard transition root assert data.shard_transition_root == Root() diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py index e34c32c0e..17d4f644f 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py @@ -1,9 +1,8 @@ from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.attestations import get_valid_attestation +from eth2spec.test.helpers.attestations import get_valid_attestation, next_epoch_with_attestations from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.state import ( next_epoch, - next_epoch_with_attestations, state_transition_and_sign_block, ) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py index f50a00a9f..3d824811c 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py @@ -4,7 +4,8 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.test.context import with_all_phases, spec_state_test from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block, transition_unsigned_block, \ build_empty_block -from eth2spec.test.helpers.state import next_epoch, next_epoch_with_attestations, state_transition_and_sign_block +from eth2spec.test.helpers.attestations import next_epoch_with_attestations +from eth2spec.test.helpers.state import next_epoch, state_transition_and_sign_block def run_on_block(spec, store, signed_block, valid=True): diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 0a6e3294d..8f11e3306 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -1,7 +1,8 @@ from typing import List -from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.context import expect_assertion_error +from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_block +from eth2spec.test.helpers.block import build_empty_block_for_next_slot, transition_unsigned_block from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.ssz.ssz_typing import Bitlist @@ -74,7 +75,48 @@ def build_attestation_data(spec, state, slot, index): ) -def get_valid_attestation(spec, state, slot=None, index=None, signed=False): +def convert_to_valid_on_time_attestation(spec, state, attestation, signed=False): + shard = spec.get_shard(state, attestation) + + next_state = state.copy() + next_slot(spec, next_state) + offset_slots = spec.get_offset_slots(next_state, spec.get_next_slot_for_shard(next_state, shard)) + for offset_slot in offset_slots: + attestation.custody_bits_blocks.append( + Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE]([0 for _ in attestation.aggregation_bits]) + ) + + if signed: + sign_attestation(spec, state, attestation) + + return attestation + + +def get_valid_on_time_attestation(spec, state, slot=None, index=None, signed=False): + ''' + Construct on-time attestation for next slot + ''' + if slot is None: + slot = state.slot + if index is None: + index = 0 + + return get_valid_attestation(spec, state, slot, index, signed, True) + + +def get_valid_late_attestation(spec, state, slot=None, index=None, signed=False): + ''' + Construct on-time attestation for next slot + ''' + if slot is None: + slot = state.slot + if index is None: + index = 0 + + return get_valid_attestation(spec, state, slot, index, signed, False) + + +def get_valid_attestation(spec, state, slot=None, index=None, signed=False, on_time=True): if slot is None: slot = state.slot if index is None: @@ -97,6 +139,10 @@ def get_valid_attestation(spec, state, slot=None, index=None, signed=False): fill_aggregate_attestation(spec, state, attestation) if signed: sign_attestation(spec, state, attestation) + + if spec.fork == 'phase1' and on_time: + attestation = convert_to_valid_on_time_attestation(spec, state, attestation, signed) + return attestation @@ -112,7 +158,6 @@ def sign_aggregate_attestation(spec, state, attestation_data, participants: List privkey ) ) - # TODO: we should try signing custody bits if spec.fork == 'phase1' return bls.Aggregate(signatures) @@ -130,7 +175,47 @@ def sign_indexed_attestation(spec, state, indexed_attestation): indexed_attestation.attestation.signature = sign_aggregate_attestation(spec, state, data, participants) +def sign_on_time_attestation(spec, state, attestation): + if not any(attestation.custody_bits_blocks): + sign_attestation(spec, state, attestation) + return + + committee = spec.get_beacon_committee(state, attestation.data.slot, attestation.data.index) + signatures = [] + for block_index, custody_bits in enumerate(attestation.custody_bits_blocks): + for participant, abit, cbit in zip(committee, attestation.aggregation_bits, custody_bits): + if not abit: + continue + signatures.append(get_attestation_custody_signature( + spec, + state, + attestation.data, + block_index, + cbit, + privkeys[participant] + )) + + attestation.signature = bls.Aggregate(signatures) + + +def get_attestation_custody_signature(spec, state, attestation_data, block_index, bit, privkey): + domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) + signing_root = spec.compute_signing_root( + spec.AttestationCustodyBitWrapper( + attestation_data.hash_tree_root(), + block_index, + bit, + ), + domain, + ) + return bls.Sign(privkey, signing_root) + + def sign_attestation(spec, state, attestation): + if spec.fork == 'phase1' and any(attestation.custody_bits_blocks): + sign_on_time_attestation(spec, state,attestation) + return + participants = spec.get_attesting_indices( state, attestation.data, @@ -163,3 +248,35 @@ def add_attestations_to_state(spec, state, attestations, slot): spec.process_slots(state, slot) for attestation in attestations: spec.process_attestation(state, attestation) + + +def next_epoch_with_attestations(spec, + state, + fill_cur_epoch, + fill_prev_epoch): + assert state.slot % spec.SLOTS_PER_EPOCH == 0 + + post_state = state.copy() + signed_blocks = [] + for _ in range(spec.SLOTS_PER_EPOCH): + block = build_empty_block_for_next_slot(spec, post_state) + if fill_cur_epoch and post_state.slot >= spec.MIN_ATTESTATION_INCLUSION_DELAY: + slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 + committees_per_slot = spec.get_committee_count_at_slot(state, slot_to_attest) + if slot_to_attest >= spec.compute_start_slot_at_epoch(spec.get_current_epoch(post_state)): + for index in range(committees_per_slot): + cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest, index=index, signed=True) + block.body.attestations.append(cur_attestation) + + if fill_prev_epoch: + slot_to_attest = post_state.slot - spec.SLOTS_PER_EPOCH + 1 + committees_per_slot = spec.get_committee_count_at_slot(state, slot_to_attest) + for index in range(committees_per_slot): + prev_attestation = get_valid_attestation( + spec, post_state, slot_to_attest, index=index, signed=True, on_time=False) + block.body.attestations.append(prev_attestation) + + signed_block = state_transition_and_sign_block(spec, post_state, block) + signed_blocks.append(signed_block) + + return state, signed_blocks, post_state diff --git a/tests/core/pyspec/eth2spec/test/helpers/state.py b/tests/core/pyspec/eth2spec/test/helpers/state.py index 5816785f7..9b5d132f9 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/state.py +++ b/tests/core/pyspec/eth2spec/test/helpers/state.py @@ -1,5 +1,4 @@ from eth2spec.test.context import expect_assertion_error -from eth2spec.test.helpers.attestations import get_valid_attestation from eth2spec.test.helpers.block import sign_block, build_empty_block_for_next_slot, transition_unsigned_block @@ -58,34 +57,3 @@ def state_transition_and_sign_block(spec, state, block, expect_fail=False): transition_unsigned_block(spec, state, block) block.state_root = state.hash_tree_root() return sign_block(spec, state, block) - - -def next_epoch_with_attestations(spec, - state, - fill_cur_epoch, - fill_prev_epoch): - assert state.slot % spec.SLOTS_PER_EPOCH == 0 - - post_state = state.copy() - signed_blocks = [] - for _ in range(spec.SLOTS_PER_EPOCH): - block = build_empty_block_for_next_slot(spec, post_state) - if fill_cur_epoch and post_state.slot >= spec.MIN_ATTESTATION_INCLUSION_DELAY: - slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 - committees_per_slot = spec.get_committee_count_at_slot(state, slot_to_attest) - if slot_to_attest >= spec.compute_start_slot_at_epoch(spec.get_current_epoch(post_state)): - for index in range(committees_per_slot): - cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest, index=index, signed=True) - block.body.attestations.append(cur_attestation) - - if fill_prev_epoch: - slot_to_attest = post_state.slot - spec.SLOTS_PER_EPOCH + 1 - committees_per_slot = spec.get_committee_count_at_slot(state, slot_to_attest) - for index in range(committees_per_slot): - prev_attestation = get_valid_attestation(spec, post_state, slot_to_attest, index=index, signed=True) - block.body.attestations.append(prev_attestation) - - signed_block = state_transition_and_sign_block(spec, post_state, block) - signed_blocks.append(signed_block) - - return state, signed_blocks, post_state diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 236fceef1..525f01fcc 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -13,7 +13,10 @@ from eth2spec.test.helpers.attestations import ( sign_attestation, ) from eth2spec.test.helpers.state import ( - next_epoch, next_slots + next_epoch, + next_slots, + next_slot, + next_epoch, ) from eth2spec.test.helpers.block import apply_empty_block from eth2spec.utils.ssz.ssz_typing import Bitlist @@ -166,7 +169,7 @@ def test_old_target_epoch(spec, state): attestation = get_valid_attestation(spec, state, signed=True) - state.slot = spec.SLOTS_PER_EPOCH * 2 # target epoch will be too old to handle + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 2) # target epoch will be too old to handle yield from run_attestation_processing(spec, state, attestation, False) @@ -222,7 +225,8 @@ def test_source_root_is_target_root(spec, state): @with_all_phases @spec_state_test def test_invalid_current_source_root(spec, state): - state.slot = spec.SLOTS_PER_EPOCH * 5 + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 5) + state.finalized_checkpoint.epoch = 2 state.previous_justified_checkpoint = spec.Checkpoint(epoch=3, root=b'\x01' * 32) @@ -259,6 +263,7 @@ def test_bad_source_root(spec, state): @with_all_phases @spec_state_test def test_empty_aggregation_bits(spec, state): + next_slot(spec, state) attestation = get_valid_attestation(spec, state) next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_attestation.py index 430ad014b..288bb8e00 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_attestation.py @@ -6,9 +6,7 @@ from eth2spec.test.context import ( from eth2spec.test.helpers.state import next_slot, transition_to from eth2spec.test.helpers.attestations import ( run_attestation_processing, - # get_valid_attestation as get_valid_late_attestation, -) -from eth2spec.test.helpers.phase1.attestations import ( + get_valid_late_attestation, get_valid_on_time_attestation, ) @@ -29,9 +27,6 @@ def test_on_time_success(spec, state): @spec_state_test @always_bls def test_on_time_empty_custody_bits_blocks(spec, state): - # Causing this test to pass causes many phase0 tests to fail - pass - """ next_slot(spec, state) attestation = get_valid_late_attestation(spec, state, signed=True) @@ -40,7 +35,6 @@ def test_on_time_empty_custody_bits_blocks(spec, state): transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) yield from run_attestation_processing(spec, state, attestation, False) - """ @with_all_phases_except(['phase0']) diff --git a/tests/core/pyspec/eth2spec/test/test_finality.py b/tests/core/pyspec/eth2spec/test/test_finality.py index 8ae50d436..d793b6656 100644 --- a/tests/core/pyspec/eth2spec/test/test_finality.py +++ b/tests/core/pyspec/eth2spec/test/test_finality.py @@ -1,5 +1,6 @@ from eth2spec.test.context import spec_state_test, never_bls, with_all_phases -from eth2spec.test.helpers.state import next_epoch, next_epoch_with_attestations +from eth2spec.test.helpers.state import next_epoch +from eth2spec.test.helpers.attestations import next_epoch_with_attestations from eth2spec.test.helpers.block import apply_empty_block From 4c1fc9bffaafaa388739cd1369804abbe342acd3 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sat, 22 Feb 2020 12:06:31 -0600 Subject: [PATCH 023/102] work through phase 1 tests --- specs/phase0/fork-choice.md | 4 ++- specs/phase1/beacon-chain.md | 3 +++ tests/core/pyspec/eth2spec/test/context.py | 4 ++- .../test/fork_choice/test_on_attestation.py | 18 +++++++------ .../test/fork_choice/test_on_block.py | 1 + .../eth2spec/test/fork_choice/test_on_tick.py | 6 +++-- .../eth2spec/test/helpers/attestations.py | 4 +-- .../pyspec/eth2spec/test/helpers/state.py | 2 +- .../test_process_attestation.py | 24 ++++++++--------- .../test_process_attester_slashing.py | 4 +++ .../test_process_rewards_and_penalties.py | 26 +++++++++++++------ .../test_process_attestation.py | 5 +--- .../eth2spec/test/sanity/test_blocks.py | 2 +- .../pyspec/eth2spec/test/test_finality.py | 4 +-- 14 files changed, 65 insertions(+), 42 deletions(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index 79d37f28d..d8aaed14f 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -104,7 +104,7 @@ def get_forkchoice_store(anchor_state: BeaconState) -> Store: justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) return Store( - time=anchor_state.genesis_time, + time=anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot, genesis_time=anchor_state.genesis_time, justified_checkpoint=justified_checkpoint, finalized_checkpoint=finalized_checkpoint, @@ -317,11 +317,13 @@ def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIn ```python def on_tick(store: Store, time: uint64) -> None: previous_slot = get_current_slot(store) + print(previous_slot) # update store time store.time = time current_slot = get_current_slot(store) + print(current_slot) # Not a new epoch, return if not (current_slot > previous_slot and compute_slots_since_epoch_start(current_slot) == 0): return diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index fe5c0d9a7..fcd703a96 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -544,6 +544,9 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe AttestationCustodyBitWrapper(hash_tree_root(attestation.data), i, cbit), domain)) else: assert not cbit + print(all_pubkeys) + print(all_signing_roots) + print(attestation.signature) return bls.AggregateVerify(zip(all_pubkeys, all_signing_roots), signature=attestation.signature) ``` diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 5338ccb9d..c4ce15ed8 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -54,7 +54,9 @@ def with_custom_state(balances_fn: Callable[[Any], Sequence[int]], # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper. # Decide based on performance/consistency results later. state = phases["phase1"].upgrade_to_phase1(state) - + # Shard state slot must lag behind BeaconState slot by at least 1 + # Will handle this more elegantly with fork mechanics + spec.process_slots(state, state.slot + 1) kw['state'] = state except KeyError: raise TypeError('Spec decorator must come within state decorator to inject spec into state.') diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 87a39a620..e77c08892 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -120,11 +120,12 @@ def test_on_attestation_mismatched_target_and_slot(spec, state): @spec_state_test def test_on_attestation_target_not_in_store(spec, state): store = spec.get_forkchoice_store(state) - time = spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH spec.on_tick(store, time) # move to immediately before next epoch to make block new target - transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH - 1) + next_epoch = spec.get_current_epoch(state) + 1 + transition_to(spec, state, spec.compute_start_slot_at_epoch(next_epoch) - 1) target_block = build_empty_block_for_next_slot(spec, state) state_transition_and_sign_block(spec, state, target_block) @@ -141,11 +142,12 @@ def test_on_attestation_target_not_in_store(spec, state): @spec_state_test def test_on_attestation_beacon_block_not_in_store(spec, state): store = spec.get_forkchoice_store(state) - time = spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH spec.on_tick(store, time) # move to immediately before next epoch to make block new target - transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH - 1) + next_epoch = spec.get_current_epoch(state) + 1 + transition_to(spec, state, spec.compute_start_slot_at_epoch(next_epoch) - 1) target_block = build_empty_block_for_next_slot(spec, state) signed_target_block = state_transition_and_sign_block(spec, state, target_block) @@ -169,7 +171,7 @@ def test_on_attestation_beacon_block_not_in_store(spec, state): @spec_state_test def test_on_attestation_future_epoch(spec, state): store = spec.get_forkchoice_store(state) - time = 3 * spec.SECONDS_PER_SLOT + time = store.time + 3 * spec.SECONDS_PER_SLOT spec.on_tick(store, time) block = build_empty_block_for_next_slot(spec, state) @@ -189,7 +191,7 @@ def test_on_attestation_future_epoch(spec, state): @spec_state_test def test_on_attestation_future_block(spec, state): store = spec.get_forkchoice_store(state) - time = spec.SECONDS_PER_SLOT * 5 + time = store.time + spec.SECONDS_PER_SLOT * 5 spec.on_tick(store, time) block = build_empty_block_for_next_slot(spec, state) @@ -209,7 +211,7 @@ def test_on_attestation_future_block(spec, state): @spec_state_test def test_on_attestation_same_slot(spec, state): store = spec.get_forkchoice_store(state) - time = 1 * spec.SECONDS_PER_SLOT + time = store.time + spec.SECONDS_PER_SLOT spec.on_tick(store, time) block = build_empty_block_for_next_slot(spec, state) @@ -225,7 +227,7 @@ def test_on_attestation_same_slot(spec, state): @spec_state_test def test_on_attestation_invalid_attestation(spec, state): store = spec.get_forkchoice_store(state) - time = 3 * spec.SECONDS_PER_SLOT + time = store.time + 3 * spec.SECONDS_PER_SLOT spec.on_tick(store, time) block = build_empty_block_for_next_slot(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py index 3d824811c..4438dff92 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py @@ -160,6 +160,7 @@ def test_on_block_finalized_skip_slots(spec, state): @spec_state_test def test_on_block_finalized_skip_slots_not_in_skip_chain(spec, state): # Initialization + next_epoch(spec, state) store = spec.get_forkchoice_store(state) store.finalized_checkpoint = spec.Checkpoint( diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_tick.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_tick.py index 27b64ac09..93f3bd9bb 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_tick.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_tick.py @@ -27,14 +27,16 @@ def test_basic(spec, state): @spec_state_test def test_update_justified_single(spec, state): store = spec.get_forkchoice_store(state) - seconds_per_epoch = spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + next_epoch = spec.get_current_epoch(state) + 1 + next_epoch_start_slot = spec.compute_start_slot_at_epoch(next_epoch) + seconds_until_next_epoch = next_epoch_start_slot * spec.SECONDS_PER_SLOT - store.time store.best_justified_checkpoint = spec.Checkpoint( epoch=store.justified_checkpoint.epoch + 1, root=b'\x55' * 32, ) - run_on_tick(spec, store, store.time + seconds_per_epoch, True) + run_on_tick(spec, store, store.time + seconds_until_next_epoch, True) @with_all_phases diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 8f11e3306..deebd0356 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -2,7 +2,7 @@ from typing import List from eth2spec.test.context import expect_assertion_error from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_block -from eth2spec.test.helpers.block import build_empty_block_for_next_slot, transition_unsigned_block +from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.ssz.ssz_typing import Bitlist @@ -213,7 +213,7 @@ def get_attestation_custody_signature(spec, state, attestation_data, block_index def sign_attestation(spec, state, attestation): if spec.fork == 'phase1' and any(attestation.custody_bits_blocks): - sign_on_time_attestation(spec, state,attestation) + sign_on_time_attestation(spec, state, attestation) return participants = spec.get_attesting_indices( diff --git a/tests/core/pyspec/eth2spec/test/helpers/state.py b/tests/core/pyspec/eth2spec/test/helpers/state.py index 9b5d132f9..46a7ce2b5 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/state.py +++ b/tests/core/pyspec/eth2spec/test/helpers/state.py @@ -1,5 +1,5 @@ from eth2spec.test.context import expect_assertion_error -from eth2spec.test.helpers.block import sign_block, build_empty_block_for_next_slot, transition_unsigned_block +from eth2spec.test.helpers.block import sign_block, transition_unsigned_block def get_balance(state, index): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 525f01fcc..9528ff6eb 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -13,10 +13,10 @@ from eth2spec.test.helpers.attestations import ( sign_attestation, ) from eth2spec.test.helpers.state import ( - next_epoch, - next_slots, next_slot, + next_slots, next_epoch, + transition_to, ) from eth2spec.test.helpers.block import apply_empty_block from eth2spec.utils.ssz.ssz_typing import Bitlist @@ -46,8 +46,8 @@ def test_success_multi_proposer_index_iterations(spec, state): @with_all_phases @spec_state_test def test_success_previous_epoch(spec, state): - attestation = get_valid_attestation(spec, state, signed=True) - state.slot = spec.SLOTS_PER_EPOCH - 1 + attestation = get_valid_attestation(spec, state, signed=True, on_time=False) + transition_to(spec, state, spec.SLOTS_PER_EPOCH - 1) next_epoch(spec, state) apply_empty_block(spec, state) @@ -76,10 +76,10 @@ def test_before_inclusion_delay(spec, state): @with_all_phases @spec_state_test def test_after_epoch_slots(spec, state): - attestation = get_valid_attestation(spec, state, signed=True) - state.slot = spec.SLOTS_PER_EPOCH - 1 + attestation = get_valid_attestation(spec, state, signed=True, on_time=False) + # increment past latest inclusion slot - spec.process_slots(state, state.slot + 2) + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH + 1) apply_empty_block(spec, state) yield from run_attestation_processing(spec, state, attestation, False) @@ -154,7 +154,7 @@ def test_mismatched_target_and_slot(spec, state): next_epoch(spec, state) next_epoch(spec, state) - attestation = get_valid_attestation(spec, state) + attestation = get_valid_attestation(spec, state, on_time=False) attestation.data.slot = attestation.data.slot - spec.SLOTS_PER_EPOCH sign_attestation(spec, state, attestation) @@ -167,9 +167,9 @@ def test_mismatched_target_and_slot(spec, state): def test_old_target_epoch(spec, state): assert spec.MIN_ATTESTATION_INCLUSION_DELAY < spec.SLOTS_PER_EPOCH * 2 - attestation = get_valid_attestation(spec, state, signed=True) + attestation = get_valid_attestation(spec, state, signed=True, on_time=False) - transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 2) # target epoch will be too old to handle + next_slots(spec, state, spec.SLOTS_PER_EPOCH * 2) # target epoch will be too old to handle yield from run_attestation_processing(spec, state, attestation, False) @@ -225,14 +225,14 @@ def test_source_root_is_target_root(spec, state): @with_all_phases @spec_state_test def test_invalid_current_source_root(spec, state): - transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 5) + next_slots(spec, state, spec.SLOTS_PER_EPOCH * 5) state.finalized_checkpoint.epoch = 2 state.previous_justified_checkpoint = spec.Checkpoint(epoch=3, root=b'\x01' * 32) state.current_justified_checkpoint = spec.Checkpoint(epoch=4, root=b'\x32' * 32) - attestation = get_valid_attestation(spec, state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1) + attestation = get_valid_attestation(spec, state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1, on_time=False) next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) # Test logic sanity checks: diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index e9665e714..c379bb815 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -143,6 +143,10 @@ def test_invalid_sig_1(spec, state): @always_bls def test_invalid_sig_2(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=False) + """ + SOMETHING WRONG WITH HASH CACHE OF SEPARATE ATTESTATIONS MAKING SIGS + LOOK CORRECT + """ yield from run_attester_slashing_processing(spec, state, attester_slashing, False) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py index 7cdeb16d9..044390b4b 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py @@ -1,7 +1,10 @@ from copy import deepcopy -from eth2spec.test.context import spec_state_test, with_all_phases, spec_test, \ +from eth2spec.test.context import ( + spec_state_test, spec_test, + with_all_phases, with_phases, misc_balances, with_custom_state, default_activation_threshold, single_phase +) from eth2spec.test.helpers.state import ( next_epoch, next_slot, @@ -19,27 +22,30 @@ def run_process_rewards_and_penalties(spec, state): def prepare_state_with_full_attestations(spec, state): + start_slot = state.slot + start_epoch = spec.get_current_epoch(state) + next_start_epoch = spec.compute_start_slot_at_epoch(start_epoch + 1) attestations = [] for slot in range(spec.SLOTS_PER_EPOCH + spec.MIN_ATTESTATION_INCLUSION_DELAY): # create an attestation for each index in each slot in epoch - if slot < spec.SLOTS_PER_EPOCH: - for committee_index in range(spec.get_committee_count_at_slot(state, slot)): + if state.slot < next_start_epoch: + for committee_index in range(spec.get_committee_count_at_slot(state, state.slot)): attestation = get_valid_attestation(spec, state, index=committee_index, signed=True) attestations.append(attestation) # fill each created slot in state after inclusion delay - if slot - spec.MIN_ATTESTATION_INCLUSION_DELAY >= 0: - inclusion_slot = slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + if state.slot >= start_slot + spec.MIN_ATTESTATION_INCLUSION_DELAY: + inclusion_slot = state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY include_attestations = [att for att in attestations if att.data.slot == inclusion_slot] add_attestations_to_state(spec, state, include_attestations, state.slot) next_slot(spec, state) - assert spec.compute_epoch_at_slot(state.slot) == spec.GENESIS_EPOCH + 1 + assert spec.compute_epoch_at_slot(state.slot) == start_epoch + 1 assert len(state.previous_epoch_attestations) == len(attestations) return attestations -@with_all_phases +@with_phases(['phase0']) @spec_state_test def test_genesis_epoch_no_attestations_no_penalties(spec, state): pre_state = deepcopy(state) @@ -52,7 +58,7 @@ def test_genesis_epoch_no_attestations_no_penalties(spec, state): assert state.balances[index] == pre_state.balances[index] -@with_all_phases +@with_phases(['phase0']) @spec_state_test def test_genesis_epoch_full_attestations_no_rewards(spec, state): attestations = [] @@ -81,6 +87,8 @@ def test_genesis_epoch_full_attestations_no_rewards(spec, state): @with_all_phases @spec_state_test def test_full_attestations(spec, state): + # Go to start of next epoch to ensure can have full participation + next_epoch(spec, state) attestations = prepare_state_with_full_attestations(spec, state) pre_state = deepcopy(state) @@ -101,6 +109,8 @@ def test_full_attestations(spec, state): @with_custom_state(balances_fn=misc_balances, threshold_fn=default_activation_threshold) @single_phase def test_full_attestations_misc_balances(spec, state): + # Go to start of next epoch to ensure can have full participation + next_epoch(spec, state) attestations = prepare_state_with_full_attestations(spec, state) pre_state = deepcopy(state) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_attestation.py index 288bb8e00..ed4328327 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_attestation.py @@ -3,7 +3,7 @@ from eth2spec.test.context import ( spec_state_test, always_bls, ) -from eth2spec.test.helpers.state import next_slot, transition_to +from eth2spec.test.helpers.state import transition_to from eth2spec.test.helpers.attestations import ( run_attestation_processing, get_valid_late_attestation, @@ -15,7 +15,6 @@ from eth2spec.test.helpers.attestations import ( @spec_state_test @always_bls def test_on_time_success(spec, state): - next_slot(spec, state) attestation = get_valid_on_time_attestation(spec, state, signed=True) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -27,7 +26,6 @@ def test_on_time_success(spec, state): @spec_state_test @always_bls def test_on_time_empty_custody_bits_blocks(spec, state): - next_slot(spec, state) attestation = get_valid_late_attestation(spec, state, signed=True) assert not any(attestation.custody_bits_blocks) @@ -41,7 +39,6 @@ def test_on_time_empty_custody_bits_blocks(spec, state): @spec_state_test @always_bls def test_late_with_custody_bits_blocks(spec, state): - next_slot(spec, state) attestation = get_valid_on_time_attestation(spec, state, signed=True) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY + 1) diff --git a/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py index e533b3b0a..156dc1581 100644 --- a/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py @@ -381,7 +381,7 @@ def test_attestation(spec, state): # Add to state via block transition pre_current_attestations_len = len(state.current_epoch_attestations) - attestation_block = build_empty_block(spec, state, state.slot + 1 + spec.MIN_ATTESTATION_INCLUSION_DELAY) + attestation_block = build_empty_block(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) attestation_block.body.attestations.append(attestation) signed_attestation_block = state_transition_and_sign_block(spec, state, attestation_block) diff --git a/tests/core/pyspec/eth2spec/test/test_finality.py b/tests/core/pyspec/eth2spec/test/test_finality.py index d793b6656..0b99cc3f9 100644 --- a/tests/core/pyspec/eth2spec/test/test_finality.py +++ b/tests/core/pyspec/eth2spec/test/test_finality.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import spec_state_test, never_bls, with_all_phases +from eth2spec.test.context import spec_state_test, never_bls, with_all_phases, with_phases from eth2spec.test.helpers.state import next_epoch from eth2spec.test.helpers.attestations import next_epoch_with_attestations from eth2spec.test.helpers.block import apply_empty_block @@ -29,7 +29,7 @@ def check_finality(spec, assert state.finalized_checkpoint == prev_state.finalized_checkpoint -@with_all_phases +@with_phases(["phase0"]) @spec_state_test @never_bls def test_finality_no_updates_at_genesis(spec, state): From 9718d206a798a503983fcff43269024e6b474181 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 26 Feb 2020 11:20:19 -0600 Subject: [PATCH 024/102] fix attester slahsing test --- specs/phase1/beacon-chain.md | 9 +++++++-- tests/core/pyspec/eth2spec/test/helpers/attestations.py | 6 +++--- .../block_processing/test_process_attester_slashing.py | 4 ---- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index fe5c0d9a7..c30810864 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -540,8 +540,12 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe if abit: all_pubkeys.append(state.validators[participant].pubkey) # Note: only 2N distinct message hashes - all_signing_roots.append(compute_signing_root( - AttestationCustodyBitWrapper(hash_tree_root(attestation.data), i, cbit), domain)) + attestation_wrapper = AttestationCustodyBitWrapper( + attestation_data_root=hash_tree_root(attestation.data), + block_index=i, + bit=cbit + ) + all_signing_roots.append(compute_signing_root(attestation_wrapper, domain)) else: assert not cbit return bls.AggregateVerify(zip(all_pubkeys, all_signing_roots), signature=attestation.signature) @@ -801,6 +805,7 @@ def get_indices_from_committee( def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSlashing) -> None: indexed_attestation_1 = attester_slashing.attestation_1 indexed_attestation_2 = attester_slashing.attestation_2 + assert is_slashable_attestation_data( indexed_attestation_1.attestation.data, indexed_attestation_2.attestation.data, diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index deebd0356..fea4ea1f6 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -202,9 +202,9 @@ def get_attestation_custody_signature(spec, state, attestation_data, block_index domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) signing_root = spec.compute_signing_root( spec.AttestationCustodyBitWrapper( - attestation_data.hash_tree_root(), - block_index, - bit, + attestation_data_root=attestation_data.hash_tree_root(), + block_index=block_index, + bit=bit, ), domain, ) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index c379bb815..e9665e714 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -143,10 +143,6 @@ def test_invalid_sig_1(spec, state): @always_bls def test_invalid_sig_2(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=False) - """ - SOMETHING WRONG WITH HASH CACHE OF SEPARATE ATTESTATIONS MAKING SIGS - LOOK CORRECT - """ yield from run_attester_slashing_processing(spec, state, attester_slashing, False) From 63687e6f34565d2b5f045cdbda711c937932a3bd Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Feb 2020 12:27:00 -0600 Subject: [PATCH 025/102] fix YAML parsed list of ints --- configs/mainnet.yaml | 2 +- configs/minimal.yaml | 2 +- tests/core/pyspec/eth2spec/config/config_util.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index c7940bceb..74f062d9b 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -179,7 +179,7 @@ MAX_SHARD_BLOCK_CHUNKS: 4 # 3 * 2**16` (= 196,608) TARGET_SHARD_BLOCK_SIZE: 196608 # Note: MAX_SHARD_BLOCKS_PER_ATTESTATION is derived from the list length. -# SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] +SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] # len(SHARD_BLOCK_OFFSETS) MAX_SHARD_BLOCKS_PER_ATTESTATION: 12 # 2**14 (= 16,384) Gwei diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 48fab475c..42c63e301 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -180,7 +180,7 @@ MAX_SHARD_BLOCK_CHUNKS: 4 # 3 * 2**16` (= 196,608) TARGET_SHARD_BLOCK_SIZE: 196608 # Note: MAX_SHARD_BLOCKS_PER_ATTESTATION is derived from the list length. -# SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] +SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] # len(SHARD_BLOCK_OFFSETS) MAX_SHARD_BLOCKS_PER_ATTESTATION: 12 # 2**14 (= 16,384) Gwei diff --git a/tests/core/pyspec/eth2spec/config/config_util.py b/tests/core/pyspec/eth2spec/config/config_util.py index 42ad76d69..64c533f2d 100644 --- a/tests/core/pyspec/eth2spec/config/config_util.py +++ b/tests/core/pyspec/eth2spec/config/config_util.py @@ -36,7 +36,8 @@ def load_config_file(configs_dir, presets_name) -> Dict[str, Any]: out = dict() for k, v in loaded.items(): if isinstance(v, list): - out[k] = v + # Clean up integer values. YAML parser renders lists of ints as list of str + out[k] = [int(item) if item.isdigit() else item for item in v] elif isinstance(v, str) and v.startswith("0x"): out[k] = bytes.fromhex(v[2:]) else: From 186d4258b697105608d2acdf24480826f7635417 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Feb 2020 13:20:37 -0600 Subject: [PATCH 026/102] fix shard offsets --- specs/phase1/beacon-chain.md | 26 +++++++++---------- .../eth2spec/test/helpers/attestations.py | 2 +- .../test/helpers/phase1/attestations.py | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index c30810864..66d94bfe9 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -46,7 +46,7 @@ - [`get_updated_gasprice`](#get_updated_gasprice) - [`get_start_shard`](#get_start_shard) - [`get_shard`](#get_shard) - - [`get_next_slot_for_shard`](#get_next_slot_for_shard) + - [`get_latest_slot_for_shard`](#get_latest_slot_for_shard) - [`get_offset_slots`](#get_offset_slots) - [Predicates](#predicates) - [Updated `is_valid_indexed_attestation`](#updated-is_valid_indexed_attestation) @@ -98,7 +98,7 @@ Configuration is not namespaced. Instead it is strictly an extension; | `SHARD_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | | `MAX_SHARD_BLOCK_SIZE` | `2**20` (= 1,048,576) | | | `TARGET_SHARD_BLOCK_SIZE` | `2**18` (= 262,144) | | -| `SHARD_BLOCK_OFFSETS` | `[0, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | | +| `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | | | `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | | | `MAX_GASPRICE` | `Gwei(2**14)` (= 16,384) | Gwei | | | `MIN_GASPRICE` | `Gwei(2**5)` (= 32) | Gwei | | @@ -493,18 +493,18 @@ def get_shard(state: BeaconState, attestation: Attestation) -> Shard: return compute_shard_from_committee_index(state, attestation.data.index, attestation.data.slot) ``` -#### `get_next_slot_for_shard` +#### `get_latest_slot_for_shard` ```python -def get_next_slot_for_shard(state: BeaconState, shard: Shard) -> Slot: - return Slot(state.shard_states[shard].slot + 1) +def get_latest_slot_for_shard(state: BeaconState, shard: Shard) -> Slot: + return state.shard_states[shard].slot ``` #### `get_offset_slots` ```python -def get_offset_slots(state: BeaconState, start_slot: Slot) -> Sequence[Slot]: - return [Slot(start_slot + x) for x in SHARD_BLOCK_OFFSETS if start_slot + x < state.slot] +def get_offset_slots(state: BeaconState, latest_shard_slot: Slot) -> Sequence[Slot]: + return [Slot(latest_shard_slot + x) for x in SHARD_BLOCK_OFFSETS if latest_shard_slot + x < state.slot] ``` ### Predicates @@ -613,14 +613,14 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: assert attestation.data.source == state.previous_justified_checkpoint shard = get_shard(state, attestation) - shard_start_slot = get_next_slot_for_shard(state, shard) + latest_shard_slot = get_latest_slot_for_shard(state, shard) # Type 1: on-time attestations if attestation.custody_bits_blocks != []: # Ensure on-time attestation assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY == state.slot # Correct data root count - assert len(attestation.custody_bits_blocks) == len(get_offset_slots(state, shard_start_slot)) + assert len(attestation.custody_bits_blocks) == len(get_offset_slots(state, latest_shard_slot)) # Correct parent block root assert data.beacon_block_root == get_block_root_at_slot(state, get_previous_slot(state.slot)) # Type 2: no shard transition, no custody bits # TODO: could only allow for older attestations. @@ -639,17 +639,17 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: ```python def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTransition) -> None: # Slot the attestation starts counting from - start_slot = get_next_slot_for_shard(state, shard) + latest_slot = get_latest_slot_for_shard(state, shard) # Correct data root count - offset_slots = get_offset_slots(state, start_slot) + offset_slots = get_offset_slots(state, latest_slot) assert ( len(transition.shard_data_roots) == len(transition.shard_states) == len(transition.shard_block_lengths) == len(offset_slots) ) - assert transition.start_slot == start_slot + assert transition.start_slot == offset_slots[0] # Reconstruct shard headers headers = [] @@ -728,7 +728,7 @@ def process_crosslink_for_shard(state: BeaconState, increase_balance(state, beacon_proposer_index, proposer_reward) states_slots_lengths = zip( shard_transition.shard_states, - get_offset_slots(state, get_next_slot_for_shard(state, shard)), + get_offset_slots(state, get_latest_slot_for_shard(state, shard)), shard_transition.shard_block_lengths ) for shard_state, slot, length in states_slots_lengths: diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index fea4ea1f6..1e7c6300d 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -80,7 +80,7 @@ def convert_to_valid_on_time_attestation(spec, state, attestation, signed=False) next_state = state.copy() next_slot(spec, next_state) - offset_slots = spec.get_offset_slots(next_state, spec.get_next_slot_for_shard(next_state, shard)) + offset_slots = spec.get_offset_slots(next_state, spec.get_latest_slot_for_shard(next_state, shard)) for offset_slot in offset_slots: attestation.custody_bits_blocks.append( Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE]([0 for _ in attestation.aggregation_bits]) diff --git a/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py index 204458342..b97cc5d46 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py @@ -18,7 +18,7 @@ def get_valid_on_time_attestation(spec, state, index=None, signed=False): next_state = state.copy() next_slot(spec, next_state) - offset_slots = spec.get_offset_slots(next_state, spec.get_next_slot_for_shard(next_state, shard)) + offset_slots = spec.get_offset_slots(next_state, spec.get_latest_slot_for_shard(next_state, shard)) for offset_slot in offset_slots: attestation.custody_bits_blocks.append( Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE]([0 for _ in attestation.aggregation_bits]) From 92eef0e00b959c42d4e2602fc38f5fb34b681a19 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 9 Mar 2020 14:52:30 -0600 Subject: [PATCH 027/102] fix light client sig verification in phase 1 --- specs/phase1/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 66d94bfe9..f1e025ed5 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -859,7 +859,7 @@ def process_light_client_signatures(state: BeaconState, block_body: BeaconBlockB slot = get_previous_slot(state.slot) signing_root = compute_signing_root(get_block_root_at_slot(state, slot), get_domain(state, DOMAIN_LIGHT_CLIENT, compute_epoch_at_slot(slot))) - return bls.FastAggregateVerify(signer_pubkeys, signing_root, signature=block_body.light_client_signature) + assert bls.FastAggregateVerify(signer_pubkeys, signing_root, signature=block_body.light_client_signature) ``` From 9b7e0ab2bede10892e187e6deff2b7cdc6b58b60 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Fri, 13 Mar 2020 17:15:25 +0000 Subject: [PATCH 028/102] Fix error in custody bit computation --- specs/phase1/custody-game.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index af3aadc96..822970a19 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -192,10 +192,10 @@ def get_custody_atoms(bytez: bytes) -> Sequence[bytes]: def compute_custody_bit(key: BLSSignature, data: bytes) -> bit: full_G2_element = bls.signature_to_G2(key) s = full_G2_element[0].coeffs - bits = [legendre_bit(sum(s[i % 2]**i * int.from_bytes(atom, "little")), BLS12_381_Q) - for i, atom in enumerate(get_custody_atoms(data))] - # XOR all atom bits - return bit(sum(bits) % 2) + custody_atoms = get_custody_atoms(data) + n = len(custody_atoms) + return legendre_bit(sum(s[i % 2]**i * int.from_bytes(atom, "little")) + for i, atom in enumerate(custody_atoms) + s[n % 2]**n, BLS12_381_Q) ``` ### `get_randao_epoch_for_custody_period` From d299b06a1c9be7af43d12dc93f49ac79de72b942 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 16 Mar 2020 09:52:27 -0600 Subject: [PATCH 029/102] fix custody bit calculation format --- specs/phase1/custody-game.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 822970a19..0d69c995a 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -194,8 +194,8 @@ def compute_custody_bit(key: BLSSignature, data: bytes) -> bit: s = full_G2_element[0].coeffs custody_atoms = get_custody_atoms(data) n = len(custody_atoms) - return legendre_bit(sum(s[i % 2]**i * int.from_bytes(atom, "little")) - for i, atom in enumerate(custody_atoms) + s[n % 2]**n, BLS12_381_Q) + a = sum(s[i % 2]**i * int.from_bytes(atom, "little") for i, atom in enumerate(custody_atoms) + s[n % 2]**n) + return legendre_bit(a, BLS12_381_Q) ``` ### `get_randao_epoch_for_custody_period` From 83851f69195707a923226d24f6c402e3ad75c322 Mon Sep 17 00:00:00 2001 From: Nathaniel Jensen Date: Tue, 24 Mar 2020 13:39:27 +1100 Subject: [PATCH 030/102] Fix inaccuracies in test format documentation. * `block_header` operation to accept input of type `BeaconBlock` * `voluntary_exit` operation to accept input of type `SignedVoluntaryExit`. * sanity/blocks to note that inputs are of type `SignedBeaconBlock` --- tests/formats/operations/README.md | 20 ++++++++++---------- tests/formats/sanity/blocks.md | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/formats/operations/README.md b/tests/formats/operations/README.md index f1ec0429a..bb4636ec0 100644 --- a/tests/formats/operations/README.md +++ b/tests/formats/operations/README.md @@ -18,11 +18,11 @@ A YAML-encoded `BeaconState`, the state before applying the operation. Also available as `pre.ssz`. -### `.yaml` +### `.yaml` A YAML-encoded operation object, e.g. a `ProposerSlashing`, or `Deposit`. -Also available as `.ssz`. +Also available as `.ssz`. ### `post.yaml` @@ -39,14 +39,14 @@ This excludes the other parts of the block-transition. Operations: -| *`operation-name`* | *`operation-object`* | *`input name`* | *`processing call`* | -|-------------------------|----------------------|----------------------|--------------------------------------------------------| -| `attestation` | `Attestation` | `attestation` | `process_attestation(state, attestation)` | -| `attester_slashing` | `AttesterSlashing` | `attester_slashing` | `process_attester_slashing(state, attester_slashing)` | -| `block_header` | `Block` | **`block`** | `process_block_header(state, block)` | -| `deposit` | `Deposit` | `deposit` | `process_deposit(state, deposit)` | -| `proposer_slashing` | `ProposerSlashing` | `proposer_slashing` | `process_proposer_slashing(state, proposer_slashing)` | -| `voluntary_exit` | `VoluntaryExit` | `voluntary_exit` | `process_voluntary_exit(state, voluntary_exit)` | +| *`operation-name`* | *`operation-object`* | *`input name`* | *`processing call`* | +|-------------------------|-----------------------|----------------------|--------------------------------------------------------| +| `attestation` | `Attestation` | `attestation` | `process_attestation(state, attestation)` | +| `attester_slashing` | `AttesterSlashing` | `attester_slashing` | `process_attester_slashing(state, attester_slashing)` | +| `block_header` | `BeaconBlock` | **`block`** | `process_block_header(state, block)` | +| `deposit` | `Deposit` | `deposit` | `process_deposit(state, deposit)` | +| `proposer_slashing` | `ProposerSlashing` | `proposer_slashing` | `process_proposer_slashing(state, proposer_slashing)` | +| `voluntary_exit` | `SignedVoluntaryExit` | `voluntary_exit` | `process_voluntary_exit(state, voluntary_exit)` | Note that `block_header` is not strictly an operation (and is a full `Block`), but processed in the same manner, and hence included here. diff --git a/tests/formats/sanity/blocks.md b/tests/formats/sanity/blocks.md index 2b50d19ca..991bc35d2 100644 --- a/tests/formats/sanity/blocks.md +++ b/tests/formats/sanity/blocks.md @@ -25,7 +25,7 @@ Also available as `pre.ssz`. A series of files, with `` in range `[0, blocks_count)`. Blocks need to be processed in order, following the main transition function (i.e. process slot and epoch transitions in between blocks as normal) -Each file is a YAML-encoded `BeaconBlock`. +Each file is a YAML-encoded `SignedBeaconBlock`. Each block is also available as `blocks_.ssz` From 68cc4c48478fc52b69365202cef27bf56e329a34 Mon Sep 17 00:00:00 2001 From: Martin Lundfall Date: Fri, 27 Mar 2020 14:02:56 +0100 Subject: [PATCH 031/102] Fix test doc links --- tests/generators/epoch_processing/README.md | 2 +- tests/generators/operations/README.md | 2 +- tests/generators/sanity/README.md | 2 +- tests/generators/ssz_static/README.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/generators/epoch_processing/README.md b/tests/generators/epoch_processing/README.md index 9b57875e2..662b0b516 100644 --- a/tests/generators/epoch_processing/README.md +++ b/tests/generators/epoch_processing/README.md @@ -5,7 +5,7 @@ Epoch processing covers the sub-transitions during an epoch change. An epoch-processing test-runner can consume these sub-transition test-suites, and handle different kinds of epoch sub-transitions by processing the cases using the specified test handler. -Information on the format of the tests can be found in the [epoch-processing test formats documentation](../../specs/test_formats/epoch_processing/README.md). +Information on the format of the tests can be found in the [epoch-processing test formats documentation](../../formats/epoch_processing/README.md). diff --git a/tests/generators/operations/README.md b/tests/generators/operations/README.md index 5cb3afc98..a5d48c11b 100644 --- a/tests/generators/operations/README.md +++ b/tests/generators/operations/README.md @@ -6,7 +6,7 @@ Operations (or "transactions" in previous spec iterations), An operation test-runner can consume these operation test-suites, and handle different kinds of operations by processing the cases using the specified test handler. -Information on the format of the tests can be found in the [operations test formats documentation](../../specs/test_formats/operations/README.md). +Information on the format of the tests can be found in the [operations test formats documentation](../../formats/operations/README.md). diff --git a/tests/generators/sanity/README.md b/tests/generators/sanity/README.md index 6d2e2f30d..cbc6aef06 100644 --- a/tests/generators/sanity/README.md +++ b/tests/generators/sanity/README.md @@ -2,7 +2,7 @@ Sanity tests cover regular state-transitions in a common block-list format, to ensure the basics work. -Information on the format of the tests can be found in the [sanity test formats documentation](../../specs/test_formats/sanity/README.md). +Information on the format of the tests can be found in the [sanity test formats documentation](../../formats/sanity/README.md). diff --git a/tests/generators/ssz_static/README.md b/tests/generators/ssz_static/README.md index 2a5040192..160d1ebb4 100644 --- a/tests/generators/ssz_static/README.md +++ b/tests/generators/ssz_static/README.md @@ -3,4 +3,4 @@ The purpose of this test-generator is to provide test-vectors for the most important applications of SSZ: the serialization and hashing of Eth2 data types. -Test-format documentation can be found [here](../../specs/test_formats/ssz_static/README.md). +Test-format documentation can be found [here](../../formats/ssz_static/README.md). From 2daa26442b0bd5e4eafa537a0e1314a91d00a109 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 30 Mar 2020 10:44:46 +1100 Subject: [PATCH 032/102] Tighten restriction on a "seen" attestation Declares that only a verified block can stop an attestation from being propagated. This achieves two things: 1. Ensures that clients don't need to scan invalid blocks for attestations and then modify their state based upon them. 1. Disallows "muting" attestations by sending around a junk block with that attestation in it. --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 841432efd..24a1d4376 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -286,7 +286,7 @@ There are two primary global topics used to propagate beacon blocks and aggregat - The block is proposed by the expected `proposer_index` for the block's slot in the context of the current shuffling (defined by `parent_root`/`slot`). If the `proposer_index` cannot immediately be verified against the expected shuffling, the block MAY be queued for later processing while proposers for the block's branch are calculated. - `beacon_aggregate_and_proof` - This topic is used to propagate aggregated attestations (as `SignedAggregateAndProof`s) to subscribing nodes (typically validators) to be included in future blocks. The following validations MUST pass before forwarding the `signed_aggregate_and_proof` on the network. (We define the following for convenience -- `aggregate_and_proof = signed_aggregate_and_proof.message` and `aggregate = aggregate_and_proof.aggregate`) - `aggregate.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= aggregate.data.slot` (a client MAY queue future aggregates for processing at the appropriate slot). - - The aggregate attestation defined by `hash_tree_root(aggregate)` has _not_ already been seen (via aggregate gossip, within a block, or through the creation of an equivalent aggregate locally). + - The aggregate attestation defined by `hash_tree_root(aggregate)` has _not_ already been seen (via aggregate gossip, within a verified block, or through the creation of an equivalent aggregate locally). - The `aggregate` is the first valid aggregate received for the aggregator with index `aggregate_and_proof.aggregator_index` for the slot `aggregate.data.slot`. - The block being voted for (`aggregate.data.beacon_block_root`) passes validation. - `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_aggregator(state, aggregate.data.slot, aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. From 071f6b51262551a7e40cd6b2b3113c92e8c60a2f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 25 Mar 2020 14:59:41 +0800 Subject: [PATCH 033/102] Gasprice rework: use `MIN_GASPRICE` as the initial gasprice and change `MIN_GASPRICE` to 8 gwei --- configs/mainnet.yaml | 7 ++----- configs/minimal.yaml | 6 ++---- specs/phase1/beacon-chain.md | 2 +- specs/phase1/phase1-fork.md | 3 +-- 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 8306bb378..6d71cfa47 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -162,9 +162,6 @@ DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 # --------------------------------------------------------------- PHASE_1_FORK_VERSION: 0x01000000 INITIAL_ACTIVE_SHARDS: 64 -# Placeholder -INITIAL_GASPRICE: 10 - # Phase 1: General # --------------------------------------------------------------- @@ -190,8 +187,8 @@ SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] MAX_SHARD_BLOCKS_PER_ATTESTATION: 12 # 2**14 (= 16,384) Gwei MAX_GASPRICE: 16384 -# 2**5 (= 32) Gwei -MIN_GASPRICE: 32 +# 2**3 (= 8) Gwei +MIN_GASPRICE: 8 # 2**3 (= 8) GASPRICE_ADJUSTMENT_COEFFICIENT: 8 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index e5d6ca308..9daf428b4 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -164,8 +164,6 @@ DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 PHASE_1_FORK_VERSION: 0x01000001 # [customized] reduced for testing INITIAL_ACTIVE_SHARDS: 4 -# Placeholder -INITIAL_GASPRICE: 10 # Phase 1: General @@ -192,8 +190,8 @@ SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] MAX_SHARD_BLOCKS_PER_ATTESTATION: 12 # 2**14 (= 16,384) Gwei MAX_GASPRICE: 16384 -# 2**5 (= 32) Gwei -MIN_GASPRICE: 32 +# 2**3 (= 8) Gwei +MIN_GASPRICE: 8 # 2**3 (= 8) GASPRICE_ADJUSTMENT_COEFFICIENT: 8 diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index e434e7eae..3db7f9537 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -101,7 +101,7 @@ Configuration is not namespaced. Instead it is strictly an extension; | `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | | | `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | | | `MAX_GASPRICE` | `Gwei(2**14)` (= 16,384) | Gwei | | -| `MIN_GASPRICE` | `Gwei(2**5)` (= 32) | Gwei | | +| `MIN_GASPRICE` | `Gwei(2**3)` (= 8) | Gwei | | | `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | | | `DOMAIN_SHARD_PROPOSAL` | `DomainType('0x80000000')` | | | `DOMAIN_SHARD_COMMITTEE` | `DomainType('0x81000000')` | | diff --git a/specs/phase1/phase1-fork.md b/specs/phase1/phase1-fork.md index adb0cd236..173fceeb4 100644 --- a/specs/phase1/phase1-fork.md +++ b/specs/phase1/phase1-fork.md @@ -36,7 +36,6 @@ Warning: this configuration is not definitive. | - | - | | `PHASE_1_FORK_VERSION` | `Version('0x01000000')` | | `INITIAL_ACTIVE_SHARDS` | `2**6` (= 64) | -| `INITIAL_GASPRICE` | `Gwei(10)` | ## Fork to Phase 1 @@ -102,7 +101,7 @@ def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: shard_states=List[ShardState, MAX_SHARDS]( ShardState( slot=pre.slot, - gasprice=INITIAL_GASPRICE, + gasprice=MIN_GASPRICE, data=Root(), latest_block_root=Root(), ) for i in range(INITIAL_ACTIVE_SHARDS) From f82cdb7e68aab1b82b45d861a73bbf1858d59379 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 31 Mar 2020 18:21:33 +0800 Subject: [PATCH 034/102] beacon-chain.md: add `unpack_compact_validator` for `light-client-sync.md` --- specs/phase1/beacon-chain.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index e434e7eae..4c0473f2d 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -34,6 +34,7 @@ - [Misc](#misc-1) - [`get_previous_slot`](#get_previous_slot) - [`pack_compact_validator`](#pack_compact_validator) + - [`unpack_compact_validator`](#unpack_compact_validator) - [`committee_to_compact_committee`](#committee_to_compact_committee) - [`compute_shard_from_committee_index`](#compute_shard_from_committee_index) - [Beacon state accessors](#beacon-state-accessors) @@ -371,15 +372,29 @@ def get_previous_slot(slot: Slot) -> Slot: #### `pack_compact_validator` ```python -def pack_compact_validator(index: int, slashed: bool, balance_in_increments: int) -> int: +def pack_compact_validator(index: ValidatorIndex, slashed: bool, balance_in_increments: uint64) -> uint64: """ - Creates a compact validator object representing index, slashed status, and compressed balance. + Create a compact validator object representing index, slashed status, and compressed balance. Takes as input balance-in-increments (// EFFECTIVE_BALANCE_INCREMENT) to preserve symmetry with the unpacking function. """ return (index << 16) + (slashed << 15) + balance_in_increments ``` +#### `unpack_compact_validator` + +```python +def unpack_compact_validator(compact_validator: uint64) -> Tuple[ValidatorIndex, bool, uint64]: + """ + Return validator index, slashed, balance // EFFECTIVE_BALANCE_INCREMENT + """ + return ( + ValidatorIndex(compact_validator >> 16), + bool((compact_validator >> 15) % 2), + compact_validator & (2**15 - 1), + ) +``` + #### `committee_to_compact_committee` ```python From 613f368c0007a9ca4671793039874f3ecc299f58 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 1 Apr 2020 12:20:32 -0600 Subject: [PATCH 035/102] fix call to get_beacon_committee in process_crosslink_for_shard --- specs/phase1/beacon-chain.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 9207e3a08..9c0cd0a6c 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -709,11 +709,12 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr ```python def process_crosslink_for_shard(state: BeaconState, - shard: Shard, + committee_index: CommitteeIndex, shard_transition: ShardTransition, attestations: Sequence[Attestation]) -> Root: - committee = get_beacon_committee(state, get_current_epoch(state), shard) + committee = get_beacon_committee(state, state.slot, committee_index) online_indices = get_online_validator_indices(state) + shard = compute_shard_from_committee_index(state, committee_index, state.slot) # Loop over all shard transition roots shard_transition_roots = set([a.data.shard_transition_root for a in attestations]) @@ -775,7 +776,7 @@ def process_crosslinks(state: BeaconState, if attestation.data.index == committee_index and attestation.data.slot == state.slot ] shard_transition = shard_transitions[shard] - winning_root = process_crosslink_for_shard(state, shard, shard_transition, shard_attestations) + winning_root = process_crosslink_for_shard(state, committee_index, shard_transition, shard_attestations) if winning_root != Root(): # Mark relevant pending attestations as creating a successful crosslink for pending_attestation in state.current_epoch_attestations: From a890d1f6a08d515c17ad45c45a35f9295ba0fed1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 2 Apr 2020 14:54:48 +0800 Subject: [PATCH 036/102] Use constant variables to define phase name/ID --- tests/core/pyspec/eth2spec/test/context.py | 45 ++++++++++++---------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 5d50612c7..c0ed723d0 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -7,7 +7,7 @@ from .helpers.genesis import create_genesis_state from .utils import vector_test, with_meta_tags from random import Random -from typing import Any, Callable, Sequence, TypedDict, Protocol +from typing import Any, Callable, NewType, Sequence, TypedDict, Protocol from importlib import reload @@ -19,25 +19,33 @@ def reload_specs(): # Some of the Spec module functionality is exposed here to deal with phase-specific changes. +SpecForkName = NewType("SpecForkName", str) + +PHASE0 = SpecForkName('phase0') +PHASE1 = SpecForkName('phase1') +ALL_PHASES = (PHASE0, PHASE1) + # TODO: currently phases are defined as python modules. # It would be better if they would be more well-defined interfaces for stronger typing. + + class Spec(Protocol): version: str -class Phase0(Spec): +class SpecPhase0(Spec): ... -class Phase1(Spec): +class SpecPhase1(Spec): def upgrade_to_phase1(self, state: spec_phase0.BeaconState) -> spec_phase1.BeaconState: ... # add transfer, bridge, etc. as the spec evolves class SpecForks(TypedDict, total=False): - phase0: Phase0 - phase1: Phase1 + PHASE0: SpecPhase0 + PHASE1: SpecPhase1 def with_custom_state(balances_fn: Callable[[Any], Sequence[int]], @@ -45,16 +53,16 @@ def with_custom_state(balances_fn: Callable[[Any], Sequence[int]], def deco(fn): def entry(*args, spec: Spec, phases: SpecForks, **kw): try: - p0 = phases["phase0"] + p0 = phases[PHASE0] balances = balances_fn(p0) activation_threshold = threshold_fn(p0) state = create_genesis_state(spec=p0, validator_balances=balances, activation_threshold=activation_threshold) - if spec.fork == 'phase1': + if spec.fork == PHASE1: # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper. # Decide based on performance/consistency results later. - state = phases["phase1"].upgrade_to_phase1(state) + state = phases[PHASE1].upgrade_to_phase1(state) kw['state'] = state except KeyError: @@ -217,14 +225,11 @@ def bls_switch(fn): return entry -all_phases = ['phase0', 'phase1'] - - def with_all_phases(fn): """ A decorator for running a test with every phase """ - return with_phases(all_phases)(fn) + return with_phases(ALL_PHASES)(fn) def with_all_phases_except(exclusion_phases): @@ -232,7 +237,7 @@ def with_all_phases_except(exclusion_phases): A decorator factory for running a tests with every phase except the ones listed """ def decorator(fn): - return with_phases([phase for phase in all_phases if phase not in exclusion_phases])(fn) + return with_phases([phase for phase in ALL_PHASES if phase not in exclusion_phases])(fn) return decorator @@ -258,18 +263,18 @@ def with_phases(phases, other_phases=None): # TODO: test state is dependent on phase0 but is immediately transitioned to phase1. # A new state-creation helper for phase 1 may be in place, and then phase1+ tests can run without phase0 - available_phases.add('phase0') + available_phases.add(PHASE0) phase_dir = {} - if 'phase0' in available_phases: - phase_dir['phase0'] = spec_phase0 - if 'phase1' in available_phases: - phase_dir['phase1'] = spec_phase1 + if PHASE0 in available_phases: + phase_dir[PHASE0] = spec_phase0 + if PHASE1 in available_phases: + phase_dir[PHASE1] = spec_phase1 # return is ignored whenever multiple phases are ran. If - if 'phase0' in run_phases: + if PHASE0 in run_phases: ret = fn(spec=spec_phase0, phases=phase_dir, *args, **kw) - if 'phase1' in run_phases: + if PHASE1 in run_phases: ret = fn(spec=spec_phase1, phases=phase_dir, *args, **kw) return ret return wrapper From 3f87cea43512886ec679eff6fa31b7a8a3768230 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 2 Apr 2020 15:01:19 +0800 Subject: [PATCH 037/102] Use constants phase names --- .../test/fork_choice/test_on_attestation.py | 4 +-- .../test/genesis/test_initialization.py | 6 ++--- .../eth2spec/test/genesis/test_validity.py | 14 +++++----- .../eth2spec/test/helpers/attestations.py | 5 ++-- .../test/helpers/attester_slashings.py | 9 ++++--- .../test_process_attester_slashing.py | 27 ++++++++++--------- .../test_process_custody_key_reveal.py | 13 ++++----- ...est_process_early_derived_secret_reveal.py | 17 ++++++------ 8 files changed, 51 insertions(+), 44 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 09248944c..265e8769a 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import with_all_phases, spec_state_test +from eth2spec.test.context import PHASE0, with_all_phases, spec_state_test from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.attestations import get_valid_attestation, sign_attestation from eth2spec.test.helpers.state import transition_to, state_transition_and_sign_block @@ -16,7 +16,7 @@ def run_on_attestation(spec, state, store, attestation, valid=True): indexed_attestation = spec.get_indexed_attestation(state, attestation) spec.on_attestation(store, attestation) - if spec.fork == 'phase0': + if spec.fork == PHASE0: sample_index = indexed_attestation.attesting_indices[0] else: attesting_indices = [ diff --git a/tests/core/pyspec/eth2spec/test/genesis/test_initialization.py b/tests/core/pyspec/eth2spec/test/genesis/test_initialization.py index 61a2ffb1e..882821337 100644 --- a/tests/core/pyspec/eth2spec/test/genesis/test_initialization.py +++ b/tests/core/pyspec/eth2spec/test/genesis/test_initialization.py @@ -1,10 +1,10 @@ -from eth2spec.test.context import spec_test, with_phases, single_phase +from eth2spec.test.context import PHASE0, spec_test, with_phases, single_phase from eth2spec.test.helpers.deposits import ( prepare_genesis_deposits, ) -@with_phases(['phase0']) +@with_phases(([PHASE0])) @spec_test @single_phase def test_initialize_beacon_state_from_eth1(spec): @@ -32,7 +32,7 @@ def test_initialize_beacon_state_from_eth1(spec): yield 'state', state -@with_phases(['phase0']) +@with_phases([PHASE0]) @spec_test @single_phase def test_initialize_beacon_state_some_small_balances(spec): diff --git a/tests/core/pyspec/eth2spec/test/genesis/test_validity.py b/tests/core/pyspec/eth2spec/test/genesis/test_validity.py index a90b4a695..dbaf3f951 100644 --- a/tests/core/pyspec/eth2spec/test/genesis/test_validity.py +++ b/tests/core/pyspec/eth2spec/test/genesis/test_validity.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import spec_test, with_phases, single_phase +from eth2spec.test.context import PHASE0, spec_test, with_phases, single_phase from eth2spec.test.helpers.deposits import ( prepare_genesis_deposits, ) @@ -25,7 +25,7 @@ def run_is_valid_genesis_state(spec, state, valid=True): assert is_valid == valid -@with_phases(['phase0']) +@with_phases([PHASE0]) @spec_test @single_phase def test_is_valid_genesis_state_true(spec): @@ -34,7 +34,7 @@ def test_is_valid_genesis_state_true(spec): yield from run_is_valid_genesis_state(spec, state, valid=True) -@with_phases(['phase0']) +@with_phases([PHASE0]) @spec_test @single_phase def test_is_valid_genesis_state_false_invalid_timestamp(spec): @@ -44,7 +44,7 @@ def test_is_valid_genesis_state_false_invalid_timestamp(spec): yield from run_is_valid_genesis_state(spec, state, valid=False) -@with_phases(['phase0']) +@with_phases([PHASE0]) @spec_test @single_phase def test_is_valid_genesis_state_true_more_balance(spec): @@ -55,7 +55,7 @@ def test_is_valid_genesis_state_true_more_balance(spec): # TODO: not part of the genesis function yet. Erroneously merged. -# @with_phases(['phase0']) +# @with_phases([PHASE0]) # @spec_test # def test_is_valid_genesis_state_false_not_enough_balance(spec): # state = create_valid_beacon_state(spec) @@ -64,7 +64,7 @@ def test_is_valid_genesis_state_true_more_balance(spec): # yield from run_is_valid_genesis_state(spec, state, valid=False) -@with_phases(['phase0']) +@with_phases([PHASE0]) @spec_test @single_phase def test_is_valid_genesis_state_true_one_more_validator(spec): @@ -78,7 +78,7 @@ def test_is_valid_genesis_state_true_one_more_validator(spec): yield from run_is_valid_genesis_state(spec, state, valid=True) -@with_phases(['phase0']) +@with_phases([PHASE0]) @spec_test @single_phase def test_is_valid_genesis_state_false_not_enough_validator(spec): diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 281d11b45..4581af24e 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -1,5 +1,6 @@ from typing import List +from eth2spec.test.context import PHASE0 from eth2spec.test.helpers.block import build_empty_block_for_next_slot, transition_unsigned_block, \ build_empty_block from eth2spec.test.helpers.keys import privkeys @@ -78,12 +79,12 @@ def sign_aggregate_attestation(spec, state, attestation_data, participants: List privkey ) ) - # TODO: we should try signing custody bits if spec.fork == 'phase1' + # TODO: we should try signing custody bits if spec.fork == PHASE1 return bls.Aggregate(signatures) def sign_indexed_attestation(spec, state, indexed_attestation): - if spec.fork == 'phase0': + if spec.fork == PHASE0: participants = indexed_attestation.attesting_indices data = indexed_attestation.data indexed_attestation.signature = sign_aggregate_attestation(spec, state, data, participants) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py b/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py index 5dfedc200..975f34c20 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py @@ -1,3 +1,4 @@ +from eth2spec.test.context import PHASE1 from eth2spec.test.helpers.attestations import get_valid_attestation, sign_attestation @@ -20,7 +21,7 @@ def get_indexed_attestation_participants(spec, indexed_att): """ Wrapper around index-attestation to return the list of participant indices, regardless of spec phase. """ - if spec.fork == "phase1": + if spec.fork == PHASE1: return list(spec.get_indices_from_committee( indexed_att.committee, indexed_att.attestation.aggregation_bits, @@ -33,21 +34,21 @@ def set_indexed_attestation_participants(spec, indexed_att, participants): """ Wrapper around index-attestation to return the list of participant indices, regardless of spec phase. """ - if spec.fork == "phase1": + if spec.fork == PHASE1: indexed_att.attestation.aggregation_bits = [bool(i in participants) for i in indexed_att.committee] else: indexed_att.attesting_indices = participants def get_attestation_1_data(spec, att_slashing): - if spec.fork == "phase1": + if spec.fork == PHASE1: return att_slashing.attestation_1.attestation.data else: return att_slashing.attestation_1.data def get_attestation_2_data(spec, att_slashing): - if spec.fork == "phase1": + if spec.fork == PHASE1: return att_slashing.attestation_2.attestation.data else: return att_slashing.attestation_2.data diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index e9665e714..48dc75fd9 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -1,4 +1,7 @@ -from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases, with_phases +from eth2spec.test.context import ( + PHASE0, PHASE1, + spec_state_test, expect_assertion_error, always_bls, with_all_phases, with_phases +) from eth2spec.test.helpers.attestations import sign_indexed_attestation from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing, \ get_indexed_attestation_participants, get_attestation_2_data, get_attestation_1_data @@ -161,7 +164,7 @@ def test_same_data(spec, state): indexed_att_1 = attester_slashing.attestation_1 att_2_data = get_attestation_2_data(spec, attester_slashing) - if spec.fork == 'phase1': + if spec.fork == PHASE1: indexed_att_1.attestation.data = att_2_data else: indexed_att_1.data = att_2_data @@ -199,7 +202,7 @@ def test_participants_already_slashed(spec, state): # Some of the following tests are phase0 only: phase 1 lists participants with bitfields instead of index list. -@with_phases(['phase0']) +@with_phases([PHASE0]) @spec_state_test @always_bls def test_att1_bad_extra_index(spec, state): @@ -215,7 +218,7 @@ def test_att1_bad_extra_index(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases(['phase0']) +@with_phases([PHASE0]) @spec_state_test @always_bls def test_att1_bad_replaced_index(spec, state): @@ -231,7 +234,7 @@ def test_att1_bad_replaced_index(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases(['phase0']) +@with_phases([PHASE0]) @spec_state_test @always_bls def test_att2_bad_extra_index(spec, state): @@ -247,7 +250,7 @@ def test_att2_bad_extra_index(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases(['phase0']) +@with_phases([PHASE0]) @spec_state_test @always_bls def test_att2_bad_replaced_index(spec, state): @@ -263,7 +266,7 @@ def test_att2_bad_replaced_index(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases(['phase0']) +@with_phases([PHASE0]) @spec_state_test @always_bls def test_att1_duplicate_index_normal_signed(spec, state): @@ -283,7 +286,7 @@ def test_att1_duplicate_index_normal_signed(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases(['phase0']) +@with_phases([PHASE0]) @spec_state_test @always_bls def test_att2_duplicate_index_normal_signed(spec, state): @@ -303,7 +306,7 @@ def test_att2_duplicate_index_normal_signed(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases(['phase0']) +@with_phases([PHASE0]) @spec_state_test @always_bls def test_att1_duplicate_index_double_signed(spec, state): @@ -318,7 +321,7 @@ def test_att1_duplicate_index_double_signed(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases(['phase0']) +@with_phases([PHASE0]) @spec_state_test @always_bls def test_att2_duplicate_index_double_signed(spec, state): @@ -333,7 +336,7 @@ def test_att2_duplicate_index_double_signed(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases(['phase0']) +@with_phases([PHASE0]) @spec_state_test def test_unsorted_att_1(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) @@ -346,7 +349,7 @@ def test_unsorted_att_1(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases(['phase0']) +@with_phases([PHASE0]) @spec_state_test def test_unsorted_att_2(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=False) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py index fb9157f2f..8c2436d5b 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py @@ -1,5 +1,6 @@ from eth2spec.test.helpers.custody import get_valid_custody_key_reveal from eth2spec.test.context import ( + PHASE0, with_all_phases_except, spec_state_test, expect_assertion_error, @@ -54,7 +55,7 @@ def run_custody_key_reveal_processing(spec, state, custody_key_reveal, valid=Tru yield 'post', state -@with_all_phases_except(['phase0']) +@with_all_phases_except([PHASE0]) @spec_state_test @always_bls def test_success(spec, state): @@ -64,7 +65,7 @@ def test_success(spec, state): yield from run_custody_key_reveal_processing(spec, state, custody_key_reveal) -@with_all_phases_except(['phase0']) +@with_all_phases_except([PHASE0]) @spec_state_test @always_bls def test_reveal_too_early(spec, state): @@ -73,7 +74,7 @@ def test_reveal_too_early(spec, state): yield from run_custody_key_reveal_processing(spec, state, custody_key_reveal, False) -@with_all_phases_except(['phase0']) +@with_all_phases_except([PHASE0]) @spec_state_test @always_bls def test_wrong_period(spec, state): @@ -82,7 +83,7 @@ def test_wrong_period(spec, state): yield from run_custody_key_reveal_processing(spec, state, custody_key_reveal, False) -@with_all_phases_except(['phase0']) +@with_all_phases_except([PHASE0]) @spec_state_test @always_bls def test_late_reveal(spec, state): @@ -92,7 +93,7 @@ def test_late_reveal(spec, state): yield from run_custody_key_reveal_processing(spec, state, custody_key_reveal) -@with_all_phases_except(['phase0']) +@with_all_phases_except([PHASE0]) @spec_state_test @always_bls def test_double_reveal(spec, state): @@ -104,7 +105,7 @@ def test_double_reveal(spec, state): yield from run_custody_key_reveal_processing(spec, state, custody_key_reveal, False) -@with_all_phases_except(['phase0']) +@with_all_phases_except([PHASE0]) @spec_state_test @always_bls def test_max_decrement(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py index c5d9c5a63..83b0fe325 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py @@ -2,6 +2,7 @@ from eth2spec.test.helpers.custody import get_valid_early_derived_secret_reveal from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.helpers.state import next_epoch, get_balance from eth2spec.test.context import ( + PHASE0, with_all_phases_except, spec_state_test, expect_assertion_error, @@ -41,7 +42,7 @@ def run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, v yield 'post', state -@with_all_phases_except(['phase0']) +@with_all_phases_except([PHASE0]) @spec_state_test @always_bls def test_success(spec, state): @@ -50,7 +51,7 @@ def test_success(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal) -@with_all_phases_except(['phase0']) +@with_all_phases_except([PHASE0]) @spec_state_test @never_bls def test_reveal_from_current_epoch(spec, state): @@ -59,7 +60,7 @@ def test_reveal_from_current_epoch(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) -@with_all_phases_except(['phase0']) +@with_all_phases_except([PHASE0]) @spec_state_test @never_bls def test_reveal_from_past_epoch(spec, state): @@ -70,7 +71,7 @@ def test_reveal_from_past_epoch(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) -@with_all_phases_except(['phase0']) +@with_all_phases_except([PHASE0]) @spec_state_test @always_bls def test_reveal_with_custody_padding(spec, state): @@ -82,7 +83,7 @@ def test_reveal_with_custody_padding(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, True) -@with_all_phases_except(['phase0']) +@with_all_phases_except([PHASE0]) @spec_state_test @always_bls def test_reveal_with_custody_padding_minus_one(spec, state): @@ -94,7 +95,7 @@ def test_reveal_with_custody_padding_minus_one(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, True) -@with_all_phases_except(['phase0']) +@with_all_phases_except([PHASE0]) @spec_state_test @never_bls def test_double_reveal(spec, state): @@ -115,7 +116,7 @@ def test_double_reveal(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal2, False) -@with_all_phases_except(['phase0']) +@with_all_phases_except([PHASE0]) @spec_state_test @never_bls def test_revealer_is_slashed(spec, state): @@ -125,7 +126,7 @@ def test_revealer_is_slashed(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) -@with_all_phases_except(['phase0']) +@with_all_phases_except([PHASE0]) @spec_state_test @never_bls def test_far_future_epoch(spec, state): From 523315bf4f9e3bfb74a4df5b5e1f4f790a882659 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 2 Apr 2020 15:23:20 +0800 Subject: [PATCH 038/102] Use phase name constants for the fork names in test generators --- tests/generators/bls/main.py | 3 ++- tests/generators/epoch_processing/main.py | 3 ++- tests/generators/genesis/main.py | 3 ++- tests/generators/operations/main.py | 3 ++- tests/generators/sanity/main.py | 3 ++- tests/generators/shuffling/main.py | 3 ++- tests/generators/ssz_generic/main.py | 3 ++- tests/generators/ssz_static/main.py | 3 ++- 8 files changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/generators/bls/main.py b/tests/generators/bls/main.py index bad4aab06..455292ae3 100644 --- a/tests/generators/bls/main.py +++ b/tests/generators/bls/main.py @@ -13,6 +13,7 @@ from gen_base import gen_runner, gen_typing from py_ecc import bls from hashlib import sha256 +from eth2spec.test.context import PHASE0 def hash(x): return sha256(x).digest() @@ -202,7 +203,7 @@ def create_provider(handler_name: str, print(data) (case_name, case_content) = data yield gen_typing.TestCase( - fork_name='phase0', + fork_name=PHASE0, runner_name='bls', handler_name=handler_name, suite_name='small', diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index 8f2a6e94f..f3bbc21e6 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -13,6 +13,7 @@ from gen_base import gen_runner, gen_typing from gen_from_tests.gen import generate_from_tests from importlib import reload from eth2spec.config import config_util +from eth2spec.test.context import PHASE0 def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: @@ -28,7 +29,7 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin runner_name='epoch_processing', handler_name=handler_name, src=tests_src, - fork_name='phase0' + fork_name=PHASE0, ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) diff --git a/tests/generators/genesis/main.py b/tests/generators/genesis/main.py index 3563c3fd9..8548b12c1 100644 --- a/tests/generators/genesis/main.py +++ b/tests/generators/genesis/main.py @@ -1,5 +1,6 @@ from typing import Iterable +from eth2spec.test.context import PHASE0 from eth2spec.test.genesis import test_initialization, test_validity from gen_base import gen_runner, gen_typing @@ -21,7 +22,7 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin runner_name='genesis', handler_name=handler_name, src=tests_src, - fork_name='phase0' + fork_name=PHASE0, ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index 6906c9df7..935c7aa63 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -15,6 +15,7 @@ from importlib import reload from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 +from eth2spec.test.context import PHASE0 def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: @@ -30,7 +31,7 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin runner_name='operations', handler_name=handler_name, src=tests_src, - fork_name='phase0' + fork_name=PHASE0, ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) diff --git a/tests/generators/sanity/main.py b/tests/generators/sanity/main.py index cfcbcfdb6..74c85a9e8 100644 --- a/tests/generators/sanity/main.py +++ b/tests/generators/sanity/main.py @@ -4,6 +4,7 @@ from importlib import reload from gen_base import gen_runner, gen_typing from gen_from_tests.gen import generate_from_tests +from eth2spec.test.context import PHASE0 from eth2spec.test.sanity import test_blocks, test_slots from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 @@ -23,7 +24,7 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin runner_name='sanity', handler_name=handler_name, src=tests_src, - fork_name='phase0' + fork_name=PHASE0, ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) diff --git a/tests/generators/shuffling/main.py b/tests/generators/shuffling/main.py index 0ef2657c4..6069de77a 100644 --- a/tests/generators/shuffling/main.py +++ b/tests/generators/shuffling/main.py @@ -6,6 +6,7 @@ from gen_base import gen_runner, gen_typing from eth2spec.config import config_util from eth2spec.phase0 import spec as spec +from eth2spec.test.context import PHASE0 def shuffling_case_fn(seed, count): @@ -37,7 +38,7 @@ def create_provider(config_name: str) -> gen_typing.TestProvider: def cases_fn() -> Iterable[gen_typing.TestCase]: for (case_name, case_fn) in shuffling_test_cases(): yield gen_typing.TestCase( - fork_name='phase0', + fork_name=PHASE0, runner_name='shuffling', handler_name='core', suite_name='shuffle', diff --git a/tests/generators/ssz_generic/main.py b/tests/generators/ssz_generic/main.py index 83e6da86d..8cfb2e3eb 100644 --- a/tests/generators/ssz_generic/main.py +++ b/tests/generators/ssz_generic/main.py @@ -6,6 +6,7 @@ import ssz_bitvector import ssz_boolean import ssz_uints import ssz_container +from eth2spec.test.context import PHASE0 def create_provider(handler_name: str, suite_name: str, case_maker) -> gen_typing.TestProvider: @@ -16,7 +17,7 @@ def create_provider(handler_name: str, suite_name: str, case_maker) -> gen_typin def cases_fn() -> Iterable[gen_typing.TestCase]: for (case_name, case_fn) in case_maker(): yield gen_typing.TestCase( - fork_name='phase0', + fork_name=PHASE0, runner_name='ssz_generic', handler_name=handler_name, suite_name=suite_name, diff --git a/tests/generators/ssz_static/main.py b/tests/generators/ssz_static/main.py index b7c948767..b9cb51db0 100644 --- a/tests/generators/ssz_static/main.py +++ b/tests/generators/ssz_static/main.py @@ -8,6 +8,7 @@ from gen_base import gen_runner, gen_typing from eth2spec.debug import random_value, encode from eth2spec.config import config_util from eth2spec.phase0 import spec +from eth2spec.test.context import PHASE0 from eth2spec.utils.ssz.ssz_typing import Container from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, @@ -44,7 +45,7 @@ def ssz_static_cases(seed: int, name, ssz_type, mode: random_value.Randomization for i in range(count): yield gen_typing.TestCase( - fork_name='phase0', + fork_name=PHASE0, runner_name='ssz_static', handler_name=name, suite_name=f"ssz_{random_mode_name}{'_chaos' if chaos else ''}", From f2c2da95ed871735d35ef757641f92bd1d31133a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 3 Apr 2020 09:19:56 -0600 Subject: [PATCH 039/102] add compute_offset_slots --- specs/phase1/beacon-chain.md | 19 +++++++++++-------- .../eth2spec/test/helpers/attestations.py | 7 ++----- .../test/helpers/phase1/attestations.py | 5 +---- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 9c0cd0a6c..5079ec5c5 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -517,11 +517,18 @@ def get_latest_slot_for_shard(state: BeaconState, shard: Shard) -> Slot: return state.shard_states[shard].slot ``` +#### `compute_offset_slots` + +```python +def compute_offset_slots(start_slot: Slot, end_slot: Slot) -> Sequence[Slot]: + return [Slot(start_slot + x) for x in SHARD_BLOCK_OFFSETS if start_slot + x < end_slot] +``` + #### `get_offset_slots` ```python -def get_offset_slots(state: BeaconState, latest_shard_slot: Slot) -> Sequence[Slot]: - return [Slot(latest_shard_slot + x) for x in SHARD_BLOCK_OFFSETS if latest_shard_slot + x < state.slot] +def get_offset_slots(state: BeaconState, shard: Shard) -> Sequence[Slot]: + return compute_offset_slots(state.shard_states[shard].slot, state.slot) ``` ### Predicates @@ -630,14 +637,13 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: assert attestation.data.source == state.previous_justified_checkpoint shard = get_shard(state, attestation) - latest_shard_slot = get_latest_slot_for_shard(state, shard) # Type 1: on-time attestations if attestation.custody_bits_blocks != []: # Ensure on-time attestation assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY == state.slot # Correct data root count - assert len(attestation.custody_bits_blocks) == len(get_offset_slots(state, latest_shard_slot)) + assert len(attestation.custody_bits_blocks) == len(get_offset_slots(state, shard)) # Correct parent block root assert data.beacon_block_root == get_block_root_at_slot(state, get_previous_slot(state.slot)) # Type 2: no shard transition, no custody bits # TODO: could only allow for older attestations. @@ -655,11 +661,8 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: ```python def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTransition) -> None: - # Slot the attestation starts counting from - latest_slot = get_latest_slot_for_shard(state, shard) - # Correct data root count - offset_slots = get_offset_slots(state, latest_slot) + offset_slots = get_offset_slots(state, shard) assert ( len(transition.shard_data_roots) == len(transition.shard_states) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index af989a9c0..7daaf159e 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -1,7 +1,7 @@ from typing import List from eth2spec.test.context import expect_assertion_error -from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_block +from eth2spec.test.helpers.state import state_transition_and_sign_block from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls @@ -77,10 +77,7 @@ def build_attestation_data(spec, state, slot, index): def convert_to_valid_on_time_attestation(spec, state, attestation, signed=False): shard = spec.get_shard(state, attestation) - - next_state = state.copy() - next_slot(spec, next_state) - offset_slots = spec.get_offset_slots(next_state, spec.get_latest_slot_for_shard(next_state, shard)) + offset_slots = spec.compute_offset_slots(spec.get_latest_slot_for_shard(state, shard), state.slot + 1) for offset_slot in offset_slots: attestation.custody_bits_blocks.append( Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE]([0 for _ in attestation.aggregation_bits]) diff --git a/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py index b97cc5d46..676908f85 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py @@ -3,7 +3,6 @@ from eth2spec.utils import bls from eth2spec.test.helpers.keys import privkeys import eth2spec.test.helpers.attestations as phase0_attestations -from eth2spec.test.helpers.state import next_slot def get_valid_on_time_attestation(spec, state, index=None, signed=False): @@ -15,10 +14,8 @@ def get_valid_on_time_attestation(spec, state, index=None, signed=False): attestation = phase0_attestations.get_valid_attestation(spec, state, state.slot, index, False) shard = spec.get_shard(state, attestation) + offset_slots = spec.compute_offset_slots(spec.get_latest_slot_for_shard(state, shard), state.slot + 1) - next_state = state.copy() - next_slot(spec, next_state) - offset_slots = spec.get_offset_slots(next_state, spec.get_latest_slot_for_shard(next_state, shard)) for offset_slot in offset_slots: attestation.custody_bits_blocks.append( Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE]([0 for _ in attestation.aggregation_bits]) From 246b46771e7b180cfdfd59b6c3812e9f44f6a60e Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 3 Apr 2020 09:46:22 -0600 Subject: [PATCH 040/102] address @hwwhww feedback --- specs/phase1/beacon-chain.md | 32 ++++++++++++++----- .../test/helpers/phase1/attestations.py | 2 +- .../test_process_rewards_and_penalties.py | 17 +++++----- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 5079ec5c5..a5c4db859 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -48,8 +48,10 @@ - [`get_start_shard`](#get_start_shard) - [`get_shard`](#get_shard) - [`get_latest_slot_for_shard`](#get_latest_slot_for_shard) + - [`compute_offset_slots`](#compute_offset_slots) - [`get_offset_slots`](#get_offset_slots) - [Predicates](#predicates) + - [`is_winning_attestation`](#is_winning_attestation) - [Updated `is_valid_indexed_attestation`](#updated-is_valid_indexed_attestation) - [Block processing](#block-processing) - [Operations](#operations) @@ -533,6 +535,24 @@ def get_offset_slots(state: BeaconState, shard: Shard) -> Sequence[Slot]: ### Predicates +#### `is_winning_attestation` + +```python +def is_winning_attestation(state: BeaconState, + attestation: PendingAttestation, + committee_index: CommitteeIndex, + winning_root: Root) -> bool: + """ + Check if ``attestation`` helped contribute to the successful crosslink of + ``winning_root`` formed by ``committee_index`` committee at the current slot. + """ + return ( + attestation.slot == state.slot + and attestation.data.index == committee_index + and attestation.data.shard_transition_root == winning_root + ) +``` + #### Updated `is_valid_indexed_attestation` Note that this replaces the Phase 0 `is_valid_indexed_attestation`. @@ -638,7 +658,7 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: shard = get_shard(state, attestation) - # Type 1: on-time attestations + # Type 1: on-time attestations, the custody bits should be non-empty. if attestation.custody_bits_blocks != []: # Ensure on-time attestation assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY == state.slot @@ -646,7 +666,7 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: assert len(attestation.custody_bits_blocks) == len(get_offset_slots(state, shard)) # Correct parent block root assert data.beacon_block_root == get_block_root_at_slot(state, get_previous_slot(state.slot)) - # Type 2: no shard transition, no custody bits # TODO: could only allow for older attestations. + # Type 2: no shard transition, no custody bits else: # Ensure delayed attestation assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY < state.slot @@ -783,11 +803,7 @@ def process_crosslinks(state: BeaconState, if winning_root != Root(): # Mark relevant pending attestations as creating a successful crosslink for pending_attestation in state.current_epoch_attestations: - if ( - pending_attestation.slot == state.slot and pending_attestation - and pending_attestation.data.index == committee_index - and pending_attestation.data.shard_transition_root == winning_root - ): + if is_winning_attestation(state, pending_attestation, committee_index, winning_root): pending_attestation.crosslink_success = True ``` @@ -801,8 +817,8 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: aggregation_bits=attestation.aggregation_bits, data=attestation.data, inclusion_delay=state.slot - attestation.data.slot, - crosslink_success=False, # To be filled in during process_crosslinks proposer_index=get_beacon_proposer_index(state), + crosslink_success=False, # To be filled in during process_crosslinks ) if attestation.data.target.epoch == get_current_epoch(state): state.current_epoch_attestations.append(pending_attestation) diff --git a/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py index 676908f85..0e16e1fac 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py @@ -16,7 +16,7 @@ def get_valid_on_time_attestation(spec, state, index=None, signed=False): shard = spec.get_shard(state, attestation) offset_slots = spec.compute_offset_slots(spec.get_latest_slot_for_shard(state, shard), state.slot + 1) - for offset_slot in offset_slots: + for _ in offset_slots: attestation.custody_bits_blocks.append( Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE]([0 for _ in attestation.aggregation_bits]) ) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py index 317c97149..af695fe69 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py @@ -22,13 +22,16 @@ def run_process_rewards_and_penalties(spec, state): def prepare_state_with_full_attestations(spec, state, empty=False): + # Go to start of next epoch to ensure can have full participation + next_epoch(spec, state) + start_slot = state.slot start_epoch = spec.get_current_epoch(state) - next_start_epoch = spec.compute_start_slot_at_epoch(start_epoch + 1) + next_epoch_start_slot = spec.compute_start_slot_at_epoch(start_epoch + 1) attestations = [] - for slot in range(spec.SLOTS_PER_EPOCH + spec.MIN_ATTESTATION_INCLUSION_DELAY): + for _ in range(spec.SLOTS_PER_EPOCH + spec.MIN_ATTESTATION_INCLUSION_DELAY): # create an attestation for each index in each slot in epoch - if state.slot < next_start_epoch: + if state.slot < next_epoch_start_slot: for committee_index in range(spec.get_committee_count_at_slot(state, state.slot)): attestation = get_valid_attestation(spec, state, index=committee_index, empty=empty, signed=True) attestations.append(attestation) @@ -39,7 +42,7 @@ def prepare_state_with_full_attestations(spec, state, empty=False): add_attestations_to_state(spec, state, include_attestations, state.slot) next_slot(spec, state) - assert spec.compute_epoch_at_slot(state.slot) == start_epoch + 1 + assert state.slot == next_epoch_start_slot + spec.MIN_ATTESTATION_INCLUSION_DELAY assert len(state.previous_epoch_attestations) == len(attestations) return attestations @@ -87,8 +90,6 @@ def test_genesis_epoch_full_attestations_no_rewards(spec, state): @with_all_phases @spec_state_test def test_full_attestations(spec, state): - # Go to start of next epoch to ensure can have full participation - next_epoch(spec, state) attestations = prepare_state_with_full_attestations(spec, state) pre_state = state.copy() @@ -132,8 +133,6 @@ def test_full_attestations_random_incorrect_fields(spec, state): @with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.MAX_EFFECTIVE_BALANCE // 2) @single_phase def test_full_attestations_misc_balances(spec, state): - # Go to start of next epoch to ensure can have full participation - next_epoch(spec, state) attestations = prepare_state_with_full_attestations(spec, state) pre_state = state.copy() @@ -178,6 +177,7 @@ def test_full_attestations_one_validaor_one_gwei(spec, state): @with_all_phases @spec_state_test def test_no_attestations_all_penalties(spec, state): + # Move to next epoch to ensure rewards/penalties are processed next_epoch(spec, state) pre_state = state.copy() @@ -254,7 +254,6 @@ def test_attestations_some_slashed(spec, state): for i in range(spec.MIN_PER_EPOCH_CHURN_LIMIT): spec.slash_validator(state, attesting_indices_before_slashings[i]) - assert spec.compute_epoch_at_slot(state.slot) == spec.GENESIS_EPOCH + 1 assert len(state.previous_epoch_attestations) == len(attestations) pre_state = state.copy() From e86c5ef41dabdb7e688d8789478d03db6c10fd26 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 3 Apr 2020 10:29:35 -0600 Subject: [PATCH 041/102] final PR nitpicks --- specs/phase1/beacon-chain.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index a5c4db859..596b3818f 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -37,6 +37,7 @@ - [`unpack_compact_validator`](#unpack_compact_validator) - [`committee_to_compact_committee`](#committee_to_compact_committee) - [`compute_shard_from_committee_index`](#compute_shard_from_committee_index) + - [`compute_offset_slots`](#compute_offset_slots) - [Beacon state accessors](#beacon-state-accessors) - [`get_active_shard_count`](#get_active_shard_count) - [`get_online_validator_indices`](#get_online_validator_indices) @@ -48,7 +49,6 @@ - [`get_start_shard`](#get_start_shard) - [`get_shard`](#get_shard) - [`get_latest_slot_for_shard`](#get_latest_slot_for_shard) - - [`compute_offset_slots`](#compute_offset_slots) - [`get_offset_slots`](#get_offset_slots) - [Predicates](#predicates) - [`is_winning_attestation`](#is_winning_attestation) @@ -421,6 +421,16 @@ def compute_shard_from_committee_index(state: BeaconState, index: CommitteeIndex return Shard((index + get_start_shard(state, slot)) % active_shards) ``` +#### `compute_offset_slots` + +```python +def compute_offset_slots(start_slot: Slot, end_slot: Slot) -> Sequence[Slot]: + """ + Return the offset slots that are greater than ``start_slot`` and less than ``end_slot``. + """ + return [Slot(start_slot + x) for x in SHARD_BLOCK_OFFSETS if start_slot + x < end_slot] +``` + ### Beacon state accessors #### `get_active_shard_count` @@ -519,13 +529,6 @@ def get_latest_slot_for_shard(state: BeaconState, shard: Shard) -> Slot: return state.shard_states[shard].slot ``` -#### `compute_offset_slots` - -```python -def compute_offset_slots(start_slot: Slot, end_slot: Slot) -> Sequence[Slot]: - return [Slot(start_slot + x) for x in SHARD_BLOCK_OFFSETS if start_slot + x < end_slot] -``` - #### `get_offset_slots` ```python From 7d4b97240b3825966c8783f6df54300f7c80684a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 6 Apr 2020 17:46:33 +1000 Subject: [PATCH 042/102] Redefine attestation propogation condition --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 24a1d4376..4c50892a3 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -316,7 +316,7 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio - The attestation's committee index (`attestation.data.index`) is for the correct subnet. - `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` (a client MAY queue future attestations for processing at the appropriate slot). - The attestation is unaggregated -- that is, it has exactly one participating validator (`len([bit for bit in attestation.aggregation_bits if bit == 0b1]) == 1`). - - The attestation is the first valid attestation received for the participating validator for the slot, `attestation.data.slot`. + - There has been no other attestation seen on an attestation subnet that has an indentical `attestation.data.slot` and participating validator index. - The block being voted for (`attestation.data.beacon_block_root`) passes validation. - The signature of `attestation` is valid. From bdf087d7f3a32133543f3a736fcbbb438ff87154 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 6 Apr 2020 09:57:23 -0600 Subject: [PATCH 043/102] add notes about how to handle peer discovery and gossip topics prior to genesis --- specs/phase0/p2p-interface.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 24a1d4376..c94b2c7e3 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -105,6 +105,7 @@ It consists of four main sections: - [Discovery](#discovery) - [Why are we using discv5 and not libp2p Kademlia DHT?](#why-are-we-using-discv5-and-not-libp2p-kademlia-dht) - [What is the difference between an ENR and a multiaddr, and why are we using ENRs?](#what-is-the-difference-between-an-enr-and-a-multiaddr-and-why-are-we-using-enrs) + - [Why do we not form ENRs and find peers until genesis block/state is known?](#why-do-we-not-form-enrs-and-find-peers-until-genesis-blockstate-is-known) - [Compression/Encoding](#compressionencoding) - [Why are we using SSZ for encoding?](#why-are-we-using-ssz-for-encoding) - [Why are we compressing, and at which layers?](#why-are-we-compressing-and-at-which-layers) @@ -247,6 +248,8 @@ Topics are plain UTF-8 strings and are encoded on the wire as determined by prot - `Name` - see table below - `Encoding` - the encoding strategy describes a specific representation of bytes that will be transmitted over the wire. See the [Encodings](#Encoding-strategies) section for further details. +*Note*: `ForkDigestValue` is composed of values that are not known until the genesis block/state are available. Due to this, clients SHOULD NOT subscribe to gossipsub topics until these genesis values are known. + Each gossipsub [message](https://github.com/libp2p/go-libp2p-pubsub/blob/master/pb/rpc.proto#L17-L24) has a maximum size of `GOSSIP_MAX_SIZE`. Clients MUST reject (fail validation) messages that are over this size limit. Likewise, clients MUST NOT emit or propagate messages larger than this limit. The `message-id` of a gossipsub message MUST be: @@ -752,6 +755,8 @@ where the fields of `ENRForkID` are defined as * `next_fork_version` is the fork version corresponding to the next planned hard fork at a future epoch. If no future fork is planned, set `next_fork_version = current_fork_version` to signal this fact * `next_fork_epoch` is the epoch at which the next fork is planned and the `current_fork_version` will be updated. If no future fork is planned, set `next_fork_epoch = FAR_FUTURE_EPOCH` to signal this fact +*Note*: `fork_digest` is composed of values that are not not known until the genesis block/state are available. Due to this, clients SHOULD NOT form ENRs and begin peer discovery until genesis values are known. + Clients SHOULD connect to peers with `fork_digest`, `next_fork_version`, and `next_fork_epoch` that match local values. Clients MAY connect to peers with the same `fork_digest` but a different `next_fork_version`/`next_fork_epoch`. Unless `ENRForkID` is manually updated to matching prior to the earlier `next_fork_epoch` of the two clients, these connecting clients will be unable to successfully interact starting at the earlier `next_fork_epoch`. @@ -1092,6 +1097,12 @@ discv5 uses ENRs and we will presumably need to: 1. Add `multiaddr` to the dictionary, so that nodes can advertise their multiaddr under a reserved namespace in ENRs. – and/or – 2. Define a bi-directional conversion function between multiaddrs and the corresponding denormalized fields in an ENR (ip, ip6, tcp, tcp6, etc.), for compatibility with nodes that do not support multiaddr natively (e.g. Eth 1.0 nodes). +### Why do we not form ENRs and find peers until genesis block/state is known? + +Although client software might very well be running locally prior to the solidification of the eth2 genesis state and block, clients cannot form valid ENRs prior to this point. ENRs contain `fork_digest` which utilizes the `genesis_validators_root` for a cleaner separation between chains so prior to knowing genesis, we cannot use `fork_digest` to cleanly find peers on our intended chain. Once genesis data is known, we can then form ENRs and safely find peers. + +When using an eth1 deposit contract for deposits, `fork_digest` will be known at least `MIN_GENESIS_DELAY` (24 hours in mainnet configuration) before `genesis_time`, providing ample time to find peers and form initial connections and gossip subnets prior to genesis. + ## Compression/Encoding ### Why are we using SSZ for encoding? From 13d1303db82610d94e5701b713be2b1012dc290a Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 Apr 2020 18:40:09 +0200 Subject: [PATCH 044/102] update remerkleable; mul/div bound checks, update config loading --- setup.py | 2 +- tests/core/pyspec/eth2spec/config/config_util.py | 13 +++++++++---- .../test_process_justification_and_finalization.py | 4 ++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 911eb65b0..d1c62fb72 100644 --- a/setup.py +++ b/setup.py @@ -499,7 +499,7 @@ setup( "pycryptodome==3.9.4", "py_ecc==2.0.0", "dataclasses==0.6", - "remerkleable==0.1.12", + "remerkleable==0.1.13", "ruamel.yaml==0.16.5", "lru-dict==1.1.6" ] diff --git a/tests/core/pyspec/eth2spec/config/config_util.py b/tests/core/pyspec/eth2spec/config/config_util.py index 64c533f2d..4c5768a29 100644 --- a/tests/core/pyspec/eth2spec/config/config_util.py +++ b/tests/core/pyspec/eth2spec/config/config_util.py @@ -8,13 +8,18 @@ config: Dict[str, Any] = {} # Access to overwrite spec constants based on configuration # This is called by the spec module after declaring its globals, and applies the loaded presets. -def apply_constants_config(spec_globals: Dict[str, Any]) -> None: +def apply_constants_config(spec_globals: Dict[str, Any], warn_if_unknown: bool = False) -> None: global config for k, v in config.items(): - if k.startswith('DOMAIN_'): - spec_globals[k] = spec_globals['DomainType'](v) # domain types are defined as bytes in the configs + # the spec should have default values for everything, if not, the config key is invalid. + if k in spec_globals: + # Keep the same type as the default value indicates (which may be an SSZ basic type subclass, e.g. 'Gwei') + spec_globals[k] = spec_globals[k].__class__(v) else: - spec_globals[k] = v + # Note: Phase 0 spec will not know the phase 1 config values. + # Yet, during debugging you can enable explicit warnings. + if warn_if_unknown: + print(f"WARNING: unknown config key: '{k}' with value: '{v}'") # Load presets from a file, and then prepares the global config setting. This does not apply the config. diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 9f9e1c316..09af2126d 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -24,7 +24,7 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support raise Exception(f"cannot include attestations in epoch ${epoch} from epoch ${current_epoch}") total_balance = spec.get_total_active_balance(state) - remaining_balance = total_balance * 2 // 3 + remaining_balance = int(total_balance * 2 // 3) # can become negative start_slot = spec.compute_start_slot_at_epoch(epoch) for slot in range(start_slot, start_slot + spec.SLOTS_PER_EPOCH): @@ -42,7 +42,7 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support aggregation_bits = [0] * len(committee) for v in range(len(committee) * 2 // 3 + 1): if remaining_balance > 0: - remaining_balance -= state.validators[v].effective_balance + remaining_balance -= int(state.validators[v].effective_balance) aggregation_bits[v] = 1 else: break From 021cb98dbb808713e8b14c37f049e7662bfbc2c0 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 7 Apr 2020 07:05:51 +1000 Subject: [PATCH 045/102] Use epoch for attestation subnet seen-ness. --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 4c50892a3..8614cd00a 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -316,7 +316,7 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio - The attestation's committee index (`attestation.data.index`) is for the correct subnet. - `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` (a client MAY queue future attestations for processing at the appropriate slot). - The attestation is unaggregated -- that is, it has exactly one participating validator (`len([bit for bit in attestation.aggregation_bits if bit == 0b1]) == 1`). - - There has been no other attestation seen on an attestation subnet that has an indentical `attestation.data.slot` and participating validator index. + - There has been no other attestation seen on an attestation subnet that has an indentical `attestation.data.target.epoch` and participating validator index. - The block being voted for (`attestation.data.beacon_block_root`) passes validation. - The signature of `attestation` is valid. From 616385a094067e89750608de5d56b7cf320df347 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 7 Apr 2020 07:45:15 +1000 Subject: [PATCH 046/102] Fix spelling mistake --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 8614cd00a..e2ca054da 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -316,7 +316,7 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio - The attestation's committee index (`attestation.data.index`) is for the correct subnet. - `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` (a client MAY queue future attestations for processing at the appropriate slot). - The attestation is unaggregated -- that is, it has exactly one participating validator (`len([bit for bit in attestation.aggregation_bits if bit == 0b1]) == 1`). - - There has been no other attestation seen on an attestation subnet that has an indentical `attestation.data.target.epoch` and participating validator index. + - There has been no other attestation seen on an attestation subnet that has an identical `attestation.data.target.epoch` and participating validator index. - The block being voted for (`attestation.data.beacon_block_root`) passes validation. - The signature of `attestation` is valid. From c96a3366fade983df68ba80850c1e409cc170c0a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 7 Apr 2020 16:07:41 +1000 Subject: [PATCH 047/102] Tighten aggregate attn propogation condition --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index c94b2c7e3..d01e4deaf 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -290,7 +290,7 @@ There are two primary global topics used to propagate beacon blocks and aggregat - `beacon_aggregate_and_proof` - This topic is used to propagate aggregated attestations (as `SignedAggregateAndProof`s) to subscribing nodes (typically validators) to be included in future blocks. The following validations MUST pass before forwarding the `signed_aggregate_and_proof` on the network. (We define the following for convenience -- `aggregate_and_proof = signed_aggregate_and_proof.message` and `aggregate = aggregate_and_proof.aggregate`) - `aggregate.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= aggregate.data.slot` (a client MAY queue future aggregates for processing at the appropriate slot). - The aggregate attestation defined by `hash_tree_root(aggregate)` has _not_ already been seen (via aggregate gossip, within a verified block, or through the creation of an equivalent aggregate locally). - - The `aggregate` is the first valid aggregate received for the aggregator with index `aggregate_and_proof.aggregator_index` for the slot `aggregate.data.slot`. + - The `aggregate` is the first valid aggregate received for the aggregator with index `aggregate_and_proof.aggregator_index` for the epoch `aggregate.data.target.epoch`. - The block being voted for (`aggregate.data.beacon_block_root`) passes validation. - `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_aggregator(state, aggregate.data.slot, aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. - The aggregator's validator index is within the aggregate's committee -- i.e. `aggregate_and_proof.aggregator_index in get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)`. From c10e59bdf7b2fad73d967061dcdc0519ef1fa06f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 8 Apr 2020 10:32:16 +0800 Subject: [PATCH 048/102] Fix `INACTIVITY_PENALTY_QUOTIENT` The amount of inactivity penalty was adjusted to half since we were applying penalty for missing FFG target and source. But now we only apply it for missing target, so `INACTIVITY_PENALTY_QUOTIENT` should be `2**24`. --- configs/mainnet.yaml | 4 ++-- configs/minimal.yaml | 4 ++-- specs/phase0/beacon-chain.md | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 6d71cfa47..3d2de75f0 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -122,8 +122,8 @@ BASE_REWARD_FACTOR: 64 WHISTLEBLOWER_REWARD_QUOTIENT: 512 # 2**3 (= 8) PROPOSER_REWARD_QUOTIENT: 8 -# 2**25 (= 33,554,432) -INACTIVITY_PENALTY_QUOTIENT: 33554432 +# 2**24 (= 16,777,216) +INACTIVITY_PENALTY_QUOTIENT: 16777216 # 2**5 (= 32) MIN_SLASHING_PENALTY_QUOTIENT: 32 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 9daf428b4..b39a4fc01 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -122,8 +122,8 @@ BASE_REWARD_FACTOR: 64 WHISTLEBLOWER_REWARD_QUOTIENT: 512 # 2**3 (= 8) PROPOSER_REWARD_QUOTIENT: 8 -# 2**25 (= 33,554,432) -INACTIVITY_PENALTY_QUOTIENT: 33554432 +# 2**24 (= 16,777,216) +INACTIVITY_PENALTY_QUOTIENT: 16777216 # 2**5 (= 32) MIN_SLASHING_PENALTY_QUOTIENT: 32 diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 23fa5ceee..c841d2dbf 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -242,10 +242,10 @@ The following values are (non-configurable) constants used throughout the specif | `BASE_REWARD_FACTOR` | `2**6` (= 64) | | `WHISTLEBLOWER_REWARD_QUOTIENT` | `2**9` (= 512) | | `PROPOSER_REWARD_QUOTIENT` | `2**3` (= 8) | -| `INACTIVITY_PENALTY_QUOTIENT` | `2**25` (= 33,554,432) | +| `INACTIVITY_PENALTY_QUOTIENT` | `2**24` (= 16,777,216) | | `MIN_SLASHING_PENALTY_QUOTIENT` | `2**5` (= 32) | -- The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**12 epochs` (about 18 days) is the time it takes the inactivity penalty to reduce the balance of non-participating validators to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline validators after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)`; so after `INVERSE_SQRT_E_DROP_TIME` epochs, it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. +- The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**12` epochs (about 18 days) is the time it takes the inactivity penalty to reduce the balance of non-participating validators to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline validators after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)`; so after `INVERSE_SQRT_E_DROP_TIME` epochs, it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. ### Max operations per block From 890c27d091bb74947ba3fdd6b86464fddf9cc1ef Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 9 Apr 2020 17:33:14 +0800 Subject: [PATCH 049/102] The input parameter `index` in `compute_shuffled_index` is the position of the given list, not `ValidatorIndex` --- specs/phase0/beacon-chain.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 23fa5ceee..8a51b5e8b 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -722,9 +722,9 @@ def is_valid_merkle_branch(leaf: Bytes32, branch: Sequence[Bytes32], depth: uint #### `compute_shuffled_index` ```python -def compute_shuffled_index(index: ValidatorIndex, index_count: uint64, seed: Bytes32) -> ValidatorIndex: +def compute_shuffled_index(index: uint64, index_count: uint64, seed: Bytes32) -> uint64: """ - Return the shuffled validator index corresponding to ``seed`` (and ``index_count``). + Return the shuffled index corresponding to ``seed`` (and ``index_count``). """ assert index < index_count @@ -732,14 +732,14 @@ def compute_shuffled_index(index: ValidatorIndex, index_count: uint64, seed: Byt # See the 'generalized domain' algorithm on page 3 for current_round in range(SHUFFLE_ROUND_COUNT): pivot = bytes_to_int(hash(seed + int_to_bytes(current_round, length=1))[0:8]) % index_count - flip = ValidatorIndex((pivot + index_count - index) % index_count) + flip = (pivot + index_count - index) % index_count position = max(index, flip) source = hash(seed + int_to_bytes(current_round, length=1) + int_to_bytes(position // 256, length=4)) byte = source[(position % 256) // 8] bit = (byte >> (position % 8)) % 2 index = flip if bit else index - return ValidatorIndex(index) + return index ``` #### `compute_proposer_index` @@ -753,11 +753,11 @@ def compute_proposer_index(state: BeaconState, indices: Sequence[ValidatorIndex] MAX_RANDOM_BYTE = 2**8 - 1 i = 0 while True: - candidate_index = indices[compute_shuffled_index(ValidatorIndex(i % len(indices)), len(indices), seed)] + candidate_index = indices[compute_shuffled_index(i % len(indices), len(indices), seed)] random_byte = hash(seed + int_to_bytes(i // 32, length=8))[i % 32] effective_balance = state.validators[candidate_index].effective_balance if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: - return ValidatorIndex(candidate_index) + return candidate_index i += 1 ``` From b2f6325db339e630feb9a459099bc8f0c5f4ce49 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 9 Apr 2020 17:48:12 +0800 Subject: [PATCH 050/102] Fix `compute_committee` --- specs/phase0/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 8a51b5e8b..82257947d 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -773,7 +773,7 @@ def compute_committee(indices: Sequence[ValidatorIndex], """ start = (len(indices) * index) // count end = (len(indices) * (index + 1)) // count - return [indices[compute_shuffled_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)] + return [indices[compute_shuffled_index(i, len(indices), seed)] for i in range(start, end)] ``` #### `compute_epoch_at_slot` From 79d6b49a904cbdba4de9b8c98eba649f381f93d4 Mon Sep 17 00:00:00 2001 From: Giuseppe Bertone Date: Fri, 10 Apr 2020 17:38:37 +0200 Subject: [PATCH 051/102] Fixed target compile_deposit_contract Path of validator_registration.vy contract was wrong --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e8f3d21bc..e53aaf8a2 100644 --- a/Makefile +++ b/Makefile @@ -117,7 +117,7 @@ install_deposit_contract_compiler: compile_deposit_contract: cd $(DEPOSIT_CONTRACT_COMPILER_DIR); . venv/bin/activate; \ - python3.7 deposit_contract/compile.py contracts/validator_registration.vy + python3.7 deposit_contract/compile.py ../contracts/validator_registration.vy test_compile_deposit_contract: cd $(DEPOSIT_CONTRACT_COMPILER_DIR); . venv/bin/activate; \ From e58cfedb6830d54ab22b08db56a1773f4b0d3797 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 16 Apr 2020 11:12:24 -0600 Subject: [PATCH 052/102] clarify ssz_snappy for gossip --- specs/phase0/p2p-interface.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index c461c2d3b..7d733e48e 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -345,7 +345,9 @@ Topics are post-fixed with an encoding. Encodings define how the payload of a go #### Mainnet -- `ssz_snappy` - All objects are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy). Example: The beacon aggregate attestation topic string is `/eth2/beacon_aggregate_and_proof/ssz_snappy`, and the data field of a gossipsub message is an `AggregateAndProof` that has been SSZ-encoded and then compressed with Snappy. +- `ssz_snappy` - All objects are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy) block compression. Example: The beacon aggregate attestation topic string is `/eth2/beacon_aggregate_and_proof/ssz_snappy`, and the data field of a gossipsub message is an `AggregateAndProof` that has been SSZ-encoded and then compressed with Snappy. + +Snappy has two formats: "block" and "frames" (streaming). Gossip messages remain relatively small (100s of bytes to 100s of kilobytes) so [basic snappy block compression](https://github.com/google/snappy/blob/master/format_description.txt) is used to avoid the additional overhead associated with snappy frames. Implementations MUST use a single encoding. Changing an encoding will require coordination between participating implementations. @@ -448,7 +450,7 @@ Here, `result` represents the 1-byte response code. The token of the negotiated protocol ID specifies the type of encoding to be used for the req/resp interaction. Two values are possible at this time: - `ssz`: the contents are [SSZ-encoded](../../ssz/simple-serialize.md). This encoding type MUST be supported by all clients. For objects containing a single field, only the field is SSZ-encoded not a container with a single field. For example, the `BeaconBlocksByRoot` request is an SSZ-encoded list of `Root`'s. -- `ssz_snappy`: The contents are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy). MAY be supported in the interoperability testnet; MUST be supported in mainnet. +- `ssz_snappy`: The contents are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy) frames compression. MAY be supported in the interoperability testnet; MUST be supported in mainnet. #### SSZ-encoding strategy (with or without Snappy) From 6fdee7547502acc8f872031cc8d6f3afb502b83c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 17 Apr 2020 17:27:57 +0800 Subject: [PATCH 053/102] Fix phase0 types --- setup.py | 10 +++++++--- specs/phase0/beacon-chain.md | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index d1c62fb72..d78161b29 100644 --- a/setup.py +++ b/setup.py @@ -108,7 +108,7 @@ SSZObject = TypeVar('SSZObject', bound=View) PHASE1_IMPORTS = '''from eth2spec.phase0 import spec as phase0 from eth2spec.config.config_util import apply_constants_config from typing import ( - Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable + Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable, Optional ) from dataclasses import ( @@ -146,8 +146,11 @@ _hash = hash hash_cache: Dict[bytes, Bytes32] = {} -def get_eth1_data(distance: uint64) -> Bytes32: - return hash(distance) +def get_eth1_data(block: Eth1Block) -> Eth1Data: + """ + A stub function return mocking Eth1Data. + """ + return Eth1Data(block_hash=hash_tree_root(block)) def hash(x: bytes) -> Bytes32: # type: ignore @@ -373,6 +376,7 @@ class PySpecCommand(Command): self.md_doc_paths = """ specs/phase0/beacon-chain.md specs/phase0/fork-choice.md + specs/phase0/validator.md specs/phase1/custody-game.md specs/phase1/beacon-chain.md specs/phase1/fraud-proofs.md diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 2ee1d46bc..5c84e41f7 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1125,7 +1125,7 @@ def slash_validator(state: BeaconState, whistleblower_reward = Gwei(validator.effective_balance // WHISTLEBLOWER_REWARD_QUOTIENT) proposer_reward = Gwei(whistleblower_reward // PROPOSER_REWARD_QUOTIENT) increase_balance(state, proposer_index, proposer_reward) - increase_balance(state, whistleblower_index, whistleblower_reward - proposer_reward) + increase_balance(state, whistleblower_index, Gwei(whistleblower_reward - proposer_reward)) ``` ## Genesis @@ -1229,7 +1229,7 @@ def process_slots(state: BeaconState, slot: Slot) -> None: # Process epoch on the start slot of the next epoch if (state.slot + 1) % SLOTS_PER_EPOCH == 0: process_epoch(state) - state.slot += Slot(1) + state.slot = Slot(state.slot + 1) ``` ```python From cafd98b9e85550acc091c797f1deb1027b802b9a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 17 Apr 2020 18:15:46 +0800 Subject: [PATCH 054/102] Fix utils.hash_function typing --- .../pyspec/eth2spec/utils/hash_function.py | 23 +++++-------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/tests/core/pyspec/eth2spec/utils/hash_function.py b/tests/core/pyspec/eth2spec/utils/hash_function.py index 2c9b5a579..627f9b990 100644 --- a/tests/core/pyspec/eth2spec/utils/hash_function.py +++ b/tests/core/pyspec/eth2spec/utils/hash_function.py @@ -1,28 +1,17 @@ from hashlib import sha256 +from typing import Dict, Union ZERO_BYTES32 = b'\x00' * 32 -def _hash(x): +def _hash(x: Union[bytes, bytearray, memoryview]) -> bytes: return sha256(x).digest() -# Minimal collection of (key, value) pairs, for fast hash-retrieval, to save on repetitive computation cost. -# Key = the hash input -# Value = the hash output -hash_cache = [] +hash_cache: Dict[bytes, bytes] = {} -def add_zero_hashes_to_cache(): - zerohashes = [(None, ZERO_BYTES32)] - for layer in range(1, 32): - k = zerohashes[layer - 1][1] + zerohashes[layer - 1][1] - zerohashes.append((k, _hash(k))) - hash_cache.extend(zerohashes[1:]) - - -def hash(x): - for (k, h) in hash_cache: - if x == k: - return h +def hash(x: bytes) -> bytes: + if x in hash_cache: + return hash_cache[x] return _hash(x) From 3575b18cd4b714427f876f646570a76df66f5f11 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 17 Apr 2020 18:18:22 +0800 Subject: [PATCH 055/102] Fix `config_util.py` typing --- tests/core/pyspec/eth2spec/config/config_util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/config/config_util.py b/tests/core/pyspec/eth2spec/config/config_util.py index 4c5768a29..c43c1521b 100644 --- a/tests/core/pyspec/eth2spec/config/config_util.py +++ b/tests/core/pyspec/eth2spec/config/config_util.py @@ -24,12 +24,12 @@ def apply_constants_config(spec_globals: Dict[str, Any], warn_if_unknown: bool = # Load presets from a file, and then prepares the global config setting. This does not apply the config. # To apply the config, reload the spec module (it will re-initialize with the config taken from here). -def prepare_config(configs_path, config_name): +def prepare_config(configs_path: str, config_name: str) -> None: global config config = load_config_file(configs_path, config_name) -def load_config_file(configs_dir, presets_name) -> Dict[str, Any]: +def load_config_file(configs_dir: str, presets_name: str) -> Dict[str, Any]: """ Loads the given preset :param presets_name: The name of the presets. (lowercase snake_case) @@ -38,7 +38,7 @@ def load_config_file(configs_dir, presets_name) -> Dict[str, Any]: path = Path(join(configs_dir, presets_name + '.yaml')) yaml = YAML(typ='base') loaded = yaml.load(path) - out = dict() + out: Dict[str, Any] = dict() for k, v in loaded.items(): if isinstance(v, list): # Clean up integer values. YAML parser renders lists of ints as list of str From 4915014a19bb8d67efac833ecb065072846e6a5d Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Mon, 20 Apr 2020 20:01:57 +0200 Subject: [PATCH 056/102] simplify block range request description There's room for ambiguity as to what `count` means - this clarifies that it always relates to the slot, and not the number of blocks in the response which allows clients to request ranges epoch by epoch (for example) without worrying about overlaps caused by empty slots. --- specs/phase0/p2p-interface.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 7d733e48e..eff6a7c4f 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -463,7 +463,7 @@ Snappy has two formats: "block" and "frames" (streaming). To support large reque Since snappy frame contents [have a maximum size of `65536` bytes](https://github.com/google/snappy/blob/master/framing_format.txt#L104) and frame headers are just `identifier (1) + checksum (4)` bytes, the expected buffering of a single frame is acceptable. -**Encoding-dependent header:** Req/Resp protocols using the `ssz` or `ssz_snappy` encoding strategies MUST encode the length of the raw SSZ bytes, encoded as an unsigned [protobuf varint](https://developers.google.com/protocol-buffers/docs/encoding#varints). +**Encoding-dependent header:** Req/Resp protocols using the `ssz` or `ssz_snappy` encoding strategies MUST encode the length of the raw SSZ bytes, encoded as an unsigned [protobuf varint](https://developers.google.com/protocol-buffers/docs/encoding#varints). *Writing*: By first computing and writing the SSZ byte length, the SSZ encoder can then directly write the chunk contents to the stream. If Snappy is applied, it can be passed through a buffered Snappy writer to compress frame by frame. @@ -577,7 +577,7 @@ Response Content: ) ``` -Requests count beacon blocks from the peer starting from `start_slot`, leading up to the current head block as selected by fork choice. `step` defines the slot increment between blocks. For example, requesting blocks starting at `start_slot` 2 with a step value of 2 would return the blocks at slots [2, 4, 6, …]. In cases where a slot is empty for a given slot number, no block is returned. For example, if slot 4 were empty in the previous example, the returned array would contain [2, 6, …]. A step value of 1 returns all blocks on the range `[start_slot, start_slot + count)`. +Requests beacon blocks in the slot range `[start_slot, start_slot + count * step)`, leading up to the current head block as selected by fork choice. `step` defines the slot increment between blocks. For example, requesting blocks starting at `start_slot` 2 with a step value of 2 would return the blocks at slots [2, 4, 6, …]. In cases where a slot is empty for a given slot number, no block is returned. For example, if slot 4 were empty in the previous example, the returned array would contain [2, 6, …]. `BeaconBlocksByRange` is primarily used to sync historical blocks. From 508811d6417640c10d4866b12965bbe9e1ed6c13 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 21 Apr 2020 08:50:42 +0100 Subject: [PATCH 057/102] =?UTF-8?q?Fix=20#1735=E2=80=94remove=20redundant?= =?UTF-8?q?=20check?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As per #1735 the check `if not len(indices) <= MAX_VALIDATORS_PER_COMMITTEE: return False` is redundant. As such this PR should be purely cosmetic. --- specs/phase0/beacon-chain.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 23fa5ceee..a093c1098 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -684,14 +684,10 @@ def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationDa ```python def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> bool: """ - Check if ``indexed_attestation`` has valid indices and signature. + Check if ``indexed_attestation`` has sorted and unique indices and a valid aggregate signature. """ - indices = indexed_attestation.attesting_indices - - # Verify max number of indices - if not len(indices) <= MAX_VALIDATORS_PER_COMMITTEE: - return False # Verify indices are sorted and unique + indices = indexed_attestation.attesting_indices if not indices == sorted(set(indices)): return False # Verify aggregate signature From e2a320ef32f0a7567a7d62008a35c2c4bcd9c312 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 21 Apr 2020 08:59:53 +0100 Subject: [PATCH 058/102] Partial fix for #1701 Clarify that state transitions with `uint64` overflows are invalid. --- specs/phase0/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 23fa5ceee..01bf36c99 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1195,7 +1195,7 @@ Let `genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state))`. ## Beacon chain state transition function -The post-state corresponding to a pre-state `state` and a signed block `signed_block` is defined as `state_transition(state, signed_block)`. State transitions that trigger an unhandled exception (e.g. a failed `assert` or an out-of-range list access) are considered invalid. +The post-state corresponding to a pre-state `state` and a signed block `signed_block` is defined as `state_transition(state, signed_block)`. State transitions that trigger an unhandled exception (e.g. a failed `assert` or an out-of-range list access) are considered invalid. State transitions that cause a `uint64` overflow are also considered invalid. ```python def state_transition(state: BeaconState, signed_block: SignedBeaconBlock, validate_result: bool=True) -> BeaconState: From 3436021e72019349f64910822ef862cddac086d4 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 21 Apr 2020 15:34:55 +0100 Subject: [PATCH 059/102] Update beacon-chain.md --- specs/phase0/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 01bf36c99..87b207686 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1195,7 +1195,7 @@ Let `genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state))`. ## Beacon chain state transition function -The post-state corresponding to a pre-state `state` and a signed block `signed_block` is defined as `state_transition(state, signed_block)`. State transitions that trigger an unhandled exception (e.g. a failed `assert` or an out-of-range list access) are considered invalid. State transitions that cause a `uint64` overflow are also considered invalid. +The post-state corresponding to a pre-state `state` and a signed block `signed_block` is defined as `state_transition(state, signed_block)`. State transitions that trigger an unhandled exception (e.g. a failed `assert` or an out-of-range list access) are considered invalid. State transitions that cause a `uint64` overflow or underflow are also considered invalid. ```python def state_transition(state: BeaconState, signed_block: SignedBeaconBlock, validate_result: bool=True) -> BeaconState: From 5929aac799e8647b6905dd829948efcdfa5849c9 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 21 Apr 2020 17:56:27 +0100 Subject: [PATCH 060/102] Cosmetic changes from #1737 --- specs/phase0/beacon-chain.md | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 5c84e41f7..111655285 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -35,7 +35,7 @@ - [`DepositMessage`](#depositmessage) - [`DepositData`](#depositdata) - [`BeaconBlockHeader`](#beaconblockheader) - - [`SigningRoot`](#signingroot) + - [`SigningData`](#signingdata) - [Beacon operations](#beacon-operations) - [`ProposerSlashing`](#proposerslashing) - [`AttesterSlashing`](#attesterslashing) @@ -191,7 +191,6 @@ The following values are (non-configurable) constants used throughout the specif | `HYSTERESIS_DOWNWARD_MULTIPLIER` | `1` | | `HYSTERESIS_UPWARD_MULTIPLIER` | `5` | - - For the safety of committees, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](http://web.archive.org/web/20190504131341/https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) ### Gwei values @@ -269,7 +268,6 @@ The following values are (non-configurable) constants used throughout the specif | `DOMAIN_SELECTION_PROOF` | `DomainType('0x05000000')` | | `DOMAIN_AGGREGATE_AND_PROOF` | `DomainType('0x06000000')` | - ## Containers The following types are [SimpleSerialize (SSZ)](../../ssz/simple-serialize.md) containers. @@ -399,10 +397,10 @@ class BeaconBlockHeader(Container): body_root: Root ``` -#### `SigningRoot` +#### `SigningData` ```python -class SigningRoot(Container): +class SigningData(Container): object_root: Root domain: Domain ``` @@ -852,13 +850,12 @@ def compute_domain(domain_type: DomainType, fork_version: Version=None, genesis_ ```python def compute_signing_root(ssz_object: SSZObject, domain: Domain) -> Root: """ - Return the signing root of an object by calculating the root of the object-domain tree. + Return the signing root for the corresponding signing data. """ - domain_wrapped_object = SigningRoot( + return hash_tree_root(SigningData( object_root=hash_tree_root(ssz_object), domain=domain, - ) - return hash_tree_root(domain_wrapped_object) + )) ``` ### Beacon state accessors From 3d4122a2f6010d393f52f63d62068ce0641068b9 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 16 Apr 2020 11:56:15 -0600 Subject: [PATCH 061/102] add note about distributing bootnode ENRs prior to genesis --- .circleci/config.yml | 22 +++++++++++----------- specs/phase0/p2p-interface.md | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5c4b77e78..3a67e5528 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,16 +79,16 @@ jobs: # Restore git repo at point close to target branch/revision, to speed up checkout - restore_cache: keys: - - v2-specs-repo-{{ .Branch }}-{{ .Revision }} - - v2-specs-repo-{{ .Branch }}- - - v2-specs-repo- + - v3-specs-repo-{{ .Branch }}-{{ .Revision }} + - v3-specs-repo-{{ .Branch }}- + - v3-specs-repo- - checkout - run: name: Clean up git repo to reduce cache size command: git gc # Save the git checkout as a cache, to make cloning next time faster. - save_cache: - key: v2-specs-repo-{{ .Branch }}-{{ .Revision }} + key: v3-specs-repo-{{ .Branch }}-{{ .Revision }} paths: - ~/specs-repo install_pyspec_test: @@ -97,7 +97,7 @@ jobs: working_directory: ~/specs-repo steps: - restore_cache: - key: v2-specs-repo-{{ .Branch }}-{{ .Revision }} + key: v3-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_pyspec_cached_venv - run: name: Install pyspec requirements @@ -109,7 +109,7 @@ jobs: working_directory: ~/specs-repo steps: - restore_cache: - key: v2-specs-repo-{{ .Branch }}-{{ .Revision }} + key: v3-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_pyspec_cached_venv - run: name: Run py-tests @@ -140,7 +140,7 @@ jobs: working_directory: ~/specs-repo steps: - restore_cache: - key: v2-specs-repo-{{ .Branch }}-{{ .Revision }} + key: v3-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_pyspec_cached_venv - run: name: Run linter @@ -152,7 +152,7 @@ jobs: working_directory: ~/specs-repo steps: - restore_cache: - key: v2-specs-repo-{{ .Branch }}-{{ .Revision }} + key: v3-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_deposit_contract_compiler_cached_venv - run: name: Install deposit contract compiler requirements @@ -164,7 +164,7 @@ jobs: working_directory: ~/specs-repo steps: - restore_cache: - key: v2-specs-repo-{{ .Branch }}-{{ .Revision }} + key: v3-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_deposit_contract_tester_cached_venv - run: name: Install deposit contract tester requirements @@ -176,7 +176,7 @@ jobs: working_directory: ~/specs-repo steps: - restore_cache: - key: v2-specs-repo-{{ .Branch }}-{{ .Revision }} + key: v3-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_deposit_contract_compiler_cached_venv - run: name: Run deposit contract compile test @@ -187,7 +187,7 @@ jobs: working_directory: ~/specs-repo steps: - restore_cache: - key: v2-specs-repo-{{ .Branch }}-{{ .Revision }} + key: v3-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_deposit_contract_tester_cached_venv - run: name: Run deposit contract test diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index c461c2d3b..9573eef11 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -755,7 +755,7 @@ where the fields of `ENRForkID` are defined as * `next_fork_version` is the fork version corresponding to the next planned hard fork at a future epoch. If no future fork is planned, set `next_fork_version = current_fork_version` to signal this fact * `next_fork_epoch` is the epoch at which the next fork is planned and the `current_fork_version` will be updated. If no future fork is planned, set `next_fork_epoch = FAR_FUTURE_EPOCH` to signal this fact -*Note*: `fork_digest` is composed of values that are not not known until the genesis block/state are available. Due to this, clients SHOULD NOT form ENRs and begin peer discovery until genesis values are known. +*Note*: `fork_digest` is composed of values that are not not known until the genesis block/state are available. Due to this, clients SHOULD NOT form ENRs and begin peer discovery until genesis values are known. One notable exception to this rule is the distribution of bootnode ENRs prior to genesis. In this case, bootnode ENRs SHOULD be initially distributed with `eth2` field set as `ENRForkID(fork_digest=compute_fork_digest(GENESIS_FORK_VERSION, b'\x00'*32), next_fork_version=GENESIS_FORK_VERSION, next_fork_epoch=FAR_FUTURE_EPOCH)`. After genesis values are known, the bootnodes SHOULD update ENRs to participate in normal discovery operations. Clients SHOULD connect to peers with `fork_digest`, `next_fork_version`, and `next_fork_epoch` that match local values. From 11d164748c58580a34edb798a91387b720e45701 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 22 Apr 2020 14:45:01 -0600 Subject: [PATCH 062/102] add 'valid' when de-deduplication of attestations on gossip subnets --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index c66ec0b6e..d3b9150d1 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -319,7 +319,7 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio - The attestation's committee index (`attestation.data.index`) is for the correct subnet. - `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` (a client MAY queue future attestations for processing at the appropriate slot). - The attestation is unaggregated -- that is, it has exactly one participating validator (`len([bit for bit in attestation.aggregation_bits if bit == 0b1]) == 1`). - - There has been no other attestation seen on an attestation subnet that has an identical `attestation.data.target.epoch` and participating validator index. + - There has been no other valid attestation seen on an attestation subnet that has an identical `attestation.data.target.epoch` and participating validator index. - The block being voted for (`attestation.data.beacon_block_root`) passes validation. - The signature of `attestation` is valid. From bf806b9efaebf957d1399adac4bd839a58db537c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 24 Apr 2020 15:01:18 +1000 Subject: [PATCH 063/102] Require "seen" aggregates to be valid --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index d3b9150d1..92194920b 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -289,7 +289,7 @@ There are two primary global topics used to propagate beacon blocks and aggregat - The block is proposed by the expected `proposer_index` for the block's slot in the context of the current shuffling (defined by `parent_root`/`slot`). If the `proposer_index` cannot immediately be verified against the expected shuffling, the block MAY be queued for later processing while proposers for the block's branch are calculated. - `beacon_aggregate_and_proof` - This topic is used to propagate aggregated attestations (as `SignedAggregateAndProof`s) to subscribing nodes (typically validators) to be included in future blocks. The following validations MUST pass before forwarding the `signed_aggregate_and_proof` on the network. (We define the following for convenience -- `aggregate_and_proof = signed_aggregate_and_proof.message` and `aggregate = aggregate_and_proof.aggregate`) - `aggregate.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= aggregate.data.slot` (a client MAY queue future aggregates for processing at the appropriate slot). - - The aggregate attestation defined by `hash_tree_root(aggregate)` has _not_ already been seen (via aggregate gossip, within a verified block, or through the creation of an equivalent aggregate locally). + - The valid aggregate attestation defined by `hash_tree_root(aggregate)` has _not_ already been seen (via aggregate gossip, within a verified block, or through the creation of an equivalent aggregate locally). - The `aggregate` is the first valid aggregate received for the aggregator with index `aggregate_and_proof.aggregator_index` for the epoch `aggregate.data.target.epoch`. - The block being voted for (`aggregate.data.beacon_block_root`) passes validation. - `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_aggregator(state, aggregate.data.slot, aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. From 56535e3dbe6127728e37fb6c871e790eaa6dc96b Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 24 Apr 2020 10:21:47 -0600 Subject: [PATCH 064/102] bump version to v0.11.2 --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index 027934ea1..a8839f70d 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -0.11.1 \ No newline at end of file +0.11.2 \ No newline at end of file From 2129f8a281ce01e722c1e7026ee938b35c64ac8c Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 24 Apr 2020 16:00:06 -0600 Subject: [PATCH 065/102] fix requirements.txt for bls gens --- tests/generators/bls/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/generators/bls/requirements.txt b/tests/generators/bls/requirements.txt index 7b1d2f4a9..24ea127c4 100644 --- a/tests/generators/bls/requirements.txt +++ b/tests/generators/bls/requirements.txt @@ -1,3 +1,4 @@ py_ecc==2.0.0 eth-utils==1.6.0 ../../core/gen_helpers +../../../ From 0c67aaa68e83d0050d4791b82d1dfce3943eef09 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 25 Apr 2020 00:05:37 +0200 Subject: [PATCH 066/102] Include fork digest in example gossip topic name --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index d3b9150d1..7197581dc 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -345,7 +345,7 @@ Topics are post-fixed with an encoding. Encodings define how the payload of a go #### Mainnet -- `ssz_snappy` - All objects are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy) block compression. Example: The beacon aggregate attestation topic string is `/eth2/beacon_aggregate_and_proof/ssz_snappy`, and the data field of a gossipsub message is an `AggregateAndProof` that has been SSZ-encoded and then compressed with Snappy. +- `ssz_snappy` - All objects are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy) block compression. Example: The beacon aggregate attestation topic string is `/eth2/446a7232/beacon_aggregate_and_proof/ssz_snappy`, the fork digest is `446a7232` and the data field of a gossipsub message is an `AggregateAndProof` that has been SSZ-encoded and then compressed with Snappy. Snappy has two formats: "block" and "frames" (streaming). Gossip messages remain relatively small (100s of bytes to 100s of kilobytes) so [basic snappy block compression](https://github.com/google/snappy/blob/master/format_description.txt) is used to avoid the additional overhead associated with snappy frames. From 1a81c873af99397d4f0cc69eb2a462d037d01b54 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 26 Apr 2020 16:24:16 +1000 Subject: [PATCH 067/102] Remove redundant check in fork choice --- specs/phase0/fork-choice.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index 18c7a1580..35e2c5f56 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -278,8 +278,6 @@ def validate_on_attestation(store: Store, attestation: Attestation) -> None: # Attestations target be for a known block. If target block is unknown, delay consideration until the block is found assert target.root in store.blocks - # Attestations cannot be from future epochs. If they are, delay consideration until the epoch arrives - assert get_current_slot(store) >= compute_start_slot_at_epoch(target.epoch) # Attestations must be for a known block. If block is unknown, delay consideration until the block is found assert attestation.data.beacon_block_root in store.blocks From c841aa102bad17c0e06d0c1bb033d44591cad619 Mon Sep 17 00:00:00 2001 From: Raw Pong Ghmoa <58883403+q9f@users.noreply.github.com> Date: Sun, 26 Apr 2020 10:09:22 +0200 Subject: [PATCH 068/102] genesis: clarify that eth1 timestamp can be less than min genesis time --- specs/phase0/beacon-chain.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index cdf38dc1e..3ef4081ce 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1170,6 +1170,8 @@ def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32, return state ``` +*Note*: The ETH1 block with `eth1_timestamp` meeting the minimum genesis active validator count criteria can also occur before `MIN_GENESIS_TIME`. + ### Genesis state Let `genesis_state = candidate_state` whenever `is_valid_genesis_state(candidate_state) is True` for the first time. From 4d980aec71f93f2f788fc8ea76bec14521d53abc Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 23 Apr 2020 13:23:31 +0800 Subject: [PATCH 069/102] Fix validator guide 1. Avoid negative computation in `is_candidate_block` 2. Fix `get_block_signature`: avoid extra casting; it's simpler to use BeaconBlock instead of BeaconHeader --- specs/phase0/validator.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index bc7510403..4576c4b90 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -281,8 +281,8 @@ def voting_period_start_time(state: BeaconState) -> uint64: ```python def is_candidate_block(block: Eth1Block, period_start: uint64) -> bool: return ( - block.timestamp <= period_start - SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE - and block.timestamp >= period_start - SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE * 2 + block.timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE <= period_start + and block.timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE * 2 >= period_start ) ``` @@ -350,9 +350,9 @@ def compute_new_state_root(state: BeaconState, block: BeaconBlock) -> Root: `signed_block = SignedBeaconBlock(message=block, signature=block_signature)`, where `block_signature` is obtained from: ```python -def get_block_signature(state: BeaconState, header: BeaconBlockHeader, privkey: int) -> BLSSignature: - domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(header.slot)) - signing_root = compute_signing_root(header, domain) +def get_block_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(block.slot)) + signing_root = compute_signing_root(block, domain) return bls.Sign(privkey, signing_root) ``` From 303d7d5adb35e51a49c570e15256763c136de8a1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 23 Apr 2020 13:27:00 +0800 Subject: [PATCH 070/102] Add validator guide tests 1. "Becoming a validator" 2. "Validator assignments" 3. "Beacon chain responsibilities: Block proposal" --- .../test/validator/test_validator_unittest.py | 182 ++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py diff --git a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py new file mode 100644 index 000000000..019221eee --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py @@ -0,0 +1,182 @@ +from eth2spec.test.context import spec_state_test, never_bls, with_all_phases +from eth2spec.test.helpers.block import build_empty_block +from eth2spec.test.helpers.deposits import prepare_state_and_deposit +from eth2spec.test.helpers.keys import privkeys, pubkeys +from eth2spec.test.helpers.state import next_epoch +from eth2spec.utils import bls + + +def run_is_candidate_block(spec, eth1_block, period_start, success): + result = spec.is_candidate_block(eth1_block, period_start) + if success: + assert result + else: + assert not result + + +def get_min_new_period_epochs(spec): + return int( + spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE * 2 # to seconds + / spec.SECONDS_PER_SLOT / spec.SLOTS_PER_EPOCH + ) + + +# +# Becoming a validator +# + + +@with_all_phases +@spec_state_test +@never_bls +def test_check_if_validator_active(spec, state): + active_validator_index = len(state.validators) - 1 + assert spec.check_if_validator_active(state, active_validator_index) + new_validator_index = len(state.validators) + amount = spec.MAX_EFFECTIVE_BALANCE + deposit = prepare_state_and_deposit(spec, state, new_validator_index, amount, signed=True) + spec.process_deposit(state, deposit) + assert not spec.check_if_validator_active(state, new_validator_index) + + +# +# Validator assignments +# + + +@with_all_phases +@spec_state_test +@never_bls +def test_get_committee_assignment(spec, state): + epoch = spec.get_current_epoch(state) + validator_index = len(state.validators) - 1 + assignment = spec.get_committee_assignment(state, epoch, validator_index) + committee, committee_index, slot = assignment + assert spec.compute_epoch_at_slot(slot) == epoch + assert committee == spec.get_beacon_committee(state, slot, committee_index) + assert committee_index < spec.get_committee_count_at_slot(state, slot) + + +@with_all_phases +@spec_state_test +@never_bls +def test_is_proposer(spec, state): + proposer_index = spec.get_beacon_proposer_index(state) + assert spec.is_proposer(state, proposer_index) + + proposer_index = proposer_index + 1 % len(state.validators) + assert not spec.is_proposer(state, proposer_index) + + +# +# Beacon chain responsibilities +# + + +# Block proposal + + +@with_all_phases +@spec_state_test +def test_get_epoch_signature(spec, state): + block = spec.BeaconBlock() + privkey = privkeys[0] + pubkey = pubkeys[0] + signature = spec.get_epoch_signature(state, block, privkey) + domain = spec.get_domain(state, spec.DOMAIN_RANDAO, spec.compute_epoch_at_slot(block.slot)) + signing_root = spec.compute_signing_root(spec.compute_epoch_at_slot(block.slot), domain) + assert bls.Verify(pubkey, signing_root, signature) + + +@with_all_phases +@spec_state_test +def test_is_candidate_block(spec, state): + period_start = spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE * 2 + 1000 + run_is_candidate_block( + spec, + spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE), + period_start, + success=True, + ) + run_is_candidate_block( + spec, + spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE + 1), + period_start, + success=False, + ) + run_is_candidate_block( + spec, + spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE * 2), + period_start, + success=True, + ) + run_is_candidate_block( + spec, + spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE * 2 - 1), + period_start, + success=False, + ) + + +@with_all_phases +@spec_state_test +def test_get_eth1_data_default_vote(spec, state): + min_new_period_epochs = get_min_new_period_epochs(spec) + for _ in range(min_new_period_epochs): + next_epoch(spec, state) + + state.eth1_data_votes = () + eth1_chain = [] + eth1_data = spec.get_eth1_vote(state, eth1_chain) + assert eth1_data == state.eth1_data + + +@with_all_phases +@spec_state_test +def test_get_eth1_data_consensus_vote(spec, state): + min_new_period_epochs = get_min_new_period_epochs(spec) + for _ in range(min_new_period_epochs): + next_epoch(spec, state) + + period_start = spec.voting_period_start_time(state) + votes_length = spec.get_current_epoch(state) % spec.EPOCHS_PER_ETH1_VOTING_PERIOD + state.eth1_data_votes = () + eth1_chain = [] + eth1_data_votes = [] + block = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE) + for i in range(votes_length): + eth1_chain.append(block) + eth1_data_votes.append(spec.get_eth1_data(block)) + + state.eth1_data_votes = eth1_data_votes + eth1_data = spec.get_eth1_vote(state, eth1_chain) + print(state.eth1_data_votes) + assert eth1_data.block_hash == block.hash_tree_root() + + +@with_all_phases +@spec_state_test +def test_compute_new_state_root(spec, state): + pre_state = state.copy() + post_state = state.copy() + block = build_empty_block(spec, state, state.slot + 1) + state_root = spec.compute_new_state_root(state, block) + + assert state_root != pre_state.hash_tree_root() + + # dumb verification + spec.process_slots(post_state, block.slot) + spec.process_block(post_state, block) + assert state_root == post_state.hash_tree_root() + + +@with_all_phases +@spec_state_test +def test_get_block_signature(spec, state): + privkey = privkeys[0] + pubkey = pubkeys[0] + block = build_empty_block(spec, state) + signature = spec.get_block_signature(state, block, privkey) + domain = spec.get_domain(state, spec.DOMAIN_BEACON_PROPOSER, spec.compute_epoch_at_slot(block.slot)) + signing_root = spec.compute_signing_root(block, domain) + assert bls.Verify(pubkey, signing_root, signature) From bdae27e317b44c1ab031a0cbdb5b4d61d0fe05a7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 23 Apr 2020 15:08:36 +0800 Subject: [PATCH 071/102] Add bls.AggregatePKs helper --- tests/core/pyspec/eth2spec/utils/bls.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/core/pyspec/eth2spec/utils/bls.py b/tests/core/pyspec/eth2spec/utils/bls.py index 83371ac62..3b648fac9 100644 --- a/tests/core/pyspec/eth2spec/utils/bls.py +++ b/tests/core/pyspec/eth2spec/utils/bls.py @@ -51,3 +51,8 @@ def Sign(SK, message): @only_with_bls(alt_return=STUB_COORDINATES) def signature_to_G2(signature): return _signature_to_G2(signature) + + +@only_with_bls(alt_return=STUB_PUBKEY) +def AggregatePKs(pubkeys): + return bls._AggregatePKs(pubkeys) From 8adc15e83de9ab6c0ac10b93aa739df2be989704 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 23 Apr 2020 15:09:23 +0800 Subject: [PATCH 072/102] Add validator guide tests 1. "Beacon chain responsibilities: Attesting" 2. "Beacon chain responsibilities: Attestation aggregation" --- .../test/validator/test_validator_unittest.py | 118 +++++++++++++++++- 1 file changed, 116 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py index 019221eee..4f6697f2f 100644 --- a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py +++ b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py @@ -1,9 +1,11 @@ from eth2spec.test.context import spec_state_test, never_bls, with_all_phases +from eth2spec.test.helpers.attestations import build_attestation_data from eth2spec.test.helpers.block import build_empty_block from eth2spec.test.helpers.deposits import prepare_state_and_deposit from eth2spec.test.helpers.keys import privkeys, pubkeys from eth2spec.test.helpers.state import next_epoch from eth2spec.utils import bls +from eth2spec.utils.ssz.ssz_typing import Bitlist def run_is_candidate_block(spec, eth1_block, period_start, success): @@ -21,6 +23,14 @@ def get_min_new_period_epochs(spec): ) +def get_mock_aggregate(spec): + return spec.Attestation( + data=spec.AttestationData( + slot=10, + ) + ) + + # # Becoming a validator # @@ -47,7 +57,7 @@ def test_check_if_validator_active(spec, state): @with_all_phases @spec_state_test @never_bls -def test_get_committee_assignment(spec, state): +def test_get_committee_assignment_current_epoch(spec, state): epoch = spec.get_current_epoch(state) validator_index = len(state.validators) - 1 assignment = spec.get_committee_assignment(state, epoch, validator_index) @@ -150,7 +160,6 @@ def test_get_eth1_data_consensus_vote(spec, state): state.eth1_data_votes = eth1_data_votes eth1_data = spec.get_eth1_vote(state, eth1_chain) - print(state.eth1_data_votes) assert eth1_data.block_hash == block.hash_tree_root() @@ -180,3 +189,108 @@ def test_get_block_signature(spec, state): domain = spec.get_domain(state, spec.DOMAIN_BEACON_PROPOSER, spec.compute_epoch_at_slot(block.slot)) signing_root = spec.compute_signing_root(block, domain) assert bls.Verify(pubkey, signing_root, signature) + + +# Attesting + + +@with_all_phases +@spec_state_test +def test_get_attestation_signature(spec, state): + privkey = privkeys[0] + pubkey = pubkeys[0] + attestation_data = spec.AttestationData(slot=10) + signature = spec.get_attestation_signature(state, attestation_data, privkey) + domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) + signing_root = spec.compute_signing_root(attestation_data, domain) + assert bls.Verify(pubkey, signing_root, signature) + + +# Attestation aggregation + + +@with_all_phases +@spec_state_test +def test_get_slot_signature(spec, state): + privkey = privkeys[0] + pubkey = pubkeys[0] + slot = 10 + signature = spec.get_slot_signature(state, spec.Slot(slot), privkey) + domain = spec.get_domain(state, spec.DOMAIN_SELECTION_PROOF, spec.compute_epoch_at_slot(slot)) + signing_root = spec.compute_signing_root(spec.Slot(slot), domain) + assert bls.Verify(pubkey, signing_root, signature) + + +@with_all_phases +@spec_state_test +def test_is_aggregator(spec, state): + # TODO: we can test the probabilistic result against `TARGET_AGGREGATORS_PER_COMMITTEE` + # if we have more validators and larger committeee size + slot = state.slot + committee_index = 0 + has_aggregator = False + beacon_committee = spec.get_beacon_committee(state, slot, committee_index) + for validator_index in beacon_committee: + privkey = privkeys[validator_index] + slot_signature = spec.get_slot_signature(state, slot, privkey) + if spec.is_aggregator(state, slot, committee_index, slot_signature): + has_aggregator = True + break + assert has_aggregator + + +@with_all_phases +@spec_state_test +def test_get_aggregate_signature(spec, state): + attestations = [] + pubkeys = [] + slot = state.slot + committee_index = 0 + attestation_data = build_attestation_data(spec, state, slot=slot, index=committee_index) + beacon_committee = spec.get_beacon_committee( + state, + attestation_data.slot, + attestation_data.index, + ) + committee_size = len(beacon_committee) + aggregation_bits = Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE](*([0] * committee_size)) + for i, validator_index in enumerate(beacon_committee): + bits = aggregation_bits + bits[i] = True + attestations.append( + spec.Attestation( + data=attestation_data, + aggregation_bits=bits, + ) + ) + pubkeys.append(state.validators[validator_index].pubkey) + pubkey = bls.AggregatePKs(pubkeys) + signature = spec.get_aggregate_signature(attestations) + domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) + signing_root = spec.compute_signing_root(attestation_data, domain) + assert bls.Verify(pubkey, signing_root, signature) + + +@with_all_phases +@spec_state_test +def test_get_aggregate_and_proof(spec, state): + privkey = privkeys[0] + aggregator_index = spec.ValidatorIndex(10) + aggregate = get_mock_aggregate(spec) + aggregate_and_proof = spec.get_aggregate_and_proof(state, aggregator_index, aggregate, privkey) + assert aggregate_and_proof.aggregator_index == aggregator_index + assert aggregate_and_proof.aggregate == aggregate + assert aggregate_and_proof.selection_proof == spec.get_slot_signature(state, aggregate.data.slot, privkey) + + +@with_all_phases +@spec_state_test +def test_get_aggregate_and_proof_signature(spec, state): + privkey = privkeys[0] + pubkey = pubkeys[0] + aggregate = get_mock_aggregate(spec) + aggregate_and_proof = spec.get_aggregate_and_proof(state, spec.ValidatorIndex(1), aggregate, privkey) + signature = spec.get_aggregate_and_proof_signature(state, aggregate_and_proof, privkey) + domain = spec.get_domain(state, spec.DOMAIN_AGGREGATE_AND_PROOF, spec.compute_epoch_at_slot(aggregate.data.slot)) + signing_root = spec.compute_signing_root(aggregate_and_proof, domain) + assert bls.Verify(pubkey, signing_root, signature) From 70bd73d2b5f4b8c3f19a9397fd983c23fd1eecc6 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 27 Apr 2020 20:47:36 +0800 Subject: [PATCH 073/102] Apply PR feedback from @djrtwo Fix get_eth1_vote test cases --- .../test/validator/test_validator_unittest.py | 164 ++++++++++++++---- 1 file changed, 131 insertions(+), 33 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py index 4f6697f2f..a655cb486 100644 --- a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py +++ b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py @@ -8,12 +8,29 @@ from eth2spec.utils import bls from eth2spec.utils.ssz.ssz_typing import Bitlist -def run_is_candidate_block(spec, eth1_block, period_start, success): - result = spec.is_candidate_block(eth1_block, period_start) - if success: - assert result +def run_get_signature_test(spec, state, obj, domain, get_signature_fn, privkey, pubkey): + signature = get_signature_fn(state, obj, privkey) + signing_root = spec.compute_signing_root(obj, domain) + assert bls.Verify(pubkey, signing_root, signature) + + +def run_get_committee_assignment(spec, state, epoch, validator_index, valid=True): + try: + assignment = spec.get_committee_assignment(state, epoch, validator_index) + committee, committee_index, slot = assignment + assert spec.compute_epoch_at_slot(slot) == epoch + assert committee == spec.get_beacon_committee(state, slot, committee_index) + assert committee_index < spec.get_committee_count_at_slot(state, slot) + assert validator_index in committee + assert valid + except AssertionError: + assert not valid else: - assert not result + assert valid + + +def run_is_candidate_block(spec, eth1_block, period_start, success=True): + assert success == spec.is_candidate_block(eth1_block, period_start) def get_min_new_period_epochs(spec): @@ -60,11 +77,25 @@ def test_check_if_validator_active(spec, state): def test_get_committee_assignment_current_epoch(spec, state): epoch = spec.get_current_epoch(state) validator_index = len(state.validators) - 1 - assignment = spec.get_committee_assignment(state, epoch, validator_index) - committee, committee_index, slot = assignment - assert spec.compute_epoch_at_slot(slot) == epoch - assert committee == spec.get_beacon_committee(state, slot, committee_index) - assert committee_index < spec.get_committee_count_at_slot(state, slot) + run_get_committee_assignment(spec, state, epoch, validator_index, valid=True) + + +@with_all_phases +@spec_state_test +@never_bls +def test_get_committee_assignment_next_epoch(spec, state): + epoch = spec.get_current_epoch(state) + 1 + validator_index = len(state.validators) - 1 + run_get_committee_assignment(spec, state, epoch, validator_index, valid=True) + + +@with_all_phases +@spec_state_test +@never_bls +def test_get_committee_assignment_out_bound_epoch(spec, state): + epoch = spec.get_current_epoch(state) + 2 + validator_index = len(state.validators) - 1 + run_get_committee_assignment(spec, state, epoch, validator_index, valid=False) @with_all_phases @@ -92,10 +123,16 @@ def test_get_epoch_signature(spec, state): block = spec.BeaconBlock() privkey = privkeys[0] pubkey = pubkeys[0] - signature = spec.get_epoch_signature(state, block, privkey) domain = spec.get_domain(state, spec.DOMAIN_RANDAO, spec.compute_epoch_at_slot(block.slot)) - signing_root = spec.compute_signing_root(spec.compute_epoch_at_slot(block.slot), domain) - assert bls.Verify(pubkey, signing_root, signature) + run_get_signature_test( + spec=spec, + state=state, + obj=block, + domain=domain, + get_signature_fn=spec.get_epoch_signature, + privkey=privkey, + pubkey=pubkey, + ) @with_all_phases @@ -130,7 +167,7 @@ def test_is_candidate_block(spec, state): @with_all_phases @spec_state_test -def test_get_eth1_data_default_vote(spec, state): +def test_get_eth1_vote_default_vote(spec, state): min_new_period_epochs = get_min_new_period_epochs(spec) for _ in range(min_new_period_epochs): next_epoch(spec, state) @@ -143,24 +180,61 @@ def test_get_eth1_data_default_vote(spec, state): @with_all_phases @spec_state_test -def test_get_eth1_data_consensus_vote(spec, state): +def test_get_eth1_vote_consensus_vote(spec, state): min_new_period_epochs = get_min_new_period_epochs(spec) - for _ in range(min_new_period_epochs): + for _ in range(min_new_period_epochs + 2): next_epoch(spec, state) period_start = spec.voting_period_start_time(state) votes_length = spec.get_current_epoch(state) % spec.EPOCHS_PER_ETH1_VOTING_PERIOD + assert votes_length >= 3 # We need to have the majority vote state.eth1_data_votes = () - eth1_chain = [] + + block_1 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE - 1) + block_2 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE) + eth1_chain = [block_1, block_2] eth1_data_votes = [] - block = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE) + + # Only the first vote is for block_1 + eth1_data_votes.append(spec.get_eth1_data(block_1)) + # Other votes are for block_2 + for _ in range(votes_length - 1): + eth1_data_votes.append(spec.get_eth1_data(block_2)) + + state.eth1_data_votes = eth1_data_votes + eth1_data = spec.get_eth1_vote(state, eth1_chain) + assert eth1_data.block_hash == block_2.hash_tree_root() + + +@with_all_phases +@spec_state_test +def test_get_eth1_vote_tie(spec, state): + min_new_period_epochs = get_min_new_period_epochs(spec) + for _ in range(min_new_period_epochs + 1): + next_epoch(spec, state) + + period_start = spec.voting_period_start_time(state) + votes_length = spec.get_current_epoch(state) % spec.EPOCHS_PER_ETH1_VOTING_PERIOD + assert votes_length > 0 and votes_length % 2 == 0 + + state.eth1_data_votes = () + block_1 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE - 1) + block_2 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE) + eth1_chain = [block_1, block_2] + eth1_data_votes = [] + # Half votes are for block_1, another half votes are for block_2 for i in range(votes_length): - eth1_chain.append(block) + if i % 2 == 0: + block = block_1 + else: + block = block_2 eth1_data_votes.append(spec.get_eth1_data(block)) state.eth1_data_votes = eth1_data_votes eth1_data = spec.get_eth1_vote(state, eth1_chain) - assert eth1_data.block_hash == block.hash_tree_root() + + # Tiebreak by smallest distance -> eth1_chain[0] + assert eth1_data.block_hash == eth1_chain[0].hash_tree_root() @with_all_phases @@ -185,10 +259,16 @@ def test_get_block_signature(spec, state): privkey = privkeys[0] pubkey = pubkeys[0] block = build_empty_block(spec, state) - signature = spec.get_block_signature(state, block, privkey) domain = spec.get_domain(state, spec.DOMAIN_BEACON_PROPOSER, spec.compute_epoch_at_slot(block.slot)) - signing_root = spec.compute_signing_root(block, domain) - assert bls.Verify(pubkey, signing_root, signature) + run_get_signature_test( + spec=spec, + state=state, + obj=block, + domain=domain, + get_signature_fn=spec.get_block_signature, + privkey=privkey, + pubkey=pubkey, + ) # Attesting @@ -200,10 +280,16 @@ def test_get_attestation_signature(spec, state): privkey = privkeys[0] pubkey = pubkeys[0] attestation_data = spec.AttestationData(slot=10) - signature = spec.get_attestation_signature(state, attestation_data, privkey) domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) - signing_root = spec.compute_signing_root(attestation_data, domain) - assert bls.Verify(pubkey, signing_root, signature) + run_get_signature_test( + spec=spec, + state=state, + obj=attestation_data, + domain=domain, + get_signature_fn=spec.get_attestation_signature, + privkey=privkey, + pubkey=pubkey, + ) # Attestation aggregation @@ -214,11 +300,17 @@ def test_get_attestation_signature(spec, state): def test_get_slot_signature(spec, state): privkey = privkeys[0] pubkey = pubkeys[0] - slot = 10 - signature = spec.get_slot_signature(state, spec.Slot(slot), privkey) + slot = spec.Slot(10) domain = spec.get_domain(state, spec.DOMAIN_SELECTION_PROOF, spec.compute_epoch_at_slot(slot)) - signing_root = spec.compute_signing_root(spec.Slot(slot), domain) - assert bls.Verify(pubkey, signing_root, signature) + run_get_signature_test( + spec=spec, + state=state, + obj=slot, + domain=domain, + get_signature_fn=spec.get_slot_signature, + privkey=privkey, + pubkey=pubkey, + ) @with_all_phases @@ -290,7 +382,13 @@ def test_get_aggregate_and_proof_signature(spec, state): pubkey = pubkeys[0] aggregate = get_mock_aggregate(spec) aggregate_and_proof = spec.get_aggregate_and_proof(state, spec.ValidatorIndex(1), aggregate, privkey) - signature = spec.get_aggregate_and_proof_signature(state, aggregate_and_proof, privkey) domain = spec.get_domain(state, spec.DOMAIN_AGGREGATE_AND_PROOF, spec.compute_epoch_at_slot(aggregate.data.slot)) - signing_root = spec.compute_signing_root(aggregate_and_proof, domain) - assert bls.Verify(pubkey, signing_root, signature) + run_get_signature_test( + spec=spec, + state=state, + obj=aggregate_and_proof, + domain=domain, + get_signature_fn=spec.get_aggregate_and_proof_signature, + privkey=privkey, + pubkey=pubkey, + ) From d311248d35684cd5aa6e47b286e075bb4feb46f4 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 27 Apr 2020 21:45:01 +0800 Subject: [PATCH 074/102] Increase `EPOCHS_PER_ETH1_VOTING_PERIOD` from `2` to `4` for testing eth1 votes consensus --- configs/minimal.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index b39a4fc01..c8b58146f 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -89,7 +89,7 @@ MIN_SEED_LOOKAHEAD: 1 # 2**2 (= 4) epochs MAX_SEED_LOOKAHEAD: 4 # [customized] higher frequency new deposits from eth1 for testing -EPOCHS_PER_ETH1_VOTING_PERIOD: 2 +EPOCHS_PER_ETH1_VOTING_PERIOD: 4 # [customized] smaller state SLOTS_PER_HISTORICAL_ROOT: 64 # 2**8 (= 256) epochs From 2dbc33327084d2814958f92eb0a838b9bc161903 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 27 Apr 2020 22:05:47 +0800 Subject: [PATCH 075/102] Make `compute_new_state_root` a pure function --- specs/phase0/validator.md | 7 ++++--- .../eth2spec/test/validator/test_validator_unittest.py | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 4576c4b90..cbe0c2d12 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -340,9 +340,10 @@ It is useful to be able to run a state transition function (working on a copy of ```python def compute_new_state_root(state: BeaconState, block: BeaconBlock) -> Root: - process_slots(state, block.slot) - process_block(state, block) - return hash_tree_root(state) + temp_state: BeaconState = state.copy() + signed_block = SignedBeaconBlock(message=block) + temp_state = state_transition(temp_state, signed_block, validate_result=False) + return hash_tree_root(temp_state) ``` ##### Signature diff --git a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py index a655cb486..5bb246ed5 100644 --- a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py +++ b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py @@ -246,6 +246,7 @@ def test_compute_new_state_root(spec, state): state_root = spec.compute_new_state_root(state, block) assert state_root != pre_state.hash_tree_root() + assert state == pre_state # dumb verification spec.process_slots(post_state, block.slot) From 3cc1fb901760f1c7ab35bcb69f4ce7c5dce78180 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Mon, 27 Apr 2020 14:34:50 -0700 Subject: [PATCH 076/102] Remove `/` --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b6e25d570..fed65eedb 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ This repository hosts the current Eth2 specifications. Discussions about design ## Specs -Core specifications for Eth2 clients be found in [specs/](specs/). These are divided into phases. Each subsequent phase depends upon the prior. The current phases specified are: +Core specifications for Eth2 clients be found in [specs](specs/). These are divided into phases. Each subsequent phase depends upon the prior. The current phases specified are: ### Phase 0 * [The Beacon Chain](specs/phase0/beacon-chain.md) From d128400da5cbcd16102fa760d4381ada07d4a577 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 27 Apr 2020 16:16:33 -0600 Subject: [PATCH 077/102] remove interop from header and gossip sections in network spec --- specs/phase0/p2p-interface.md | 55 +++++------------------------------ 1 file changed, 7 insertions(+), 48 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index f4f2f3d22..a3a9b48fe 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -4,7 +4,7 @@ This document contains the networking specification for Ethereum 2.0 clients. It consists of four main sections: -1. A specification of the network fundamentals detailing the two network configurations: interoperability test network and mainnet launch. +1. A specification of the network fundamentals. 2. A specification of the three network interaction *domains* of Eth2: (a) the gossip domain, (b) the discovery domain, and (c) the Req/Resp domain. 3. The rationale and further explanation for the design choices made in the previous two sections. 4. An analysis of the maturity/state of the libp2p features required by this spec across the languages in which Eth2 clients are being developed. @@ -120,42 +120,20 @@ It consists of four main sections: This section outlines the specification for the networking stack in Ethereum 2.0 clients. -Sections that have differing parameters for mainnet launch and interoperability testing are split into subsections. Sections that are not split have the same parameters for interoperability testing as mainnet launch. - ## Transport Even though libp2p is a multi-transport stack (designed to listen on multiple simultaneous transports and endpoints transparently), we hereby define a profile for basic interoperability. -#### Interop - All implementations MUST support the TCP libp2p transport, and it MUST be enabled for both dialing and listening (i.e. outbound and inbound connections). The libp2p TCP transport supports listening on IPv4 and IPv6 addresses (and on multiple simultaneously). -To facilitate connectivity and avert possible IPv6 routability/support issues, clients participating in the interoperability testnet MUST expose at least ONE IPv4 endpoint. +Clients must support listening on at least one of IPv4 or IPv6. Clients that do _not_ have support for listening on IPv4 SHOULD be cognizant of the potential disadvantages in terms of Internet-wide routability/support. Clients MAY choose to listen only on IPv6, but MUST be capable of dialing both IPv4 and IPv6 addresses. -All listening endpoints must be publicly dialable, and thus not rely on libp2p circuit relay, AutoNAT, or AutoRelay facilities. +All listening endpoints must be publicly dialable, and thus not rely on libp2p circuit relay, AutoNAT, or AutoRelay facilities. (Usage of circuit relay, AutoNAT, or AutoRelay will be specifically re-examined soon.) Nodes operating behind a NAT, or otherwise undialable by default (e.g. container runtime, firewall, etc.), MUST have their infrastructure configured to enable inbound traffic on the announced public listening endpoint. -#### Mainnet - -All requirements from the interoperability testnet apply, except for the IPv4 addressing scheme requirement. - -At this stage, clients are licensed to drop IPv4 support if they wish to do so, cognizant of the potential disadvantages in terms of Internet-wide routability/support. Clients MAY choose to listen only on IPv6, but MUST retain capability to dial both IPv4 and IPv6 addresses. - -Usage of circuit relay, AutoNAT, or AutoRelay will be specifically re-examined closer to the time. - ## Encryption and identification -#### Interop - -[SecIO](https://github.com/libp2p/specs/tree/master/secio) with `secp256k1` identities will be used for initial interoperability testing. - -The following SecIO parameters MUST be supported by all stacks: - -- Key agreement: ECDH-P256. -- Cipher: AES-128. -- Digest: SHA-256. - #### Mainnet The [Libp2p-noise](https://github.com/libp2p/specs/tree/master/noise) secure @@ -167,13 +145,7 @@ As specified in the libp2p specification, clients MUST support the `XX` handshak Clients MUST use exact equality when negotiating protocol versions to use and MAY use the version to give priority to higher version numbers. -#### Interop - -Connection-level and stream-level (see the [Rationale](#design-decision-rationale) section below for explanations) protocol negotiation MUST be conducted using [multistream-select v1.0](https://github.com/multiformats/multistream-select/). Its protocol ID is: `/multistream/1.0.0`. - -#### Mainnet - -Clients MUST support [multistream-select 1.0](https://github.com/multiformats/multistream-select/) and MAY support [multiselect 2.0](https://github.com/libp2p/specs/pull/95). Depending on the number of clients that have implementations for multiselect 2.0 by mainnet, [multistream-select 1.0](https://github.com/multiformats/multistream-select/) may be phased out. +Clients MUST support [multistream-select 1.0](https://github.com/multiformats/multistream-select/) and MAY support [multiselect 2.0](https://github.com/libp2p/specs/pull/95) when the spec solidifies. Once all clients have implementations for multiselect 2.0, multistream-select 1.0 MAY be phased out. ## Multiplexing @@ -181,7 +153,7 @@ During connection bootstrapping, libp2p dynamically negotiates a mutually suppor Two multiplexers are commonplace in libp2p implementations: [mplex](https://github.com/libp2p/specs/tree/master/mplex) and [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md). Their protocol IDs are, respectively: `/mplex/6.7.0` and `/yamux/1.0.0`. -Clients MUST support [mplex](https://github.com/libp2p/specs/tree/master/mplex) and MAY support [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md). If both are supported by the client, yamux must take precedence during negotiation. See the [Rationale](#design-decision-rationale) section below for tradeoffs. +Clients MUST support [mplex](https://github.com/libp2p/specs/tree/master/mplex) and MAY support [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md). If both are supported by the client, yamux MUST take precedence during negotiation. See the [Rationale](#design-decision-rationale) section below for tradeoffs. # Eth2 network interaction domains @@ -265,7 +237,6 @@ The payload is carried in the `data` field of a gossipsub message, and varies de |------------------------------------------------|-------------------------| | beacon_block | SignedBeaconBlock | | beacon_aggregate_and_proof | SignedAggregateAndProof | -| beacon_attestation\* | Attestation | | committee_index{subnet_id}\_beacon_attestation | Attestation | | voluntary_exit | SignedVoluntaryExit | | proposer_slashing | ProposerSlashing | @@ -275,8 +246,6 @@ Clients MUST reject (fail validation) messages containing an incorrect type, or When processing incoming gossip, clients MAY descore or disconnect peers who fail to observe these constraints. -\* The `beacon_attestation` topic is only for interop and will be removed prior to mainnet. - #### Global topics There are two primary global topics used to propagate beacon blocks and aggregate attestations to all nodes on the network. Their `Name`s are: @@ -323,11 +292,7 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio - The block being voted for (`attestation.data.beacon_block_root`) passes validation. - The signature of `attestation` is valid. -#### Interop - -Unaggregated and aggregated attestations from all shards are sent as `Attestation`s to the `beacon_attestation` topic. Clients are not required to publish aggregate attestations but must be able to process them. All validating clients SHOULD try to perform local attestation aggregation to prepare for block proposing. - -#### Mainnet +#### Attestations and Aggregation Attestation broadcasting is grouped into subnets defined by a topic. The number of subnets is defined via `ATTESTATION_SUBNET_COUNT`. For the `committee_index{subnet_id}_beacon_attestation` topics, `subnet_id` is set to `index % ATTESTATION_SUBNET_COUNT`, where `index` is the `CommitteeIndex` of the given committee. @@ -339,17 +304,11 @@ Aggregated attestations are sent to the `beacon_aggregate_and_proof` topic as `A Topics are post-fixed with an encoding. Encodings define how the payload of a gossipsub message is encoded. -#### Interop - -- `ssz` - All objects are [SSZ-encoded](#ssz-encoding). Example: The beacon block topic string is `/eth2/beacon_block/ssz`, and the data field of a gossipsub message is an ssz-encoded `SignedBeaconBlock`. - -#### Mainnet - - `ssz_snappy` - All objects are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy) block compression. Example: The beacon aggregate attestation topic string is `/eth2/446a7232/beacon_aggregate_and_proof/ssz_snappy`, the fork digest is `446a7232` and the data field of a gossipsub message is an `AggregateAndProof` that has been SSZ-encoded and then compressed with Snappy. Snappy has two formats: "block" and "frames" (streaming). Gossip messages remain relatively small (100s of bytes to 100s of kilobytes) so [basic snappy block compression](https://github.com/google/snappy/blob/master/format_description.txt) is used to avoid the additional overhead associated with snappy frames. -Implementations MUST use a single encoding. Changing an encoding will require coordination between participating implementations. +Implementations MUST use a single encoding for gossip. Changing an encoding will require coordination between participating implementations. ## The Req/Resp domain From 87586837c388a4005d09d7baf56213fc418211e9 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 27 Apr 2020 17:34:26 -0600 Subject: [PATCH 078/102] remove interop from phase 0 p2p specs --- specs/phase0/p2p-interface.md | 73 ++++++++++------------------------- 1 file changed, 21 insertions(+), 52 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index a3a9b48fe..f76502bb4 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -17,14 +17,8 @@ It consists of four main sections: - [Network fundamentals](#network-fundamentals) - [Transport](#transport) - - [Interop](#interop) - - [Mainnet](#mainnet) - [Encryption and identification](#encryption-and-identification) - - [Interop](#interop-1) - - [Mainnet](#mainnet-1) - [Protocol Negotiation](#protocol-negotiation) - - [Interop](#interop-2) - - [Mainnet](#mainnet-2) - [Multiplexing](#multiplexing) - [Eth2 network interaction domains](#eth2-network-interaction-domains) - [Configuration](#configuration) @@ -33,11 +27,8 @@ It consists of four main sections: - [Topics and messages](#topics-and-messages) - [Global topics](#global-topics) - [Attestation subnets](#attestation-subnets) - - [Interop](#interop-3) - - [Mainnet](#mainnet-3) + - [Attestations and Aggregation](#attestations-and-aggregation) - [Encodings](#encodings) - - [Interop](#interop-4) - - [Mainnet](#mainnet-4) - [The Req/Resp domain](#the-reqresp-domain) - [Protocol identification](#protocol-identification) - [Req/Resp interaction](#reqresp-interaction) @@ -56,29 +47,25 @@ It consists of four main sections: - [Integration into libp2p stacks](#integration-into-libp2p-stacks) - [ENR structure](#enr-structure) - [Attestation subnet bitfield](#attestation-subnet-bitfield) - - [Interop](#interop-5) - - [Mainnet](#mainnet-5) - - [`eth2` field](#eth2-field) - - [General capabilities](#general-capabilities) + - [`eth2` field](#eth2-field) + - [General capabilities](#general-capabilities) - [Topic advertisement](#topic-advertisement) - - [Mainnet](#mainnet-6) - [Design decision rationale](#design-decision-rationale) - [Transport](#transport-1) - [Why are we defining specific transports?](#why-are-we-defining-specific-transports) - [Can clients support other transports/handshakes than the ones mandated by the spec?](#can-clients-support-other-transportshandshakes-than-the-ones-mandated-by-the-spec) - [What are the advantages of using TCP/QUIC/Websockets?](#what-are-the-advantages-of-using-tcpquicwebsockets) - [Why do we not just support a single transport?](#why-do-we-not-just-support-a-single-transport) - - [Why are we not using QUIC for mainnet from the start?](#why-are-we-not-using-quic-for-mainnet-from-the-start) + - [Why are we not using QUIC from the start?](#why-are-we-not-using-quic-from-the-start) - [Multiplexing](#multiplexing-1) - [Why are we using mplex/yamux?](#why-are-we-using-mplexyamux) - [Protocol Negotiation](#protocol-negotiation-1) - - [When is multiselect 2.0 due and why are we using it for mainnet?](#when-is-multiselect-20-due-and-why-are-we-using-it-for-mainnet) + - [When is multiselect 2.0 due and why do we plan to migrate to it?](#when-is-multiselect-20-due-and-why-do-we-plan-to-migrate-to-it) - [What is the difference between connection-level and stream-level protocol negotiation?](#what-is-the-difference-between-connection-level-and-stream-level-protocol-negotiation) - [Encryption](#encryption) - - [Why are we using SecIO for interop? Why not for mainnet?](#why-are-we-using-secio-for-interop-why-not-for-mainnet) - - [Why are we using Noise/TLS 1.3 for mainnet?](#why-are-we-using-noisetls-13-for-mainnet) + - [Why are we not supporting SecIO?](#why-are-we-not-supporting-secio) + - [Why are we using Noise/TLS 1.3?](#why-are-we-using-noisetls-13) - [Why are we using encryption at all?](#why-are-we-using-encryption-at-all) - - [Will mainnnet networking be untested when it launches?](#will-mainnnet-networking-be-untested-when-it-launches) - [Gossipsub](#gossipsub) - [Why are we using a pub/sub algorithm for block and attestation propagation?](#why-are-we-using-a-pubsub-algorithm-for-block-and-attestation-propagation) - [Why are we using topics to segregate encodings, yet only support one encoding?](#why-are-we-using-topics-to-segregate-encodings-yet-only-support-one-encoding) @@ -134,10 +121,8 @@ Nodes operating behind a NAT, or otherwise undialable by default (e.g. container ## Encryption and identification -#### Mainnet - The [Libp2p-noise](https://github.com/libp2p/specs/tree/master/noise) secure -channel handshake with `secp256k1` identities will be used for mainnet. +channel handshake with `secp256k1` identities will be used for encryption. As specified in the libp2p specification, clients MUST support the `XX` handshake pattern. @@ -409,7 +394,7 @@ Here, `result` represents the 1-byte response code. The token of the negotiated protocol ID specifies the type of encoding to be used for the req/resp interaction. Two values are possible at this time: - `ssz`: the contents are [SSZ-encoded](../../ssz/simple-serialize.md). This encoding type MUST be supported by all clients. For objects containing a single field, only the field is SSZ-encoded not a container with a single field. For example, the `BeaconBlocksByRoot` request is an SSZ-encoded list of `Root`'s. -- `ssz_snappy`: The contents are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy) frames compression. MAY be supported in the interoperability testnet; MUST be supported in mainnet. +- `ssz_snappy`: The contents are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy) frames compression. This encoding type MUST be supported by all clients. #### SSZ-encoding strategy (with or without Snappy) @@ -642,7 +627,7 @@ The response MUST consist of a single `response_chunk`. ## The discovery domain: discv5 -Discovery Version 5 ([discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md)) is used for peer discovery, both in the interoperability testnet and mainnet. +Discovery Version 5 ([discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md)) is used for peer discovery. `discv5` is a standalone protocol, running on UDP on a dedicated port, meant for peer discovery only. `discv5` supports self-certified, flexible peer records (ENRs) and topic-based advertisement, both of which are (or will be) requirements in this context. @@ -682,15 +667,7 @@ If a node's `MetaData.attnets` has any non-zero bit, the ENR MUST include the `a If a node's `MetaData.attnets` is composed of all zeros, the ENR MAY optionally include the `attnets` entry or leave it out entirely. -#### Interop - -In the interoperability testnet, all peers will support all capabilities defined in this document (gossip, full Req/Resp suite, discovery protocol), therefore the ENR record does not need to carry Eth2 capability information, as it would be superfluous. - -Nonetheless, ENRs MUST carry a generic `eth2` key with nil value, denoting that the peer is indeed an Eth2 peer, in order to eschew connecting to Eth 1.0 peers. - -#### Mainnet - -##### `eth2` field +#### `eth2` field ENRs MUST carry a generic `eth2` key with an 16-byte value of the node's current fork digest, next fork version, and next fork epoch to ensure connections are made with peers on the intended eth2 network. @@ -722,14 +699,12 @@ Clients SHOULD connect to peers with `fork_digest`, `next_fork_version`, and `ne Clients MAY connect to peers with the same `fork_digest` but a different `next_fork_version`/`next_fork_epoch`. Unless `ENRForkID` is manually updated to matching prior to the earlier `next_fork_epoch` of the two clients, these connecting clients will be unable to successfully interact starting at the earlier `next_fork_epoch`. -##### General capabilities +#### General capabilities -On mainnet, ENRs MUST include a structure enumerating the capabilities offered by the peer in an efficient manner. The concrete solution is currently undefined. Proposals include using namespaced bloom filters mapping capabilities to specific protocol IDs supported under that capability. +ENRs MUST include a structure enumerating the capabilities offered by the peer in an efficient manner. The concrete solution is currently undefined. Proposals include using namespaced bloom filters mapping capabilities to specific protocol IDs supported under that capability. ### Topic advertisement -#### Mainnet - discv5's topic advertisement feature is not expected to be ready for mainnet launch of Phase 0. Once this feature is built out and stable, we expect to use topic advertisement as a rendezvous facility for peers on shards. Until then, the ENR [attestation subnet bitfield](#attestation-subnet-bitfield) will be used for discovery of peers on particular subnets. @@ -775,7 +750,7 @@ Modeling for upgradeability and dynamic transport selection from the get-go lays Clients can adopt new transports without breaking old ones, and the multi-transport ability enables constrained and sandboxed environments (e.g. browsers, embedded devices) to interact with the network as first-class citizens via suitable/native transports (e.g. WSS), without the need for proxying or trust delegation to servers. -### Why are we not using QUIC for mainnet from the start? +### Why are we not using QUIC from the start? The QUIC standard is still not finalized (at working draft 22 at the time of writing), and not all mainstream runtimes/languages have mature, standard, and/or fully-interoperable [QUIC support](https://github.com/quicwg/base-drafts/wiki/Implementations). One remarkable example is node.js, where the QUIC implementation is [in early development](https://github.com/nodejs/quic). @@ -791,13 +766,13 @@ Overlay multiplexers are not necessary with QUIC since the protocol provides nat ## Protocol Negotiation -### When is multiselect 2.0 due and why are we using it for mainnet? +### When is multiselect 2.0 due and why do we plan to migrate to it? multiselect 2.0 is currently being conceptualized. The debate started [on this issue](https://github.com/libp2p/specs/pull/95), but it got overloaded—as it tends to happen with large conceptual OSS discussions that touch the heart and core of a system. -In the following weeks (August 2019), there will be a renewed initiative to first define the requirements, constraints, assumptions, and features, in order to lock in basic consensus upfront and subsequently build on that consensus by submitting a specification for implementation. +At some point in 2020, we expect a renewed initiative to first define the requirements, constraints, assumptions, and features, in order to lock in basic consensus upfront and subsequently build on that consensus by submitting a specification for implementation. -We plan to use multiselect 2.0 for mainnet because it will: +We plan to eventually migrate to multiselect 2.0 because it will: 1. Reduce round trips during connection bootstrapping and stream protocol negotiation. 2. Enable efficient one-stream-per-request interaction patterns. @@ -819,17 +794,15 @@ At present, multistream-select 1.0 is used for both types of negotiation, but mu ## Encryption -### Why are we using SecIO for interop? Why not for mainnet? +### Why are we not supporting SecIO? SecIO has been the default encryption layer for libp2p for years. It is used in IPFS and Filecoin. And although it will be superseded shortly, it is proven to work at scale. -SecIO is the common denominator across the various language libraries at this stage. It is widely implemented. That’s why we have chosen to use it for initial interop to minimize overhead in getting to a basic interoperability testnet. - -We won’t be using it for mainnet because, amongst other things, it requires several round trips to be sound, and doesn’t support early data (0-RTT data), a mechanism that multiselect 2.0 will leverage to reduce round trips during connection bootstrapping. +Although SecIO has wide language support, we won’t be using it for mainnet because, amongst other things, it requires several round trips to be sound, and doesn’t support early data (0-RTT data), a mechanism that multiselect 2.0 will leverage to reduce round trips during connection bootstrapping. SecIO is not considered secure for the purposes of this spec. -### Why are we using Noise/TLS 1.3 for mainnet? +### Why are we using Noise/TLS 1.3? Copied from the Noise Protocol Framework [website](http://www.noiseprotocol.org): @@ -855,10 +828,6 @@ Transport level encryption secures message exchange and provides properties that Note that transport-level encryption is not exclusive of application-level encryption or cryptography. Transport-level encryption secures the communication itself, while application-level cryptography is necessary for the application’s use cases (e.g. signatures, randomness, etc.). -### Will mainnnet networking be untested when it launches? - -Before launching mainnet, the testnet will be switched over to mainnet networking parameters, including Noise handshakes, and other new protocols. This gives us an opportunity to drill coordinated network upgrades and verifying that there are no significant upgradeability gaps. - ## Gossipsub ### Why are we using a pub/sub algorithm for block and attestation propagation? @@ -967,7 +936,7 @@ Requests are segregated by protocol ID to: 2. Affording this level of granularity with a top-level protocol would imply creating as many variants (e.g. /protocol/43-{a,b,c,d,...}) as the cartesian product of RFCs inflight, O(n^2). 7. Allow us to simplify the payload of requests. Request-id’s and method-ids no longer need to be sent. The encoding/request type and version can all be handled by the framework. -**Caveat**: The protocol negotiation component in the current version of libp2p is called multistream-select 1.0. It is somewhat naïve and introduces overhead on every request when negotiating streams, although implementation-specific optimizations are possible to save this cost. Multiselect 2.0 will remove this overhead by memoizing previously selected protocols, and modeling shared protocol tables. Fortunately, this req/resp protocol is not the expected network bottleneck in the protocol so the additional overhead is not expected to hinder interop testing. More info is to be released from the libp2p community in the coming weeks. +**Caveat**: The protocol negotiation component in the current version of libp2p is called multistream-select 1.0. It is somewhat naïve and introduces overhead on every request when negotiating streams, although implementation-specific optimizations are possible to save this cost. Multiselect 2.0 will eventually remove this overhead by memoizing previously selected protocols, and modeling shared protocol tables. Fortunately, this req/resp protocol is not the expected network bottleneck in the protocol so the additional overhead is not expected to significantly hinder this domain. ### Why are messages length-prefixed with a protobuf varint in the SSZ-encoding? From fa66475da46d334ed2214a7b72143829af9cafab Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 29 Apr 2020 00:04:44 +0800 Subject: [PATCH 079/102] Move `sanity` to under `phase_0` --- .../core/pyspec/eth2spec/test/{ => phase_0}/sanity/__init__.py | 0 .../pyspec/eth2spec/test/{ => phase_0}/sanity/test_blocks.py | 0 .../pyspec/eth2spec/test/{ => phase_0}/sanity/test_slots.py | 0 tests/generators/sanity/main.py | 2 +- 4 files changed, 1 insertion(+), 1 deletion(-) rename tests/core/pyspec/eth2spec/test/{ => phase_0}/sanity/__init__.py (100%) rename tests/core/pyspec/eth2spec/test/{ => phase_0}/sanity/test_blocks.py (100%) rename tests/core/pyspec/eth2spec/test/{ => phase_0}/sanity/test_slots.py (100%) diff --git a/tests/core/pyspec/eth2spec/test/sanity/__init__.py b/tests/core/pyspec/eth2spec/test/phase_0/sanity/__init__.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/sanity/__init__.py rename to tests/core/pyspec/eth2spec/test/phase_0/sanity/__init__.py diff --git a/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/sanity/test_blocks.py rename to tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py diff --git a/tests/core/pyspec/eth2spec/test/sanity/test_slots.py b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_slots.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/sanity/test_slots.py rename to tests/core/pyspec/eth2spec/test/phase_0/sanity/test_slots.py diff --git a/tests/generators/sanity/main.py b/tests/generators/sanity/main.py index 74c85a9e8..89fb89838 100644 --- a/tests/generators/sanity/main.py +++ b/tests/generators/sanity/main.py @@ -5,7 +5,7 @@ from gen_base import gen_runner, gen_typing from gen_from_tests.gen import generate_from_tests from eth2spec.test.context import PHASE0 -from eth2spec.test.sanity import test_blocks, test_slots +from eth2spec.test.phase_0.sanity import test_blocks, test_slots from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 From 6a40f71a31724aab57ca90ff0a00fc5a536f62d6 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 29 Apr 2020 20:29:48 -0600 Subject: [PATCH 080/102] add note about beacon committees not going into attnets --- specs/phase0/validator.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index bc7510403..69be71279 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -524,6 +524,8 @@ Because Phase 0 does not have shards and thus does not have Shard Committees, th * Maintain advertisement of the randomly selected subnets in their node's ENR `attnets` entry by setting the randomly selected `subnet_id` bits to `True` (e.g. `ENR["attnets"][subnet_id] = True`) for all persistent attestation subnets * Set the lifetime of each random subscription to a random number of epochs between `EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION` and `2 * EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION]`. At the end of life for a subscription, select a new random subnet, update subnet subscriptions, and publish an updated ENR +*Note*: Short lived beacon committee assignments should not be added in into the ENR `attnets` entry. + *Note*: When preparing for a hard fork, a validator must select and subscribe to random subnets of the future fork versioning at least `EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION` epochs in advance of the fork. These new subnets for the fork are maintained in addition to those for the current fork until the fork occurs. After the fork occurs, let the subnets from the previous fork reach the end of life with no replacements. ## How to avoid slashing From 2dc515665181c5cb30e60148fb3be57073cb3a77 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 30 Apr 2020 16:27:02 +1000 Subject: [PATCH 081/102] Add message about delaying consideration --- specs/phase0/fork-choice.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index 35e2c5f56..60d398dd5 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -273,6 +273,7 @@ def validate_on_attestation(store: Store, attestation: Attestation) -> None: current_epoch = compute_epoch_at_slot(get_current_slot(store)) # Use GENESIS_EPOCH for previous when genesis to avoid underflow previous_epoch = current_epoch - 1 if current_epoch > GENESIS_EPOCH else GENESIS_EPOCH + # If attestation target is from a future epoch, delay consideration until the epoch arrives assert target.epoch in [current_epoch, previous_epoch] assert target.epoch == compute_epoch_at_slot(attestation.data.slot) From feb27a14be1ae2b64a96f0469ffaf0bd68223c67 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 31 Mar 2020 12:37:36 +0800 Subject: [PATCH 082/102] beacon-chain.md: Replace block wrapper with signable pattern --- specs/phase1/beacon-chain.md | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 596b3818f..d90afda16 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -24,8 +24,9 @@ - [Extended `SignedBeaconBlock`](#extended-signedbeaconblock) - [Extended `BeaconState`](#extended-beaconstate) - [New containers](#new-containers) - - [`ShardBlockWrapper`](#shardblockwrapper) - - [`ShardSignableHeader`](#shardsignableheader) + - [`ShardBlock`](#shardblock) + - [`SignedShardBlock`](#signedshardblock) + - [`ShardBlockHeader`](#shardblockheader) - [`ShardState`](#shardstate) - [`ShardTransition`](#shardtransition) - [`CompactCommittee`](#compactcommittee) @@ -291,23 +292,28 @@ class BeaconState(Container): The following containers are new in Phase 1. -### `ShardBlockWrapper` - -_Wrapper for being broadcasted over the network._ +### `ShardBlock` ```python -class ShardBlockWrapper(Container): +class ShardBlock(Container): shard_parent_root: Root beacon_parent_root: Root slot: Slot body: ByteList[MAX_SHARD_BLOCK_SIZE] +``` + +### `SignedShardBlock` + +```python +class SignedShardBlock(Container): + message: ShardBlock signature: BLSSignature ``` -### `ShardSignableHeader` +### `ShardBlockHeader` ```python -class ShardSignableHeader(Container): +class ShardBlockHeader(Container): shard_parent_root: Root beacon_parent_root: Root slot: Slot @@ -700,7 +706,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr shard_parent_root = state.shard_states[shard].latest_block_root for i in range(len(offset_slots)): if any(transition.shard_data_roots): - headers.append(ShardSignableHeader( + headers.append(ShardBlockHeader( shard_parent_root=shard_parent_root, parent_hash=get_block_root_at_slot(state, get_previous_slot(state.slot)), slot=offset_slots[i], From e9f1e4186d0e6cf70f32e68cd97f33a61eec5a2e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 2 Apr 2020 10:43:50 +0800 Subject: [PATCH 083/102] Add `proposer_index` to shard block --- specs/phase1/beacon-chain.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index d90afda16..0652fecad 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -299,6 +299,7 @@ class ShardBlock(Container): shard_parent_root: Root beacon_parent_root: Root slot: Slot + proposer_index: ValidatorIndex body: ByteList[MAX_SHARD_BLOCK_SIZE] ``` @@ -317,6 +318,7 @@ class ShardBlockHeader(Container): shard_parent_root: Root beacon_parent_root: Root slot: Slot + proposer_index: ValidatorIndex body_root: Root ``` From 5f69afea382740c696052a11e939329f46810f82 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 2 Apr 2020 10:53:46 +0800 Subject: [PATCH 084/102] Make `shard_state_transition` more like beacon state_transition function --- specs/phase1/fraud-proofs.md | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/specs/phase1/fraud-proofs.md b/specs/phase1/fraud-proofs.md index 0688f5f47..19077ba88 100644 --- a/specs/phase1/fraud-proofs.md +++ b/specs/phase1/fraud-proofs.md @@ -45,14 +45,30 @@ The proof verifies that one of the two conditions is false: ## Shard state transition function ```python -def shard_state_transition(shard: Shard, +def shard_state_transition(beacon_state: BeaconState, + shard: Shard, slot: Slot, pre_state: Root, previous_beacon_root: Root, - proposer_pubkey: BLSPubkey, - block_data: ByteList[MAX_SHARD_BLOCK_SIZE]) -> Root: + proposer_index: ValidatorIndex, + signed_block: SignedShardBlock, + validate_result: bool=True) -> Root: # We will add something more substantive in phase 2 - return hash(pre_state + hash_tree_root(previous_beacon_root) + hash_tree_root(block_data)) + + # Verify the proposer_index and signature + assert proposer_index == signed_block.message.proposer_index + if validate_result: + assert verify_shard_block_signature(beacon_state, signed_block) + + return hash(pre_state + hash_tree_root(previous_beacon_root) + hash_tree_root(signed_block.message.data)) +``` + +```python +def verify_shard_block_signature(beacon_state: BeaconState, + signed_block: SignedShardBlock) -> bool: + proposer = beacon_state.validators[signed_block.message.proposer_index] + signing_root = compute_signing_root(signed_block.message, get_domain(beacon_state, DOMAIN_SHARD_PROPOSAL)) + return bls.Verify(proposer.pubkey, signing_root, signed_block.signature) ``` ## Honest committee member behavior @@ -61,10 +77,10 @@ Suppose you are a committee member on shard `shard` at slot `current_slot`. Let * Initialize `proposals = []`, `shard_states = []`, `shard_state = state.shard_states[shard][-1]`, `start_slot = shard_state.slot`. * For `slot in get_offset_slots(state, start_slot)`, do the following: - * Look for all valid proposals for `slot`; that is, a Bytes `proposal` where `shard_state_transition(shard, slot, shard_state, get_block_root_at_slot(state, state.slot - 1), get_shard_proposer_index(state, shard, slot), proposal)` returns a result and does not throw an exception. Let `choices` be the set of non-empty valid proposals you discover. + * Look for all valid proposals for `slot`; that is, a SignedShardBlock `proposal` where `shard_state_transition(shard, slot, shard_state, get_block_root_at_slot(state, state.slot - 1), get_shard_proposer_index(state, slot, shard), proposal)` returns a result and does not throw an exception. Let `choices` be the set of non-empty valid proposals you discover. * If `len(choices) == 0`, do `proposals.append(make_empty_proposal(shard_state, slot))` * If `len(choices) == 1`, do `proposals.append(choices[0])` * If `len(choices) > 1`, let `winning_proposal` be the proposal with the largest number of total attestations from slots in `state.shard_next_slots[shard]....slot-1` supporting it or any of its descendants, breaking ties by choosing the first proposal locally seen. Do `proposals.append(winning_proposal)`. - * If `proposals[-1]` is NOT an empty proposal, set `shard_state = shard_state_transition(shard, slot, shard_state, get_block_root_at_slot(state, state.slot - 1), get_shard_proposer_index(state, shard, slot), proposals[-1])` and do `shard_states.append(shard_state)`. If it is an empty proposal, leave `shard_state` unchanged. + * If `proposals.message.data[-1]` is NOT an empty proposal, set `shard_state = shard_state_transition(shard, slot, shard_state, get_block_root_at_slot(state, state.slot - 1), get_shard_proposer_index(state, slot, shard), proposals[-1])` and do `shard_states.append(shard_state)`. If it is an empty proposal, leave `shard_state` unchanged. -Make an attestation using `shard_data_roots = [hash_tree_root(proposal) for proposal in proposals]` and `shard_state_roots = shard_states`. +Make an attestation using `shard_data_roots = [hash_tree_root(proposal.message.data) for proposal in proposals]` and `shard_state_roots = shard_states`. From be50020bf8134b82c5d7e46add6713f193ccddde Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 2 Apr 2020 11:20:22 +0800 Subject: [PATCH 085/102] Refactor `get_light_client_committee` to similar to `get_shard_committee` --- specs/phase1/beacon-chain.md | 40 ++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 0652fecad..fbfdce70e 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -43,8 +43,8 @@ - [`get_active_shard_count`](#get_active_shard_count) - [`get_online_validator_indices`](#get_online_validator_indices) - [`get_shard_committee`](#get_shard_committee) - - [`get_shard_proposer_index`](#get_shard_proposer_index) - [`get_light_client_committee`](#get_light_client_committee) + - [`get_shard_proposer_index`](#get_shard_proposer_index) - [`get_indexed_attestation`](#get_indexed_attestation) - [`get_updated_gasprice`](#get_updated_gasprice) - [`get_start_shard`](#get_start_shard) @@ -465,7 +465,30 @@ def get_shard_committee(beacon_state: BeaconState, epoch: Epoch, shard: Shard) - source_epoch -= SHARD_COMMITTEE_PERIOD active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) seed = get_seed(beacon_state, source_epoch, DOMAIN_SHARD_COMMITTEE) - return compute_committee(active_validator_indices, seed, shard, get_active_shard_count(beacon_state)) + return compute_committee( + indices=active_validator_indices, + seed=seed, + index=shard, + count=get_active_shard_count(beacon_state) + ) +``` + +#### `get_light_client_committee` + +```python +def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: + source_epoch = epoch - epoch % LIGHT_CLIENT_COMMITTEE_PERIOD + if source_epoch > 0: + source_epoch -= LIGHT_CLIENT_COMMITTEE_PERIOD + active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) + seed = get_seed(beacon_state, source_epoch, DOMAIN_LIGHT_CLIENT) + active_shards_count = get_active_shard_count(beacon_state) + return compute_committee( + indices=active_validator_indices, + seed=seed, + index=0, + count=active_shards_count, + )[:TARGET_COMMITTEE_SIZE] ``` #### `get_shard_proposer_index` @@ -477,19 +500,6 @@ def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard return committee[r % len(committee)] ``` -#### `get_light_client_committee` - -```python -def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: - source_epoch = epoch - epoch % LIGHT_CLIENT_COMMITTEE_PERIOD - if source_epoch > 0: - source_epoch -= LIGHT_CLIENT_COMMITTEE_PERIOD - active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) - seed = get_seed(beacon_state, source_epoch, DOMAIN_LIGHT_CLIENT) - active_shards = get_active_shard_count(beacon_state) - return compute_committee(active_validator_indices, seed, 0, active_shards)[:TARGET_COMMITTEE_SIZE] -``` - #### `get_indexed_attestation` ```python From 247a6c8fca24e98489295d56db58c32180158c88 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 2 Apr 2020 11:22:21 +0800 Subject: [PATCH 086/102] Add `verify_fraud_proof` function --- specs/phase1/fraud-proofs.md | 60 ++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 10 deletions(-) diff --git a/specs/phase1/fraud-proofs.md b/specs/phase1/fraud-proofs.md index 19077ba88..e72a8bb2a 100644 --- a/specs/phase1/fraud-proofs.md +++ b/specs/phase1/fraud-proofs.md @@ -6,7 +6,8 @@ - [Table of contents](#table-of-contents) - [Introduction](#introduction) - [Fraud proofs](#fraud-proofs) - - [Shard state transition function](#shard-state-transition-function) + - [Shard state transition function](#shard-state-transition-function) + - [Verifying the proof](#verifying-the-proof) - [Honest committee member behavior](#honest-committee-member-behavior) @@ -32,17 +33,12 @@ This document describes the shard transition function and fraud proofs as part o TODO. The intent is to have a single universal fraud proof type, which contains the following parts: 1. An on-time attestation on some `shard` signing a `ShardTransition` -2. An index `i` of a particular position to focus on +2. An index `index` of a particular position to focus on 3. The `ShardTransition` itself -4. The full body of the block -5. A Merkle proof to the `shard_states` in the parent block the attestation is referencing +4. The full body of the block `ShardBlock` +5. A Merkle proof to the `shard_states` in the parent block `parent_block` the attestation is referencing -The proof verifies that one of the two conditions is false: - -1. `custody_bits[i][j] != generate_custody_bit(subkey, block_contents)` for any `j` -2. `execute_state_transition(shard, slot, transition.shard_states[i-1].data, hash_tree_root(parent), get_shard_proposer_index(state, shard, slot), block_contents) != transition.shard_states[i].data` (if `i=0` then instead use `parent.shard_states[shard][-1].data`) - -## Shard state transition function +### Shard state transition function ```python def shard_state_transition(beacon_state: BeaconState, @@ -71,6 +67,50 @@ def verify_shard_block_signature(beacon_state: BeaconState, return bls.Verify(proposer.pubkey, signing_root, signed_block.signature) ``` +### Verifying the proof + +```python +def verify_fraud_proof(beacon_state: BeaconState, + subkey: BLSPubkey, + attestation: Attestation, + index: uint64, + transition: ShardTransition, + signed_block: SignedShardBlock, + parent_block: ShardBlock) -> bool: + # 1. Check if `custody_bits[index][j] != generate_custody_bit(subkey, block_contents)` for any `j` + shard = get_shard(beacon_state, attestation) + slot = attestation.data.slot + custody_bits = attestation.custody_bits_blocks + for j in range(custody_bits[index]): + if custody_bits[index][j] != generate_custody_bit(subkey, signed_block): + return True + + # 2. Verify the shard state transition + if index == 0: + parent_data = parent_block.shard_states[shard][-1].data + else: + parent_data = parent_block.shard_states[shard][index].data + + if shard_state_transition( + beacon_state, + shard, + slot, + transition.shard_states[index - 1].data, + hash_tree_root(parent_block), + get_shard_proposer_index(beacon_state, slot, shard), + signed_block, + ) != parent_data: + return True + + return False +``` + +```python +def generate_custody_bit(subkey: BLSPubkey, block: ShardBlock) -> bool: + # TODO + ... +``` + ## Honest committee member behavior Suppose you are a committee member on shard `shard` at slot `current_slot`. Let `state` be the head beacon state you are building on, and let `QUARTER_PERIOD = SECONDS_PER_SLOT // 4`. `2 * QUARTER_PERIOD` seconds into slot `slot`, run the following procedure: From 849d3f83bf4fd95ef47fd1fcee45494dc4a49e83 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 6 Apr 2020 17:48:24 +0800 Subject: [PATCH 087/102] Apply @terencechain 's review feedback Co-Authored-By: terence tsao --- specs/phase1/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index fbfdce70e..7c3ae786b 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -720,7 +720,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr if any(transition.shard_data_roots): headers.append(ShardBlockHeader( shard_parent_root=shard_parent_root, - parent_hash=get_block_root_at_slot(state, get_previous_slot(state.slot)), + beacon_parent_root=get_block_root_at_slot(state, get_previous_slot(state.slot)), slot=offset_slots[i], body_root=transition.shard_data_roots[i] )) From 4e8a7ff1156dc3a12ccb718476782bc685de3a4f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 6 Apr 2020 19:30:32 +0800 Subject: [PATCH 088/102] [squashed] shard transition wip Fix the wrong `get_shard_proposer_index` parameters order Phase 1 WIP Add shard transition basic test Fix lint error Fix --- specs/phase1/beacon-chain.md | 63 ++--- specs/phase1/fraud-proofs.md | 229 ++++++++++++++---- .../eth2spec/test/helpers/attestations.py | 40 ++- .../eth2spec/test/helpers/crosslinks.py | 28 +++ .../eth2spec/test/helpers/phase1/__init__.py | 0 .../test/helpers/phase1/attestations.py | 63 ----- .../test/helpers/phase1/shard_block.py | 71 ------ .../test/helpers/phase1/shard_state.py | 18 -- .../eth2spec/test/helpers/shard_block.py | 47 ++++ .../test_process_crosslink.py | 52 ++++ 10 files changed, 374 insertions(+), 237 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/helpers/crosslinks.py delete mode 100644 tests/core/pyspec/eth2spec/test/helpers/phase1/__init__.py delete mode 100644 tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py delete mode 100644 tests/core/pyspec/eth2spec/test/helpers/phase1/shard_block.py delete mode 100644 tests/core/pyspec/eth2spec/test/helpers/phase1/shard_state.py create mode 100644 tests/core/pyspec/eth2spec/test/helpers/shard_block.py create mode 100644 tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 7c3ae786b..4b22f8633 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -460,16 +460,17 @@ def get_online_validator_indices(state: BeaconState) -> Set[ValidatorIndex]: ```python def get_shard_committee(beacon_state: BeaconState, epoch: Epoch, shard: Shard) -> Sequence[ValidatorIndex]: - source_epoch = epoch - epoch % SHARD_COMMITTEE_PERIOD + source_epoch = epoch - epoch % SHARD_COMMITTEE_PERIOD if source_epoch > 0: source_epoch -= SHARD_COMMITTEE_PERIOD active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) seed = get_seed(beacon_state, source_epoch, DOMAIN_SHARD_COMMITTEE) + active_shards_count = get_active_shard_count(beacon_state) return compute_committee( indices=active_validator_indices, seed=seed, index=shard, - count=get_active_shard_count(beacon_state) + count=active_shards_count, ) ``` @@ -712,29 +713,35 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr ) assert transition.start_slot == offset_slots[0] - # Reconstruct shard headers headers = [] + header = ShardBlockHeader() proposers = [] - shard_parent_root = state.shard_states[shard].latest_block_root - for i in range(len(offset_slots)): - if any(transition.shard_data_roots): - headers.append(ShardBlockHeader( - shard_parent_root=shard_parent_root, - beacon_parent_root=get_block_root_at_slot(state, get_previous_slot(state.slot)), - slot=offset_slots[i], - body_root=transition.shard_data_roots[i] - )) - proposers.append(get_shard_proposer_index(state, shard, offset_slots[i])) - shard_parent_root = hash_tree_root(headers[-1]) - - # Verify correct calculation of gas prices and slots prev_gasprice = state.shard_states[shard].gasprice + shard_parent_root = state.shard_states[shard].latest_block_root + beacon_parent_root = get_block_root_at_slot(state, get_previous_slot(state.slot)) for i in range(len(offset_slots)): + shard_block_length = transition.shard_block_lengths[i] + is_empty_proposal = (shard_block_length == 0) shard_state = transition.shard_states[i] - block_length = transition.shard_block_lengths[i] - assert shard_state.gasprice == get_updated_gasprice(prev_gasprice, block_length) - assert shard_state.slot == offset_slots[i] - prev_gasprice = shard_state.gasprice + proposal_index = get_shard_proposer_index(state, offset_slots[i], shard) + # Reconstruct shard headers + header = ShardBlockHeader( + shard_parent_root=shard_parent_root, + beacon_parent_root=beacon_parent_root, + proposer_index=proposal_index, + slot=offset_slots[i], + body_root=transition.shard_data_roots[i] + ) + shard_parent_root = hash_tree_root(header) + + if not is_empty_proposal: + # Only add non-empty signature + headers.append(header) + proposers.append(proposal_index) + # Verify correct calculation of gas prices and slots + assert shard_state.gasprice == get_updated_gasprice(prev_gasprice, shard_block_length) + assert shard_state.slot == offset_slots[i] + prev_gasprice = shard_state.gasprice pubkeys = [state.validators[proposer].pubkey for proposer in proposers] signing_roots = [ @@ -745,7 +752,8 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr assert bls.AggregateVerify(zip(pubkeys, signing_roots), signature=transition.proposer_signature_aggregate) # Save updated state - state.shard_states[shard] = transition.shard_states[-1] + state.shard_states[shard] = transition.shard_states[len(transition.shard_states) - 1] + assert state.slot > 0 state.shard_states[shard].slot = state.slot - 1 ``` @@ -779,7 +787,9 @@ def process_crosslink_for_shard(state: BeaconState, # Attestation <-> shard transition consistency assert shard_transition_root == hash_tree_root(shard_transition) - assert attestation.data.head_shard_root == shard_transition.shard_data_roots[-1] + assert attestation.data.head_shard_root == shard_transition.shard_data_roots[ + len(shard_transition.shard_data_roots) - 1 + ] # Apply transition apply_shard_transition(state, shard, shard_transition) @@ -790,11 +800,11 @@ def process_crosslink_for_shard(state: BeaconState, increase_balance(state, beacon_proposer_index, proposer_reward) states_slots_lengths = zip( shard_transition.shard_states, - get_offset_slots(state, get_latest_slot_for_shard(state, shard)), + get_offset_slots(state, shard), shard_transition.shard_block_lengths ) for shard_state, slot, length in states_slots_lengths: - proposer_index = get_shard_proposer_index(state, shard, slot) + proposer_index = get_shard_proposer_index(state, slot, shard) decrease_balance(state, proposer_index, shard_state.gasprice * length) # Return winning transition root @@ -913,14 +923,13 @@ def process_light_client_signatures(state: BeaconState, block_body: BeaconBlockB total_reward += get_base_reward(state, participant_index) increase_balance(state, get_beacon_proposer_index(state), Gwei(total_reward // PROPOSER_REWARD_QUOTIENT)) - + slot = get_previous_slot(state.slot) - signing_root = compute_signing_root(get_block_root_at_slot(state, slot), + signing_root = compute_signing_root(get_block_root_at_slot(state, slot), get_domain(state, DOMAIN_LIGHT_CLIENT, compute_epoch_at_slot(slot))) assert bls.FastAggregateVerify(signer_pubkeys, signing_root, signature=block_body.light_client_signature) ``` - ### Epoch transition This epoch transition overrides the phase0 epoch transition: diff --git a/specs/phase1/fraud-proofs.md b/specs/phase1/fraud-proofs.md index e72a8bb2a..30b94d810 100644 --- a/specs/phase1/fraud-proofs.md +++ b/specs/phase1/fraud-proofs.md @@ -9,6 +9,8 @@ - [Shard state transition function](#shard-state-transition-function) - [Verifying the proof](#verifying-the-proof) - [Honest committee member behavior](#honest-committee-member-behavior) + - [Helper functions](#helper-functions) + - [Make attestations](#make-attestations) @@ -30,76 +32,76 @@ This document describes the shard transition function and fraud proofs as part o ## Fraud proofs -TODO. The intent is to have a single universal fraud proof type, which contains the following parts: - -1. An on-time attestation on some `shard` signing a `ShardTransition` -2. An index `index` of a particular position to focus on -3. The `ShardTransition` itself -4. The full body of the block `ShardBlock` -5. A Merkle proof to the `shard_states` in the parent block `parent_block` the attestation is referencing - ### Shard state transition function ```python def shard_state_transition(beacon_state: BeaconState, shard: Shard, slot: Slot, - pre_state: Root, - previous_beacon_root: Root, - proposer_index: ValidatorIndex, - signed_block: SignedShardBlock, - validate_result: bool=True) -> Root: - # We will add something more substantive in phase 2 - - # Verify the proposer_index and signature - assert proposer_index == signed_block.message.proposer_index - if validate_result: - assert verify_shard_block_signature(beacon_state, signed_block) - - return hash(pre_state + hash_tree_root(previous_beacon_root) + hash_tree_root(signed_block.message.data)) + shard_state: ShardState, + beacon_parent_root: Root, + signed_block: SignedShardBlock) -> None: + # Update shard state + shard_state.data = hash( + hash_tree_root(shard_state) + hash_tree_root(beacon_parent_root) + hash_tree_root(signed_block.message.body) + ) + shard_state.slot = slot + shard_state.latest_block_root = hash_tree_root(signed_block.message) ``` ```python def verify_shard_block_signature(beacon_state: BeaconState, signed_block: SignedShardBlock) -> bool: proposer = beacon_state.validators[signed_block.message.proposer_index] - signing_root = compute_signing_root(signed_block.message, get_domain(beacon_state, DOMAIN_SHARD_PROPOSAL)) + domain = get_domain(beacon_state, DOMAIN_SHARD_PROPOSAL, compute_epoch_at_slot(signed_block.message.slot)) + signing_root = compute_signing_root(signed_block.message, domain) return bls.Verify(proposer.pubkey, signing_root, signed_block.signature) ``` ### Verifying the proof +TODO. The intent is to have a single universal fraud proof type, which contains the following parts: + +1. An on-time attestation `attestation` on some shard `shard` signing a `transition: ShardTransition` +2. An index `offset_index` of a particular position to focus on +3. The `transition: ShardTransition` itself +4. The full body of the shard block `shard_block` +5. A Merkle proof to the `shard_states` in the parent block the attestation is referencing + +Call the following function to verify the proof: + ```python def verify_fraud_proof(beacon_state: BeaconState, - subkey: BLSPubkey, attestation: Attestation, - index: uint64, + offset_index: uint64, transition: ShardTransition, signed_block: SignedShardBlock, - parent_block: ShardBlock) -> bool: - # 1. Check if `custody_bits[index][j] != generate_custody_bit(subkey, block_contents)` for any `j` + subkey: BLSPubkey, + beacon_parent_block: BeaconBlock) -> bool: + # 1. Check if `custody_bits[offset_index][j] != generate_custody_bit(subkey, block_contents)` for any `j`. shard = get_shard(beacon_state, attestation) slot = attestation.data.slot custody_bits = attestation.custody_bits_blocks - for j in range(custody_bits[index]): - if custody_bits[index][j] != generate_custody_bit(subkey, signed_block): + for j in range(custody_bits[offset_index]): + if custody_bits[offset_index][j] != generate_custody_bit(subkey, signed_block): return True - # 2. Verify the shard state transition - if index == 0: - parent_data = parent_block.shard_states[shard][-1].data + # 2. Check if the shard state transition result is wrong between + # `transition.shard_states[offset_index - 1]` to `transition.shard_states[offset_index]`. + if offset_index == 0: + shard_state = beacon_parent_block.shard_transitions[shard][-1] else: - parent_data = parent_block.shard_states[shard][index].data + shard_state = transition.shard_states[offset_index - 1].copy() # Not doing the actual state updates here. - if shard_state_transition( - beacon_state, - shard, - slot, - transition.shard_states[index - 1].data, - hash_tree_root(parent_block), - get_shard_proposer_index(beacon_state, slot, shard), - signed_block, - ) != parent_data: + shard_state_transition( + beacon_state=beacon_state, + shard=shard, + slot=slot, + shard_state=shard_state, + beacon_parent_root=hash_tree_root(beacon_parent_block), + signed_block=signed_block, + ) + if shard_state.latest_block_root != transition.shard_states[offset_index].data: return True return False @@ -113,14 +115,141 @@ def generate_custody_bit(subkey: BLSPubkey, block: ShardBlock) -> bool: ## Honest committee member behavior -Suppose you are a committee member on shard `shard` at slot `current_slot`. Let `state` be the head beacon state you are building on, and let `QUARTER_PERIOD = SECONDS_PER_SLOT // 4`. `2 * QUARTER_PERIOD` seconds into slot `slot`, run the following procedure: +### Helper functions -* Initialize `proposals = []`, `shard_states = []`, `shard_state = state.shard_states[shard][-1]`, `start_slot = shard_state.slot`. -* For `slot in get_offset_slots(state, start_slot)`, do the following: - * Look for all valid proposals for `slot`; that is, a SignedShardBlock `proposal` where `shard_state_transition(shard, slot, shard_state, get_block_root_at_slot(state, state.slot - 1), get_shard_proposer_index(state, slot, shard), proposal)` returns a result and does not throw an exception. Let `choices` be the set of non-empty valid proposals you discover. - * If `len(choices) == 0`, do `proposals.append(make_empty_proposal(shard_state, slot))` - * If `len(choices) == 1`, do `proposals.append(choices[0])` - * If `len(choices) > 1`, let `winning_proposal` be the proposal with the largest number of total attestations from slots in `state.shard_next_slots[shard]....slot-1` supporting it or any of its descendants, breaking ties by choosing the first proposal locally seen. Do `proposals.append(winning_proposal)`. - * If `proposals.message.data[-1]` is NOT an empty proposal, set `shard_state = shard_state_transition(shard, slot, shard_state, get_block_root_at_slot(state, state.slot - 1), get_shard_proposer_index(state, slot, shard), proposals[-1])` and do `shard_states.append(shard_state)`. If it is an empty proposal, leave `shard_state` unchanged. +```python +def get_winning_proposal(beacon_state: BeaconState, proposals: Sequence[SignedShardBlock]) -> SignedShardBlock: + # TODO: Let `winning_proposal` be the proposal with the largest number of total attestationsfrom slots in + # `state.shard_next_slots[shard]....slot-1` supporting it or any of its descendants, breaking ties by choosing + # the first proposal locally seen. Do `proposals.append(winning_proposal)`. + return proposals[-1] # stub +``` -Make an attestation using `shard_data_roots = [hash_tree_root(proposal.message.data) for proposal in proposals]` and `shard_state_roots = shard_states`. +```python +def get_empty_body_block(shard_parent_root: Root, + beacon_parent_root: Root, + slot: Slot, + proposer_index: ValidatorIndex) -> ShardBlock: + return ShardBlock( + shard_parent_root=shard_parent_root, + beacon_parent_root=beacon_parent_root, + slot=slot, + proposer_index=proposer_index, + ) +``` + +```python +def is_empty_body(proposal: ShardBlock) -> bool: + # TODO + return len(proposal.body) == 0 +``` + +```python +def compute_shard_data_roots(proposals: Sequence[SignedShardBlock]) -> Sequence[Root]: + return [hash_tree_root(proposal.message.body) for proposal in proposals] +``` + +### Make attestations + +Suppose you are a committee member on shard `shard` at slot `current_slot` and you have received shard blocks `shard_blocks`. Let `state` be the head beacon state you are building on, and let `QUARTER_PERIOD = SECONDS_PER_SLOT // 4`. `2 * QUARTER_PERIOD` seconds into slot `current_slot`, run `get_shard_transition(beacon_state, shard, shard_blocks)` to get `shard_transition`. + +```python +def get_shard_transition(beacon_state: BeaconState, + shard: Shard, + shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition: + offset_slots = get_offset_slots(beacon_state, shard) + start_slot = offset_slots[0] + proposals, shard_states, shard_data_roots = get_shard_state_transition_result(beacon_state, shard, shard_blocks) + + assert len(proposals) > 0 + assert len(shard_data_roots) > 0 + + shard_block_lengths = [] + proposer_signatures = [] + for proposal in proposals: + shard_block_lengths.append(len(proposal.message.body)) + if proposal.signature != BLSSignature(): + proposer_signatures.append(proposal.signature) + + proposer_signature_aggregate = bls.Aggregate(proposer_signatures) + + return ShardTransition( + start_slot=start_slot, + shard_block_lengths=shard_block_lengths, + shard_data_roots=shard_data_roots, + shard_states=shard_states, + proposer_signature_aggregate=proposer_signature_aggregate, + ) +``` + +```python +def get_shard_state_transition_result( + beacon_state: BeaconState, + shard: Shard, + shard_blocks: Sequence[SignedShardBlock], + validate_result: bool=True, +) -> Tuple[Sequence[SignedShardBlock], Sequence[ShardState], Sequence[Root]]: + proposals = [] + shard_states = [] + shard_state = beacon_state.shard_states[shard].copy() + shard_parent_root = beacon_state.shard_states[shard].latest_block_root + for slot in get_offset_slots(beacon_state, shard): + choices = [] + beacon_parent_root = get_block_root_at_slot(beacon_state, get_previous_slot(beacon_state.slot)) + proposer_index = get_shard_proposer_index(beacon_state, slot, shard) + shard_blocks_at_slot = [block for block in shard_blocks if block.message.slot == slot] + for block in shard_blocks_at_slot: + temp_shard_state = shard_state.copy() # Not doing the actual state updates here. + # Try to apply state transition to temp_shard_state. + try: + # Verify the proposer_index and signature + assert block.message.proposer_index == proposer_index + if validate_result: + assert verify_shard_block_signature(beacon_state, block) + + shard_state_transition( + beacon_state=beacon_state, + shard=shard, + slot=slot, + shard_state=temp_shard_state, + beacon_parent_root=beacon_parent_root, + signed_block=block, + ) + except Exception: + pass # TODO: throw error in the test helper + else: + choices.append(block) + + if len(choices) == 0: + block_header = get_empty_body_block( + shard_parent_root=shard_parent_root, + beacon_parent_root=beacon_parent_root, + slot=slot, + proposer_index=proposer_index, + ) + block = SignedShardBlock(message=block_header) + proposals.append(block) + elif len(choices) == 1: + proposals.append(choices[0]) + else: + proposals.append(get_winning_proposal(beacon_state, choices)) + + shard_parent_root = hash_tree_root(proposals[-1].message) + + if not is_empty_body(proposals[-1].message): + # Apply state transition to shard_state. + shard_state_transition( + beacon_state=beacon_state, + shard=shard, + slot=slot, + shard_state=shard_state, + beacon_parent_root=beacon_parent_root, + signed_block=block, + ) + + shard_states.append(shard_state) + + shard_data_roots = compute_shard_data_roots(proposals) + + return proposals, shard_states, shard_data_roots +``` diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 8215f5c5b..c203b737a 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -1,6 +1,6 @@ from typing import List -from eth2spec.test.context import expect_assertion_error, PHASE0 +from eth2spec.test.context import expect_assertion_error, PHASE0, PHASE1 from eth2spec.test.helpers.state import state_transition_and_sign_block from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.keys import privkeys @@ -43,7 +43,7 @@ def run_attestation_processing(spec, state, attestation, valid=True): yield 'post', state -def build_attestation_data(spec, state, slot, index): +def build_attestation_data(spec, state, slot, index, shard_transition=None): assert state.slot >= slot if slot == state.slot: @@ -66,12 +66,21 @@ def build_attestation_data(spec, state, slot, index): source_epoch = state.current_justified_checkpoint.epoch source_root = state.current_justified_checkpoint.root + if spec.fork == PHASE1 and shard_transition is not None: + shard_transition_root = shard_transition.hash_tree_root() + head_shard_root = shard_transition.shard_data_roots[len(shard_transition.shard_data_roots) - 1] + else: + shard_transition_root = spec.Root() + head_shard_root = spec.Root() + return spec.AttestationData( slot=slot, index=index, beacon_block_root=block_root, source=spec.Checkpoint(epoch=source_epoch, root=source_root), target=spec.Checkpoint(epoch=spec.compute_epoch_at_slot(slot), root=epoch_boundary_root), + head_shard_root=head_shard_root, + shard_transition_root=shard_transition_root, ) @@ -89,7 +98,7 @@ def convert_to_valid_on_time_attestation(spec, state, attestation, signed=False) return attestation -def get_valid_on_time_attestation(spec, state, slot=None, index=None, signed=False): +def get_valid_on_time_attestation(spec, state, slot=None, index=None, shard_transition=None, signed=False): ''' Construct on-time attestation for next slot ''' @@ -98,7 +107,15 @@ def get_valid_on_time_attestation(spec, state, slot=None, index=None, signed=Fal if index is None: index = 0 - return get_valid_attestation(spec, state, slot=slot, index=index, signed=signed, on_time=True) + return get_valid_attestation( + spec, + state, + slot=slot, + index=index, + shard_transition=shard_transition, + signed=signed, + on_time=True, + ) def get_valid_late_attestation(spec, state, slot=None, index=None, signed=False): @@ -113,13 +130,20 @@ def get_valid_late_attestation(spec, state, slot=None, index=None, signed=False) return get_valid_attestation(spec, state, slot=slot, index=index, signed=signed, on_time=False) -def get_valid_attestation(spec, state, slot=None, index=None, empty=False, signed=False, on_time=True): +def get_valid_attestation(spec, + state, + slot=None, + index=None, + shard_transition=None, + empty=False, + signed=False, + on_time=True): if slot is None: slot = state.slot if index is None: index = 0 - attestation_data = build_attestation_data(spec, state, slot, index) + attestation_data = build_attestation_data(spec, state, slot=slot, index=index, shard_transition=shard_transition) beacon_committee = spec.get_beacon_committee( state, @@ -138,7 +162,7 @@ def get_valid_attestation(spec, state, slot=None, index=None, empty=False, signe if signed: sign_attestation(spec, state, attestation) - if spec.fork == 'phase1' and on_time: + if spec.fork == PHASE1 and on_time: attestation = convert_to_valid_on_time_attestation(spec, state, attestation, signed) return attestation @@ -210,7 +234,7 @@ def get_attestation_custody_signature(spec, state, attestation_data, block_index def sign_attestation(spec, state, attestation): - if spec.fork == 'phase1' and any(attestation.custody_bits_blocks): + if spec.fork == PHASE1 and any(attestation.custody_bits_blocks): sign_on_time_attestation(spec, state, attestation) return diff --git a/tests/core/pyspec/eth2spec/test/helpers/crosslinks.py b/tests/core/pyspec/eth2spec/test/helpers/crosslinks.py new file mode 100644 index 000000000..ea5da89d9 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/crosslinks.py @@ -0,0 +1,28 @@ +from eth2spec.test.context import expect_assertion_error + + +def run_crosslinks_processing(spec, state, shard_transitions, attestations, valid=True): + """ + Run ``process_attestation``, yielding: + - pre-state ('pre') + - shard_transitions ('shard_transitions') + - attestations ('attestations') + - post-state ('post'). + If ``valid == False``, run expecting ``AssertionError`` + """ + # yield pre-state + yield 'pre', state + yield 'shard_transitions', shard_transitions + yield 'attestations', attestations + + # If the attestation is invalid, processing is aborted, and there is no post-state. + if not valid: + expect_assertion_error(lambda: spec.process_crosslinks(state, shard_transitions, attestations)) + yield 'post', None + return + + # process crosslinks + spec.process_crosslinks(state, shard_transitions, attestations) + + # yield post-state + yield 'post', state diff --git a/tests/core/pyspec/eth2spec/test/helpers/phase1/__init__.py b/tests/core/pyspec/eth2spec/test/helpers/phase1/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py deleted file mode 100644 index 0e16e1fac..000000000 --- a/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py +++ /dev/null @@ -1,63 +0,0 @@ -from eth2spec.utils.ssz.ssz_typing import Bitlist -from eth2spec.utils import bls - -from eth2spec.test.helpers.keys import privkeys -import eth2spec.test.helpers.attestations as phase0_attestations - - -def get_valid_on_time_attestation(spec, state, index=None, signed=False): - ''' - Construct on-time attestation for next slot - ''' - if index is None: - index = 0 - - attestation = phase0_attestations.get_valid_attestation(spec, state, state.slot, index, False) - shard = spec.get_shard(state, attestation) - offset_slots = spec.compute_offset_slots(spec.get_latest_slot_for_shard(state, shard), state.slot + 1) - - for _ in offset_slots: - attestation.custody_bits_blocks.append( - Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE]([0 for _ in attestation.aggregation_bits]) - ) - - if signed: - sign_attestation(spec, state, attestation) - - return attestation - - -def sign_attestation(spec, state, attestation): - if not any(attestation.custody_bits_blocks): - phase0_attestations.sign_attestation(spec, state, attestation) - return - - committee = spec.get_beacon_committee(state, attestation.data.slot, attestation.data.index) - signatures = [] - for block_index, custody_bits in enumerate(attestation.custody_bits_blocks): - for participant, abit, cbit in zip(committee, attestation.aggregation_bits, custody_bits): - if not abit: - continue - signatures.append(get_attestation_custody_signature( - spec, - state, - attestation.data, - block_index, - cbit, - privkeys[participant] - )) - - attestation.signature = bls.Aggregate(signatures) - - -def get_attestation_custody_signature(spec, state, attestation_data, block_index, bit, privkey): - domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) - signing_root = spec.compute_signing_root( - spec.AttestationCustodyBitWrapper( - attestation_data.hash_tree_root(), - block_index, - bit, - ), - domain, - ) - return bls.Sign(privkey, signing_root) diff --git a/tests/core/pyspec/eth2spec/test/helpers/phase1/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/phase1/shard_block.py deleted file mode 100644 index 6ef0cf79b..000000000 --- a/tests/core/pyspec/eth2spec/test/helpers/phase1/shard_block.py +++ /dev/null @@ -1,71 +0,0 @@ -from eth2spec.test.helpers.keys import privkeys -from eth2spec.utils import bls -from eth2spec.utils.bls import only_with_bls -from eth2spec.utils.ssz.ssz_impl import ( - hash_tree_root, -) - -from .attestations import ( - sign_shard_attestation, -) - - -@only_with_bls() -def sign_shard_block(spec, beacon_state, shard_state, block, proposer_index=None): - if proposer_index is None: - proposer_index = spec.get_shard_proposer_index(beacon_state, shard_state.shard, block.slot) - - privkey = privkeys[proposer_index] - domain = spec.get_domain(beacon_state, spec.DOMAIN_SHARD_PROPOSER, spec.compute_epoch_of_shard_slot(block.slot)) - signing_root = spec.compute_signing_root(block, domain) - block.signature = bls.Sign(privkey, signing_root) - - -def build_empty_shard_block(spec, - beacon_state, - shard_state, - slot, - signed=False, - full_attestation=False): - if slot is None: - slot = shard_state.slot - - previous_beacon_header = beacon_state.latest_block_header.copy() - if previous_beacon_header.state_root == spec.Bytes32(): - previous_beacon_header.state_root = beacon_state.hash_tree_root() - beacon_block_root = hash_tree_root(previous_beacon_header) - - previous_block_header = shard_state.latest_block_header.copy() - if previous_block_header.state_root == spec.Bytes32(): - previous_block_header.state_root = shard_state.hash_tree_root() - parent_root = hash_tree_root(previous_block_header) - - block = spec.ShardBlock( - shard=shard_state.shard, - slot=slot, - beacon_block_root=beacon_block_root, - parent_root=parent_root, - block_size_sum=shard_state.block_size_sum + spec.SHARD_HEADER_SIZE, - ) - - if full_attestation: - shard_committee = spec.get_shard_committee(beacon_state, shard_state.shard, block.slot) - block.aggregation_bits = list( - (True,) * len(shard_committee) + - (False,) * (spec.MAX_PERIOD_COMMITTEE_SIZE * 2 - len(shard_committee)) - ) - else: - shard_committee = [] - - block.attestations = sign_shard_attestation( - spec, - beacon_state, - shard_state, - block, - participants=shard_committee, - ) - - if signed: - sign_shard_block(spec, beacon_state, shard_state, block) - - return block diff --git a/tests/core/pyspec/eth2spec/test/helpers/phase1/shard_state.py b/tests/core/pyspec/eth2spec/test/helpers/phase1/shard_state.py deleted file mode 100644 index 24240b5fa..000000000 --- a/tests/core/pyspec/eth2spec/test/helpers/phase1/shard_state.py +++ /dev/null @@ -1,18 +0,0 @@ -from eth2spec.test.helpers.phase1.shard_block import sign_shard_block - - -def configure_shard_state(spec, beacon_state, shard=0): - beacon_state.slot = spec.Slot(spec.SHARD_GENESIS_EPOCH * spec.SLOTS_PER_EPOCH) - shard_state = spec.get_genesis_shard_state(spec.Shard(shard)) - shard_state.slot = spec.ShardSlot(spec.SHARD_GENESIS_EPOCH * spec.SHARD_SLOTS_PER_EPOCH) - return beacon_state, shard_state - - -def shard_state_transition_and_sign_block(spec, beacon_state, shard_state, block): - """ - Shard state transition via the provided ``block`` - then package the block with the state root and signature. - """ - spec.shard_state_transition(beacon_state, shard_state, block) - block.state_root = shard_state.hash_tree_root() - sign_shard_block(spec, beacon_state, shard_state, block) diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py new file mode 100644 index 000000000..0cb8a3dfd --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -0,0 +1,47 @@ +from eth2spec.test.helpers.keys import privkeys +from eth2spec.utils import bls +from eth2spec.utils.bls import only_with_bls + + +@only_with_bls() +def sign_shard_block(spec, beacon_state, shard, block, proposer_index=None): + slot = block.message.slot + if proposer_index is None: + proposer_index = spec.get_shard_proposer_index(beacon_state, slot, shard) + + privkey = privkeys[proposer_index] + domain = spec.get_domain(beacon_state, spec.DOMAIN_SHARD_PROPOSAL, spec.compute_epoch_at_slot(slot)) + signing_root = spec.compute_signing_root(block.message, domain) + block.signature = bls.Sign(privkey, signing_root) + + +def build_empty_shard_block(spec, + beacon_state, + shard, + slot, + body=None, + signed=False): + shard_state = beacon_state.shard_states[shard] + if slot is None: + slot = shard_state.slot + + if body is None: + body = [] + + proposer_index = spec.get_shard_proposer_index(beacon_state, slot, shard) + + block = spec.ShardBlock( + shard_parent_root=shard_state.latest_block_root, + beacon_parent_root=spec.get_block_root_at_slot(beacon_state, spec.get_previous_slot(beacon_state.slot)), + slot=slot, + proposer_index=proposer_index, + body=body, + ) + signed_block = spec.SignedShardBlock( + message=block, + ) + + if signed: + sign_shard_block(spec, beacon_state, shard, signed_block, proposer_index=proposer_index) + + return signed_block diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py new file mode 100644 index 000000000..1d5fe7c02 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py @@ -0,0 +1,52 @@ +from eth2spec.test.context import ( + with_all_phases_except, + spec_state_test, + always_bls, +) +from eth2spec.test.helpers.attestations import ( + get_valid_on_time_attestation, +) +from eth2spec.test.helpers.crosslinks import ( + run_crosslinks_processing, +) +from eth2spec.test.helpers.shard_block import build_empty_shard_block +from eth2spec.test.helpers.state import next_epoch, next_slot + + +@with_all_phases_except(['phase0']) +@spec_state_test +@always_bls +def test_basic_crosslinks(spec, state): + next_epoch(spec, state) + next_epoch(spec, state) + state = spec.upgrade_to_phase1(state) + next_slot(spec, state) + + committee_index = spec.CommitteeIndex(0) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard_block = build_empty_shard_block(spec, state, shard, body=b'\x12' * 10, slot=state.slot, signed=True) + shard_blocks = [shard_block] + + next_slot(spec, state) + + shard_transition = spec.get_shard_transition(state, shard, shard_blocks) + shard_transitions = [spec.ShardTransition()] * len(state.shard_states) + shard_transitions[shard] = shard_transition + + attestation = get_valid_on_time_attestation( + spec, + state, + slot=state.slot, + index=committee_index, + shard_transition=shard_transition, + signed=True, + ) + attestations = [attestation] + + offset_slots = spec.get_offset_slots(state, shard) + + yield from run_crosslinks_processing(spec, state, shard_transitions, attestations) + + shard_state = state.shard_states[shard] + assert shard_state.slot == offset_slots[-1] + assert shard_state.latest_block_root == shard_block.message.hash_tree_root() From afa12caf1f3d2595c79e1a53e901c9639e98f500 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 11 Apr 2020 01:12:40 +0800 Subject: [PATCH 089/102] Refactor `get_shard_state_transition_result` --- specs/phase1/fraud-proofs.md | 184 +++++++++++++++++++++-------------- 1 file changed, 112 insertions(+), 72 deletions(-) diff --git a/specs/phase1/fraud-proofs.md b/specs/phase1/fraud-proofs.md index 30b94d810..5af904ff5 100644 --- a/specs/phase1/fraud-proofs.md +++ b/specs/phase1/fraud-proofs.md @@ -149,6 +149,118 @@ def compute_shard_data_roots(proposals: Sequence[SignedShardBlock]) -> Sequence[ return [hash_tree_root(proposal.message.body) for proposal in proposals] ``` +```python +def get_proposal_choices_at_slot(beacon_state: BeaconState, + shard_state: ShardState, + slot: Slot, + shard: Shard, + shard_blocks: Sequence[SignedShardBlock], + validate_result: bool=True) -> Sequence[SignedShardBlock]: + choices = [] + beacon_parent_root = get_block_root_at_slot(beacon_state, get_previous_slot(beacon_state.slot)) + proposer_index = get_shard_proposer_index(beacon_state, slot, shard) + shard_blocks_at_slot = [block for block in shard_blocks if block.message.slot == slot] + for block in shard_blocks_at_slot: + temp_shard_state = shard_state.copy() # Not doing the actual state updates here. + # Try to apply state transition to temp_shard_state. + try: + # Verify the proposer_index and signature + assert block.message.proposer_index == proposer_index + if validate_result: + assert verify_shard_block_signature(beacon_state, block) + + shard_state_transition( + beacon_state=beacon_state, + shard=shard, + slot=slot, + shard_state=temp_shard_state, + beacon_parent_root=beacon_parent_root, + signed_block=block, + ) + except Exception: + pass # TODO: throw error in the test helper + else: + choices.append(block) + return choices +``` + +```python +def get_proposal_at_slot(beacon_state: BeaconState, + shard_state: ShardState, + slot: Shard, + shard: Shard, + shard_parent_root: Root, + shard_blocks: Sequence[SignedShardBlock], + validate_result: bool=True) -> Tuple[SignedShardBlock, ShardState, Root]: + beacon_parent_root = get_block_root_at_slot(beacon_state, get_previous_slot(beacon_state.slot)) + proposer_index = get_shard_proposer_index(beacon_state, slot, shard) + shard_state = shard_state.copy() # Don't update the given shard_state + choices = get_proposal_choices_at_slot( + beacon_state=beacon_state, + shard_state=shard_state, + slot=slot, + shard=shard, + shard_blocks=shard_blocks, + validate_result=validate_result, + ) + if len(choices) == 0: + block_header = get_empty_body_block( + shard_parent_root=shard_parent_root, + beacon_parent_root=beacon_parent_root, + slot=slot, + proposer_index=proposer_index, + ) + proposal = SignedShardBlock(message=block_header) + elif len(choices) == 1: + proposal = choices[0] + else: + proposal = get_winning_proposal(beacon_state, choices) + + shard_parent_root = hash_tree_root(proposal.message) + + if not is_empty_body(proposal.message): + # Apply state transition to shard_state. + shard_state_transition( + beacon_state=beacon_state, + shard=shard, + slot=slot, + shard_state=shard_state, + beacon_parent_root=beacon_parent_root, + signed_block=proposal, + ) + + return proposal, shard_state, shard_parent_root +``` + +```python +def get_shard_state_transition_result( + beacon_state: BeaconState, + shard: Shard, + shard_blocks: Sequence[SignedShardBlock], + validate_result: bool=True, +) -> Tuple[Sequence[SignedShardBlock], Sequence[ShardState], Sequence[Root]]: + proposals = [] + shard_states = [] + shard_state = beacon_state.shard_states[shard].copy() + shard_parent_root = beacon_state.shard_states[shard].latest_block_root + for slot in get_offset_slots(beacon_state, shard): + proposal, shard_state, shard_parent_root = get_proposal_at_slot( + beacon_state=beacon_state, + shard_state=shard_state, + slot=slot, + shard=shard, + shard_parent_root=shard_parent_root, + shard_blocks=shard_blocks, + validate_result=validate_result, + ) + shard_states.append(shard_state) + proposals.append(proposal) + + shard_data_roots = compute_shard_data_roots(proposals) + + return proposals, shard_states, shard_data_roots +``` + ### Make attestations Suppose you are a committee member on shard `shard` at slot `current_slot` and you have received shard blocks `shard_blocks`. Let `state` be the head beacon state you are building on, and let `QUARTER_PERIOD = SECONDS_PER_SLOT // 4`. `2 * QUARTER_PERIOD` seconds into slot `current_slot`, run `get_shard_transition(beacon_state, shard, shard_blocks)` to get `shard_transition`. @@ -181,75 +293,3 @@ def get_shard_transition(beacon_state: BeaconState, proposer_signature_aggregate=proposer_signature_aggregate, ) ``` - -```python -def get_shard_state_transition_result( - beacon_state: BeaconState, - shard: Shard, - shard_blocks: Sequence[SignedShardBlock], - validate_result: bool=True, -) -> Tuple[Sequence[SignedShardBlock], Sequence[ShardState], Sequence[Root]]: - proposals = [] - shard_states = [] - shard_state = beacon_state.shard_states[shard].copy() - shard_parent_root = beacon_state.shard_states[shard].latest_block_root - for slot in get_offset_slots(beacon_state, shard): - choices = [] - beacon_parent_root = get_block_root_at_slot(beacon_state, get_previous_slot(beacon_state.slot)) - proposer_index = get_shard_proposer_index(beacon_state, slot, shard) - shard_blocks_at_slot = [block for block in shard_blocks if block.message.slot == slot] - for block in shard_blocks_at_slot: - temp_shard_state = shard_state.copy() # Not doing the actual state updates here. - # Try to apply state transition to temp_shard_state. - try: - # Verify the proposer_index and signature - assert block.message.proposer_index == proposer_index - if validate_result: - assert verify_shard_block_signature(beacon_state, block) - - shard_state_transition( - beacon_state=beacon_state, - shard=shard, - slot=slot, - shard_state=temp_shard_state, - beacon_parent_root=beacon_parent_root, - signed_block=block, - ) - except Exception: - pass # TODO: throw error in the test helper - else: - choices.append(block) - - if len(choices) == 0: - block_header = get_empty_body_block( - shard_parent_root=shard_parent_root, - beacon_parent_root=beacon_parent_root, - slot=slot, - proposer_index=proposer_index, - ) - block = SignedShardBlock(message=block_header) - proposals.append(block) - elif len(choices) == 1: - proposals.append(choices[0]) - else: - proposals.append(get_winning_proposal(beacon_state, choices)) - - shard_parent_root = hash_tree_root(proposals[-1].message) - - if not is_empty_body(proposals[-1].message): - # Apply state transition to shard_state. - shard_state_transition( - beacon_state=beacon_state, - shard=shard, - slot=slot, - shard_state=shard_state, - beacon_parent_root=beacon_parent_root, - signed_block=block, - ) - - shard_states.append(shard_state) - - shard_data_roots = compute_shard_data_roots(proposals) - - return proposals, shard_states, shard_data_roots -``` From e645d6b5fa7490134c1b30d98ae29aaa4cb46f00 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 13 Apr 2020 18:23:01 +0800 Subject: [PATCH 090/102] Rename `build_empty_shard_block` to `build_shard_block` --- .../core/pyspec/eth2spec/test/helpers/shard_block.py | 12 ++++++------ .../block_processing/test_process_crosslink.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 0cb8a3dfd..8f95cf73a 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -15,12 +15,12 @@ def sign_shard_block(spec, beacon_state, shard, block, proposer_index=None): block.signature = bls.Sign(privkey, signing_root) -def build_empty_shard_block(spec, - beacon_state, - shard, - slot, - body=None, - signed=False): +def build_shard_block(spec, + beacon_state, + shard, + slot, + body=None, + signed=False): shard_state = beacon_state.shard_states[shard] if slot is None: slot = shard_state.slot diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py index 1d5fe7c02..1585a1a45 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py @@ -9,7 +9,7 @@ from eth2spec.test.helpers.attestations import ( from eth2spec.test.helpers.crosslinks import ( run_crosslinks_processing, ) -from eth2spec.test.helpers.shard_block import build_empty_shard_block +from eth2spec.test.helpers.shard_block import build_shard_block from eth2spec.test.helpers.state import next_epoch, next_slot @@ -24,7 +24,7 @@ def test_basic_crosslinks(spec, state): committee_index = spec.CommitteeIndex(0) shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) - shard_block = build_empty_shard_block(spec, state, shard, body=b'\x12' * 10, slot=state.slot, signed=True) + shard_block = build_shard_block(spec, state, shard, body=b'\x12' * 10, slot=state.slot, signed=True) shard_blocks = [shard_block] next_slot(spec, state) From 9724cb832d9f909b21f88683721389ff90e70756 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 Apr 2020 17:18:11 +0800 Subject: [PATCH 091/102] Apply suggestions from code review from @djrtwo Co-Authored-By: Danny Ryan --- specs/phase1/beacon-chain.md | 8 ++++---- specs/phase1/fraud-proofs.md | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 4b22f8633..2410c534d 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -465,12 +465,12 @@ def get_shard_committee(beacon_state: BeaconState, epoch: Epoch, shard: Shard) - source_epoch -= SHARD_COMMITTEE_PERIOD active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) seed = get_seed(beacon_state, source_epoch, DOMAIN_SHARD_COMMITTEE) - active_shards_count = get_active_shard_count(beacon_state) + active_shard_count = get_active_shard_count(beacon_state) return compute_committee( indices=active_validator_indices, seed=seed, index=shard, - count=active_shards_count, + count=active_shard_count, ) ``` @@ -483,12 +483,12 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque source_epoch -= LIGHT_CLIENT_COMMITTEE_PERIOD active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) seed = get_seed(beacon_state, source_epoch, DOMAIN_LIGHT_CLIENT) - active_shards_count = get_active_shard_count(beacon_state) + active_shard_count = get_active_shard_count(beacon_state) return compute_committee( indices=active_validator_indices, seed=seed, index=0, - count=active_shards_count, + count=active_shard_count, )[:TARGET_COMMITTEE_SIZE] ``` diff --git a/specs/phase1/fraud-proofs.md b/specs/phase1/fraud-proofs.md index 5af904ff5..b476508a4 100644 --- a/specs/phase1/fraud-proofs.md +++ b/specs/phase1/fraud-proofs.md @@ -89,7 +89,7 @@ def verify_fraud_proof(beacon_state: BeaconState, # 2. Check if the shard state transition result is wrong between # `transition.shard_states[offset_index - 1]` to `transition.shard_states[offset_index]`. if offset_index == 0: - shard_state = beacon_parent_block.shard_transitions[shard][-1] + shard_state = beacon_parent_block.shard_transitions[shard].shard_states[-1] else: shard_state = transition.shard_states[offset_index - 1].copy() # Not doing the actual state updates here. @@ -119,7 +119,7 @@ def generate_custody_bit(subkey: BLSPubkey, block: ShardBlock) -> bool: ```python def get_winning_proposal(beacon_state: BeaconState, proposals: Sequence[SignedShardBlock]) -> SignedShardBlock: - # TODO: Let `winning_proposal` be the proposal with the largest number of total attestationsfrom slots in + # TODO: Let `winning_proposal` be the proposal with the largest number of total attestations from slots in # `state.shard_next_slots[shard]....slot-1` supporting it or any of its descendants, breaking ties by choosing # the first proposal locally seen. Do `proposals.append(winning_proposal)`. return proposals[-1] # stub @@ -263,7 +263,7 @@ def get_shard_state_transition_result( ### Make attestations -Suppose you are a committee member on shard `shard` at slot `current_slot` and you have received shard blocks `shard_blocks`. Let `state` be the head beacon state you are building on, and let `QUARTER_PERIOD = SECONDS_PER_SLOT // 4`. `2 * QUARTER_PERIOD` seconds into slot `current_slot`, run `get_shard_transition(beacon_state, shard, shard_blocks)` to get `shard_transition`. +Suppose you are a committee member on shard `shard` at slot `current_slot` and you have received shard blocks `shard_blocks` since the latest successful crosslink for `shard` into the beacon chain. Let `state` be the head beacon state you are building on, and let `QUARTER_PERIOD = SECONDS_PER_SLOT // 4`. `2 * QUARTER_PERIOD` seconds into slot `current_slot`, run `get_shard_transition(beacon_state, shard, shard_blocks)` to get `shard_transition`. ```python def get_shard_transition(beacon_state: BeaconState, From 85d5a9abaf0dfbf9d652da3be3f606880b54943d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 Apr 2020 19:43:48 +0800 Subject: [PATCH 092/102] [squashed] shard chain updates wip PR feedback from Danny and some refactor 1. Add stub `PHASE_1_GENESIS_SLOT` 2. Rename `get_updated_gasprice` to `compute_updated_gasprice` 3. Rename `compute_shard_data_roots` to `compute_shard_body_roots` Apply shard transition for the skipped slots Refactor `shard_state_transition` Get `beacon_parent_root` from offset slot Add more test Add `verify_shard_block_message` Add `> 0` Keep `beacon_parent_block` unchanged in `is_valid_fraud_proof` Remove some lines Fix type Refactor + simplify skipped slot processing --- configs/mainnet.yaml | 2 + configs/minimal.yaml | 2 + specs/phase1/beacon-chain.md | 68 ++++---- specs/phase1/fraud-proofs.md | 164 ++++++++---------- specs/phase1/phase1-fork.md | 9 +- .../eth2spec/test/helpers/shard_block.py | 2 +- .../test_process_crosslink.py | 53 +++++- 7 files changed, 166 insertions(+), 134 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 6d71cfa47..f5f38de5e 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -161,6 +161,8 @@ DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 # Phase 1: Upgrade from Phase 0 # --------------------------------------------------------------- PHASE_1_FORK_VERSION: 0x01000000 +# [STUB] +PHASE_1_GENESIS_SLOT: 32 INITIAL_ACTIVE_SHARDS: 64 # Phase 1: General diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 9daf428b4..7fc255f59 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -162,6 +162,8 @@ DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 # --------------------------------------------------------------- # [customized] for testnet distinction PHASE_1_FORK_VERSION: 0x01000001 +# [customized] for testing +PHASE_1_GENESIS_SLOT: 8 # [customized] reduced for testing INITIAL_ACTIVE_SHARDS: 4 diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 2410c534d..5c67fe4f2 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -39,6 +39,7 @@ - [`committee_to_compact_committee`](#committee_to_compact_committee) - [`compute_shard_from_committee_index`](#compute_shard_from_committee_index) - [`compute_offset_slots`](#compute_offset_slots) + - [`compute_updated_gasprice`](#compute_updated_gasprice) - [Beacon state accessors](#beacon-state-accessors) - [`get_active_shard_count`](#get_active_shard_count) - [`get_online_validator_indices`](#get_online_validator_indices) @@ -46,7 +47,6 @@ - [`get_light_client_committee`](#get_light_client_committee) - [`get_shard_proposer_index`](#get_shard_proposer_index) - [`get_indexed_attestation`](#get_indexed_attestation) - - [`get_updated_gasprice`](#get_updated_gasprice) - [`get_start_shard`](#get_start_shard) - [`get_shard`](#get_shard) - [`get_latest_slot_for_shard`](#get_latest_slot_for_shard) @@ -439,6 +439,20 @@ def compute_offset_slots(start_slot: Slot, end_slot: Slot) -> Sequence[Slot]: return [Slot(start_slot + x) for x in SHARD_BLOCK_OFFSETS if start_slot + x < end_slot] ``` +#### `compute_updated_gasprice` + +```python +def compute_updated_gasprice(prev_gasprice: Gwei, length: uint8) -> Gwei: + if length > TARGET_SHARD_BLOCK_SIZE: + delta = (prev_gasprice * (length - TARGET_SHARD_BLOCK_SIZE) + // TARGET_SHARD_BLOCK_SIZE // GASPRICE_ADJUSTMENT_COEFFICIENT) + return min(prev_gasprice + delta, MAX_GASPRICE) + else: + delta = (prev_gasprice * (TARGET_SHARD_BLOCK_SIZE - length) + // TARGET_SHARD_BLOCK_SIZE // GASPRICE_ADJUSTMENT_COEFFICIENT) + return max(prev_gasprice, MIN_GASPRICE + delta) - delta +``` + ### Beacon state accessors #### `get_active_shard_count` @@ -512,20 +526,6 @@ def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation) ) ``` -#### `get_updated_gasprice` - -```python -def get_updated_gasprice(prev_gasprice: Gwei, length: uint8) -> Gwei: - if length > TARGET_SHARD_BLOCK_SIZE: - delta = (prev_gasprice * (length - TARGET_SHARD_BLOCK_SIZE) - // TARGET_SHARD_BLOCK_SIZE // GASPRICE_ADJUSTMENT_COEFFICIENT) - return min(prev_gasprice + delta, MAX_GASPRICE) - else: - delta = (prev_gasprice * (TARGET_SHARD_BLOCK_SIZE - length) - // TARGET_SHARD_BLOCK_SIZE // GASPRICE_ADJUSTMENT_COEFFICIENT) - return max(prev_gasprice, MIN_GASPRICE + delta) - delta -``` - #### `get_start_shard` ```python @@ -617,7 +617,6 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe return bls.AggregateVerify(zip(all_pubkeys, all_signing_roots), signature=attestation.signature) ``` - ### Block processing ```python @@ -630,7 +629,6 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_operations(state, block.body) ``` - #### Operations ```python @@ -714,34 +712,31 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr assert transition.start_slot == offset_slots[0] headers = [] - header = ShardBlockHeader() proposers = [] prev_gasprice = state.shard_states[shard].gasprice shard_parent_root = state.shard_states[shard].latest_block_root - beacon_parent_root = get_block_root_at_slot(state, get_previous_slot(state.slot)) for i in range(len(offset_slots)): shard_block_length = transition.shard_block_lengths[i] - is_empty_proposal = (shard_block_length == 0) + is_empty_proposal = shard_block_length == 0 shard_state = transition.shard_states[i] - proposal_index = get_shard_proposer_index(state, offset_slots[i], shard) - # Reconstruct shard headers - header = ShardBlockHeader( - shard_parent_root=shard_parent_root, - beacon_parent_root=beacon_parent_root, - proposer_index=proposal_index, - slot=offset_slots[i], - body_root=transition.shard_data_roots[i] - ) - shard_parent_root = hash_tree_root(header) - if not is_empty_proposal: - # Only add non-empty signature + proposal_index = get_shard_proposer_index(state, offset_slots[i], shard) + # Reconstruct shard headers + header = ShardBlockHeader( + shard_parent_root=shard_parent_root, + beacon_parent_root=get_block_root_at_slot(state, offset_slots[i]), + proposer_index=proposal_index, + slot=offset_slots[i], + body_root=transition.shard_data_roots[i] + ) + shard_parent_root = hash_tree_root(header) headers.append(header) proposers.append(proposal_index) - # Verify correct calculation of gas prices and slots - assert shard_state.gasprice == get_updated_gasprice(prev_gasprice, shard_block_length) - assert shard_state.slot == offset_slots[i] - prev_gasprice = shard_state.gasprice + + # Verify correct calculation of gas prices and slots + assert shard_state.gasprice == compute_updated_gasprice(prev_gasprice, shard_block_length) + assert shard_state.slot == offset_slots[i] + prev_gasprice = shard_state.gasprice pubkeys = [state.validators[proposer].pubkey for proposer in proposers] signing_roots = [ @@ -753,7 +748,6 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr # Save updated state state.shard_states[shard] = transition.shard_states[len(transition.shard_states) - 1] - assert state.slot > 0 state.shard_states[shard].slot = state.slot - 1 ``` diff --git a/specs/phase1/fraud-proofs.md b/specs/phase1/fraud-proofs.md index b476508a4..c2178c10e 100644 --- a/specs/phase1/fraud-proofs.md +++ b/specs/phase1/fraud-proofs.md @@ -36,17 +36,48 @@ This document describes the shard transition function and fraud proofs as part o ```python def shard_state_transition(beacon_state: BeaconState, - shard: Shard, - slot: Slot, shard_state: ShardState, - beacon_parent_root: Root, signed_block: SignedShardBlock) -> None: # Update shard state - shard_state.data = hash( - hash_tree_root(shard_state) + hash_tree_root(beacon_parent_root) + hash_tree_root(signed_block.message.body) + prev_gasprice = shard_state.gasprice + if len(signed_block.message.body) == 0: + latest_block_root = shard_state.latest_block_root + else: + latest_block_root = hash_tree_root(signed_block.message) + + shard_state.data = compute_shard_transition_data( + beacon_state, + shard_state, + signed_block.message.beacon_parent_root, + signed_block.message.body, ) - shard_state.slot = slot - shard_state.latest_block_root = hash_tree_root(signed_block.message) + shard_state.gasprice = compute_updated_gasprice(prev_gasprice, len(signed_block.message.body)) + shard_state.slot = signed_block.message.slot + shard_state.latest_block_root = latest_block_root +``` + +```python +def compute_shard_transition_data(beacon_state: BeaconState, + shard_state: ShardState, + beacon_parent_root: Root, + shard_body_root: Root) -> Bytes32: + return hash( + hash_tree_root(shard_state) + beacon_parent_root + shard_body_root + ) +``` + +```python +def verify_shard_block_message(beacon_state: BeaconState, + shard_state: ShardState, + block: ShardBlock, + slot: Slot, + shard: Shard) -> bool: + assert block.shard_parent_root == shard_state.latest_block_root + assert block.beacon_parent_root == get_block_root_at_slot(beacon_state, slot) + assert block.slot == slot + assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard) + assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE + return True ``` ```python @@ -67,20 +98,20 @@ TODO. The intent is to have a single universal fraud proof type, which contains 3. The `transition: ShardTransition` itself 4. The full body of the shard block `shard_block` 5. A Merkle proof to the `shard_states` in the parent block the attestation is referencing +6. The `subkey` to generate the custody bit Call the following function to verify the proof: ```python -def verify_fraud_proof(beacon_state: BeaconState, - attestation: Attestation, - offset_index: uint64, - transition: ShardTransition, - signed_block: SignedShardBlock, - subkey: BLSPubkey, - beacon_parent_block: BeaconBlock) -> bool: +def is_valid_fraud_proof(beacon_state: BeaconState, + attestation: Attestation, + offset_index: uint64, + transition: ShardTransition, + signed_block: SignedShardBlock, + subkey: BLSPubkey, + beacon_parent_block: BeaconBlock) -> bool: # 1. Check if `custody_bits[offset_index][j] != generate_custody_bit(subkey, block_contents)` for any `j`. shard = get_shard(beacon_state, attestation) - slot = attestation.data.slot custody_bits = attestation.custody_bits_blocks for j in range(custody_bits[offset_index]): if custody_bits[offset_index][j] != generate_custody_bit(subkey, signed_block): @@ -89,19 +120,12 @@ def verify_fraud_proof(beacon_state: BeaconState, # 2. Check if the shard state transition result is wrong between # `transition.shard_states[offset_index - 1]` to `transition.shard_states[offset_index]`. if offset_index == 0: - shard_state = beacon_parent_block.shard_transitions[shard].shard_states[-1] + shard_state = beacon_parent_block.shard_transitions[shard].shard_states[-1].copy() else: shard_state = transition.shard_states[offset_index - 1].copy() # Not doing the actual state updates here. - shard_state_transition( - beacon_state=beacon_state, - shard=shard, - slot=slot, - shard_state=shard_state, - beacon_parent_root=hash_tree_root(beacon_parent_block), - signed_block=signed_block, - ) - if shard_state.latest_block_root != transition.shard_states[offset_index].data: + shard_state_transition(beacon_state, shard_state, signed_block) + if shard_state.data != transition.shard_states[offset_index].data: return True return False @@ -126,26 +150,7 @@ def get_winning_proposal(beacon_state: BeaconState, proposals: Sequence[SignedSh ``` ```python -def get_empty_body_block(shard_parent_root: Root, - beacon_parent_root: Root, - slot: Slot, - proposer_index: ValidatorIndex) -> ShardBlock: - return ShardBlock( - shard_parent_root=shard_parent_root, - beacon_parent_root=beacon_parent_root, - slot=slot, - proposer_index=proposer_index, - ) -``` - -```python -def is_empty_body(proposal: ShardBlock) -> bool: - # TODO - return len(proposal.body) == 0 -``` - -```python -def compute_shard_data_roots(proposals: Sequence[SignedShardBlock]) -> Sequence[Root]: +def compute_shard_body_roots(proposals: Sequence[SignedShardBlock]) -> Sequence[Root]: return [hash_tree_root(proposal.message.body) for proposal in proposals] ``` @@ -155,28 +160,23 @@ def get_proposal_choices_at_slot(beacon_state: BeaconState, slot: Slot, shard: Shard, shard_blocks: Sequence[SignedShardBlock], - validate_result: bool=True) -> Sequence[SignedShardBlock]: + validate_signature: bool=True) -> Sequence[SignedShardBlock]: + """ + Return the valid shard blocks at the given ``slot``. + Note that this function doesn't change the state. + """ choices = [] - beacon_parent_root = get_block_root_at_slot(beacon_state, get_previous_slot(beacon_state.slot)) - proposer_index = get_shard_proposer_index(beacon_state, slot, shard) shard_blocks_at_slot = [block for block in shard_blocks if block.message.slot == slot] for block in shard_blocks_at_slot: temp_shard_state = shard_state.copy() # Not doing the actual state updates here. # Try to apply state transition to temp_shard_state. try: - # Verify the proposer_index and signature - assert block.message.proposer_index == proposer_index - if validate_result: + # Verify block message and signature + assert verify_shard_block_message(beacon_state, temp_shard_state, block.message, slot, shard) + if validate_signature: assert verify_shard_block_signature(beacon_state, block) - shard_state_transition( - beacon_state=beacon_state, - shard=shard, - slot=slot, - shard_state=temp_shard_state, - beacon_parent_root=beacon_parent_root, - signed_block=block, - ) + shard_state_transition(beacon_state, temp_shard_state, block) except Exception: pass # TODO: throw error in the test helper else: @@ -189,11 +189,12 @@ def get_proposal_at_slot(beacon_state: BeaconState, shard_state: ShardState, slot: Shard, shard: Shard, - shard_parent_root: Root, shard_blocks: Sequence[SignedShardBlock], - validate_result: bool=True) -> Tuple[SignedShardBlock, ShardState, Root]: - beacon_parent_root = get_block_root_at_slot(beacon_state, get_previous_slot(beacon_state.slot)) - proposer_index = get_shard_proposer_index(beacon_state, slot, shard) + validate_signature: bool=True) -> Tuple[SignedShardBlock, ShardState]: + """ + Return ``proposal``, ``shard_state`` of the given ``slot``. + Note that this function doesn't change the state. + """ shard_state = shard_state.copy() # Don't update the given shard_state choices = get_proposal_choices_at_slot( beacon_state=beacon_state, @@ -201,35 +202,20 @@ def get_proposal_at_slot(beacon_state: BeaconState, slot=slot, shard=shard, shard_blocks=shard_blocks, - validate_result=validate_result, + validate_signature=validate_signature, ) if len(choices) == 0: - block_header = get_empty_body_block( - shard_parent_root=shard_parent_root, - beacon_parent_root=beacon_parent_root, - slot=slot, - proposer_index=proposer_index, - ) - proposal = SignedShardBlock(message=block_header) + block = ShardBlock(slot=slot) + proposal = SignedShardBlock(message=block) elif len(choices) == 1: proposal = choices[0] else: proposal = get_winning_proposal(beacon_state, choices) - shard_parent_root = hash_tree_root(proposal.message) + # Apply state transition + shard_state_transition(beacon_state, shard_state, proposal) - if not is_empty_body(proposal.message): - # Apply state transition to shard_state. - shard_state_transition( - beacon_state=beacon_state, - shard=shard, - slot=slot, - shard_state=shard_state, - beacon_parent_root=beacon_parent_root, - signed_block=proposal, - ) - - return proposal, shard_state, shard_parent_root + return proposal, shard_state ``` ```python @@ -237,26 +223,24 @@ def get_shard_state_transition_result( beacon_state: BeaconState, shard: Shard, shard_blocks: Sequence[SignedShardBlock], - validate_result: bool=True, + validate_signature: bool=True, ) -> Tuple[Sequence[SignedShardBlock], Sequence[ShardState], Sequence[Root]]: proposals = [] shard_states = [] shard_state = beacon_state.shard_states[shard].copy() - shard_parent_root = beacon_state.shard_states[shard].latest_block_root for slot in get_offset_slots(beacon_state, shard): - proposal, shard_state, shard_parent_root = get_proposal_at_slot( + proposal, shard_state = get_proposal_at_slot( beacon_state=beacon_state, shard_state=shard_state, slot=slot, shard=shard, - shard_parent_root=shard_parent_root, shard_blocks=shard_blocks, - validate_result=validate_result, + validate_signature=validate_signature, ) shard_states.append(shard_state) proposals.append(proposal) - shard_data_roots = compute_shard_data_roots(proposals) + shard_data_roots = compute_shard_body_roots(proposals) return proposals, shard_states, shard_data_roots ``` diff --git a/specs/phase1/phase1-fork.md b/specs/phase1/phase1-fork.md index 173fceeb4..e95c12c72 100644 --- a/specs/phase1/phase1-fork.md +++ b/specs/phase1/phase1-fork.md @@ -7,7 +7,7 @@ - [Introduction](#introduction) - [Configuration](#configuration) - [Fork to Phase 1](#fork-to-phase-1) - - [Fork trigger.](#fork-trigger) + - [Fork trigger](#fork-trigger) - [Upgrading the state](#upgrading-the-state) @@ -35,17 +35,18 @@ Warning: this configuration is not definitive. | Name | Value | | - | - | | `PHASE_1_FORK_VERSION` | `Version('0x01000000')` | +| `PHASE_1_GENESIS_SLOT` | `2**5` **TBD** | | `INITIAL_ACTIVE_SHARDS` | `2**6` (= 64) | ## Fork to Phase 1 -### Fork trigger. +### Fork trigger -TBD. Social consensus, along with state conditions such as epoch boundary, finality, deposits, active validator count, etc. may be part of the decision process to trigger the fork. +TBD. Social consensus, along with state conditions such as epoch boundary, finality, deposits, active validator count, etc. may be part of the decision process to trigger the fork. For now we assume the condition will be triggered at slot `PHASE_1_GENESIS_SLOT`, where `PHASE_1_GENESIS_SLOT % SLOTS_PER_EPOCH == 0`. ### Upgrading the state -After `process_slots` of Phase 0 finishes, but before the first Phase 1 block is processed, an irregular state change is made to upgrade to Phase 1. +After `process_slots` of Phase 0 finishes, if `state.slot == PHASE_1_GENESIS_SLOT`, an irregular state change is made to upgrade to Phase 1. ```python def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 8f95cf73a..25d868b65 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -32,7 +32,7 @@ def build_shard_block(spec, block = spec.ShardBlock( shard_parent_root=shard_state.latest_block_root, - beacon_parent_root=spec.get_block_root_at_slot(beacon_state, spec.get_previous_slot(beacon_state.slot)), + beacon_parent_root=spec.get_block_root_at_slot(beacon_state, spec.get_previous_slot(slot)), slot=slot, proposer_index=proposer_index, body=body, diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py index 1585a1a45..6d1e65678 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py @@ -10,7 +10,7 @@ from eth2spec.test.helpers.crosslinks import ( run_crosslinks_processing, ) from eth2spec.test.helpers.shard_block import build_shard_block -from eth2spec.test.helpers.state import next_epoch, next_slot +from eth2spec.test.helpers.state import next_epoch, next_slot, next_slots @with_all_phases_except(['phase0']) @@ -24,7 +24,8 @@ def test_basic_crosslinks(spec, state): committee_index = spec.CommitteeIndex(0) shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) - shard_block = build_shard_block(spec, state, shard, body=b'\x12' * 10, slot=state.slot, signed=True) + body = b'1' * spec.MAX_SHARD_BLOCK_SIZE + shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) shard_blocks = [shard_block] next_slot(spec, state) @@ -43,10 +44,58 @@ def test_basic_crosslinks(spec, state): ) attestations = [attestation] + pre_gasprice = state.shard_states[shard].gasprice offset_slots = spec.get_offset_slots(state, shard) + assert len(offset_slots) == 1 yield from run_crosslinks_processing(spec, state, shard_transitions, attestations) shard_state = state.shard_states[shard] assert shard_state.slot == offset_slots[-1] assert shard_state.latest_block_root == shard_block.message.hash_tree_root() + assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1] + assert shard_state.gasprice > pre_gasprice + + +@with_all_phases_except(['phase0']) +@spec_state_test +@always_bls +def test_multiple_offset_slots(spec, state): + next_epoch(spec, state) + next_epoch(spec, state) + state = spec.upgrade_to_phase1(state) + next_slot(spec, state) + + committee_index = spec.CommitteeIndex(0) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + body = b'1' * spec.MAX_SHARD_BLOCK_SIZE + shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) + shard_blocks = [shard_block] + + next_slots(spec, state, 3) + + shard_transition = spec.get_shard_transition(state, shard, shard_blocks) + shard_transitions = [spec.ShardTransition()] * len(state.shard_states) + shard_transitions[shard] = shard_transition + + attestation = get_valid_on_time_attestation( + spec, + state, + slot=state.slot, + index=committee_index, + shard_transition=shard_transition, + signed=True, + ) + attestations = [attestation] + + pre_gasprice = state.shard_states[shard].gasprice + offset_slots = spec.get_offset_slots(state, shard) + assert len(offset_slots) == 3 + + yield from run_crosslinks_processing(spec, state, shard_transitions, attestations) + + shard_state = state.shard_states[shard] + assert shard_state.slot == offset_slots[-1] + assert shard_state.latest_block_root == shard_block.message.hash_tree_root() + assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1] + assert shard_state.gasprice > pre_gasprice From 40483b587b1945eace6a1aeb187608782179e3a6 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 20 Apr 2020 16:57:29 +0800 Subject: [PATCH 093/102] [squashed] shard chain updates wip Use `ShardBlock` in `shard_state_transition` PR feedback 1. Rename `ShardState.data` -> `ShardState.transition_digest` 2. Rename `compute_shard_transition_data` to `compute_shard_transition_digest` 3. Add `assert state.slot > PHASE_1_GENESIS_SLOT` just in case, may move it later Add `get_post_shard_state` as a pure function wrapper --- specs/phase1/beacon-chain.md | 5 ++- specs/phase1/fraud-proofs.md | 61 +++++++++++++++++++++--------------- 2 files changed, 40 insertions(+), 26 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 5c67fe4f2..1fdf7ec4c 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -328,7 +328,7 @@ class ShardBlockHeader(Container): class ShardState(Container): slot: Slot gasprice: Gwei - data: Bytes32 + transition_digest: Bytes32 latest_block_root: Root ``` @@ -701,6 +701,9 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: ```python def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTransition) -> None: + # TODO: only need to check it once when phase 1 starts + assert state.slot > PHASE_1_GENESIS_SLOT + # Correct data root count offset_slots = get_offset_slots(state, shard) assert ( diff --git a/specs/phase1/fraud-proofs.md b/specs/phase1/fraud-proofs.md index c2178c10e..a5068cd07 100644 --- a/specs/phase1/fraud-proofs.md +++ b/specs/phase1/fraud-proofs.md @@ -37,30 +37,43 @@ This document describes the shard transition function and fraud proofs as part o ```python def shard_state_transition(beacon_state: BeaconState, shard_state: ShardState, - signed_block: SignedShardBlock) -> None: + block: ShardBlock) -> None: # Update shard state prev_gasprice = shard_state.gasprice - if len(signed_block.message.body) == 0: + if len(block.body) == 0: latest_block_root = shard_state.latest_block_root else: - latest_block_root = hash_tree_root(signed_block.message) + latest_block_root = hash_tree_root(block) - shard_state.data = compute_shard_transition_data( + shard_state.transition_digest = compute_shard_transition_digest( beacon_state, shard_state, - signed_block.message.beacon_parent_root, - signed_block.message.body, + block.beacon_parent_root, + block.body, ) - shard_state.gasprice = compute_updated_gasprice(prev_gasprice, len(signed_block.message.body)) - shard_state.slot = signed_block.message.slot + shard_state.gasprice = compute_updated_gasprice(prev_gasprice, len(block.body)) + shard_state.slot = block.slot shard_state.latest_block_root = latest_block_root ``` ```python -def compute_shard_transition_data(beacon_state: BeaconState, - shard_state: ShardState, - beacon_parent_root: Root, - shard_body_root: Root) -> Bytes32: +def get_post_shard_state(beacon_state: BeaconState, + shard_state: ShardState, + block: ShardBlock) -> ShardState: + """ + A pure function that returns a new post ShardState instead of modifying the given `shard_state`. + """ + post_state = shard_state.copy() + shard_state_transition(beacon_state, post_state, block) + return post_state +``` + +```python +def compute_shard_transition_digest(beacon_state: BeaconState, + shard_state: ShardState, + beacon_parent_root: Root, + shard_body_root: Root) -> Bytes32: + # TODO: use SSZ hash tree root return hash( hash_tree_root(shard_state) + beacon_parent_root + shard_body_root ) @@ -107,25 +120,25 @@ def is_valid_fraud_proof(beacon_state: BeaconState, attestation: Attestation, offset_index: uint64, transition: ShardTransition, - signed_block: SignedShardBlock, + block: ShardBlock, subkey: BLSPubkey, beacon_parent_block: BeaconBlock) -> bool: # 1. Check if `custody_bits[offset_index][j] != generate_custody_bit(subkey, block_contents)` for any `j`. shard = get_shard(beacon_state, attestation) custody_bits = attestation.custody_bits_blocks for j in range(custody_bits[offset_index]): - if custody_bits[offset_index][j] != generate_custody_bit(subkey, signed_block): + if custody_bits[offset_index][j] != generate_custody_bit(subkey, block): return True # 2. Check if the shard state transition result is wrong between # `transition.shard_states[offset_index - 1]` to `transition.shard_states[offset_index]`. if offset_index == 0: - shard_state = beacon_parent_block.shard_transitions[shard].shard_states[-1].copy() + shard_state = beacon_parent_block.shard_transitions[shard].shard_states[-1] else: - shard_state = transition.shard_states[offset_index - 1].copy() # Not doing the actual state updates here. + shard_state = transition.shard_states[offset_index - 1] # Not doing the actual state updates here. - shard_state_transition(beacon_state, shard_state, signed_block) - if shard_state.data != transition.shard_states[offset_index].data: + shard_state = get_post_shard_state(beacon_state, shard_state, block) + if shard_state.transition_digest != transition.shard_states[offset_index].transition_digest: return True return False @@ -168,15 +181,14 @@ def get_proposal_choices_at_slot(beacon_state: BeaconState, choices = [] shard_blocks_at_slot = [block for block in shard_blocks if block.message.slot == slot] for block in shard_blocks_at_slot: - temp_shard_state = shard_state.copy() # Not doing the actual state updates here. - # Try to apply state transition to temp_shard_state. try: # Verify block message and signature - assert verify_shard_block_message(beacon_state, temp_shard_state, block.message, slot, shard) + # TODO these validations should have been checked upon receiving shard blocks. + assert verify_shard_block_message(beacon_state, shard_state, block.message, slot, shard) if validate_signature: assert verify_shard_block_signature(beacon_state, block) - shard_state_transition(beacon_state, temp_shard_state, block) + shard_state = get_post_shard_state(beacon_state, shard_state, block.message) except Exception: pass # TODO: throw error in the test helper else: @@ -195,7 +207,6 @@ def get_proposal_at_slot(beacon_state: BeaconState, Return ``proposal``, ``shard_state`` of the given ``slot``. Note that this function doesn't change the state. """ - shard_state = shard_state.copy() # Don't update the given shard_state choices = get_proposal_choices_at_slot( beacon_state=beacon_state, shard_state=shard_state, @@ -213,7 +224,7 @@ def get_proposal_at_slot(beacon_state: BeaconState, proposal = get_winning_proposal(beacon_state, choices) # Apply state transition - shard_state_transition(beacon_state, shard_state, proposal) + shard_state = get_post_shard_state(beacon_state, shard_state, proposal.message) return proposal, shard_state ``` @@ -227,7 +238,7 @@ def get_shard_state_transition_result( ) -> Tuple[Sequence[SignedShardBlock], Sequence[ShardState], Sequence[Root]]: proposals = [] shard_states = [] - shard_state = beacon_state.shard_states[shard].copy() + shard_state = beacon_state.shard_states[shard] for slot in get_offset_slots(beacon_state, shard): proposal, shard_state = get_proposal_at_slot( beacon_state=beacon_state, From c8a473ba24e130bd44c2cd28b64ce58e23b1814a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 28 Apr 2020 10:46:13 +0800 Subject: [PATCH 094/102] Apply suggestions from code review Co-Authored-By: Danny Ryan --- specs/phase1/beacon-chain.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 1fdf7ec4c..fc9687545 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -497,12 +497,11 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque source_epoch -= LIGHT_CLIENT_COMMITTEE_PERIOD active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) seed = get_seed(beacon_state, source_epoch, DOMAIN_LIGHT_CLIENT) - active_shard_count = get_active_shard_count(beacon_state) return compute_committee( indices=active_validator_indices, seed=seed, index=0, - count=active_shard_count, + count=get_active_shard_count(beacon_state), )[:TARGET_COMMITTEE_SIZE] ``` From 524ba166d13d954e5d512b44bb9a5855f6f77116 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 28 Apr 2020 11:28:21 +0800 Subject: [PATCH 095/102] [squashed] shard chain updates wip Fix wrong field names Fix `build_attestation_data` and other PR feedback from Danny and Terence 1. Rename `get_previous_slot` to `compute_previous_slot` 2. Break down `build_empty_block` into `get_state_and_beacon_parent_root_at_slot`, use it in `build_shard_block` 3. Set defult `slot` to `shard_state.slot + 1` in `build_shard_block` Update `verify_shard_block_message`: check beacon_parent_root at fork choice rule stage instead of state transition Fix `beacon-chain.md` 1. Fix typo `attestation.slot == state.slot` -> `attestation.data.slot == state.slot` in `is_winning_attestation` 2. Check `verify_shard_transition_false_positives` **after** `process_operations` 3. Fix `shard_attestations` filter in `process_crosslinks`: since attestations come from block, should use `attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY == state.slot` 4. [TBD] Allow empty `light_client_signature` to make the tests pass 5. [TBD] Add `is_shard_attestation`, filter out empty `ShardTransition()` Rework `test_process_crosslink` Add basic phase 1 `test_blocks` Add more test cases Revert `is_shard_attestation` and fix test cases backward compatibility. Remove `test_process_beacon_block_no_shard_transition` and consider it as invalid case. --- specs/phase1/beacon-chain.md | 41 +++-- specs/phase1/fraud-proofs.md | 3 +- specs/phase1/phase1-fork.md | 4 +- .../eth2spec/test/helpers/attestations.py | 60 +++++-- .../pyspec/eth2spec/test/helpers/block.py | 27 ++-- .../eth2spec/test/helpers/shard_block.py | 49 +++++- .../test/phase_0/sanity/test_blocks.py | 11 +- .../test_process_crosslink.py | 147 ++++++++---------- .../eth2spec/test/phase_1/sanity/__init__.py | 0 .../test/phase_1/sanity/test_blocks.py | 141 +++++++++++++++++ 10 files changed, 352 insertions(+), 131 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/phase_1/sanity/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index fc9687545..611cc1955 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -33,7 +33,7 @@ - [`AttestationCustodyBitWrapper`](#attestationcustodybitwrapper) - [Helper functions](#helper-functions) - [Misc](#misc-1) - - [`get_previous_slot`](#get_previous_slot) + - [`compute_previous_slot`](#compute_previous_slot) - [`pack_compact_validator`](#pack_compact_validator) - [`unpack_compact_validator`](#unpack_compact_validator) - [`committee_to_compact_committee`](#committee_to_compact_committee) @@ -52,6 +52,7 @@ - [`get_latest_slot_for_shard`](#get_latest_slot_for_shard) - [`get_offset_slots`](#get_offset_slots) - [Predicates](#predicates) + - [`is_shard_attestation`](#is_shard_attestation) - [`is_winning_attestation`](#is_winning_attestation) - [Updated `is_valid_indexed_attestation`](#updated-is_valid_indexed_attestation) - [Block processing](#block-processing) @@ -369,10 +370,10 @@ class AttestationCustodyBitWrapper(Container): ### Misc -#### `get_previous_slot` +#### `compute_previous_slot` ```python -def get_previous_slot(slot: Slot) -> Slot: +def compute_previous_slot(slot: Slot) -> Slot: if slot > 0: return Slot(slot - 1) else: @@ -556,6 +557,21 @@ def get_offset_slots(state: BeaconState, shard: Shard) -> Sequence[Slot]: ### Predicates +#### `is_shard_attestation` + +```python +def is_shard_attestation(state: BeaconState, + attestation: Attestation, + committee_index: CommitteeIndex) -> bool: + if not ( + attestation.data.index == committee_index + and attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY == state.slot + ): + return False + + return True +``` + #### `is_winning_attestation` ```python @@ -568,7 +584,7 @@ def is_winning_attestation(state: BeaconState, ``winning_root`` formed by ``committee_index`` committee at the current slot. """ return ( - attestation.slot == state.slot + attestation.data.slot == state.slot and attestation.data.index == committee_index and attestation.data.shard_transition_root == winning_root ) @@ -623,9 +639,9 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) process_randao(state, block.body) process_eth1_data(state, block.body) - verify_shard_transition_false_positives(state, block.body) process_light_client_signatures(state, block.body) process_operations(state, block.body) + verify_shard_transition_false_positives(state, block.body) ``` #### Operations @@ -684,7 +700,7 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: # Correct data root count assert len(attestation.custody_bits_blocks) == len(get_offset_slots(state, shard)) # Correct parent block root - assert data.beacon_block_root == get_block_root_at_slot(state, get_previous_slot(state.slot)) + assert data.beacon_block_root == get_block_root_at_slot(state, compute_previous_slot(state.slot)) # Type 2: no shard transition, no custody bits else: # Ensure delayed attestation @@ -821,11 +837,12 @@ def process_crosslinks(state: BeaconState, for committee_index in map(CommitteeIndex, range(committee_count)): shard = compute_shard_from_committee_index(state, committee_index, state.slot) # All attestations in the block for this committee/shard and current slot + shard_transition = shard_transitions[shard] shard_attestations = [ attestation for attestation in attestations - if attestation.data.index == committee_index and attestation.data.slot == state.slot + if is_shard_attestation(state, attestation, committee_index) ] - shard_transition = shard_transitions[shard] + winning_root = process_crosslink_for_shard(state, committee_index, shard_transition, shard_attestations) if winning_root != Root(): # Mark relevant pending attestations as creating a successful crosslink @@ -920,10 +937,14 @@ def process_light_client_signatures(state: BeaconState, block_body: BeaconBlockB increase_balance(state, get_beacon_proposer_index(state), Gwei(total_reward // PROPOSER_REWARD_QUOTIENT)) - slot = get_previous_slot(state.slot) + slot = compute_previous_slot(state.slot) signing_root = compute_signing_root(get_block_root_at_slot(state, slot), get_domain(state, DOMAIN_LIGHT_CLIENT, compute_epoch_at_slot(slot))) - assert bls.FastAggregateVerify(signer_pubkeys, signing_root, signature=block_body.light_client_signature) + if len(signer_pubkeys) == 0: + # TODO: Or disallow empty light_client_signature? + return + else: + assert bls.FastAggregateVerify(signer_pubkeys, signing_root, signature=block_body.light_client_signature) ``` ### Epoch transition diff --git a/specs/phase1/fraud-proofs.md b/specs/phase1/fraud-proofs.md index a5068cd07..c676e142a 100644 --- a/specs/phase1/fraud-proofs.md +++ b/specs/phase1/fraud-proofs.md @@ -86,7 +86,6 @@ def verify_shard_block_message(beacon_state: BeaconState, slot: Slot, shard: Shard) -> bool: assert block.shard_parent_root == shard_state.latest_block_root - assert block.beacon_parent_root == get_block_root_at_slot(beacon_state, slot) assert block.slot == slot assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard) assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE @@ -124,7 +123,6 @@ def is_valid_fraud_proof(beacon_state: BeaconState, subkey: BLSPubkey, beacon_parent_block: BeaconBlock) -> bool: # 1. Check if `custody_bits[offset_index][j] != generate_custody_bit(subkey, block_contents)` for any `j`. - shard = get_shard(beacon_state, attestation) custody_bits = attestation.custody_bits_blocks for j in range(custody_bits[offset_index]): if custody_bits[offset_index][j] != generate_custody_bit(subkey, block): @@ -133,6 +131,7 @@ def is_valid_fraud_proof(beacon_state: BeaconState, # 2. Check if the shard state transition result is wrong between # `transition.shard_states[offset_index - 1]` to `transition.shard_states[offset_index]`. if offset_index == 0: + shard = get_shard(beacon_state, attestation) shard_state = beacon_parent_block.shard_transitions[shard].shard_states[-1] else: shard_state = transition.shard_states[offset_index - 1] # Not doing the actual state updates here. diff --git a/specs/phase1/phase1-fork.md b/specs/phase1/phase1-fork.md index e95c12c72..cc7d8f33e 100644 --- a/specs/phase1/phase1-fork.md +++ b/specs/phase1/phase1-fork.md @@ -103,7 +103,7 @@ def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: ShardState( slot=pre.slot, gasprice=MIN_GASPRICE, - data=Root(), + transition_digest=Root(), latest_block_root=Root(), ) for i in range(INITIAL_ACTIVE_SHARDS) ), @@ -111,7 +111,7 @@ def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: current_light_committee=CompactCommittee(), # computed after state creation next_light_committee=CompactCommittee(), # Custody game - custody_challenge_index=0, + exposed_derived_secrets=[] * EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS, # exposed_derived_secrets will fully default to zeroes ) next_epoch = Epoch(epoch + 1) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index c203b737a..9e98e83ea 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -1,7 +1,7 @@ from typing import List from eth2spec.test.context import expect_assertion_error, PHASE0, PHASE1 -from eth2spec.test.helpers.state import state_transition_and_sign_block +from eth2spec.test.helpers.state import state_transition_and_sign_block, next_slot, transition_to from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls @@ -43,7 +43,7 @@ def run_attestation_processing(spec, state, attestation, valid=True): yield 'post', state -def build_attestation_data(spec, state, slot, index, shard_transition=None): +def build_attestation_data(spec, state, slot, index, shard_transition=None, on_time=True): assert state.slot >= slot if slot == state.slot: @@ -66,28 +66,42 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None): source_epoch = state.current_justified_checkpoint.epoch source_root = state.current_justified_checkpoint.root - if spec.fork == PHASE1 and shard_transition is not None: - shard_transition_root = shard_transition.hash_tree_root() - head_shard_root = shard_transition.shard_data_roots[len(shard_transition.shard_data_roots) - 1] - else: - shard_transition_root = spec.Root() - head_shard_root = spec.Root() - - return spec.AttestationData( + attestation_data = spec.AttestationData( slot=slot, index=index, beacon_block_root=block_root, source=spec.Checkpoint(epoch=source_epoch, root=source_root), target=spec.Checkpoint(epoch=spec.compute_epoch_at_slot(slot), root=epoch_boundary_root), - head_shard_root=head_shard_root, - shard_transition_root=shard_transition_root, ) + if spec.fork == PHASE1: + if shard_transition is not None: + lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 + attestation_data.head_shard_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] + attestation_data.shard_transition_root = shard_transition.hash_tree_root() + else: + # No shard transition + shard = spec.get_shard(state, spec.Attestation(data=attestation_data)) + if on_time: + temp_state = state.copy() + next_slot(spec, temp_state) + shard_transition = spec.get_shard_transition(temp_state, shard, []) + lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 + attestation_data.head_shard_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] + attestation_data.shard_transition_root = shard_transition.hash_tree_root() + else: + attestation_data.head_shard_root = state.shard_states[shard].transition_digest + attestation_data.shard_transition_root = spec.Root() + return attestation_data + def convert_to_valid_on_time_attestation(spec, state, attestation, signed=False): shard = spec.get_shard(state, attestation) - offset_slots = spec.compute_offset_slots(spec.get_latest_slot_for_shard(state, shard), state.slot + 1) - for offset_slot in offset_slots: + offset_slots = spec.compute_offset_slots( + spec.get_latest_slot_for_shard(state, shard), + attestation.data.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY, + ) + for _ in offset_slots: attestation.custody_bits_blocks.append( Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE]([0 for _ in attestation.aggregation_bits]) ) @@ -143,7 +157,9 @@ def get_valid_attestation(spec, if index is None: index = 0 - attestation_data = build_attestation_data(spec, state, slot=slot, index=index, shard_transition=shard_transition) + attestation_data = build_attestation_data( + spec, state, slot=slot, index=index, shard_transition=shard_transition, on_time=on_time + ) beacon_committee = spec.get_beacon_committee( state, @@ -298,7 +314,21 @@ def next_epoch_with_attestations(spec, spec, post_state, slot_to_attest, index=index, signed=True, on_time=False) block.body.attestations.append(prev_attestation) + if spec.fork == PHASE1: + fill_block_shard_transitions_by_attestations(spec, post_state, block) + signed_block = state_transition_and_sign_block(spec, post_state, block) signed_blocks.append(signed_block) return state, signed_blocks, post_state + + +def fill_block_shard_transitions_by_attestations(spec, state, block): + block.body.shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS + for attestation in block.body.attestations: + shard = spec.get_shard(state, attestation) + if attestation.data.slot == state.slot: + temp_state = state.copy() + transition_to(spec, temp_state, slot=block.slot) + shard_transition = spec.get_shard_transition(temp_state, shard, []) + block.body.shard_transitions[shard] = shard_transition diff --git a/tests/core/pyspec/eth2spec/test/helpers/block.py b/tests/core/pyspec/eth2spec/test/helpers/block.py index 96cc30e35..e40d5e7d8 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/block.py @@ -71,24 +71,31 @@ def build_empty_block(spec, state, slot=None): """ if slot is None: slot = state.slot - if slot < state.slot: - raise Exception("build_empty_block cannot build blocks for past slots") - if slot > state.slot: - # transition forward in copied state to grab relevant data from state - state = state.copy() - spec.process_slots(state, slot) + state, parent_block_root = get_state_and_beacon_parent_root_at_slot(spec, state, slot) empty_block = spec.BeaconBlock() empty_block.slot = slot empty_block.proposer_index = spec.get_beacon_proposer_index(state) empty_block.body.eth1_data.deposit_count = state.eth1_deposit_index - previous_block_header = state.latest_block_header.copy() - if previous_block_header.state_root == spec.Root(): - previous_block_header.state_root = hash_tree_root(state) - empty_block.parent_root = hash_tree_root(previous_block_header) + empty_block.parent_root = parent_block_root apply_randao_reveal(spec, state, empty_block) return empty_block def build_empty_block_for_next_slot(spec, state): return build_empty_block(spec, state, state.slot + 1) + + +def get_state_and_beacon_parent_root_at_slot(spec, state, slot): + if slot < state.slot: + raise Exception("Cannot build blocks for past slots") + if slot > state.slot: + # transition forward in copied state to grab relevant data from state + state = state.copy() + spec.process_slots(state, slot) + + previous_block_header = state.latest_block_header.copy() + if previous_block_header.state_root == spec.Root(): + previous_block_header.state_root = hash_tree_root(state) + beacon_parent_root = hash_tree_root(previous_block_header) + return state, beacon_parent_root diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 25d868b65..922f4d748 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -1,3 +1,6 @@ +from eth2spec.test.helpers.attestations import get_valid_on_time_attestation +from eth2spec.test.helpers.block import get_state_and_beacon_parent_root_at_slot +from eth2spec.test.helpers.state import next_slots from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.bls import only_with_bls @@ -18,21 +21,22 @@ def sign_shard_block(spec, beacon_state, shard, block, proposer_index=None): def build_shard_block(spec, beacon_state, shard, - slot, + slot=None, body=None, signed=False): shard_state = beacon_state.shard_states[shard] if slot is None: - slot = shard_state.slot + slot = shard_state.slot + 1 if body is None: - body = [] + body = b'\x56' * 128 proposer_index = spec.get_shard_proposer_index(beacon_state, slot, shard) + beacon_state, beacon_parent_root = get_state_and_beacon_parent_root_at_slot(spec, beacon_state, slot) block = spec.ShardBlock( shard_parent_root=shard_state.latest_block_root, - beacon_parent_root=spec.get_block_root_at_slot(beacon_state, spec.get_previous_slot(slot)), + beacon_parent_root=beacon_parent_root, slot=slot, proposer_index=proposer_index, body=body, @@ -45,3 +49,40 @@ def build_shard_block(spec, sign_shard_block(spec, beacon_state, shard, signed_block, proposer_index=proposer_index) return signed_block + + +def build_shard_transitions_till_slot(spec, state, shards, shard_blocks, target_len_offset_slot): + state = state.copy() + next_slots(spec, state, target_len_offset_slot) + shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS + for shard in shards: + offset_slots = spec.get_offset_slots(state, shard) + len_offset_slots = len(offset_slots) + assert len_offset_slots == target_len_offset_slot + shard_blocks_of_shard = shard_blocks[shard] + shard_transition = spec.get_shard_transition(state, shard, shard_blocks_of_shard) + if len(shard_blocks_of_shard) > 0: + shard_block_root = shard_blocks_of_shard[-1].message.hash_tree_root() + assert shard_transition.shard_states[len_offset_slots - 1].latest_block_root == shard_block_root + assert shard_transition.shard_states[len_offset_slots - 1].slot == offset_slots[-1] + shard_transitions[shard] = shard_transition + + return shard_transitions + + +def build_attestation_with_shard_transition(spec, state, slot, index, target_len_offset_slot, shard_transition=None): + state = state.copy() + next_slots(spec, state, target_len_offset_slot) + attestation = get_valid_on_time_attestation( + spec, + state, + slot=slot, + index=index, + shard_transition=shard_transition, + signed=True, + ) + assert attestation.data.slot == slot + if shard_transition is not None: + assert target_len_offset_slot == len(shard_transition.shard_states) + assert attestation.data.shard_transition_root == shard_transition.hash_tree_root() + return attestation diff --git a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py index b6b671872..29a9dcca2 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py @@ -8,10 +8,13 @@ from eth2spec.test.helpers.block import build_empty_block_for_next_slot, build_e from eth2spec.test.helpers.keys import privkeys, pubkeys from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing, get_indexed_attestation_participants from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing -from eth2spec.test.helpers.attestations import get_valid_attestation +from eth2spec.test.helpers.attestations import get_valid_attestation, fill_block_shard_transitions_by_attestations from eth2spec.test.helpers.deposits import prepare_state_and_deposit -from eth2spec.test.context import spec_state_test, with_all_phases, expect_assertion_error, always_bls, with_phases +from eth2spec.test.context import ( + spec_state_test, with_all_phases, expect_assertion_error, always_bls, with_phases, + PHASE1 +) @with_all_phases @@ -420,12 +423,14 @@ def test_attestation(spec, state): yield 'pre', state - attestation = get_valid_attestation(spec, state, signed=True) + attestation = get_valid_attestation(spec, state, signed=True, on_time=True) # Add to state via block transition pre_current_attestations_len = len(state.current_epoch_attestations) attestation_block = build_empty_block(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) attestation_block.body.attestations.append(attestation) + if spec.fork == PHASE1: + fill_block_shard_transitions_by_attestations(spec, state, attestation_block) signed_attestation_block = state_transition_and_sign_block(spec, state, attestation_block) assert len(state.current_epoch_attestations) == pre_current_attestations_len + 1 diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py index 6d1e65678..912770da8 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py @@ -1,101 +1,78 @@ from eth2spec.test.context import ( + PHASE0, with_all_phases_except, spec_state_test, always_bls, ) -from eth2spec.test.helpers.attestations import ( - get_valid_on_time_attestation, +from eth2spec.test.helpers.crosslinks import run_crosslinks_processing +from eth2spec.test.helpers.shard_block import ( + build_attestation_with_shard_transition, + build_shard_block, + build_shard_transitions_till_slot, ) -from eth2spec.test.helpers.crosslinks import ( - run_crosslinks_processing, -) -from eth2spec.test.helpers.shard_block import build_shard_block -from eth2spec.test.helpers.state import next_epoch, next_slot, next_slots +from eth2spec.test.helpers.state import next_epoch, next_slot, transition_to -@with_all_phases_except(['phase0']) +def run_basic_crosslink_tests(spec, state, target_len_offset_slot): + next_epoch(spec, state) + next_epoch(spec, state) + state = spec.upgrade_to_phase1(state) + next_slot(spec, state) + + # At the beginning, let `x = state.slot`, `state.shard_states[shard].slot == x - 1` + slot_x = state.slot + committee_index = spec.CommitteeIndex(0) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + assert state.shard_states[shard].slot == slot_x - 1 + + # Create SignedShardBlock at slot `shard_state.slot + 1` -> x + body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE + shard_block = build_shard_block(spec, state, shard, body=body, signed=True) + shard_blocks = [shard_block] + + # Attester creates `attestation` at slot x + # Use temporary next state to get ShardTransition of shard block + shard_transitions = build_shard_transitions_till_slot( + spec, + state, + shards=[shard, ], + shard_blocks={shard: shard_blocks}, + target_len_offset_slot=target_len_offset_slot, + ) + shard_transition = shard_transitions[shard] + attestation = build_attestation_with_shard_transition( + spec, + state, + slot=slot_x + target_len_offset_slot - 1, + index=committee_index, + target_len_offset_slot=target_len_offset_slot, + shard_transition=shard_transition, + ) + pre_gasprice = state.shard_states[shard].gasprice + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + pre_shard_state = state.shard_states[shard] + yield from run_crosslinks_processing(spec, state, shard_transitions, [attestation]) + + # After state transition, + assert state.slot == slot_x + target_len_offset_slot + shard_state = state.shard_states[shard] + assert shard_state != pre_shard_state + assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1] + + if target_len_offset_slot == 1: + assert shard_state.gasprice > pre_gasprice + + +@with_all_phases_except([PHASE0]) @spec_state_test @always_bls def test_basic_crosslinks(spec, state): - next_epoch(spec, state) - next_epoch(spec, state) - state = spec.upgrade_to_phase1(state) - next_slot(spec, state) - - committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) - body = b'1' * spec.MAX_SHARD_BLOCK_SIZE - shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) - shard_blocks = [shard_block] - - next_slot(spec, state) - - shard_transition = spec.get_shard_transition(state, shard, shard_blocks) - shard_transitions = [spec.ShardTransition()] * len(state.shard_states) - shard_transitions[shard] = shard_transition - - attestation = get_valid_on_time_attestation( - spec, - state, - slot=state.slot, - index=committee_index, - shard_transition=shard_transition, - signed=True, - ) - attestations = [attestation] - - pre_gasprice = state.shard_states[shard].gasprice - offset_slots = spec.get_offset_slots(state, shard) - assert len(offset_slots) == 1 - - yield from run_crosslinks_processing(spec, state, shard_transitions, attestations) - - shard_state = state.shard_states[shard] - assert shard_state.slot == offset_slots[-1] - assert shard_state.latest_block_root == shard_block.message.hash_tree_root() - assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1] - assert shard_state.gasprice > pre_gasprice + run_basic_crosslink_tests(spec, state, target_len_offset_slot=1) -@with_all_phases_except(['phase0']) +@with_all_phases_except([PHASE0]) @spec_state_test @always_bls def test_multiple_offset_slots(spec, state): - next_epoch(spec, state) - next_epoch(spec, state) - state = spec.upgrade_to_phase1(state) - next_slot(spec, state) - - committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) - body = b'1' * spec.MAX_SHARD_BLOCK_SIZE - shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) - shard_blocks = [shard_block] - - next_slots(spec, state, 3) - - shard_transition = spec.get_shard_transition(state, shard, shard_blocks) - shard_transitions = [spec.ShardTransition()] * len(state.shard_states) - shard_transitions[shard] = shard_transition - - attestation = get_valid_on_time_attestation( - spec, - state, - slot=state.slot, - index=committee_index, - shard_transition=shard_transition, - signed=True, - ) - attestations = [attestation] - - pre_gasprice = state.shard_states[shard].gasprice - offset_slots = spec.get_offset_slots(state, shard) - assert len(offset_slots) == 3 - - yield from run_crosslinks_processing(spec, state, shard_transitions, attestations) - - shard_state = state.shard_states[shard] - assert shard_state.slot == offset_slots[-1] - assert shard_state.latest_block_root == shard_block.message.hash_tree_root() - assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1] - assert shard_state.gasprice > pre_gasprice + run_basic_crosslink_tests(spec, state, target_len_offset_slot=3) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/sanity/__init__.py b/tests/core/pyspec/eth2spec/test/phase_1/sanity/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py new file mode 100644 index 000000000..168915d02 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py @@ -0,0 +1,141 @@ +from eth2spec.test.context import ( + PHASE0, + with_all_phases_except, + spec_state_test, + always_bls, +) +from eth2spec.test.helpers.block import build_empty_block +from eth2spec.test.helpers.shard_block import ( + build_attestation_with_shard_transition, + build_shard_block, + build_shard_transitions_till_slot, +) +from eth2spec.test.helpers.state import next_epoch, next_slot, state_transition_and_sign_block + + +@with_all_phases_except([PHASE0]) +@spec_state_test +@always_bls +def test_process_beacon_block_with_normal_shard_transition(spec, state): + next_epoch(spec, state) + next_epoch(spec, state) + state = spec.upgrade_to_phase1(state) + next_slot(spec, state) + + target_len_offset_slot = 1 + + # At the beginning, let `x = state.slot`, `state.shard_states[shard].slot == x - 1` + slot_x = state.slot + committee_index = spec.CommitteeIndex(0) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + assert state.shard_states[shard].slot == slot_x - 1 + + # Create SignedShardBlock at slot `shard_state.slot + 1` -> x + body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE + shard_block = build_shard_block(spec, state, shard, body=body, signed=True) + shard_blocks = [shard_block] + + # Attester creates `attestation` at slot x + # Use temporary next state to get ShardTransition of shard block + shard_transitions = build_shard_transitions_till_slot( + spec, + state, + shards=[shard, ], + shard_blocks={shard: shard_blocks}, + target_len_offset_slot=target_len_offset_slot, + ) + shard_transition = shard_transitions[shard] + attestation = build_attestation_with_shard_transition( + spec, + state, + slot=slot_x + target_len_offset_slot - 1, + index=committee_index, + target_len_offset_slot=target_len_offset_slot, + shard_transition=shard_transition, + ) + pre_gasprice = state.shard_states[shard].gasprice + + # Propose beacon block at slot `x + 1` + pre_shard_state = state.shard_states[shard] + beacon_block_1 = build_empty_block(spec, state, slot=slot_x + target_len_offset_slot) + beacon_block_1.body.attestations = [attestation] + beacon_block_1.body.shard_transitions = shard_transitions + assert ( + beacon_block_1.slot == slot_x + target_len_offset_slot + == shard_transition.shard_states[0].slot + target_len_offset_slot + ) + state_transition_and_sign_block(spec, state, beacon_block_1) + + # After state transition + assert state.slot == slot_x + target_len_offset_slot + shard_state = state.shard_states[shard] + # latest_block_root has changed + assert shard_state.latest_block_root == shard_block.message.hash_tree_root() + assert shard_state != pre_shard_state + assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1] + + if target_len_offset_slot == 1 and len(shard_blocks) > 0: + assert shard_state.gasprice > pre_gasprice + + +@with_all_phases_except([PHASE0]) +@spec_state_test +@always_bls +def test_process_beacon_block_with_empty_proposal_transition(spec, state): + next_epoch(spec, state) + next_epoch(spec, state) + state = spec.upgrade_to_phase1(state) + next_slot(spec, state) + + target_len_offset_slot = 1 + + # At the beginning, let `x = state.slot`, `state.shard_states[shard].slot == x - 1` + slot_x = state.slot + committee_index = spec.CommitteeIndex(0) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + assert state.shard_states[shard].slot == slot_x - 1 + + # No new shard block + shard_blocks = [] + + # Attester creates `attestation` at slot x + # Use temporary next state to get ShardTransition of shard block + shard_transitions = build_shard_transitions_till_slot( + spec, + state, + shards=[shard, ], + shard_blocks={shard: shard_blocks}, + target_len_offset_slot=target_len_offset_slot, + ) + shard_transition = shard_transitions[shard] + attestation = build_attestation_with_shard_transition( + spec, + state, + slot=slot_x + target_len_offset_slot - 1, + index=committee_index, + target_len_offset_slot=target_len_offset_slot, + shard_transition=shard_transition, + ) + pre_gasprice = state.shard_states[shard].gasprice + + # Propose beacon block at slot `x + 1` + pre_shard_state = state.shard_states[shard] + beacon_block_1 = build_empty_block(spec, state, slot=slot_x + target_len_offset_slot) + beacon_block_1.body.attestations = [attestation] + beacon_block_1.body.shard_transitions = shard_transitions + assert ( + beacon_block_1.slot == slot_x + target_len_offset_slot + == shard_transition.shard_states[0].slot + target_len_offset_slot + ) + state_transition_and_sign_block(spec, state, beacon_block_1) + + # After state transition + assert state.slot == slot_x + target_len_offset_slot + shard_state = state.shard_states[shard] + # latest_block_root hasn't changed + assert shard_state.latest_block_root == pre_shard_state.latest_block_root + assert shard_state != pre_shard_state + assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1] + + if target_len_offset_slot == 1 and len(shard_blocks) > 0: + assert shard_state.gasprice > pre_gasprice From e758fb76c2ee67afa63a581c5b03022c6a739875 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 2 May 2020 01:39:03 +0800 Subject: [PATCH 096/102] Check `head_shard_root` of all `transition_attestations` --- specs/phase1/beacon-chain.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 611cc1955..f58a95fb7 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -788,6 +788,9 @@ def process_crosslink_for_shard(state: BeaconState, for attestation in transition_attestations: participants = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) transition_participants = transition_participants.union(participants) + assert attestation.data.head_shard_root == shard_transition.shard_data_roots[ + len(shard_transition.shard_data_roots) - 1 + ] enough_online_stake = ( get_total_balance(state, online_indices.intersection(transition_participants)) * 3 >= @@ -799,9 +802,6 @@ def process_crosslink_for_shard(state: BeaconState, # Attestation <-> shard transition consistency assert shard_transition_root == hash_tree_root(shard_transition) - assert attestation.data.head_shard_root == shard_transition.shard_data_roots[ - len(shard_transition.shard_data_roots) - 1 - ] # Apply transition apply_shard_transition(state, shard, shard_transition) From ff850251130c807b1403a5c9a9cfd0ef17a28a01 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 2 May 2020 01:56:25 +0800 Subject: [PATCH 097/102] PR feedback from terence --- specs/phase1/beacon-chain.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index f58a95fb7..c9d8eeb54 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -735,8 +735,12 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr shard_parent_root = state.shard_states[shard].latest_block_root for i in range(len(offset_slots)): shard_block_length = transition.shard_block_lengths[i] - is_empty_proposal = shard_block_length == 0 shard_state = transition.shard_states[i] + # Verify correct calculation of gas prices and slots + assert shard_state.gasprice == compute_updated_gasprice(prev_gasprice, shard_block_length) + assert shard_state.slot == offset_slots[i] + # Collect the non-empty proposals result + is_empty_proposal = shard_block_length == 0 if not is_empty_proposal: proposal_index = get_shard_proposer_index(state, offset_slots[i], shard) # Reconstruct shard headers @@ -751,9 +755,6 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr headers.append(header) proposers.append(proposal_index) - # Verify correct calculation of gas prices and slots - assert shard_state.gasprice == compute_updated_gasprice(prev_gasprice, shard_block_length) - assert shard_state.slot == offset_slots[i] prev_gasprice = shard_state.gasprice pubkeys = [state.validators[proposer].pubkey for proposer in proposers] From 977cd73379e383cb4f1b08d342f0d790ab513915 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 2 May 2020 01:57:34 +0800 Subject: [PATCH 098/102] Refactor the tests --- .../eth2spec/test/helpers/shard_block.py | 33 ++-- .../pyspec/eth2spec/test/helpers/state.py | 10 ++ .../test_process_crosslink.py | 47 +++--- .../test/phase_1/sanity/test_blocks.py | 154 +++++++----------- 4 files changed, 106 insertions(+), 138 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 922f4d748..ef65d2427 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -1,6 +1,6 @@ from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import get_state_and_beacon_parent_root_at_slot -from eth2spec.test.helpers.state import next_slots +from eth2spec.test.helpers.state import transition_to from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.bls import only_with_bls @@ -51,18 +51,17 @@ def build_shard_block(spec, return signed_block -def build_shard_transitions_till_slot(spec, state, shards, shard_blocks, target_len_offset_slot): - state = state.copy() - next_slots(spec, state, target_len_offset_slot) +def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): + temp_state = state.copy() + transition_to(spec, temp_state, on_time_slot) shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS - for shard in shards: - offset_slots = spec.get_offset_slots(state, shard) + for shard, blocks in shard_blocks.items(): + offset_slots = spec.get_offset_slots(temp_state, shard) len_offset_slots = len(offset_slots) - assert len_offset_slots == target_len_offset_slot - shard_blocks_of_shard = shard_blocks[shard] - shard_transition = spec.get_shard_transition(state, shard, shard_blocks_of_shard) - if len(shard_blocks_of_shard) > 0: - shard_block_root = shard_blocks_of_shard[-1].message.hash_tree_root() + assert len_offset_slots == on_time_slot - state.shard_states[shard].slot - 1 + shard_transition = spec.get_shard_transition(temp_state, shard, blocks) + if len(blocks) > 0: + shard_block_root = blocks[-1].message.hash_tree_root() assert shard_transition.shard_states[len_offset_slots - 1].latest_block_root == shard_block_root assert shard_transition.shard_states[len_offset_slots - 1].slot == offset_slots[-1] shard_transitions[shard] = shard_transition @@ -70,19 +69,17 @@ def build_shard_transitions_till_slot(spec, state, shards, shard_blocks, target_ return shard_transitions -def build_attestation_with_shard_transition(spec, state, slot, index, target_len_offset_slot, shard_transition=None): - state = state.copy() - next_slots(spec, state, target_len_offset_slot) +def build_attestation_with_shard_transition(spec, state, index, on_time_slot, shard_transition=None): + temp_state = state.copy() + transition_to(spec, temp_state, on_time_slot - 1) attestation = get_valid_on_time_attestation( spec, - state, - slot=slot, + temp_state, index=index, shard_transition=shard_transition, signed=True, ) - assert attestation.data.slot == slot + assert attestation.data.slot == temp_state.slot if shard_transition is not None: - assert target_len_offset_slot == len(shard_transition.shard_states) assert attestation.data.shard_transition_root == shard_transition.hash_tree_root() return attestation diff --git a/tests/core/pyspec/eth2spec/test/helpers/state.py b/tests/core/pyspec/eth2spec/test/helpers/state.py index 46a7ce2b5..f08f22602 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/state.py +++ b/tests/core/pyspec/eth2spec/test/helpers/state.py @@ -30,6 +30,16 @@ def transition_to(spec, state, slot): assert state.slot == slot +def transition_to_valid_shard_slot(spec, state): + """ + Transition to slot `spec.PHASE_1_GENESIS_SLOT + 1` and fork at `spec.PHASE_1_GENESIS_SLOT`. + """ + transition_to(spec, state, spec.PHASE_1_GENESIS_SLOT) + state = spec.upgrade_to_phase1(state) # `upgrade_to_phase1` is a pure function + next_slot(spec, state) + return state + + def next_epoch(spec, state): """ Transition to the start slot of the next epoch diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py index 912770da8..1f066b344 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py @@ -10,69 +10,64 @@ from eth2spec.test.helpers.shard_block import ( build_shard_block, build_shard_transitions_till_slot, ) -from eth2spec.test.helpers.state import next_epoch, next_slot, transition_to +from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot -def run_basic_crosslink_tests(spec, state, target_len_offset_slot): - next_epoch(spec, state) - next_epoch(spec, state) - state = spec.upgrade_to_phase1(state) - next_slot(spec, state) - +def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): + state = transition_to_valid_shard_slot(spec, state) # At the beginning, let `x = state.slot`, `state.shard_states[shard].slot == x - 1` slot_x = state.slot committee_index = spec.CommitteeIndex(0) shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) assert state.shard_states[shard].slot == slot_x - 1 - # Create SignedShardBlock at slot `shard_state.slot + 1` -> x + # Create SignedShardBlock body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE shard_block = build_shard_block(spec, state, shard, body=body, signed=True) shard_blocks = [shard_block] - - # Attester creates `attestation` at slot x - # Use temporary next state to get ShardTransition of shard block + # Create a shard_transitions that would be included at beacon block `state.slot + target_len_offset_slot` shard_transitions = build_shard_transitions_till_slot( spec, state, - shards=[shard, ], shard_blocks={shard: shard_blocks}, - target_len_offset_slot=target_len_offset_slot, + on_time_slot=state.slot + target_len_offset_slot, ) shard_transition = shard_transitions[shard] + # Create an attestation that would be included at beacon block `state.slot + target_len_offset_slot` attestation = build_attestation_with_shard_transition( spec, state, - slot=slot_x + target_len_offset_slot - 1, index=committee_index, - target_len_offset_slot=target_len_offset_slot, + on_time_slot=state.slot + target_len_offset_slot, shard_transition=shard_transition, ) pre_gasprice = state.shard_states[shard].gasprice - transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + transition_to(spec, state, state.slot + target_len_offset_slot) pre_shard_state = state.shard_states[shard] - yield from run_crosslinks_processing(spec, state, shard_transitions, [attestation]) - # After state transition, - assert state.slot == slot_x + target_len_offset_slot - shard_state = state.shard_states[shard] - assert shard_state != pre_shard_state - assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1] + yield from run_crosslinks_processing(spec, state, shard_transitions, [attestation], valid=valid) - if target_len_offset_slot == 1: - assert shard_state.gasprice > pre_gasprice + if valid: + # After state transition, + assert state.slot == slot_x + target_len_offset_slot + shard_state = state.shard_states[shard] + assert shard_state != pre_shard_state + assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1] + + if target_len_offset_slot == 1: + assert shard_state.gasprice > pre_gasprice @with_all_phases_except([PHASE0]) @spec_state_test @always_bls def test_basic_crosslinks(spec, state): - run_basic_crosslink_tests(spec, state, target_len_offset_slot=1) + yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=1, valid=True) @with_all_phases_except([PHASE0]) @spec_state_test @always_bls def test_multiple_offset_slots(spec, state): - run_basic_crosslink_tests(spec, state, target_len_offset_slot=3) + yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=3, valid=True) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py index 168915d02..60af35d45 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py @@ -1,3 +1,5 @@ +from typing import Dict, Sequence + from eth2spec.test.context import ( PHASE0, with_all_phases_except, @@ -10,69 +12,74 @@ from eth2spec.test.helpers.shard_block import ( build_shard_block, build_shard_transitions_till_slot, ) -from eth2spec.test.helpers.state import next_epoch, next_slot, state_transition_and_sign_block +from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot + + +def run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index, valid=True): + shard_transitions = build_shard_transitions_till_slot( + spec, state, shard_blocks, on_time_slot=state.slot + target_len_offset_slot + ) + attestations = [ + build_attestation_with_shard_transition( + spec, + state, + on_time_slot=state.slot + target_len_offset_slot, + index=committee_index, + shard_transition=shard_transitions[shard], + ) + for shard in shard_blocks.keys() + ] + + # Propose beacon block at slot `x + 1` + beacon_block = build_empty_block(spec, state, slot=state.slot + target_len_offset_slot) + beacon_block.body.attestations = attestations + beacon_block.body.shard_transitions = shard_transitions + + pre_shard_states = state.shard_states.copy() + yield 'pre', state.copy() + yield 'block', beacon_block + state_transition_and_sign_block(spec, state, beacon_block) + if valid: + yield 'post', state + else: + yield 'post', None + return + + for shard in range(spec.get_active_shard_count(state)): + post_shard_state = state.shard_states[shard] + if shard in shard_blocks: + # Shard state has been changed to state_transition result + assert post_shard_state == shard_transitions[shard].shard_states[ + len(shard_transitions[shard].shard_states) - 1 + ] + assert beacon_block.slot == shard_transitions[shard].shard_states[0].slot + target_len_offset_slot + assert post_shard_state.slot == state.slot - 1 + if len(shard_blocks[shard]) == 0: + # `latest_block_root` is the same + assert post_shard_state.latest_block_root == pre_shard_states[shard].latest_block_root @with_all_phases_except([PHASE0]) @spec_state_test @always_bls def test_process_beacon_block_with_normal_shard_transition(spec, state): - next_epoch(spec, state) - next_epoch(spec, state) - state = spec.upgrade_to_phase1(state) - next_slot(spec, state) + state = transition_to_valid_shard_slot(spec, state) target_len_offset_slot = 1 - - # At the beginning, let `x = state.slot`, `state.shard_states[shard].slot == x - 1` - slot_x = state.slot committee_index = spec.CommitteeIndex(0) shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) - assert state.shard_states[shard].slot == slot_x - 1 + assert state.shard_states[shard].slot == state.slot - 1 - # Create SignedShardBlock at slot `shard_state.slot + 1` -> x - body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE - shard_block = build_shard_block(spec, state, shard, body=body, signed=True) - shard_blocks = [shard_block] - - # Attester creates `attestation` at slot x - # Use temporary next state to get ShardTransition of shard block - shard_transitions = build_shard_transitions_till_slot( - spec, - state, - shards=[shard, ], - shard_blocks={shard: shard_blocks}, - target_len_offset_slot=target_len_offset_slot, - ) - shard_transition = shard_transitions[shard] - attestation = build_attestation_with_shard_transition( - spec, - state, - slot=slot_x + target_len_offset_slot - 1, - index=committee_index, - target_len_offset_slot=target_len_offset_slot, - shard_transition=shard_transition, - ) pre_gasprice = state.shard_states[shard].gasprice - # Propose beacon block at slot `x + 1` - pre_shard_state = state.shard_states[shard] - beacon_block_1 = build_empty_block(spec, state, slot=slot_x + target_len_offset_slot) - beacon_block_1.body.attestations = [attestation] - beacon_block_1.body.shard_transitions = shard_transitions - assert ( - beacon_block_1.slot == slot_x + target_len_offset_slot - == shard_transition.shard_states[0].slot + target_len_offset_slot - ) - state_transition_and_sign_block(spec, state, beacon_block_1) + # Create SignedShardBlock at slot `shard_state.slot + 1` + body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE + shard_block = build_shard_block(spec, state, shard, body=body, signed=True) + shard_blocks: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]} + + yield from run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index) - # After state transition - assert state.slot == slot_x + target_len_offset_slot shard_state = state.shard_states[shard] - # latest_block_root has changed - assert shard_state.latest_block_root == shard_block.message.hash_tree_root() - assert shard_state != pre_shard_state - assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1] if target_len_offset_slot == 1 and len(shard_blocks) > 0: assert shard_state.gasprice > pre_gasprice @@ -82,60 +89,19 @@ def test_process_beacon_block_with_normal_shard_transition(spec, state): @spec_state_test @always_bls def test_process_beacon_block_with_empty_proposal_transition(spec, state): - next_epoch(spec, state) - next_epoch(spec, state) - state = spec.upgrade_to_phase1(state) - next_slot(spec, state) + state = transition_to_valid_shard_slot(spec, state) target_len_offset_slot = 1 - - # At the beginning, let `x = state.slot`, `state.shard_states[shard].slot == x - 1` - slot_x = state.slot committee_index = spec.CommitteeIndex(0) shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) - assert state.shard_states[shard].slot == slot_x - 1 + assert state.shard_states[shard].slot == state.slot - 1 # No new shard block - shard_blocks = [] + shard_blocks = {} - # Attester creates `attestation` at slot x - # Use temporary next state to get ShardTransition of shard block - shard_transitions = build_shard_transitions_till_slot( - spec, - state, - shards=[shard, ], - shard_blocks={shard: shard_blocks}, - target_len_offset_slot=target_len_offset_slot, - ) - shard_transition = shard_transitions[shard] - attestation = build_attestation_with_shard_transition( - spec, - state, - slot=slot_x + target_len_offset_slot - 1, - index=committee_index, - target_len_offset_slot=target_len_offset_slot, - shard_transition=shard_transition, - ) pre_gasprice = state.shard_states[shard].gasprice - # Propose beacon block at slot `x + 1` - pre_shard_state = state.shard_states[shard] - beacon_block_1 = build_empty_block(spec, state, slot=slot_x + target_len_offset_slot) - beacon_block_1.body.attestations = [attestation] - beacon_block_1.body.shard_transitions = shard_transitions - assert ( - beacon_block_1.slot == slot_x + target_len_offset_slot - == shard_transition.shard_states[0].slot + target_len_offset_slot - ) - state_transition_and_sign_block(spec, state, beacon_block_1) - - # After state transition - assert state.slot == slot_x + target_len_offset_slot - shard_state = state.shard_states[shard] - # latest_block_root hasn't changed - assert shard_state.latest_block_root == pre_shard_state.latest_block_root - assert shard_state != pre_shard_state - assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1] + yield from run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index) if target_len_offset_slot == 1 and len(shard_blocks) > 0: - assert shard_state.gasprice > pre_gasprice + assert state.shard_states[shard].gasprice > pre_gasprice From b43e24acb610ae75497fdb10633a110bf6cddcfc Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 2 May 2020 02:04:46 +0800 Subject: [PATCH 099/102] specs/phase1/fraud-proofs.md -> specs/phase1/shard-transition.md --- specs/phase1/{fraud-proofs.md => shard-transition.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename specs/phase1/{fraud-proofs.md => shard-transition.md} (100%) diff --git a/specs/phase1/fraud-proofs.md b/specs/phase1/shard-transition.md similarity index 100% rename from specs/phase1/fraud-proofs.md rename to specs/phase1/shard-transition.md From 4558c7db4ed4600895909420b4d0629cc93e0d56 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 2 May 2020 02:22:31 +0800 Subject: [PATCH 100/102] Reorg the file structure --- setup.py | 2 +- specs/phase1/shard-transition.md | 109 ++++++++++++++++--------------- 2 files changed, 57 insertions(+), 54 deletions(-) diff --git a/setup.py b/setup.py index d1c62fb72..e0d6561dd 100644 --- a/setup.py +++ b/setup.py @@ -375,7 +375,7 @@ class PySpecCommand(Command): specs/phase0/fork-choice.md specs/phase1/custody-game.md specs/phase1/beacon-chain.md - specs/phase1/fraud-proofs.md + specs/phase1/shard-transition.md specs/phase1/fork-choice.md specs/phase1/phase1-fork.md """ diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index c676e142a..a8de508fb 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -1,38 +1,70 @@ - - -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - -- [Ethereum 2.0 Phase 1 -- Shard Transition and Fraud Proofs](#ethereum-20-phase-1----shard-transition-and-fraud-proofs) - - [Table of contents](#table-of-contents) - - [Introduction](#introduction) - - [Fraud proofs](#fraud-proofs) - - [Shard state transition function](#shard-state-transition-function) - - [Verifying the proof](#verifying-the-proof) - - [Honest committee member behavior](#honest-committee-member-behavior) - - [Helper functions](#helper-functions) - - [Make attestations](#make-attestations) - - - # Ethereum 2.0 Phase 1 -- Shard Transition and Fraud Proofs **Notice**: This document is a work-in-progress for researchers and implementers. ## Table of contents - + + +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - TODO +- [Introduction](#introduction) +- [Helper functions](#helper-functions) + - [Misc](#misc) + - [Shard block verification functions](#shard-block-verification-functions) +- [Shard state transition](#shard-state-transition) +- [Fraud proofs](#fraud-proofs) + - [Verifying the proof](#verifying-the-proof) +- [Honest committee member behavior](#honest-committee-member-behavior) + - [Helper functions](#helper-functions-1) + - [Make attestations](#make-attestations) - + ## Introduction This document describes the shard transition function and fraud proofs as part of Phase 1 of Ethereum 2.0. -## Fraud proofs +## Helper functions -### Shard state transition function +### Misc + +```python +def compute_shard_transition_digest(beacon_state: BeaconState, + shard_state: ShardState, + beacon_parent_root: Root, + shard_body_root: Root) -> Bytes32: + # TODO: use SSZ hash tree root + return hash( + hash_tree_root(shard_state) + beacon_parent_root + shard_body_root + ) +``` + +### Shard block verification functions + +```python +def verify_shard_block_message(beacon_state: BeaconState, + shard_state: ShardState, + block: ShardBlock, + slot: Slot, + shard: Shard) -> bool: + assert block.shard_parent_root == shard_state.latest_block_root + assert block.slot == slot + assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard) + assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE + return True +``` + +```python +def verify_shard_block_signature(beacon_state: BeaconState, + signed_block: SignedShardBlock) -> bool: + proposer = beacon_state.validators[signed_block.message.proposer_index] + domain = get_domain(beacon_state, DOMAIN_SHARD_PROPOSAL, compute_epoch_at_slot(signed_block.message.slot)) + signing_root = compute_signing_root(signed_block.message, domain) + return bls.Verify(proposer.pubkey, signing_root, signed_block.signature) +``` + +## Shard state transition ```python def shard_state_transition(beacon_state: BeaconState, @@ -56,6 +88,8 @@ def shard_state_transition(beacon_state: BeaconState, shard_state.latest_block_root = latest_block_root ``` +We have a pure function `get_post_shard_state` for describing the fraud proof verification and honest validator behavior. + ```python def get_post_shard_state(beacon_state: BeaconState, shard_state: ShardState, @@ -68,38 +102,7 @@ def get_post_shard_state(beacon_state: BeaconState, return post_state ``` -```python -def compute_shard_transition_digest(beacon_state: BeaconState, - shard_state: ShardState, - beacon_parent_root: Root, - shard_body_root: Root) -> Bytes32: - # TODO: use SSZ hash tree root - return hash( - hash_tree_root(shard_state) + beacon_parent_root + shard_body_root - ) -``` - -```python -def verify_shard_block_message(beacon_state: BeaconState, - shard_state: ShardState, - block: ShardBlock, - slot: Slot, - shard: Shard) -> bool: - assert block.shard_parent_root == shard_state.latest_block_root - assert block.slot == slot - assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard) - assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE - return True -``` - -```python -def verify_shard_block_signature(beacon_state: BeaconState, - signed_block: SignedShardBlock) -> bool: - proposer = beacon_state.validators[signed_block.message.proposer_index] - domain = get_domain(beacon_state, DOMAIN_SHARD_PROPOSAL, compute_epoch_at_slot(signed_block.message.slot)) - signing_root = compute_signing_root(signed_block.message, domain) - return bls.Verify(proposer.pubkey, signing_root, signed_block.signature) -``` +## Fraud proofs ### Verifying the proof From 7a770186b5ba576bf14ce496dc2b0381d169840e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 2 May 2020 02:25:11 +0800 Subject: [PATCH 101/102] Reorg beacon-chain spec a bit --- specs/phase1/beacon-chain.md | 80 ++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index c9d8eeb54..35f6a6425 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -16,8 +16,8 @@ - [Extended `AttestationData`](#extended-attestationdata) - [Extended `Attestation`](#extended-attestation) - [Extended `PendingAttestation`](#extended-pendingattestation) - - [`IndexedAttestation`](#indexedattestation) - - [Extended `AttesterSlashing`](#extended-attesterslashing) + - [Extended `IndexedAttestation`](#extended-indexedattestation) + - [Extended `AttesterSlashing`](#extended-attesterslashing) - [Extended `Validator`](#extended-validator) - [Extended `BeaconBlockBody`](#extended-beaconblockbody) - [Extended `BeaconBlock`](#extended-beaconblock) @@ -52,9 +52,9 @@ - [`get_latest_slot_for_shard`](#get_latest_slot_for_shard) - [`get_offset_slots`](#get_offset_slots) - [Predicates](#predicates) + - [Updated `is_valid_indexed_attestation`](#updated-is_valid_indexed_attestation) - [`is_shard_attestation`](#is_shard_attestation) - [`is_winning_attestation`](#is_winning_attestation) - - [Updated `is_valid_indexed_attestation`](#updated-is_valid_indexed_attestation) - [Block processing](#block-processing) - [Operations](#operations) - [New Attestation processing](#new-attestation-processing) @@ -154,7 +154,7 @@ class PendingAttestation(Container): crosslink_success: boolean ``` -### `IndexedAttestation` +### Extended `IndexedAttestation` ```python class IndexedAttestation(Container): @@ -162,7 +162,7 @@ class IndexedAttestation(Container): attestation: Attestation ``` -#### Extended `AttesterSlashing` +### Extended `AttesterSlashing` Note that the `attestation_1` and `attestation_2` have a new `IndexedAttestation` definition. @@ -557,39 +557,6 @@ def get_offset_slots(state: BeaconState, shard: Shard) -> Sequence[Slot]: ### Predicates -#### `is_shard_attestation` - -```python -def is_shard_attestation(state: BeaconState, - attestation: Attestation, - committee_index: CommitteeIndex) -> bool: - if not ( - attestation.data.index == committee_index - and attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY == state.slot - ): - return False - - return True -``` - -#### `is_winning_attestation` - -```python -def is_winning_attestation(state: BeaconState, - attestation: PendingAttestation, - committee_index: CommitteeIndex, - winning_root: Root) -> bool: - """ - Check if ``attestation`` helped contribute to the successful crosslink of - ``winning_root`` formed by ``committee_index`` committee at the current slot. - """ - return ( - attestation.data.slot == state.slot - and attestation.data.index == committee_index - and attestation.data.shard_transition_root == winning_root - ) -``` - #### Updated `is_valid_indexed_attestation` Note that this replaces the Phase 0 `is_valid_indexed_attestation`. @@ -632,6 +599,40 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe return bls.AggregateVerify(zip(all_pubkeys, all_signing_roots), signature=attestation.signature) ``` +#### `is_shard_attestation` + +```python +def is_shard_attestation(state: BeaconState, + attestation: Attestation, + committee_index: CommitteeIndex) -> bool: + if not ( + attestation.data.index == committee_index + and attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY == state.slot # Must be on-time attestation + # TODO: MIN_ATTESTATION_INCLUSION_DELAY should always be 1 + ): + return False + + return True +``` + +#### `is_winning_attestation` + +```python +def is_winning_attestation(state: BeaconState, + attestation: PendingAttestation, + committee_index: CommitteeIndex, + winning_root: Root) -> bool: + """ + Check if ``attestation`` helped contribute to the successful crosslink of + ``winning_root`` formed by ``committee_index`` committee at the current slot. + """ + return ( + attestation.data.slot == state.slot + and attestation.data.index == committee_index + and attestation.data.shard_transition_root == winning_root + ) +``` + ### Block processing ```python @@ -942,7 +943,8 @@ def process_light_client_signatures(state: BeaconState, block_body: BeaconBlockB signing_root = compute_signing_root(get_block_root_at_slot(state, slot), get_domain(state, DOMAIN_LIGHT_CLIENT, compute_epoch_at_slot(slot))) if len(signer_pubkeys) == 0: - # TODO: Or disallow empty light_client_signature? + # TODO: handle the empty light_client_signature case? + assert block_body.light_client_signature == BLSSignature() return else: assert bls.FastAggregateVerify(signer_pubkeys, signing_root, signature=block_body.light_client_signature) From 5b3ed8a3e73b0a3825a1ac7ffa469e3759a03922 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 5 May 2020 09:45:24 -0600 Subject: [PATCH 102/102] bump VERSION.txt to 0.12.0 --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index a8839f70d..d33c3a212 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -0.11.2 \ No newline at end of file +0.12.0 \ No newline at end of file