diff --git a/Makefile b/Makefile index 7ca2cc909..b39538791 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,6 @@ SPEC_DIR = ./specs SCRIPT_DIR = ./scripts TEST_LIBS_DIR = ./test_libs PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec -PY_TEST_DIR = ./py_tests YAML_TEST_DIR = ./yaml_tests GENERATOR_DIR = ./test_generators CONFIGS_DIR = ./configs @@ -24,7 +23,7 @@ all: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR) $(YAML_TEST_TARGETS) clean: rm -rf $(YAML_TEST_DIR) rm -rf $(GENERATOR_VENVS) - rm -rf $(PY_TEST_DIR)/venv $(PY_TEST_DIR)/.pytest_cache + rm -rf $(PY_SPEC_DIR)/venv $(PY_SPEC_DIR)/.pytest_cache rm -rf $(PY_SPEC_ALL_TARGETS) # "make gen_yaml_tests" to run generators @@ -32,7 +31,7 @@ gen_yaml_tests: $(YAML_TEST_DIR) $(YAML_TEST_TARGETS) # runs a limited set of tests against a minimal config test: $(PY_SPEC_ALL_TARGETS) - cd $(PY_TEST_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt; pytest -m minimal_config . + cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt; python -m pytest -m minimal_config . # "make pyspec" to create the pyspec for all phases. pyspec: $(PY_SPEC_ALL_TARGETS) diff --git a/README.md b/README.md index d1aaab74e..aa5b7e302 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,5 @@ The following are the broad design goals for Ethereum 2.0: Documentation on the different components used during spec writing can be found here: * [YAML Test Generators](test_generators/README.md) -* [Executable Python Spec](test_libs/pyspec/README.md) -* [Py-tests](py_tests/README.md) +* [Executable Python Spec, with Py-tests](test_libs/pyspec/README.md) diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index e67cf79ce..d06febb77 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -62,8 +62,8 @@ SLOTS_PER_EPOCH: 64 MIN_SEED_LOOKAHEAD: 1 # 2**2 (= 4) epochs 25.6 minutes ACTIVATION_EXIT_DELAY: 4 -# 2**4 (= 16) epochs ~1.7 hours -EPOCHS_PER_ETH1_VOTING_PERIOD: 16 +# 2**10 (= 1,024) slots ~1.7 hours +SLOTS_PER_ETH1_VOTING_PERIOD: 1024 # 2**13 (= 8,192) slots ~13 hours SLOTS_PER_HISTORICAL_ROOT: 8192 # 2**8 (= 256) epochs ~27 hours diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index 91ab7b358..80af5398c 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -63,7 +63,7 @@ MIN_SEED_LOOKAHEAD: 1 # 2**2 (= 4) epochs 25.6 minutes ACTIVATION_EXIT_DELAY: 4 # [customized] higher frequency new deposits from eth1 for testing -EPOCHS_PER_ETH1_VOTING_PERIOD: 2 +SLOTS_PER_ETH1_VOTING_PERIOD: 16 # [customized] smaller state SLOTS_PER_HISTORICAL_ROOT: 64 # 2**8 (= 256) epochs ~27 hours diff --git a/py_tests/README.md b/py_tests/README.md deleted file mode 100644 index ca2bed4cc..000000000 --- a/py_tests/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# ETH 2.0 py-tests - -These tests are not intended for client-consumption. -These tests are sanity tests, to verify if the spec itself is consistent. - -There are ideas to port these tests to the YAML test suite, - but we are still looking for inputs on how this should work. - -## How to run tests - -### Automated - -Run `make test` from the root of the spec repository. - -### Manual - -From within the py_tests folder: - -Install dependencies: -```bash -python3 -m venv venv -. venv/bin/activate -pip3 install -r requirements.txt -``` -Note: make sure to run `make pyspec` from the root of the specs repository, to build the pyspec requirement. - -Run the tests: -``` -pytest -m minimal_config . -``` diff --git a/py_tests/requirements.txt b/py_tests/requirements.txt deleted file mode 100644 index 27b3f22d8..000000000 --- a/py_tests/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -eth-utils>=1.3.0,<2 -eth-typing>=2.1.0,<3.0.0 -oyaml==0.7 -pycryptodome==3.7.3 -py_ecc>=1.6.0 -pytest>=3.6,<3.7 -../test_libs/pyspec diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 864411c9f..fcd177917 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -25,7 +25,6 @@ - [`Fork`](#fork) - [`Crosslink`](#crosslink) - [`Eth1Data`](#eth1data) - - [`Eth1DataVote`](#eth1datavote) - [`AttestationData`](#attestationdata) - [`AttestationDataAndCustodyBit`](#attestationdataandcustodybit) - [`IndexedAttestation`](#indexedattestation) @@ -114,22 +113,17 @@ - [State caching](#state-caching) - [Per-epoch processing](#per-epoch-processing) - [Helper functions](#helper-functions-1) - - [Justification](#justification) + - [Justification and finalization](#justification-and-finalization) - [Crosslinks](#crosslinks) - - [Eth1 data](#eth1-data) - [Rewards and penalties](#rewards-and-penalties) - - [Justification and finalization](#justification-and-finalization) - - [Crosslinks](#crosslinks-1) - - [Apply rewards](#apply-rewards) - - [Balance-driven status transitions](#balance-driven-status-transitions) - - [Activation queue and start shard](#activation-queue-and-start-shard) + - [Registry updates](#registry-updates) - [Slashings](#slashings) - [Final updates](#final-updates) - [Per-slot processing](#per-slot-processing) - [Per-block processing](#per-block-processing) - [Block header](#block-header) - [RANDAO](#randao) - - [Eth1 data](#eth1-data-1) + - [Eth1 data](#eth1-data) - [Operations](#operations) - [Proposer slashings](#proposer-slashings) - [Attester slashings](#attester-slashings) @@ -211,16 +205,14 @@ These configurations are updated for releases, but may be out of sync during `de | Name | Value | | - | - | | `GENESIS_FORK_VERSION` | `int_to_bytes4(0)` | -| `GENESIS_SLOT` | `2**32` | -| `GENESIS_EPOCH` | `slot_to_epoch(GENESIS_SLOT)` | +| `GENESIS_SLOT` | `0` | +| `GENESIS_EPOCH` | `0` | | `GENESIS_START_SHARD` | `0` | | `FAR_FUTURE_EPOCH` | `2**64 - 1` | | `ZERO_HASH` | `int_to_bytes32(0)` | | `EMPTY_SIGNATURE` | `int_to_bytes96(0)` | | `BLS_WITHDRAWAL_PREFIX_BYTE` | `int_to_bytes1(0)` | -* `GENESIS_SLOT` should be at least as large in terms of time as the largest of the time parameters or state list lengths below (ie. it should be at least as large as any value measured in slots, and at least `SLOTS_PER_EPOCH` times as large as any value measured in epochs). - ### Time parameters | Name | Value | Unit | Duration | @@ -230,7 +222,7 @@ These configurations are updated for releases, but may be out of sync during `de | `SLOTS_PER_EPOCH` | `2**6` (= 64) | slots | 6.4 minutes | | `MIN_SEED_LOOKAHEAD` | `2**0` (= 1) | epochs | 6.4 minutes | | `ACTIVATION_EXIT_DELAY` | `2**2` (= 4) | epochs | 25.6 minutes | -| `EPOCHS_PER_ETH1_VOTING_PERIOD` | `2**4` (= 16) | epochs | ~1.7 hours | +| `SLOTS_PER_ETH1_VOTING_PERIOD` | `2**10` (= 1,024) | slots | ~1.7 hours | | `SLOTS_PER_HISTORICAL_ROOT` | `2**13` (= 8,192) | slots | ~13 hours | | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours | | `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | 9 days | @@ -327,17 +319,6 @@ The types are defined topologically to aid in facilitating an executable version } ``` -#### `Eth1DataVote` - -```python -{ - # Data being voted for - 'eth1_data': Eth1Data, - # Vote count - 'vote_count': 'uint64', -} -``` - #### `AttestationData` ```python @@ -440,8 +421,6 @@ The types are defined topologically to aid in facilitating an executable version 'aggregation_bitfield': 'bytes', # Attestation data 'data': AttestationData, - # Custody bitfield - 'custody_bitfield': 'bytes', # Inclusion slot 'inclusion_slot': 'uint64', } @@ -594,7 +573,7 @@ The types are defined topologically to aid in facilitating an executable version # Randomness and committees 'latest_randao_mixes': ['bytes32', LATEST_RANDAO_MIXES_LENGTH], 'latest_start_shard': 'uint64', - + # Finality 'previous_epoch_attestations': [PendingAttestation], 'current_epoch_attestations': [PendingAttestation], @@ -617,7 +596,7 @@ The types are defined topologically to aid in facilitating an executable version # Ethereum 1.0 chain data 'latest_eth1_data': Eth1Data, - 'eth1_data_votes': [Eth1DataVote], + 'eth1_data_votes': [Eth1Data], 'deposit_index': 'uint64', } ``` @@ -667,7 +646,7 @@ Note: We aim to migrate to a S[T/N]ARK-friendly hash function in a future Ethere ```python def get_temporary_block_header(block: BeaconBlock) -> BeaconBlockHeader: """ - Return the block header corresponding to a block with ``state_root`` set to ``ZERO_HASH``. + Return the block header corresponding to a block with ``state_root`` set to ``ZERO_HASH``. """ return BeaconBlockHeader( slot=block.slot, @@ -695,8 +674,10 @@ def slot_to_epoch(slot: Slot) -> Epoch: def get_previous_epoch(state: BeaconState) -> Epoch: """` Return the previous epoch of the given ``state``. + Return the current epoch if it's genesis epoch. """ - return get_current_epoch(state) - 1 + current_epoch = get_current_epoch(state) + return (current_epoch - 1) if current_epoch > GENESIS_EPOCH else current_epoch ``` ### `get_current_epoch` @@ -810,7 +791,7 @@ def get_permuted_index(index: int, list_size: int, seed: Bytes32) -> int: """ assert index < list_size assert list_size <= 2**40 - + for round in range(SHUFFLE_ROUND_COUNT): pivot = bytes_to_int(hash(seed + int_to_bytes1(round))[0:8]) % list_size flip = (pivot - index) % list_size @@ -931,7 +912,7 @@ def get_block_root(state: BeaconState, return state.latest_block_roots[slot % SLOTS_PER_HISTORICAL_ROOT] ``` -`get_block_root(_, s)` should always return `hash_tree_root` of the block in the beacon chain at slot `s`, and `get_crosslink_committees_at_slot(_, s)` should not change unless the [validator](#dfn-validator) registry changes. +`get_block_root(_, s)` should always return `signed_root` of the block in the beacon chain at slot `s`, and `get_crosslink_committees_at_slot(_, s)` should not change unless the [validator](#dfn-validator) registry changes. ### `get_state_root` @@ -991,6 +972,7 @@ def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: Return the beacon proposer index at ``state.slot``. """ current_epoch = get_current_epoch(state) + first_committee, _ = get_crosslink_committees_at_slot(state, state.slot)[0] i = 0 while True: @@ -1277,14 +1259,13 @@ Note: All functions in this section mutate `state`. #### `activate_validator` ```python -def activate_validator(state: BeaconState, index: ValidatorIndex, is_genesis: bool) -> None: +def activate_validator(state: BeaconState, index: ValidatorIndex) -> None: """ Activate the validator of the given ``index``. Note that this function mutates ``state``. """ validator = state.validator_registry[index] - - if is_genesis: + if state.slot == GENESIS_SLOT: validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH else: @@ -1452,7 +1433,7 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], # Finality previous_epoch_attestations=[], current_epoch_attestations=[], - previous_justified_epoch=GENESIS_EPOCH - 1, + previous_justified_epoch=GENESIS_EPOCH, current_justified_epoch=GENESIS_EPOCH, previous_justified_root=ZERO_HASH, current_justified_root=ZERO_HASH, @@ -1482,7 +1463,7 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], # Process genesis activations for index in range(len(state.validator_registry)): if get_effective_balance(state, index) >= MAX_DEPOSIT_AMOUNT: - activate_validator(state, index, is_genesis=True) + activate_validator(state, index) genesis_active_index_root = hash_tree_root(get_active_validator_indices(state, GENESIS_EPOCH)) for index in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH): @@ -1505,7 +1486,7 @@ For a beacon chain block, `block`, to be processed by a node, the following cond * The parent block with root `block.previous_block_root` has been processed and accepted. * An Ethereum 1.0 block pointed to by the `state.latest_eth1_data.block_hash` has been processed and accepted. -* The node's Unix time is greater than or equal to `state.genesis_time + (block.slot - GENESIS_SLOT) * SECONDS_PER_SLOT`. (Note that leap seconds mean that slots will occasionally last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds, possibly several times a year.) +* The node's Unix time is greater than or equal to `state.genesis_time + block.slot * SECONDS_PER_SLOT`. (Note that leap seconds mean that slots will occasionally last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds, possibly several times a year.) If these conditions are not met, the client should delay processing the beacon block until the conditions are all satisfied. @@ -1592,17 +1573,17 @@ At every `slot > GENESIS_SLOT` run the following function: ```python def cache_state(state: BeaconState) -> None: - previous_slot_state_root = hash_tree_root(state) + # Cache latest known state root (for previous slot) + latest_state_root = hash_tree_root(state) + state.latest_state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = latest_state_root - # store the previous slot's post state transition root - state.latest_state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_slot_state_root - - # cache state root in stored latest_block_header if empty + # Store latest known state root (for previous slot) in latest_block_header if it is empty if state.latest_block_header.state_root == ZERO_HASH: - state.latest_block_header.state_root = previous_slot_state_root + state.latest_block_header.state_root = latest_state_root - # store latest known block for previous slot - state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = signing_root(state.latest_block_header) + # Cache latest known block root (for previous slot) + latest_block_root = signing_root(state.latest_block_header) + state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = latest_block_root ``` ### Per-epoch processing @@ -1692,13 +1673,17 @@ def get_earliest_attestation(state: BeaconState, attestations: List[PendingAttes ], key=lambda a: a.inclusion_slot) ``` -#### Justification +#### Justification and finalization Run the following function: ```python -def update_justification_and_finalization(state: BeaconState) -> None: - antepenultimate_justified_epoch = state.previous_justified_epoch +def process_justification_and_finalization(state: BeaconState) -> None: + if get_current_epoch(state) <= GENESIS_EPOCH + 1: + return + + old_previous_justified_epoch = state.previous_justified_epoch + old_current_justified_epoch = state.current_justified_epoch # Process justifications state.previous_justified_epoch = state.current_justified_epoch @@ -1719,20 +1704,20 @@ def update_justification_and_finalization(state: BeaconState) -> None: bitfield = state.justification_bitfield current_epoch = get_current_epoch(state) # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source - if (bitfield >> 1) % 8 == 0b111 and antepenultimate_justified_epoch == current_epoch - 3: - state.finalized_epoch = antepenultimate_justified_epoch + if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_epoch == current_epoch - 3: + state.finalized_epoch = old_previous_justified_epoch state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source - if (bitfield >> 1) % 4 == 0b11 and antepenultimate_justified_epoch == current_epoch - 2: - state.finalized_epoch = antepenultimate_justified_epoch + if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_epoch == current_epoch - 2: + state.finalized_epoch = old_previous_justified_epoch state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source - if (bitfield >> 0) % 8 == 0b111 and state.previous_justified_root == current_epoch - 2: - state.finalized_epoch = state.previous_justified_root + if (bitfield >> 0) % 8 == 0b111 and old_current_justified_epoch == current_epoch - 2: + state.finalized_epoch = old_current_justified_epoch state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source - if (bitfield >> 0) % 4 == 0b11 and state.previous_justified_root == current_epoch - 1: - state.finalized_epoch = state.previous_justified_root + if (bitfield >> 0) % 4 == 0b11 and old_current_justified_epoch == current_epoch - 1: + state.finalized_epoch = old_current_justified_epoch state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) ``` @@ -1742,9 +1727,8 @@ Run the following function: ```python def process_crosslinks(state: BeaconState) -> None: - current_epoch = get_current_epoch(state) - previous_epoch = max(current_epoch - 1, GENESIS_EPOCH) - next_epoch = current_epoch + 1 + previous_epoch = get_previous_epoch(state) + next_epoch = get_current_epoch(state) + 1 for slot in range(get_epoch_start_slot(previous_epoch), get_epoch_start_slot(next_epoch)): for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot): winning_root, participants = get_winning_root_and_participants(state, shard) @@ -1757,24 +1741,9 @@ def process_crosslinks(state: BeaconState) -> None: ) ``` -#### Eth1 data - -Run the following function: - -```python -def maybe_reset_eth1_period(state: BeaconState) -> None: - if (get_current_epoch(state) + 1) % EPOCHS_PER_ETH1_VOTING_PERIOD == 0: - for eth1_data_vote in state.eth1_data_votes: - # If a majority of all votes were for a particular eth1_data value, - # then set that as the new canonical value - if eth1_data_vote.vote_count * 2 > EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH: - state.latest_eth1_data = eth1_data_vote.eth1_data - state.eth1_data_votes = [] -``` - #### Rewards and penalties -We first define a helper: +First, we define additional helpers: ```python def get_base_reward(state: BeaconState, total_balance: Gwei, index: ValidatorIndex) -> Gwei: @@ -1785,8 +1754,6 @@ def get_base_reward(state: BeaconState, total_balance: Gwei, index: ValidatorInd return get_effective_balance(state, index) // adjusted_quotient // BASE_REWARDS_PER_EPOCH ``` -##### Justification and finalization - ```python def get_justification_and_finalization_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: previous_epoch = get_previous_epoch(state) @@ -1829,8 +1796,6 @@ def get_justification_and_finalization_deltas(state: BeaconState) -> Tuple[List[ return [rewards, penalties] ``` -##### Crosslinks - ```python def get_crosslink_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: rewards = [0 for index in range(len(state.validator_registry))] @@ -1851,12 +1816,13 @@ def get_crosslink_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: return [rewards, penalties] ``` -#### Apply rewards - -Run the following: +Run the following function: ```python -def apply_rewards(state: BeaconState) -> None: +def process_rewards_and_penalties(state: BeaconState) -> None: + if get_current_epoch(state) == GENESIS_EPOCH: + return + rewards1, penalties1 = get_justification_and_finalization_deltas(state) rewards2, penalties2 = get_crosslink_deltas(state) for i in range(len(state.validator_registry)): @@ -1864,16 +1830,13 @@ def apply_rewards(state: BeaconState) -> None: decrease_balance(state, i, penalties1[i] + penalties2[i]) ``` -#### Balance-driven status transitions +#### Registry updates -Run `process_balance_driven_status_transitions(state)`. +Run the following function: ```python -def process_balance_driven_status_transitions(state: BeaconState) -> None: - """ - Iterate through the validator registry - and deposit or eject active validators with sufficiently high or low balances - """ +def process_registry_updates(state: BeaconState) -> None: + # Process activation eligibility and ejections for index, validator in enumerate(state.validator_registry): balance = get_balance(state, index) if validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH and balance >= MAX_DEPOSIT_AMOUNT: @@ -1881,39 +1844,23 @@ def process_balance_driven_status_transitions(state: BeaconState) -> None: if is_active_validator(validator, get_current_epoch(state)) and balance < EJECTION_BALANCE: initiate_validator_exit(state, index) -``` -#### Activation queue and start shard - -Run the following function: - -```python -def update_registry(state: BeaconState) -> None: + # Process activations activation_queue = sorted([ index for index, validator in enumerate(state.validator_registry) if validator.activation_eligibility_epoch != FAR_FUTURE_EPOCH and validator.activation_epoch >= get_delayed_activation_exit_epoch(state.finalized_epoch) ], key=lambda index: state.validator_registry[index].activation_eligibility_epoch) - for index in activation_queue[:get_churn_limit(state)]: - activate_validator(state, index, is_genesis=False) - - state.latest_start_shard = ( - state.latest_start_shard + - get_shard_delta(state, get_current_epoch(state)) - ) % SHARD_COUNT + activate_validator(state, index) ``` #### Slashings -Run `process_slashings(state)`: +Run the following function: ```python def process_slashings(state: BeaconState) -> None: - """ - Process the slashings. - Note that this function mutates ``state``. - """ current_epoch = get_current_epoch(state) active_validator_indices = get_active_validator_indices(state, current_epoch) total_balance = get_total_balance(state, active_validator_indices) @@ -1937,9 +1884,14 @@ def process_slashings(state: BeaconState) -> None: Run the following function: ```python -def finish_epoch_update(state: BeaconState) -> None: +def process_final_updates(state: BeaconState) -> None: current_epoch = get_current_epoch(state) next_epoch = current_epoch + 1 + # Reset eth1 data votes + if state.slot % SLOTS_PER_ETH1_VOTING_PERIOD == 0: + state.eth1_data_votes = [] + # Update start shard + state.latest_start_shard = (state.latest_start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT # Set active index root index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % LATEST_ACTIVE_INDEX_ROOTS_LENGTH state.latest_active_index_roots[index_root_position] = hash_tree_root( @@ -2021,13 +1973,9 @@ def process_randao(state: BeaconState, block: BeaconBlock) -> None: ```python def process_eth1_data(state: BeaconState, block: BeaconBlock) -> None: - for eth1_data_vote in state.eth1_data_votes: - # If someone else has already voted for the same hash, add to its counter - if eth1_data_vote.eth1_data == block.body.eth1_data: - eth1_data_vote.vote_count += 1 - return - # If we're seeing this hash for the first time, make a new counter - state.eth1_data_votes.append(Eth1DataVote(eth1_data=block.body.eth1_data, vote_count=1)) + state.eth1_data_votes.append(block.body.eth1_data) + if state.eth1_data_votes.count(block.body.eth1_data) * 2 > SLOTS_PER_ETH1_VOTING_PERIOD: + state.latest_eth1_data = block.body.eth1_data ``` #### Operations @@ -2113,8 +2061,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: Process ``Attestation`` operation. Note that this function mutates ``state``. """ - assert max(GENESIS_SLOT, state.slot - SLOTS_PER_EPOCH) <= attestation.data.slot - assert attestation.data.slot <= state.slot - MIN_ATTESTATION_INCLUSION_DELAY + assert attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= attestation.data.slot + SLOTS_PER_EPOCH # Check target epoch, source epoch, and source root target_epoch = slot_to_epoch(attestation.data.slot) @@ -2141,7 +2088,6 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: pending_attestation = PendingAttestation( data=attestation.data, aggregation_bitfield=attestation.aggregation_bitfield, - custody_bitfield=attestation.custody_bitfield, inclusion_slot=state.slot ) if target_epoch == get_current_epoch(state): @@ -2164,7 +2110,7 @@ def process_proposer_attestation_rewards(state: BeaconState) -> None: ##### Deposits -Verify that `len(block.body.deposits) == min(MAX_DEPOSITS, latest_eth1_data.deposit_count - state.deposit_index)`. +Verify that `len(block.body.deposits) == min(MAX_DEPOSITS, state.latest_eth1_data.deposit_count - state.deposit_index)`. For each `deposit` in `block.body.deposits`, run the following function: diff --git a/specs/light_client/merkle_proofs.md b/specs/light_client/merkle_proofs.md index 285445ca8..63c018f2f 100644 --- a/specs/light_client/merkle_proofs.md +++ b/specs/light_client/merkle_proofs.md @@ -1,12 +1,26 @@ **NOTICE**: This document is a work-in-progress for researchers and implementers. -### Constants +## Table of Contents + + +- [Table of Contents](#table-of-contents) +- [Constants](#constants) +- [Generalized Merkle tree index](#generalized-merkle-tree-index) +- [SSZ object to index](#ssz-object-to-index) +- [Merkle multiproofs](#merkle-multiproofs) +- [MerklePartial](#merklepartial) + - [`SSZMerklePartial`](#sszmerklepartial) + - [Proofs for execution](#proofs-for-execution) + + + +## Constants | Name | Value | | - | - | | `LENGTH_FLAG` | `2**64 - 1` | -### Generalized Merkle tree index +## Generalized Merkle tree index In a binary Merkle tree, we define a "generalized index" of a node as `2**depth + index`. Visually, this looks as follows: @@ -20,16 +34,16 @@ In a binary Merkle tree, we define a "generalized index" of a node as `2**depth Note that the generalized index has the convenient property that the two children of node `k` are `2k` and `2k+1`, and also that it equals the position of a node in the linear representation of the Merkle tree that's computed by this function: ```python -def merkle_tree(leaves): +def merkle_tree(leaves: List[Bytes32]) -> List[Bytes32]: o = [0] * len(leaves) + leaves - for i in range(len(leaves)-1, 0, -1): - o[i] = hash(o[i*2] + o[i*2+1]) + for i in range(len(leaves) - 1, 0, -1): + o[i] = hash(o[i * 2] + o[i * 2 + 1]) return o ``` We will define Merkle proofs in terms of generalized indices. -### SSZ object to index +## SSZ object to index We can describe the hash tree of any SSZ object, rooted in `hash_tree_root(object)`, as a binary Merkle tree whose depth may vary. For example, an object `{x: bytes32, y: List[uint64]}` would look as follows: @@ -47,33 +61,33 @@ y_data_root len(y) We can now define a concept of a "path", a way of describing a function that takes as input an SSZ object and outputs some specific (possibly deeply nested) member. For example, `foo -> foo.x` is a path, as are `foo -> len(foo.y)` and `foo -> foo.y[5].w`. We'll describe paths as lists, which can have two representations. In "human-readable form", they are `["x"]`, `["y", "__len__"]` and `["y", 5, "w"]` respectively. In "encoded form", they are lists of `uint64` values, in these cases (assuming the fields of `foo` in order are `x` then `y`, and `w` is the first field of `y[i]`) `[0]`, `[1, 2**64-1]`, `[1, 5, 0]`. ```python -def path_to_encoded_form(obj: Any, path: List[str or int]) -> List[int]: +def path_to_encoded_form(obj: Any, path: List[Union[str, int]]) -> List[int]: if len(path) == 0: return [] - if isinstance(path[0], "__len__"): + elif isinstance(path[0], "__len__"): assert len(path) == 1 return [LENGTH_FLAG] elif isinstance(path[0], str) and hasattr(obj, "fields"): return [list(obj.fields.keys()).index(path[0])] + path_to_encoded_form(getattr(obj, path[0]), path[1:]) - elif isinstance(obj, (StaticList, DynamicList)): + elif isinstance(obj, (Vector, List)): return [path[0]] + path_to_encoded_form(obj[path[0]], path[1:]) else: raise Exception("Unknown type / path") ``` -We can now define a function `get_generalized_indices(object: Any, path: List[int], root=1: int) -> int` that converts an object and a path to a set of generalized indices (note that for constant-sized objects, there is only one generalized index and it only depends on the path, but for dynamically sized objects the indices may depend on the object itself too). For dynamically-sized objects, the set of indices will have more than one member because of the need to access an array's length to determine the correct generalized index for some array access. +We can now define a function `get_generalized_indices(object: Any, path: List[int], root: int=1) -> List[int]` that converts an object and a path to a set of generalized indices (note that for constant-sized objects, there is only one generalized index and it only depends on the path, but for dynamically sized objects the indices may depend on the object itself too). For dynamically-sized objects, the set of indices will have more than one member because of the need to access an array's length to determine the correct generalized index for some array access. ```python -def get_generalized_indices(obj: Any, path: List[int], root=1) -> List[int]: +def get_generalized_indices(obj: Any, path: List[int], root: int=1) -> List[int]: if len(path) == 0: return [root] - elif isinstance(obj, StaticList): + elif isinstance(obj, Vector): items_per_chunk = (32 // len(serialize(x))) if isinstance(x, int) else 1 new_root = root * next_power_of_2(len(obj) // items_per_chunk) + path[0] // items_per_chunk return get_generalized_indices(obj[path[0]], path[1:], new_root) - elif isinstance(obj, DynamicList) and path[0] == LENGTH_FLAG: + elif isinstance(obj, List) and path[0] == LENGTH_FLAG: return [root * 2 + 1] - elif isinstance(obj, DynamicList) and isinstance(path[0], int): + elif isinstance(obj, List) and isinstance(path[0], int): assert path[0] < len(obj) items_per_chunk = (32 // len(serialize(x))) if isinstance(x, int) else 1 new_root = root * 2 * next_power_of_2(len(obj) // items_per_chunk) + path[0] // items_per_chunk @@ -86,7 +100,7 @@ def get_generalized_indices(obj: Any, path: List[int], root=1) -> List[int]: raise Exception("Unknown type / path") ``` -### Merkle multiproofs +## Merkle multiproofs We define a Merkle multiproof as a minimal subset of nodes in a Merkle tree needed to fully authenticate that a set of nodes actually are part of a Merkle tree with some specified root, at a particular set of generalized indices. For example, here is the Merkle multiproof for positions 0, 1, 6 in an 8-node Merkle tree (ie. generalized indices 8, 9, 14): @@ -99,19 +113,12 @@ x x . . . . x * . are unused nodes, * are used nodes, x are the values we are trying to prove. Notice how despite being a multiproof for 3 values, it requires only 3 auxiliary nodes, only one node more than would be required to prove a single value. Normally the efficiency gains are not quite that extreme, but the savings relative to individual Merkle proofs are still significant. As a rule of thumb, a multiproof for k nodes at the same level of an n-node tree has size `k * (n/k + log(n/k))`. -Here is code for creating and verifying a multiproof. First a helper: - -```python -def log2(x): - return 0 if x == 1 else 1 + log2(x//2) -``` - -First, a method for computing the generalized indices of the auxiliary tree nodes that a proof of a given set of generalized indices will require: +Here is code for creating and verifying a multiproof. First, a method for computing the generalized indices of the auxiliary tree nodes that a proof of a given set of generalized indices will require: ```python def get_proof_indices(tree_indices: List[int]) -> List[int]: # Get all indices touched by the proof - maximal_indices = set({}) + maximal_indices = set() for i in tree_indices: x = i while x > 1: @@ -119,7 +126,7 @@ def get_proof_indices(tree_indices: List[int]) -> List[int]: x //= 2 maximal_indices = tree_indices + sorted(list(maximal_indices))[::-1] # Get indices that cannot be recalculated from earlier indices - redundant_indices = set({}) + redundant_indices = set() proof = [] for index in maximal_indices: if index not in redundant_indices: @@ -130,35 +137,35 @@ def get_proof_indices(tree_indices: List[int]) -> List[int]: break index //= 2 return [i for i in proof if i not in tree_indices] -```` +``` Generating a proof is simply a matter of taking the node of the SSZ hash tree with the union of the given generalized indices for each index given by `get_proof_indices`, and outputting the list of nodes in the same order. Here is the verification function: ```python -def verify_multi_proof(root, indices, leaves, proof): +def verify_multi_proof(root: Bytes32, indices: List[int], leaves: List[Bytes32], proof: List[Bytes32]) -> bool: tree = {} for index, leaf in zip(indices, leaves): tree[index] = leaf - for index, proofitem in zip(get_proof_indices(indices), proof): - tree[index] = proofitem - indexqueue = sorted(tree.keys())[:-1] + for index, proof_item in zip(get_proof_indices(indices), proof): + tree[index] = proof_item + index_queue = sorted(tree.keys())[:-1] i = 0 - while i < len(indexqueue): - index = indexqueue[i] - if index >= 2 and index^1 in tree: - tree[index//2] = hash(tree[index - index%2] + tree[index - index%2 + 1]) - indexqueue.append(index//2) + while i < len(index_queue): + index = index_queue[i] + if index >= 2 and index ^ 1 in tree: + tree[index // 2] = hash(tree[index - index % 2] + tree[index - index % 2 + 1]) + index_queue.append(index // 2) i += 1 return (indices == []) or (1 in tree and tree[1] == root) ``` -### MerklePartial +## MerklePartial We define: -#### `MerklePartial` +### `SSZMerklePartial` ```python @@ -170,8 +177,8 @@ We define: } ``` -#### Proofs for execution +### Proofs for execution -We define `MerklePartial(f, arg1, arg2..., focus=0)` as being a `MerklePartial` object wrapping a Merkle multiproof of the set of nodes in the hash tree of the SSZ object `arg[focus]` that is needed to authenticate the parts of the object needed to compute `f(arg1, arg2...)`. +We define `MerklePartial(f, arg1, arg2..., focus=0)` as being a `SSZMerklePartial` object wrapping a Merkle multiproof of the set of nodes in the hash tree of the SSZ object `arg[focus]` that is needed to authenticate the parts of the object needed to compute `f(arg1, arg2...)`. -Ideally, any function which accepts an SSZ object should also be able to accept a `MerklePartial` object as a substitute. +Ideally, any function which accepts an SSZ object should also be able to accept a `SSZMerklePartial` object as a substitute. diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index af1e0d5b8..900b2e64f 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -5,52 +5,78 @@ __NOTICE__: This document is a work-in-progress for researchers and implementers ## Table of Contents + - [Beacon Chain Light Client Syncing](#beacon-chain-light-client-syncing) - [Table of Contents](#table-of-contents) - - [Light client state](#light-client-state) - - [Updating the shuffled committee](#updating-the-shuffled-committee) - - [Computing the current committee](#computing-the-current-committee) - - [Verifying blocks](#verifying-blocks) + - [Preliminaries](#preliminaries) + - [Expansions](#expansions) + - [`get_active_validator_indices`](#get_active_validator_indices) + - [`MerklePartial`](#merklepartial) + - [`PeriodData`](#perioddata) + - [`get_earlier_start_epoch`](#get_earlier_start_epoch) + - [`get_later_start_epoch`](#get_later_start_epoch) + - [`get_period_data`](#get_period_data) + - [Light client state](#light-client-state) + - [Updating the shuffled committee](#updating-the-shuffled-committee) + - [Computing the current committee](#computing-the-current-committee) + - [Verifying blocks](#verifying-blocks) + +## Preliminaries -### Preliminaries +### Expansions We define an "expansion" of an object as an object where a field in an object that is meant to represent the `hash_tree_root` of another object is replaced by the object. Note that defining expansions is not a consensus-layer-change; it is merely a "re-interpretation" of the object. Particularly, the `hash_tree_root` of an expansion of an object is identical to that of the original object, and we can define expansions where, given a complete history, it is always possible to compute the expansion of any object in the history. The opposite of an expansion is a "summary" (eg. `BeaconBlockHeader` is a summary of `BeaconBlock`). We define two expansions: -* `ExtendedBeaconBlock`, which is identical to a `BeaconBlock` except `state_root` is replaced with the corresponding `state: ExtendedBeaconState` -* `ExtendedBeaconState`, which is identical to a `BeaconState` except `latest_active_index_roots: List[Bytes32]` is replaced by `latest_active_indices: List[List[ValidatorIndex]]`, where `BeaconState.latest_active_index_roots[i] = hash_tree_root(ExtendedBeaconState.latest_active_indices[i])` +* `ExtendedBeaconState`, which is identical to a `BeaconState` except `latest_active_index_roots: List[Bytes32]` is replaced by `latest_active_indices: List[List[ValidatorIndex]]`, where `BeaconState.latest_active_index_roots[i] = hash_tree_root(ExtendedBeaconState.latest_active_indices[i])`. +* `ExtendedBeaconBlock`, which is identical to a `BeaconBlock` except `state_root` is replaced with the corresponding `state: ExtendedBeaconState`. + +### `get_active_validator_indices` Note that there is now a new way to compute `get_active_validator_indices`: ```python -def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> List[ValidatorIndex]: +def get_active_validator_indices(state: ExtendedBeaconState, epoch: Epoch) -> List[ValidatorIndex]: return state.latest_active_indices[epoch % LATEST_ACTIVE_INDEX_ROOTS_LENGTH] ``` Note that it takes `state` instead of `state.validator_registry` as an argument. This does not affect its use in `get_shuffled_committee`, because `get_shuffled_committee` has access to the full `state` as one of its arguments. + +### `MerklePartial` + A `MerklePartial(f, *args)` is an object that contains a minimal Merkle proof needed to compute `f(*args)`. A `MerklePartial` can be used in place of a regular SSZ object, though a computation would return an error if it attempts to access part of the object that is not contained in the proof. -We add a data type `PeriodData` and four helpers: +### `PeriodData` ```python { 'validator_count': 'uint64', 'seed': 'bytes32', - 'committee': [Validator] + 'committee': [Validator], } ``` +### `get_earlier_start_epoch` + ```python def get_earlier_start_epoch(slot: Slot) -> int: return slot - slot % PERSISTENT_COMMITTEE_PERIOD - PERSISTENT_COMMITTEE_PERIOD * 2 - +``` + +### `get_later_start_epoch` + +```python def get_later_start_epoch(slot: Slot) -> int: return slot - slot % PERSISTENT_COMMITTEE_PERIOD - PERSISTENT_COMMITTEE_PERIOD - +``` + +### `get_period_data` + +```python def get_period_data(block: ExtendedBeaconBlock, shard_id: Shard, later: bool) -> PeriodData: period_start = get_later_start_epoch(header.slot) if later else get_earlier_start_epoch(header.slot) validator_count = len(get_active_validator_indices(state, period_start)) @@ -59,7 +85,7 @@ def get_period_data(block: ExtendedBeaconBlock, shard_id: Shard, later: bool) -> return PeriodData( validator_count, generate_seed(block.state, period_start), - [block.state.validator_registry[i] for i in indices] + [block.state.validator_registry[i] for i in indices], ) ``` @@ -72,7 +98,7 @@ A light client will keep track of: * `later_period_data = get_period_data(finalized_header, shard_id, later=True)` * `earlier_period_data = get_period_data(finalized_header, shard_id, later=False)` -We use the struct `validator_memory` to keep track of these variables. +We use the struct `ValidatorMemory` to keep track of these variables. ### Updating the shuffled committee @@ -85,14 +111,13 @@ later_period_data = get_period_data(new_committee_proof, finalized_header, shard The maximum size of a proof is `128 * ((22-7) * 32 + 110) = 75520` bytes for validator records and `(22-7) * 32 + 128 * 8 = 1504` for the active index proof (much smaller because the relevant active indices are all beside each other in the Merkle tree). This needs to be done once per `PERSISTENT_COMMITTEE_PERIOD` epochs (2048 epochs / 9 days), or ~38 bytes per epoch. -### Computing the current committee +## Computing the current committee Here is a helper to compute the committee at a slot given the maximal earlier and later committees: ```python def compute_committee(header: BeaconBlockHeader, - validator_memory: ValidatorMemory): - + validator_memory: ValidatorMemory) -> List[ValidatorIndex]: earlier_validator_count = validator_memory.earlier_period_data.validator_count later_validator_count = validator_memory.later_period_data.validator_count maximal_earlier_committee = validator_memory.earlier_period_data.committee @@ -100,17 +125,19 @@ def compute_committee(header: BeaconBlockHeader, earlier_start_epoch = get_earlier_start_epoch(header.slot) later_start_epoch = get_later_start_epoch(header.slot) epoch = slot_to_epoch(header.slot) - + committee_count = max( earlier_validator_count // (SHARD_COUNT * TARGET_COMMITTEE_SIZE), later_validator_count // (SHARD_COUNT * TARGET_COMMITTEE_SIZE), ) + 1 - - def get_offset(count, end:bool): - return get_split_offset(count, - SHARD_COUNT * committee_count, - validator_memory.shard_id * committee_count + (1 if end else 0)) - + + def get_offset(count: int, end: bool) -> int: + return get_split_offset( + count, + SHARD_COUNT * committee_count, + validator_memory.shard_id * committee_count + (1 if end else 0), + ) + actual_earlier_committee = maximal_earlier_committee[ 0:get_offset(earlier_validator_count, True) - get_offset(earlier_validator_count, False) ] @@ -119,31 +146,30 @@ def compute_committee(header: BeaconBlockHeader, ] def get_switchover_epoch(index): return ( - bytes_to_int(hash(validator_memory.earlier_period_data.seed + bytes3(index))[0:8]) % + bytes_to_int(hash(validator_memory.earlier_period_data.seed + int_to_bytes3(index))[0:8]) % PERSISTENT_COMMITTEE_PERIOD ) # Take not-yet-cycled-out validators from earlier committee and already-cycled-in validators from # later committee; return a sorted list of the union of the two, deduplicated return sorted(list(set( - [i for i in earlier_committee if epoch % PERSISTENT_COMMITTEE_PERIOD < get_switchover_epoch(i)] + - [i for i in later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= get_switchover_epoch(i)] + [i for i in actual_earlier_committee if epoch % PERSISTENT_COMMITTEE_PERIOD < get_switchover_epoch(i)] + + [i for i in actual_later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= get_switchover_epoch(i)] ))) - ``` Note that this method makes use of the fact that the committee for any given shard always starts and ends at the same validator index independently of the committee count (this is because the validator set is split into `SHARD_COUNT * committee_count` slices but the first slice of a shard is a multiple `committee_count * i`, so the start of the slice is `n * committee_count * i // (SHARD_COUNT * committee_count) = n * i // SHARD_COUNT`, using the slightly nontrivial algebraic identity `(x * a) // ab == x // b`). -### Verifying blocks +## Verifying blocks If a client wants to update its `finalized_header` it asks the network for a `BlockValidityProof`, which is simply: ```python { - 'header': BlockHeader, + 'header': BeaconBlockHeader, 'shard_aggregate_signature': 'bytes96', 'shard_bitfield': 'bytes', - 'shard_parent_block': ShardBlock + 'shard_parent_block': ShardBlock, } ``` @@ -151,23 +177,23 @@ The verification procedure is as follows: ```python def verify_block_validity_proof(proof: BlockValidityProof, validator_memory: ValidatorMemory) -> bool: - assert proof.shard_parent_block.beacon_chain_ref == hash_tree_root(proof.header) + assert proof.shard_parent_block.beacon_chain_root == hash_tree_root(proof.header) committee = compute_committee(proof.header, validator_memory) # Verify that we have >=50% support - support_balance = sum([c.high_balance for i, c in enumerate(committee) if get_bitfield_bit(proof.shard_bitfield, i) is True]) - total_balance = sum([c.high_balance for i, c in enumerate(committee)] + support_balance = sum([v.high_balance for i, v in enumerate(committee) if get_bitfield_bit(proof.shard_bitfield, i) is True]) + total_balance = sum([v.high_balance for i, v in enumerate(committee)]) assert support_balance * 2 > total_balance # Verify shard attestations group_public_key = bls_aggregate_pubkeys([ - v.pubkey for v, index in enumerate(committee) if - get_bitfield_bit(proof.shard_bitfield, i) is True + v.pubkey for v, index in enumerate(committee) + if get_bitfield_bit(proof.shard_bitfield, index) is True ]) assert bls_verify( pubkey=group_public_key, message_hash=hash_tree_root(shard_parent_block), - signature=shard_aggregate_signature, - domain=get_domain(state, slot_to_epoch(shard_block.slot), DOMAIN_SHARD_ATTESTER) + signature=proof.shard_aggregate_signature, + domain=get_domain(state, slot_to_epoch(shard_block.slot), DOMAIN_SHARD_ATTESTER), ) ``` -The size of this proof is only 200 (header) + 96 (signature) + 16 (bitfield) + 352 (shard block) = 664 bytes. It can be reduced further by replacing `ShardBlock` with `MerklePartial(lambda x: x.beacon_chain_ref, ShardBlock)`, which would cut off ~220 bytes. +The size of this proof is only 200 (header) + 96 (signature) + 16 (bitfield) + 352 (shard block) = 664 bytes. It can be reduced further by replacing `ShardBlock` with `MerklePartial(lambda x: x.beacon_chain_root, ShardBlock)`, which would cut off ~220 bytes. diff --git a/test_libs/pyspec/README.md b/test_libs/pyspec/README.md index 08042e746..b3cab11d2 100644 --- a/test_libs/pyspec/README.md +++ b/test_libs/pyspec/README.md @@ -7,6 +7,7 @@ With this executable spec, test-generators can easily create test-vectors for client implementations, and the spec itself can be verified to be consistent and coherent, through sanity tests implemented with pytest. + ## Building All the dynamic parts of the spec can be build at once with `make pyspec`. @@ -15,12 +16,42 @@ Alternatively, you can build a sub-set of the pyspec: `make phase0`. Or, to build a single file, specify the path, e.g. `make test_libs/pyspec/eth2spec/phase0/spec.py` + +## Py-tests + +These tests are not intended for client-consumption. +These tests are sanity tests, to verify if the spec itself is consistent. + +### How to run tests + +#### Automated + +Run `make test` from the root of the spec repository. + +#### Manual + +From within the `pyspec` folder: + +Install dependencies: +```bash +python3 -m venv venv +. venv/bin/activate +pip3 install -r requirements.txt +``` +Note: make sure to run `make pyspec` from the root of the specs repository, + to build the parts of the pyspec module derived from the markdown specs. + +Run the tests: +``` +pytest -m minimal_config . +``` + + ## Contributing Contributions are welcome, but consider implementing your idea as part of the spec itself first. The pyspec is not a replacement. -If you see opportunity to include any of the `pyspec/eth2spec/utils/` code in the spec, - please submit an issue or PR. + ## License diff --git a/test_libs/pyspec/eth2spec/phase0/state_transition.py b/test_libs/pyspec/eth2spec/phase0/state_transition.py index 94cd35b99..38ecd2a02 100644 --- a/test_libs/pyspec/eth2spec/phase0/state_transition.py +++ b/test_libs/pyspec/eth2spec/phase0/state_transition.py @@ -91,14 +91,12 @@ def process_block(state: BeaconState, def process_epoch_transition(state: BeaconState) -> None: - spec.update_justification_and_finalization(state) + spec.process_justification_and_finalization(state) spec.process_crosslinks(state) - spec.maybe_reset_eth1_period(state) - spec.apply_rewards(state) - spec.process_balance_driven_status_transitions(state) - spec.update_registry(state) + spec.process_rewards_and_penalties(state) + spec.process_registry_updates(state) spec.process_slashings(state) - spec.finish_epoch_update(state) + spec.process_final_updates(state) def state_transition_to(state: BeaconState, up_to: Slot) -> BeaconState: diff --git a/test_libs/pyspec/requirements.txt b/test_libs/pyspec/requirements.txt index 78d41708d..3296ef807 100644 --- a/test_libs/pyspec/requirements.txt +++ b/test_libs/pyspec/requirements.txt @@ -2,3 +2,4 @@ eth-utils>=1.3.0,<2 eth-typing>=2.1.0,<3.0.0 pycryptodome==3.7.3 py_ecc>=1.6.0 +pytest>=3.6,<3.7 diff --git a/test_libs/pyspec/setup.py b/test_libs/pyspec/setup.py index b04847d37..1a131a417 100644 --- a/test_libs/pyspec/setup.py +++ b/test_libs/pyspec/setup.py @@ -3,6 +3,7 @@ from setuptools import setup, find_packages setup( name='pyspec', packages=find_packages(), + tests_require=["pytest"], install_requires=[ "eth-utils>=1.3.0,<2", "eth-typing>=2.1.0,<3.0.0", diff --git a/py_tests/phase0/__init__.py b/test_libs/pyspec/tests/README.md similarity index 100% rename from py_tests/phase0/__init__.py rename to test_libs/pyspec/tests/README.md diff --git a/test_libs/pyspec/tests/__init__.py b/test_libs/pyspec/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/py_tests/phase0/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/block_processing/test_process_attestation.py similarity index 99% rename from py_tests/phase0/block_processing/test_process_attestation.py rename to test_libs/pyspec/tests/block_processing/test_process_attestation.py index 2e3f24dd6..d66434c2c 100644 --- a/py_tests/phase0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attestation.py @@ -11,7 +11,7 @@ from eth2spec.phase0.spec import ( process_attestation, slot_to_epoch, ) -from phase0.helpers import ( +from tests.helpers import ( build_empty_block_for_next_slot, get_valid_attestation, ) diff --git a/py_tests/phase0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py similarity index 97% rename from py_tests/phase0/block_processing/test_process_attester_slashing.py rename to test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py index 8db71deb9..84c19145a 100644 --- a/py_tests/phase0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py @@ -7,8 +7,9 @@ from eth2spec.phase0.spec import ( get_beacon_proposer_index, process_attester_slashing, ) -from phase0.helpers import ( +from tests.helpers import ( get_valid_attester_slashing, + next_epoch, ) # mark entire file as 'attester_slashing' @@ -58,6 +59,8 @@ def test_success_double(state): def test_success_surround(state): + next_epoch(state) + state.current_justified_epoch += 1 attester_slashing = get_valid_attester_slashing(state) # set attestion1 to surround attestation 2 diff --git a/py_tests/phase0/block_processing/test_process_block_header.py b/test_libs/pyspec/tests/block_processing/test_process_block_header.py similarity index 98% rename from py_tests/phase0/block_processing/test_process_block_header.py rename to test_libs/pyspec/tests/block_processing/test_process_block_header.py index 3b99f2ad4..b35b0a9c1 100644 --- a/py_tests/phase0/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/tests/block_processing/test_process_block_header.py @@ -8,7 +8,7 @@ from eth2spec.phase0.spec import ( advance_slot, process_block_header, ) -from phase0.helpers import ( +from tests.helpers import ( build_empty_block_for_next_slot, next_slot, ) diff --git a/py_tests/phase0/block_processing/test_process_deposit.py b/test_libs/pyspec/tests/block_processing/test_process_deposit.py similarity index 99% rename from py_tests/phase0/block_processing/test_process_deposit.py rename to test_libs/pyspec/tests/block_processing/test_process_deposit.py index cd682a4d4..4031e650d 100644 --- a/py_tests/phase0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/tests/block_processing/test_process_deposit.py @@ -8,7 +8,7 @@ from eth2spec.phase0.spec import ( ZERO_HASH, process_deposit, ) -from phase0.helpers import ( +from tests.helpers import ( build_deposit, privkeys, pubkeys, diff --git a/py_tests/phase0/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py similarity index 99% rename from py_tests/phase0/block_processing/test_process_proposer_slashing.py rename to test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py index d7afd2750..6f9dee262 100644 --- a/py_tests/phase0/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py @@ -7,7 +7,7 @@ from eth2spec.phase0.spec import ( get_current_epoch, process_proposer_slashing, ) -from phase0.helpers import ( +from tests.helpers import ( get_valid_proposer_slashing, ) diff --git a/py_tests/phase0/block_processing/test_voluntary_exit.py b/test_libs/pyspec/tests/block_processing/test_voluntary_exit.py similarity index 99% rename from py_tests/phase0/block_processing/test_voluntary_exit.py rename to test_libs/pyspec/tests/block_processing/test_voluntary_exit.py index 4404e7255..c58c5238a 100644 --- a/py_tests/phase0/block_processing/test_voluntary_exit.py +++ b/test_libs/pyspec/tests/block_processing/test_voluntary_exit.py @@ -9,7 +9,7 @@ from eth2spec.phase0.spec import ( get_current_epoch, process_voluntary_exit, ) -from phase0.helpers import ( +from tests.helpers import ( build_voluntary_exit, pubkey_to_privkey, ) diff --git a/py_tests/phase0/conftest.py b/test_libs/pyspec/tests/conftest.py similarity index 97% rename from py_tests/phase0/conftest.py rename to test_libs/pyspec/tests/conftest.py index ec499862c..21f5e18e6 100644 --- a/py_tests/phase0/conftest.py +++ b/test_libs/pyspec/tests/conftest.py @@ -14,7 +14,6 @@ MINIMAL_CONFIG = { "MIN_ATTESTATION_INCLUSION_DELAY": 2, "TARGET_COMMITTEE_SIZE": 4, "SLOTS_PER_EPOCH": 8, - "GENESIS_EPOCH": spec.GENESIS_SLOT // 8, "SLOTS_PER_HISTORICAL_ROOT": 64, "LATEST_RANDAO_MIXES_LENGTH": 64, "LATEST_ACTIVE_INDEX_ROOTS_LENGTH": 64, diff --git a/py_tests/phase0/helpers.py b/test_libs/pyspec/tests/helpers.py similarity index 83% rename from py_tests/phase0/helpers.py rename to test_libs/pyspec/tests/helpers.py index 044386696..c44934950 100644 --- a/py_tests/phase0/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -33,6 +33,8 @@ from eth2spec.phase0.spec import ( get_empty_block, get_epoch_start_slot, get_genesis_beacon_state, + get_previous_epoch, + get_shard_delta, slot_to_epoch, verify_merkle_branch, hash, @@ -49,6 +51,19 @@ pubkeys = [bls.privtopub(privkey) for privkey in privkeys] pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} +def set_bitfield_bit(bitfield, i): + """ + Set the bit in ``bitfield`` at position ``i`` to ``1``. + """ + byte_index = i // 8 + bit_index = i % 8 + return ( + bitfield[:byte_index] + + bytes([bitfield[byte_index] | (1 << bit_index)]) + + bitfield[byte_index+1:] + ) + + def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves=None): if not deposit_data_leaves: deposit_data_leaves = [] @@ -132,24 +147,31 @@ def build_deposit_data(state, pubkey, privkey, amount): def build_attestation_data(state, slot, shard): assert state.slot >= slot - block_root = build_empty_block_for_next_slot(state).previous_block_root + if slot == state.slot: + block_root = build_empty_block_for_next_slot(state).previous_block_root + else: + block_root = get_block_root(state, slot) - epoch_start_slot = get_epoch_start_slot(get_current_epoch(state)) - if epoch_start_slot == slot: + current_epoch_start_slot = get_epoch_start_slot(get_current_epoch(state)) + if slot < current_epoch_start_slot: + epoch_boundary_root = get_block_root(state, get_epoch_start_slot(get_previous_epoch(state))) + elif slot == current_epoch_start_slot: epoch_boundary_root = block_root else: - get_block_root(state, epoch_start_slot) + epoch_boundary_root = get_block_root(state, current_epoch_start_slot) - if slot < epoch_start_slot: + if slot < current_epoch_start_slot: + justified_epoch = state.previous_justified_epoch justified_block_root = state.previous_justified_root else: + justified_epoch = state.current_justified_epoch justified_block_root = state.current_justified_root return AttestationData( slot=slot, shard=shard, beacon_block_root=block_root, - source_epoch=state.current_justified_epoch, + source_epoch=justified_epoch, source_root=justified_block_root, target_root=epoch_boundary_root, crosslink_data_root=spec.ZERO_HASH, @@ -254,7 +276,13 @@ def get_valid_attester_slashing(state): def get_valid_attestation(state, slot=None): if slot is None: slot = state.slot - shard = state.latest_start_shard + + if slot_to_epoch(slot) == get_current_epoch(state): + shard = (state.latest_start_shard + slot) % spec.SLOTS_PER_EPOCH + else: + previous_shard_delta = get_shard_delta(state, get_previous_epoch(state)) + shard = (state.latest_start_shard - previous_shard_delta + slot) % spec.SHARD_COUNT + attestation_data = build_attestation_data(state, slot, shard) crosslink_committee = get_crosslink_committee_for_attestation(state, attestation_data) @@ -308,6 +336,18 @@ def get_attestation_signature(state, attestation_data, privkey, custody_bit=0b0) ) +def fill_aggregate_attestation(state, attestation): + crosslink_committee = get_crosslink_committee_for_attestation(state, attestation.data) + for i in range(len(crosslink_committee)): + attestation.aggregation_bitfield = set_bitfield_bit(attestation.aggregation_bitfield, i) + + def next_slot(state): block = build_empty_block_for_next_slot(state) state_transition(state, block) + + +def next_epoch(state): + block = build_empty_block_for_next_slot(state) + block.slot += spec.SLOTS_PER_EPOCH - (state.slot % spec.SLOTS_PER_EPOCH) + state_transition(state, block) diff --git a/py_tests/phase0/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py similarity index 86% rename from py_tests/phase0/test_sanity.py rename to test_libs/pyspec/tests/test_sanity.py index 04af507d7..236b409ac 100644 --- a/py_tests/phase0/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -39,9 +39,11 @@ from eth2spec.utils.merkle_minimal import ( from .helpers import ( build_deposit_data, build_empty_block_for_next_slot, + fill_aggregate_attestation, get_valid_attestation, get_valid_attester_slashing, get_valid_proposer_slashing, + next_slot, privkeys, pubkeys, ) @@ -51,6 +53,33 @@ from .helpers import ( pytestmark = pytest.mark.sanity +def check_finality(state, + prev_state, + current_justified_changed, + previous_justified_changed, + finalized_changed): + if current_justified_changed: + assert state.current_justified_epoch > prev_state.current_justified_epoch + assert state.current_justified_root != prev_state.current_justified_root + else: + assert state.current_justified_epoch == prev_state.current_justified_epoch + assert state.current_justified_root == prev_state.current_justified_root + + if previous_justified_changed: + assert state.previous_justified_epoch > prev_state.previous_justified_epoch + assert state.previous_justified_root != prev_state.previous_justified_root + else: + assert state.previous_justified_epoch == prev_state.previous_justified_epoch + assert state.previous_justified_root == prev_state.previous_justified_root + + if finalized_changed: + assert state.finalized_epoch > prev_state.finalized_epoch + assert state.finalized_root != prev_state.finalized_root + else: + assert state.finalized_epoch == prev_state.finalized_epoch + assert state.finalized_root == prev_state.finalized_root + + def test_slot_transition(state): test_state = deepcopy(state) cache_state(test_state) @@ -115,6 +144,33 @@ def test_empty_epoch_transition_not_finalizing(state): return state, [block], test_state +def test_full_attestations_finalizing(state): + test_state = deepcopy(state) + + for slot in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): + next_slot(test_state) + + for epoch in range(5): + for slot in range(spec.SLOTS_PER_EPOCH): + print(test_state.slot) + attestation = get_valid_attestation(test_state, test_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY) + fill_aggregate_attestation(test_state, attestation) + block = build_empty_block_for_next_slot(test_state) + block.body.attestations.append(attestation) + state_transition(test_state, block) + + if epoch == 0: + check_finality(test_state, state, False, False, False) + elif epoch == 1: + check_finality(test_state, state, False, False, False) + elif epoch == 2: + check_finality(test_state, state, True, False, False) + elif epoch == 3: + check_finality(test_state, state, True, True, False) + elif epoch == 4: + check_finality(test_state, state, True, True, True) + + def test_proposer_slashing(state): test_state = deepcopy(state) proposer_slashing = get_valid_proposer_slashing(state)