From ce987c4a64d392af1b6c33d4ec010cda4e10a42c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 27 Jan 2023 09:12:39 +0100 Subject: [PATCH 001/110] Delete `is_merge_transition_block` check from Capella --- specs/capella/fork-choice.md | 55 ++++++++++++++++++++++++++++++++++++ specs/eip4844/fork-choice.md | 4 --- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/specs/capella/fork-choice.md b/specs/capella/fork-choice.md index 0e0a393c3..c08be1c3f 100644 --- a/specs/capella/fork-choice.md +++ b/specs/capella/fork-choice.md @@ -14,6 +14,8 @@ - [`notify_forkchoice_updated`](#notify_forkchoice_updated) - [Helpers](#helpers) - [Extended `PayloadAttributes`](#extended-payloadattributes) +- [Updated fork-choice handlers](#updated-fork-choice-handlers) + - [`on_block`](#on_block) @@ -60,3 +62,56 @@ class PayloadAttributes(object): suggested_fee_recipient: ExecutionAddress withdrawals: Sequence[Withdrawal] # [New in Capella] ``` + +## Updated fork-choice handlers + +### `on_block` + +*Note*: The only modification is the deletion of the verification of merge transition block conditions. + +```python +def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: + """ + Run ``on_block`` upon receiving a new block. + """ + block = signed_block.message + # Parent block must be known + assert block.parent_root in store.block_states + # Make a copy of the state to avoid mutability issues + pre_state = copy(store.block_states[block.parent_root]) + # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past. + assert get_current_slot(store) >= block.slot + + # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert block.slot > finalized_slot + # Check block is a descendant of the finalized block at the checkpoint finalized slot + assert get_ancestor(store, block.parent_root, finalized_slot) == store.finalized_checkpoint.root + + # Check the block is valid and compute the post-state + state = pre_state.copy() + state_transition(state, signed_block, True) + + # Add new block to the store + store.blocks[hash_tree_root(block)] = block + # Add new state for this block to the store + store.block_states[hash_tree_root(block)] = state + + # Add proposer score boost if the block is timely + time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT + is_before_attesting_interval = time_into_slot < SECONDS_PER_SLOT // INTERVALS_PER_SLOT + if get_current_slot(store) == block.slot and is_before_attesting_interval: + store.proposer_boost_root = hash_tree_root(block) + + # Update justified checkpoint + if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + if state.current_justified_checkpoint.epoch > store.best_justified_checkpoint.epoch: + store.best_justified_checkpoint = state.current_justified_checkpoint + if should_update_justified_checkpoint(store, state.current_justified_checkpoint): + store.justified_checkpoint = state.current_justified_checkpoint + + # Update finalized checkpoint + if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: + store.finalized_checkpoint = state.finalized_checkpoint + store.justified_checkpoint = state.current_justified_checkpoint +``` diff --git a/specs/eip4844/fork-choice.md b/specs/eip4844/fork-choice.md index 8dea28ded..865443e04 100644 --- a/specs/eip4844/fork-choice.md +++ b/specs/eip4844/fork-choice.md @@ -108,10 +108,6 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: state = pre_state.copy() state_transition(state, signed_block, True) - # Check the merge transition - if is_merge_transition_block(pre_state, block.body): - validate_merge_block(block) - # Add new block to the store store.blocks[hash_tree_root(block)] = block # Add new state for this block to the store From 21cf61ea8f339c36c276003844f002e70f491741 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 15 Dec 2022 11:28:20 +0700 Subject: [PATCH 002/110] De-duplicate range references in BlobSidecarsByRange --- specs/deneb/p2p-interface.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 107c90b9a..51180f8a2 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -256,18 +256,18 @@ The request MUST be encoded as an SSZ-container. The response MUST consist of zero or more `response_chunk`. Each _successful_ `response_chunk` MUST contain a single `BlobSidecar` payload. -Clients MUST keep a record of signed blobs sidecars seen on the epoch range -`[max(current_epoch - MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS, DENEB_FORK_EPOCH), current_epoch]` +Let `blob_serve_range` be `[max(current_epoch - MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS, DENEB_FORK_EPOCH), current_epoch]`. +Clients MUST keep a record of signed blob sidecars seen on the epoch range `blob_serve_range` where `current_epoch` is defined by the current wall-clock time, and clients MUST support serving requests of blobs on this range. -Peers that are unable to reply to blob sidecar requests within the `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` -epoch range SHOULD respond with error code `3: ResourceUnavailable`. +Peers that are unable to reply to blob sidecar requests within the +range `blob_serve_range` SHOULD respond with error code `3: ResourceUnavailable`. Such peers that are unable to successfully reply to this range of requests MAY get descored or disconnected at any time. *Note*: The above requirement implies that nodes that start from a recent weak subjectivity checkpoint -MUST backfill the local blobs database to at least epoch `current_epoch - MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` +MUST backfill the local blobs database to at least the range `blob_serve_range` to be fully compliant with `BlobSidecarsByRange` requests. *Note*: Although clients that bootstrap from a weak subjectivity checkpoint can begin From 1c35eb1c3303fe1e0b101323106d766d8f848cd6 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Fri, 10 Mar 2023 14:37:18 +0800 Subject: [PATCH 003/110] Lock voluntary_exit domain on capella --- specs/phase0/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 7e14fa951..e5eb12006 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1903,7 +1903,7 @@ def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVolu # Verify the validator has been active long enough assert get_current_epoch(state) >= validator.activation_epoch + SHARD_COMMITTEE_PERIOD # Verify signature - domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch) + domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, min(voluntary_exit.epoch, CAPELLA_FORK_EPOCH)) signing_root = compute_signing_root(voluntary_exit, domain) assert bls.Verify(validator.pubkey, signing_root, signed_voluntary_exit.signature) # Initiate exit From 47f078fc1fdbcc6274d42ee891718ed73bc731d9 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Sat, 11 Mar 2023 20:28:49 +0800 Subject: [PATCH 004/110] Move change to deneb --- specs/deneb/beacon-chain.md | 44 +++++++++++++++++++++++++++++++++++- specs/phase0/beacon-chain.md | 2 +- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index a0ac783b7..672d0a633 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -203,7 +203,7 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in Deneb] process_randao(state, block.body) process_eth1_data(state, block.body) - process_operations(state, block.body) + process_operations(state, block.body) # [Modified in Deneb] process_sync_aggregate(state, block.body.sync_aggregate) process_blob_kzg_commitments(state, block.body) # [New in Deneb] ``` @@ -245,6 +245,48 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe ) ``` +#### Modified `process_operations` + +```python +def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: + # Verify that outstanding deposits are processed up to the maximum number of deposits + assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) + + def for_ops(operations: Sequence[Any], fn: Callable[[BeaconState, Any], None]) -> None: + for operation in operations: + fn(state, operation) + + for_ops(body.proposer_slashings, process_proposer_slashing) + for_ops(body.attester_slashings, process_attester_slashing) + for_ops(body.attestations, process_attestation) + for_ops(body.deposits, process_deposit) + for_ops(body.voluntary_exits, process_voluntary_exit) # [Modified in Deneb] + for_ops(body.bls_to_execution_changes, process_bls_to_execution_change) +``` + +##### Modified `process_voluntary_exit` + +```python +def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVoluntaryExit) -> None: + voluntary_exit = signed_voluntary_exit.message + validator = state.validators[voluntary_exit.validator_index] + # Verify the validator is active + assert is_active_validator(validator, get_current_epoch(state)) + # Verify exit has not been initiated + assert validator.exit_epoch == FAR_FUTURE_EPOCH + # Exits must specify an epoch when they become valid; they are not valid before then + assert get_current_epoch(state) >= voluntary_exit.epoch + # Verify the validator has been active long enough + assert get_current_epoch(state) >= validator.activation_epoch + SHARD_COMMITTEE_PERIOD + # Verify signature + domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, min(voluntary_exit.epoch, CAPELLA_FORK_EPOCH)) # [Modified in Deneb] + signing_root = compute_signing_root(voluntary_exit, domain) + assert bls.Verify(validator.pubkey, signing_root, signed_voluntary_exit.signature) + # Initiate exit + initiate_validator_exit(state, voluntary_exit.validator_index) +``` + + #### Blob KZG commitments ```python diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index e5eb12006..7e14fa951 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1903,7 +1903,7 @@ def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVolu # Verify the validator has been active long enough assert get_current_epoch(state) >= validator.activation_epoch + SHARD_COMMITTEE_PERIOD # Verify signature - domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, min(voluntary_exit.epoch, CAPELLA_FORK_EPOCH)) + domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch) signing_root = compute_signing_root(voluntary_exit, domain) assert bls.Verify(validator.pubkey, signing_root, signed_voluntary_exit.signature) # Initiate exit From df4ba47e633dff7707d7a531136cfa785a65f5b0 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 13 Mar 2023 08:51:02 +0800 Subject: [PATCH 005/110] Update beacon-chain.md --- specs/deneb/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 672d0a633..fd983f29e 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -279,7 +279,7 @@ def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVolu # Verify the validator has been active long enough assert get_current_epoch(state) >= validator.activation_epoch + SHARD_COMMITTEE_PERIOD # Verify signature - domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, min(voluntary_exit.epoch, CAPELLA_FORK_EPOCH)) # [Modified in Deneb] + domain = compute_domain(DOMAIN_VOLUNTARY_EXIT, CAPELLA_FORK_VERSION, state.genesis_validators_root) # [Modified in Deneb] signing_root = compute_signing_root(voluntary_exit, domain) assert bls.Verify(validator.pubkey, signing_root, signed_voluntary_exit.signature) # Initiate exit From 7cb163090246e152927bbc5857b4fd8c9ad53bf9 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Thu, 30 Mar 2023 14:09:16 +1100 Subject: [PATCH 006/110] Attnet revamp draft --- specs/phase0/validator.md | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 54b344791..2ed047f0f 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -88,10 +88,11 @@ All terminology, constants, functions, and protocol mechanics defined in the [Ph | Name | Value | Unit | Duration | | - | - | :-: | :-: | -| `TARGET_AGGREGATORS_PER_COMMITTEE` | `2**4` (= 16) | validators | | -| `RANDOM_SUBNETS_PER_VALIDATOR` | `2**0` (= 1) | subnets | | -| `EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | epochs | ~27 hours | +| `TARGET_AGGREGATORS_PER_COMMITTEE` | `2**4` (= 16) | validators | +| `EPOCHS_PER_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | epochs | ~27 hours | | `ATTESTATION_SUBNET_COUNT` | `64` | The number of attestation subnets used in the gossipsub protocol. | +| `ATTESTATION_SUBNET_EXTRA_BITS` | 0 | The number of extra bits of a NodeId to use when mapping to a subscribed subnet | +| `SUBNETS_PER_NODE` | 2 | The number of long-lived subnets a beacon node should be subscribed to. | ## Containers @@ -606,15 +607,29 @@ def get_aggregate_and_proof_signature(state: BeaconState, ## Phase 0 attestation subnet stability -Because Phase 0 does not have shards and thus does not have Shard Committees, there is no stable backbone to the attestation subnets (`beacon_attestation_{subnet_id}`). To provide this stability, each validator must: +Because Phase 0 does not have shards and thus does not have Shard Committees, there is no stable backbone to the attestation subnets (`beacon_attestation_{subnet_id}`). To provide this stability, each beacon node should: -* Randomly select and remain subscribed to `RANDOM_SUBNETS_PER_VALIDATOR` attestation subnets -* Maintain advertisement of the randomly selected subnets in their node's ENR `attnets` entry by setting the randomly selected `subnet_id` bits to `True` (e.g. `ENR["attnets"][subnet_id] = True`) for all persistent attestation subnets -* Set the lifetime of each random subscription to a random number of epochs between `EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION` and `2 * EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION]`. At the end of life for a subscription, select a new random subnet, update subnet subscriptions, and publish an updated ENR +* Remain subscribed to `SUBNETS_PER_NODE` for `SUBNET_DURATION_IN_EPOCHS` epochs. +* Maintain advertisement of the selected subnets in their node's ENR `attnets` entry by setting the selected `subnet_id` bits to `True` (e.g. `ENR["attnets"][subnet_id] = True`) for all persistent attestation subnets. +* Select these subnets based on their node-id as specified by the following + `compute_subnets(node_id,epoch)` function. -*Note*: Short lived beacon committee assignments should not be added in into the ENR `attnets` entry. +```python +ATTESTATION_SUBNET_PREFIX_BITS = ceil(log2(ATTESTATION_SUBNET_COUNT)) + ATTESTATION_SUBNET_EXTRA_BITS -*Note*: When preparing for a hard fork, a validator must select and subscribe to random subnets of the future fork versioning at least `EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION` epochs in advance of the fork. These new subnets for the fork are maintained in addition to those for the current fork until the fork occurs. After the fork occurs, let the subnets from the previous fork reach the end of life with no replacements. +def compute_subnet(node_id, epoch, index): + node_id_prefix = node_id >> (256 - ATTESTATION_SUBNET_PREFIX_BITS) + permutation_seed = hash(uint_to_bytes(epoch // SUBNET_DURATION_IN_EPOCHS)) + permutated_prefix = compute_shuffled_index(node_id_prefix, 1 << ATTESTATION_SUBNET_PREFIX_BITS, permutation_seed) + return (permutated_prefix + index) % ATTESTATION_SUBNET_COUNT + +def compute_subnets(node_id, epoch): + return [compute_subnet(node_id, epoch, idx) for idx in range(SUBNETS_PER_NODE)] +``` + +*Note*: Nodes should subscribe to new subnets and remain subscribed to old subnets for at least one epoch. Nodes should pick a random duration to unsubscribe from old subnets to smooth the transition on the exact epoch boundary of which the shuffling changes. + +*Note*: When preparing for a hard fork, a validator must select and subscribe to subnets of the future fork versioning at least `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs in advance of the fork. These new subnets for the fork are maintained in addition to those for the current fork until the fork occurs. After the fork occurs, let the subnets from the previous fork reach the end of life with no replacements. ## How to avoid slashing From 0dd8db76cd7d21a2853f0aad5995d027daf8c0e3 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 30 Mar 2023 14:51:41 +0800 Subject: [PATCH 007/110] Make linter happy. Add `SUBNET_DURATION_IN_EPOCHS` definition. --- specs/phase0/validator.md | 27 ++++++++++--------- .../unittests/test_config_invariants.py | 2 +- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 2ed047f0f..4df4437d0 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -91,8 +91,10 @@ All terminology, constants, functions, and protocol mechanics defined in the [Ph | `TARGET_AGGREGATORS_PER_COMMITTEE` | `2**4` (= 16) | validators | | `EPOCHS_PER_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | epochs | ~27 hours | | `ATTESTATION_SUBNET_COUNT` | `64` | The number of attestation subnets used in the gossipsub protocol. | -| `ATTESTATION_SUBNET_EXTRA_BITS` | 0 | The number of extra bits of a NodeId to use when mapping to a subscribed subnet | -| `SUBNETS_PER_NODE` | 2 | The number of long-lived subnets a beacon node should be subscribed to. | +| `ATTESTATION_SUBNET_EXTRA_BITS` | `0` | The number of extra bits of a NodeId to use when mapping to a subscribed subnet | +| `SUBNETS_PER_NODE` | `2` | The number of long-lived subnets a beacon node should be subscribed to. | +| `ATTESTATION_SUBNET_PREFIX_BITS` | `(ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS)` | | +| `SUBNET_DURATION_IN_EPOCHS` | `2` | | ## Containers @@ -611,20 +613,19 @@ Because Phase 0 does not have shards and thus does not have Shard Committees, th * Remain subscribed to `SUBNETS_PER_NODE` for `SUBNET_DURATION_IN_EPOCHS` epochs. * Maintain advertisement of the selected subnets in their node's ENR `attnets` entry by setting the selected `subnet_id` bits to `True` (e.g. `ENR["attnets"][subnet_id] = True`) for all persistent attestation subnets. -* Select these subnets based on their node-id as specified by the following - `compute_subnets(node_id,epoch)` function. +* Select these subnets based on their node-id as specified by the following `compute_subnets(node_id,epoch)` function. ```python -ATTESTATION_SUBNET_PREFIX_BITS = ceil(log2(ATTESTATION_SUBNET_COUNT)) + ATTESTATION_SUBNET_EXTRA_BITS +def compute_subnet(node_id: int, epoch: Epoch, index: int) -> int: + node_id_prefix = node_id >> (256 - ATTESTATION_SUBNET_PREFIX_BITS) + permutation_seed = hash(uint_to_bytes(epoch // SUBNET_DURATION_IN_EPOCHS)) + permutated_prefix = compute_shuffled_index(node_id_prefix, 1 << ATTESTATION_SUBNET_PREFIX_BITS, permutation_seed) + return (permutated_prefix + index) % ATTESTATION_SUBNET_COUNT +``` -def compute_subnet(node_id, epoch, index): - node_id_prefix = node_id >> (256 - ATTESTATION_SUBNET_PREFIX_BITS) - permutation_seed = hash(uint_to_bytes(epoch // SUBNET_DURATION_IN_EPOCHS)) - permutated_prefix = compute_shuffled_index(node_id_prefix, 1 << ATTESTATION_SUBNET_PREFIX_BITS, permutation_seed) - return (permutated_prefix + index) % ATTESTATION_SUBNET_COUNT - -def compute_subnets(node_id, epoch): - return [compute_subnet(node_id, epoch, idx) for idx in range(SUBNETS_PER_NODE)] +```python +def compute_subnets(node_id: int, epoch: Epoch) -> Sequence[int]: + return [compute_subnet(node_id, epoch, idx) for idx in range(SUBNETS_PER_NODE)] ``` *Note*: Nodes should subscribe to new subnets and remain subscribed to old subnets for at least one epoch. Nodes should pick a random duration to unsubscribe from old subnets to smooth the transition on the exact epoch boundary of which the shuffling changes. diff --git a/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py b/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py index 9b27d1deb..69aa3eb2a 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py +++ b/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py @@ -75,7 +75,7 @@ def test_time(spec, state): @with_all_phases @spec_state_test def test_networking(spec, state): - assert spec.RANDOM_SUBNETS_PER_VALIDATOR <= spec.ATTESTATION_SUBNET_COUNT + assert spec.SUBNETS_PER_NODE <= spec.ATTESTATION_SUBNET_COUNT @with_all_phases From a0d03378fabf76cb91897ffe17310050f3996ee2 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Thu, 6 Apr 2023 12:40:55 +1000 Subject: [PATCH 008/110] Correct subnet subscription duration variable --- specs/phase0/validator.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 4df4437d0..1b06aecfb 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -94,7 +94,6 @@ All terminology, constants, functions, and protocol mechanics defined in the [Ph | `ATTESTATION_SUBNET_EXTRA_BITS` | `0` | The number of extra bits of a NodeId to use when mapping to a subscribed subnet | | `SUBNETS_PER_NODE` | `2` | The number of long-lived subnets a beacon node should be subscribed to. | | `ATTESTATION_SUBNET_PREFIX_BITS` | `(ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS)` | | -| `SUBNET_DURATION_IN_EPOCHS` | `2` | | ## Containers @@ -611,14 +610,14 @@ def get_aggregate_and_proof_signature(state: BeaconState, Because Phase 0 does not have shards and thus does not have Shard Committees, there is no stable backbone to the attestation subnets (`beacon_attestation_{subnet_id}`). To provide this stability, each beacon node should: -* Remain subscribed to `SUBNETS_PER_NODE` for `SUBNET_DURATION_IN_EPOCHS` epochs. +* Remain subscribed to `SUBNETS_PER_NODE` for `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs. * Maintain advertisement of the selected subnets in their node's ENR `attnets` entry by setting the selected `subnet_id` bits to `True` (e.g. `ENR["attnets"][subnet_id] = True`) for all persistent attestation subnets. * Select these subnets based on their node-id as specified by the following `compute_subnets(node_id,epoch)` function. ```python def compute_subnet(node_id: int, epoch: Epoch, index: int) -> int: node_id_prefix = node_id >> (256 - ATTESTATION_SUBNET_PREFIX_BITS) - permutation_seed = hash(uint_to_bytes(epoch // SUBNET_DURATION_IN_EPOCHS)) + permutation_seed = hash(uint_to_bytes(epoch // EPOCHS_PER_SUBNET_SUBSCRIPTION)) permutated_prefix = compute_shuffled_index(node_id_prefix, 1 << ATTESTATION_SUBNET_PREFIX_BITS, permutation_seed) return (permutated_prefix + index) % ATTESTATION_SUBNET_COUNT ``` From 6e423f6c4275608515158b5c483c351f0eb61b19 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 12 Apr 2023 11:29:48 +1000 Subject: [PATCH 009/110] Stagger node rotations --- specs/phase0/validator.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 1b06aecfb..56ca50732 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -612,23 +612,22 @@ Because Phase 0 does not have shards and thus does not have Shard Committees, th * Remain subscribed to `SUBNETS_PER_NODE` for `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs. * Maintain advertisement of the selected subnets in their node's ENR `attnets` entry by setting the selected `subnet_id` bits to `True` (e.g. `ENR["attnets"][subnet_id] = True`) for all persistent attestation subnets. -* Select these subnets based on their node-id as specified by the following `compute_subnets(node_id,epoch)` function. +* Select these subnets based on their node-id as specified by the following `compute_subscribed_subnets(node_id,epoch)` function. ```python -def compute_subnet(node_id: int, epoch: Epoch, index: int) -> int: +def compute_subscribed_subnet(node_id: int, epoch: Epoch, index: int) -> int: node_id_prefix = node_id >> (256 - ATTESTATION_SUBNET_PREFIX_BITS) - permutation_seed = hash(uint_to_bytes(epoch // EPOCHS_PER_SUBNET_SUBSCRIPTION)) + node_offset = node_id % EPOCHS_PER_SUBNET_SUBSCRIPTION + permutation_seed = hash(uint_to_bytes((epoch + node_offset) // EPOCHS_PER_SUBNET_SUBSCRIPTION)) permutated_prefix = compute_shuffled_index(node_id_prefix, 1 << ATTESTATION_SUBNET_PREFIX_BITS, permutation_seed) return (permutated_prefix + index) % ATTESTATION_SUBNET_COUNT ``` ```python -def compute_subnets(node_id: int, epoch: Epoch) -> Sequence[int]: - return [compute_subnet(node_id, epoch, idx) for idx in range(SUBNETS_PER_NODE)] +def compute_subscribed_subnets(node_id: int, epoch: Epoch) -> Sequence[int]: + return [compute_subscribed_subnet(node_id, epoch, idx) for idx in range(SUBNETS_PER_NODE)] ``` -*Note*: Nodes should subscribe to new subnets and remain subscribed to old subnets for at least one epoch. Nodes should pick a random duration to unsubscribe from old subnets to smooth the transition on the exact epoch boundary of which the shuffling changes. - *Note*: When preparing for a hard fork, a validator must select and subscribe to subnets of the future fork versioning at least `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs in advance of the fork. These new subnets for the fork are maintained in addition to those for the current fork until the fork occurs. After the fork occurs, let the subnets from the previous fork reach the end of life with no replacements. ## How to avoid slashing From 745d529598632029fee9820590b033b7e0e23935 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 25 Apr 2023 12:57:42 +0800 Subject: [PATCH 010/110] Add `compute_subscribed_subnets` unittests and fix typing errors --- specs/phase0/validator.md | 10 +++-- .../validator/test_validator_unittest.py | 39 ++++++++++++++++++- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 56ca50732..5266fec7a 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -616,10 +616,14 @@ Because Phase 0 does not have shards and thus does not have Shard Committees, th ```python def compute_subscribed_subnet(node_id: int, epoch: Epoch, index: int) -> int: - node_id_prefix = node_id >> (256 - ATTESTATION_SUBNET_PREFIX_BITS) + node_id_prefix = node_id >> (256 - int(ATTESTATION_SUBNET_PREFIX_BITS)) node_offset = node_id % EPOCHS_PER_SUBNET_SUBSCRIPTION - permutation_seed = hash(uint_to_bytes((epoch + node_offset) // EPOCHS_PER_SUBNET_SUBSCRIPTION)) - permutated_prefix = compute_shuffled_index(node_id_prefix, 1 << ATTESTATION_SUBNET_PREFIX_BITS, permutation_seed) + permutation_seed = hash(uint_to_bytes(uint64((epoch + node_offset) // EPOCHS_PER_SUBNET_SUBSCRIPTION))) + permutated_prefix = compute_shuffled_index( + node_id_prefix, + 1 << int(ATTESTATION_SUBNET_PREFIX_BITS), + permutation_seed, + ) return (permutated_prefix + index) % ATTESTATION_SUBNET_COUNT ``` diff --git a/tests/core/pyspec/eth2spec/test/phase0/unittests/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/phase0/unittests/validator/test_validator_unittest.py index cf7ef392f..177748eac 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/unittests/validator/test_validator_unittest.py +++ b/tests/core/pyspec/eth2spec/test/phase0/unittests/validator/test_validator_unittest.py @@ -1,6 +1,12 @@ +import random + from eth2spec.test.context import ( + single_phase, spec_state_test, - always_bls, with_phases, with_all_phases, + spec_test, + always_bls, + with_phases, + with_all_phases, ) from eth2spec.test.helpers.constants import PHASE0 from eth2spec.test.helpers.attestations import build_attestation_data, get_valid_attestation @@ -476,3 +482,34 @@ def test_get_aggregate_and_proof_signature(spec, state): privkey=privkey, pubkey=pubkey, ) + + +def run_compute_subscribed_subnets_arguments(spec, rng=random.Random(1111)): + node_id = rng.randint(0, 2**40 - 1) # try VALIDATOR_REGISTRY_LIMIT + epoch = rng.randint(0, 2**64 - 1) + subnets = spec.compute_subscribed_subnets(node_id, epoch) + assert len(subnets) == spec.SUBNETS_PER_NODE + + +@with_all_phases +@spec_test +@single_phase +def test_compute_subscribed_subnets_random_1(spec): + rng = random.Random(1111) + run_compute_subscribed_subnets_arguments(spec, rng) + + +@with_all_phases +@spec_test +@single_phase +def test_compute_subscribed_subnets_random_2(spec): + rng = random.Random(2222) + run_compute_subscribed_subnets_arguments(spec, rng) + + +@with_all_phases +@spec_test +@single_phase +def test_compute_subscribed_subnets_random_3(spec): + rng = random.Random(3333) + run_compute_subscribed_subnets_arguments(spec, rng) From 655094ee4a7026185f1962fc245dfa763735828a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 28 Apr 2023 17:28:11 +0800 Subject: [PATCH 011/110] Add `test_zero_blob` test case --- .../eth2spec/test/deneb/sanity/test_blocks.py | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index 5e65dbd4e..9a6c6a45b 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -16,13 +16,11 @@ from eth2spec.test.helpers.sharding import ( ) -@with_deneb_and_later -@spec_state_test -def test_one_blob(spec, state): +def run_block_with_blobs(spec, state, blob_count): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=blob_count) block.body.blob_kzg_commitments = blob_kzg_commitments block.body.execution_payload.transactions = [opaque_tx] block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) @@ -32,17 +30,19 @@ def test_one_blob(spec, state): yield 'post', state +@with_deneb_and_later +@spec_state_test +def test_zero_blob(spec, state): + yield from run_block_with_blobs(spec, state, blob_count=0) + + +@with_deneb_and_later +@spec_state_test +def test_one_blob(spec, state): + yield from run_block_with_blobs(spec, state, blob_count=1) + + @with_deneb_and_later @spec_state_test def test_max_blobs(spec, state): - yield 'pre', state - - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=spec.MAX_BLOBS_PER_BLOCK) - block.body.blob_kzg_commitments = blob_kzg_commitments - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block) - - yield 'blocks', [signed_block] - yield 'post', state + yield from run_block_with_blobs(spec, state, blob_count=spec.MAX_BLOBS_PER_BLOCK) From 327ef095d0fe8f309dc42126a9b3abf64f31175d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 28 Apr 2023 17:37:50 +0800 Subject: [PATCH 012/110] Add `test_incorrect_blob_tx_type` --- .../eth2spec/test/deneb/sanity/test_blocks.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index 9a6c6a45b..50b975d3b 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -46,3 +46,20 @@ def test_one_blob(spec, state): @spec_state_test def test_max_blobs(spec, state): yield from run_block_with_blobs(spec, state, blob_count=spec.MAX_BLOBS_PER_BLOCK) + + +@with_deneb_and_later +@spec_state_test +def test_incorrect_blob_tx_type(spec, state): + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + block.body.blob_kzg_commitments = blob_kzg_commitments + opaque_tx[0] == spec.uint8(0x04) # incorrect tx type + block.body.execution_payload.transactions = [opaque_tx] + block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state From 22da92e5979b72d6b2c37719c84ee66b9d909cc8 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 28 Apr 2023 18:26:42 +0800 Subject: [PATCH 013/110] Refactor the spec and add more test cases --- specs/deneb/beacon-chain.md | 7 +- .../eth2spec/test/deneb/sanity/test_blocks.py | 90 ++++++++++++++++++- 2 files changed, 92 insertions(+), 5 deletions(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index df1da8e2a..359c7fc95 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -173,6 +173,8 @@ def tx_peek_blob_versioned_hashes(opaque_tx: Transaction) -> Sequence[VersionedH message_offset + uint32.decode_bytes(opaque_tx[(message_offset + 188):(message_offset + 192)]) ) + # `VersionedHash` is a 32-byte object + assert (len(opaque_tx) - blob_versioned_hashes_offset) % 32 == 0 return [ VersionedHash(opaque_tx[x:(x + 32)]) for x in range(blob_versioned_hashes_offset, len(opaque_tx), 32) @@ -205,7 +207,7 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_eth1_data(state, block.body) process_operations(state, block.body) process_sync_aggregate(state, block.body.sync_aggregate) - process_blob_kzg_commitments(state, block.body) # [New in Deneb] + process_blob_kzg_commitments(block.body) # [New in Deneb] ``` #### Execution payload @@ -248,8 +250,7 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe #### Blob KZG commitments ```python -def process_blob_kzg_commitments(state: BeaconState, body: BeaconBlockBody) -> None: - # pylint: disable=unused-argument +def process_blob_kzg_commitments(body: BeaconBlockBody) -> None: assert verify_kzg_commitments_against_transactions(body.execution_payload.transactions, body.blob_kzg_commitments) ``` diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index 50b975d3b..0f8f8b3a4 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -1,3 +1,5 @@ +import random + from eth2spec.test.helpers.state import ( state_transition_and_sign_block ) @@ -50,15 +52,99 @@ def test_max_blobs(spec, state): @with_deneb_and_later @spec_state_test -def test_incorrect_blob_tx_type(spec, state): +def test_invalid_incorrect_blob_tx_type(spec, state): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) block.body.blob_kzg_commitments = blob_kzg_commitments - opaque_tx[0] == spec.uint8(0x04) # incorrect tx type + opaque_tx = b'\x04' + opaque_tx[1:] # incorrect tx type block.body.execution_payload.transactions = [opaque_tx] block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) + signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + + yield 'blocks', [signed_block] + yield 'post', None + + +@with_deneb_and_later +@spec_state_test +def test_invalid_incorrect_transaction_length_1_byte(spec, state): + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + block.body.blob_kzg_commitments = blob_kzg_commitments + opaque_tx = opaque_tx + b'\x12' # incorrect tx length + block.body.execution_payload.transactions = [opaque_tx] + block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) + signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + + yield 'blocks', [signed_block] + yield 'post', None + + +@with_deneb_and_later +@spec_state_test +def test_invalid_incorrect_transaction_length_32_bytes(spec, state): + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + block.body.blob_kzg_commitments = blob_kzg_commitments + opaque_tx = opaque_tx + b'\x12' * 32 # incorrect tx length + block.body.execution_payload.transactions = [opaque_tx] + block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) + signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + + yield 'blocks', [signed_block] + yield 'post', None + + +@with_deneb_and_later +@spec_state_test +def test_invalid_incorrect_commitment(spec, state): + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + blob_kzg_commitments[0] = b'\x12' * 48 # incorrect commitment + block.body.blob_kzg_commitments = blob_kzg_commitments + block.body.execution_payload.transactions = [opaque_tx] + block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) + signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + + yield 'blocks', [signed_block] + yield 'post', None + + +@with_deneb_and_later +@spec_state_test +def test_invalid_incorrect_commitments_order(spec, state): + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=2, rng=random.Random(1111)) + block.body.blob_kzg_commitments = [blob_kzg_commitments[1], blob_kzg_commitments[0]] # incorrect order + block.body.execution_payload.transactions = [opaque_tx] + block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) + signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + + yield 'blocks', [signed_block] + yield 'post', None + + +@with_deneb_and_later +@spec_state_test +def test_incorrect_block_hash(spec, state): + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + block.body.blob_kzg_commitments = blob_kzg_commitments + block.body.execution_payload.transactions = [opaque_tx] + block.body.execution_payload.block_hash = b'\x12' * 32 # incorrect block hash + # CL itself doesn't verify EL block hash signed_block = state_transition_and_sign_block(spec, state, block) yield 'blocks', [signed_block] From 2a1d998594ba795c22ffd657292e2b0b87fe8850 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 28 Apr 2023 19:01:19 +0800 Subject: [PATCH 014/110] Test non zero `excess_data_gas` --- tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py | 1 + tests/core/pyspec/eth2spec/test/helpers/execution_payload.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index 0f8f8b3a4..2b99d6a72 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -25,6 +25,7 @@ def run_block_with_blobs(spec, state, blob_count): opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=blob_count) block.body.blob_kzg_commitments = blob_kzg_commitments block.body.execution_payload.transactions = [opaque_tx] + block.body.execution_payload.excess_data_gas = 2 block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) signed_block = state_transition_and_sign_block(spec, state, block) diff --git a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py index c0a70aca1..c8ef1cbf0 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py @@ -165,6 +165,8 @@ def build_empty_execution_payload(spec, state, randao_mix=None): ) if is_post_capella(spec): payload.withdrawals = spec.get_expected_withdrawals(state) + if is_post_deneb(spec): + payload.excess_data_gas = 0 payload.block_hash = compute_el_block_hash(spec, payload) From e31fcbd6a9f795100ec6f1de434ffd4555a0f0e2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 28 Apr 2023 23:09:10 +0800 Subject: [PATCH 015/110] Add `GetPayloadResponse` for `get_payload` API --- setup.py | 2 +- specs/_features/eip4788/validator.md | 4 +-- specs/bellatrix/validator.md | 19 ++++++++--- specs/capella/validator.md | 14 ++++++-- specs/deneb/validator.md | 50 ++++++++++++++++++++++------ tests/formats/fork_choice/README.md | 4 +-- 6 files changed, 70 insertions(+), 23 deletions(-) diff --git a/setup.py b/setup.py index fc3acb806..a3e94642e 100644 --- a/setup.py +++ b/setup.py @@ -588,7 +588,7 @@ class NoopExecutionEngine(ExecutionEngine): payload_attributes: Optional[PayloadAttributes]) -> Optional[PayloadId]: pass - def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> ExecutionPayload: + def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> GetPayloadResponse: # pylint: disable=unused-argument raise NotImplementedError("no default block production") diff --git a/specs/_features/eip4788/validator.md b/specs/_features/eip4788/validator.md index 421e297ce..3140cdb21 100644 --- a/specs/_features/eip4788/validator.md +++ b/specs/_features/eip4788/validator.md @@ -13,7 +13,7 @@ - [Helpers](#helpers) - [Protocols](#protocols) - [`ExecutionEngine`](#executionengine) - - [`get_payload`](#get_payload) + - [Modified `get_payload`](#modified-get_payload) - [Beacon chain responsibilities](#beacon-chain-responsibilities) - [Block proposal](#block-proposal) - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody) @@ -40,7 +40,7 @@ Please see related Beacon Chain doc before continuing and use them as a referenc ### `ExecutionEngine` -#### `get_payload` +#### Modified `get_payload` `get_payload` returns the upgraded EIP-4788 `ExecutionPayload` type. diff --git a/specs/bellatrix/validator.md b/specs/bellatrix/validator.md index a176d7534..dea763cde 100644 --- a/specs/bellatrix/validator.md +++ b/specs/bellatrix/validator.md @@ -9,6 +9,7 @@ - [Introduction](#introduction) - [Prerequisites](#prerequisites) - [Helpers](#helpers) + - [`GetPayloadResponse`](#getpayloadresponse) - [`get_pow_block_at_terminal_total_difficulty`](#get_pow_block_at_terminal_total_difficulty) - [`get_terminal_pow_block`](#get_terminal_pow_block) - [Protocols](#protocols) @@ -36,6 +37,14 @@ Please see related Beacon Chain doc before continuing and use them as a referenc ## Helpers +### `GetPayloadResponse` + +```python +@dataclass +class GetPayloadResponse(object): + execution_payload: ExecutionPayload +``` + ### `get_pow_block_at_terminal_total_difficulty` ```python @@ -83,13 +92,13 @@ The Engine API may be used to implement it with an external execution engine. #### `get_payload` -Given the `payload_id`, `get_payload` returns the most recent version of the execution payload that -has been built since the corresponding call to `notify_forkchoice_updated` method. +Given the `payload_id`, `get_payload` returns `GetPayloadResponse` with the most recent version of +the execution payload that has been built since the corresponding call to `notify_forkchoice_updated` method. ```python -def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> ExecutionPayload: +def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> GetPayloadResponse: """ - Return ``execution_payload`` object. + Return ``GetPayloadResponse`` object. """ ... ``` @@ -162,7 +171,7 @@ def get_execution_payload(payload_id: Optional[PayloadId], execution_engine: Exe # Pre-merge, empty payload return ExecutionPayload() else: - return execution_engine.get_payload(payload_id) + return execution_engine.get_payload(payload_id).execution_payload ``` *Note*: It is recommended for a validator to call `prepare_execution_payload` as soon as input parameters become known, diff --git a/specs/capella/validator.md b/specs/capella/validator.md index 644ee476f..29cff8c61 100644 --- a/specs/capella/validator.md +++ b/specs/capella/validator.md @@ -11,9 +11,10 @@ - [Introduction](#introduction) - [Prerequisites](#prerequisites) - [Helpers](#helpers) + - [Modified `GetPayloadResponse`](#modified-getpayloadresponse) - [Protocols](#protocols) - [`ExecutionEngine`](#executionengine) - - [`get_payload`](#get_payload) + - [Modified `get_payload`](#modified-get_payload) - [Beacon chain responsibilities](#beacon-chain-responsibilities) - [Block proposal](#block-proposal) - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody) @@ -39,11 +40,20 @@ Please see related Beacon Chain doc before continuing and use them as a referenc ## Helpers +### Modified `GetPayloadResponse` + +```python +@dataclass +class GetPayloadResponse(object): + execution_payload: ExecutionPayload + block_value: uint256 +``` + ## Protocols ### `ExecutionEngine` -#### `get_payload` +#### Modified `get_payload` `get_payload` returns the upgraded Capella `ExecutionPayload` type. diff --git a/specs/deneb/validator.md b/specs/deneb/validator.md index b627de023..6562c91dd 100644 --- a/specs/deneb/validator.md +++ b/specs/deneb/validator.md @@ -11,7 +11,11 @@ - [Introduction](#introduction) - [Prerequisites](#prerequisites) - [Helpers](#helpers) - - [`get_blobs_and_kzg_commitments`](#get_blobs_and_kzg_commitments) + - [`BlobsBundle`](#blobsbundle) + - [Modified `GetPayloadResponse`](#modified-getpayloadresponse) +- [Protocol](#protocol) + - [`ExecutionEngine`](#executionengine) + - [Modified `get_payload`](#modified-get_payload) - [Beacon chain responsibilities](#beacon-chain-responsibilities) - [Block and sidecar proposal](#block-and-sidecar-proposal) - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody) @@ -36,17 +40,40 @@ Please see related Beacon Chain doc before continuing and use them as a referenc ## Helpers -### `get_blobs_and_kzg_commitments` - -The interface to retrieve blobs and corresponding kzg commitments. - -Note: This API is *unstable*. `get_blobs_and_kzg_commitments` and `get_payload` may be unified. -Implementers may also retrieve blobs individually per transaction. +### `BlobsBundle` ```python -def get_blobs_and_kzg_commitments( - payload_id: PayloadId -) -> Tuple[Sequence[Blob], Sequence[KZGCommitment], Sequence[KZGProof]]: +@dataclass +class BlobsBundle(object): + commitments: Sequence[KZGCommitment] + proofs: Sequence[KZGProof] + blobs: Sequence[Blob] +``` + +### Modified `GetPayloadResponse` + +```python +@dataclass +class GetPayloadResponse(object): + execution_payload: ExecutionPayload + block_value: uint256 + blobs_bundle: BlobsBundle +``` + +## Protocol + +### `ExecutionEngine` + +#### Modified `get_payload` + +Given the `payload_id`, `get_payload` returns the most recent version of the execution payload that +has been built since the corresponding call to `notify_forkchoice_updated` method. + +```python +def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> GetPayloadResponse: + """ + Return ExecutionPayload, uint256, BlobsBundle objects. + """ # pylint: disable=unused-argument ... ``` @@ -62,7 +89,8 @@ All validator responsibilities remain unchanged other than those noted below. ##### Blob KZG commitments 1. After retrieving the execution payload from the execution engine as specified in Capella, -use the `payload_id` to retrieve `blobs` and `blob_kzg_commitments` via `get_blobs_and_kzg_commitments(payload_id)`. +use the `payload_id` to retrieve `blobs`, `blob_kzg_commitments`, and `blob_kzg_proofs` +via `get_payload(payload_id).blobs_bundle`. 2. Validate `blobs` and `blob_kzg_commitments`: ```python diff --git a/tests/formats/fork_choice/README.md b/tests/formats/fork_choice/README.md index c94b95933..3b28837de 100644 --- a/tests/formats/fork_choice/README.md +++ b/tests/formats/fork_choice/README.md @@ -114,8 +114,8 @@ Optional step for optimistic sync tests. This step sets the [`payloadStatus`](https://github.com/ethereum/execution-apis/blob/main/src/engine/specification.md#PayloadStatusV1) value that Execution Layer client mock returns in responses to the following Engine API calls: -* [`engine_newPayloadV1(payload)`](https://github.com/ethereum/execution-apis/blob/main/src/engine/specification.md#engine_newpayloadv1) if `payload.blockHash == payload_info.block_hash` -* [`engine_forkchoiceUpdatedV1(forkchoiceState, ...)`](https://github.com/ethereum/execution-apis/blob/main/src/engine/specification.md#engine_forkchoiceupdatedv1) if `forkchoiceState.headBlockHash == payload_info.block_hash` +* [`engine_newPayloadV1(payload)`](https://github.com/ethereum/execution-apis/blob/main/src/engine/paris.md#engine_newpayloadv1) if `payload.blockHash == payload_info.block_hash` +* [`engine_forkchoiceUpdatedV1(forkchoiceState, ...)`](https://github.com/ethereum/execution-apis/blob/main/src/engine/paris.md#engine_forkchoiceupdatedv1) if `forkchoiceState.headBlockHash == payload_info.block_hash` *Note:* Status of a payload must be *initialized* via `on_payload_info` before the corresponding `on_block` execution step. From 4244dc8e139f299b0b0a2cdaf1f045b69a9b0917 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 May 2023 17:55:14 +0800 Subject: [PATCH 016/110] Make `excess_data_gas` a param of `run_block_with_blobs` --- tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index 2b99d6a72..111565cce 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -18,14 +18,14 @@ from eth2spec.test.helpers.sharding import ( ) -def run_block_with_blobs(spec, state, blob_count): +def run_block_with_blobs(spec, state, blob_count, excess_data_gas=1): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=blob_count) block.body.blob_kzg_commitments = blob_kzg_commitments block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.excess_data_gas = 2 + block.body.execution_payload.excess_data_gas = excess_data_gas block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) signed_block = state_transition_and_sign_block(spec, state, block) From 79b8a9abecded921179d2b2854d8dc7b8c570d5d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 May 2023 18:09:01 +0800 Subject: [PATCH 017/110] Apply suggestions from code review --- specs/phase0/validator.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 5266fec7a..92eadde5f 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -612,7 +612,7 @@ Because Phase 0 does not have shards and thus does not have Shard Committees, th * Remain subscribed to `SUBNETS_PER_NODE` for `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs. * Maintain advertisement of the selected subnets in their node's ENR `attnets` entry by setting the selected `subnet_id` bits to `True` (e.g. `ENR["attnets"][subnet_id] = True`) for all persistent attestation subnets. -* Select these subnets based on their node-id as specified by the following `compute_subscribed_subnets(node_id,epoch)` function. +* Select these subnets based on their node-id as specified by the following `compute_subscribed_subnets(node_id, epoch)` function. ```python def compute_subscribed_subnet(node_id: int, epoch: Epoch, index: int) -> int: @@ -629,7 +629,7 @@ def compute_subscribed_subnet(node_id: int, epoch: Epoch, index: int) -> int: ```python def compute_subscribed_subnets(node_id: int, epoch: Epoch) -> Sequence[int]: - return [compute_subscribed_subnet(node_id, epoch, idx) for idx in range(SUBNETS_PER_NODE)] + return [compute_subscribed_subnet(node_id, epoch, index) for index in range(SUBNETS_PER_NODE)] ``` *Note*: When preparing for a hard fork, a validator must select and subscribe to subnets of the future fork versioning at least `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs in advance of the fork. These new subnets for the fork are maintained in addition to those for the current fork until the fork occurs. After the fork occurs, let the subnets from the previous fork reach the end of life with no replacements. From 5cb2733ed5271c582fb2235367558ff8950dd7a2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 May 2023 18:50:13 +0800 Subject: [PATCH 018/110] Add custom types `NodeID` and `SubnetID` and constant `NODE_ID_BITS` --- setup.py | 4 +-- specs/phase0/validator.md | 25 ++++++++++++++----- .../unittests/test_config_invariants.py | 2 ++ 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 52bad2b71..f1130eb58 100644 --- a/setup.py +++ b/setup.py @@ -383,7 +383,7 @@ from typing import ( from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes from eth2spec.utils.ssz.ssz_typing import ( - View, boolean, Container, List, Vector, uint8, uint32, uint64, + View, boolean, Container, List, Vector, uint8, uint32, uint64, uint256, Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist) from eth2spec.utils.ssz.ssz_typing import Bitvector # noqa: F401 from eth2spec.utils import bls @@ -551,7 +551,7 @@ class BellatrixSpecBuilder(AltairSpecBuilder): return super().imports(preset_name) + f''' from typing import Protocol from eth2spec.altair import {preset_name} as altair -from eth2spec.utils.ssz.ssz_typing import Bytes8, Bytes20, ByteList, ByteVector, uint256 +from eth2spec.utils.ssz.ssz_typing import Bytes8, Bytes20, ByteList, ByteVector ''' @classmethod diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 92eadde5f..604350ed8 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -10,6 +10,7 @@ This is an accompanying document to [Phase 0 -- The Beacon Chain](./beacon-chain - [Introduction](#introduction) - [Prerequisites](#prerequisites) +- [Custom types](#custom-types) - [Constants](#constants) - [Misc](#misc) - [Containers](#containers) @@ -82,6 +83,15 @@ A validator is an entity that participates in the consensus of the Ethereum proo All terminology, constants, functions, and protocol mechanics defined in the [Phase 0 -- The Beacon Chain](./beacon-chain.md) and [Phase 0 -- Deposit Contract](./deposit-contract.md) doc are requisite for this document and used throughout. Please see the Phase 0 doc before continuing and use as a reference throughout. +## Custom types + +We define the following Python custom types for type hinting and readability: + +| Name | SSZ equivalent | Description | +| - | - | - | +| `NodeID` | `uint256` | node identifier | +| `SubnetID` | `uint64` | subnet identifier | + ## Constants ### Misc @@ -94,6 +104,7 @@ All terminology, constants, functions, and protocol mechanics defined in the [Ph | `ATTESTATION_SUBNET_EXTRA_BITS` | `0` | The number of extra bits of a NodeId to use when mapping to a subscribed subnet | | `SUBNETS_PER_NODE` | `2` | The number of long-lived subnets a beacon node should be subscribed to. | | `ATTESTATION_SUBNET_PREFIX_BITS` | `(ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS)` | | +| `NODE_ID_BITS` | `256` | The bit length of uint256 is 256 | ## Containers @@ -515,7 +526,9 @@ The `subnet_id` for the `attestation` is calculated with: - Let `subnet_id = compute_subnet_for_attestation(committees_per_slot, attestation.data.slot, attestation.data.index)`. ```python -def compute_subnet_for_attestation(committees_per_slot: uint64, slot: Slot, committee_index: CommitteeIndex) -> uint64: +def compute_subnet_for_attestation(committees_per_slot: uint64, + slot: Slot, + committee_index: CommitteeIndex) -> SubnetID: """ Compute the correct subnet for an attestation for Phase 0. Note, this mimics expected future behavior where attestations will be mapped to their shard subnet. @@ -523,7 +536,7 @@ def compute_subnet_for_attestation(committees_per_slot: uint64, slot: Slot, comm slots_since_epoch_start = uint64(slot % SLOTS_PER_EPOCH) committees_since_epoch_start = committees_per_slot * slots_since_epoch_start - return uint64((committees_since_epoch_start + committee_index) % ATTESTATION_SUBNET_COUNT) + return SubnetID((committees_since_epoch_start + committee_index) % ATTESTATION_SUBNET_COUNT) ``` ### Attestation aggregation @@ -615,8 +628,8 @@ Because Phase 0 does not have shards and thus does not have Shard Committees, th * Select these subnets based on their node-id as specified by the following `compute_subscribed_subnets(node_id, epoch)` function. ```python -def compute_subscribed_subnet(node_id: int, epoch: Epoch, index: int) -> int: - node_id_prefix = node_id >> (256 - int(ATTESTATION_SUBNET_PREFIX_BITS)) +def compute_subscribed_subnet(node_id: NodeID, epoch: Epoch, index: int) -> SubnetID: + node_id_prefix = node_id >> (NODE_ID_BITS - int(ATTESTATION_SUBNET_PREFIX_BITS)) node_offset = node_id % EPOCHS_PER_SUBNET_SUBSCRIPTION permutation_seed = hash(uint_to_bytes(uint64((epoch + node_offset) // EPOCHS_PER_SUBNET_SUBSCRIPTION))) permutated_prefix = compute_shuffled_index( @@ -624,11 +637,11 @@ def compute_subscribed_subnet(node_id: int, epoch: Epoch, index: int) -> int: 1 << int(ATTESTATION_SUBNET_PREFIX_BITS), permutation_seed, ) - return (permutated_prefix + index) % ATTESTATION_SUBNET_COUNT + return SubnetID((permutated_prefix + index) % ATTESTATION_SUBNET_COUNT) ``` ```python -def compute_subscribed_subnets(node_id: int, epoch: Epoch) -> Sequence[int]: +def compute_subscribed_subnets(node_id: NodeID, epoch: Epoch) -> Sequence[SubnetID]: return [compute_subscribed_subnet(node_id, epoch, index) for index in range(SUBNETS_PER_NODE)] ``` diff --git a/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py b/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py index 69aa3eb2a..b0fd06374 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py +++ b/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py @@ -76,6 +76,8 @@ def test_time(spec, state): @spec_state_test def test_networking(spec, state): assert spec.SUBNETS_PER_NODE <= spec.ATTESTATION_SUBNET_COUNT + node_id_length = spec.NodeID(1).type_byte_length() # in bytes + assert node_id_length * 8 == spec.NODE_ID_BITS # in bits @with_all_phases From 9f5bb03cb4fea394a0d64963c3e3b820a5da5813 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 10 Nov 2022 16:33:50 -0500 Subject: [PATCH 019/110] Refactor `run_generator` --- .../gen_helpers/gen_base/gen_runner.py | 296 +++++++++++------- 1 file changed, 175 insertions(+), 121 deletions(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index 0a831a592..5e6ea93d3 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -1,4 +1,7 @@ -from eth_utils import encode_hex +from dataclasses import ( + dataclass, + field, +) import os import time import shutil @@ -15,6 +18,8 @@ from ruamel.yaml import ( from filelock import FileLock from snappy import compress +from eth_utils import encode_hex + from eth2spec.test import context from eth2spec.test.exceptions import SkippedTest @@ -28,6 +33,14 @@ context.is_pytest = False TIME_THRESHOLD_TO_PRINT = 1.0 # seconds +@dataclass +class Diagnostics(object): + collected_test_count: int = 0 + generated_test_count: int = 0 + skipped_test_count: int = 0 + test_identifiers: list = field(default_factory=list) + + def validate_output_dir(path_str): path = Path(path_str) @@ -40,6 +53,47 @@ def validate_output_dir(path_str): return path +def get_test_case_dir(test_case, output_dir): + return ( + Path(output_dir) / Path(test_case.preset_name) / Path(test_case.fork_name) + / Path(test_case.runner_name) / Path(test_case.handler_name) + / Path(test_case.suite_name) / Path(test_case.case_name) + ) + + +def get_test_identifier(test_case): + return "::".join([ + test_case.preset_name, + test_case.fork_name, + test_case.runner_name, + test_case.handler_name, + test_case.suite_name, + test_case.case_name + ]) + + +def get_incomplete_tag_file(case_dir): + return case_dir / "INCOMPLETE" + + +def should_skip_case_dir(case_dir, is_force, diagnostics_obj): + is_skip = False + incomplete_tag_file = get_incomplete_tag_file(case_dir) + + if case_dir.exists(): + if not is_force and not incomplete_tag_file.exists(): + diagnostics_obj.skipped_test_count += 1 + print(f'Skipping already existing test: {case_dir}') + is_skip = True + else: + print(f'Warning, output directory {case_dir} already exist,' + ' old files will be deleted and it will generate test vector files with the latest version') + # Clear the existing case_dir folder + shutil.rmtree(case_dir) + + return is_skip, diagnostics_obj + + def run_generator(generator_name, test_providers: Iterable[TestProvider]): """ Implementation for a general test generator. @@ -129,10 +183,8 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): print(f"Filtering test-generator runs to only include presets: {', '.join(presets)}") collect_only = args.collect_only - collected_test_count = 0 - generated_test_count = 0 - skipped_test_count = 0 - test_identifiers = [] + + diagnostics_obj = Diagnostics() provider_start = time.time() for tprov in test_providers: @@ -145,146 +197,114 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): if len(presets) != 0 and test_case.preset_name not in presets: continue - case_dir = ( - Path(output_dir) / Path(test_case.preset_name) / Path(test_case.fork_name) - / Path(test_case.runner_name) / Path(test_case.handler_name) - / Path(test_case.suite_name) / Path(test_case.case_name) - ) - collected_test_count += 1 + case_dir = get_test_case_dir(test_case, output_dir) print(f"Collected test at: {case_dir}") + diagnostics_obj.collected_test_count += 1 - incomplete_tag_file = case_dir / "INCOMPLETE" + is_skip, diagnostics_obj = should_skip_case_dir(case_dir, args.force, diagnostics_obj) + if is_skip: + continue - if case_dir.exists(): - if not args.force and not incomplete_tag_file.exists(): - skipped_test_count += 1 - print(f'Skipping already existing test: {case_dir}') - continue - else: - print(f'Warning, output directory {case_dir} already exist,' - f' old files will be deleted and it will generate test vector files with the latest version') - # Clear the existing case_dir folder - shutil.rmtree(case_dir) - - print(f'Generating test: {case_dir}') - test_start = time.time() - - written_part = False - - # Add `INCOMPLETE` tag file to indicate that the test generation has not completed. - case_dir.mkdir(parents=True, exist_ok=True) - with incomplete_tag_file.open("w") as f: - f.write("\n") - - try: - def output_part(out_kind: str, name: str, fn: Callable[[Path, ], None]): - # make sure the test case directory is created before any test part is written. - case_dir.mkdir(parents=True, exist_ok=True) - try: - fn(case_dir) - except IOError as e: - error_message = ( - f'[Error] error when dumping test "{case_dir}", part "{name}", kind "{out_kind}": {e}' - ) - # Write to error log file - with log_file.open("a+") as f: - f.write(error_message) - traceback.print_exc(file=f) - f.write('\n') - - sys.exit(error_message) - - meta = dict() - - try: - for (name, out_kind, data) in test_case.case_fn(): - written_part = True - if out_kind == "meta": - meta[name] = data - elif out_kind == "cfg": - output_part(out_kind, name, dump_yaml_fn(data, name, file_mode, cfg_yaml)) - elif out_kind == "data": - output_part(out_kind, name, dump_yaml_fn(data, name, file_mode, yaml)) - elif out_kind == "ssz": - output_part(out_kind, name, dump_ssz_fn(data, name, file_mode)) - else: - assert False # Unknown kind - except SkippedTest as e: - print(e) - skipped_test_count += 1 - shutil.rmtree(case_dir) - continue - - # Once all meta data is collected (if any), write it to a meta data file. - if len(meta) != 0: - written_part = True - output_part("data", "meta", dump_yaml_fn(meta, "meta", file_mode, yaml)) - - if not written_part: - print(f"test case {case_dir} did not produce any test case parts") - except Exception as e: - error_message = f"[ERROR] failed to generate vector(s) for test {case_dir}: {e}" - # Write to error log file - with log_file.open("a+") as f: - f.write(error_message) - traceback.print_exc(file=f) - f.write('\n') - traceback.print_exc() - else: - # If no written_part, the only file was incomplete_tag_file. Clear the existing case_dir folder. - if not written_part: - shutil.rmtree(case_dir) - else: - generated_test_count += 1 - test_identifier = "::".join([ - test_case.preset_name, - test_case.fork_name, - test_case.runner_name, - test_case.handler_name, - test_case.suite_name, - test_case.case_name - ]) - test_identifiers.append(test_identifier) - # Only remove `INCOMPLETE` tag file - os.remove(incomplete_tag_file) - test_end = time.time() - span = round(test_end - test_start, 2) - if span > TIME_THRESHOLD_TO_PRINT: - print(f' - generated in {span} seconds') + # generate test vector + is_skip, diagnostics_obj = generate_test_vector_and_diagnose( + test_case, case_dir, log_file, file_mode, + cfg_yaml, yaml, diagnostics_obj, + ) + if is_skip: + continue provider_end = time.time() span = round(provider_end - provider_start, 2) if collect_only: - print(f"Collected {collected_test_count} tests in total") + print(f"Collected {diagnostics_obj.collected_test_count} tests in total") else: - summary_message = f"completed generation of {generator_name} with {generated_test_count} tests" - summary_message += f" ({skipped_test_count} skipped tests)" + summary_message = f"completed generation of {generator_name} with {diagnostics_obj.generated_test_count} tests" + summary_message += f" ({diagnostics_obj.skipped_test_count} skipped tests)" if span > TIME_THRESHOLD_TO_PRINT: summary_message += f" in {span} seconds" print(summary_message) - diagnostics = { - "collected_test_count": collected_test_count, - "generated_test_count": generated_test_count, - "skipped_test_count": skipped_test_count, - "test_identifiers": test_identifiers, + + diagnostics_output = { + "collected_test_count": diagnostics_obj.collected_test_count, + "generated_test_count": diagnostics_obj.generated_test_count, + "skipped_test_count": diagnostics_obj.skipped_test_count, + "test_identifiers": diagnostics_obj.test_identifiers, "durations": [f"{span} seconds"], } - diagnostics_path = Path(os.path.join(output_dir, "diagnostics.json")) - diagnostics_lock = FileLock(os.path.join(output_dir, "diagnostics.json.lock")) + diagnostics_path = Path(os.path.join(output_dir, "diagnostics_obj.json")) + diagnostics_lock = FileLock(os.path.join(output_dir, "diagnostics_obj.json.lock")) with diagnostics_lock: diagnostics_path.touch(exist_ok=True) if os.path.getsize(diagnostics_path) == 0: with open(diagnostics_path, "w+") as f: - json.dump(diagnostics, f) + json.dump(diagnostics_output, f) else: with open(diagnostics_path, "r+") as f: existing_diagnostics = json.load(f) - for k, v in diagnostics.items(): + for k, v in diagnostics_output.items(): existing_diagnostics[k] += v with open(diagnostics_path, "w+") as f: json.dump(existing_diagnostics, f) - print(f"wrote diagnostics to {diagnostics_path}") + print(f"wrote diagnostics_obj to {diagnostics_path}") + + +def generate_test_vector_and_diagnose(test_case, case_dir, log_file, file_mode, cfg_yaml, yaml, diagnostics_obj): + is_skip = False + + print(f'Generating test: {case_dir}') + test_start = time.time() + + written_part = False + + # Add `INCOMPLETE` tag file to indicate that the test generation has not completed. + incomplete_tag_file = get_incomplete_tag_file(case_dir) + case_dir.mkdir(parents=True, exist_ok=True) + with incomplete_tag_file.open("w") as f: + f.write("\n") + + try: + meta = dict() + try: + written_part, meta = execute_test(test_case, case_dir, meta, log_file, file_mode, cfg_yaml, yaml) + except SkippedTest as e: + print(e) + diagnostics_obj.skipped_test_count += 1 + shutil.rmtree(case_dir) + is_skip = True + return is_skip, diagnostics_obj + + # Once all meta data is collected (if any), write it to a meta data file. + if len(meta) != 0: + written_part = True + output_part(case_dir, log_file, "data", "meta", dump_yaml_fn(meta, "meta", file_mode, yaml)) + + if not written_part: + print(f"test case {case_dir} did not produce any test case parts") + except Exception as e: + error_message = f"[ERROR] failed to generate vector(s) for test {case_dir}: {e}" + # Write to error log file + with log_file.open("a+") as f: + f.write(error_message) + traceback.print_exc(file=f) + f.write('\n') + traceback.print_exc() + else: + # If no written_part, the only file was incomplete_tag_file. Clear the existing case_dir folder. + if not written_part: + shutil.rmtree(case_dir) + else: + diagnostics_obj.generated_test_count += 1 + test_identifier = get_test_identifier(test_case) + diagnostics_obj.test_identifiers.append(test_identifier) + # Only remove `INCOMPLETE` tag file + os.remove(incomplete_tag_file) + test_end = time.time() + span = round(test_end - test_start, 2) + if span > TIME_THRESHOLD_TO_PRINT: + print(f' - generated in {span} seconds') + + return is_skip, diagnostics_obj def dump_yaml_fn(data: Any, name: str, file_mode: str, yaml_encoder: YAML): @@ -295,6 +315,40 @@ def dump_yaml_fn(data: Any, name: str, file_mode: str, yaml_encoder: YAML): return dump +def output_part(case_dir, log_file, out_kind: str, name: str, fn: Callable[[Path, ], None]): + # make sure the test case directory is created before any test part is written. + case_dir.mkdir(parents=True, exist_ok=True) + try: + fn(case_dir) + except IOError as e: + error_message = f'[Error] error when dumping test "{case_dir}", part "{name}", kind "{out_kind}": {e}' + # Write to error log file + with log_file.open("a+") as f: + f.write(error_message) + traceback.print_exc(file=f) + f.write('\n') + + sys.exit(error_message) + + +def execute_test(test_case, case_dir, meta, log_file, file_mode, cfg_yaml, yaml): + result = test_case.case_fn() + for (name, out_kind, data) in result: + written_part = True + if out_kind == "meta": + meta[name] = data + elif out_kind == "cfg": + output_part(case_dir, log_file, out_kind, name, dump_yaml_fn(data, name, file_mode, cfg_yaml)) + elif out_kind == "data": + output_part(case_dir, log_file, out_kind, name, dump_yaml_fn(data, name, file_mode, yaml)) + elif out_kind == "ssz": + output_part(case_dir, log_file, out_kind, name, dump_ssz_fn(data, name, file_mode)) + else: + raise ValueError("Unknown out_kind %s" % out_kind) + + return written_part, meta + + def dump_ssz_fn(data: AnyStr, name: str, file_mode: str): def dump(case_path: Path): out_path = case_path / Path(name + '.ssz_snappy') From aeccd20fd102021984073162e48fc84abc900601 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 5 May 2023 23:03:25 +0800 Subject: [PATCH 020/110] Try multiprocessing --- setup.py | 2 +- .../gen_helpers/gen_base/gen_runner.py | 141 ++++++++++++------ tests/core/pyspec/eth2spec/test/context.py | 6 +- 3 files changed, 100 insertions(+), 49 deletions(-) diff --git a/setup.py b/setup.py index 5d2736979..a82f55e02 100644 --- a/setup.py +++ b/setup.py @@ -1184,7 +1184,7 @@ setup( extras_require={ "test": ["pytest>=4.4", "pytest-cov", "pytest-xdist"], "lint": ["flake8==5.0.4", "mypy==0.981", "pylint==2.15.3"], - "generator": ["python-snappy==0.6.1", "filelock"], + "generator": ["python-snappy==0.6.1", "filelock", "pathos==0.3.0"], "docs": ["mkdocs==1.4.2", "mkdocs-material==9.1.5", "mdx-truly-sane-lists==1.3", "mkdocs-awesome-pages-plugin==2.8.0"] }, install_requires=[ diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index 5e6ea93d3..56b1bcab1 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -11,12 +11,16 @@ import sys import json from typing import Iterable, AnyStr, Any, Callable import traceback +import multiprocessing +from collections import namedtuple + from ruamel.yaml import ( YAML, ) from filelock import FileLock from snappy import compress +from pathos.multiprocessing import ProcessingPool as Pool from eth_utils import encode_hex @@ -32,6 +36,12 @@ context.is_pytest = False TIME_THRESHOLD_TO_PRINT = 1.0 # seconds +# Generator mode setting +MODE_SINGLE_PROCESS = 'MODE_SINGLE_PROCESS' +MODE_MULTIPROCESSING = 'MODE_MULTIPROCESSING' + +GENERATOR_MODE = MODE_SINGLE_PROCESS + @dataclass class Diagnostics(object): @@ -41,6 +51,45 @@ class Diagnostics(object): test_identifiers: list = field(default_factory=list) +TestCaseParams = namedtuple( + 'TestCaseParams', [ + 'test_case', 'case_dir', 'log_file', 'file_mode', + ]) + + +def worker_function(item): + return generate_test_vector(*item) + + +def get_default_yaml(): + yaml = YAML(pure=True) + yaml.default_flow_style = None + + def _represent_none(self, _): + return self.represent_scalar('tag:yaml.org,2002:null', 'null') + + yaml.representer.add_representer(type(None), _represent_none) + + return yaml + + +def get_cfg_yaml(): + # Spec config is using a YAML subset + cfg_yaml = YAML(pure=True) + cfg_yaml.default_flow_style = False # Emit separate line for each key + + def cfg_represent_bytes(self, data): + return self.represent_int(encode_hex(data)) + + cfg_yaml.representer.add_representer(bytes, cfg_represent_bytes) + + def cfg_represent_quoted_str(self, data): + return self.represent_scalar(u'tag:yaml.org,2002:str', data, style="'") + + cfg_yaml.representer.add_representer(context.quoted_str, cfg_represent_quoted_str) + return cfg_yaml + + def validate_output_dir(path_str): path = Path(path_str) @@ -148,28 +197,6 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): else: file_mode = "w" - yaml = YAML(pure=True) - yaml.default_flow_style = None - - def _represent_none(self, _): - return self.represent_scalar('tag:yaml.org,2002:null', 'null') - - yaml.representer.add_representer(type(None), _represent_none) - - # Spec config is using a YAML subset - cfg_yaml = YAML(pure=True) - cfg_yaml.default_flow_style = False # Emit separate line for each key - - def cfg_represent_bytes(self, data): - return self.represent_int(encode_hex(data)) - - cfg_yaml.representer.add_representer(bytes, cfg_represent_bytes) - - def cfg_represent_quoted_str(self, data): - return self.represent_scalar(u'tag:yaml.org,2002:str', data, style="'") - - cfg_yaml.representer.add_representer(context.quoted_str, cfg_represent_quoted_str) - log_file = Path(output_dir) / 'testgen_error_log.txt' print(f"Generating tests into {output_dir}") @@ -185,8 +212,11 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): collect_only = args.collect_only diagnostics_obj = Diagnostics() - provider_start = time.time() + + if GENERATOR_MODE == MODE_MULTIPROCESSING: + all_test_case_params = [] + for tprov in test_providers: if not collect_only: # runs anything that we don't want to repeat for every test case. @@ -205,13 +235,20 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): if is_skip: continue - # generate test vector - is_skip, diagnostics_obj = generate_test_vector_and_diagnose( - test_case, case_dir, log_file, file_mode, - cfg_yaml, yaml, diagnostics_obj, - ) - if is_skip: - continue + if GENERATOR_MODE == MODE_SINGLE_PROCESS: + result = generate_test_vector(test_case, case_dir, log_file, file_mode) + write_result_into_diagnostics_obj(result, diagnostics_obj) + elif GENERATOR_MODE == MODE_MULTIPROCESSING: + item = TestCaseParams(test_case, case_dir, log_file, file_mode) + all_test_case_params.append(item) + + if GENERATOR_MODE == MODE_MULTIPROCESSING: + num_process = multiprocessing.cpu_count() // 2 - 1 + with Pool(processes=num_process) as pool: + results = pool.map(worker_function, iter(all_test_case_params)) + + for result in results: + write_result_into_diagnostics_obj(result, diagnostics_obj) provider_end = time.time() span = round(provider_end - provider_start, 2) @@ -249,54 +286,55 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): print(f"wrote diagnostics_obj to {diagnostics_path}") -def generate_test_vector_and_diagnose(test_case, case_dir, log_file, file_mode, cfg_yaml, yaml, diagnostics_obj): - is_skip = False +def generate_test_vector(test_case, case_dir, log_file, file_mode): + cfg_yaml = get_cfg_yaml() + yaml = get_default_yaml() + + written_part = False print(f'Generating test: {case_dir}') test_start = time.time() - written_part = False - # Add `INCOMPLETE` tag file to indicate that the test generation has not completed. incomplete_tag_file = get_incomplete_tag_file(case_dir) case_dir.mkdir(parents=True, exist_ok=True) with incomplete_tag_file.open("w") as f: f.write("\n") + result = None try: meta = dict() try: written_part, meta = execute_test(test_case, case_dir, meta, log_file, file_mode, cfg_yaml, yaml) except SkippedTest as e: + result = 0 # 0 means skipped print(e) - diagnostics_obj.skipped_test_count += 1 shutil.rmtree(case_dir) - is_skip = True - return is_skip, diagnostics_obj + return result # Once all meta data is collected (if any), write it to a meta data file. if len(meta) != 0: written_part = True output_part(case_dir, log_file, "data", "meta", dump_yaml_fn(meta, "meta", file_mode, yaml)) - if not written_part: - print(f"test case {case_dir} did not produce any test case parts") except Exception as e: + result = -1 # -1 means error error_message = f"[ERROR] failed to generate vector(s) for test {case_dir}: {e}" # Write to error log file with log_file.open("a+") as f: f.write(error_message) traceback.print_exc(file=f) f.write('\n') + print(error_message) traceback.print_exc() else: # If no written_part, the only file was incomplete_tag_file. Clear the existing case_dir folder. if not written_part: + print(f"test case {case_dir} did not produce any written_part") shutil.rmtree(case_dir) + result = -1 else: - diagnostics_obj.generated_test_count += 1 - test_identifier = get_test_identifier(test_case) - diagnostics_obj.test_identifiers.append(test_identifier) + result = get_test_identifier(test_case) # Only remove `INCOMPLETE` tag file os.remove(incomplete_tag_file) test_end = time.time() @@ -304,7 +342,19 @@ def generate_test_vector_and_diagnose(test_case, case_dir, log_file, file_mode, if span > TIME_THRESHOLD_TO_PRINT: print(f' - generated in {span} seconds') - return is_skip, diagnostics_obj + return result + + +def write_result_into_diagnostics_obj(result, diagnostics_obj): + if result == -1: # error + pass + elif result == 0: + diagnostics_obj.skipped_test_count += 1 + elif result is not None: + diagnostics_obj.generated_test_count += 1 + diagnostics_obj.test_identifiers.append(result) + else: + raise Exception(f"Unexpected result: {result}") def dump_yaml_fn(data: Any, name: str, file_mode: str, yaml_encoder: YAML): @@ -312,6 +362,7 @@ def dump_yaml_fn(data: Any, name: str, file_mode: str, yaml_encoder: YAML): out_path = case_path / Path(name + '.yaml') with out_path.open(file_mode) as f: yaml_encoder.dump(data, f) + f.close() return dump @@ -320,14 +371,14 @@ def output_part(case_dir, log_file, out_kind: str, name: str, fn: Callable[[Path case_dir.mkdir(parents=True, exist_ok=True) try: fn(case_dir) - except IOError as e: + except (IOError, ValueError) as e: error_message = f'[Error] error when dumping test "{case_dir}", part "{name}", kind "{out_kind}": {e}' # Write to error log file with log_file.open("a+") as f: f.write(error_message) traceback.print_exc(file=f) f.write('\n') - + print(error_message) sys.exit(error_message) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 901fd273a..626ffc1db 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -560,7 +560,7 @@ def _get_basic_dict(ssz_dict: Dict[str, Any]) -> Dict[str, Any]: return result -def _get_copy_of_spec(spec): +def get_copy_of_spec(spec): fork = spec.fork preset = spec.config.PRESET_BASE module_path = f"eth2spec.{fork}.{preset}" @@ -601,14 +601,14 @@ def with_config_overrides(config_overrides, emitted_fork=None, emit=True): def decorator(fn): def wrapper(*args, spec: Spec, **kw): # Apply config overrides to spec - spec, output_config = spec_with_config_overrides(_get_copy_of_spec(spec), config_overrides) + spec, output_config = spec_with_config_overrides(get_copy_of_spec(spec), config_overrides) # Apply config overrides to additional phases, if present if 'phases' in kw: phases = {} for fork in kw['phases']: phases[fork], output = spec_with_config_overrides( - _get_copy_of_spec(kw['phases'][fork]), config_overrides) + get_copy_of_spec(kw['phases'][fork]), config_overrides) if emitted_fork == fork: output_config = output kw['phases'] = phases From 98d0ca48b8730f05ed180182971f629bfedd8d0d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 6 May 2023 17:45:22 +0800 Subject: [PATCH 021/110] Fix `test_randomized_state` and `test_randomized_state_leaking` --- .../test_process_inactivity_updates.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_inactivity_updates.py b/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_inactivity_updates.py index 0816dfad6..57fe8b9ca 100644 --- a/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_inactivity_updates.py +++ b/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_inactivity_updates.py @@ -54,7 +54,15 @@ def test_genesis_random_scores(spec, state): # def run_inactivity_scores_test(spec, state, participation_fn=None, inactivity_scores_fn=None, rng=Random(10101)): - next_epoch_via_block(spec, state) + while True: + try: + next_epoch_via_block(spec, state) + except AssertionError: + # If the proposer is slashed, we skip this epoch and try to propose block at the next epoch + next_epoch(spec, state) + else: + break + if participation_fn is not None: participation_fn(spec, state, rng=rng) if inactivity_scores_fn is not None: @@ -363,7 +371,7 @@ def test_randomized_state(spec, state): their inactivity score does not change. """ rng = Random(10011001) - _run_randomized_state_test_for_inactivity_updates(spec, state, rng=rng) + yield from _run_randomized_state_test_for_inactivity_updates(spec, state, rng=rng) @with_altair_and_later @@ -377,6 +385,6 @@ def test_randomized_state_leaking(spec, state): (refer ``get_eligible_validator_indices`). """ rng = Random(10011002) - _run_randomized_state_test_for_inactivity_updates(spec, state, rng=rng) + yield from _run_randomized_state_test_for_inactivity_updates(spec, state, rng=rng) # Check still in leak assert spec.is_in_inactivity_leak(state) From 3ae4bf14f1d607a19f5b3bd694b841fff0545d98 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 6 May 2023 20:11:50 +0800 Subject: [PATCH 022/110] Fix and set to `MODE_MULTIPROCESSING` --- .../core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index 56b1bcab1..c8b7f0c03 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -40,7 +40,7 @@ TIME_THRESHOLD_TO_PRINT = 1.0 # seconds MODE_SINGLE_PROCESS = 'MODE_SINGLE_PROCESS' MODE_MULTIPROCESSING = 'MODE_MULTIPROCESSING' -GENERATOR_MODE = MODE_SINGLE_PROCESS +GENERATOR_MODE = MODE_MULTIPROCESSING @dataclass @@ -330,7 +330,7 @@ def generate_test_vector(test_case, case_dir, log_file, file_mode): else: # If no written_part, the only file was incomplete_tag_file. Clear the existing case_dir folder. if not written_part: - print(f"test case {case_dir} did not produce any written_part") + print(f"[Error] test case {case_dir} did not produce any written_part") shutil.rmtree(case_dir) result = -1 else: @@ -384,6 +384,7 @@ def output_part(case_dir, log_file, out_kind: str, name: str, fn: Callable[[Path def execute_test(test_case, case_dir, meta, log_file, file_mode, cfg_yaml, yaml): result = test_case.case_fn() + written_part = False for (name, out_kind, data) in result: written_part = True if out_kind == "meta": From 1008714e545b5a1e60cca772271f7df0592db394 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 May 2023 21:49:41 +0800 Subject: [PATCH 023/110] Add `settings.py` of testgen --- .../gen_helpers/gen_base/gen_runner.py | 20 ++++++++----------- .../eth2spec/gen_helpers/gen_base/settings.py | 13 ++++++++++++ 2 files changed, 21 insertions(+), 12 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/gen_helpers/gen_base/settings.py diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index c8b7f0c03..2562c7fad 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -11,7 +11,6 @@ import sys import json from typing import Iterable, AnyStr, Any, Callable import traceback -import multiprocessing from collections import namedtuple from ruamel.yaml import ( @@ -28,21 +27,19 @@ from eth2spec.test import context from eth2spec.test.exceptions import SkippedTest from .gen_typing import TestProvider +from .settings import ( + GENERATOR_MODE, + MODE_MULTIPROCESSING, + MODE_SINGLE_PROCESS, + NUM_PROCESS, + TIME_THRESHOLD_TO_PRINT, +) # Flag that the runner does NOT run test via pytest context.is_pytest = False -TIME_THRESHOLD_TO_PRINT = 1.0 # seconds - -# Generator mode setting -MODE_SINGLE_PROCESS = 'MODE_SINGLE_PROCESS' -MODE_MULTIPROCESSING = 'MODE_MULTIPROCESSING' - -GENERATOR_MODE = MODE_MULTIPROCESSING - - @dataclass class Diagnostics(object): collected_test_count: int = 0 @@ -243,8 +240,7 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): all_test_case_params.append(item) if GENERATOR_MODE == MODE_MULTIPROCESSING: - num_process = multiprocessing.cpu_count() // 2 - 1 - with Pool(processes=num_process) as pool: + with Pool(processes=NUM_PROCESS) as pool: results = pool.map(worker_function, iter(all_test_case_params)) for result in results: diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/settings.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/settings.py new file mode 100644 index 000000000..b8c3e591f --- /dev/null +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/settings.py @@ -0,0 +1,13 @@ +import multiprocessing + + +# Generator mode setting +MODE_SINGLE_PROCESS = 'MODE_SINGLE_PROCESS' +MODE_MULTIPROCESSING = 'MODE_MULTIPROCESSING' +# Test generator mode +GENERATOR_MODE = MODE_MULTIPROCESSING +# Number of subprocesses when using MODE_MULTIPROCESSING +NUM_PROCESS = multiprocessing.cpu_count() // 2 - 1 + +# Diagnostics +TIME_THRESHOLD_TO_PRINT = 1.0 # seconds From 0f5ac1186e29ec883abd11842530f91c74d5bc52 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 10 May 2023 02:29:00 +0800 Subject: [PATCH 024/110] Remove `is_execution_enabled` condition since Capella --- specs/_features/eip6110/beacon-chain.md | 8 +++---- specs/_features/sharding/beacon-chain.md | 3 +-- specs/capella/beacon-chain.md | 6 ++--- specs/deneb/beacon-chain.md | 5 ++-- .../test/bellatrix/sanity/test_blocks.py | 7 ++++-- .../test/capella/sanity/test_blocks.py | 23 +++++++++++++++++++ 6 files changed, 37 insertions(+), 15 deletions(-) diff --git a/specs/_features/eip6110/beacon-chain.md b/specs/_features/eip6110/beacon-chain.md index 708418e1c..4d69fb4e0 100644 --- a/specs/_features/eip6110/beacon-chain.md +++ b/specs/_features/eip6110/beacon-chain.md @@ -176,9 +176,8 @@ class BeaconState(Container): ```python def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) - if is_execution_enabled(state, block.body): - process_withdrawals(state, block.body.execution_payload) - process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in EIP6110] + process_withdrawals(state, block.body.execution_payload) + process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in EIP6110] process_randao(state, block.body) process_eth1_data(state, block.body) process_operations(state, block.body) # [Modified in EIP6110] @@ -212,8 +211,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: for_ops(body.bls_to_execution_changes, process_bls_to_execution_change) # [New in EIP6110] - if is_execution_enabled(state, body): - for_ops(body.execution_payload.deposit_receipts, process_deposit_receipt) + for_ops(body.execution_payload.deposit_receipts, process_deposit_receipt) ``` #### New `process_deposit_receipt` diff --git a/specs/_features/sharding/beacon-chain.md b/specs/_features/sharding/beacon-chain.md index 7d6df51aa..f7de7af65 100644 --- a/specs/_features/sharding/beacon-chain.md +++ b/specs/_features/sharding/beacon-chain.md @@ -236,8 +236,7 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) verify_builder_block_bid(state, block) process_sharded_data(state, block) - if is_execution_enabled(state, block.body): - process_execution_payload(state, block, EXECUTION_ENGINE) + process_execution_payload(state, block, EXECUTION_ENGINE) if not is_builder_block_slot(block.slot): process_randao(state, block.body) diff --git a/specs/capella/beacon-chain.md b/specs/capella/beacon-chain.md index 1df617daa..72260108f 100644 --- a/specs/capella/beacon-chain.md +++ b/specs/capella/beacon-chain.md @@ -331,9 +331,9 @@ def process_historical_summaries_update(state: BeaconState) -> None: ```python def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) - if is_execution_enabled(state, block.body): - process_withdrawals(state, block.body.execution_payload) # [New in Capella] - process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in Capella] + # Removed `is_execution_enabled` check + process_withdrawals(state, block.body.execution_payload) # [New in Capella] + process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in Capella] process_randao(state, block.body) process_eth1_data(state, block.body) process_operations(state, block.body) # [Modified in Capella] diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 359c7fc95..6aaa2567f 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -200,9 +200,8 @@ def verify_kzg_commitments_against_transactions(transactions: Sequence[Transacti ```python def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) - if is_execution_enabled(state, block.body): - process_withdrawals(state, block.body.execution_payload) - process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in Deneb] + process_withdrawals(state, block.body.execution_payload) + process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in Deneb] process_randao(state, block.body) process_eth1_data(state, block.body) process_operations(state, block.body) diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/bellatrix/sanity/test_blocks.py index ef6bb75a9..75dfa0c9c 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/sanity/test_blocks.py @@ -10,7 +10,10 @@ from eth2spec.test.helpers.execution_payload import ( build_randomized_execution_payload ) from eth2spec.test.context import ( - with_bellatrix_and_later, spec_state_test + BELLATRIX, + with_bellatrix_and_later, + with_phases, + spec_state_test, ) @@ -44,7 +47,7 @@ def test_empty_block_transition_randomized_payload(spec, state): yield 'post', state -@with_bellatrix_and_later +@with_phases([BELLATRIX]) @spec_state_test def test_is_execution_enabled_false(spec, state): # Set `latest_execution_payload_header` to empty diff --git a/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py index d62e458be..11d6f4e91 100644 --- a/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py @@ -31,6 +31,29 @@ from eth2spec.test.helpers.deposits import ( from eth2spec.test.helpers.voluntary_exits import prepare_signed_exits +# +# `is_execution_enabled` has been removed from Capella +# + +@with_capella_and_later +@spec_state_test +def test_invalid_is_execution_enabled_false(spec, state): + # Set `latest_execution_payload_header` to empty + state.latest_execution_payload_header = spec.ExecutionPayloadHeader() + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + + # Set `execution_payload` to empty + block.body.execution_payload = spec.ExecutionPayload() + assert len(block.body.execution_payload.transactions) == 0 + + signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + + yield 'blocks', [signed_block] + yield 'post', None + + # # BLSToExecutionChange # From 5b983f40972e44a07a9022390898a48fab37e599 Mon Sep 17 00:00:00 2001 From: gajinder Date: Fri, 12 May 2023 21:44:43 +0530 Subject: [PATCH 025/110] Update the endianess of the polynomial commitments to be big endian --- specs/deneb/polynomial-commitments.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/specs/deneb/polynomial-commitments.md b/specs/deneb/polynomial-commitments.md index e23c31fab..fb2d70237 100644 --- a/specs/deneb/polynomial-commitments.md +++ b/specs/deneb/polynomial-commitments.md @@ -77,6 +77,7 @@ Public functions MUST accept raw bytes as input and perform the required cryptog | `BYTES_PER_FIELD_ELEMENT` | `uint64(32)` | Bytes used to encode a BLS scalar field element | | `BYTES_PER_BLOB` | `uint64(BYTES_PER_FIELD_ELEMENT * FIELD_ELEMENTS_PER_BLOB)` | The number of bytes in a blob | | `G1_POINT_AT_INFINITY` | `Bytes48(b'\xc0' + b'\x00' * 47)` | Serialized form of the point at infinity on the G1 group | +| `KZG_ENDIANNESS` | `'big'` | The endianess of the field elements including blobs | ## Preset @@ -161,7 +162,7 @@ def hash_to_bls_field(data: bytes) -> BLSFieldElement: The output is not uniform over the BLS field. """ hashed_data = hash(data) - return BLSFieldElement(int.from_bytes(hashed_data, ENDIANNESS) % BLS_MODULUS) + return BLSFieldElement(int.from_bytes(hashed_data, KZG_ENDIANNESS) % BLS_MODULUS) ``` #### `bytes_to_bls_field` @@ -172,7 +173,7 @@ def bytes_to_bls_field(b: Bytes32) -> BLSFieldElement: Convert untrusted bytes to a trusted and validated BLS scalar field element. This function does not accept inputs greater than the BLS modulus. """ - field_element = int.from_bytes(b, ENDIANNESS) + field_element = int.from_bytes(b, KZG_ENDIANNESS) assert field_element < BLS_MODULUS return BLSFieldElement(field_element) ``` @@ -237,7 +238,7 @@ def compute_challenge(blob: Blob, """ # Append the degree of the polynomial as a domain separator - degree_poly = int.to_bytes(FIELD_ELEMENTS_PER_BLOB, 16, ENDIANNESS) + degree_poly = int.to_bytes(FIELD_ELEMENTS_PER_BLOB, 16, KZG_ENDIANNESS) data = FIAT_SHAMIR_PROTOCOL_DOMAIN + degree_poly data += blob @@ -406,15 +407,15 @@ def verify_kzg_proof_batch(commitments: Sequence[KZGCommitment], # Compute a random challenge. Note that it does not have to be computed from a hash, # r just has to be random. - degree_poly = int.to_bytes(FIELD_ELEMENTS_PER_BLOB, 8, ENDIANNESS) - num_commitments = int.to_bytes(len(commitments), 8, ENDIANNESS) + degree_poly = int.to_bytes(FIELD_ELEMENTS_PER_BLOB, 8, KZG_ENDIANNESS) + num_commitments = int.to_bytes(len(commitments), 8, KZG_ENDIANNESS) data = RANDOM_CHALLENGE_KZG_BATCH_DOMAIN + degree_poly + num_commitments # Append all inputs to the transcript before we hash for commitment, z, y, proof in zip(commitments, zs, ys, proofs): data += commitment \ - + int.to_bytes(z, BYTES_PER_FIELD_ELEMENT, ENDIANNESS) \ - + int.to_bytes(y, BYTES_PER_FIELD_ELEMENT, ENDIANNESS) \ + + int.to_bytes(z, BYTES_PER_FIELD_ELEMENT, KZG_ENDIANNESS) \ + + int.to_bytes(y, BYTES_PER_FIELD_ELEMENT, KZG_ENDIANNESS) \ + proof r = hash_to_bls_field(data) @@ -451,7 +452,7 @@ def compute_kzg_proof(blob: Blob, z_bytes: Bytes32) -> Tuple[KZGProof, Bytes32]: assert len(z_bytes) == BYTES_PER_FIELD_ELEMENT polynomial = blob_to_polynomial(blob) proof, y = compute_kzg_proof_impl(polynomial, bytes_to_bls_field(z_bytes)) - return proof, y.to_bytes(BYTES_PER_FIELD_ELEMENT, ENDIANNESS) + return proof, y.to_bytes(BYTES_PER_FIELD_ELEMENT, KZG_ENDIANNESS) ``` #### `compute_quotient_eval_within_domain` From 847553783b786686e7f24c69d104c055cbf9508b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 May 2023 17:11:50 +0800 Subject: [PATCH 026/110] Add `test_zeroed_commitment` --- .../eth2spec/test/deneb/sanity/test_blocks.py | 20 +++++++++++++++++++ .../pyspec/eth2spec/test/helpers/sharding.py | 19 +++++++++--------- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index 111565cce..b518f2b55 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -150,3 +150,23 @@ def test_incorrect_block_hash(spec, state): yield 'blocks', [signed_block] yield 'post', state + + +@with_deneb_and_later +@spec_state_test +def test_zeroed_commitment(spec, state): + """ + The blob is invalid, but the commitment is in correct form. + """ + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=1, is_valid_blob=False) + assert all(commitment == b'\x00' * 48 for commitment in blob_kzg_commitments) + block.body.blob_kzg_commitments = blob_kzg_commitments + block.body.execution_payload.transactions = [opaque_tx] + block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state diff --git a/tests/core/pyspec/eth2spec/test/helpers/sharding.py b/tests/core/pyspec/eth2spec/test/helpers/sharding.py index 6b913b90e..16b5a0234 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/sharding.py +++ b/tests/core/pyspec/eth2spec/test/helpers/sharding.py @@ -50,12 +50,9 @@ class SignedBlobTransaction(Container): signature: ECDSASignature -def get_sample_blob(spec, rng=None): - if rng is None: - rng = random.Random(5566) - +def get_sample_blob(spec, rng=random.Random(5566), is_valid_blob=True): values = [ - rng.randint(0, spec.BLS_MODULUS - 1) + rng.randint(0, spec.BLS_MODULUS - 1) if is_valid_blob else spec.BLS_MODULUS for _ in range(spec.FIELD_ELEMENTS_PER_BLOB) ] @@ -98,15 +95,19 @@ def get_poly_in_both_forms(spec, rng=None): return coeffs, evals -def get_sample_opaque_tx(spec, blob_count=1, rng=None): +def get_sample_opaque_tx(spec, blob_count=1, rng=random.Random(5566), is_valid_blob=True): blobs = [] blob_kzg_commitments = [] blob_kzg_proofs = [] blob_versioned_hashes = [] for _ in range(blob_count): - blob = get_sample_blob(spec, rng) - blob_commitment = spec.KZGCommitment(spec.blob_to_kzg_commitment(blob)) - blob_kzg_proof = spec.compute_blob_kzg_proof(blob, blob_commitment) + blob = get_sample_blob(spec, rng, is_valid_blob=is_valid_blob) + if is_valid_blob: + blob_commitment = spec.KZGCommitment(spec.blob_to_kzg_commitment(blob)) + blob_kzg_proof = spec.compute_blob_kzg_proof(blob, blob_commitment) + else: + blob_commitment = spec.KZGCommitment() + blob_kzg_proof = spec.KZGProof() blob_versioned_hash = spec.kzg_commitment_to_versioned_hash(blob_commitment) blobs.append(blob) blob_kzg_commitments.append(blob_commitment) From eea04704d48e8ddb7626f9835d9d81c548f15498 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 May 2023 17:16:41 +0800 Subject: [PATCH 027/110] Fix tests --- setup.py | 2 +- .../polynomial_commitments/test_polynomial_commitments.py | 6 +++--- tests/core/pyspec/eth2spec/test/helpers/sharding.py | 2 +- tests/generators/kzg_4844/main.py | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index b2316ed95..c6a9ebaa9 100644 --- a/setup.py +++ b/setup.py @@ -279,7 +279,7 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr elif name in config: config_vars[name] = VariableDefinition(value_def.type_name, config[name], value_def.comment, None) else: - if name == 'ENDIANNESS': + if name in ('ENDIANNESS', 'KZG_ENDIANNESS'): # Deal with mypy Literal typing check value_def = _parse_value(name, value, type_hint='Final') constant_vars[name] = value_def diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/polynomial_commitments/test_polynomial_commitments.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/polynomial_commitments/test_polynomial_commitments.py index 6d3f377a3..5ee8097c8 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/polynomial_commitments/test_polynomial_commitments.py +++ b/tests/core/pyspec/eth2spec/test/deneb/unittests/polynomial_commitments/test_polynomial_commitments.py @@ -33,7 +33,7 @@ def bls_add_one(x): def field_element_bytes(x): - return int.to_bytes(x % BLS_MODULUS, 32, "little") + return int.to_bytes(x % BLS_MODULUS, 32, "big") @with_deneb_and_later @@ -304,7 +304,7 @@ def test_bytes_to_bls_field_modulus_minus_one(spec): Verify that `bytes_to_bls_field` handles modulus minus one """ - spec.bytes_to_bls_field((BLS_MODULUS - 1).to_bytes(spec.BYTES_PER_FIELD_ELEMENT, spec.ENDIANNESS)) + spec.bytes_to_bls_field((BLS_MODULUS - 1).to_bytes(spec.BYTES_PER_FIELD_ELEMENT, spec.KZG_ENDIANNESS)) @with_deneb_and_later @@ -316,7 +316,7 @@ def test_bytes_to_bls_field_modulus(spec): """ expect_assertion_error(lambda: spec.bytes_to_bls_field( - BLS_MODULUS.to_bytes(spec.BYTES_PER_FIELD_ELEMENT, spec.ENDIANNESS) + BLS_MODULUS.to_bytes(spec.BYTES_PER_FIELD_ELEMENT, spec.KZG_ENDIANNESS) )) diff --git a/tests/core/pyspec/eth2spec/test/helpers/sharding.py b/tests/core/pyspec/eth2spec/test/helpers/sharding.py index 6b913b90e..cd88ae3ea 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/sharding.py +++ b/tests/core/pyspec/eth2spec/test/helpers/sharding.py @@ -61,7 +61,7 @@ def get_sample_blob(spec, rng=None): b = bytes() for v in values: - b += v.to_bytes(32, spec.ENDIANNESS) + b += v.to_bytes(32, spec.KZG_ENDIANNESS) return spec.Blob(b) diff --git a/tests/generators/kzg_4844/main.py b/tests/generators/kzg_4844/main.py index 2f3efeb21..9297e524a 100644 --- a/tests/generators/kzg_4844/main.py +++ b/tests/generators/kzg_4844/main.py @@ -27,11 +27,11 @@ def expect_exception(func, *args): def field_element_bytes(x): - return int.to_bytes(x % spec.BLS_MODULUS, 32, spec.ENDIANNESS) + return int.to_bytes(x % spec.BLS_MODULUS, 32, spec.KZG_ENDIANNESS) def field_element_bytes_unchecked(x): - return int.to_bytes(x, 32, spec.ENDIANNESS) + return int.to_bytes(x, 32, spec.KZG_ENDIANNESS) def encode_hex_list(a): @@ -54,7 +54,7 @@ def evaluate_blob_at(blob, z): ) -BLS_MODULUS_BYTES = spec.BLS_MODULUS.to_bytes(32, spec.ENDIANNESS) +BLS_MODULUS_BYTES = spec.BLS_MODULUS.to_bytes(32, spec.KZG_ENDIANNESS) G1 = bls.G1_to_bytes48(bls.G1()) G1_INVALID_TOO_FEW_BYTES = G1[:-1] From 340f3cc1a3c54bac1892050d7555b53d305ecd56 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 May 2023 17:25:48 +0800 Subject: [PATCH 028/110] Rebase the fc changes --- specs/capella/fork-choice.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/specs/capella/fork-choice.md b/specs/capella/fork-choice.md index c08be1c3f..87fec02f8 100644 --- a/specs/capella/fork-choice.md +++ b/specs/capella/fork-choice.md @@ -86,16 +86,22 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert block.slot > finalized_slot # Check block is a descendant of the finalized block at the checkpoint finalized slot - assert get_ancestor(store, block.parent_root, finalized_slot) == store.finalized_checkpoint.root + finalized_checkpoint_block = get_checkpoint_block( + store, + block.parent_root, + store.finalized_checkpoint.epoch, + ) + assert store.finalized_checkpoint.root == finalized_checkpoint_block # Check the block is valid and compute the post-state state = pre_state.copy() + block_root = hash_tree_root(block) state_transition(state, signed_block, True) # Add new block to the store - store.blocks[hash_tree_root(block)] = block + store.blocks[block_root] = block # Add new state for this block to the store - store.block_states[hash_tree_root(block)] = state + store.block_states[block_root] = state # Add proposer score boost if the block is timely time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT @@ -103,15 +109,9 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: if get_current_slot(store) == block.slot and is_before_attesting_interval: store.proposer_boost_root = hash_tree_root(block) - # Update justified checkpoint - if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: - if state.current_justified_checkpoint.epoch > store.best_justified_checkpoint.epoch: - store.best_justified_checkpoint = state.current_justified_checkpoint - if should_update_justified_checkpoint(store, state.current_justified_checkpoint): - store.justified_checkpoint = state.current_justified_checkpoint + # Update checkpoints in store if necessary + update_checkpoints(store, state.current_justified_checkpoint, state.finalized_checkpoint) - # Update finalized checkpoint - if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: - store.finalized_checkpoint = state.finalized_checkpoint - store.justified_checkpoint = state.current_justified_checkpoint + # Eagerly compute unrealized justification and finality. + compute_pulled_up_tip(store, block_root) ``` From 928fd42517b6ffc41ffd96e2e06066f47762ad4d Mon Sep 17 00:00:00 2001 From: g11tech Date: Mon, 15 May 2023 15:53:18 +0530 Subject: [PATCH 029/110] fix typo Co-authored-by: Hsiao-Wei Wang --- specs/deneb/polynomial-commitments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/polynomial-commitments.md b/specs/deneb/polynomial-commitments.md index fb2d70237..fe5ae845a 100644 --- a/specs/deneb/polynomial-commitments.md +++ b/specs/deneb/polynomial-commitments.md @@ -77,7 +77,7 @@ Public functions MUST accept raw bytes as input and perform the required cryptog | `BYTES_PER_FIELD_ELEMENT` | `uint64(32)` | Bytes used to encode a BLS scalar field element | | `BYTES_PER_BLOB` | `uint64(BYTES_PER_FIELD_ELEMENT * FIELD_ELEMENTS_PER_BLOB)` | The number of bytes in a blob | | `G1_POINT_AT_INFINITY` | `Bytes48(b'\xc0' + b'\x00' * 47)` | Serialized form of the point at infinity on the G1 group | -| `KZG_ENDIANNESS` | `'big'` | The endianess of the field elements including blobs | +| `KZG_ENDIANNESS` | `'big'` | The endianness of the field elements including blobs | ## Preset From 058137327a4cb9163c4202e1ed3bd4d886d745cf Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 16 May 2023 00:27:00 +0800 Subject: [PATCH 030/110] Use new `engine_newPayloadV3` --- setup.py | 2 +- specs/_features/eip6110/beacon-chain.md | 10 ++- specs/bellatrix/beacon-chain.md | 25 +++++- specs/capella/beacon-chain.md | 8 +- specs/deneb/beacon-chain.md | 85 ++++++++----------- specs/deneb/validator.md | 4 +- .../test_process_execution_payload.py | 4 +- .../test/phase0/sanity/test_blocks.py | 2 +- 8 files changed, 74 insertions(+), 66 deletions(-) diff --git a/setup.py b/setup.py index b2316ed95..8ef66df23 100644 --- a/setup.py +++ b/setup.py @@ -578,7 +578,7 @@ def get_pow_chain_head() -> PowBlock: class NoopExecutionEngine(ExecutionEngine): - def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: + def notify_new_payload(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: return True def notify_forkchoice_updated(self: ExecutionEngine, diff --git a/specs/_features/eip6110/beacon-chain.md b/specs/_features/eip6110/beacon-chain.md index 708418e1c..c9dccbdeb 100644 --- a/specs/_features/eip6110/beacon-chain.md +++ b/specs/_features/eip6110/beacon-chain.md @@ -178,12 +178,11 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) if is_execution_enabled(state, block.body): process_withdrawals(state, block.body.execution_payload) - process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in EIP6110] + process_execution_payload(state, block.body, EXECUTION_ENGINE) # [Modified in EIP6110] process_randao(state, block.body) process_eth1_data(state, block.body) process_operations(state, block.body) # [Modified in EIP6110] process_sync_aggregate(state, block.body.sync_aggregate) - process_blob_kzg_commitments(block.body) ``` #### Modified `process_operations` @@ -238,7 +237,9 @@ def process_deposit_receipt(state: BeaconState, deposit_receipt: DepositReceipt) *Note*: The function `process_execution_payload` is modified to use the new `ExecutionPayloadHeader` type. ```python -def process_execution_payload(state: BeaconState, payload: ExecutionPayload, execution_engine: ExecutionEngine) -> None: +def process_execution_payload(state: BeaconState, body: BeaconBlockBody, execution_engine: ExecutionEngine) -> None: + payload = body.execution_payload + # Verify consistency of the parent hash with respect to the previous execution payload header if is_merge_transition_complete(state): assert payload.parent_hash == state.latest_execution_payload_header.block_hash @@ -247,7 +248,8 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid - assert execution_engine.notify_new_payload(payload) + versioned_hashes = [kzg_commitment_to_versioned_hash(commitment) for commitment in body.blob_kzg_commitments] + assert execution_engine.notify_new_payload(NewPayloadRequest(execution_payload=payload)) # Cache execution payload header state.latest_execution_payload_header = ExecutionPayloadHeader( parent_hash=payload.parent_hash, diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index 1133cba06..a4c7be16e 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -33,6 +33,9 @@ - [Modified `slash_validator`](#modified-slash_validator) - [Beacon chain state transition function](#beacon-chain-state-transition-function) - [Execution engine](#execution-engine) + - [Request data](#request-data) + - [`NewPayloadRequest`](#newpayloadrequest) + - [Engine APIs](#engine-apis) - [`notify_new_payload`](#notify_new_payload) - [Block processing](#block-processing) - [Execution payload](#execution-payload) @@ -300,6 +303,18 @@ def slash_validator(state: BeaconState, ### Execution engine +#### Request data + +##### `NewPayloadRequest` + +```python +@dataclass +class NewPayloadRequest(object): + execution_payload: ExecutionPayload +``` + +#### Engine APIs + The implementation-dependent `ExecutionEngine` protocol encapsulates the execution sub-system logic via: * a state object `self.execution_state` of type `ExecutionState` @@ -313,7 +328,7 @@ The Engine API may be used to implement this and similarly defined functions via #### `notify_new_payload` ```python -def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: +def notify_new_payload(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: """ Return ``True`` if and only if ``execution_payload`` is valid with respect to ``self.execution_state``. """ @@ -328,7 +343,7 @@ def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayloa def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) if is_execution_enabled(state, block.body): - process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [New in Bellatrix] + process_execution_payload(state, block.body, EXECUTION_ENGINE) # [New in Bellatrix] process_randao(state, block.body) process_eth1_data(state, block.body) process_operations(state, block.body) @@ -340,7 +355,9 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: ##### `process_execution_payload` ```python -def process_execution_payload(state: BeaconState, payload: ExecutionPayload, execution_engine: ExecutionEngine) -> None: +def process_execution_payload(state: BeaconState, body: BeaconBlockBody, execution_engine: ExecutionEngine) -> None: + payload = body.execution_payload + # Verify consistency of the parent hash with respect to the previous execution payload header if is_merge_transition_complete(state): assert payload.parent_hash == state.latest_execution_payload_header.block_hash @@ -349,7 +366,7 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid - assert execution_engine.notify_new_payload(payload) + assert execution_engine.notify_new_payload(NewPayloadRequest(execution_payload=payload)) # Cache execution payload header state.latest_execution_payload_header = ExecutionPayloadHeader( parent_hash=payload.parent_hash, diff --git a/specs/capella/beacon-chain.md b/specs/capella/beacon-chain.md index 1df617daa..1799f4e6e 100644 --- a/specs/capella/beacon-chain.md +++ b/specs/capella/beacon-chain.md @@ -333,7 +333,7 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) if is_execution_enabled(state, block.body): process_withdrawals(state, block.body.execution_payload) # [New in Capella] - process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in Capella] + process_execution_payload(state, block.body, EXECUTION_ENGINE) # [Modified in Capella] process_randao(state, block.body) process_eth1_data(state, block.body) process_operations(state, block.body) # [Modified in Capella] @@ -407,7 +407,9 @@ def process_withdrawals(state: BeaconState, payload: ExecutionPayload) -> None: *Note*: The function `process_execution_payload` is modified to use the new `ExecutionPayloadHeader` type. ```python -def process_execution_payload(state: BeaconState, payload: ExecutionPayload, execution_engine: ExecutionEngine) -> None: +def process_execution_payload(state: BeaconState, body: BeaconBlockBody, execution_engine: ExecutionEngine) -> None: + payload = body.execution_payload + # Verify consistency of the parent hash with respect to the previous execution payload header if is_merge_transition_complete(state): assert payload.parent_hash == state.latest_execution_payload_header.block_hash @@ -416,7 +418,7 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid - assert execution_engine.notify_new_payload(payload) + assert execution_engine.notify_new_payload(NewPayloadRequest(execution_payload=payload)) # Cache execution payload header state.latest_execution_payload_header = ExecutionPayloadHeader( parent_hash=payload.parent_hash, diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 359c7fc95..52719dcf6 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -24,13 +24,15 @@ - [Helper functions](#helper-functions) - [Misc](#misc) - [`kzg_commitment_to_versioned_hash`](#kzg_commitment_to_versioned_hash) - - [`tx_peek_blob_versioned_hashes`](#tx_peek_blob_versioned_hashes) - - [`verify_kzg_commitments_against_transactions`](#verify_kzg_commitments_against_transactions) - [Beacon chain state transition function](#beacon-chain-state-transition-function) + - [Execution engine](#execution-engine) + - [Request data](#request-data) + - [Modified `NewPayloadRequest`](#modified-newpayloadrequest) + - [Engine APIs](#engine-apis) + - [Modified `notify_new_payload`](#modified-notify_new_payload) - [Block processing](#block-processing) - [Execution payload](#execution-payload) - [`process_execution_payload`](#process_execution_payload) - - [Blob KZG commitments](#blob-kzg-commitments) - [Testing](#testing) @@ -158,43 +160,33 @@ def kzg_commitment_to_versioned_hash(kzg_commitment: KZGCommitment) -> Versioned return VERSIONED_HASH_VERSION_KZG + hash(kzg_commitment)[1:] ``` -#### `tx_peek_blob_versioned_hashes` - -This function retrieves the hashes from the `SignedBlobTransaction` as defined in Deneb, using SSZ offsets. -Offsets are little-endian `uint32` values, as defined in the [SSZ specification](../../ssz/simple-serialize.md). -See [the full details of `blob_versioned_hashes` offset calculation](https://gist.github.com/protolambda/23bd106b66f6d4bb854ce46044aa3ca3). - -```python -def tx_peek_blob_versioned_hashes(opaque_tx: Transaction) -> Sequence[VersionedHash]: - assert opaque_tx[0] == BLOB_TX_TYPE - message_offset = 1 + uint32.decode_bytes(opaque_tx[1:5]) - # field offset: 32 + 8 + 32 + 32 + 8 + 4 + 32 + 4 + 4 + 32 = 188 - blob_versioned_hashes_offset = ( - message_offset - + uint32.decode_bytes(opaque_tx[(message_offset + 188):(message_offset + 192)]) - ) - # `VersionedHash` is a 32-byte object - assert (len(opaque_tx) - blob_versioned_hashes_offset) % 32 == 0 - return [ - VersionedHash(opaque_tx[x:(x + 32)]) - for x in range(blob_versioned_hashes_offset, len(opaque_tx), 32) - ] -``` - -#### `verify_kzg_commitments_against_transactions` - -```python -def verify_kzg_commitments_against_transactions(transactions: Sequence[Transaction], - kzg_commitments: Sequence[KZGCommitment]) -> bool: - all_versioned_hashes: List[VersionedHash] = [] - for tx in transactions: - if tx[0] == BLOB_TX_TYPE: - all_versioned_hashes += tx_peek_blob_versioned_hashes(tx) - return all_versioned_hashes == [kzg_commitment_to_versioned_hash(commitment) for commitment in kzg_commitments] -``` - ## Beacon chain state transition function +### Execution engine + +#### Request data + +##### Modified `NewPayloadRequest` + +```python +@dataclass +class NewPayloadRequest(object): + execution_payload: ExecutionPayload + versioned_hashes: Sequence[VersionedHash] +``` + +#### Engine APIs + +#### Modified `notify_new_payload` + +```python +def notify_new_payload(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: + """ + Return ``True`` if and only if ``execution_payload`` is valid with respect to ``self.execution_state``. + """ + ... +``` + ### Block processing ```python @@ -202,12 +194,11 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) if is_execution_enabled(state, block.body): process_withdrawals(state, block.body.execution_payload) - process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in Deneb] + process_execution_payload(state, block.body, EXECUTION_ENGINE) # [Modified in Deneb] process_randao(state, block.body) process_eth1_data(state, block.body) process_operations(state, block.body) process_sync_aggregate(state, block.body.sync_aggregate) - process_blob_kzg_commitments(block.body) # [New in Deneb] ``` #### Execution payload @@ -215,7 +206,9 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: ##### `process_execution_payload` ```python -def process_execution_payload(state: BeaconState, payload: ExecutionPayload, execution_engine: ExecutionEngine) -> None: +def process_execution_payload(state: BeaconState, body: BeaconBlockBody, execution_engine: ExecutionEngine) -> None: + payload = body.execution_payload + # Verify consistency of the parent hash with respect to the previous execution payload header if is_merge_transition_complete(state): assert payload.parent_hash == state.latest_execution_payload_header.block_hash @@ -224,7 +217,8 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid - assert execution_engine.notify_new_payload(payload) + versioned_hashes = [kzg_commitment_to_versioned_hash(commitment) for commitment in body.blob_kzg_commitments] + assert execution_engine.notify_new_payload(NewPayloadRequest(execution_payload=payload, versioned_hashes=versioned_hashes)) # Cache execution payload header state.latest_execution_payload_header = ExecutionPayloadHeader( @@ -247,13 +241,6 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe ) ``` -#### Blob KZG commitments - -```python -def process_blob_kzg_commitments(body: BeaconBlockBody) -> None: - assert verify_kzg_commitments_against_transactions(body.execution_payload.transactions, body.blob_kzg_commitments) -``` - ## Testing *Note*: The function `initialize_beacon_state_from_eth1` is modified for pure Deneb testing only. diff --git a/specs/deneb/validator.md b/specs/deneb/validator.md index 6562c91dd..e52ce9eaa 100644 --- a/specs/deneb/validator.md +++ b/specs/deneb/validator.md @@ -98,8 +98,8 @@ def validate_blobs_and_kzg_commitments(execution_payload: ExecutionPayload, blobs: Sequence[Blob], blob_kzg_commitments: Sequence[KZGCommitment], blob_kzg_proofs: Sequence[KZGProof]) -> None: - # Optionally sanity-check that the KZG commitments match the versioned hashes in the transactions - assert verify_kzg_commitments_against_transactions(execution_payload.transactions, blob_kzg_commitments) + # TODO: can we just remove it? + # assert verify_kzg_commitments_against_transactions(execution_payload.transactions, blob_kzg_commitments) # Optionally sanity-check that the KZG commitments match the blobs (as produced by the execution engine) assert len(blob_kzg_commitments) == len(blobs) == len(blob_kzg_proofs) diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py index 3ec58b31e..05e16f1d1 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py @@ -29,10 +29,10 @@ def run_execution_payload_processing(spec, state, execution_payload, valid=True, called_new_block = False class TestEngine(spec.NoopExecutionEngine): - def notify_new_payload(self, payload) -> bool: + def notify_new_payload(self, new_payload_request) -> bool: nonlocal called_new_block, execution_valid called_new_block = True - assert payload == execution_payload + assert new_payload_request.execution_payload == execution_payload return execution_valid if not valid: diff --git a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py index 2e1a2a369..d31955e82 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py @@ -143,7 +143,7 @@ def process_and_sign_block_without_header_validations(spec, state, block): ) if is_post_bellatrix(spec): if spec.is_execution_enabled(state, block.body): - spec.process_execution_payload(state, block.body.execution_payload, spec.EXECUTION_ENGINE) + spec.process_execution_payload(state, block.body, spec.EXECUTION_ENGINE) # Perform rest of process_block transitions spec.process_randao(state, block.body) From 5a6052f46caa540ccceaaf6fab6a2d4c91354dfd Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Mon, 15 May 2023 16:51:52 -0600 Subject: [PATCH 031/110] Update fork-choice.md Stylistic change to be in line with validations in other specifications --- specs/deneb/fork-choice.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/specs/deneb/fork-choice.md b/specs/deneb/fork-choice.md index 9faa11077..ce9973f13 100644 --- a/specs/deneb/fork-choice.md +++ b/specs/deneb/fork-choice.md @@ -30,9 +30,8 @@ This is the modification of the fork choice accompanying the Deneb upgrade. def validate_blobs(expected_kzg_commitments: Sequence[KZGCommitment], blobs: Sequence[Blob], proofs: Sequence[KZGProof]) -> None: - assert len(expected_kzg_commitments) == len(blobs) - assert len(blobs) == len(proofs) - + assert len(expected_kzg_commitments) == len(blobs) == len(proofs) + assert verify_blob_kzg_proof_batch(blobs, expected_kzg_commitments, proofs) ``` From bb38c56ddd0b6c29d472e6eb96b63a44d7c89abc Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 16 May 2023 20:07:21 +0800 Subject: [PATCH 032/110] Fix `dump_kzg_trusted_setup_files`. Use Fastest BLS lib (#3358) --- tests/core/pyspec/eth2spec/utils/kzg.py | 29 ++++++++++--------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/tests/core/pyspec/eth2spec/utils/kzg.py b/tests/core/pyspec/eth2spec/utils/kzg.py index 4b5454f66..cef1faa66 100644 --- a/tests/core/pyspec/eth2spec/utils/kzg.py +++ b/tests/core/pyspec/eth2spec/utils/kzg.py @@ -10,20 +10,13 @@ from typing import ( from pathlib import Path from eth_utils import encode_hex -from py_ecc.optimized_bls12_381 import ( # noqa: F401 - G1, - G2, - Z1, - Z2, - curve_order as BLS_MODULUS, - add, - multiply, - neg, -) from py_ecc.typing import ( Optimized_Point3D, ) from eth2spec.utils import bls +from eth2spec.utils.bls import ( + BLS_MODULUS, +) PRIMITIVE_ROOT_OF_UNITY = 7 @@ -35,7 +28,7 @@ def generate_setup(generator: Optimized_Point3D, secret: int, length: int) -> Tu """ result = [generator] for _ in range(1, length): - result.append(multiply(result[-1], secret)) + result.append(bls.multiply(result[-1], secret)) return tuple(result) @@ -49,9 +42,9 @@ def fft(vals: Sequence[Optimized_Point3D], modulus: int, domain: int) -> Sequenc R = fft(vals[1::2], modulus, domain[::2]) o = [0] * len(vals) for i, (x, y) in enumerate(zip(L, R)): - y_times_root = multiply(y, domain[i]) - o[i] = add(x, y_times_root) - o[i + len(L)] = add(x, neg(y_times_root)) + y_times_root = bls.multiply(y, domain[i]) + o[i] = bls.add(x, y_times_root) + o[i + len(L)] = bls.add(x, bls.neg(y_times_root)) return o @@ -90,12 +83,14 @@ def get_lagrange(setup: Sequence[Optimized_Point3D]) -> Tuple[bytes]: # TODO: introduce an IFFT function for simplicity fft_output = fft(setup, BLS_MODULUS, domain) inv_length = pow(len(setup), BLS_MODULUS - 2, BLS_MODULUS) - return tuple(bls.G1_to_bytes48(multiply(fft_output[-i], inv_length)) for i in range(len(fft_output))) + return tuple(bls.G1_to_bytes48(bls.multiply(fft_output[-i], inv_length)) for i in range(len(fft_output))) def dump_kzg_trusted_setup_files(secret: int, g1_length: int, g2_length: int, output_dir: str) -> None: - setup_g1 = generate_setup(bls.G1, secret, g1_length) - setup_g2 = generate_setup(bls.G2, secret, g2_length) + bls.use_fastest() + + setup_g1 = generate_setup(bls.G1(), secret, g1_length) + setup_g2 = generate_setup(bls.G2(), secret, g2_length) setup_g1_lagrange = get_lagrange(setup_g1) roots_of_unity = compute_roots_of_unity(g1_length) From 0b2f604f86a037d5b92e82aa79ab8bfc3a55d812 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 16 May 2023 23:58:01 +0800 Subject: [PATCH 033/110] Fix tests --- specs/_features/eip6110/beacon-chain.md | 4 +- specs/deneb/beacon-chain.md | 5 ++- specs/deneb/validator.md | 6 +-- .../test_process_execution_payload.py | 12 ++--- .../test_process_withdrawals.py | 2 +- .../eth2spec/test/deneb/sanity/test_blocks.py | 45 ++++++++++++------- .../test/deneb/unittests/test_offset.py | 23 ---------- .../unittests/validator/test_validator.py | 6 +-- tests/formats/operations/README.md | 2 +- 9 files changed, 48 insertions(+), 57 deletions(-) delete mode 100644 tests/core/pyspec/eth2spec/test/deneb/unittests/test_offset.py diff --git a/specs/_features/eip6110/beacon-chain.md b/specs/_features/eip6110/beacon-chain.md index c9dccbdeb..591d8dc86 100644 --- a/specs/_features/eip6110/beacon-chain.md +++ b/specs/_features/eip6110/beacon-chain.md @@ -249,7 +249,9 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid versioned_hashes = [kzg_commitment_to_versioned_hash(commitment) for commitment in body.blob_kzg_commitments] - assert execution_engine.notify_new_payload(NewPayloadRequest(execution_payload=payload)) + assert execution_engine.notify_new_payload( + NewPayloadRequest(execution_payload=payload, versioned_hashes=versioned_hashes) + ) # Cache execution payload header state.latest_execution_payload_header = ExecutionPayloadHeader( parent_hash=payload.parent_hash, diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 52719dcf6..2c4bb6133 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -217,8 +217,11 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid + # [Modified in Deneb] versioned_hashes = [kzg_commitment_to_versioned_hash(commitment) for commitment in body.blob_kzg_commitments] - assert execution_engine.notify_new_payload(NewPayloadRequest(execution_payload=payload, versioned_hashes=versioned_hashes)) + assert execution_engine.notify_new_payload( + NewPayloadRequest(execution_payload=payload, versioned_hashes=versioned_hashes) + ) # Cache execution payload header state.latest_execution_payload_header = ExecutionPayloadHeader( diff --git a/specs/deneb/validator.md b/specs/deneb/validator.md index e52ce9eaa..03297c59e 100644 --- a/specs/deneb/validator.md +++ b/specs/deneb/validator.md @@ -94,13 +94,9 @@ via `get_payload(payload_id).blobs_bundle`. 2. Validate `blobs` and `blob_kzg_commitments`: ```python -def validate_blobs_and_kzg_commitments(execution_payload: ExecutionPayload, - blobs: Sequence[Blob], +def validate_blobs_and_kzg_commitments(blobs: Sequence[Blob], blob_kzg_commitments: Sequence[KZGCommitment], blob_kzg_proofs: Sequence[KZGProof]) -> None: - # TODO: can we just remove it? - # assert verify_kzg_commitments_against_transactions(execution_payload.transactions, blob_kzg_commitments) - # Optionally sanity-check that the KZG commitments match the blobs (as produced by the execution engine) assert len(blob_kzg_commitments) == len(blobs) == len(blob_kzg_proofs) assert verify_blob_kzg_proof_batch(blobs, blob_kzg_commitments, blob_kzg_proofs) diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py index 05e16f1d1..b83563235 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py @@ -21,10 +21,12 @@ def run_execution_payload_processing(spec, state, execution_payload, valid=True, - post-state ('post'). If ``valid == False``, run expecting ``AssertionError`` """ + # Before Deneb, only `body.execution_payload` matters. `BeaconBlockBody` is just a wrapper. + body = spec.BeaconBlockBody(execution_payload=execution_payload) yield 'pre', state yield 'execution', {'execution_valid': execution_valid} - yield 'execution_payload', execution_payload + yield 'body', body called_new_block = False @@ -32,22 +34,22 @@ def run_execution_payload_processing(spec, state, execution_payload, valid=True, def notify_new_payload(self, new_payload_request) -> bool: nonlocal called_new_block, execution_valid called_new_block = True - assert new_payload_request.execution_payload == execution_payload + assert new_payload_request.execution_payload == body.execution_payload return execution_valid if not valid: - expect_assertion_error(lambda: spec.process_execution_payload(state, execution_payload, TestEngine())) + expect_assertion_error(lambda: spec.process_execution_payload(state, body, TestEngine())) yield 'post', None return - spec.process_execution_payload(state, execution_payload, TestEngine()) + spec.process_execution_payload(state, body, TestEngine()) # Make sure we called the engine assert called_new_block yield 'post', state - assert state.latest_execution_payload_header == get_execution_payload_header(spec, execution_payload) + assert state.latest_execution_payload_header == get_execution_payload_header(spec, body.execution_payload) def run_success_test(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_withdrawals.py b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_withdrawals.py index d7813fb1f..ad71e2c73 100644 --- a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_withdrawals.py +++ b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_withdrawals.py @@ -56,7 +56,7 @@ def verify_post_state(state, spec, expected_withdrawals, def run_withdrawals_processing(spec, state, execution_payload, num_expected_withdrawals=None, fully_withdrawable_indices=None, partial_withdrawals_indices=None, valid=True): """ - Run ``process_execution_payload``, yielding: + Run ``process_withdrawals``, yielding: - pre-state ('pre') - execution payload ('execution_payload') - post-state ('post'). diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index 111565cce..f27a774ab 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -53,7 +53,10 @@ def test_max_blobs(spec, state): @with_deneb_and_later @spec_state_test -def test_invalid_incorrect_blob_tx_type(spec, state): +def test_incorrect_blob_tx_type(spec, state): + """ + The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. + """ yield 'pre', state block = build_empty_block_for_next_slot(spec, state) @@ -62,15 +65,18 @@ def test_invalid_incorrect_blob_tx_type(spec, state): opaque_tx = b'\x04' + opaque_tx[1:] # incorrect tx type block.body.execution_payload.transactions = [opaque_tx] block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + signed_block = state_transition_and_sign_block(spec, state, block) yield 'blocks', [signed_block] - yield 'post', None + yield 'post', state @with_deneb_and_later @spec_state_test -def test_invalid_incorrect_transaction_length_1_byte(spec, state): +def test_incorrect_transaction_length_1_byte(spec, state): + """ + The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. + """ yield 'pre', state block = build_empty_block_for_next_slot(spec, state) @@ -79,15 +85,18 @@ def test_invalid_incorrect_transaction_length_1_byte(spec, state): opaque_tx = opaque_tx + b'\x12' # incorrect tx length block.body.execution_payload.transactions = [opaque_tx] block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + signed_block = state_transition_and_sign_block(spec, state, block) yield 'blocks', [signed_block] - yield 'post', None + yield 'post', state @with_deneb_and_later @spec_state_test -def test_invalid_incorrect_transaction_length_32_bytes(spec, state): +def test_incorrect_transaction_length_32_bytes(spec, state): + """ + The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. + """ yield 'pre', state block = build_empty_block_for_next_slot(spec, state) @@ -96,15 +105,18 @@ def test_invalid_incorrect_transaction_length_32_bytes(spec, state): opaque_tx = opaque_tx + b'\x12' * 32 # incorrect tx length block.body.execution_payload.transactions = [opaque_tx] block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + signed_block = state_transition_and_sign_block(spec, state, block) yield 'blocks', [signed_block] - yield 'post', None + yield 'post', state @with_deneb_and_later @spec_state_test -def test_invalid_incorrect_commitment(spec, state): +def test_incorrect_commitment(spec, state): + """ + The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. + """ yield 'pre', state block = build_empty_block_for_next_slot(spec, state) @@ -113,15 +125,18 @@ def test_invalid_incorrect_commitment(spec, state): block.body.blob_kzg_commitments = blob_kzg_commitments block.body.execution_payload.transactions = [opaque_tx] block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + signed_block = state_transition_and_sign_block(spec, state, block) yield 'blocks', [signed_block] - yield 'post', None + yield 'post', state @with_deneb_and_later @spec_state_test -def test_invalid_incorrect_commitments_order(spec, state): +def test_incorrect_commitments_order(spec, state): + """ + The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. + """ yield 'pre', state block = build_empty_block_for_next_slot(spec, state) @@ -129,10 +144,10 @@ def test_invalid_incorrect_commitments_order(spec, state): block.body.blob_kzg_commitments = [blob_kzg_commitments[1], blob_kzg_commitments[0]] # incorrect order block.body.execution_payload.transactions = [opaque_tx] block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + signed_block = state_transition_and_sign_block(spec, state, block) yield 'blocks', [signed_block] - yield 'post', None + yield 'post', state @with_deneb_and_later diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/test_offset.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/test_offset.py deleted file mode 100644 index 3c3b51ff1..000000000 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/test_offset.py +++ /dev/null @@ -1,23 +0,0 @@ - -from eth2spec.test.helpers.constants import ( - DENEB, - MINIMAL, -) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, -) -from eth2spec.test.context import ( - with_phases, - spec_state_test, - with_presets, -) - - -@with_phases([DENEB]) -@spec_state_test -@with_presets([MINIMAL]) -def test_tx_peek_blob_versioned_hashes(spec, state): - otx, _, commitments, _ = get_sample_opaque_tx(spec) - data_hashes = spec.tx_peek_blob_versioned_hashes(otx) - expected = [spec.kzg_commitment_to_versioned_hash(blob_commitment) for blob_commitment in commitments] - assert expected == data_hashes diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py index 07039ccfe..62b8e9dc1 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py +++ b/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py @@ -31,8 +31,7 @@ def test_validate_blobs_and_kzg_commitments(spec, state): block.body.execution_payload.transactions = [opaque_tx] block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - spec.validate_blobs_and_kzg_commitments(block.body.execution_payload, - blobs, + spec.validate_blobs_and_kzg_commitments(blobs, blob_kzg_commitments, proofs) @@ -52,7 +51,6 @@ def test_validate_blobs_and_kzg_commitments_missing_blob(spec, state): expect_assertion_error( lambda: spec.validate_blobs_and_kzg_commitments( - block.body.execution_payload, blobs[:-1], blob_kzg_commitments, proofs @@ -75,7 +73,6 @@ def test_validate_blobs_and_kzg_commitments_missing_proof(spec, state): expect_assertion_error( lambda: spec.validate_blobs_and_kzg_commitments( - block.body.execution_payload, blobs, blob_kzg_commitments, proofs[:-1] @@ -100,7 +97,6 @@ def test_validate_blobs_and_kzg_commitments_incorrect_blob(spec, state): expect_assertion_error( lambda: spec.validate_blobs_and_kzg_commitments( - block.body.execution_payload, blobs, blob_kzg_commitments, proofs diff --git a/tests/formats/operations/README.md b/tests/formats/operations/README.md index 245ce8565..ba764879b 100644 --- a/tests/formats/operations/README.md +++ b/tests/formats/operations/README.md @@ -42,7 +42,7 @@ Operations: | `proposer_slashing` | `ProposerSlashing` | `proposer_slashing` | `process_proposer_slashing(state, proposer_slashing)` | | `voluntary_exit` | `SignedVoluntaryExit` | `voluntary_exit` | `process_voluntary_exit(state, voluntary_exit)` | | `sync_aggregate` | `SyncAggregate` | `sync_aggregate` | `process_sync_aggregate(state, sync_aggregate)` (new in Altair) | -| `execution_payload` | `ExecutionPayload` | `execution_payload` | `process_execution_payload(state, execution_payload)` (new in Bellatrix) | +| `execution_payload` | `BeaconBlockBody` | **`body`** | `process_execution_payload(state, body)` (new in Bellatrix) | | `withdrawals` | `ExecutionPayload` | `execution_payload` | `process_withdrawals(state, execution_payload)` (new in Capella) | | `bls_to_execution_change` | `SignedBLSToExecutionChange` | `address_change` | `process_bls_to_execution_change(state, address_change)` (new in Capella) | | `deposit_receipt` | `DepositReceipt` | `deposit_receipt` | `process_deposit_receipt(state, deposit_receipt)` (new in EIP6110) | From 32358e8fad3425eb23e406413d60744e0f274d40 Mon Sep 17 00:00:00 2001 From: Justin Traglia <95511699+jtraglia@users.noreply.github.com> Date: Wed, 17 May 2023 11:24:48 -0500 Subject: [PATCH 034/110] Add comment about zero elements in batch verification (#3367) --- specs/deneb/polynomial-commitments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/polynomial-commitments.md b/specs/deneb/polynomial-commitments.md index e23c31fab..39758ac88 100644 --- a/specs/deneb/polynomial-commitments.md +++ b/specs/deneb/polynomial-commitments.md @@ -566,7 +566,7 @@ def verify_blob_kzg_proof_batch(blobs: Sequence[Blob], proofs_bytes: Sequence[Bytes48]) -> bool: """ Given a list of blobs and blob KZG proofs, verify that they correspond to the provided commitments. - + Will return True if there are zero blobs/commitments/proofs. Public method. """ From 1a4db24919dbdf438c5f45b2bb4c51874266d6f6 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Wed, 17 May 2023 11:49:46 -0600 Subject: [PATCH 035/110] Update fork-choice.md Remove duplicate check --- specs/deneb/fork-choice.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/specs/deneb/fork-choice.md b/specs/deneb/fork-choice.md index ce9973f13..1390b3d4e 100644 --- a/specs/deneb/fork-choice.md +++ b/specs/deneb/fork-choice.md @@ -30,8 +30,6 @@ This is the modification of the fork choice accompanying the Deneb upgrade. def validate_blobs(expected_kzg_commitments: Sequence[KZGCommitment], blobs: Sequence[Blob], proofs: Sequence[KZGProof]) -> None: - assert len(expected_kzg_commitments) == len(blobs) == len(proofs) - assert verify_blob_kzg_proof_batch(blobs, expected_kzg_commitments, proofs) ``` From db2e613aac14012234edf94d42cc34b8b5eaa49e Mon Sep 17 00:00:00 2001 From: terencechain Date: Wed, 17 May 2023 11:08:33 -0700 Subject: [PATCH 036/110] Nitpick: blob -> blob sidecar --- specs/deneb/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 3c6f3c88a..507cd4c0c 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -226,7 +226,7 @@ No more than `MAX_REQUEST_BLOB_SIDECARS` may be requested at a time. The response MUST consist of zero or more `response_chunk`. Each _successful_ `response_chunk` MUST contain a single `BlobSidecar` payload. -Clients MUST support requesting sidecars since `minimum_request_epoch`, where `minimum_request_epoch = max(finalized_epoch, current_epoch - MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS, DENEB_FORK_EPOCH)`. If any root in the request content references a block earlier than `minimum_request_epoch`, peers MAY respond with error code `3: ResourceUnavailable` or not include the blob in the response. +Clients MUST support requesting sidecars since `minimum_request_epoch`, where `minimum_request_epoch = max(finalized_epoch, current_epoch - MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS, DENEB_FORK_EPOCH)`. If any root in the request content references a block earlier than `minimum_request_epoch`, peers MAY respond with error code `3: ResourceUnavailable` or not include the blob sidecar in the response. Clients MUST respond with at least one sidecar, if they have it. Clients MAY limit the number of blocks and sidecars in the response. From 44394ad1216b68dbcb5e00c9239e8b74998ee0de Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 18 May 2023 20:58:24 +0800 Subject: [PATCH 037/110] Fix CircleCI Python version --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5958a2fc6..fcdf483d5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -157,7 +157,7 @@ jobs: path: tests/core/pyspec/test-reports test-eip6110: docker: - - image: circleci/python:3.8 + - image: circleci/python:3.9 working_directory: ~/specs-repo steps: - restore_cache: From 2f218f83368562abd83fa236e542033e5ebcbafc Mon Sep 17 00:00:00 2001 From: Suphanat Chunhapanya Date: Fri, 5 May 2023 15:09:43 +0700 Subject: [PATCH 038/110] Specify the number of sidecar subnets Previously the number of subnets is equal to MAX_BLOBS_PER_BLOCK which specifies the number of blobs per block. This commit now makes the number of subnets equal to BLOB_SIDECAR_SUBNET_COUNT instead. The advantage of doing this is that we can change MAX_BLOBS_PER_BLOCK without worrying about the p2p network structure and the number of subnets. --- specs/deneb/p2p-interface.md | 8 ++++---- specs/deneb/validator.md | 21 ++++++++++++++++++++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 507cd4c0c..3136f190d 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -22,7 +22,7 @@ The specification of these changes continues in the same format as the network s - [Topics and messages](#topics-and-messages) - [Global topics](#global-topics) - [`beacon_block`](#beacon_block) - - [`blob_sidecar_{index}`](#blob_sidecar_index) + - [`blob_sidecar_{subnet_id}`](#blob_sidecar_subnet_id) - [Transitioning the gossip](#transitioning-the-gossip) - [The Req/Resp domain](#the-reqresp-domain) - [Messages](#messages) @@ -107,7 +107,7 @@ The new topics along with the type of the `data` field of a gossipsub message ar | Name | Message Type | | - | - | -| `blob_sidecar_{index}` | `SignedBlobSidecar` (new) | +| `blob_sidecar_{subnet_id}` | `SignedBlobSidecar` (new) | ##### Global topics @@ -117,13 +117,13 @@ Deneb introduces new global topics for blob sidecars. The *type* of the payload of this topic changes to the (modified) `SignedBeaconBlock` found in deneb. -###### `blob_sidecar_{index}` +###### `blob_sidecar_{subnet_id}` This topic is used to propagate signed blob sidecars, one for each sidecar index. The number of indices is defined by `MAX_BLOBS_PER_BLOCK`. The following validations MUST pass before forwarding the `signed_blob_sidecar` on the network, assuming the alias `sidecar = signed_blob_sidecar.message`: -- _[REJECT]_ The sidecar is for the correct topic -- i.e. `sidecar.index` matches the topic `{index}`. +- _[REJECT]_ The sidecar is for the correct subnet -- i.e. `compute_subnet_for_blob_sidecar(sidecar.index) == subnet_id`. - _[IGNORE]_ The sidecar is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `sidecar.slot <= current_slot` (a client MAY queue future sidecars for processing at the appropriate slot). - _[IGNORE]_ The sidecar is from a slot greater than the latest finalized slot -- i.e. validate that `sidecar.slot > compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)` - _[IGNORE]_ The sidecar's block's parent (defined by `sidecar.block_parent_root`) has been seen (via both gossip and non-gossip sources) (a client MAY queue sidecars for processing once the parent block is retrieved). diff --git a/specs/deneb/validator.md b/specs/deneb/validator.md index 6562c91dd..4bd9ab4a5 100644 --- a/specs/deneb/validator.md +++ b/specs/deneb/validator.md @@ -10,6 +10,8 @@ - [Introduction](#introduction) - [Prerequisites](#prerequisites) +- [Constants](#constants) + - [Misc](#misc) - [Helpers](#helpers) - [`BlobsBundle`](#blobsbundle) - [Modified `GetPayloadResponse`](#modified-getpayloadresponse) @@ -38,6 +40,14 @@ All behaviors and definitions defined in this document, and documents it extends All terminology, constants, functions, and protocol mechanics defined in the updated [Beacon Chain doc of Deneb](./beacon-chain.md) are requisite for this document and used throughout. Please see related Beacon Chain doc before continuing and use them as a reference throughout. +## Constants + +### Misc + +| Name | Value | Unit | +| - | - | :-: | +| `BLOB_SIDECAR_SUBNET_COUNT` | `4` | The number of blob sidecar subnets used in the gossipsub protocol. | + ## Helpers ### `BlobsBundle` @@ -136,7 +146,7 @@ def get_blob_sidecars(block: BeaconBlock, ``` -Then for each sidecar, `signed_sidecar = SignedBlobSidecar(message=sidecar, signature=signature)` is constructed and published to the `blob_sidecar_{index}` topics according to its index. +Then for each sidecar, `signed_sidecar = SignedBlobSidecar(message=sidecar, signature=signature)` is constructed and published to the associated sidecar topic, the `blob_sidecar_{subnet_id}` pubsub topic. `signature` is obtained from: @@ -149,6 +159,15 @@ def get_blob_sidecar_signature(state: BeaconState, return bls.Sign(privkey, signing_root) ``` +The `subnet_id` for the `signed_sidecar` is calculated with: +- Let `blob_index = signed_sidecar.message.index`. +- Let `subnet_id = compute_subnet_for_blob_sidecar(blob_index)`. + +```python +def compute_subnet_for_blob_sidecar(blob_index: BlobIndex) -> uint64: + return uint64(blob_index % BLOB_SIDECAR_SUBNET_COUNT) +``` + After publishing the peers on the network may request the sidecar through sync-requests, or a local user may be interested. The validator MUST hold on to sidecars for `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` epochs and serve when capable, From 08a13261c209ca2ebb7d099b4cc6240a7b6ad8b7 Mon Sep 17 00:00:00 2001 From: Suphanat Chunhapanya Date: Tue, 9 May 2023 21:03:59 +0700 Subject: [PATCH 039/110] Use SubnetID instead of uint64 --- specs/deneb/validator.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/deneb/validator.md b/specs/deneb/validator.md index 4bd9ab4a5..4303c90b5 100644 --- a/specs/deneb/validator.md +++ b/specs/deneb/validator.md @@ -164,8 +164,8 @@ The `subnet_id` for the `signed_sidecar` is calculated with: - Let `subnet_id = compute_subnet_for_blob_sidecar(blob_index)`. ```python -def compute_subnet_for_blob_sidecar(blob_index: BlobIndex) -> uint64: - return uint64(blob_index % BLOB_SIDECAR_SUBNET_COUNT) +def compute_subnet_for_blob_sidecar(blob_index: BlobIndex) -> SubnetID: + return SubnetID(blob_index % BLOB_SIDECAR_SUBNET_COUNT) ``` After publishing the peers on the network may request the sidecar through sync-requests, or a local user may be interested. From 7097dcf27ae357a57c107f6939d1b96e202665f4 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 18 May 2023 15:27:47 -0600 Subject: [PATCH 040/110] Clarify blob subnets --- specs/deneb/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 3136f190d..d67c144b2 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -119,7 +119,7 @@ The *type* of the payload of this topic changes to the (modified) `SignedBeaconB ###### `blob_sidecar_{subnet_id}` -This topic is used to propagate signed blob sidecars, one for each sidecar index. The number of indices is defined by `MAX_BLOBS_PER_BLOCK`. +This topic is used to propagate signed blob sidecars, where each blob index maps to some `subnet_id`. The following validations MUST pass before forwarding the `signed_blob_sidecar` on the network, assuming the alias `sidecar = signed_blob_sidecar.message`: From a7f8659725ad4372ae7f5be970c235b2e4f6dfbc Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Fri, 19 May 2023 17:17:24 +0300 Subject: [PATCH 041/110] KZG test vector format also uses big endian now --- tests/formats/kzg/compute_kzg_proof.md | 4 ++-- tests/formats/kzg/verify_kzg_proof.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/formats/kzg/compute_kzg_proof.md b/tests/formats/kzg/compute_kzg_proof.md index 0713d50d8..b10105129 100644 --- a/tests/formats/kzg/compute_kzg_proof.md +++ b/tests/formats/kzg/compute_kzg_proof.md @@ -14,8 +14,8 @@ output: Tuple[KZGProof, Bytes32] -- The KZG proof and the value y = f(z) ``` - `blob` here is encoded as a string: hexadecimal encoding of `4096 * 32 = 131072` bytes, prefixed with `0x`. -- `z` here is encoded as a string: hexadecimal encoding of `32` bytes representing a little endian encoded field element, prefixed with `0x`. -- `y` here is encoded as a string: hexadecimal encoding of `32` bytes representing a little endian encoded field element, prefixed with `0x`. +- `z` here is encoded as a string: hexadecimal encoding of `32` bytes representing a big endian encoded field element, prefixed with `0x`. +- `y` here is encoded as a string: hexadecimal encoding of `32` bytes representing a big endian encoded field element, prefixed with `0x`. All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. diff --git a/tests/formats/kzg/verify_kzg_proof.md b/tests/formats/kzg/verify_kzg_proof.md index 143466b66..18e02710c 100644 --- a/tests/formats/kzg/verify_kzg_proof.md +++ b/tests/formats/kzg/verify_kzg_proof.md @@ -15,8 +15,8 @@ input: output: bool -- true (valid proof) or false (incorrect proof) ``` -- `z` here is encoded as a string: hexadecimal encoding of `32` bytes representing a little endian encoded field element, prefixed with `0x`. -- `y` here is encoded as a string: hexadecimal encoding of `32` bytes representing a little endian encoded field element, prefixed with `0x`. +- `z` here is encoded as a string: hexadecimal encoding of `32` bytes representing a big endian encoded field element, prefixed with `0x`. +- `y` here is encoded as a string: hexadecimal encoding of `32` bytes representing a big endian encoded field element, prefixed with `0x`. All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. From f87e1436bfbc1f51a18aedc337c6013cd3b3305e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 16 May 2023 23:36:52 +0800 Subject: [PATCH 042/110] Add networking configs to config files --- configs/mainnet.yaml | 28 ++++++++ configs/minimal.yaml | 28 ++++++++ presets/mainnet/phase0.yaml | 5 ++ presets/minimal/phase0.yaml | 5 ++ setup.py | 1 + specs/phase0/p2p-interface.md | 72 +++++++++++++++++-- specs/phase0/validator.md | 45 ------------ .../unittests/test_config_invariants.py | 8 ++- .../validator/test_validator_unittest.py | 4 +- 9 files changed, 141 insertions(+), 55 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 5ad394c08..1a1537b26 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -94,3 +94,31 @@ PROPOSER_SCORE_BOOST: 40 DEPOSIT_CHAIN_ID: 1 DEPOSIT_NETWORK_ID: 1 DEPOSIT_CONTRACT_ADDRESS: 0x00000000219ab540356cBB839Cbe05303d7705Fa + + +# Networking +# --------------------------------------------------------------- +# `2**20` (= 1048576, 1 MiB) +GOSSIP_MAX_SIZE: 1048576 +# `2**10` (= 1024) +MAX_REQUEST_BLOCKS: 1024 +# `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) +MIN_EPOCHS_FOR_BLOCK_REQUESTS: 33024 +# `2**20` (=1048576, 1 MiB) +MAX_CHUNK_SIZE: 1048576 +# 5s +TTFB_TIMEOUT: 5 +# 10s +RESP_TIMEOUT: 10 +ATTESTATION_PROPAGATION_SLOT_RANGE: 32 +# 500ms +MAXIMUM_GOSSIP_CLOCK_DISPARITY: 500 +MESSAGE_DOMAIN_INVALID_SNAPPY: 0x00000000 +MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000 +# 2 subnets per node +SUBNETS_PER_NODE: 2 +# 2**8 (= 64) +ATTESTATION_SUBNET_COUNT: 64 +ATTESTATION_SUBNET_EXTRA_BITS: 0 +# ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS +ATTESTATION_SUBNET_PREFIX_BITS: 6 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 5895cfc70..32d5682a8 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -95,3 +95,31 @@ DEPOSIT_CHAIN_ID: 5 DEPOSIT_NETWORK_ID: 5 # Configured on a per testnet basis DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123456789012345678901234567890 + + +# Networking +# --------------------------------------------------------------- +# `2**20` (= 1048576, 1 MiB) +GOSSIP_MAX_SIZE: 1048576 +# `2**10` (= 1024) +MAX_REQUEST_BLOCKS: 1024 +# [customized] `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 272) +MIN_EPOCHS_FOR_BLOCK_REQUESTS: 272 +# `2**20` (=1048576, 1 MiB) +MAX_CHUNK_SIZE: 1048576 +# 5s +TTFB_TIMEOUT: 5 +# 10s +RESP_TIMEOUT: 10 +ATTESTATION_PROPAGATION_SLOT_RANGE: 32 +# 500ms +MAXIMUM_GOSSIP_CLOCK_DISPARITY: 500 +MESSAGE_DOMAIN_INVALID_SNAPPY: 0x00000000 +MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000 +# 2 subnets per node +SUBNETS_PER_NODE: 2 +# 2**8 (= 64) +ATTESTATION_SUBNET_COUNT: 64 +ATTESTATION_SUBNET_EXTRA_BITS: 0 +# ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS +ATTESTATION_SUBNET_PREFIX_BITS: 6 diff --git a/presets/mainnet/phase0.yaml b/presets/mainnet/phase0.yaml index 02bc96c8c..c84a93595 100644 --- a/presets/mainnet/phase0.yaml +++ b/presets/mainnet/phase0.yaml @@ -86,3 +86,8 @@ MAX_ATTESTATIONS: 128 MAX_DEPOSITS: 16 # 2**4 (= 16) MAX_VOLUNTARY_EXITS: 16 + +# Networking +# --------------------------------------------------------------- +# 2**8 (= 256) +EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 diff --git a/presets/minimal/phase0.yaml b/presets/minimal/phase0.yaml index e7028f5a4..c55f7de7b 100644 --- a/presets/minimal/phase0.yaml +++ b/presets/minimal/phase0.yaml @@ -86,3 +86,8 @@ MAX_ATTESTATIONS: 128 MAX_DEPOSITS: 16 # 2**4 (= 16) MAX_VOLUNTARY_EXITS: 16 + +# Networking +# --------------------------------------------------------------- +# 2**8 (= 256) +EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 diff --git a/setup.py b/setup.py index f053412b5..4f338f27f 100644 --- a/setup.py +++ b/setup.py @@ -988,6 +988,7 @@ class PySpecCommand(Command): specs/phase0/fork-choice.md specs/phase0/validator.md specs/phase0/weak-subjectivity.md + specs/phase0/p2p-interface.md """ if self.spec_fork in (ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110): self.md_doc_paths += """ diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 2503d906c..3b8a91cc9 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -20,6 +20,9 @@ It consists of four main sections: - [Protocol Negotiation](#protocol-negotiation) - [Multiplexing](#multiplexing) - [Consensus-layer network interaction domains](#consensus-layer-network-interaction-domains) + - [Custom types](#custom-types) + - [Constants](#constants) + - [Preset](#preset) - [Configuration](#configuration) - [MetaData](#metadata) - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) @@ -53,6 +56,7 @@ It consists of four main sections: - [ENR structure](#enr-structure) - [Attestation subnet bitfield](#attestation-subnet-bitfield) - [`eth2` field](#eth2-field) + - [Attestation subnet subcription](#attestation-subnet-subcription) - [Design decision rationale](#design-decision-rationale) - [Transport](#transport-1) - [Why are we defining specific transports?](#why-are-we-defining-specific-transports) @@ -165,22 +169,46 @@ See the [Rationale](#design-decision-rationale) section below for tradeoffs. ## Consensus-layer network interaction domains +### Custom types + +We define the following Python custom types for type hinting and readability: + +| Name | SSZ equivalent | Description | +| - | - | - | +| `NodeID` | `uint256` | node identifier | +| `SubnetID` | `uint64` | subnet identifier | + +### Constants + +| Name | Value | Unit | Duration | +| - | - | :-: | :-: | +| `NODE_ID_BITS` | `256` | The bit length of uint256 is 256 | + +### Preset +| Name | Value | Unit | Duration | +| - | - | :-: | :-: | +| `EPOCHS_PER_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | epochs | ~27 hours | + ### Configuration -This section outlines constants that are used in this spec. +This section outlines configurations that are used in this spec. | Name | Value | Description | |---|---|---| | `GOSSIP_MAX_SIZE` | `2**20` (= 1048576, 1 MiB) | The maximum allowed size of uncompressed gossip messages. | | `MAX_REQUEST_BLOCKS` | `2**10` (= 1024) | Maximum number of blocks in a single request | | `MIN_EPOCHS_FOR_BLOCK_REQUESTS` | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) | The minimum epoch range over which a node must serve blocks | -| `MAX_CHUNK_SIZE` | `2**20` (1048576, 1 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | -| `TTFB_TIMEOUT` | `5s` | The maximum time to wait for first byte of request response (time-to-first-byte). | -| `RESP_TIMEOUT` | `10s` | The maximum time for complete response transfer. | +| `MAX_CHUNK_SIZE` | `2**20` (=1048576, 1 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | +| `TTFB_TIMEOUT` | `5` | The maximum duration in **seconds** to wait for first byte of request response (time-to-first-byte). | +| `RESP_TIMEOUT` | `10` | The maximum duration in **seconds** for complete response transfer. | | `ATTESTATION_PROPAGATION_SLOT_RANGE` | `32` | The maximum number of slots during which an attestation can be propagated. | -| `MAXIMUM_GOSSIP_CLOCK_DISPARITY` | `500ms` | The maximum milliseconds of clock disparity assumed between honest nodes. | -| `MESSAGE_DOMAIN_INVALID_SNAPPY` | `0x00000000` | 4-byte domain for gossip message-id isolation of *invalid* snappy messages | -| `MESSAGE_DOMAIN_VALID_SNAPPY` | `0x01000000` | 4-byte domain for gossip message-id isolation of *valid* snappy messages | +| `MAXIMUM_GOSSIP_CLOCK_DISPARITY` | `500` | The maximum **milliseconds** of clock disparity assumed between honest nodes. | +| `MESSAGE_DOMAIN_INVALID_SNAPPY` | `DomainType('0x00000000')` | 4-byte domain for gossip message-id isolation of *invalid* snappy messages | +| `MESSAGE_DOMAIN_VALID_SNAPPY` | `DomainType('0x01000000')` | 4-byte domain for gossip message-id isolation of *valid* snappy messages | +| `SUBNETS_PER_NODE` | `2` | The number of long-lived subnets a beacon node should be subscribed to. | +| `ATTESTATION_SUBNET_COUNT` | `2**6` (= 64) | The number of attestation subnets used in the gossipsub protocol. | +| `ATTESTATION_SUBNET_EXTRA_BITS` | `0` | The number of extra bits of a NodeId to use when mapping to a subscribed subnet | +| `ATTESTATION_SUBNET_PREFIX_BITS` | `int(ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS)` | | ### MetaData @@ -979,6 +1007,34 @@ Clients MAY connect to peers with the same `fork_digest` but a different `next_f Unless `ENRForkID` is manually updated to matching prior to the earlier `next_fork_epoch` of the two clients, these connecting clients will be unable to successfully interact starting at the earlier `next_fork_epoch`. +### Attestation subnet subcription + +Because Phase 0 does not have shards and thus does not have Shard Committees, there is no stable backbone to the attestation subnets (`beacon_attestation_{subnet_id}`). To provide this stability, each beacon node should: + +* Remain subscribed to `SUBNETS_PER_NODE` for `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs. +* Maintain advertisement of the selected subnets in their node's ENR `attnets` entry by setting the selected `subnet_id` bits to `True` (e.g. `ENR["attnets"][subnet_id] = True`) for all persistent attestation subnets. +* Select these subnets based on their node-id as specified by the following `compute_subscribed_subnets(node_id, epoch)` function. + +```python +def compute_subscribed_subnet(node_id: NodeID, epoch: Epoch, index: int) -> SubnetID: + node_id_prefix = node_id >> (NODE_ID_BITS - ATTESTATION_SUBNET_PREFIX_BITS) + node_offset = node_id % EPOCHS_PER_SUBNET_SUBSCRIPTION + permutation_seed = hash(uint_to_bytes(uint64((epoch + node_offset) // EPOCHS_PER_SUBNET_SUBSCRIPTION))) + permutated_prefix = compute_shuffled_index( + node_id_prefix, + 1 << ATTESTATION_SUBNET_PREFIX_BITS, + permutation_seed, + ) + return SubnetID((permutated_prefix + index) % ATTESTATION_SUBNET_COUNT) +``` + +```python +def compute_subscribed_subnets(node_id: NodeID, epoch: Epoch) -> Sequence[SubnetID]: + return [compute_subscribed_subnet(node_id, epoch, index) for index in range(SUBNETS_PER_NODE)] +``` + +*Note*: When preparing for a hard fork, a node must select and subscribe to subnets of the future fork versioning at least `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs in advance of the fork. These new subnets for the fork are maintained in addition to those for the current fork until the fork occurs. After the fork occurs, let the subnets from the previous fork reach the end of life with no replacements. + ## Design decision rationale ### Transport @@ -1438,6 +1494,8 @@ the epoch range that a new node syncing from a checkpoint must backfill. [weak subjectivity guide](./weak-subjectivity.md). Specifically to find this max epoch range, we use the worst case event of a very large validator size (`>= MIN_PER_EPOCH_CHURN_LIMIT * CHURN_LIMIT_QUOTIENT`). +[0]: # (eth2spec: skip) + ```python MIN_EPOCHS_FOR_BLOCK_REQUESTS = ( MIN_VALIDATOR_WITHDRAWABILITY_DELAY diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index b0a9ac507..602df0973 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -10,7 +10,6 @@ This is an accompanying document to [Phase 0 -- The Beacon Chain](./beacon-chain - [Introduction](#introduction) - [Prerequisites](#prerequisites) -- [Custom types](#custom-types) - [Constants](#constants) - [Misc](#misc) - [Containers](#containers) @@ -64,7 +63,6 @@ This is an accompanying document to [Phase 0 -- The Beacon Chain](./beacon-chain - [Aggregation bits](#aggregation-bits-1) - [Aggregate signature](#aggregate-signature-1) - [Broadcast aggregate](#broadcast-aggregate) -- [Phase 0 attestation subnet stability](#phase-0-attestation-subnet-stability) - [How to avoid slashing](#how-to-avoid-slashing) - [Proposer slashing](#proposer-slashing) - [Attester slashing](#attester-slashing) @@ -83,15 +81,6 @@ A validator is an entity that participates in the consensus of the Ethereum proo All terminology, constants, functions, and protocol mechanics defined in the [Phase 0 -- The Beacon Chain](./beacon-chain.md) and [Phase 0 -- Deposit Contract](./deposit-contract.md) doc are requisite for this document and used throughout. Please see the Phase 0 doc before continuing and use as a reference throughout. -## Custom types - -We define the following Python custom types for type hinting and readability: - -| Name | SSZ equivalent | Description | -| - | - | - | -| `NodeID` | `uint256` | node identifier | -| `SubnetID` | `uint64` | subnet identifier | - ## Constants ### Misc @@ -99,12 +88,6 @@ We define the following Python custom types for type hinting and readability: | Name | Value | Unit | Duration | | - | - | :-: | :-: | | `TARGET_AGGREGATORS_PER_COMMITTEE` | `2**4` (= 16) | validators | -| `EPOCHS_PER_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | epochs | ~27 hours | -| `ATTESTATION_SUBNET_COUNT` | `64` | The number of attestation subnets used in the gossipsub protocol. | -| `ATTESTATION_SUBNET_EXTRA_BITS` | `0` | The number of extra bits of a NodeId to use when mapping to a subscribed subnet | -| `SUBNETS_PER_NODE` | `2` | The number of long-lived subnets a beacon node should be subscribed to. | -| `ATTESTATION_SUBNET_PREFIX_BITS` | `(ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS)` | | -| `NODE_ID_BITS` | `256` | The bit length of uint256 is 256 | ## Containers @@ -619,34 +602,6 @@ def get_aggregate_and_proof_signature(state: BeaconState, return bls.Sign(privkey, signing_root) ``` -## Phase 0 attestation subnet stability - -Because Phase 0 does not have shards and thus does not have Shard Committees, there is no stable backbone to the attestation subnets (`beacon_attestation_{subnet_id}`). To provide this stability, each beacon node should: - -* Remain subscribed to `SUBNETS_PER_NODE` for `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs. -* Maintain advertisement of the selected subnets in their node's ENR `attnets` entry by setting the selected `subnet_id` bits to `True` (e.g. `ENR["attnets"][subnet_id] = True`) for all persistent attestation subnets. -* Select these subnets based on their node-id as specified by the following `compute_subscribed_subnets(node_id, epoch)` function. - -```python -def compute_subscribed_subnet(node_id: NodeID, epoch: Epoch, index: int) -> SubnetID: - node_id_prefix = node_id >> (NODE_ID_BITS - int(ATTESTATION_SUBNET_PREFIX_BITS)) - node_offset = node_id % EPOCHS_PER_SUBNET_SUBSCRIPTION - permutation_seed = hash(uint_to_bytes(uint64((epoch + node_offset) // EPOCHS_PER_SUBNET_SUBSCRIPTION))) - permutated_prefix = compute_shuffled_index( - node_id_prefix, - 1 << int(ATTESTATION_SUBNET_PREFIX_BITS), - permutation_seed, - ) - return SubnetID((permutated_prefix + index) % ATTESTATION_SUBNET_COUNT) -``` - -```python -def compute_subscribed_subnets(node_id: NodeID, epoch: Epoch) -> Sequence[SubnetID]: - return [compute_subscribed_subnet(node_id, epoch, index) for index in range(SUBNETS_PER_NODE)] -``` - -*Note*: When preparing for a hard fork, a validator must select and subscribe to subnets of the future fork versioning at least `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs in advance of the fork. These new subnets for the fork are maintained in addition to those for the current fork until the fork occurs. After the fork occurs, let the subnets from the previous fork reach the end of life with no replacements. - ## How to avoid slashing "Slashing" is the burning of some amount of validator funds and immediate ejection from the active validator set. In Phase 0, there are two ways in which funds can be slashed: [proposer slashing](#proposer-slashing) and [attester slashing](#attester-slashing). Although being slashed has serious repercussions, it is simple enough to avoid being slashed all together by remaining _consistent_ with respect to the messages a validator has previously signed. diff --git a/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py b/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py index b0fd06374..6216ea2f0 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py +++ b/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py @@ -75,7 +75,13 @@ def test_time(spec, state): @with_all_phases @spec_state_test def test_networking(spec, state): - assert spec.SUBNETS_PER_NODE <= spec.ATTESTATION_SUBNET_COUNT + assert spec.config.MIN_EPOCHS_FOR_BLOCK_REQUESTS == ( + spec.config.MIN_VALIDATOR_WITHDRAWABILITY_DELAY + spec.config.CHURN_LIMIT_QUOTIENT // 2 + ) + assert spec.config.ATTESTATION_SUBNET_PREFIX_BITS == ( + spec.ceillog2(spec.config.ATTESTATION_SUBNET_COUNT) + spec.config.ATTESTATION_SUBNET_EXTRA_BITS + ) + assert spec.config.SUBNETS_PER_NODE <= spec.config.ATTESTATION_SUBNET_COUNT node_id_length = spec.NodeID(1).type_byte_length() # in bytes assert node_id_length * 8 == spec.NODE_ID_BITS # in bits diff --git a/tests/core/pyspec/eth2spec/test/phase0/unittests/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/phase0/unittests/validator/test_validator_unittest.py index 177748eac..918ab96e2 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/unittests/validator/test_validator_unittest.py +++ b/tests/core/pyspec/eth2spec/test/phase0/unittests/validator/test_validator_unittest.py @@ -371,7 +371,7 @@ def test_compute_subnet_for_attestation(spec, state): slots_since_epoch_start = slot % spec.SLOTS_PER_EPOCH committees_since_epoch_start = committees_per_slot * slots_since_epoch_start - expected_subnet_id = (committees_since_epoch_start + committee_idx) % spec.ATTESTATION_SUBNET_COUNT + expected_subnet_id = (committees_since_epoch_start + committee_idx) % spec.config.ATTESTATION_SUBNET_COUNT assert actual_subnet_id == expected_subnet_id @@ -488,7 +488,7 @@ def run_compute_subscribed_subnets_arguments(spec, rng=random.Random(1111)): node_id = rng.randint(0, 2**40 - 1) # try VALIDATOR_REGISTRY_LIMIT epoch = rng.randint(0, 2**64 - 1) subnets = spec.compute_subscribed_subnets(node_id, epoch) - assert len(subnets) == spec.SUBNETS_PER_NODE + assert len(subnets) == spec.config.SUBNETS_PER_NODE @with_all_phases From 32036d84a3542ce5b9bd4a2f4e288edfb6267a25 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 19 May 2023 23:32:49 +0800 Subject: [PATCH 043/110] Fix tests --- specs/deneb/beacon-chain.md | 5 ++- .../test_process_voluntary_exit.py | 28 +++++++----- .../test/deneb/block_processing/__init__.py | 0 .../test_process_voluntary_exit.py | 45 +++++++++++++++++++ .../test_process_voluntary_exit.py | 44 ++++++++++++++++++ .../eth2spec/test/helpers/voluntary_exits.py | 20 +++++---- tests/generators/operations/main.py | 6 ++- 7 files changed, 127 insertions(+), 21 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/deneb/block_processing/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py create mode 100644 tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_process_voluntary_exit.py diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index f1fe48e82..7d2f9bdc7 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -30,6 +30,8 @@ - [Block processing](#block-processing) - [Execution payload](#execution-payload) - [`process_execution_payload`](#process_execution_payload) + - [Modified `process_operations`](#modified-process_operations) + - [Modified `process_voluntary_exit`](#modified-process_voluntary_exit) - [Blob KZG commitments](#blob-kzg-commitments) - [Testing](#testing) @@ -281,7 +283,8 @@ def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVolu # Verify the validator has been active long enough assert get_current_epoch(state) >= validator.activation_epoch + SHARD_COMMITTEE_PERIOD # Verify signature - domain = compute_domain(DOMAIN_VOLUNTARY_EXIT, CAPELLA_FORK_VERSION, state.genesis_validators_root) # [Modified in Deneb] + # [Modified in Deneb] + domain = compute_domain(DOMAIN_VOLUNTARY_EXIT, CAPELLA_FORK_VERSION, state.genesis_validators_root) signing_root = compute_signing_root(voluntary_exit, domain) assert bls.Verify(validator.pubkey, signing_root, signed_voluntary_exit.signature) # Initiate exit diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_voluntary_exit.py b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_voluntary_exit.py index f4fcaac68..12b554da5 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_voluntary_exit.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_voluntary_exit.py @@ -2,6 +2,12 @@ from eth2spec.test.context import ( spec_state_test, always_bls, with_bellatrix_and_later, + with_phases, +) +from eth2spec.test.helpers.constants import ( + BELLATRIX, + CAPELLA, + DENEB, ) from eth2spec.test.helpers.keys import pubkey_to_privkey from eth2spec.test.helpers.state import ( @@ -12,8 +18,10 @@ from eth2spec.test.helpers.voluntary_exits import ( sign_voluntary_exit, ) +BELLATRIX_AND_CAPELLA = [BELLATRIX, CAPELLA] -def _run_voluntary_exit_processing_test( + +def run_voluntary_exit_processing_test( spec, state, fork_version, @@ -51,7 +59,7 @@ def _run_voluntary_exit_processing_test( @spec_state_test @always_bls def test_invalid_voluntary_exit_with_current_fork_version_is_before_fork_epoch(spec, state): - yield from _run_voluntary_exit_processing_test( + yield from run_voluntary_exit_processing_test( spec, state, fork_version=state.fork.current_version, @@ -60,11 +68,11 @@ def test_invalid_voluntary_exit_with_current_fork_version_is_before_fork_epoch(s ) -@with_bellatrix_and_later +@with_phases(BELLATRIX_AND_CAPELLA) @spec_state_test @always_bls def test_voluntary_exit_with_current_fork_version_not_is_before_fork_epoch(spec, state): - yield from _run_voluntary_exit_processing_test( + yield from run_voluntary_exit_processing_test( spec, state, fork_version=state.fork.current_version, @@ -72,13 +80,13 @@ def test_voluntary_exit_with_current_fork_version_not_is_before_fork_epoch(spec, ) -@with_bellatrix_and_later +@with_phases([BELLATRIX, CAPELLA, DENEB]) @spec_state_test @always_bls def test_voluntary_exit_with_previous_fork_version_is_before_fork_epoch(spec, state): assert state.fork.previous_version != state.fork.current_version - yield from _run_voluntary_exit_processing_test( + yield from run_voluntary_exit_processing_test( spec, state, fork_version=state.fork.previous_version, @@ -86,13 +94,13 @@ def test_voluntary_exit_with_previous_fork_version_is_before_fork_epoch(spec, st ) -@with_bellatrix_and_later +@with_phases(BELLATRIX_AND_CAPELLA) @spec_state_test @always_bls def test_invalid_voluntary_exit_with_previous_fork_version_not_is_before_fork_epoch(spec, state): assert state.fork.previous_version != state.fork.current_version - yield from _run_voluntary_exit_processing_test( + yield from run_voluntary_exit_processing_test( spec, state, fork_version=state.fork.previous_version, @@ -107,7 +115,7 @@ def test_invalid_voluntary_exit_with_previous_fork_version_not_is_before_fork_ep def test_invalid_voluntary_exit_with_genesis_fork_version_is_before_fork_epoch(spec, state): assert spec.config.GENESIS_FORK_VERSION not in (state.fork.previous_version, state.fork.current_version) - yield from _run_voluntary_exit_processing_test( + yield from run_voluntary_exit_processing_test( spec, state, fork_version=spec.config.GENESIS_FORK_VERSION, @@ -122,7 +130,7 @@ def test_invalid_voluntary_exit_with_genesis_fork_version_is_before_fork_epoch(s def test_invalid_voluntary_exit_with_genesis_fork_version_not_is_before_fork_epoch(spec, state): assert spec.config.GENESIS_FORK_VERSION not in (state.fork.previous_version, state.fork.current_version) - yield from _run_voluntary_exit_processing_test( + yield from run_voluntary_exit_processing_test( spec, state, fork_version=spec.config.GENESIS_FORK_VERSION, diff --git a/tests/core/pyspec/eth2spec/test/deneb/block_processing/__init__.py b/tests/core/pyspec/eth2spec/test/deneb/block_processing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py new file mode 100644 index 000000000..371fcfed4 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py @@ -0,0 +1,45 @@ +from eth2spec.test.context import ( + always_bls, + spec_state_test, + with_phases, + with_deneb_and_later, +) +from eth2spec.test.helpers.constants import ( + DENEB, +) +from eth2spec.test.bellatrix.block_processing.test_process_voluntary_exit import ( + run_voluntary_exit_processing_test, +) + + +@with_deneb_and_later +@spec_state_test +@always_bls +def test_invalid_voluntary_exit_with_current_fork_version_not_is_before_fork_epoch(spec, state): + """ + Since Deneb, the VoluntaryExit domain is fixed to `CAPELLA_FORK_VERSION` + """ + yield from run_voluntary_exit_processing_test( + spec, + state, + fork_version=state.fork.current_version, + is_before_fork_epoch=False, + valid=False, + ) + + +@with_phases([DENEB]) +@spec_state_test +@always_bls +def test_voluntary_exit_with_previous_fork_version_not_is_before_fork_epoch(spec, state): + """ + Since Deneb, the VoluntaryExit domain is fixed to `CAPELLA_FORK_VERSION` + """ + assert state.fork.previous_version != state.fork.current_version + + yield from run_voluntary_exit_processing_test( + spec, + state, + fork_version=state.fork.previous_version, + is_before_fork_epoch=False, + ) diff --git a/tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_process_voluntary_exit.py b/tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_process_voluntary_exit.py new file mode 100644 index 000000000..4128a1181 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_process_voluntary_exit.py @@ -0,0 +1,44 @@ +from eth2spec.test.context import ( + always_bls, + spec_state_test, + with_eip6110_and_later, +) +from eth2spec.test.bellatrix.block_processing.test_process_voluntary_exit import ( + run_voluntary_exit_processing_test, +) + + +@with_eip6110_and_later +@spec_state_test +@always_bls +def test_invalid_voluntary_exit_with_previous_fork_version_not_is_before_fork_epoch(spec, state): + """ + Since Deneb, the VoluntaryExit domain is fixed to `CAPELLA_FORK_VERSION` + """ + assert state.fork.previous_version != state.fork.current_version + + yield from run_voluntary_exit_processing_test( + spec, + state, + fork_version=state.fork.previous_version, + is_before_fork_epoch=False, + valid=False, + ) + + +@with_eip6110_and_later +@spec_state_test +@always_bls +def test_invalid_voluntary_exit_with_previous_fork_version_is_before_fork_epoch(spec, state): + """ + Since Deneb, the VoluntaryExit domain is fixed to `CAPELLA_FORK_VERSION` + """ + assert state.fork.previous_version != state.fork.current_version + + yield from run_voluntary_exit_processing_test( + spec, + state, + fork_version=state.fork.previous_version, + is_before_fork_epoch=True, + valid=False, + ) diff --git a/tests/core/pyspec/eth2spec/test/helpers/voluntary_exits.py b/tests/core/pyspec/eth2spec/test/helpers/voluntary_exits.py index cac101dff..2e8139db6 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/voluntary_exits.py +++ b/tests/core/pyspec/eth2spec/test/helpers/voluntary_exits.py @@ -1,29 +1,31 @@ from random import Random from eth2spec.utils import bls from eth2spec.test.context import expect_assertion_error +from eth2spec.test.helpers.forks import is_post_deneb from eth2spec.test.helpers.keys import privkeys def prepare_signed_exits(spec, state, indices, fork_version=None): - if fork_version is None: - domain = spec.get_domain(state, spec.DOMAIN_VOLUNTARY_EXIT) - else: - domain = spec.compute_domain(spec.DOMAIN_VOLUNTARY_EXIT, fork_version, state.genesis_validators_root) - def create_signed_exit(index): - exit = spec.VoluntaryExit( + voluntary_exit = spec.VoluntaryExit( epoch=spec.get_current_epoch(state), validator_index=index, ) - signing_root = spec.compute_signing_root(exit, domain) - return spec.SignedVoluntaryExit(message=exit, signature=bls.Sign(privkeys[index], signing_root)) + return sign_voluntary_exit(spec, state, voluntary_exit, privkeys[index], fork_version=fork_version) return [create_signed_exit(index) for index in indices] def sign_voluntary_exit(spec, state, voluntary_exit, privkey, fork_version=None): if fork_version is None: - domain = spec.get_domain(state, spec.DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch) + if is_post_deneb(spec): + domain = spec.compute_domain( + spec.DOMAIN_VOLUNTARY_EXIT, + spec.config.CAPELLA_FORK_VERSION, + state.genesis_validators_root, + ) + else: + domain = spec.get_domain(state, spec.DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch) else: domain = spec.compute_domain(spec.DOMAIN_VOLUNTARY_EXIT, fork_version, state.genesis_validators_root) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index fc2217917..2cff8bdf6 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -36,10 +36,14 @@ if __name__ == "__main__": ]} capella_mods = combine_mods(_new_capella_mods, bellatrix_mods) - deneb_mods = capella_mods + _new_deneb_mods = {key: 'eth2spec.test.deneb.block_processing.test_process_' + key for key in [ + 'voluntary_exit', + ]} + deneb_mods = combine_mods(_new_deneb_mods, capella_mods) _new_eip6110_mods = {key: 'eth2spec.test.eip6110.block_processing.test_process_' + key for key in [ 'deposit_receipt', + 'voluntary_exit', ]} eip6110_mods = combine_mods(_new_eip6110_mods, deneb_mods) From 92324ca845dee97b3c817d075352b85fa7a1e9c7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 20 May 2023 01:14:10 +0800 Subject: [PATCH 044/110] Remove `is_merge_transition_complete` check from Capella --- specs/_features/eip4788/validator.md | 19 ++----------------- specs/_features/eip6110/beacon-chain.md | 3 +-- specs/_features/sharding/beacon-chain.md | 3 +-- specs/bellatrix/validator.md | 5 +++-- specs/capella/beacon-chain.md | 6 +++--- specs/capella/validator.md | 19 ++----------------- specs/deneb/beacon-chain.md | 3 +-- .../unittests/validator/test_validator.py | 11 +++++++---- 8 files changed, 20 insertions(+), 49 deletions(-) diff --git a/specs/_features/eip4788/validator.md b/specs/_features/eip4788/validator.md index 421e297ce..f24c97353 100644 --- a/specs/_features/eip4788/validator.md +++ b/specs/_features/eip4788/validator.md @@ -64,27 +64,12 @@ parameter to the `PayloadAttributes`. ```python def prepare_execution_payload(state: BeaconState, - pow_chain: Dict[Hash32, PowBlock], safe_block_hash: Hash32, finalized_block_hash: Hash32, suggested_fee_recipient: ExecutionAddress, execution_engine: ExecutionEngine) -> Optional[PayloadId]: - if not is_merge_transition_complete(state): - is_terminal_block_hash_set = TERMINAL_BLOCK_HASH != Hash32() - is_activation_epoch_reached = get_current_epoch(state) >= TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH - if is_terminal_block_hash_set and not is_activation_epoch_reached: - # Terminal block hash is set but activation epoch is not yet reached, no prepare payload call is needed - return None - - terminal_pow_block = get_terminal_pow_block(pow_chain) - if terminal_pow_block is None: - # Pre-merge, no prepare payload call is needed - return None - # Signify merge via producing on top of the terminal PoW block - parent_hash = terminal_pow_block.block_hash - else: - # Post-merge, normal payload - parent_hash = state.latest_execution_payload_header.block_hash + # Verify consistency of the parent hash with respect to the previous execution payload header + parent_hash = state.latest_execution_payload_header.block_hash # Set the forkchoice head and initiate the payload build process payload_attributes = PayloadAttributes( diff --git a/specs/_features/eip6110/beacon-chain.md b/specs/_features/eip6110/beacon-chain.md index 4d69fb4e0..04ba5d3a7 100644 --- a/specs/_features/eip6110/beacon-chain.md +++ b/specs/_features/eip6110/beacon-chain.md @@ -238,8 +238,7 @@ def process_deposit_receipt(state: BeaconState, deposit_receipt: DepositReceipt) ```python def process_execution_payload(state: BeaconState, payload: ExecutionPayload, execution_engine: ExecutionEngine) -> None: # Verify consistency of the parent hash with respect to the previous execution payload header - if is_merge_transition_complete(state): - assert payload.parent_hash == state.latest_execution_payload_header.block_hash + assert payload.parent_hash == state.latest_execution_payload_header.block_hash # Verify prev_randao assert payload.prev_randao == get_randao_mix(state, get_current_epoch(state)) # Verify timestamp diff --git a/specs/_features/sharding/beacon-chain.md b/specs/_features/sharding/beacon-chain.md index f7de7af65..e67bc09ed 100644 --- a/specs/_features/sharding/beacon-chain.md +++ b/specs/_features/sharding/beacon-chain.md @@ -370,8 +370,7 @@ def process_execution_payload(state: BeaconState, block: BeaconBlock, execution_ payload = block.body.payload_data.value.execution_payload # Verify consistency of the parent hash with respect to the previous execution payload header - if is_merge_transition_complete(state): - assert payload.parent_hash == state.latest_execution_payload_header.block_hash + assert payload.parent_hash == state.latest_execution_payload_header.block_hash # Verify random assert payload.random == get_randao_mix(state, get_current_epoch(state)) # Verify timestamp diff --git a/specs/bellatrix/validator.md b/specs/bellatrix/validator.md index a176d7534..e6fd5443a 100644 --- a/specs/bellatrix/validator.md +++ b/specs/bellatrix/validator.md @@ -118,12 +118,13 @@ To obtain an execution payload, a block proposer building a block on top of a `s ```python def prepare_execution_payload(state: BeaconState, - pow_chain: Dict[Hash32, PowBlock], safe_block_hash: Hash32, finalized_block_hash: Hash32, suggested_fee_recipient: ExecutionAddress, - execution_engine: ExecutionEngine) -> Optional[PayloadId]: + execution_engine: ExecutionEngine, + pow_chain: Optional[Dict[Hash32, PowBlock]]=None) -> Optional[PayloadId]: if not is_merge_transition_complete(state): + assert pow_chain is not None is_terminal_block_hash_set = TERMINAL_BLOCK_HASH != Hash32() is_activation_epoch_reached = get_current_epoch(state) >= TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH if is_terminal_block_hash_set and not is_activation_epoch_reached: diff --git a/specs/capella/beacon-chain.md b/specs/capella/beacon-chain.md index 72260108f..3ea54ab12 100644 --- a/specs/capella/beacon-chain.md +++ b/specs/capella/beacon-chain.md @@ -331,7 +331,7 @@ def process_historical_summaries_update(state: BeaconState) -> None: ```python def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) - # Removed `is_execution_enabled` check + # [Modified in Capella] Removed `is_execution_enabled` check in Capella process_withdrawals(state, block.body.execution_payload) # [New in Capella] process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in Capella] process_randao(state, block.body) @@ -408,9 +408,9 @@ def process_withdrawals(state: BeaconState, payload: ExecutionPayload) -> None: ```python def process_execution_payload(state: BeaconState, payload: ExecutionPayload, execution_engine: ExecutionEngine) -> None: + # [Modified in Capella] Removed `is_merge_transition_complete` check in Capella # Verify consistency of the parent hash with respect to the previous execution payload header - if is_merge_transition_complete(state): - assert payload.parent_hash == state.latest_execution_payload_header.block_hash + assert payload.parent_hash == state.latest_execution_payload_header.block_hash # Verify prev_randao assert payload.prev_randao == get_randao_mix(state, get_current_epoch(state)) # Verify timestamp diff --git a/specs/capella/validator.md b/specs/capella/validator.md index 644ee476f..12c93b6b0 100644 --- a/specs/capella/validator.md +++ b/specs/capella/validator.md @@ -69,27 +69,12 @@ That is, `state` is the `previous_state` processed through any empty slots up to ```python def prepare_execution_payload(state: BeaconState, - pow_chain: Dict[Hash32, PowBlock], safe_block_hash: Hash32, finalized_block_hash: Hash32, suggested_fee_recipient: ExecutionAddress, execution_engine: ExecutionEngine) -> Optional[PayloadId]: - if not is_merge_transition_complete(state): - is_terminal_block_hash_set = TERMINAL_BLOCK_HASH != Hash32() - is_activation_epoch_reached = get_current_epoch(state) >= TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH - if is_terminal_block_hash_set and not is_activation_epoch_reached: - # Terminal block hash is set but activation epoch is not yet reached, no prepare payload call is needed - return None - - terminal_pow_block = get_terminal_pow_block(pow_chain) - if terminal_pow_block is None: - # Pre-merge, no prepare payload call is needed - return None - # Signify merge via producing on top of the terminal PoW block - parent_hash = terminal_pow_block.block_hash - else: - # Post-merge, normal payload - parent_hash = state.latest_execution_payload_header.block_hash + # [Modified in Capella] Removed `is_merge_transition_complete` check in Capella + parent_hash = state.latest_execution_payload_header.block_hash # Set the forkchoice head and initiate the payload build process payload_attributes = PayloadAttributes( diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 6aaa2567f..5834d5a37 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -216,8 +216,7 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: ```python def process_execution_payload(state: BeaconState, payload: ExecutionPayload, execution_engine: ExecutionEngine) -> None: # Verify consistency of the parent hash with respect to the previous execution payload header - if is_merge_transition_complete(state): - assert payload.parent_hash == state.latest_execution_payload_header.block_hash + assert payload.parent_hash == state.latest_execution_payload_header.block_hash # Verify prev_randao assert payload.prev_randao == get_randao_mix(state, get_current_epoch(state)) # Verify timestamp diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/unittests/validator/test_validator.py b/tests/core/pyspec/eth2spec/test/bellatrix/unittests/validator/test_validator.py index 6398aedd8..c55b6e3c3 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/unittests/validator/test_validator.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/unittests/validator/test_validator.py @@ -4,9 +4,12 @@ from typing import Optional from eth2spec.test.helpers.pow_block import ( prepare_random_pow_chain, ) +from eth2spec.test.helpers.constants import ( + BELLATRIX, +) from eth2spec.test.context import ( spec_state_test, - with_bellatrix_and_later, + with_phases, ) @@ -30,7 +33,7 @@ expected_results = [ # it would return the first block (IS_HEAD_PARENT_BLOCK). -@with_bellatrix_and_later +@with_phases([BELLATRIX]) @spec_state_test def test_get_pow_block_at_terminal_total_difficulty(spec, state): for result in expected_results: @@ -90,7 +93,7 @@ prepare_execution_payload_expected_results = [ ] -@with_bellatrix_and_later +@with_phases([BELLATRIX]) @spec_state_test def test_prepare_execution_payload(spec, state): for result in prepare_execution_payload_expected_results: @@ -157,11 +160,11 @@ def test_prepare_execution_payload(spec, state): payload_id = spec.prepare_execution_payload( state=state, - pow_chain=pow_chain.to_dict(), safe_block_hash=safe_block_hash, finalized_block_hash=finalized_block_hash, suggested_fee_recipient=suggested_fee_recipient, execution_engine=TestEngine(), + pow_chain=pow_chain.to_dict(), ) assert payload_id == result_payload_id From 17dac8cab92d2982d7940966d0de40c5e3283828 Mon Sep 17 00:00:00 2001 From: gajinder Date: Tue, 25 Apr 2023 15:36:48 +0530 Subject: [PATCH 045/110] Update MAX_BLOBS_PER_BLOCK to a higher bound --- presets/mainnet/deneb.yaml | 4 ++-- specs/deneb/beacon-chain.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/presets/mainnet/deneb.yaml b/presets/mainnet/deneb.yaml index ebe33f2d1..7cf41bbf4 100644 --- a/presets/mainnet/deneb.yaml +++ b/presets/mainnet/deneb.yaml @@ -4,5 +4,5 @@ # --------------------------------------------------------------- # `uint64(4096)` FIELD_ELEMENTS_PER_BLOB: 4096 -# `uint64(2**2)` (= 4) -MAX_BLOBS_PER_BLOCK: 4 +# `uint64(2**10)` (= 1024) +MAX_BLOBS_PER_BLOCK: 1024 diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 359c7fc95..a123f921b 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -68,7 +68,7 @@ This upgrade adds blobs to the beacon chain as part of Deneb. This is an extensi | Name | Value | | - | - | -| `MAX_BLOBS_PER_BLOCK` | `uint64(2**2)` (= 4) | +| `MAX_BLOBS_PER_BLOCK` | `uint64(2**10)` (= 1024) | ## Configuration From 5e43f43df135a576cbdff51fa3aec6d85a08cf24 Mon Sep 17 00:00:00 2001 From: gajinder Date: Fri, 28 Apr 2023 10:55:35 +0530 Subject: [PATCH 046/110] update limit to 4844 friendly 16 blobs --- presets/mainnet/deneb.yaml | 4 ++-- specs/deneb/beacon-chain.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/presets/mainnet/deneb.yaml b/presets/mainnet/deneb.yaml index 7cf41bbf4..d5dafe0d0 100644 --- a/presets/mainnet/deneb.yaml +++ b/presets/mainnet/deneb.yaml @@ -4,5 +4,5 @@ # --------------------------------------------------------------- # `uint64(4096)` FIELD_ELEMENTS_PER_BLOB: 4096 -# `uint64(2**10)` (= 1024) -MAX_BLOBS_PER_BLOCK: 1024 +# `uint64(2**4)` (= 16) +MAX_BLOBS_PER_BLOCK: 16 diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index a123f921b..fc59e3be8 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -68,7 +68,7 @@ This upgrade adds blobs to the beacon chain as part of Deneb. This is an extensi | Name | Value | | - | - | -| `MAX_BLOBS_PER_BLOCK` | `uint64(2**10)` (= 1024) | +| `MAX_BLOBS_PER_BLOCK` | `uint64(2**4)` (= 16) | ## Configuration From 1aad9b5fa0dce3a0043710bd6595646260459fb2 Mon Sep 17 00:00:00 2001 From: gajinder Date: Sat, 6 May 2023 17:19:28 +0530 Subject: [PATCH 047/110] adding a fixed theoretical limit for commitments in a block --- specs/deneb/beacon-chain.md | 4 ++-- specs/deneb/p2p-interface.md | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index fc59e3be8..934a84013 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -68,7 +68,7 @@ This upgrade adds blobs to the beacon chain as part of Deneb. This is an extensi | Name | Value | | - | - | -| `MAX_BLOBS_PER_BLOCK` | `uint64(2**4)` (= 16) | +| `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**10)` (= 1024) | hardfork independent fixed theoretical limit | ## Configuration @@ -96,7 +96,7 @@ class BeaconBlockBody(Container): # Execution execution_payload: ExecutionPayload # [Modified in Deneb] bls_to_execution_changes: List[SignedBLSToExecutionChange, MAX_BLS_TO_EXECUTION_CHANGES] - blob_kzg_commitments: List[KZGCommitment, MAX_BLOBS_PER_BLOCK] # [New in Deneb] + blob_kzg_commitments: List[KZGCommitment, MAX_BLOB_COMMITMENTS_PER_BLOCK] # [New in Deneb] ``` #### `ExecutionPayload` diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index d67c144b2..514991bb7 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -43,6 +43,7 @@ The specification of these changes continues in the same format as the network s | Name | Value | Description | |------------------------------------------|-----------------------------------|---------------------------------------------------------------------| | `MAX_REQUEST_BLOCKS_DENEB` | `2**7` (= 128) | Maximum number of blocks in a single request | +| `MAX_BLOBS_PER_BLOCK` | `2**4` (= 16) | Maximum number of blobs in a single request, to be always <= `MAX_BLOB_COMMITMENTS_PER_BLOCK` | | `MAX_REQUEST_BLOB_SIDECARS` | `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK` | Maximum number of blob sidecars in a single request | | `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` | `2**12` (= 4096 epochs, ~18 days) | The minimum epoch range over which a node must serve blob sidecars | @@ -119,7 +120,7 @@ The *type* of the payload of this topic changes to the (modified) `SignedBeaconB ###### `blob_sidecar_{subnet_id}` -This topic is used to propagate signed blob sidecars, where each blob index maps to some `subnet_id`. +This topic is used to propagate signed blob sidecars, where each blob index maps to some `subnet_id`. However the actual blobs that will be generated by the EL will be limited by `MAX_DATA_GAS_PER_BLOCK // DATA_GAS_PER_BLOB` and will require an EL hardfork to upgrade the limit. The following validations MUST pass before forwarding the `signed_blob_sidecar` on the network, assuming the alias `sidecar = signed_blob_sidecar.message`: From 9f530a7741e220b2c2611452a98e8ee195e6f02c Mon Sep 17 00:00:00 2001 From: gajinder Date: Sun, 7 May 2023 19:52:59 +0530 Subject: [PATCH 048/110] update max commitments per block limit to blobs per tx limit from eip4844 --- specs/deneb/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 934a84013..971b67f0e 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -68,7 +68,7 @@ This upgrade adds blobs to the beacon chain as part of Deneb. This is an extensi | Name | Value | | - | - | -| `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**10)` (= 1024) | hardfork independent fixed theoretical limit | +| `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**12)` (= 4096) | hardfork independent fixed theoretical limit same as `LIMIT_BLOBS_PER_TX` (see EIP 4844) | ## Configuration From a75292beebdea362b70d65e7ceb3b78ae03d5be4 Mon Sep 17 00:00:00 2001 From: gajinder Date: Thu, 11 May 2023 19:03:43 +0530 Subject: [PATCH 049/110] restore the 4844 max limit to 4 --- presets/mainnet/deneb.yaml | 4 ++-- specs/deneb/p2p-interface.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/presets/mainnet/deneb.yaml b/presets/mainnet/deneb.yaml index d5dafe0d0..ebe33f2d1 100644 --- a/presets/mainnet/deneb.yaml +++ b/presets/mainnet/deneb.yaml @@ -4,5 +4,5 @@ # --------------------------------------------------------------- # `uint64(4096)` FIELD_ELEMENTS_PER_BLOB: 4096 -# `uint64(2**4)` (= 16) -MAX_BLOBS_PER_BLOCK: 16 +# `uint64(2**2)` (= 4) +MAX_BLOBS_PER_BLOCK: 4 diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 514991bb7..4de7f4a83 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -43,7 +43,7 @@ The specification of these changes continues in the same format as the network s | Name | Value | Description | |------------------------------------------|-----------------------------------|---------------------------------------------------------------------| | `MAX_REQUEST_BLOCKS_DENEB` | `2**7` (= 128) | Maximum number of blocks in a single request | -| `MAX_BLOBS_PER_BLOCK` | `2**4` (= 16) | Maximum number of blobs in a single request, to be always <= `MAX_BLOB_COMMITMENTS_PER_BLOCK` | +| `MAX_BLOBS_PER_BLOCK` | `2**2` (= 4) | Maximum number of blobs in a single request, can never exceed `MAX_BLOB_COMMITMENTS_PER_BLOCK` limit | | `MAX_REQUEST_BLOB_SIDECARS` | `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK` | Maximum number of blob sidecars in a single request | | `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` | `2**12` (= 4096 epochs, ~18 days) | The minimum epoch range over which a node must serve blob sidecars | From 8ccc2570d15774b474b2151337304be727750cf1 Mon Sep 17 00:00:00 2001 From: gajinder Date: Sat, 20 May 2023 19:35:22 +0530 Subject: [PATCH 050/110] apply feedback --- specs/deneb/beacon-chain.md | 3 +++ specs/deneb/p2p-interface.md | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 971b67f0e..1926cc8ce 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -40,6 +40,8 @@ This upgrade adds blobs to the beacon chain as part of Deneb. This is an extension of the Capella upgrade. +The blob transactions are packed into the execution payload by the EL/builder with their corresponding blobs being independently transmitted and are limited by `MAX_DATA_GAS_PER_BLOCK // DATA_GAS_PER_BLOB`. However the CL limit is independently defined by `MAX_BLOBS_PER_BLOCK`. + ## Custom types | Name | SSZ equivalent | Description | @@ -69,6 +71,7 @@ This upgrade adds blobs to the beacon chain as part of Deneb. This is an extensi | Name | Value | | - | - | | `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**12)` (= 4096) | hardfork independent fixed theoretical limit same as `LIMIT_BLOBS_PER_TX` (see EIP 4844) | +| `MAX_BLOBS_PER_BLOCK` | `uint64(2**2)` (= 4) | Maximum number of blobs in a single block limited by `MAX_BLOB_COMMITMENTS_PER_BLOCK` | ## Configuration diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 4de7f4a83..d67c144b2 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -43,7 +43,6 @@ The specification of these changes continues in the same format as the network s | Name | Value | Description | |------------------------------------------|-----------------------------------|---------------------------------------------------------------------| | `MAX_REQUEST_BLOCKS_DENEB` | `2**7` (= 128) | Maximum number of blocks in a single request | -| `MAX_BLOBS_PER_BLOCK` | `2**2` (= 4) | Maximum number of blobs in a single request, can never exceed `MAX_BLOB_COMMITMENTS_PER_BLOCK` limit | | `MAX_REQUEST_BLOB_SIDECARS` | `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK` | Maximum number of blob sidecars in a single request | | `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` | `2**12` (= 4096 epochs, ~18 days) | The minimum epoch range over which a node must serve blob sidecars | @@ -120,7 +119,7 @@ The *type* of the payload of this topic changes to the (modified) `SignedBeaconB ###### `blob_sidecar_{subnet_id}` -This topic is used to propagate signed blob sidecars, where each blob index maps to some `subnet_id`. However the actual blobs that will be generated by the EL will be limited by `MAX_DATA_GAS_PER_BLOCK // DATA_GAS_PER_BLOB` and will require an EL hardfork to upgrade the limit. +This topic is used to propagate signed blob sidecars, where each blob index maps to some `subnet_id`. The following validations MUST pass before forwarding the `signed_blob_sidecar` on the network, assuming the alias `sidecar = signed_blob_sidecar.message`: From 4458645f0cffc65133618690ff97ace401f83f27 Mon Sep 17 00:00:00 2001 From: gajinder Date: Mon, 22 May 2023 18:41:40 +0530 Subject: [PATCH 051/110] add check --- specs/deneb/beacon-chain.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 1926cc8ce..b2f6b1763 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -254,6 +254,7 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe ```python def process_blob_kzg_commitments(body: BeaconBlockBody) -> None: + assert len(body.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK assert verify_kzg_commitments_against_transactions(body.execution_payload.transactions, body.blob_kzg_commitments) ``` From feb1968e433daa1a4c540d9226a98c6c4f9b238b Mon Sep 17 00:00:00 2001 From: gajinder Date: Mon, 22 May 2023 18:44:12 +0530 Subject: [PATCH 052/110] add comment --- specs/deneb/beacon-chain.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index b2f6b1763..f492a948c 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -254,7 +254,9 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe ```python def process_blob_kzg_commitments(body: BeaconBlockBody) -> None: + # Verify commitments are under limit assert len(body.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK + # Verify commitments match with corresponding blob transactions assert verify_kzg_commitments_against_transactions(body.execution_payload.transactions, body.blob_kzg_commitments) ``` From 3247bcf8f721cef4b88deac6b3e22c18fbf815eb Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 23 May 2023 15:16:12 +0800 Subject: [PATCH 053/110] PR feedback from @ppopth --- specs/bellatrix/beacon-chain.md | 2 +- specs/deneb/beacon-chain.md | 17 +---------------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index a4c7be16e..9df07d69f 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -330,7 +330,7 @@ The Engine API may be used to implement this and similarly defined functions via ```python def notify_new_payload(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: """ - Return ``True`` if and only if ``execution_payload`` is valid with respect to ``self.execution_state``. + Return ``True`` if and only if ``new_payload_request`` is valid with respect to ``self.execution_state``. """ ... ``` diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 2c4bb6133..20455ab0d 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -30,7 +30,6 @@ - [Modified `NewPayloadRequest`](#modified-newpayloadrequest) - [Engine APIs](#engine-apis) - [Modified `notify_new_payload`](#modified-notify_new_payload) - - [Block processing](#block-processing) - [Execution payload](#execution-payload) - [`process_execution_payload`](#process_execution_payload) - [Testing](#testing) @@ -182,25 +181,11 @@ class NewPayloadRequest(object): ```python def notify_new_payload(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: """ - Return ``True`` if and only if ``execution_payload`` is valid with respect to ``self.execution_state``. + Return ``True`` if and only if ``new_payload_request`` is valid with respect to ``self.execution_state``. """ ... ``` -### Block processing - -```python -def process_block(state: BeaconState, block: BeaconBlock) -> None: - process_block_header(state, block) - if is_execution_enabled(state, block.body): - process_withdrawals(state, block.body.execution_payload) - process_execution_payload(state, block.body, EXECUTION_ENGINE) # [Modified in Deneb] - process_randao(state, block.body) - process_eth1_data(state, block.body) - process_operations(state, block.body) - process_sync_aggregate(state, block.body.sync_aggregate) -``` - #### Execution payload ##### `process_execution_payload` From fc45220a7d1ee5bc75be4c5a0a834512f812f8f1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 23 May 2023 15:38:54 +0800 Subject: [PATCH 054/110] Move old Deneb sanity tests to block_processing (operations) tests --- .../test/deneb/block_processing/__init__.py | 0 .../test_process_execution_payload.py | 175 ++++++++++++++++++ .../eth2spec/test/deneb/sanity/test_blocks.py | 138 -------------- tests/generators/operations/main.py | 5 +- 4 files changed, 179 insertions(+), 139 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/deneb/block_processing/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py diff --git a/tests/core/pyspec/eth2spec/test/deneb/block_processing/__init__.py b/tests/core/pyspec/eth2spec/test/deneb/block_processing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py new file mode 100644 index 000000000..68da478b7 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py @@ -0,0 +1,175 @@ +from random import Random + +from eth2spec.test.helpers.execution_payload import ( + build_empty_execution_payload, + compute_el_block_hash, + get_execution_payload_header, +) +from eth2spec.test.context import ( + spec_state_test, + expect_assertion_error, + with_deneb_and_later +) +from eth2spec.test.helpers.sharding import ( + get_sample_opaque_tx, +) + + +def run_execution_payload_processing(spec, state, execution_payload, blob_kzg_commitments, + valid=True, execution_valid=True): + """ + Run ``process_execution_payload``, yielding: + - pre-state ('pre') + - execution payload ('execution_payload') + - execution details, to mock EVM execution ('execution.yml', a dict with 'execution_valid' key and boolean value) + - post-state ('post'). + If ``valid == False``, run expecting ``AssertionError`` + """ + # Before Deneb, only `body.execution_payload` matters. `BeaconBlockBody` is just a wrapper. + body = spec.BeaconBlockBody( + blob_kzg_commitments=blob_kzg_commitments, + execution_payload=execution_payload + ) + + yield 'pre', state + yield 'execution', {'execution_valid': execution_valid} + yield 'body', body + + called_new_block = False + + class TestEngine(spec.NoopExecutionEngine): + def notify_new_payload(self, new_payload_request) -> bool: + nonlocal called_new_block, execution_valid + called_new_block = True + assert new_payload_request.execution_payload == body.execution_payload + return execution_valid + + if not valid: + expect_assertion_error(lambda: spec.process_execution_payload(state, body, TestEngine())) + yield 'post', None + return + + spec.process_execution_payload(state, body, TestEngine()) + + # Make sure we called the engine + assert called_new_block + + yield 'post', state + + assert state.latest_execution_payload_header == get_execution_payload_header(spec, body.execution_payload) + + +@with_deneb_and_later +@spec_state_test +def test_incorrect_blob_tx_type(spec, state): + """ + The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. + """ + execution_payload = build_empty_execution_payload(spec, state) + + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx = b'\x04' + opaque_tx[1:] # incorrect tx type + + execution_payload.transactions = [opaque_tx] + execution_payload.block_hash = compute_el_block_hash(spec, execution_payload) + + yield from run_execution_payload_processing(spec, state, execution_payload, blob_kzg_commitments) + + +@with_deneb_and_later +@spec_state_test +def test_incorrect_transaction_length_1_byte(spec, state): + """ + The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. + """ + execution_payload = build_empty_execution_payload(spec, state) + + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx = opaque_tx + b'\x12' # incorrect tx length + + execution_payload.transactions = [opaque_tx] + execution_payload.block_hash = compute_el_block_hash(spec, execution_payload) + + yield from run_execution_payload_processing(spec, state, execution_payload, blob_kzg_commitments) + + +@with_deneb_and_later +@spec_state_test +def test_incorrect_transaction_length_32_bytes(spec, state): + """ + The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. + """ + execution_payload = build_empty_execution_payload(spec, state) + + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx = opaque_tx + b'\x12' * 32 # incorrect tx length + + execution_payload.transactions = [opaque_tx] + execution_payload.block_hash = compute_el_block_hash(spec, execution_payload) + + yield from run_execution_payload_processing(spec, state, execution_payload, blob_kzg_commitments) + + +@with_deneb_and_later +@spec_state_test +def test_incorrect_commitment(spec, state): + """ + The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. + """ + execution_payload = build_empty_execution_payload(spec, state) + + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + blob_kzg_commitments[0] = b'\x12' * 48 # incorrect commitment + + execution_payload.transactions = [opaque_tx] + execution_payload.block_hash = compute_el_block_hash(spec, execution_payload) + + yield from run_execution_payload_processing(spec, state, execution_payload, blob_kzg_commitments) + + +@with_deneb_and_later +@spec_state_test +def test_incorrect_commitments_order(spec, state): + """ + The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. + """ + execution_payload = build_empty_execution_payload(spec, state) + + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=2, rng=Random(1111)) + blob_kzg_commitments = [blob_kzg_commitments[1], blob_kzg_commitments[0]] # incorrect order + + execution_payload.transactions = [opaque_tx] + execution_payload.block_hash = compute_el_block_hash(spec, execution_payload) + + yield from run_execution_payload_processing(spec, state, execution_payload, blob_kzg_commitments) + + +@with_deneb_and_later +@spec_state_test +def test_incorrect_block_hash(spec, state): + execution_payload = build_empty_execution_payload(spec, state) + + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + + execution_payload.transactions = [opaque_tx] + execution_payload.block_hash = b'\x12' * 32 # incorrect block hash + + # CL itself doesn't verify EL block hash + yield from run_execution_payload_processing(spec, state, execution_payload, blob_kzg_commitments) + + +@with_deneb_and_later +@spec_state_test +def test_zeroed_commitment(spec, state): + """ + The blob is invalid, but the commitment is in correct form. + """ + execution_payload = build_empty_execution_payload(spec, state) + + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=1, is_valid_blob=False) + assert all(commitment == b'\x00' * 48 for commitment in blob_kzg_commitments) + + execution_payload.transactions = [opaque_tx] + execution_payload.block_hash = compute_el_block_hash(spec, execution_payload) + + yield from run_execution_payload_processing(spec, state, execution_payload, blob_kzg_commitments) diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index f65514f23..36caacd3f 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -1,5 +1,3 @@ -import random - from eth2spec.test.helpers.state import ( state_transition_and_sign_block ) @@ -49,139 +47,3 @@ def test_one_blob(spec, state): @spec_state_test def test_max_blobs(spec, state): yield from run_block_with_blobs(spec, state, blob_count=spec.MAX_BLOBS_PER_BLOCK) - - -@with_deneb_and_later -@spec_state_test -def test_incorrect_blob_tx_type(spec, state): - """ - The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. - """ - yield 'pre', state - - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) - block.body.blob_kzg_commitments = blob_kzg_commitments - opaque_tx = b'\x04' + opaque_tx[1:] # incorrect tx type - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block) - - yield 'blocks', [signed_block] - yield 'post', state - - -@with_deneb_and_later -@spec_state_test -def test_incorrect_transaction_length_1_byte(spec, state): - """ - The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. - """ - yield 'pre', state - - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) - block.body.blob_kzg_commitments = blob_kzg_commitments - opaque_tx = opaque_tx + b'\x12' # incorrect tx length - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block) - - yield 'blocks', [signed_block] - yield 'post', state - - -@with_deneb_and_later -@spec_state_test -def test_incorrect_transaction_length_32_bytes(spec, state): - """ - The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. - """ - yield 'pre', state - - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) - block.body.blob_kzg_commitments = blob_kzg_commitments - opaque_tx = opaque_tx + b'\x12' * 32 # incorrect tx length - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block) - - yield 'blocks', [signed_block] - yield 'post', state - - -@with_deneb_and_later -@spec_state_test -def test_incorrect_commitment(spec, state): - """ - The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. - """ - yield 'pre', state - - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) - blob_kzg_commitments[0] = b'\x12' * 48 # incorrect commitment - block.body.blob_kzg_commitments = blob_kzg_commitments - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block) - - yield 'blocks', [signed_block] - yield 'post', state - - -@with_deneb_and_later -@spec_state_test -def test_incorrect_commitments_order(spec, state): - """ - The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. - """ - yield 'pre', state - - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=2, rng=random.Random(1111)) - block.body.blob_kzg_commitments = [blob_kzg_commitments[1], blob_kzg_commitments[0]] # incorrect order - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block) - - yield 'blocks', [signed_block] - yield 'post', state - - -@with_deneb_and_later -@spec_state_test -def test_incorrect_block_hash(spec, state): - yield 'pre', state - - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) - block.body.blob_kzg_commitments = blob_kzg_commitments - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = b'\x12' * 32 # incorrect block hash - # CL itself doesn't verify EL block hash - signed_block = state_transition_and_sign_block(spec, state, block) - - yield 'blocks', [signed_block] - yield 'post', state - - -@with_deneb_and_later -@spec_state_test -def test_zeroed_commitment(spec, state): - """ - The blob is invalid, but the commitment is in correct form. - """ - yield 'pre', state - - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=1, is_valid_blob=False) - assert all(commitment == b'\x00' * 48 for commitment in blob_kzg_commitments) - block.body.blob_kzg_commitments = blob_kzg_commitments - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block) - - yield 'blocks', [signed_block] - yield 'post', state diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index fc2217917..293bcdea7 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -36,7 +36,10 @@ if __name__ == "__main__": ]} capella_mods = combine_mods(_new_capella_mods, bellatrix_mods) - deneb_mods = capella_mods + _new_deneb_mods = {key: 'eth2spec.test.deneb.block_processing.test_process_' + key for key in [ + 'execution_payload', + ]} + deneb_mods = combine_mods(_new_deneb_mods, capella_mods) _new_eip6110_mods = {key: 'eth2spec.test.eip6110.block_processing.test_process_' + key for key in [ 'deposit_receipt', From bee8fa16e567cec88f68372a3768e4fec476a6b2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 23 May 2023 18:34:00 +0800 Subject: [PATCH 055/110] Add test_invalid_correct_input__execution_invalid --- .../test_process_execution_payload.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py index 68da478b7..d3a04ddfe 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py @@ -173,3 +173,20 @@ def test_zeroed_commitment(spec, state): execution_payload.block_hash = compute_el_block_hash(spec, execution_payload) yield from run_execution_payload_processing(spec, state, execution_payload, blob_kzg_commitments) + + +@with_deneb_and_later +@spec_state_test +def test_invalid_correct_input__execution_invalid(spec, state): + """ + The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. + """ + execution_payload = build_empty_execution_payload(spec, state) + + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + + execution_payload.transactions = [opaque_tx] + execution_payload.block_hash = compute_el_block_hash(spec, execution_payload) + + yield from run_execution_payload_processing(spec, state, execution_payload, blob_kzg_commitments, + valid=False, execution_valid=False) From 73df1935b1dbc09036f793e2759e3c4f794fc04b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 23 May 2023 22:25:43 +0800 Subject: [PATCH 056/110] Use `verify_and_notify_new_payload` approach --- setup.py | 58 ++++++++++++++++++- specs/_features/eip6110/beacon-chain.md | 2 +- specs/bellatrix/beacon-chain.md | 6 +- specs/capella/beacon-chain.md | 2 +- specs/deneb/beacon-chain.md | 40 +++++++++++-- .../test_process_execution_payload.py | 8 ++- .../test_process_execution_payload.py | 2 +- 7 files changed, 104 insertions(+), 14 deletions(-) diff --git a/setup.py b/setup.py index e8bbeefbb..609d2acf3 100644 --- a/setup.py +++ b/setup.py @@ -336,6 +336,10 @@ class SpecBuilder(ABC): """ raise NotImplementedError() + @classmethod + def execution_engine_cls(cls) -> str: + raise NotImplementedError() + @classmethod @abstractmethod def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: @@ -469,6 +473,12 @@ get_attesting_indices = cache_this( ), _get_attesting_indices, lru_size=SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT * 3)''' + + @classmethod + def execution_engine_cls(cls) -> str: + return "" + + @classmethod def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: return {} @@ -573,12 +583,14 @@ def get_execution_state(_execution_state_root: Bytes32) -> ExecutionState: def get_pow_chain_head() -> PowBlock: - pass - + pass""" + @classmethod + def execution_engine_cls(cls) -> str: + return "\n\n" + """ class NoopExecutionEngine(ExecutionEngine): - def notify_new_payload(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: + def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: return True def notify_forkchoice_updated(self: ExecutionEngine, @@ -658,6 +670,39 @@ def retrieve_blobs_and_proofs(beacon_block_root: Root) -> PyUnion[Tuple[Blob, KZ # pylint: disable=unused-argument return ("TEST", "TEST")''' + @classmethod + def execution_engine_cls(cls) -> str: + return "\n\n" + """ +class NoopExecutionEngine(ExecutionEngine): + + def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: + return True + + def notify_forkchoice_updated(self: ExecutionEngine, + head_block_hash: Hash32, + safe_block_hash: Hash32, + finalized_block_hash: Hash32, + payload_attributes: Optional[PayloadAttributes]) -> Optional[PayloadId]: + pass + + def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> GetPayloadResponse: + # pylint: disable=unused-argument + raise NotImplementedError("no default block production") + + def is_valid_block_hash(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: + return True + + def is_valid_versioned_hashes(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: + return True + + def verify_and_notify_new_payload(self: ExecutionEngine, + new_payload_request: NewPayloadRequest) -> bool: + return True + + +EXECUTION_ENGINE = NoopExecutionEngine()""" + + @classmethod def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: constants = { @@ -708,6 +753,12 @@ def objects_to_spec(preset_name: str, ) def format_protocol(protocol_name: str, protocol_def: ProtocolDefinition) -> str: + if "verify_and_notify_new_payload" in protocol_def.functions: + # del protocol_def.functions['verify_and_notify_new_payload'] + protocol_def.functions['verify_and_notify_new_payload'] = """def verify_and_notify_new_payload(self: ExecutionEngine, + new_payload_request: NewPayloadRequest) -> bool: + ...""" + protocol = f"class {protocol_name}(Protocol):" for fn_source in protocol_def.functions.values(): fn_source = fn_source.replace("self: "+protocol_name, "self") @@ -783,6 +834,7 @@ def objects_to_spec(preset_name: str, + ('\n\n\n' + protocols_spec if protocols_spec != '' else '') + '\n\n\n' + functions_spec + '\n\n' + builder.sundry_functions() + + builder.execution_engine_cls() # Since some constants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are # as same as the spec definition. + ('\n\n\n' + ssz_dep_constants_verification if ssz_dep_constants_verification != '' else '') diff --git a/specs/_features/eip6110/beacon-chain.md b/specs/_features/eip6110/beacon-chain.md index 591d8dc86..084f88f14 100644 --- a/specs/_features/eip6110/beacon-chain.md +++ b/specs/_features/eip6110/beacon-chain.md @@ -249,7 +249,7 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid versioned_hashes = [kzg_commitment_to_versioned_hash(commitment) for commitment in body.blob_kzg_commitments] - assert execution_engine.notify_new_payload( + assert execution_engine.verify_and_notify_new_payload( NewPayloadRequest(execution_payload=payload, versioned_hashes=versioned_hashes) ) # Cache execution payload header diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index 9df07d69f..9ae1e5a71 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -328,9 +328,9 @@ The Engine API may be used to implement this and similarly defined functions via #### `notify_new_payload` ```python -def notify_new_payload(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: +def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: """ - Return ``True`` if and only if ``new_payload_request`` is valid with respect to ``self.execution_state``. + Return ``True`` if and only if ``execution_payload`` is valid with respect to ``self.execution_state``. """ ... ``` @@ -366,7 +366,7 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid - assert execution_engine.notify_new_payload(NewPayloadRequest(execution_payload=payload)) + assert execution_engine.notify_new_payload(payload) # Cache execution payload header state.latest_execution_payload_header = ExecutionPayloadHeader( parent_hash=payload.parent_hash, diff --git a/specs/capella/beacon-chain.md b/specs/capella/beacon-chain.md index 1799f4e6e..c8e969d25 100644 --- a/specs/capella/beacon-chain.md +++ b/specs/capella/beacon-chain.md @@ -418,7 +418,7 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid - assert execution_engine.notify_new_payload(NewPayloadRequest(execution_payload=payload)) + assert execution_engine.notify_new_payload(payload) # Cache execution payload header state.latest_execution_payload_header = ExecutionPayloadHeader( parent_hash=payload.parent_hash, diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 20455ab0d..bed64db51 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -29,7 +29,9 @@ - [Request data](#request-data) - [Modified `NewPayloadRequest`](#modified-newpayloadrequest) - [Engine APIs](#engine-apis) - - [Modified `notify_new_payload`](#modified-notify_new_payload) + - [`is_valid_block_hash`](#is_valid_block_hash) + - [`is_valid_versioned_hashes`](#is_valid_versioned_hashes) + - [`verify_and_notify_new_payload`](#verify_and_notify_new_payload) - [Execution payload](#execution-payload) - [`process_execution_payload`](#process_execution_payload) - [Testing](#testing) @@ -176,14 +178,44 @@ class NewPayloadRequest(object): #### Engine APIs -#### Modified `notify_new_payload` +#### `is_valid_block_hash` + +[0]: # (eth2spec: skip) ```python -def notify_new_payload(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: +def is_valid_block_hash(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: + """ + Return ``True`` if and only if ``execution_payload.block_hash`` is computed correctly. + """ + ... +``` + +#### `is_valid_versioned_hashes` + +[0]: # (eth2spec: skip) + +```python +def is_valid_versioned_hashes(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: + """ + Return ``True`` if and only if the version hashes computed by the blob transactions of + ``new_payload_request.execution_payload`` matches ``new_payload_request.version_hashes``. + """ + ... +``` + +#### `verify_and_notify_new_payload` + +```python +def verify_and_notify_new_payload(self: ExecutionEngine, + new_payload_request: NewPayloadRequest) -> bool: """ Return ``True`` if and only if ``new_payload_request`` is valid with respect to ``self.execution_state``. """ + assert self.is_valid_block_hash(new_payload_request.execution_payload) + assert self.is_valid_versioned_hashes(new_payload_request) + assert self.notify_new_payload(new_payload_request.execution_payload) ... + return True ``` #### Execution payload @@ -204,7 +236,7 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi # Verify the execution payload is valid # [Modified in Deneb] versioned_hashes = [kzg_commitment_to_versioned_hash(commitment) for commitment in body.blob_kzg_commitments] - assert execution_engine.notify_new_payload( + assert execution_engine.verify_and_notify_new_payload( NewPayloadRequest(execution_payload=payload, versioned_hashes=versioned_hashes) ) diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py index b83563235..b54afafc3 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py @@ -31,7 +31,13 @@ def run_execution_payload_processing(spec, state, execution_payload, valid=True, called_new_block = False class TestEngine(spec.NoopExecutionEngine): - def notify_new_payload(self, new_payload_request) -> bool: + def notify_new_payload(self, payload) -> bool: + nonlocal called_new_block, execution_valid + called_new_block = True + assert payload == execution_payload + return execution_valid + + def verify_and_notify_new_payload(self, new_payload_request) -> bool: nonlocal called_new_block, execution_valid called_new_block = True assert new_payload_request.execution_payload == body.execution_payload diff --git a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py index d3a04ddfe..df59385ab 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py @@ -38,7 +38,7 @@ def run_execution_payload_processing(spec, state, execution_payload, blob_kzg_co called_new_block = False class TestEngine(spec.NoopExecutionEngine): - def notify_new_payload(self, new_payload_request) -> bool: + def verify_and_notify_new_payload(self, new_payload_request) -> bool: nonlocal called_new_block, execution_valid called_new_block = True assert new_payload_request.execution_payload == body.execution_payload From 7ec5efb10666af3ccff9a3a84cbe46ba6578ac1a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 23 May 2023 23:55:35 +0800 Subject: [PATCH 057/110] Add `### Block processing` back --- specs/deneb/beacon-chain.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 20455ab0d..548d493cc 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -30,6 +30,7 @@ - [Modified `NewPayloadRequest`](#modified-newpayloadrequest) - [Engine APIs](#engine-apis) - [Modified `notify_new_payload`](#modified-notify_new_payload) + - [Block processing](#block-processing) - [Execution payload](#execution-payload) - [`process_execution_payload`](#process_execution_payload) - [Testing](#testing) @@ -186,6 +187,8 @@ def notify_new_payload(self: ExecutionEngine, new_payload_request: NewPayloadReq ... ``` +### Block processing + #### Execution payload ##### `process_execution_payload` From 48618fe866383c07f4fc814b819af72bdff8b4b7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 May 2023 00:08:45 +0800 Subject: [PATCH 058/110] Fix tests --- .../test_process_execution_payload.py | 10 ++++++-- .../test_process_execution_payload.py | 24 +++++++++++++++++++ tests/generators/operations/main.py | 3 ++- 3 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_execution_payload.py diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py index 3ec58b31e..d960ceb0a 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py @@ -8,7 +8,13 @@ from eth2spec.test.helpers.execution_payload import ( build_state_with_incomplete_transition, build_state_with_complete_transition, ) -from eth2spec.test.context import spec_state_test, expect_assertion_error, with_bellatrix_and_later +from eth2spec.test.context import ( + BELLATRIX, + expect_assertion_error, + spec_state_test, + with_bellatrix_and_later, + with_phases, +) from eth2spec.test.helpers.state import next_slot @@ -117,7 +123,7 @@ def test_invalid_bad_execution_regular_payload(spec, state): yield from run_bad_execution_test(spec, state) -@with_bellatrix_and_later +@with_phases([BELLATRIX]) @spec_state_test def test_bad_parent_hash_first_payload(spec, state): state = build_state_with_incomplete_transition(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_execution_payload.py new file mode 100644 index 000000000..1a2c1771a --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_execution_payload.py @@ -0,0 +1,24 @@ +from eth2spec.test.helpers.execution_payload import ( + build_empty_execution_payload, + compute_el_block_hash, + build_state_with_incomplete_transition, +) +from eth2spec.test.context import ( + spec_state_test, + with_capella_and_later, +) +from eth2spec.test.helpers.state import next_slot +from eth2spec.test.bellatrix.block_processing.test_process_execution_payload import run_execution_payload_processing + + +@with_capella_and_later +@spec_state_test +def test_invalid_bad_parent_hash_first_payload(spec, state): + state = build_state_with_incomplete_transition(spec, state) + next_slot(spec, state) + + execution_payload = build_empty_execution_payload(spec, state) + execution_payload.parent_hash = b'\x55' * 32 + execution_payload.block_hash = compute_el_block_hash(spec, execution_payload) + + yield from run_execution_payload_processing(spec, state, execution_payload, valid=False) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index fc2217917..3983931b3 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -30,8 +30,9 @@ if __name__ == "__main__": bellatrix_mods = combine_mods(_new_bellatrix_mods, altair_mods) _new_capella_mods = {key: 'eth2spec.test.capella.block_processing.test_process_' + key for key in [ - 'deposit', 'bls_to_execution_change', + 'deposit', + 'execution_payload', 'withdrawals', ]} capella_mods = combine_mods(_new_capella_mods, bellatrix_mods) From dd5e6f813f82cd20636dfb36950ec281a428d48f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 May 2023 01:58:49 +0800 Subject: [PATCH 059/110] Add `make_function_abstract` to make it more general --- setup.py | 14 +++++++++----- specs/deneb/beacon-chain.md | 4 ---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 609d2acf3..22a05ee32 100644 --- a/setup.py +++ b/setup.py @@ -736,6 +736,11 @@ def is_byte_vector(value: str) -> bool: return value.startswith(('ByteVector')) +def make_function_abstract(protocol_def: ProtocolDefinition, key: str): + function = protocol_def.functions[key].split('"""') + protocol_def.functions[key] = function[0] + "..." + + def objects_to_spec(preset_name: str, spec_object: SpecObject, builder: SpecBuilder, @@ -753,11 +758,10 @@ def objects_to_spec(preset_name: str, ) def format_protocol(protocol_name: str, protocol_def: ProtocolDefinition) -> str: - if "verify_and_notify_new_payload" in protocol_def.functions: - # del protocol_def.functions['verify_and_notify_new_payload'] - protocol_def.functions['verify_and_notify_new_payload'] = """def verify_and_notify_new_payload(self: ExecutionEngine, - new_payload_request: NewPayloadRequest) -> bool: - ...""" + abstract_functions = ["verify_and_notify_new_payload"] + for key in protocol_def.functions.keys(): + if key in abstract_functions: + make_function_abstract(protocol_def, key) protocol = f"class {protocol_name}(Protocol):" for fn_source in protocol_def.functions.values(): diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index bed64db51..615af60d4 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -180,8 +180,6 @@ class NewPayloadRequest(object): #### `is_valid_block_hash` -[0]: # (eth2spec: skip) - ```python def is_valid_block_hash(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: """ @@ -192,8 +190,6 @@ def is_valid_block_hash(self: ExecutionEngine, execution_payload: ExecutionPaylo #### `is_valid_versioned_hashes` -[0]: # (eth2spec: skip) - ```python def is_valid_versioned_hashes(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: """ From 68ce45b3058186ff9e8bdecb19d51327851a4259 Mon Sep 17 00:00:00 2001 From: djrtwo Date: Tue, 23 May 2023 13:52:36 -0600 Subject: [PATCH 060/110] move epoch subscription length to config --- configs/mainnet.yaml | 2 ++ configs/minimal.yaml | 2 ++ presets/mainnet/phase0.yaml | 5 ----- presets/minimal/phase0.yaml | 5 ----- specs/phase0/p2p-interface.md | 7 +------ 5 files changed, 5 insertions(+), 16 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 1a1537b26..baf489739 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -102,6 +102,8 @@ DEPOSIT_CONTRACT_ADDRESS: 0x00000000219ab540356cBB839Cbe05303d7705Fa GOSSIP_MAX_SIZE: 1048576 # `2**10` (= 1024) MAX_REQUEST_BLOCKS: 1024 +# `2**8` (= 256) +EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 # `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) MIN_EPOCHS_FOR_BLOCK_REQUESTS: 33024 # `2**20` (=1048576, 1 MiB) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 32d5682a8..43f1fc83f 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -103,6 +103,8 @@ DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123456789012345678901234567890 GOSSIP_MAX_SIZE: 1048576 # `2**10` (= 1024) MAX_REQUEST_BLOCKS: 1024 +# `2**8` (= 256) +EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 # [customized] `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 272) MIN_EPOCHS_FOR_BLOCK_REQUESTS: 272 # `2**20` (=1048576, 1 MiB) diff --git a/presets/mainnet/phase0.yaml b/presets/mainnet/phase0.yaml index c84a93595..02bc96c8c 100644 --- a/presets/mainnet/phase0.yaml +++ b/presets/mainnet/phase0.yaml @@ -86,8 +86,3 @@ MAX_ATTESTATIONS: 128 MAX_DEPOSITS: 16 # 2**4 (= 16) MAX_VOLUNTARY_EXITS: 16 - -# Networking -# --------------------------------------------------------------- -# 2**8 (= 256) -EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 diff --git a/presets/minimal/phase0.yaml b/presets/minimal/phase0.yaml index c55f7de7b..e7028f5a4 100644 --- a/presets/minimal/phase0.yaml +++ b/presets/minimal/phase0.yaml @@ -86,8 +86,3 @@ MAX_ATTESTATIONS: 128 MAX_DEPOSITS: 16 # 2**4 (= 16) MAX_VOLUNTARY_EXITS: 16 - -# Networking -# --------------------------------------------------------------- -# 2**8 (= 256) -EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 3b8a91cc9..de3e0e529 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -22,7 +22,6 @@ It consists of four main sections: - [Consensus-layer network interaction domains](#consensus-layer-network-interaction-domains) - [Custom types](#custom-types) - [Constants](#constants) - - [Preset](#preset) - [Configuration](#configuration) - [MetaData](#metadata) - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) @@ -184,11 +183,6 @@ We define the following Python custom types for type hinting and readability: | - | - | :-: | :-: | | `NODE_ID_BITS` | `256` | The bit length of uint256 is 256 | -### Preset -| Name | Value | Unit | Duration | -| - | - | :-: | :-: | -| `EPOCHS_PER_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | epochs | ~27 hours | - ### Configuration This section outlines configurations that are used in this spec. @@ -197,6 +191,7 @@ This section outlines configurations that are used in this spec. |---|---|---| | `GOSSIP_MAX_SIZE` | `2**20` (= 1048576, 1 MiB) | The maximum allowed size of uncompressed gossip messages. | | `MAX_REQUEST_BLOCKS` | `2**10` (= 1024) | Maximum number of blocks in a single request | +| `EPOCHS_PER_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | Number of epochs on a subnet subscription (~27 hours) | | `MIN_EPOCHS_FOR_BLOCK_REQUESTS` | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) | The minimum epoch range over which a node must serve blocks | | `MAX_CHUNK_SIZE` | `2**20` (=1048576, 1 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | | `TTFB_TIMEOUT` | `5` | The maximum duration in **seconds** to wait for first byte of request response (time-to-first-byte). | From 1f75acadd979734988b07c018ce6d454ccf8d0e9 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 23 May 2023 14:10:35 -0600 Subject: [PATCH 061/110] Update VERSION.txt to 1.4.0-alpha.0 --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index f0bb29e76..779260991 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -1.3.0 +1.4.0-alpha.0 From 289d8147be38438fac7b196622e2a133b9f2adf9 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 May 2023 11:12:03 +0800 Subject: [PATCH 062/110] Move `verify_and_notify_new_payload` to Bellatrix --- setup.py | 7 ++++ specs/bellatrix/beacon-chain.md | 32 ++++++++++++++++--- specs/capella/beacon-chain.md | 2 +- specs/deneb/beacon-chain.md | 14 ++------ sync/optimistic.md | 4 +-- .../test_process_execution_payload.py | 6 ---- .../eth2spec/test/helpers/optimistic_sync.py | 2 +- 7 files changed, 41 insertions(+), 26 deletions(-) diff --git a/setup.py b/setup.py index 22a05ee32..12a561514 100644 --- a/setup.py +++ b/setup.py @@ -604,6 +604,13 @@ class NoopExecutionEngine(ExecutionEngine): # pylint: disable=unused-argument raise NotImplementedError("no default block production") + def is_valid_block_hash(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: + return True + + def verify_and_notify_new_payload(self: ExecutionEngine, + new_payload_request: NewPayloadRequest) -> bool: + return True + EXECUTION_ENGINE = NoopExecutionEngine()""" diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index 9ae1e5a71..e8eef346d 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -320,13 +320,13 @@ The implementation-dependent `ExecutionEngine` protocol encapsulates the executi * a state object `self.execution_state` of type `ExecutionState` * a notification function `self.notify_new_payload` which may apply changes to the `self.execution_state` -*Note*: `notify_new_payload` is a function accessed through the `EXECUTION_ENGINE` module which instantiates the `ExecutionEngine` protocol. - -The body of this function is implementation dependent. +The body of these functions are implementation dependent. The Engine API may be used to implement this and similarly defined functions via an external execution engine. #### `notify_new_payload` +`notify_new_payload` is a function accessed through the `EXECUTION_ENGINE` module which instantiates the `ExecutionEngine` protocol. + ```python def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: """ @@ -335,6 +335,30 @@ def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayloa ... ``` +#### `is_valid_block_hash` + +```python +def is_valid_block_hash(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: + """ + Return ``True`` if and only if ``execution_payload.block_hash`` is computed correctly. + """ + ... +``` + +#### `verify_and_notify_new_payload` + +```python +def verify_and_notify_new_payload(self: ExecutionEngine, + new_payload_request: NewPayloadRequest) -> bool: + """ + Return ``True`` if and only if ``new_payload_request`` is valid with respect to ``self.execution_state``. + """ + assert self.is_valid_block_hash(new_payload_request.execution_payload) + assert self.notify_new_payload(new_payload_request.execution_payload) + ... + return True +``` + ### Block processing *Note*: The call to the `process_execution_payload` must happen before the call to the `process_randao` as the former depends on the `randao_mix` computed with the reveal of the previous block. @@ -366,7 +390,7 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid - assert execution_engine.notify_new_payload(payload) + assert execution_engine.verify_and_notify_new_payload(NewPayloadRequest(execution_payload=payload)) # Cache execution payload header state.latest_execution_payload_header = ExecutionPayloadHeader( parent_hash=payload.parent_hash, diff --git a/specs/capella/beacon-chain.md b/specs/capella/beacon-chain.md index c8e969d25..af4083017 100644 --- a/specs/capella/beacon-chain.md +++ b/specs/capella/beacon-chain.md @@ -418,7 +418,7 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid - assert execution_engine.notify_new_payload(payload) + assert execution_engine.verify_and_notify_new_payload(NewPayloadRequest(execution_payload=payload)) # Cache execution payload header state.latest_execution_payload_header = ExecutionPayloadHeader( parent_hash=payload.parent_hash, diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 517913a36..bffa6a456 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -179,16 +179,6 @@ class NewPayloadRequest(object): #### Engine APIs -#### `is_valid_block_hash` - -```python -def is_valid_block_hash(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: - """ - Return ``True`` if and only if ``execution_payload.block_hash`` is computed correctly. - """ - ... -``` - #### `is_valid_versioned_hashes` ```python @@ -200,7 +190,7 @@ def is_valid_versioned_hashes(self: ExecutionEngine, new_payload_request: NewPay ... ``` -#### `verify_and_notify_new_payload` +#### Modified `verify_and_notify_new_payload` ```python def verify_and_notify_new_payload(self: ExecutionEngine, @@ -209,7 +199,7 @@ def verify_and_notify_new_payload(self: ExecutionEngine, Return ``True`` if and only if ``new_payload_request`` is valid with respect to ``self.execution_state``. """ assert self.is_valid_block_hash(new_payload_request.execution_payload) - assert self.is_valid_versioned_hashes(new_payload_request) + assert self.is_valid_versioned_hashes(new_payload_request) # [Modified in Deneb] assert self.notify_new_payload(new_payload_request.execution_payload) ... return True diff --git a/sync/optimistic.md b/sync/optimistic.md index 14eb99fb1..1fd1ba46e 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -159,7 +159,7 @@ these conditions.* To optimistically import a block: -- The [`notify_new_payload`](../specs/bellatrix/beacon-chain.md#notify_new_payload) function MUST return `True` if the execution +- The [`verify_and_notify_new_payload`](../specs/bellatrix/beacon-chain.md#verify_and_notify_new_payload) function MUST return `True` if the execution engine returns `NOT_VALIDATED` or `VALID`. An `INVALIDATED` response MUST return `False`. - The [`validate_merge_block`](../specs/bellatrix/fork-choice.md#validate_merge_block) function MUST NOT raise an assertion if both the @@ -172,7 +172,7 @@ In addition to this change in validation, the consensus engine MUST track which blocks returned `NOT_VALIDATED` and which returned `VALID` for subsequent processing. Optimistically imported blocks MUST pass all verifications included in -`process_block` (withstanding the modifications to `notify_new_payload`). +`process_block` (withstanding the modifications to `verify_and_notify_new_payload`). A consensus engine MUST be able to retrospectively (i.e., after import) modify the status of `NOT_VALIDATED` blocks to be either `VALID` or `INVALIDATED` based upon responses diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py index b54afafc3..1aa4a2c37 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py @@ -31,12 +31,6 @@ def run_execution_payload_processing(spec, state, execution_payload, valid=True, called_new_block = False class TestEngine(spec.NoopExecutionEngine): - def notify_new_payload(self, payload) -> bool: - nonlocal called_new_block, execution_valid - called_new_block = True - assert payload == execution_payload - return execution_valid - def verify_and_notify_new_payload(self, new_payload_request) -> bool: nonlocal called_new_block, execution_valid called_new_block = True diff --git a/tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py b/tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py index 816c7a10b..ad23dbebc 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py +++ b/tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py @@ -91,7 +91,7 @@ def add_optimistic_block(spec, mega_store, signed_block, test_steps, Add a block with optimistic sync logic ``valid`` indicates if the given ``signed_block.message.body.execution_payload`` is valid/invalid - from ``notify_new_payload`` method response. + from ``verify_and_notify_new_payload`` method response. """ block = signed_block.message block_root = block.hash_tree_root() From 212a314287568a1414f389569a0bda0a8343354f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 May 2023 11:27:26 +0800 Subject: [PATCH 063/110] Fix ToC and remove `validate_blobs_and_kzg_commitments` --- specs/bellatrix/beacon-chain.md | 2 + specs/deneb/beacon-chain.md | 3 +- specs/deneb/validator.md | 13 +-- .../unittests/validator/test_validator.py | 87 ------------------- 4 files changed, 4 insertions(+), 101 deletions(-) diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index e8eef346d..6cc69d747 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -37,6 +37,8 @@ - [`NewPayloadRequest`](#newpayloadrequest) - [Engine APIs](#engine-apis) - [`notify_new_payload`](#notify_new_payload) + - [`is_valid_block_hash`](#is_valid_block_hash) + - [`verify_and_notify_new_payload`](#verify_and_notify_new_payload) - [Block processing](#block-processing) - [Execution payload](#execution-payload) - [`process_execution_payload`](#process_execution_payload) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 5c31f1fc0..85c1fbdce 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -29,9 +29,8 @@ - [Request data](#request-data) - [Modified `NewPayloadRequest`](#modified-newpayloadrequest) - [Engine APIs](#engine-apis) - - [`is_valid_block_hash`](#is_valid_block_hash) - [`is_valid_versioned_hashes`](#is_valid_versioned_hashes) - - [`verify_and_notify_new_payload`](#verify_and_notify_new_payload) + - [Modified `verify_and_notify_new_payload`](#modified-verify_and_notify_new_payload) - [Block processing](#block-processing) - [Execution payload](#execution-payload) - [`process_execution_payload`](#process_execution_payload) diff --git a/specs/deneb/validator.md b/specs/deneb/validator.md index fa6f1651e..ae5f26673 100644 --- a/specs/deneb/validator.md +++ b/specs/deneb/validator.md @@ -101,18 +101,7 @@ All validator responsibilities remain unchanged other than those noted below. 1. After retrieving the execution payload from the execution engine as specified in Capella, use the `payload_id` to retrieve `blobs`, `blob_kzg_commitments`, and `blob_kzg_proofs` via `get_payload(payload_id).blobs_bundle`. -2. Validate `blobs` and `blob_kzg_commitments`: - -```python -def validate_blobs_and_kzg_commitments(blobs: Sequence[Blob], - blob_kzg_commitments: Sequence[KZGCommitment], - blob_kzg_proofs: Sequence[KZGProof]) -> None: - # Optionally sanity-check that the KZG commitments match the blobs (as produced by the execution engine) - assert len(blob_kzg_commitments) == len(blobs) == len(blob_kzg_proofs) - assert verify_blob_kzg_proof_batch(blobs, blob_kzg_commitments, blob_kzg_proofs) -``` - -3. If valid, set `block.body.blob_kzg_commitments = blob_kzg_commitments`. +2. Set `block.body.blob_kzg_commitments = blob_kzg_commitments`. #### Constructing the `SignedBlobSidecar`s diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py index 62b8e9dc1..876824107 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py +++ b/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py @@ -2,7 +2,6 @@ from eth2spec.test.context import ( always_bls, spec_state_test, with_deneb_and_later, - expect_assertion_error ) from eth2spec.test.helpers.execution_payload import ( compute_el_block_hash, @@ -18,92 +17,6 @@ from eth2spec.test.helpers.keys import ( ) -@with_deneb_and_later -@spec_state_test -def test_validate_blobs_and_kzg_commitments(spec, state): - """ - Test `validate_blobs_and_kzg_commitments` - """ - blob_count = 4 - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, blobs, blob_kzg_commitments, proofs = get_sample_opaque_tx(spec, blob_count=blob_count) - block.body.blob_kzg_commitments = blob_kzg_commitments - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - - spec.validate_blobs_and_kzg_commitments(blobs, - blob_kzg_commitments, - proofs) - - -@with_deneb_and_later -@spec_state_test -def test_validate_blobs_and_kzg_commitments_missing_blob(spec, state): - """ - Test `validate_blobs_and_kzg_commitments` - """ - blob_count = 4 - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, blobs, blob_kzg_commitments, proofs = get_sample_opaque_tx(spec, blob_count=blob_count) - block.body.blob_kzg_commitments = blob_kzg_commitments - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - - expect_assertion_error( - lambda: spec.validate_blobs_and_kzg_commitments( - blobs[:-1], - blob_kzg_commitments, - proofs - ) - ) - - -@with_deneb_and_later -@spec_state_test -def test_validate_blobs_and_kzg_commitments_missing_proof(spec, state): - """ - Test `validate_blobs_and_kzg_commitments` - """ - blob_count = 4 - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, blobs, blob_kzg_commitments, proofs = get_sample_opaque_tx(spec, blob_count=blob_count) - block.body.blob_kzg_commitments = blob_kzg_commitments - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - - expect_assertion_error( - lambda: spec.validate_blobs_and_kzg_commitments( - blobs, - blob_kzg_commitments, - proofs[:-1] - ) - ) - - -@with_deneb_and_later -@spec_state_test -def test_validate_blobs_and_kzg_commitments_incorrect_blob(spec, state): - """ - Test `validate_blobs_and_kzg_commitments` - """ - blob_count = 4 - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, blobs, blob_kzg_commitments, proofs = get_sample_opaque_tx(spec, blob_count=blob_count) - block.body.blob_kzg_commitments = blob_kzg_commitments - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - - blobs[1] = spec.Blob(blobs[1][:13] + bytes([(blobs[1][13] + 1) % 256]) + blobs[1][14:]) - - expect_assertion_error( - lambda: spec.validate_blobs_and_kzg_commitments( - blobs, - blob_kzg_commitments, - proofs - ) - ) - - @with_deneb_and_later @spec_state_test def test_blob_sidecar_signature(spec, state): From 53a9221d1a67799549f55438a2b1ce5e533d3b5b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 May 2023 11:55:22 +0800 Subject: [PATCH 064/110] Fix ToC --- specs/deneb/beacon-chain.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 85c1fbdce..1ab1df611 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -29,8 +29,8 @@ - [Request data](#request-data) - [Modified `NewPayloadRequest`](#modified-newpayloadrequest) - [Engine APIs](#engine-apis) - - [`is_valid_versioned_hashes`](#is_valid_versioned_hashes) - - [Modified `verify_and_notify_new_payload`](#modified-verify_and_notify_new_payload) + - [`is_valid_versioned_hashes`](#is_valid_versioned_hashes) + - [Modified `verify_and_notify_new_payload`](#modified-verify_and_notify_new_payload) - [Block processing](#block-processing) - [Execution payload](#execution-payload) - [`process_execution_payload`](#process_execution_payload) @@ -178,7 +178,7 @@ class NewPayloadRequest(object): #### Engine APIs -#### `is_valid_versioned_hashes` +##### `is_valid_versioned_hashes` ```python def is_valid_versioned_hashes(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: @@ -189,7 +189,7 @@ def is_valid_versioned_hashes(self: ExecutionEngine, new_payload_request: NewPay ... ``` -#### Modified `verify_and_notify_new_payload` +##### Modified `verify_and_notify_new_payload` ```python def verify_and_notify_new_payload(self: ExecutionEngine, From 57378c87d3e33e6cccfd2a3505e48f3d483adcf2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 May 2023 12:13:02 +0800 Subject: [PATCH 065/110] Update fork constants --- .../pyspec/eth2spec/test/helpers/constants.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/constants.py b/tests/core/pyspec/eth2spec/test/helpers/constants.py index 2140c96e4..8fb88d0be 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/constants.py +++ b/tests/core/pyspec/eth2spec/test/helpers/constants.py @@ -4,6 +4,7 @@ from .typing import SpecForkName, PresetBaseName # # SpecForkName # + # Some of the Spec module functionality is exposed here to deal with phase-specific changes. PHASE0 = SpecForkName('phase0') ALTAIR = SpecForkName('altair') @@ -17,15 +18,24 @@ CUSTODY_GAME = SpecForkName('custody_game') DAS = SpecForkName('das') EIP6110 = SpecForkName('eip6110') +# +# SpecFork settings +# + +# The forks that are deployed on Mainnet +MAINNET_FORKS = (PHASE0, ALTAIR, BELLATRIX, CAPELLA) # The forks that pytest can run with. ALL_PHASES = ( # Formal forks - PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, - # Experimental patches + *MAINNET_FORKS, + DENEB, + # Experimental features EIP6110, ) +# The forks that have light client specs +LIGHT_CLIENT_TESTING_FORKS = (*MAINNET_FORKS, DENEB) # The forks that output to the test vectors. -TESTGEN_FORKS = (PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110) +TESTGEN_FORKS = (*MAINNET_FORKS, DENEB, EIP6110) ALL_FORK_UPGRADES = { # pre_fork_name: post_fork_name @@ -46,7 +56,7 @@ AFTER_DENEB_UPGRADES = {key: value for key, value in ALL_FORK_UPGRADES.items() AFTER_DENEB_PRE_POST_FORKS = AFTER_DENEB_UPGRADES.items() # -# Config +# Config and Preset # MAINNET = PresetBaseName('mainnet') MINIMAL = PresetBaseName('minimal') From ec1ee74edb97112fdd0361a6d7c46d8a509684a6 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 May 2023 12:17:07 +0800 Subject: [PATCH 066/110] Fix typo Co-authored-by: Danny Ryan --- specs/deneb/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 1ab1df611..735dcf628 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -198,7 +198,7 @@ def verify_and_notify_new_payload(self: ExecutionEngine, Return ``True`` if and only if ``new_payload_request`` is valid with respect to ``self.execution_state``. """ assert self.is_valid_block_hash(new_payload_request.execution_payload) - assert self.is_valid_versioned_hashes(new_payload_request) # [Modified in Deneb] + assert self.is_valid_versioned_hashes(new_payload_request) # [New in Deneb] assert self.notify_new_payload(new_payload_request.execution_payload) ... return True From 7a827638e66c982a341e2b10b303f772ea9d1f70 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 May 2023 12:55:49 +0800 Subject: [PATCH 067/110] Ensure `verify_and_notify_new_payload` returns bool --- specs/bellatrix/beacon-chain.md | 7 ++++--- specs/deneb/beacon-chain.md | 14 ++++++++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index 6cc69d747..6f67be96a 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -355,9 +355,10 @@ def verify_and_notify_new_payload(self: ExecutionEngine, """ Return ``True`` if and only if ``new_payload_request`` is valid with respect to ``self.execution_state``. """ - assert self.is_valid_block_hash(new_payload_request.execution_payload) - assert self.notify_new_payload(new_payload_request.execution_payload) - ... + if not self.is_valid_block_hash(new_payload_request.execution_payload): + return False + if not self.notify_new_payload(new_payload_request.execution_payload): + return False return True ``` diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 735dcf628..bcbf4d1d0 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -197,10 +197,16 @@ def verify_and_notify_new_payload(self: ExecutionEngine, """ Return ``True`` if and only if ``new_payload_request`` is valid with respect to ``self.execution_state``. """ - assert self.is_valid_block_hash(new_payload_request.execution_payload) - assert self.is_valid_versioned_hashes(new_payload_request) # [New in Deneb] - assert self.notify_new_payload(new_payload_request.execution_payload) - ... + if not self.is_valid_block_hash(new_payload_request.execution_payload): + return False + + # [New in Deneb] + if not self.is_valid_versioned_hashes(new_payload_request): + return False + + if not self.notify_new_payload(new_payload_request.execution_payload): + return False + return True ``` From f0a4281afdeeea8cc41c85405fb35203cf60463a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 May 2023 16:11:32 +0800 Subject: [PATCH 068/110] Add tests. Add validation in the p2p beacon block gossiping --- presets/mainnet/deneb.yaml | 2 ++ presets/minimal/deneb.yaml | 2 ++ specs/_features/eip6110/beacon-chain.md | 2 ++ specs/deneb/p2p-interface.md | 5 +++++ .../test_process_execution_payload.py | 13 ++++++++++++ .../eth2spec/test/deneb/sanity/test_blocks.py | 20 +++++++++++++++---- 6 files changed, 40 insertions(+), 4 deletions(-) diff --git a/presets/mainnet/deneb.yaml b/presets/mainnet/deneb.yaml index ebe33f2d1..10c5025ed 100644 --- a/presets/mainnet/deneb.yaml +++ b/presets/mainnet/deneb.yaml @@ -4,5 +4,7 @@ # --------------------------------------------------------------- # `uint64(4096)` FIELD_ELEMENTS_PER_BLOB: 4096 +# `uint64(2**12)` (= 4096) +MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096 # `uint64(2**2)` (= 4) MAX_BLOBS_PER_BLOCK: 4 diff --git a/presets/minimal/deneb.yaml b/presets/minimal/deneb.yaml index e51b5587d..91120f9da 100644 --- a/presets/minimal/deneb.yaml +++ b/presets/minimal/deneb.yaml @@ -4,5 +4,7 @@ # --------------------------------------------------------------- # [customized] FIELD_ELEMENTS_PER_BLOB: 4 +# [customized] +MAX_BLOB_COMMITMENTS_PER_BLOCK: 16 # `uint64(2**2)` (= 4) MAX_BLOBS_PER_BLOCK: 4 diff --git a/specs/_features/eip6110/beacon-chain.md b/specs/_features/eip6110/beacon-chain.md index 5f6fd99cf..e1384fdae 100644 --- a/specs/_features/eip6110/beacon-chain.md +++ b/specs/_features/eip6110/beacon-chain.md @@ -244,6 +244,8 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi assert payload.prev_randao == get_randao_mix(state, get_current_epoch(state)) # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) + # Verify commitments are under limit + assert len(body.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK # Verify the execution payload is valid versioned_hashes = [kzg_commitment_to_versioned_hash(commitment) for commitment in body.blob_kzg_commitments] assert execution_engine.verify_and_notify_new_payload( diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index d67c144b2..2d6454737 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -117,6 +117,11 @@ Deneb introduces new global topics for blob sidecars. The *type* of the payload of this topic changes to the (modified) `SignedBeaconBlock` found in deneb. +New validation: + +- _[REJECT]_ The length of KZG commitments is less than or equal to the limitation defined in Consensus Layer -- + i.e. validate that `len(body.signed_beacon_block.message.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK` + ###### `blob_sidecar_{subnet_id}` This topic is used to propagate signed blob sidecars, where each blob index maps to some `subnet_id`. diff --git a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py index df59385ab..988070278 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py @@ -190,3 +190,16 @@ def test_invalid_correct_input__execution_invalid(spec, state): yield from run_execution_payload_processing(spec, state, execution_payload, blob_kzg_commitments, valid=False, execution_valid=False) + + +@with_deneb_and_later +@spec_state_test +def test_invalid_exceed_max_blobs_per_block(spec, state): + execution_payload = build_empty_execution_payload(spec, state) + + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=spec.MAX_BLOBS_PER_BLOCK + 1) + + execution_payload.transactions = [opaque_tx] + execution_payload.block_hash = compute_el_block_hash(spec, execution_payload) + + yield from run_execution_payload_processing(spec, state, execution_payload, blob_kzg_commitments, valid=False) diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index 36caacd3f..13a042b9f 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -16,7 +16,7 @@ from eth2spec.test.helpers.sharding import ( ) -def run_block_with_blobs(spec, state, blob_count, excess_data_gas=1): +def run_block_with_blobs(spec, state, blob_count, excess_data_gas=1, valid=True): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) @@ -25,10 +25,16 @@ def run_block_with_blobs(spec, state, blob_count, excess_data_gas=1): block.body.execution_payload.transactions = [opaque_tx] block.body.execution_payload.excess_data_gas = excess_data_gas block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block) + + print(len(block.body.blob_kzg_commitments)) + + if valid: + signed_block = state_transition_and_sign_block(spec, state, block) + else: + signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) yield 'blocks', [signed_block] - yield 'post', state + yield 'post', state if valid else None @with_deneb_and_later @@ -45,5 +51,11 @@ def test_one_blob(spec, state): @with_deneb_and_later @spec_state_test -def test_max_blobs(spec, state): +def test_max_blobs_per_block(spec, state): yield from run_block_with_blobs(spec, state, blob_count=spec.MAX_BLOBS_PER_BLOCK) + + +@with_deneb_and_later +@spec_state_test +def test_invalid_exceed_max_blobs_per_block(spec, state): + yield from run_block_with_blobs(spec, state, blob_count=spec.MAX_BLOBS_PER_BLOCK + 1, valid=False) From e9cc8dcc051ccff90916547027e6088b55083da5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 May 2023 16:32:39 +0800 Subject: [PATCH 069/110] PR feedback of Danny + verify `MAX_BLOBS_PER_BLOCK` size in unittest --- specs/deneb/beacon-chain.md | 2 +- .../test/deneb/unittests/test_config_invariants.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index c8b79f919..cd02d2494 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -73,7 +73,7 @@ The blob transactions are packed into the execution payload by the EL/builder wi | Name | Value | | - | - | -| `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**12)` (= 4096) | hardfork independent fixed theoretical limit same as `LIMIT_BLOBS_PER_TX` (see EIP 4844) | +| `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**12)` (= 4096) | hardfork independent fixed theoretical limit same as `LIMIT_BLOBS_PER_TX` (see EIP 4844) | | `MAX_BLOBS_PER_BLOCK` | `uint64(2**2)` (= 4) | Maximum number of blobs in a single block limited by `MAX_BLOB_COMMITMENTS_PER_BLOCK` | ## Configuration diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py new file mode 100644 index 000000000..13a54225e --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py @@ -0,0 +1,12 @@ +from eth2spec.test.context import ( + single_phase, + spec_test, + with_deneb_and_later, +) + + +@with_deneb_and_later +@spec_test +@single_phase +def test_length(spec): + assert spec.MAX_BLOBS_PER_BLOCK < spec.MAX_BLOB_COMMITMENTS_PER_BLOCK From 9bc27bad306c5b5be7c6e3381c8a01e10232bd91 Mon Sep 17 00:00:00 2001 From: djrtwo Date: Wed, 24 May 2023 05:41:42 -0600 Subject: [PATCH 070/110] a couple of minor cleanups --- specs/deneb/beacon-chain.md | 2 +- tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index cd02d2494..e1690c062 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -234,7 +234,7 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi assert len(body.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK # Verify the execution payload is valid - # [Modified in Deneb] Pass `versioned_hashes` to Engine API + # [Modified in Deneb] Pass `versioned_hashes` to Execution Engine versioned_hashes = [kzg_commitment_to_versioned_hash(commitment) for commitment in body.blob_kzg_commitments] assert execution_engine.verify_and_notify_new_payload( NewPayloadRequest(execution_payload=payload, versioned_hashes=versioned_hashes) diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index 13a042b9f..ba0797a04 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -26,8 +26,6 @@ def run_block_with_blobs(spec, state, blob_count, excess_data_gas=1, valid=True) block.body.execution_payload.excess_data_gas = excess_data_gas block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - print(len(block.body.blob_kzg_commitments)) - if valid: signed_block = state_transition_and_sign_block(spec, state, block) else: From 8d32e79b8d40993eeb6fa2cb67183682518bd3aa Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 May 2023 20:54:22 +0800 Subject: [PATCH 071/110] Fix multiprocessing --- tests/generators/shuffling/main.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/tests/generators/shuffling/main.py b/tests/generators/shuffling/main.py index b85fd42a2..88c586b6c 100644 --- a/tests/generators/shuffling/main.py +++ b/tests/generators/shuffling/main.py @@ -1,5 +1,5 @@ -from eth_utils import to_tuple from typing import Iterable +import random from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.test.helpers.typing import PresetBaseName @@ -8,6 +8,16 @@ from eth2spec.phase0 import mainnet as spec_mainnet, minimal as spec_minimal from eth2spec.test.helpers.constants import PHASE0, MINIMAL, MAINNET +def generate_random_bytes(rng=random.Random(5566)): + random_bytes = bytes(rng.randint(0, 255) for _ in range(32)) + return random_bytes + + +# NOTE: somehow the random.Random generated seeds do not have pickle issue. +rng = random.Random(1234) +seeds = [generate_random_bytes(rng) for i in range(30)] + + def shuffling_case_fn(spec, seed, count): yield 'mapping', 'data', { 'seed': '0x' + seed.hex(), @@ -20,9 +30,8 @@ def shuffling_case(spec, seed, count): return f'shuffle_0x{seed.hex()}_{count}', lambda: shuffling_case_fn(spec, seed, count) -@to_tuple def shuffling_test_cases(spec): - for seed in [spec.hash(seed_init_value.to_bytes(length=4, byteorder='little')) for seed_init_value in range(30)]: + for seed in seeds: for count in [0, 1, 2, 3, 5, 10, 33, 100, 1000, 9999]: yield shuffling_case(spec, seed, count) @@ -47,11 +56,13 @@ def create_provider(preset_name: PresetBaseName) -> gen_typing.TestProvider: handler_name='core', suite_name='shuffle', case_name=case_name, - case_fn=case_fn + case_fn=case_fn, ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) if __name__ == "__main__": - gen_runner.run_generator("shuffling", [create_provider(MINIMAL), create_provider(MAINNET)]) + gen_runner.run_generator("shuffling", [ + create_provider(MINIMAL), create_provider(MAINNET)] + ) From a10affa24580602670b319ffc5e349664b53b92e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 May 2023 21:15:01 +0800 Subject: [PATCH 072/110] Fix preset table --- specs/deneb/beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index e1690c062..175a80edf 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -71,8 +71,8 @@ The blob transactions are packed into the execution payload by the EL/builder wi ### Execution -| Name | Value | -| - | - | +| Name | Value | Description | +| - | - | - | | `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**12)` (= 4096) | hardfork independent fixed theoretical limit same as `LIMIT_BLOBS_PER_TX` (see EIP 4844) | | `MAX_BLOBS_PER_BLOCK` | `uint64(2**2)` (= 4) | Maximum number of blobs in a single block limited by `MAX_BLOB_COMMITMENTS_PER_BLOCK` | From b2771c67c31f5635b4d87eefe7f3a03aeeddeb7c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 25 May 2023 00:16:19 +0800 Subject: [PATCH 073/110] Fallback to MODE_SINGLE_PROCESS --- tests/core/pyspec/eth2spec/gen_helpers/gen_base/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/settings.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/settings.py index b8c3e591f..2f462dddb 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/settings.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/settings.py @@ -5,7 +5,7 @@ import multiprocessing MODE_SINGLE_PROCESS = 'MODE_SINGLE_PROCESS' MODE_MULTIPROCESSING = 'MODE_MULTIPROCESSING' # Test generator mode -GENERATOR_MODE = MODE_MULTIPROCESSING +GENERATOR_MODE = MODE_SINGLE_PROCESS # Number of subprocesses when using MODE_MULTIPROCESSING NUM_PROCESS = multiprocessing.cpu_count() // 2 - 1 From 89e45c6454a494fdb9750db56f2a64fe8167a71a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 25 May 2023 02:04:17 +0800 Subject: [PATCH 074/110] do not use setup.py to install mkdocs modules --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index eab3bba17..209bb4a80 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -20,5 +20,5 @@ jobs: with: key: ${{ github.ref }} path: .cache - - run: pip install -e .[docs] + - run: pip install mkdocs==1.4.2 mkdocs-material==9.1.5 mdx-truly-sane-lists==1.3 mkdocs-awesome-pages-plugin==2.8.0 - run: mkdocs gh-deploy --force From 30da14e9b4931e95ef08e41a9e8441a23f326e70 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 May 2023 16:39:58 +0800 Subject: [PATCH 075/110] Create `LIGHT_CLIENT_TESTING_FORKS` to limit the light client tests --- setup.py | 4 - specs/_features/eip6110/light-client/fork.md | 112 ------------------ .../eip6110/light-client/full-node.md | 77 ------------ .../eip6110/light-client/p2p-interface.md | 111 ----------------- .../eip6110/light-client/sync-protocol.md | 89 -------------- .../light_client/test_single_merkle_proof.py | 8 +- .../test/altair/light_client/test_sync.py | 19 +-- .../light_client/test_update_ranking.py | 4 +- .../light_client/test_sync_protocol.py | 10 +- tests/core/pyspec/eth2spec/test/context.py | 16 +-- .../pyspec/eth2spec/test/helpers/constants.py | 3 +- 11 files changed, 26 insertions(+), 427 deletions(-) delete mode 100644 specs/_features/eip6110/light-client/fork.md delete mode 100644 specs/_features/eip6110/light-client/full-node.md delete mode 100644 specs/_features/eip6110/light-client/p2p-interface.md delete mode 100644 specs/_features/eip6110/light-client/sync-protocol.md diff --git a/setup.py b/setup.py index 75e396ee2..f3943c9ac 100644 --- a/setup.py +++ b/setup.py @@ -1038,10 +1038,6 @@ class PySpecCommand(Command): """ if self.spec_fork == EIP6110: self.md_doc_paths += """ - specs/_features/eip6110/light-client/fork.md - specs/_features/eip6110/light-client/full-node.md - specs/_features/eip6110/light-client/p2p-interface.md - specs/_features/eip6110/light-client/sync-protocol.md specs/_features/eip6110/beacon-chain.md specs/_features/eip6110/fork.md """ diff --git a/specs/_features/eip6110/light-client/fork.md b/specs/_features/eip6110/light-client/fork.md deleted file mode 100644 index 34f0fef8c..000000000 --- a/specs/_features/eip6110/light-client/fork.md +++ /dev/null @@ -1,112 +0,0 @@ -# EIP-6110 Light Client -- Fork Logic - -## Table of contents - - - - - -- [Introduction](#introduction) - - [Upgrading light client data](#upgrading-light-client-data) - - [Upgrading the store](#upgrading-the-store) - - - - -## Introduction - -This document describes how to upgrade existing light client objects based on the [Deneb specification](../../deneb/light-client/sync-protocol.md) to eip6110. This is necessary when processing pre-eip6110 data with a post-eip6110 `LightClientStore`. Note that the data being exchanged over the network protocols uses the original format. - -### Upgrading light client data - -A eip6110 `LightClientStore` can still process earlier light client data. In order to do so, that pre-eip6110 data needs to be locally upgraded to eip6110 before processing. - -```python -def upgrade_lc_header_to_eip6110(pre: deneb.LightClientHeader) -> LightClientHeader: - return LightClientHeader( - beacon=pre.beacon, - execution=ExecutionPayloadHeader( - parent_hash=pre.execution.parent_hash, - fee_recipient=pre.execution.fee_recipient, - state_root=pre.execution.state_root, - receipts_root=pre.execution.receipts_root, - logs_bloom=pre.execution.logs_bloom, - prev_randao=pre.execution.prev_randao, - block_number=pre.execution.block_number, - gas_limit=pre.execution.gas_limit, - gas_used=pre.execution.gas_used, - timestamp=pre.execution.timestamp, - extra_data=pre.execution.extra_data, - base_fee_per_gas=pre.execution.base_fee_per_gas, - block_hash=pre.execution.block_hash, - transactions_root=pre.execution.transactions_root, - withdrawals_root=pre.execution.withdrawals_root, - excess_data_gas=pre.execution.excess_data_gas, - deposit_receipts_root=Root(), # [New in EIP6110] - ), - execution_branch=pre.execution_branch, - ) -``` - -```python -def upgrade_lc_bootstrap_to_eip6110(pre: deneb.LightClientBootstrap) -> LightClientBootstrap: - return LightClientBootstrap( - header=upgrade_lc_header_to_eip6110(pre.header), - current_sync_committee=pre.current_sync_committee, - current_sync_committee_branch=pre.current_sync_committee_branch, - ) -``` - -```python -def upgrade_lc_update_to_eip6110(pre: deneb.LightClientUpdate) -> LightClientUpdate: - return LightClientUpdate( - attested_header=upgrade_lc_header_to_eip6110(pre.attested_header), - next_sync_committee=pre.next_sync_committee, - next_sync_committee_branch=pre.next_sync_committee_branch, - finalized_header=upgrade_lc_header_to_eip6110(pre.finalized_header), - finality_branch=pre.finality_branch, - sync_aggregate=pre.sync_aggregate, - signature_slot=pre.signature_slot, - ) -``` - -```python -def upgrade_lc_finality_update_to_eip6110(pre: deneb.LightClientFinalityUpdate) -> LightClientFinalityUpdate: - return LightClientFinalityUpdate( - attested_header=upgrade_lc_header_to_eip6110(pre.attested_header), - finalized_header=upgrade_lc_header_to_eip6110(pre.finalized_header), - finality_branch=pre.finality_branch, - sync_aggregate=pre.sync_aggregate, - signature_slot=pre.signature_slot, - ) -``` - -```python -def upgrade_lc_optimistic_update_to_eip6110(pre: deneb.LightClientOptimisticUpdate) -> LightClientOptimisticUpdate: - return LightClientOptimisticUpdate( - attested_header=upgrade_lc_header_to_eip6110(pre.attested_header), - sync_aggregate=pre.sync_aggregate, - signature_slot=pre.signature_slot, - ) -``` - -### Upgrading the store - -Existing `LightClientStore` objects based on Deneb MUST be upgraded to eip6110 before eip6110 based light client data can be processed. The `LightClientStore` upgrade MAY be performed before `EIP6110_FORK_EPOCH`. - -```python -def upgrade_lc_store_to_eip6110(pre: deneb.LightClientStore) -> LightClientStore: - if pre.best_valid_update is None: - best_valid_update = None - else: - best_valid_update = upgrade_lc_update_to_eip6110(pre.best_valid_update) - return LightClientStore( - finalized_header=upgrade_lc_header_to_eip6110(pre.finalized_header), - current_sync_committee=pre.current_sync_committee, - next_sync_committee=pre.next_sync_committee, - best_valid_update=best_valid_update, - optimistic_header=upgrade_lc_header_to_eip6110(pre.optimistic_header), - previous_max_active_participants=pre.previous_max_active_participants, - current_max_active_participants=pre.current_max_active_participants, - ) -``` diff --git a/specs/_features/eip6110/light-client/full-node.md b/specs/_features/eip6110/light-client/full-node.md deleted file mode 100644 index 03c0f17bd..000000000 --- a/specs/_features/eip6110/light-client/full-node.md +++ /dev/null @@ -1,77 +0,0 @@ -# EIP-6110 Light Client -- Full Node - -**Notice**: This document is a work-in-progress for researchers and implementers. - -## Table of contents - - - - - -- [Introduction](#introduction) -- [Helper functions](#helper-functions) - - [Modified `block_to_light_client_header`](#modified-block_to_light_client_header) - - - - -## Introduction - -This upgrade adds information about the execution payload to light client data as part of the EIP-6110 upgrade. - -## Helper functions - -### Modified `block_to_light_client_header` - -```python -def block_to_light_client_header(block: SignedBeaconBlock) -> LightClientHeader: - epoch = compute_epoch_at_slot(block.message.slot) - - if epoch >= CAPELLA_FORK_EPOCH: - payload = block.message.body.execution_payload - execution_header = ExecutionPayloadHeader( - parent_hash=payload.parent_hash, - fee_recipient=payload.fee_recipient, - state_root=payload.state_root, - receipts_root=payload.receipts_root, - logs_bloom=payload.logs_bloom, - prev_randao=payload.prev_randao, - block_number=payload.block_number, - gas_limit=payload.gas_limit, - gas_used=payload.gas_used, - timestamp=payload.timestamp, - extra_data=payload.extra_data, - base_fee_per_gas=payload.base_fee_per_gas, - block_hash=payload.block_hash, - transactions_root=hash_tree_root(payload.transactions), - withdrawals_root=hash_tree_root(payload.withdrawals), - ) - - if epoch >= DENEB_FORK_EPOCH: - execution_header.excess_data_gas = payload.excess_data_gas - - # [New in EIP6110] - if epoch >= EIP6110_FORK_EPOCH: - execution_header.deposit_receipts_root = hash_tree_root(payload.deposit_receipts) - - execution_branch = compute_merkle_proof_for_block_body(block.message.body, EXECUTION_PAYLOAD_INDEX) - else: - # Note that during fork transitions, `finalized_header` may still point to earlier forks. - # While Bellatrix blocks also contain an `ExecutionPayload` (minus `withdrawals_root`), - # it was not included in the corresponding light client data. To ensure compatibility - # with legacy data going through `upgrade_lc_header_to_capella`, leave out execution data. - execution_header = ExecutionPayloadHeader() - execution_branch = [Bytes32() for _ in range(floorlog2(EXECUTION_PAYLOAD_INDEX))] - - return LightClientHeader( - beacon=BeaconBlockHeader( - slot=block.message.slot, - proposer_index=block.message.proposer_index, - parent_root=block.message.parent_root, - state_root=block.message.state_root, - body_root=hash_tree_root(block.message.body), - ), - execution=execution_header, - execution_branch=execution_branch, - ) -``` diff --git a/specs/_features/eip6110/light-client/p2p-interface.md b/specs/_features/eip6110/light-client/p2p-interface.md deleted file mode 100644 index f55fb2f77..000000000 --- a/specs/_features/eip6110/light-client/p2p-interface.md +++ /dev/null @@ -1,111 +0,0 @@ -# EIP-6110 Light Client -- Networking - -**Notice**: This document is a work-in-progress for researchers and implementers. - -## Table of contents - - - - - -- [Networking](#networking) - - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) - - [Topics and messages](#topics-and-messages) - - [Global topics](#global-topics) - - [`light_client_finality_update`](#light_client_finality_update) - - [`light_client_optimistic_update`](#light_client_optimistic_update) - - [The Req/Resp domain](#the-reqresp-domain) - - [Messages](#messages) - - [GetLightClientBootstrap](#getlightclientbootstrap) - - [LightClientUpdatesByRange](#lightclientupdatesbyrange) - - [GetLightClientFinalityUpdate](#getlightclientfinalityupdate) - - [GetLightClientOptimisticUpdate](#getlightclientoptimisticupdate) - - - - -## Networking - -The [Deneb light client networking specification](../../deneb/light-client/p2p-interface.md) is extended to exchange [EIP-6110 light client data](./sync-protocol.md). - -### The gossip domain: gossipsub - -#### Topics and messages - -##### Global topics - -###### `light_client_finality_update` - -[0]: # (eth2spec: skip) - -| `fork_version` | Message SSZ type | -|--------------------------------------------------------|-------------------------------------| -| `GENESIS_FORK_VERSION` | n/a | -| `ALTAIR_FORK_VERSION` through `BELLATRIX_FORK_VERSION` | `altair.LightClientFinalityUpdate` | -| `CAPELLA_FORK_VERSION` | `capella.LightClientFinalityUpdate` | -| `DENEB_FORK_VERSION` | `deneb.LightClientFinalityUpdate` | -| `EIP6110_FORK_VERSION` and later | `eip6110.LightClientFinalityUpdate` | - -###### `light_client_optimistic_update` - -[0]: # (eth2spec: skip) - -| `fork_version` | Message SSZ type | -|--------------------------------------------------------|---------------------------------------| -| `GENESIS_FORK_VERSION` | n/a | -| `ALTAIR_FORK_VERSION` through `BELLATRIX_FORK_VERSION` | `altair.LightClientOptimisticUpdate` | -| `CAPELLA_FORK_VERSION` | `capella.LightClientOptimisticUpdate` | -| `DENEB_FORK_VERSION` | `deneb.LightClientOptimisticUpdate` | -| `EIP6110_FORK_VERSION` and later | `eip6110.LightClientOptimisticUpdate` | - -### The Req/Resp domain - -#### Messages - -##### GetLightClientBootstrap - -[0]: # (eth2spec: skip) - -| `fork_version` | Response SSZ type | -|--------------------------------------------------------|------------------------------------| -| `GENESIS_FORK_VERSION` | n/a | -| `ALTAIR_FORK_VERSION` through `BELLATRIX_FORK_VERSION` | `altair.LightClientBootstrap` | -| `CAPELLA_FORK_VERSION` | `capella.LightClientBootstrap` | -| `DENEB_FORK_VERSION` | `deneb.LightClientBootstrap` | -| `EIP6110_FORK_VERSION` and later | `eip6110.LightClientBootstrap` | - -##### LightClientUpdatesByRange - -[0]: # (eth2spec: skip) - -| `fork_version` | Response chunk SSZ type | -|--------------------------------------------------------|----------------------------------| -| `GENESIS_FORK_VERSION` | n/a | -| `ALTAIR_FORK_VERSION` through `BELLATRIX_FORK_VERSION` | `altair.LightClientUpdate` | -| `CAPELLA_FORK_VERSION` | `capella.LightClientUpdate` | -| `DENEB_FORK_VERSION` | `deneb.LightClientUpdate` | -| `EIP6110_FORK_VERSION` and later | `eip6110.LightClientUpdate` | - -##### GetLightClientFinalityUpdate - -[0]: # (eth2spec: skip) - -| `fork_version` | Response SSZ type | -|--------------------------------------------------------|-------------------------------------| -| `GENESIS_FORK_VERSION` | n/a | -| `ALTAIR_FORK_VERSION` through `BELLATRIX_FORK_VERSION` | `altair.LightClientFinalityUpdate` | -| `CAPELLA_FORK_VERSION` | `capella.LightClientFinalityUpdate` | -| `DENEB_FORK_VERSION` | `deneb.LightClientFinalityUpdate` | -| `EIP6110_FORK_VERSION` and later | `eip6110.LightClientFinalityUpdate` | - -##### GetLightClientOptimisticUpdate - -[0]: # (eth2spec: skip) - -| `fork_version` | Response SSZ type | -|--------------------------------------------------------|---------------------------------------| -| `GENESIS_FORK_VERSION` | n/a | -| `ALTAIR_FORK_VERSION` through `BELLATRIX_FORK_VERSION` | `altair.LightClientOptimisticUpdate` | -| `CAPELLA_FORK_VERSION` | `capella.LightClientOptimisticUpdate` | -| `DENEB_FORK_VERSION` | `deneb.LightClientOptimisticUpdate` | -| `EIP6110_FORK_VERSION` and later | `eip6110.LightClientOptimisticUpdate` | diff --git a/specs/_features/eip6110/light-client/sync-protocol.md b/specs/_features/eip6110/light-client/sync-protocol.md deleted file mode 100644 index bcb9d50e4..000000000 --- a/specs/_features/eip6110/light-client/sync-protocol.md +++ /dev/null @@ -1,89 +0,0 @@ -# EIP-6110 Light Client -- Sync Protocol - -**Notice**: This document is a work-in-progress for researchers and implementers. - -## Table of contents - - - - - -- [Introduction](#introduction) -- [Helper functions](#helper-functions) - - [Modified `get_lc_execution_root`](#modified-get_lc_execution_root) - - [Modified `is_valid_light_client_header`](#modified-is_valid_light_client_header) - - - - -## Introduction - -This upgrade updates light client data to include the EIP-6110 changes to the [`ExecutionPayload`](../beacon-chain.md) structure. It extends the [Deneb Light Client specifications](../../deneb/light-client/sync-protocol.md). The [fork document](./fork.md) explains how to upgrade existing Deneb based deployments to EIP-6110. - -Additional documents describes the impact of the upgrade on certain roles: -- [Full node](./full-node.md) -- [Networking](./p2p-interface.md) - -## Helper functions - -### Modified `get_lc_execution_root` - -```python -def get_lc_execution_root(header: LightClientHeader) -> Root: - epoch = compute_epoch_at_slot(header.beacon.slot) - - if epoch >= DENEB_FORK_EPOCH: - return hash_tree_root(header.execution) - - if epoch >= CAPELLA_FORK_EPOCH: - execution_header = capella.ExecutionPayloadHeader( - parent_hash=header.execution.parent_hash, - fee_recipient=header.execution.fee_recipient, - state_root=header.execution.state_root, - receipts_root=header.execution.receipts_root, - logs_bloom=header.execution.logs_bloom, - prev_randao=header.execution.prev_randao, - block_number=header.execution.block_number, - gas_limit=header.execution.gas_limit, - gas_used=header.execution.gas_used, - timestamp=header.execution.timestamp, - extra_data=header.execution.extra_data, - base_fee_per_gas=header.execution.base_fee_per_gas, - block_hash=header.execution.block_hash, - transactions_root=header.execution.transactions_root, - withdrawals_root=header.execution.withdrawals_root, - ) - return hash_tree_root(execution_header) - - return Root() -``` - -### Modified `is_valid_light_client_header` - -```python -def is_valid_light_client_header(header: LightClientHeader) -> bool: - epoch = compute_epoch_at_slot(header.beacon.slot) - - # [New in EIP-6110] - if epoch < EIP6110_FORK_EPOCH: - if header.execution.deposit_receipts_root != Root(): - return False - - if epoch < DENEB_FORK_EPOCH: - if header.execution.excess_data_gas != uint256(0): - return False - - if epoch < CAPELLA_FORK_EPOCH: - return ( - header.execution == ExecutionPayloadHeader() - and header.execution_branch == [Bytes32() for _ in range(floorlog2(EXECUTION_PAYLOAD_INDEX))] - ) - - return is_valid_merkle_branch( - leaf=get_lc_execution_root(header), - branch=header.execution_branch, - depth=floorlog2(EXECUTION_PAYLOAD_INDEX), - index=get_subtree_index(EXECUTION_PAYLOAD_INDEX), - root=header.beacon.body_root, - ) -``` diff --git a/tests/core/pyspec/eth2spec/test/altair/light_client/test_single_merkle_proof.py b/tests/core/pyspec/eth2spec/test/altair/light_client/test_single_merkle_proof.py index 5d802bbb3..7e9c6b7e2 100644 --- a/tests/core/pyspec/eth2spec/test/altair/light_client/test_single_merkle_proof.py +++ b/tests/core/pyspec/eth2spec/test/altair/light_client/test_single_merkle_proof.py @@ -1,12 +1,12 @@ from eth2spec.test.context import ( spec_state_test, - with_altair_and_later, + with_light_client, with_test_suite_name, ) @with_test_suite_name("BeaconState") -@with_altair_and_later +@with_light_client @spec_state_test def test_current_sync_committee_merkle_proof(spec, state): yield "object", state @@ -27,7 +27,7 @@ def test_current_sync_committee_merkle_proof(spec, state): @with_test_suite_name("BeaconState") -@with_altair_and_later +@with_light_client @spec_state_test def test_next_sync_committee_merkle_proof(spec, state): yield "object", state @@ -48,7 +48,7 @@ def test_next_sync_committee_merkle_proof(spec, state): @with_test_suite_name("BeaconState") -@with_altair_and_later +@with_light_client @spec_state_test def test_finality_root_merkle_proof(spec, state): yield "object", state diff --git a/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py b/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py index 1a527a767..f8faa2fdd 100644 --- a/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py +++ b/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py @@ -9,7 +9,7 @@ from eth2spec.test.context import ( with_phases, with_presets, with_state, - with_altair_and_later, + with_light_client, ) from eth2spec.test.helpers.attestations import ( next_slots_with_attestations, @@ -26,7 +26,6 @@ from eth2spec.test.helpers.fork_transition import ( from eth2spec.test.helpers.forks import ( is_post_capella, is_post_deneb, is_post_fork, - is_post_eip6110, ) from eth2spec.test.helpers.light_client import ( get_sync_aggregate, @@ -58,10 +57,6 @@ def needs_upgrade_to_deneb(d_spec, s_spec): return is_post_deneb(s_spec) and not is_post_deneb(d_spec) -def needs_upgrade_to_eip6110(d_spec, s_spec): - return is_post_eip6110(s_spec) and not is_post_eip6110(d_spec) - - def check_lc_header_equal(d_spec, s_spec, data, upgraded): assert upgraded.beacon.slot == data.beacon.slot assert upgraded.beacon.hash_tree_root() == data.beacon.hash_tree_root() @@ -89,10 +84,6 @@ def upgrade_lc_bootstrap_to_store(d_spec, s_spec, data): upgraded = s_spec.upgrade_lc_bootstrap_to_deneb(upgraded) check_lc_bootstrap_equal(d_spec, s_spec, data, upgraded) - if needs_upgrade_to_eip6110(d_spec, s_spec): - upgraded = s_spec.upgrade_lc_bootstrap_to_eip6110(upgraded) - check_lc_bootstrap_equal(d_spec, s_spec, data, upgraded) - return upgraded @@ -154,8 +145,6 @@ class LightClientSyncTest(object): def get_store_fork_version(s_spec): - if is_post_eip6110(s_spec): - return s_spec.config.EIP6110_FORK_VERSION if is_post_deneb(s_spec): return s_spec.config.DENEB_FORK_VERSION if is_post_capella(s_spec): @@ -301,7 +290,7 @@ def compute_start_slot_at_next_sync_committee_period(spec, state): return compute_start_slot_at_sync_committee_period(spec, sync_committee_period + 1) -@with_altair_and_later +@with_light_client @spec_state_test_with_matching_config @with_presets([MINIMAL], reason="too slow") def test_light_client_sync(spec, state): @@ -523,7 +512,7 @@ def test_light_client_sync(spec, state): yield from finish_test(test) -@with_altair_and_later +@with_light_client @spec_state_test_with_matching_config @with_presets([MINIMAL], reason="too slow") def test_supply_sync_committee_from_past_update(spec, state): @@ -553,7 +542,7 @@ def test_supply_sync_committee_from_past_update(spec, state): yield from finish_test(test) -@with_altair_and_later +@with_light_client @spec_state_test_with_matching_config @with_presets([MINIMAL], reason="too slow") def test_advance_finality_without_sync_committee(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/altair/light_client/test_update_ranking.py b/tests/core/pyspec/eth2spec/test/altair/light_client/test_update_ranking.py index 75f1a568a..5bca370ab 100644 --- a/tests/core/pyspec/eth2spec/test/altair/light_client/test_update_ranking.py +++ b/tests/core/pyspec/eth2spec/test/altair/light_client/test_update_ranking.py @@ -1,7 +1,7 @@ from eth2spec.test.context import ( spec_state_test, with_presets, - with_altair_and_later, + with_light_client, ) from eth2spec.test.helpers.attestations import ( next_slots_with_attestations, @@ -29,7 +29,7 @@ def create_test_update(spec, test, with_next, with_finality, participation_rate) ) -@with_altair_and_later +@with_light_client @spec_state_test @with_presets([MINIMAL], reason="too slow") def test_update_ranking(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/altair/unittests/light_client/test_sync_protocol.py b/tests/core/pyspec/eth2spec/test/altair/unittests/light_client/test_sync_protocol.py index 235f9a8b1..a7fc7d928 100644 --- a/tests/core/pyspec/eth2spec/test/altair/unittests/light_client/test_sync_protocol.py +++ b/tests/core/pyspec/eth2spec/test/altair/unittests/light_client/test_sync_protocol.py @@ -3,7 +3,7 @@ from copy import deepcopy from eth2spec.test.context import ( spec_state_test_with_matching_config, with_presets, - with_altair_and_later, + with_light_client, ) from eth2spec.test.helpers.attestations import ( next_epoch_with_attestations, @@ -29,7 +29,7 @@ def setup_test(spec, state): return (trusted_block, store) -@with_altair_and_later +@with_light_client @spec_state_test_with_matching_config def test_process_light_client_update_not_timeout(spec, state): genesis_block, store = setup_test(spec, state) @@ -61,7 +61,7 @@ def test_process_light_client_update_not_timeout(spec, state): assert store.current_max_active_participants > 0 -@with_altair_and_later +@with_light_client @spec_state_test_with_matching_config @with_presets([MINIMAL], reason="too slow") def test_process_light_client_update_at_period_boundary(spec, state): @@ -96,7 +96,7 @@ def test_process_light_client_update_at_period_boundary(spec, state): assert store.current_max_active_participants > 0 -@with_altair_and_later +@with_light_client @spec_state_test_with_matching_config @with_presets([MINIMAL], reason="too slow") def test_process_light_client_update_timeout(spec, state): @@ -131,7 +131,7 @@ def test_process_light_client_update_timeout(spec, state): assert store.current_max_active_participants > 0 -@with_altair_and_later +@with_light_client @spec_state_test_with_matching_config @with_presets([MINIMAL], reason="too slow") def test_process_light_client_update_finality_updated(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 626ffc1db..00589c872 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -18,6 +18,7 @@ from .helpers.constants import ( MINIMAL, MAINNET, ALL_PHASES, ALL_FORK_UPGRADES, + LIGHT_CLIENT_TESTING_FORKS, ) from .helpers.forks import is_post_fork from .helpers.typing import SpecForkName, PresetBaseName @@ -428,13 +429,6 @@ def with_all_phases_except(exclusion_phases): return decorator -with_altair_and_later = with_all_phases_from(ALTAIR) -with_bellatrix_and_later = with_all_phases_from(BELLATRIX) -with_capella_and_later = with_all_phases_from(CAPELLA) -with_deneb_and_later = with_all_phases_from(DENEB) -with_eip6110_and_later = with_all_phases_from(EIP6110) - - def _get_preset_targets(kw): preset_name = DEFAULT_TEST_PRESET if 'preset' in kw: @@ -540,6 +534,14 @@ def with_presets(preset_bases, reason=None): return decorator +with_altair_and_later = with_all_phases_from(ALTAIR) +with_bellatrix_and_later = with_all_phases_from(BELLATRIX) +with_capella_and_later = with_all_phases_from(CAPELLA) +with_deneb_and_later = with_all_phases_from(DENEB) +with_eip6110_and_later = with_all_phases_from(EIP6110) +with_light_client = with_phases(LIGHT_CLIENT_TESTING_FORKS) + + class quoted_str(str): pass diff --git a/tests/core/pyspec/eth2spec/test/helpers/constants.py b/tests/core/pyspec/eth2spec/test/helpers/constants.py index 8fb88d0be..601650e97 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/constants.py +++ b/tests/core/pyspec/eth2spec/test/helpers/constants.py @@ -24,6 +24,7 @@ EIP6110 = SpecForkName('eip6110') # The forks that are deployed on Mainnet MAINNET_FORKS = (PHASE0, ALTAIR, BELLATRIX, CAPELLA) +LATEST_FORK = MAINNET_FORKS[-1] # The forks that pytest can run with. ALL_PHASES = ( # Formal forks @@ -33,7 +34,7 @@ ALL_PHASES = ( EIP6110, ) # The forks that have light client specs -LIGHT_CLIENT_TESTING_FORKS = (*MAINNET_FORKS, DENEB) +LIGHT_CLIENT_TESTING_FORKS = (*[item for item in MAINNET_FORKS if item != PHASE0], DENEB) # The forks that output to the test vectors. TESTGEN_FORKS = (*MAINNET_FORKS, DENEB, EIP6110) From 296efec2561a44580a7c321bf22c51aff2e52714 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 30 May 2023 00:39:13 +0800 Subject: [PATCH 076/110] Add `data_gas_used` field to `ExecutionPayload` --- specs/_features/eip6110/beacon-chain.md | 3 +++ specs/_features/eip6110/fork.md | 1 + specs/deneb/beacon-chain.md | 3 +++ specs/deneb/fork.md | 1 + specs/deneb/light-client/fork.md | 1 + specs/deneb/light-client/full-node.md | 1 + specs/deneb/light-client/sync-protocol.md | 2 +- tests/core/pyspec/eth2spec/test/helpers/execution_payload.py | 3 +++ 8 files changed, 14 insertions(+), 1 deletion(-) diff --git a/specs/_features/eip6110/beacon-chain.md b/specs/_features/eip6110/beacon-chain.md index e1384fdae..95dc7fd3c 100644 --- a/specs/_features/eip6110/beacon-chain.md +++ b/specs/_features/eip6110/beacon-chain.md @@ -92,6 +92,7 @@ class ExecutionPayload(Container): transactions: List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD] withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] excess_data_gas: uint256 + data_gas_used: uint256 deposit_receipts: List[DepositReceipt, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD] # [New in EIP6110] ``` @@ -117,6 +118,7 @@ class ExecutionPayloadHeader(Container): transactions_root: Root withdrawals_root: Root excess_data_gas: uint256 + data_gas_used: uint256 deposit_receipts_root: Root # [New in EIP6110] ``` @@ -269,6 +271,7 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi transactions_root=hash_tree_root(payload.transactions), withdrawals_root=hash_tree_root(payload.withdrawals), excess_data_gas=payload.excess_data_gas, + data_gas_used=payload.data_gas_used, deposit_receipts_root=hash_tree_root(payload.deposit_receipts), # [New in EIP6110] ) ``` diff --git a/specs/_features/eip6110/fork.md b/specs/_features/eip6110/fork.md index 2145a9d1a..8a603b530 100644 --- a/specs/_features/eip6110/fork.md +++ b/specs/_features/eip6110/fork.md @@ -89,6 +89,7 @@ def upgrade_to_eip6110(pre: deneb.BeaconState) -> BeaconState: transactions_root=pre.latest_execution_payload_header.transactions_root, withdrawals_root=pre.latest_execution_payload_header.withdrawals_root, excess_data_gas=uint256(0), + data_gas_used=uint256(0), deposit_receipts_root=Root(), # [New in EIP-6110] ) post = BeaconState( diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 175a80edf..26206384a 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -127,6 +127,7 @@ class ExecutionPayload(Container): transactions: List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD] withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] excess_data_gas: uint256 # [New in Deneb] + data_gas_used: uint256 # [New in Deneb] ``` #### `ExecutionPayloadHeader` @@ -151,6 +152,7 @@ class ExecutionPayloadHeader(Container): transactions_root: Root withdrawals_root: Root excess_data_gas: uint256 # [New in Deneb] + data_gas_used: uint256 # [New in Deneb] ``` ## Helper functions @@ -258,6 +260,7 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi transactions_root=hash_tree_root(payload.transactions), withdrawals_root=hash_tree_root(payload.withdrawals), excess_data_gas=payload.excess_data_gas, # [New in Deneb] + data_gas_used=payload.data_gas_used, # [New in Deneb] ) ``` diff --git a/specs/deneb/fork.md b/specs/deneb/fork.md index 23b3f23c7..8c7fd5319 100644 --- a/specs/deneb/fork.md +++ b/specs/deneb/fork.md @@ -84,6 +84,7 @@ def upgrade_to_deneb(pre: capella.BeaconState) -> BeaconState: transactions_root=pre.latest_execution_payload_header.transactions_root, withdrawals_root=pre.latest_execution_payload_header.withdrawals_root, excess_data_gas=uint256(0), # [New in Deneb] + data_gas_used=uint256(0), # [New in Deneb] ) post = BeaconState( # Versioning diff --git a/specs/deneb/light-client/fork.md b/specs/deneb/light-client/fork.md index 46a093028..2ae59cc7e 100644 --- a/specs/deneb/light-client/fork.md +++ b/specs/deneb/light-client/fork.md @@ -42,6 +42,7 @@ def upgrade_lc_header_to_deneb(pre: capella.LightClientHeader) -> LightClientHea transactions_root=pre.execution.transactions_root, withdrawals_root=pre.execution.withdrawals_root, excess_data_gas=uint256(0), # [New in Deneb] + data_gas_used=uint256(0), # [New in Deneb] ), execution_branch=pre.execution_branch, ) diff --git a/specs/deneb/light-client/full-node.md b/specs/deneb/light-client/full-node.md index 275194036..0ef0ee96f 100644 --- a/specs/deneb/light-client/full-node.md +++ b/specs/deneb/light-client/full-node.md @@ -50,6 +50,7 @@ def block_to_light_client_header(block: SignedBeaconBlock) -> LightClientHeader: # [New in Deneb] if epoch >= DENEB_FORK_EPOCH: execution_header.excess_data_gas = payload.excess_data_gas + execution_header.data_gas_used = payload.data_gas_used execution_branch = compute_merkle_proof_for_block_body(block.message.body, EXECUTION_PAYLOAD_INDEX) else: diff --git a/specs/deneb/light-client/sync-protocol.md b/specs/deneb/light-client/sync-protocol.md index c691a113d..194be0ae4 100644 --- a/specs/deneb/light-client/sync-protocol.md +++ b/specs/deneb/light-client/sync-protocol.md @@ -68,7 +68,7 @@ def is_valid_light_client_header(header: LightClientHeader) -> bool: # [New in Deneb] if epoch < DENEB_FORK_EPOCH: - if header.execution.excess_data_gas != uint256(0): + if header.execution.excess_data_gas != uint256(0) or header.execution.data_gas_used != uint256(0): return False if epoch < CAPELLA_FORK_EPOCH: diff --git a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py index 747d678ef..1fa3f4c4b 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py @@ -32,6 +32,7 @@ def get_execution_payload_header(spec, execution_payload): payload_header.withdrawals_root = spec.hash_tree_root(execution_payload.withdrawals) if is_post_deneb(spec): payload_header.excess_data_gas = execution_payload.excess_data_gas + payload_header.data_gas_used = execution_payload.data_gas_used if is_post_eip6110(spec): payload_header.deposit_receipts_root = spec.hash_tree_root(execution_payload.deposit_receipts) return payload_header @@ -99,6 +100,7 @@ def compute_el_header_block_hash(spec, if is_post_deneb(spec): # excess_data_gas execution_payload_header_rlp.append((big_endian_int, payload_header.excess_data_gas)) + execution_payload_header_rlp.append((big_endian_int, payload_header.data_gas_used)) if is_post_eip6110(spec): # deposit_receipts_root assert deposit_receipts_trie_root is not None @@ -202,6 +204,7 @@ def build_empty_execution_payload(spec, state, randao_mix=None): payload.withdrawals = spec.get_expected_withdrawals(state) if is_post_deneb(spec): payload.excess_data_gas = 0 + payload.data_gas_used = 0 if is_post_eip6110(spec): # just to be clear payload.deposit_receipts = [] From f6fe8e27292c121c133a882998b1dacf900556ee Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 30 May 2023 20:36:58 +0800 Subject: [PATCH 077/110] Change `ExecutionPayload.excess_data_gas` type from `uint256` to `uint64` --- specs/_features/eip6110/beacon-chain.md | 4 ++-- specs/_features/eip6110/fork.md | 2 +- specs/_features/eip6110/light-client/sync-protocol.md | 2 +- specs/deneb/beacon-chain.md | 4 ++-- specs/deneb/fork.md | 2 +- specs/deneb/light-client/fork.md | 2 +- specs/deneb/light-client/sync-protocol.md | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/specs/_features/eip6110/beacon-chain.md b/specs/_features/eip6110/beacon-chain.md index e1384fdae..3e260006f 100644 --- a/specs/_features/eip6110/beacon-chain.md +++ b/specs/_features/eip6110/beacon-chain.md @@ -91,7 +91,7 @@ class ExecutionPayload(Container): block_hash: Hash32 transactions: List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD] withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] - excess_data_gas: uint256 + excess_data_gas: uint64 deposit_receipts: List[DepositReceipt, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD] # [New in EIP6110] ``` @@ -116,7 +116,7 @@ class ExecutionPayloadHeader(Container): block_hash: Hash32 transactions_root: Root withdrawals_root: Root - excess_data_gas: uint256 + excess_data_gas: uint64 deposit_receipts_root: Root # [New in EIP6110] ``` diff --git a/specs/_features/eip6110/fork.md b/specs/_features/eip6110/fork.md index 2145a9d1a..468f432e6 100644 --- a/specs/_features/eip6110/fork.md +++ b/specs/_features/eip6110/fork.md @@ -88,7 +88,7 @@ def upgrade_to_eip6110(pre: deneb.BeaconState) -> BeaconState: block_hash=pre.latest_execution_payload_header.block_hash, transactions_root=pre.latest_execution_payload_header.transactions_root, withdrawals_root=pre.latest_execution_payload_header.withdrawals_root, - excess_data_gas=uint256(0), + excess_data_gas=uint64(0), deposit_receipts_root=Root(), # [New in EIP-6110] ) post = BeaconState( diff --git a/specs/_features/eip6110/light-client/sync-protocol.md b/specs/_features/eip6110/light-client/sync-protocol.md index bcb9d50e4..b454a2ebe 100644 --- a/specs/_features/eip6110/light-client/sync-protocol.md +++ b/specs/_features/eip6110/light-client/sync-protocol.md @@ -70,7 +70,7 @@ def is_valid_light_client_header(header: LightClientHeader) -> bool: return False if epoch < DENEB_FORK_EPOCH: - if header.execution.excess_data_gas != uint256(0): + if header.execution.excess_data_gas != uint64(0): return False if epoch < CAPELLA_FORK_EPOCH: diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 175a80edf..ba84d33c0 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -126,7 +126,7 @@ class ExecutionPayload(Container): block_hash: Hash32 # Hash of execution block transactions: List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD] withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] - excess_data_gas: uint256 # [New in Deneb] + excess_data_gas: uint64 # [New in Deneb] ``` #### `ExecutionPayloadHeader` @@ -150,7 +150,7 @@ class ExecutionPayloadHeader(Container): block_hash: Hash32 # Hash of execution block transactions_root: Root withdrawals_root: Root - excess_data_gas: uint256 # [New in Deneb] + excess_data_gas: uint64 # [New in Deneb] ``` ## Helper functions diff --git a/specs/deneb/fork.md b/specs/deneb/fork.md index 23b3f23c7..a4014b10a 100644 --- a/specs/deneb/fork.md +++ b/specs/deneb/fork.md @@ -83,7 +83,7 @@ def upgrade_to_deneb(pre: capella.BeaconState) -> BeaconState: block_hash=pre.latest_execution_payload_header.block_hash, transactions_root=pre.latest_execution_payload_header.transactions_root, withdrawals_root=pre.latest_execution_payload_header.withdrawals_root, - excess_data_gas=uint256(0), # [New in Deneb] + excess_data_gas=uint64(0), # [New in Deneb] ) post = BeaconState( # Versioning diff --git a/specs/deneb/light-client/fork.md b/specs/deneb/light-client/fork.md index 46a093028..dba50beb1 100644 --- a/specs/deneb/light-client/fork.md +++ b/specs/deneb/light-client/fork.md @@ -41,7 +41,7 @@ def upgrade_lc_header_to_deneb(pre: capella.LightClientHeader) -> LightClientHea block_hash=pre.execution.block_hash, transactions_root=pre.execution.transactions_root, withdrawals_root=pre.execution.withdrawals_root, - excess_data_gas=uint256(0), # [New in Deneb] + excess_data_gas=uint64(0), # [New in Deneb] ), execution_branch=pre.execution_branch, ) diff --git a/specs/deneb/light-client/sync-protocol.md b/specs/deneb/light-client/sync-protocol.md index c691a113d..697d12c04 100644 --- a/specs/deneb/light-client/sync-protocol.md +++ b/specs/deneb/light-client/sync-protocol.md @@ -68,7 +68,7 @@ def is_valid_light_client_header(header: LightClientHeader) -> bool: # [New in Deneb] if epoch < DENEB_FORK_EPOCH: - if header.execution.excess_data_gas != uint256(0): + if header.execution.excess_data_gas != uint64(0): return False if epoch < CAPELLA_FORK_EPOCH: From 653e03e70223335d270940cc81640ff875080c7d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 30 May 2023 21:12:47 +0800 Subject: [PATCH 078/110] Fix ToC --- specs/deneb/beacon-chain.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index a929db859..335723626 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -33,8 +33,8 @@ - [Modified `verify_and_notify_new_payload`](#modified-verify_and_notify_new_payload) - [Block processing](#block-processing) - [Execution payload](#execution-payload) - - [`process_execution_payload`](#process_execution_payload) - - [Modified `process_voluntary_exit`](#modified-process_voluntary_exit) + - [Modified `process_execution_payload`](#modified-process_execution_payload) + - [Modified `process_voluntary_exit`](#modified-process_voluntary_exit) - [Testing](#testing) @@ -218,7 +218,7 @@ def verify_and_notify_new_payload(self: ExecutionEngine, #### Execution payload -##### `process_execution_payload` +##### Modified `process_execution_payload` ```python def process_execution_payload(state: BeaconState, body: BeaconBlockBody, execution_engine: ExecutionEngine) -> None: @@ -262,7 +262,7 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi ) ``` -##### Modified `process_voluntary_exit` +#### Modified `process_voluntary_exit` ```python def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVoluntaryExit) -> None: From 7bc5d51ce84d47e9b495225c02ebade181438e0e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 1 Jun 2023 18:09:23 +0800 Subject: [PATCH 079/110] Add non-zero data_gas_used case --- tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index ba0797a04..a6025092b 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -16,7 +16,7 @@ from eth2spec.test.helpers.sharding import ( ) -def run_block_with_blobs(spec, state, blob_count, excess_data_gas=1, valid=True): +def run_block_with_blobs(spec, state, blob_count, excess_data_gas=1, data_gas_used=1, valid=True): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) @@ -24,6 +24,7 @@ def run_block_with_blobs(spec, state, blob_count, excess_data_gas=1, valid=True) block.body.blob_kzg_commitments = blob_kzg_commitments block.body.execution_payload.transactions = [opaque_tx] block.body.execution_payload.excess_data_gas = excess_data_gas + block.body.execution_payload.data_gas_used = data_gas_used block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) if valid: From 60721b9d8aeab0f83dd317edceb84c437cfc3848 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 1 Jun 2023 22:21:50 +0800 Subject: [PATCH 080/110] switch order --- specs/_features/eip6110/beacon-chain.md | 6 +++--- specs/_features/eip6110/fork.md | 2 +- specs/deneb/beacon-chain.md | 6 +++--- specs/deneb/fork.md | 2 +- specs/deneb/light-client/fork.md | 2 +- specs/deneb/light-client/full-node.md | 2 +- tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py | 4 ++-- .../core/pyspec/eth2spec/test/helpers/execution_payload.py | 6 +++--- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/specs/_features/eip6110/beacon-chain.md b/specs/_features/eip6110/beacon-chain.md index 95dc7fd3c..dc68d4ab0 100644 --- a/specs/_features/eip6110/beacon-chain.md +++ b/specs/_features/eip6110/beacon-chain.md @@ -91,8 +91,8 @@ class ExecutionPayload(Container): block_hash: Hash32 transactions: List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD] withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] - excess_data_gas: uint256 data_gas_used: uint256 + excess_data_gas: uint256 deposit_receipts: List[DepositReceipt, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD] # [New in EIP6110] ``` @@ -117,8 +117,8 @@ class ExecutionPayloadHeader(Container): block_hash: Hash32 transactions_root: Root withdrawals_root: Root - excess_data_gas: uint256 data_gas_used: uint256 + excess_data_gas: uint256 deposit_receipts_root: Root # [New in EIP6110] ``` @@ -270,8 +270,8 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi block_hash=payload.block_hash, transactions_root=hash_tree_root(payload.transactions), withdrawals_root=hash_tree_root(payload.withdrawals), - excess_data_gas=payload.excess_data_gas, data_gas_used=payload.data_gas_used, + excess_data_gas=payload.excess_data_gas, deposit_receipts_root=hash_tree_root(payload.deposit_receipts), # [New in EIP6110] ) ``` diff --git a/specs/_features/eip6110/fork.md b/specs/_features/eip6110/fork.md index 8a603b530..b702472b8 100644 --- a/specs/_features/eip6110/fork.md +++ b/specs/_features/eip6110/fork.md @@ -88,8 +88,8 @@ def upgrade_to_eip6110(pre: deneb.BeaconState) -> BeaconState: block_hash=pre.latest_execution_payload_header.block_hash, transactions_root=pre.latest_execution_payload_header.transactions_root, withdrawals_root=pre.latest_execution_payload_header.withdrawals_root, - excess_data_gas=uint256(0), data_gas_used=uint256(0), + excess_data_gas=uint256(0), deposit_receipts_root=Root(), # [New in EIP-6110] ) post = BeaconState( diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 26206384a..d8815762e 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -126,8 +126,8 @@ class ExecutionPayload(Container): block_hash: Hash32 # Hash of execution block transactions: List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD] withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] - excess_data_gas: uint256 # [New in Deneb] data_gas_used: uint256 # [New in Deneb] + excess_data_gas: uint256 # [New in Deneb] ``` #### `ExecutionPayloadHeader` @@ -151,8 +151,8 @@ class ExecutionPayloadHeader(Container): block_hash: Hash32 # Hash of execution block transactions_root: Root withdrawals_root: Root - excess_data_gas: uint256 # [New in Deneb] data_gas_used: uint256 # [New in Deneb] + excess_data_gas: uint256 # [New in Deneb] ``` ## Helper functions @@ -259,8 +259,8 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi block_hash=payload.block_hash, transactions_root=hash_tree_root(payload.transactions), withdrawals_root=hash_tree_root(payload.withdrawals), - excess_data_gas=payload.excess_data_gas, # [New in Deneb] data_gas_used=payload.data_gas_used, # [New in Deneb] + excess_data_gas=payload.excess_data_gas, # [New in Deneb] ) ``` diff --git a/specs/deneb/fork.md b/specs/deneb/fork.md index 8c7fd5319..910a6dbff 100644 --- a/specs/deneb/fork.md +++ b/specs/deneb/fork.md @@ -83,8 +83,8 @@ def upgrade_to_deneb(pre: capella.BeaconState) -> BeaconState: block_hash=pre.latest_execution_payload_header.block_hash, transactions_root=pre.latest_execution_payload_header.transactions_root, withdrawals_root=pre.latest_execution_payload_header.withdrawals_root, - excess_data_gas=uint256(0), # [New in Deneb] data_gas_used=uint256(0), # [New in Deneb] + excess_data_gas=uint256(0), # [New in Deneb] ) post = BeaconState( # Versioning diff --git a/specs/deneb/light-client/fork.md b/specs/deneb/light-client/fork.md index 2ae59cc7e..c10bec2b9 100644 --- a/specs/deneb/light-client/fork.md +++ b/specs/deneb/light-client/fork.md @@ -41,8 +41,8 @@ def upgrade_lc_header_to_deneb(pre: capella.LightClientHeader) -> LightClientHea block_hash=pre.execution.block_hash, transactions_root=pre.execution.transactions_root, withdrawals_root=pre.execution.withdrawals_root, - excess_data_gas=uint256(0), # [New in Deneb] data_gas_used=uint256(0), # [New in Deneb] + excess_data_gas=uint256(0), # [New in Deneb] ), execution_branch=pre.execution_branch, ) diff --git a/specs/deneb/light-client/full-node.md b/specs/deneb/light-client/full-node.md index 0ef0ee96f..69ba0d3ba 100644 --- a/specs/deneb/light-client/full-node.md +++ b/specs/deneb/light-client/full-node.md @@ -49,8 +49,8 @@ def block_to_light_client_header(block: SignedBeaconBlock) -> LightClientHeader: # [New in Deneb] if epoch >= DENEB_FORK_EPOCH: - execution_header.excess_data_gas = payload.excess_data_gas execution_header.data_gas_used = payload.data_gas_used + execution_header.excess_data_gas = payload.excess_data_gas execution_branch = compute_merkle_proof_for_block_body(block.message.body, EXECUTION_PAYLOAD_INDEX) else: diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index a6025092b..2af330efb 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -16,15 +16,15 @@ from eth2spec.test.helpers.sharding import ( ) -def run_block_with_blobs(spec, state, blob_count, excess_data_gas=1, data_gas_used=1, valid=True): +def run_block_with_blobs(spec, state, blob_count, data_gas_used=1, excess_data_gas=1, valid=True): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=blob_count) block.body.blob_kzg_commitments = blob_kzg_commitments block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.excess_data_gas = excess_data_gas block.body.execution_payload.data_gas_used = data_gas_used + block.body.execution_payload.excess_data_gas = excess_data_gas block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) if valid: diff --git a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py index 1fa3f4c4b..b08b1975e 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py @@ -31,8 +31,8 @@ def get_execution_payload_header(spec, execution_payload): if is_post_capella(spec): payload_header.withdrawals_root = spec.hash_tree_root(execution_payload.withdrawals) if is_post_deneb(spec): - payload_header.excess_data_gas = execution_payload.excess_data_gas payload_header.data_gas_used = execution_payload.data_gas_used + payload_header.excess_data_gas = execution_payload.excess_data_gas if is_post_eip6110(spec): payload_header.deposit_receipts_root = spec.hash_tree_root(execution_payload.deposit_receipts) return payload_header @@ -99,8 +99,8 @@ def compute_el_header_block_hash(spec, execution_payload_header_rlp.append((Binary(32, 32), withdrawals_trie_root)) if is_post_deneb(spec): # excess_data_gas - execution_payload_header_rlp.append((big_endian_int, payload_header.excess_data_gas)) execution_payload_header_rlp.append((big_endian_int, payload_header.data_gas_used)) + execution_payload_header_rlp.append((big_endian_int, payload_header.excess_data_gas)) if is_post_eip6110(spec): # deposit_receipts_root assert deposit_receipts_trie_root is not None @@ -203,8 +203,8 @@ def build_empty_execution_payload(spec, state, randao_mix=None): if is_post_capella(spec): payload.withdrawals = spec.get_expected_withdrawals(state) if is_post_deneb(spec): - payload.excess_data_gas = 0 payload.data_gas_used = 0 + payload.excess_data_gas = 0 if is_post_eip6110(spec): # just to be clear payload.deposit_receipts = [] From c9b84ecc1a75d3a3d3d028955a2e5646cc62b54a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 1 Jun 2023 23:40:46 +0800 Subject: [PATCH 081/110] Use `verify_blob_kzg_proof_batch` directly in `is_data_available` and remove `validate_blob` wrapper --- specs/deneb/fork-choice.md | 14 +---- specs/deneb/p2p-interface.md | 2 +- .../deneb/unittests/fork_choice/__init__.py | 0 .../fork_choice/test_validate_blobs.py | 54 ------------------- 4 files changed, 3 insertions(+), 67 deletions(-) delete mode 100644 tests/core/pyspec/eth2spec/test/deneb/unittests/fork_choice/__init__.py delete mode 100644 tests/core/pyspec/eth2spec/test/deneb/unittests/fork_choice/test_validate_blobs.py diff --git a/specs/deneb/fork-choice.md b/specs/deneb/fork-choice.md index 1390b3d4e..4636b42a1 100644 --- a/specs/deneb/fork-choice.md +++ b/specs/deneb/fork-choice.md @@ -8,7 +8,6 @@ - [Introduction](#introduction) - [Containers](#containers) - [Helpers](#helpers) - - [`validate_blobs`](#validate_blobs) - [`is_data_available`](#is_data_available) - [Updated fork-choice handlers](#updated-fork-choice-handlers) - [`on_block`](#on_block) @@ -24,19 +23,10 @@ This is the modification of the fork choice accompanying the Deneb upgrade. ## Helpers -#### `validate_blobs` - -```python -def validate_blobs(expected_kzg_commitments: Sequence[KZGCommitment], - blobs: Sequence[Blob], - proofs: Sequence[KZGProof]) -> None: - assert verify_blob_kzg_proof_batch(blobs, expected_kzg_commitments, proofs) -``` - #### `is_data_available` The implementation of `is_data_available` will become more sophisticated during later scaling upgrades. -Initially, verification requires every verifying actor to retrieve all matching `Blob`s and `KZGProof`s, and validate them with `validate_blobs`. +Initially, verification requires every verifying actor to retrieve all matching `Blob`s and `KZGProof`s, and validate them with `verify_blob_kzg_proof_batch`. The block MUST NOT be considered valid until all valid `Blob`s have been downloaded. Blocks that have been previously validated as available SHOULD be considered available even if the associated `Blob`s have subsequently been pruned. @@ -52,7 +42,7 @@ def is_data_available(beacon_block_root: Root, blob_kzg_commitments: Sequence[KZ if isinstance(blobs, str) or isinstance(proofs, str): return True - validate_blobs(blob_kzg_commitments, blobs, proofs) + assert verify_blob_kzg_proof_batch(blobs, blob_kzg_commitments, proofs) return True ``` diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 3c6f3c88a..f7c962e46 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -264,7 +264,7 @@ Requests blob sidecars in the slot range `[start_slot, start_slot + count)`, lea The response is unsigned, i.e. `BlobSidecarsByRange`, as the signature of the beacon block proposer may not be available beyond the initial distribution via gossip. -Before consuming the next response chunk, the response reader SHOULD verify the blob sidecar is well-formatted and correct w.r.t. the expected KZG commitments through `validate_blobs`. +Before consuming the next response chunk, the response reader SHOULD verify the blob sidecar is well-formatted and correct w.r.t. the expected KZG commitments through `verify_blob_kzg_proof_batch`. `BlobSidecarsByRange` is primarily used to sync blobs that may have been missed on gossip and to sync within the `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` window. diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/fork_choice/__init__.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/fork_choice/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/fork_choice/test_validate_blobs.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/fork_choice/test_validate_blobs.py deleted file mode 100644 index 0d7bd53e5..000000000 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/fork_choice/test_validate_blobs.py +++ /dev/null @@ -1,54 +0,0 @@ -from eth2spec.test.helpers.state import ( - state_transition_and_sign_block, -) -from eth2spec.test.helpers.block import ( - build_empty_block_for_next_slot -) -from eth2spec.test.context import ( - spec_state_test, - with_deneb_and_later, -) -from eth2spec.test.helpers.execution_payload import ( - compute_el_block_hash, -) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, -) - - -def _run_validate_blobs(spec, state, blob_count): - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, blobs, blob_kzg_commitments, kzg_proofs = get_sample_opaque_tx(spec, blob_count=blob_count) - block.body.blob_kzg_commitments = blob_kzg_commitments - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - state_transition_and_sign_block(spec, state, block) - - blob_sidecars = spec.get_blob_sidecars(block, blobs, kzg_proofs) - blobs = [sidecar.blob for sidecar in blob_sidecars] - kzg_proofs = [sidecar.kzg_proof for sidecar in blob_sidecars] - spec.validate_blobs(blob_kzg_commitments, blobs, kzg_proofs) - - -@with_deneb_and_later -@spec_state_test -def test_validate_blobs_zero_blobs(spec, state): - _run_validate_blobs(spec, state, blob_count=0) - - -@with_deneb_and_later -@spec_state_test -def test_validate_blobs_one_blob(spec, state): - _run_validate_blobs(spec, state, blob_count=1) - - -@with_deneb_and_later -@spec_state_test -def test_validate_blobs_two_blobs(spec, state): - _run_validate_blobs(spec, state, blob_count=2) - - -@with_deneb_and_later -@spec_state_test -def test_validate_blobs_max_blobs(spec, state): - _run_validate_blobs(spec, state, blob_count=spec.MAX_BLOBS_PER_BLOCK) From 4bf3895864ceb670a4dfba16b7c4bf3ead0aea55 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 1 Jun 2023 23:53:01 +0800 Subject: [PATCH 082/110] Update VERSION.txt to 1.4.0-alpha.1 --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index 779260991..2a6fa03c0 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -1.4.0-alpha.0 +1.4.0-alpha.1 From e1a55238f2576aa4829cdf32357d530eb7fa0301 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 2 Jun 2023 00:06:47 +0800 Subject: [PATCH 083/110] Delete eip6110_mods --- tests/generators/light_client/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/generators/light_client/main.py b/tests/generators/light_client/main.py index c6b0e01b9..178ef4f44 100644 --- a/tests/generators/light_client/main.py +++ b/tests/generators/light_client/main.py @@ -15,14 +15,12 @@ if __name__ == "__main__": ]} capella_mods = combine_mods(_new_capella_mods, bellatrix_mods) deneb_mods = capella_mods - eip6110_mods = deneb_mods all_mods = { ALTAIR: altair_mods, BELLATRIX: bellatrix_mods, CAPELLA: capella_mods, DENEB: deneb_mods, - EIP6110: eip6110_mods, } run_state_test_generators(runner_name="light_client", all_mods=all_mods) From dde9301610797cbbeb45278b5511f03c382140d5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 2 Jun 2023 00:15:45 +0800 Subject: [PATCH 084/110] fix lint --- tests/generators/light_client/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/light_client/main.py b/tests/generators/light_client/main.py index 178ef4f44..cfe34aee4 100644 --- a/tests/generators/light_client/main.py +++ b/tests/generators/light_client/main.py @@ -1,4 +1,4 @@ -from eth2spec.test.helpers.constants import ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110 +from eth2spec.test.helpers.constants import ALTAIR, BELLATRIX, CAPELLA, DENEB from eth2spec.gen_helpers.gen_from_tests.gen import combine_mods, run_state_test_generators From f5e00848d0f74cd440b996ae716d58947b868173 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Wed, 7 Jun 2023 11:59:51 +0600 Subject: [PATCH 085/110] Remove assert from is_data_available --- specs/deneb/fork-choice.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/specs/deneb/fork-choice.md b/specs/deneb/fork-choice.md index 283e76b1d..8b3c22423 100644 --- a/specs/deneb/fork-choice.md +++ b/specs/deneb/fork-choice.md @@ -42,8 +42,7 @@ def is_data_available(beacon_block_root: Root, blob_kzg_commitments: Sequence[KZ if isinstance(blobs, str) or isinstance(proofs, str): return True - assert verify_blob_kzg_proof_batch(blobs, blob_kzg_commitments, proofs) - return True + return verify_blob_kzg_proof_batch(blobs, blob_kzg_commitments, proofs) ``` ## Updated fork-choice handlers From a547d47264e40adca4b06fce1954e22ba84e0c27 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 7 Jun 2023 17:45:39 +0800 Subject: [PATCH 086/110] specially mark the EIP4844 changes --- specs/deneb/beacon-chain.md | 38 ++++++++++++----------- specs/deneb/fork-choice.md | 4 ++- specs/deneb/fork.md | 6 ++-- specs/deneb/light-client/fork.md | 4 +-- specs/deneb/light-client/full-node.md | 2 +- specs/deneb/light-client/sync-protocol.md | 2 +- specs/deneb/p2p-interface.md | 14 +++++++-- specs/deneb/polynomial-commitments.md | 2 +- specs/deneb/validator.md | 2 ++ 9 files changed, 44 insertions(+), 30 deletions(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 489ae6151..5c3b4e8a4 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -41,16 +41,15 @@ ## Introduction -This upgrade adds blobs to the beacon chain as part of Deneb. This is an extension of the Capella upgrade. - -The blob transactions are packed into the execution payload by the EL/builder with their corresponding blobs being independently transmitted and are limited by `MAX_DATA_GAS_PER_BLOCK // DATA_GAS_PER_BLOB`. However the CL limit is independently defined by `MAX_BLOBS_PER_BLOCK`. +Deneb is a consensus-layer upgrade containing a number of features. Including: +* [EIP-4844](https://eips.ethereum.org/EIPS/eip-4844): Shard Blob Transactions scale data-availability of Ethereum in a simple, forwards-compatible manner. ## Custom types | Name | SSZ equivalent | Description | | - | - | - | -| `VersionedHash` | `Bytes32` | | -| `BlobIndex` | `uint64` | | +| `VersionedHash` | `Bytes32` | [New in Deneb:EIP4844] | +| `BlobIndex` | `uint64` | [New in Deneb:EIP4844] | ## Constants @@ -73,8 +72,11 @@ The blob transactions are packed into the execution payload by the EL/builder wi | Name | Value | Description | | - | - | - | -| `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**12)` (= 4096) | hardfork independent fixed theoretical limit same as `LIMIT_BLOBS_PER_TX` (see EIP 4844) | -| `MAX_BLOBS_PER_BLOCK` | `uint64(2**2)` (= 4) | Maximum number of blobs in a single block limited by `MAX_BLOB_COMMITMENTS_PER_BLOCK` | +| `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**12)` (= 4096) | [New in Deneb:EIP4844] hardfork independent fixed theoretical limit same as `LIMIT_BLOBS_PER_TX` (see EIP 4844) | +| `MAX_BLOBS_PER_BLOCK` | `uint64(2**2)` (= 4) | [New in Deneb:EIP4844] Maximum number of blobs in a single block limited by `MAX_BLOB_COMMITMENTS_PER_BLOCK` | + +*Note*: The blob transactions are packed into the execution payload by the EL/builder with their corresponding blobs being independently transmitted +and are limited by `MAX_DATA_GAS_PER_BLOCK // DATA_GAS_PER_BLOB`. However the CL limit is independently defined by `MAX_BLOBS_PER_BLOCK`. ## Configuration @@ -100,9 +102,9 @@ class BeaconBlockBody(Container): voluntary_exits: List[SignedVoluntaryExit, MAX_VOLUNTARY_EXITS] sync_aggregate: SyncAggregate # Execution - execution_payload: ExecutionPayload # [Modified in Deneb] + execution_payload: ExecutionPayload # [Modified in Deneb:EIP4844] bls_to_execution_changes: List[SignedBLSToExecutionChange, MAX_BLS_TO_EXECUTION_CHANGES] - blob_kzg_commitments: List[KZGCommitment, MAX_BLOB_COMMITMENTS_PER_BLOCK] # [New in Deneb] + blob_kzg_commitments: List[KZGCommitment, MAX_BLOB_COMMITMENTS_PER_BLOCK] # [New in Deneb:EIP4844] ``` #### `ExecutionPayload` @@ -126,8 +128,8 @@ class ExecutionPayload(Container): block_hash: Hash32 # Hash of execution block transactions: List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD] withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] - data_gas_used: uint64 # [New in Deneb] - excess_data_gas: uint64 # [New in Deneb] + data_gas_used: uint64 # [New in Deneb:EIP4844] + excess_data_gas: uint64 # [New in Deneb:EIP4844] ``` #### `ExecutionPayloadHeader` @@ -151,8 +153,8 @@ class ExecutionPayloadHeader(Container): block_hash: Hash32 # Hash of execution block transactions_root: Root withdrawals_root: Root - data_gas_used: uint64 # [New in Deneb] - excess_data_gas: uint64 # [New in Deneb] + data_gas_used: uint64 # [New in Deneb:EIP4844] + excess_data_gas: uint64 # [New in Deneb:EIP4844] ``` ## Helper functions @@ -205,7 +207,7 @@ def verify_and_notify_new_payload(self: ExecutionEngine, if not self.is_valid_block_hash(new_payload_request.execution_payload): return False - # [New in Deneb] + # [New in Deneb:EIP4844] if not self.is_valid_versioned_hashes(new_payload_request): return False @@ -232,11 +234,11 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) - # [New in Deneb] Verify commitments are under limit + # [New in Deneb:EIP4844] Verify commitments are under limit assert len(body.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK # Verify the execution payload is valid - # [Modified in Deneb] Pass `versioned_hashes` to Execution Engine + # [Modified in Deneb:EIP4844] Pass `versioned_hashes` to Execution Engine versioned_hashes = [kzg_commitment_to_versioned_hash(commitment) for commitment in body.blob_kzg_commitments] assert execution_engine.verify_and_notify_new_payload( NewPayloadRequest(execution_payload=payload, versioned_hashes=versioned_hashes) @@ -259,8 +261,8 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi block_hash=payload.block_hash, transactions_root=hash_tree_root(payload.transactions), withdrawals_root=hash_tree_root(payload.withdrawals), - data_gas_used=payload.data_gas_used, # [New in Deneb] - excess_data_gas=payload.excess_data_gas, # [New in Deneb] + data_gas_used=payload.data_gas_used, # [New in Deneb:EIP4844] + excess_data_gas=payload.excess_data_gas, # [New in Deneb:EIP4844] ) ``` diff --git a/specs/deneb/fork-choice.md b/specs/deneb/fork-choice.md index 283e76b1d..60e05f826 100644 --- a/specs/deneb/fork-choice.md +++ b/specs/deneb/fork-choice.md @@ -25,6 +25,8 @@ This is the modification of the fork choice accompanying the Deneb upgrade. #### `is_data_available` +[New in Deneb:EIP4844] + The implementation of `is_data_available` will become more sophisticated during later scaling upgrades. Initially, verification requires every verifying actor to retrieve all matching `Blob`s and `KZGProof`s, and validate them with `verify_blob_kzg_proof_batch`. @@ -76,7 +78,7 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: ) assert store.finalized_checkpoint.root == finalized_checkpoint_block - # [New in Deneb] + # [New in Deneb:EIP4844] # Check if blob data is available # If not, this block MAY be queued and subsequently considered when blob data becomes available assert is_data_available(hash_tree_root(block), block.body.blob_kzg_commitments) diff --git a/specs/deneb/fork.md b/specs/deneb/fork.md index 7fc80275d..ffced6a59 100644 --- a/specs/deneb/fork.md +++ b/specs/deneb/fork.md @@ -83,8 +83,8 @@ def upgrade_to_deneb(pre: capella.BeaconState) -> BeaconState: block_hash=pre.latest_execution_payload_header.block_hash, transactions_root=pre.latest_execution_payload_header.transactions_root, withdrawals_root=pre.latest_execution_payload_header.withdrawals_root, - data_gas_used=uint64(0), # [New in Deneb] - excess_data_gas=uint64(0), # [New in Deneb] + data_gas_used=uint64(0), # [New in Deneb:EIP4844] + excess_data_gas=uint64(0), # [New in Deneb:EIP4844] ) post = BeaconState( # Versioning @@ -126,7 +126,7 @@ def upgrade_to_deneb(pre: capella.BeaconState) -> BeaconState: current_sync_committee=pre.current_sync_committee, next_sync_committee=pre.next_sync_committee, # Execution-layer - latest_execution_payload_header=latest_execution_payload_header, # [Modified in Deneb] + latest_execution_payload_header=latest_execution_payload_header, # [Modified in Deneb:EIP4844] # Withdrawals next_withdrawal_index=pre.next_withdrawal_index, next_withdrawal_validator_index=pre.next_withdrawal_validator_index, diff --git a/specs/deneb/light-client/fork.md b/specs/deneb/light-client/fork.md index 2fd410fc5..f4fd1b396 100644 --- a/specs/deneb/light-client/fork.md +++ b/specs/deneb/light-client/fork.md @@ -41,8 +41,8 @@ def upgrade_lc_header_to_deneb(pre: capella.LightClientHeader) -> LightClientHea block_hash=pre.execution.block_hash, transactions_root=pre.execution.transactions_root, withdrawals_root=pre.execution.withdrawals_root, - data_gas_used=uint64(0), # [New in Deneb] - excess_data_gas=uint64(0), # [New in Deneb] + data_gas_used=uint64(0), # [New in Deneb:EIP4844] + excess_data_gas=uint64(0), # [New in Deneb:EIP4844] ), execution_branch=pre.execution_branch, ) diff --git a/specs/deneb/light-client/full-node.md b/specs/deneb/light-client/full-node.md index 69ba0d3ba..876a6c258 100644 --- a/specs/deneb/light-client/full-node.md +++ b/specs/deneb/light-client/full-node.md @@ -47,7 +47,7 @@ def block_to_light_client_header(block: SignedBeaconBlock) -> LightClientHeader: withdrawals_root=hash_tree_root(payload.withdrawals), ) - # [New in Deneb] + # [New in Deneb:EIP4844] if epoch >= DENEB_FORK_EPOCH: execution_header.data_gas_used = payload.data_gas_used execution_header.excess_data_gas = payload.excess_data_gas diff --git a/specs/deneb/light-client/sync-protocol.md b/specs/deneb/light-client/sync-protocol.md index b7b9b5ab1..38909ddbf 100644 --- a/specs/deneb/light-client/sync-protocol.md +++ b/specs/deneb/light-client/sync-protocol.md @@ -66,7 +66,7 @@ def get_lc_execution_root(header: LightClientHeader) -> Root: def is_valid_light_client_header(header: LightClientHeader) -> bool: epoch = compute_epoch_at_slot(header.beacon.slot) - # [New in Deneb] + # [New in Deneb:EIP4844] if epoch < DENEB_FORK_EPOCH: if header.execution.data_gas_used != uint64(0) or header.execution.excess_data_gas != uint64(0): return False diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index caf2b5bae..e9123c309 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -50,6 +50,8 @@ The specification of these changes continues in the same format as the network s #### `BlobSidecar` +[New in Deneb:EIP4844] + ```python class BlobSidecar(Container): block_root: Root @@ -64,6 +66,8 @@ class BlobSidecar(Container): #### `SignedBlobSidecar` +[New in Deneb:EIP4844] + ```python class SignedBlobSidecar(Container): message: BlobSidecar @@ -72,6 +76,8 @@ class SignedBlobSidecar(Container): #### `BlobIdentifier` +[New in Deneb:EIP4844] + ```python class BlobIdentifier(Container): block_root: Root @@ -107,7 +113,7 @@ The new topics along with the type of the `data` field of a gossipsub message ar | Name | Message Type | | - | - | -| `blob_sidecar_{subnet_id}` | `SignedBlobSidecar` (new) | +| `blob_sidecar_{subnet_id}` | `SignedBlobSidecar` [New in Deneb:EIP4844] | ##### Global topics @@ -124,6 +130,8 @@ New validation: ###### `blob_sidecar_{subnet_id}` +[New in Deneb:EIP4844] + This topic is used to propagate signed blob sidecars, where each blob index maps to some `subnet_id`. The following validations MUST pass before forwarding the `signed_blob_sidecar` on the network, assuming the alias `sidecar = signed_blob_sidecar.message`: @@ -191,7 +199,7 @@ No more than `MAX_REQUEST_BLOCKS_DENEB` may be requested at a time. **Protocol ID:** `/eth2/beacon_chain/req/blob_sidecars_by_root/1/` -New in deneb. +[New in Deneb:EIP4844] The `` field is calculated as `context = compute_fork_digest(fork_version, genesis_validators_root)`: @@ -240,7 +248,7 @@ Clients MAY limit the number of blocks and sidecars in the response. **Protocol ID:** `/eth2/beacon_chain/req/blob_sidecars_by_range/1/` -New in deneb. +[New in Deneb:EIP4844] The `` field is calculated as `context = compute_fork_digest(fork_version, genesis_validators_root)`: diff --git a/specs/deneb/polynomial-commitments.md b/specs/deneb/polynomial-commitments.md index e800d8cc2..8a26b9ade 100644 --- a/specs/deneb/polynomial-commitments.md +++ b/specs/deneb/polynomial-commitments.md @@ -336,7 +336,7 @@ def evaluate_polynomial_in_evaluation_form(polynomial: Polynomial, ### KZG -KZG core functions. These are also defined in Deneb execution specs. +KZG core functions. These are also defined in Deneb execution specs for EIP-4844. #### `blob_to_kzg_commitment` diff --git a/specs/deneb/validator.md b/specs/deneb/validator.md index ae5f26673..5af7c4938 100644 --- a/specs/deneb/validator.md +++ b/specs/deneb/validator.md @@ -98,6 +98,8 @@ All validator responsibilities remain unchanged other than those noted below. ##### Blob KZG commitments +[New in Deneb:EIP4844] + 1. After retrieving the execution payload from the execution engine as specified in Capella, use the `payload_id` to retrieve `blobs`, `blob_kzg_commitments`, and `blob_kzg_proofs` via `get_payload(payload_id).blobs_bundle`. From bab93e8d44b27ccf41a0ddc4993eaae95b144939 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 7 Jun 2023 18:40:02 +0800 Subject: [PATCH 087/110] specially mark EIP7044 changes --- specs/deneb/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index ccf0d6933..7ce8bb1e7 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -282,7 +282,7 @@ def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVolu # Verify the validator has been active long enough assert get_current_epoch(state) >= validator.activation_epoch + SHARD_COMMITTEE_PERIOD # Verify signature - # [Modified in Deneb] + # [Modified in Deneb:EIP7044] domain = compute_domain(DOMAIN_VOLUNTARY_EXIT, CAPELLA_FORK_VERSION, state.genesis_validators_root) signing_root = compute_signing_root(voluntary_exit, domain) assert bls.Verify(validator.pubkey, signing_root, signed_voluntary_exit.signature) From b7e21d6caf94980318f4892d35096c2061b10f2c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 8 Jun 2023 15:05:46 +0800 Subject: [PATCH 088/110] PR feedback and make the marks italic --- specs/deneb/beacon-chain.md | 8 ++++---- specs/deneb/fork-choice.md | 2 +- specs/deneb/p2p-interface.md | 12 ++++++------ specs/deneb/polynomial-commitments.md | 4 ++-- specs/deneb/validator.md | 10 +++++++--- 5 files changed, 20 insertions(+), 16 deletions(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 5c3b4e8a4..41ce76bd2 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -48,8 +48,8 @@ Deneb is a consensus-layer upgrade containing a number of features. Including: | Name | SSZ equivalent | Description | | - | - | - | -| `VersionedHash` | `Bytes32` | [New in Deneb:EIP4844] | -| `BlobIndex` | `uint64` | [New in Deneb:EIP4844] | +| `VersionedHash` | `Bytes32` | *[New in Deneb:EIP4844]* | +| `BlobIndex` | `uint64` | *[New in Deneb:EIP4844]* | ## Constants @@ -72,8 +72,8 @@ Deneb is a consensus-layer upgrade containing a number of features. Including: | Name | Value | Description | | - | - | - | -| `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**12)` (= 4096) | [New in Deneb:EIP4844] hardfork independent fixed theoretical limit same as `LIMIT_BLOBS_PER_TX` (see EIP 4844) | -| `MAX_BLOBS_PER_BLOCK` | `uint64(2**2)` (= 4) | [New in Deneb:EIP4844] Maximum number of blobs in a single block limited by `MAX_BLOB_COMMITMENTS_PER_BLOCK` | +| `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**12)` (= 4096) | *[New in Deneb:EIP4844]* hardfork independent fixed theoretical limit same as `LIMIT_BLOBS_PER_TX` (see EIP 4844) | +| `MAX_BLOBS_PER_BLOCK` | `uint64(2**2)` (= 4) | *[New in Deneb:EIP4844]* Maximum number of blobs in a single block limited by `MAX_BLOB_COMMITMENTS_PER_BLOCK` | *Note*: The blob transactions are packed into the execution payload by the EL/builder with their corresponding blobs being independently transmitted and are limited by `MAX_DATA_GAS_PER_BLOCK // DATA_GAS_PER_BLOB`. However the CL limit is independently defined by `MAX_BLOBS_PER_BLOCK`. diff --git a/specs/deneb/fork-choice.md b/specs/deneb/fork-choice.md index 60e05f826..cc7373c09 100644 --- a/specs/deneb/fork-choice.md +++ b/specs/deneb/fork-choice.md @@ -25,7 +25,7 @@ This is the modification of the fork choice accompanying the Deneb upgrade. #### `is_data_available` -[New in Deneb:EIP4844] +*[New in Deneb:EIP4844]* The implementation of `is_data_available` will become more sophisticated during later scaling upgrades. Initially, verification requires every verifying actor to retrieve all matching `Blob`s and `KZGProof`s, and validate them with `verify_blob_kzg_proof_batch`. diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index e9123c309..0a61ef800 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -50,7 +50,7 @@ The specification of these changes continues in the same format as the network s #### `BlobSidecar` -[New in Deneb:EIP4844] +*[New in Deneb:EIP4844]* ```python class BlobSidecar(Container): @@ -66,7 +66,7 @@ class BlobSidecar(Container): #### `SignedBlobSidecar` -[New in Deneb:EIP4844] +*[New in Deneb:EIP4844]* ```python class SignedBlobSidecar(Container): @@ -76,7 +76,7 @@ class SignedBlobSidecar(Container): #### `BlobIdentifier` -[New in Deneb:EIP4844] +*[New in Deneb:EIP4844]* ```python class BlobIdentifier(Container): @@ -130,7 +130,7 @@ New validation: ###### `blob_sidecar_{subnet_id}` -[New in Deneb:EIP4844] +*[New in Deneb:EIP4844]* This topic is used to propagate signed blob sidecars, where each blob index maps to some `subnet_id`. @@ -199,7 +199,7 @@ No more than `MAX_REQUEST_BLOCKS_DENEB` may be requested at a time. **Protocol ID:** `/eth2/beacon_chain/req/blob_sidecars_by_root/1/` -[New in Deneb:EIP4844] +*[New in Deneb:EIP4844]* The `` field is calculated as `context = compute_fork_digest(fork_version, genesis_validators_root)`: @@ -248,7 +248,7 @@ Clients MAY limit the number of blocks and sidecars in the response. **Protocol ID:** `/eth2/beacon_chain/req/blob_sidecars_by_range/1/` -[New in Deneb:EIP4844] +*[New in Deneb:EIP4844]* The `` field is calculated as `context = compute_fork_digest(fork_version, genesis_validators_root)`: diff --git a/specs/deneb/polynomial-commitments.md b/specs/deneb/polynomial-commitments.md index 8a26b9ade..94299aaa2 100644 --- a/specs/deneb/polynomial-commitments.md +++ b/specs/deneb/polynomial-commitments.md @@ -49,7 +49,7 @@ ## Introduction -This document specifies basic polynomial operations and KZG polynomial commitment operations as they are needed for the Deneb specification. The implementations are not optimized for performance, but readability. All practical implementations should optimize the polynomial operations. +This document specifies basic polynomial operations and KZG polynomial commitment operations that are essential for the implementation of the EIP-4844 feature in the Deneb specification. The implementations are not optimized for performance, but readability. All practical implementations should optimize the polynomial operations. Functions flagged as "Public method" MUST be provided by the underlying KZG library as public functions. All other functions are private functions used internally by the KZG library. @@ -336,7 +336,7 @@ def evaluate_polynomial_in_evaluation_form(polynomial: Polynomial, ### KZG -KZG core functions. These are also defined in Deneb execution specs for EIP-4844. +KZG core functions. These are also defined in Deneb execution specs. #### `blob_to_kzg_commitment` diff --git a/specs/deneb/validator.md b/specs/deneb/validator.md index 5af7c4938..b53c0a042 100644 --- a/specs/deneb/validator.md +++ b/specs/deneb/validator.md @@ -46,12 +46,14 @@ Please see related Beacon Chain doc before continuing and use them as a referenc | Name | Value | Unit | | - | - | :-: | -| `BLOB_SIDECAR_SUBNET_COUNT` | `4` | The number of blob sidecar subnets used in the gossipsub protocol. | +| `BLOB_SIDECAR_SUBNET_COUNT` | `4` | *[New in Deneb:EIP4844]* The number of blob sidecar subnets used in the gossipsub protocol. | ## Helpers ### `BlobsBundle` +*[New in Deneb:EIP4844]* + ```python @dataclass class BlobsBundle(object): @@ -67,7 +69,7 @@ class BlobsBundle(object): class GetPayloadResponse(object): execution_payload: ExecutionPayload block_value: uint256 - blobs_bundle: BlobsBundle + blobs_bundle: BlobsBundle # [New in Deneb:EIP4844] ``` ## Protocol @@ -98,7 +100,7 @@ All validator responsibilities remain unchanged other than those noted below. ##### Blob KZG commitments -[New in Deneb:EIP4844] +*[New in Deneb:EIP4844]* 1. After retrieving the execution payload from the execution engine as specified in Capella, use the `payload_id` to retrieve `blobs`, `blob_kzg_commitments`, and `blob_kzg_proofs` @@ -107,6 +109,8 @@ via `get_payload(payload_id).blobs_bundle`. #### Constructing the `SignedBlobSidecar`s +*[New in Deneb:EIP4844]* + To construct a `SignedBlobSidecar`, a `signed_blob_sidecar` is defined with the necessary context for block and sidecar proposal. ##### Sidecar From 241e52a20230a970a156b7d9501d9a98ed8954c1 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 8 Jun 2023 10:35:03 +0300 Subject: [PATCH 089/110] Whisk (SSLE) with Curdleproofs - rebased (#3342) * Introduce consensus code for Whisk * polish, simplify, clean up (~100 fewer lines) @asn-d6: As discussed, I fixed a few bugs along the way but likely also introduced some bugs :) * minor cleanups and fixes * simplify is_k_commitment_unique * Update beacon-chain.md * Update beacon-chain.md * Initialize `k` in `get_validator_from_deposit()` * minor cleanups * Update beacon-chain.md * Create beacon-chain.md This PR changes the Whisk tracker format to be of the form `(r * pubkey, r * BLS_GT_GENERATOR)` instead of `(r * k * BLS_G1_GENERATOR, r * BLS_G1_GENERATOR)`. This allows for non-interactive tracker registrations from validator pubkeys, removing ~50 lines the code. It also significantly reduces the amount of state overhead. This PR also removes permutation commitments, though those can be easily readded if deemed necessary. * A couple of fixes to the no-registration simplification @asn-d6: Readded a consistency check for `IsValidWhiskOpeningProof` (involving `pubkey` instead of `k_commitment`). * remove unused helpers * use Mary's suggested tracker * Update beacon-chain.md * Revert G_t element optimization This needs its own ethresearch post, and some additional analysis to see if we can do the shuffle ZKP in the allowed timeframe. This reverts commit 8517acabfc1dbb1a35789e6170b5db0bb2c19c9a. * Implement new shuffling strategy Ditch the Feistel logic and instead have each shuffler pick the row they shuffle using their RANDAO reveal. * Curdleproofs edits * working whisk eth2spec * working whisk dummy test * add more boilerplate set up code * rebase constants * Implement even newer and simplified shuffling strategy This commit further simplifies 0faef30fc131d1b471b63a7f16772eeeef548ef8 by removing the entire squareshuffle. The latest version of https://eprint.iacr.org/2022/560 proposes that each shuffler picks random indices from the entire candidate set instead of organizing validators into a square. * Move to _features * remove dummy test * Run doctoc * Change Whisk's previous fork to Capella instead of Bellatrix. Make linter happier. * Fix lint * Fix pylint * Fix mypy issues * Clean-up get_beacon_proposer_index * Fix doc headers * Fix capella link * Update apply_deposit * Rename process_shuffled_trackers --------- Co-authored-by: George Kadianakis Co-authored-by: Justin Co-authored-by: Nalin Bhardwaj Co-authored-by: Hsiao-Wei Wang --- .gitignore | 1 + Makefile | 2 +- setup.py | 39 ++- specs/_features/whisk/beacon-chain.md | 482 ++++++++++++++++++++++++++ specs/_features/whisk/fork.md | 138 ++++++++ 5 files changed, 656 insertions(+), 6 deletions(-) create mode 100644 specs/_features/whisk/beacon-chain.md create mode 100644 specs/_features/whisk/fork.md diff --git a/.gitignore b/.gitignore index 82026c27b..c7012d51b 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ tests/core/pyspec/eth2spec/bellatrix/ tests/core/pyspec/eth2spec/capella/ tests/core/pyspec/eth2spec/deneb/ tests/core/pyspec/eth2spec/eip6110/ +tests/core/pyspec/eth2spec/whisk/ # coverage reports .htmlcov diff --git a/Makefile b/Makefile index ab5521663..5c25d59d3 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/*/*.md) \ $(wildcard $(SPEC_DIR)/_features/*/*/*.md) \ $(wildcard $(SSZ_DIR)/*.md) -ALL_EXECUTABLE_SPECS = phase0 altair bellatrix capella deneb eip6110 +ALL_EXECUTABLE_SPECS = phase0 altair bellatrix capella deneb eip6110 whisk # The parameters for commands. Use `foreach` to avoid listing specs again. COVERAGE_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPECS), --cov=eth2spec.$S.$(TEST_PRESET_TYPE)) PYLINT_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPECS), ./eth2spec/$S) diff --git a/setup.py b/setup.py index c9041010d..f4d0c8719 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ BELLATRIX = 'bellatrix' CAPELLA = 'capella' DENEB = 'deneb' EIP6110 = 'eip6110' +WHISK = 'whisk' # The helper functions that are used when defining constants @@ -733,9 +734,31 @@ from eth2spec.deneb import {preset_name} as deneb ''' +# +# WhiskSpecBuilder +# +class WhiskSpecBuilder(CapellaSpecBuilder): + fork: str = WHISK + + @classmethod + def imports(cls, preset_name: str): + return super().imports(preset_name) + f''' +from eth2spec.capella import {preset_name} as capella +''' + + @classmethod + def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: + # Necessary for custom types `WhiskShuffleProof` and `WhiskTrackerProof` + constants = { + 'WHISK_MAX_SHUFFLE_PROOF_SIZE': spec_object.constant_vars['WHISK_MAX_SHUFFLE_PROOF_SIZE'].value, + 'WHISK_MAX_OPENING_PROOF_SIZE': spec_object.constant_vars['WHISK_MAX_OPENING_PROOF_SIZE'].value, + } + return {**super().hardcoded_custom_type_dep_constants(spec_object), **constants} + + spec_builders = { builder.fork: builder - for builder in (Phase0SpecBuilder, AltairSpecBuilder, BellatrixSpecBuilder, CapellaSpecBuilder, DenebSpecBuilder, EIP6110SpecBuilder) + for builder in (Phase0SpecBuilder, AltairSpecBuilder, BellatrixSpecBuilder, CapellaSpecBuilder, DenebSpecBuilder, EIP6110SpecBuilder, WhiskSpecBuilder) } @@ -1045,7 +1068,7 @@ class PySpecCommand(Command): if len(self.md_doc_paths) == 0: print("no paths were specified, using default markdown file paths for pyspec" " build (spec fork: %s)" % self.spec_fork) - if self.spec_fork in (PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110): + if self.spec_fork in (PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110, WHISK): self.md_doc_paths = """ specs/phase0/beacon-chain.md specs/phase0/fork-choice.md @@ -1053,7 +1076,7 @@ class PySpecCommand(Command): specs/phase0/weak-subjectivity.md specs/phase0/p2p-interface.md """ - if self.spec_fork in (ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110): + if self.spec_fork in (ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110, WHISK): self.md_doc_paths += """ specs/altair/light-client/full-node.md specs/altair/light-client/light-client.md @@ -1065,7 +1088,7 @@ class PySpecCommand(Command): specs/altair/validator.md specs/altair/p2p-interface.md """ - if self.spec_fork in (BELLATRIX, CAPELLA, DENEB, EIP6110): + if self.spec_fork in (BELLATRIX, CAPELLA, DENEB, EIP6110, WHISK): self.md_doc_paths += """ specs/bellatrix/beacon-chain.md specs/bellatrix/fork.md @@ -1074,7 +1097,7 @@ class PySpecCommand(Command): specs/bellatrix/p2p-interface.md sync/optimistic.md """ - if self.spec_fork in (CAPELLA, DENEB, EIP6110): + if self.spec_fork in (CAPELLA, DENEB, EIP6110, WHISK): self.md_doc_paths += """ specs/capella/light-client/fork.md specs/capella/light-client/full-node.md @@ -1104,6 +1127,11 @@ class PySpecCommand(Command): specs/_features/eip6110/beacon-chain.md specs/_features/eip6110/fork.md """ + if self.spec_fork == WHISK: + self.md_doc_paths += """ + specs/_features/whisk/beacon-chain.md + specs/_features/whisk/fork.md + """ if len(self.md_doc_paths) == 0: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) @@ -1259,5 +1287,6 @@ setup( "lru-dict==1.1.8", MARKO_VERSION, "py_arkworks_bls12381==0.3.4", + "curdleproofs @ git+https://github.com/nalinbhardwaj/curdleproofs.pie@master#egg=curdleproofs&subdirectory=curdleproofs", ] ) diff --git a/specs/_features/whisk/beacon-chain.md b/specs/_features/whisk/beacon-chain.md new file mode 100644 index 000000000..7d2ee078f --- /dev/null +++ b/specs/_features/whisk/beacon-chain.md @@ -0,0 +1,482 @@ +# Whisk -- The Beacon Chain + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + + +- [Introduction](#introduction) +- [Constants](#constants) +- [Cryptography](#cryptography) + - [BLS](#bls) + - [Curdleproofs and opening proofs](#curdleproofs-and-opening-proofs) +- [Epoch processing](#epoch-processing) + - [`WhiskTracker`](#whisktracker) + - [`Validator`](#validator) + - [`BeaconState`](#beaconstate) +- [Block processing](#block-processing) + - [Block header](#block-header) + - [`BeaconBlock`](#beaconblock) + - [Whisk](#whisk) + - [`BeaconBlockBody`](#beaconblockbody) + - [Deposits](#deposits) + - [`get_beacon_proposer_index`](#get_beacon_proposer_index) + + + + +## Introduction + +This document details the beacon chain additions and changes of to support the Whisk SSLE, + +*Note:* This specification is built upon [Capella](../../capella/beacon-chain.md) and is under active development. + +## Constants + +| Name | Value | Description | +| ---------------------------------- | -------------------------- | ----------------------------------------------------------- | +| `WHISK_CANDIDATE_TRACKERS_COUNT` | `uint64(2**14)` (= 16,384) | number of candidate trackers | +| `WHISK_PROPOSER_TRACKERS_COUNT` | `uint64(2**13)` (= 8,192) | number of proposer trackers | +| `WHISK_EPOCHS_PER_SHUFFLING_PHASE` | `Epoch(2**8)` (= 256) | epochs per shuffling phase | +| `WHISK_VALIDATORS_PER_SHUFFLE` | `uint64(2**7)` (= 128) | number of validators shuffled per shuffle step | +| `WHISK_PROPOSER_SELECTION_GAP` | `Epoch(2)` | gap between proposer selection and the block proposal phase | +| `WHISK_MAX_SHUFFLE_PROOF_SIZE` | `uint64(2**15)` | max size of a shuffle proof | +| `WHISK_MAX_OPENING_PROOF_SIZE` | `uint64(2**10)` | max size of a opening proof | + +| Name | Value | +| ---------------------------------- | -------------------------- | +| `DOMAIN_WHISK_CANDIDATE_SELECTION` | `DomainType('0x07000000')` | +| `DOMAIN_WHISK_SHUFFLE` | `DomainType('0x07100000')` | +| `DOMAIN_WHISK_PROPOSER_SELECTION` | `DomainType('0x07200000')` | + +## Cryptography + +### BLS + +| Name | SSZ equivalent | Description | +| ------------------- | ---------------------------------------- | ----------------------------- | +| `BLSFieldElement` | `uint256` | BLS12-381 scalar | +| `BLSG1Point` | `Bytes48` | compressed BLS12-381 G1 point | +| `WhiskShuffleProof` | `ByteList[WHISK_MAX_SHUFFLE_PROOF_SIZE]` | Serialized shuffle proof | +| `WhiskTrackerProof` | `ByteList[WHISK_MAX_OPENING_PROOF_SIZE]` | Serialized tracker proof | + +*Note*: A subgroup check MUST be performed when deserializing a `BLSG1Point` for use in any of the functions below. + +```python +def BLSG1ScalarMultiply(scalar: BLSFieldElement, point: BLSG1Point) -> BLSG1Point: + return bls.G1_to_bytes48(bls.multiply(bls.bytes48_to_G1(point), scalar)) +``` + +```python +def bytes_to_bls_field(b: Bytes32) -> BLSFieldElement: + """ + Convert bytes to a BLS field scalar. The output is not uniform over the BLS field. + TODO: Deneb will introduces this helper too. Should delete it once it's rebased to post-Deneb. + """ + field_element = int.from_bytes(b, ENDIANNESS) + assert field_element < BLS_MODULUS + return BLSFieldElement(field_element) +``` + +| Name | Value | +| ------------------ | ------------------------------------------------------------------------------- | +| `BLS_G1_GENERATOR` | `bls.G1_to_bytes48(bls.G1)` | +| `BLS_MODULUS` | `52435875175126190479447740508185965837690552500527637822603658699938581184513` | + +### Curdleproofs and opening proofs + +Note that Curdleproofs (Whisk Shuffle Proofs), the tracker opening proofs and all related data structures and verifier code (along with tests) is specified in [curdleproofs.pie](https://github.com/nalinbhardwaj/curdleproofs.pie/tree/verifier-only) repository. + +```python +def IsValidWhiskShuffleProof(pre_shuffle_trackers: Sequence[WhiskTracker], + post_shuffle_trackers: Sequence[WhiskTracker], + M: BLSG1Point, + shuffle_proof: WhiskShuffleProof) -> bool: + """ + Verify `post_shuffle_trackers` is a permutation of `pre_shuffle_trackers`. + Defined in https://github.com/nalinbhardwaj/curdleproofs.pie/tree/verifier-only. + """ + # pylint: disable=unused-argument + return True +``` + +```python +def IsValidWhiskOpeningProof(tracker: WhiskTracker, + k_commitment: BLSG1Point, + tracker_proof: WhiskTrackerProof) -> bool: + """ + Verify knowledge of `k` such that `tracker.k_r_G == k * tracker.r_G` and `k_commitment == k * BLS_G1_GENERATOR`. + Defined in https://github.com/nalinbhardwaj/curdleproofs.pie/tree/verifier-only. + """ + # pylint: disable=unused-argument + return True +``` + +## Epoch processing + +### `WhiskTracker` + +```python +class WhiskTracker(Container): + r_G: BLSG1Point # r * G + k_r_G: BLSG1Point # k * r * G +``` + +### `Validator` + +```python +class Validator(Container): + pubkey: BLSPubkey + withdrawal_credentials: Bytes32 # Commitment to pubkey for withdrawals + effective_balance: Gwei # Balance at stake + slashed: boolean + # Status epochs + activation_eligibility_epoch: Epoch # When criteria for activation were met + activation_epoch: Epoch + exit_epoch: Epoch + withdrawable_epoch: Epoch # When validator can withdraw funds + whisk_tracker: WhiskTracker # Whisk tracker (r * G, k * r * G) [New in Whisk] + whisk_k_commitment: BLSG1Point # Whisk k commitment k * BLS_G1_GENERATOR [New in Whisk] +``` + +### `BeaconState` + +```python +class BeaconState(Container): + # Versioning + genesis_time: uint64 + genesis_validators_root: Root + slot: Slot + fork: Fork + # History + latest_block_header: BeaconBlockHeader + block_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] + state_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] + historical_roots: List[Root, HISTORICAL_ROOTS_LIMIT] # Frozen in Capella, replaced by historical_summaries + # Eth1 + eth1_data: Eth1Data + eth1_data_votes: List[Eth1Data, EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH] + eth1_deposit_index: uint64 + # Registry + validators: List[Validator, VALIDATOR_REGISTRY_LIMIT] # [Modified in Whisk] + balances: List[Gwei, VALIDATOR_REGISTRY_LIMIT] + # Randomness + randao_mixes: Vector[Bytes32, EPOCHS_PER_HISTORICAL_VECTOR] + # Slashings + slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR] # Per-epoch sums of slashed effective balances + # Participation + previous_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT] + current_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT] + # Finality + justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH] # Bit set for every recent justified epoch + previous_justified_checkpoint: Checkpoint + current_justified_checkpoint: Checkpoint + finalized_checkpoint: Checkpoint + # Inactivity + inactivity_scores: List[uint64, VALIDATOR_REGISTRY_LIMIT] + # Sync + current_sync_committee: SyncCommittee + next_sync_committee: SyncCommittee + # Execution + latest_execution_payload_header: ExecutionPayloadHeader + # Withdrawals + next_withdrawal_index: WithdrawalIndex + next_withdrawal_validator_index: ValidatorIndex + # Deep history valid from Capella onwards + historical_summaries: List[HistoricalSummary, HISTORICAL_ROOTS_LIMIT] + whisk_candidate_trackers: Vector[WhiskTracker, WHISK_CANDIDATE_TRACKERS_COUNT] # [New in Whisk] + whisk_proposer_trackers: Vector[WhiskTracker, WHISK_PROPOSER_TRACKERS_COUNT] # [New in Whisk] +``` + +```python +def select_whisk_trackers(state: BeaconState, epoch: Epoch) -> None: + # Select proposer trackers from candidate trackers + proposer_seed = get_seed(state, epoch - WHISK_PROPOSER_SELECTION_GAP, DOMAIN_WHISK_PROPOSER_SELECTION) + for i in range(WHISK_PROPOSER_TRACKERS_COUNT): + index = compute_shuffled_index(uint64(i), uint64(len(state.whisk_candidate_trackers)), proposer_seed) + state.whisk_proposer_trackers[i] = state.whisk_candidate_trackers[index] + + # Select candidate trackers from active validator trackers + active_validator_indices = get_active_validator_indices(state, epoch) + for i in range(WHISK_CANDIDATE_TRACKERS_COUNT): + seed = hash(get_seed(state, epoch, DOMAIN_WHISK_CANDIDATE_SELECTION) + uint_to_bytes(i)) + candidate_index = compute_proposer_index(state, active_validator_indices, seed) # sample by effective balance + state.whisk_candidate_trackers[i] = state.validators[candidate_index].whisk_tracker +``` + +```python +def process_whisk_updates(state: BeaconState) -> None: + next_epoch = Epoch(get_current_epoch(state) + 1) + if next_epoch % WHISK_EPOCHS_PER_SHUFFLING_PHASE == 0: # select trackers at the start of shuffling phases + select_whisk_trackers(state, next_epoch) +``` + +```python +def process_epoch(state: BeaconState) -> None: + process_justification_and_finalization(state) + process_inactivity_updates(state) + process_rewards_and_penalties(state) + process_registry_updates(state) + process_slashings(state) + process_eth1_data_reset(state) + process_effective_balance_updates(state) + process_slashings_reset(state) + process_randao_mixes_reset(state) + process_historical_summaries_update(state) + process_participation_flag_updates(state) + process_sync_committee_updates(state) + process_whisk_updates(state) # [New in Whisk] +``` + +## Block processing + +### Block header + +#### `BeaconBlock` + +```python +class BeaconBlock(Container): + slot: Slot + proposer_index: ValidatorIndex + parent_root: Root + state_root: Root + body: BeaconBlockBody + whisk_opening_proof: WhiskTrackerProof # [New in Whisk] +``` + +```python +def process_whisk_opening_proof(state: BeaconState, block: BeaconBlock) -> None: + tracker = state.whisk_proposer_trackers[state.slot % WHISK_PROPOSER_TRACKERS_COUNT] + k_commitment = state.validators[block.proposer_index].whisk_k_commitment + assert IsValidWhiskOpeningProof(tracker, k_commitment, block.whisk_opening_proof) +``` + +Removed `assert block.proposer_index == get_beacon_proposer_index(state)` check in Whisk. + +```python +def process_block_header(state: BeaconState, block: BeaconBlock) -> None: + # Verify that the slots match + assert block.slot == state.slot + # Verify that the block is newer than latest block header + assert block.slot > state.latest_block_header.slot + + # # Verify that proposer index is the correct index + # assert block.proposer_index == get_beacon_proposer_index(state) + + # Verify that the parent matches + assert block.parent_root == hash_tree_root(state.latest_block_header) + # Cache current block as the new latest block + state.latest_block_header = BeaconBlockHeader( + slot=block.slot, + proposer_index=block.proposer_index, + parent_root=block.parent_root, + state_root=Bytes32(), # Overwritten in the next process_slot call + body_root=hash_tree_root(block.body), + ) + + # Verify proposer is not slashed + proposer = state.validators[block.proposer_index] + assert not proposer.slashed + process_whisk_opening_proof(state, block) # [New in Whisk] +``` + +### Whisk + +#### `BeaconBlockBody` + +```python +class BeaconBlockBody(capella.BeaconBlockBody): + randao_reveal: BLSSignature + eth1_data: Eth1Data # Eth1 data vote + graffiti: Bytes32 # Arbitrary data + # Operations + proposer_slashings: List[ProposerSlashing, MAX_PROPOSER_SLASHINGS] + attester_slashings: List[AttesterSlashing, MAX_ATTESTER_SLASHINGS] + attestations: List[Attestation, MAX_ATTESTATIONS] + deposits: List[Deposit, MAX_DEPOSITS] + voluntary_exits: List[SignedVoluntaryExit, MAX_VOLUNTARY_EXITS] + sync_aggregate: SyncAggregate + # Execution + execution_payload: ExecutionPayload + # Capella operations + bls_to_execution_changes: List[SignedBLSToExecutionChange, MAX_BLS_TO_EXECUTION_CHANGES] + whisk_post_shuffle_trackers: Vector[WhiskTracker, WHISK_VALIDATORS_PER_SHUFFLE] # [New in Whisk] + whisk_shuffle_proof: WhiskShuffleProof # [New in Whisk] + whisk_shuffle_proof_M_commitment: BLSG1Point # [New in Whisk] + whisk_registration_proof: WhiskTrackerProof # [New in Whisk] + whisk_tracker: WhiskTracker # [New in Whisk] + whisk_k_commitment: BLSG1Point # [New in Whisk] +``` + +```python +def get_shuffle_indices(randao_reveal: BLSSignature) -> Sequence[uint64]: + """ + Given a `randao_reveal` return the list of indices that got shuffled from the entire candidate set + """ + indices = [] + for i in WHISK_VALIDATORS_PER_SHUFFLE: + # XXX ensure we are not suffering from modulo bias + shuffle_index = uint256(hash(randao_reveal + uint_to_bytes(i))) % WHISK_CANDIDATE_TRACKERS_COUNT + indices.append(shuffle_index) + + return indices +``` + +```python +def process_shuffled_trackers(state: BeaconState, body: BeaconBlockBody) -> None: + # Check the shuffle proof + shuffle_indices = get_shuffle_indices(body.randao_reveal) + pre_shuffle_trackers = [state.whisk_candidate_trackers[i] for i in shuffle_indices] + post_shuffle_trackers = body.whisk_post_shuffle_trackers + + shuffle_epoch = get_current_epoch(state) % WHISK_EPOCHS_PER_SHUFFLING_PHASE + if shuffle_epoch + WHISK_PROPOSER_SELECTION_GAP + 1 >= WHISK_EPOCHS_PER_SHUFFLING_PHASE: + # Require unchanged trackers during cooldown + assert pre_shuffle_trackers == post_shuffle_trackers + else: + # Require shuffled trackers during shuffle + assert IsValidWhiskShuffleProof( + pre_shuffle_trackers, + post_shuffle_trackers, + body.whisk_shuffle_proof_M_commitment, + body.whisk_shuffle_proof, + ) + + # Shuffle candidate trackers + for i, shuffle_index in enumerate(shuffle_indices): + state.whisk_candidate_trackers[shuffle_index] = post_shuffle_trackers[i] +``` + +```python +def is_k_commitment_unique(state: BeaconState, k_commitment: BLSG1Point) -> bool: + return all([validator.whisk_k_commitment != k_commitment for validator in state.validators]) +``` + +```python +def process_whisk(state: BeaconState, body: BeaconBlockBody) -> None: + process_shuffled_trackers(state, body) + + # Overwrite all validator Whisk fields (first Whisk proposal) or just the permutation commitment (next proposals) + proposer = state.validators[get_beacon_proposer_index(state)] + if proposer.whisk_tracker.r_G == BLS_G1_GENERATOR: # first Whisk proposal + assert body.whisk_tracker.r_G != BLS_G1_GENERATOR + assert is_k_commitment_unique(state, body.whisk_k_commitment) + assert IsValidWhiskOpeningProof( + body.whisk_tracker, + body.whisk_k_commitment, + body.whisk_registration_proof, + ) + proposer.whisk_tracker = body.whisk_tracker + proposer.whisk_k_commitment = body.whisk_k_commitment + else: # next Whisk proposals + assert body.whisk_registration_proof == WhiskTrackerProof() + assert body.whisk_tracker == WhiskTracker() + assert body.whisk_k_commitment == BLSG1Point() + assert body.whisk_shuffle_proof_M_commitment == BLSG1Point() +``` + +```python +def process_block(state: BeaconState, block: BeaconBlock) -> None: + process_block_header(state, block) + if is_execution_enabled(state, block.body): + process_withdrawals(state, block.body.execution_payload) + process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) + process_randao(state, block.body) + process_eth1_data(state, block.body) + process_operations(state, block.body) + process_sync_aggregate(state, block.body.sync_aggregate) + process_whisk(state, block.body) # [New in Whisk] +``` + +### Deposits + +```python +def get_unique_whisk_k(state: BeaconState, validator_index: ValidatorIndex) -> BLSFieldElement: + counter = 0 + while True: + # hash `validator_index || counter` + k = BLSFieldElement(bytes_to_bls_field(hash(uint_to_bytes(validator_index) + uint_to_bytes(uint64(counter))))) + if is_k_commitment_unique(state, BLSG1ScalarMultiply(k, BLS_G1_GENERATOR)): + return k # unique by trial and error + counter += 1 +``` + +```python +def get_initial_commitments(k: BLSFieldElement) -> Tuple[BLSG1Point, WhiskTracker]: + return ( + BLSG1ScalarMultiply(k, BLS_G1_GENERATOR), + WhiskTracker(r_G=BLS_G1_GENERATOR, k_r_G=BLSG1ScalarMultiply(k, BLS_G1_GENERATOR)) + ) +``` + +```python +def get_validator_from_deposit_whisk( + state: BeaconState, + pubkey: BLSPubkey, + withdrawal_credentials: Bytes32, + amount: uint64 +) -> Validator: + effective_balance = min(amount - amount % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) + k = get_unique_whisk_k(state, ValidatorIndex(len(state.validators))) + whisk_k_commitment, whisk_tracker = get_initial_commitments(k) + + validator = Validator( + pubkey=pubkey, + withdrawal_credentials=withdrawal_credentials, + activation_eligibility_epoch=FAR_FUTURE_EPOCH, + activation_epoch=FAR_FUTURE_EPOCH, + exit_epoch=FAR_FUTURE_EPOCH, + withdrawable_epoch=FAR_FUTURE_EPOCH, + effective_balance=effective_balance, + # Whisk fields + whisk_tracker=whisk_tracker, + whisk_k_commitment=whisk_k_commitment, + ) + return validator +``` + +```python +def apply_deposit(state: BeaconState, + pubkey: BLSPubkey, + withdrawal_credentials: Bytes32, + amount: uint64, + signature: BLSSignature) -> None: + validator_pubkeys = [validator.pubkey for validator in state.validators] + if pubkey not in validator_pubkeys: + # Verify the deposit signature (proof of possession) which is not checked by the deposit contract + deposit_message = DepositMessage( + pubkey=pubkey, + withdrawal_credentials=withdrawal_credentials, + amount=amount, + ) + domain = compute_domain(DOMAIN_DEPOSIT) # Fork-agnostic domain since deposits are valid across forks + signing_root = compute_signing_root(deposit_message, domain) + # Initialize validator if the deposit signature is valid + if bls.Verify(pubkey, signing_root, signature): + index = get_index_for_new_validator(state) + validator = get_validator_from_deposit_whisk(state, pubkey, withdrawal_credentials, amount) + set_or_append_list(state.validators, index, validator) + set_or_append_list(state.balances, index, amount) + # [New in Altair] + set_or_append_list(state.previous_epoch_participation, index, ParticipationFlags(0b0000_0000)) + set_or_append_list(state.current_epoch_participation, index, ParticipationFlags(0b0000_0000)) + set_or_append_list(state.inactivity_scores, index, uint64(0)) + else: + # Increase balance by deposit amount + index = ValidatorIndex(validator_pubkeys.index(pubkey)) + increase_balance(state, index, amount) +``` + +### `get_beacon_proposer_index` + +```python +def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: + """ + Return the beacon proposer index at the current slot. + """ + assert state.latest_block_header.slot == state.slot # sanity check `process_block_header` has been called + return state.latest_block_header.proposer_index +``` diff --git a/specs/_features/whisk/fork.md b/specs/_features/whisk/fork.md new file mode 100644 index 000000000..189d8c5b3 --- /dev/null +++ b/specs/_features/whisk/fork.md @@ -0,0 +1,138 @@ +# Whisk -- Fork Logic + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + + +- [Introduction](#introduction) +- [Configuration](#configuration) +- [Fork to Whisk](#fork-to-whisk) + + + + +## Introduction + +This document describes the process of Whisk upgrade. + + +``` +""" + WHISK_FORK_EPOCH + | cooldown + | | || + v vsvv + --+~~~~~~~~~~~~~~~~~~~~~----+- + shuffling ^ + | + | + proposer selection + candidate selection +""" +``` + +## Configuration + +Warning: this configuration is not definitive. + +| Name | Value | +| -------------------- | ----------------------- | +| `WHISK_FORK_VERSION` | `Version('0x05000000')` | +| `WHISK_FORK_EPOCH` | `Epoch(18446744073709551615)` **TBD** | + +## Fork to Whisk + +If `state.slot % SLOTS_PER_EPOCH == 0` and `compute_epoch_at_slot(state.slot) == WHISK_FORK_EPOCH`, an irregular state change is made to upgrade to Whisk. `WHISK_FORK_EPOCH` must be a multiple of `WHISK_RUN_DURATION_IN_EPOCHS`. + +The upgrade occurs after the completion of the inner loop of `process_slots` that sets `state.slot` equal to `WHISK_FORK_EPOCH * SLOTS_PER_EPOCH`. + +This ensures that we drop right into the beginning of the shuffling phase but without `process_whisk_epoch()` triggering for this Whisk run. Hence we handle all the setup ourselves in `upgrade_to_whisk()` below. + +```python +def whisk_candidate_selection(state: BeaconState, epoch: Epoch) -> None: + # TODO + # pylint: disable=unused-argument + pass +``` + +```python +def whisk_proposer_selection(state: BeaconState, epoch: Epoch) -> None: + # TODO + # pylint: disable=unused-argument + pass +``` + +```python +def upgrade_to_whisk(pre: bellatrix.BeaconState) -> BeaconState: + epoch = bellatrix.get_current_epoch(pre) + post = BeaconState( + # Versioning + genesis_time=pre.genesis_time, + genesis_validators_root=pre.genesis_validators_root, + slot=pre.slot, + fork=Fork( + previous_version=pre.fork.current_version, + current_version=WHISK_FORK_VERSION, + epoch=epoch, + ), + # History + latest_block_header=pre.latest_block_header, + block_roots=pre.block_roots, + state_roots=pre.state_roots, + historical_roots=pre.historical_roots, + # Eth1 + eth1_data=pre.eth1_data, + eth1_data_votes=pre.eth1_data_votes, + eth1_deposit_index=pre.eth1_deposit_index, + # Registry + validators=[], + balances=pre.balances, + # Randomness + randao_mixes=pre.randao_mixes, + # Slashings + slashings=pre.slashings, + # Participation + previous_epoch_participation=pre.previous_epoch_participation, + current_epoch_participation=pre.current_epoch_participation, + # Finality + justification_bits=pre.justification_bits, + previous_justified_checkpoint=pre.previous_justified_checkpoint, + current_justified_checkpoint=pre.current_justified_checkpoint, + finalized_checkpoint=pre.finalized_checkpoint, + # Inactivity + inactivity_scores=pre.inactivity_Scores, + ) + + # Initialize all validators with predictable commitments + for val_index, pre_validator in enumerate(pre.validators): + whisk_commitment, whisk_tracker = get_initial_commitments(get_unique_whisk_k(post, ValidatorIndex(val_index))) + + post_validator = Validator( + pubkey=pre_validator.pubkey, + withdrawal_credentials=pre_validator.withdrawal_credentials, + effective_balance=pre_validator.effective_balance, + slashed=pre_validator.slashed, + activation_eligibility_epoch=pre_validator.activation_eligibility_epoch, + activation_epoch=pre_validator.activation_epoch, + exit_epoch=pre_validator.exit_epoch, + withdrawable_epoch=pre_validator.withdrawable_epoch, + whisk_commitment=whisk_commitment, + whisk_tracker=whisk_tracker, + ) + post.validators.append(post_validator) + + # Do a candidate selection followed by a proposer selection so that we have proposers for the upcoming day + # Use an old epoch when selecting candidates so that we don't get the same seed as in the next candidate selection + whisk_candidate_selection(post, epoch - WHISK_PROPOSER_SELECTION_GAP - 1) + whisk_proposer_selection(post, epoch) + + # Do a final round of candidate selection. + # We need it so that we have something to shuffle over the upcoming shuffling phase. + whisk_candidate_selection(post, epoch) + + return post +``` From 68bb5ee5079ababf7be4cf7b6a1e0a9cada09f43 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 8 Jun 2023 15:00:49 +0300 Subject: [PATCH 090/110] Ensure make lint resolves to the right pylint and mypy versions --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 5c25d59d3..6c852a1e9 100644 --- a/Makefile +++ b/Makefile @@ -146,8 +146,8 @@ codespell: lint: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ flake8 --config $(LINTER_CONFIG_FILE) ./eth2spec \ - && pylint --rcfile $(LINTER_CONFIG_FILE) $(PYLINT_SCOPE) \ - && mypy --config-file $(LINTER_CONFIG_FILE) $(MYPY_SCOPE) + && python -m pylint --rcfile $(LINTER_CONFIG_FILE) $(PYLINT_SCOPE) \ + && python -m mypy --config-file $(LINTER_CONFIG_FILE) $(MYPY_SCOPE) lint_generators: pyspec . venv/bin/activate; cd $(TEST_GENERATORS_DIR); \ From 706824cf959df3aaa5aef543d573aaae00c411a3 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 8 Jun 2023 20:25:03 +0300 Subject: [PATCH 091/110] Drop need to declare spec feature files --- setup.py | 121 ++++++++++++++++++++++++++----------------------------- 1 file changed, 57 insertions(+), 64 deletions(-) diff --git a/setup.py b/setup.py index f4d0c8719..f9b2438d4 100644 --- a/setup.py +++ b/setup.py @@ -50,6 +50,25 @@ DENEB = 'deneb' EIP6110 = 'eip6110' WHISK = 'whisk' +PREVIOUS_FORK_OF = { + PHASE0: None, + ALTAIR: PHASE0, + BELLATRIX: ALTAIR, + CAPELLA: BELLATRIX, + DENEB: CAPELLA, + EIP6110: DENEB, + WHISK: CAPELLA, +} + +ALL_FORKS = list(PREVIOUS_FORK_OF.keys()) + +IGNORE_SPEC_FILES = [ + "specs/phase0/deposit-contract.md" +] + +EXTRA_SPEC_FILES = { + BELLATRIX: "sync/optimistic.md" +} # The helper functions that are used when defining constants CONSTANT_DEP_SUNDRY_CONSTANTS_FUNCTIONS = ''' @@ -96,6 +115,30 @@ class SpecObject(NamedTuple): dataclasses: Dict[str, str] +def is_post_fork(a, b) -> bool: + """ + Returns true if fork a is after b, or if a == b + """ + if a == b: + return True + + prev_fork = PREVIOUS_FORK_OF[a] + if prev_fork == b: + return True + elif prev_fork == None: + return False + else: + return is_post_fork(prev_fork, b) + +def get_fork_directory(fork): + dir1 = f'specs/{fork}' + if os.path.exists(dir1): + return dir1 + dir2 = f'specs/_features/{fork}' + if os.path.exists(dir2): + return dir2 + raise FileNotFoundError(f"No directory found for fork: {fork}") + def _get_name_from_heading(heading: Heading) -> Optional[str]: last_child = heading.children[-1] if isinstance(last_child, CodeSpan): @@ -1068,70 +1111,20 @@ class PySpecCommand(Command): if len(self.md_doc_paths) == 0: print("no paths were specified, using default markdown file paths for pyspec" " build (spec fork: %s)" % self.spec_fork) - if self.spec_fork in (PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110, WHISK): - self.md_doc_paths = """ - specs/phase0/beacon-chain.md - specs/phase0/fork-choice.md - specs/phase0/validator.md - specs/phase0/weak-subjectivity.md - specs/phase0/p2p-interface.md - """ - if self.spec_fork in (ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110, WHISK): - self.md_doc_paths += """ - specs/altair/light-client/full-node.md - specs/altair/light-client/light-client.md - specs/altair/light-client/p2p-interface.md - specs/altair/light-client/sync-protocol.md - specs/altair/beacon-chain.md - specs/altair/bls.md - specs/altair/fork.md - specs/altair/validator.md - specs/altair/p2p-interface.md - """ - if self.spec_fork in (BELLATRIX, CAPELLA, DENEB, EIP6110, WHISK): - self.md_doc_paths += """ - specs/bellatrix/beacon-chain.md - specs/bellatrix/fork.md - specs/bellatrix/fork-choice.md - specs/bellatrix/validator.md - specs/bellatrix/p2p-interface.md - sync/optimistic.md - """ - if self.spec_fork in (CAPELLA, DENEB, EIP6110, WHISK): - self.md_doc_paths += """ - specs/capella/light-client/fork.md - specs/capella/light-client/full-node.md - specs/capella/light-client/p2p-interface.md - specs/capella/light-client/sync-protocol.md - specs/capella/beacon-chain.md - specs/capella/fork.md - specs/capella/fork-choice.md - specs/capella/validator.md - specs/capella/p2p-interface.md - """ - if self.spec_fork in (DENEB, EIP6110): - self.md_doc_paths += """ - specs/deneb/light-client/fork.md - specs/deneb/light-client/full-node.md - specs/deneb/light-client/p2p-interface.md - specs/deneb/light-client/sync-protocol.md - specs/deneb/beacon-chain.md - specs/deneb/fork.md - specs/deneb/fork-choice.md - specs/deneb/polynomial-commitments.md - specs/deneb/p2p-interface.md - specs/deneb/validator.md - """ - if self.spec_fork == EIP6110: - self.md_doc_paths += """ - specs/_features/eip6110/beacon-chain.md - specs/_features/eip6110/fork.md - """ - if self.spec_fork == WHISK: - self.md_doc_paths += """ - specs/_features/whisk/beacon-chain.md - specs/_features/whisk/fork.md - """ + self.md_doc_paths = "" + + for fork in ALL_FORKS: + if is_post_fork(self.spec_fork, fork): + # Append all files in fork directory recursively + for root, dirs, files in os.walk(get_fork_directory(fork)): + for filename in files: + filepath = os.path.join(root, filename) + if filepath.endswith('.md') and filepath not in IGNORE_SPEC_FILES: + self.md_doc_paths += filepath + "\n" + # Append extra files if any + if fork in EXTRA_SPEC_FILES: + self.md_doc_paths += EXTRA_SPEC_FILES[fork] + "\n" + if len(self.md_doc_paths) == 0: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) From efd2429b3f14f7bccaad342a75fdf843b21c0951 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Fri, 9 Jun 2023 11:28:06 +0300 Subject: [PATCH 092/110] Move whisk_opening_proof to block body (#3408) --- specs/_features/whisk/beacon-chain.md | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/specs/_features/whisk/beacon-chain.md b/specs/_features/whisk/beacon-chain.md index 7d2ee078f..bad818837 100644 --- a/specs/_features/whisk/beacon-chain.md +++ b/specs/_features/whisk/beacon-chain.md @@ -19,7 +19,6 @@ - [`BeaconState`](#beaconstate) - [Block processing](#block-processing) - [Block header](#block-header) - - [`BeaconBlock`](#beaconblock) - [Whisk](#whisk) - [`BeaconBlockBody`](#beaconblockbody) - [Deposits](#deposits) @@ -235,23 +234,11 @@ def process_epoch(state: BeaconState) -> None: ### Block header -#### `BeaconBlock` - -```python -class BeaconBlock(Container): - slot: Slot - proposer_index: ValidatorIndex - parent_root: Root - state_root: Root - body: BeaconBlockBody - whisk_opening_proof: WhiskTrackerProof # [New in Whisk] -``` - ```python def process_whisk_opening_proof(state: BeaconState, block: BeaconBlock) -> None: tracker = state.whisk_proposer_trackers[state.slot % WHISK_PROPOSER_TRACKERS_COUNT] k_commitment = state.validators[block.proposer_index].whisk_k_commitment - assert IsValidWhiskOpeningProof(tracker, k_commitment, block.whisk_opening_proof) + assert IsValidWhiskOpeningProof(tracker, k_commitment, block.body.whisk_opening_proof) ``` Removed `assert block.proposer_index == get_beacon_proposer_index(state)` check in Whisk. @@ -303,6 +290,8 @@ class BeaconBlockBody(capella.BeaconBlockBody): execution_payload: ExecutionPayload # Capella operations bls_to_execution_changes: List[SignedBLSToExecutionChange, MAX_BLS_TO_EXECUTION_CHANGES] + # Whisk + whisk_opening_proof: WhiskTrackerProof # [New in Whisk] whisk_post_shuffle_trackers: Vector[WhiskTracker, WHISK_VALIDATORS_PER_SHUFFLE] # [New in Whisk] whisk_shuffle_proof: WhiskShuffleProof # [New in Whisk] whisk_shuffle_proof_M_commitment: BLSG1Point # [New in Whisk] From 7b69f176e2e836ccff5b58f655b39b9c583f00cd Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 9 Jun 2023 08:54:16 -0600 Subject: [PATCH 093/110] bump VERSION.txt --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index 2a6fa03c0..54dd11e46 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -1.4.0-alpha.1 +1.4.0-alpha.2 From 45f3432019336b5cb1196037e6f5d92a6dc5aeba Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 9 Jun 2023 23:10:00 +0800 Subject: [PATCH 094/110] Update `MAX_BLOBS_PER_BLOCK` to `6` and add Deneb networking configs to yaml files (#3410) --- configs/mainnet.yaml | 8 ++++++++ configs/minimal.yaml | 8 ++++++++ presets/mainnet/deneb.yaml | 4 ++-- presets/minimal/deneb.yaml | 4 ++-- specs/deneb/beacon-chain.md | 2 +- specs/deneb/fork-choice.md | 3 ++- .../test/deneb/unittests/test_config_invariants.py | 8 ++++++++ 7 files changed, 31 insertions(+), 6 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index baf489739..80a240185 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -124,3 +124,11 @@ ATTESTATION_SUBNET_COUNT: 64 ATTESTATION_SUBNET_EXTRA_BITS: 0 # ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS ATTESTATION_SUBNET_PREFIX_BITS: 6 + +# Deneb +# `2**7` (=128) +MAX_REQUEST_BLOCKS_DENEB: 128 +# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK +MAX_REQUEST_BLOB_SIDECARS: 768 +# `2**12` (= 4096 epochs, ~18 days) +MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 43f1fc83f..82a813a9b 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -125,3 +125,11 @@ ATTESTATION_SUBNET_COUNT: 64 ATTESTATION_SUBNET_EXTRA_BITS: 0 # ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS ATTESTATION_SUBNET_PREFIX_BITS: 6 + +# Deneb +# `2**7` (=128) +MAX_REQUEST_BLOCKS_DENEB: 128 +# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK +MAX_REQUEST_BLOB_SIDECARS: 768 +# `2**12` (= 4096 epochs, ~18 days) +MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 diff --git a/presets/mainnet/deneb.yaml b/presets/mainnet/deneb.yaml index 10c5025ed..23889fd18 100644 --- a/presets/mainnet/deneb.yaml +++ b/presets/mainnet/deneb.yaml @@ -6,5 +6,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096 # `uint64(2**12)` (= 4096) MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096 -# `uint64(2**2)` (= 4) -MAX_BLOBS_PER_BLOCK: 4 +# `uint64(6)` +MAX_BLOBS_PER_BLOCK: 6 diff --git a/presets/minimal/deneb.yaml b/presets/minimal/deneb.yaml index 91120f9da..e21d38777 100644 --- a/presets/minimal/deneb.yaml +++ b/presets/minimal/deneb.yaml @@ -6,5 +6,5 @@ FIELD_ELEMENTS_PER_BLOB: 4 # [customized] MAX_BLOB_COMMITMENTS_PER_BLOCK: 16 -# `uint64(2**2)` (= 4) -MAX_BLOBS_PER_BLOCK: 4 +# `uint64(6)` +MAX_BLOBS_PER_BLOCK: 6 diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 41ce76bd2..148dc96d8 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -73,7 +73,7 @@ Deneb is a consensus-layer upgrade containing a number of features. Including: | Name | Value | Description | | - | - | - | | `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**12)` (= 4096) | *[New in Deneb:EIP4844]* hardfork independent fixed theoretical limit same as `LIMIT_BLOBS_PER_TX` (see EIP 4844) | -| `MAX_BLOBS_PER_BLOCK` | `uint64(2**2)` (= 4) | *[New in Deneb:EIP4844]* Maximum number of blobs in a single block limited by `MAX_BLOB_COMMITMENTS_PER_BLOCK` | +| `MAX_BLOBS_PER_BLOCK` | `uint64(6)` | *[New in Deneb:EIP4844]* maximum number of blobs in a single block limited by `MAX_BLOB_COMMITMENTS_PER_BLOCK` | *Note*: The blob transactions are packed into the execution payload by the EL/builder with their corresponding blobs being independently transmitted and are limited by `MAX_DATA_GAS_PER_BLOCK // DATA_GAS_PER_BLOB`. However the CL limit is independently defined by `MAX_BLOBS_PER_BLOCK`. diff --git a/specs/deneb/fork-choice.md b/specs/deneb/fork-choice.md index c67906259..bbc7fa0f8 100644 --- a/specs/deneb/fork-choice.md +++ b/specs/deneb/fork-choice.md @@ -36,7 +36,8 @@ The block MUST NOT be considered valid until all valid `Blob`s have been downloa def is_data_available(beacon_block_root: Root, blob_kzg_commitments: Sequence[KZGCommitment]) -> bool: # `retrieve_blobs_and_proofs` is implementation and context dependent # It returns all the blobs for the given block root, and raises an exception if not available - # Note: the p2p network does not guarantee sidecar retrieval outside of `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` + # Note: the p2p network does not guarantee sidecar retrieval outside of + # `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` blobs, proofs = retrieve_blobs_and_proofs(beacon_block_root) # For testing, `retrieve_blobs_and_proofs` returns ("TEST", "TEST"). diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py index 13a54225e..75955ed16 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py +++ b/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py @@ -10,3 +10,11 @@ from eth2spec.test.context import ( @single_phase def test_length(spec): assert spec.MAX_BLOBS_PER_BLOCK < spec.MAX_BLOB_COMMITMENTS_PER_BLOCK + + +@with_deneb_and_later +@spec_test +@single_phase +def test_networking(spec): + assert spec.MAX_BLOBS_PER_BLOCK < spec.MAX_BLOB_COMMITMENTS_PER_BLOCK + assert spec.config.MAX_REQUEST_BLOB_SIDECARS == spec.config.MAX_REQUEST_BLOCKS_DENEB * spec.MAX_BLOBS_PER_BLOCK From 5576d0e6850b0ddbffd1ec122841b35e31e66fee Mon Sep 17 00:00:00 2001 From: Paul Harris Date: Sat, 10 Jun 2023 01:12:38 +1000 Subject: [PATCH 095/110] Moved configuration into network preset instead of constants. (#3394) * Moved configuration into network preset instead of constants. Now that `MAX_CHUNK_SIZE` and `GOSSIP_MAX_SIZE` are in configuration, we no longer need separate constants to represent them in the spec when they change in Bellatrix. I've changed the usage, and put the values into the presets, but I'm not sure if I've updated the descriptions in the best way... This is following on from the work in #3375 where a number of constants got moved into configuration, so we no longer need these constants to be separately represented, they can simply be updated in presets. * Update presets/minimal/bellatrix.yaml Co-authored-by: Hsiao-Wei Wang * Update presets/mainnet/bellatrix.yaml Co-authored-by: Hsiao-Wei Wang * Moved preset items into the correct section and updated TOC. It looked like the items listed in configuration about the max size and chunk size were no longer needed since we're updating preset values now and the preset changes seem to only be listed in the changes at the top. * review feedback * hopefully correct this time! Moved the 2 fields from configs into presets completely as suggested. * WIP - changing back to being in config and updating the phase 0 value... I think this should be close but want to see what's outstanding. * fix intellij's formatting of table. * more fixes --------- Co-authored-by: Hsiao-Wei Wang --- configs/mainnet.yaml | 8 ++++---- configs/minimal.yaml | 8 ++++---- presets/mainnet/phase0.yaml | 2 +- presets/minimal/phase0.yaml | 2 +- specs/bellatrix/p2p-interface.md | 28 ++-------------------------- specs/phase0/p2p-interface.md | 4 ++-- 6 files changed, 14 insertions(+), 38 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 80a240185..fe405cb38 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -98,16 +98,16 @@ DEPOSIT_CONTRACT_ADDRESS: 0x00000000219ab540356cBB839Cbe05303d7705Fa # Networking # --------------------------------------------------------------- -# `2**20` (= 1048576, 1 MiB) -GOSSIP_MAX_SIZE: 1048576 +# `10 * 2**20` (= 10485760, 10 MiB) +GOSSIP_MAX_SIZE: 10485760 # `2**10` (= 1024) MAX_REQUEST_BLOCKS: 1024 # `2**8` (= 256) EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 # `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) MIN_EPOCHS_FOR_BLOCK_REQUESTS: 33024 -# `2**20` (=1048576, 1 MiB) -MAX_CHUNK_SIZE: 1048576 +# `10 * 2**20` (=10485760, 10 MiB) +MAX_CHUNK_SIZE: 10485760 # 5s TTFB_TIMEOUT: 5 # 10s diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 82a813a9b..dcc834963 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -99,16 +99,16 @@ DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123456789012345678901234567890 # Networking # --------------------------------------------------------------- -# `2**20` (= 1048576, 1 MiB) -GOSSIP_MAX_SIZE: 1048576 +# `10 * 2**20` (= 10485760, 10 MiB) +GOSSIP_MAX_SIZE: 10485760 # `2**10` (= 1024) MAX_REQUEST_BLOCKS: 1024 # `2**8` (= 256) EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 # [customized] `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 272) MIN_EPOCHS_FOR_BLOCK_REQUESTS: 272 -# `2**20` (=1048576, 1 MiB) -MAX_CHUNK_SIZE: 1048576 +# `10 * 2**20` (=10485760, 10 MiB) +MAX_CHUNK_SIZE: 10485760 # 5s TTFB_TIMEOUT: 5 # 10s diff --git a/presets/mainnet/phase0.yaml b/presets/mainnet/phase0.yaml index 02bc96c8c..00133ba36 100644 --- a/presets/mainnet/phase0.yaml +++ b/presets/mainnet/phase0.yaml @@ -85,4 +85,4 @@ MAX_ATTESTATIONS: 128 # 2**4 (= 16) MAX_DEPOSITS: 16 # 2**4 (= 16) -MAX_VOLUNTARY_EXITS: 16 +MAX_VOLUNTARY_EXITS: 16 \ No newline at end of file diff --git a/presets/minimal/phase0.yaml b/presets/minimal/phase0.yaml index e7028f5a4..d9a6a2b6c 100644 --- a/presets/minimal/phase0.yaml +++ b/presets/minimal/phase0.yaml @@ -85,4 +85,4 @@ MAX_ATTESTATIONS: 128 # 2**4 (= 16) MAX_DEPOSITS: 16 # 2**4 (= 16) -MAX_VOLUNTARY_EXITS: 16 +MAX_VOLUNTARY_EXITS: 16 \ No newline at end of file diff --git a/specs/bellatrix/p2p-interface.md b/specs/bellatrix/p2p-interface.md index b8b3a11d6..7d80d40a8 100644 --- a/specs/bellatrix/p2p-interface.md +++ b/specs/bellatrix/p2p-interface.md @@ -14,7 +14,6 @@ Readers should understand the Phase 0 and Altair documents and use them as a bas - [Warning](#warning) - [Modifications in Bellatrix](#modifications-in-bellatrix) - - [Configuration](#configuration) - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) - [Topics and messages](#topics-and-messages) - [Global topics](#global-topics) @@ -41,15 +40,6 @@ Refer to the note in the [validator guide](./validator.md) for further details. ## Modifications in Bellatrix -### Configuration - -This section outlines modifications constants that are used in this spec. - -| Name | Value | Description | -|---|---|---| -| `GOSSIP_MAX_SIZE_BELLATRIX` | `10 * 2**20` (= 10,485,760, 10 MiB) | The maximum allowed size of uncompressed gossip messages starting at Bellatrix upgrade. | -| `MAX_CHUNK_SIZE_BELLATRIX` | `10 * 2**20` (= 10,485,760, 10 MiB) | The maximum allowed size of uncompressed req/resp chunked responses starting at Bellatrix upgrade. | - ### The gossip domain: gossipsub Some gossip meshes are upgraded in Bellatrix to support upgraded types. @@ -61,11 +51,6 @@ All topics remain stable except the beacon block topic which is updated with the The specification around the creation, validation, and dissemination of messages has not changed from the Phase 0 and Altair documents unless explicitly noted here. -Starting at Bellatrix upgrade, each gossipsub [message](https://github.com/libp2p/go-libp2p-pubsub/blob/master/pb/rpc.proto#L17-L24) -has a maximum size of `GOSSIP_MAX_SIZE_BELLATRIX`. -Clients MUST reject (fail validation) messages that are over this size limit. -Likewise, clients MUST NOT emit or propagate messages larger than this limit. - The derivation of the `message-id` remains stable. The new topics along with the type of the `data` field of a gossipsub message are given in this table: @@ -130,10 +115,6 @@ down-scoring or disconnection. Request and Response remain unchanged unless explicitly noted here. -Starting at Bellatrix upgrade, -a global maximum uncompressed byte size of `MAX_CHUNK_SIZE_BELLATRIX` MUST be applied to all method response chunks -regardless of type specific bounds that *MUST* also be respected. - Bellatrix fork-digest is introduced to the `context` enum to specify Bellatrix block type. Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: @@ -171,17 +152,12 @@ Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: With the addition of `ExecutionPayload` to `BeaconBlock`s, there is a dynamic field -- `transactions` -- which can validly exceed the `GOSSIP_MAX_SIZE` limit (1 MiB) put in -place at Phase 0. At the `GAS_LIMIT` (~30M) currently seen on mainnet in 2021, a single transaction +place at Phase 0, so GOSSIP_MAX_SIZE has increased to 10 Mib on the network. +At the `GAS_LIMIT` (~30M) currently seen on mainnet in 2021, a single transaction filled entirely with data at a cost of 16 gas per byte can create a valid `ExecutionPayload` of ~2 MiB. Thus we need a size limit to at least account for current mainnet conditions. -Geth currently has a [max gossip message size](https://github.com/ethereum/go-ethereum/blob/3ce9f6d96f38712f5d6756e97b59ccc20cc403b3/eth/protocols/eth/protocol.go#L49) of 10 MiB. -To support backward compatibility with this previously defined network limit, -we adopt `GOSSIP_MAX_SIZE_BELLATRIX` of 10 MiB for maximum gossip sizes at the -point of Bellatrix and beyond. Note, that clients SHOULD still reject objects -that exceed their maximum theoretical bounds which in most cases is less than `GOSSIP_MAX_SIZE_BELLATRIX`. - Note, that due to additional size induced by the `BeaconBlock` contents (e.g. proposer signature, operations lists, etc) this does reduce the theoretical max valid `ExecutionPayload` (and `transactions` list) size as diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index de3e0e529..c0d18b08f 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -189,11 +189,11 @@ This section outlines configurations that are used in this spec. | Name | Value | Description | |---|---|---| -| `GOSSIP_MAX_SIZE` | `2**20` (= 1048576, 1 MiB) | The maximum allowed size of uncompressed gossip messages. | +| `GOSSIP_MAX_SIZE` | `10 * 2**20` (= 10485760, 10 MiB) | The maximum allowed size of uncompressed gossip messages. | | `MAX_REQUEST_BLOCKS` | `2**10` (= 1024) | Maximum number of blocks in a single request | | `EPOCHS_PER_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | Number of epochs on a subnet subscription (~27 hours) | | `MIN_EPOCHS_FOR_BLOCK_REQUESTS` | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) | The minimum epoch range over which a node must serve blocks | -| `MAX_CHUNK_SIZE` | `2**20` (=1048576, 1 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | +| `MAX_CHUNK_SIZE` | `10 * 2**20` (=10485760, 10 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | | `TTFB_TIMEOUT` | `5` | The maximum duration in **seconds** to wait for first byte of request response (time-to-first-byte). | | `RESP_TIMEOUT` | `10` | The maximum duration in **seconds** for complete response transfer. | | `ATTESTATION_PROPAGATION_SLOT_RANGE` | `32` | The maximum number of slots during which an attestation can be propagated. | From cfd44dac56996b8195434277392686fc319b70d2 Mon Sep 17 00:00:00 2001 From: William Dowling Date: Sun, 11 Jun 2023 15:07:39 +0200 Subject: [PATCH 096/110] Update lru-dict package to latest stable release. lru-dict build fails when building under clang version 16 (https://github.com/amitdev/lru-dict/commit/6badf6376d12a2e0498f67bbc7d5c0332b96a4d7). This bumps the version to the latest stable release. Tested on MacOS M2 and Ubuntu Jammy x86-64. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f4d0c8719..adf24ba36 100644 --- a/setup.py +++ b/setup.py @@ -1284,7 +1284,7 @@ setup( "remerkleable==0.1.27", "trie==2.0.2", RUAMEL_YAML_VERSION, - "lru-dict==1.1.8", + "lru-dict==1.2.0", MARKO_VERSION, "py_arkworks_bls12381==0.3.4", "curdleproofs @ git+https://github.com/nalinbhardwaj/curdleproofs.pie@master#egg=curdleproofs&subdirectory=curdleproofs", From 99f294cdd8a2d4461576eacaabbf4dc7ad60169c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 12 Jun 2023 16:02:28 +0800 Subject: [PATCH 097/110] Add link to EIP PR7044. Need to change it to eips.ethereum.org path once the EIP is merged --- specs/deneb/beacon-chain.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index e25751232..929ac39f6 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -43,7 +43,8 @@ ## Introduction Deneb is a consensus-layer upgrade containing a number of features. Including: -* [EIP-4844](https://eips.ethereum.org/EIPS/eip-4844): Shard Blob Transactions scale data-availability of Ethereum in a simple, forwards-compatible manner. +* [EIP-4844](https://eips.ethereum.org/EIPS/eip-4844): Shard Blob Transactions scale data-availability of Ethereum in a simple, forwards-compatible manner +* [EIP-7044](https://github.com/ethereum/EIPs/pull/7044): Perpetually Valid Signed Voluntary Exits ## Custom types From 7cf55c138ceb4bcbad889c362b9dce6d1c130afb Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 12 Jun 2023 15:34:05 +0300 Subject: [PATCH 098/110] Lock nalinbhardwaj/curdleproofs.pie version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f4d0c8719..f6d69cc13 100644 --- a/setup.py +++ b/setup.py @@ -1287,6 +1287,6 @@ setup( "lru-dict==1.1.8", MARKO_VERSION, "py_arkworks_bls12381==0.3.4", - "curdleproofs @ git+https://github.com/nalinbhardwaj/curdleproofs.pie@master#egg=curdleproofs&subdirectory=curdleproofs", + "curdleproofs @ git+https://github.com/nalinbhardwaj/curdleproofs.pie@805d06785b6ff35fde7148762277dd1ae678beeb#egg=curdleproofs&subdirectory=curdleproofs", ] ) From 512d2ca516d3aa6c3b65a831ff89c2bacaa6f9da Mon Sep 17 00:00:00 2001 From: gajinder Date: Mon, 12 Jun 2023 20:24:22 +0530 Subject: [PATCH 099/110] Update blob side car subnet count to 6 in line with max blobs limit update --- specs/deneb/validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/validator.md b/specs/deneb/validator.md index b53c0a042..ec9c54784 100644 --- a/specs/deneb/validator.md +++ b/specs/deneb/validator.md @@ -46,7 +46,7 @@ Please see related Beacon Chain doc before continuing and use them as a referenc | Name | Value | Unit | | - | - | :-: | -| `BLOB_SIDECAR_SUBNET_COUNT` | `4` | *[New in Deneb:EIP4844]* The number of blob sidecar subnets used in the gossipsub protocol. | +| `BLOB_SIDECAR_SUBNET_COUNT` | `6` | *[New in Deneb:EIP4844]* The number of blob sidecar subnets used in the gossipsub protocol. | ## Helpers From 210c4827bc6824c02c7c0cfcc7aad26294767560 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 12 Jun 2023 23:57:21 +0800 Subject: [PATCH 100/110] Move `BLOB_SIDECAR_SUBNET_COUNT` to networking config --- configs/mainnet.yaml | 2 ++ configs/minimal.yaml | 2 ++ specs/deneb/p2p-interface.md | 7 +++++-- specs/deneb/validator.md | 10 ---------- .../test/deneb/unittests/test_config_invariants.py | 2 ++ 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index fe405cb38..365bc1136 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -132,3 +132,5 @@ MAX_REQUEST_BLOCKS_DENEB: 128 MAX_REQUEST_BLOB_SIDECARS: 768 # `2**12` (= 4096 epochs, ~18 days) MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 +# `6` +BLOB_SIDECAR_SUBNET_COUNT: 6 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index dcc834963..b22a7165e 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -133,3 +133,5 @@ MAX_REQUEST_BLOCKS_DENEB: 128 MAX_REQUEST_BLOB_SIDECARS: 768 # `2**12` (= 4096 epochs, ~18 days) MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 +# `6` +BLOB_SIDECAR_SUBNET_COUNT: 6 diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 32e0ee9bf..809e405a6 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -40,11 +40,14 @@ The specification of these changes continues in the same format as the network s ### Configuration +*[New in Deneb:EIP4844]* + | Name | Value | Description | |------------------------------------------|-----------------------------------|---------------------------------------------------------------------| | `MAX_REQUEST_BLOCKS_DENEB` | `2**7` (= 128) | Maximum number of blocks in a single request | -| `MAX_REQUEST_BLOB_SIDECARS` | `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK` | Maximum number of blob sidecars in a single request | -| `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` | `2**12` (= 4096 epochs, ~18 days) | The minimum epoch range over which a node must serve blob sidecars | +| `MAX_REQUEST_BLOB_SIDECARS` | `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK` | Maximum number of blob sidecars in a single request | +| `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` | `2**12` (= 4096 epochs, ~18 days) | The minimum epoch range over which a node must serve blob sidecars | +| `BLOB_SIDECAR_SUBNET_COUNT` | `6` | The number of blob sidecar subnets used in the gossipsub protocol. | ### Containers diff --git a/specs/deneb/validator.md b/specs/deneb/validator.md index ec9c54784..3157ccf21 100644 --- a/specs/deneb/validator.md +++ b/specs/deneb/validator.md @@ -10,8 +10,6 @@ - [Introduction](#introduction) - [Prerequisites](#prerequisites) -- [Constants](#constants) - - [Misc](#misc) - [Helpers](#helpers) - [`BlobsBundle`](#blobsbundle) - [Modified `GetPayloadResponse`](#modified-getpayloadresponse) @@ -40,14 +38,6 @@ All behaviors and definitions defined in this document, and documents it extends All terminology, constants, functions, and protocol mechanics defined in the updated [Beacon Chain doc of Deneb](./beacon-chain.md) are requisite for this document and used throughout. Please see related Beacon Chain doc before continuing and use them as a reference throughout. -## Constants - -### Misc - -| Name | Value | Unit | -| - | - | :-: | -| `BLOB_SIDECAR_SUBNET_COUNT` | `6` | *[New in Deneb:EIP4844]* The number of blob sidecar subnets used in the gossipsub protocol. | - ## Helpers ### `BlobsBundle` diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py index 75955ed16..087bd63c6 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py +++ b/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py @@ -18,3 +18,5 @@ def test_length(spec): def test_networking(spec): assert spec.MAX_BLOBS_PER_BLOCK < spec.MAX_BLOB_COMMITMENTS_PER_BLOCK assert spec.config.MAX_REQUEST_BLOB_SIDECARS == spec.config.MAX_REQUEST_BLOCKS_DENEB * spec.MAX_BLOBS_PER_BLOCK + # Start with the same size, but `BLOB_SIDECAR_SUBNET_COUNT` could potentially increase later. + assert spec.config.BLOB_SIDECAR_SUBNET_COUNT == spec.MAX_BLOBS_PER_BLOCK From 87f77a3a6d5d5cb240e9961611514606b6d44f2f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 13 Jun 2023 01:42:18 +0800 Subject: [PATCH 101/110] bump VERSION.txt --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index 54dd11e46..45eafc27d 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -1.4.0-alpha.2 +1.4.0-alpha.3 From 530924020f84198675268830f54a879c21784d24 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 14 Jun 2023 15:29:59 +0300 Subject: [PATCH 102/110] Lock doctoc version --- .circleci/config.yml | 2 +- .github/workflows/run-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fcdf483d5..157c56ca5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -176,7 +176,7 @@ jobs: - checkout - run: name: Check table of contents - command: sudo npm install -g doctoc@2 && make check_toc + command: sudo npm install -g doctoc@2.2.0 && make check_toc codespell: docker: - image: circleci/python:3.9 diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 41a80ab92..b27c90765 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -50,7 +50,7 @@ jobs: with: ref: ${{ github.event.inputs.commitRef || env.DEFAULT_BRANCH }} - name: Check table of contents - run: sudo npm install -g doctoc@2 && make check_toc + run: sudo npm install -g doctoc@2.2.0 && make check_toc codespell: runs-on: self-hosted From 12fabf5854622cec2b01cf652a9277534ea6c59a Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 14 Jun 2023 16:47:17 +0300 Subject: [PATCH 103/110] Update specs/deneb/beacon-chain.md Co-authored-by: Danny Ryan --- specs/deneb/beacon-chain.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 929ac39f6..0d71e4ae7 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -270,6 +270,8 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi #### Modified `process_voluntary_exit` +Note: The function `process_voluntary_exit` is modified to use the a fixed fork version -- `CAPELLA_FORK_VERSION` -- for EIP-7044 + ```python def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVoluntaryExit) -> None: voluntary_exit = signed_voluntary_exit.message From 420f8baf6777cbf35e63ffe4ceb9029b168f65ad Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 14 Jun 2023 23:04:46 +0800 Subject: [PATCH 104/110] Rework tests. Move all `process_voluntary_exit` tests to Deneb --- specs/deneb/beacon-chain.md | 2 +- .../test_process_voluntary_exit.py | 3 +- .../test_process_voluntary_exit.py | 54 ++++++++++++++++--- .../test_process_voluntary_exit.py | 44 --------------- tests/generators/operations/main.py | 1 - 5 files changed, 49 insertions(+), 55 deletions(-) delete mode 100644 tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_process_voluntary_exit.py diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 0d71e4ae7..3189ee190 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -270,7 +270,7 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi #### Modified `process_voluntary_exit` -Note: The function `process_voluntary_exit` is modified to use the a fixed fork version -- `CAPELLA_FORK_VERSION` -- for EIP-7044 +*Note*: The function `process_voluntary_exit` is modified to use the a fixed fork version -- `CAPELLA_FORK_VERSION` -- for EIP-7044 ```python def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVoluntaryExit) -> None: diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_voluntary_exit.py b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_voluntary_exit.py index 12b554da5..ea3b57a97 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_voluntary_exit.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_voluntary_exit.py @@ -7,7 +7,6 @@ from eth2spec.test.context import ( from eth2spec.test.helpers.constants import ( BELLATRIX, CAPELLA, - DENEB, ) from eth2spec.test.helpers.keys import pubkey_to_privkey from eth2spec.test.helpers.state import ( @@ -80,7 +79,7 @@ def test_voluntary_exit_with_current_fork_version_not_is_before_fork_epoch(spec, ) -@with_phases([BELLATRIX, CAPELLA, DENEB]) +@with_phases([BELLATRIX, CAPELLA]) @spec_state_test @always_bls def test_voluntary_exit_with_previous_fork_version_is_before_fork_epoch(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py index 371fcfed4..06a111c86 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py +++ b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py @@ -19,6 +19,7 @@ def test_invalid_voluntary_exit_with_current_fork_version_not_is_before_fork_epo """ Since Deneb, the VoluntaryExit domain is fixed to `CAPELLA_FORK_VERSION` """ + assert state.fork.current_version != spec.config.CAPELLA_FORK_VERSION yield from run_voluntary_exit_processing_test( spec, state, @@ -28,7 +29,7 @@ def test_invalid_voluntary_exit_with_current_fork_version_not_is_before_fork_epo ) -@with_phases([DENEB]) +@with_deneb_and_later @spec_state_test @always_bls def test_voluntary_exit_with_previous_fork_version_not_is_before_fork_epoch(spec, state): @@ -37,9 +38,48 @@ def test_voluntary_exit_with_previous_fork_version_not_is_before_fork_epoch(spec """ assert state.fork.previous_version != state.fork.current_version - yield from run_voluntary_exit_processing_test( - spec, - state, - fork_version=state.fork.previous_version, - is_before_fork_epoch=False, - ) + if spec.fork == DENEB: + assert state.fork.previous_version == spec.config.CAPELLA_FORK_VERSION + yield from run_voluntary_exit_processing_test( + spec, + state, + fork_version=state.fork.previous_version, + is_before_fork_epoch=False, + ) + else: + assert state.fork.previous_version != spec.config.CAPELLA_FORK_VERSION + yield from run_voluntary_exit_processing_test( + spec, + state, + fork_version=state.fork.previous_version, + is_before_fork_epoch=False, + valid=False, + ) + + +@with_deneb_and_later +@spec_state_test +@always_bls +def test_voluntary_exit_with_previous_fork_version_is_before_fork_epoch(spec, state): + """ + Since Deneb, the VoluntaryExit domain is fixed to `CAPELLA_FORK_VERSION` + """ + assert state.fork.previous_version != state.fork.current_version + + if spec.fork == DENEB: + assert state.fork.previous_version == spec.config.CAPELLA_FORK_VERSION + yield from run_voluntary_exit_processing_test( + spec, + state, + fork_version=state.fork.previous_version, + is_before_fork_epoch=True, + ) + else: + assert state.fork.previous_version != spec.config.CAPELLA_FORK_VERSION + yield from run_voluntary_exit_processing_test( + spec, + state, + fork_version=state.fork.previous_version, + is_before_fork_epoch=True, + valid=False, + ) diff --git a/tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_process_voluntary_exit.py b/tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_process_voluntary_exit.py deleted file mode 100644 index 4128a1181..000000000 --- a/tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_process_voluntary_exit.py +++ /dev/null @@ -1,44 +0,0 @@ -from eth2spec.test.context import ( - always_bls, - spec_state_test, - with_eip6110_and_later, -) -from eth2spec.test.bellatrix.block_processing.test_process_voluntary_exit import ( - run_voluntary_exit_processing_test, -) - - -@with_eip6110_and_later -@spec_state_test -@always_bls -def test_invalid_voluntary_exit_with_previous_fork_version_not_is_before_fork_epoch(spec, state): - """ - Since Deneb, the VoluntaryExit domain is fixed to `CAPELLA_FORK_VERSION` - """ - assert state.fork.previous_version != state.fork.current_version - - yield from run_voluntary_exit_processing_test( - spec, - state, - fork_version=state.fork.previous_version, - is_before_fork_epoch=False, - valid=False, - ) - - -@with_eip6110_and_later -@spec_state_test -@always_bls -def test_invalid_voluntary_exit_with_previous_fork_version_is_before_fork_epoch(spec, state): - """ - Since Deneb, the VoluntaryExit domain is fixed to `CAPELLA_FORK_VERSION` - """ - assert state.fork.previous_version != state.fork.current_version - - yield from run_voluntary_exit_processing_test( - spec, - state, - fork_version=state.fork.previous_version, - is_before_fork_epoch=True, - valid=False, - ) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index 3b4e5d1f7..053236c8d 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -45,7 +45,6 @@ if __name__ == "__main__": _new_eip6110_mods = {key: 'eth2spec.test.eip6110.block_processing.test_process_' + key for key in [ 'deposit_receipt', - 'voluntary_exit', ]} eip6110_mods = combine_mods(_new_eip6110_mods, deneb_mods) From 11ab19c90a6228a696dea3df27b8834674fef348 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 14 Jun 2023 09:25:54 -0600 Subject: [PATCH 105/110] Apply suggestions from code review --- .../deneb/block_processing/test_process_voluntary_exit.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py index 06a111c86..711d27eb9 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py +++ b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py @@ -35,6 +35,8 @@ def test_invalid_voluntary_exit_with_current_fork_version_not_is_before_fork_epo def test_voluntary_exit_with_previous_fork_version_not_is_before_fork_epoch(spec, state): """ Since Deneb, the VoluntaryExit domain is fixed to `CAPELLA_FORK_VERSION` + + Note: This test is valid for ``spec.fork == DENEB`` and invalid for subsequent forks """ assert state.fork.previous_version != state.fork.current_version @@ -63,6 +65,8 @@ def test_voluntary_exit_with_previous_fork_version_not_is_before_fork_epoch(spec def test_voluntary_exit_with_previous_fork_version_is_before_fork_epoch(spec, state): """ Since Deneb, the VoluntaryExit domain is fixed to `CAPELLA_FORK_VERSION` + + Note: This test is valid for ``spec.fork == DENEB`` and invalid for subsequent forks """ assert state.fork.previous_version != state.fork.current_version From 7b132c20d1d4347df9bb262753be81407c410279 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 14 Jun 2023 18:52:22 +0300 Subject: [PATCH 106/110] Fix typos in get_shuffle_indices (#3426) --- specs/_features/whisk/beacon-chain.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/_features/whisk/beacon-chain.md b/specs/_features/whisk/beacon-chain.md index bad818837..6c8db4cec 100644 --- a/specs/_features/whisk/beacon-chain.md +++ b/specs/_features/whisk/beacon-chain.md @@ -306,9 +306,10 @@ def get_shuffle_indices(randao_reveal: BLSSignature) -> Sequence[uint64]: Given a `randao_reveal` return the list of indices that got shuffled from the entire candidate set """ indices = [] - for i in WHISK_VALIDATORS_PER_SHUFFLE: + for i in range(0, WHISK_VALIDATORS_PER_SHUFFLE): # XXX ensure we are not suffering from modulo bias - shuffle_index = uint256(hash(randao_reveal + uint_to_bytes(i))) % WHISK_CANDIDATE_TRACKERS_COUNT + pre_image = randao_reveal + uint_to_bytes(uint64(i)) + shuffle_index = bytes_to_uint64(hash(pre_image)[0:8]) % WHISK_CANDIDATE_TRACKERS_COUNT indices.append(shuffle_index) return indices From 0ab160bc2801be1ecb5ca2f65a2202f632b75bf5 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 14 Jun 2023 18:55:07 +0300 Subject: [PATCH 107/110] Add initialize_beacon_state_from_eth1 (#3428) --- configs/mainnet.yaml | 3 +++ configs/minimal.yaml | 3 +++ specs/_features/whisk/beacon-chain.md | 23 +++++++++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 365bc1136..9206ab77d 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -53,6 +53,9 @@ DENEB_FORK_EPOCH: 18446744073709551615 # EIP6110 EIP6110_FORK_VERSION: 0x05000000 # temporary stub EIP6110_FORK_EPOCH: 18446744073709551615 +# WHISK +WHISK_FORK_VERSION: 0x06000000 # temporary stub +WHISK_FORK_EPOCH: 18446744073709551615 # Time parameters diff --git a/configs/minimal.yaml b/configs/minimal.yaml index b22a7165e..256a39d1c 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -52,6 +52,9 @@ DENEB_FORK_EPOCH: 18446744073709551615 # EIP6110 EIP6110_FORK_VERSION: 0x05000001 EIP6110_FORK_EPOCH: 18446744073709551615 +# WHISK +WHISK_FORK_VERSION: 0x06000001 +WHISK_FORK_EPOCH: 18446744073709551615 # Time parameters diff --git a/specs/_features/whisk/beacon-chain.md b/specs/_features/whisk/beacon-chain.md index 6c8db4cec..c8adf945d 100644 --- a/specs/_features/whisk/beacon-chain.md +++ b/specs/_features/whisk/beacon-chain.md @@ -23,6 +23,7 @@ - [`BeaconBlockBody`](#beaconblockbody) - [Deposits](#deposits) - [`get_beacon_proposer_index`](#get_beacon_proposer_index) +- [Testing](#testing) @@ -470,3 +471,25 @@ def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: assert state.latest_block_header.slot == state.slot # sanity check `process_block_header` has been called return state.latest_block_header.proposer_index ``` + +## Testing + +*Note*: The function `initialize_beacon_state_from_eth1` is modified for pure Whisk testing only. + +```python +def initialize_beacon_state_from_eth1(eth1_block_hash: Hash32, + eth1_timestamp: uint64, + deposits: Sequence[Deposit], + execution_payload_header: ExecutionPayloadHeader=ExecutionPayloadHeader() + ) -> BeaconState: + state_capella = capella.initialize_beacon_state_from_eth1( + eth1_block_hash, + eth1_timestamp, + deposits, + execution_payload_header, + ) + state = upgrade_to_whisk(state_capella) + state.fork.previous_version = WHISK_FORK_VERSION + state.fork.current_version = WHISK_FORK_VERSION + return state +``` From 65a28b6d69a85135a8b9907b53d40f2f0182cb32 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 14 Jun 2023 18:58:57 +0300 Subject: [PATCH 108/110] Whisk: Move validator whisk trackers and commitments to state (#3407) * Move validator whisk trackers and commitments to state * Move comment --- specs/_features/whisk/beacon-chain.md | 77 ++++++++------------------- specs/_features/whisk/fork.md | 35 ++++++------ 2 files changed, 39 insertions(+), 73 deletions(-) diff --git a/specs/_features/whisk/beacon-chain.md b/specs/_features/whisk/beacon-chain.md index c8adf945d..34e79c1e4 100644 --- a/specs/_features/whisk/beacon-chain.md +++ b/specs/_features/whisk/beacon-chain.md @@ -15,7 +15,6 @@ - [Curdleproofs and opening proofs](#curdleproofs-and-opening-proofs) - [Epoch processing](#epoch-processing) - [`WhiskTracker`](#whisktracker) - - [`Validator`](#validator) - [`BeaconState`](#beaconstate) - [Block processing](#block-processing) - [Block header](#block-header) @@ -125,23 +124,6 @@ class WhiskTracker(Container): k_r_G: BLSG1Point # k * r * G ``` -### `Validator` - -```python -class Validator(Container): - pubkey: BLSPubkey - withdrawal_credentials: Bytes32 # Commitment to pubkey for withdrawals - effective_balance: Gwei # Balance at stake - slashed: boolean - # Status epochs - activation_eligibility_epoch: Epoch # When criteria for activation were met - activation_epoch: Epoch - exit_epoch: Epoch - withdrawable_epoch: Epoch # When validator can withdraw funds - whisk_tracker: WhiskTracker # Whisk tracker (r * G, k * r * G) [New in Whisk] - whisk_k_commitment: BLSG1Point # Whisk k commitment k * BLS_G1_GENERATOR [New in Whisk] -``` - ### `BeaconState` ```python @@ -187,8 +169,11 @@ class BeaconState(Container): next_withdrawal_validator_index: ValidatorIndex # Deep history valid from Capella onwards historical_summaries: List[HistoricalSummary, HISTORICAL_ROOTS_LIMIT] + # Whisk whisk_candidate_trackers: Vector[WhiskTracker, WHISK_CANDIDATE_TRACKERS_COUNT] # [New in Whisk] whisk_proposer_trackers: Vector[WhiskTracker, WHISK_PROPOSER_TRACKERS_COUNT] # [New in Whisk] + whisk_trackers: List[WhiskTracker, VALIDATOR_REGISTRY_LIMIT] # [New in Whisk] + whisk_k_commitments: List[BLSG1Point, VALIDATOR_REGISTRY_LIMIT] # [New in Whisk] ``` ```python @@ -204,7 +189,7 @@ def select_whisk_trackers(state: BeaconState, epoch: Epoch) -> None: for i in range(WHISK_CANDIDATE_TRACKERS_COUNT): seed = hash(get_seed(state, epoch, DOMAIN_WHISK_CANDIDATE_SELECTION) + uint_to_bytes(i)) candidate_index = compute_proposer_index(state, active_validator_indices, seed) # sample by effective balance - state.whisk_candidate_trackers[i] = state.validators[candidate_index].whisk_tracker + state.whisk_candidate_trackers[i] = state.whisk_trackers[candidate_index] ``` ```python @@ -238,7 +223,7 @@ def process_epoch(state: BeaconState) -> None: ```python def process_whisk_opening_proof(state: BeaconState, block: BeaconBlock) -> None: tracker = state.whisk_proposer_trackers[state.slot % WHISK_PROPOSER_TRACKERS_COUNT] - k_commitment = state.validators[block.proposer_index].whisk_k_commitment + k_commitment = state.whisk_k_commitments[block.proposer_index] assert IsValidWhiskOpeningProof(tracker, k_commitment, block.body.whisk_opening_proof) ``` @@ -298,7 +283,7 @@ class BeaconBlockBody(capella.BeaconBlockBody): whisk_shuffle_proof_M_commitment: BLSG1Point # [New in Whisk] whisk_registration_proof: WhiskTrackerProof # [New in Whisk] whisk_tracker: WhiskTracker # [New in Whisk] - whisk_k_commitment: BLSG1Point # [New in Whisk] + whisk_k_commitment: BLSG1Point # k * BLS_G1_GENERATOR [New in Whisk] ``` ```python @@ -343,7 +328,7 @@ def process_shuffled_trackers(state: BeaconState, body: BeaconBlockBody) -> None ```python def is_k_commitment_unique(state: BeaconState, k_commitment: BLSG1Point) -> bool: - return all([validator.whisk_k_commitment != k_commitment for validator in state.validators]) + return all([whisk_k_commitment != k_commitment for whisk_k_commitment in state.whisk_k_commitments]) ``` ```python @@ -384,49 +369,30 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: ### Deposits +```python +def get_initial_whisk_k(validator_index: ValidatorIndex, counter: int) -> BLSFieldElement: + # hash `validator_index || counter` + return BLSFieldElement(bytes_to_bls_field(hash(uint_to_bytes(validator_index) + uint_to_bytes(uint64(counter))))) +``` + ```python def get_unique_whisk_k(state: BeaconState, validator_index: ValidatorIndex) -> BLSFieldElement: counter = 0 while True: - # hash `validator_index || counter` - k = BLSFieldElement(bytes_to_bls_field(hash(uint_to_bytes(validator_index) + uint_to_bytes(uint64(counter))))) + k = get_initial_whisk_k(validator_index, counter) if is_k_commitment_unique(state, BLSG1ScalarMultiply(k, BLS_G1_GENERATOR)): return k # unique by trial and error counter += 1 ``` ```python -def get_initial_commitments(k: BLSFieldElement) -> Tuple[BLSG1Point, WhiskTracker]: - return ( - BLSG1ScalarMultiply(k, BLS_G1_GENERATOR), - WhiskTracker(r_G=BLS_G1_GENERATOR, k_r_G=BLSG1ScalarMultiply(k, BLS_G1_GENERATOR)) - ) +def get_k_commitment(k: BLSFieldElement) -> BLSG1Point: + return BLSG1ScalarMultiply(k, BLS_G1_GENERATOR) ``` ```python -def get_validator_from_deposit_whisk( - state: BeaconState, - pubkey: BLSPubkey, - withdrawal_credentials: Bytes32, - amount: uint64 -) -> Validator: - effective_balance = min(amount - amount % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) - k = get_unique_whisk_k(state, ValidatorIndex(len(state.validators))) - whisk_k_commitment, whisk_tracker = get_initial_commitments(k) - - validator = Validator( - pubkey=pubkey, - withdrawal_credentials=withdrawal_credentials, - activation_eligibility_epoch=FAR_FUTURE_EPOCH, - activation_epoch=FAR_FUTURE_EPOCH, - exit_epoch=FAR_FUTURE_EPOCH, - withdrawable_epoch=FAR_FUTURE_EPOCH, - effective_balance=effective_balance, - # Whisk fields - whisk_tracker=whisk_tracker, - whisk_k_commitment=whisk_k_commitment, - ) - return validator +def get_initial_tracker(k: BLSFieldElement) -> WhiskTracker: + return WhiskTracker(r_G=BLS_G1_GENERATOR, k_r_G=BLSG1ScalarMultiply(k, BLS_G1_GENERATOR)) ``` ```python @@ -448,13 +414,16 @@ def apply_deposit(state: BeaconState, # Initialize validator if the deposit signature is valid if bls.Verify(pubkey, signing_root, signature): index = get_index_for_new_validator(state) - validator = get_validator_from_deposit_whisk(state, pubkey, withdrawal_credentials, amount) + validator = get_validator_from_deposit(pubkey, withdrawal_credentials, amount) set_or_append_list(state.validators, index, validator) set_or_append_list(state.balances, index, amount) - # [New in Altair] set_or_append_list(state.previous_epoch_participation, index, ParticipationFlags(0b0000_0000)) set_or_append_list(state.current_epoch_participation, index, ParticipationFlags(0b0000_0000)) set_or_append_list(state.inactivity_scores, index, uint64(0)) + # [New in Whisk] + k = get_unique_whisk_k(state, ValidatorIndex(len(state.validators) - 1)) + state.whisk_trackers.append(get_initial_tracker(k)) + state.whisk_k_commitments.append(get_k_commitment(k)) else: # Increase balance by deposit amount index = ValidatorIndex(validator_pubkeys.index(pubkey)) diff --git a/specs/_features/whisk/fork.md b/specs/_features/whisk/fork.md index 189d8c5b3..f66026037 100644 --- a/specs/_features/whisk/fork.md +++ b/specs/_features/whisk/fork.md @@ -68,6 +68,11 @@ def whisk_proposer_selection(state: BeaconState, epoch: Epoch) -> None: ```python def upgrade_to_whisk(pre: bellatrix.BeaconState) -> BeaconState: + # Compute initial unsafe trackers for all validators + ks = [get_initial_whisk_k(ValidatorIndex(validator_index), 0) for validator_index in range(len(pre.validators))] + whisk_k_commitments = [get_k_commitment(k) for k in ks] + whisk_trackers = [get_initial_tracker(k) for k in ks] + epoch = bellatrix.get_current_epoch(pre) post = BeaconState( # Versioning @@ -104,27 +109,19 @@ def upgrade_to_whisk(pre: bellatrix.BeaconState) -> BeaconState: current_justified_checkpoint=pre.current_justified_checkpoint, finalized_checkpoint=pre.finalized_checkpoint, # Inactivity - inactivity_scores=pre.inactivity_Scores, + inactivity_scores=pre.inactivity_scores, + # Sync + current_sync_committee=pre.current_sync_committee, + next_sync_committee=pre.next_sync_committee, + # Execution-layer + latest_execution_payload_header=pre.latest_execution_payload_header, + # Whisk + whisk_proposer_trackers=[WhiskTracker() for _ in range(WHISK_PROPOSER_TRACKERS_COUNT)], # [New in Whisk] + whisk_candidate_trackers=[WhiskTracker() for _ in range(WHISK_CANDIDATE_TRACKERS_COUNT)], # [New in Whisk] + whisk_trackers=whisk_trackers, # [New in Whisk] + whisk_k_commitments=whisk_k_commitments, # [New in Whisk] ) - # Initialize all validators with predictable commitments - for val_index, pre_validator in enumerate(pre.validators): - whisk_commitment, whisk_tracker = get_initial_commitments(get_unique_whisk_k(post, ValidatorIndex(val_index))) - - post_validator = Validator( - pubkey=pre_validator.pubkey, - withdrawal_credentials=pre_validator.withdrawal_credentials, - effective_balance=pre_validator.effective_balance, - slashed=pre_validator.slashed, - activation_eligibility_epoch=pre_validator.activation_eligibility_epoch, - activation_epoch=pre_validator.activation_epoch, - exit_epoch=pre_validator.exit_epoch, - withdrawable_epoch=pre_validator.withdrawable_epoch, - whisk_commitment=whisk_commitment, - whisk_tracker=whisk_tracker, - ) - post.validators.append(post_validator) - # Do a candidate selection followed by a proposer selection so that we have proposers for the upcoming day # Use an old epoch when selecting candidates so that we don't get the same seed as in the next candidate selection whisk_candidate_selection(post, epoch - WHISK_PROPOSER_SELECTION_GAP - 1) From 9e50c74a372c0d8349764ef411b22cfa71f2a656 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 15 Jun 2023 00:45:43 +0800 Subject: [PATCH 109/110] Fix linter (#3430) --- .../test/deneb/block_processing/test_process_voluntary_exit.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py index 711d27eb9..b01eaab0e 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py +++ b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py @@ -1,7 +1,6 @@ from eth2spec.test.context import ( always_bls, spec_state_test, - with_phases, with_deneb_and_later, ) from eth2spec.test.helpers.constants import ( From 834f6f70e7b864dde91641191f20e0dd1c3b0331 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 15 Jun 2023 14:33:28 +0300 Subject: [PATCH 110/110] Whisk: assert zeroed values during selection gap (#3425) * Assert zeroed values during selection gap * Update comment --- specs/_features/whisk/beacon-chain.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/specs/_features/whisk/beacon-chain.md b/specs/_features/whisk/beacon-chain.md index 34e79c1e4..b10611a8d 100644 --- a/specs/_features/whisk/beacon-chain.md +++ b/specs/_features/whisk/beacon-chain.md @@ -306,20 +306,23 @@ def process_shuffled_trackers(state: BeaconState, body: BeaconBlockBody) -> None # Check the shuffle proof shuffle_indices = get_shuffle_indices(body.randao_reveal) pre_shuffle_trackers = [state.whisk_candidate_trackers[i] for i in shuffle_indices] - post_shuffle_trackers = body.whisk_post_shuffle_trackers shuffle_epoch = get_current_epoch(state) % WHISK_EPOCHS_PER_SHUFFLING_PHASE if shuffle_epoch + WHISK_PROPOSER_SELECTION_GAP + 1 >= WHISK_EPOCHS_PER_SHUFFLING_PHASE: - # Require unchanged trackers during cooldown - assert pre_shuffle_trackers == post_shuffle_trackers + # Require trackers set to zero during cooldown + assert body.whisk_post_shuffle_trackers == Vector[WhiskTracker, WHISK_VALIDATORS_PER_SHUFFLE]() + assert body.whisk_shuffle_proof_M_commitment == BLSG1Point() + assert body.whisk_shuffle_proof == WhiskShuffleProof() + post_shuffle_trackers = pre_shuffle_trackers else: # Require shuffled trackers during shuffle assert IsValidWhiskShuffleProof( pre_shuffle_trackers, - post_shuffle_trackers, + body.whisk_post_shuffle_trackers, body.whisk_shuffle_proof_M_commitment, body.whisk_shuffle_proof, ) + post_shuffle_trackers = body.whisk_post_shuffle_trackers # Shuffle candidate trackers for i, shuffle_index in enumerate(shuffle_indices):