From 68d6e4319a5a7292653606b9ff2dcaac2c3c0c5c Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Sat, 1 May 2021 16:30:23 -0700 Subject: [PATCH 01/36] Adjust sync committee size and duration --- configs/mainnet/altair.yaml | 8 ++++---- specs/altair/beacon-chain.md | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/configs/mainnet/altair.yaml b/configs/mainnet/altair.yaml index 3cd4b8419..32b75fc9c 100644 --- a/configs/mainnet/altair.yaml +++ b/configs/mainnet/altair.yaml @@ -12,8 +12,8 @@ PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR: 2 # Misc # --------------------------------------------------------------- -# 2**10 (= 1,024) -SYNC_COMMITTEE_SIZE: 1024 +# 2**9 (= 512) +SYNC_COMMITTEE_SIZE: 512 # 2**6 (= 64) SYNC_PUBKEYS_PER_AGGREGATE: 64 # 2**2 (= 4) @@ -22,8 +22,8 @@ INACTIVITY_SCORE_BIAS: 4 # Time parameters # --------------------------------------------------------------- -# 2**8 (= 256) -EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 256 +# 2**9 (= 512) +EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 512 # Signature domains diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index f498c40c5..aa961d956 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -118,7 +118,7 @@ This patch updates a few configuration values to move penalty parameters toward | Name | Value | | - | - | -| `SYNC_COMMITTEE_SIZE` | `uint64(2**10)` (= 1,024) | +| `SYNC_COMMITTEE_SIZE` | `uint64(2**9)` (= 512) | | `SYNC_PUBKEYS_PER_AGGREGATE` | `uint64(2**6)` (= 64) | | `INACTIVITY_SCORE_BIAS` | `uint64(4)` | @@ -126,7 +126,7 @@ This patch updates a few configuration values to move penalty parameters toward | Name | Value | Unit | Duration | | - | - | :-: | :-: | -| `EPOCHS_PER_SYNC_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | +| `EPOCHS_PER_SYNC_COMMITTEE_PERIOD` | `Epoch(2**9)` (= 512) | epochs | ~54 hours | ### Domain types From 0438f2f27c5c3eb1b43c940962fec702361ee1b2 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Sat, 1 May 2021 16:07:43 -0700 Subject: [PATCH 02/36] whitespace --- specs/altair/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 726747c59..25b8bae54 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -409,7 +409,7 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S #### Modified `slash_validator` -*Note*: The function `slash_validator` is modified to use `MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR` +*Note*: The function `slash_validator` is modified to use `MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR` and use `PROPOSER_WEIGHT` when calculating the proposer reward. ```python From 7b33c1119a383ea693ada3793699e0bb1d43c8de Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Sat, 1 May 2021 16:08:36 -0700 Subject: [PATCH 03/36] simplify sync committee pubkey aggregation in altair --- configs/mainnet/altair.yaml | 2 -- configs/minimal/altair.yaml | 2 -- specs/altair/beacon-chain.md | 8 +++----- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/configs/mainnet/altair.yaml b/configs/mainnet/altair.yaml index 3cd4b8419..46b74b3aa 100644 --- a/configs/mainnet/altair.yaml +++ b/configs/mainnet/altair.yaml @@ -14,8 +14,6 @@ PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR: 2 # --------------------------------------------------------------- # 2**10 (= 1,024) SYNC_COMMITTEE_SIZE: 1024 -# 2**6 (= 64) -SYNC_PUBKEYS_PER_AGGREGATE: 64 # 2**2 (= 4) INACTIVITY_SCORE_BIAS: 4 diff --git a/configs/minimal/altair.yaml b/configs/minimal/altair.yaml index f9b30eea2..42e464ab4 100644 --- a/configs/minimal/altair.yaml +++ b/configs/minimal/altair.yaml @@ -14,8 +14,6 @@ PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR: 2 # --------------------------------------------------------------- # [customized] SYNC_COMMITTEE_SIZE: 32 -# [customized] -SYNC_PUBKEYS_PER_AGGREGATE: 16 # 2**2 (= 4) INACTIVITY_SCORE_BIAS: 4 diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 25b8bae54..96a14fe21 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -119,7 +119,6 @@ This patch updates a few configuration values to move penalty parameters toward | Name | Value | | - | - | | `SYNC_COMMITTEE_SIZE` | `uint64(2**10)` (= 1,024) | -| `SYNC_PUBKEYS_PER_AGGREGATE` | `uint64(2**6)` (= 64) | | `INACTIVITY_SCORE_BIAS` | `uint64(4)` | ### Time parameters @@ -212,7 +211,7 @@ class SyncAggregate(Container): ```python class SyncCommittee(Container): pubkeys: Vector[BLSPubkey, SYNC_COMMITTEE_SIZE] - pubkey_aggregates: Vector[BLSPubkey, SYNC_COMMITTEE_SIZE // SYNC_PUBKEYS_PER_AGGREGATE] + aggregate_pubkey: BLSPubkey ``` ## Helper functions @@ -306,9 +305,8 @@ def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: """ indices = get_sync_committee_indices(state, epoch) pubkeys = [state.validators[index].pubkey for index in indices] - partition = [pubkeys[i:i + SYNC_PUBKEYS_PER_AGGREGATE] for i in range(0, len(pubkeys), SYNC_PUBKEYS_PER_AGGREGATE)] - pubkey_aggregates = [bls.AggregatePKs(preaggregate) for preaggregate in partition] - return SyncCommittee(pubkeys=pubkeys, pubkey_aggregates=pubkey_aggregates) + aggregate_pubkey = bls.AggregatePKs(pubkeys) + return SyncCommittee(pubkeys=pubkeys, aggregate_pubkey=aggregate_pubkey) ``` #### `get_base_reward_per_increment` From 9c3d5982cfbe9a52b02e2bd028a873c9226a34c9 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 4 May 2021 12:16:39 -0700 Subject: [PATCH 04/36] add documentation about duplicate pubkeys --- specs/altair/beacon-chain.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 96a14fe21..b75835d55 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -302,6 +302,14 @@ def get_sync_committee_indices(state: BeaconState, epoch: Epoch) -> Sequence[Val def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: """ Return the sync committee for a given ``state`` and ``epoch``. + + ``SyncCommittee`` contains an aggregate pubkey that enables + resource-constrained clients to save some computation when verifying + the sync committee's signature. + + ``SyncCommittee`` can also contain duplicate pubkeys, when ``get_sync_committee_indices`` + returns duplicate indices. Implementations must take care when handling + optimizations relating to aggregation and verification in the presence of duplicates. """ indices = get_sync_committee_indices(state, epoch) pubkeys = [state.validators[index].pubkey for index in indices] From 7d236561bd39bb92f3d0c7662c5d4d2762a5fe45 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 4 May 2021 12:41:19 -0700 Subject: [PATCH 05/36] adjust subnet count to reflect smaller sync committees --- specs/altair/validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/altair/validator.md b/specs/altair/validator.md index c69e490b1..c58086b8e 100644 --- a/specs/altair/validator.md +++ b/specs/altair/validator.md @@ -74,7 +74,7 @@ This document is currently illustrative for early Altair testnets and some parts | Name | Value | Unit | | - | - | :-: | | `TARGET_AGGREGATORS_PER_SYNC_SUBCOMMITTEE` | `2**2` (= 4) | validators | -| `SYNC_COMMITTEE_SUBNET_COUNT` | `8` | The number of sync committee subnets used in the gossipsub aggregation protocol. | +| `SYNC_COMMITTEE_SUBNET_COUNT` | `4` | The number of sync committee subnets used in the gossipsub aggregation protocol. | ## Containers From eae64fd18c94cfa74fad5edf09bc72ba09fe8cf6 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 4 May 2021 12:43:21 -0700 Subject: [PATCH 06/36] clean up whitespace on altair files --- specs/altair/p2p-interface.md | 28 ++++++++++---------- specs/altair/validator.md | 50 +++++++++++++++++------------------ 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/specs/altair/p2p-interface.md b/specs/altair/p2p-interface.md index 4f2fd6aa8..85e859191 100644 --- a/specs/altair/p2p-interface.md +++ b/specs/altair/p2p-interface.md @@ -40,8 +40,8 @@ Altair adds new messages, topics and data to the Req-Resp, Gossip and Discovery ## Warning -This document is currently illustrative for early Altair testnets and some parts are subject to change. -Refer to the note in the [validator guide](./validator.md) for further details. +This document is currently illustrative for early Altair testnets and some parts are subject to change. +Refer to the note in the [validator guide](./validator.md) for further details. # Modifications in Altair @@ -64,12 +64,12 @@ Where ## The gossip domain: gossipsub -Gossip meshes are added in Altair to support the consensus activities of the sync committees. +Gossip meshes are added in Altair to support the consensus activities of the sync committees. Validators use an aggregation scheme to balance the processing and networking load across all of the relevant actors. ### Topics and messages -Topics follow the same specification as in the Phase 0 document. +Topics follow the same specification as in the Phase 0 document. New topics are added in Altair to support the sync committees and the beacon block topic is updated with the modified type. The specification around the creation, validation, and dissemination of messages has not changed from the Phase 0 document. @@ -93,13 +93,13 @@ Altair changes the type of the global beacon block topic and adds one global top ##### `beacon_block` The existing specification for this topic does not change from the Phase 0 document, -but the type of the payload does change to the (modified) `SignedBeaconBlock`. +but the type of the payload does change to the (modified) `SignedBeaconBlock`. This type changes due to the inclusion of the inner `BeaconBlockBody` that is modified in Altair. See the [state transition document](./beacon-chain.md#beaconblockbody) for Altair for further details. ##### `sync_committee_contribution_and_proof` - + This topic is used to propagate partially aggregated sync committee signatures to be included in future blocks. The following validations MUST pass before forwarding the `signed_contribution_and_proof` on the network; define `contribution_and_proof = signed_contribution_and_proof.message`, `contribution = contribution_and_proof.contribution`, and the following function `get_sync_subcommittee_pubkeys` for convenience: @@ -135,16 +135,16 @@ The following validations MUST pass before forwarding the `sync_committee_signat - _[IGNORE]_ The signature's slot is for the current slot, i.e. `sync_committee_signature.slot == current_slot`. - _[IGNORE]_ The block being signed over (`sync_committee_signature.beacon_block_root`) has been seen (via both gossip and non-gossip sources). - _[IGNORE]_ There has been no other valid sync committee signature for the declared `slot` for the validator referenced by `sync_committee_signature.validator_index`. -- _[REJECT]_ The `subnet_id` is valid for the given validator, i.e. `subnet_id in compute_subnets_for_sync_committee(state, sync_committee_signature.validator_index)`. +- _[REJECT]_ The `subnet_id` is valid for the given validator, i.e. `subnet_id in compute_subnets_for_sync_committee(state, sync_committee_signature.validator_index)`. Note this validation implies the validator is part of the broader current sync committee along with the correct subcommittee. - _[REJECT]_ The `signature` is valid for the message `beacon_block_root` for the validator referenced by `validator_index`. #### Sync committees and aggregation -The aggregation scheme closely follows the design of the attestation aggregation scheme. -Sync committee signatures are broadcast into "subnets" defined by a topic. -The number of subnets is defined by `SYNC_COMMITTEE_SUBNET_COUNT` in the [Altair validator guide](./validator.md#constants). -Sync committee members are divided into "subcommittees" which are then assigned to a subnet for the duration of tenure in the sync committee. +The aggregation scheme closely follows the design of the attestation aggregation scheme. +Sync committee signatures are broadcast into "subnets" defined by a topic. +The number of subnets is defined by `SYNC_COMMITTEE_SUBNET_COUNT` in the [Altair validator guide](./validator.md#constants). +Sync committee members are divided into "subcommittees" which are then assigned to a subnet for the duration of tenure in the sync committee. Individual validators can be duplicated in the broader sync committee such that they are included multiple times in a given subcommittee or across multiple subcommittees. Unaggregated signatures (along with metadata) are sent as `SyncCommitteeSignature`s on the `sync_committee_{subnet_id}` topics. @@ -244,7 +244,7 @@ Response Content: ) ``` -Requests the MetaData of a peer, using the new `MetaData` definition given above +Requests the MetaData of a peer, using the new `MetaData` definition given above that is extended from phase 0 in Altair. Other conditions for the `GetMetaData` protocol are unchanged from the phase 0 p2p networking document. @@ -253,7 +253,7 @@ protocol are unchanged from the phase 0 p2p networking document. In advance of the fork, implementations can opt in to both run the v1 and v2 for a smooth transition. This is non-breaking, and is recommended as soon as the fork specification is stable. -The v1 variants will be deprecated, and implementations should use v2 when available +The v1 variants will be deprecated, and implementations should use v2 when available (as negotiated with peers via LibP2P multistream-select). The v1 method MAY be unregistered at the fork boundary. @@ -265,7 +265,7 @@ the responder MUST return the **InvalidRequest** response code. The `attnets` key of the ENR is used as defined in the Phase 0 document. An additional bitfield is added to the ENR under the key `syncnets` to facilitate sync committee subnet discovery. -The length of this bitfield is `SYNC_COMMITTEE_SUBNET_COUNT` where each bit corresponds to a distinct `subnet_id` for a specific sync committee subnet. +The length of this bitfield is `SYNC_COMMITTEE_SUBNET_COUNT` where each bit corresponds to a distinct `subnet_id` for a specific sync committee subnet. The `i`th bit is set in this bitfield if the validator is currently subscribed to the `sync_committee_{i}` topic. See the [validator document](./validator.md#sync-committee-subnet-stability) for further details on how the new bits are used. diff --git a/specs/altair/validator.md b/specs/altair/validator.md index c69e490b1..8cd9fc404 100644 --- a/specs/altair/validator.md +++ b/specs/altair/validator.md @@ -49,18 +49,18 @@ This is an accompanying document to [Ethereum 2.0 Altair -- The Beacon Chain](./ ## Introduction -This document represents the expected behavior of an "honest validator" with respect to the Altair upgrade of the Ethereum 2.0 protocol. -It builds on the [previous document for the behavior of an "honest validator" from Phase 0](../phase0/validator.md) of the Ethereum 2.0 protocol. +This document represents the expected behavior of an "honest validator" with respect to the Altair upgrade of the Ethereum 2.0 protocol. +It builds on the [previous document for the behavior of an "honest validator" from Phase 0](../phase0/validator.md) of the Ethereum 2.0 protocol. This previous document is referred to below as the "Phase 0 document". -Altair introduces a new type of committee: the sync committee. Sync committees are responsible for signing each block of the canonical chain and there exists an efficient algorithm for light clients to sync the chain using the output of the sync committees. -See the [sync protocol](./sync-protocol.md) for further details on the light client sync. -Under this network upgrade, validators track their participation in this new committee type and produce the relevant signatures as required. +Altair introduces a new type of committee: the sync committee. Sync committees are responsible for signing each block of the canonical chain and there exists an efficient algorithm for light clients to sync the chain using the output of the sync committees. +See the [sync protocol](./sync-protocol.md) for further details on the light client sync. +Under this network upgrade, validators track their participation in this new committee type and produce the relevant signatures as required. Block proposers incorporate the (aggregated) sync committee signatures into each block they produce. ## Prerequisites -All terminology, constants, functions, and protocol mechanics defined in the [Altair -- The Beacon Chain](./beacon-chain.md) doc are requisite for this document and used throughout. +All terminology, constants, functions, and protocol mechanics defined in the [Altair -- The Beacon Chain](./beacon-chain.md) doc are requisite for this document and used throughout. Please see this document before continuing and use as a reference throughout. ## Warning @@ -168,11 +168,11 @@ def is_assigned_to_sync_committee(state: BeaconState, ### Lookahead The sync committee shufflings give validators 1 sync committee period of lookahead which amounts to `EPOCHS_PER_SYNC_COMMITTEE_PERIOD` epochs. -At any given `epoch`, the `BeaconState` contains the current `SyncCommittee` and the next `SyncCommittee`. +At any given `epoch`, the `BeaconState` contains the current `SyncCommittee` and the next `SyncCommittee`. Once every `EPOCHS_PER_SYNC_COMMITTEE_PERIOD` epochs, the next `SyncCommittee` becomes the current `SyncCommittee` and the next committee is computed and stored. -*Note*: The data required to compute a given committee is not cached in the `BeaconState` after committees are calculated at the period boundaries. -This means that calling `get_sync_commitee()` in a given `epoch` can return a different result than what was computed during the relevant epoch transition. +*Note*: The data required to compute a given committee is not cached in the `BeaconState` after committees are calculated at the period boundaries. +This means that calling `get_sync_commitee()` in a given `epoch` can return a different result than what was computed during the relevant epoch transition. For this reason, *always* get committee assignments via the fields of the `BeaconState` (`current_sync_committee` and `next_sync_committee`) or use the above reference code. A validator should plan for future sync committee assignments by noting which sync committee periods they are selected for participation. @@ -188,7 +188,7 @@ A validator maintains the responsibilities given in the Phase 0 document. Block proposals are modified to incorporate the sync committee signatures as detailed below. When assigned to a sync committee, validators have a new responsibility to sign and broadcast beacon block roots during each slot of the sync committee period. -These signatures are aggregated and routed to the proposer over gossip for inclusion into a beacon block. +These signatures are aggregated and routed to the proposer over gossip for inclusion into a beacon block. Assignments to a particular sync committee are infrequent at normal validator counts; however, an action every slot is required when in the current active sync committee. ### Block proposal @@ -202,25 +202,25 @@ No change to [Preparing for a `BeaconBlock`](../phase0/validator.md#preparing-fo #### Constructing the `BeaconBlockBody` -Each section of [Constructing the `BeaconBlockBody`](../phase0/validator.md#constructing-the-beaconblockbody) should be followed. +Each section of [Constructing the `BeaconBlockBody`](../phase0/validator.md#constructing-the-beaconblockbody) should be followed. After constructing the `BeaconBlockBody` as per that section, the proposer has an additional task to include the sync committee signatures: ##### Sync committee The proposer receives a number of `SyncCommitteeContribution`s (wrapped in `SignedContributionAndProof`s on the wire) from validators in the sync committee who are selected to partially aggregate signatures from independent subcommittees formed by breaking the full sync committee into `SYNC_COMMITTEE_SUBNET_COUNT` pieces (see below for details). -The proposer collects the contributions that match their local view of the chain (i.e. `contribution.beacon_block_root == block.parent_root`) for further aggregation when preparing a block. +The proposer collects the contributions that match their local view of the chain (i.e. `contribution.beacon_block_root == block.parent_root`) for further aggregation when preparing a block. Of these contributions, proposers should select the best contribution seen across all aggregators for each subnet/subcommittee. A contribution with more valid signatures is better than a contribution with fewer signatures. -Recall `block.body.sync_aggregate.sync_committee_bits` is a `Bitvector` where the `i`th bit is `True` if the corresponding validator in the sync committee has produced a valid signature, +Recall `block.body.sync_aggregate.sync_committee_bits` is a `Bitvector` where the `i`th bit is `True` if the corresponding validator in the sync committee has produced a valid signature, and that `block.body.sync_aggregate.sync_committee_signature` is the aggregate BLS signature combining all of the valid signatures. -Given a collection of the best seen `contributions` (with no repeating `subcommittee_index` values) and the `BeaconBlock` under construction, +Given a collection of the best seen `contributions` (with no repeating `subcommittee_index` values) and the `BeaconBlock` under construction, the proposer processes them as follows: ```python -def process_sync_committee_contributions(block: BeaconBlock, +def process_sync_committee_contributions(block: BeaconBlock, contributions: Set[SyncCommitteeContribution]) -> None: sync_aggregate = SyncAggregate() signatures = [] @@ -248,7 +248,7 @@ No change to [Packaging into a `SignedBeaconBlock`](../phase0/validator.md#packa ### Attesting and attestation aggregation -Refer to the phase 0 document for the [attesting](../phase0/validator.md#attesting) and [attestation aggregation](../phase0/validator.md#attestation-aggregation) responsibilities. +Refer to the phase 0 document for the [attesting](../phase0/validator.md#attesting) and [attestation aggregation](../phase0/validator.md#attestation-aggregation) responsibilities. There is no change compared to the phase 0 document. ### Sync committees @@ -263,15 +263,15 @@ This process occurs each slot. If a validator is in the current sync committee (i.e. `is_assigned_to_sync_committee()` above returns `True`), then for every slot in the current sync committee period, the validator should prepare a `SyncCommitteeSignature` according to the logic in `get_sync_committee_signature` as soon as they have determined the head block of the current slot. -This logic is triggered upon the same conditions as when producing an attestation. +This logic is triggered upon the same conditions as when producing an attestation. Meaning, a sync committee member should produce and broadcast a `SyncCommitteeSignature` either when (a) the validator has received a valid block from the expected block proposer for the current `slot` or (b) one-third of the slot has transpired (`SECONDS_PER_SLOT / 3` seconds after the start of the slot) -- whichever comes first. `get_sync_committee_signature()` assumes `state` is the head state corresponding to processing the block up to the current slot as determined by the fork choice (including any empty slots up to the current slot processed with `process_slots` on top of the latest block), `block_root` is the root of the head block, `validator_index` is the index of the validator in the registry `state.validators` controlled by `privkey`, and `privkey` is the BLS private key for the validator. ```python -def get_sync_committee_signature(state: BeaconState, +def get_sync_committee_signature(state: BeaconState, block_root: Root, - validator_index: ValidatorIndex, + validator_index: ValidatorIndex, privkey: int) -> SyncCommitteeSignature: epoch = get_current_epoch(state) domain = get_domain(state, DOMAIN_SYNC_COMMITTEE, epoch) @@ -286,7 +286,7 @@ def get_sync_committee_signature(state: BeaconState, The validator broadcasts the assembled signature to the assigned subnet, the `sync_committee_{subnet_id}` pubsub topic. The `subnet_id` is derived from the position in the sync committee such that the sync committee is divided into "subcommittees". -`subnet_id` can be computed via `compute_subnets_for_sync_committee()` where `state` is a `BeaconState` during the matching sync committee period. +`subnet_id` can be computed via `compute_subnets_for_sync_committee()` where `state` is a `BeaconState` during the matching sync committee period. *Note*: This function returns multiple subnets if a given validator index is included multiple times in a given sync committee across multiple subcommittees. @@ -340,7 +340,7 @@ def is_sync_committee_aggregator(signature: BLSSignature) -> bool: ##### Construct sync committee contribution -If a validator is selected to aggregate the `SyncCommitteeSignature`s produced on a subnet during a given `slot`, they construct an aggregated `SyncCommitteeContribution`. +If a validator is selected to aggregate the `SyncCommitteeSignature`s produced on a subnet during a given `slot`, they construct an aggregated `SyncCommitteeContribution`. Given all of the (valid) collected `sync_committee_signatures: Set[SyncCommitteeSignature]` from the `sync_committee_{subnet_id}` gossip during the selected `slot` with an equivalent `beacon_block_root` to that of the aggregator, the aggregator creates a `contribution: SyncCommitteeContribution` with the following fields: @@ -361,13 +361,13 @@ Set `contribution.subcommittee_index` to the index for the subcommittee index co Let `contribution.aggregation_bits` be a `Bitvector[SYNC_COMMITTEE_SIZE // SYNC_COMMITTEE_SUBNET_COUNT]`, where the `index`th bit is set in the `Bitvector` for each corresponding validator included in this aggregate from the corresponding subcommittee. An aggregator finds the index in the sync committee (as returned by `get_sync_committee_indices()`) for a given validator referenced by `sync_committee_signature.validator_index` and maps the sync committee index to an index in the subcommittee (along with the prior `subcommittee_index`). This index within the subcommittee is set in `contribution.aggegration_bits`. -For example, if a validator with index `2044` is pseudo-randomly sampled to sync committee index `135`. This sync committee index maps to `subcommittee_index` `1` with position `7` in the `Bitvector` for the contribution. +For example, if a validator with index `2044` is pseudo-randomly sampled to sync committee index `135`. This sync committee index maps to `subcommittee_index` `1` with position `7` in the `Bitvector` for the contribution. *Note*: A validator **could be included multiple times** in a given subcommittee such that multiple bits are set for a single `SyncCommitteeSignature`. ###### Signature -Set `contribution.signature = aggregate_signature` where `aggregate_signature` is obtained by assembling the appropriate collection of `BLSSignature`s from the set of `sync_committee_signatures` and using the `bls.Aggregate()` function to produce an aggregate `BLSSignature`. +Set `contribution.signature = aggregate_signature` where `aggregate_signature` is obtained by assembling the appropriate collection of `BLSSignature`s from the set of `sync_committee_signatures` and using the `bls.Aggregate()` function to produce an aggregate `BLSSignature`. The collection of input signatures should include one signature per validator who had a bit set in the `aggregation_bits` bitfield, with repeated signatures if one validator maps to multiple indices within the subcommittee. @@ -402,8 +402,8 @@ def get_contribution_and_proof(state: BeaconState, Then `signed_contribution_and_proof = SignedContributionAndProof(message=contribution_and_proof, signature=signature)` is constructed and broadcast. Where `signature` is obtained from: ```python -def get_contribution_and_proof_signature(state: BeaconState, - contribution_and_proof: ContributionAndProof, +def get_contribution_and_proof_signature(state: BeaconState, + contribution_and_proof: ContributionAndProof, privkey: int) -> BLSSignature: contribution = contribution_and_proof.contribution domain = get_domain(state, DOMAIN_CONTRIBUTION_AND_PROOF, compute_epoch_at_slot(contribution.slot)) From 7a168be862294d580003126d881cc7f4ba4e2584 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 4 May 2021 13:25:34 -0700 Subject: [PATCH 07/36] allow fault tolerance equal to threshold, not just above --- specs/altair/sync-protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index 925c882ef..fba263bb1 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -185,7 +185,7 @@ def process_light_client_update(store: LightClientStore, update: LightClientUpda store.valid_updates.append(update) if ( - sum(update.sync_committee_bits) * 3 > len(update.sync_committee_bits) * 2 + sum(update.sync_committee_bits) * 3 >= len(update.sync_committee_bits) * 2 and update.finality_header != BeaconBlockHeader() ): # Apply update if (1) 2/3 quorum is reached and (2) we have a finality proof. From 3b803241193083696c2922f5983712a269d8d62b Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 4 May 2021 13:42:02 -0700 Subject: [PATCH 08/36] Compute `LIGHT_CLIENT_UPDATE_TIMEOUT` in lieu of maintaining a constant --- configs/mainnet/altair.yaml | 2 -- configs/minimal/altair.yaml | 2 -- specs/altair/sync-protocol.md | 10 ++-------- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/configs/mainnet/altair.yaml b/configs/mainnet/altair.yaml index 3cd4b8419..6db260cca 100644 --- a/configs/mainnet/altair.yaml +++ b/configs/mainnet/altair.yaml @@ -47,8 +47,6 @@ ALTAIR_FORK_EPOCH: 18446744073709551615 MIN_SYNC_COMMITTEE_PARTICIPANTS: 1 # 2**13 MAX_VALID_LIGHT_CLIENT_UPDATES: 8192 -# 2**13 (=8192) -LIGHT_CLIENT_UPDATE_TIMEOUT: 8192 # Validator diff --git a/configs/minimal/altair.yaml b/configs/minimal/altair.yaml index f9b30eea2..5c5781010 100644 --- a/configs/minimal/altair.yaml +++ b/configs/minimal/altair.yaml @@ -47,8 +47,6 @@ ALTAIR_FORK_EPOCH: 18446744073709551615 MIN_SYNC_COMMITTEE_PARTICIPANTS: 1 # [customized] MAX_VALID_LIGHT_CLIENT_UPDATES: 32 -# [customized] -LIGHT_CLIENT_UPDATE_TIMEOUT: 32 # Validator # --------------------------------------------------------------- diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index fba263bb1..ca55c92f7 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -12,7 +12,6 @@ - [Constants](#constants) - [Configuration](#configuration) - [Misc](#misc) - - [Time parameters](#time-parameters) - [Containers](#containers) - [`LightClientSnapshot`](#lightclientsnapshot) - [`LightClientUpdate`](#lightclientupdate) @@ -53,12 +52,6 @@ uses sync committees introduced in [this beacon chain extension](./beacon-chain. | `MIN_SYNC_COMMITTEE_PARTICIPANTS` | `1` | | `MAX_VALID_LIGHT_CLIENT_UPDATES` | `uint64(2**64 - 1)` | -### Time parameters - -| Name | Value | Unit | Duration | -| - | - | :-: | :-: | -| `LIGHT_CLIENT_UPDATE_TIMEOUT` | `Slot(2**13)` | slots | ~27 hours | - ## Containers ### `LightClientSnapshot` @@ -184,6 +177,7 @@ def process_light_client_update(store: LightClientStore, update: LightClientUpda validate_light_client_update(store.snapshot, update, genesis_validators_root) store.valid_updates.append(update) + update_timeout = SLOTS_PER_EPOCH * EPOCHS_PER_SYNC_COMMITTEE_PERIOD if ( sum(update.sync_committee_bits) * 3 >= len(update.sync_committee_bits) * 2 and update.finality_header != BeaconBlockHeader() @@ -193,7 +187,7 @@ def process_light_client_update(store: LightClientStore, update: LightClientUpda # It may be changed to re-organizable light client design. See the on-going issue eth2.0-specs#2182. apply_light_client_update(store.snapshot, update) store.valid_updates = [] - elif current_slot > store.snapshot.header.slot + LIGHT_CLIENT_UPDATE_TIMEOUT: + elif current_slot > store.snapshot.header.slot + update_timeout: # Forced best update when the update timeout has elapsed apply_light_client_update(store.snapshot, max(store.valid_updates, key=lambda update: sum(update.sync_committee_bits))) From 165c960cf15b63c30bed2dd2ce9b1a36c7734093 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 4 May 2021 13:46:30 -0700 Subject: [PATCH 09/36] Update config value to reflect spec --- configs/mainnet/altair.yaml | 4 ++-- configs/minimal/altair.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/mainnet/altair.yaml b/configs/mainnet/altair.yaml index 6db260cca..b7b7603a2 100644 --- a/configs/mainnet/altair.yaml +++ b/configs/mainnet/altair.yaml @@ -45,8 +45,8 @@ ALTAIR_FORK_EPOCH: 18446744073709551615 # --------------------------------------------------------------- # 1 MIN_SYNC_COMMITTEE_PARTICIPANTS: 1 -# 2**13 -MAX_VALID_LIGHT_CLIENT_UPDATES: 8192 +# 2**64 - 1 +MAX_VALID_LIGHT_CLIENT_UPDATES: 18446744073709551615 # Validator diff --git a/configs/minimal/altair.yaml b/configs/minimal/altair.yaml index 5c5781010..5618af8ff 100644 --- a/configs/minimal/altair.yaml +++ b/configs/minimal/altair.yaml @@ -45,8 +45,8 @@ ALTAIR_FORK_EPOCH: 18446744073709551615 # --------------------------------------------------------------- # 1 MIN_SYNC_COMMITTEE_PARTICIPANTS: 1 -# [customized] -MAX_VALID_LIGHT_CLIENT_UPDATES: 32 +# 2**64 - 1 +MAX_VALID_LIGHT_CLIENT_UPDATES: 18446744073709551615 # Validator # --------------------------------------------------------------- From f37f9a367d4f55957ab70c59e6502993cf43ad3a Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 5 May 2021 00:15:15 +0200 Subject: [PATCH 10/36] include merge in generators --- tests/generators/epoch_processing/main.py | 10 ++++++++-- tests/generators/finality/main.py | 9 ++++++--- tests/generators/fork_choice/main.py | 8 ++++++-- tests/generators/operations/main.py | 13 +++++++++++-- tests/generators/rewards/main.py | 11 +++++++++-- tests/generators/sanity/main.py | 11 +++++++++-- tests/generators/ssz_static/main.py | 2 +- 7 files changed, 50 insertions(+), 14 deletions(-) diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index 9efd96534..2aa6381ff 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -1,10 +1,11 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.altair import spec as spec_altair -from eth2spec.test.helpers.constants import PHASE0, ALTAIR +from eth2spec.merge import spec as spec_merge +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE -specs = (spec_phase0, spec_altair) +specs = (spec_phase0, spec_altair, spec_merge) if __name__ == "__main__": @@ -27,6 +28,10 @@ if __name__ == "__main__": **phase_0_mods, } # also run the previous phase 0 tests + # No epoch-processing changes in Merge and previous testing repeats with new types, so no additional tests required. + # TODO: rebase onto Altair testing later. + merge_mods = phase_0_mods + # TODO Custody Game testgen is disabled for now # custody_game_mods = {**{key: 'eth2spec.test.custody_game.epoch_processing.test_process_' + key for key in [ # 'reveal_deadlines', @@ -37,6 +42,7 @@ if __name__ == "__main__": all_mods = { PHASE0: phase_0_mods, ALTAIR: altair_mods, + MERGE: merge_mods, } run_state_test_generators(runner_name="epoch_processing", specs=specs, all_mods=all_mods) diff --git a/tests/generators/finality/main.py b/tests/generators/finality/main.py index e50425da4..148ddef96 100644 --- a/tests/generators/finality/main.py +++ b/tests/generators/finality/main.py @@ -1,19 +1,22 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.altair import spec as spec_altair -from eth2spec.test.helpers.constants import PHASE0, ALTAIR +from eth2spec.merge import spec as spec_merge +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE -specs = (spec_phase0, spec_altair) +specs = (spec_phase0, spec_altair, spec_merge) if __name__ == "__main__": phase_0_mods = {'finality': 'eth2spec.test.phase0.finality.test_finality'} - altair_mods = phase_0_mods # No additional altair specific finality tests + altair_mods = phase_0_mods # No additional Altair specific finality tests + merge_mods = phase_0_mods # No additional Merge specific finality tests all_mods = { PHASE0: phase_0_mods, ALTAIR: altair_mods, + MERGE: spec_merge, } run_state_test_generators(runner_name="finality", specs=specs, all_mods=all_mods) diff --git a/tests/generators/fork_choice/main.py b/tests/generators/fork_choice/main.py index 445ee629c..ae15caa1d 100644 --- a/tests/generators/fork_choice/main.py +++ b/tests/generators/fork_choice/main.py @@ -1,10 +1,11 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.altair import spec as spec_altair -from eth2spec.test.helpers.constants import PHASE0, ALTAIR +from eth2spec.merge import spec as spec_merge +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE -specs = (spec_phase0, spec_altair) +specs = (spec_phase0, spec_altair, spec_merge) if __name__ == "__main__": @@ -13,10 +14,13 @@ if __name__ == "__main__": ]} # No additional Altair specific finality tests, yet. altair_mods = phase_0_mods + # No specific Merge tests yet. TODO: rebase onto Altair testing later. + merge_mods = phase_0_mods all_mods = { PHASE0: phase_0_mods, ALTAIR: altair_mods, + MERGE: merge_mods, } run_state_test_generators(runner_name="fork_choice", specs=specs, all_mods=all_mods) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index 0c1b84c24..316ccdf10 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -1,10 +1,11 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.altair import spec as spec_altair -from eth2spec.test.helpers.constants import PHASE0, ALTAIR +from eth2spec.merge import spec as spec_merge +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE -specs = (spec_phase0, spec_altair) +specs = (spec_phase0, spec_altair, spec_merge) if __name__ == "__main__": @@ -23,6 +24,13 @@ if __name__ == "__main__": **phase_0_mods, } # also run the previous phase 0 tests + merge_mods = { + **{key: 'eth2spec.test.merge.block_processing.test_process_' + key for key in [ + 'execution_payload', + ]}, + **phase_0_mods, # TODO: runs phase0 tests. Rebase to include `altair_mods` testing later. + } + # TODO Custody Game testgen is disabled for now # custody_game_mods = {**{key: 'eth2spec.test.custody_game.block_processing.test_process_' + key for key in [ # 'attestation', @@ -35,6 +43,7 @@ if __name__ == "__main__": all_mods = { PHASE0: phase_0_mods, ALTAIR: altair_mods, + MERGE: merge_mods, } run_state_test_generators(runner_name="operations", specs=specs, all_mods=all_mods) diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index 2a0f14863..8e50732e1 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -1,10 +1,11 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.altair import spec as spec_altair -from eth2spec.test.helpers.constants import PHASE0, ALTAIR +from eth2spec.merge import spec as spec_merge +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE -specs = (spec_phase0, spec_altair) +specs = (spec_phase0, spec_altair, spec_merge) if __name__ == "__main__": @@ -16,9 +17,15 @@ if __name__ == "__main__": # No additional altair specific rewards tests, yet. altair_mods = phase_0_mods + # No additional merge specific rewards tests, yet. + # Note: Block rewards are non-epoch rewards and are tested as part of block processing tests. + # Transaction fees are part of the execution-layer. + merge_mods = phase_0_mods + all_mods = { PHASE0: phase_0_mods, ALTAIR: altair_mods, + MERGE: merge_mods, } run_state_test_generators(runner_name="rewards", specs=specs, all_mods=all_mods) diff --git a/tests/generators/sanity/main.py b/tests/generators/sanity/main.py index 7f7fecc67..3bed6672d 100644 --- a/tests/generators/sanity/main.py +++ b/tests/generators/sanity/main.py @@ -1,11 +1,12 @@ from eth2spec.phase0 import spec as spec_phase0 from eth2spec.altair import spec as spec_altair -from eth2spec.test.helpers.constants import PHASE0, ALTAIR +from eth2spec.merge import spec as spec_merge +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators -specs = (spec_phase0, spec_altair) +specs = (spec_phase0, spec_altair, spec_merge) if __name__ == "__main__": @@ -17,9 +18,15 @@ if __name__ == "__main__": 'blocks', ]}, **phase_0_mods} # also run the previous phase 0 tests + # Altair-specific test cases are ignored, but should be included after the Merge is rebased onto Altair work. + merge_mods = {**{key: 'eth2spec.test.merge.sanity.test_' + key for key in [ + 'blocks', + ]}, **phase_0_mods} # TODO: Merge inherits phase0 tests for now. + all_mods = { PHASE0: phase_0_mods, ALTAIR: altair_mods, + MERGE: merge_mods, } run_state_test_generators(runner_name="sanity", specs=specs, all_mods=all_mods) diff --git a/tests/generators/ssz_static/main.py b/tests/generators/ssz_static/main.py index 94ce02335..1e810f71e 100644 --- a/tests/generators/ssz_static/main.py +++ b/tests/generators/ssz_static/main.py @@ -94,7 +94,7 @@ if __name__ == "__main__": settings.append((seed, MAINNET, random_value.RandomizationMode.mode_random, False, 5)) seed += 1 # TODO: enable testing for the whole merge spec. - for fork in TESTGEN_FORKS + (MERGE,): + for fork in TESTGEN_FORKS: gen_runner.run_generator("ssz_static", [ create_provider(fork, config_name, seed, mode, chaos, cases_if_random) for (seed, config_name, mode, chaos, cases_if_random) in settings From ab693c972445a7ba362d2f527447461f377ce82a Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 5 May 2021 00:16:22 +0200 Subject: [PATCH 11/36] update spec test constants for merge --- tests/core/pyspec/eth2spec/test/helpers/constants.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/constants.py b/tests/core/pyspec/eth2spec/test/helpers/constants.py index ccd7b20a2..d8f2a37ba 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/constants.py +++ b/tests/core/pyspec/eth2spec/test/helpers/constants.py @@ -7,21 +7,24 @@ from .typing import SpecForkName, ConfigName # Some of the Spec module functionality is exposed here to deal with phase-specific changes. PHASE0 = SpecForkName('phase0') ALTAIR = SpecForkName('altair') +MERGE = SpecForkName('merge') # Experimental phases (not included in default "ALL_PHASES"): -MERGE = SpecForkName('merge') SHARDING = SpecForkName('sharding') CUSTODY_GAME = SpecForkName('custody_game') DAS = SpecForkName('das') # The forks that pytest runs with. -ALL_PHASES = (PHASE0, ALTAIR) +ALL_PHASES = (PHASE0, ALTAIR, MERGE) # The forks that output to the test vectors. -TESTGEN_FORKS = (PHASE0, ALTAIR) +TESTGEN_FORKS = (PHASE0, ALTAIR, MERGE) # TODO: everything runs in parallel to Altair. # After features are rebased on the Altair fork, this can be reduced to just PHASE0. FORKS_BEFORE_ALTAIR = (PHASE0, MERGE, SHARDING, CUSTODY_GAME, DAS) +# TODO: when rebasing Merge onto Altair, add ALTAIR to this tuple. +FORKS_BEFORE_MERGE = (PHASE0,) + # # Config # From 521cffc3e9c4de950489470eb1eefbf2871a5c33 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 5 May 2021 00:17:10 +0200 Subject: [PATCH 12/36] update execution-payload processing to isolate payload from block body --- specs/merge/beacon-chain.md | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/specs/merge/beacon-chain.md b/specs/merge/beacon-chain.md index b9c443962..d665f1151 100644 --- a/specs/merge/beacon-chain.md +++ b/specs/merge/beacon-chain.md @@ -146,8 +146,8 @@ def is_transition_completed(state: BeaconState) -> bool: #### `is_transition_block` ```python -def is_transition_block(state: BeaconState, block_body: BeaconBlockBody) -> bool: - return not is_transition_completed(state) and block_body.execution_payload != ExecutionPayload() +def is_transition_block(state: BeaconState, block: BeaconBlock) -> bool: + return not is_transition_completed(state) and block.body.execution_payload != ExecutionPayload() ``` #### `compute_time_at_slot` @@ -168,7 +168,9 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_randao(state, block.body) process_eth1_data(state, block.body) process_operations(state, block.body) - process_execution_payload(state, block.body) # [New in Merge] + # Pre-merge, skip execution payload processing + if is_transition_completed(state) or is_transition_block(state, block): + process_execution_payload(state, block.body.execution_payload) # [New in Merge] ``` #### Execution payload processing @@ -181,16 +183,10 @@ The body of the function is implementation dependent. ##### `process_execution_payload` ```python -def process_execution_payload(state: BeaconState, body: BeaconBlockBody) -> None: +def process_execution_payload(state: BeaconState, execution_payload: ExecutionPayload) -> None: """ Note: This function is designed to be able to be run in parallel with the other `process_block` sub-functions """ - # Pre-merge, skip processing - if not is_transition_completed(state) and not is_transition_block(state, body): - return - - execution_payload = body.execution_payload - if is_transition_completed(state): assert execution_payload.parent_hash == state.latest_execution_payload_header.block_hash assert execution_payload.number == state.latest_execution_payload_header.number + 1 From 25d0d673a9b0950c0aa2d0b2406f849c9dca0470 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 5 May 2021 00:18:01 +0200 Subject: [PATCH 13/36] start testing of merge functionality --- tests/core/pyspec/eth2spec/test/context.py | 8 +- .../pyspec/eth2spec/test/helpers/block.py | 6 +- .../test/helpers/execution_payload.py | 26 ++++ .../pyspec/eth2spec/test/merge/__init__.py | 0 .../test/merge/block_processing/__init__.py | 0 .../test_process_execution_payload.py | 147 ++++++++++++++++++ .../eth2spec/test/merge/sanity/__init__.py | 0 .../eth2spec/test/merge/sanity/test_blocks.py | 25 +++ 8 files changed, 210 insertions(+), 2 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/helpers/execution_payload.py create mode 100644 tests/core/pyspec/eth2spec/test/merge/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/merge/block_processing/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py create mode 100644 tests/core/pyspec/eth2spec/test/merge/sanity/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/merge/sanity/test_blocks.py diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 438e611cf..8699c65ad 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -8,7 +8,7 @@ from eth2spec.utils import bls from .exceptions import SkippedTest from .helpers.constants import ( PHASE0, ALTAIR, - ALL_PHASES, FORKS_BEFORE_ALTAIR, + ALL_PHASES, FORKS_BEFORE_ALTAIR, FORKS_BEFORE_MERGE, ) from .helpers.genesis import create_genesis_state from .utils import vector_test, with_meta_tags @@ -365,3 +365,9 @@ def is_post_altair(spec): if spec.fork in FORKS_BEFORE_ALTAIR: return False return True + + +def is_post_merge(spec): + if spec.fork in FORKS_BEFORE_MERGE: + return False + return True diff --git a/tests/core/pyspec/eth2spec/test/helpers/block.py b/tests/core/pyspec/eth2spec/test/helpers/block.py index 6949b54bc..64f406633 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/block.py @@ -1,4 +1,5 @@ -from eth2spec.test.context import is_post_altair +from eth2spec.test.context import is_post_altair, is_post_merge +from eth2spec.test.helpers.execution_payload import build_empty_execution_payload from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.bls import only_with_bls @@ -94,6 +95,9 @@ def build_empty_block(spec, state, slot=None): if is_post_altair(spec): empty_block.body.sync_aggregate.sync_committee_signature = spec.G2_POINT_AT_INFINITY + if is_post_merge(spec): + empty_block.body.execution_payload = build_empty_execution_payload(spec, state) + apply_randao_reveal(spec, state, empty_block) return empty_block diff --git a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py new file mode 100644 index 000000000..e829e07c7 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py @@ -0,0 +1,26 @@ + +def build_empty_execution_payload(spec, state): + """ + Assuming a pre-state of the same slot, build a valid ExecutionPayload without any transactions. + """ + latest = state.latest_execution_payload_header + timestamp = spec.compute_time_at_slot(state, state.slot) + empty_txs = spec.List[spec.OpaqueTransaction, spec.MAX_EXECUTION_TRANSACTIONS]() + + payload = spec.ExecutionPayload( + block_hash=spec.Hash32(), + parent_hash=latest.block_hash, + coinbase=spec.Bytes20(), + state_root=latest.state_root, # no changes to the state + number=latest.number + 1, + gas_limit=latest.gas_limit, # retain same limit + gas_used=0, # empty block, 0 gas + timestamp=timestamp, + receipt_root=b"no receipts here" + b"\x00"*16, # TODO: root of empty MPT may be better. + logs_bloom=spec.ByteVector[spec.BYTES_PER_LOGS_BLOOM](), # TODO: zeroed logs bloom for empty logs ok? + transactions_root=empty_txs, + ) + # TODO: real RLP + block hash logic would be nice, requires RLP and keccak256 dependency however. + payload.block_hash = spec.Hash32(spec.hash(payload.hash_tree_root() + b"FAKE RLP HASH")) + + return payload diff --git a/tests/core/pyspec/eth2spec/test/merge/__init__.py b/tests/core/pyspec/eth2spec/test/merge/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/merge/block_processing/__init__.py b/tests/core/pyspec/eth2spec/test/merge/block_processing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py new file mode 100644 index 000000000..0cf5cef5f --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py @@ -0,0 +1,147 @@ +from eth2spec.test.helpers.constants import PHASE0, ALTAIR +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases_except + +with_merge_and_later = with_all_phases_except([PHASE0, ALTAIR]) + + +def run_execution_payload_processing(spec, state, execution_payload, valid=True, execution_valid=True): + """ + Run ``process_execution_payload``, yielding: + - pre-state ('pre') + - execution payload ('execution_payload') + - execution details, to mock EVM execution ('execution.yml', a dict with 'execution_valid' key and boolean value) + - post-state ('post'). + If ``valid == False``, run expecting ``AssertionError`` + """ + + yield 'pre', state + yield 'execution', {'execution_valid': execution_valid} + yield 'execution_payload', execution_payload + + if not valid: + expect_assertion_error(lambda: spec.process_execution_payload(state, execution_payload)) + yield 'post', None + return + + spec.process_execution_payload(state, execution_payload) + + yield 'post', state + + # TODO: any assertions to make? + + +@with_merge_and_later +@spec_state_test +def test_success_first_payload(spec, state): + assert not spec.is_transition_completed(state) + + # TODO: execution payload + execution_payload = spec.ExecutionPayload() + + yield from run_execution_payload_processing(spec, state, execution_payload) + + +@with_merge_and_later +@spec_state_test +def test_success_regular_payload(spec, state): + # TODO: setup state + assert spec.is_transition_completed(state) + + # TODO: execution payload + execution_payload = spec.ExecutionPayload() + + yield from run_execution_payload_processing(spec, state, execution_payload) + + +@with_merge_and_later +@spec_state_test +def test_success_first_payload_with_gap_slot(spec, state): + # TODO: transition gap slot + + assert not spec.is_transition_completed(state) + + # TODO: execution payload + execution_payload = spec.ExecutionPayload() + + yield from run_execution_payload_processing(spec, state, execution_payload) + + +@with_merge_and_later +@spec_state_test +def test_success_regular_payload_with_gap_slot(spec, state): + # TODO: setup state + assert spec.is_transition_completed(state) + # TODO: transition gap slot + + # TODO: execution payload + execution_payload = spec.ExecutionPayload() + + yield from run_execution_payload_processing(spec, state, execution_payload) + + +@with_merge_and_later +@spec_state_test +def test_bad_execution_first_payload(spec, state): + # completely valid payload, but execution itself fails (e.g. block exceeds gas limit) + + # TODO: execution payload. + execution_payload = spec.ExecutionPayload() + + yield from run_execution_payload_processing(spec, state, execution_payload, valid=False, execution_valid=False) + + +@with_merge_and_later +@spec_state_test +def test_bad_execution_regular_payload(spec, state): + # completely valid payload, but execution itself fails (e.g. block exceeds gas limit) + + # TODO: execution payload + execution_payload = spec.ExecutionPayload() + + yield from run_execution_payload_processing(spec, state, execution_payload, valid=False, execution_valid=False) + + +@with_merge_and_later +@spec_state_test +def test_bad_parent_hash_first_payload(spec, state): + # TODO: execution payload + execution_payload = spec.ExecutionPayload() + + yield from run_execution_payload_processing(spec, state, execution_payload, valid=False) + + +@with_merge_and_later +@spec_state_test +def test_bad_number_first_payload(spec, state): + # TODO: execution payload + execution_payload = spec.ExecutionPayload() + + yield from run_execution_payload_processing(spec, state, execution_payload, valid=False) + + +@with_merge_and_later +@spec_state_test +def test_bad_everything_first_payload(spec, state): + # TODO: execution payload + execution_payload = spec.ExecutionPayload() + + yield from run_execution_payload_processing(spec, state, execution_payload, valid=False) + + +@with_merge_and_later +@spec_state_test +def test_bad_timestamp_first_payload(spec, state): + # TODO: execution payload + execution_payload = spec.ExecutionPayload() + + yield from run_execution_payload_processing(spec, state, execution_payload, valid=False) + + +@with_merge_and_later +@spec_state_test +def test_bad_timestamp_regular_payload(spec, state): + # TODO: execution payload + execution_payload = spec.ExecutionPayload() + + yield from run_execution_payload_processing(spec, state, execution_payload, valid=False) + diff --git a/tests/core/pyspec/eth2spec/test/merge/sanity/__init__.py b/tests/core/pyspec/eth2spec/test/merge/sanity/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/merge/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/merge/sanity/test_blocks.py new file mode 100644 index 000000000..1f9e69167 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/merge/sanity/test_blocks.py @@ -0,0 +1,25 @@ +from eth2spec.test.helpers.state import ( + state_transition_and_sign_block +) +from eth2spec.test.helpers.block import ( + build_empty_block_for_next_slot +) +from eth2spec.test.context import ( + with_all_phases, spec_state_test +) + + +@with_all_phases +@spec_state_test +def test_empty_block_transition(spec, state): + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + assert len(block.body.execution_payload.transactions) == 0 + + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state + +# TODO: tests with EVM, mock or replacement? From 36032fd115c1be0eef52ac4a68c3cc8672b5ed70 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 5 May 2021 00:37:00 +0200 Subject: [PATCH 14/36] update doc about format --- tests/formats/operations/README.md | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/formats/operations/README.md b/tests/formats/operations/README.md index 033c5fdef..f562a6f2a 100644 --- a/tests/formats/operations/README.md +++ b/tests/formats/operations/README.md @@ -33,17 +33,23 @@ This excludes the other parts of the block-transition. Operations: -| *`operation-name`* | *`operation-object`* | *`input name`* | *`processing call`* | -|-------------------------|-----------------------|----------------------|-----------------------------------------------------------------| -| `attestation` | `Attestation` | `attestation` | `process_attestation(state, attestation)` | -| `attester_slashing` | `AttesterSlashing` | `attester_slashing` | `process_attester_slashing(state, attester_slashing)` | -| `block_header` | `BeaconBlock` | **`block`** | `process_block_header(state, block)` | -| `deposit` | `Deposit` | `deposit` | `process_deposit(state, deposit)` | -| `proposer_slashing` | `ProposerSlashing` | `proposer_slashing` | `process_proposer_slashing(state, proposer_slashing)` | -| `voluntary_exit` | `SignedVoluntaryExit` | `voluntary_exit` | `process_voluntary_exit(state, voluntary_exit)` | -| `sync_aggregate` | `SyncAggregate` | `sync_aggregate` | `process_sync_committee(state, sync_aggregate)` (new in Altair) | +| *`operation-name`* | *`operation-object`* | *`input name`* | *`processing call`* | +|-------------------------|-----------------------|----------------------|----------------------------------------------------------------------| +| `attestation` | `Attestation` | `attestation` | `process_attestation(state, attestation)` | +| `attester_slashing` | `AttesterSlashing` | `attester_slashing` | `process_attester_slashing(state, attester_slashing)` | +| `block_header` | `BeaconBlock` | **`block`** | `process_block_header(state, block)` | +| `deposit` | `Deposit` | `deposit` | `process_deposit(state, deposit)` | +| `proposer_slashing` | `ProposerSlashing` | `proposer_slashing` | `process_proposer_slashing(state, proposer_slashing)` | +| `voluntary_exit` | `SignedVoluntaryExit` | `voluntary_exit` | `process_voluntary_exit(state, voluntary_exit)` | +| `sync_aggregate` | `SyncAggregate` | `sync_aggregate` | `process_sync_committee(state, sync_aggregate)` (new in Altair) | +| `execution_payload` | `ExecutionPayload` | `execution_payload` | `process_execution_payload(state, execution_payload)` (new in Merge) | Note that `block_header` is not strictly an operation (and is a full `Block`), but processed in the same manner, and hence included here. +The `execution_payload` processing normally requires a `verify_execution_state_transition(execution_payload)`, +a responsibility of an (external) execution engine. +During testing this execution is mocked, an `execution.yml` is provided instead: +a dict containing an `execution_valid` boolean field with the verification result. + The resulting state should match the expected `post` state, or if the `post` state is left blank, the handler should reject the input operation as invalid. From a562f2aeb45026902f6d9ae3e92e338ddcd87d6c Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 4 May 2021 17:05:15 -0700 Subject: [PATCH 15/36] "toward" -> "closer to" for penalty adjustments Makes it clearer that even the Altair values are not final --- specs/altair/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 726747c59..7bf0c2d99 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -104,7 +104,7 @@ Altair is the first beacon chain hard fork. Its main features are: ### Updated penalty values -This patch updates a few configuration values to move penalty parameters toward their final, maximum security values. +This patch updates a few configuration values to move penalty parameters closer to their final, maximum security values. *Note*: The spec does *not* override previous configuration values but instead creates new values and replaces usage throughout. From 79fc41146d898d80358ff6c746217d827587ee2c Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Wed, 5 May 2021 13:37:07 +0600 Subject: [PATCH 16/36] Adjust is_transition_block call in fork-choice --- specs/merge/fork-choice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/merge/fork-choice.md b/specs/merge/fork-choice.md index 34647f45d..f478dd7e6 100644 --- a/specs/merge/fork-choice.md +++ b/specs/merge/fork-choice.md @@ -75,7 +75,7 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: assert get_ancestor(store, block.parent_root, finalized_slot) == store.finalized_checkpoint.root # [New in Merge] - if is_transition_block(pre_state, block.body): + if is_transition_block(pre_state, block): # Delay consideration of block until PoW block is processed by the PoW node pow_block = get_pow_block(block.body.execution_payload.parent_hash) assert pow_block.is_processed From e2be7614cc2224c75ac6c0356dff48a9b31563e0 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 5 May 2021 15:35:36 +0200 Subject: [PATCH 17/36] introduce merge fork version --- specs/merge/beacon-chain.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/specs/merge/beacon-chain.md b/specs/merge/beacon-chain.md index d665f1151..50b9551b0 100644 --- a/specs/merge/beacon-chain.md +++ b/specs/merge/beacon-chain.md @@ -13,8 +13,8 @@ - [Introduction](#introduction) - [Custom types](#custom-types) - [Constants](#constants) - - [Transition](#transition) - [Execution](#execution) +- [Configuration](#configuration) - [Containers](#containers) - [Extended containers](#extended-containers) - [`BeaconBlockBody`](#beaconblockbody) @@ -50,12 +50,6 @@ We define the following Python custom types for type hinting and readability: ## Constants -### Transition - -| Name | Value | -| - | - | -| `TRANSITION_TOTAL_DIFFICULTY` | **TBD** | - ### Execution | Name | Value | @@ -64,6 +58,16 @@ We define the following Python custom types for type hinting and readability: | `MAX_EXECUTION_TRANSACTIONS` | `uint64(2**14)` (= 16,384) | | `BYTES_PER_LOGS_BLOOM` | `uint64(2**8)` (= 256) | +## Configuration + +Warning: this configuration is not definitive. + +| Name | Value | +| - | - | +| `MERGE_FORK_VERSION` | `Version('0x02000000')` | +| `MERGE_FORK_EPOCH` | `Epoch(18446744073709551615)` **TBD** | +| `TRANSITION_TOTAL_DIFFICULTY` | **TBD** | + ## Containers ### Extended containers From 470c6dcc6f38093a6991ee8bb78d1385d11d0d08 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 5 May 2021 15:35:52 +0200 Subject: [PATCH 18/36] update test runner to handle merge phase --- tests/core/pyspec/eth2spec/test/context.py | 9 +++++++-- tests/core/pyspec/eth2spec/test/helpers/genesis.py | 3 +++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 8699c65ad..65201c0c4 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -7,7 +7,7 @@ from eth2spec.utils import bls from .exceptions import SkippedTest from .helpers.constants import ( - PHASE0, ALTAIR, + PHASE0, ALTAIR, MERGE, ALL_PHASES, FORKS_BEFORE_ALTAIR, FORKS_BEFORE_MERGE, ) from .helpers.genesis import create_genesis_state @@ -312,7 +312,7 @@ def with_phases(phases, other_phases=None): return None run_phases = [phase] - if PHASE0 not in run_phases and ALTAIR not in run_phases: + if PHASE0 not in run_phases and ALTAIR not in run_phases and MERGE not in run_phases: dump_skipping_message("none of the recognized phases are executable, skipping test.") return None @@ -331,12 +331,17 @@ def with_phases(phases, other_phases=None): if ALTAIR in available_phases: phase_dir[ALTAIR] = spec_altair + if MERGE in available_phases: + phase_dir[MERGE] = spec_merge + # return is ignored whenever multiple phases are ran. # This return is for test generators to emit python generators (yielding test vector outputs) if PHASE0 in run_phases: ret = fn(spec=spec_phase0, phases=phase_dir, *args, **kw) if ALTAIR in run_phases: ret = fn(spec=spec_altair, phases=phase_dir, *args, **kw) + if MERGE in run_phases: + ret = fn(spec=spec_merge, phases=phase_dir, *args, **kw) # TODO: merge, sharding, custody_game and das are not executable yet. # Tests that specify these features will not run, and get ignored for these specific phases. diff --git a/tests/core/pyspec/eth2spec/test/helpers/genesis.py b/tests/core/pyspec/eth2spec/test/helpers/genesis.py index 49af43ec1..4a34a5eb3 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/genesis.py +++ b/tests/core/pyspec/eth2spec/test/helpers/genesis.py @@ -1,6 +1,7 @@ from eth2spec.test.helpers.constants import ( ALTAIR, FORKS_BEFORE_ALTAIR, + MERGE, ) from eth2spec.test.helpers.keys import pubkeys @@ -28,6 +29,8 @@ def create_genesis_state(spec, validator_balances, activation_threshold): if spec.fork == ALTAIR: current_version = spec.ALTAIR_FORK_VERSION + elif spec.fork == MERGE: + current_version = spec.MERGE_FORK_VERSION state = spec.BeaconState( genesis_time=0, From ff3a82e0f38de19559bbba52e2693e01d4dad44f Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 5 May 2021 15:40:56 +0200 Subject: [PATCH 19/36] fix transactions field in exec payload helper --- tests/core/pyspec/eth2spec/test/helpers/execution_payload.py | 2 +- .../merge/block_processing/test_process_execution_payload.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py index e829e07c7..0d035350d 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py @@ -18,7 +18,7 @@ def build_empty_execution_payload(spec, state): timestamp=timestamp, receipt_root=b"no receipts here" + b"\x00"*16, # TODO: root of empty MPT may be better. logs_bloom=spec.ByteVector[spec.BYTES_PER_LOGS_BLOOM](), # TODO: zeroed logs bloom for empty logs ok? - transactions_root=empty_txs, + transactions=empty_txs, ) # TODO: real RLP + block hash logic would be nice, requires RLP and keccak256 dependency however. payload.block_hash = spec.Hash32(spec.hash(payload.hash_tree_root() + b"FAKE RLP HASH")) diff --git a/tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py index 0cf5cef5f..f3f521c4b 100644 --- a/tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py @@ -1,4 +1,5 @@ from eth2spec.test.helpers.constants import PHASE0, ALTAIR +from eth2spec.test.helpers.execution_payload import build_empty_execution_payload from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases_except with_merge_and_later = with_all_phases_except([PHASE0, ALTAIR]) @@ -36,7 +37,7 @@ def test_success_first_payload(spec, state): assert not spec.is_transition_completed(state) # TODO: execution payload - execution_payload = spec.ExecutionPayload() + execution_payload = build_empty_execution_payload(spec, state) yield from run_execution_payload_processing(spec, state, execution_payload) From 865d7db5cad448e8d2ac37859a74eff9d8a2d633 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 5 May 2021 16:03:52 +0200 Subject: [PATCH 20/36] update altair tests to not collide with Merge + fix merge test triggers --- .../test_process_sync_committee.py | 20 +++++++++---------- .../test_process_sync_committee_updates.py | 8 ++++---- .../test/altair/sanity/test_blocks.py | 17 ++++++++-------- .../unittests/validator/test_validator.py | 8 ++++---- tests/core/pyspec/eth2spec/test/context.py | 8 ++++++++ .../pyspec/eth2spec/test/helpers/block.py | 2 ++ .../test_process_execution_payload.py | 5 +---- .../eth2spec/test/merge/sanity/test_blocks.py | 4 ++-- 8 files changed, 39 insertions(+), 33 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py index e0faf3d0d..9b415cfbd 100644 --- a/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py @@ -17,7 +17,7 @@ from eth2spec.test.helpers.sync_committee import ( ) from eth2spec.test.context import ( expect_assertion_error, - with_all_phases_except, + with_altair_and_later, with_configs, spec_state_test, always_bls, @@ -62,7 +62,7 @@ def get_committee_indices(spec, state, duplicates=False): state.randao_mixes[randao_index] = hash(state.randao_mixes[randao_index]) -@with_all_phases_except([PHASE0]) +@with_altair_and_later @spec_state_test @always_bls def test_invalid_signature_missing_participant(spec, state): @@ -84,7 +84,7 @@ def test_invalid_signature_missing_participant(spec, state): yield from run_sync_committee_processing(spec, state, block, expect_exception=True) -@with_all_phases_except([PHASE0]) +@with_altair_and_later @spec_state_test @always_bls def test_invalid_signature_extra_participant(spec, state): @@ -197,7 +197,7 @@ def run_successful_sync_committee_test(spec, state, committee, committee_bits): ) -@with_all_phases_except([PHASE0]) +@with_altair_and_later @with_configs([MINIMAL], reason="to create nonduplicate committee") @spec_state_test def test_sync_committee_rewards_nonduplicate_committee(spec, state): @@ -213,7 +213,7 @@ def test_sync_committee_rewards_nonduplicate_committee(spec, state): yield from run_successful_sync_committee_test(spec, state, committee, committee_bits) -@with_all_phases_except([PHASE0]) +@with_altair_and_later @with_configs([MAINNET], reason="to create duplicate committee") @spec_state_test def test_sync_committee_rewards_duplicate_committee(spec, state): @@ -229,7 +229,7 @@ def test_sync_committee_rewards_duplicate_committee(spec, state): yield from run_successful_sync_committee_test(spec, state, committee, committee_bits) -@with_all_phases_except([PHASE0]) +@with_altair_and_later @spec_state_test @always_bls def test_sync_committee_rewards_not_full_participants(spec, state): @@ -240,7 +240,7 @@ def test_sync_committee_rewards_not_full_participants(spec, state): yield from run_successful_sync_committee_test(spec, state, committee, committee_bits) -@with_all_phases_except([PHASE0]) +@with_altair_and_later @spec_state_test @always_bls def test_sync_committee_rewards_empty_participants(spec, state): @@ -250,7 +250,7 @@ def test_sync_committee_rewards_empty_participants(spec, state): yield from run_successful_sync_committee_test(spec, state, committee, committee_bits) -@with_all_phases_except([PHASE0]) +@with_altair_and_later @spec_state_test @always_bls def test_invalid_signature_past_block(spec, state): @@ -289,7 +289,7 @@ def test_invalid_signature_past_block(spec, state): yield from run_sync_committee_processing(spec, state, invalid_block, expect_exception=True) -@with_all_phases_except([PHASE0]) +@with_altair_and_later @with_configs([MINIMAL], reason="to produce different committee sets") @spec_state_test @always_bls @@ -326,7 +326,7 @@ def test_invalid_signature_previous_committee(spec, state): yield from run_sync_committee_processing(spec, state, block, expect_exception=True) -@with_all_phases_except([PHASE0]) +@with_altair_and_later @spec_state_test @always_bls @with_configs([MINIMAL], reason="too slow") diff --git a/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py b/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py index 8ffc51507..c914c476d 100644 --- a/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py +++ b/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py @@ -2,7 +2,7 @@ from eth2spec.test.context import ( always_bls, spec_state_test, spec_test, - with_all_phases_except, + with_altair_and_later, with_configs, with_custom_state, single_phase, @@ -49,7 +49,7 @@ def run_sync_committees_progress_test(spec, state): assert state.next_sync_committee == third_sync_committee -@with_all_phases_except([PHASE0]) +@with_altair_and_later @spec_state_test @always_bls @with_configs([MINIMAL], reason="too slow") @@ -60,7 +60,7 @@ def test_sync_committees_progress_genesis(spec, state): yield from run_sync_committees_progress_test(spec, state) -@with_all_phases_except([PHASE0]) +@with_altair_and_later @spec_state_test @always_bls @with_configs([MINIMAL], reason="too slow") @@ -73,7 +73,7 @@ def test_sync_committees_progress_not_genesis(spec, state): yield from run_sync_committees_progress_test(spec, state) -@with_all_phases_except([PHASE0]) +@with_altair_and_later @with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test @single_phase diff --git a/tests/core/pyspec/eth2spec/test/altair/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/altair/sanity/test_blocks.py index 48ab6956b..ffe743531 100644 --- a/tests/core/pyspec/eth2spec/test/altair/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/altair/sanity/test_blocks.py @@ -11,9 +11,8 @@ from eth2spec.test.helpers.block import ( from eth2spec.test.helpers.sync_committee import ( compute_aggregate_sync_committee_signature, ) -from eth2spec.test.helpers.constants import PHASE0 from eth2spec.test.context import ( - with_all_phases_except, + with_altair_and_later, spec_state_test, ) @@ -40,46 +39,46 @@ def run_sync_committee_sanity_test(spec, state, fraction_full=1.0): yield 'post', state -@with_all_phases_except([PHASE0]) +@with_altair_and_later @spec_state_test def test_full_sync_committee_committee(spec, state): next_epoch(spec, state) yield from run_sync_committee_sanity_test(spec, state, fraction_full=1.0) -@with_all_phases_except([PHASE0]) +@with_altair_and_later @spec_state_test def test_half_sync_committee_committee(spec, state): next_epoch(spec, state) yield from run_sync_committee_sanity_test(spec, state, fraction_full=0.5) -@with_all_phases_except([PHASE0]) +@with_altair_and_later @spec_state_test def test_empty_sync_committee_committee(spec, state): next_epoch(spec, state) yield from run_sync_committee_sanity_test(spec, state, fraction_full=0.0) -@with_all_phases_except([PHASE0]) +@with_altair_and_later @spec_state_test def test_full_sync_committee_committee_genesis(spec, state): yield from run_sync_committee_sanity_test(spec, state, fraction_full=1.0) -@with_all_phases_except([PHASE0]) +@with_altair_and_later @spec_state_test def test_half_sync_committee_committee_genesis(spec, state): yield from run_sync_committee_sanity_test(spec, state, fraction_full=0.5) -@with_all_phases_except([PHASE0]) +@with_altair_and_later @spec_state_test def test_empty_sync_committee_committee_genesis(spec, state): yield from run_sync_committee_sanity_test(spec, state, fraction_full=0.0) -@with_all_phases_except([PHASE0]) +@with_altair_and_later @spec_state_test def test_inactivity_scores(spec, state): for _ in range(spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + 2): diff --git a/tests/core/pyspec/eth2spec/test/altair/unittests/validator/test_validator.py b/tests/core/pyspec/eth2spec/test/altair/unittests/validator/test_validator.py index 244e1aa7e..e1f5d3860 100644 --- a/tests/core/pyspec/eth2spec/test/altair/unittests/validator/test_validator.py +++ b/tests/core/pyspec/eth2spec/test/altair/unittests/validator/test_validator.py @@ -8,7 +8,7 @@ from eth2spec.utils import bls from eth2spec.utils.bls import only_with_bls from eth2spec.test.context import ( PHASE0, - with_all_phases_except, + with_altair_and_later, with_state, ) @@ -25,7 +25,7 @@ def ensure_assignments_in_sync_committee( assert spec.is_assigned_to_sync_committee(state, epoch, validator_index) -@with_all_phases_except([PHASE0]) +@with_altair_and_later @with_state def test_is_assigned_to_sync_committee(phases, spec, state): epoch = spec.get_current_epoch(state) @@ -91,7 +91,7 @@ def _get_sync_committee_signature( @only_with_bls() -@with_all_phases_except([PHASE0]) +@with_altair_and_later @with_state def test_process_sync_committee_contributions(phases, spec, state): # skip over slots at genesis @@ -144,7 +144,7 @@ def _subnet_for_sync_committee_index(spec, i): return i // (spec.SYNC_COMMITTEE_SIZE // spec.SYNC_COMMITTEE_SUBNET_COUNT) -@with_all_phases_except([PHASE0]) +@with_altair_and_later @with_state def test_compute_subnets_for_sync_committee(state, spec, phases): some_sync_committee_members = list( diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 65201c0c4..775efdcf3 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -367,12 +367,20 @@ def with_configs(configs, reason=None): def is_post_altair(spec): + if spec.fork == MERGE: # TODO: remove parallel Altair-Merge condition after rebase. + return False if spec.fork in FORKS_BEFORE_ALTAIR: return False return True def is_post_merge(spec): + if spec.fork == ALTAIR: # TODO: remove parallel Altair-Merge condition after rebase. + return False if spec.fork in FORKS_BEFORE_MERGE: return False return True + + +with_altair_and_later = with_phases([ALTAIR]) # TODO: include Merge, but not until Merge work is rebased. +with_merge_and_later = with_phases([MERGE]) diff --git a/tests/core/pyspec/eth2spec/test/helpers/block.py b/tests/core/pyspec/eth2spec/test/helpers/block.py index 64f406633..41d0a9131 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/block.py @@ -96,6 +96,8 @@ def build_empty_block(spec, state, slot=None): empty_block.body.sync_aggregate.sync_committee_signature = spec.G2_POINT_AT_INFINITY if is_post_merge(spec): + if not hasattr(state, 'latest_execution_payload_header'): + raise Exception("panic!!!") empty_block.body.execution_payload = build_empty_execution_payload(spec, state) apply_randao_reveal(spec, state, empty_block) diff --git a/tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py index f3f521c4b..96a657d7a 100644 --- a/tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py @@ -1,8 +1,5 @@ -from eth2spec.test.helpers.constants import PHASE0, ALTAIR from eth2spec.test.helpers.execution_payload import build_empty_execution_payload -from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases_except - -with_merge_and_later = with_all_phases_except([PHASE0, ALTAIR]) +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_merge_and_later def run_execution_payload_processing(spec, state, execution_payload, valid=True, execution_valid=True): diff --git a/tests/core/pyspec/eth2spec/test/merge/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/merge/sanity/test_blocks.py index 1f9e69167..4a6db4106 100644 --- a/tests/core/pyspec/eth2spec/test/merge/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/merge/sanity/test_blocks.py @@ -5,11 +5,11 @@ from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot ) from eth2spec.test.context import ( - with_all_phases, spec_state_test + with_merge_and_later, spec_state_test ) -@with_all_phases +@with_merge_and_later @spec_state_test def test_empty_block_transition(spec, state): yield 'pre', state From cc11328f74d77bf3a914f71c6cbc606620bf0c24 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 5 May 2021 16:24:44 +0200 Subject: [PATCH 21/36] fix merge forkchoice tests with mock get_pow_block --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 37fbd7795..9e5a546fa 100644 --- a/setup.py +++ b/setup.py @@ -441,7 +441,7 @@ ExecutionState = Any def get_pow_block(hash: Bytes32) -> PowBlock: - pass + return PowBlock(block_hash=hash, is_valid=True, is_processed=True, total_difficulty=TRANSITION_TOTAL_DIFFICULTY) def get_execution_state(execution_state_root: Bytes32) -> ExecutionState: From 00cd1c3db76c77fe468721075347df683adb8aa3 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 5 May 2021 16:31:28 +0200 Subject: [PATCH 22/36] fix forkchoice unittest not recognizing merge spec --- .../test/phase0/unittests/fork_choice/test_on_attestation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/phase0/unittests/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/phase0/unittests/fork_choice/test_on_attestation.py index fcc5e2d21..8ff6bbb38 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/unittests/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase0/unittests/fork_choice/test_on_attestation.py @@ -1,7 +1,7 @@ from eth2spec.test.context import with_all_phases, spec_state_test from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.attestations import get_valid_attestation, sign_attestation -from eth2spec.test.helpers.constants import PHASE0, ALTAIR +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE from eth2spec.test.helpers.state import transition_to, state_transition_and_sign_block, next_epoch, next_slot from eth2spec.test.helpers.fork_choice import get_genesis_forkchoice_store @@ -19,7 +19,7 @@ def run_on_attestation(spec, state, store, attestation, valid=True): spec.on_attestation(store, attestation) sample_index = indexed_attestation.attesting_indices[0] - if spec.fork in (PHASE0, ALTAIR): + if spec.fork in (PHASE0, ALTAIR, MERGE): latest_message = spec.LatestMessage( epoch=attestation.data.target.epoch, root=attestation.data.beacon_block_root, From ed4b8d5f184d6eb38480899457f8b6ecd9200fb7 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Wed, 5 May 2021 07:40:23 -0700 Subject: [PATCH 23/36] `get_shard_proposer_index` to use `DOMAIN_SHARD_PROPOSER` --- specs/sharding/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index bddc24a40..e13ca6c5b 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -402,7 +402,7 @@ def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard """ epoch = compute_epoch_at_slot(slot) committee = get_shard_committee(beacon_state, epoch, shard) - seed = hash(get_seed(beacon_state, epoch, DOMAIN_BEACON_PROPOSER) + uint_to_bytes(slot)) + seed = hash(get_seed(beacon_state, epoch, DOMAIN_SHARD_PROPOSER) + uint_to_bytes(slot)) # Proposer must have sufficient balance to pay for worst case fee burn EFFECTIVE_BALANCE_MAX_DOWNWARD_DEVIATION = ( From 2ef6291cbc74237c364c22c4f29f8debafe1d4b0 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 5 May 2021 16:41:59 +0200 Subject: [PATCH 24/36] Minimal execution payload test, more merge-specific testing in later PR --- .../test_process_execution_payload.py | 109 +----------------- 1 file changed, 2 insertions(+), 107 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py index 96a657d7a..acb88b944 100644 --- a/tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py @@ -1,6 +1,6 @@ from eth2spec.test.helpers.execution_payload import build_empty_execution_payload from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_merge_and_later - +from eth2spec.test.helpers.state import next_slot def run_execution_payload_processing(spec, state, execution_payload, valid=True, execution_valid=True): """ @@ -31,115 +31,10 @@ def run_execution_payload_processing(spec, state, execution_payload, valid=True, @with_merge_and_later @spec_state_test def test_success_first_payload(spec, state): + next_slot(spec, state) assert not spec.is_transition_completed(state) - # TODO: execution payload execution_payload = build_empty_execution_payload(spec, state) yield from run_execution_payload_processing(spec, state, execution_payload) - -@with_merge_and_later -@spec_state_test -def test_success_regular_payload(spec, state): - # TODO: setup state - assert spec.is_transition_completed(state) - - # TODO: execution payload - execution_payload = spec.ExecutionPayload() - - yield from run_execution_payload_processing(spec, state, execution_payload) - - -@with_merge_and_later -@spec_state_test -def test_success_first_payload_with_gap_slot(spec, state): - # TODO: transition gap slot - - assert not spec.is_transition_completed(state) - - # TODO: execution payload - execution_payload = spec.ExecutionPayload() - - yield from run_execution_payload_processing(spec, state, execution_payload) - - -@with_merge_and_later -@spec_state_test -def test_success_regular_payload_with_gap_slot(spec, state): - # TODO: setup state - assert spec.is_transition_completed(state) - # TODO: transition gap slot - - # TODO: execution payload - execution_payload = spec.ExecutionPayload() - - yield from run_execution_payload_processing(spec, state, execution_payload) - - -@with_merge_and_later -@spec_state_test -def test_bad_execution_first_payload(spec, state): - # completely valid payload, but execution itself fails (e.g. block exceeds gas limit) - - # TODO: execution payload. - execution_payload = spec.ExecutionPayload() - - yield from run_execution_payload_processing(spec, state, execution_payload, valid=False, execution_valid=False) - - -@with_merge_and_later -@spec_state_test -def test_bad_execution_regular_payload(spec, state): - # completely valid payload, but execution itself fails (e.g. block exceeds gas limit) - - # TODO: execution payload - execution_payload = spec.ExecutionPayload() - - yield from run_execution_payload_processing(spec, state, execution_payload, valid=False, execution_valid=False) - - -@with_merge_and_later -@spec_state_test -def test_bad_parent_hash_first_payload(spec, state): - # TODO: execution payload - execution_payload = spec.ExecutionPayload() - - yield from run_execution_payload_processing(spec, state, execution_payload, valid=False) - - -@with_merge_and_later -@spec_state_test -def test_bad_number_first_payload(spec, state): - # TODO: execution payload - execution_payload = spec.ExecutionPayload() - - yield from run_execution_payload_processing(spec, state, execution_payload, valid=False) - - -@with_merge_and_later -@spec_state_test -def test_bad_everything_first_payload(spec, state): - # TODO: execution payload - execution_payload = spec.ExecutionPayload() - - yield from run_execution_payload_processing(spec, state, execution_payload, valid=False) - - -@with_merge_and_later -@spec_state_test -def test_bad_timestamp_first_payload(spec, state): - # TODO: execution payload - execution_payload = spec.ExecutionPayload() - - yield from run_execution_payload_processing(spec, state, execution_payload, valid=False) - - -@with_merge_and_later -@spec_state_test -def test_bad_timestamp_regular_payload(spec, state): - # TODO: execution payload - execution_payload = spec.ExecutionPayload() - - yield from run_execution_payload_processing(spec, state, execution_payload, valid=False) - From 56bcb630dbfa487a4797c718e5a0b5c3ac04abc2 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 5 May 2021 17:03:29 +0200 Subject: [PATCH 25/36] Lint fixes for merge testing update --- .../altair/block_processing/test_process_sync_committee.py | 1 - .../epoch_processing/test_process_sync_committee_updates.py | 5 +---- .../test/altair/unittests/validator/test_validator.py | 1 - tests/core/pyspec/eth2spec/test/helpers/execution_payload.py | 2 +- .../merge/block_processing/test_process_execution_payload.py | 4 ++-- 5 files changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py index 9b415cfbd..db2664b21 100644 --- a/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py @@ -9,7 +9,6 @@ from eth2spec.test.helpers.state import ( transition_to, ) from eth2spec.test.helpers.constants import ( - PHASE0, MAINNET, MINIMAL, ) from eth2spec.test.helpers.sync_committee import ( diff --git a/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py b/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py index c914c476d..c909c791c 100644 --- a/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py +++ b/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py @@ -8,10 +8,7 @@ from eth2spec.test.context import ( single_phase, misc_balances, ) -from eth2spec.test.helpers.constants import ( - PHASE0, - MINIMAL, -) +from eth2spec.test.helpers.constants import MINIMAL from eth2spec.test.helpers.state import transition_to from eth2spec.test.helpers.epoch_processing import ( run_epoch_processing_with, diff --git a/tests/core/pyspec/eth2spec/test/altair/unittests/validator/test_validator.py b/tests/core/pyspec/eth2spec/test/altair/unittests/validator/test_validator.py index e1f5d3860..c8a894da6 100644 --- a/tests/core/pyspec/eth2spec/test/altair/unittests/validator/test_validator.py +++ b/tests/core/pyspec/eth2spec/test/altair/unittests/validator/test_validator.py @@ -7,7 +7,6 @@ from eth2spec.test.helpers.state import transition_to from eth2spec.utils import bls from eth2spec.utils.bls import only_with_bls from eth2spec.test.context import ( - PHASE0, with_altair_and_later, with_state, ) diff --git a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py index 0d035350d..093b7cf2e 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py @@ -16,7 +16,7 @@ def build_empty_execution_payload(spec, state): gas_limit=latest.gas_limit, # retain same limit gas_used=0, # empty block, 0 gas timestamp=timestamp, - receipt_root=b"no receipts here" + b"\x00"*16, # TODO: root of empty MPT may be better. + receipt_root=b"no receipts here" + b"\x00" * 16, # TODO: root of empty MPT may be better. logs_bloom=spec.ByteVector[spec.BYTES_PER_LOGS_BLOOM](), # TODO: zeroed logs bloom for empty logs ok? transactions=empty_txs, ) diff --git a/tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py index acb88b944..d45a2689b 100644 --- a/tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py @@ -1,7 +1,8 @@ from eth2spec.test.helpers.execution_payload import build_empty_execution_payload -from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_merge_and_later +from eth2spec.test.context import spec_state_test, expect_assertion_error, with_merge_and_later from eth2spec.test.helpers.state import next_slot + def run_execution_payload_processing(spec, state, execution_payload, valid=True, execution_valid=True): """ Run ``process_execution_payload``, yielding: @@ -37,4 +38,3 @@ def test_success_first_payload(spec, state): execution_payload = build_empty_execution_payload(spec, state) yield from run_execution_payload_processing(spec, state, execution_payload) - From a2cf833437dd5ceba194de702a00ff6243313fe9 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Wed, 5 May 2021 09:32:28 -0700 Subject: [PATCH 26/36] Update specs/altair/beacon-chain.md Co-authored-by: Hsiao-Wei Wang --- specs/altair/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index aa961d956..62f6174cf 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -126,7 +126,7 @@ This patch updates a few configuration values to move penalty parameters toward | Name | Value | Unit | Duration | | - | - | :-: | :-: | -| `EPOCHS_PER_SYNC_COMMITTEE_PERIOD` | `Epoch(2**9)` (= 512) | epochs | ~54 hours | +| `EPOCHS_PER_SYNC_COMMITTEE_PERIOD` | `uint64(2**9)` (= 512) | epochs | ~54 hours | ### Domain types From 8ac59b73170d452c5a8e16f65b8bb14bbe8c5c75 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 5 May 2021 22:38:16 +0200 Subject: [PATCH 27/36] fix old ssz-static todo comment --- tests/generators/ssz_static/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/generators/ssz_static/main.py b/tests/generators/ssz_static/main.py index 1e810f71e..d86636e85 100644 --- a/tests/generators/ssz_static/main.py +++ b/tests/generators/ssz_static/main.py @@ -93,7 +93,6 @@ if __name__ == "__main__": seed += 1 settings.append((seed, MAINNET, random_value.RandomizationMode.mode_random, False, 5)) seed += 1 - # TODO: enable testing for the whole merge spec. for fork in TESTGEN_FORKS: gen_runner.run_generator("ssz_static", [ create_provider(fork, config_name, seed, mode, chaos, cases_if_random) From 76b5974d11692326cbbf513e9c04130aceabeba4 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 6 May 2021 02:21:52 +0200 Subject: [PATCH 28/36] is_execution_enabled function + misc review fixes Co-Authored-By: Danny Ryan --- specs/merge/beacon-chain.md | 10 +++++++++- tests/core/pyspec/eth2spec/test/context.py | 1 - .../block_processing/test_process_execution_payload.py | 5 ++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/specs/merge/beacon-chain.md b/specs/merge/beacon-chain.md index 50b9551b0..626a86724 100644 --- a/specs/merge/beacon-chain.md +++ b/specs/merge/beacon-chain.md @@ -24,6 +24,7 @@ - [`ExecutionPayloadHeader`](#executionpayloadheader) - [Helper functions](#helper-functions) - [Misc](#misc) + - [`is_execution_enabled`](#is_execution_enabled) - [`is_transition_completed`](#is_transition_completed) - [`is_transition_block`](#is_transition_block) - [`compute_time_at_slot`](#compute_time_at_slot) @@ -140,6 +141,13 @@ class ExecutionPayloadHeader(Container): ### Misc +#### `is_execution_enabled` + +```python +def is_execution_enabled(state: BeaconState, block: BeaconBlock) -> bool: + return is_transition_completed(state) or is_transition_block(state, block) +``` + #### `is_transition_completed` ```python @@ -173,7 +181,7 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_eth1_data(state, block.body) process_operations(state, block.body) # Pre-merge, skip execution payload processing - if is_transition_completed(state) or is_transition_block(state, block): + if is_execution_enabled(state, block): process_execution_payload(state, block.body.execution_payload) # [New in Merge] ``` diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 775efdcf3..7a2e61c22 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -330,7 +330,6 @@ def with_phases(phases, other_phases=None): phase_dir[PHASE0] = spec_phase0 if ALTAIR in available_phases: phase_dir[ALTAIR] = spec_altair - if MERGE in available_phases: phase_dir[MERGE] = spec_merge diff --git a/tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py index d45a2689b..fb1da8758 100644 --- a/tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py @@ -13,6 +13,8 @@ def run_execution_payload_processing(spec, state, execution_payload, valid=True, If ``valid == False``, run expecting ``AssertionError`` """ + pre_exec_header = state.latest_execution_payload_header.copy() + yield 'pre', state yield 'execution', {'execution_valid': execution_valid} yield 'execution_payload', execution_payload @@ -26,7 +28,8 @@ def run_execution_payload_processing(spec, state, execution_payload, valid=True, yield 'post', state - # TODO: any assertions to make? + assert pre_exec_header != state.latest_execution_payload_header + # TODO: any more assertions to make? @with_merge_and_later From 42733b7e34fa9f0e7fbd9eda0415171e843d0de7 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 6 May 2021 02:27:05 +0200 Subject: [PATCH 29/36] remove merge-test exec-payload trigger debug helper --- tests/core/pyspec/eth2spec/test/helpers/block.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/block.py b/tests/core/pyspec/eth2spec/test/helpers/block.py index 41d0a9131..64f406633 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/block.py @@ -96,8 +96,6 @@ def build_empty_block(spec, state, slot=None): empty_block.body.sync_aggregate.sync_committee_signature = spec.G2_POINT_AT_INFINITY if is_post_merge(spec): - if not hasattr(state, 'latest_execution_payload_header'): - raise Exception("panic!!!") empty_block.body.execution_payload = build_empty_execution_payload(spec, state) apply_randao_reveal(spec, state, empty_block) From 4c73fec88e61e83240d0d45270e7afafb0d17d0f Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 6 May 2021 10:00:04 -0700 Subject: [PATCH 30/36] convert `LightClientStore` to python object instead of SSZ object this avoids the type overhead of having to define a max size for the object's data and skips the overhead of serialization/consensus for a type that does not need it --- configs/mainnet/altair.yaml | 2 -- configs/minimal/altair.yaml | 3 +-- setup.py | 2 +- specs/altair/sync-protocol.md | 12 ++++++------ .../test/altair/unittests/test_sync_protocol.py | 8 ++++---- 5 files changed, 12 insertions(+), 15 deletions(-) diff --git a/configs/mainnet/altair.yaml b/configs/mainnet/altair.yaml index b7b7603a2..eac4e1c14 100644 --- a/configs/mainnet/altair.yaml +++ b/configs/mainnet/altair.yaml @@ -45,8 +45,6 @@ ALTAIR_FORK_EPOCH: 18446744073709551615 # --------------------------------------------------------------- # 1 MIN_SYNC_COMMITTEE_PARTICIPANTS: 1 -# 2**64 - 1 -MAX_VALID_LIGHT_CLIENT_UPDATES: 18446744073709551615 # Validator diff --git a/configs/minimal/altair.yaml b/configs/minimal/altair.yaml index 5618af8ff..0300ff1b8 100644 --- a/configs/minimal/altair.yaml +++ b/configs/minimal/altair.yaml @@ -45,8 +45,7 @@ ALTAIR_FORK_EPOCH: 18446744073709551615 # --------------------------------------------------------------- # 1 MIN_SYNC_COMMITTEE_PARTICIPANTS: 1 -# 2**64 - 1 -MAX_VALID_LIGHT_CLIENT_UPDATES: 18446744073709551615 + # Validator # --------------------------------------------------------------- diff --git a/setup.py b/setup.py index 37fbd7795..aad834e2c 100644 --- a/setup.py +++ b/setup.py @@ -548,7 +548,7 @@ ignored_dependencies = [ 'Bytes1', 'Bytes4', 'Bytes20', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector', 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', 'bytes', 'byte', 'ByteList', 'ByteVector', - 'Dict', 'dict', 'field', 'ceillog2', 'floorlog2', + 'Dict', 'dict', 'field', 'ceillog2', 'floorlog2', 'Set' ] diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index ca55c92f7..784ea63b2 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -50,7 +50,6 @@ uses sync committees introduced in [this beacon chain extension](./beacon-chain. | Name | Value | | - | - | | `MIN_SYNC_COMMITTEE_PARTICIPANTS` | `1` | -| `MAX_VALID_LIGHT_CLIENT_UPDATES` | `uint64(2**64 - 1)` | ## Containers @@ -87,9 +86,10 @@ class LightClientUpdate(Container): ### `LightClientStore` ```python -class LightClientStore(Container): +@dataclass +class LightClientStore(object): snapshot: LightClientSnapshot - valid_updates: List[LightClientUpdate, MAX_VALID_LIGHT_CLIENT_UPDATES] + valid_updates: Set[LightClientUpdate] ``` ## Helper functions @@ -175,7 +175,7 @@ def apply_light_client_update(snapshot: LightClientSnapshot, update: LightClient def process_light_client_update(store: LightClientStore, update: LightClientUpdate, current_slot: Slot, genesis_validators_root: Root) -> None: validate_light_client_update(store.snapshot, update, genesis_validators_root) - store.valid_updates.append(update) + store.valid_updates.add(update) update_timeout = SLOTS_PER_EPOCH * EPOCHS_PER_SYNC_COMMITTEE_PERIOD if ( @@ -186,10 +186,10 @@ def process_light_client_update(store: LightClientStore, update: LightClientUpda # Note that (2) means that the current light client design needs finality. # It may be changed to re-organizable light client design. See the on-going issue eth2.0-specs#2182. apply_light_client_update(store.snapshot, update) - store.valid_updates = [] + store.valid_updates = set() elif current_slot > store.snapshot.header.slot + update_timeout: # Forced best update when the update timeout has elapsed apply_light_client_update(store.snapshot, max(store.valid_updates, key=lambda update: sum(update.sync_committee_bits))) - store.valid_updates = [] + store.valid_updates = set() ``` diff --git a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py index 9b8c35e76..932a46ca5 100644 --- a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py +++ b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py @@ -32,7 +32,7 @@ def test_process_light_client_update_not_updated(spec, state): ) store = spec.LightClientStore( snapshot=pre_snapshot, - valid_updates=[] + valid_updates=set(), ) # Block at slot 1 doesn't increase sync committee period, so it won't update snapshot @@ -76,7 +76,7 @@ def test_process_light_client_update_not_updated(spec, state): spec.process_light_client_update(store, update, state.slot, state.genesis_validators_root) assert len(store.valid_updates) == 1 - assert store.valid_updates[0] == update + assert store.valid_updates.pop() == update assert store.snapshot == pre_snapshot @@ -91,7 +91,7 @@ def test_process_light_client_update_timeout(spec, state): ) store = spec.LightClientStore( snapshot=pre_snapshot, - valid_updates=[] + valid_updates=set(), ) # Forward to next sync committee period @@ -156,7 +156,7 @@ def test_process_light_client_update_finality_updated(spec, state): ) store = spec.LightClientStore( snapshot=pre_snapshot, - valid_updates=[] + valid_updates=set(), ) # Change finality From 953b0278a10bbf98ed1f6e921a7accf9b87b62c8 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 6 May 2021 10:23:50 -0700 Subject: [PATCH 31/36] whitespace --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index aad834e2c..aaed891c4 100644 --- a/setup.py +++ b/setup.py @@ -167,7 +167,7 @@ def get_spec(file_name: str) -> SpecObject: comment = _get_eth2_spec_comment(child) if comment == "skip": should_skip = True - + return SpecObject( functions=functions, custom_types=custom_types, From 4b27b076f6c2b3e90b69f6ace83e89c13f3f30fe Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 6 May 2021 12:36:08 -0600 Subject: [PATCH 32/36] add missing comma --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index aaed891c4..039e8ab63 100644 --- a/setup.py +++ b/setup.py @@ -548,7 +548,7 @@ ignored_dependencies = [ 'Bytes1', 'Bytes4', 'Bytes20', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector', 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', 'bytes', 'byte', 'ByteList', 'ByteVector', - 'Dict', 'dict', 'field', 'ceillog2', 'floorlog2', 'Set' + 'Dict', 'dict', 'field', 'ceillog2', 'floorlog2', 'Set', ] From 86104ea361921cd6b952bd13a22c6a198648404a Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Fri, 7 May 2021 09:55:21 -0700 Subject: [PATCH 33/36] Use stable sync committee indices when processing block rewards --- specs/altair/beacon-chain.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index b2ee93557..59b11c8e4 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -571,7 +571,11 @@ def process_sync_committee(state: BeaconState, aggregate: SyncAggregate) -> None proposer_reward = Gwei(participant_reward * PROPOSER_WEIGHT // (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT)) # Apply participant and proposer rewards - committee_indices = get_sync_committee_indices(state, get_current_epoch(state)) + committee_indices = [] + pubkeys = [v.pubkey for v in state.validators] + for pubkey in state.current_sync_committee.pubkeys: + index = pubkeys.index(pubkey) + committee_indices.append(ValidatorIndex(index)) participant_indices = [index for index, bit in zip(committee_indices, aggregate.sync_committee_bits) if bit] for participant_index in participant_indices: increase_balance(state, participant_index, participant_reward) From 04a9595415678013442460ef4a0ee90d0799924b Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Fri, 7 May 2021 10:06:44 -0700 Subject: [PATCH 34/36] Add notes about sync committee stability --- specs/altair/beacon-chain.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 59b11c8e4..0855d6f11 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -277,6 +277,9 @@ def get_sync_committee_indices(state: BeaconState, epoch: Epoch) -> Sequence[Val """ Return the sequence of sync committee indices (which may include duplicate indices) for a given ``state`` and ``epoch``. + + Note: This function is not stable during a sync committee period as + a validator's effective balance may change enough to affect the sampling. """ MAX_RANDOM_BYTE = 2**8 - 1 base_epoch = Epoch((max(epoch // EPOCHS_PER_SYNC_COMMITTEE_PERIOD, 1) - 1) * EPOCHS_PER_SYNC_COMMITTEE_PERIOD) @@ -310,6 +313,9 @@ def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: ``SyncCommittee`` can also contain duplicate pubkeys, when ``get_sync_committee_indices`` returns duplicate indices. Implementations must take care when handling optimizations relating to aggregation and verification in the presence of duplicates. + + Note: This function should only be called at sync committee period boundaries, as + ``get_sync_committee_indices`` is not stable within a given period. """ indices = get_sync_committee_indices(state, epoch) pubkeys = [state.validators[index].pubkey for index in indices] From b336b710e9bd08594af87a16f53932bbe4672fe9 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Fri, 7 May 2021 15:54:01 -0700 Subject: [PATCH 35/36] Update specs/altair/beacon-chain.md Co-authored-by: vbuterin --- specs/altair/beacon-chain.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 0855d6f11..ac4457b22 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -577,11 +577,8 @@ def process_sync_committee(state: BeaconState, aggregate: SyncAggregate) -> None proposer_reward = Gwei(participant_reward * PROPOSER_WEIGHT // (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT)) # Apply participant and proposer rewards - committee_indices = [] - pubkeys = [v.pubkey for v in state.validators] - for pubkey in state.current_sync_committee.pubkeys: - index = pubkeys.index(pubkey) - committee_indices.append(ValidatorIndex(index)) + all_pubkeys = [v.pubkey for v in state.validators] + committee_indices = [ValidatorIndex(all_pubkeys.index(pubkey)) for pubkey in state.current_sync_committee.pubkeys] participant_indices = [index for index, bit in zip(committee_indices, aggregate.sync_committee_bits) if bit] for participant_index in participant_indices: increase_balance(state, participant_index, participant_reward) From 72a4ff803b3be39da3a29730a273b04ec6f35a87 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Fri, 7 May 2021 17:02:52 -0700 Subject: [PATCH 36/36] add test to ensure sync committees are referenced from the state --- .../test_process_sync_committee.py | 55 ++++++++++++++++--- 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py index e0faf3d0d..3e4d77c0f 100644 --- a/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py @@ -7,6 +7,7 @@ from eth2spec.test.helpers.block_processing import run_block_processing_to from eth2spec.test.helpers.state import ( state_transition_and_sign_block, transition_to, + next_epoch, ) from eth2spec.test.helpers.constants import ( PHASE0, @@ -160,13 +161,13 @@ def validate_sync_committee_rewards(spec, pre_state, post_state, committee, comm committee_bits, ) - if proposer_index == index: - reward += compute_sync_committee_proposer_reward( - spec, - pre_state, - committee, - committee_bits, - ) + if proposer_index == index: + reward += compute_sync_committee_proposer_reward( + spec, + pre_state, + committee, + committee_bits, + ) assert post_state.balances[index] == pre_state.balances[index] + reward @@ -367,3 +368,43 @@ def test_valid_signature_future_committee(spec, state): ) yield from run_sync_committee_processing(spec, state, block) + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_sync_committee_is_only_computed_at_epoch_boundary(spec, state): + """ + Sync committees can only be computed at sync committee period boundaries. + Ensure a client respects the committee in the state (assumed to be derived + in the correct way). + """ + current_epoch = spec.get_current_epoch(state) + + # use a "synthetic" committee to simulate the situation + # where ``spec.get_sync_committee`` at the sync committee + # period epoch boundary would have diverged some epochs into the + # period; ``aggregate_pubkey`` is not relevant to this test + pubkeys = [] + committee_indices = [] + i = 0 + active_validator_count = len(spec.get_active_validator_indices(state, current_epoch)) + while len(pubkeys) < spec.SYNC_COMMITTEE_SIZE: + v = state.validators[i % active_validator_count] + if spec.is_active_validator(v, current_epoch): + pubkeys.append(v.pubkey) + committee_indices.append(i) + i += 1 + + synthetic_committee = spec.SyncCommittee(pubkeys=pubkeys, aggregate_pubkey=spec.BLSPubkey()) + state.current_sync_committee = synthetic_committee + + assert spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD > 3 + for _ in range(3): + next_epoch(spec, state) + + committee = get_committee_indices(spec, state) + assert committee != committee_indices + committee_size = len(committee_indices) + committee_bits = [True] * committee_size + + yield from run_successful_sync_committee_test(spec, state, committee_indices, committee_bits)