From 5653649ca8cc80fb732f68c9e7f4aafb3f3a1ff6 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Tue, 26 Apr 2022 22:32:25 +0200 Subject: [PATCH 001/105] Allow light client to verify signatures at period boundary As the sync committee signs the previous block, the situation arises at every sync committee period boundary, that the new sync committee signs a block in the previous sync committee period. The light client cannot reliably detect this condition (e.g., assume that this is the case when it is currently on the last slot of a sync committee period), because the last couple slots of a sync committee period may not have a block. For example, when receiving a `LightClientUpdate` that is constructed as in the following illustration, it is unknown whether `sync_aggregate` was signed by the current or next sync committee at `attested_header`. ``` slot N N + 1 | N + 2 (slot not sent!) | +-----------------+ \ / | +----------------+ | attested_header | <--- X ----|---- | sync_aggregate | +-----------------+ / \ | +----------------+ missed | | sync committee period boundary ``` This patch addresses this edge case by including the slot at which the `sync_aggregate` was created into the `LightClientUpdate` object. Note that the `signature_slot` cannot be trusted beyond the purpose of signature verification, as it could be manipulated to any other slot within the same sync committee period and fork version, without making the `sync_aggregate` invalid. --- specs/altair/sync-protocol.md | 17 ++-- .../altair/unittests/test_sync_protocol.py | 81 +++++++++++++++---- .../eth2spec/test/helpers/light_client.py | 34 +++++--- 3 files changed, 102 insertions(+), 30 deletions(-) diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index 8751ba6e8..1aa9316d3 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -52,7 +52,7 @@ uses sync committees introduced in [this beacon chain extension](./beacon-chain. | Name | Value | Unit | Duration | | - | - | - | - | -| `MIN_SYNC_COMMITTEE_PARTICIPANTS` | `1` | validators | +| `MIN_SYNC_COMMITTEE_PARTICIPANTS` | `1` | validators | | | `UPDATE_TIMEOUT` | `SLOTS_PER_EPOCH * EPOCHS_PER_SYNC_COMMITTEE_PERIOD` | slots | ~27.3 hours | ## Containers @@ -73,6 +73,8 @@ class LightClientUpdate(Container): sync_aggregate: SyncAggregate # Fork version for the aggregate signature fork_version: Version + # Slot at which the aggregate signature was created (untrusted) + signature_slot: Slot ``` ### `LightClientStore` @@ -162,15 +164,16 @@ def validate_light_client_update(store: LightClientStore, genesis_validators_root: Root) -> None: # Verify update slot is larger than slot of current best finalized header active_header = get_active_header(update) - assert current_slot >= active_header.slot > store.finalized_header.slot + assert current_slot >= update.signature_slot > active_header.slot > store.finalized_header.slot # Verify update does not skip a sync committee period finalized_period = compute_sync_committee_period(compute_epoch_at_slot(store.finalized_header.slot)) update_period = compute_sync_committee_period(compute_epoch_at_slot(active_header.slot)) - assert update_period in (finalized_period, finalized_period + 1) + signature_period = compute_sync_committee_period(compute_epoch_at_slot(update.signature_slot)) + assert signature_period in (finalized_period, finalized_period + 1) # Verify that the `finalized_header`, if present, actually is the finalized header saved in the - # state of the `attested header` + # state of the `attested_header` if not is_finality_update(update): assert update.finality_branch == [Bytes32() for _ in range(floorlog2(FINALIZED_ROOT_INDEX))] else: @@ -184,10 +187,8 @@ def validate_light_client_update(store: LightClientStore, # Verify update next sync committee if the update period incremented if update_period == finalized_period: - sync_committee = store.current_sync_committee assert update.next_sync_committee_branch == [Bytes32() for _ in range(floorlog2(NEXT_SYNC_COMMITTEE_INDEX))] else: - sync_committee = store.next_sync_committee assert is_valid_merkle_branch( leaf=hash_tree_root(update.next_sync_committee), branch=update.next_sync_committee_branch, @@ -202,6 +203,10 @@ def validate_light_client_update(store: LightClientStore, assert sum(sync_aggregate.sync_committee_bits) >= MIN_SYNC_COMMITTEE_PARTICIPANTS # Verify sync committee aggregate signature + if signature_period == finalized_period: + sync_committee = store.current_sync_committee + else: + sync_committee = store.next_sync_committee participant_pubkeys = [ pubkey for (bit, pubkey) in zip(sync_aggregate.sync_committee_bits, sync_committee.pubkeys) if bit diff --git a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py index 04553b3f8..6411ecb6a 100644 --- a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py +++ b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py @@ -39,8 +39,9 @@ def test_process_light_client_update_not_timeout(spec, state): state_root=signed_block.message.state_root, body_root=signed_block.message.body.hash_tree_root(), ) - # Sync committee signing the header - sync_aggregate = get_sync_aggregate(spec, state, block_header, block_root=None) + + # Sync committee signing the block_header + sync_aggregate, fork_version, signature_slot = get_sync_aggregate(spec, state, block_header) next_sync_committee_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.NEXT_SYNC_COMMITTEE_INDEX))] # Ensure that finality checkpoint is genesis @@ -56,12 +57,13 @@ def test_process_light_client_update_not_timeout(spec, state): finalized_header=finality_header, finality_branch=finality_branch, sync_aggregate=sync_aggregate, - fork_version=state.fork.current_version, + fork_version=fork_version, + signature_slot=signature_slot, ) pre_store = deepcopy(store) - spec.process_light_client_update(store, update, state.slot, state.genesis_validators_root) + spec.process_light_client_update(store, update, signature_slot, state.genesis_validators_root) assert store.current_max_active_participants > 0 assert store.optimistic_header == update.attested_header @@ -69,6 +71,57 @@ def test_process_light_client_update_not_timeout(spec, state): assert store.best_valid_update == update +@with_altair_and_later +@spec_state_test +@with_presets([MINIMAL], reason="too slow") +def test_process_light_client_update_at_period_boundary(spec, state): + store = initialize_light_client_store(spec, state) + + # Forward to slot before next sync committee period so that next block is final one in period + next_slots(spec, state, spec.UPDATE_TIMEOUT - 2) + snapshot_period = spec.compute_sync_committee_period(spec.compute_epoch_at_slot(store.optimistic_header.slot)) + update_period = spec.compute_sync_committee_period(spec.compute_epoch_at_slot(state.slot)) + assert snapshot_period == update_period + + block = build_empty_block_for_next_slot(spec, state) + signed_block = state_transition_and_sign_block(spec, state, block) + block_header = spec.BeaconBlockHeader( + slot=signed_block.message.slot, + proposer_index=signed_block.message.proposer_index, + parent_root=signed_block.message.parent_root, + state_root=signed_block.message.state_root, + body_root=signed_block.message.body.hash_tree_root(), + ) + + # Sync committee signing the block_header + sync_aggregate, fork_version, signature_slot = get_sync_aggregate(spec, state, block_header) + next_sync_committee_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.NEXT_SYNC_COMMITTEE_INDEX))] + + # Finality is unchanged + finality_header = spec.BeaconBlockHeader() + finality_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.FINALIZED_ROOT_INDEX))] + + update = spec.LightClientUpdate( + attested_header=block_header, + next_sync_committee=state.next_sync_committee, + next_sync_committee_branch=next_sync_committee_branch, + finalized_header=finality_header, + finality_branch=finality_branch, + sync_aggregate=sync_aggregate, + fork_version=fork_version, + signature_slot=signature_slot, + ) + + pre_store = deepcopy(store) + + spec.process_light_client_update(store, update, signature_slot, state.genesis_validators_root) + + assert store.current_max_active_participants > 0 + assert store.optimistic_header == update.attested_header + assert store.best_valid_update == update + assert store.finalized_header == pre_store.finalized_header + + @with_altair_and_later @spec_state_test @with_presets([MINIMAL], reason="too slow") @@ -91,9 +144,8 @@ def test_process_light_client_update_timeout(spec, state): body_root=signed_block.message.body.hash_tree_root(), ) - # Sync committee signing the finalized_block_header - sync_aggregate = get_sync_aggregate( - spec, state, block_header, block_root=spec.Root(block_header.hash_tree_root())) + # Sync committee signing the block_header + sync_aggregate, fork_version, signature_slot = get_sync_aggregate(spec, state, block_header) # Sync committee is updated next_sync_committee_branch = build_proof(state.get_backing(), spec.NEXT_SYNC_COMMITTEE_INDEX) @@ -108,12 +160,13 @@ def test_process_light_client_update_timeout(spec, state): finalized_header=finality_header, finality_branch=finality_branch, sync_aggregate=sync_aggregate, - fork_version=state.fork.current_version, + fork_version=fork_version, + signature_slot=signature_slot, ) pre_store = deepcopy(store) - spec.process_light_client_update(store, update, state.slot, state.genesis_validators_root) + spec.process_light_client_update(store, update, signature_slot, state.genesis_validators_root) assert store.current_max_active_participants > 0 assert store.optimistic_header == update.attested_header @@ -157,9 +210,8 @@ def test_process_light_client_update_finality_updated(spec, state): body_root=block.body.hash_tree_root(), ) - # Sync committee signing the finalized_block_header - sync_aggregate = get_sync_aggregate( - spec, state, block_header, block_root=spec.Root(block_header.hash_tree_root())) + # Sync committee signing the block_header + sync_aggregate, fork_version, signature_slot = get_sync_aggregate(spec, state, block_header) update = spec.LightClientUpdate( attested_header=block_header, @@ -168,10 +220,11 @@ def test_process_light_client_update_finality_updated(spec, state): finalized_header=finalized_block_header, finality_branch=finality_branch, sync_aggregate=sync_aggregate, - fork_version=state.fork.current_version, + fork_version=fork_version, + signature_slot=signature_slot, ) - spec.process_light_client_update(store, update, state.slot, state.genesis_validators_root) + spec.process_light_client_update(store, update, signature_slot, state.genesis_validators_root) assert store.current_max_active_participants > 0 assert store.optimistic_header == update.attested_header diff --git a/tests/core/pyspec/eth2spec/test/helpers/light_client.py b/tests/core/pyspec/eth2spec/test/helpers/light_client.py index 15c764fc4..b35c6c687 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/light_client.py +++ b/tests/core/pyspec/eth2spec/test/helpers/light_client.py @@ -1,5 +1,9 @@ +from eth2spec.test.helpers.state import ( + transition_to, +) from eth2spec.test.helpers.sync_committee import ( compute_aggregate_sync_committee_signature, + compute_committee_indices, ) @@ -15,21 +19,31 @@ def initialize_light_client_store(spec, state): ) -def get_sync_aggregate(spec, state, block_header, block_root=None, signature_slot=None): +def get_sync_aggregate(spec, state, block_header, signature_slot=None): + # By default, the sync committee signs the previous slot if signature_slot is None: - signature_slot = block_header.slot + signature_slot = block_header.slot + 1 - all_pubkeys = [v.pubkey for v in state.validators] - committee = [all_pubkeys.index(pubkey) for pubkey in state.current_sync_committee.pubkeys] - sync_committee_bits = [True] * len(committee) + # Ensure correct sync committee and fork version are selected + signature_state = state.copy() + transition_to(spec, signature_state, signature_slot) + + # Fetch sync committee + committee_indices = compute_committee_indices(spec, signature_state) + committee_size = len(committee_indices) + + # Compute sync aggregate + sync_committee_bits = [True] * committee_size sync_committee_signature = compute_aggregate_sync_committee_signature( spec, - state, - block_header.slot, - committee, - block_root=block_root, + signature_state, + signature_slot, + committee_indices, + block_root=spec.Root(block_header.hash_tree_root()), ) - return spec.SyncAggregate( + sync_aggregate = spec.SyncAggregate( sync_committee_bits=sync_committee_bits, sync_committee_signature=sync_committee_signature, ) + fork_version = signature_state.fork.current_version + return sync_aggregate, fork_version, signature_slot From 895cce5aeb263f2c6946b21d2fee73ac356fd4b0 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 23 May 2022 05:23:16 -0600 Subject: [PATCH 002/105] bump version.txt --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index 9695327b3..a94c1f6e2 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -1.1.10 \ No newline at end of file +1.2.0-rc.1 From a3128b8c5708767212a6b71f82f4cf4a0b8cb34e Mon Sep 17 00:00:00 2001 From: Murphy Law Date: Tue, 31 May 2022 10:40:06 -0400 Subject: [PATCH 003/105] EIP-4844: fix a typo (#2898) --- specs/eip4844/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/eip4844/p2p-interface.md b/specs/eip4844/p2p-interface.md index ff2a11e25..68c984950 100644 --- a/specs/eip4844/p2p-interface.md +++ b/specs/eip4844/p2p-interface.md @@ -204,7 +204,7 @@ correct w.r.t. the expected KZG commitments through `verify_blobs_sidecar`. The request MUST be encoded as an SSZ-container. The response MUST consist of zero or more `response_chunk`. -Each _successful_ `response_chunk` MUST contain a single `SignedBlobsSidecar` payload. +Each _successful_ `response_chunk` MUST contain a single `BlobsSidecar` payload. Clients MUST keep a record of signed blobs sidecars seen on the epoch range `[max(GENESIS_EPOCH, current_epoch - MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS), current_epoch]` From 986f7ee34c4459148bd6a9347b8744c8aee5665d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 1 Jun 2022 02:33:12 +0800 Subject: [PATCH 004/105] Use table to list the specs. Add `Capella` and `EIP4844` to README --- README.md | 62 +++++++++++++------------------------------------------ 1 file changed, 14 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index fb58edf8d..c92dd65dd 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,6 @@ To learn more about proof-of-stake and sharding, see the [PoS FAQ](https://eth.w This repository hosts the current Ethereum proof-of-stake specifications. Discussions about design rationale and proposed changes can be brought up and discussed as issues. Solidified, agreed-upon changes to the spec can be made through pull requests. - ## Specs [![GitHub release](https://img.shields.io/github/v/release/ethereum/eth2.0-specs)](https://github.com/ethereum/eth2.0-specs/releases/) [![PyPI version](https://badge.fury.io/py/eth2spec.svg)](https://badge.fury.io/py/eth2spec) @@ -14,55 +13,22 @@ This repository hosts the current Ethereum proof-of-stake specifications. Discus Core specifications for Ethereum proof-of-stake clients can be found in [specs](specs/). These are divided into features. Features are researched and developed in parallel, and then consolidated into sequential upgrades when ready. -The current features are: +### Stable Specifications -### Phase 0 +| Code Name | Fork Version | Previous Fork | Specs | Notes | +| - | - | - | - | - | +| **Phase0** | `0x00000000` | - |
  • Core
    • [The beacon chain](specs/phase0/beacon-chain.md)
    • [Deposit contract](specs/phase0/deposit-contract.md)
    • [Beacon chain fork choice](specs/phase0/fork-choice.md)
  • Additions
    • [Honest validator guide](specs/phase0/validator.md)
    • [P2P networking](specs/phase0/p2p-interface.md)
    • [Weak subjectivity](specs/phase0/weak-subjectivity.md)
| | +| **Altair** | `0x01000000` | Phase0 |
  • Core
    • [Beacon chain changes](specs/altair/beacon-chain.md)
    • [Altair fork](specs/altair/fork.md)
  • Additions
    • [Light client sync protocol](specs/altair/sync-protocol.md)
    • [Honest validator guide changes](specs/altair/validator.md)
    • [P2P networking](specs/altair/p2p-interface.md)
| | +| **Bellatrix** | `0x02000000` | Altair |
  • Core
    • [Beacon Chain changes](specs/bellatrix/beacon-chain.md)
    • [Bellatrix fork](specs/bellatrix/fork.md)
    • [Fork choice changes](specs/bellatrix/fork-choice.md)
  • Additions
    • [Honest validator guide changes](specs/bellatrix/validator.md)
    • [P2P networking](specs/bellatrix/p2p-interface.md)
| [ethereum.org - The Merge intro](https://ethereum.org/en/upgrades/merge/) | -* [The Beacon Chain](specs/phase0/beacon-chain.md) -* [Beacon Chain Fork Choice](specs/phase0/fork-choice.md) -* [Deposit Contract](specs/phase0/deposit-contract.md) -* [Honest Validator](specs/phase0/validator.md) -* [P2P Networking](specs/phase0/p2p-interface.md) -* [Weak Subjectivity](specs/phase0/weak-subjectivity.md) - -### Altair - -* [Beacon chain changes](specs/altair/beacon-chain.md) -* [Altair fork](specs/altair/fork.md) -* [Light client sync protocol](specs/altair/sync-protocol.md) -* [Honest Validator guide changes](specs/altair/validator.md) -* [P2P Networking](specs/altair/p2p-interface.md) - -### Bellatrix (also known as The Merge) - -The Bellatrix protocol upgrade is still actively in development. The exact specification has not been formally accepted as final and details are still subject to change. - -* Background material: - * An [ethresear.ch](https://ethresear.ch) post [describing the basic mechanism of the CL+EL merge](https://ethresear.ch/t/the-eth1-eth2-transition/6265) - * [ethereum.org](https://ethereum.org) high-level description of the CL+EL merge [here](https://ethereum.org/en/eth2/docking/) -* Specifications: - * [Beacon Chain changes](specs/bellatrix/beacon-chain.md) - * [Bellatrix fork](specs/bellatrix/fork.md) - * [Fork Choice changes](specs/bellatrix/fork-choice.md) - * [Validator additions](specs/bellatrix/validator.md) - * [P2P Networking](specs/bellatrix/p2p-interface.md) - -### Sharding - -Sharding follows Bellatrix, and is divided into three parts: - -* Sharding base functionality - In early engineering phase - * [Beacon Chain changes](specs/sharding/beacon-chain.md) - * [P2P Network changes](specs/sharding/p2p-interface.md) -* Custody Game - Ready, dependent on sharding - * [Beacon Chain changes](specs/custody_game/beacon-chain.md) - * [Validator custody work](specs/custody_game/validator.md) -* Data Availability Sampling - In active R&D - * Technical details [here](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/B1YJPGkpD). - * [Core types and functions](specs/das/das-core.md) - * [P2P Networking](specs/das/p2p-interface.md) - * [Fork Choice](specs/das/fork-choice.md) - * [Sampling process](specs/das/sampling.md) +### In-development Specifications +| Code Name or Topic | Specs | Notes | +| - | - | - | +| Capella (tentative) |
  • Core
    • [Beacon chain changes](specs/capella/beacon-chain.md)
    • [Capella fork](specs/capella/fork.md)
  • Additions
    • [Validator additions](specs/capella/validator.md)
| +| EIP4844 (tentative) |
  • Core
    • [Beacon Chain changes](specs/eip4844/beacon-chain.md)
    • [EIP-4844 fork](specs/eip4844/fork.md)
  • Additions
    • [Honest validator guide changes](specs/eip4844/validator.md)
    • [P2P networking](specs/eip4844/p2p-interface.md)
| +| Sharding (outdated) |
  • Core
    • [Beacon Chain changes](specs/sharding/beacon-chain.md)
  • Additions
    • [P2P networking](specs/sharding/p2p-interface.md)
| +| Custody Game (outdated) |
  • Core
    • [Beacon Chain changes](specs/custody_game/beacon-chain.md)
  • Additions
    • [Honest validator guide changes](specs/custody_game/validator.md)
| dependent on sharding | +| Data Availability Sampling (outdated) |
  • Core
    • [Core types and functions](specs/das/das-core.md)
    • [Fork choice changes](specs/das/fork-choice.md)
  • Additions
    • [P2P Networking](specs/das/p2p-interface.md)
    • [Sampling process](specs/das/sampling.md)
|
  • dependent on sharding
  • [Technical explainer](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/B1YJPGkpD)
| ### Accompanying documents can be found in [specs](specs) and include: From 0d305f0ae3f1aca986eb253964a332b7c4bcbb93 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Wed, 1 Jun 2022 22:00:11 -0600 Subject: [PATCH 005/105] Update README.md Some edits to remove stale information --- tests/formats/README.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/formats/README.md b/tests/formats/README.md index e4f2bcb29..ec495daa5 100644 --- a/tests/formats/README.md +++ b/tests/formats/README.md @@ -29,7 +29,7 @@ This document defines the YAML format and structure used for consensus spec test ## About -Ethereum 2.0 uses YAML as the format for all cross client tests. This document describes at a high level the general format to which all test files should conform. +The consensus layer uses YAML as the format for all cross client tests. This document describes at a high level the general format to which all test files should conform. ### Test-case formats @@ -115,7 +115,6 @@ some tests of earlier forks repeat with updated state data. The well known bls/shuffling/ssz_static/operations/epoch_processing/etc. Handlers can change the format, but there is a general target to test. - ### `/` Specialization within category. All suites in here will have the same test case format. @@ -171,9 +170,6 @@ bls_setting: int -- optional, can have 3 different values: but there is no change of outcome when running the test if BLS is ON or OFF. 1: known as "BLS required" - if the test validity is strictly dependent on BLS being ON 2: known as "BLS ignored" - if the test validity is strictly dependent on BLS being OFF -reveal_deadlines_setting: -- optional, can have 2 different values: - 0: default, `process_reveal_deadlines` is ON. - 1: `process_reveal_deadlines` is OFF. ``` ##### `config.yaml` From df1464728bfea9fbfd18f20f00cd69cbebc90ca7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 2 Jun 2022 22:14:39 +0800 Subject: [PATCH 006/105] Remove fork version and previous fork. Add seq. --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index c92dd65dd..c581b89eb 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,11 @@ Features are researched and developed in parallel, and then consolidated into se ### Stable Specifications -| Code Name | Fork Version | Previous Fork | Specs | Notes | -| - | - | - | - | - | -| **Phase0** | `0x00000000` | - |
  • Core
    • [The beacon chain](specs/phase0/beacon-chain.md)
    • [Deposit contract](specs/phase0/deposit-contract.md)
    • [Beacon chain fork choice](specs/phase0/fork-choice.md)
  • Additions
    • [Honest validator guide](specs/phase0/validator.md)
    • [P2P networking](specs/phase0/p2p-interface.md)
    • [Weak subjectivity](specs/phase0/weak-subjectivity.md)
| | -| **Altair** | `0x01000000` | Phase0 |
  • Core
    • [Beacon chain changes](specs/altair/beacon-chain.md)
    • [Altair fork](specs/altair/fork.md)
  • Additions
    • [Light client sync protocol](specs/altair/sync-protocol.md)
    • [Honest validator guide changes](specs/altair/validator.md)
    • [P2P networking](specs/altair/p2p-interface.md)
| | -| **Bellatrix** | `0x02000000` | Altair |
  • Core
    • [Beacon Chain changes](specs/bellatrix/beacon-chain.md)
    • [Bellatrix fork](specs/bellatrix/fork.md)
    • [Fork choice changes](specs/bellatrix/fork-choice.md)
  • Additions
    • [Honest validator guide changes](specs/bellatrix/validator.md)
    • [P2P networking](specs/bellatrix/p2p-interface.md)
| [ethereum.org - The Merge intro](https://ethereum.org/en/upgrades/merge/) | +| Seq. | Code Name | Specs | Notes | +| - | - | - | - | +| 0 | **Phase0** |
  • Core
    • [The beacon chain](specs/phase0/beacon-chain.md)
    • [Deposit contract](specs/phase0/deposit-contract.md)
    • [Beacon chain fork choice](specs/phase0/fork-choice.md)
  • Additions
    • [Honest validator guide](specs/phase0/validator.md)
    • [P2P networking](specs/phase0/p2p-interface.md)
    • [Weak subjectivity](specs/phase0/weak-subjectivity.md)
| | +| 1 | **Altair** |
  • Core
    • [Beacon chain changes](specs/altair/beacon-chain.md)
    • [Altair fork](specs/altair/fork.md)
  • Additions
    • [Light client sync protocol](specs/altair/sync-protocol.md)
    • [Honest validator guide changes](specs/altair/validator.md)
    • [P2P networking](specs/altair/p2p-interface.md)
| | +| 2 | **Bellatrix** |
  • Core
    • [Beacon Chain changes](specs/bellatrix/beacon-chain.md)
    • [Bellatrix fork](specs/bellatrix/fork.md)
    • [Fork choice changes](specs/bellatrix/fork-choice.md)
  • Additions
    • [Honest validator guide changes](specs/bellatrix/validator.md)
    • [P2P networking](specs/bellatrix/p2p-interface.md)
| [ethereum.org - The Merge intro](https://ethereum.org/en/upgrades/merge/) | ### In-development Specifications | Code Name or Topic | Specs | Notes | @@ -27,8 +27,8 @@ Features are researched and developed in parallel, and then consolidated into se | Capella (tentative) |
  • Core
    • [Beacon chain changes](specs/capella/beacon-chain.md)
    • [Capella fork](specs/capella/fork.md)
  • Additions
    • [Validator additions](specs/capella/validator.md)
| | EIP4844 (tentative) |
  • Core
    • [Beacon Chain changes](specs/eip4844/beacon-chain.md)
    • [EIP-4844 fork](specs/eip4844/fork.md)
  • Additions
    • [Honest validator guide changes](specs/eip4844/validator.md)
    • [P2P networking](specs/eip4844/p2p-interface.md)
| | Sharding (outdated) |
  • Core
    • [Beacon Chain changes](specs/sharding/beacon-chain.md)
  • Additions
    • [P2P networking](specs/sharding/p2p-interface.md)
| -| Custody Game (outdated) |
  • Core
    • [Beacon Chain changes](specs/custody_game/beacon-chain.md)
  • Additions
    • [Honest validator guide changes](specs/custody_game/validator.md)
| dependent on sharding | -| Data Availability Sampling (outdated) |
  • Core
    • [Core types and functions](specs/das/das-core.md)
    • [Fork choice changes](specs/das/fork-choice.md)
  • Additions
    • [P2P Networking](specs/das/p2p-interface.md)
    • [Sampling process](specs/das/sampling.md)
|
  • dependent on sharding
  • [Technical explainer](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/B1YJPGkpD)
| +| Custody Game (outdated) |
  • Core
    • [Beacon Chain changes](specs/custody_game/beacon-chain.md)
  • Additions
    • [Honest validator guide changes](specs/custody_game/validator.md)
| Dependent on sharding | +| Data Availability Sampling (outdated) |
  • Core
    • [Core types and functions](specs/das/das-core.md)
    • [Fork choice changes](specs/das/fork-choice.md)
  • Additions
    • [P2P Networking](specs/das/p2p-interface.md)
    • [Sampling process](specs/das/sampling.md)
|
  • Dependent on sharding
  • [Technical explainer](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/B1YJPGkpD)
| ### Accompanying documents can be found in [specs](specs) and include: From dde1313a9d068cba0b7ea3c2edc9c446f388978c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 6 Jun 2022 16:03:46 +0800 Subject: [PATCH 007/105] Add Fork Epoch column --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c581b89eb..fcae73a32 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,11 @@ Features are researched and developed in parallel, and then consolidated into se ### Stable Specifications -| Seq. | Code Name | Specs | Notes | +| Seq. | Code Name | Fork Epoch | Specs | | - | - | - | - | -| 0 | **Phase0** |
  • Core
    • [The beacon chain](specs/phase0/beacon-chain.md)
    • [Deposit contract](specs/phase0/deposit-contract.md)
    • [Beacon chain fork choice](specs/phase0/fork-choice.md)
  • Additions
    • [Honest validator guide](specs/phase0/validator.md)
    • [P2P networking](specs/phase0/p2p-interface.md)
    • [Weak subjectivity](specs/phase0/weak-subjectivity.md)
| | -| 1 | **Altair** |
  • Core
    • [Beacon chain changes](specs/altair/beacon-chain.md)
    • [Altair fork](specs/altair/fork.md)
  • Additions
    • [Light client sync protocol](specs/altair/sync-protocol.md)
    • [Honest validator guide changes](specs/altair/validator.md)
    • [P2P networking](specs/altair/p2p-interface.md)
| | -| 2 | **Bellatrix** |
  • Core
    • [Beacon Chain changes](specs/bellatrix/beacon-chain.md)
    • [Bellatrix fork](specs/bellatrix/fork.md)
    • [Fork choice changes](specs/bellatrix/fork-choice.md)
  • Additions
    • [Honest validator guide changes](specs/bellatrix/validator.md)
    • [P2P networking](specs/bellatrix/p2p-interface.md)
| [ethereum.org - The Merge intro](https://ethereum.org/en/upgrades/merge/) | +| 0 | **Phase0** |`0` |
  • Core
    • [The beacon chain](specs/phase0/beacon-chain.md)
    • [Deposit contract](specs/phase0/deposit-contract.md)
    • [Beacon chain fork choice](specs/phase0/fork-choice.md)
  • Additions
    • [Honest validator guide](specs/phase0/validator.md)
    • [P2P networking](specs/phase0/p2p-interface.md)
    • [Weak subjectivity](specs/phase0/weak-subjectivity.md)
| +| 1 | **Altair** | `74240` |
  • Core
    • [Beacon chain changes](specs/altair/beacon-chain.md)
    • [Altair fork](specs/altair/fork.md)
  • Additions
    • [Light client sync protocol](specs/altair/sync-protocol.md)
    • [Honest validator guide changes](specs/altair/validator.md)
    • [P2P networking](specs/altair/p2p-interface.md)
| +| 2 | **Bellatrix**
(["The Merge"](https://ethereum.org/en/upgrades/merge/)) | TBD |
  • Core
    • [Beacon Chain changes](specs/bellatrix/beacon-chain.md)
    • [Bellatrix fork](specs/bellatrix/fork.md)
    • [Fork choice changes](specs/bellatrix/fork-choice.md)
  • Additions
    • [Honest validator guide changes](specs/bellatrix/validator.md)
    • [P2P networking](specs/bellatrix/p2p-interface.md)
| ### In-development Specifications | Code Name or Topic | Specs | Notes | From c165d31ccb7f417ab41c4abbcc7c7ec08e538656 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 6 Jun 2022 16:24:44 +0800 Subject: [PATCH 008/105] Remove `@disable_process_reveal_deadlines` --- tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py index 34a053c86..44d663422 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py @@ -34,7 +34,6 @@ from eth2spec.test.context import ( spec_test, spec_state_test, dump_skipping_message, with_phases, with_all_phases, single_phase, expect_assertion_error, always_bls, - disable_process_reveal_deadlines, with_presets, with_custom_state, large_validator_set, @@ -840,7 +839,6 @@ def test_attestation(spec, state): @with_all_phases @spec_state_test -@disable_process_reveal_deadlines def test_voluntary_exit(spec, state): validator_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] @@ -890,7 +888,6 @@ def test_double_validator_exit_same_block(spec, state): @with_all_phases @spec_state_test -@disable_process_reveal_deadlines def test_multiple_different_validator_exits_same_block(spec, state): validator_indices = [ spec.get_active_validator_indices(state, spec.get_current_epoch(state))[i] @@ -923,7 +920,6 @@ def test_multiple_different_validator_exits_same_block(spec, state): @with_all_phases @spec_state_test -@disable_process_reveal_deadlines def test_slash_and_exit_same_index(spec, state): validator_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] yield from run_slash_and_exit(spec, state, validator_index, validator_index, valid=False) @@ -931,7 +927,6 @@ def test_slash_and_exit_same_index(spec, state): @with_all_phases @spec_state_test -@disable_process_reveal_deadlines def test_slash_and_exit_diff_index(spec, state): slash_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] exit_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-2] From 74489d552304b9246d59e48b643ed03962e6a9e8 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 8 Jun 2022 13:16:12 -0600 Subject: [PATCH 009/105] Partial withdrawals (#2862) * t push base design for partial withdrawals * moor tests * clean up withdrawals naming * make partial withdrawal randomized tests better * Apply suggestions from code review Co-authored-by: Alex Stokes Co-authored-by: Hsiao-Wei Wang * fix mainnet brokn test * name swap * lint Co-authored-by: Alex Stokes Co-authored-by: Hsiao-Wei Wang --- presets/mainnet/capella.yaml | 25 +- presets/minimal/capella.yaml | 23 ++ specs/capella/beacon-chain.md | 93 ++++++-- specs/capella/fork.md | 5 +- specs/capella/validator.md | 4 +- .../test_process_withdrawals.py | 50 ++-- .../test_process_full_withdrawals.py | 14 +- .../test_process_partial_withdrawals.py | 224 ++++++++++++++++++ .../eth2spec/test/helpers/epoch_processing.py | 1 + .../test/helpers/execution_payload.py | 4 +- 10 files changed, 390 insertions(+), 53 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_partial_withdrawals.py diff --git a/presets/mainnet/capella.yaml b/presets/mainnet/capella.yaml index c5dfe1d4b..c3eedc5c1 100644 --- a/presets/mainnet/capella.yaml +++ b/presets/mainnet/capella.yaml @@ -1 +1,24 @@ -# Minimal preset - Capella +# Mainnet preset - Capella + +# Misc +# --------------------------------------------------------------- +# 2**8 (= 256) withdrawals +MAX_PARTIAL_WITHDRAWALS_PER_EPOCH: 256 + + +# State list lengths +# --------------------------------------------------------------- +# 2**40 (= 1,099,511,627,776) withdrawals +WITHDRAWALS_QUEUE_LIMIT: 1099511627776 + + +# Max operations per block +# --------------------------------------------------------------- +# 2**4 (= 16) +MAX_BLS_TO_EXECUTION_CHANGES: 16 + + +# Execution +# --------------------------------------------------------------- +# 2**4 (= 16) withdrawals +MAX_WITHDRAWALS_PER_PAYLOAD: 16 diff --git a/presets/minimal/capella.yaml b/presets/minimal/capella.yaml index c5dfe1d4b..a3c5e8f1d 100644 --- a/presets/minimal/capella.yaml +++ b/presets/minimal/capella.yaml @@ -1 +1,24 @@ # Minimal preset - Capella + +# Misc +# --------------------------------------------------------------- +# [customized] 16 for more interesting tests at low validator count +MAX_PARTIAL_WITHDRAWALS_PER_EPOCH: 16 + + +# State list lengths +# --------------------------------------------------------------- +# 2**40 (= 1,099,511,627,776) withdrawals +WITHDRAWALS_QUEUE_LIMIT: 1099511627776 + + +# Max operations per block +# --------------------------------------------------------------- +# 2**4 (= 16) +MAX_BLS_TO_EXECUTION_CHANGES: 16 + + +# Execution +# --------------------------------------------------------------- +# [customized] Lower than MAX_PARTIAL_WITHDRAWALS_PER_EPOCH so not all processed in one block +MAX_WITHDRAWALS_PER_PAYLOAD: 16 diff --git a/specs/capella/beacon-chain.md b/specs/capella/beacon-chain.md index f940b3a27..da67a8fcb 100644 --- a/specs/capella/beacon-chain.md +++ b/specs/capella/beacon-chain.md @@ -11,6 +11,7 @@ - [Constants](#constants) - [Domain types](#domain-types) - [Preset](#preset) + - [Misc](#misc) - [State list lengths](#state-list-lengths) - [Max operations per block](#max-operations-per-block) - [Execution](#execution) @@ -30,10 +31,13 @@ - [Beacon state mutators](#beacon-state-mutators) - [`withdraw`](#withdraw) - [Predicates](#predicates) + - [`has_eth1_withdrawal_credential`](#has_eth1_withdrawal_credential) - [`is_fully_withdrawable_validator`](#is_fully_withdrawable_validator) + - [`is_partially_withdrawable_validator`](#is_partially_withdrawable_validator) - [Beacon chain state transition function](#beacon-chain-state-transition-function) - [Epoch processing](#epoch-processing) - - [Withdrawals](#withdrawals) + - [Full withdrawals](#full-withdrawals) + - [Partial withdrawals](#partial-withdrawals) - [Block processing](#block-processing) - [New `process_withdrawals`](#new-process_withdrawals) - [Modified `process_execution_payload`](#modified-process_execution_payload) @@ -48,7 +52,8 @@ Capella is a consensus-layer upgrade containing a number of features related to validator withdrawals. Including: * Automatic withdrawals of `withdrawable` validators -* Partial withdrawals during block proposal +* Partial withdrawals sweep for validators with 0x01 withdrawal + credentials and balances in exceess of `MAX_EFFECTIVE_BALANCE` * Operation to change from `BLS_WITHDRAWAL_PREFIX` to `ETH1_ADDRESS_WITHDRAWAL_PREFIX` versioned withdrawal credentials to enable withdrawals for a validator @@ -70,11 +75,17 @@ We define the following Python custom types for type hinting and readability: ## Preset +### Misc + +| Name | Value | +| - | - | +| `MAX_PARTIAL_WITHDRAWALS_PER_EPOCH` | `uint64(2**8)` (= 256) | + ### State list lengths | Name | Value | Unit | Duration | | - | - | :-: | :-: | -| `WITHDRAWALS_QUEUE_LIMIT` | `uint64(2**40)` (= 1,099,511,627,776) | withdrawals enqueued in state| +| `WITHDRAWAL_QUEUE_LIMIT` | `uint64(2**40)` (= 1,099,511,627,776) | withdrawals enqueued in state| ### Max operations per block @@ -245,8 +256,9 @@ class BeaconState(Container): # Execution latest_execution_payload_header: ExecutionPayloadHeader # Withdrawals - withdrawal_index: WithdrawalIndex - withdrawals_queue: List[Withdrawal, WITHDRAWALS_QUEUE_LIMIT] # [New in Capella] + withdrawal_queue: List[Withdrawal, WITHDRAWAL_QUEUE_LIMIT] # [New in Capella] + next_withdrawal_index: WithdrawalIndex # [New in Capella] + next_partial_withdrawal_validator_index: ValidatorIndex # [New in Capella] ``` ## Helpers @@ -261,16 +273,26 @@ def withdraw_balance(state: BeaconState, index: ValidatorIndex, amount: Gwei) -> decrease_balance(state, index, amount) # Create a corresponding withdrawal receipt withdrawal = Withdrawal( - index=state.withdrawal_index, - address=state.validators[index].withdrawal_credentials[12:], + index=state.next_withdrawal_index, + address=ExecutionAddress(state.validators[index].withdrawal_credentials[12:]), amount=amount, ) - state.withdrawal_index = WithdrawalIndex(state.withdrawal_index + 1) - state.withdrawals_queue.append(withdrawal) + state.next_withdrawal_index = WithdrawalIndex(state.next_withdrawal_index + 1) + state.withdrawal_queue.append(withdrawal) ``` ### Predicates +#### `has_eth1_withdrawal_credential` + +```python +def has_eth1_withdrawal_credential(validator: Validator) -> bool: + """ + Check if ``validator`` has an 0x01 prefixed "eth1" withdrawal credential + """ + return validator.withdrawal_credentials[:1] == ETH1_ADDRESS_WITHDRAWAL_PREFIX +``` + #### `is_fully_withdrawable_validator` ```python @@ -278,8 +300,22 @@ def is_fully_withdrawable_validator(validator: Validator, epoch: Epoch) -> bool: """ Check if ``validator`` is fully withdrawable. """ - is_eth1_withdrawal_prefix = validator.withdrawal_credentials[:1] == ETH1_ADDRESS_WITHDRAWAL_PREFIX - return is_eth1_withdrawal_prefix and validator.withdrawable_epoch <= epoch < validator.fully_withdrawn_epoch + return ( + has_eth1_withdrawal_credential(validator) + and validator.withdrawable_epoch <= epoch < validator.fully_withdrawn_epoch + ) +``` + +#### `is_partially_withdrawable_validator` + +```python +def is_partially_withdrawable_validator(validator: Validator, balance: Gwei) -> bool: + """ + Check if ``validator`` is partially withdrawable. + """ + has_max_effective_balance = validator.effective_balance == MAX_EFFECTIVE_BALANCE + has_excess_balance = balance > MAX_EFFECTIVE_BALANCE + return has_eth1_withdrawal_credential(validator) and has_max_effective_balance and has_excess_balance ``` ## Beacon chain state transition function @@ -301,9 +337,11 @@ def process_epoch(state: BeaconState) -> None: process_participation_flag_updates(state) process_sync_committee_updates(state) process_full_withdrawals(state) # [New in Capella] + process_partial_withdrawals(state) # [New in Capella] + ``` -#### Withdrawals +#### Full withdrawals *Note*: The function `process_full_withdrawals` is new. @@ -317,6 +355,31 @@ def process_full_withdrawals(state: BeaconState) -> None: validator.fully_withdrawn_epoch = current_epoch ``` +#### Partial withdrawals + +*Note*: The function `process_partial_withdrawals` is new. + +```python +def process_partial_withdrawals(state: BeaconState) -> None: + partial_withdrawals_count = 0 + # Begin where we left off last time + validator_index = state.next_partial_withdrawal_validator_index + for _ in range(len(state.validators)): + balance = state.balances[validator_index] + validator = state.validators[validator_index] + if is_partially_withdrawable_validator(validator, balance): + withdraw_balance(state, validator_index, balance - MAX_EFFECTIVE_BALANCE) + partial_withdrawals_count += 1 + + # Iterate to next validator to check for partial withdrawal + validator_index = ValidatorIndex((validator_index + 1) % len(state.validators)) + # Exit if performed maximum allowable withdrawals + if partial_withdrawals_count == MAX_PARTIAL_WITHDRAWALS_PER_EPOCH: + break + + state.next_partial_withdrawal_validator_index = validator_index +``` + ### Block processing ```python @@ -335,15 +398,15 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: ```python def process_withdrawals(state: BeaconState, payload: ExecutionPayload) -> None: - num_withdrawals = min(MAX_WITHDRAWALS_PER_PAYLOAD, len(state.withdrawals_queue)) - dequeued_withdrawals = state.withdrawals_queue[:num_withdrawals] + num_withdrawals = min(MAX_WITHDRAWALS_PER_PAYLOAD, len(state.withdrawal_queue)) + dequeued_withdrawals = state.withdrawal_queue[:num_withdrawals] assert len(dequeued_withdrawals) == len(payload.withdrawals) for dequeued_withdrawal, withdrawal in zip(dequeued_withdrawals, payload.withdrawals): assert dequeued_withdrawal == withdrawal # Remove dequeued withdrawals from state - state.withdrawals_queue = state.withdrawals_queue[num_withdrawals:] + state.withdrawal_queue = state.withdrawal_queue[num_withdrawals:] ``` #### Modified `process_execution_payload` diff --git a/specs/capella/fork.md b/specs/capella/fork.md index 5f015a4ff..0d8283e56 100644 --- a/specs/capella/fork.md +++ b/specs/capella/fork.md @@ -89,8 +89,9 @@ def upgrade_to_capella(pre: bellatrix.BeaconState) -> BeaconState: # Execution-layer latest_execution_payload_header=pre.latest_execution_payload_header, # Withdrawals - withdrawal_index=WithdrawalIndex(0), - withdrawals_queue=[], + withdrawal_queue=[], + next_withdrawal_index=WithdrawalIndex(0), + next_partial_withdrawal_validator_index=ValidatorIndex(0), ) for pre_validator in pre.validators: diff --git a/specs/capella/validator.md b/specs/capella/validator.md index 8c6c860a3..85dbd7e00 100644 --- a/specs/capella/validator.md +++ b/specs/capella/validator.md @@ -61,8 +61,8 @@ helper `get_expected_withdrawals`) and passed into the `ExecutionEngine` within ```python def get_expected_withdrawals(state: BeaconState) -> Sequence[Withdrawal]: - num_withdrawals = min(MAX_WITHDRAWALS_PER_PAYLOAD, len(state.withdrawals_queue)) - return state.withdrawals_queue[:num_withdrawals] + num_withdrawals = min(MAX_WITHDRAWALS_PER_PAYLOAD, len(state.withdrawal_queue)) + return state.withdrawal_queue[:num_withdrawals] ``` *Note*: The only change made to `prepare_execution_payload` is to call diff --git a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_withdrawals.py b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_withdrawals.py index 204816c99..26ace24b3 100644 --- a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_withdrawals.py +++ b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_withdrawals.py @@ -7,8 +7,8 @@ from eth2spec.test.context import spec_state_test, expect_assertion_error, with_ from eth2spec.test.helpers.state import next_slot -def prepare_withdrawals_queue(spec, state, num_withdrawals): - pre_queue_len = len(state.withdrawals_queue) +def prepare_withdrawal_queue(spec, state, num_withdrawals): + pre_queue_len = len(state.withdrawal_queue) for i in range(num_withdrawals): withdrawal = spec.Withdrawal( @@ -16,9 +16,9 @@ def prepare_withdrawals_queue(spec, state, num_withdrawals): address=b'\x42' * 20, amount=200000 + i, ) - state.withdrawals_queue.append(withdrawal) + state.withdrawal_queue.append(withdrawal) - assert len(state.withdrawals_queue) == num_withdrawals + pre_queue_len + assert len(state.withdrawal_queue) == num_withdrawals + pre_queue_len def run_withdrawals_processing(spec, state, execution_payload, valid=True): @@ -30,8 +30,8 @@ def run_withdrawals_processing(spec, state, execution_payload, valid=True): If ``valid == False``, run expecting ``AssertionError`` """ - pre_withdrawals_queue = state.withdrawals_queue.copy() - num_withdrawals = min(spec.MAX_WITHDRAWALS_PER_PAYLOAD, len(pre_withdrawals_queue)) + pre_withdrawal_queue = state.withdrawal_queue.copy() + num_withdrawals = min(spec.MAX_WITHDRAWALS_PER_PAYLOAD, len(pre_withdrawal_queue)) yield 'pre', state yield 'execution_payload', execution_payload @@ -45,18 +45,18 @@ def run_withdrawals_processing(spec, state, execution_payload, valid=True): yield 'post', state - if len(pre_withdrawals_queue) == 0: - assert len(state.withdrawals_queue) == 0 - elif len(pre_withdrawals_queue) <= num_withdrawals: - assert len(state.withdrawals_queue) == 0 + if len(pre_withdrawal_queue) == 0: + assert len(state.withdrawal_queue) == 0 + elif len(pre_withdrawal_queue) <= num_withdrawals: + assert len(state.withdrawal_queue) == 0 else: - assert state.withdrawals_queue == pre_withdrawals_queue[num_withdrawals:] + assert state.withdrawal_queue == pre_withdrawal_queue[num_withdrawals:] @with_capella_and_later @spec_state_test def test_success_empty_queue(spec, state): - assert len(state.withdrawals_queue) == 0 + assert len(state.withdrawal_queue) == 0 next_slot(spec, state) execution_payload = build_empty_execution_payload(spec, state) @@ -67,7 +67,7 @@ def test_success_empty_queue(spec, state): @with_capella_and_later @spec_state_test def test_success_one_in_queue(spec, state): - prepare_withdrawals_queue(spec, state, 1) + prepare_withdrawal_queue(spec, state, 1) next_slot(spec, state) execution_payload = build_empty_execution_payload(spec, state) @@ -78,7 +78,7 @@ def test_success_one_in_queue(spec, state): @with_capella_and_later @spec_state_test def test_success_max_per_slot_in_queue(spec, state): - prepare_withdrawals_queue(spec, state, spec.MAX_WITHDRAWALS_PER_PAYLOAD) + prepare_withdrawal_queue(spec, state, spec.MAX_WITHDRAWALS_PER_PAYLOAD) next_slot(spec, state) execution_payload = build_empty_execution_payload(spec, state) @@ -89,7 +89,7 @@ def test_success_max_per_slot_in_queue(spec, state): @with_capella_and_later @spec_state_test def test_success_a_lot_in_queue(spec, state): - prepare_withdrawals_queue(spec, state, spec.MAX_WITHDRAWALS_PER_PAYLOAD * 4) + prepare_withdrawal_queue(spec, state, spec.MAX_WITHDRAWALS_PER_PAYLOAD * 4) next_slot(spec, state) execution_payload = build_empty_execution_payload(spec, state) @@ -104,7 +104,7 @@ def test_success_a_lot_in_queue(spec, state): @with_capella_and_later @spec_state_test def test_fail_empty_queue_non_empty_withdrawals(spec, state): - assert len(state.withdrawals_queue) == 0 + assert len(state.withdrawal_queue) == 0 next_slot(spec, state) execution_payload = build_empty_execution_payload(spec, state) @@ -121,7 +121,7 @@ def test_fail_empty_queue_non_empty_withdrawals(spec, state): @with_capella_and_later @spec_state_test def test_fail_one_in_queue_none_in_withdrawals(spec, state): - prepare_withdrawals_queue(spec, state, 1) + prepare_withdrawal_queue(spec, state, 1) next_slot(spec, state) execution_payload = build_empty_execution_payload(spec, state) @@ -133,7 +133,7 @@ def test_fail_one_in_queue_none_in_withdrawals(spec, state): @with_capella_and_later @spec_state_test def test_fail_one_in_queue_two_in_withdrawals(spec, state): - prepare_withdrawals_queue(spec, state, 1) + prepare_withdrawal_queue(spec, state, 1) next_slot(spec, state) execution_payload = build_empty_execution_payload(spec, state) @@ -145,7 +145,7 @@ def test_fail_one_in_queue_two_in_withdrawals(spec, state): @with_capella_and_later @spec_state_test def test_fail_max_per_slot_in_queue_one_less_in_withdrawals(spec, state): - prepare_withdrawals_queue(spec, state, spec.MAX_WITHDRAWALS_PER_PAYLOAD) + prepare_withdrawal_queue(spec, state, spec.MAX_WITHDRAWALS_PER_PAYLOAD) next_slot(spec, state) execution_payload = build_empty_execution_payload(spec, state) @@ -157,7 +157,7 @@ def test_fail_max_per_slot_in_queue_one_less_in_withdrawals(spec, state): @with_capella_and_later @spec_state_test def test_fail_a_lot_in_queue_too_few_in_withdrawals(spec, state): - prepare_withdrawals_queue(spec, state, spec.MAX_WITHDRAWALS_PER_PAYLOAD * 4) + prepare_withdrawal_queue(spec, state, spec.MAX_WITHDRAWALS_PER_PAYLOAD * 4) next_slot(spec, state) execution_payload = build_empty_execution_payload(spec, state) @@ -173,7 +173,7 @@ def test_fail_a_lot_in_queue_too_few_in_withdrawals(spec, state): @with_capella_and_later @spec_state_test def test_fail_incorrect_dequeue_index(spec, state): - prepare_withdrawals_queue(spec, state, 1) + prepare_withdrawal_queue(spec, state, 1) next_slot(spec, state) execution_payload = build_empty_execution_payload(spec, state) @@ -185,7 +185,7 @@ def test_fail_incorrect_dequeue_index(spec, state): @with_capella_and_later @spec_state_test def test_fail_incorrect_dequeue_address(spec, state): - prepare_withdrawals_queue(spec, state, 1) + prepare_withdrawal_queue(spec, state, 1) next_slot(spec, state) execution_payload = build_empty_execution_payload(spec, state) @@ -197,7 +197,7 @@ def test_fail_incorrect_dequeue_address(spec, state): @with_capella_and_later @spec_state_test def test_fail_incorrect_dequeue_amount(spec, state): - prepare_withdrawals_queue(spec, state, 1) + prepare_withdrawal_queue(spec, state, 1) next_slot(spec, state) execution_payload = build_empty_execution_payload(spec, state) @@ -209,7 +209,7 @@ def test_fail_incorrect_dequeue_amount(spec, state): @with_capella_and_later @spec_state_test def test_fail_one_of_many_dequeued_incorrectly(spec, state): - prepare_withdrawals_queue(spec, state, spec.MAX_WITHDRAWALS_PER_PAYLOAD * 4) + prepare_withdrawal_queue(spec, state, spec.MAX_WITHDRAWALS_PER_PAYLOAD * 4) next_slot(spec, state) execution_payload = build_empty_execution_payload(spec, state) @@ -227,7 +227,7 @@ def test_fail_one_of_many_dequeued_incorrectly(spec, state): @with_capella_and_later @spec_state_test def test_fail_many_dequeued_incorrectly(spec, state): - prepare_withdrawals_queue(spec, state, spec.MAX_WITHDRAWALS_PER_PAYLOAD * 4) + prepare_withdrawal_queue(spec, state, spec.MAX_WITHDRAWALS_PER_PAYLOAD * 4) next_slot(spec, state) execution_payload = build_empty_execution_payload(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_full_withdrawals.py b/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_full_withdrawals.py index 305f6e1ba..1498666bb 100644 --- a/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_full_withdrawals.py +++ b/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_full_withdrawals.py @@ -17,8 +17,8 @@ def set_validator_withdrawable(spec, state, index, withdrawable_epoch=None): def run_process_full_withdrawals(spec, state, num_expected_withdrawals=None): - pre_withdrawal_index = state.withdrawal_index - pre_withdrawals_queue = state.withdrawals_queue + pre_next_withdrawal_index = state.next_withdrawal_index + pre_withdrawal_queue = state.withdrawal_queue.copy() to_be_withdrawn_indices = [ index for index, validator in enumerate(state.validators) if spec.is_fully_withdrawable_validator(validator, spec.get_current_epoch(state)) @@ -26,6 +26,8 @@ def run_process_full_withdrawals(spec, state, num_expected_withdrawals=None): if num_expected_withdrawals is not None: assert len(to_be_withdrawn_indices) == num_expected_withdrawals + else: + num_expected_withdrawals = len(to_be_withdrawn_indices) yield from run_epoch_processing_with(spec, state, 'process_full_withdrawals') @@ -34,8 +36,8 @@ def run_process_full_withdrawals(spec, state, num_expected_withdrawals=None): assert validator.fully_withdrawn_epoch == spec.get_current_epoch(state) assert state.balances[index] == 0 - assert len(state.withdrawals_queue) == len(pre_withdrawals_queue) + num_expected_withdrawals - assert state.withdrawal_index == pre_withdrawal_index + num_expected_withdrawals + assert len(state.withdrawal_queue) == len(pre_withdrawal_queue) + num_expected_withdrawals + assert state.next_withdrawal_index == pre_next_withdrawal_index + num_expected_withdrawals @with_capella_and_later @@ -65,10 +67,10 @@ def test_single_withdrawal(spec, state): # Make one validator withdrawable set_validator_withdrawable(spec, state, 0) - assert state.withdrawal_index == 0 + assert state.next_withdrawal_index == 0 yield from run_process_full_withdrawals(spec, state, 1) - assert state.withdrawal_index == 1 + assert state.next_withdrawal_index == 1 @with_capella_and_later diff --git a/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_partial_withdrawals.py b/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_partial_withdrawals.py new file mode 100644 index 000000000..bf2e73fa1 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_partial_withdrawals.py @@ -0,0 +1,224 @@ +import random +from eth2spec.test.helpers.constants import MINIMAL +from eth2spec.test.context import ( + with_capella_and_later, + spec_state_test, + with_presets, +) +from eth2spec.test.helpers.epoch_processing import run_epoch_processing_to +from eth2spec.test.helpers.state import next_epoch +from eth2spec.test.helpers.random import randomize_state + + +def set_validator_partially_withdrawable(spec, state, index, rng=random.Random(666)): + validator = state.validators[index] + validator.withdrawal_credentials = spec.ETH1_ADDRESS_WITHDRAWAL_PREFIX + validator.withdrawal_credentials[1:] + validator.effective_balance = spec.MAX_EFFECTIVE_BALANCE + state.balances[index] = spec.MAX_EFFECTIVE_BALANCE + rng.randint(1, 100000000) + + assert spec.is_partially_withdrawable_validator(validator, state.balances[index]) + + +def run_process_partial_withdrawals(spec, state, num_expected_withdrawals=None): + # Run rest of epoch processing before predicting partial withdrawals as + # balance changes can affect withdrawability + run_epoch_processing_to(spec, state, 'process_partial_withdrawals') + + pre_next_withdrawal_index = state.next_withdrawal_index + pre_withdrawal_queue = state.withdrawal_queue.copy() + + partially_withdrawable_indices = [ + index for index, validator in enumerate(state.validators) + if spec.is_partially_withdrawable_validator(validator, state.balances[index]) + ] + num_partial_withdrawals = min(len(partially_withdrawable_indices), spec.MAX_PARTIAL_WITHDRAWALS_PER_EPOCH) + + if num_expected_withdrawals is not None: + assert num_partial_withdrawals == num_expected_withdrawals + else: + num_expected_withdrawals = num_partial_withdrawals + + yield 'pre', state + spec.process_partial_withdrawals(state) + yield 'post', state + + post_partially_withdrawable_indices = [ + index for index, validator in enumerate(state.validators) + if spec.is_partially_withdrawable_validator(validator, state.balances[index]) + ] + + assert len(partially_withdrawable_indices) - num_partial_withdrawals == len(post_partially_withdrawable_indices) + + assert len(state.withdrawal_queue) == len(pre_withdrawal_queue) + num_expected_withdrawals + assert state.next_withdrawal_index == pre_next_withdrawal_index + num_expected_withdrawals + + +@with_capella_and_later +@spec_state_test +def test_success_no_withdrawable(spec, state): + pre_validators = state.validators.copy() + yield from run_process_partial_withdrawals(spec, state, 0) + + assert pre_validators == state.validators + + +@with_capella_and_later +@spec_state_test +def test_success_one_partial_withdrawable(spec, state): + validator_index = len(state.validators) // 2 + set_validator_partially_withdrawable(spec, state, validator_index) + + yield from run_process_partial_withdrawals(spec, state, 1) + + +@with_capella_and_later +@spec_state_test +def test_success_one_partial_withdrawable_not_yet_active(spec, state): + validator_index = len(state.validators) // 2 + state.validators[validator_index].activation_epoch += 4 + set_validator_partially_withdrawable(spec, state, validator_index) + + assert not spec.is_active_validator(state.validators[validator_index], spec.get_current_epoch(state)) + + yield from run_process_partial_withdrawals(spec, state, 1) + + +@with_capella_and_later +@spec_state_test +def test_success_one_partial_withdrawable_in_exit_queue(spec, state): + validator_index = len(state.validators) // 2 + state.validators[validator_index].exit_epoch = spec.get_current_epoch(state) + 1 + set_validator_partially_withdrawable(spec, state, validator_index) + + assert spec.is_active_validator(state.validators[validator_index], spec.get_current_epoch(state)) + assert not spec.is_active_validator(state.validators[validator_index], spec.get_current_epoch(state) + 1) + + yield from run_process_partial_withdrawals(spec, state, 1) + + +@with_capella_and_later +@spec_state_test +def test_success_one_partial_withdrawable_exited(spec, state): + validator_index = len(state.validators) // 2 + state.validators[validator_index].exit_epoch = spec.get_current_epoch(state) + set_validator_partially_withdrawable(spec, state, validator_index) + + assert not spec.is_active_validator(state.validators[validator_index], spec.get_current_epoch(state)) + + yield from run_process_partial_withdrawals(spec, state, 1) + + +@with_capella_and_later +@spec_state_test +def test_success_one_partial_withdrawable_active_and_slashed(spec, state): + validator_index = len(state.validators) // 2 + state.validators[validator_index].slashed = True + set_validator_partially_withdrawable(spec, state, validator_index) + + assert spec.is_active_validator(state.validators[validator_index], spec.get_current_epoch(state)) + + yield from run_process_partial_withdrawals(spec, state, 1) + + +@with_capella_and_later +@spec_state_test +def test_success_one_partial_withdrawable_exited_and_slashed(spec, state): + validator_index = len(state.validators) // 2 + state.validators[validator_index].slashed = True + state.validators[validator_index].exit_epoch = spec.get_current_epoch(state) + set_validator_partially_withdrawable(spec, state, validator_index) + + assert not spec.is_active_validator(state.validators[validator_index], spec.get_current_epoch(state)) + + yield from run_process_partial_withdrawals(spec, state, 1) + + +@with_capella_and_later +@spec_state_test +def test_success_two_partial_withdrawable(spec, state): + set_validator_partially_withdrawable(spec, state, 0) + set_validator_partially_withdrawable(spec, state, 1) + + yield from run_process_partial_withdrawals(spec, state, 2) + + +@with_capella_and_later +@spec_state_test +def test_success_max_partial_withdrawable(spec, state): + # Sanity check that this test works for this state + assert len(state.validators) >= spec.MAX_PARTIAL_WITHDRAWALS_PER_EPOCH + + for i in range(spec.MAX_PARTIAL_WITHDRAWALS_PER_EPOCH): + set_validator_partially_withdrawable(spec, state, i) + + yield from run_process_partial_withdrawals(spec, state, spec.MAX_PARTIAL_WITHDRAWALS_PER_EPOCH) + + +@with_capella_and_later +@with_presets([MINIMAL], reason="not no enough validators with mainnet config") +@spec_state_test +def test_success_max_plus_one_withdrawable(spec, state): + # Sanity check that this test works for this state + assert len(state.validators) >= spec.MAX_PARTIAL_WITHDRAWALS_PER_EPOCH + 1 + + # More than MAX_PARTIAL_WITHDRAWALS_PER_EPOCH partially withdrawable + for i in range(spec.MAX_PARTIAL_WITHDRAWALS_PER_EPOCH + 1): + set_validator_partially_withdrawable(spec, state, i) + + # Should only have MAX_PARTIAL_WITHDRAWALS_PER_EPOCH withdrawals created + yield from run_process_partial_withdrawals(spec, state, spec.MAX_PARTIAL_WITHDRAWALS_PER_EPOCH) + + +def run_random_partial_withdrawals_test(spec, state, rng): + for _ in range(rng.randint(0, 2)): + next_epoch(spec, state) + randomize_state(spec, state, rng) + + num_validators = len(state.validators) + state.next_partial_withdrawal_validator_index = rng.randint(0, num_validators - 1) + + num_partially_withdrawable = rng.randint(0, num_validators - 1) + partially_withdrawable_indices = rng.sample(range(num_validators), num_partially_withdrawable) + for index in partially_withdrawable_indices: + set_validator_partially_withdrawable(spec, state, index) + + # Note: due to the randomness and other epoch processing, some of these set as "partially withdrawable" + # may not be partially withdrawable once we get to ``process_partial_withdrawals``, + # thus *not* using the optional third param in this call + yield from run_process_partial_withdrawals(spec, state) + + +@with_capella_and_later +@spec_state_test +def test_random_0(spec, state): + yield from run_random_partial_withdrawals_test(spec, state, random.Random(0)) + + +@with_capella_and_later +@spec_state_test +def test_random_1(spec, state): + yield from run_random_partial_withdrawals_test(spec, state, random.Random(1)) + + +@with_capella_and_later +@spec_state_test +def test_random_2(spec, state): + yield from run_random_partial_withdrawals_test(spec, state, random.Random(2)) + + +@with_capella_and_later +@spec_state_test +def test_random_3(spec, state): + yield from run_random_partial_withdrawals_test(spec, state, random.Random(3)) + + +@with_capella_and_later +@spec_state_test +def test_random_4(spec, state): + yield from run_random_partial_withdrawals_test(spec, state, random.Random(4)) + + +@with_capella_and_later +@spec_state_test +def test_random_5(spec, state): + yield from run_random_partial_withdrawals_test(spec, state, random.Random(5)) diff --git a/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py b/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py index 8c27480c0..e18c239f1 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py +++ b/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py @@ -29,6 +29,7 @@ def get_process_calls(spec): ), 'process_sync_committee_updates', # altair 'process_full_withdrawals', # capella + 'process_partial_withdrawals', # capella # TODO: add sharding processing functions when spec stabilizes. ] diff --git a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py index 5e7066632..30f7a0394 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py @@ -29,8 +29,8 @@ def build_empty_execution_payload(spec, state, randao_mix=None): transactions=empty_txs, ) if spec.fork not in FORKS_BEFORE_CAPELLA: - num_withdrawals = min(spec.MAX_WITHDRAWALS_PER_PAYLOAD, len(state.withdrawals_queue)) - payload.withdrawals = state.withdrawals_queue[:num_withdrawals] + num_withdrawals = min(spec.MAX_WITHDRAWALS_PER_PAYLOAD, len(state.withdrawal_queue)) + payload.withdrawals = state.withdrawal_queue[:num_withdrawals] # TODO: real RLP + block hash logic would be nice, requires RLP and keccak256 dependency however. payload.block_hash = spec.Hash32(spec.hash(payload.hash_tree_root() + b"FAKE RLP HASH")) From dbac7b15824edeaadc349aa4967582f06720c559 Mon Sep 17 00:00:00 2001 From: inphi Date: Mon, 13 Jun 2022 16:08:25 -0400 Subject: [PATCH 010/105] EIP-4844: Remove shard field from BlobsSidecar --- specs/eip4844/validator.md | 1 - 1 file changed, 1 deletion(-) diff --git a/specs/eip4844/validator.md b/specs/eip4844/validator.md index 2083934c5..e9fe05d8b 100644 --- a/specs/eip4844/validator.md +++ b/specs/eip4844/validator.md @@ -110,7 +110,6 @@ Before publishing a prepared beacon block proposal, the corresponding blobs are blobs_sidecar = BlobsSidecar( beacon_block_root=hash_tree_root(beacon_block) beacon_block_slot=beacon_block.slot - shard=0, blobs=blobs, ) ``` From e4c75d67ceee50236dd867f91b6fe5ddf4c0d16d Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 13 Jun 2022 15:30:12 +0300 Subject: [PATCH 011/105] Introduce high-level logic of new efficient block validation --- specs/eip4844/beacon-chain.md | 1 + specs/eip4844/p2p-interface.md | 2 ++ specs/eip4844/validator.md | 20 ++++++++++++++++++-- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/specs/eip4844/beacon-chain.md b/specs/eip4844/beacon-chain.md index 8c84a2862..6011e5832 100644 --- a/specs/eip4844/beacon-chain.md +++ b/specs/eip4844/beacon-chain.md @@ -45,6 +45,7 @@ This upgrade adds blobs to the beacon chain as part of EIP-4844. | `Blob` | `Vector[BLSFieldElement, FIELD_ELEMENTS_PER_BLOB]` | | | `VersionedHash` | `Bytes32` | | | `KZGCommitment` | `Bytes48` | Same as BLS standard "is valid pubkey" check but also allows `0x00..00` for point-at-infinity | +| `KZGProof` | Bytes48 | Same as for `KZGCommitment` | ## Constants diff --git a/specs/eip4844/p2p-interface.md b/specs/eip4844/p2p-interface.md index 68c984950..7a447dbd4 100644 --- a/specs/eip4844/p2p-interface.md +++ b/specs/eip4844/p2p-interface.md @@ -57,6 +57,7 @@ class BlobsSidecar(Container): beacon_block_root: Root beacon_block_slot: Slot blobs: List[Blob, MAX_BLOBS_PER_BLOCK] + kzg_aggregated_proof: KZGProof ``` ### `SignedBlobsSidecar` @@ -114,6 +115,7 @@ The following validations MUST pass before forwarding the `signed_blobs_sidecar` Alias `sidecar = signed_blobs_sidecar.message`. - _[IGNORE]_ the `sidecar.beacon_block_slot` is for the current slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `blobs_sidecar.beacon_block_slot == current_slot`. - _[REJECT]_ the `sidecar.blobs` are all well formatted, i.e. the `BLSFieldElement` in valid range (`x < BLS_MODULUS`). +- _[REJECT]_ The KZG proof is a correctly encoded compressed BLS G1 Point -- i.e. `bls.KeyValidate(blobs_sidecar.kzg_aggregated_proof) - _[REJECT]_ the beacon proposer signature, `signed_blobs_sidecar.signature`, is valid -- i.e. ```python domain = get_domain(state, DOMAIN_BLOBS_SIDECAR, blobs_sidecar.beacon_block_slot // SLOTS_PER_EPOCH) diff --git a/specs/eip4844/validator.md b/specs/eip4844/validator.md index 2083934c5..aed38639f 100644 --- a/specs/eip4844/validator.md +++ b/specs/eip4844/validator.md @@ -59,9 +59,25 @@ def verify_blobs_sidecar(slot: Slot, beacon_block_root: Root, assert slot == blobs_sidecar.beacon_block_slot assert beacon_block_root == blobs_sidecar.beacon_block_root blobs = blobs_sidecar.blobs + kzg_aggregated_proof = blobs_sidecar.kzg_aggregated_proof assert len(expected_kzgs) == len(blobs) - for kzg, blob in zip(expected_kzgs, blobs): - assert blob_to_kzg(blob) == kzg + + # Generate random linear combination challenges + r = hash_to_bls_field([blobs, expected_kzgs]) + r_powers = compute_powers(r, len(expected_kzgs)) + + # Compute commitment to aggregated polynomial + aggregated_poly_commitment = lincomb(expected_kzgs, r_powers) + + # Create aggregated polynomial in evaluation form + aggregated_poly = vector_lincomb(blobs, r_powers) + + # Generate challenge `x` and evaluate the aggregated polynomial at `x` + x = hash_to_bls_field([aggregated_poly, aggregated_poly_commitment]) + y = evaluate_polynomial_in_evaluation_form(aggregated_poly, x) + + # Verify aggregated proof + assert verify_kzg_proof(aggregated_poly_commitment, x, y, kzg_aggregated_proof) ``` From 4be0b9d6bc7bc964ce15502759bc637882a757dc Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 13 Jun 2022 15:36:31 +0300 Subject: [PATCH 012/105] Add needed math/crypto functions to validate KZG aggregated proofs All code pretty much straight up copied from https://github.com/ethereum/EIPs/pull/5088 --- specs/eip4844/beacon-chain.md | 19 ++++---- specs/eip4844/validator.md | 81 +++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 8 deletions(-) diff --git a/specs/eip4844/beacon-chain.md b/specs/eip4844/beacon-chain.md index 6011e5832..ad8731736 100644 --- a/specs/eip4844/beacon-chain.md +++ b/specs/eip4844/beacon-chain.md @@ -54,6 +54,7 @@ This upgrade adds blobs to the beacon chain as part of EIP-4844. | `BLOB_TX_TYPE` | `uint8(0x05)` | | `FIELD_ELEMENTS_PER_BLOB` | `4096` | | `BLS_MODULUS` | `52435875175126190479447740508185965837690552500527637822603658699938581184513` | +| `ROOTS_OF_UNITY` | `Vector[BLSFieldElement, FIELD_ELEMENTS_PER_BLOB]` | ### Domain types @@ -110,15 +111,17 @@ KZG core functions. These are also defined in EIP-4844 execution specs. #### `blob_to_kzg` ```python +def lincomb(points: List[KZGCommitment], scalars: List[BLSFieldElement]) -> KZGCommitment: + """ + BLS multiscalar multiplication. This function can be optimized using Pippenger's algorithm and variants. + """ + r = bls.Z1 + for x, a in zip(points, scalars): + r = bls.add(r, bls.multiply(x, a)) + return r + def blob_to_kzg(blob: Blob) -> KZGCommitment: - computed_kzg = bls.Z1 - for value, point_kzg in zip(blob, KZG_SETUP_LAGRANGE): - assert value < BLS_MODULUS - computed_kzg = bls.add( - computed_kzg, - bls.multiply(point_kzg, value) - ) - return computed_kzg + return lincomb(blob, KZG_SETUP_LAGRANGE) ``` #### `kzg_to_versioned_hash` diff --git a/specs/eip4844/validator.md b/specs/eip4844/validator.md index aed38639f..c3dd83a41 100644 --- a/specs/eip4844/validator.md +++ b/specs/eip4844/validator.md @@ -54,6 +54,87 @@ def is_data_available(slot: Slot, beacon_block_root: Root, kzgs: Sequence[KZGCom ### `verify_blobs_sidecar` ```python +def hash_to_bls_field(x: Container) -> BLSFieldElement: + """ + This function is used to generate Fiat-Shamir challenges. The output is not uniform over the BLS field. + """ + return int.from_bytes(hash_tree_root(x), "little") % BLS_MODULUS + + +def compute_powers(x: BLSFieldElement, n: uint64) -> List[BLSFieldElement]: + current_power = 1 + powers = [] + for _ in range(n): + powers.append(BLSFieldElement(current_power)) + current_power = current_power * int(x) % BLS_MODULUS + return powers + + +def vector_lincomb(vectors: List[List[BLSFieldElement]], scalars: List[BLSFieldElement]) -> List[BLSFieldElement]: + """ + Given a list of vectors, compute the linear combination of each column with `scalars`, and return the resulting + vector. + """ + r = [0]*len(vectors[0]) + for v, a in zip(vectors, scalars): + for i, x in enumerate(v): + r[i] = (r[i] + a * x) % BLS_MODULUS + return [BLSFieldElement(x) for x in r] + + +def bls_modular_inverse(x: BLSFieldElement) -> BLSFieldElement: + """ + Compute the modular inverse of x using the eGCD algorithm + i.e. return y such that x * y % BLS_MODULUS == 1 and return 0 for x == 0 + """ + if x == 0: + return 0 + + lm, hm = 1, 0 + low, high = x % BLS_MODULUS, BLS_MODULUS + while low > 1: + r = high // low + nm, new = hm - lm * r, high - low * r + lm, low, hm, high = nm, new, lm, low + return lm % BLS_MODULUS + + +def div(x, y): + """Divide two field elements: `x` by `y`""" + return x * inv(y) % MODULUS + + +def verify_kzg_proof(polynomial_kzg: KZGCommitment, + x: BLSFieldElement, + y: BLSFieldElement, + quotient_kzg: KZGProof) -> bool: + """Verify KZG proof that `p(x) == y` where `p(x)` is the polynomial represented by `polynomial_kzg`""" + # Verify: P - y = Q * (X - x) + X_minus_x = bls.add(KZG_SETUP_G2[1], bls.multiply(bls.G2, BLS_MODULUS - x)) + P_minus_y = bls.add(polynomial_kzg, bls.multiply(bls.G1, BLS_MODULUS - y)) + return bls.pairing_check([ + [P_minus_y, bls.neg(bls.G2)], + [quotient_kzg, X_minus_x] + ]) + + +def evaluate_polynomial_in_evaluation_form(poly: List[BLSFieldElement], x: BLSFieldElement) -> BLSFieldElement: + """ + Evaluate a polynomial (in evaluation form) at an arbitrary point `x` + Uses the barycentric formula: + f(x) = (1 - x**WIDTH) / WIDTH * sum_(i=0)^WIDTH (f(DOMAIN[i]) * DOMAIN[i]) / (x - DOMAIN[i]) + """ + width = len(poly) + assert width == FIELD_ELEMENTS_PER_BLOB + inverse_width = bls_modular_inverse(width) + + for i in range(width): + r += div(poly[i] * ROOTS_OF_UNITY[i], (x - ROOTS_OF_UNITY[i]) ) + r = r * (pow(x, width, BLS_MODULUS) - 1) * inverse_width % BLS_MODULUS + + return r + + def verify_blobs_sidecar(slot: Slot, beacon_block_root: Root, expected_kzgs: Sequence[KZGCommitment], blobs_sidecar: BlobsSidecar): assert slot == blobs_sidecar.beacon_block_slot From 0b8ab23bd4fdb38a1bc38e0f242fc892cee71663 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Wed, 27 Apr 2022 09:50:27 +0200 Subject: [PATCH 013/105] Remove `fork_version` from `LightClientUpdate` The `fork_version` field in `LightClientUpdate` can be derived from the `update.signature_slot` value by consulting the locally configured fork schedule. The light client already needs access to the fork schedule to determine the `GeneralizedIndex` values used for merkle proofs, and the memory layouts of the structures (including `LightClientUpdate`). The `fork_version` itself is network dependent and doesn't reveal that info. --- specs/altair/beacon-chain.md | 13 +++++++ specs/altair/sync-protocol.md | 5 ++- specs/bellatrix/beacon-chain.md | 15 ++++++++ specs/capella/beacon-chain.md | 20 +++++++++++ .../altair/unittests/test_sync_protocol.py | 33 +++++++++++++----- .../eth2spec/test/helpers/light_client.py | 34 +++++++++++++++++-- 6 files changed, 107 insertions(+), 13 deletions(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index a14ddb4f6..fadce93ef 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -28,6 +28,7 @@ - [Helper functions](#helper-functions) - [Crypto](#crypto) - [Misc](#misc-1) + - [`compute_fork_version`](#compute_fork_version) - [`add_flag`](#add_flag) - [`has_flag`](#has_flag) - [Beacon state accessors](#beacon-state-accessors) @@ -226,6 +227,18 @@ the functionality described in those documents. ### Misc +#### `compute_fork_version` + +```python +def compute_fork_version(epoch: Epoch) -> Version: + """ + Return the fork version at the given ``epoch``. + """ + if epoch >= ALTAIR_FORK_EPOCH: + return ALTAIR_FORK_VERSION + return GENESIS_FORK_VERSION +``` + #### `add_flag` ```python diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index 1aa9316d3..a3b10efa2 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -71,8 +71,6 @@ class LightClientUpdate(Container): finality_branch: Vector[Bytes32, floorlog2(FINALIZED_ROOT_INDEX)] # Sync committee aggregate signature sync_aggregate: SyncAggregate - # Fork version for the aggregate signature - fork_version: Version # Slot at which the aggregate signature was created (untrusted) signature_slot: Slot ``` @@ -211,7 +209,8 @@ def validate_light_client_update(store: LightClientStore, pubkey for (bit, pubkey) in zip(sync_aggregate.sync_committee_bits, sync_committee.pubkeys) if bit ] - domain = compute_domain(DOMAIN_SYNC_COMMITTEE, update.fork_version, genesis_validators_root) + fork_version = compute_fork_version(compute_epoch_at_slot(update.signature_slot)) + domain = compute_domain(DOMAIN_SYNC_COMMITTEE, fork_version, genesis_validators_root) signing_root = compute_signing_root(update.attested_header, domain) assert bls.FastAggregateVerify(participant_pubkeys, signing_root, sync_aggregate.sync_committee_signature) ``` diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index b37a8ab71..f92c4cbf5 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -29,6 +29,7 @@ - [`is_execution_enabled`](#is_execution_enabled) - [Misc](#misc) - [`compute_timestamp_at_slot`](#compute_timestamp_at_slot) + - [Modified `compute_fork_version`](#modified-compute_fork_version) - [Beacon state accessors](#beacon-state-accessors) - [Modified `get_inactivity_penalty_deltas`](#modified-get_inactivity_penalty_deltas) - [Beacon state mutators](#beacon-state-mutators) @@ -242,6 +243,20 @@ def compute_timestamp_at_slot(state: BeaconState, slot: Slot) -> uint64: return uint64(state.genesis_time + slots_since_genesis * SECONDS_PER_SLOT) ``` +#### Modified `compute_fork_version` + +```python +def compute_fork_version(epoch: Epoch) -> Version: + """ + Return the fork version at the given ``epoch``. + """ + if epoch >= BELLATRIX_FORK_EPOCH: + return BELLATRIX_FORK_VERSION + if epoch >= ALTAIR_FORK_EPOCH: + return ALTAIR_FORK_VERSION + return GENESIS_FORK_VERSION +``` + ### Beacon state accessors #### Modified `get_inactivity_penalty_deltas` diff --git a/specs/capella/beacon-chain.md b/specs/capella/beacon-chain.md index da67a8fcb..e2b47dd1f 100644 --- a/specs/capella/beacon-chain.md +++ b/specs/capella/beacon-chain.md @@ -34,6 +34,8 @@ - [`has_eth1_withdrawal_credential`](#has_eth1_withdrawal_credential) - [`is_fully_withdrawable_validator`](#is_fully_withdrawable_validator) - [`is_partially_withdrawable_validator`](#is_partially_withdrawable_validator) + - [Misc](#misc-1) + - [Modified `compute_fork_version`](#modified-compute_fork_version) - [Beacon chain state transition function](#beacon-chain-state-transition-function) - [Epoch processing](#epoch-processing) - [Full withdrawals](#full-withdrawals) @@ -318,6 +320,24 @@ def is_partially_withdrawable_validator(validator: Validator, balance: Gwei) -> return has_eth1_withdrawal_credential(validator) and has_max_effective_balance and has_excess_balance ``` +### Misc + +#### Modified `compute_fork_version` + +```python +def compute_fork_version(epoch: Epoch) -> Version: + """ + Return the fork version at the given ``epoch``. + """ + if epoch >= CAPELLA_FORK_EPOCH: + return CAPELLA_FORK_VERSION + if epoch >= BELLATRIX_FORK_EPOCH: + return BELLATRIX_FORK_VERSION + if epoch >= ALTAIR_FORK_EPOCH: + return ALTAIR_FORK_VERSION + return GENESIS_FORK_VERSION +``` + ## Beacon chain state transition function ### Epoch processing diff --git a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py index 6411ecb6a..4acfa6424 100644 --- a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py +++ b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py @@ -16,6 +16,7 @@ from eth2spec.test.helpers.constants import MINIMAL from eth2spec.test.helpers.light_client import ( get_sync_aggregate, initialize_light_client_store, + override_config_fork_epochs, ) from eth2spec.test.helpers.state import ( next_slots, @@ -27,6 +28,9 @@ from eth2spec.test.helpers.merkle import build_proof @with_altair_and_later @spec_state_test def test_process_light_client_update_not_timeout(spec, state): + old_config = spec.config + override_config_fork_epochs(spec, state) + store = initialize_light_client_store(spec, state) # Block at slot 1 doesn't increase sync committee period, so it won't force update store.finalized_header @@ -41,7 +45,7 @@ def test_process_light_client_update_not_timeout(spec, state): ) # Sync committee signing the block_header - sync_aggregate, fork_version, signature_slot = get_sync_aggregate(spec, state, block_header) + sync_aggregate, signature_slot = get_sync_aggregate(spec, state, block_header) next_sync_committee_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.NEXT_SYNC_COMMITTEE_INDEX))] # Ensure that finality checkpoint is genesis @@ -57,7 +61,6 @@ def test_process_light_client_update_not_timeout(spec, state): finalized_header=finality_header, finality_branch=finality_branch, sync_aggregate=sync_aggregate, - fork_version=fork_version, signature_slot=signature_slot, ) @@ -70,11 +73,16 @@ def test_process_light_client_update_not_timeout(spec, state): assert store.finalized_header == pre_store.finalized_header assert store.best_valid_update == update + spec.config = old_config + @with_altair_and_later @spec_state_test @with_presets([MINIMAL], reason="too slow") def test_process_light_client_update_at_period_boundary(spec, state): + old_config = spec.config + override_config_fork_epochs(spec, state) + store = initialize_light_client_store(spec, state) # Forward to slot before next sync committee period so that next block is final one in period @@ -94,7 +102,7 @@ def test_process_light_client_update_at_period_boundary(spec, state): ) # Sync committee signing the block_header - sync_aggregate, fork_version, signature_slot = get_sync_aggregate(spec, state, block_header) + sync_aggregate, signature_slot = get_sync_aggregate(spec, state, block_header) next_sync_committee_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.NEXT_SYNC_COMMITTEE_INDEX))] # Finality is unchanged @@ -108,7 +116,6 @@ def test_process_light_client_update_at_period_boundary(spec, state): finalized_header=finality_header, finality_branch=finality_branch, sync_aggregate=sync_aggregate, - fork_version=fork_version, signature_slot=signature_slot, ) @@ -121,11 +128,16 @@ def test_process_light_client_update_at_period_boundary(spec, state): assert store.best_valid_update == update assert store.finalized_header == pre_store.finalized_header + spec.config = old_config + @with_altair_and_later @spec_state_test @with_presets([MINIMAL], reason="too slow") def test_process_light_client_update_timeout(spec, state): + old_config = spec.config + override_config_fork_epochs(spec, state) + store = initialize_light_client_store(spec, state) # Forward to next sync committee period @@ -145,7 +157,7 @@ def test_process_light_client_update_timeout(spec, state): ) # Sync committee signing the block_header - sync_aggregate, fork_version, signature_slot = get_sync_aggregate(spec, state, block_header) + sync_aggregate, signature_slot = get_sync_aggregate(spec, state, block_header) # Sync committee is updated next_sync_committee_branch = build_proof(state.get_backing(), spec.NEXT_SYNC_COMMITTEE_INDEX) @@ -160,7 +172,6 @@ def test_process_light_client_update_timeout(spec, state): finalized_header=finality_header, finality_branch=finality_branch, sync_aggregate=sync_aggregate, - fork_version=fork_version, signature_slot=signature_slot, ) @@ -173,11 +184,16 @@ def test_process_light_client_update_timeout(spec, state): assert store.best_valid_update == update assert store.finalized_header == pre_store.finalized_header + spec.config = old_config + @with_altair_and_later @spec_state_test @with_presets([MINIMAL], reason="too slow") def test_process_light_client_update_finality_updated(spec, state): + old_config = spec.config + override_config_fork_epochs(spec, state) + store = initialize_light_client_store(spec, state) # Change finality @@ -211,7 +227,7 @@ def test_process_light_client_update_finality_updated(spec, state): ) # Sync committee signing the block_header - sync_aggregate, fork_version, signature_slot = get_sync_aggregate(spec, state, block_header) + sync_aggregate, signature_slot = get_sync_aggregate(spec, state, block_header) update = spec.LightClientUpdate( attested_header=block_header, @@ -220,7 +236,6 @@ def test_process_light_client_update_finality_updated(spec, state): finalized_header=finalized_block_header, finality_branch=finality_branch, sync_aggregate=sync_aggregate, - fork_version=fork_version, signature_slot=signature_slot, ) @@ -230,3 +245,5 @@ def test_process_light_client_update_finality_updated(spec, state): assert store.optimistic_header == update.attested_header assert store.finalized_header == update.finalized_header assert store.best_valid_update is None + + spec.config = old_config diff --git a/tests/core/pyspec/eth2spec/test/helpers/light_client.py b/tests/core/pyspec/eth2spec/test/helpers/light_client.py index b35c6c687..6868cad86 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/light_client.py +++ b/tests/core/pyspec/eth2spec/test/helpers/light_client.py @@ -1,3 +1,5 @@ +from copy import deepcopy + from eth2spec.test.helpers.state import ( transition_to, ) @@ -7,6 +9,35 @@ from eth2spec.test.helpers.sync_committee import ( ) +def override_config_fork_epochs(spec, state): + # Test framework adjusts state fork but leaves spec config constants inconsistent + config_overrides = {} + if state.fork.current_version == spec.config.GENESIS_FORK_VERSION: + pass + elif state.fork.current_version == spec.config.ALTAIR_FORK_VERSION: + config_overrides['ALTAIR_FORK_EPOCH'] = spec.GENESIS_EPOCH + elif state.fork.current_version == spec.config.BELLATRIX_FORK_VERSION: + config_overrides['ALTAIR_FORK_EPOCH'] = spec.GENESIS_EPOCH + config_overrides['BELLATRIX_FORK_EPOCH'] = spec.GENESIS_EPOCH + elif state.fork.current_version == spec.config.CAPELLA_FORK_VERSION: + config_overrides['ALTAIR_FORK_EPOCH'] = spec.GENESIS_EPOCH + config_overrides['BELLATRIX_FORK_EPOCH'] = spec.GENESIS_EPOCH + config_overrides['CAPELLA_FORK_EPOCH'] = spec.GENESIS_EPOCH + elif state.fork.current_version == spec.config.SHARDING_FORK_VERSION: + config_overrides['ALTAIR_FORK_EPOCH'] = spec.GENESIS_EPOCH + config_overrides['BELLATRIX_FORK_EPOCH'] = spec.GENESIS_EPOCH + config_overrides['CAPELLA_FORK_EPOCH'] = spec.GENESIS_EPOCH + config_overrides['SHARDING_FORK_EPOCH'] = spec.GENESIS_EPOCH + else: + assert False + + tmp_config = deepcopy(spec.config._asdict()) + tmp_config.update(config_overrides) + config_types = spec.Configuration.__annotations__ + test_config = {k: config_types[k](v) for k, v in tmp_config.items()} + spec.config = spec.Configuration(**test_config) + + def initialize_light_client_store(spec, state): return spec.LightClientStore( finalized_header=spec.BeaconBlockHeader(), @@ -45,5 +76,4 @@ def get_sync_aggregate(spec, state, block_header, signature_slot=None): sync_committee_bits=sync_committee_bits, sync_committee_signature=sync_committee_signature, ) - fork_version = signature_state.fork.current_version - return sync_aggregate, fork_version, signature_slot + return sync_aggregate, signature_slot From 8a70295a84f1774bb1d5aec8130b14a0ee02008d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 20 Jun 2022 21:26:41 +0800 Subject: [PATCH 014/105] Fix ToC --- specs/eip4844/beacon-chain.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/specs/eip4844/beacon-chain.md b/specs/eip4844/beacon-chain.md index ad8731736..2aa5723e6 100644 --- a/specs/eip4844/beacon-chain.md +++ b/specs/eip4844/beacon-chain.md @@ -20,6 +20,7 @@ - [`BeaconBlockBody`](#beaconblockbody) - [Helper functions](#helper-functions) - [KZG core](#kzg-core) + - [`lincomb`](#lincomb) - [`blob_to_kzg`](#blob_to_kzg) - [`kzg_to_versioned_hash`](#kzg_to_versioned_hash) - [Misc](#misc) @@ -108,7 +109,7 @@ class BeaconBlockBody(Container): KZG core functions. These are also defined in EIP-4844 execution specs. -#### `blob_to_kzg` +#### `lincomb` ```python def lincomb(points: List[KZGCommitment], scalars: List[BLSFieldElement]) -> KZGCommitment: @@ -119,7 +120,11 @@ def lincomb(points: List[KZGCommitment], scalars: List[BLSFieldElement]) -> KZGC for x, a in zip(points, scalars): r = bls.add(r, bls.multiply(x, a)) return r +``` +#### `blob_to_kzg` + +```python def blob_to_kzg(blob: Blob) -> KZGCommitment: return lincomb(blob, KZG_SETUP_LAGRANGE) ``` From 534a9d1d958ff4fcb4db2f6be808b9892b009d4b Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 22 Jun 2022 15:13:41 +0300 Subject: [PATCH 015/105] Move EIP-4844 cryptography code to its own file --- specs/eip4844/beacon-chain.md | 50 +------- specs/eip4844/polynomial-commitments.md | 153 ++++++++++++++++++++++++ specs/eip4844/validator.md | 53 -------- 3 files changed, 155 insertions(+), 101 deletions(-) create mode 100644 specs/eip4844/polynomial-commitments.md diff --git a/specs/eip4844/beacon-chain.md b/specs/eip4844/beacon-chain.md index 2aa5723e6..537b294c7 100644 --- a/specs/eip4844/beacon-chain.md +++ b/specs/eip4844/beacon-chain.md @@ -12,18 +12,13 @@ - [Custom types](#custom-types) - [Constants](#constants) - [Domain types](#domain-types) -- [Preset](#preset) - - [Trusted setup](#trusted-setup) - [Configuration](#configuration) - [Containers](#containers) - [Extended containers](#extended-containers) - [`BeaconBlockBody`](#beaconblockbody) - [Helper functions](#helper-functions) - - [KZG core](#kzg-core) - - [`lincomb`](#lincomb) - - [`blob_to_kzg`](#blob_to_kzg) - - [`kzg_to_versioned_hash`](#kzg_to_versioned_hash) - [Misc](#misc) + - [`kzg_to_versioned_hash`](#kzg_to_versioned_hash) - [`tx_peek_blob_versioned_hashes`](#tx_peek_blob_versioned_hashes) - [`verify_kzgs_against_transactions`](#verify_kzgs_against_transactions) - [Beacon chain state transition function](#beacon-chain-state-transition-function) @@ -42,11 +37,8 @@ This upgrade adds blobs to the beacon chain as part of EIP-4844. | Name | SSZ equivalent | Description | | - | - | - | -| `BLSFieldElement` | `uint256` | `x < BLS_MODULUS` | | `Blob` | `Vector[BLSFieldElement, FIELD_ELEMENTS_PER_BLOB]` | | | `VersionedHash` | `Bytes32` | | -| `KZGCommitment` | `Bytes48` | Same as BLS standard "is valid pubkey" check but also allows `0x00..00` for point-at-infinity | -| `KZGProof` | Bytes48 | Same as for `KZGCommitment` | ## Constants @@ -54,8 +46,6 @@ This upgrade adds blobs to the beacon chain as part of EIP-4844. | - | - | | `BLOB_TX_TYPE` | `uint8(0x05)` | | `FIELD_ELEMENTS_PER_BLOB` | `4096` | -| `BLS_MODULUS` | `52435875175126190479447740508185965837690552500527637822603658699938581184513` | -| `ROOTS_OF_UNITY` | `Vector[BLSFieldElement, FIELD_ELEMENTS_PER_BLOB]` | ### Domain types @@ -63,18 +53,6 @@ This upgrade adds blobs to the beacon chain as part of EIP-4844. | - | - | | `DOMAIN_BLOBS_SIDECAR` | `DomainType('0x0a000000')` | -## Preset - -### Trusted setup - -The trusted setup is part of the preset: during testing a `minimal` insecure variant may be used, -but reusing the `mainnet` settings in public networks is a critical security requirement. - -| Name | Value | -| - | - | -| `KZG_SETUP_G2` | `Vector[G2Point, FIELD_ELEMENTS_PER_BLOB]`, contents TBD | -| `KZG_SETUP_LAGRANGE` | `Vector[KZGCommitment, FIELD_ELEMENTS_PER_BLOB]`, contents TBD | - ## Configuration @@ -105,29 +83,7 @@ class BeaconBlockBody(Container): ## Helper functions -### KZG core - -KZG core functions. These are also defined in EIP-4844 execution specs. - -#### `lincomb` - -```python -def lincomb(points: List[KZGCommitment], scalars: List[BLSFieldElement]) -> KZGCommitment: - """ - BLS multiscalar multiplication. This function can be optimized using Pippenger's algorithm and variants. - """ - r = bls.Z1 - for x, a in zip(points, scalars): - r = bls.add(r, bls.multiply(x, a)) - return r -``` - -#### `blob_to_kzg` - -```python -def blob_to_kzg(blob: Blob) -> KZGCommitment: - return lincomb(blob, KZG_SETUP_LAGRANGE) -``` +### Misc #### `kzg_to_versioned_hash` @@ -136,8 +92,6 @@ def kzg_to_versioned_hash(kzg: KZGCommitment) -> VersionedHash: return BLOB_COMMITMENT_VERSION_KZG + hash(kzg)[1:] ``` -### Misc - #### `tx_peek_blob_versioned_hashes` This function retrieves the hashes from the `SignedBlobTransaction` as defined in EIP-4844, using SSZ offsets. diff --git a/specs/eip4844/polynomial-commitments.md b/specs/eip4844/polynomial-commitments.md new file mode 100644 index 000000000..df5aad146 --- /dev/null +++ b/specs/eip4844/polynomial-commitments.md @@ -0,0 +1,153 @@ +# EIP-4844 -- Polynomial Commitments + +## Table of contents + + + + + +- [Introduction](#introduction) +- [Custom types](#custom-types) +- [Constants](#constants) +- [Preset](#preset) + - [Trusted setup](#trusted-setup) +- [Helper functions](#helper-functions) + - [BLS12-381 helpers](#bls12-381-helpers) + - [`bls_modular_inverse`](#bls_modular_inverse) + - [`div`](#div) + - [`lincomb`](#lincomb) + - [KZG](#kzg) + - [`blob_to_kzg`](#blob_to_kzg) + - [`verify_kzg_proof`](#verify_kzg_proof) + - [Polynomials](#polynomials) + - [`evaluate_polynomial_in_evaluation_form`](#evaluate_polynomial_in_evaluation_form) + + + + + +## Introduction + +This document specifies basic polynomial operations and KZG polynomial commitment operations as they are needed for the EIP-4844 specification. The implementations are not optimized for performance, but readability. All practical implementations should optimize the polynomial operations. + +## Custom types + +| Name | SSZ equivalent | Description | +| - | - | - | +| `BLSFieldElement` | `uint256` | `x < BLS_MODULUS` | +| `KZGCommitment` | `Bytes48` | Same as BLS standard "is valid pubkey" check but also allows `0x00..00` for point-at-infinity | +| `KZGProof` | `Bytes48` | Same as for `KZGCommitment` | + +## Constants + +| Name | Value | Notes | +| - | - | - | +| `BLS_MODULUS` | `52435875175126190479447740508185965837690552500527637822603658699938581184513` | Scalar field modulus of BLS12-381 | +| `ROOTS_OF_UNITY` | `Vector[BLSFieldElement, FIELD_ELEMENTS_PER_BLOB]` | Roots of unity of order FIELD_ELEMENTS_PER_BLOB over the BLS12-381 field | + +## Preset + +### Trusted setup + +The trusted setup is part of the preset: during testing a `minimal` insecure variant may be used, +but reusing the `mainnet` settings in public networks is a critical security requirement. + +| Name | Value | +| - | - | +| `KZG_SETUP_G2` | `Vector[G2Point, FIELD_ELEMENTS_PER_BLOB]`, contents TBD | +| `KZG_SETUP_LAGRANGE` | `Vector[KZGCommitment, FIELD_ELEMENTS_PER_BLOB]`, contents TBD | + +## Helper functions + +### BLS12-381 helpers + +#### `bls_modular_inverse` + +```python +def bls_modular_inverse(x: BLSFieldElement) -> BLSFieldElement: + """ + Compute the modular inverse of x using the eGCD algorithm + i.e. return y such that x * y % BLS_MODULUS == 1 and return 0 for x == 0 + """ + if x == 0: + return 0 + + lm, hm = 1, 0 + low, high = x % BLS_MODULUS, BLS_MODULUS + while low > 1: + r = high // low + nm, new = hm - lm * r, high - low * r + lm, low, hm, high = nm, new, lm, low + return lm % BLS_MODULUS +``` + +#### `div` + +```python +def div(x, y): + """Divide two field elements: `x` by `y`""" + return x * inv(y) % BLS_MODULUS +``` + +#### `lincomb` + +```python +def lincomb(points: List[KZGCommitment], scalars: List[BLSFieldElement]) -> KZGCommitment: + """ + BLS multiscalar multiplication. This function can be optimized using Pippenger's algorithm and variants. + """ + r = bls.Z1 + for x, a in zip(points, scalars): + r = bls.add(r, bls.multiply(x, a)) + return r +``` + +### KZG + +KZG core functions. These are also defined in EIP-4844 execution specs. + +#### `blob_to_kzg` + +```python +def blob_to_kzg(blob: Blob) -> KZGCommitment: + return lincomb(KZG_SETUP_LAGRANGE, blob) +``` + +#### `verify_kzg_proof` + +```python +def verify_kzg_proof(polynomial_kzg: KZGCommitment, + x: BLSFieldElement, + y: BLSFieldElement, + quotient_kzg: KZGProof) -> bool: + """Verify KZG proof that `p(x) == y` where `p(x)` is the polynomial represented by `polynomial_kzg`""" + # Verify: P - y = Q * (X - x) + X_minus_x = bls.add(KZG_SETUP_G2[1], bls.multiply(bls.G2, BLS_MODULUS - x)) + P_minus_y = bls.add(polynomial_kzg, bls.multiply(bls.G1, BLS_MODULUS - y)) + return bls.pairing_check([ + [P_minus_y, bls.neg(bls.G2)], + [quotient_kzg, X_minus_x] + ]) +``` + +### Polynomials + +#### `evaluate_polynomial_in_evaluation_form` + +```python +def evaluate_polynomial_in_evaluation_form(poly: List[BLSFieldElement], x: BLSFieldElement) -> BLSFieldElement: + """ + Evaluate a polynomial (in evaluation form) at an arbitrary point `x` + Uses the barycentric formula: + f(x) = (1 - x**WIDTH) / WIDTH * sum_(i=0)^WIDTH (f(DOMAIN[i]) * DOMAIN[i]) / (x - DOMAIN[i]) + """ + width = len(poly) + assert width == FIELD_ELEMENTS_PER_BLOB + inverse_width = bls_modular_inverse(width) + + for i in range(width): + r += div(poly[i] * ROOTS_OF_UNITY[i], (x - ROOTS_OF_UNITY[i]) ) + r = r * (pow(x, width, BLS_MODULUS) - 1) * inverse_width % BLS_MODULUS + + return r +``` diff --git a/specs/eip4844/validator.md b/specs/eip4844/validator.md index c3dd83a41..d636b39e4 100644 --- a/specs/eip4844/validator.md +++ b/specs/eip4844/validator.md @@ -82,59 +82,6 @@ def vector_lincomb(vectors: List[List[BLSFieldElement]], scalars: List[BLSFieldE return [BLSFieldElement(x) for x in r] -def bls_modular_inverse(x: BLSFieldElement) -> BLSFieldElement: - """ - Compute the modular inverse of x using the eGCD algorithm - i.e. return y such that x * y % BLS_MODULUS == 1 and return 0 for x == 0 - """ - if x == 0: - return 0 - - lm, hm = 1, 0 - low, high = x % BLS_MODULUS, BLS_MODULUS - while low > 1: - r = high // low - nm, new = hm - lm * r, high - low * r - lm, low, hm, high = nm, new, lm, low - return lm % BLS_MODULUS - - -def div(x, y): - """Divide two field elements: `x` by `y`""" - return x * inv(y) % MODULUS - - -def verify_kzg_proof(polynomial_kzg: KZGCommitment, - x: BLSFieldElement, - y: BLSFieldElement, - quotient_kzg: KZGProof) -> bool: - """Verify KZG proof that `p(x) == y` where `p(x)` is the polynomial represented by `polynomial_kzg`""" - # Verify: P - y = Q * (X - x) - X_minus_x = bls.add(KZG_SETUP_G2[1], bls.multiply(bls.G2, BLS_MODULUS - x)) - P_minus_y = bls.add(polynomial_kzg, bls.multiply(bls.G1, BLS_MODULUS - y)) - return bls.pairing_check([ - [P_minus_y, bls.neg(bls.G2)], - [quotient_kzg, X_minus_x] - ]) - - -def evaluate_polynomial_in_evaluation_form(poly: List[BLSFieldElement], x: BLSFieldElement) -> BLSFieldElement: - """ - Evaluate a polynomial (in evaluation form) at an arbitrary point `x` - Uses the barycentric formula: - f(x) = (1 - x**WIDTH) / WIDTH * sum_(i=0)^WIDTH (f(DOMAIN[i]) * DOMAIN[i]) / (x - DOMAIN[i]) - """ - width = len(poly) - assert width == FIELD_ELEMENTS_PER_BLOB - inverse_width = bls_modular_inverse(width) - - for i in range(width): - r += div(poly[i] * ROOTS_OF_UNITY[i], (x - ROOTS_OF_UNITY[i]) ) - r = r * (pow(x, width, BLS_MODULUS) - 1) * inverse_width % BLS_MODULUS - - return r - - def verify_blobs_sidecar(slot: Slot, beacon_block_root: Root, expected_kzgs: Sequence[KZGCommitment], blobs_sidecar: BlobsSidecar): assert slot == blobs_sidecar.beacon_block_slot From e7e520791d27dad6c82d63611d89b8ddcf3b6aac Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 22 Jun 2022 15:19:24 +0300 Subject: [PATCH 016/105] Use Python's pow() in bls_modular_inverse(). --- specs/eip4844/polynomial-commitments.md | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/specs/eip4844/polynomial-commitments.md b/specs/eip4844/polynomial-commitments.md index df5aad146..9558641c6 100644 --- a/specs/eip4844/polynomial-commitments.md +++ b/specs/eip4844/polynomial-commitments.md @@ -66,19 +66,10 @@ but reusing the `mainnet` settings in public networks is a critical security req ```python def bls_modular_inverse(x: BLSFieldElement) -> BLSFieldElement: """ - Compute the modular inverse of x using the eGCD algorithm + Compute the modular inverse of x i.e. return y such that x * y % BLS_MODULUS == 1 and return 0 for x == 0 """ - if x == 0: - return 0 - - lm, hm = 1, 0 - low, high = x % BLS_MODULUS, BLS_MODULUS - while low > 1: - r = high // low - nm, new = hm - lm * r, high - low * r - lm, low, hm, high = nm, new, lm, low - return lm % BLS_MODULUS + return pow(x, -1, BLS_MODULUS) if x != 0 else 0 ``` #### `div` From 0ab280d396ca6566deda294466e92b87f34a5d59 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 23 Jun 2022 18:40:09 +0800 Subject: [PATCH 017/105] minor suggestions --- specs/eip4844/polynomial-commitments.md | 8 +++++--- specs/eip4844/validator.md | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/specs/eip4844/polynomial-commitments.md b/specs/eip4844/polynomial-commitments.md index 9558641c6..fc2339659 100644 --- a/specs/eip4844/polynomial-commitments.md +++ b/specs/eip4844/polynomial-commitments.md @@ -75,7 +75,7 @@ def bls_modular_inverse(x: BLSFieldElement) -> BLSFieldElement: #### `div` ```python -def div(x, y): +def div(x: BLSFieldElement, y: BLSFieldElement) -> BLSFieldElement: """Divide two field elements: `x` by `y`""" return x * inv(y) % BLS_MODULUS ``` @@ -111,7 +111,9 @@ def verify_kzg_proof(polynomial_kzg: KZGCommitment, x: BLSFieldElement, y: BLSFieldElement, quotient_kzg: KZGProof) -> bool: - """Verify KZG proof that `p(x) == y` where `p(x)` is the polynomial represented by `polynomial_kzg`""" + """ + Verify KZG proof that ``p(x) == y`` where ``p(x)`` is the polynomial represented by ``polynomial_kzg``. + """ # Verify: P - y = Q * (X - x) X_minus_x = bls.add(KZG_SETUP_G2[1], bls.multiply(bls.G2, BLS_MODULUS - x)) P_minus_y = bls.add(polynomial_kzg, bls.multiply(bls.G1, BLS_MODULUS - y)) @@ -137,7 +139,7 @@ def evaluate_polynomial_in_evaluation_form(poly: List[BLSFieldElement], x: BLSFi inverse_width = bls_modular_inverse(width) for i in range(width): - r += div(poly[i] * ROOTS_OF_UNITY[i], (x - ROOTS_OF_UNITY[i]) ) + r += div(poly[i] * ROOTS_OF_UNITY[i], (x - ROOTS_OF_UNITY[i])) r = r * (pow(x, width, BLS_MODULUS) - 1) * inverse_width % BLS_MODULUS return r diff --git a/specs/eip4844/validator.md b/specs/eip4844/validator.md index d636b39e4..fb96282c0 100644 --- a/specs/eip4844/validator.md +++ b/specs/eip4844/validator.md @@ -83,7 +83,7 @@ def vector_lincomb(vectors: List[List[BLSFieldElement]], scalars: List[BLSFieldE def verify_blobs_sidecar(slot: Slot, beacon_block_root: Root, - expected_kzgs: Sequence[KZGCommitment], blobs_sidecar: BlobsSidecar): + expected_kzgs: Sequence[KZGCommitment], blobs_sidecar: BlobsSidecar) -> None: assert slot == blobs_sidecar.beacon_block_slot assert beacon_block_root == blobs_sidecar.beacon_block_root blobs = blobs_sidecar.blobs From 94da861d9c1b6b7d1424337c187aa5d924be0266 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 23 Jun 2022 18:44:37 +0800 Subject: [PATCH 018/105] Fix ToC --- specs/eip4844/validator.md | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/specs/eip4844/validator.md b/specs/eip4844/validator.md index fb96282c0..fba328df1 100644 --- a/specs/eip4844/validator.md +++ b/specs/eip4844/validator.md @@ -12,6 +12,9 @@ - [Prerequisites](#prerequisites) - [Helpers](#helpers) - [`is_data_available`](#is_data_available) + - [`hash_to_bls_field`](#hash_to_bls_field) + - [`compute_powers`](#compute_powers) + - [`vector_lincomb`](#vector_lincomb) - [`verify_blobs_sidecar`](#verify_blobs_sidecar) - [Beacon chain responsibilities](#beacon-chain-responsibilities) - [Block proposal](#block-proposal) @@ -40,7 +43,7 @@ Please see related Beacon Chain doc before continuing and use them as a referenc The implementation of `is_data_available` is meant to change with later sharding upgrades. Initially, it requires every verifying actor to retrieve the matching `BlobsSidecar`, -and verify the sidecar with `verify_blobs`. +and verify the sidecar with `verify_blobs_sidecar`. Without the sidecar the block may be processed further optimistically, but MUST NOT be considered valid until a valid `BlobsSidecar` has been downloaded. @@ -51,7 +54,7 @@ def is_data_available(slot: Slot, beacon_block_root: Root, kzgs: Sequence[KZGCom verify_blobs_sidecar(slot, beacon_block_root, kzgs, sidecar) ``` -### `verify_blobs_sidecar` +### `hash_to_bls_field` ```python def hash_to_bls_field(x: Container) -> BLSFieldElement: @@ -59,8 +62,10 @@ def hash_to_bls_field(x: Container) -> BLSFieldElement: This function is used to generate Fiat-Shamir challenges. The output is not uniform over the BLS field. """ return int.from_bytes(hash_tree_root(x), "little") % BLS_MODULUS +``` - +### `compute_powers` +```python def compute_powers(x: BLSFieldElement, n: uint64) -> List[BLSFieldElement]: current_power = 1 powers = [] @@ -68,8 +73,11 @@ def compute_powers(x: BLSFieldElement, n: uint64) -> List[BLSFieldElement]: powers.append(BLSFieldElement(current_power)) current_power = current_power * int(x) % BLS_MODULUS return powers +``` +### `vector_lincomb` +```python def vector_lincomb(vectors: List[List[BLSFieldElement]], scalars: List[BLSFieldElement]) -> List[BLSFieldElement]: """ Given a list of vectors, compute the linear combination of each column with `scalars`, and return the resulting @@ -80,8 +88,11 @@ def vector_lincomb(vectors: List[List[BLSFieldElement]], scalars: List[BLSFieldE for i, x in enumerate(v): r[i] = (r[i] + a * x) % BLS_MODULUS return [BLSFieldElement(x) for x in r] +``` +### `verify_blobs_sidecar` +```python def verify_blobs_sidecar(slot: Slot, beacon_block_root: Root, expected_kzgs: Sequence[KZGCommitment], blobs_sidecar: BlobsSidecar) -> None: assert slot == blobs_sidecar.beacon_block_slot @@ -108,7 +119,6 @@ def verify_blobs_sidecar(slot: Slot, beacon_block_root: Root, assert verify_kzg_proof(aggregated_poly_commitment, x, y, kzg_aggregated_proof) ``` - ## Beacon chain responsibilities All validator responsibilities remain unchanged other than those noted below. @@ -175,4 +185,3 @@ The validator MUST hold on to blobs for `MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS` to ensure the data-availability of these blobs throughout the network. After `MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS` nodes MAY prune the blobs and/or stop serving them. - From 840733ffd2994d2ef361175fe98b9cff4e7403e5 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Thu, 23 Jun 2022 20:55:18 +0200 Subject: [PATCH 019/105] Move `compute_fork_version` to `fork.md` --- specs/altair/beacon-chain.md | 13 ------------- specs/altair/fork.md | 19 +++++++++++++++++++ specs/bellatrix/beacon-chain.md | 15 --------------- specs/bellatrix/fork.md | 21 +++++++++++++++++++++ specs/capella/beacon-chain.md | 20 -------------------- specs/capella/fork.md | 23 +++++++++++++++++++++++ 6 files changed, 63 insertions(+), 48 deletions(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index fadce93ef..a14ddb4f6 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -28,7 +28,6 @@ - [Helper functions](#helper-functions) - [Crypto](#crypto) - [Misc](#misc-1) - - [`compute_fork_version`](#compute_fork_version) - [`add_flag`](#add_flag) - [`has_flag`](#has_flag) - [Beacon state accessors](#beacon-state-accessors) @@ -227,18 +226,6 @@ the functionality described in those documents. ### Misc -#### `compute_fork_version` - -```python -def compute_fork_version(epoch: Epoch) -> Version: - """ - Return the fork version at the given ``epoch``. - """ - if epoch >= ALTAIR_FORK_EPOCH: - return ALTAIR_FORK_VERSION - return GENESIS_FORK_VERSION -``` - #### `add_flag` ```python diff --git a/specs/altair/fork.md b/specs/altair/fork.md index 6228022b8..bf8499a21 100644 --- a/specs/altair/fork.md +++ b/specs/altair/fork.md @@ -7,6 +7,9 @@ - [Introduction](#introduction) - [Configuration](#configuration) +- [Helper functions](#helper-functions) + - [Misc](#misc) + - [`compute_fork_version`](#compute_fork_version) - [Fork to Altair](#fork-to-altair) - [Fork trigger](#fork-trigger) - [Upgrading the state](#upgrading-the-state) @@ -26,6 +29,22 @@ Warning: this configuration is not definitive. | `ALTAIR_FORK_VERSION` | `Version('0x01000000')` | | `ALTAIR_FORK_EPOCH` | `Epoch(74240)` (Oct 27, 2021, 10:56:23am UTC) | +## Helper functions + +### Misc + +#### `compute_fork_version` + +```python +def compute_fork_version(epoch: Epoch) -> Version: + """ + Return the fork version at the given ``epoch``. + """ + if epoch >= ALTAIR_FORK_EPOCH: + return ALTAIR_FORK_VERSION + return GENESIS_FORK_VERSION +``` + ## Fork to Altair ### Fork trigger diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index f92c4cbf5..b37a8ab71 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -29,7 +29,6 @@ - [`is_execution_enabled`](#is_execution_enabled) - [Misc](#misc) - [`compute_timestamp_at_slot`](#compute_timestamp_at_slot) - - [Modified `compute_fork_version`](#modified-compute_fork_version) - [Beacon state accessors](#beacon-state-accessors) - [Modified `get_inactivity_penalty_deltas`](#modified-get_inactivity_penalty_deltas) - [Beacon state mutators](#beacon-state-mutators) @@ -243,20 +242,6 @@ def compute_timestamp_at_slot(state: BeaconState, slot: Slot) -> uint64: return uint64(state.genesis_time + slots_since_genesis * SECONDS_PER_SLOT) ``` -#### Modified `compute_fork_version` - -```python -def compute_fork_version(epoch: Epoch) -> Version: - """ - Return the fork version at the given ``epoch``. - """ - if epoch >= BELLATRIX_FORK_EPOCH: - return BELLATRIX_FORK_VERSION - if epoch >= ALTAIR_FORK_EPOCH: - return ALTAIR_FORK_VERSION - return GENESIS_FORK_VERSION -``` - ### Beacon state accessors #### Modified `get_inactivity_penalty_deltas` diff --git a/specs/bellatrix/fork.md b/specs/bellatrix/fork.md index 8cb34099b..5c0e174da 100644 --- a/specs/bellatrix/fork.md +++ b/specs/bellatrix/fork.md @@ -9,6 +9,9 @@ - [Introduction](#introduction) - [Configuration](#configuration) +- [Helper functions](#helper-functions) + - [Misc](#misc) + - [Modified `compute_fork_version`](#modified-compute_fork_version) - [Fork to Bellatrix](#fork-to-bellatrix) - [Fork trigger](#fork-trigger) - [Upgrading the state](#upgrading-the-state) @@ -28,6 +31,24 @@ Warning: this configuration is not definitive. | `BELLATRIX_FORK_VERSION` | `Version('0x02000000')` | | `BELLATRIX_FORK_EPOCH` | `Epoch(18446744073709551615)` **TBD** | +## Helper functions + +### Misc + +#### Modified `compute_fork_version` + +```python +def compute_fork_version(epoch: Epoch) -> Version: + """ + Return the fork version at the given ``epoch``. + """ + if epoch >= BELLATRIX_FORK_EPOCH: + return BELLATRIX_FORK_VERSION + if epoch >= ALTAIR_FORK_EPOCH: + return ALTAIR_FORK_VERSION + return GENESIS_FORK_VERSION +``` + ## Fork to Bellatrix ### Fork trigger diff --git a/specs/capella/beacon-chain.md b/specs/capella/beacon-chain.md index e2b47dd1f..da67a8fcb 100644 --- a/specs/capella/beacon-chain.md +++ b/specs/capella/beacon-chain.md @@ -34,8 +34,6 @@ - [`has_eth1_withdrawal_credential`](#has_eth1_withdrawal_credential) - [`is_fully_withdrawable_validator`](#is_fully_withdrawable_validator) - [`is_partially_withdrawable_validator`](#is_partially_withdrawable_validator) - - [Misc](#misc-1) - - [Modified `compute_fork_version`](#modified-compute_fork_version) - [Beacon chain state transition function](#beacon-chain-state-transition-function) - [Epoch processing](#epoch-processing) - [Full withdrawals](#full-withdrawals) @@ -320,24 +318,6 @@ def is_partially_withdrawable_validator(validator: Validator, balance: Gwei) -> return has_eth1_withdrawal_credential(validator) and has_max_effective_balance and has_excess_balance ``` -### Misc - -#### Modified `compute_fork_version` - -```python -def compute_fork_version(epoch: Epoch) -> Version: - """ - Return the fork version at the given ``epoch``. - """ - if epoch >= CAPELLA_FORK_EPOCH: - return CAPELLA_FORK_VERSION - if epoch >= BELLATRIX_FORK_EPOCH: - return BELLATRIX_FORK_VERSION - if epoch >= ALTAIR_FORK_EPOCH: - return ALTAIR_FORK_VERSION - return GENESIS_FORK_VERSION -``` - ## Beacon chain state transition function ### Epoch processing diff --git a/specs/capella/fork.md b/specs/capella/fork.md index 0d8283e56..c22387ee7 100644 --- a/specs/capella/fork.md +++ b/specs/capella/fork.md @@ -7,6 +7,9 @@ - [Introduction](#introduction) - [Configuration](#configuration) +- [Helper functions](#helper-functions) + - [Misc](#misc) + - [Modified `compute_fork_version`](#modified-compute_fork_version) - [Fork to Capella](#fork-to-capella) - [Fork trigger](#fork-trigger) - [Upgrading the state](#upgrading-the-state) @@ -27,6 +30,26 @@ Warning: this configuration is not definitive. | `CAPELLA_FORK_EPOCH` | `Epoch(18446744073709551615)` **TBD** | +## Helper functions + +### Misc + +#### Modified `compute_fork_version` + +```python +def compute_fork_version(epoch: Epoch) -> Version: + """ + Return the fork version at the given ``epoch``. + """ + if epoch >= CAPELLA_FORK_EPOCH: + return CAPELLA_FORK_VERSION + if epoch >= BELLATRIX_FORK_EPOCH: + return BELLATRIX_FORK_VERSION + if epoch >= ALTAIR_FORK_EPOCH: + return ALTAIR_FORK_VERSION + return GENESIS_FORK_VERSION +``` + ## Fork to Capella ### Fork trigger From 86e2f8fd7de25a7478c240f0cf5ba3c5264e96bf Mon Sep 17 00:00:00 2001 From: marioevz Date: Thu, 23 Jun 2022 22:02:39 +0000 Subject: [PATCH 020/105] Fix name discrepancy --- presets/mainnet/capella.yaml | 2 +- presets/minimal/capella.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/presets/mainnet/capella.yaml b/presets/mainnet/capella.yaml index c3eedc5c1..f04bdd06f 100644 --- a/presets/mainnet/capella.yaml +++ b/presets/mainnet/capella.yaml @@ -9,7 +9,7 @@ MAX_PARTIAL_WITHDRAWALS_PER_EPOCH: 256 # State list lengths # --------------------------------------------------------------- # 2**40 (= 1,099,511,627,776) withdrawals -WITHDRAWALS_QUEUE_LIMIT: 1099511627776 +WITHDRAWAL_QUEUE_LIMIT: 1099511627776 # Max operations per block diff --git a/presets/minimal/capella.yaml b/presets/minimal/capella.yaml index a3c5e8f1d..bf78685cd 100644 --- a/presets/minimal/capella.yaml +++ b/presets/minimal/capella.yaml @@ -9,7 +9,7 @@ MAX_PARTIAL_WITHDRAWALS_PER_EPOCH: 16 # State list lengths # --------------------------------------------------------------- # 2**40 (= 1,099,511,627,776) withdrawals -WITHDRAWALS_QUEUE_LIMIT: 1099511627776 +WITHDRAWAL_QUEUE_LIMIT: 1099511627776 # Max operations per block From 9a253e437d4991e3ca7e8aa3cf4adf095d424321 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 24 Jun 2022 01:05:11 +0200 Subject: [PATCH 021/105] create decorator for tests with matching config --- .../altair/unittests/test_config_override.py | 32 +++++++++++++++++- .../altair/unittests/test_sync_protocol.py | 31 +++-------------- tests/core/pyspec/eth2spec/test/context.py | 33 +++++++++++++++++++ .../eth2spec/test/helpers/light_client.py | 31 ----------------- 4 files changed, 69 insertions(+), 58 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/unittests/test_config_override.py b/tests/core/pyspec/eth2spec/test/altair/unittests/test_config_override.py index f1503c39f..5c940eafc 100644 --- a/tests/core/pyspec/eth2spec/test/altair/unittests/test_config_override.py +++ b/tests/core/pyspec/eth2spec/test/altair/unittests/test_config_override.py @@ -1,4 +1,9 @@ -from eth2spec.test.context import spec_configured_state_test, with_phases +from eth2spec.test.context import ( + spec_configured_state_test, + spec_state_test_with_matching_config, + with_all_phases, + with_phases +) from eth2spec.test.helpers.constants import ALTAIR @@ -17,3 +22,28 @@ def test_config_override(spec, state): # TODO: it would be nice if the create_genesis_state actually outputs a state # for the fork with a slot that matches at least the fork boundary. # assert spec.get_current_epoch(state) >= 4 + + +@with_all_phases +@spec_state_test_with_matching_config +def test_override_config_fork_epoch(spec, state): + if state.fork.current_version == spec.config.GENESIS_FORK_VERSION: + return + + assert spec.config.ALTAIR_FORK_EPOCH == spec.GENESIS_EPOCH + if state.fork.current_version == spec.config.ALTAIR_FORK_VERSION: + return + + assert spec.config.BELLATRIX_FORK_EPOCH == spec.GENESIS_EPOCH + if state.fork.current_version == spec.config.BELLATRIX_FORK_VERSION: + return + + assert spec.config.CAPELLA_FORK_EPOCH == spec.GENESIS_EPOCH + if state.fork.current_version == spec.config.CAPELLA_FORK_VERSION: + return + + assert spec.config.SHARDING_FORK_EPOCH == spec.GENESIS_EPOCH + if state.fork.current_version == spec.config.SHARDING_FORK_VERSION: + return + + assert False # Fork is missing diff --git a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py index 4acfa6424..69d9db0f2 100644 --- a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py +++ b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py @@ -1,7 +1,7 @@ from copy import deepcopy from eth2spec.test.context import ( - spec_state_test, + spec_state_test_with_matching_config, with_presets, with_altair_and_later, ) @@ -16,7 +16,6 @@ from eth2spec.test.helpers.constants import MINIMAL from eth2spec.test.helpers.light_client import ( get_sync_aggregate, initialize_light_client_store, - override_config_fork_epochs, ) from eth2spec.test.helpers.state import ( next_slots, @@ -26,11 +25,8 @@ from eth2spec.test.helpers.merkle import build_proof @with_altair_and_later -@spec_state_test +@spec_state_test_with_matching_config def test_process_light_client_update_not_timeout(spec, state): - old_config = spec.config - override_config_fork_epochs(spec, state) - store = initialize_light_client_store(spec, state) # Block at slot 1 doesn't increase sync committee period, so it won't force update store.finalized_header @@ -73,16 +69,11 @@ def test_process_light_client_update_not_timeout(spec, state): assert store.finalized_header == pre_store.finalized_header assert store.best_valid_update == update - spec.config = old_config - @with_altair_and_later -@spec_state_test +@spec_state_test_with_matching_config @with_presets([MINIMAL], reason="too slow") def test_process_light_client_update_at_period_boundary(spec, state): - old_config = spec.config - override_config_fork_epochs(spec, state) - store = initialize_light_client_store(spec, state) # Forward to slot before next sync committee period so that next block is final one in period @@ -128,16 +119,11 @@ def test_process_light_client_update_at_period_boundary(spec, state): assert store.best_valid_update == update assert store.finalized_header == pre_store.finalized_header - spec.config = old_config - @with_altair_and_later -@spec_state_test +@spec_state_test_with_matching_config @with_presets([MINIMAL], reason="too slow") def test_process_light_client_update_timeout(spec, state): - old_config = spec.config - override_config_fork_epochs(spec, state) - store = initialize_light_client_store(spec, state) # Forward to next sync committee period @@ -184,16 +170,11 @@ def test_process_light_client_update_timeout(spec, state): assert store.best_valid_update == update assert store.finalized_header == pre_store.finalized_header - spec.config = old_config - @with_altair_and_later -@spec_state_test +@spec_state_test_with_matching_config @with_presets([MINIMAL], reason="too slow") def test_process_light_client_update_finality_updated(spec, state): - old_config = spec.config - override_config_fork_epochs(spec, state) - store = initialize_light_client_store(spec, state) # Change finality @@ -245,5 +226,3 @@ def test_process_light_client_update_finality_updated(spec, state): assert store.optimistic_header == update.attested_header assert store.finalized_header == update.finalized_header assert store.best_valid_update is None - - spec.config = old_config diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 35e4f1bcb..06313c195 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -277,6 +277,39 @@ def spec_configured_state_test(conf): return decorator +def config_fork_epoch_overrides(spec, state): + overrides = {} + if state.fork.current_version == spec.config.GENESIS_FORK_VERSION: + pass + elif state.fork.current_version == spec.config.ALTAIR_FORK_VERSION: + overrides['ALTAIR_FORK_EPOCH'] = spec.GENESIS_EPOCH + elif state.fork.current_version == spec.config.BELLATRIX_FORK_VERSION: + overrides['ALTAIR_FORK_EPOCH'] = spec.GENESIS_EPOCH + overrides['BELLATRIX_FORK_EPOCH'] = spec.GENESIS_EPOCH + elif state.fork.current_version == spec.config.CAPELLA_FORK_VERSION: + overrides['ALTAIR_FORK_EPOCH'] = spec.GENESIS_EPOCH + overrides['BELLATRIX_FORK_EPOCH'] = spec.GENESIS_EPOCH + overrides['CAPELLA_FORK_EPOCH'] = spec.GENESIS_EPOCH + elif state.fork.current_version == spec.config.SHARDING_FORK_VERSION: + overrides['ALTAIR_FORK_EPOCH'] = spec.GENESIS_EPOCH + overrides['BELLATRIX_FORK_EPOCH'] = spec.GENESIS_EPOCH + overrides['CAPELLA_FORK_EPOCH'] = spec.GENESIS_EPOCH + overrides['SHARDING_FORK_EPOCH'] = spec.GENESIS_EPOCH + else: + assert False # Fork is missing + return overrides + + +def spec_state_test_with_matching_config(fn): + def decorator(fn): + def wrapper(*args, spec: Spec, **kw): + conf = config_fork_epoch_overrides(spec, kw['state']) + overrides = with_config_overrides(conf) + return overrides(fn)(*args, spec=spec, **kw) + return wrapper + return spec_test(with_state(decorator(single_phase(fn)))) + + def expect_assertion_error(fn): bad = False try: diff --git a/tests/core/pyspec/eth2spec/test/helpers/light_client.py b/tests/core/pyspec/eth2spec/test/helpers/light_client.py index 6868cad86..7dcad319a 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/light_client.py +++ b/tests/core/pyspec/eth2spec/test/helpers/light_client.py @@ -1,5 +1,3 @@ -from copy import deepcopy - from eth2spec.test.helpers.state import ( transition_to, ) @@ -9,35 +7,6 @@ from eth2spec.test.helpers.sync_committee import ( ) -def override_config_fork_epochs(spec, state): - # Test framework adjusts state fork but leaves spec config constants inconsistent - config_overrides = {} - if state.fork.current_version == spec.config.GENESIS_FORK_VERSION: - pass - elif state.fork.current_version == spec.config.ALTAIR_FORK_VERSION: - config_overrides['ALTAIR_FORK_EPOCH'] = spec.GENESIS_EPOCH - elif state.fork.current_version == spec.config.BELLATRIX_FORK_VERSION: - config_overrides['ALTAIR_FORK_EPOCH'] = spec.GENESIS_EPOCH - config_overrides['BELLATRIX_FORK_EPOCH'] = spec.GENESIS_EPOCH - elif state.fork.current_version == spec.config.CAPELLA_FORK_VERSION: - config_overrides['ALTAIR_FORK_EPOCH'] = spec.GENESIS_EPOCH - config_overrides['BELLATRIX_FORK_EPOCH'] = spec.GENESIS_EPOCH - config_overrides['CAPELLA_FORK_EPOCH'] = spec.GENESIS_EPOCH - elif state.fork.current_version == spec.config.SHARDING_FORK_VERSION: - config_overrides['ALTAIR_FORK_EPOCH'] = spec.GENESIS_EPOCH - config_overrides['BELLATRIX_FORK_EPOCH'] = spec.GENESIS_EPOCH - config_overrides['CAPELLA_FORK_EPOCH'] = spec.GENESIS_EPOCH - config_overrides['SHARDING_FORK_EPOCH'] = spec.GENESIS_EPOCH - else: - assert False - - tmp_config = deepcopy(spec.config._asdict()) - tmp_config.update(config_overrides) - config_types = spec.Configuration.__annotations__ - test_config = {k: config_types[k](v) for k, v in tmp_config.items()} - spec.config = spec.Configuration(**test_config) - - def initialize_light_client_store(spec, state): return spec.LightClientStore( finalized_header=spec.BeaconBlockHeader(), From a3339c7b7fdc4eb80e5a700542acd90ade424d50 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 25 Jun 2022 00:13:20 +0800 Subject: [PATCH 022/105] PR suggestion from @Inphi --- specs/eip4844/polynomial-commitments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/eip4844/polynomial-commitments.md b/specs/eip4844/polynomial-commitments.md index fc2339659..5d1b86895 100644 --- a/specs/eip4844/polynomial-commitments.md +++ b/specs/eip4844/polynomial-commitments.md @@ -77,7 +77,7 @@ def bls_modular_inverse(x: BLSFieldElement) -> BLSFieldElement: ```python def div(x: BLSFieldElement, y: BLSFieldElement) -> BLSFieldElement: """Divide two field elements: `x` by `y`""" - return x * inv(y) % BLS_MODULUS + return x * bls_modular_inverse(y) % BLS_MODULUS ``` #### `lincomb` From c10daf1709b973ce2bbc3948d82db0b165d51574 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Mon, 9 May 2022 14:37:54 +0200 Subject: [PATCH 023/105] Allow `LightClientUpdate` with genesis finality When `state.finalized_checkpoint` references the genesis slot, it points to an empty `root`, instead of the actual genesis block hash. This patch updates the `LightClientUpdate` logic to allow including finality proofs for genesis `finalized_checkpoint.root`, better supporting non-mainnet. When including such a finality proof, the proof is for the empty `root`, but `finalized_header` is kept zeroed out to signify this edge case. --- specs/altair/sync-protocol.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index a3b10efa2..c03db304f 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -100,7 +100,7 @@ class LightClientStore(object): ```python def is_finality_update(update: LightClientUpdate) -> bool: - return update.finalized_header != BeaconBlockHeader() + return update.finality_branch != [Bytes32() for _ in range(floorlog2(FINALIZED_ROOT_INDEX))] ``` ### `get_subtree_index` @@ -173,10 +173,15 @@ def validate_light_client_update(store: LightClientStore, # Verify that the `finalized_header`, if present, actually is the finalized header saved in the # state of the `attested_header` if not is_finality_update(update): - assert update.finality_branch == [Bytes32() for _ in range(floorlog2(FINALIZED_ROOT_INDEX))] + assert update.finalized_header == BeaconBlockHeader() else: + if update.finalized_header.slot != GENESIS_SLOT: + finalized_root = hash_tree_root(update.finalized_header) + else: + finalized_root = Bytes32() + assert update.finalized_header == BeaconBlockHeader() assert is_valid_merkle_branch( - leaf=hash_tree_root(update.finalized_header), + leaf=finalized_root, branch=update.finality_branch, depth=floorlog2(FINALIZED_ROOT_INDEX), index=get_subtree_index(FINALIZED_ROOT_INDEX), From 67d62025d93c4fa62c44f4d79c3415a004762adf Mon Sep 17 00:00:00 2001 From: fradamt <104826920+fradamt@users.noreply.github.com> Date: Mon, 27 Jun 2022 14:20:25 +0200 Subject: [PATCH 024/105] fix typo and specify something - committee_index is used as an input to compute_subnet_for_attestation but it's not previously defined - attestation.data.committee_index is incorrect, the field is "index" --- specs/phase0/validator.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 1b7124e92..54b344791 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -260,7 +260,7 @@ A validator should plan for future assignments by noting their assigned attestat slot and joining the committee index attestation subnet related to their committee assignment. Specifically a validator should: -* Call `get_committee_assignment(state, next_epoch, validator_index)` when checking for next epoch assignments. +* Call `_, committee_index, _ = get_committee_assignment(state, next_epoch, validator_index)` when checking for next epoch assignments. * Calculate the committees per slot for the next epoch: `committees_per_slot = get_committee_count_per_slot(state, next_epoch)` * Calculate the subnet index: `subnet_id = compute_subnet_for_attestation(committees_per_slot, slot, committee_index)` * Find peers of the pubsub topic `beacon_attestation_{subnet_id}`. @@ -510,7 +510,7 @@ Finally, the validator broadcasts `attestation` to the associated attestation su The `subnet_id` for the `attestation` is calculated with: - Let `committees_per_slot = get_committee_count_per_slot(state, attestation.data.target.epoch)`. -- Let `subnet_id = compute_subnet_for_attestation(committees_per_slot, attestation.data.slot, attestation.data.committee_index)`. +- Let `subnet_id = compute_subnet_for_attestation(committees_per_slot, attestation.data.slot, attestation.data.index)`. ```python def compute_subnet_for_attestation(committees_per_slot: uint64, slot: Slot, committee_index: CommitteeIndex) -> uint64: From 99e30328a8a51de55789ca4421da42d4705230d1 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Mon, 27 Jun 2022 22:26:35 +0200 Subject: [PATCH 025/105] Switch condition order for readability Co-authored-by: Hsiao-Wei Wang --- specs/altair/sync-protocol.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index c03db304f..23ef1ce66 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -175,11 +175,11 @@ def validate_light_client_update(store: LightClientStore, if not is_finality_update(update): assert update.finalized_header == BeaconBlockHeader() else: - if update.finalized_header.slot != GENESIS_SLOT: - finalized_root = hash_tree_root(update.finalized_header) - else: + if update.finalized_header.slot == GENESIS_SLOT: finalized_root = Bytes32() assert update.finalized_header == BeaconBlockHeader() + else: + finalized_root = hash_tree_root(update.finalized_header) assert is_valid_merkle_branch( leaf=finalized_root, branch=update.finality_branch, From 65cfedef22cd90607ac4a8b63d83545f7820015c Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Mon, 27 Jun 2022 22:36:55 +0200 Subject: [PATCH 026/105] Update finality update documentation --- specs/altair/sync-protocol.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index 23ef1ce66..5340a2015 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -170,8 +170,9 @@ def validate_light_client_update(store: LightClientStore, signature_period = compute_sync_committee_period(compute_epoch_at_slot(update.signature_slot)) assert signature_period in (finalized_period, finalized_period + 1) - # Verify that the `finalized_header`, if present, actually is the finalized header saved in the - # state of the `attested_header` + # Verify that the `finality_branch`, if present, confirms `finalized_header` + # to match the finalized checkpoint root saved in the state of `attested_header`. + # Note that the genesis finalized checkpoint root is represented as a zero hash. if not is_finality_update(update): assert update.finalized_header == BeaconBlockHeader() else: From f5f3031c5e7f5049eae117817894b522b8ebb3ba Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Wed, 27 Apr 2022 16:07:49 +0200 Subject: [PATCH 027/105] Allow passing redundant `next_sync_committee` The producer of `LightClientUpdate` structures usually does not know how far the `LightClientStore` on the client side has advanced. Updates are currently rejected when including a redundant `next_sync_committee` not advancing the `LightClientStore`. Behaviour is changed to allow this. --- specs/altair/sync-protocol.md | 18 +++++++++++++++--- .../altair/unittests/test_sync_protocol.py | 12 ++++++++---- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index 5340a2015..fdfa68da8 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -16,6 +16,7 @@ - [`LightClientUpdate`](#lightclientupdate) - [`LightClientStore`](#lightclientstore) - [Helper functions](#helper-functions) + - [`is_sync_committee_update`](#is_sync_committee_update) - [`is_finality_update`](#is_finality_update) - [`get_subtree_index`](#get_subtree_index) - [`get_active_header`](#get_active_header) @@ -96,6 +97,13 @@ class LightClientStore(object): ## Helper functions +### `is_sync_committee_update` + +```python +def is_sync_committee_update(update: LightClientUpdate) -> bool: + return update.next_sync_committee_branch != [Bytes32() for _ in range(floorlog2(NEXT_SYNC_COMMITTEE_INDEX))] +``` + ### `is_finality_update` ```python @@ -189,10 +197,14 @@ def validate_light_client_update(store: LightClientStore, root=update.attested_header.state_root, ) - # Verify update next sync committee if the update period incremented - if update_period == finalized_period: - assert update.next_sync_committee_branch == [Bytes32() for _ in range(floorlog2(NEXT_SYNC_COMMITTEE_INDEX))] + # Verify that the `next_sync_committee`, if present, actually is the next sync committee saved in the + # state of the `active_header` + if not is_sync_committee_update(update): + assert update_period == finalized_period + assert update.next_sync_committee == SyncCommittee() else: + if update_period == finalized_period: + assert update.next_sync_committee == store.next_sync_committee assert is_valid_merkle_branch( leaf=hash_tree_root(update.next_sync_committee), branch=update.next_sync_committee_branch, diff --git a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py index 69d9db0f2..a252f0e68 100644 --- a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py +++ b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py @@ -42,6 +42,7 @@ def test_process_light_client_update_not_timeout(spec, state): # Sync committee signing the block_header sync_aggregate, signature_slot = get_sync_aggregate(spec, state, block_header) + next_sync_committee = spec.SyncCommittee() next_sync_committee_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.NEXT_SYNC_COMMITTEE_INDEX))] # Ensure that finality checkpoint is genesis @@ -52,7 +53,7 @@ def test_process_light_client_update_not_timeout(spec, state): update = spec.LightClientUpdate( attested_header=block_header, - next_sync_committee=state.next_sync_committee, + next_sync_committee=next_sync_committee, next_sync_committee_branch=next_sync_committee_branch, finalized_header=finality_header, finality_branch=finality_branch, @@ -94,6 +95,7 @@ def test_process_light_client_update_at_period_boundary(spec, state): # Sync committee signing the block_header sync_aggregate, signature_slot = get_sync_aggregate(spec, state, block_header) + next_sync_committee = spec.SyncCommittee() next_sync_committee_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.NEXT_SYNC_COMMITTEE_INDEX))] # Finality is unchanged @@ -102,7 +104,7 @@ def test_process_light_client_update_at_period_boundary(spec, state): update = spec.LightClientUpdate( attested_header=block_header, - next_sync_committee=state.next_sync_committee, + next_sync_committee=next_sync_committee, next_sync_committee_branch=next_sync_committee_branch, finalized_header=finality_header, finality_branch=finality_branch, @@ -146,6 +148,7 @@ def test_process_light_client_update_timeout(spec, state): sync_aggregate, signature_slot = get_sync_aggregate(spec, state, block_header) # Sync committee is updated + next_sync_committee = state.next_sync_committee next_sync_committee_branch = build_proof(state.get_backing(), spec.NEXT_SYNC_COMMITTEE_INDEX) # Finality is unchanged finality_header = spec.BeaconBlockHeader() @@ -153,7 +156,7 @@ def test_process_light_client_update_timeout(spec, state): update = spec.LightClientUpdate( attested_header=block_header, - next_sync_committee=state.next_sync_committee, + next_sync_committee=next_sync_committee, next_sync_committee_branch=next_sync_committee_branch, finalized_header=finality_header, finality_branch=finality_branch, @@ -191,6 +194,7 @@ def test_process_light_client_update_finality_updated(spec, state): assert snapshot_period == update_period # Updated sync_committee and finality + next_sync_committee = spec.SyncCommittee() next_sync_committee_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.NEXT_SYNC_COMMITTEE_INDEX))] finalized_block_header = blocks[spec.SLOTS_PER_EPOCH - 1].message assert finalized_block_header.slot == spec.compute_start_slot_at_epoch(state.finalized_checkpoint.epoch) @@ -212,7 +216,7 @@ def test_process_light_client_update_finality_updated(spec, state): update = spec.LightClientUpdate( attested_header=block_header, - next_sync_committee=state.next_sync_committee, + next_sync_committee=next_sync_committee, next_sync_committee_branch=next_sync_committee_branch, finalized_header=finalized_block_header, finality_branch=finality_branch, From 2dd9a52fe44b6f3741f1a207fa0bbfc2a99d3e76 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Wed, 29 Jun 2022 10:24:53 -0600 Subject: [PATCH 028/105] remove data logging from BLS test gen --- tests/generators/bls/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/generators/bls/main.py b/tests/generators/bls/main.py index 6fc86d10f..60c31d9d9 100644 --- a/tests/generators/bls/main.py +++ b/tests/generators/bls/main.py @@ -513,7 +513,6 @@ def create_provider(fork_name: SpecForkName, def cases_fn() -> Iterable[gen_typing.TestCase]: for data in test_case_fn(): - print(data) (case_name, case_content) = data yield gen_typing.TestCase( fork_name=fork_name, From 529ce301b8244ae4957418fd8b1af58a962ff5d9 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Mon, 27 Jun 2022 12:33:54 -0600 Subject: [PATCH 029/105] write test generator diagnostics to a summary file --- setup.py | 2 +- .../gen_helpers/gen_base/gen_runner.py | 43 ++++++++++++++++--- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 57d8d8fe1..e02759bfc 100644 --- a/setup.py +++ b/setup.py @@ -1041,7 +1041,7 @@ setup( extras_require={ "test": ["pytest>=4.4", "pytest-cov", "pytest-xdist"], "lint": ["flake8==3.7.7", "mypy==0.812", "pylint==2.12.2"], - "generator": ["python-snappy==0.5.4"], + "generator": ["python-snappy==0.5.4", "filelock"], }, install_requires=[ "eth-utils>=1.3.0,<2", diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index 6850af188..3e5d71d14 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -3,7 +3,9 @@ import time import shutil import argparse from pathlib import Path +from filelock import FileLock import sys +import json from typing import Iterable, AnyStr, Any, Callable import traceback @@ -111,6 +113,8 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): collected_test_count = 0 generated_test_count = 0 skipped_test_count = 0 + test_identifiers = [] + provider_start = time.time() for tprov in test_providers: if not collect_only: @@ -123,12 +127,10 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): / Path(test_case.runner_name) / Path(test_case.handler_name) / Path(test_case.suite_name) / Path(test_case.case_name) ) - incomplete_tag_file = case_dir / "INCOMPLETE" - collected_test_count += 1 - if collect_only: - print(f"Collected test at: {case_dir}") - continue + print(f"Collected test at: {case_dir}") + + incomplete_tag_file = case_dir / "INCOMPLETE" if case_dir.exists(): if not args.force and not incomplete_tag_file.exists(): @@ -198,6 +200,15 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): shutil.rmtree(case_dir) else: generated_test_count += 1 + test_identifier = "::".join([ + test_case.preset_name, + test_case.fork_name, + test_case.runner_name, + test_case.handler_name, + test_case.suite_name, + test_case.case_name + ]) + test_identifiers.append(test_identifier) # Only remove `INCOMPLETE` tag file os.remove(incomplete_tag_file) test_end = time.time() @@ -216,6 +227,28 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): if span > TIME_THRESHOLD_TO_PRINT: summary_message += f" in {span} seconds" print(summary_message) + diagnostics = { + "collected_test_count": collected_test_count, + "generated_test_count": generated_test_count, + "skipped_test_count": skipped_test_count, + "test_identifiers": test_identifiers, + "durations": [f"{span} seconds"], + } + diagnostics_path = Path(os.path.join(output_dir, "diagnostics.json")) + diagnostics_lock = FileLock(os.path.join(output_dir, "diagnostics.json.lock")) + with diagnostics_lock: + diagnostics_path.touch(exist_ok=True) + if os.path.getsize(diagnostics_path) == 0: + with open(diagnostics_path, "w+") as f: + json.dump(diagnostics, f) + else: + with open(diagnostics_path, "r+") as f: + existing_diagnostics = json.load(f) + for k, v in diagnostics.items(): + existing_diagnostics[k] += v + with open(diagnostics_path, "w+") as f: + json.dump(existing_diagnostics, f) + print(f"wrote diagnostics to {diagnostics_path}") def dump_yaml_fn(data: Any, name: str, file_mode: str, yaml_encoder: YAML): From e269b305b9530239844861569f0d4d0909baf468 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Mon, 2 May 2022 18:21:26 +0200 Subject: [PATCH 030/105] Cleanup light client tests Various cleanups and minor fixes: - Consistent terminology: - `signed_block` -> `attested_block` - `finalized_block_header` -> `finalized_header` - `snapshot_period` -> `store_period` - Use correct block in finality test (`blocks[-1]` instead of new one) - Add `signed_block_header` func to get header from `SignedBeaconBlock` - Remove `block_header` from `get_sync_aggregate` helper arguments - Use `state_transition_with_full_block` as shortcut for multiple calls - Have `finalized_header` actually be header instead of full block body - Consistent ordering of `assert` to match structure definition --- .../altair/unittests/test_sync_protocol.py | 120 +++++++----------- .../eth2spec/test/helpers/light_client.py | 15 ++- 2 files changed, 57 insertions(+), 78 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py index a252f0e68..17c450f17 100644 --- a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py +++ b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py @@ -7,19 +7,16 @@ from eth2spec.test.context import ( ) from eth2spec.test.helpers.attestations import ( next_epoch_with_attestations, -) -from eth2spec.test.helpers.block import ( - build_empty_block, - build_empty_block_for_next_slot, + state_transition_with_full_block, ) from eth2spec.test.helpers.constants import MINIMAL from eth2spec.test.helpers.light_client import ( get_sync_aggregate, initialize_light_client_store, + signed_block_to_header, ) from eth2spec.test.helpers.state import ( next_slots, - state_transition_and_sign_block, ) from eth2spec.test.helpers.merkle import build_proof @@ -30,18 +27,11 @@ def test_process_light_client_update_not_timeout(spec, state): store = initialize_light_client_store(spec, state) # Block at slot 1 doesn't increase sync committee period, so it won't force update store.finalized_header - block = build_empty_block_for_next_slot(spec, state) - signed_block = state_transition_and_sign_block(spec, state, block) - block_header = spec.BeaconBlockHeader( - slot=signed_block.message.slot, - proposer_index=signed_block.message.proposer_index, - parent_root=signed_block.message.parent_root, - state_root=signed_block.message.state_root, - body_root=signed_block.message.body.hash_tree_root(), - ) + attested_block = state_transition_with_full_block(spec, state, False, False) + attested_header = signed_block_to_header(spec, attested_block) - # Sync committee signing the block_header - sync_aggregate, signature_slot = get_sync_aggregate(spec, state, block_header) + # Sync committee signing the attested_header + sync_aggregate, signature_slot = get_sync_aggregate(spec, state) next_sync_committee = spec.SyncCommittee() next_sync_committee_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.NEXT_SYNC_COMMITTEE_INDEX))] @@ -52,7 +42,7 @@ def test_process_light_client_update_not_timeout(spec, state): finality_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.FINALIZED_ROOT_INDEX))] update = spec.LightClientUpdate( - attested_header=block_header, + attested_header=attested_header, next_sync_committee=next_sync_committee, next_sync_committee_branch=next_sync_committee_branch, finalized_header=finality_header, @@ -65,10 +55,10 @@ def test_process_light_client_update_not_timeout(spec, state): spec.process_light_client_update(store, update, signature_slot, state.genesis_validators_root) - assert store.current_max_active_participants > 0 - assert store.optimistic_header == update.attested_header assert store.finalized_header == pre_store.finalized_header assert store.best_valid_update == update + assert store.optimistic_header == update.attested_header + assert store.current_max_active_participants > 0 @with_altair_and_later @@ -79,22 +69,15 @@ def test_process_light_client_update_at_period_boundary(spec, state): # Forward to slot before next sync committee period so that next block is final one in period next_slots(spec, state, spec.UPDATE_TIMEOUT - 2) - snapshot_period = spec.compute_sync_committee_period(spec.compute_epoch_at_slot(store.optimistic_header.slot)) + store_period = spec.compute_sync_committee_period(spec.compute_epoch_at_slot(store.optimistic_header.slot)) update_period = spec.compute_sync_committee_period(spec.compute_epoch_at_slot(state.slot)) - assert snapshot_period == update_period + assert store_period == update_period - block = build_empty_block_for_next_slot(spec, state) - signed_block = state_transition_and_sign_block(spec, state, block) - block_header = spec.BeaconBlockHeader( - slot=signed_block.message.slot, - proposer_index=signed_block.message.proposer_index, - parent_root=signed_block.message.parent_root, - state_root=signed_block.message.state_root, - body_root=signed_block.message.body.hash_tree_root(), - ) + attested_block = state_transition_with_full_block(spec, state, False, False) + attested_header = signed_block_to_header(spec, attested_block) - # Sync committee signing the block_header - sync_aggregate, signature_slot = get_sync_aggregate(spec, state, block_header) + # Sync committee signing the attested_header + sync_aggregate, signature_slot = get_sync_aggregate(spec, state) next_sync_committee = spec.SyncCommittee() next_sync_committee_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.NEXT_SYNC_COMMITTEE_INDEX))] @@ -103,7 +86,7 @@ def test_process_light_client_update_at_period_boundary(spec, state): finality_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.FINALIZED_ROOT_INDEX))] update = spec.LightClientUpdate( - attested_header=block_header, + attested_header=attested_header, next_sync_committee=next_sync_committee, next_sync_committee_branch=next_sync_committee_branch, finalized_header=finality_header, @@ -116,10 +99,10 @@ def test_process_light_client_update_at_period_boundary(spec, state): spec.process_light_client_update(store, update, signature_slot, state.genesis_validators_root) - assert store.current_max_active_participants > 0 - assert store.optimistic_header == update.attested_header - assert store.best_valid_update == update assert store.finalized_header == pre_store.finalized_header + assert store.best_valid_update == update + assert store.optimistic_header == update.attested_header + assert store.current_max_active_participants > 0 @with_altair_and_later @@ -130,22 +113,15 @@ def test_process_light_client_update_timeout(spec, state): # Forward to next sync committee period next_slots(spec, state, spec.UPDATE_TIMEOUT) - snapshot_period = spec.compute_sync_committee_period(spec.compute_epoch_at_slot(store.optimistic_header.slot)) + store_period = spec.compute_sync_committee_period(spec.compute_epoch_at_slot(store.optimistic_header.slot)) update_period = spec.compute_sync_committee_period(spec.compute_epoch_at_slot(state.slot)) - assert snapshot_period + 1 == update_period + assert store_period + 1 == update_period - block = build_empty_block_for_next_slot(spec, state) - signed_block = state_transition_and_sign_block(spec, state, block) - block_header = spec.BeaconBlockHeader( - slot=signed_block.message.slot, - proposer_index=signed_block.message.proposer_index, - parent_root=signed_block.message.parent_root, - state_root=signed_block.message.state_root, - body_root=signed_block.message.body.hash_tree_root(), - ) + attested_block = state_transition_with_full_block(spec, state, False, False) + attested_header = signed_block_to_header(spec, attested_block) - # Sync committee signing the block_header - sync_aggregate, signature_slot = get_sync_aggregate(spec, state, block_header) + # Sync committee signing the attested_header + sync_aggregate, signature_slot = get_sync_aggregate(spec, state) # Sync committee is updated next_sync_committee = state.next_sync_committee @@ -155,7 +131,7 @@ def test_process_light_client_update_timeout(spec, state): finality_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.FINALIZED_ROOT_INDEX))] update = spec.LightClientUpdate( - attested_header=block_header, + attested_header=attested_header, next_sync_committee=next_sync_committee, next_sync_committee_branch=next_sync_committee_branch, finalized_header=finality_header, @@ -168,10 +144,10 @@ def test_process_light_client_update_timeout(spec, state): spec.process_light_client_update(store, update, signature_slot, state.genesis_validators_root) - assert store.current_max_active_participants > 0 - assert store.optimistic_header == update.attested_header - assert store.best_valid_update == update assert store.finalized_header == pre_store.finalized_header + assert store.best_valid_update == update + assert store.optimistic_header == update.attested_header + assert store.current_max_active_participants > 0 @with_altair_and_later @@ -189,36 +165,30 @@ def test_process_light_client_update_finality_updated(spec, state): # Ensure that finality checkpoint has changed assert state.finalized_checkpoint.epoch == 3 # Ensure that it's same period - snapshot_period = spec.compute_sync_committee_period(spec.compute_epoch_at_slot(store.optimistic_header.slot)) + store_period = spec.compute_sync_committee_period(spec.compute_epoch_at_slot(store.optimistic_header.slot)) update_period = spec.compute_sync_committee_period(spec.compute_epoch_at_slot(state.slot)) - assert snapshot_period == update_period + assert store_period == update_period + + attested_block = blocks[-1] + attested_header = signed_block_to_header(spec, attested_block) + + # Sync committee signing the attested_header + sync_aggregate, signature_slot = get_sync_aggregate(spec, state) # Updated sync_committee and finality next_sync_committee = spec.SyncCommittee() next_sync_committee_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.NEXT_SYNC_COMMITTEE_INDEX))] - finalized_block_header = blocks[spec.SLOTS_PER_EPOCH - 1].message - assert finalized_block_header.slot == spec.compute_start_slot_at_epoch(state.finalized_checkpoint.epoch) - assert finalized_block_header.hash_tree_root() == state.finalized_checkpoint.root + finalized_block = blocks[spec.SLOTS_PER_EPOCH - 1] + finalized_header = signed_block_to_header(spec, finalized_block) + assert finalized_header.slot == spec.compute_start_slot_at_epoch(state.finalized_checkpoint.epoch) + assert finalized_header.hash_tree_root() == state.finalized_checkpoint.root finality_branch = build_proof(state.get_backing(), spec.FINALIZED_ROOT_INDEX) - # Build block header - block = build_empty_block(spec, state) - block_header = spec.BeaconBlockHeader( - slot=block.slot, - proposer_index=block.proposer_index, - parent_root=block.parent_root, - state_root=state.hash_tree_root(), - body_root=block.body.hash_tree_root(), - ) - - # Sync committee signing the block_header - sync_aggregate, signature_slot = get_sync_aggregate(spec, state, block_header) - update = spec.LightClientUpdate( - attested_header=block_header, + attested_header=attested_header, next_sync_committee=next_sync_committee, next_sync_committee_branch=next_sync_committee_branch, - finalized_header=finalized_block_header, + finalized_header=finalized_header, finality_branch=finality_branch, sync_aggregate=sync_aggregate, signature_slot=signature_slot, @@ -226,7 +196,7 @@ def test_process_light_client_update_finality_updated(spec, state): spec.process_light_client_update(store, update, signature_slot, state.genesis_validators_root) - assert store.current_max_active_participants > 0 - assert store.optimistic_header == update.attested_header assert store.finalized_header == update.finalized_header assert store.best_valid_update is None + assert store.optimistic_header == update.attested_header + assert store.current_max_active_participants > 0 diff --git a/tests/core/pyspec/eth2spec/test/helpers/light_client.py b/tests/core/pyspec/eth2spec/test/helpers/light_client.py index 7dcad319a..090036094 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/light_client.py +++ b/tests/core/pyspec/eth2spec/test/helpers/light_client.py @@ -7,6 +7,16 @@ from eth2spec.test.helpers.sync_committee import ( ) +def signed_block_to_header(spec, block): + return spec.BeaconBlockHeader( + slot=block.message.slot, + proposer_index=block.message.proposer_index, + parent_root=block.message.parent_root, + state_root=block.message.state_root, + body_root=block.message.body.hash_tree_root(), + ) + + def initialize_light_client_store(spec, state): return spec.LightClientStore( finalized_header=spec.BeaconBlockHeader(), @@ -19,10 +29,10 @@ def initialize_light_client_store(spec, state): ) -def get_sync_aggregate(spec, state, block_header, signature_slot=None): +def get_sync_aggregate(spec, state, signature_slot=None): # By default, the sync committee signs the previous slot if signature_slot is None: - signature_slot = block_header.slot + 1 + signature_slot = state.slot + 1 # Ensure correct sync committee and fork version are selected signature_state = state.copy() @@ -39,7 +49,6 @@ def get_sync_aggregate(spec, state, block_header, signature_slot=None): signature_state, signature_slot, committee_indices, - block_root=spec.Root(block_header.hash_tree_root()), ) sync_aggregate = spec.SyncAggregate( sync_committee_bits=sync_committee_bits, From 4b5a4ad51f95424575c32ad77201a27bf477134a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 30 Jun 2022 22:12:09 +0800 Subject: [PATCH 031/105] Include Bellatrix p2p interface specs into pyspec --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 57d8d8fe1..d83931850 100644 --- a/setup.py +++ b/setup.py @@ -891,6 +891,7 @@ class PySpecCommand(Command): specs/bellatrix/fork.md specs/bellatrix/fork-choice.md specs/bellatrix/validator.md + specs/bellatrix/p2p-interface.md sync/optimistic.md """ if self.spec_fork == CAPELLA: From c4dbd241acc93d29d82e06d18e0828a264c5bf29 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 29 Apr 2022 16:22:26 +0200 Subject: [PATCH 032/105] Root `next_sync_committee` in `attested_header` `LightClientUpdate` structures currently use different merkle proof root depending on the presence of `finalized_header`. By always rooting it in the same state (the `attested_header.state_root`), logic gets simpler. Caveats: - In periods of extended non-finality, `update.finalized_header` may now be outdated by several sync committee periods. The old implementation rejected such updates as the `next_sync_committee` in them was stale, but the new implementation can properly handle this case. - The `next_sync_committee` can no longer be considered finalized based on `is_finality_update`. Instead, waiting until `finalized_header` is in the `attested_header`'s sync committee period is now necessary. - Because `update.finalized_header > store.finalized_header` no longer holds (for updates with finality), an `is_better_update` helper is added to improve `best_valid_update` tracking (in the past, finalized updates with supermajority participation would always directly apply) This PR builds on prior work from: - @hwwhww at https://github.com/ethereum/consensus-specs/pull/2829 --- specs/altair/sync-protocol.md | 107 +++++++++----- .../test/altair/sync_protocol/__init__.py | 0 .../sync_protocol/test_update_ranking.py | 132 ++++++++++++++++++ .../eth2spec/test/helpers/light_client.py | 11 +- tests/formats/sync_protocol/README.md | 6 + tests/formats/sync_protocol/update_ranking.md | 21 +++ tests/generators/sync_protocol/README.md | 5 + tests/generators/sync_protocol/__init__.py | 0 tests/generators/sync_protocol/main.py | 16 +++ .../generators/sync_protocol/requirements.txt | 2 + 10 files changed, 259 insertions(+), 41 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/altair/sync_protocol/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_update_ranking.py create mode 100644 tests/formats/sync_protocol/README.md create mode 100644 tests/formats/sync_protocol/update_ranking.md create mode 100644 tests/generators/sync_protocol/README.md create mode 100644 tests/generators/sync_protocol/__init__.py create mode 100644 tests/generators/sync_protocol/main.py create mode 100644 tests/generators/sync_protocol/requirements.txt diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index fdfa68da8..7864ac6e2 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -19,8 +19,8 @@ - [`is_sync_committee_update`](#is_sync_committee_update) - [`is_finality_update`](#is_finality_update) - [`get_subtree_index`](#get_subtree_index) - - [`get_active_header`](#get_active_header) - [`get_safety_threshold`](#get_safety_threshold) + - [`is_better_update`](#is_better_update) - [Light client state updates](#light-client-state-updates) - [`process_slot_for_light_client_store`](#process_slot_for_light_client_store) - [`validate_light_client_update`](#validate_light_client_update) @@ -64,7 +64,7 @@ uses sync committees introduced in [this beacon chain extension](./beacon-chain. class LightClientUpdate(Container): # The beacon block header that is attested to by the sync committee attested_header: BeaconBlockHeader - # Next sync committee corresponding to the active header + # Next sync committee corresponding to `attested_header` next_sync_committee: SyncCommittee next_sync_committee_branch: Vector[Bytes32, floorlog2(NEXT_SYNC_COMMITTEE_INDEX)] # The finalized beacon block header attested to by Merkle branch @@ -118,19 +118,6 @@ def get_subtree_index(generalized_index: GeneralizedIndex) -> uint64: return uint64(generalized_index % 2**(floorlog2(generalized_index))) ``` -### `get_active_header` - -```python -def get_active_header(update: LightClientUpdate) -> BeaconBlockHeader: - # The "active header" is the header that the update is trying to convince us - # to accept. If a finalized header is present, it's the finalized header, - # otherwise it's the attested header - if is_finality_update(update): - return update.finalized_header - else: - return update.attested_header -``` - ### `get_safety_threshold` ```python @@ -141,6 +128,50 @@ def get_safety_threshold(store: LightClientStore) -> uint64: ) // 2 ``` +### `is_better_update` + +```python +def is_better_update(new_update: LightClientUpdate, old_update: LightClientUpdate) -> bool: + # Compare supermajority (> 2/3) sync committee participation + max_active_participants = len(new_update.sync_aggregate.sync_committee_bits) + new_num_active_participants = sum(new_update.sync_aggregate.sync_committee_bits) + old_num_active_participants = sum(old_update.sync_aggregate.sync_committee_bits) + new_has_supermajority = new_num_active_participants * 3 >= max_active_participants * 2 + old_has_supermajority = old_num_active_participants * 3 >= max_active_participants * 2 + if new_has_supermajority != old_has_supermajority: + return new_has_supermajority > old_has_supermajority + if not new_has_supermajority and new_num_active_participants != old_num_active_participants: + return new_num_active_participants > old_num_active_participants + + # Compare indication of any finality + new_has_finality = is_finality_update(new_update) + old_has_finality = is_finality_update(old_update) + if new_has_finality != old_has_finality: + return new_has_finality > old_has_finality + + # Compare sync committee finality + if new_has_finality: + new_has_sync_committee_finality = ( + compute_sync_committee_period(compute_epoch_at_slot(new_update.finalized_header.slot)) == + compute_sync_committee_period(compute_epoch_at_slot(new_update.attested_header.slot)) + ) + old_has_sync_committee_finality = ( + compute_sync_committee_period(compute_epoch_at_slot(old_update.finalized_header.slot)) == + compute_sync_committee_period(compute_epoch_at_slot(old_update.attested_header.slot)) + ) + if new_has_sync_committee_finality != old_has_sync_committee_finality: + return new_has_sync_committee_finality > old_has_sync_committee_finality + + # Tiebreaker 1: Sync committee participation beyond supermajority + if new_num_active_participants != old_num_active_participants: + return new_num_active_participants > old_num_active_participants + + # Tiebreaker 2: Prefer older data (fewer changes to best) + if new_update.attested_header.slot != old_update.attested_header.slot: + return new_update.attested_header.slot < old_update.attested_header.slot + return new_update.signature_slot < old_update.signature_slot +``` + ## Light client state updates A light client maintains its state in a `store` object of type `LightClientStore` and receives `update` objects of type `LightClientUpdate`. Every `update` triggers `process_light_client_update(store, update, current_slot, genesis_validators_root)` where `current_slot` is the current slot based on a local clock. `process_slot_for_light_client_store` is triggered every time the current slot increments. @@ -157,6 +188,8 @@ def process_slot_for_light_client_store(store: LightClientStore, current_slot: S and store.best_valid_update is not None ): # Forced best update when the update timeout has elapsed + if store.best_valid_update.finalized_header.slot <= store.finalized_header.slot: + store.best_valid_update.finalized_header = store.best_valid_update.attested_header apply_light_client_update(store, store.best_valid_update) store.best_valid_update = None ``` @@ -168,15 +201,19 @@ def validate_light_client_update(store: LightClientStore, update: LightClientUpdate, current_slot: Slot, genesis_validators_root: Root) -> None: - # Verify update slot is larger than slot of current best finalized header - active_header = get_active_header(update) - assert current_slot >= update.signature_slot > active_header.slot > store.finalized_header.slot + # Verify sync committee has sufficient participants + sync_aggregate = update.sync_aggregate + assert sum(sync_aggregate.sync_committee_bits) >= MIN_SYNC_COMMITTEE_PARTICIPANTS # Verify update does not skip a sync committee period - finalized_period = compute_sync_committee_period(compute_epoch_at_slot(store.finalized_header.slot)) - update_period = compute_sync_committee_period(compute_epoch_at_slot(active_header.slot)) + assert current_slot >= update.signature_slot > update.attested_header.slot >= update.finalized_header.slot + store_period = compute_sync_committee_period(compute_epoch_at_slot(store.finalized_header.slot)) signature_period = compute_sync_committee_period(compute_epoch_at_slot(update.signature_slot)) - assert signature_period in (finalized_period, finalized_period + 1) + assert signature_period in (store_period, store_period + 1) + + # Verify update is relevant + attested_period = compute_sync_committee_period(compute_epoch_at_slot(update.attested_header.slot)) + assert update.attested_header.slot > store.finalized_header.slot # Verify that the `finality_branch`, if present, confirms `finalized_header` # to match the finalized checkpoint root saved in the state of `attested_header`. @@ -198,28 +235,23 @@ def validate_light_client_update(store: LightClientStore, ) # Verify that the `next_sync_committee`, if present, actually is the next sync committee saved in the - # state of the `active_header` + # state of the `attested_header` if not is_sync_committee_update(update): - assert update_period == finalized_period + assert attested_period == store_period assert update.next_sync_committee == SyncCommittee() else: - if update_period == finalized_period: + if attested_period == store_period: assert update.next_sync_committee == store.next_sync_committee assert is_valid_merkle_branch( leaf=hash_tree_root(update.next_sync_committee), branch=update.next_sync_committee_branch, depth=floorlog2(NEXT_SYNC_COMMITTEE_INDEX), index=get_subtree_index(NEXT_SYNC_COMMITTEE_INDEX), - root=active_header.state_root, + root=update.attested_header.state_root, ) - sync_aggregate = update.sync_aggregate - - # Verify sync committee has sufficient participants - assert sum(sync_aggregate.sync_committee_bits) >= MIN_SYNC_COMMITTEE_PARTICIPANTS - # Verify sync committee aggregate signature - if signature_period == finalized_period: + if signature_period == store_period: sync_committee = store.current_sync_committee else: sync_committee = store.next_sync_committee @@ -237,13 +269,12 @@ def validate_light_client_update(store: LightClientStore, ```python def apply_light_client_update(store: LightClientStore, update: LightClientUpdate) -> None: - active_header = get_active_header(update) - finalized_period = compute_sync_committee_period(compute_epoch_at_slot(store.finalized_header.slot)) - update_period = compute_sync_committee_period(compute_epoch_at_slot(active_header.slot)) - if update_period == finalized_period + 1: + store_period = compute_sync_committee_period(compute_epoch_at_slot(store.finalized_header.slot)) + finalized_period = compute_sync_committee_period(compute_epoch_at_slot(update.finalized_header.slot)) + if finalized_period == store_period + 1: store.current_sync_committee = store.next_sync_committee store.next_sync_committee = update.next_sync_committee - store.finalized_header = active_header + store.finalized_header = update.finalized_header if store.finalized_header.slot > store.optimistic_header.slot: store.optimistic_header = store.finalized_header ``` @@ -262,7 +293,7 @@ def process_light_client_update(store: LightClientStore, # Update the best update in case we have to force-update to it if the timeout elapses if ( store.best_valid_update is None - or sum(sync_committee_bits) > sum(store.best_valid_update.sync_aggregate.sync_committee_bits) + or is_better_update(update, store.best_valid_update) ): store.best_valid_update = update @@ -282,7 +313,7 @@ def process_light_client_update(store: LightClientStore, # Update finalized header if ( sum(sync_committee_bits) * 3 >= len(sync_committee_bits) * 2 - and is_finality_update(update) + and update.finalized_header.slot > store.finalized_header.slot ): # Normal update through 2/3 threshold apply_light_client_update(store, update) diff --git a/tests/core/pyspec/eth2spec/test/altair/sync_protocol/__init__.py b/tests/core/pyspec/eth2spec/test/altair/sync_protocol/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_update_ranking.py b/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_update_ranking.py new file mode 100644 index 000000000..453d7235b --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_update_ranking.py @@ -0,0 +1,132 @@ +from eth2spec.test.context import ( + spec_state_test, + with_presets, + with_altair_and_later, +) +from eth2spec.test.helpers.attestations import ( + next_slots_with_attestations, + state_transition_with_full_block, +) +from eth2spec.test.helpers.constants import MINIMAL +from eth2spec.test.helpers.light_client import ( + get_sync_aggregate, + signed_block_to_header, +) +from eth2spec.test.helpers.state import ( + next_slots, +) +from eth2spec.test.helpers.merkle import build_proof +from math import floor + + +def create_update(spec, test, with_next_sync_committee, with_finality, participation_rate): + attested_state, attested_block, finalized_block = test + num_participants = floor(spec.SYNC_COMMITTEE_SIZE * participation_rate) + + attested_header = signed_block_to_header(spec, attested_block) + + if with_next_sync_committee: + next_sync_committee = attested_state.next_sync_committee + next_sync_committee_branch = build_proof(attested_state.get_backing(), spec.NEXT_SYNC_COMMITTEE_INDEX) + else: + next_sync_committee = spec.SyncCommittee() + next_sync_committee_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.NEXT_SYNC_COMMITTEE_INDEX))] + + if with_finality: + finalized_header = signed_block_to_header(spec, finalized_block) + finality_branch = build_proof(attested_state.get_backing(), spec.FINALIZED_ROOT_INDEX) + else: + finalized_header = spec.BeaconBlockHeader() + finality_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.FINALIZED_ROOT_INDEX))] + + sync_aggregate, signature_slot = get_sync_aggregate(spec, attested_state, num_participants) + + return spec.LightClientUpdate( + attested_header=attested_header, + next_sync_committee=next_sync_committee, + next_sync_committee_branch=next_sync_committee_branch, + finalized_header=finalized_header, + finality_branch=finality_branch, + sync_aggregate=sync_aggregate, + signature_slot=signature_slot, + ) + + +@with_altair_and_later +@spec_state_test +@with_presets([MINIMAL], reason="too slow") +def test_update_ranking(spec, state): + # Set up blocks and states: + # - `sig_finalized` / `sig_attested` --> Only signature in next sync committee period + # - `att_finalized` / `att_attested` --> Attested header also in next sync committee period + # - `fin_finalized` / `fin_attested` --> Finalized header also in next sync committee period + # - `lat_finalized` / `lat_attested` --> Like `fin`, but at a later `attested_header.slot` + next_slots(spec, state, spec.compute_start_slot_at_epoch(spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD - 3) - 1) + sig_finalized_block = state_transition_with_full_block(spec, state, True, True) + _, _, state = next_slots_with_attestations(spec, state, spec.SLOTS_PER_EPOCH - 1, True, True) + att_finalized_block = state_transition_with_full_block(spec, state, True, True) + _, _, state = next_slots_with_attestations(spec, state, 2 * spec.SLOTS_PER_EPOCH - 2, True, True) + sig_attested_block = state_transition_with_full_block(spec, state, True, True) + sig_attested_state = state.copy() + att_attested_block = state_transition_with_full_block(spec, state, True, True) + att_attested_state = state.copy() + fin_finalized_block = att_attested_block + _, _, state = next_slots_with_attestations(spec, state, 2 * spec.SLOTS_PER_EPOCH - 1, True, True) + fin_attested_block = state_transition_with_full_block(spec, state, True, True) + fin_attested_state = state.copy() + lat_finalized_block = fin_finalized_block + lat_attested_block = state_transition_with_full_block(spec, state, True, True) + lat_attested_state = state.copy() + sig = (sig_attested_state, sig_attested_block, sig_finalized_block) + att = (att_attested_state, att_attested_block, att_finalized_block) + fin = (fin_attested_state, fin_attested_block, fin_finalized_block) + lat = (lat_attested_state, lat_attested_block, lat_finalized_block) + + # Create updates (in descending order of quality) + updates = [ + # Updates with sync committee finality + create_update(spec, sig, with_next_sync_committee=0, with_finality=1, participation_rate=1.0), + create_update(spec, fin, with_next_sync_committee=1, with_finality=1, participation_rate=1.0), + create_update(spec, lat, with_next_sync_committee=1, with_finality=1, participation_rate=1.0), + create_update(spec, sig, with_next_sync_committee=0, with_finality=1, participation_rate=0.8), + create_update(spec, fin, with_next_sync_committee=1, with_finality=1, participation_rate=0.8), + create_update(spec, lat, with_next_sync_committee=1, with_finality=1, participation_rate=0.8), + + # Updates without sync committee finality + create_update(spec, att, with_next_sync_committee=1, with_finality=1, participation_rate=1.0), + create_update(spec, att, with_next_sync_committee=1, with_finality=1, participation_rate=0.8), + + # Updates without indication of any finality + create_update(spec, sig, with_next_sync_committee=0, with_finality=0, participation_rate=1.0), + create_update(spec, att, with_next_sync_committee=1, with_finality=0, participation_rate=1.0), + create_update(spec, fin, with_next_sync_committee=1, with_finality=0, participation_rate=1.0), + create_update(spec, lat, with_next_sync_committee=1, with_finality=0, participation_rate=1.0), + create_update(spec, sig, with_next_sync_committee=0, with_finality=0, participation_rate=0.8), + create_update(spec, att, with_next_sync_committee=1, with_finality=0, participation_rate=0.8), + create_update(spec, fin, with_next_sync_committee=1, with_finality=0, participation_rate=0.8), + create_update(spec, lat, with_next_sync_committee=1, with_finality=0, participation_rate=0.8), + + # Updates with low sync committee participation + create_update(spec, sig, with_next_sync_committee=0, with_finality=1, participation_rate=0.4), + create_update(spec, fin, with_next_sync_committee=1, with_finality=1, participation_rate=0.4), + create_update(spec, lat, with_next_sync_committee=1, with_finality=1, participation_rate=0.4), + create_update(spec, att, with_next_sync_committee=1, with_finality=1, participation_rate=0.4), + create_update(spec, sig, with_next_sync_committee=0, with_finality=0, participation_rate=0.4), + create_update(spec, att, with_next_sync_committee=1, with_finality=0, participation_rate=0.4), + create_update(spec, fin, with_next_sync_committee=1, with_finality=0, participation_rate=0.4), + create_update(spec, lat, with_next_sync_committee=1, with_finality=0, participation_rate=0.4), + + # Updates with very low sync committee participation + create_update(spec, sig, with_next_sync_committee=0, with_finality=1, participation_rate=0.2), + create_update(spec, fin, with_next_sync_committee=1, with_finality=1, participation_rate=0.2), + create_update(spec, lat, with_next_sync_committee=1, with_finality=1, participation_rate=0.2), + create_update(spec, att, with_next_sync_committee=1, with_finality=1, participation_rate=0.2), + create_update(spec, sig, with_next_sync_committee=0, with_finality=0, participation_rate=0.2), + create_update(spec, att, with_next_sync_committee=1, with_finality=0, participation_rate=0.2), + create_update(spec, fin, with_next_sync_committee=1, with_finality=0, participation_rate=0.2), + create_update(spec, lat, with_next_sync_committee=1, with_finality=0, participation_rate=0.2), + ] + yield "updates", updates + + for i in range(len(updates) - 1): + assert spec.is_better_update(updates[i], updates[i + 1]) diff --git a/tests/core/pyspec/eth2spec/test/helpers/light_client.py b/tests/core/pyspec/eth2spec/test/helpers/light_client.py index 090036094..144b9719c 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/light_client.py +++ b/tests/core/pyspec/eth2spec/test/helpers/light_client.py @@ -29,7 +29,7 @@ def initialize_light_client_store(spec, state): ) -def get_sync_aggregate(spec, state, signature_slot=None): +def get_sync_aggregate(spec, state, num_participants=None, signature_slot=None): # By default, the sync committee signs the previous slot if signature_slot is None: signature_slot = state.slot + 1 @@ -42,13 +42,18 @@ def get_sync_aggregate(spec, state, signature_slot=None): committee_indices = compute_committee_indices(spec, signature_state) committee_size = len(committee_indices) + # By default, use full participation + if num_participants is None: + num_participants = committee_size + assert committee_size >= num_participants >= 0 + # Compute sync aggregate - sync_committee_bits = [True] * committee_size + sync_committee_bits = [True] * num_participants + [False] * (committee_size - num_participants) sync_committee_signature = compute_aggregate_sync_committee_signature( spec, signature_state, signature_slot, - committee_indices, + committee_indices[:num_participants], ) sync_aggregate = spec.SyncAggregate( sync_committee_bits=sync_committee_bits, diff --git a/tests/formats/sync_protocol/README.md b/tests/formats/sync_protocol/README.md new file mode 100644 index 000000000..da93972d6 --- /dev/null +++ b/tests/formats/sync_protocol/README.md @@ -0,0 +1,6 @@ +# Light client sync protocol tests + +This series of tests provides reference test vectors for the light client sync protocol spec. + +Handlers: +- `update_ranking`: see [`LightClientUpdate` ranking test format](./update_ranking.md) diff --git a/tests/formats/sync_protocol/update_ranking.md b/tests/formats/sync_protocol/update_ranking.md new file mode 100644 index 000000000..fe73fb9df --- /dev/null +++ b/tests/formats/sync_protocol/update_ranking.md @@ -0,0 +1,21 @@ +# `LightClientUpdate` ranking tests + +This series of tests provides reference test vectors for validating that `LightClientUpdate` instances are ranked in a canonical order. + +## Test case format + +### `meta.yaml` + +```yaml +updates_count: int -- integer, decimal +``` + +### `updates_.ssz_snappy` + +A series of files, with `` in range `[0, updates_count)`, ordered by descending precedence according to `is_better_update` (best update at index 0). + +Each file is a SSZ-snappy encoded `LightClientUpdate`. + +## Condition + +A test-runner should load the provided `update` objects and verify that the local implementation ranks them in the same order. Note that the `update` objects are not restricted to a single sync committee period for the scope of this test. diff --git a/tests/generators/sync_protocol/README.md b/tests/generators/sync_protocol/README.md new file mode 100644 index 000000000..f3c8ea339 --- /dev/null +++ b/tests/generators/sync_protocol/README.md @@ -0,0 +1,5 @@ +# Light client sync protocol tests + +The purpose of this test-generator is to provide test-vectors for validating the correct implementation of the light client sync protocol. + +Test-format documentation can be found [here](../../formats/sync_protocol/README.md). diff --git a/tests/generators/sync_protocol/__init__.py b/tests/generators/sync_protocol/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/generators/sync_protocol/main.py b/tests/generators/sync_protocol/main.py new file mode 100644 index 000000000..0fe8e0fe1 --- /dev/null +++ b/tests/generators/sync_protocol/main.py @@ -0,0 +1,16 @@ +from eth2spec.test.helpers.constants import ALTAIR, BELLATRIX +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators + + +if __name__ == "__main__": + altair_mods = {key: 'eth2spec.test.altair.sync_protocol.test_' + key for key in [ + 'update_ranking', + ]} + bellatrix_mods = altair_mods + + all_mods = { + ALTAIR: altair_mods, + BELLATRIX: bellatrix_mods, + } + + run_state_test_generators(runner_name="sync_protocol", all_mods=all_mods) diff --git a/tests/generators/sync_protocol/requirements.txt b/tests/generators/sync_protocol/requirements.txt new file mode 100644 index 000000000..182248686 --- /dev/null +++ b/tests/generators/sync_protocol/requirements.txt @@ -0,0 +1,2 @@ +pytest>=4.4 +../../../[generator] From 55202a4f00b4122b08bd819b3603f6ccc1472c0c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 4 Jul 2022 23:06:42 +0800 Subject: [PATCH 033/105] Use the correct `TEST_REPORT_DIR` --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index e3ff8e455..a1c2c80cf 100644 --- a/Makefile +++ b/Makefile @@ -107,13 +107,13 @@ find_test: pyspec python3 -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.minimal --cov=eth2spec.altair.minimal --cov=eth2spec.bellatrix.minimal --cov=eth2spec.capella.minimal --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec citest: pyspec - mkdir -p tests/core/pyspec/test-reports/eth2spec; + mkdir -p $(TEST_REPORT_DIR); ifdef fork . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python3 -m pytest -n 4 --bls-type=milagro --fork=$(fork) --junitxml=eth2spec/test_results.xml eth2spec + python3 -m pytest -n 4 --bls-type=milagro --fork=$(fork) --junitxml=test-reports/test_results.xml eth2spec else . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python3 -m pytest -n 4 --bls-type=milagro --junitxml=eth2spec/test_results.xml eth2spec + python3 -m pytest -n 4 --bls-type=milagro --junitxml=test-reports/test_results.xml eth2spec endif From c67ad597f0c38f7eb83dea733b1bdf2490029bcb Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 8 Jul 2022 21:46:34 -0700 Subject: [PATCH 034/105] Reorder helpers to group ones operating on updates --- specs/altair/sync-protocol.md | 38 +++++++++++++++++------------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index 7864ac6e2..6f6e728e4 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -18,9 +18,9 @@ - [Helper functions](#helper-functions) - [`is_sync_committee_update`](#is_sync_committee_update) - [`is_finality_update`](#is_finality_update) - - [`get_subtree_index`](#get_subtree_index) - - [`get_safety_threshold`](#get_safety_threshold) - [`is_better_update`](#is_better_update) + - [`get_safety_threshold`](#get_safety_threshold) + - [`get_subtree_index`](#get_subtree_index) - [Light client state updates](#light-client-state-updates) - [`process_slot_for_light_client_store`](#process_slot_for_light_client_store) - [`validate_light_client_update`](#validate_light_client_update) @@ -111,23 +111,6 @@ def is_finality_update(update: LightClientUpdate) -> bool: return update.finality_branch != [Bytes32() for _ in range(floorlog2(FINALIZED_ROOT_INDEX))] ``` -### `get_subtree_index` - -```python -def get_subtree_index(generalized_index: GeneralizedIndex) -> uint64: - return uint64(generalized_index % 2**(floorlog2(generalized_index))) -``` - -### `get_safety_threshold` - -```python -def get_safety_threshold(store: LightClientStore) -> uint64: - return max( - store.previous_max_active_participants, - store.current_max_active_participants, - ) // 2 -``` - ### `is_better_update` ```python @@ -172,6 +155,23 @@ def is_better_update(new_update: LightClientUpdate, old_update: LightClientUpdat return new_update.signature_slot < old_update.signature_slot ``` +### `get_safety_threshold` + +```python +def get_safety_threshold(store: LightClientStore) -> uint64: + return max( + store.previous_max_active_participants, + store.current_max_active_participants, + ) // 2 +``` + +### `get_subtree_index` + +```python +def get_subtree_index(generalized_index: GeneralizedIndex) -> uint64: + return uint64(generalized_index % 2**(floorlog2(generalized_index))) +``` + ## Light client state updates A light client maintains its state in a `store` object of type `LightClientStore` and receives `update` objects of type `LightClientUpdate`. Every `update` triggers `process_light_client_update(store, update, current_slot, genesis_validators_root)` where `current_slot` is the current slot based on a local clock. `process_slot_for_light_client_store` is triggered every time the current slot increments. From aa8caee2e39bd618bc5baf7c35f978cc204657c1 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 8 Jul 2022 21:59:09 -0700 Subject: [PATCH 035/105] Add `compute_sync_committee_period_at_slot` --- specs/altair/sync-protocol.md | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index 6f6e728e4..44fe978f8 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -21,6 +21,7 @@ - [`is_better_update`](#is_better_update) - [`get_safety_threshold`](#get_safety_threshold) - [`get_subtree_index`](#get_subtree_index) + - [`compute_sync_committee_period_at_slot`](#compute_sync_committee_period_at_slot) - [Light client state updates](#light-client-state-updates) - [`process_slot_for_light_client_store`](#process_slot_for_light_client_store) - [`validate_light_client_update`](#validate_light_client_update) @@ -135,12 +136,12 @@ def is_better_update(new_update: LightClientUpdate, old_update: LightClientUpdat # Compare sync committee finality if new_has_finality: new_has_sync_committee_finality = ( - compute_sync_committee_period(compute_epoch_at_slot(new_update.finalized_header.slot)) == - compute_sync_committee_period(compute_epoch_at_slot(new_update.attested_header.slot)) + compute_sync_committee_period_at_slot(new_update.finalized_header.slot) == + compute_sync_committee_period_at_slot(new_update.attested_header.slot) ) old_has_sync_committee_finality = ( - compute_sync_committee_period(compute_epoch_at_slot(old_update.finalized_header.slot)) == - compute_sync_committee_period(compute_epoch_at_slot(old_update.attested_header.slot)) + compute_sync_committee_period_at_slot(old_update.finalized_header.slot) == + compute_sync_committee_period_at_slot(old_update.attested_header.slot) ) if new_has_sync_committee_finality != old_has_sync_committee_finality: return new_has_sync_committee_finality > old_has_sync_committee_finality @@ -172,6 +173,13 @@ def get_subtree_index(generalized_index: GeneralizedIndex) -> uint64: return uint64(generalized_index % 2**(floorlog2(generalized_index))) ``` +### `compute_sync_committee_period_at_slot` + +```python +def compute_sync_committee_period_at_slot(slot: Slot) -> uint64: + return compute_sync_committee_period(compute_epoch_at_slot(slot)) +``` + ## Light client state updates A light client maintains its state in a `store` object of type `LightClientStore` and receives `update` objects of type `LightClientUpdate`. Every `update` triggers `process_light_client_update(store, update, current_slot, genesis_validators_root)` where `current_slot` is the current slot based on a local clock. `process_slot_for_light_client_store` is triggered every time the current slot increments. @@ -207,12 +215,12 @@ def validate_light_client_update(store: LightClientStore, # Verify update does not skip a sync committee period assert current_slot >= update.signature_slot > update.attested_header.slot >= update.finalized_header.slot - store_period = compute_sync_committee_period(compute_epoch_at_slot(store.finalized_header.slot)) - signature_period = compute_sync_committee_period(compute_epoch_at_slot(update.signature_slot)) + store_period = compute_sync_committee_period_at_slot(store.finalized_header.slot) + signature_period = compute_sync_committee_period_at_slot(update.signature_slot) assert signature_period in (store_period, store_period + 1) # Verify update is relevant - attested_period = compute_sync_committee_period(compute_epoch_at_slot(update.attested_header.slot)) + attested_period = compute_sync_committee_period_at_slot(update.attested_header.slot) assert update.attested_header.slot > store.finalized_header.slot # Verify that the `finality_branch`, if present, confirms `finalized_header` @@ -269,8 +277,8 @@ def validate_light_client_update(store: LightClientStore, ```python def apply_light_client_update(store: LightClientStore, update: LightClientUpdate) -> None: - store_period = compute_sync_committee_period(compute_epoch_at_slot(store.finalized_header.slot)) - finalized_period = compute_sync_committee_period(compute_epoch_at_slot(update.finalized_header.slot)) + store_period = compute_sync_committee_period_at_slot(store.finalized_header.slot) + finalized_period = compute_sync_committee_period_at_slot(update.finalized_header.slot) if finalized_period == store_period + 1: store.current_sync_committee = store.next_sync_committee store.next_sync_committee = update.next_sync_committee From 35f5010de745ac1d96fe0c02389e374da43bbf5b Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 8 Jul 2022 22:01:56 -0700 Subject: [PATCH 036/105] More concise finality ranking Co-authored-by: Hsiao-Wei Wang --- specs/altair/sync-protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index 44fe978f8..ab9ffe8af 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -131,7 +131,7 @@ def is_better_update(new_update: LightClientUpdate, old_update: LightClientUpdat new_has_finality = is_finality_update(new_update) old_has_finality = is_finality_update(old_update) if new_has_finality != old_has_finality: - return new_has_finality > old_has_finality + return new_has_finality # Compare sync committee finality if new_has_finality: From fa85648385ebe8f6378779fa3822799d3ae3efb6 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 8 Jul 2022 22:03:31 -0700 Subject: [PATCH 037/105] `==` on new line Co-authored-by: Hsiao-Wei Wang --- specs/altair/sync-protocol.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index ab9ffe8af..b9290e789 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -136,12 +136,12 @@ def is_better_update(new_update: LightClientUpdate, old_update: LightClientUpdat # Compare sync committee finality if new_has_finality: new_has_sync_committee_finality = ( - compute_sync_committee_period_at_slot(new_update.finalized_header.slot) == - compute_sync_committee_period_at_slot(new_update.attested_header.slot) + compute_sync_committee_period_at_slot(new_update.finalized_header.slot) + == compute_sync_committee_period_at_slot(new_update.attested_header.slot) ) old_has_sync_committee_finality = ( - compute_sync_committee_period_at_slot(old_update.finalized_header.slot) == - compute_sync_committee_period_at_slot(old_update.attested_header.slot) + compute_sync_committee_period_at_slot(old_update.finalized_header.slot) + == compute_sync_committee_period_at_slot(old_update.attested_header.slot) ) if new_has_sync_committee_finality != old_has_sync_committee_finality: return new_has_sync_committee_finality > old_has_sync_committee_finality From 33f572c98de16b4881436045bdd19493c83d39db Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 8 Jul 2022 22:05:48 -0700 Subject: [PATCH 038/105] More concise sync committee finality ranking Co-authored-by: Hsiao-Wei Wang --- specs/altair/sync-protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index b9290e789..278f49ee0 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -144,7 +144,7 @@ def is_better_update(new_update: LightClientUpdate, old_update: LightClientUpdat == compute_sync_committee_period_at_slot(old_update.attested_header.slot) ) if new_has_sync_committee_finality != old_has_sync_committee_finality: - return new_has_sync_committee_finality > old_has_sync_committee_finality + return new_has_sync_committee_finality # Tiebreaker 1: Sync committee participation beyond supermajority if new_num_active_participants != old_num_active_participants: From 2ddb3833ff493638a6a329b582cb03c1431e9a61 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 8 Jul 2022 22:25:22 -0700 Subject: [PATCH 039/105] Extend documentation for forced updates --- specs/altair/sync-protocol.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index 278f49ee0..82de921e6 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -195,7 +195,10 @@ def process_slot_for_light_client_store(store: LightClientStore, current_slot: S current_slot > store.finalized_header.slot + UPDATE_TIMEOUT and store.best_valid_update is not None ): - # Forced best update when the update timeout has elapsed + # Forced best update when the update timeout has elapsed. + # Because the apply logic waits for `finalized_header.slot` to indicate sync committee finality, + # the `attested_header` may be treated as `finalized_header` in extended periods of non-finality + # to guarantee progression into later sync committee periods according to `is_better_update`. if store.best_valid_update.finalized_header.slot <= store.finalized_header.slot: store.best_valid_update.finalized_header = store.best_valid_update.attested_header apply_light_client_update(store, store.best_valid_update) From b7a0e532e476b3bbc8ffdebdaf06c172b7b911c4 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Sun, 10 Jul 2022 22:05:18 -0700 Subject: [PATCH 040/105] Prefix update `period` variables --- specs/altair/sync-protocol.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index 82de921e6..423414357 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -219,11 +219,11 @@ def validate_light_client_update(store: LightClientStore, # Verify update does not skip a sync committee period assert current_slot >= update.signature_slot > update.attested_header.slot >= update.finalized_header.slot store_period = compute_sync_committee_period_at_slot(store.finalized_header.slot) - signature_period = compute_sync_committee_period_at_slot(update.signature_slot) - assert signature_period in (store_period, store_period + 1) + update_signature_period = compute_sync_committee_period_at_slot(update.signature_slot) + assert update_signature_period in (store_period, store_period + 1) # Verify update is relevant - attested_period = compute_sync_committee_period_at_slot(update.attested_header.slot) + update_attested_period = compute_sync_committee_period_at_slot(update.attested_header.slot) assert update.attested_header.slot > store.finalized_header.slot # Verify that the `finality_branch`, if present, confirms `finalized_header` @@ -248,10 +248,10 @@ def validate_light_client_update(store: LightClientStore, # Verify that the `next_sync_committee`, if present, actually is the next sync committee saved in the # state of the `attested_header` if not is_sync_committee_update(update): - assert attested_period == store_period + assert update_attested_period == store_period assert update.next_sync_committee == SyncCommittee() else: - if attested_period == store_period: + if update_attested_period == store_period: assert update.next_sync_committee == store.next_sync_committee assert is_valid_merkle_branch( leaf=hash_tree_root(update.next_sync_committee), @@ -262,7 +262,7 @@ def validate_light_client_update(store: LightClientStore, ) # Verify sync committee aggregate signature - if signature_period == store_period: + if update_signature_period == store_period: sync_committee = store.current_sync_committee else: sync_committee = store.next_sync_committee @@ -281,8 +281,8 @@ def validate_light_client_update(store: LightClientStore, ```python def apply_light_client_update(store: LightClientStore, update: LightClientUpdate) -> None: store_period = compute_sync_committee_period_at_slot(store.finalized_header.slot) - finalized_period = compute_sync_committee_period_at_slot(update.finalized_header.slot) - if finalized_period == store_period + 1: + update_finalized_period = compute_sync_committee_period_at_slot(update.finalized_header.slot) + if update_finalized_period == store_period + 1: store.current_sync_committee = store.next_sync_committee store.next_sync_committee = update.next_sync_committee store.finalized_header = update.finalized_header From 654970c6057011e407299a61610c697662c335bd Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Mon, 2 May 2022 23:24:26 +0200 Subject: [PATCH 041/105] Add `LightClientBootstrap` Introduces a new `LightClientBootstrap` structure to allow setting up a `LightClientStore` with the initial sync committee and block header from a user-configured trusted block root. This leads to new cases where the `LightClientStore` is only aware of the current but not the next sync committee. As a side effect of these new cases, the store's `finalized_header` may now advance into the next sync committee period before a corresponding `LightClientUpdate` with the new sync committee is obtained, improving responsiveness. Note that so far, `LightClientUpdate.attested_header.slot` needed to be newer than `LightClientStore.finalized_header.slot`. However, it is now necessary to also consider certain older updates to try and backfill the `next_sync_committee`. The `is_better_update` helper is also updated to improve `best_valid_update` tracking. --- setup.py | 1 + specs/altair/sync-protocol.md | 124 +++++++++++++++--- .../test/altair/merkle/test_single_proof.py | 19 +++ .../sync_protocol/test_update_ranking.py | 46 +++++-- 4 files changed, 163 insertions(+), 27 deletions(-) diff --git a/setup.py b/setup.py index d83931850..973c49fd4 100644 --- a/setup.py +++ b/setup.py @@ -481,6 +481,7 @@ def get_generalized_index(ssz_class: Any, *path: Sequence[PyUnion[int, SSZVariab def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: constants = { 'FINALIZED_ROOT_INDEX': 'GeneralizedIndex(105)', + 'CURRENT_SYNC_COMMITTEE_INDEX': 'GeneralizedIndex(54)', 'NEXT_SYNC_COMMITTEE_INDEX': 'GeneralizedIndex(55)', } return {**super().hardcoded_ssz_dep_constants(), **constants} diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index 423414357..7a5905063 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -13,20 +13,24 @@ - [Preset](#preset) - [Misc](#misc) - [Containers](#containers) + - [`LightClientBootstrap`](#lightclientbootstrap) - [`LightClientUpdate`](#lightclientupdate) - [`LightClientStore`](#lightclientstore) - [Helper functions](#helper-functions) - [`is_sync_committee_update`](#is_sync_committee_update) - [`is_finality_update`](#is_finality_update) - [`is_better_update`](#is_better_update) + - [`is_next_sync_committee_known`](#is_next_sync_committee_known) - [`get_safety_threshold`](#get_safety_threshold) - [`get_subtree_index`](#get_subtree_index) - [`compute_sync_committee_period_at_slot`](#compute_sync_committee_period_at_slot) +- [Light client initialization](#light-client-initialization) + - [`initialize_light_client_store`](#initialize_light_client_store) - [Light client state updates](#light-client-state-updates) - - [`process_slot_for_light_client_store`](#process_slot_for_light_client_store) - - [`validate_light_client_update`](#validate_light_client_update) - - [`apply_light_client_update`](#apply_light_client_update) - - [`process_light_client_update`](#process_light_client_update) + - [`process_slot_for_light_client_store`](#process_slot_for_light_client_store) + - [`validate_light_client_update`](#validate_light_client_update) + - [`apply_light_client_update`](#apply_light_client_update) + - [`process_light_client_update`](#process_light_client_update) @@ -35,7 +39,7 @@ The beacon chain is designed to be light client friendly for constrained environments to access Ethereum with reasonable safety and liveness. -Such environments include resource-constrained devices (e.g. phones for trust-minimised wallets) +Such environments include resource-constrained devices (e.g. phones for trust-minimized wallets) and metered VMs (e.g. blockchain VMs for cross-chain bridges). This document suggests a minimal light client design for the beacon chain that @@ -46,6 +50,7 @@ uses sync committees introduced in [this beacon chain extension](./beacon-chain. | Name | Value | | - | - | | `FINALIZED_ROOT_INDEX` | `get_generalized_index(BeaconState, 'finalized_checkpoint', 'root')` (= 105) | +| `CURRENT_SYNC_COMMITTEE_INDEX` | `get_generalized_index(BeaconState, 'current_sync_committee')` (= 54) | | `NEXT_SYNC_COMMITTEE_INDEX` | `get_generalized_index(BeaconState, 'next_sync_committee')` (= 55) | ## Preset @@ -59,6 +64,17 @@ uses sync committees introduced in [this beacon chain extension](./beacon-chain. ## Containers +### `LightClientBootstrap` + +```python +class LightClientBootstrap(Container): + # The requested beacon block header + header: BeaconBlockHeader + # Current sync committee corresponding to `header` + current_sync_committee: SyncCommittee + current_sync_committee_branch: Vector[Bytes32, floorlog2(CURRENT_SYNC_COMMITTEE_INDEX)] +``` + ### `LightClientUpdate` ```python @@ -127,6 +143,18 @@ def is_better_update(new_update: LightClientUpdate, old_update: LightClientUpdat if not new_has_supermajority and new_num_active_participants != old_num_active_participants: return new_num_active_participants > old_num_active_participants + # Compare presence of relevant sync committee + new_has_relevant_sync_committee = is_sync_committee_update(new_update) and ( + compute_sync_committee_period_at_slot(new_update.attested_header.slot) + == compute_sync_committee_period_at_slot(new_update.signature_slot) + ) + old_has_relevant_sync_committee = is_sync_committee_update(old_update) and ( + compute_sync_committee_period_at_slot(old_update.attested_header.slot) + == compute_sync_committee_period_at_slot(old_update.signature_slot) + ) + if new_has_relevant_sync_committee != old_has_relevant_sync_committee: + return new_has_relevant_sync_committee > old_has_relevant_sync_committee + # Compare indication of any finality new_has_finality = is_finality_update(new_update) old_has_finality = is_finality_update(old_update) @@ -156,6 +184,13 @@ def is_better_update(new_update: LightClientUpdate, old_update: LightClientUpdat return new_update.signature_slot < old_update.signature_slot ``` +### `is_next_sync_committee_known` + +```python +def is_next_sync_committee_known(store: LightClientStore) -> bool: + return store.next_sync_committee != SyncCommittee() +``` + ### `get_safety_threshold` ```python @@ -180,11 +215,41 @@ def compute_sync_committee_period_at_slot(slot: Slot) -> uint64: return compute_sync_committee_period(compute_epoch_at_slot(slot)) ``` +## Light client initialization + +A light client maintains its state in a `store` object of type `LightClientStore`. `initialize_light_client_store` initializes a new `store` with a received `LightClientBootstrap` derived from a given `trusted_block_root`. + +### `initialize_light_client_store` + +```python +def initialize_light_client_store(trusted_block_root: Root, + bootstrap: LightClientBootstrap) -> LightClientStore: + assert hash_tree_root(bootstrap.header) == trusted_block_root + + assert is_valid_merkle_branch( + leaf=hash_tree_root(bootstrap.current_sync_committee), + branch=bootstrap.current_sync_committee_branch, + depth=floorlog2(CURRENT_SYNC_COMMITTEE_INDEX), + index=get_subtree_index(CURRENT_SYNC_COMMITTEE_INDEX), + root=bootstrap.header.state_root, + ) + + return LightClientStore( + finalized_header=bootstrap.header, + current_sync_committee=bootstrap.current_sync_committee, + next_sync_committee=SyncCommittee(), + best_valid_update=None, + optimistic_header=bootstrap.header, + previous_max_active_participants=0, + current_max_active_participants=0, + ) +``` + ## Light client state updates -A light client maintains its state in a `store` object of type `LightClientStore` and receives `update` objects of type `LightClientUpdate`. Every `update` triggers `process_light_client_update(store, update, current_slot, genesis_validators_root)` where `current_slot` is the current slot based on a local clock. `process_slot_for_light_client_store` is triggered every time the current slot increments. +A light client receives `update` objects of type `LightClientUpdate`. Every `update` triggers `process_light_client_update(store, update, current_slot, genesis_validators_root)` where `current_slot` is the current slot based on a local clock. `process_slot_for_light_client_store` is triggered every time the current slot increments. -#### `process_slot_for_light_client_store` +### `process_slot_for_light_client_store` ```python def process_slot_for_light_client_store(store: LightClientStore, current_slot: Slot) -> None: @@ -205,7 +270,7 @@ def process_slot_for_light_client_store(store: LightClientStore, current_slot: S store.best_valid_update = None ``` -#### `validate_light_client_update` +### `validate_light_client_update` ```python def validate_light_client_update(store: LightClientStore, @@ -220,11 +285,20 @@ def validate_light_client_update(store: LightClientStore, assert current_slot >= update.signature_slot > update.attested_header.slot >= update.finalized_header.slot store_period = compute_sync_committee_period_at_slot(store.finalized_header.slot) update_signature_period = compute_sync_committee_period_at_slot(update.signature_slot) - assert update_signature_period in (store_period, store_period + 1) + if is_next_sync_committee_known(store): + assert update_signature_period in (store_period, store_period + 1) + else: + assert update_signature_period == store_period # Verify update is relevant update_attested_period = compute_sync_committee_period_at_slot(update.attested_header.slot) - assert update.attested_header.slot > store.finalized_header.slot + assert ( + update.attested_header.slot > store.finalized_header.slot + or ( + not is_next_sync_committee_known(store) + and update_attested_period == store_period and is_sync_committee_update(update) + ) + ) # Verify that the `finality_branch`, if present, confirms `finalized_header` # to match the finalized checkpoint root saved in the state of `attested_header`. @@ -248,10 +322,9 @@ def validate_light_client_update(store: LightClientStore, # Verify that the `next_sync_committee`, if present, actually is the next sync committee saved in the # state of the `attested_header` if not is_sync_committee_update(update): - assert update_attested_period == store_period assert update.next_sync_committee == SyncCommittee() else: - if update_attested_period == store_period: + if update_attested_period == store_period and is_next_sync_committee_known(store): assert update.next_sync_committee == store.next_sync_committee assert is_valid_merkle_branch( leaf=hash_tree_root(update.next_sync_committee), @@ -276,21 +349,25 @@ def validate_light_client_update(store: LightClientStore, assert bls.FastAggregateVerify(participant_pubkeys, signing_root, sync_aggregate.sync_committee_signature) ``` -#### `apply_light_client_update` +### `apply_light_client_update` ```python def apply_light_client_update(store: LightClientStore, update: LightClientUpdate) -> None: store_period = compute_sync_committee_period_at_slot(store.finalized_header.slot) update_finalized_period = compute_sync_committee_period_at_slot(update.finalized_header.slot) - if update_finalized_period == store_period + 1: + if not is_next_sync_committee_known(store): + assert update_finalized_period == store_period + store.next_sync_committee = update.next_sync_committee + elif update_finalized_period == store_period + 1: store.current_sync_committee = store.next_sync_committee store.next_sync_committee = update.next_sync_committee - store.finalized_header = update.finalized_header - if store.finalized_header.slot > store.optimistic_header.slot: - store.optimistic_header = store.finalized_header + if update.finalized_header.slot > store.finalized_header.slot: + store.finalized_header = update.finalized_header + if store.finalized_header.slot > store.optimistic_header.slot: + store.optimistic_header = store.finalized_header ``` -#### `process_light_client_update` +### `process_light_client_update` ```python def process_light_client_update(store: LightClientStore, @@ -324,7 +401,16 @@ def process_light_client_update(store: LightClientStore, # Update finalized header if ( sum(sync_committee_bits) * 3 >= len(sync_committee_bits) * 2 - and update.finalized_header.slot > store.finalized_header.slot + and ( + update.finalized_header.slot > store.finalized_header.slot + or ( + not is_next_sync_committee_known(store) + and is_sync_committee_update(update) and is_finality_update(update) and ( + compute_sync_committee_period_at_slot(update.finalized_header.slot) + == compute_sync_committee_period_at_slot(update.attested_header.slot) + ) + ) + ) ): # Normal update through 2/3 threshold apply_light_client_update(store, update) diff --git a/tests/core/pyspec/eth2spec/test/altair/merkle/test_single_proof.py b/tests/core/pyspec/eth2spec/test/altair/merkle/test_single_proof.py index 31cdd13bb..d90feb146 100644 --- a/tests/core/pyspec/eth2spec/test/altair/merkle/test_single_proof.py +++ b/tests/core/pyspec/eth2spec/test/altair/merkle/test_single_proof.py @@ -5,6 +5,25 @@ from eth2spec.test.context import ( from eth2spec.test.helpers.merkle import build_proof +@with_altair_and_later +@spec_state_test +def test_current_sync_committee_merkle_proof(spec, state): + yield "state", state + current_sync_committee_branch = build_proof(state.get_backing(), spec.CURRENT_SYNC_COMMITTEE_INDEX) + yield "proof", { + "leaf": "0x" + state.current_sync_committee.hash_tree_root().hex(), + "leaf_index": spec.CURRENT_SYNC_COMMITTEE_INDEX, + "branch": ['0x' + root.hex() for root in current_sync_committee_branch] + } + assert spec.is_valid_merkle_branch( + leaf=state.current_sync_committee.hash_tree_root(), + branch=current_sync_committee_branch, + depth=spec.floorlog2(spec.CURRENT_SYNC_COMMITTEE_INDEX), + index=spec.get_subtree_index(spec.CURRENT_SYNC_COMMITTEE_INDEX), + root=state.hash_tree_root(), + ) + + @with_altair_and_later @spec_state_test def test_next_sync_committee_merkle_proof(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_update_ranking.py b/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_update_ranking.py index 453d7235b..4b73a15ba 100644 --- a/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_update_ranking.py +++ b/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_update_ranking.py @@ -85,10 +85,8 @@ def test_update_ranking(spec, state): # Create updates (in descending order of quality) updates = [ # Updates with sync committee finality - create_update(spec, sig, with_next_sync_committee=0, with_finality=1, participation_rate=1.0), create_update(spec, fin, with_next_sync_committee=1, with_finality=1, participation_rate=1.0), create_update(spec, lat, with_next_sync_committee=1, with_finality=1, participation_rate=1.0), - create_update(spec, sig, with_next_sync_committee=0, with_finality=1, participation_rate=0.8), create_update(spec, fin, with_next_sync_committee=1, with_finality=1, participation_rate=0.8), create_update(spec, lat, with_next_sync_committee=1, with_finality=1, participation_rate=0.8), @@ -97,34 +95,66 @@ def test_update_ranking(spec, state): create_update(spec, att, with_next_sync_committee=1, with_finality=1, participation_rate=0.8), # Updates without indication of any finality - create_update(spec, sig, with_next_sync_committee=0, with_finality=0, participation_rate=1.0), create_update(spec, att, with_next_sync_committee=1, with_finality=0, participation_rate=1.0), create_update(spec, fin, with_next_sync_committee=1, with_finality=0, participation_rate=1.0), create_update(spec, lat, with_next_sync_committee=1, with_finality=0, participation_rate=1.0), - create_update(spec, sig, with_next_sync_committee=0, with_finality=0, participation_rate=0.8), create_update(spec, att, with_next_sync_committee=1, with_finality=0, participation_rate=0.8), create_update(spec, fin, with_next_sync_committee=1, with_finality=0, participation_rate=0.8), create_update(spec, lat, with_next_sync_committee=1, with_finality=0, participation_rate=0.8), + # Updates with sync committee finality but no `next_sync_committee` + create_update(spec, sig, with_next_sync_committee=0, with_finality=1, participation_rate=1.0), + create_update(spec, fin, with_next_sync_committee=0, with_finality=1, participation_rate=1.0), + create_update(spec, lat, with_next_sync_committee=0, with_finality=1, participation_rate=1.0), + create_update(spec, sig, with_next_sync_committee=0, with_finality=1, participation_rate=0.8), + create_update(spec, fin, with_next_sync_committee=0, with_finality=1, participation_rate=0.8), + create_update(spec, lat, with_next_sync_committee=0, with_finality=1, participation_rate=0.8), + + # Updates without sync committee finality and also no `next_sync_committee` + create_update(spec, att, with_next_sync_committee=0, with_finality=1, participation_rate=1.0), + create_update(spec, att, with_next_sync_committee=0, with_finality=1, participation_rate=0.8), + + # Updates without indication of any finality nor `next_sync_committee` + create_update(spec, sig, with_next_sync_committee=0, with_finality=0, participation_rate=1.0), + create_update(spec, att, with_next_sync_committee=0, with_finality=0, participation_rate=1.0), + create_update(spec, fin, with_next_sync_committee=0, with_finality=0, participation_rate=1.0), + create_update(spec, lat, with_next_sync_committee=0, with_finality=0, participation_rate=1.0), + create_update(spec, sig, with_next_sync_committee=0, with_finality=0, participation_rate=0.8), + create_update(spec, att, with_next_sync_committee=0, with_finality=0, participation_rate=0.8), + create_update(spec, fin, with_next_sync_committee=0, with_finality=0, participation_rate=0.8), + create_update(spec, lat, with_next_sync_committee=0, with_finality=0, participation_rate=0.8), + # Updates with low sync committee participation - create_update(spec, sig, with_next_sync_committee=0, with_finality=1, participation_rate=0.4), create_update(spec, fin, with_next_sync_committee=1, with_finality=1, participation_rate=0.4), create_update(spec, lat, with_next_sync_committee=1, with_finality=1, participation_rate=0.4), create_update(spec, att, with_next_sync_committee=1, with_finality=1, participation_rate=0.4), - create_update(spec, sig, with_next_sync_committee=0, with_finality=0, participation_rate=0.4), create_update(spec, att, with_next_sync_committee=1, with_finality=0, participation_rate=0.4), create_update(spec, fin, with_next_sync_committee=1, with_finality=0, participation_rate=0.4), create_update(spec, lat, with_next_sync_committee=1, with_finality=0, participation_rate=0.4), + create_update(spec, sig, with_next_sync_committee=0, with_finality=1, participation_rate=0.4), + create_update(spec, fin, with_next_sync_committee=0, with_finality=1, participation_rate=0.4), + create_update(spec, lat, with_next_sync_committee=0, with_finality=1, participation_rate=0.4), + create_update(spec, att, with_next_sync_committee=0, with_finality=1, participation_rate=0.4), + create_update(spec, sig, with_next_sync_committee=0, with_finality=0, participation_rate=0.4), + create_update(spec, att, with_next_sync_committee=0, with_finality=0, participation_rate=0.4), + create_update(spec, fin, with_next_sync_committee=0, with_finality=0, participation_rate=0.4), + create_update(spec, lat, with_next_sync_committee=0, with_finality=0, participation_rate=0.4), # Updates with very low sync committee participation - create_update(spec, sig, with_next_sync_committee=0, with_finality=1, participation_rate=0.2), create_update(spec, fin, with_next_sync_committee=1, with_finality=1, participation_rate=0.2), create_update(spec, lat, with_next_sync_committee=1, with_finality=1, participation_rate=0.2), create_update(spec, att, with_next_sync_committee=1, with_finality=1, participation_rate=0.2), - create_update(spec, sig, with_next_sync_committee=0, with_finality=0, participation_rate=0.2), create_update(spec, att, with_next_sync_committee=1, with_finality=0, participation_rate=0.2), create_update(spec, fin, with_next_sync_committee=1, with_finality=0, participation_rate=0.2), create_update(spec, lat, with_next_sync_committee=1, with_finality=0, participation_rate=0.2), + create_update(spec, sig, with_next_sync_committee=0, with_finality=1, participation_rate=0.2), + create_update(spec, fin, with_next_sync_committee=0, with_finality=1, participation_rate=0.2), + create_update(spec, lat, with_next_sync_committee=0, with_finality=1, participation_rate=0.2), + create_update(spec, att, with_next_sync_committee=0, with_finality=1, participation_rate=0.2), + create_update(spec, sig, with_next_sync_committee=0, with_finality=0, participation_rate=0.2), + create_update(spec, att, with_next_sync_committee=0, with_finality=0, participation_rate=0.2), + create_update(spec, fin, with_next_sync_committee=0, with_finality=0, participation_rate=0.2), + create_update(spec, lat, with_next_sync_committee=0, with_finality=0, participation_rate=0.2), ] yield "updates", updates From 6048f1abf6ea64a546993f76b89fd35d28921702 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Tue, 12 Jul 2022 07:37:12 +0200 Subject: [PATCH 042/105] More concise finality ranking --- specs/altair/sync-protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index 7a5905063..144c5c980 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -153,7 +153,7 @@ def is_better_update(new_update: LightClientUpdate, old_update: LightClientUpdat == compute_sync_committee_period_at_slot(old_update.signature_slot) ) if new_has_relevant_sync_committee != old_has_relevant_sync_committee: - return new_has_relevant_sync_committee > old_has_relevant_sync_committee + return new_has_relevant_sync_committee # Compare indication of any finality new_has_finality = is_finality_update(new_update) From 221f3687bdb797cacae5f1ba3c88e8949e621247 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 13 Jul 2022 13:12:31 +0300 Subject: [PATCH 043/105] EIP-4844: Improve the cryptographic parts of the spec - Move more code into polynomial-commitments.md - Implement aggregated sidecar verification logic from PR #2915 - Rename `kzgs` to `kzg_commitments` Co-authored-by: Hsiao-Wei Wang --- specs/eip4844/beacon-chain.md | 32 +++--- specs/eip4844/p2p-interface.md | 10 +- specs/eip4844/polynomial-commitments.md | 92 ++++++++++++----- specs/eip4844/validator.md | 126 +++++++++++++++++------- 4 files changed, 181 insertions(+), 79 deletions(-) diff --git a/specs/eip4844/beacon-chain.md b/specs/eip4844/beacon-chain.md index 537b294c7..1ac4b879c 100644 --- a/specs/eip4844/beacon-chain.md +++ b/specs/eip4844/beacon-chain.md @@ -39,6 +39,7 @@ This upgrade adds blobs to the beacon chain as part of EIP-4844. | - | - | - | | `Blob` | `Vector[BLSFieldElement, FIELD_ELEMENTS_PER_BLOB]` | | | `VersionedHash` | `Bytes32` | | +| `KZGCommitment` | `Bytes48` | Same as BLS standard "is valid pubkey" check but also allows `0x00..00` for point-at-infinity | ## Constants @@ -78,18 +79,18 @@ class BeaconBlockBody(Container): sync_aggregate: SyncAggregate # Execution execution_payload: ExecutionPayload - blob_kzgs: List[KZGCommitment, MAX_BLOBS_PER_BLOCK] # [New in EIP-4844] + blob_kzg_commitments: List[KZGCommitment, MAX_BLOBS_PER_BLOCK] # [New in EIP-4844] ``` ## Helper functions ### Misc -#### `kzg_to_versioned_hash` +#### `kzg_commitment_to_versioned_hash` ```python -def kzg_to_versioned_hash(kzg: KZGCommitment) -> VersionedHash: - return BLOB_COMMITMENT_VERSION_KZG + hash(kzg)[1:] +def kzg_commitment_to_versioned_hash(kzg_commitment: KZGCommitment) -> VersionedHash: + return VERSIONED_HASH_VERSION_KZG + hash(kzg_commitment)[1:] ``` #### `tx_peek_blob_versioned_hashes` @@ -106,15 +107,16 @@ def tx_peek_blob_versioned_hashes(opaque_tx: Transaction) -> Sequence[VersionedH return [VersionedHash(opaque_tx[x:x+32]) for x in range(blob_versioned_hashes_offset, len(opaque_tx), 32)] ``` -#### `verify_kzgs_against_transactions` +#### `verify_kzg_commitments_against_transactions` ```python -def verify_kzgs_against_transactions(transactions: Sequence[Transaction], blob_kzgs: Sequence[KZGCommitment]) -> bool: - all_versioned_hashes = [] - for tx in transactions: - if tx[0] == BLOB_TX_TYPE: - all_versioned_hashes.extend(tx_peek_blob_versioned_hashes(tx)) - return all_versioned_hashes == [kzg_to_versioned_hash(kzg) for kzg in blob_kzgs] +def verify_kzg_commitments_against_transactions(transactions: Sequence[Transaction], + kzg_commitments: Sequence[KZGCommitment]) -> bool: + all_versioned_hashes = [] + for tx in transactions: + if tx[0] == BLOB_TX_TYPE: + all_versioned_hashes.extend(tx_peek_blob_versioned_hashes(tx)) + return all_versioned_hashes == [kzg_commitment_to_versioned_hash(commitment) for commitment in kzg_commitments] ``` ## Beacon chain state transition function @@ -130,14 +132,14 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_eth1_data(state, block.body) process_operations(state, block.body) process_sync_aggregate(state, block.body.sync_aggregate) - process_blob_kzgs(state, block.body) # [New in EIP-4844] + process_blob_kzg_commitments(state, block.body) # [New in EIP-4844] ``` -#### Blob KZGs +#### Blob KZG commitments ```python -def process_blob_kzgs(state: BeaconState, body: BeaconBlockBody): - assert verify_kzgs_against_transactions(body.execution_payload.transactions, body.blob_kzgs) +def process_blob_kzg_commitments(state: BeaconState, body: BeaconBlockBody): + assert verify_kzg_commitments_against_transactions(body.execution_payload.transactions, body.blob_kzg_commitments) ``` ## Testing diff --git a/specs/eip4844/p2p-interface.md b/specs/eip4844/p2p-interface.md index 7a447dbd4..2141d557a 100644 --- a/specs/eip4844/p2p-interface.md +++ b/specs/eip4844/p2p-interface.md @@ -103,9 +103,9 @@ In addition to the gossip validations for this topic from prior specifications, the following validations MUST pass before forwarding the `signed_beacon_block` on the network. Alias `block = signed_beacon_block.message`, `execution_payload = block.body.execution_payload`. - _[REJECT]_ The KZG commitments of the blobs are all correctly encoded compressed BLS G1 Points. - -- i.e. `all(bls.KeyValidate(commitment) for commitment in block.body.blob_kzgs)` + -- i.e. `all(bls.KeyValidate(commitment) for commitment in block.body.blob_kzg_commitments)` - _[REJECT]_ The KZG commitments correspond to the versioned hashes in the transactions list. - -- i.e. `verify_kzgs_against_transactions(block.body.execution_payload.transactions, block.body.blob_kzgs)` + -- i.e. `verify_kzg_commitments_against_transactions(block.body.execution_payload.transactions, block.body.blob_kzg_commitments)` ##### `blobs_sidecar` @@ -117,12 +117,14 @@ Alias `sidecar = signed_blobs_sidecar.message`. - _[REJECT]_ the `sidecar.blobs` are all well formatted, i.e. the `BLSFieldElement` in valid range (`x < BLS_MODULUS`). - _[REJECT]_ The KZG proof is a correctly encoded compressed BLS G1 Point -- i.e. `bls.KeyValidate(blobs_sidecar.kzg_aggregated_proof) - _[REJECT]_ the beacon proposer signature, `signed_blobs_sidecar.signature`, is valid -- i.e. -```python + +``` domain = get_domain(state, DOMAIN_BLOBS_SIDECAR, blobs_sidecar.beacon_block_slot // SLOTS_PER_EPOCH) signing_root = compute_signing_root(blobs_sidecar, domain) assert bls.Verify(proposer_pubkey, signing_root, signed_blob_header.signature) ``` - where `proposer_pubkey` is the pubkey of the beacon block proposer of `blobs_sidecar.beacon_block_slot` + +where `proposer_pubkey` is the pubkey of the beacon block proposer of `blobs_sidecar.beacon_block_slot` - _[IGNORE]_ The sidecar is the first sidecar with valid signature received for the `(proposer_index, sidecar.beacon_block_slot)` combination, where `proposer_index` is the validator index of the beacon block proposer of `blobs_sidecar.beacon_block_slot` diff --git a/specs/eip4844/polynomial-commitments.md b/specs/eip4844/polynomial-commitments.md index 5d1b86895..00ac76203 100644 --- a/specs/eip4844/polynomial-commitments.md +++ b/specs/eip4844/polynomial-commitments.md @@ -34,6 +34,8 @@ This document specifies basic polynomial operations and KZG polynomial commitmen | Name | SSZ equivalent | Description | | - | - | - | +| `G1Point` | `Bytes48` | | +| `G2Point` | `Bytes96` | | | `BLSFieldElement` | `uint256` | `x < BLS_MODULUS` | | `KZGCommitment` | `Bytes48` | Same as BLS standard "is valid pubkey" check but also allows `0x00..00` for point-at-infinity | | `KZGProof` | `Bytes48` | Same as for `KZGCommitment` | @@ -54,6 +56,7 @@ but reusing the `mainnet` settings in public networks is a critical security req | Name | Value | | - | - | +| `KZG_SETUP_G1` | `Vector[G1Point, FIELD_ELEMENTS_PER_BLOB]`, contents TBD | | `KZG_SETUP_G2` | `Vector[G2Point, FIELD_ELEMENTS_PER_BLOB]`, contents TBD | | `KZG_SETUP_LAGRANGE` | `Vector[KZGCommitment, FIELD_ELEMENTS_PER_BLOB]`, contents TBD | @@ -77,30 +80,47 @@ def bls_modular_inverse(x: BLSFieldElement) -> BLSFieldElement: ```python def div(x: BLSFieldElement, y: BLSFieldElement) -> BLSFieldElement: """Divide two field elements: `x` by `y`""" - return x * bls_modular_inverse(y) % BLS_MODULUS + return (int(x) * int(bls_modular_inverse(y))) % BLS_MODULUS ``` #### `lincomb` ```python -def lincomb(points: List[KZGCommitment], scalars: List[BLSFieldElement]) -> KZGCommitment: +def lincomb(points: Sequence[KZGCommitment], scalars: Sequence[BLSFieldElement]) -> KZGCommitment: """ BLS multiscalar multiplication. This function can be optimized using Pippenger's algorithm and variants. """ - r = bls.Z1 + assert len(points) == len(scalars) + result = bls.Z1 for x, a in zip(points, scalars): - r = bls.add(r, bls.multiply(x, a)) - return r + result = bls.add(result, bls.multiply(bls.bytes48_to_G1(x), a)) + return KZGCommitment(bls.G1_to_bytes48(result)) +``` + +#### `matrix_lincomb` + +```python +def matrix_lincomb(vectors: Sequence[Sequence[BLSFieldElement]], + scalars: Sequence[BLSFieldElement]) -> Sequence[BLSFieldElement]: + """ + Given a list of ``vectors``, interpret it as a 2D matrix and compute the linear combination + of each column with `scalars`: return the resulting vector. + """ + result = [0] * len(vectors[0]) + for v, s in zip(vectors, scalars): + for i, x in enumerate(v): + result[i] = (result[i] + int(s) * int(x)) % BLS_MODULUS + return [BLSFieldElement(x) for x in result] ``` ### KZG KZG core functions. These are also defined in EIP-4844 execution specs. -#### `blob_to_kzg` +#### `blob_to_kzg_commitment` ```python -def blob_to_kzg(blob: Blob) -> KZGCommitment: +def blob_to_kzg_commitment(blob: Blob) -> KZGCommitment: return lincomb(KZG_SETUP_LAGRANGE, blob) ``` @@ -108,39 +128,67 @@ def blob_to_kzg(blob: Blob) -> KZGCommitment: ```python def verify_kzg_proof(polynomial_kzg: KZGCommitment, - x: BLSFieldElement, + z: BLSFieldElement, y: BLSFieldElement, - quotient_kzg: KZGProof) -> bool: + kzg_proof: KZGProof) -> bool: """ - Verify KZG proof that ``p(x) == y`` where ``p(x)`` is the polynomial represented by ``polynomial_kzg``. + Verify KZG proof that ``p(z) == y`` where ``p(z)`` is the polynomial represented by ``polynomial_kzg``. """ - # Verify: P - y = Q * (X - x) - X_minus_x = bls.add(KZG_SETUP_G2[1], bls.multiply(bls.G2, BLS_MODULUS - x)) - P_minus_y = bls.add(polynomial_kzg, bls.multiply(bls.G1, BLS_MODULUS - y)) + # Verify: P - y = Q * (X - z) + X_minus_z = bls.add(bls.bytes96_to_G2(KZG_SETUP_G2[1]), bls.multiply(bls.G2, BLS_MODULUS - z)) + P_minus_y = bls.add(bls.bytes48_to_G1(polynomial_kzg), bls.multiply(bls.G1, BLS_MODULUS - y)) return bls.pairing_check([ [P_minus_y, bls.neg(bls.G2)], - [quotient_kzg, X_minus_x] + [bls.bytes48_to_G1(kzg_proof), X_minus_z] ]) ``` +#### `compute_kzg_proof` + +```python +def compute_kzg_proof(polynomial: Sequence[BLSFieldElement], z: BLSFieldElement) -> KZGProof: + """Compute KZG proof at point `z` with `polynomial` being in evaluation form""" + + # To avoid SSZ overflow/underflow, convert element into int + polynomial = [int(i) for i in polynomial] + z = int(z) + + # Shift our polynomial first (in evaluation form we can't handle the division remainder) + y = evaluate_polynomial_in_evaluation_form(polynomial, z) + polynomial_shifted = [(p - int(y)) % BLS_MODULUS for p in polynomial] + + # Make sure we won't divide by zero during division + assert z not in ROOTS_OF_UNITY + denominator_poly = [(x - z) % BLS_MODULUS for x in ROOTS_OF_UNITY] + + # Calculate quotient polynomial by doing point-by-point division + quotient_polynomial = [div(a, b) for a, b in zip(polynomial_shifted, denominator_poly)] + return KZGProof(lincomb(KZG_SETUP_LAGRANGE, quotient_polynomial)) +``` + ### Polynomials #### `evaluate_polynomial_in_evaluation_form` ```python -def evaluate_polynomial_in_evaluation_form(poly: List[BLSFieldElement], x: BLSFieldElement) -> BLSFieldElement: +def evaluate_polynomial_in_evaluation_form(polynomial: Sequence[BLSFieldElement], + z: BLSFieldElement) -> BLSFieldElement: """ - Evaluate a polynomial (in evaluation form) at an arbitrary point `x` + Evaluate a polynomial (in evaluation form) at an arbitrary point `z` Uses the barycentric formula: - f(x) = (1 - x**WIDTH) / WIDTH * sum_(i=0)^WIDTH (f(DOMAIN[i]) * DOMAIN[i]) / (x - DOMAIN[i]) + f(z) = (1 - z**WIDTH) / WIDTH * sum_(i=0)^WIDTH (f(DOMAIN[i]) * DOMAIN[i]) / (z - DOMAIN[i]) """ - width = len(poly) + width = len(polynomial) assert width == FIELD_ELEMENTS_PER_BLOB inverse_width = bls_modular_inverse(width) - for i in range(width): - r += div(poly[i] * ROOTS_OF_UNITY[i], (x - ROOTS_OF_UNITY[i])) - r = r * (pow(x, width, BLS_MODULUS) - 1) * inverse_width % BLS_MODULUS + # Make sure we won't divide by zero during division + assert z not in ROOTS_OF_UNITY - return r + result = 0 + for i in range(width): + result += div(int(polynomial[i]) * int(ROOTS_OF_UNITY[i]), (z - ROOTS_OF_UNITY[i])) + result = result * (pow(z, width, BLS_MODULUS) - 1) * inverse_width % BLS_MODULUS + return result ``` + diff --git a/specs/eip4844/validator.md b/specs/eip4844/validator.md index bd391ad83..216f38435 100644 --- a/specs/eip4844/validator.md +++ b/specs/eip4844/validator.md @@ -37,6 +37,31 @@ All behaviors and definitions defined in this document, and documents it extends All terminology, constants, functions, and protocol mechanics defined in the updated [Beacon Chain doc of EIP4844](./beacon-chain.md) are requisite for this document and used throughout. Please see related Beacon Chain doc before continuing and use them as a reference throughout. +## Custom types + +| Name | SSZ equivalent | Description | +| - | - | - | +| `Polynomial` | `List[BLSFieldElement, MAX_BLOBS_PER_BLOCK]` | a polynomial in evaluation form | + +## Containers + +### `BlobsAndCommmitments` + +```python +class BlobsAndCommmitments(Container): + blobs: List[Blob, MAX_BLOBS_PER_BLOCK] + kzg_commitments: List[KZGCommitment, MAX_BLOBS_PER_BLOCK] +``` + +### `PolynomialAndCommitment` + +```python +class PolynomialAndCommitment(Container): + polynomial: Polynomial + kzg_commitment: KZGCommitment +``` + + ## Helpers ### `is_data_available` @@ -66,7 +91,10 @@ def hash_to_bls_field(x: Container) -> BLSFieldElement: ### `compute_powers` ```python -def compute_powers(x: BLSFieldElement, n: uint64) -> List[BLSFieldElement]: +def compute_powers(x: BLSFieldElement, n: uint64) -> Sequence[BLSFieldElement]: + """ + Return ``x`` to power of [0, n-1]. + """ current_power = 1 powers = [] for _ in range(n): @@ -75,48 +103,66 @@ def compute_powers(x: BLSFieldElement, n: uint64) -> List[BLSFieldElement]: return powers ``` -### `vector_lincomb` +### `compute_aggregated_poly_and_commitment` ```python -def vector_lincomb(vectors: List[List[BLSFieldElement]], scalars: List[BLSFieldElement]) -> List[BLSFieldElement]: +def compute_aggregated_poly_and_commitment( + blobs: Sequence[BLSFieldElement], + kzg_commitments: Sequence[KZGCommitment]) -> Tuple[Polynomial, KZGCommitment]: """ - Given a list of vectors, compute the linear combination of each column with `scalars`, and return the resulting - vector. + Return the aggregated polynomial and aggregated KZG commitment. """ - r = [0]*len(vectors[0]) - for v, a in zip(vectors, scalars): - for i, x in enumerate(v): - r[i] = (r[i] + a * x) % BLS_MODULUS - return [BLSFieldElement(x) for x in r] + # Generate random linear combination challenges + r = hash_to_bls_field(BlobsAndCommmitments(blobs=blobs, kzg_commitments=kzg_commitments)) + r_powers = compute_powers(r, len(kzg_commitments)) + + # Create aggregated polynomial in evaluation form + aggregated_poly = Polynomial(matrix_lincomb(blobs, r_powers)) + + # Compute commitment to aggregated polynomial + aggregated_poly_commitment = KZGCommitment(lincomb(kzg_commitments, r_powers)) + + return aggregated_poly, aggregated_poly_commitment ``` ### `verify_blobs_sidecar` ```python def verify_blobs_sidecar(slot: Slot, beacon_block_root: Root, - expected_kzgs: Sequence[KZGCommitment], blobs_sidecar: BlobsSidecar) -> None: + expected_kzg_commitments: Sequence[KZGCommitment], blobs_sidecar: BlobsSidecar) -> bool: assert slot == blobs_sidecar.beacon_block_slot assert beacon_block_root == blobs_sidecar.beacon_block_root blobs = blobs_sidecar.blobs kzg_aggregated_proof = blobs_sidecar.kzg_aggregated_proof - assert len(expected_kzgs) == len(blobs) + assert len(expected_kzg_commitments) == len(blobs) - # Generate random linear combination challenges - r = hash_to_bls_field([blobs, expected_kzgs]) - r_powers = compute_powers(r, len(expected_kzgs)) - - # Compute commitment to aggregated polynomial - aggregated_poly_commitment = lincomb(expected_kzgs, r_powers) - - # Create aggregated polynomial in evaluation form - aggregated_poly = vector_lincomb(blobs, r_powers) + aggregated_poly, aggregated_poly_commitment = compute_aggregated_poly_and_commitment( + blobs, + expected_kzg_commitments, + ) # Generate challenge `x` and evaluate the aggregated polynomial at `x` - x = hash_to_bls_field([aggregated_poly, aggregated_poly_commitment]) + x = hash_to_bls_field( + PolynomialAndCommitment(polynomial=aggregated_poly, kzg_commitment=aggregated_poly_commitment) + ) + # Evaluate aggregated polynomial at `x` (evaluation function checks for div-by-zero) y = evaluate_polynomial_in_evaluation_form(aggregated_poly, x) # Verify aggregated proof - assert verify_kzg_proof(aggregated_poly_commitment, x, y, kzg_aggregated_proof) + return verify_kzg_proof(aggregated_poly_commitment, x, y, kzg_aggregated_proof) +``` + +### `compute_proof_from_blobs` + +```python +def compute_proof_from_blobs(blobs: Sequence[BLSFieldElement]) -> KZGProof: + commitments = [blob_to_kzg_commitment(blob) for blob in blobs] + aggregated_poly, aggregated_poly_commitment = compute_aggregated_poly_and_commitment(blobs, commitments) + x = hash_to_bls_field(PolynomialAndCommitment( + polynomial=aggregated_poly, + kzg_commitment=aggregated_poly_commitment, + )) + return compute_kzg_proof(aggregated_poly, x) ``` ## Beacon chain responsibilities @@ -128,26 +174,27 @@ Namely, the blob handling and the addition of `BlobsSidecar`. #### Constructing the `BeaconBlockBody` -##### Blob commitments +##### Blob KZG commitments After retrieving the execution payload from the execution engine as specified in Bellatrix, the blobs are retrieved and processed: -```python +``` # execution_payload = execution_engine.get_payload(payload_id) # block.body.execution_payload = execution_payload # ... -kzgs, blobs = get_blobs(payload_id) +blobs, blob_kzg_commitments = get_blobs(payload_id) # Optionally sanity-check that the KZG commitments match the versioned hashes in the transactions -assert verify_kzgs_against_transactions(execution_payload.transactions, kzgs) +assert verify_kzg_commitments_against_transactions(execution_payload.transactions, blob_kzg_commitments) # Optionally sanity-check that the KZG commitments match the blobs (as produced by the execution engine) -assert len(kzgs) == len(blobs) and [blob_to_kzg(blob) == kzg for blob, kzg in zip(blobs, kzgs)] +assert len(blob_kzg_commitments) == len(blobs) +assert [blob_to_kzg_commitment(blob) == commitment for blob, commitment in zip(blobs, blob_kzg_commitments)] # Update the block body -block.body.blob_kzgs = kzgs +block.body.blob_kzg_commitments = blob_kzg_commitments ``` The `blobs` should be held with the block in preparation of publishing. @@ -161,20 +208,23 @@ Implementers may also retrieve blobs individually per transaction. Before publishing a prepared beacon block proposal, the corresponding blobs are packaged into a sidecar object for distribution to the network: ```python -blobs_sidecar = BlobsSidecar( - beacon_block_root=hash_tree_root(beacon_block) - beacon_block_slot=beacon_block.slot - blobs=blobs, -) +def get_blobs_sidecar(block: BeaconBlock, blobs: Sequence[Blob]) -> BlobsSidecar: + return BlobsSidecar( + beacon_block_root=hash_tree_root(block), + beacon_block_slot=block.slot, + blobs=blobs, + kzg_aggregated_proof=compute_proof_from_blobs(blobs), + ) ``` And then signed: ```python -domain = get_domain(state, DOMAIN_BLOBS_SIDECAR, blobs_sidecar.beacon_block_slot / SLOTS_PER_EPOCH) -signing_root = compute_signing_root(blobs_sidecar, domain) -signature = bls.Sign(privkey, signing_root) -signed_blobs_sidecar = SignedBlobsSidecar(message=blobs_sidecar, signature=signature) +def get_signed_blobs_sidecar(state: BeaconState, blobs_sidecar: BlobsSidecar, privkey: int) -> SignedBlobsSidecar: + domain = get_domain(state, DOMAIN_BLOBS_SIDECAR, blobs_sidecar.beacon_block_slot // SLOTS_PER_EPOCH) + signing_root = compute_signing_root(blobs_sidecar, domain) + signature = bls.Sign(privkey, signing_root) + return SignedBlobsSidecar(message=blobs_sidecar, signature=signature) ``` This `signed_blobs_sidecar` is then published to the global `blobs_sidecar` topic as soon as the `beacon_block` is published. From 5b9bf41de6c31731cd3d9dca05bb7ccd90c612ca Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 13 Jul 2022 13:13:30 +0300 Subject: [PATCH 044/105] EIP-4844: Further improvements on the spec - Move constants around - Implement missing functions to make the spec executable Co-authored-by: Hsiao-Wei Wang --- specs/eip4844/beacon-chain.md | 83 ++++++++++++++++++++++--- specs/eip4844/fork.md | 51 ++++++++++++++- specs/eip4844/p2p-interface.md | 11 ---- specs/eip4844/polynomial-commitments.md | 4 +- specs/eip4844/validator.md | 16 +++-- 5 files changed, 138 insertions(+), 27 deletions(-) diff --git a/specs/eip4844/beacon-chain.md b/specs/eip4844/beacon-chain.md index 1ac4b879c..38c08453e 100644 --- a/specs/eip4844/beacon-chain.md +++ b/specs/eip4844/beacon-chain.md @@ -11,19 +11,22 @@ - [Introduction](#introduction) - [Custom types](#custom-types) - [Constants](#constants) + - [Blob](#blob) - [Domain types](#domain-types) +- [Preset](#preset) + - [Execution](#execution) - [Configuration](#configuration) - [Containers](#containers) - [Extended containers](#extended-containers) - [`BeaconBlockBody`](#beaconblockbody) - [Helper functions](#helper-functions) - [Misc](#misc) - - [`kzg_to_versioned_hash`](#kzg_to_versioned_hash) + - [`kzg_commitment_to_versioned_hash`](#kzg_commitment_to_versioned_hash) - [`tx_peek_blob_versioned_hashes`](#tx_peek_blob_versioned_hashes) - - [`verify_kzgs_against_transactions`](#verify_kzgs_against_transactions) + - [`verify_kzg_commitments_against_transactions`](#verify_kzg_commitments_against_transactions) - [Beacon chain state transition function](#beacon-chain-state-transition-function) - [Block processing](#block-processing) - - [Blob KZGs](#blob-kzgs) + - [Blob KZG commitments](#blob-kzg-commitments) - [Testing](#testing) @@ -43,10 +46,13 @@ This upgrade adds blobs to the beacon chain as part of EIP-4844. ## Constants +### Blob + | Name | Value | | - | - | | `BLOB_TX_TYPE` | `uint8(0x05)` | -| `FIELD_ELEMENTS_PER_BLOB` | `4096` | +| `FIELD_ELEMENTS_PER_BLOB` | `uint64(4096)` | +| `VERSIONED_HASH_VERSION_KZG` | `Bytes1(0x01)` | ### Domain types @@ -54,6 +60,14 @@ This upgrade adds blobs to the beacon chain as part of EIP-4844. | - | - | | `DOMAIN_BLOBS_SIDECAR` | `DomainType('0x0a000000')` | +## Preset + +### Execution + +| Name | Value | +| - | - | +| `MAX_BLOBS_PER_BLOCK` | `uint64(2**4)` (= 16) | + ## Configuration @@ -97,14 +111,21 @@ def kzg_commitment_to_versioned_hash(kzg_commitment: KZGCommitment) -> Versioned This function retrieves the hashes from the `SignedBlobTransaction` as defined in EIP-4844, using SSZ offsets. Offsets are little-endian `uint32` values, as defined in the [SSZ specification](../../ssz/simple-serialize.md). +See [the full details of `blob_versioned_hashes` offset calculation](https://gist.github.com/protolambda/23bd106b66f6d4bb854ce46044aa3ca3). ```python def tx_peek_blob_versioned_hashes(opaque_tx: Transaction) -> Sequence[VersionedHash]: assert opaque_tx[0] == BLOB_TX_TYPE message_offset = 1 + uint32.decode_bytes(opaque_tx[1:5]) # field offset: 32 + 8 + 32 + 32 + 8 + 4 + 32 + 4 + 4 = 156 - blob_versioned_hashes_offset = uint32.decode_bytes(opaque_tx[message_offset+156:message_offset+160]) - return [VersionedHash(opaque_tx[x:x+32]) for x in range(blob_versioned_hashes_offset, len(opaque_tx), 32)] + blob_versioned_hashes_offset = ( + message_offset + + uint32.decode_bytes(opaque_tx[(message_offset + 156):(message_offset + 160)]) + ) + return [ + VersionedHash(opaque_tx[x:(x + 32)]) + for x in range(blob_versioned_hashes_offset, len(opaque_tx), 32) + ] ``` #### `verify_kzg_commitments_against_transactions` @@ -147,9 +168,53 @@ def process_blob_kzg_commitments(state: BeaconState, body: BeaconBlockBody): *Note*: The function `initialize_beacon_state_from_eth1` is modified for pure EIP-4844 testing only. The `BeaconState` initialization is unchanged, except for the use of the updated `eip4844.BeaconBlockBody` type -when initializing the first body-root: +when initializing the first body-root. ```python -state.latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), -``` +def initialize_beacon_state_from_eth1(eth1_block_hash: Hash32, + eth1_timestamp: uint64, + deposits: Sequence[Deposit], + execution_payload_header: ExecutionPayloadHeader=ExecutionPayloadHeader() + ) -> BeaconState: + fork = Fork( + previous_version=EIP4844_FORK_VERSION, # [Modified in EIP-4844] for testing only + current_version=EIP4844_FORK_VERSION, # [Modified in EIP-4844] + epoch=GENESIS_EPOCH, + ) + state = BeaconState( + genesis_time=eth1_timestamp + GENESIS_DELAY, + fork=fork, + eth1_data=Eth1Data(block_hash=eth1_block_hash, deposit_count=uint64(len(deposits))), + latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), + randao_mixes=[eth1_block_hash] * EPOCHS_PER_HISTORICAL_VECTOR, # Seed RANDAO with Eth1 entropy + ) + # Process deposits + leaves = list(map(lambda deposit: deposit.data, deposits)) + for index, deposit in enumerate(deposits): + deposit_data_list = List[DepositData, 2**DEPOSIT_CONTRACT_TREE_DEPTH](*leaves[:index + 1]) + state.eth1_data.deposit_root = hash_tree_root(deposit_data_list) + process_deposit(state, deposit) + + # Process activations + for index, validator in enumerate(state.validators): + balance = state.balances[index] + validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) + if validator.effective_balance == MAX_EFFECTIVE_BALANCE: + validator.activation_eligibility_epoch = GENESIS_EPOCH + validator.activation_epoch = GENESIS_EPOCH + + # Set genesis validators root for domain separation and chain versioning + state.genesis_validators_root = hash_tree_root(state.validators) + + # Fill in sync committees + # Note: A duplicate committee is assigned for the current and next committee at genesis + state.current_sync_committee = get_next_sync_committee(state) + state.next_sync_committee = get_next_sync_committee(state) + + # Initialize the execution payload header + # If empty, will initialize a chain that has not yet gone through the Merge transition + state.latest_execution_payload_header = execution_payload_header + + return state +``` diff --git a/specs/eip4844/fork.md b/specs/eip4844/fork.md index ad1b00b79..6473f3c85 100644 --- a/specs/eip4844/fork.md +++ b/specs/eip4844/fork.md @@ -39,5 +39,54 @@ Note that for the pure EIP-4844 networks, we don't apply `upgrade_to_eip4844` si ### Upgrading the state -The `eip4844.BeaconState` format is equal to the `bellatrix.BeaconState` format, no upgrade has to be performed. +Since the `eip4844.BeaconState` format is equal to the `bellatrix.BeaconState` format, we only have to update `BeaconState.fork`. +```python +def upgrade_to_eip4844(pre: bellatrix.BeaconState) -> BeaconState: + # TODO: if Capella gets scheduled, add sync it with Capella.BeaconState + epoch = bellatrix.get_current_epoch(pre) + post = BeaconState( + # Versioning + genesis_time=pre.genesis_time, + genesis_validators_root=pre.genesis_validators_root, + slot=pre.slot, + fork=Fork( + previous_version=pre.fork.current_version, + current_version=EIP4844_FORK_VERSION, # [Modified in EIP4844] + epoch=epoch, + ), + # History + latest_block_header=pre.latest_block_header, + block_roots=pre.block_roots, + state_roots=pre.state_roots, + historical_roots=pre.historical_roots, + # Eth1 + eth1_data=pre.eth1_data, + eth1_data_votes=pre.eth1_data_votes, + eth1_deposit_index=pre.eth1_deposit_index, + # Registry + validators=pre.validators, + balances=pre.balances, + # Randomness + randao_mixes=pre.randao_mixes, + # Slashings + slashings=pre.slashings, + # Participation + previous_epoch_participation=pre.previous_epoch_participation, + current_epoch_participation=pre.current_epoch_participation, + # Finality + justification_bits=pre.justification_bits, + previous_justified_checkpoint=pre.previous_justified_checkpoint, + current_justified_checkpoint=pre.current_justified_checkpoint, + finalized_checkpoint=pre.finalized_checkpoint, + # Inactivity + inactivity_scores=pre.inactivity_scores, + # Sync + current_sync_committee=pre.current_sync_committee, + next_sync_committee=pre.next_sync_committee, + # Execution-layer + latest_execution_payload_header=pre.latest_execution_payload_header, + ) + + return post +``` diff --git a/specs/eip4844/p2p-interface.md b/specs/eip4844/p2p-interface.md index 2141d557a..0c6f7bf37 100644 --- a/specs/eip4844/p2p-interface.md +++ b/specs/eip4844/p2p-interface.md @@ -10,7 +10,6 @@ The specification of these changes continues in the same format as the network s - - [Preset](#preset) - [Configuration](#configuration) - [Containers](#containers) - [`BlobsSidecar`](#blobssidecar) @@ -32,13 +31,6 @@ The specification of these changes continues in the same format as the network s - -## Preset - -| Name | Value | -| - | - | -| `MAX_BLOBS_PER_BLOCK` | `uint64(2**4)` (= 16) | - ## Configuration | Name | Value | Description | @@ -46,8 +38,6 @@ The specification of these changes continues in the same format as the network s | `MAX_REQUEST_BLOBS_SIDECARS` | `2**7` (= 128) | Maximum number of blobs sidecars in a single request | | `MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS` | `2**13` (= 8192, ~1.2 months) | The minimum epoch range over which a node must serve blobs sidecars | - - ## Containers ### `BlobsSidecar` @@ -68,7 +58,6 @@ class SignedBlobsSidecar(Container): signature: BLSSignature ``` - ## The gossip domain: gossipsub Some gossip meshes are upgraded in the fork of EIP4844 to support upgraded types. diff --git a/specs/eip4844/polynomial-commitments.md b/specs/eip4844/polynomial-commitments.md index 00ac76203..f66e3eb2e 100644 --- a/specs/eip4844/polynomial-commitments.md +++ b/specs/eip4844/polynomial-commitments.md @@ -16,9 +16,11 @@ - [`bls_modular_inverse`](#bls_modular_inverse) - [`div`](#div) - [`lincomb`](#lincomb) + - [`matrix_lincomb`](#matrix_lincomb) - [KZG](#kzg) - - [`blob_to_kzg`](#blob_to_kzg) + - [`blob_to_kzg_commitment`](#blob_to_kzg_commitment) - [`verify_kzg_proof`](#verify_kzg_proof) + - [`compute_kzg_proof`](#compute_kzg_proof) - [Polynomials](#polynomials) - [`evaluate_polynomial_in_evaluation_form`](#evaluate_polynomial_in_evaluation_form) diff --git a/specs/eip4844/validator.md b/specs/eip4844/validator.md index 216f38435..213c3c000 100644 --- a/specs/eip4844/validator.md +++ b/specs/eip4844/validator.md @@ -10,16 +10,21 @@ - [Introduction](#introduction) - [Prerequisites](#prerequisites) +- [Custom types](#custom-types) +- [Containers](#containers) + - [`BlobsAndCommmitments`](#blobsandcommmitments) + - [`PolynomialAndCommitment`](#polynomialandcommitment) - [Helpers](#helpers) - [`is_data_available`](#is_data_available) - [`hash_to_bls_field`](#hash_to_bls_field) - [`compute_powers`](#compute_powers) - - [`vector_lincomb`](#vector_lincomb) + - [`compute_aggregated_poly_and_commitment`](#compute_aggregated_poly_and_commitment) - [`verify_blobs_sidecar`](#verify_blobs_sidecar) + - [`compute_proof_from_blobs`](#compute_proof_from_blobs) - [Beacon chain responsibilities](#beacon-chain-responsibilities) - [Block proposal](#block-proposal) - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody) - - [Blob commitments](#blob-commitments) + - [Blob KZG commitments](#blob-kzg-commitments) - [Beacon Block publishing time](#beacon-block-publishing-time) @@ -74,9 +79,10 @@ Without the sidecar the block may be processed further optimistically, but MUST NOT be considered valid until a valid `BlobsSidecar` has been downloaded. ```python -def is_data_available(slot: Slot, beacon_block_root: Root, kzgs: Sequence[KZGCommitment]): - sidecar = retrieve_blobs_sidecar(slot, beacon_block_root) # implementation dependent, raises an exception if not available - verify_blobs_sidecar(slot, beacon_block_root, kzgs, sidecar) +def is_data_available(slot: Slot, beacon_block_root: Root, blob_kzg_commitments: Sequence[KZGCommitment]) -> bool: + # `retrieve_blobs_sidecar` is implementation dependent, raises an exception if not available. + sidecar = retrieve_blobs_sidecar(slot, beacon_block_root) + return verify_blobs_sidecar(slot, beacon_block_root, blob_kzg_commitments, sidecar) ``` ### `hash_to_bls_field` From 567a25f88356fcfa4748c8c5b04e6d275dc31b75 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 13 Jul 2022 13:14:05 +0300 Subject: [PATCH 045/105] EIP-4844: Make the spec executable - Implement all the required glue code to make things executable - Implement a dummy KZG trusted setup Co-authored-by: Hsiao-Wei Wang --- .circleci/config.yml | 17 +++- .gitignore | 1 + Makefile | 2 +- configs/mainnet.yaml | 6 +- configs/minimal.yaml | 6 +- presets/mainnet/eip4844.yaml | 8 ++ presets/minimal/eip4844.yaml | 8 ++ setup.py | 79 +++++++++++++++--- tests/core/pyspec/eth2spec/test/conftest.py | 2 +- tests/core/pyspec/eth2spec/test/context.py | 12 ++- .../pyspec/eth2spec/test/helpers/constants.py | 10 ++- .../test/helpers/execution_payload.py | 6 +- .../eth2spec/test/helpers/fork_transition.py | 3 + .../pyspec/eth2spec/test/helpers/genesis.py | 6 +- .../pyspec/eth2spec/test/helpers/sharding.py | 81 +++++++++++++++++++ .../test/phase0/sanity/test_blocks.py | 1 + .../fork_choice/test_on_attestation.py | 4 +- tests/core/pyspec/eth2spec/utils/bls.py | 29 +++++++ tests/core/pyspec/eth2spec/utils/kzg.py | 80 ++++++++++++++++++ 19 files changed, 326 insertions(+), 35 deletions(-) create mode 100644 presets/mainnet/eip4844.yaml create mode 100644 presets/minimal/eip4844.yaml create mode 100644 tests/core/pyspec/eth2spec/test/helpers/sharding.py create mode 100644 tests/core/pyspec/eth2spec/utils/kzg.py diff --git a/.circleci/config.yml b/.circleci/config.yml index bdb3f5bc6..94065d0bb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -142,7 +142,19 @@ jobs: command: make citest fork=capella - store_test_results: path: tests/core/pyspec/test-reports - + test-eip4844: + docker: + - image: circleci/python:3.8 + working_directory: ~/specs-repo + steps: + - restore_cache: + key: v3-specs-repo-{{ .Branch }}-{{ .Revision }} + - restore_pyspec_cached_venv + - run: + name: Run py-tests + command: make citest fork=eip4844 + - store_test_results: + path: tests/core/pyspec/test-reports table_of_contents: docker: - image: circleci/node:10.16.3 @@ -260,6 +272,9 @@ workflows: - test-capella: requires: - install_pyspec_test + - test-eip4844: + requires: + - install_pyspec_test - table_of_contents - codespell - lint: diff --git a/.gitignore b/.gitignore index 101cb0b08..219251599 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ tests/core/pyspec/eth2spec/phase0/ tests/core/pyspec/eth2spec/altair/ tests/core/pyspec/eth2spec/bellatrix/ tests/core/pyspec/eth2spec/capella/ +tests/core/pyspec/eth2spec/eip4844/ # coverage reports .htmlcov diff --git a/Makefile b/Makefile index a1c2c80cf..6afe42993 100644 --- a/Makefile +++ b/Makefile @@ -136,7 +136,7 @@ codespell: lint: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ flake8 --config $(LINTER_CONFIG_FILE) ./eth2spec \ - && pylint --disable=all --enable unused-argument ./eth2spec/phase0 ./eth2spec/altair ./eth2spec/bellatrix \ + && pylint --disable=all --enable unused-argument ./eth2spec/phase0 ./eth2spec/altair ./eth2spec/bellatrix ./eth2spec/capella \ && mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.altair -p eth2spec.bellatrix -p eth2spec.capella lint_generators: pyspec diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 1c0a12d4e..e86f5b5c6 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -47,9 +47,9 @@ BELLATRIX_FORK_EPOCH: 18446744073709551615 # Capella CAPELLA_FORK_VERSION: 0x03000000 CAPELLA_FORK_EPOCH: 18446744073709551615 -# Sharding -SHARDING_FORK_VERSION: 0x04000000 -SHARDING_FORK_EPOCH: 18446744073709551615 +# EIP4844 +EIP4844_FORK_VERSION: 0x04000000 +EIP4844_FORK_EPOCH: 18446744073709551615 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index e619ee931..0917cfb57 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -46,9 +46,9 @@ BELLATRIX_FORK_EPOCH: 18446744073709551615 # Capella CAPELLA_FORK_VERSION: 0x03000001 CAPELLA_FORK_EPOCH: 18446744073709551615 -# Sharding -SHARDING_FORK_VERSION: 0x04000001 -SHARDING_FORK_EPOCH: 18446744073709551615 +# EIP4844 +EIP4844_FORK_VERSION: 0x04000001 +EIP4844_FORK_EPOCH: 18446744073709551615 # Time parameters diff --git a/presets/mainnet/eip4844.yaml b/presets/mainnet/eip4844.yaml new file mode 100644 index 000000000..dac58a9f0 --- /dev/null +++ b/presets/mainnet/eip4844.yaml @@ -0,0 +1,8 @@ +# Mainnet preset - Phase0 + +# Misc +# --------------------------------------------------------------- +# `uint64(4096)` +FIELD_ELEMENTS_PER_BLOB: 4096 +# `uint64(2**4)` (= 16) +MAX_BLOBS_PER_BLOCK: 16 \ No newline at end of file diff --git a/presets/minimal/eip4844.yaml b/presets/minimal/eip4844.yaml new file mode 100644 index 000000000..fbb676819 --- /dev/null +++ b/presets/minimal/eip4844.yaml @@ -0,0 +1,8 @@ +# Minimal preset - Phase0 + +# Misc +# --------------------------------------------------------------- +# [customized] +FIELD_ELEMENTS_PER_BLOB: 4 +# `uint64(2**4)` (= 16) +MAX_BLOBS_PER_BLOCK: 16 diff --git a/setup.py b/setup.py index d83931850..36400e2ab 100644 --- a/setup.py +++ b/setup.py @@ -45,6 +45,7 @@ PHASE0 = 'phase0' ALTAIR = 'altair' BELLATRIX = 'bellatrix' CAPELLA = 'capella' +EIP4844 = 'eip4844' # The helper functions that are used when defining constants @@ -208,6 +209,9 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str]) -> elif source.startswith("class"): class_name, parent_class = _get_class_info_from_source(source) # check consistency with spec + if class_name != current_name: + print('class_name', class_name, 'current_name', current_name) + assert class_name == current_name if parent_class: assert parent_class == "Container" @@ -230,7 +234,7 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str]) -> if not _is_constant_id(name): # Check for short type declarations - if value.startswith(("uint", "Bytes", "ByteList", "Union")): + if value.startswith(("uint", "Bytes", "ByteList", "Union", "Vector", "List")): custom_types[name] = value continue @@ -304,7 +308,7 @@ class SpecBuilder(ABC): @classmethod @abstractmethod - def hardcoded_custom_type_dep_constants(cls) -> Dict[str, str]: # TODO + def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]: # TODO """ The constants that are required for custom types. """ @@ -432,7 +436,7 @@ get_attesting_indices = cache_this( return {} @classmethod - def hardcoded_custom_type_dep_constants(cls) -> Dict[str, str]: + def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]: return {} @classmethod @@ -547,11 +551,11 @@ EXECUTION_ENGINE = NoopExecutionEngine()""" @classmethod - def hardcoded_custom_type_dep_constants(cls) -> str: + def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: constants = { - 'MAX_BYTES_PER_TRANSACTION': 'uint64(2**30)', + 'MAX_BYTES_PER_TRANSACTION': spec_object.preset_vars['MAX_BYTES_PER_TRANSACTION'].value, } - return {**super().hardcoded_custom_type_dep_constants(), **constants} + return {**super().hardcoded_custom_type_dep_constants(spec_object), **constants} # @@ -567,14 +571,57 @@ from eth2spec.bellatrix import {preset_name} as bellatrix ''' +# +# EIP4844SpecBuilder +# +class EIP4844SpecBuilder(BellatrixSpecBuilder): + fork: str = EIP4844 + + @classmethod + def imports(cls, preset_name: str): + return super().imports(preset_name) + f''' +from eth2spec.utils import kzg +from eth2spec.bellatrix import {preset_name} as bellatrix +''' + + @classmethod + def sundry_functions(cls) -> str: + return super().sundry_functions() + ''' +# TODO: for mainnet, load pre-generated trusted setup file to reduce building time. +# TESTING_FIELD_ELEMENTS_PER_BLOB is hardcoded copy from minimal presets +TESTING_FIELD_ELEMENTS_PER_BLOB = 4 +TESTING_SECRET = 1337 +TESTING_KZG_SETUP_G1 = kzg.generate_setup(bls.G1, TESTING_SECRET, TESTING_FIELD_ELEMENTS_PER_BLOB) +TESTING_KZG_SETUP_G2 = kzg.generate_setup(bls.G2, TESTING_SECRET, TESTING_FIELD_ELEMENTS_PER_BLOB) +TESTING_KZG_SETUP_LAGRANGE = kzg.get_lagrange(TESTING_KZG_SETUP_G1) + +KZG_SETUP_G1 = [bls.G1_to_bytes48(p) for p in TESTING_KZG_SETUP_G1] +KZG_SETUP_G2 = [bls.G2_to_bytes96(p) for p in TESTING_KZG_SETUP_G2] +KZG_SETUP_LAGRANGE = TESTING_KZG_SETUP_LAGRANGE +ROOTS_OF_UNITY = kzg.compute_roots_of_unity(TESTING_FIELD_ELEMENTS_PER_BLOB) + + +def retrieve_blobs_sidecar(slot: Slot, beacon_block_root: Root) -> BlobsSidecar: + pass''' + + @classmethod + def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: + constants = { + 'FIELD_ELEMENTS_PER_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_BLOB'].value, + 'MAX_BLOBS_PER_BLOCK': spec_object.preset_vars['MAX_BLOBS_PER_BLOCK'].value, + } + return {**super().hardcoded_custom_type_dep_constants(spec_object), **constants} + + + spec_builders = { builder.fork: builder - for builder in (Phase0SpecBuilder, AltairSpecBuilder, BellatrixSpecBuilder, CapellaSpecBuilder) + for builder in (Phase0SpecBuilder, AltairSpecBuilder, BellatrixSpecBuilder, CapellaSpecBuilder, EIP4844SpecBuilder) } def is_spec_defined_type(value: str) -> bool: - return value.startswith('ByteList') or value.startswith('Union') + return value.startswith(('ByteList', 'Union', 'Vector', 'List')) def objects_to_spec(preset_name: str, @@ -652,7 +699,7 @@ def objects_to_spec(preset_name: str, ordered_class_objects_spec = '\n\n\n'.join(ordered_class_objects.values()) ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, builder.hardcoded_ssz_dep_constants()[x]), builder.hardcoded_ssz_dep_constants())) ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), builder.hardcoded_ssz_dep_constants())) - custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, builder.hardcoded_custom_type_dep_constants()[x]), builder.hardcoded_custom_type_dep_constants())) + custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, builder.hardcoded_custom_type_dep_constants(spec_object)[x]), builder.hardcoded_custom_type_dep_constants(spec_object))) spec = ( builder.imports(preset_name) + builder.preparations() @@ -869,14 +916,14 @@ class PySpecCommand(Command): if len(self.md_doc_paths) == 0: print("no paths were specified, using default markdown file paths for pyspec" " build (spec fork: %s)" % self.spec_fork) - if self.spec_fork in (PHASE0, ALTAIR, BELLATRIX, CAPELLA): + if self.spec_fork in (PHASE0, ALTAIR, BELLATRIX, CAPELLA, EIP4844): self.md_doc_paths = """ specs/phase0/beacon-chain.md specs/phase0/fork-choice.md specs/phase0/validator.md specs/phase0/weak-subjectivity.md """ - if self.spec_fork in (ALTAIR, BELLATRIX, CAPELLA): + if self.spec_fork in (ALTAIR, BELLATRIX, CAPELLA, EIP4844): self.md_doc_paths += """ specs/altair/beacon-chain.md specs/altair/bls.md @@ -885,7 +932,7 @@ class PySpecCommand(Command): specs/altair/p2p-interface.md specs/altair/sync-protocol.md """ - if self.spec_fork in (BELLATRIX, CAPELLA): + if self.spec_fork in (BELLATRIX, CAPELLA, EIP4844): self.md_doc_paths += """ specs/bellatrix/beacon-chain.md specs/bellatrix/fork.md @@ -902,6 +949,14 @@ class PySpecCommand(Command): specs/capella/validator.md specs/capella/p2p-interface.md """ + if self.spec_fork == EIP4844: + self.md_doc_paths += """ + specs/eip4844/beacon-chain.md + specs/eip4844/fork.md + specs/eip4844/polynomial-commitments.md + specs/eip4844/p2p-interface.md + specs/eip4844/validator.md + """ if len(self.md_doc_paths) == 0: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) diff --git a/tests/core/pyspec/eth2spec/test/conftest.py b/tests/core/pyspec/eth2spec/test/conftest.py index b3c250c11..a5f19e20c 100644 --- a/tests/core/pyspec/eth2spec/test/conftest.py +++ b/tests/core/pyspec/eth2spec/test/conftest.py @@ -51,7 +51,7 @@ def pytest_addoption(parser): def _validate_fork_name(forks): for fork in forks: - if fork not in ALL_PHASES: + if fork not in set(ALL_PHASES): raise ValueError( f'The given --fork argument "{fork}" is not an available fork.' f' The available forks: {ALL_PHASES}' diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 06313c195..ec3032b28 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -7,13 +7,14 @@ from eth2spec.phase0 import mainnet as spec_phase0_mainnet, minimal as spec_phas from eth2spec.altair import mainnet as spec_altair_mainnet, minimal as spec_altair_minimal from eth2spec.bellatrix import mainnet as spec_bellatrix_mainnet, minimal as spec_bellatrix_minimal from eth2spec.capella import mainnet as spec_capella_mainnet, minimal as spec_capella_minimal +from eth2spec.eip4844 import mainnet as spec_eip4844_mainnet, minimal as spec_eip4844_minimal from eth2spec.utils import bls from .exceptions import SkippedTest from .helpers.constants import ( - PHASE0, ALTAIR, BELLATRIX, CAPELLA, + PHASE0, ALTAIR, BELLATRIX, CAPELLA, EIP4844, MINIMAL, MAINNET, - ALL_PHASES, FORKS_BEFORE_ALTAIR, FORKS_BEFORE_BELLATRIX, FORKS_BEFORE_CAPELLA, + ALL_PHASES, FORKS_BEFORE_ALTAIR, FORKS_BEFORE_BELLATRIX, ALL_FORK_UPGRADES, ) from .helpers.typing import SpecForkName, PresetBaseName @@ -76,12 +77,14 @@ spec_targets: Dict[PresetBaseName, Dict[SpecForkName, Spec]] = { ALTAIR: spec_altair_minimal, BELLATRIX: spec_bellatrix_minimal, CAPELLA: spec_capella_minimal, + EIP4844: spec_eip4844_minimal, }, MAINNET: { PHASE0: spec_phase0_mainnet, ALTAIR: spec_altair_mainnet, BELLATRIX: spec_bellatrix_mainnet, CAPELLA: spec_capella_mainnet, + EIP4844: spec_eip4844_mainnet }, } @@ -576,12 +579,13 @@ def is_post_bellatrix(spec): def is_post_capella(spec): - return spec.fork not in FORKS_BEFORE_CAPELLA + return spec.fork == CAPELLA with_altair_and_later = with_all_phases_except([PHASE0]) with_bellatrix_and_later = with_all_phases_except([PHASE0, ALTAIR]) -with_capella_and_later = with_all_phases_except([PHASE0, ALTAIR, BELLATRIX]) +with_capella_and_later = with_all_phases_except([PHASE0, ALTAIR, BELLATRIX, EIP4844]) +with_eip4844_and_later = with_all_phases_except([PHASE0, ALTAIR, BELLATRIX, CAPELLA]) def only_generator(reason): diff --git a/tests/core/pyspec/eth2spec/test/helpers/constants.py b/tests/core/pyspec/eth2spec/test/helpers/constants.py index 0bc6b2e08..b1463b97b 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/constants.py +++ b/tests/core/pyspec/eth2spec/test/helpers/constants.py @@ -14,9 +14,15 @@ CAPELLA = SpecForkName('capella') SHARDING = SpecForkName('sharding') CUSTODY_GAME = SpecForkName('custody_game') DAS = SpecForkName('das') +EIP4844 = SpecForkName('eip4844') -# The forks that pytest runs with. -ALL_PHASES = (PHASE0, ALTAIR, BELLATRIX, CAPELLA) +# The forks that pytest can run with. +ALL_PHASES = ( + # Formal forks + PHASE0, ALTAIR, BELLATRIX, CAPELLA, + # Experimental patches + EIP4844, +) # The forks that output to the test vectors. TESTGEN_FORKS = (PHASE0, ALTAIR, BELLATRIX) diff --git a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py index 30f7a0394..5225d4efe 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py @@ -1,4 +1,4 @@ -from eth2spec.test.helpers.constants import FORKS_BEFORE_CAPELLA +from eth2spec.test.context import is_post_capella def build_empty_execution_payload(spec, state, randao_mix=None): @@ -28,7 +28,7 @@ def build_empty_execution_payload(spec, state, randao_mix=None): block_hash=spec.Hash32(), transactions=empty_txs, ) - if spec.fork not in FORKS_BEFORE_CAPELLA: + if is_post_capella(spec): num_withdrawals = min(spec.MAX_WITHDRAWALS_PER_PAYLOAD, len(state.withdrawal_queue)) payload.withdrawals = state.withdrawal_queue[:num_withdrawals] @@ -55,7 +55,7 @@ def get_execution_payload_header(spec, execution_payload): block_hash=execution_payload.block_hash, transactions_root=spec.hash_tree_root(execution_payload.transactions) ) - if spec.fork not in FORKS_BEFORE_CAPELLA: + if is_post_capella(spec): payload_header.withdrawals_root = spec.hash_tree_root(execution_payload.withdrawals) return payload_header diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py b/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py index ca248d8a5..0280bc7fb 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py @@ -13,6 +13,7 @@ from eth2spec.test.helpers.constants import ( ALTAIR, BELLATRIX, CAPELLA, + EIP4844, ) from eth2spec.test.helpers.deposits import ( prepare_state_and_deposit, @@ -150,6 +151,8 @@ def do_fork(state, spec, post_spec, fork_epoch, with_block=True, operation_dict= state = post_spec.upgrade_to_bellatrix(state) elif post_spec.fork == CAPELLA: state = post_spec.upgrade_to_capella(state) + elif post_spec.fork == EIP4844: + state = post_spec.upgrade_to_eip4844(state) assert state.fork.epoch == fork_epoch diff --git a/tests/core/pyspec/eth2spec/test/helpers/genesis.py b/tests/core/pyspec/eth2spec/test/helpers/genesis.py index 83994c409..67ff2d4a8 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/genesis.py +++ b/tests/core/pyspec/eth2spec/test/helpers/genesis.py @@ -1,6 +1,6 @@ from eth2spec.test.helpers.constants import ( - ALTAIR, BELLATRIX, - FORKS_BEFORE_ALTAIR, FORKS_BEFORE_BELLATRIX, FORKS_BEFORE_CAPELLA, + ALTAIR, BELLATRIX, CAPELLA, + FORKS_BEFORE_ALTAIR, FORKS_BEFORE_BELLATRIX, ) from eth2spec.test.helpers.keys import pubkeys @@ -20,7 +20,7 @@ def build_mock_validator(spec, i: int, balance: int): effective_balance=min(balance - balance % spec.EFFECTIVE_BALANCE_INCREMENT, spec.MAX_EFFECTIVE_BALANCE) ) - if spec.fork not in FORKS_BEFORE_CAPELLA: + if spec.fork in (CAPELLA): validator.fully_withdrawn_epoch = spec.FAR_FUTURE_EPOCH return validator diff --git a/tests/core/pyspec/eth2spec/test/helpers/sharding.py b/tests/core/pyspec/eth2spec/test/helpers/sharding.py new file mode 100644 index 000000000..6c90153fc --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/sharding.py @@ -0,0 +1,81 @@ +import random +from eth2spec.utils.ssz.ssz_typing import ( + Container, + Bytes20, Bytes32, + ByteList, + List, + Union, + boolean, + uint256, uint64, +) +from eth2spec.utils.ssz.ssz_impl import serialize + + +# +# Containers from EIP-4844 +# +MAX_CALLDATA_SIZE = 2**24 +MAX_VERSIONED_HASHES_LIST_SIZE = 2**24 +MAX_ACCESS_LIST_STORAGE_KEYS = 2**24 +MAX_ACCESS_LIST_SIZE = 2**24 + + +class AccessTuple(Container): + address: Bytes20 # Address = Bytes20 + storage_keys: List[Bytes32, MAX_ACCESS_LIST_STORAGE_KEYS] + + +class ECDSASignature(Container): + y_parity: boolean + r: uint256 + s: uint256 + + +class BlobTransaction(Container): + chain_id: uint256 + nonce: uint64 + priority_fee_per_gas: uint256 + max_basefee_per_gas: uint256 + gas: uint64 + to: Union[None, Bytes20] # Address = Bytes20 + value: uint256 + data: ByteList[MAX_CALLDATA_SIZE] + access_list: List[AccessTuple, MAX_ACCESS_LIST_SIZE] + blob_versioned_hashes: List[Bytes32, MAX_VERSIONED_HASHES_LIST_SIZE] + + +class SignedBlobTransaction(Container): + message: BlobTransaction + signature: ECDSASignature + + +def get_sample_blob(spec, rng=None): + if rng is None: + rng = random.Random(5566) + + return spec.Blob([ + rng.randint(0, spec.BLS_MODULUS - 1) + for _ in range(spec.FIELD_ELEMENTS_PER_BLOB) + ]) + + +def get_sample_opaque_tx(spec, blob_count=1, rng=None): + blobs = [] + blob_kzg_commitments = [] + blob_versioned_hashes = [] + for _ in range(blob_count): + blob = get_sample_blob(spec, rng) + blob_commitment = spec.KZGCommitment(spec.blob_to_kzg_commitment(blob)) + blob_versioned_hash = spec.kzg_commitment_to_versioned_hash(blob_commitment) + blobs.append(blob) + blob_kzg_commitments.append(blob_commitment) + blob_versioned_hashes.append(blob_versioned_hash) + + signed_blob_tx = SignedBlobTransaction( + message=BlobTransaction( + blob_versioned_hashes=blob_versioned_hashes, + ) + ) + serialized_tx = serialize(signed_blob_tx) + opaque_tx = spec.uint_to_bytes(spec.BLOB_TX_TYPE) + serialized_tx + return opaque_tx, blobs, blob_kzg_commitments diff --git a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py index 44d663422..907e73b8d 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py @@ -45,6 +45,7 @@ from eth2spec.test.context import ( @with_all_phases @spec_state_test def test_prev_slot_block_transition(spec, state): + print('spec.fork', spec.fork) # Go to clean slot spec.process_slots(state, state.slot + 1) # Make a block for it diff --git a/tests/core/pyspec/eth2spec/test/phase0/unittests/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/phase0/unittests/fork_choice/test_on_attestation.py index 4e8d4bbaa..6e545cef7 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/unittests/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase0/unittests/fork_choice/test_on_attestation.py @@ -1,7 +1,7 @@ from eth2spec.test.context import with_all_phases, spec_state_test from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.attestations import get_valid_attestation, sign_attestation -from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA +from eth2spec.test.helpers.constants import ALL_PHASES from eth2spec.test.helpers.state import transition_to, state_transition_and_sign_block, next_epoch, next_slot from eth2spec.test.helpers.fork_choice import get_genesis_forkchoice_store @@ -19,7 +19,7 @@ def run_on_attestation(spec, state, store, attestation, valid=True): spec.on_attestation(store, attestation) sample_index = indexed_attestation.attesting_indices[0] - if spec.fork in (PHASE0, ALTAIR, BELLATRIX, CAPELLA): + if spec.fork in ALL_PHASES: latest_message = spec.LatestMessage( epoch=attestation.data.target.epoch, root=attestation.data.beacon_block_root, diff --git a/tests/core/pyspec/eth2spec/utils/bls.py b/tests/core/pyspec/eth2spec/utils/bls.py index 9211e0ff0..e33017ade 100644 --- a/tests/core/pyspec/eth2spec/utils/bls.py +++ b/tests/core/pyspec/eth2spec/utils/bls.py @@ -1,5 +1,25 @@ from py_ecc.bls import G2ProofOfPossession as py_ecc_bls from py_ecc.bls.g2_primatives import signature_to_G2 as _signature_to_G2 +from py_ecc.optimized_bls12_381 import ( # noqa: F401 + G1, + G2, + Z1, + Z2, + add, + multiply, + neg, + pairing, + final_exponentiate, + FQ12 +) +from py_ecc.bls.g2_primitives import ( # noqa: F401 + G1_to_pubkey as G1_to_bytes48, + pubkey_to_G1 as bytes48_to_G1, + G2_to_signature as G2_to_bytes96, + signature_to_G2 as bytes96_to_G2, +) + + import milagro_bls_binding as milagro_bls # noqa: F401 for BLS switching option # Flag to make BLS active or not. Used for testing, do not ignore BLS in production unless you know what you are doing. @@ -109,3 +129,12 @@ def SkToPk(SK): return bls.SkToPk(SK) else: return bls.SkToPk(SK.to_bytes(32, 'big')) + + +def pairing_check(values): + p_q_1, p_q_2 = values + final_exponentiation = final_exponentiate( + pairing(p_q_1[1], p_q_1[0], final_exponentiate=False) + * pairing(p_q_2[1], p_q_2[0], final_exponentiate=False) + ) + return final_exponentiation == FQ12.one() diff --git a/tests/core/pyspec/eth2spec/utils/kzg.py b/tests/core/pyspec/eth2spec/utils/kzg.py new file mode 100644 index 000000000..e174e69ab --- /dev/null +++ b/tests/core/pyspec/eth2spec/utils/kzg.py @@ -0,0 +1,80 @@ +# Ref: +# - https://github.com/ethereum/research/blob/8f084630528ba33d92b2bc05edf5338dd193c6f1/trusted_setup/trusted_setup.py +# - https://github.com/asn-d6/kzgverify +from py_ecc.optimized_bls12_381 import ( # noqa: F401 + G1, + G2, + Z1, + Z2, + curve_order as BLS_MODULUS, + add, + multiply, + neg, +) +from eth2spec.utils import bls + + +PRIMITIVE_ROOT_OF_UNITY = 7 + + +def generate_setup(generator, secret, length): + """ + Generate trusted setup of ``generator`` in ``length``. + """ + result = [generator] + for _ in range(1, length): + result.append(multiply(result[-1], secret)) + return tuple(result) + + +def fft(vals, modulus, domain): + """ + FFT for group elements + """ + if len(vals) == 1: + return vals + L = fft(vals[::2], modulus, domain[::2]) + R = fft(vals[1::2], modulus, domain[::2]) + o = [0] * len(vals) + for i, (x, y) in enumerate(zip(L, R)): + y_times_root = multiply(y, domain[i]) + o[i] = add(x, y_times_root) + o[i + len(L)] = add(x, neg(y_times_root)) + return o + + +def compute_root_of_unity(length) -> int: + """ + Generate a w such that ``w**length = 1``. + """ + assert (BLS_MODULUS - 1) % length == 0 + return pow(PRIMITIVE_ROOT_OF_UNITY, (BLS_MODULUS - 1) // length, BLS_MODULUS) + + +def compute_roots_of_unity(field_elements_per_blob): + """ + Compute a list of roots of unity for a given order. + The order must divide the BLS multiplicative group order, i.e. BLS_MODULUS - 1 + """ + assert (BLS_MODULUS - 1) % field_elements_per_blob == 0 + root_of_unity = compute_root_of_unity(length=field_elements_per_blob) + + roots = [] + current_root_of_unity = 1 + for _ in range(field_elements_per_blob): + roots.append(current_root_of_unity) + current_root_of_unity = current_root_of_unity * root_of_unity % BLS_MODULUS + return roots + + +def get_lagrange(setup): + """ + Convert a G1 or G2 portion of a setup into the Lagrange basis. + """ + root_of_unity = compute_root_of_unity(len(setup)) + assert pow(root_of_unity, len(setup), BLS_MODULUS) == 1 + domain = [pow(root_of_unity, i, BLS_MODULUS) for i in range(len(setup))] + # TODO: introduce an IFFT function for simplicity + fft_output = fft(setup, BLS_MODULUS, domain) + inv_length = pow(len(setup), BLS_MODULUS - 2, BLS_MODULUS) + return [bls.G1_to_bytes48(multiply(fft_output[-i], inv_length)) for i in range(len(fft_output))] From 2558282e6dd051c2d352921884976140a684c6a7 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 13 Jul 2022 13:14:31 +0300 Subject: [PATCH 046/105] EIP-4844: Introduce KZG and blob verification unittests Co-authored-by: Hsiao-Wei Wang --- .../eth2spec/test/eip4844/sanity/__init__.py | 0 .../test/eip4844/sanity/test_blocks.py | 43 +++++++++++++ .../test/eip4844/unittests/__init__.py | 0 .../test/eip4844/unittests/test_kzg.py | 21 +++++++ .../eip4844/unittests/validator/__init__.py | 0 .../unittests/validator/test_validator.py | 62 +++++++++++++++++++ 6 files changed, 126 insertions(+) create mode 100644 tests/core/pyspec/eth2spec/test/eip4844/sanity/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/eip4844/sanity/test_blocks.py create mode 100644 tests/core/pyspec/eth2spec/test/eip4844/unittests/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/eip4844/unittests/test_kzg.py create mode 100644 tests/core/pyspec/eth2spec/test/eip4844/unittests/validator/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/eip4844/unittests/validator/test_validator.py diff --git a/tests/core/pyspec/eth2spec/test/eip4844/sanity/__init__.py b/tests/core/pyspec/eth2spec/test/eip4844/sanity/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/eip4844/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/eip4844/sanity/test_blocks.py new file mode 100644 index 000000000..f08a7fda9 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/eip4844/sanity/test_blocks.py @@ -0,0 +1,43 @@ +from eth2spec.test.helpers.state import ( + state_transition_and_sign_block +) +from eth2spec.test.helpers.block import ( + build_empty_block_for_next_slot +) +from eth2spec.test.context import ( + spec_state_test, + with_eip4844_and_later, +) +from eth2spec.test.helpers.sharding import ( + get_sample_opaque_tx, +) + + +@with_eip4844_and_later +@spec_state_test +def test_one_blob(spec, state): + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + opaque_tx, _, blob_kzg_commitments = get_sample_opaque_tx(spec) + block.body.blob_kzg_commitments = blob_kzg_commitments + block.body.execution_payload.transactions = [opaque_tx] + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state + + +@with_eip4844_and_later +@spec_state_test +def test_multiple_blobs(spec, state): + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + opaque_tx, _, blob_kzg_commitments = get_sample_opaque_tx(spec, blob_count=5) + block.body.blob_kzg_commitments = blob_kzg_commitments + block.body.execution_payload.transactions = [opaque_tx] + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state diff --git a/tests/core/pyspec/eth2spec/test/eip4844/unittests/__init__.py b/tests/core/pyspec/eth2spec/test/eip4844/unittests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/eip4844/unittests/test_kzg.py b/tests/core/pyspec/eth2spec/test/eip4844/unittests/test_kzg.py new file mode 100644 index 000000000..7474707b9 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/eip4844/unittests/test_kzg.py @@ -0,0 +1,21 @@ + +from eth2spec.test.helpers.constants import ( + EIP4844, + MINIMAL, +) +from eth2spec.test.helpers.sharding import ( + get_sample_blob, +) +from eth2spec.test.context import ( + with_phases, + spec_state_test, + with_presets, +) + + +@with_phases([EIP4844]) +@spec_state_test +@with_presets([MINIMAL]) +def test_blob_to_kzg_commitment(spec, state): + blob = get_sample_blob(spec) + spec.blob_to_kzg_commitment(blob) diff --git a/tests/core/pyspec/eth2spec/test/eip4844/unittests/validator/__init__.py b/tests/core/pyspec/eth2spec/test/eip4844/unittests/validator/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/eip4844/unittests/validator/test_validator.py b/tests/core/pyspec/eth2spec/test/eip4844/unittests/validator/test_validator.py new file mode 100644 index 000000000..bd6374f3f --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/eip4844/unittests/validator/test_validator.py @@ -0,0 +1,62 @@ +from eth2spec.test.helpers.state import ( + state_transition_and_sign_block, +) +from eth2spec.test.helpers.block import ( + build_empty_block_for_next_slot +) +from eth2spec.test.context import ( + spec_state_test, + with_eip4844_and_later, +) +from eth2spec.test.helpers.sharding import ( + get_sample_opaque_tx, + get_sample_blob, +) +from eth2spec.test.helpers.keys import privkeys + + +@with_eip4844_and_later +@spec_state_test +def test_verify_kzg_proof(spec, state): + x = 3 + polynomial = get_sample_blob(spec) + polynomial = [int(i) for i in polynomial] + commitment = spec.blob_to_kzg_commitment(polynomial) + + # Get the proof + proof = spec.compute_kzg_proof(polynomial, x) + + y = spec.evaluate_polynomial_in_evaluation_form(polynomial, x) + assert spec.verify_kzg_proof(commitment, x, y, proof) + + +def _run_verify_blobs_sidecar_test(spec, state, blob_count): + block = build_empty_block_for_next_slot(spec, state) + opaque_tx, blobs, blob_kzg_commitments = get_sample_opaque_tx(spec, blob_count=blob_count) + block.body.blob_kzg_commitments = blob_kzg_commitments + block.body.execution_payload.transactions = [opaque_tx] + state_transition_and_sign_block(spec, state, block) + + blobs_sidecar = spec.get_blobs_sidecar(block, blobs) + privkey = privkeys[1] + spec.get_signed_blobs_sidecar(state, blobs_sidecar, privkey) + expected_commitments = [spec.blob_to_kzg_commitment(blobs[i]) for i in range(blob_count)] + assert spec.verify_blobs_sidecar(block.slot, block.hash_tree_root(), expected_commitments, blobs_sidecar) + + +@with_eip4844_and_later +@spec_state_test +def test_verify_blobs_sidecar_one_blob(spec, state): + _run_verify_blobs_sidecar_test(spec, state, blob_count=1) + + +@with_eip4844_and_later +@spec_state_test +def test_verify_blobs_sidecar_two_blobs(spec, state): + _run_verify_blobs_sidecar_test(spec, state, blob_count=2) + + +@with_eip4844_and_later +@spec_state_test +def test_verify_blobs_sidecar_ten_blobs(spec, state): + _run_verify_blobs_sidecar_test(spec, state, blob_count=10) From 66325d0e9bbd0bd72588102a4aa3d34cf65df3da Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Wed, 13 Jul 2022 14:08:51 +0200 Subject: [PATCH 047/105] Use helper variables to make ifs more readable Co-authored-by: Hsiao-Wei Wang --- specs/altair/sync-protocol.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index 144c5c980..8d420eda5 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -292,12 +292,12 @@ def validate_light_client_update(store: LightClientStore, # Verify update is relevant update_attested_period = compute_sync_committee_period_at_slot(update.attested_header.slot) + update_has_next_sync_committee = not is_next_sync_committee_known(store) and ( + is_sync_committee_update(update) and update_attested_period == store_period + ) assert ( update.attested_header.slot > store.finalized_header.slot - or ( - not is_next_sync_committee_known(store) - and update_attested_period == store_period and is_sync_committee_update(update) - ) + or update_has_next_sync_committee ) # Verify that the `finality_branch`, if present, confirms `finalized_header` @@ -399,17 +399,18 @@ def process_light_client_update(store: LightClientStore, store.optimistic_header = update.attested_header # Update finalized header + update_has_finalized_next_sync_committee = ( + not is_next_sync_committee_known(store) + and is_sync_committee_update(update) and is_finality_update(update) and ( + compute_sync_committee_period_at_slot(update.finalized_header.slot) + == compute_sync_committee_period_at_slot(update.attested_header.slot) + ) + ) if ( sum(sync_committee_bits) * 3 >= len(sync_committee_bits) * 2 and ( update.finalized_header.slot > store.finalized_header.slot - or ( - not is_next_sync_committee_known(store) - and is_sync_committee_update(update) and is_finality_update(update) and ( - compute_sync_committee_period_at_slot(update.finalized_header.slot) - == compute_sync_committee_period_at_slot(update.attested_header.slot) - ) - ) + or update_has_finalized_next_sync_committee ) ): # Normal update through 2/3 threshold From 86fe93ca96e55a74184a10264af3b919b2c7c936 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Tue, 3 May 2022 01:17:46 +0200 Subject: [PATCH 048/105] Add functions for deriving light client data Adds `create_light_client_bootstrap` and `create_light_client_update` functions as a reference implementation for serving light client data. This also enables a new test harness to verify that light client data gets applied to a `LightClientStore` as expected. --- setup.py | 1 + specs/altair/sync-protocol.md | 105 +++++ .../sync_protocol/test_light_client_sync.py | 424 ++++++++++++++++++ .../eth2spec/test/helpers/attestations.py | 9 +- tests/formats/sync_protocol/README.md | 1 + .../sync_protocol/light_client_sync.md | 59 +++ tests/generators/sync_protocol/main.py | 1 + 7 files changed, 599 insertions(+), 1 deletion(-) create mode 100644 tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_light_client_sync.py create mode 100644 tests/formats/sync_protocol/light_client_sync.md diff --git a/setup.py b/setup.py index 973c49fd4..cd6a5264f 100644 --- a/setup.py +++ b/setup.py @@ -457,6 +457,7 @@ class AltairSpecBuilder(Phase0SpecBuilder): from typing import NewType, Union as PyUnion from eth2spec.phase0 import {preset_name} as phase0 +from eth2spec.test.helpers.merkle import build_proof from eth2spec.utils.ssz.ssz_typing import Path ''' diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index 8d420eda5..a07adbd67 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -31,6 +31,9 @@ - [`validate_light_client_update`](#validate_light_client_update) - [`apply_light_client_update`](#apply_light_client_update) - [`process_light_client_update`](#process_light_client_update) +- [Deriving light client data](#deriving-light-client-data) + - [`create_light_client_bootstrap`](#create_light_client_bootstrap) + - [`create_light_client_update`](#create_light_client_update) @@ -417,3 +420,105 @@ def process_light_client_update(store: LightClientStore, apply_light_client_update(store, update) store.best_valid_update = None ``` + +## Deriving light client data + +Full nodes are expected to derive light client data from historic blocks and states and provide it to other clients. + +### `create_light_client_bootstrap` + +```python +def create_light_client_bootstrap(state: BeaconState) -> LightClientBootstrap: + assert compute_epoch_at_slot(state.slot) >= ALTAIR_FORK_EPOCH + assert state.slot == state.latest_block_header.slot + + return LightClientBootstrap( + header=BeaconBlockHeader( + slot=state.latest_block_header.slot, + proposer_index=state.latest_block_header.proposer_index, + parent_root=state.latest_block_header.parent_root, + state_root=hash_tree_root(state), + body_root=state.latest_block_header.body_root, + ), + current_sync_committee=state.current_sync_committee, + current_sync_committee_branch=build_proof(state.get_backing(), CURRENT_SYNC_COMMITTEE_INDEX) + ) +``` + +Full nodes SHOULD provide `LightClientBootstrap` for all finalized epoch boundary blocks in the epoch range `[max(ALTAIR_FORK_EPOCH, current_epoch - MIN_EPOCHS_FOR_BLOCK_REQUESTS), current_epoch]` where `current_epoch` is defined by the current wall-clock time. Full nodes MAY also provide `LightClientBootstrap` for other blocks. + +Blocks are considered to be epoch boundary blocks if their block root can occur as part of a valid `Checkpoint`, i.e., if their slot is the initial slot of an epoch, or if all following slots through the initial slot of the next epoch are empty (no block proposed / orphaned). + +`LightClientBootstrap` is computed from the block's immediate post state (without applying empty slots). + +### `create_light_client_update` + +To form a `LightClientUpdate`, the following historical states and blocks are needed: +- `state`: the post state of any block with a post-Altair parent block +- `block`: the corresponding block +- `attested_state`: the post state of the block referred to by `block.parent_root` +- `finalized_block`: the block referred to by `attested_state.finalized_checkpoint.root` + +```python +def create_light_client_update(state: BeaconState, + block: SignedBeaconBlock, + attested_state: BeaconState, + finalized_block: Optional[SignedBeaconBlock]) -> LightClientUpdate: + assert compute_epoch_at_slot(attested_state.slot) >= ALTAIR_FORK_EPOCH + assert sum(block.message.body.sync_aggregate.sync_committee_bits) >= MIN_SYNC_COMMITTEE_PARTICIPANTS + + assert state.slot == state.latest_block_header.slot + header = state.latest_block_header.copy() + header.state_root = hash_tree_root(state) + assert hash_tree_root(header) == hash_tree_root(block.message) + update_signature_period = compute_sync_committee_period(compute_epoch_at_slot(block.message.slot)) + + assert attested_state.slot == attested_state.latest_block_header.slot + attested_header = attested_state.latest_block_header.copy() + attested_header.state_root = hash_tree_root(attested_state) + assert hash_tree_root(attested_header) == block.message.parent_root + update_attested_period = compute_sync_committee_period(compute_epoch_at_slot(attested_header.slot)) + + # `next_sync_committee` is only useful if the message is signed by the current sync committee + if update_attested_period == update_signature_period: + next_sync_committee = attested_state.next_sync_committee + next_sync_committee_branch = build_proof(attested_state.get_backing(), NEXT_SYNC_COMMITTEE_INDEX) + else: + next_sync_committee = SyncCommittee() + next_sync_committee_branch = [Bytes32() for _ in range(floorlog2(NEXT_SYNC_COMMITTEE_INDEX))] + + # Indicate finality whenever possible + if finalized_block is not None: + if finalized_block.message.slot != GENESIS_SLOT: + finalized_header = BeaconBlockHeader( + slot=finalized_block.message.slot, + proposer_index=finalized_block.message.proposer_index, + parent_root=finalized_block.message.parent_root, + state_root=finalized_block.message.state_root, + body_root=hash_tree_root(finalized_block.message.body), + ) + assert hash_tree_root(finalized_header) == attested_state.finalized_checkpoint.root + else: + finalized_header = BeaconBlockHeader() + assert attested_state.finalized_checkpoint.root == Bytes32() + finality_branch = build_proof(attested_state.get_backing(), FINALIZED_ROOT_INDEX) + else: + finalized_header = BeaconBlockHeader() + finality_branch = [Bytes32() for _ in range(floorlog2(FINALIZED_ROOT_INDEX))] + + return LightClientUpdate( + attested_header=attested_header, + next_sync_committee=next_sync_committee, + next_sync_committee_branch=next_sync_committee_branch, + finalized_header=finalized_header, + finality_branch=finality_branch, + sync_aggregate=block.message.body.sync_aggregate, + signature_slot=block.message.slot, + ) +``` + +Full nodes SHOULD provide the best derivable `LightClientUpdate` (according to `is_better_update`) for each sync committee period covering any epochs in range `[max(ALTAIR_FORK_EPOCH, current_epoch - MIN_EPOCHS_FOR_BLOCK_REQUESTS), current_epoch]` where `current_epoch` is defined by the current wall-clock time. Full nodes MAY also provide `LightClientUpdate` for other sync committee periods. + +- `LightClientUpdate` are assigned to sync committee periods based on their `attested_header.slot` +- `LightClientUpdate` are only considered if `compute_sync_committee_period(compute_epoch_at_slot(update.attested_header.slot)) == compute_sync_committee_period(compute_epoch_at_slot(update.signature_slot))` +- Only `LightClientUpdate` with `next_sync_committee` as selected by fork choice are provided, regardless of ranking by `is_better_update`. To uniquely identify a non-finalized sync committee fork, all of `period`, `current_sync_committee` and `next_sync_committee` need to be incorporated, as sync committees may reappear over time. diff --git a/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_light_client_sync.py b/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_light_client_sync.py new file mode 100644 index 000000000..765d42968 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_light_client_sync.py @@ -0,0 +1,424 @@ +from typing import (Any, Dict, List) + +from eth_utils import encode_hex +from eth2spec.test.context import ( + spec_state_test_with_matching_config, + with_presets, + with_altair_and_later, +) +from eth2spec.test.helpers.attestations import ( + next_slots_with_attestations, + state_transition_with_full_block, +) +from eth2spec.test.helpers.constants import MINIMAL +from eth2spec.test.helpers.light_client import ( + get_sync_aggregate, +) +from eth2spec.test.helpers.state import ( + next_slots, + transition_to, +) + + +def setup_test(spec, state): + class LightClientSyncTest(object): + steps: List[Dict[str, Any]] + genesis_validators_root: spec.Root + store: spec.LightClientStore + + test = LightClientSyncTest() + test.steps = [] + + yield "genesis_validators_root", "meta", "0x" + state.genesis_validators_root.hex() + test.genesis_validators_root = state.genesis_validators_root + + next_slots(spec, state, spec.SLOTS_PER_EPOCH * 2 - 1) + trusted_block = state_transition_with_full_block(spec, state, True, True) + trusted_block_root = trusted_block.message.hash_tree_root() + bootstrap = spec.create_light_client_bootstrap(state) + yield "trusted_block_root", "meta", "0x" + trusted_block_root.hex() + yield "bootstrap", bootstrap + test.store = spec.initialize_light_client_store(trusted_block_root, bootstrap) + + return test + + +def finalize_test(test): + yield "steps", test.steps + yield "expected_finalized_header", test.store.finalized_header + yield "expected_optimistic_header", test.store.optimistic_header + + +def get_update_file_name(spec, update): + if spec.is_sync_committee_update(update): + suffix1 = "s" + else: + suffix1 = "x" + if spec.is_finality_update(update): + suffix2 = "f" + else: + suffix2 = "x" + return f"update_{encode_hex(update.attested_header.hash_tree_root())}_{suffix1}{suffix2}" + + +def emit_slot(test, spec, state): + current_slot = state.slot + yield from [] + test.steps.append({ + "process_slot": { + "current_slot": int(current_slot), + } + }) + spec.process_slot_for_light_client_store(test.store, current_slot) + + +def emit_update(test, spec, state, block, attested_state, finalized_block, with_next_sync_committee=True): + update = spec.create_light_client_update(state, block, attested_state, finalized_block) + if not with_next_sync_committee: + update.next_sync_committee = spec.SyncCommittee() + update.next_sync_committee_branch = \ + [spec.Bytes32() for _ in range(spec.floorlog2(spec.NEXT_SYNC_COMMITTEE_INDEX))] + current_slot = state.slot + yield get_update_file_name(spec, update), update + test.steps.append({ + "process_update": { + "update": get_update_file_name(spec, update), + "current_slot": int(current_slot), + } + }) + spec.process_light_client_update(test.store, update, current_slot, test.genesis_validators_root) + return update + + +def compute_start_slot_at_sync_committee_period(spec, sync_committee_period): + return spec.compute_start_slot_at_epoch(sync_committee_period * spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD) + + +def compute_start_slot_at_next_sync_committee_period(spec, state): + sync_committee_period = spec.compute_sync_committee_period(spec.compute_epoch_at_slot(state.slot)) + return compute_start_slot_at_sync_committee_period(spec, sync_committee_period + 1) + + +@with_altair_and_later +@spec_state_test_with_matching_config +@with_presets([MINIMAL], reason="too slow") +def test_light_client_sync(spec, state): + if state.fork.current_version == spec.config.GENESIS_FORK_VERSION: + return # Test requires Altair or later + + # Start test + test = yield from setup_test(spec, state) + + # Initial `LightClientUpdate`, populating `store.next_sync_committee` + # ``` + # | + # +-----------+ +----------+ +-----------+ | + # | finalized | <-- (2 epochs) -- | attested | <-- | signature | | + # +-----------+ +----------+ +-----------+ | + # | + # | + # sync committee + # period boundary + # ``` + next_slots(spec, state, spec.SLOTS_PER_EPOCH - 1) + finalized_block = state_transition_with_full_block(spec, state, True, True) + finalized_state = state.copy() + _, _, state = next_slots_with_attestations(spec, state, 2 * spec.SLOTS_PER_EPOCH, True, True) + attested_state = state.copy() + sync_aggregate, _ = get_sync_aggregate(spec, state) + block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) + yield from emit_update(test, spec, state, block, attested_state, finalized_block) + assert test.store.finalized_header.slot == finalized_state.slot + assert test.store.next_sync_committee == finalized_state.next_sync_committee + assert test.store.best_valid_update is None + assert test.store.optimistic_header.slot == attested_state.slot + + # Advance to next sync committee period + # ``` + # | + # +-----------+ +----------+ +-----------+ | + # | finalized | <-- (2 epochs) -- | attested | <-- | signature | | + # +-----------+ +----------+ +-----------+ | + # | + # | + # sync committee + # period boundary + # ``` + transition_to(spec, state, compute_start_slot_at_next_sync_committee_period(spec, state)) + next_slots(spec, state, spec.SLOTS_PER_EPOCH - 1) + finalized_block = state_transition_with_full_block(spec, state, True, True) + finalized_state = state.copy() + _, _, state = next_slots_with_attestations(spec, state, 2 * spec.SLOTS_PER_EPOCH, True, True) + attested_state = state.copy() + sync_aggregate, _ = get_sync_aggregate(spec, state) + block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) + yield from emit_update(test, spec, state, block, attested_state, finalized_block) + assert test.store.finalized_header.slot == finalized_state.slot + assert test.store.next_sync_committee == finalized_state.next_sync_committee + assert test.store.best_valid_update is None + assert test.store.optimistic_header.slot == attested_state.slot + + # Edge case: Signature in next period + # ``` + # | + # +-----------+ +----------+ | +-----------+ + # | finalized | <-- (2 epochs) -- | attested | <-- | signature | + # +-----------+ +----------+ | +-----------+ + # | + # | + # sync committee + # period boundary + # ``` + next_slots(spec, state, spec.SLOTS_PER_EPOCH - 2) + finalized_block = state_transition_with_full_block(spec, state, True, True) + finalized_state = state.copy() + _, _, state = next_slots_with_attestations(spec, state, 2 * spec.SLOTS_PER_EPOCH, True, True) + attested_state = state.copy() + transition_to(spec, state, compute_start_slot_at_next_sync_committee_period(spec, state)) + sync_aggregate, _ = get_sync_aggregate(spec, state) + block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) + yield from emit_update(test, spec, state, block, attested_state, finalized_block) + assert test.store.finalized_header.slot == finalized_state.slot + assert test.store.next_sync_committee == finalized_state.next_sync_committee + assert test.store.best_valid_update is None + assert test.store.optimistic_header.slot == attested_state.slot + + # Edge case: Finalized header not included + # ``` + # | + # + - - - - - + | +----------+ +-----------+ + # ¦ finalized ¦ <-- (2 epochs) -- | attested | <-- | signature | + # + - - - - - + | +----------+ +-----------+ + # | + # | + # sync committee + # period boundary + # ``` + attested_state = state.copy() + sync_aggregate, _ = get_sync_aggregate(spec, state) + block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) + update = yield from emit_update(test, spec, state, block, attested_state, finalized_block=None) + assert test.store.finalized_header.slot == finalized_state.slot + assert test.store.next_sync_committee == finalized_state.next_sync_committee + assert test.store.best_valid_update == update + assert test.store.optimistic_header.slot == attested_state.slot + + # Non-finalized case: Attested `next_sync_committee` is not finalized + # ``` + # | + # +-----------+ | +----------+ +-----------+ + # | finalized | <-- (2 epochs) -- | attested | <-- | signature | + # +-----------+ | +----------+ +-----------+ + # | + # | + # sync committee + # period boundary + # ``` + attested_state = state.copy() + store_state = attested_state.copy() + sync_aggregate, _ = get_sync_aggregate(spec, state) + block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) + update = yield from emit_update(test, spec, state, block, attested_state, finalized_block) + assert test.store.finalized_header.slot == finalized_state.slot + assert test.store.next_sync_committee == finalized_state.next_sync_committee + assert test.store.best_valid_update == update + assert test.store.optimistic_header.slot == attested_state.slot + + # Force-update using timeout + # ``` + # | + # +-----------+ | +----------+ + # | finalized | <-- (2 epochs) -- | attested | + # +-----------+ | +----------+ + # | ^ + # | \ + # sync committee `--- store.finalized_header + # period boundary + # ``` + attested_state = state.copy() + next_slots(spec, state, spec.UPDATE_TIMEOUT - 1) + yield from emit_slot(test, spec, state) + assert test.store.finalized_header.slot == store_state.slot + assert test.store.next_sync_committee == store_state.next_sync_committee + assert test.store.best_valid_update is None + assert test.store.optimistic_header.slot == store_state.slot + + # Edge case: Finalized header not included, after force-update + # ``` + # | | + # + - - - - - + | +--+ +----------+ | +-----------+ + # ¦ finalized ¦ <-- (2 epochs) -- | | <-- | attested | <-- | signature | + # + - - - - - + | +--+ +----------+ | +-----------+ + # | / | + # | store.fin | + # sync committee sync committee + # period boundary period boundary + # ``` + sync_aggregate, _ = get_sync_aggregate(spec, state) + block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) + update = yield from emit_update(test, spec, state, block, attested_state, finalized_block=None) + assert test.store.finalized_header.slot == store_state.slot + assert test.store.next_sync_committee == store_state.next_sync_committee + assert test.store.best_valid_update == update + assert test.store.optimistic_header.slot == attested_state.slot + + # Edge case: Finalized header older than store + # ``` + # | | + # +-----------+ | +--+ | +----------+ +-----------+ + # | finalized | <-- (2 epochs) -- | | <-- | attested | <-- | signature | + # +-----------+ | +--+ | +----------+ +-----------+ + # | / | + # | store.fin | + # sync committee sync committee + # period boundary period boundary + # ``` + attested_state = state.copy() + sync_aggregate, _ = get_sync_aggregate(spec, state) + block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) + update = yield from emit_update(test, spec, state, block, attested_state, finalized_block) + assert test.store.finalized_header.slot == store_state.slot + assert test.store.next_sync_committee == store_state.next_sync_committee + assert test.store.best_valid_update == update + assert test.store.optimistic_header.slot == attested_state.slot + yield from emit_slot(test, spec, state) + assert test.store.finalized_header.slot == attested_state.slot + assert test.store.next_sync_committee == attested_state.next_sync_committee + assert test.store.best_valid_update is None + assert test.store.optimistic_header.slot == attested_state.slot + + # Advance to next sync committee period + # ``` + # | + # +-----------+ +----------+ +-----------+ | + # | finalized | <-- (2 epochs) -- | attested | <-- | signature | | + # +-----------+ +----------+ +-----------+ | + # | + # | + # sync committee + # period boundary + # ``` + transition_to(spec, state, compute_start_slot_at_next_sync_committee_period(spec, state)) + next_slots(spec, state, spec.SLOTS_PER_EPOCH - 1) + finalized_block = state_transition_with_full_block(spec, state, True, True) + finalized_state = state.copy() + _, _, state = next_slots_with_attestations(spec, state, 2 * spec.SLOTS_PER_EPOCH, True, True) + attested_state = state.copy() + sync_aggregate, _ = get_sync_aggregate(spec, state) + block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) + yield from emit_update(test, spec, state, block, attested_state, finalized_block) + assert test.store.finalized_header.slot == finalized_state.slot + assert test.store.next_sync_committee == finalized_state.next_sync_committee + assert test.store.best_valid_update is None + assert test.store.optimistic_header.slot == attested_state.slot + + # Finish test + yield from finalize_test(test) + + +@with_altair_and_later +@spec_state_test_with_matching_config +@with_presets([MINIMAL], reason="too slow") +def test_supply_sync_committee_from_past_update(spec, state): + if state.fork.current_version == spec.config.GENESIS_FORK_VERSION: + return # Test requires Altair or later + + # Advance the chain, so that a `LightClientUpdate` from the past is available + next_slots(spec, state, spec.SLOTS_PER_EPOCH * 2 - 1) + finalized_block = state_transition_with_full_block(spec, state, True, True) + finalized_state = state.copy() + _, _, state = next_slots_with_attestations(spec, state, 2 * spec.SLOTS_PER_EPOCH, True, True) + attested_state = state.copy() + sync_aggregate, _ = get_sync_aggregate(spec, state) + block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) + past_state = state.copy() + + # Start test + test = yield from setup_test(spec, state) + assert not spec.is_next_sync_committee_known(test.store) + + # Apply `LightClientUpdate` from the past, populating `store.next_sync_committee` + yield from emit_update(test, spec, past_state, block, attested_state, finalized_block) + assert test.store.finalized_header.slot == state.slot + assert test.store.next_sync_committee == finalized_state.next_sync_committee + assert test.store.best_valid_update is None + assert test.store.optimistic_header.slot == state.slot + + # Finish test + yield from finalize_test(test) + + +@with_altair_and_later +@spec_state_test_with_matching_config +@with_presets([MINIMAL], reason="too slow") +def test_advance_finality_without_sync_committee(spec, state): + if state.fork.current_version == spec.config.GENESIS_FORK_VERSION: + return # Test requires Altair or later + + # Start test + test = yield from setup_test(spec, state) + + # Initial `LightClientUpdate`, populating `store.next_sync_committee` + next_slots(spec, state, spec.SLOTS_PER_EPOCH - 1) + finalized_block = state_transition_with_full_block(spec, state, True, True) + finalized_state = state.copy() + _, _, state = next_slots_with_attestations(spec, state, 2 * spec.SLOTS_PER_EPOCH, True, True) + attested_state = state.copy() + sync_aggregate, _ = get_sync_aggregate(spec, state) + block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) + yield from emit_update(test, spec, state, block, attested_state, finalized_block) + assert test.store.finalized_header.slot == finalized_state.slot + assert test.store.next_sync_committee == finalized_state.next_sync_committee + assert test.store.best_valid_update is None + assert test.store.optimistic_header.slot == attested_state.slot + + # Advance finality into next sync committee period, but omit `next_sync_committee` + transition_to(spec, state, compute_start_slot_at_next_sync_committee_period(spec, state)) + next_slots(spec, state, spec.SLOTS_PER_EPOCH - 1) + finalized_block = state_transition_with_full_block(spec, state, True, True) + finalized_state = state.copy() + _, _, state = next_slots_with_attestations(spec, state, spec.SLOTS_PER_EPOCH - 1, True, True) + justified_block = state_transition_with_full_block(spec, state, True, True) + justified_state = state.copy() + _, _, state = next_slots_with_attestations(spec, state, spec.SLOTS_PER_EPOCH, True, True) + attested_state = state.copy() + sync_aggregate, _ = get_sync_aggregate(spec, state) + block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) + yield from emit_update(test, spec, state, block, attested_state, finalized_block, with_next_sync_committee=False) + assert test.store.finalized_header.slot == finalized_state.slot + assert not spec.is_next_sync_committee_known(test.store) + assert test.store.best_valid_update is None + assert test.store.optimistic_header.slot == attested_state.slot + + # Advance finality once more, with `next_sync_committee` still unknown + past_state = finalized_state + finalized_block = justified_block + finalized_state = justified_state + _, _, state = next_slots_with_attestations(spec, state, spec.SLOTS_PER_EPOCH - 1, True, True) + attested_state = state.copy() + sync_aggregate, _ = get_sync_aggregate(spec, state) + block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) + + # Apply `LightClientUpdate` without `finalized_header` nor `next_sync_committee` + update = yield from emit_update(test, spec, state, block, attested_state, None, with_next_sync_committee=False) + assert test.store.finalized_header.slot == past_state.slot + assert not spec.is_next_sync_committee_known(test.store) + assert test.store.best_valid_update == update + assert test.store.optimistic_header.slot == attested_state.slot + + # Apply `LightClientUpdate` with `finalized_header` but no `next_sync_committee` + yield from emit_update(test, spec, state, block, attested_state, finalized_block, with_next_sync_committee=False) + assert test.store.finalized_header.slot == finalized_state.slot + assert not spec.is_next_sync_committee_known(test.store) + assert test.store.best_valid_update is None + assert test.store.optimistic_header.slot == attested_state.slot + + # Apply full `LightClientUpdate`, supplying `next_sync_committee` + yield from emit_update(test, spec, state, block, attested_state, finalized_block) + assert test.store.finalized_header.slot == finalized_state.slot + assert test.store.next_sync_committee == finalized_state.next_sync_committee + assert test.store.best_valid_update is None + assert test.store.optimistic_header.slot == attested_state.slot + + # Finish test + yield from finalize_test(test) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index ffd484ecd..7ba71a969 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -246,7 +246,12 @@ def next_epoch_with_attestations(spec, ) -def state_transition_with_full_block(spec, state, fill_cur_epoch, fill_prev_epoch, participation_fn=None): +def state_transition_with_full_block(spec, + state, + fill_cur_epoch, + fill_prev_epoch, + participation_fn=None, + sync_aggregate=None): """ Build and apply a block with attestions at the calculated `slot_to_attest` of current epoch and/or previous epoch. """ @@ -272,6 +277,8 @@ def state_transition_with_full_block(spec, state, fill_cur_epoch, fill_prev_epoc ) for attestation in attestations: block.body.attestations.append(attestation) + if sync_aggregate is not None: + block.body.sync_aggregate = sync_aggregate signed_block = state_transition_and_sign_block(spec, state, block) return signed_block diff --git a/tests/formats/sync_protocol/README.md b/tests/formats/sync_protocol/README.md index da93972d6..c189cbc23 100644 --- a/tests/formats/sync_protocol/README.md +++ b/tests/formats/sync_protocol/README.md @@ -3,4 +3,5 @@ This series of tests provides reference test vectors for the light client sync protocol spec. Handlers: +- `light_client_sync`: see [Light client sync test format](./light_client_sync.md) - `update_ranking`: see [`LightClientUpdate` ranking test format](./update_ranking.md) diff --git a/tests/formats/sync_protocol/light_client_sync.md b/tests/formats/sync_protocol/light_client_sync.md new file mode 100644 index 000000000..1d1506936 --- /dev/null +++ b/tests/formats/sync_protocol/light_client_sync.md @@ -0,0 +1,59 @@ +# Light client sync tests + +This series of tests provides reference test vectors for validating that a light client implementing the sync protocol can sync to the latest block header. + +## Test case format + +### `meta.yaml` + +```yaml +genesis_validators_root: Bytes32 -- string, hex encoded, with 0x prefix +trusted_block_root: Bytes32 -- string, hex encoded, with 0x prefix +``` + +### `bootstrap.ssz_snappy` + +An SSZ-snappy encoded `bootstrap` object of type `LightClientBootstrap` to initialize a local `store` object of type `LightClientStore` using `initialize_light_client_store(trusted_block_rooot, bootstrap)`. + +### `steps.yaml` + +The steps to execute in sequence. There may be multiple items of the following types: + +#### `process_slot` execution step + +The function `process_slot_for_light_client_store(store, current_slot)` +should be executed with the specified parameters: + +```yaml +{ + current_slot: int -- integer, decimal +} +``` + +After this step, the `store` object may have been updated. + +#### `process_update` execution step + +The function `process_light_client_update(store, update, current_slot, genesis_validators_root)` should be executed with the specified parameters: + +```yaml +{ + update: string -- name of the `*.ssz_snappy` file to load + as a `LightClientUpdate` object + current_slot: int -- integer, decimal +} +``` + +After this step, the `store` object may have been updated. + +### `expected_finalized_header.ssz_snappy` + +An SSZ-snappy encoded `expected_finalized_header` object of type `BeaconBlockHeader` that represents the expected `store.finalized_header` after applying all the test steps. + +### `expected_optimistic_header.ssz_snappy` + +An SSZ-snappy encoded `expected_optimistic_header` object of type `BeaconBlockHeader` that represents the expected `store.finalized_header` after applying all the test steps. + +## Condition + +A test-runner should initialize a local `LightClientStore` using the provided `bootstrap` object. It should then proceed to execute all the test steps in sequence. Finally, it should verify that the resulting `store` refers to the provided `expected_finalized_header` and `expected_optimistic_header`. diff --git a/tests/generators/sync_protocol/main.py b/tests/generators/sync_protocol/main.py index 0fe8e0fe1..607587d93 100644 --- a/tests/generators/sync_protocol/main.py +++ b/tests/generators/sync_protocol/main.py @@ -4,6 +4,7 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators if __name__ == "__main__": altair_mods = {key: 'eth2spec.test.altair.sync_protocol.test_' + key for key in [ + 'light_client_sync', 'update_ranking', ]} bellatrix_mods = altair_mods From 95c07575b160720eb0559d4306ffc3ef8e928f5d Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 15 Jul 2022 11:54:47 +0200 Subject: [PATCH 049/105] Hide `build_proof` and `get_backing` from spec --- setup.py | 13 +++++++++-- specs/altair/sync-protocol.md | 23 +++++++++++++------ .../test/altair/merkle/test_single_proof.py | 7 +++--- .../sync_protocol/test_update_ranking.py | 5 ++-- .../altair/unittests/test_sync_protocol.py | 5 ++-- 5 files changed, 34 insertions(+), 19 deletions(-) diff --git a/setup.py b/setup.py index cd6a5264f..0c37e33c6 100644 --- a/setup.py +++ b/setup.py @@ -475,7 +475,12 @@ def get_generalized_index(ssz_class: Any, *path: Sequence[PyUnion[int, SSZVariab ssz_path = Path(ssz_class) for item in path: ssz_path = ssz_path / item - return GeneralizedIndex(ssz_path.gindex())''' + return GeneralizedIndex(ssz_path.gindex()) + + +def compute_merkle_proof_for_state(state: BeaconState, + index: GeneralizedIndex) -> Sequence[Bytes32]: + return build_proof(state.get_backing(), index)''' @classmethod @@ -613,7 +618,11 @@ def objects_to_spec(preset_name: str, protocols_spec = '\n\n\n'.join(format_protocol(k, v) for k, v in spec_object.protocols.items()) for k in list(spec_object.functions): - if "ceillog2" in k or "floorlog2" in k: + if k in [ + "ceillog2", + "floorlog2", + "compute_merkle_proof_for_state", + ]: del spec_object.functions[k] functions = builder.implement_optimizations(spec_object.functions) functions_spec = '\n\n\n'.join(functions.values()) diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index a07adbd67..ed1b02152 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -22,8 +22,9 @@ - [`is_better_update`](#is_better_update) - [`is_next_sync_committee_known`](#is_next_sync_committee_known) - [`get_safety_threshold`](#get_safety_threshold) - - [`get_subtree_index`](#get_subtree_index) - [`compute_sync_committee_period_at_slot`](#compute_sync_committee_period_at_slot) + - [`get_subtree_index`](#get_subtree_index) + - [`compute_merkle_proof_for_state`](#compute_merkle_proof_for_state) - [Light client initialization](#light-client-initialization) - [`initialize_light_client_store`](#initialize_light_client_store) - [Light client state updates](#light-client-state-updates) @@ -204,6 +205,13 @@ def get_safety_threshold(store: LightClientStore) -> uint64: ) // 2 ``` +### `compute_sync_committee_period_at_slot` + +```python +def compute_sync_committee_period_at_slot(slot: Slot) -> uint64: + return compute_sync_committee_period(compute_epoch_at_slot(slot)) +``` + ### `get_subtree_index` ```python @@ -211,11 +219,12 @@ def get_subtree_index(generalized_index: GeneralizedIndex) -> uint64: return uint64(generalized_index % 2**(floorlog2(generalized_index))) ``` -### `compute_sync_committee_period_at_slot` +### `compute_merkle_proof_for_state` ```python -def compute_sync_committee_period_at_slot(slot: Slot) -> uint64: - return compute_sync_committee_period(compute_epoch_at_slot(slot)) +def compute_merkle_proof_for_state(state: BeaconState, + index: GeneralizedIndex) -> Sequence[Bytes32]: + ... ``` ## Light client initialization @@ -441,7 +450,7 @@ def create_light_client_bootstrap(state: BeaconState) -> LightClientBootstrap: body_root=state.latest_block_header.body_root, ), current_sync_committee=state.current_sync_committee, - current_sync_committee_branch=build_proof(state.get_backing(), CURRENT_SYNC_COMMITTEE_INDEX) + current_sync_committee_branch=compute_merkle_proof_for_state(state, CURRENT_SYNC_COMMITTEE_INDEX) ) ``` @@ -482,7 +491,7 @@ def create_light_client_update(state: BeaconState, # `next_sync_committee` is only useful if the message is signed by the current sync committee if update_attested_period == update_signature_period: next_sync_committee = attested_state.next_sync_committee - next_sync_committee_branch = build_proof(attested_state.get_backing(), NEXT_SYNC_COMMITTEE_INDEX) + next_sync_committee_branch = compute_merkle_proof_for_state(attested_state, NEXT_SYNC_COMMITTEE_INDEX) else: next_sync_committee = SyncCommittee() next_sync_committee_branch = [Bytes32() for _ in range(floorlog2(NEXT_SYNC_COMMITTEE_INDEX))] @@ -501,7 +510,7 @@ def create_light_client_update(state: BeaconState, else: finalized_header = BeaconBlockHeader() assert attested_state.finalized_checkpoint.root == Bytes32() - finality_branch = build_proof(attested_state.get_backing(), FINALIZED_ROOT_INDEX) + finality_branch = compute_merkle_proof_for_state(attested_state, FINALIZED_ROOT_INDEX) else: finalized_header = BeaconBlockHeader() finality_branch = [Bytes32() for _ in range(floorlog2(FINALIZED_ROOT_INDEX))] diff --git a/tests/core/pyspec/eth2spec/test/altair/merkle/test_single_proof.py b/tests/core/pyspec/eth2spec/test/altair/merkle/test_single_proof.py index d90feb146..064760bf0 100644 --- a/tests/core/pyspec/eth2spec/test/altair/merkle/test_single_proof.py +++ b/tests/core/pyspec/eth2spec/test/altair/merkle/test_single_proof.py @@ -2,14 +2,13 @@ from eth2spec.test.context import ( spec_state_test, with_altair_and_later, ) -from eth2spec.test.helpers.merkle import build_proof @with_altair_and_later @spec_state_test def test_current_sync_committee_merkle_proof(spec, state): yield "state", state - current_sync_committee_branch = build_proof(state.get_backing(), spec.CURRENT_SYNC_COMMITTEE_INDEX) + current_sync_committee_branch = spec.compute_merkle_proof_for_state(state, spec.CURRENT_SYNC_COMMITTEE_INDEX) yield "proof", { "leaf": "0x" + state.current_sync_committee.hash_tree_root().hex(), "leaf_index": spec.CURRENT_SYNC_COMMITTEE_INDEX, @@ -28,7 +27,7 @@ def test_current_sync_committee_merkle_proof(spec, state): @spec_state_test def test_next_sync_committee_merkle_proof(spec, state): yield "state", state - next_sync_committee_branch = build_proof(state.get_backing(), spec.NEXT_SYNC_COMMITTEE_INDEX) + next_sync_committee_branch = spec.compute_merkle_proof_for_state(state, spec.NEXT_SYNC_COMMITTEE_INDEX) yield "proof", { "leaf": "0x" + state.next_sync_committee.hash_tree_root().hex(), "leaf_index": spec.NEXT_SYNC_COMMITTEE_INDEX, @@ -47,7 +46,7 @@ def test_next_sync_committee_merkle_proof(spec, state): @spec_state_test def test_finality_root_merkle_proof(spec, state): yield "state", state - finality_branch = build_proof(state.get_backing(), spec.FINALIZED_ROOT_INDEX) + finality_branch = spec.compute_merkle_proof_for_state(state, spec.FINALIZED_ROOT_INDEX) yield "proof", { "leaf": "0x" + state.finalized_checkpoint.root.hex(), "leaf_index": spec.FINALIZED_ROOT_INDEX, diff --git a/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_update_ranking.py b/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_update_ranking.py index 4b73a15ba..ae5d5296a 100644 --- a/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_update_ranking.py +++ b/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_update_ranking.py @@ -15,7 +15,6 @@ from eth2spec.test.helpers.light_client import ( from eth2spec.test.helpers.state import ( next_slots, ) -from eth2spec.test.helpers.merkle import build_proof from math import floor @@ -27,14 +26,14 @@ def create_update(spec, test, with_next_sync_committee, with_finality, participa if with_next_sync_committee: next_sync_committee = attested_state.next_sync_committee - next_sync_committee_branch = build_proof(attested_state.get_backing(), spec.NEXT_SYNC_COMMITTEE_INDEX) + next_sync_committee_branch = spec.compute_merkle_proof_for_state(attested_state, spec.NEXT_SYNC_COMMITTEE_INDEX) else: next_sync_committee = spec.SyncCommittee() next_sync_committee_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.NEXT_SYNC_COMMITTEE_INDEX))] if with_finality: finalized_header = signed_block_to_header(spec, finalized_block) - finality_branch = build_proof(attested_state.get_backing(), spec.FINALIZED_ROOT_INDEX) + finality_branch = spec.compute_merkle_proof_for_state(attested_state, spec.FINALIZED_ROOT_INDEX) else: finalized_header = spec.BeaconBlockHeader() finality_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.FINALIZED_ROOT_INDEX))] diff --git a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py index 17c450f17..e7845292a 100644 --- a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py +++ b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py @@ -18,7 +18,6 @@ from eth2spec.test.helpers.light_client import ( from eth2spec.test.helpers.state import ( next_slots, ) -from eth2spec.test.helpers.merkle import build_proof @with_altair_and_later @@ -125,7 +124,7 @@ def test_process_light_client_update_timeout(spec, state): # Sync committee is updated next_sync_committee = state.next_sync_committee - next_sync_committee_branch = build_proof(state.get_backing(), spec.NEXT_SYNC_COMMITTEE_INDEX) + next_sync_committee_branch = spec.compute_merkle_proof_for_state(state, spec.NEXT_SYNC_COMMITTEE_INDEX) # Finality is unchanged finality_header = spec.BeaconBlockHeader() finality_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.FINALIZED_ROOT_INDEX))] @@ -182,7 +181,7 @@ def test_process_light_client_update_finality_updated(spec, state): finalized_header = signed_block_to_header(spec, finalized_block) assert finalized_header.slot == spec.compute_start_slot_at_epoch(state.finalized_checkpoint.epoch) assert finalized_header.hash_tree_root() == state.finalized_checkpoint.root - finality_branch = build_proof(state.get_backing(), spec.FINALIZED_ROOT_INDEX) + finality_branch = spec.compute_merkle_proof_for_state(state, spec.FINALIZED_ROOT_INDEX) update = spec.LightClientUpdate( attested_header=attested_header, From dbefbcf3261ff41f6d904946d7413749e5c3adcc Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 15 Jul 2022 11:59:11 +0200 Subject: [PATCH 050/105] Explicitly mention optional `finalized_block` --- specs/altair/sync-protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index ed1b02152..1fe375012 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -466,7 +466,7 @@ To form a `LightClientUpdate`, the following historical states and blocks are ne - `state`: the post state of any block with a post-Altair parent block - `block`: the corresponding block - `attested_state`: the post state of the block referred to by `block.parent_root` -- `finalized_block`: the block referred to by `attested_state.finalized_checkpoint.root` +- `finalized_block`: the block referred to by `attested_state.finalized_checkpoint.root`, if locally available (may be unavailable, e.g., when using checkpoint sync, or if it was pruned locally) ```python def create_light_client_update(state: BeaconState, From 6146f01400e5b54c025829da12423054ee191311 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 15 Jul 2022 12:03:09 +0200 Subject: [PATCH 051/105] Flip order of `assert` and assignment Co-authored-by: Hsiao-Wei Wang --- specs/altair/sync-protocol.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index 1fe375012..c8697d13c 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -319,8 +319,8 @@ def validate_light_client_update(store: LightClientStore, assert update.finalized_header == BeaconBlockHeader() else: if update.finalized_header.slot == GENESIS_SLOT: - finalized_root = Bytes32() assert update.finalized_header == BeaconBlockHeader() + finalized_root = Bytes32() else: finalized_root = hash_tree_root(update.finalized_header) assert is_valid_merkle_branch( @@ -508,8 +508,8 @@ def create_light_client_update(state: BeaconState, ) assert hash_tree_root(finalized_header) == attested_state.finalized_checkpoint.root else: - finalized_header = BeaconBlockHeader() assert attested_state.finalized_checkpoint.root == Bytes32() + finalized_header = BeaconBlockHeader() finality_branch = compute_merkle_proof_for_state(attested_state, FINALIZED_ROOT_INDEX) else: finalized_header = BeaconBlockHeader() From 88a69512b2700aa0d8ace729be12f420f1e100fb Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 15 Jul 2022 12:04:56 +0200 Subject: [PATCH 052/105] `finalize_test` --> `finish_test` --- .../test/altair/sync_protocol/test_light_client_sync.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_light_client_sync.py b/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_light_client_sync.py index 765d42968..aa8c9950a 100644 --- a/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_light_client_sync.py +++ b/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_light_client_sync.py @@ -43,7 +43,7 @@ def setup_test(spec, state): return test -def finalize_test(test): +def finish_test(test): yield "steps", test.steps yield "expected_finalized_header", test.store.finalized_header yield "expected_optimistic_header", test.store.optimistic_header @@ -313,7 +313,7 @@ def test_light_client_sync(spec, state): assert test.store.optimistic_header.slot == attested_state.slot # Finish test - yield from finalize_test(test) + yield from finish_test(test) @with_altair_and_later @@ -345,7 +345,7 @@ def test_supply_sync_committee_from_past_update(spec, state): assert test.store.optimistic_header.slot == state.slot # Finish test - yield from finalize_test(test) + yield from finish_test(test) @with_altair_and_later @@ -421,4 +421,4 @@ def test_advance_finality_without_sync_committee(spec, state): assert test.store.optimistic_header.slot == attested_state.slot # Finish test - yield from finalize_test(test) + yield from finish_test(test) From 73279f8382230bc3eb5a48ffd28ac0c27f57b359 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 15 Jul 2022 12:21:48 +0200 Subject: [PATCH 053/105] Explain why the tests check for Altair twice --- .../test/altair/sync_protocol/test_light_client_sync.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_light_client_sync.py b/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_light_client_sync.py index aa8c9950a..0df77cda4 100644 --- a/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_light_client_sync.py +++ b/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_light_client_sync.py @@ -104,7 +104,7 @@ def compute_start_slot_at_next_sync_committee_period(spec, state): @with_presets([MINIMAL], reason="too slow") def test_light_client_sync(spec, state): if state.fork.current_version == spec.config.GENESIS_FORK_VERSION: - return # Test requires Altair or later + return # Test requires Altair or later (`make test` does not honor phases) # Start test test = yield from setup_test(spec, state) @@ -321,7 +321,7 @@ def test_light_client_sync(spec, state): @with_presets([MINIMAL], reason="too slow") def test_supply_sync_committee_from_past_update(spec, state): if state.fork.current_version == spec.config.GENESIS_FORK_VERSION: - return # Test requires Altair or later + return # Test requires Altair or later (`make test` does not honor phases) # Advance the chain, so that a `LightClientUpdate` from the past is available next_slots(spec, state, spec.SLOTS_PER_EPOCH * 2 - 1) @@ -353,7 +353,7 @@ def test_supply_sync_committee_from_past_update(spec, state): @with_presets([MINIMAL], reason="too slow") def test_advance_finality_without_sync_committee(spec, state): if state.fork.current_version == spec.config.GENESIS_FORK_VERSION: - return # Test requires Altair or later + return # Test requires Altair or later (`make test` does not honor phases) # Start test test = yield from setup_test(spec, state) From 3c5d347cdc66e3cb40f248ce1dfe4def4b7c6ad8 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 15 Jul 2022 13:13:33 +0200 Subject: [PATCH 054/105] Emit checks after each individual test step Co-authored-by: Hsiao-Wei Wang --- .../sync_protocol/test_light_client_sync.py | 25 ++++++++++--- .../sync_protocol/light_client_sync.md | 37 ++++++++++++------- 2 files changed, 43 insertions(+), 19 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_light_client_sync.py b/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_light_client_sync.py index 0df77cda4..d2eed0627 100644 --- a/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_light_client_sync.py +++ b/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_light_client_sync.py @@ -45,8 +45,6 @@ def setup_test(spec, state): def finish_test(test): yield "steps", test.steps - yield "expected_finalized_header", test.store.finalized_header - yield "expected_optimistic_header", test.store.optimistic_header def get_update_file_name(spec, update): @@ -61,15 +59,30 @@ def get_update_file_name(spec, update): return f"update_{encode_hex(update.attested_header.hash_tree_root())}_{suffix1}{suffix2}" +def get_checks(store): + return { + "finalized_header": { + 'slot': int(store.finalized_header.slot), + 'root': encode_hex(store.finalized_header.hash_tree_root()), + }, + "optimistic_header": { + 'slot': int(store.optimistic_header.slot), + 'root': encode_hex(store.optimistic_header.hash_tree_root()), + }, + } + + def emit_slot(test, spec, state): current_slot = state.slot - yield from [] + spec.process_slot_for_light_client_store(test.store, current_slot) + + yield from [] # Consistently enable `yield from` syntax in calling tests test.steps.append({ "process_slot": { "current_slot": int(current_slot), + "checks": get_checks(test.store), } }) - spec.process_slot_for_light_client_store(test.store, current_slot) def emit_update(test, spec, state, block, attested_state, finalized_block, with_next_sync_committee=True): @@ -79,14 +92,16 @@ def emit_update(test, spec, state, block, attested_state, finalized_block, with_ update.next_sync_committee_branch = \ [spec.Bytes32() for _ in range(spec.floorlog2(spec.NEXT_SYNC_COMMITTEE_INDEX))] current_slot = state.slot + spec.process_light_client_update(test.store, update, current_slot, test.genesis_validators_root) + yield get_update_file_name(spec, update), update test.steps.append({ "process_update": { "update": get_update_file_name(spec, update), "current_slot": int(current_slot), + "checks": get_checks(test.store), } }) - spec.process_light_client_update(test.store, update, current_slot, test.genesis_validators_root) return update diff --git a/tests/formats/sync_protocol/light_client_sync.md b/tests/formats/sync_protocol/light_client_sync.md index 1d1506936..6940ced87 100644 --- a/tests/formats/sync_protocol/light_client_sync.md +++ b/tests/formats/sync_protocol/light_client_sync.md @@ -17,7 +17,22 @@ An SSZ-snappy encoded `bootstrap` object of type `LightClientBootstrap` to initi ### `steps.yaml` -The steps to execute in sequence. There may be multiple items of the following types: +The steps to execute in sequence. + +#### `checks` execution step + +Each step includes checks to verify the expected impact on the `store` object. + +```yaml +finalized_header: { + slot: int, -- Integer value from store.finalized_header.slot + root: string, -- Encoded 32-byte value from store.finalized_header.hash_tree_root() +} +optimistic_header: { + slot: int, -- Integer value from store.optimistic_header.slot + root: string, -- Encoded 32-byte value from store.optimistic_header.hash_tree_root() +} +``` #### `process_slot` execution step @@ -26,7 +41,8 @@ should be executed with the specified parameters: ```yaml { - current_slot: int -- integer, decimal + current_slot: int -- integer, decimal + checks: {: value} -- the assertions. } ``` @@ -38,22 +54,15 @@ The function `process_light_client_update(store, update, current_slot, genesis_v ```yaml { - update: string -- name of the `*.ssz_snappy` file to load - as a `LightClientUpdate` object - current_slot: int -- integer, decimal + update: string -- name of the `*.ssz_snappy` file to load + as a `LightClientUpdate` object + current_slot: int -- integer, decimal + checks: {: value} -- the assertions. } ``` After this step, the `store` object may have been updated. -### `expected_finalized_header.ssz_snappy` - -An SSZ-snappy encoded `expected_finalized_header` object of type `BeaconBlockHeader` that represents the expected `store.finalized_header` after applying all the test steps. - -### `expected_optimistic_header.ssz_snappy` - -An SSZ-snappy encoded `expected_optimistic_header` object of type `BeaconBlockHeader` that represents the expected `store.finalized_header` after applying all the test steps. - ## Condition -A test-runner should initialize a local `LightClientStore` using the provided `bootstrap` object. It should then proceed to execute all the test steps in sequence. Finally, it should verify that the resulting `store` refers to the provided `expected_finalized_header` and `expected_optimistic_header`. +A test-runner should initialize a local `LightClientStore` using the provided `bootstrap` object. It should then proceed to execute all the test steps in sequence. After each step, it should verify that the resulting `store` verifies against the provided `checks`. From d06f44ef55ddb69bf618e9e913f02bd89c4e01da Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 15 Jul 2022 13:15:22 +0200 Subject: [PATCH 055/105] Move `sync-protocol.md` into `light-client` subdir --- README.md | 4 ++-- setup.py | 2 +- specs/altair/{ => light-client}/sync-protocol.md | 0 specs/altair/validator.md | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) rename specs/altair/{ => light-client}/sync-protocol.md (100%) diff --git a/README.md b/README.md index fcae73a32..6b8198373 100644 --- a/README.md +++ b/README.md @@ -18,11 +18,11 @@ Features are researched and developed in parallel, and then consolidated into se | Seq. | Code Name | Fork Epoch | Specs | | - | - | - | - | | 0 | **Phase0** |`0` |
  • Core
    • [The beacon chain](specs/phase0/beacon-chain.md)
    • [Deposit contract](specs/phase0/deposit-contract.md)
    • [Beacon chain fork choice](specs/phase0/fork-choice.md)
  • Additions
    • [Honest validator guide](specs/phase0/validator.md)
    • [P2P networking](specs/phase0/p2p-interface.md)
    • [Weak subjectivity](specs/phase0/weak-subjectivity.md)
| -| 1 | **Altair** | `74240` |
  • Core
    • [Beacon chain changes](specs/altair/beacon-chain.md)
    • [Altair fork](specs/altair/fork.md)
  • Additions
    • [Light client sync protocol](specs/altair/sync-protocol.md)
    • [Honest validator guide changes](specs/altair/validator.md)
    • [P2P networking](specs/altair/p2p-interface.md)
| +| 1 | **Altair** | `74240` |
  • Core
    • [Beacon chain changes](specs/altair/beacon-chain.md)
    • [Altair fork](specs/altair/fork.md)
  • Additions
    • [Light client sync protocol](specs/altair/light-client/sync-protocol.md)
    • [Honest validator guide changes](specs/altair/validator.md)
    • [P2P networking](specs/altair/p2p-interface.md)
| | 2 | **Bellatrix**
(["The Merge"](https://ethereum.org/en/upgrades/merge/)) | TBD |
  • Core
    • [Beacon Chain changes](specs/bellatrix/beacon-chain.md)
    • [Bellatrix fork](specs/bellatrix/fork.md)
    • [Fork choice changes](specs/bellatrix/fork-choice.md)
  • Additions
    • [Honest validator guide changes](specs/bellatrix/validator.md)
    • [P2P networking](specs/bellatrix/p2p-interface.md)
| ### In-development Specifications -| Code Name or Topic | Specs | Notes | +| Code Name or Topic | Specs | Notes | | - | - | - | | Capella (tentative) |
  • Core
    • [Beacon chain changes](specs/capella/beacon-chain.md)
    • [Capella fork](specs/capella/fork.md)
  • Additions
    • [Validator additions](specs/capella/validator.md)
| | EIP4844 (tentative) |
  • Core
    • [Beacon Chain changes](specs/eip4844/beacon-chain.md)
    • [EIP-4844 fork](specs/eip4844/fork.md)
  • Additions
    • [Honest validator guide changes](specs/eip4844/validator.md)
    • [P2P networking](specs/eip4844/p2p-interface.md)
| diff --git a/setup.py b/setup.py index 0c37e33c6..16b002e2d 100644 --- a/setup.py +++ b/setup.py @@ -889,12 +889,12 @@ class PySpecCommand(Command): """ if self.spec_fork in (ALTAIR, BELLATRIX, CAPELLA): self.md_doc_paths += """ + specs/altair/light-client/sync-protocol.md specs/altair/beacon-chain.md specs/altair/bls.md specs/altair/fork.md specs/altair/validator.md specs/altair/p2p-interface.md - specs/altair/sync-protocol.md """ if self.spec_fork in (BELLATRIX, CAPELLA): self.md_doc_paths += """ diff --git a/specs/altair/sync-protocol.md b/specs/altair/light-client/sync-protocol.md similarity index 100% rename from specs/altair/sync-protocol.md rename to specs/altair/light-client/sync-protocol.md diff --git a/specs/altair/validator.md b/specs/altair/validator.md index c59aa29f7..626535374 100644 --- a/specs/altair/validator.md +++ b/specs/altair/validator.md @@ -54,7 +54,7 @@ It builds on the [previous document for the behavior of an "honest validator" fr This previous document is referred to below as the "Phase 0 document". Altair introduces a new type of committee: the sync committee. Sync committees are responsible for signing each block of the canonical chain and there exists an efficient algorithm for light clients to sync the chain using the output of the sync committees. -See the [sync protocol](./sync-protocol.md) for further details on the light client sync. +See the [sync protocol](./light-client/sync-protocol.md) for further details on the light client sync. Under this network upgrade, validators track their participation in this new committee type and produce the relevant signatures as required. Block proposers incorporate the (aggregated) sync committee signatures into each block they produce. @@ -265,7 +265,7 @@ This process occurs each slot. ##### Prepare sync committee message -If a validator is in the current sync committee (i.e. `is_assigned_to_sync_committee()` above returns `True`), then for every `slot` in the current sync committee period, the validator should prepare a `SyncCommitteeMessage` for the previous slot (`slot - 1`) according to the logic in `get_sync_committee_message` as soon as they have determined the head block of `slot - 1`. This means that when assigned to `slot` a `SyncCommitteeMessage` is prepared and broadcast in `slot-1 ` instead of `slot`. +If a validator is in the current sync committee (i.e. `is_assigned_to_sync_committee()` above returns `True`), then for every `slot` in the current sync committee period, the validator should prepare a `SyncCommitteeMessage` for the previous slot (`slot - 1`) according to the logic in `get_sync_committee_message` as soon as they have determined the head block of `slot - 1`. This means that when assigned to `slot` a `SyncCommitteeMessage` is prepared and broadcast in `slot-1 ` instead of `slot`. This logic is triggered upon the same conditions as when producing an attestation. Meaning, a sync committee member should produce and broadcast a `SyncCommitteeMessage` either when (a) the validator has received a valid block from the expected block proposer for the current `slot` or (b) one-third of the slot has transpired (`SECONDS_PER_SLOT / INTERVALS_PER_SLOT` seconds after the start of the slot) -- whichever comes first. From 88206117c6d7be6752455f8fa1209cce05da4287 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 15 Jul 2022 13:37:27 +0200 Subject: [PATCH 056/105] Extract full node specifics to separate doc --- README.md | 2 +- setup.py | 1 + specs/altair/light-client/full-node.md | 135 +++++++++++++++++++++ specs/altair/light-client/sync-protocol.md | 119 +----------------- 4 files changed, 141 insertions(+), 116 deletions(-) create mode 100644 specs/altair/light-client/full-node.md diff --git a/README.md b/README.md index 6b8198373..9431dd201 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Features are researched and developed in parallel, and then consolidated into se | Seq. | Code Name | Fork Epoch | Specs | | - | - | - | - | | 0 | **Phase0** |`0` |
  • Core
    • [The beacon chain](specs/phase0/beacon-chain.md)
    • [Deposit contract](specs/phase0/deposit-contract.md)
    • [Beacon chain fork choice](specs/phase0/fork-choice.md)
  • Additions
    • [Honest validator guide](specs/phase0/validator.md)
    • [P2P networking](specs/phase0/p2p-interface.md)
    • [Weak subjectivity](specs/phase0/weak-subjectivity.md)
| -| 1 | **Altair** | `74240` |
  • Core
    • [Beacon chain changes](specs/altair/beacon-chain.md)
    • [Altair fork](specs/altair/fork.md)
  • Additions
    • [Light client sync protocol](specs/altair/light-client/sync-protocol.md)
    • [Honest validator guide changes](specs/altair/validator.md)
    • [P2P networking](specs/altair/p2p-interface.md)
| +| 1 | **Altair** | `74240` |
  • Core
    • [Beacon chain changes](specs/altair/beacon-chain.md)
    • [Altair fork](specs/altair/fork.md)
  • Additions
    • [Light client sync protocol](specs/altair/light-client/sync-protocol.md) ([full node](specs/altair/light-client/full-node.md))
    • [Honest validator guide changes](specs/altair/validator.md)
    • [P2P networking](specs/altair/p2p-interface.md)
| | 2 | **Bellatrix**
(["The Merge"](https://ethereum.org/en/upgrades/merge/)) | TBD |
  • Core
    • [Beacon Chain changes](specs/bellatrix/beacon-chain.md)
    • [Bellatrix fork](specs/bellatrix/fork.md)
    • [Fork choice changes](specs/bellatrix/fork-choice.md)
  • Additions
    • [Honest validator guide changes](specs/bellatrix/validator.md)
    • [P2P networking](specs/bellatrix/p2p-interface.md)
| ### In-development Specifications diff --git a/setup.py b/setup.py index 16b002e2d..36e9dc656 100644 --- a/setup.py +++ b/setup.py @@ -889,6 +889,7 @@ class PySpecCommand(Command): """ if self.spec_fork in (ALTAIR, BELLATRIX, CAPELLA): self.md_doc_paths += """ + specs/altair/light-client/full-node.md specs/altair/light-client/sync-protocol.md specs/altair/beacon-chain.md specs/altair/bls.md diff --git a/specs/altair/light-client/full-node.md b/specs/altair/light-client/full-node.md new file mode 100644 index 000000000..a7dd55101 --- /dev/null +++ b/specs/altair/light-client/full-node.md @@ -0,0 +1,135 @@ +# Altair Light Client -- Full Node + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + + +- [Introduction](#introduction) +- [Helper functions](#helper-functions) + - [`compute_merkle_proof_for_state`](#compute_merkle_proof_for_state) +- [Deriving light client data](#deriving-light-client-data) + - [`create_light_client_bootstrap`](#create_light_client_bootstrap) + - [`create_light_client_update`](#create_light_client_update) + + + + +## Introduction + +This document provides helper functions to enable full nodes to serve light client data. Full nodes SHOULD implement the described functionality to enable light clients to sync with the network. + +## Helper functions + +### `compute_merkle_proof_for_state` + +```python +def compute_merkle_proof_for_state(state: BeaconState, + index: GeneralizedIndex) -> Sequence[Bytes32]: + ... +``` + +## Deriving light client data + +Full nodes are expected to derive light client data from historic blocks and states and provide it to other clients. + +### `create_light_client_bootstrap` + +```python +def create_light_client_bootstrap(state: BeaconState) -> LightClientBootstrap: + assert compute_epoch_at_slot(state.slot) >= ALTAIR_FORK_EPOCH + assert state.slot == state.latest_block_header.slot + + return LightClientBootstrap( + header=BeaconBlockHeader( + slot=state.latest_block_header.slot, + proposer_index=state.latest_block_header.proposer_index, + parent_root=state.latest_block_header.parent_root, + state_root=hash_tree_root(state), + body_root=state.latest_block_header.body_root, + ), + current_sync_committee=state.current_sync_committee, + current_sync_committee_branch=compute_merkle_proof_for_state(state, CURRENT_SYNC_COMMITTEE_INDEX) + ) +``` + +Full nodes SHOULD provide `LightClientBootstrap` for all finalized epoch boundary blocks in the epoch range `[max(ALTAIR_FORK_EPOCH, current_epoch - MIN_EPOCHS_FOR_BLOCK_REQUESTS), current_epoch]` where `current_epoch` is defined by the current wall-clock time. Full nodes MAY also provide `LightClientBootstrap` for other blocks. + +Blocks are considered to be epoch boundary blocks if their block root can occur as part of a valid `Checkpoint`, i.e., if their slot is the initial slot of an epoch, or if all following slots through the initial slot of the next epoch are empty (no block proposed / orphaned). + +`LightClientBootstrap` is computed from the block's immediate post state (without applying empty slots). + +### `create_light_client_update` + +To form a `LightClientUpdate`, the following historical states and blocks are needed: +- `state`: the post state of any block with a post-Altair parent block +- `block`: the corresponding block +- `attested_state`: the post state of the block referred to by `block.parent_root` +- `finalized_block`: the block referred to by `attested_state.finalized_checkpoint.root`, if locally available (may be unavailable, e.g., when using checkpoint sync, or if it was pruned locally) + +```python +def create_light_client_update(state: BeaconState, + block: SignedBeaconBlock, + attested_state: BeaconState, + finalized_block: Optional[SignedBeaconBlock]) -> LightClientUpdate: + assert compute_epoch_at_slot(attested_state.slot) >= ALTAIR_FORK_EPOCH + assert sum(block.message.body.sync_aggregate.sync_committee_bits) >= MIN_SYNC_COMMITTEE_PARTICIPANTS + + assert state.slot == state.latest_block_header.slot + header = state.latest_block_header.copy() + header.state_root = hash_tree_root(state) + assert hash_tree_root(header) == hash_tree_root(block.message) + update_signature_period = compute_sync_committee_period(compute_epoch_at_slot(block.message.slot)) + + assert attested_state.slot == attested_state.latest_block_header.slot + attested_header = attested_state.latest_block_header.copy() + attested_header.state_root = hash_tree_root(attested_state) + assert hash_tree_root(attested_header) == block.message.parent_root + update_attested_period = compute_sync_committee_period(compute_epoch_at_slot(attested_header.slot)) + + # `next_sync_committee` is only useful if the message is signed by the current sync committee + if update_attested_period == update_signature_period: + next_sync_committee = attested_state.next_sync_committee + next_sync_committee_branch = compute_merkle_proof_for_state(attested_state, NEXT_SYNC_COMMITTEE_INDEX) + else: + next_sync_committee = SyncCommittee() + next_sync_committee_branch = [Bytes32() for _ in range(floorlog2(NEXT_SYNC_COMMITTEE_INDEX))] + + # Indicate finality whenever possible + if finalized_block is not None: + if finalized_block.message.slot != GENESIS_SLOT: + finalized_header = BeaconBlockHeader( + slot=finalized_block.message.slot, + proposer_index=finalized_block.message.proposer_index, + parent_root=finalized_block.message.parent_root, + state_root=finalized_block.message.state_root, + body_root=hash_tree_root(finalized_block.message.body), + ) + assert hash_tree_root(finalized_header) == attested_state.finalized_checkpoint.root + else: + assert attested_state.finalized_checkpoint.root == Bytes32() + finalized_header = BeaconBlockHeader() + finality_branch = compute_merkle_proof_for_state(attested_state, FINALIZED_ROOT_INDEX) + else: + finalized_header = BeaconBlockHeader() + finality_branch = [Bytes32() for _ in range(floorlog2(FINALIZED_ROOT_INDEX))] + + return LightClientUpdate( + attested_header=attested_header, + next_sync_committee=next_sync_committee, + next_sync_committee_branch=next_sync_committee_branch, + finalized_header=finalized_header, + finality_branch=finality_branch, + sync_aggregate=block.message.body.sync_aggregate, + signature_slot=block.message.slot, + ) +``` + +Full nodes SHOULD provide the best derivable `LightClientUpdate` (according to `is_better_update`) for each sync committee period covering any epochs in range `[max(ALTAIR_FORK_EPOCH, current_epoch - MIN_EPOCHS_FOR_BLOCK_REQUESTS), current_epoch]` where `current_epoch` is defined by the current wall-clock time. Full nodes MAY also provide `LightClientUpdate` for other sync committee periods. + +- `LightClientUpdate` are assigned to sync committee periods based on their `attested_header.slot` +- `LightClientUpdate` are only considered if `compute_sync_committee_period(compute_epoch_at_slot(update.attested_header.slot)) == compute_sync_committee_period(compute_epoch_at_slot(update.signature_slot))` +- Only `LightClientUpdate` with `next_sync_committee` as selected by fork choice are provided, regardless of ranking by `is_better_update`. To uniquely identify a non-finalized sync committee fork, all of `period`, `current_sync_committee` and `next_sync_committee` need to be incorporated, as sync committees may reappear over time. diff --git a/specs/altair/light-client/sync-protocol.md b/specs/altair/light-client/sync-protocol.md index c8697d13c..a7ac8e367 100644 --- a/specs/altair/light-client/sync-protocol.md +++ b/specs/altair/light-client/sync-protocol.md @@ -1,4 +1,4 @@ -# Altair -- Minimal Light Client +# Altair Light Client -- Sync Protocol **Notice**: This document is a work-in-progress for researchers and implementers. @@ -24,7 +24,6 @@ - [`get_safety_threshold`](#get_safety_threshold) - [`compute_sync_committee_period_at_slot`](#compute_sync_committee_period_at_slot) - [`get_subtree_index`](#get_subtree_index) - - [`compute_merkle_proof_for_state`](#compute_merkle_proof_for_state) - [Light client initialization](#light-client-initialization) - [`initialize_light_client_store`](#initialize_light_client_store) - [Light client state updates](#light-client-state-updates) @@ -32,9 +31,6 @@ - [`validate_light_client_update`](#validate_light_client_update) - [`apply_light_client_update`](#apply_light_client_update) - [`process_light_client_update`](#process_light_client_update) -- [Deriving light client data](#deriving-light-client-data) - - [`create_light_client_bootstrap`](#create_light_client_bootstrap) - - [`create_light_client_update`](#create_light_client_update) @@ -49,6 +45,9 @@ and metered VMs (e.g. blockchain VMs for cross-chain bridges). This document suggests a minimal light client design for the beacon chain that uses sync committees introduced in [this beacon chain extension](./beacon-chain.md). +Additional documents describe how the light client sync protocol can be used: +- [Full node](./full-node.md) + ## Constants | Name | Value | @@ -219,14 +218,6 @@ def get_subtree_index(generalized_index: GeneralizedIndex) -> uint64: return uint64(generalized_index % 2**(floorlog2(generalized_index))) ``` -### `compute_merkle_proof_for_state` - -```python -def compute_merkle_proof_for_state(state: BeaconState, - index: GeneralizedIndex) -> Sequence[Bytes32]: - ... -``` - ## Light client initialization A light client maintains its state in a `store` object of type `LightClientStore`. `initialize_light_client_store` initializes a new `store` with a received `LightClientBootstrap` derived from a given `trusted_block_root`. @@ -429,105 +420,3 @@ def process_light_client_update(store: LightClientStore, apply_light_client_update(store, update) store.best_valid_update = None ``` - -## Deriving light client data - -Full nodes are expected to derive light client data from historic blocks and states and provide it to other clients. - -### `create_light_client_bootstrap` - -```python -def create_light_client_bootstrap(state: BeaconState) -> LightClientBootstrap: - assert compute_epoch_at_slot(state.slot) >= ALTAIR_FORK_EPOCH - assert state.slot == state.latest_block_header.slot - - return LightClientBootstrap( - header=BeaconBlockHeader( - slot=state.latest_block_header.slot, - proposer_index=state.latest_block_header.proposer_index, - parent_root=state.latest_block_header.parent_root, - state_root=hash_tree_root(state), - body_root=state.latest_block_header.body_root, - ), - current_sync_committee=state.current_sync_committee, - current_sync_committee_branch=compute_merkle_proof_for_state(state, CURRENT_SYNC_COMMITTEE_INDEX) - ) -``` - -Full nodes SHOULD provide `LightClientBootstrap` for all finalized epoch boundary blocks in the epoch range `[max(ALTAIR_FORK_EPOCH, current_epoch - MIN_EPOCHS_FOR_BLOCK_REQUESTS), current_epoch]` where `current_epoch` is defined by the current wall-clock time. Full nodes MAY also provide `LightClientBootstrap` for other blocks. - -Blocks are considered to be epoch boundary blocks if their block root can occur as part of a valid `Checkpoint`, i.e., if their slot is the initial slot of an epoch, or if all following slots through the initial slot of the next epoch are empty (no block proposed / orphaned). - -`LightClientBootstrap` is computed from the block's immediate post state (without applying empty slots). - -### `create_light_client_update` - -To form a `LightClientUpdate`, the following historical states and blocks are needed: -- `state`: the post state of any block with a post-Altair parent block -- `block`: the corresponding block -- `attested_state`: the post state of the block referred to by `block.parent_root` -- `finalized_block`: the block referred to by `attested_state.finalized_checkpoint.root`, if locally available (may be unavailable, e.g., when using checkpoint sync, or if it was pruned locally) - -```python -def create_light_client_update(state: BeaconState, - block: SignedBeaconBlock, - attested_state: BeaconState, - finalized_block: Optional[SignedBeaconBlock]) -> LightClientUpdate: - assert compute_epoch_at_slot(attested_state.slot) >= ALTAIR_FORK_EPOCH - assert sum(block.message.body.sync_aggregate.sync_committee_bits) >= MIN_SYNC_COMMITTEE_PARTICIPANTS - - assert state.slot == state.latest_block_header.slot - header = state.latest_block_header.copy() - header.state_root = hash_tree_root(state) - assert hash_tree_root(header) == hash_tree_root(block.message) - update_signature_period = compute_sync_committee_period(compute_epoch_at_slot(block.message.slot)) - - assert attested_state.slot == attested_state.latest_block_header.slot - attested_header = attested_state.latest_block_header.copy() - attested_header.state_root = hash_tree_root(attested_state) - assert hash_tree_root(attested_header) == block.message.parent_root - update_attested_period = compute_sync_committee_period(compute_epoch_at_slot(attested_header.slot)) - - # `next_sync_committee` is only useful if the message is signed by the current sync committee - if update_attested_period == update_signature_period: - next_sync_committee = attested_state.next_sync_committee - next_sync_committee_branch = compute_merkle_proof_for_state(attested_state, NEXT_SYNC_COMMITTEE_INDEX) - else: - next_sync_committee = SyncCommittee() - next_sync_committee_branch = [Bytes32() for _ in range(floorlog2(NEXT_SYNC_COMMITTEE_INDEX))] - - # Indicate finality whenever possible - if finalized_block is not None: - if finalized_block.message.slot != GENESIS_SLOT: - finalized_header = BeaconBlockHeader( - slot=finalized_block.message.slot, - proposer_index=finalized_block.message.proposer_index, - parent_root=finalized_block.message.parent_root, - state_root=finalized_block.message.state_root, - body_root=hash_tree_root(finalized_block.message.body), - ) - assert hash_tree_root(finalized_header) == attested_state.finalized_checkpoint.root - else: - assert attested_state.finalized_checkpoint.root == Bytes32() - finalized_header = BeaconBlockHeader() - finality_branch = compute_merkle_proof_for_state(attested_state, FINALIZED_ROOT_INDEX) - else: - finalized_header = BeaconBlockHeader() - finality_branch = [Bytes32() for _ in range(floorlog2(FINALIZED_ROOT_INDEX))] - - return LightClientUpdate( - attested_header=attested_header, - next_sync_committee=next_sync_committee, - next_sync_committee_branch=next_sync_committee_branch, - finalized_header=finalized_header, - finality_branch=finality_branch, - sync_aggregate=block.message.body.sync_aggregate, - signature_slot=block.message.slot, - ) -``` - -Full nodes SHOULD provide the best derivable `LightClientUpdate` (according to `is_better_update`) for each sync committee period covering any epochs in range `[max(ALTAIR_FORK_EPOCH, current_epoch - MIN_EPOCHS_FOR_BLOCK_REQUESTS), current_epoch]` where `current_epoch` is defined by the current wall-clock time. Full nodes MAY also provide `LightClientUpdate` for other sync committee periods. - -- `LightClientUpdate` are assigned to sync committee periods based on their `attested_header.slot` -- `LightClientUpdate` are only considered if `compute_sync_committee_period(compute_epoch_at_slot(update.attested_header.slot)) == compute_sync_committee_period(compute_epoch_at_slot(update.signature_slot))` -- Only `LightClientUpdate` with `next_sync_committee` as selected by fork choice are provided, regardless of ranking by `is_better_update`. To uniquely identify a non-finalized sync committee fork, all of `period`, `current_sync_committee` and `next_sync_committee` need to be incorporated, as sync committees may reappear over time. From de15971ae81b7422e41546ee5a266c71d37f7fe4 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 15 Jul 2022 13:45:34 +0200 Subject: [PATCH 057/105] Cleanups --- specs/altair/light-client/sync-protocol.md | 14 +++++++------- tests/formats/sync_protocol/light_client_sync.md | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/specs/altair/light-client/sync-protocol.md b/specs/altair/light-client/sync-protocol.md index a7ac8e367..feda4dd5f 100644 --- a/specs/altair/light-client/sync-protocol.md +++ b/specs/altair/light-client/sync-protocol.md @@ -204,13 +204,6 @@ def get_safety_threshold(store: LightClientStore) -> uint64: ) // 2 ``` -### `compute_sync_committee_period_at_slot` - -```python -def compute_sync_committee_period_at_slot(slot: Slot) -> uint64: - return compute_sync_committee_period(compute_epoch_at_slot(slot)) -``` - ### `get_subtree_index` ```python @@ -218,6 +211,13 @@ def get_subtree_index(generalized_index: GeneralizedIndex) -> uint64: return uint64(generalized_index % 2**(floorlog2(generalized_index))) ``` +### `compute_sync_committee_period_at_slot` + +```python +def compute_sync_committee_period_at_slot(slot: Slot) -> uint64: + return compute_sync_committee_period(compute_epoch_at_slot(slot)) +``` + ## Light client initialization A light client maintains its state in a `store` object of type `LightClientStore`. `initialize_light_client_store` initializes a new `store` with a received `LightClientBootstrap` derived from a given `trusted_block_root`. diff --git a/tests/formats/sync_protocol/light_client_sync.md b/tests/formats/sync_protocol/light_client_sync.md index 6940ced87..31ecf6036 100644 --- a/tests/formats/sync_protocol/light_client_sync.md +++ b/tests/formats/sync_protocol/light_client_sync.md @@ -19,7 +19,7 @@ An SSZ-snappy encoded `bootstrap` object of type `LightClientBootstrap` to initi The steps to execute in sequence. -#### `checks` execution step +#### Checks to run after each step Each step includes checks to verify the expected impact on the `store` object. From a126f22cd9f9b24abeea28c6e4f52be84c30bbd5 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 15 Jul 2022 13:49:19 +0200 Subject: [PATCH 058/105] Sync tocs --- specs/altair/light-client/sync-protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/altair/light-client/sync-protocol.md b/specs/altair/light-client/sync-protocol.md index feda4dd5f..217171db1 100644 --- a/specs/altair/light-client/sync-protocol.md +++ b/specs/altair/light-client/sync-protocol.md @@ -22,8 +22,8 @@ - [`is_better_update`](#is_better_update) - [`is_next_sync_committee_known`](#is_next_sync_committee_known) - [`get_safety_threshold`](#get_safety_threshold) - - [`compute_sync_committee_period_at_slot`](#compute_sync_committee_period_at_slot) - [`get_subtree_index`](#get_subtree_index) + - [`compute_sync_committee_period_at_slot`](#compute_sync_committee_period_at_slot) - [Light client initialization](#light-client-initialization) - [`initialize_light_client_store`](#initialize_light_client_store) - [Light client state updates](#light-client-state-updates) From e75e35fad1a456831537e80a60ad0c68962af36a Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 15 Jul 2022 14:03:18 +0200 Subject: [PATCH 059/105] Update `make check_toc` list --- Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a1c2c80cf..d138028cd 100644 --- a/Makefile +++ b/Makefile @@ -23,13 +23,15 @@ GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENER # To check generator matching: #$(info $$GENERATOR_TARGETS is [${GENERATOR_TARGETS}]) -MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/phase0/*.md) $(wildcard $(SPEC_DIR)/altair/*.md) $(wildcard $(SSZ_DIR)/*.md) \ +MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/phase0/*.md) \ + $(wildcard $(SPEC_DIR)/altair/*.md) $(wildcard $(SPEC_DIR)/altair/**/*.md) \ $(wildcard $(SPEC_DIR)/bellatrix/*.md) \ $(wildcard $(SPEC_DIR)/capella/*.md) \ $(wildcard $(SPEC_DIR)/custody/*.md) \ $(wildcard $(SPEC_DIR)/das/*.md) \ $(wildcard $(SPEC_DIR)/sharding/*.md) \ - $(wildcard $(SPEC_DIR)/eip4844/*.md) + $(wildcard $(SPEC_DIR)/eip4844/*.md) \ + $(wildcard $(SSZ_DIR)/*.md) COV_HTML_OUT=.htmlcov COV_HTML_OUT_DIR=$(PY_SPEC_DIR)/$(COV_HTML_OUT) From c6cd35aea44260fe1819067afeb6c6522d9a47c3 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 15 Jul 2022 14:05:45 +0200 Subject: [PATCH 060/105] Tabs --> Spaces --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d138028cd..0956363e0 100644 --- a/Makefile +++ b/Makefile @@ -24,14 +24,14 @@ GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENER #$(info $$GENERATOR_TARGETS is [${GENERATOR_TARGETS}]) MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/phase0/*.md) \ - $(wildcard $(SPEC_DIR)/altair/*.md) $(wildcard $(SPEC_DIR)/altair/**/*.md) \ + $(wildcard $(SPEC_DIR)/altair/*.md) $(wildcard $(SPEC_DIR)/altair/**/*.md) \ $(wildcard $(SPEC_DIR)/bellatrix/*.md) \ $(wildcard $(SPEC_DIR)/capella/*.md) \ $(wildcard $(SPEC_DIR)/custody/*.md) \ $(wildcard $(SPEC_DIR)/das/*.md) \ $(wildcard $(SPEC_DIR)/sharding/*.md) \ $(wildcard $(SPEC_DIR)/eip4844/*.md) \ - $(wildcard $(SSZ_DIR)/*.md) + $(wildcard $(SSZ_DIR)/*.md) COV_HTML_OUT=.htmlcov COV_HTML_OUT_DIR=$(PY_SPEC_DIR)/$(COV_HTML_OUT) From 9c798806355f67a32704a8cd5a98d90deccb8f69 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 15 Jul 2022 22:18:33 +0800 Subject: [PATCH 061/105] Fix genesis testing tool and add Capella version `initialize_beacon_state_from_eth1` helper --- specs/capella/beacon-chain.md | 57 +++++++++++++++++++ .../pyspec/eth2spec/test/helpers/genesis.py | 5 +- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/specs/capella/beacon-chain.md b/specs/capella/beacon-chain.md index da67a8fcb..ca0305f04 100644 --- a/specs/capella/beacon-chain.md +++ b/specs/capella/beacon-chain.md @@ -43,6 +43,7 @@ - [Modified `process_execution_payload`](#modified-process_execution_payload) - [Modified `process_operations`](#modified-process_operations) - [New `process_bls_to_execution_change`](#new-process_bls_to_execution_change) +- [Testing](#testing) @@ -489,3 +490,59 @@ def process_bls_to_execution_change(state: BeaconState, + address_change.to_execution_address ) ``` + +## Testing + +*Note*: The function `initialize_beacon_state_from_eth1` is modified for pure Capella testing only. +Modifications include: +1. Use `CAPELLA_FORK_VERSION` as the current fork version. +2. Utilize the Capella `BeaconBlockBody` when constructing the initial `latest_block_header`. + +```python +def initialize_beacon_state_from_eth1(eth1_block_hash: Hash32, + eth1_timestamp: uint64, + deposits: Sequence[Deposit], + execution_payload_header: ExecutionPayloadHeader=ExecutionPayloadHeader() + ) -> BeaconState: + fork = Fork( + previous_version=CAPELLA_FORK_VERSION, # [Modified in Capella] for testing only + current_version=CAPELLA_FORK_VERSION, # [Modified in Capella] + epoch=GENESIS_EPOCH, + ) + state = BeaconState( + genesis_time=eth1_timestamp + GENESIS_DELAY, + fork=fork, + eth1_data=Eth1Data(block_hash=eth1_block_hash, deposit_count=uint64(len(deposits))), + latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), + randao_mixes=[eth1_block_hash] * EPOCHS_PER_HISTORICAL_VECTOR, # Seed RANDAO with Eth1 entropy + ) + + # Process deposits + leaves = list(map(lambda deposit: deposit.data, deposits)) + for index, deposit in enumerate(deposits): + deposit_data_list = List[DepositData, 2**DEPOSIT_CONTRACT_TREE_DEPTH](*leaves[:index + 1]) + state.eth1_data.deposit_root = hash_tree_root(deposit_data_list) + process_deposit(state, deposit) + + # Process activations + for index, validator in enumerate(state.validators): + balance = state.balances[index] + validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) + if validator.effective_balance == MAX_EFFECTIVE_BALANCE: + validator.activation_eligibility_epoch = GENESIS_EPOCH + validator.activation_epoch = GENESIS_EPOCH + + # Set genesis validators root for domain separation and chain versioning + state.genesis_validators_root = hash_tree_root(state.validators) + + # Fill in sync committees + # Note: A duplicate committee is assigned for the current and next committee at genesis + state.current_sync_committee = get_next_sync_committee(state) + state.next_sync_committee = get_next_sync_committee(state) + + # Initialize the execution payload header + # If empty, will initialize a chain that has not yet gone through the Merge transition + state.latest_execution_payload_header = execution_payload_header + + return state +``` diff --git a/tests/core/pyspec/eth2spec/test/helpers/genesis.py b/tests/core/pyspec/eth2spec/test/helpers/genesis.py index 83994c409..e9ef66711 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/genesis.py +++ b/tests/core/pyspec/eth2spec/test/helpers/genesis.py @@ -1,5 +1,5 @@ from eth2spec.test.helpers.constants import ( - ALTAIR, BELLATRIX, + ALTAIR, BELLATRIX, CAPELLA, FORKS_BEFORE_ALTAIR, FORKS_BEFORE_BELLATRIX, FORKS_BEFORE_CAPELLA, ) from eth2spec.test.helpers.keys import pubkeys @@ -57,6 +57,9 @@ def create_genesis_state(spec, validator_balances, activation_threshold): elif spec.fork == BELLATRIX: previous_version = spec.config.ALTAIR_FORK_VERSION current_version = spec.config.BELLATRIX_FORK_VERSION + elif spec.fork == CAPELLA: + previous_version = spec.config.BELLATRIX_FORK_VERSION + current_version = spec.config.CAPELLA_FORK_VERSION state = spec.BeaconState( genesis_time=0, From a205e9314d38e7bc12fea1b37078a9362a1f984c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 15 Jul 2022 23:29:22 +0800 Subject: [PATCH 062/105] Make block proposal direction more clear --- specs/eip4844/validator.md | 50 ++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/specs/eip4844/validator.md b/specs/eip4844/validator.md index 213c3c000..89389380f 100644 --- a/specs/eip4844/validator.md +++ b/specs/eip4844/validator.md @@ -21,6 +21,7 @@ - [`compute_aggregated_poly_and_commitment`](#compute_aggregated_poly_and_commitment) - [`verify_blobs_sidecar`](#verify_blobs_sidecar) - [`compute_proof_from_blobs`](#compute_proof_from_blobs) + - [`get_blobs_and_kzg_commitments`](#get_blobs_and_kzg_commitments) - [Beacon chain responsibilities](#beacon-chain-responsibilities) - [Block proposal](#block-proposal) - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody) @@ -171,6 +172,18 @@ def compute_proof_from_blobs(blobs: Sequence[BLSFieldElement]) -> KZGProof: return compute_kzg_proof(aggregated_poly, x) ``` +### `get_blobs_and_kzg_commitments` + +The interface to retrieve blobs and corresponding kzg commitments. + +Note: This API is *unstable*. `get_blobs_and_kzg_commitments` and `get_payload` may be unified. +Implementers may also retrieve blobs individually per transaction. + +```python +def get_blobs_and_kzg_commitments(payload_id: PayloadId) -> Tuple[Sequence[BLSFieldElement], Sequence[KZGCommitment]]: + ... +``` + ## Beacon chain responsibilities All validator responsibilities remain unchanged other than those noted below. @@ -182,33 +195,28 @@ Namely, the blob handling and the addition of `BlobsSidecar`. ##### Blob KZG commitments -After retrieving the execution payload from the execution engine as specified in Bellatrix, -the blobs are retrieved and processed: +1. After retrieving the execution payload from the execution engine as specified in Bellatrix, +use the `payload_id` to retrieve `blobs` and `blob_kzg_commitments` via `get_blobs_and_kzg_commitments(payload_id)`. +2. Verify `blobs` and `blob_kzg_commitments`: -``` -# execution_payload = execution_engine.get_payload(payload_id) -# block.body.execution_payload = execution_payload -# ... +```python +def verify_blobs_and_kzg_commitments(execution_payload: ExecutionPayload, + blobs: Sequence[BLSFieldElement], + blob_kzg_commitments: Sequence[KZGCommitment]) -> bool: + # Optionally sanity-check that the KZG commitments match the versioned hashes in the transactions + assert verify_kzg_commitments_against_transactions(execution_payload.transactions, blob_kzg_commitments) -blobs, blob_kzg_commitments = get_blobs(payload_id) - -# Optionally sanity-check that the KZG commitments match the versioned hashes in the transactions -assert verify_kzg_commitments_against_transactions(execution_payload.transactions, blob_kzg_commitments) - -# Optionally sanity-check that the KZG commitments match the blobs (as produced by the execution engine) -assert len(blob_kzg_commitments) == len(blobs) -assert [blob_to_kzg_commitment(blob) == commitment for blob, commitment in zip(blobs, blob_kzg_commitments)] - -# Update the block body -block.body.blob_kzg_commitments = blob_kzg_commitments + # Optionally sanity-check that the KZG commitments match the blobs (as produced by the execution engine) + assert len(blob_kzg_commitments) == len(blobs) + assert [blob_to_kzg_commitment(blob) == commitment for blob, commitment in zip(blobs, blob_kzg_commitments)] + return True ``` -The `blobs` should be held with the block in preparation of publishing. +3. If valid, set `block.body.blob_kzg_commitments = blob_kzg_commitments`. + +Note that the `blobs` should be held with the block in preparation of publishing. Without the `blobs`, the published block will effectively be ignored by honest validators. -Note: This API is *unstable*. `get_blobs` and `get_payload` may be unified. -Implementers may also retrieve blobs individually per transaction. - ### Beacon Block publishing time Before publishing a prepared beacon block proposal, the corresponding blobs are packaged into a sidecar object for distribution to the network: From 41767811234b55196d85dbc84eef74ece89ac16c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 15 Jul 2022 23:37:32 +0800 Subject: [PATCH 063/105] PR feedback from @djrtwo --- presets/mainnet/eip4844.yaml | 2 +- setup.py | 3 --- specs/eip4844/beacon-chain.md | 2 +- specs/eip4844/p2p-interface.md | 5 +---- 4 files changed, 3 insertions(+), 9 deletions(-) diff --git a/presets/mainnet/eip4844.yaml b/presets/mainnet/eip4844.yaml index dac58a9f0..c40908fde 100644 --- a/presets/mainnet/eip4844.yaml +++ b/presets/mainnet/eip4844.yaml @@ -5,4 +5,4 @@ # `uint64(4096)` FIELD_ELEMENTS_PER_BLOB: 4096 # `uint64(2**4)` (= 16) -MAX_BLOBS_PER_BLOCK: 16 \ No newline at end of file +MAX_BLOBS_PER_BLOCK: 16 diff --git a/setup.py b/setup.py index 36400e2ab..f25a3eea7 100644 --- a/setup.py +++ b/setup.py @@ -209,9 +209,6 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str]) -> elif source.startswith("class"): class_name, parent_class = _get_class_info_from_source(source) # check consistency with spec - if class_name != current_name: - print('class_name', class_name, 'current_name', current_name) - assert class_name == current_name if parent_class: assert parent_class == "Container" diff --git a/specs/eip4844/beacon-chain.md b/specs/eip4844/beacon-chain.md index 38c08453e..a1385d9e2 100644 --- a/specs/eip4844/beacon-chain.md +++ b/specs/eip4844/beacon-chain.md @@ -136,7 +136,7 @@ def verify_kzg_commitments_against_transactions(transactions: Sequence[Transacti all_versioned_hashes = [] for tx in transactions: if tx[0] == BLOB_TX_TYPE: - all_versioned_hashes.extend(tx_peek_blob_versioned_hashes(tx)) + all_versioned_hashes += tx_peek_blob_versioned_hashes(tx) return all_versioned_hashes == [kzg_commitment_to_versioned_hash(commitment) for commitment in kzg_commitments] ``` diff --git a/specs/eip4844/p2p-interface.md b/specs/eip4844/p2p-interface.md index 0c6f7bf37..147017eb7 100644 --- a/specs/eip4844/p2p-interface.md +++ b/specs/eip4844/p2p-interface.md @@ -107,7 +107,7 @@ Alias `sidecar = signed_blobs_sidecar.message`. - _[REJECT]_ The KZG proof is a correctly encoded compressed BLS G1 Point -- i.e. `bls.KeyValidate(blobs_sidecar.kzg_aggregated_proof) - _[REJECT]_ the beacon proposer signature, `signed_blobs_sidecar.signature`, is valid -- i.e. -``` +```py domain = get_domain(state, DOMAIN_BLOBS_SIDECAR, blobs_sidecar.beacon_block_slot // SLOTS_PER_EPOCH) signing_root = compute_signing_root(blobs_sidecar, domain) assert bls.Verify(proposer_pubkey, signing_root, signed_blob_header.signature) @@ -238,8 +238,6 @@ Clients MUST respond with blobs sidecars that are consistent from a single chain After the initial blobs sidecar, clients MAY stop in the process of responding if their fork choice changes the view of the chain in the context of the request. - - # Design decision rationale ## Why are blobs relayed as a sidecar, separate from beacon blocks? @@ -250,4 +248,3 @@ thus avoiding all blobs being downloaded by all beacon nodes on the network. Such sharding design may introduce an updated `BlobsSidecar` to identify the shard, but does not affect the `BeaconBlock` structure. - From 808f9c7f7f1f634d794c8aef048ff664aac61f94 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 15 Jul 2022 23:54:53 +0800 Subject: [PATCH 064/105] Use `validate_*` for the valdiation functions that return `None` --- specs/eip4844/p2p-interface.md | 4 +-- specs/eip4844/validator.md | 27 ++++++++++--------- .../unittests/validator/test_validator.py | 16 +++++------ 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/specs/eip4844/p2p-interface.md b/specs/eip4844/p2p-interface.md index 147017eb7..279594903 100644 --- a/specs/eip4844/p2p-interface.md +++ b/specs/eip4844/p2p-interface.md @@ -119,7 +119,7 @@ where `proposer_pubkey` is the pubkey of the beacon block proposer of `blobs_sid Note that a sidecar may be propagated before or after the corresponding beacon block. -Once both sidecar and beacon block are received, `verify_blobs_sidecar` can unlock the data-availability fork-choice dependency. +Once both sidecar and beacon block are received, `validate_blobs_sidecar` can unlock the data-availability fork-choice dependency. ### Transitioning the gossip @@ -190,7 +190,7 @@ The response is unsigned, i.e. `BlobsSidecarsByRange`, as the signature of the b may not be available beyond the initial distribution via gossip. Before consuming the next response chunk, the response reader SHOULD verify the blobs sidecar is well-formatted and -correct w.r.t. the expected KZG commitments through `verify_blobs_sidecar`. +correct w.r.t. the expected KZG commitments through `validate_blobs_sidecar`. `BlobsSidecarsByRange` is primarily used to sync blobs that may have been missed on gossip. diff --git a/specs/eip4844/validator.md b/specs/eip4844/validator.md index 89389380f..1a45342d0 100644 --- a/specs/eip4844/validator.md +++ b/specs/eip4844/validator.md @@ -19,7 +19,7 @@ - [`hash_to_bls_field`](#hash_to_bls_field) - [`compute_powers`](#compute_powers) - [`compute_aggregated_poly_and_commitment`](#compute_aggregated_poly_and_commitment) - - [`verify_blobs_sidecar`](#verify_blobs_sidecar) + - [`validate_blobs_sidecar`](#validate_blobs_sidecar) - [`compute_proof_from_blobs`](#compute_proof_from_blobs) - [`get_blobs_and_kzg_commitments`](#get_blobs_and_kzg_commitments) - [Beacon chain responsibilities](#beacon-chain-responsibilities) @@ -74,7 +74,7 @@ class PolynomialAndCommitment(Container): The implementation of `is_data_available` is meant to change with later sharding upgrades. Initially, it requires every verifying actor to retrieve the matching `BlobsSidecar`, -and verify the sidecar with `verify_blobs_sidecar`. +and validate the sidecar with `validate_blobs_sidecar`. Without the sidecar the block may be processed further optimistically, but MUST NOT be considered valid until a valid `BlobsSidecar` has been downloaded. @@ -83,7 +83,9 @@ but MUST NOT be considered valid until a valid `BlobsSidecar` has been downloade def is_data_available(slot: Slot, beacon_block_root: Root, blob_kzg_commitments: Sequence[KZGCommitment]) -> bool: # `retrieve_blobs_sidecar` is implementation dependent, raises an exception if not available. sidecar = retrieve_blobs_sidecar(slot, beacon_block_root) - return verify_blobs_sidecar(slot, beacon_block_root, blob_kzg_commitments, sidecar) + validate_blobs_sidecar(slot, beacon_block_root, blob_kzg_commitments, sidecar) + + return True ``` ### `hash_to_bls_field` @@ -132,11 +134,13 @@ def compute_aggregated_poly_and_commitment( return aggregated_poly, aggregated_poly_commitment ``` -### `verify_blobs_sidecar` +### `validate_blobs_sidecar` ```python -def verify_blobs_sidecar(slot: Slot, beacon_block_root: Root, - expected_kzg_commitments: Sequence[KZGCommitment], blobs_sidecar: BlobsSidecar) -> bool: +def validate_blobs_sidecar(slot: Slot, + beacon_block_root: Root, + expected_kzg_commitments: Sequence[KZGCommitment], + blobs_sidecar: BlobsSidecar) -> None: assert slot == blobs_sidecar.beacon_block_slot assert beacon_block_root == blobs_sidecar.beacon_block_root blobs = blobs_sidecar.blobs @@ -156,7 +160,7 @@ def verify_blobs_sidecar(slot: Slot, beacon_block_root: Root, y = evaluate_polynomial_in_evaluation_form(aggregated_poly, x) # Verify aggregated proof - return verify_kzg_proof(aggregated_poly_commitment, x, y, kzg_aggregated_proof) + assert verify_kzg_proof(aggregated_poly_commitment, x, y, kzg_aggregated_proof) ``` ### `compute_proof_from_blobs` @@ -197,19 +201,18 @@ Namely, the blob handling and the addition of `BlobsSidecar`. 1. After retrieving the execution payload from the execution engine as specified in Bellatrix, use the `payload_id` to retrieve `blobs` and `blob_kzg_commitments` via `get_blobs_and_kzg_commitments(payload_id)`. -2. Verify `blobs` and `blob_kzg_commitments`: +2. Validate `blobs` and `blob_kzg_commitments`: ```python -def verify_blobs_and_kzg_commitments(execution_payload: ExecutionPayload, - blobs: Sequence[BLSFieldElement], - blob_kzg_commitments: Sequence[KZGCommitment]) -> bool: +def validate_blobs_and_kzg_commitments(execution_payload: ExecutionPayload, + blobs: Sequence[BLSFieldElement], + blob_kzg_commitments: Sequence[KZGCommitment]) -> None: # Optionally sanity-check that the KZG commitments match the versioned hashes in the transactions assert verify_kzg_commitments_against_transactions(execution_payload.transactions, blob_kzg_commitments) # Optionally sanity-check that the KZG commitments match the blobs (as produced by the execution engine) assert len(blob_kzg_commitments) == len(blobs) assert [blob_to_kzg_commitment(blob) == commitment for blob, commitment in zip(blobs, blob_kzg_commitments)] - return True ``` 3. If valid, set `block.body.blob_kzg_commitments = blob_kzg_commitments`. diff --git a/tests/core/pyspec/eth2spec/test/eip4844/unittests/validator/test_validator.py b/tests/core/pyspec/eth2spec/test/eip4844/unittests/validator/test_validator.py index bd6374f3f..d0250f3df 100644 --- a/tests/core/pyspec/eth2spec/test/eip4844/unittests/validator/test_validator.py +++ b/tests/core/pyspec/eth2spec/test/eip4844/unittests/validator/test_validator.py @@ -30,7 +30,7 @@ def test_verify_kzg_proof(spec, state): assert spec.verify_kzg_proof(commitment, x, y, proof) -def _run_verify_blobs_sidecar_test(spec, state, blob_count): +def _run_validate_blobs_sidecar_test(spec, state, blob_count): block = build_empty_block_for_next_slot(spec, state) opaque_tx, blobs, blob_kzg_commitments = get_sample_opaque_tx(spec, blob_count=blob_count) block.body.blob_kzg_commitments = blob_kzg_commitments @@ -41,22 +41,22 @@ def _run_verify_blobs_sidecar_test(spec, state, blob_count): privkey = privkeys[1] spec.get_signed_blobs_sidecar(state, blobs_sidecar, privkey) expected_commitments = [spec.blob_to_kzg_commitment(blobs[i]) for i in range(blob_count)] - assert spec.verify_blobs_sidecar(block.slot, block.hash_tree_root(), expected_commitments, blobs_sidecar) + spec.validate_blobs_sidecar(block.slot, block.hash_tree_root(), expected_commitments, blobs_sidecar) @with_eip4844_and_later @spec_state_test -def test_verify_blobs_sidecar_one_blob(spec, state): - _run_verify_blobs_sidecar_test(spec, state, blob_count=1) +def test_validate_blobs_sidecar_one_blob(spec, state): + _run_validate_blobs_sidecar_test(spec, state, blob_count=1) @with_eip4844_and_later @spec_state_test -def test_verify_blobs_sidecar_two_blobs(spec, state): - _run_verify_blobs_sidecar_test(spec, state, blob_count=2) +def test_validate_blobs_sidecar_two_blobs(spec, state): + _run_validate_blobs_sidecar_test(spec, state, blob_count=2) @with_eip4844_and_later @spec_state_test -def test_verify_blobs_sidecar_ten_blobs(spec, state): - _run_verify_blobs_sidecar_test(spec, state, blob_count=10) +def test_validate_blobs_sidecar_ten_blobs(spec, state): + _run_validate_blobs_sidecar_test(spec, state, blob_count=10) From 400e1e54c14954fafea6db91823b5338ef35fb5b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 16 Jul 2022 00:01:33 +0800 Subject: [PATCH 065/105] Fix EIP4844 `create_genesis_state`, `EIP4844_FORK_VERSION`, and `config_fork_epoch_overrides` `config_fork_epoch_overrides`: since Capella and EIP4844 are in parallel, need to check if the field exists Update `compute_fork_version` --- specs/eip4844/fork.md | 25 ++++++++++++++++++- tests/core/pyspec/eth2spec/test/context.py | 24 ++++++++++++++---- .../pyspec/eth2spec/test/helpers/genesis.py | 5 +++- 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/specs/eip4844/fork.md b/specs/eip4844/fork.md index 6473f3c85..eaabba916 100644 --- a/specs/eip4844/fork.md +++ b/specs/eip4844/fork.md @@ -9,6 +9,9 @@ - [Introduction](#introduction) - [Configuration](#configuration) +- [Helper functions](#helper-functions) + - [Misc](#misc) + - [Modified `compute_fork_version`](#modified-compute_fork_version) - [Fork to EIP-4844](#fork-to-eip-4844) - [Fork trigger](#fork-trigger) - [Upgrading the state](#upgrading-the-state) @@ -25,9 +28,29 @@ Warning: this configuration is not definitive. | Name | Value | | - | - | -| `EIP4844_FORK_VERSION` | `Version('0x03000000')` | +| `EIP4844_FORK_VERSION` | `Version('0x04000000')` | | `EIP4844_FORK_EPOCH` | `Epoch(18446744073709551615)` **TBD** | +## Helper functions + +### Misc + +#### Modified `compute_fork_version` + +```python +def compute_fork_version(epoch: Epoch) -> Version: + """ + Return the fork version at the given ``epoch``. + """ + if epoch >= EIP4844_FORK_EPOCH: + return EIP4844_FORK_VERSION + if epoch >= BELLATRIX_FORK_EPOCH: + return BELLATRIX_FORK_VERSION + if epoch >= ALTAIR_FORK_EPOCH: + return ALTAIR_FORK_VERSION + return GENESIS_FORK_VERSION +``` + ## Fork to EIP-4844 ### Fork trigger diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index ec3032b28..0f62612e7 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -12,7 +12,7 @@ from eth2spec.utils import bls from .exceptions import SkippedTest from .helpers.constants import ( - PHASE0, ALTAIR, BELLATRIX, CAPELLA, EIP4844, + PHASE0, ALTAIR, BELLATRIX, CAPELLA, EIP4844, SHARDING, MINIMAL, MAINNET, ALL_PHASES, FORKS_BEFORE_ALTAIR, FORKS_BEFORE_BELLATRIX, ALL_FORK_UPGRADES, @@ -280,20 +280,34 @@ def spec_configured_state_test(conf): return decorator +def _check_current_version(spec, state, version_name): + fork_version_field = version_name.upper() + '_FORK_VERSION' + try: + fork_version = getattr(spec.config, fork_version_field) + except Exception: + return False + else: + return state.fork.current_version == fork_version + + def config_fork_epoch_overrides(spec, state): overrides = {} if state.fork.current_version == spec.config.GENESIS_FORK_VERSION: pass - elif state.fork.current_version == spec.config.ALTAIR_FORK_VERSION: + elif _check_current_version(spec, state, ALTAIR): overrides['ALTAIR_FORK_EPOCH'] = spec.GENESIS_EPOCH - elif state.fork.current_version == spec.config.BELLATRIX_FORK_VERSION: + elif _check_current_version(spec, state, BELLATRIX): overrides['ALTAIR_FORK_EPOCH'] = spec.GENESIS_EPOCH overrides['BELLATRIX_FORK_EPOCH'] = spec.GENESIS_EPOCH - elif state.fork.current_version == spec.config.CAPELLA_FORK_VERSION: + elif _check_current_version(spec, state, CAPELLA): overrides['ALTAIR_FORK_EPOCH'] = spec.GENESIS_EPOCH overrides['BELLATRIX_FORK_EPOCH'] = spec.GENESIS_EPOCH overrides['CAPELLA_FORK_EPOCH'] = spec.GENESIS_EPOCH - elif state.fork.current_version == spec.config.SHARDING_FORK_VERSION: + elif _check_current_version(spec, state, EIP4844): + overrides['ALTAIR_FORK_EPOCH'] = spec.GENESIS_EPOCH + overrides['BELLATRIX_FORK_EPOCH'] = spec.GENESIS_EPOCH + overrides['EIP4844_FORK_EPOCH'] = spec.GENESIS_EPOCH + elif _check_current_version(spec, state, SHARDING): overrides['ALTAIR_FORK_EPOCH'] = spec.GENESIS_EPOCH overrides['BELLATRIX_FORK_EPOCH'] = spec.GENESIS_EPOCH overrides['CAPELLA_FORK_EPOCH'] = spec.GENESIS_EPOCH diff --git a/tests/core/pyspec/eth2spec/test/helpers/genesis.py b/tests/core/pyspec/eth2spec/test/helpers/genesis.py index 67ff2d4a8..4d81de537 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/genesis.py +++ b/tests/core/pyspec/eth2spec/test/helpers/genesis.py @@ -1,5 +1,5 @@ from eth2spec.test.helpers.constants import ( - ALTAIR, BELLATRIX, CAPELLA, + ALTAIR, BELLATRIX, CAPELLA, EIP4844, FORKS_BEFORE_ALTAIR, FORKS_BEFORE_BELLATRIX, ) from eth2spec.test.helpers.keys import pubkeys @@ -57,6 +57,9 @@ def create_genesis_state(spec, validator_balances, activation_threshold): elif spec.fork == BELLATRIX: previous_version = spec.config.ALTAIR_FORK_VERSION current_version = spec.config.BELLATRIX_FORK_VERSION + elif spec.fork == EIP4844: + previous_version = spec.config.BELLATRIX_FORK_VERSION + current_version = spec.config.EIP4844_FORK_VERSION state = spec.BeaconState( genesis_time=0, From ec980dae4c1c4ef21a35dfd2d75fa59875b04d9f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 16 Jul 2022 00:14:52 +0800 Subject: [PATCH 066/105] Rework `blobs_sidecar` propagation guide --- specs/eip4844/p2p-interface.md | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/specs/eip4844/p2p-interface.md b/specs/eip4844/p2p-interface.md index 279594903..9bd206127 100644 --- a/specs/eip4844/p2p-interface.md +++ b/specs/eip4844/p2p-interface.md @@ -102,20 +102,16 @@ This topic is used to propagate data blobs included in any given beacon block. The following validations MUST pass before forwarding the `signed_blobs_sidecar` on the network; Alias `sidecar = signed_blobs_sidecar.message`. -- _[IGNORE]_ the `sidecar.beacon_block_slot` is for the current slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `blobs_sidecar.beacon_block_slot == current_slot`. +- _[IGNORE]_ the `sidecar.beacon_block_slot` is for the current slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `sidecar.beacon_block_slot == current_slot`. - _[REJECT]_ the `sidecar.blobs` are all well formatted, i.e. the `BLSFieldElement` in valid range (`x < BLS_MODULUS`). - _[REJECT]_ The KZG proof is a correctly encoded compressed BLS G1 Point -- i.e. `bls.KeyValidate(blobs_sidecar.kzg_aggregated_proof) - _[REJECT]_ the beacon proposer signature, `signed_blobs_sidecar.signature`, is valid -- i.e. - -```py -domain = get_domain(state, DOMAIN_BLOBS_SIDECAR, blobs_sidecar.beacon_block_slot // SLOTS_PER_EPOCH) -signing_root = compute_signing_root(blobs_sidecar, domain) -assert bls.Verify(proposer_pubkey, signing_root, signed_blob_header.signature) -``` - -where `proposer_pubkey` is the pubkey of the beacon block proposer of `blobs_sidecar.beacon_block_slot` + - Let `domain = get_domain(state, DOMAIN_BLOBS_SIDECAR, sidecar.beacon_block_slot // SLOTS_PER_EPOCH)` + - Let `signing_root = compute_signing_root(sidecar, domain)` + - Verify `bls.Verify(proposer_pubkey, signing_root, signed_blob_header.signature) is True`, + where `proposer_pubkey` is the pubkey of the beacon block proposer of `sidecar.beacon_block_slot` - _[IGNORE]_ The sidecar is the first sidecar with valid signature received for the `(proposer_index, sidecar.beacon_block_slot)` combination, - where `proposer_index` is the validator index of the beacon block proposer of `blobs_sidecar.beacon_block_slot` + where `proposer_index` is the validator index of the beacon block proposer of `sidecar.beacon_block_slot` Note that a sidecar may be propagated before or after the corresponding beacon block. From 5356fee28266223c977d9587d42c0777797ecfbd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 16 Jul 2022 01:28:19 +0800 Subject: [PATCH 067/105] Fix `test_override_config_fork_epoch` --- .../altair/unittests/test_config_override.py | 16 ++++++++++++---- tests/core/pyspec/eth2spec/test/context.py | 4 ++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/unittests/test_config_override.py b/tests/core/pyspec/eth2spec/test/altair/unittests/test_config_override.py index 5c940eafc..b7df49790 100644 --- a/tests/core/pyspec/eth2spec/test/altair/unittests/test_config_override.py +++ b/tests/core/pyspec/eth2spec/test/altair/unittests/test_config_override.py @@ -1,8 +1,10 @@ from eth2spec.test.context import ( + is_post_capella, + is_post_eip4844, spec_configured_state_test, spec_state_test_with_matching_config, with_all_phases, - with_phases + with_phases, ) from eth2spec.test.helpers.constants import ALTAIR @@ -38,9 +40,15 @@ def test_override_config_fork_epoch(spec, state): if state.fork.current_version == spec.config.BELLATRIX_FORK_VERSION: return - assert spec.config.CAPELLA_FORK_EPOCH == spec.GENESIS_EPOCH - if state.fork.current_version == spec.config.CAPELLA_FORK_VERSION: - return + if is_post_capella(spec): + assert spec.config.CAPELLA_FORK_EPOCH == spec.GENESIS_EPOCH + if state.fork.current_version == spec.config.CAPELLA_FORK_VERSION: + return + + if is_post_eip4844(spec): + assert spec.config.EIP4844_FORK_EPOCH == spec.GENESIS_EPOCH + if state.fork.current_version == spec.config.EIP4844_FORK_VERSION: + return assert spec.config.SHARDING_FORK_EPOCH == spec.GENESIS_EPOCH if state.fork.current_version == spec.config.SHARDING_FORK_VERSION: diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 0f62612e7..e46e41a70 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -596,6 +596,10 @@ def is_post_capella(spec): return spec.fork == CAPELLA +def is_post_eip4844(spec): + return spec.fork == EIP4844 + + with_altair_and_later = with_all_phases_except([PHASE0]) with_bellatrix_and_later = with_all_phases_except([PHASE0, ALTAIR]) with_capella_and_later = with_all_phases_except([PHASE0, ALTAIR, BELLATRIX, EIP4844]) From ec7c7c85893ba4d49d794da28d1dcf6e99753d00 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 16 Jul 2022 01:41:56 +0800 Subject: [PATCH 068/105] Remove leftover print --- tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py index 907e73b8d..44d663422 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py @@ -45,7 +45,6 @@ from eth2spec.test.context import ( @with_all_phases @spec_state_test def test_prev_slot_block_transition(spec, state): - print('spec.fork', spec.fork) # Go to clean slot spec.process_slots(state, state.slot + 1) # Make a block for it From 30d635259be4e79a92b3d7e044d659674544d85f Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 15 Jul 2022 21:36:26 +0200 Subject: [PATCH 069/105] Move test files to `light_client` subdir --- .../test/altair/{sync_protocol => light_client}/__init__.py | 0 .../test_light_client_sync.py => light_client/test_sync.py} | 0 .../{sync_protocol => light_client}/test_update_ranking.py | 0 .../test/altair/unittests/light_client}/__init__.py | 0 .../unittests/{ => light_client}/test_sync_protocol.py | 0 tests/formats/{sync_protocol => light_client}/README.md | 2 +- .../light_client_sync.md => light_client/sync.md} | 0 .../{sync_protocol => light_client}/update_ranking.md | 0 tests/generators/{sync_protocol => light_client}/README.md | 4 ++-- tests/generators/light_client/__init__.py | 0 tests/generators/{sync_protocol => light_client}/main.py | 6 +++--- .../{sync_protocol => light_client}/requirements.txt | 0 12 files changed, 6 insertions(+), 6 deletions(-) rename tests/core/pyspec/eth2spec/test/altair/{sync_protocol => light_client}/__init__.py (100%) rename tests/core/pyspec/eth2spec/test/altair/{sync_protocol/test_light_client_sync.py => light_client/test_sync.py} (100%) rename tests/core/pyspec/eth2spec/test/altair/{sync_protocol => light_client}/test_update_ranking.py (100%) rename tests/{generators/sync_protocol => core/pyspec/eth2spec/test/altair/unittests/light_client}/__init__.py (100%) rename tests/core/pyspec/eth2spec/test/altair/unittests/{ => light_client}/test_sync_protocol.py (100%) rename tests/formats/{sync_protocol => light_client}/README.md (73%) rename tests/formats/{sync_protocol/light_client_sync.md => light_client/sync.md} (100%) rename tests/formats/{sync_protocol => light_client}/update_ranking.md (100%) rename tests/generators/{sync_protocol => light_client}/README.md (53%) create mode 100644 tests/generators/light_client/__init__.py rename tests/generators/{sync_protocol => light_client}/main.py (62%) rename tests/generators/{sync_protocol => light_client}/requirements.txt (100%) diff --git a/tests/core/pyspec/eth2spec/test/altair/sync_protocol/__init__.py b/tests/core/pyspec/eth2spec/test/altair/light_client/__init__.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/altair/sync_protocol/__init__.py rename to tests/core/pyspec/eth2spec/test/altair/light_client/__init__.py diff --git a/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_light_client_sync.py b/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_light_client_sync.py rename to tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py diff --git a/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_update_ranking.py b/tests/core/pyspec/eth2spec/test/altair/light_client/test_update_ranking.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_update_ranking.py rename to tests/core/pyspec/eth2spec/test/altair/light_client/test_update_ranking.py diff --git a/tests/generators/sync_protocol/__init__.py b/tests/core/pyspec/eth2spec/test/altair/unittests/light_client/__init__.py similarity index 100% rename from tests/generators/sync_protocol/__init__.py rename to tests/core/pyspec/eth2spec/test/altair/unittests/light_client/__init__.py diff --git a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py b/tests/core/pyspec/eth2spec/test/altair/unittests/light_client/test_sync_protocol.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py rename to tests/core/pyspec/eth2spec/test/altair/unittests/light_client/test_sync_protocol.py diff --git a/tests/formats/sync_protocol/README.md b/tests/formats/light_client/README.md similarity index 73% rename from tests/formats/sync_protocol/README.md rename to tests/formats/light_client/README.md index c189cbc23..8c7571d4b 100644 --- a/tests/formats/sync_protocol/README.md +++ b/tests/formats/light_client/README.md @@ -3,5 +3,5 @@ This series of tests provides reference test vectors for the light client sync protocol spec. Handlers: -- `light_client_sync`: see [Light client sync test format](./light_client_sync.md) +- `sync`: see [Light client sync test format](./sync.md) - `update_ranking`: see [`LightClientUpdate` ranking test format](./update_ranking.md) diff --git a/tests/formats/sync_protocol/light_client_sync.md b/tests/formats/light_client/sync.md similarity index 100% rename from tests/formats/sync_protocol/light_client_sync.md rename to tests/formats/light_client/sync.md diff --git a/tests/formats/sync_protocol/update_ranking.md b/tests/formats/light_client/update_ranking.md similarity index 100% rename from tests/formats/sync_protocol/update_ranking.md rename to tests/formats/light_client/update_ranking.md diff --git a/tests/generators/sync_protocol/README.md b/tests/generators/light_client/README.md similarity index 53% rename from tests/generators/sync_protocol/README.md rename to tests/generators/light_client/README.md index f3c8ea339..7eabc2520 100644 --- a/tests/generators/sync_protocol/README.md +++ b/tests/generators/light_client/README.md @@ -1,5 +1,5 @@ -# Light client sync protocol tests +# Light client tests The purpose of this test-generator is to provide test-vectors for validating the correct implementation of the light client sync protocol. -Test-format documentation can be found [here](../../formats/sync_protocol/README.md). +Test-format documentation can be found [here](../../formats/light_client/README.md). diff --git a/tests/generators/light_client/__init__.py b/tests/generators/light_client/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/generators/sync_protocol/main.py b/tests/generators/light_client/main.py similarity index 62% rename from tests/generators/sync_protocol/main.py rename to tests/generators/light_client/main.py index 607587d93..ef6c2cbca 100644 --- a/tests/generators/sync_protocol/main.py +++ b/tests/generators/light_client/main.py @@ -3,8 +3,8 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators if __name__ == "__main__": - altair_mods = {key: 'eth2spec.test.altair.sync_protocol.test_' + key for key in [ - 'light_client_sync', + altair_mods = {key: 'eth2spec.test.altair.light_client.test_' + key for key in [ + 'sync', 'update_ranking', ]} bellatrix_mods = altair_mods @@ -14,4 +14,4 @@ if __name__ == "__main__": BELLATRIX: bellatrix_mods, } - run_state_test_generators(runner_name="sync_protocol", all_mods=all_mods) + run_state_test_generators(runner_name="light_client", all_mods=all_mods) diff --git a/tests/generators/sync_protocol/requirements.txt b/tests/generators/light_client/requirements.txt similarity index 100% rename from tests/generators/sync_protocol/requirements.txt rename to tests/generators/light_client/requirements.txt From 9d41c2306ffb4abce2094142bedddae570fb7758 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 15 Jul 2022 21:37:18 +0200 Subject: [PATCH 070/105] Align format of override `config.yaml` with base cfg --- setup.py | 2 +- .../gen_helpers/gen_base/gen_runner.py | 27 ++++++++++++++++--- .../gen_helpers/gen_base/gen_typing.py | 3 ++- tests/core/pyspec/eth2spec/test/context.py | 13 +++++---- .../core/pyspec/eth2spec/test/utils/utils.py | 1 + 5 files changed, 35 insertions(+), 11 deletions(-) diff --git a/setup.py b/setup.py index 36e9dc656..5dbcff276 100644 --- a/setup.py +++ b/setup.py @@ -783,7 +783,7 @@ def parse_config_vars(conf: Dict[str, str]) -> Dict[str, str]: for k, v in conf.items(): if isinstance(v, str) and (v.startswith("0x") or k == 'PRESET_BASE' or k == 'CONFIG_NAME'): # Represent byte data with string, to avoid misinterpretation as big-endian int. - # Everything is either byte data or an integer, with PRESET_BASE as one exception. + # Everything except PRESET_BASE and CONFIG_NAME is either byte data or an integer. out[k] = f"'{v}'" else: out[k] = str(int(v)) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index 6850af188..07daa9839 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -1,3 +1,4 @@ +from eth_utils import encode_hex import os import time import shutil @@ -95,6 +96,20 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): yaml = YAML(pure=True) yaml.default_flow_style = None + # Spec config is using a YAML subset + cfg_yaml = YAML(pure=True) + cfg_yaml.default_flow_style = False # Emit separate line for each key + + def cfg_represent_bytes(self, data): + return self.represent_int(encode_hex(data)) + + cfg_yaml.representer.add_representer(bytes, cfg_represent_bytes) + + def cfg_represent_quoted_str(self, data): + return self.represent_scalar(u'tag:yaml.org,2002:str', data, style="'") + + cfg_yaml.representer.add_representer(context.quoted_str, cfg_represent_quoted_str) + log_file = Path(output_dir) / 'testgen_error_log.txt' print(f"Generating tests into {output_dir}") @@ -167,10 +182,14 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): written_part = True if out_kind == "meta": meta[name] = data - if out_kind == "data": - output_part("data", name, dump_yaml_fn(data, name, file_mode, yaml)) - if out_kind == "ssz": - output_part("ssz", name, dump_ssz_fn(data, name, file_mode)) + elif out_kind == "cfg": + output_part(out_kind, name, dump_yaml_fn(data, name, file_mode, cfg_yaml)) + elif out_kind == "data": + output_part(out_kind, name, dump_yaml_fn(data, name, file_mode, yaml)) + elif out_kind == "ssz": + output_part(out_kind, name, dump_ssz_fn(data, name, file_mode)) + else: + assert False # Unknown kind except SkippedTest as e: print(e) skipped_test_count += 1 diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_typing.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_typing.py index 669238d1c..2bb66d06c 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_typing.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_typing.py @@ -10,9 +10,10 @@ from dataclasses import dataclass # Elements: name, out_kind, data # # out_kind is the type of data: +# - "meta" for generic data to collect into a meta data dict +# - "cfg" for a spec config dictionary # - "data" for generic # - "ssz" for SSZ encoded bytes -# - "meta" for generic data to collect into a meta data dict. TestCasePart = NewType("TestCasePart", Tuple[str, str, Any]) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 06313c195..3a0ee5ebb 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -1,7 +1,6 @@ import pytest from dataclasses import dataclass import importlib -from eth_utils import encode_hex from eth2spec.phase0 import mainnet as spec_phase0_mainnet, minimal as spec_phase0_minimal from eth2spec.altair import mainnet as spec_altair_mainnet, minimal as spec_altair_minimal @@ -506,18 +505,22 @@ def with_presets(preset_bases, reason=None): return decorator +class quoted_str(str): + pass + + def _get_basic_dict(ssz_dict: Dict[str, Any]) -> Dict[str, Any]: """ - Get dict of Python built-in types from a dict of SSZ objects. + Get dict of basic types from a dict of SSZ objects. """ result = {} for k, v in ssz_dict.items(): if isinstance(v, int): value = int(v) elif isinstance(v, bytes): - value = encode_hex(v) + value = bytes(bytearray(v)) else: - value = str(v) + value = quoted_str(v) result[k] = value return result @@ -553,7 +556,7 @@ def with_config_overrides(config_overrides): # To output the changed config to could be serialized with yaml test vectors, # the dict SSZ objects have to be converted into Python built-in types. output_config = _get_basic_dict(modified_config) - yield 'config', 'data', output_config + yield 'config', 'cfg', output_config spec.config = spec.Configuration(**modified_config) diff --git a/tests/core/pyspec/eth2spec/test/utils/utils.py b/tests/core/pyspec/eth2spec/test/utils/utils.py index bad6c867b..14a7ceeb9 100644 --- a/tests/core/pyspec/eth2spec/test/utils/utils.py +++ b/tests/core/pyspec/eth2spec/test/utils/utils.py @@ -17,6 +17,7 @@ def vector_test(description: str = None): # this wraps the function, to yield type-annotated entries of data. # Valid types are: # - "meta": all key-values with this type can be collected by the generator, to put somewhere together. + # - "cfg": spec config dictionary # - "ssz": raw SSZ bytes # - "data": a python structure to be encoded by the user. def entry(*args, **kw): From 261c5317e1929aaf64d0029ddb306308b0020ea2 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 15 Jul 2022 21:40:13 +0200 Subject: [PATCH 071/105] Remove genesis check now that Capella is fixed Co-authored-by: Hsiao-Wei Wang --- .../eth2spec/test/altair/light_client/test_sync.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py b/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py index d2eed0627..9448302e7 100644 --- a/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py +++ b/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py @@ -118,9 +118,6 @@ def compute_start_slot_at_next_sync_committee_period(spec, state): @spec_state_test_with_matching_config @with_presets([MINIMAL], reason="too slow") def test_light_client_sync(spec, state): - if state.fork.current_version == spec.config.GENESIS_FORK_VERSION: - return # Test requires Altair or later (`make test` does not honor phases) - # Start test test = yield from setup_test(spec, state) @@ -335,9 +332,6 @@ def test_light_client_sync(spec, state): @spec_state_test_with_matching_config @with_presets([MINIMAL], reason="too slow") def test_supply_sync_committee_from_past_update(spec, state): - if state.fork.current_version == spec.config.GENESIS_FORK_VERSION: - return # Test requires Altair or later (`make test` does not honor phases) - # Advance the chain, so that a `LightClientUpdate` from the past is available next_slots(spec, state, spec.SLOTS_PER_EPOCH * 2 - 1) finalized_block = state_transition_with_full_block(spec, state, True, True) @@ -367,9 +361,6 @@ def test_supply_sync_committee_from_past_update(spec, state): @spec_state_test_with_matching_config @with_presets([MINIMAL], reason="too slow") def test_advance_finality_without_sync_committee(spec, state): - if state.fork.current_version == spec.config.GENESIS_FORK_VERSION: - return # Test requires Altair or later (`make test` does not honor phases) - # Start test test = yield from setup_test(spec, state) From 647e9d0ec0adf0acab687e7b6fb4f5be02ae7190 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 18 Jul 2022 14:45:00 +0800 Subject: [PATCH 072/105] PR feedback from @ralexstokes --- specs/altair/beacon-chain.md | 2 +- specs/bellatrix/beacon-chain.md | 2 +- specs/capella/beacon-chain.md | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index a14ddb4f6..79ab8d022 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -682,7 +682,7 @@ def process_sync_committee_updates(state: BeaconState) -> None: This helper function is only for initializing the state for pure Altair testnets and tests. -*Note*: The function `initialize_beacon_state_from_eth1` is modified: (1) using `ALTAIR_FORK_VERSION` as the current fork version, (2) utilizing the Altair `BeaconBlockBody` when constructing the initial `latest_block_header`, and (3) adding initial sync committees. +*Note*: The function `initialize_beacon_state_from_eth1` is modified: (1) using `ALTAIR_FORK_VERSION` as the previous and current fork version, (2) utilizing the Altair `BeaconBlockBody` when constructing the initial `latest_block_header`, and (3) adding initial sync committees. ```python def initialize_beacon_state_from_eth1(eth1_block_hash: Hash32, diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index b37a8ab71..983ecda23 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -399,7 +399,7 @@ def process_slashings(state: BeaconState) -> None: *Note*: The function `initialize_beacon_state_from_eth1` is modified for pure Bellatrix testing only. Modifications include: -1. Use `BELLATRIX_FORK_VERSION` as the current fork version. +1. Use `BELLATRIX_FORK_VERSION` as the previous and current fork version. 2. Utilize the Bellatrix `BeaconBlockBody` when constructing the initial `latest_block_header`. 3. Initialize `latest_execution_payload_header`. If `execution_payload_header == ExecutionPayloadHeader()`, then the Merge has not yet occurred. diff --git a/specs/capella/beacon-chain.md b/specs/capella/beacon-chain.md index ca0305f04..06659349a 100644 --- a/specs/capella/beacon-chain.md +++ b/specs/capella/beacon-chain.md @@ -495,7 +495,7 @@ def process_bls_to_execution_change(state: BeaconState, *Note*: The function `initialize_beacon_state_from_eth1` is modified for pure Capella testing only. Modifications include: -1. Use `CAPELLA_FORK_VERSION` as the current fork version. +1. Use `CAPELLA_FORK_VERSION` as the previous and current fork version. 2. Utilize the Capella `BeaconBlockBody` when constructing the initial `latest_block_header`. ```python @@ -541,7 +541,6 @@ def initialize_beacon_state_from_eth1(eth1_block_hash: Hash32, state.next_sync_committee = get_next_sync_committee(state) # Initialize the execution payload header - # If empty, will initialize a chain that has not yet gone through the Merge transition state.latest_execution_payload_header = execution_payload_header return state From 02a2b71d64fcf5023a8bd890dabce774a6e9802e Mon Sep 17 00:00:00 2001 From: dankrad Date: Mon, 18 Jul 2022 18:20:25 +0100 Subject: [PATCH 073/105] WIP implementation of Danksharding (#2792) * Rough structure * Most of the KZG checks in * Fixes suggested by Vitalik * Add low degree check for commitments * Remove -1 check from degree proof (not needed) * Require block builders to be validators to simplify things * remove verify_intermediate_block_bid_commitment * Rename back to process_block * Degree check formula corrections * Updated TOC, bid processing corrections * Link to latest sharding doc * Add shard samples + P2P * Add validator guide for attestations and reconstruction * Update specs/sharding/beacon-chain.md Co-authored-by: terence tsao * Update specs/sharding/beacon-chain.md Co-authored-by: vbuterin * Update specs/sharding/beacon-chain.md Co-authored-by: vbuterin * Refactor polynomial operations into separate file * Add missing polynomial functions * Fix polynomial commitment file toc levels * Refactor the payload to make better use of unions * Add reverse bit order convention * Correct inequality in verify_degree_proof * Small fix * Fix polynomial evaluation * Update specs/sharding/beacon-chain.md Co-authored-by: George Kadianakis * MAX_BEACON_BLOCKS_BETWEEN_INTERMEDIATE_BLOCKS definition added * Sample reconstruction estimate * Update specs/sharding/beacon-chain.md Co-authored-by: Hsiao-Wei Wang * Update specs/sharding/polynomial-commitments.md Co-authored-by: Hsiao-Wei Wang * Fix return value of roots_of_unity() * Update specs/sharding/polynomial-commitments.md Co-authored-by: Hsiao-Wei Wang * Update specs/sharding/polynomial-commitments.md Co-authored-by: Hsiao-Wei Wang * Update specs/sharding/polynomial-commitments.md Co-authored-by: Hsiao-Wei Wang * Update specs/sharding/polynomial-commitments.md Co-authored-by: Hsiao-Wei Wang * Update specs/sharding/polynomial-commitments.md Co-authored-by: Hsiao-Wei Wang * Update specs/sharding/beacon-chain.md Co-authored-by: Hsiao-Wei Wang * Update specs/sharding/beacon-chain.md Co-authored-by: Hsiao-Wei Wang * Update specs/sharding/polynomial-commitments.md Co-authored-by: Hsiao-Wei Wang * Update specs/sharding/beacon-chain.md Co-authored-by: Hsiao-Wei Wang * Intermediate block -> Builder block * Some small omissions in intermediate -> builder * Update specs/sharding/polynomial-commitments.md Co-authored-by: Hsiao-Wei Wang * Update specs/sharding/polynomial-commitments.md Co-authored-by: George Kadianakis * Update specs/sharding/polynomial-commitments.md Co-authored-by: George Kadianakis * Update specs/sharding/polynomial-commitments.md Co-authored-by: Hsiao-Wei Wang * Update specs/sharding/polynomial-commitments.md Co-authored-by: Hsiao-Wei Wang * Update specs/sharding/polynomial-commitments.md Co-authored-by: Hsiao-Wei Wang * Update specs/sharding/polynomial-commitments.md Co-authored-by: Hsiao-Wei Wang * Update specs/sharding/polynomial-commitments.md Co-authored-by: Hsiao-Wei Wang * Update specs/sharding/polynomial-commitments.md Co-authored-by: Hsiao-Wei Wang * Update specs/sharding/polynomial-commitments.md Co-authored-by: Hsiao-Wei Wang * Update specs/sharding/polynomial-commitments.md Co-authored-by: Hsiao-Wei Wang * Update specs/sharding/polynomial-commitments.md Co-authored-by: Hsiao-Wei Wang * Z1 -> inf_G1 * Fix degree proof bound * SignedShardSample -> ShardSample * Typo * Don't allow 0 for `next_power_of_two` * Remove unused `ROOT_OF_UNITY` constant * Throwaway variable name * Fix function documentation of `bls_modular_inverse` * Builder block bid increase by at least 1%, no RANDAO processing in builder blocks * Fix tocs * Fix tocs * Fix typo * Update specs/sharding/polynomial-commitments.md Co-authored-by: George Kadianakis Co-authored-by: Danny Ryan Co-authored-by: Hsiao-Wei Wang Co-authored-by: terence tsao Co-authored-by: vbuterin Co-authored-by: George Kadianakis --- specs/sharding/beacon-chain.md | 1014 ++++++---------------- specs/sharding/p2p-interface.md | 170 +--- specs/sharding/polynomial-commitments.md | 396 +++++++++ specs/sharding/validator.md | 141 +++ 4 files changed, 850 insertions(+), 871 deletions(-) create mode 100644 specs/sharding/polynomial-commitments.md create mode 100644 specs/sharding/validator.md diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index ede369b95..7d6df51aa 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -10,60 +10,37 @@ - [Introduction](#introduction) - [Glossary](#glossary) -- [Custom types](#custom-types) - [Constants](#constants) - [Misc](#misc) - [Domain types](#domain-types) - - [Shard Work Status](#shard-work-status) - - [Misc](#misc-1) - - [Participation flag indices](#participation-flag-indices) - - [Incentivization weights](#incentivization-weights) - [Preset](#preset) - - [Misc](#misc-2) + - [Misc](#misc-1) + - [Time parameters](#time-parameters) - [Shard blob samples](#shard-blob-samples) - - [Precomputed size verification points](#precomputed-size-verification-points) - - [Gwei values](#gwei-values) - [Configuration](#configuration) -- [Updated containers](#updated-containers) - - [`AttestationData`](#attestationdata) - - [`BeaconBlockBody`](#beaconblockbody) - - [`BeaconState`](#beaconstate) -- [New containers](#new-containers) - - [`Builder`](#builder) - - [`DataCommitment`](#datacommitment) - - [`AttestedDataCommitment`](#attesteddatacommitment) - - [`ShardBlobBody`](#shardblobbody) - - [`ShardBlobBodySummary`](#shardblobbodysummary) - - [`ShardBlob`](#shardblob) - - [`ShardBlobHeader`](#shardblobheader) - - [`SignedShardBlob`](#signedshardblob) - - [`SignedShardBlobHeader`](#signedshardblobheader) - - [`PendingShardHeader`](#pendingshardheader) - - [`ShardBlobReference`](#shardblobreference) - - [`ShardProposerSlashing`](#shardproposerslashing) - - [`ShardWork`](#shardwork) + - [Time parameters](#time-parameters-1) +- [Containers](#containers) + - [New Containers](#new-containers) + - [`BuilderBlockBid`](#builderblockbid) + - [`BuilderBlockBidWithRecipientAddress`](#builderblockbidwithrecipientaddress) + - [`ShardedCommitmentsContainer`](#shardedcommitmentscontainer) + - [`ShardSample`](#shardsample) + - [Extended Containers](#extended-containers) + - [`BeaconState`](#beaconstate) + - [`BuilderBlockData`](#builderblockdata) + - [`BeaconBlockBody`](#beaconblockbody) - [Helper functions](#helper-functions) - - [Misc](#misc-3) - - [`next_power_of_two`](#next_power_of_two) - - [`compute_previous_slot`](#compute_previous_slot) - - [`compute_updated_sample_price`](#compute_updated_sample_price) - - [`compute_committee_source_epoch`](#compute_committee_source_epoch) - - [`batch_apply_participation_flag`](#batch_apply_participation_flag) - - [Beacon state accessors](#beacon-state-accessors) - - [Updated `get_committee_count_per_slot`](#updated-get_committee_count_per_slot) - - [`get_active_shard_count`](#get_active_shard_count) - - [`get_shard_proposer_index`](#get_shard_proposer_index) - - [`get_start_shard`](#get_start_shard) - - [`compute_shard_from_committee_index`](#compute_shard_from_committee_index) - - [`compute_committee_index_from_shard`](#compute_committee_index_from_shard) - [Block processing](#block-processing) - - [Operations](#operations) - - [Extended Attestation processing](#extended-attestation-processing) - - [`process_shard_header`](#process_shard_header) - - [`process_shard_proposer_slashing`](#process_shard_proposer_slashing) - - [Epoch transition](#epoch-transition) - - [`process_pending_shard_confirmations`](#process_pending_shard_confirmations) - - [`reset_pending_shard_work`](#reset_pending_shard_work) + - [`is_builder_block_slot`](#is_builder_block_slot) + - [Beacon state accessors](#beacon-state-accessors) + - [`get_active_shard_count`](#get_active_shard_count) +- [Beacon chain state transition function](#beacon-chain-state-transition-function) + - [Block processing](#block-processing-1) + - [`process_block`](#process_block) + - [Block header](#block-header) + - [Builder Block Bid](#builder-block-bid) + - [Sharded data](#sharded-data) + - [Execution payload](#execution-payload) @@ -72,26 +49,14 @@ ## Introduction This document describes the extensions made to the Phase 0 design of The Beacon Chain to support data sharding, -based on the ideas [here](https://hackmd.io/G-Iy5jqyT7CXWEz8Ssos8g) and more broadly [here](https://arxiv.org/abs/1809.09044), +based on the ideas [here](https://notes.ethereum.org/@dankrad/new_sharding) and more broadly [here](https://arxiv.org/abs/1809.09044), using KZG10 commitments to commit to data to remove any need for fraud proofs (and hence, safety-critical synchrony assumptions) in the design. ### Glossary - **Data**: A list of KZG points, to translate a byte string into - **Blob**: Data with commitments and meta-data, like a flattened bundle of L2 transactions. -- **Builder**: Independent actor that builds blobs and bids for proposal slots via fee-paying blob-headers, responsible for availability. -- **Shard proposer**: Validator taking bids from blob builders for shard data opportunity, co-signs with builder to propose the blob. -## Custom types - -We define the following Python custom types for type hinting and readability: - -| Name | SSZ equivalent | Description | -| - | - | - | -| `Shard` | `uint64` | A shard number | -| `BLSCommitment` | `Bytes48` | A G1 curve point | -| `BLSPoint` | `uint256` | A number `x` in the range `0 <= x < MODULUS` | -| `BuilderIndex` | `uint64` | Builder registry index | ## Constants @@ -101,48 +66,13 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | Notes | | - | - | - | -| `PRIMITIVE_ROOT_OF_UNITY` | `7` | Primitive root of unity of the BLS12_381 (inner) modulus | -| `DATA_AVAILABILITY_INVERSE_CODING_RATE` | `2**1` (= 2) | Factor by which samples are extended for data availability encoding | -| `POINTS_PER_SAMPLE` | `uint64(2**3)` (= 8) | 31 * 8 = 248 bytes | -| `MODULUS` | `0x73eda753299d7d483339d80809a1d80553bda402fffe5bfeffffffff00000001` (curve order of BLS12_381) | +| `FIELD_ELEMENTS_PER_SAMPLE` | `uint64(2**4)` (= 16) | 31 * 16 = 496 bytes | ### Domain types | Name | Value | | - | - | -| `DOMAIN_SHARD_BLOB` | `DomainType('0x80000000')` | - -### Shard Work Status - -| Name | Value | Notes | -| - | - | - | -| `SHARD_WORK_UNCONFIRMED` | `0` | Unconfirmed, nullified after confirmation time elapses | -| `SHARD_WORK_CONFIRMED` | `1` | Confirmed, reduced to just the commitment | -| `SHARD_WORK_PENDING` | `2` | Pending, a list of competing headers | - -### Misc - -TODO: `PARTICIPATION_FLAG_WEIGHTS` backwards-compatibility is difficult, depends on usage. - -| Name | Value | -| - | - | -| `PARTICIPATION_FLAG_WEIGHTS` | `[TIMELY_SOURCE_WEIGHT, TIMELY_TARGET_WEIGHT, TIMELY_HEAD_WEIGHT, TIMELY_SHARD_WEIGHT]` | - -### Participation flag indices - -| Name | Value | -| - | - | -| `TIMELY_SHARD_FLAG_INDEX` | `3` | - -### Incentivization weights - -TODO: determine weight for shard attestations - -| Name | Value | -| - | - | -| `TIMELY_SHARD_WEIGHT` | `uint64(8)` | - -TODO: `WEIGHT_DENOMINATOR` needs to be adjusted, but this breaks a lot of Altair code. +| `DOMAIN_SHARD_SAMPLE` | `DomainType('0x10000000')` | ## Preset @@ -150,341 +80,140 @@ TODO: `WEIGHT_DENOMINATOR` needs to be adjusted, but this breaks a lot of Altair | Name | Value | Notes | | - | - | - | -| `MAX_SHARDS` | `uint64(2**10)` (= 1,024) | Theoretical max shard count (used to determine data structure sizes) | -| `INITIAL_ACTIVE_SHARDS` | `uint64(2**6)` (= 64) | Initial shard count | -| `SAMPLE_PRICE_ADJUSTMENT_COEFFICIENT` | `uint64(2**3)` (= 8) | Sample price may decrease/increase by at most exp(1 / this value) *per epoch* | -| `MAX_SHARD_PROPOSER_SLASHINGS` | `2**4` (= 16) | Maximum amount of shard proposer slashing operations per block | -| `MAX_SHARD_HEADERS_PER_SHARD` | `4` | | -| `SHARD_STATE_MEMORY_SLOTS` | `uint64(2**8)` (= 256) | Number of slots for which shard commitments and confirmation status is directly available in the state | -| `BLOB_BUILDER_REGISTRY_LIMIT` | `uint64(2**40)` (= 1,099,511,627,776) | shard blob builders | +| `MAX_SHARDS` | `uint64(2**12)` (= 4,096) | Theoretical max shard count (used to determine data structure sizes) | +| `ACTIVE_SHARDS` | `uint64(2**8)` (= 256) | Initial shard count | +| `MAX_PROPOSER_BLOCKS_BETWEEN_BUILDER_BLOCKS` | `uint64(2**4)` (= 16) | TODO: Need to define what happens if there were more blocks without builder blocks | + +### Time parameters + +With the introduction of builder blocks the number of slots per epoch is doubled (it counts beacon blocks and builder blocks). + +| Name | Value | Unit | Duration | +| - | - | :-: | :-: | +| `SLOTS_PER_EPOCH` | `uint64(2**6)` (= 64) | slots | 8:32 minutes | ### Shard blob samples | Name | Value | Notes | | - | - | - | -| `MAX_SAMPLES_PER_BLOB` | `uint64(2**11)` (= 2,048) | 248 * 2,048 = 507,904 bytes | -| `TARGET_SAMPLES_PER_BLOB` | `uint64(2**10)` (= 1,024) | 248 * 1,024 = 253,952 bytes | - -### Precomputed size verification points - -| Name | Value | -| - | - | -| `G1_SETUP` | Type `List[G1]`. The G1-side trusted setup `[G, G*s, G*s**2....]`; note that the first point is the generator. | -| `G2_SETUP` | Type `List[G2]`. The G2-side trusted setup `[G, G*s, G*s**2....]` | -| `ROOT_OF_UNITY` | `pow(PRIMITIVE_ROOT_OF_UNITY, (MODULUS - 1) // int(MAX_SAMPLES_PER_BLOB * POINTS_PER_SAMPLE), MODULUS)` | - -### Gwei values - -| Name | Value | Unit | Description | -| - | - | - | - | -| `MAX_SAMPLE_PRICE` | `Gwei(2**33)` (= 8,589,934,592) | Gwei | Max sample charged for a TARGET-sized shard blob | -| `MIN_SAMPLE_PRICE` | `Gwei(2**3)` (= 8) | Gwei | Min sample price charged for a TARGET-sized shard blob | +| `SAMPLES_PER_BLOB` | `uint64(2**9)` (= 512) | 248 * 512 = 126,976 bytes | ## Configuration Note: Some preset variables may become run-time configurable for testnets, but default to a preset while the spec is unstable. -E.g. `INITIAL_ACTIVE_SHARDS`, `MAX_SAMPLES_PER_BLOB` and `TARGET_SAMPLES_PER_BLOB`. +E.g. `ACTIVE_SHARDS` and `SAMPLES_PER_BLOB`. -## Updated containers +### Time parameters -The following containers have updated definitions to support Sharding. +| Name | Value | Unit | Duration | +| - | - | :-: | :-: | +| `SECONDS_PER_SLOT` | `uint64(8)` | seconds | 8 seconds | -### `AttestationData` +## Containers + +### New Containers + +#### `BuilderBlockBid` ```python -class AttestationData(Container): +class BuilderBlockBid(Container): slot: Slot - index: CommitteeIndex - # LMD GHOST vote - beacon_block_root: Root - # FFG vote - source: Checkpoint - target: Checkpoint - # Hash-tree-root of ShardBlob - shard_blob_root: Root # [New in Sharding] + parent_block_root: Root + + execution_payload_root: Root + + sharded_data_commitment_root: Root # Root of the sharded data (only data, not beacon/builder block commitments) + + sharded_data_commitment_count: uint64 # Count of sharded data commitments + + bid: Gwei # Block builder bid paid to proposer + + validator_index: ValidatorIndex # Validator index for this bid + + # Block builders use an Eth1 address -- need signature as + # block bid and data gas base fees will be charged to this address + signature_y_parity: bool + signature_r: uint256 + signature_s: uint256 ``` -### `BeaconBlockBody` +#### `BuilderBlockBidWithRecipientAddress` ```python -class BeaconBlockBody(bellatrix.BeaconBlockBody): # [extends Bellatrix block body] - shard_proposer_slashings: List[ShardProposerSlashing, MAX_SHARD_PROPOSER_SLASHINGS] - shard_headers: List[SignedShardBlobHeader, MAX_SHARDS * MAX_SHARD_HEADERS_PER_SHARD] +class BuilderBlockBidWithRecipientAddress(Container): + builder_block_bid: Union[None, BuilderBlockBid] + recipient_address: ExecutionAddress # Address to receive the block builder bid ``` -### `BeaconState` +#### `ShardedCommitmentsContainer` + +```python +class ShardedCommitmentsContainer(Container): + sharded_commitments: List[KZGCommitment, 2 * MAX_SHARDS] + + # Aggregate degree proof for all sharded_commitments + degree_proof: KZGCommitment + + # The sizes of the blocks encoded in the commitments (last builder and all beacon blocks since) + included_block_sizes: List[uint64, MAX_PROPOSER_BLOCKS_BETWEEN_BUILDER_BLOCKS + 1] + + # Number of commitments that are for sharded data (no blocks) + included_sharded_data_commitments: uint64 + + # Random evaluation of beacon blocks + execution payload (this helps with quick verification) + block_verification_kzg_proof: KZGCommitment +``` + +#### `ShardSample` + +```python +class ShardSample(Container): + slot: Slot + row: uint64 + column: uint64 + data: Vector[BLSFieldElement, FIELD_ELEMENTS_PER_SAMPLE] + proof: KZGCommitment + builder: ValidatorIndex + signature: BLSSignature +``` + +### Extended Containers + +#### `BeaconState` ```python class BeaconState(bellatrix.BeaconState): - # Blob builder registry. - blob_builders: List[Builder, BLOB_BUILDER_REGISTRY_LIMIT] - blob_builder_balances: List[Gwei, BLOB_BUILDER_REGISTRY_LIMIT] - # A ring buffer of the latest slots, with information per active shard. - shard_buffer: Vector[List[ShardWork, MAX_SHARDS], SHARD_STATE_MEMORY_SLOTS] - shard_sample_price: uint64 + blocks_since_builder_block: List[BeaconBlock, MAX_PROPOSER_BLOCKS_BETWEEN_BUILDER_BLOCKS] ``` -## New containers - -### `Builder` +#### `BuilderBlockData` ```python -class Builder(Container): - pubkey: BLSPubkey - # TODO: fields for either an expiry mechanism (refunding execution account with remaining balance) - # and/or a builder-transaction mechanism. -``` +class BuilderBlockData(Container): + execution_payload: ExecutionPayload + sharded_commitments_container: ShardedCommitmentsContainer +``` -### `DataCommitment` +#### `BeaconBlockBody` ```python -class DataCommitment(Container): - # KZG10 commitment to the data - point: BLSCommitment - # Length of the data in samples - samples_count: uint64 -``` - -### `AttestedDataCommitment` - -```python -class AttestedDataCommitment(Container): - # KZG10 commitment to the data, and length - commitment: DataCommitment - # hash_tree_root of the ShardBlobHeader (stored so that attestations can be checked against it) - root: Root - # The proposer who included the shard-header - includer_index: ValidatorIndex -``` - -### `ShardBlobBody` - -Unsigned shard data, bundled by a shard-builder. -Unique, signing different bodies as shard proposer for the same `(slot, shard)` is slashable. - -```python -class ShardBlobBody(Container): - # The actual data commitment - commitment: DataCommitment - # Proof that the degree < commitment.samples_count * POINTS_PER_SAMPLE - degree_proof: BLSCommitment - # The actual data. Should match the commitment and degree proof. - data: List[BLSPoint, POINTS_PER_SAMPLE * MAX_SAMPLES_PER_BLOB] - # fee payment fields (EIP 1559 like) - # TODO: express in MWei instead? - max_priority_fee_per_sample: Gwei - max_fee_per_sample: Gwei -``` - -### `ShardBlobBodySummary` - -Summary version of the `ShardBlobBody`, omitting the data payload, while preserving the data-commitments. - -The commitments are not further collapsed to a single hash, -to avoid an extra network roundtrip between proposer and builder, to include the header on-chain more quickly. - -```python -class ShardBlobBodySummary(Container): - # The actual data commitment - commitment: DataCommitment - # Proof that the degree < commitment.samples_count * POINTS_PER_SAMPLE - degree_proof: BLSCommitment - # Hash-tree-root as summary of the data field - data_root: Root - # fee payment fields (EIP 1559 like) - # TODO: express in MWei instead? - max_priority_fee_per_sample: Gwei - max_fee_per_sample: Gwei -``` - -### `ShardBlob` - -`ShardBlobBody` wrapped with the header data that is unique to the shard blob proposal. - -```python -class ShardBlob(Container): - slot: Slot - shard: Shard - # Builder of the data, pays data-fee to proposer - builder_index: BuilderIndex - # Proposer of the shard-blob - proposer_index: ValidatorIndex - # Blob contents - body: ShardBlobBody -``` - -### `ShardBlobHeader` - -Header version of `ShardBlob`. - -```python -class ShardBlobHeader(Container): - slot: Slot - shard: Shard - # Builder of the data, pays data-fee to proposer - builder_index: BuilderIndex - # Proposer of the shard-blob - proposer_index: ValidatorIndex - # Blob contents, without the full data - body_summary: ShardBlobBodySummary -``` - -### `SignedShardBlob` - -Full blob data, signed by the shard builder (ensuring fee payment) and shard proposer (ensuring a single proposal). - -```python -class SignedShardBlob(Container): - message: ShardBlob - signature: BLSSignature -``` - -### `SignedShardBlobHeader` - -Header of the blob, the signature is equally applicable to `SignedShardBlob`. -Shard proposers can accept `SignedShardBlobHeader` as a data-transaction by co-signing the header. - -```python -class SignedShardBlobHeader(Container): - message: ShardBlobHeader - # Signature by builder. - # Once accepted by proposer, the signatures is the aggregate of both. - signature: BLSSignature -``` - -### `PendingShardHeader` - -```python -class PendingShardHeader(Container): - # The commitment that is attested - attested: AttestedDataCommitment - # Who voted for the header - votes: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] - # Sum of effective balances of votes - weight: Gwei - # When the header was last updated, as reference for weight accuracy - update_slot: Slot -``` - -### `ShardBlobReference` - -Reference version of `ShardBlobHeader`, substituting the body for just a hash-tree-root. - -```python -class ShardBlobReference(Container): - slot: Slot - shard: Shard - # Builder of the data - builder_index: BuilderIndex - # Proposer of the shard-blob - proposer_index: ValidatorIndex - # Blob hash-tree-root for slashing reference - body_root: Root -``` - -### `ShardProposerSlashing` - -```python -class ShardProposerSlashing(Container): - slot: Slot - shard: Shard - proposer_index: ValidatorIndex - builder_index_1: BuilderIndex - builder_index_2: BuilderIndex - body_root_1: Root - body_root_2: Root - signature_1: BLSSignature - signature_2: BLSSignature -``` - -### `ShardWork` - -```python -class ShardWork(Container): - # Upon confirmation the data is reduced to just the commitment. - status: Union[ # See Shard Work Status enum - None, # SHARD_WORK_UNCONFIRMED - AttestedDataCommitment, # SHARD_WORK_CONFIRMED - List[PendingShardHeader, MAX_SHARD_HEADERS_PER_SHARD] # SHARD_WORK_PENDING - ] +class BeaconBlockBody(altair.BeaconBlockBody): + payload_data: Union[BuilderBlockBid, BuilderBlockData] ``` ## Helper functions -### Misc +### Block processing -#### `next_power_of_two` +#### `is_builder_block_slot` ```python -def next_power_of_two(x: int) -> int: - return 2 ** ((x - 1).bit_length()) -``` - -#### `compute_previous_slot` - -```python -def compute_previous_slot(slot: Slot) -> Slot: - if slot > 0: - return Slot(slot - 1) - else: - return Slot(0) -``` - -#### `compute_updated_sample_price` - -```python -def compute_updated_sample_price(prev_price: Gwei, samples_length: uint64, active_shards: uint64) -> Gwei: - adjustment_quotient = active_shards * SLOTS_PER_EPOCH * SAMPLE_PRICE_ADJUSTMENT_COEFFICIENT - if samples_length > TARGET_SAMPLES_PER_BLOB: - delta = max(1, prev_price * (samples_length - TARGET_SAMPLES_PER_BLOB) // TARGET_SAMPLES_PER_BLOB // adjustment_quotient) - return min(prev_price + delta, MAX_SAMPLE_PRICE) - else: - delta = max(1, prev_price * (TARGET_SAMPLES_PER_BLOB - samples_length) // TARGET_SAMPLES_PER_BLOB // adjustment_quotient) - return max(prev_price, MIN_SAMPLE_PRICE + delta) - delta -``` - -#### `compute_committee_source_epoch` - -```python -def compute_committee_source_epoch(epoch: Epoch, period: uint64) -> Epoch: - """ - Return the source epoch for computing the committee. - """ - source_epoch = Epoch(epoch - epoch % period) - if source_epoch >= period: - source_epoch -= period # `period` epochs lookahead - return source_epoch -``` - -#### `batch_apply_participation_flag` - -```python -def batch_apply_participation_flag(state: BeaconState, bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE], - epoch: Epoch, full_committee: Sequence[ValidatorIndex], flag_index: int): - if epoch == get_current_epoch(state): - epoch_participation = state.current_epoch_participation - else: - epoch_participation = state.previous_epoch_participation - for bit, index in zip(bits, full_committee): - if bit: - epoch_participation[index] = add_flag(epoch_participation[index], flag_index) +def is_builder_block_slot(slot: Slot) -> bool: + return slot % 2 == 1 ``` ### Beacon state accessors -#### Updated `get_committee_count_per_slot` - -```python -def get_committee_count_per_slot(state: BeaconState, epoch: Epoch) -> uint64: - """ - Return the number of committees in each slot for the given ``epoch``. - """ - return max(uint64(1), min( - get_active_shard_count(state, epoch), - uint64(len(get_active_validator_indices(state, epoch))) // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE, - )) -``` - #### `get_active_shard_count` ```python @@ -493,396 +222,195 @@ def get_active_shard_count(state: BeaconState, epoch: Epoch) -> uint64: Return the number of active shards. Note that this puts an upper bound on the number of committees per slot. """ - return INITIAL_ACTIVE_SHARDS -``` - -#### `get_shard_proposer_index` - -```python -def get_shard_proposer_index(state: BeaconState, slot: Slot, shard: Shard) -> ValidatorIndex: - """ - Return the proposer's index of shard block at ``slot``. - """ - epoch = compute_epoch_at_slot(slot) - seed = hash(get_seed(state, epoch, DOMAIN_SHARD_BLOB) + uint_to_bytes(slot) + uint_to_bytes(shard)) - indices = get_active_validator_indices(state, epoch) - return compute_proposer_index(state, indices, seed) -``` - -#### `get_start_shard` - -```python -def get_start_shard(state: BeaconState, slot: Slot) -> Shard: - """ - Return the start shard at ``slot``. - """ - epoch = compute_epoch_at_slot(Slot(slot)) - committee_count = get_committee_count_per_slot(state, epoch) - active_shard_count = get_active_shard_count(state, epoch) - return committee_count * slot % active_shard_count -``` - -#### `compute_shard_from_committee_index` - -```python -def compute_shard_from_committee_index(state: BeaconState, slot: Slot, index: CommitteeIndex) -> Shard: - active_shards = get_active_shard_count(state, compute_epoch_at_slot(slot)) - assert index < active_shards - return Shard((index + get_start_shard(state, slot)) % active_shards) -``` - -#### `compute_committee_index_from_shard` - -```python -def compute_committee_index_from_shard(state: BeaconState, slot: Slot, shard: Shard) -> CommitteeIndex: - epoch = compute_epoch_at_slot(slot) - active_shards = get_active_shard_count(state, epoch) - index = CommitteeIndex((active_shards + shard - get_start_shard(state, slot)) % active_shards) - assert index < get_committee_count_per_slot(state, epoch) - return index + return ACTIVE_SHARDS ``` +## Beacon chain state transition function ### Block processing +#### `process_block` + ```python def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) - # is_execution_enabled is omitted, execution is enabled by default. - process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) - process_randao(state, block.body) + verify_builder_block_bid(state, block) + process_sharded_data(state, block) + if is_execution_enabled(state, block.body): + process_execution_payload(state, block, EXECUTION_ENGINE) + + if not is_builder_block_slot(block.slot): + process_randao(state, block.body) + process_eth1_data(state, block.body) - process_operations(state, block.body) # [Modified in Sharding] + process_operations(state, block.body) process_sync_aggregate(state, block.body.sync_aggregate) + + if is_builder_block_slot(block.slot): + state.blocks_since_builder_block = [] + state.blocks_since_builder_block.append(block) ``` -#### Operations +#### Block header ```python -def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: - # Verify that outstanding deposits are processed up to the maximum number of deposits - assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) - - def for_ops(operations: Sequence[Any], fn: Callable[[BeaconState, Any], None]) -> None: - for operation in operations: - fn(state, operation) - - for_ops(body.proposer_slashings, process_proposer_slashing) - for_ops(body.attester_slashings, process_attester_slashing) - # New shard proposer slashing processing - for_ops(body.shard_proposer_slashings, process_shard_proposer_slashing) - - # Limit is dynamic: based on active shard count - assert len(body.shard_headers) <= MAX_SHARD_HEADERS_PER_SHARD * get_active_shard_count(state, get_current_epoch(state)) - for_ops(body.shard_headers, process_shard_header) - - # New attestation processing - for_ops(body.attestations, process_attestation) - for_ops(body.deposits, process_deposit) - for_ops(body.voluntary_exits, process_voluntary_exit) - - # TODO: to avoid parallel shards racing, and avoid inclusion-order problems, - # update the fee price per slot, instead of per header. - # state.shard_sample_price = compute_updated_sample_price(state.shard_sample_price, ?, shard_count) -``` - -##### Extended Attestation processing - -```python -def process_attestation(state: BeaconState, attestation: Attestation) -> None: - altair.process_attestation(state, attestation) - process_attested_shard_work(state, attestation) -``` - -```python -def process_attested_shard_work(state: BeaconState, attestation: Attestation) -> None: - attestation_shard = compute_shard_from_committee_index( - state, - attestation.data.slot, - attestation.data.index, - ) - full_committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index) - - buffer_index = attestation.data.slot % SHARD_STATE_MEMORY_SLOTS - committee_work = state.shard_buffer[buffer_index][attestation_shard] - - # Skip attestation vote accounting if the header is not pending - if committee_work.status.selector != SHARD_WORK_PENDING: - # If the data was already confirmed, check if this matches, to apply the flag to the attesters. - if committee_work.status.selector == SHARD_WORK_CONFIRMED: - attested: AttestedDataCommitment = committee_work.status.value - if attested.root == attestation.data.shard_blob_root: - batch_apply_participation_flag(state, attestation.aggregation_bits, - attestation.data.target.epoch, - full_committee, TIMELY_SHARD_FLAG_INDEX) - return - - current_headers: Sequence[PendingShardHeader] = committee_work.status.value - - # Find the corresponding header, abort if it cannot be found - header_index = len(current_headers) - for i, header in enumerate(current_headers): - if attestation.data.shard_blob_root == header.attested.root: - header_index = i - break - - # Attestations for an unknown header do not count towards shard confirmations, but can otherwise be valid. - if header_index == len(current_headers): - # Note: Attestations may be re-included if headers are included late. - return - - pending_header: PendingShardHeader = current_headers[header_index] - - # The weight may be outdated if it is not the initial weight, and from a previous epoch - if pending_header.weight != 0 and compute_epoch_at_slot(pending_header.update_slot) < get_current_epoch(state): - pending_header.weight = sum(state.validators[index].effective_balance for index, bit - in zip(full_committee, pending_header.votes) if bit) - - pending_header.update_slot = state.slot - - full_committee_balance = Gwei(0) - # Update votes bitfield in the state, update weights - for i, bit in enumerate(attestation.aggregation_bits): - weight = state.validators[full_committee[i]].effective_balance - full_committee_balance += weight - if bit: - if not pending_header.votes[i]: - pending_header.weight += weight - pending_header.votes[i] = True - - # Check if the PendingShardHeader is eligible for expedited confirmation, requiring 2/3 of balance attesting - if pending_header.weight * 3 >= full_committee_balance * 2: - # participants of the winning header are remembered with participation flags - batch_apply_participation_flag(state, pending_header.votes, attestation.data.target.epoch, - full_committee, TIMELY_SHARD_FLAG_INDEX) - - if pending_header.attested.commitment == DataCommitment(): - # The committee voted to not confirm anything - state.shard_buffer[buffer_index][attestation_shard].status.change( - selector=SHARD_WORK_UNCONFIRMED, - value=None, - ) - else: - state.shard_buffer[buffer_index][attestation_shard].status.change( - selector=SHARD_WORK_CONFIRMED, - value=pending_header.attested, - ) -``` - -##### `process_shard_header` - -```python -def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeader) -> None: - header: ShardBlobHeader = signed_header.message - slot = header.slot - shard = header.shard - - # Verify the header is not 0, and not from the future. - assert Slot(0) < slot <= state.slot - header_epoch = compute_epoch_at_slot(slot) - # Verify that the header is within the processing time window - assert header_epoch in [get_previous_epoch(state), get_current_epoch(state)] - # Verify that the shard is valid - shard_count = get_active_shard_count(state, header_epoch) - assert shard < shard_count - # Verify that a committee is able to attest this (slot, shard) - start_shard = get_start_shard(state, slot) - committee_index = (shard_count + shard - start_shard) % shard_count - committees_per_slot = get_committee_count_per_slot(state, header_epoch) - assert committee_index <= committees_per_slot - - # Check that this data is still pending - committee_work = state.shard_buffer[slot % SHARD_STATE_MEMORY_SLOTS][shard] - assert committee_work.status.selector == SHARD_WORK_PENDING - - # Check that this header is not yet in the pending list - current_headers: List[PendingShardHeader, MAX_SHARD_HEADERS_PER_SHARD] = committee_work.status.value - header_root = hash_tree_root(header) - assert header_root not in [pending_header.attested.root for pending_header in current_headers] - - # Verify proposer matches - assert header.proposer_index == get_shard_proposer_index(state, slot, shard) - - # Verify builder and proposer aggregate signature - blob_signing_root = compute_signing_root(header, get_domain(state, DOMAIN_SHARD_BLOB)) - builder_pubkey = state.blob_builders[header.builder_index].pubkey - proposer_pubkey = state.validators[header.proposer_index].pubkey - assert bls.FastAggregateVerify([builder_pubkey, proposer_pubkey], blob_signing_root, signed_header.signature) - - # Verify the length by verifying the degree. - body_summary = header.body_summary - points_count = body_summary.commitment.samples_count * POINTS_PER_SAMPLE - if points_count == 0: - assert body_summary.degree_proof == G1_SETUP[0] - assert ( - bls.Pairing(body_summary.degree_proof, G2_SETUP[0]) - == bls.Pairing(body_summary.commitment.point, G2_SETUP[-points_count]) +def process_block_header(state: BeaconState, block: BeaconBlock) -> None: + # Verify that the slots match + assert block.slot == state.slot + # Verify that the block is newer than latest block header + assert block.slot > state.latest_block_header.slot + # Verify that proposer index is the correct index + if not is_builder_block_slot(block.slot): + assert block.proposer_index == get_beacon_proposer_index(state) + # Verify that the parent matches + assert block.parent_root == hash_tree_root(state.latest_block_header) + # Cache current block as the new latest block + state.latest_block_header = BeaconBlockHeader( + slot=block.slot, + proposer_index=block.proposer_index, + parent_root=block.parent_root, + state_root=Bytes32(), # Overwritten in the next process_slot call + body_root=hash_tree_root(block.body), ) - # Charge EIP 1559 fee, builder pays for opportunity, and is responsible for later availability, - # or fail to publish at their own expense. - samples = body_summary.commitment.samples_count - # TODO: overflows, need bigger int type - max_fee = body_summary.max_fee_per_sample * samples - - # Builder must have sufficient balance, even if max_fee is not completely utilized - assert state.blob_builder_balances[header.builder_index] >= max_fee - - base_fee = state.shard_sample_price * samples - # Base fee must be paid - assert max_fee >= base_fee - - # Remaining fee goes towards proposer for prioritizing, up to a maximum - max_priority_fee = body_summary.max_priority_fee_per_sample * samples - priority_fee = min(max_fee - base_fee, max_priority_fee) - - # Burn base fee, take priority fee - # priority_fee <= max_fee - base_fee, thus priority_fee + base_fee <= max_fee, thus sufficient balance. - state.blob_builder_balances[header.builder_index] -= base_fee + priority_fee - # Pay out priority fee - increase_balance(state, header.proposer_index, priority_fee) - - # Initialize the pending header - index = compute_committee_index_from_shard(state, slot, shard) - committee_length = len(get_beacon_committee(state, slot, index)) - initial_votes = Bitlist[MAX_VALIDATORS_PER_COMMITTEE]([0] * committee_length) - pending_header = PendingShardHeader( - attested=AttestedDataCommitment( - commitment=body_summary.commitment, - root=header_root, - includer_index=get_beacon_proposer_index(state), - ), - votes=initial_votes, - weight=0, - update_slot=state.slot, - ) - - # Include it in the pending list - current_headers.append(pending_header) + # Verify proposer is not slashed + proposer = state.validators[block.proposer_index] + assert not proposer.slashed ``` -The degree proof works as follows. For a block `B` with length `l` (so `l` values in `[0...l - 1]`, seen as a polynomial `B(X)` which takes these values), -the length proof is the commitment to the polynomial `B(X) * X**(MAX_DEGREE + 1 - l)`, -where `MAX_DEGREE` is the maximum power of `s` available in the setup, which is `MAX_DEGREE = len(G2_SETUP) - 1`. -The goal is to ensure that a proof can only be constructed if `deg(B) < l` (there are not hidden higher-order terms in the polynomial, which would thwart reconstruction). - -##### `process_shard_proposer_slashing` +#### Builder Block Bid ```python -def process_shard_proposer_slashing(state: BeaconState, proposer_slashing: ShardProposerSlashing) -> None: - slot = proposer_slashing.slot - shard = proposer_slashing.shard - proposer_index = proposer_slashing.proposer_index +def verify_builder_block_bid(state: BeaconState, block: BeaconBlock) -> None: + if is_builder_block_slot(block.slot): + # Get last builder block bid + assert state.blocks_since_builder_block[-1].body.payload_data.selector == 0 + builder_block_bid = state.blocks_since_builder_block[-1].body.payload_data.value.builder_block_bid + assert builder_block_bid.slot + 1 == block.slot - reference_1 = ShardBlobReference(slot=slot, shard=shard, - proposer_index=proposer_index, - builder_index=proposer_slashing.builder_index_1, - body_root=proposer_slashing.body_root_1) - reference_2 = ShardBlobReference(slot=slot, shard=shard, - proposer_index=proposer_index, - builder_index=proposer_slashing.builder_index_2, - body_root=proposer_slashing.body_root_2) + assert block.body.payload_data.selector == 1 # Verify that builder block does not contain bid - # Verify the signed messages are different - assert reference_1 != reference_2 + builder_block_data = block.body.payload_data.value - # Verify the proposer is slashable - proposer = state.validators[proposer_index] - assert is_slashable_validator(proposer, get_current_epoch(state)) + assert builder_block_bid.execution_payload_root == hash_tree_root(builder_block_data.execution_payload) - # The builders are not slashed, the proposer co-signed with them - builder_pubkey_1 = state.blob_builders[proposer_slashing.builder_index_1].pubkey - builder_pubkey_2 = state.blob_builders[proposer_slashing.builder_index_2].pubkey - domain = get_domain(state, DOMAIN_SHARD_PROPOSER, compute_epoch_at_slot(slot)) - signing_root_1 = compute_signing_root(reference_1, domain) - signing_root_2 = compute_signing_root(reference_2, domain) - assert bls.FastAggregateVerify([builder_pubkey_1, proposer.pubkey], signing_root_1, proposer_slashing.signature_1) - assert bls.FastAggregateVerify([builder_pubkey_2, proposer.pubkey], signing_root_2, proposer_slashing.signature_2) + assert builder_block_bid.sharded_data_commitment_count == builder_block_data.included_sharded_data_commitments - slash_validator(state, proposer_index) + assert builder_block_bid.sharded_data_commitment_root == hash_tree_root(builder_block_data.sharded_commitments[-builder_block_bid.included_sharded_data_commitments:]) + + assert builder_block_bid.validator_index == block.proposer_index + + else: + assert block.body.payload_data.selector == 0 + + builder_block_bid = block.body.payload_data.value.builder_block_bid + assert builder_block_bid.slot == block.slot + assert builder_block_bid.parent_block_root == block.parent_root + # We do not check that the builder address exists or has sufficient balance here. + # If it does not have sufficient balance, the block proposer loses out, so it is their + # responsibility to check. + + # Check that the builder is a slashable validator. We can probably reduce this requirement and only + # ensure that they have 1 ETH in their account as a DOS protection. + builder = state.validators[builder_block_bid.validator_index] + assert is_slashable_validator(builder, get_current_epoch(state)) ``` -### Epoch transition - -This epoch transition overrides Bellatrix epoch transition: +#### Sharded data ```python -def process_epoch(state: BeaconState) -> None: - # Sharding pre-processing - process_pending_shard_confirmations(state) - reset_pending_shard_work(state) +def process_sharded_data(state: BeaconState, block: BeaconBlock) -> None: + if is_builder_block_slot(block.slot): + assert block.body.payload_data.selector == 1 + sharded_commitments_container = block.body.payload_data.value.sharded_commitments_container - # Base functionality - process_justification_and_finalization(state) - process_inactivity_updates(state) - process_rewards_and_penalties(state) # Note: modified, see new TIMELY_SHARD_FLAG_INDEX - process_registry_updates(state) - process_slashings(state) - process_eth1_data_reset(state) - process_effective_balance_updates(state) - process_slashings_reset(state) - process_randao_mixes_reset(state) - process_historical_roots_update(state) - process_participation_flag_updates(state) - process_sync_committee_updates(state) + # Verify not too many commitments + assert len(sharded_commitments_container.sharded_commitments) // 2 <= get_active_shard_count(state, get_current_epoch(state)) + + # Verify the degree proof + r = hash_to_bls_field(sharded_commitments_container.sharded_commitments, 0) + r_powers = compute_powers(r, len(sharded_commitments_container.sharded_commitments)) + combined_commitment = elliptic_curve_lincomb(sharded_commitments_container.sharded_commitments, r_powers) + + payload_field_elements_per_blob = SAMPLES_PER_BLOB * FIELD_ELEMENTS_PER_SAMPLE // 2 + + verify_degree_proof(combined_commitment, payload_field_elements_per_blob, sharded_commitments_container.degree_proof) + + # Verify that the 2*N commitments lie on a degree < N polynomial + low_degree_check(sharded_commitments_container.sharded_commitments) + + # Verify that blocks since the last builder block have been included + blocks_chunked = [bytes_to_field_elements(ssz_serialize(block)) for block in state.blocks_since_builder_block] + block_vectors = [] + + for block_chunked in blocks_chunked: + for i in range(0, len(block_chunked), payload_field_elements_per_blob): + block_vectors.append(block_chunked[i:i + payload_field_elements_per_blob]) + + number_of_blobs = len(block_vectors) + r = hash_to_bls_field(sharded_commitments_container.sharded_commitments[:number_of_blobs], 0) + x = hash_to_bls_field(sharded_commitments_container.sharded_commitments[:number_of_blobs], 1) + + r_powers = compute_powers(r, number_of_blobs) + combined_vector = vector_lincomb(block_vectors, r_powers) + combined_commitment = elliptic_curve_lincomb(sharded_commitments_container.sharded_commitments[:number_of_blobs], r_powers) + y = evaluate_polynomial_in_evaluation_form(combined_vector, x) + + verify_kzg_proof(combined_commitment, x, y, sharded_commitments_container.block_verification_kzg_proof) + + # Verify that number of sharded data commitments is correctly indicated + assert 2 * (number_of_blobs + included_sharded_data_commitments) == len(sharded_commitments_container.sharded_commitments) ``` -#### `process_pending_shard_confirmations` +#### Execution payload ```python -def process_pending_shard_confirmations(state: BeaconState) -> None: - # Pending header processing applies to the previous epoch. - # Skip if `GENESIS_EPOCH` because no prior epoch to process. - if get_current_epoch(state) == GENESIS_EPOCH: - return +def process_execution_payload(state: BeaconState, block: BeaconBlock, execution_engine: ExecutionEngine) -> None: + if is_builder_block_slot(block.slot): + assert block.body.payload_data.selector == 1 + payload = block.body.payload_data.value.execution_payload - previous_epoch = get_previous_epoch(state) - previous_epoch_start_slot = compute_start_slot_at_epoch(previous_epoch) + # Verify consistency of the parent hash with respect to the previous execution payload header + if is_merge_transition_complete(state): + assert payload.parent_hash == state.latest_execution_payload_header.block_hash + # Verify random + assert payload.random == get_randao_mix(state, get_current_epoch(state)) + # Verify timestamp + assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) - # Mark stale headers as unconfirmed - for slot in range(previous_epoch_start_slot, previous_epoch_start_slot + SLOTS_PER_EPOCH): - buffer_index = slot % SHARD_STATE_MEMORY_SLOTS - for shard_index in range(len(state.shard_buffer[buffer_index])): - committee_work = state.shard_buffer[buffer_index][shard_index] - if committee_work.status.selector == SHARD_WORK_PENDING: - winning_header = max(committee_work.status.value, key=lambda header: header.weight) - if winning_header.attested.commitment == DataCommitment(): - committee_work.status.change(selector=SHARD_WORK_UNCONFIRMED, value=None) - else: - committee_work.status.change(selector=SHARD_WORK_CONFIRMED, value=winning_header.attested) -``` + # Get sharded data commitments + sharded_commitments_container = block.body.sharded_commitments_container + sharded_data_commitments = sharded_commitments_container.sharded_commitments[-sharded_commitments_container.included_sharded_data_commitments:] -#### `reset_pending_shard_work` + # Get all unprocessed builder block bids + unprocessed_builder_block_bid_with_recipient_addresses = [] + for block in state.blocks_since_builder_block[1:]: + unprocessed_builder_block_bid_with_recipient_addresses.append(block.body.builder_block_bid_with_recipient_address.value) -```python -def reset_pending_shard_work(state: BeaconState) -> None: - # Add dummy "empty" PendingShardHeader (default vote if no shard header is available) - next_epoch = get_current_epoch(state) + 1 - next_epoch_start_slot = compute_start_slot_at_epoch(next_epoch) - committees_per_slot = get_committee_count_per_slot(state, next_epoch) - active_shards = get_active_shard_count(state, next_epoch) + # Verify the execution payload is valid + # The execution engine gets two extra payloads: One for the sharded data commitments (these are needed to verify type 3 transactions) + # and one for all so far unprocessed builder block bids: + # * The execution engine needs to transfer the balance from the bidder to the proposer. + # * The execution engine needs to deduct data gas fees from the bidder balances + assert execution_engine.execute_payload(payload, + sharded_data_commitments, + unprocessed_builder_block_bid_with_recipient_addresses) - for slot in range(next_epoch_start_slot, next_epoch_start_slot + SLOTS_PER_EPOCH): - buffer_index = slot % SHARD_STATE_MEMORY_SLOTS - - # Reset the shard work tracking - state.shard_buffer[buffer_index] = [ShardWork() for _ in range(active_shards)] - - start_shard = get_start_shard(state, slot) - for committee_index in range(committees_per_slot): - shard = (start_shard + committee_index) % active_shards - # a committee is available, initialize a pending shard-header list - committee_length = len(get_beacon_committee(state, slot, CommitteeIndex(committee_index))) - state.shard_buffer[buffer_index][shard].status.change( - selector=SHARD_WORK_PENDING, - value=List[PendingShardHeader, MAX_SHARD_HEADERS_PER_SHARD]( - PendingShardHeader( - attested=AttestedDataCommitment(), - votes=Bitlist[MAX_VALIDATORS_PER_COMMITTEE]([0] * committee_length), - weight=0, - update_slot=slot, - ) - ) - ) - # a shard without committee available defaults to SHARD_WORK_UNCONFIRMED. -``` + # Cache execution payload header + state.latest_execution_payload_header = ExecutionPayloadHeader( + parent_hash=payload.parent_hash, + fee_recipient=payload.fee_recipient, + state_root=payload.state_root, + receipt_root=payload.receipt_root, + logs_bloom=payload.logs_bloom, + random=payload.random, + block_number=payload.block_number, + gas_limit=payload.gas_limit, + gas_used=payload.gas_used, + timestamp=payload.timestamp, + extra_data=payload.extra_data, + base_fee_per_gas=payload.base_fee_per_gas, + block_hash=payload.block_hash, + transactions_root=hash_tree_root(payload.transactions), + ) +``` \ No newline at end of file diff --git a/specs/sharding/p2p-interface.md b/specs/sharding/p2p-interface.md index ab32c37fa..3b627a339 100644 --- a/specs/sharding/p2p-interface.md +++ b/specs/sharding/p2p-interface.md @@ -13,17 +13,14 @@ - [Misc](#misc) - [Gossip domain](#gossip-domain) - [Topics and messages](#topics-and-messages) - - [Shard blob subnets](#shard-blob-subnets) - - [`shard_blob_{subnet_id}`](#shard_blob_subnet_id) - - [Global topics](#global-topics) - - [`shard_blob_header`](#shard_blob_header) - - [`shard_blob_tx`](#shard_blob_tx) - - [`shard_proposer_slashing`](#shard_proposer_slashing) + - [Builder block bid](#builder-block-bid) + - [`builder_block_bid`](#builder_block_bid) + - [Shard sample subnets](#shard-sample-subnets) + - [`shard_row_{subnet_id}`](#shard_row_subnet_id) - ## Introduction The specification of these changes continues in the same format as the network specifications of previous upgrades, and assumes them as pre-requisite. @@ -33,12 +30,10 @@ The adjustments and additions for Shards are outlined in this document. ### Misc -| Name | Value | Description | -| ---- | ----- | ----------- | -| `SHARD_BLOB_SUBNET_COUNT` | `64` | The number of `shard_blob_{subnet_id}` subnets used in the gossipsub protocol. | -| `SHARD_TX_PROPAGATION_GRACE_SLOTS` | `4` | The number of slots for a late transaction to propagate | -| `SHARD_TX_PROPAGATION_BUFFER_SLOTS` | `8` | The number of slots for an early transaction to propagate | - +| Name | Value | Description | +| --------------------------- | ----- | -------------------------------------------------------------------------------- | +| `SHARD_ROW_SUBNET_COUNT` | `512` | The number of `shard_row_{subnet_id}` subnets used in the gossipsub protocol. | +| `SHARD_COLUMN_SUBNET_COUNT` | `512` | The number of `shard_column_{subnet_id}` subnets used in the gossipsub protocol. | ## Gossip domain @@ -48,130 +43,49 @@ Following the same scheme as the [Phase0 gossip topics](../phase0/p2p-interface. | Name | Message Type | |---------------------------------|--------------------------| -| `shard_blob_{subnet_id}` | `SignedShardBlob` | -| `shard_blob_header` | `SignedShardBlobHeader` | -| `shard_blob_tx` | `SignedShardBlobHeader` | -| `shard_proposer_slashing` | `ShardProposerSlashing` | +| `shard_row_{subnet_id}` | `SignedShardSample` | +| `shard_column_{subnet_id}` | `SignedShardSample` | +| `builder_block_bid` | `BuilderBlockBid` | The [DAS network specification](./das-p2p.md) defines additional topics. -#### Shard blob subnets +#### Builder block bid -Shard blob subnets are used by builders to make their blobs available after selection by shard proposers. +##### `builder_block_bid` -##### `shard_blob_{subnet_id}` +- _[IGNORE]_ The `bid` is published 1 slot early or later (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- + i.e. validate that `bid.slot <= current_slot + 1` + (a client MAY queue future samples for propagation at the appropriate slot). +- _[IGNORE]_ The `bid` is for the current or next block + i.e. validate that `bid.slot >= current_slot` +- _[IGNORE]_ The `bid` is the first `bid` valid bid for `bid.slot`, or the bid is at least 1% higher than the previous known `bid` +- _[REJECT]_ The validator defined by `bid.validator_index` exists and is slashable. +- _[REJECT]_ The bid signature, which is an Eth1 signature, needs to be valid and the address needs to contain enough Ether to cover the bid and the data gas base fee. -Shard blob data, in the form of a `SignedShardBlob` is published to the `shard_blob_{subnet_id}` subnets. +#### Shard sample subnets -```python -def compute_subnet_for_shard_blob(state: BeaconState, slot: Slot, shard: Shard) -> uint64: - """ - Compute the correct subnet for a shard blob publication. - Note, this mimics compute_subnet_for_attestation(). - """ - committee_index = compute_committee_index_from_shard(state, slot, shard) - committees_per_slot = get_committee_count_per_slot(state, compute_epoch_at_slot(slot)) - slots_since_epoch_start = Slot(slot % SLOTS_PER_EPOCH) - committees_since_epoch_start = committees_per_slot * slots_since_epoch_start +Shard sample (row/column) subnets are used by builders to make their samples available as part of their intermediate block release after selection by beacon block proposers. - return uint64((committees_since_epoch_start + committee_index) % SHARD_BLOB_SUBNET_COUNT) -``` +##### `shard_row_{subnet_id}` -The following validations MUST pass before forwarding the `signed_blob`, -on the horizontal subnet or creating samples for it. Alias `blob = signed_blob.message`. +Shard sample data, in the form of a `SignedShardSample` is published to the `shard_row_{subnet_id}` and `shard_column_{subnet_id}` subnets. -- _[IGNORE]_ The `blob` is published 1 slot early or later (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- - i.e. validate that `blob.slot <= current_slot + 1` - (a client MAY queue future blobs for propagation at the appropriate slot). -- _[IGNORE]_ The `blob` is new enough to still be processed -- - i.e. validate that `compute_epoch_at_slot(blob.slot) >= get_previous_epoch(state)` -- _[REJECT]_ The shard blob is for an active shard -- - i.e. `blob.shard < get_active_shard_count(state, compute_epoch_at_slot(blob.slot))` -- _[REJECT]_ The `blob.shard` MUST have a committee at the `blob.slot` -- - i.e. validate that `compute_committee_index_from_shard(state, blob.slot, blob.shard)` doesn't raise an error -- _[REJECT]_ The shard blob is for the correct subnet -- - i.e. `compute_subnet_for_shard_blob(state, blob.slot, blob.shard) == subnet_id` -- _[IGNORE]_ The blob is the first blob with valid signature received for the `(blob.proposer_index, blob.slot, blob.shard)` combination. -- _[REJECT]_ The blob is not too large -- the data MUST NOT be larger than the SSZ list-limit, and a client MAY apply stricter bounds. -- _[REJECT]_ The `blob.body.data` MUST NOT contain any point `p >= MODULUS`. Although it is a `uint256`, not the full 256 bit range is valid. -- _[REJECT]_ The blob builder defined by `blob.builder_index` exists and has sufficient balance to back the fee payment. -- _[REJECT]_ The blob signature, `signed_blob.signature`, is valid for the aggregate of proposer and builder -- - i.e. `bls.FastAggregateVerify([builder_pubkey, proposer_pubkey], blob_signing_root, signed_blob.signature)`. -- _[REJECT]_ The blob is proposed by the expected `proposer_index` for the blob's `slot` and `shard`, - in the context of the current shuffling (defined by the current node head state and `blob.slot`). - If the `proposer_index` cannot immediately be verified against the expected shuffling, - the blob MAY be queued for later processing while proposers for the blob's branch are calculated -- - in such a case _do not_ `REJECT`, instead `IGNORE` this message. +The following validations MUST pass before forwarding the `sample`. -#### Global topics +- _[IGNORE]_ The `sample` is published 1 slot early or later (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- + i.e. validate that `sample.slot <= current_slot + 1` + (a client MAY queue future samples for propagation at the appropriate slot). +- _[IGNORE]_ The `sample` is new enough to still be processed -- + i.e. validate that `compute_epoch_at_slot(sample.slot) >= get_previous_epoch(state)` +- _[REJECT]_ The shard sample is for the correct subnet -- + i.e. `sample.row == subnet_id` for `shard_row_{subnet_id}` and `sample.column == subnet_id` for `shard_column_{subnet_id}` +- _[IGNORE]_ The sample is the first sample with valid signature received for the `(sample.builder, sample.slot, sample.row, sample.column)` combination. +- _[REJECT]_ The `sample.data` MUST NOT contain any point `x >= BLS_MODULUS`. Although it is a `uint256`, not the full 256 bit range is valid. +- _[REJECT]_ The validator defined by `sample.builder` exists and is slashable. +- _[REJECT]_ The sample is proposed by the expected `builder` for the sample's `slot`. + i.e., the beacon block at `sample.slot - 1` according to the node's fork choice contains an `IntermediateBlockBid` + with `intermediate_block_bid.validator_index == sample.builder` +- _[REJECT]_ The sample signature, `sample.signature`, is valid for the builder -- + i.e. `bls.Verify(builder_pubkey, sample_signing_root, sample.signature)` OR `sample.signature == Bytes96(b"\0" * 96)` AND + the sample verification `verify_sample` passes -There are three additional global topics for Sharding. - -- `shard_blob_header`: co-signed headers to be included on-chain and to serve as a signal to the builder to publish full data. -- `shard_blob_tx`: builder-signed headers, also known as "data transaction". -- `shard_proposer_slashing`: slashings of duplicate shard proposals. - -##### `shard_blob_header` - -Shard header data, in the form of a `SignedShardBlobHeader` is published to the global `shard_blob_header` subnet. -Shard blob headers select shard blob bids by builders -and should be timely to ensure builders can publish the full shard blob before subsequent attestations. - -The following validations MUST pass before forwarding the `signed_blob_header` on the network. Alias `header = signed_blob_header.message`. - -- _[IGNORE]_ The `header` is published 1 slot early or later (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- - i.e. validate that `header.slot <= current_slot + 1` - (a client MAY queue future headers for propagation at the appropriate slot). -- _[IGNORE]_ The header is new enough to still be processed -- - i.e. validate that `compute_epoch_at_slot(header.slot) >= get_previous_epoch(state)` -- _[REJECT]_ The shard header is for an active shard -- - i.e. `header.shard < get_active_shard_count(state, compute_epoch_at_slot(header.slot))` -- _[REJECT]_ The `header.shard` MUST have a committee at the `header.slot` -- - i.e. validate that `compute_committee_index_from_shard(state, header.slot, header.shard)` doesn't raise an error. -- _[IGNORE]_ The header is the first header with valid signature received for the `(header.proposer_index, header.slot, header.shard)` combination. -- _[REJECT]_ The blob builder defined by `blob.builder_index` exists and has sufficient balance to back the fee payment. -- _[REJECT]_ The header signature, `signed_blob_header.signature`, is valid for the aggregate of proposer and builder -- - i.e. `bls.FastAggregateVerify([builder_pubkey, proposer_pubkey], blob_signing_root, signed_blob_header.signature)`. -- _[REJECT]_ The header is proposed by the expected `proposer_index` for the blob's `header.slot` and `header.shard` - in the context of the current shuffling (defined by the current node head state and `header.slot`). - If the `proposer_index` cannot immediately be verified against the expected shuffling, - the blob MAY be queued for later processing while proposers for the blob's branch are calculated -- - in such a case _do not_ `REJECT`, instead `IGNORE` this message. - -##### `shard_blob_tx` - -Shard data-transactions in the form of a `SignedShardBlobHeader` are published to the global `shard_blob_tx` subnet. -These shard blob headers are signed solely by the blob-builder. - -The following validations MUST pass before forwarding the `signed_blob_header` on the network. Alias `header = signed_blob_header.message`. - -- _[IGNORE]_ The header is not propagating more than `SHARD_TX_PROPAGATION_BUFFER_SLOTS` slots ahead of time -- - i.e. validate that `header.slot <= current_slot + SHARD_TX_PROPAGATION_BUFFER_SLOTS`. -- _[IGNORE]_ The header is not propagating later than `SHARD_TX_PROPAGATION_GRACE_SLOTS` slots too late -- - i.e. validate that `header.slot + SHARD_TX_PROPAGATION_GRACE_SLOTS >= current_slot` -- _[REJECT]_ The shard header is for an active shard -- - i.e. `header.shard < get_active_shard_count(state, compute_epoch_at_slot(header.slot))` -- _[REJECT]_ The `header.shard` MUST have a committee at the `header.slot` -- - i.e. validate that `compute_committee_index_from_shard(state, header.slot, header.shard)` doesn't raise an error. -- _[IGNORE]_ The header is not stale -- i.e. the corresponding shard proposer has not already selected a header for `(header.slot, header.shard)`. -- _[IGNORE]_ The header is the first header with valid signature received for the `(header.builder_index, header.slot, header.shard)` combination. -- _[REJECT]_ The blob builder, define by `header.builder_index`, exists and has sufficient balance to back the fee payment. -- _[IGNORE]_ The header fee SHOULD be higher than previously seen headers for `(header.slot, header.shard)`, from any builder. - Propagating nodes MAY increase fee increments in case of spam. -- _[REJECT]_ The header signature, `signed_blob_header.signature`, is valid for ONLY the builder -- - i.e. `bls.Verify(builder_pubkey, blob_signing_root, signed_blob_header.signature)`. The signature is not an aggregate with the proposer. -- _[REJECT]_ The header is designated for proposal by the expected `proposer_index` for the blob's `header.slot` and `header.shard` - in the context of the current shuffling (defined by the current node head state and `header.slot`). - If the `proposer_index` cannot immediately be verified against the expected shuffling, - the blob MAY be queued for later processing while proposers for the blob's branch are calculated -- - in such a case _do not_ `REJECT`, instead `IGNORE` this message. - -##### `shard_proposer_slashing` - -Shard proposer slashings, in the form of `ShardProposerSlashing`, are published to the global `shard_proposer_slashing` topic. - -The following validations MUST pass before forwarding the `shard_proposer_slashing` on to the network. -- _[IGNORE]_ The shard proposer slashing is the first valid shard proposer slashing received - for the proposer with index `proposer_slashing.proposer_index`. - The `proposer_slashing.slot` and `proposer_slashing.shard` are ignored, there are no repeated or per-shard slashings. -- _[REJECT]_ All of the conditions within `process_shard_proposer_slashing` pass validation. diff --git a/specs/sharding/polynomial-commitments.md b/specs/sharding/polynomial-commitments.md new file mode 100644 index 000000000..e2a4285ca --- /dev/null +++ b/specs/sharding/polynomial-commitments.md @@ -0,0 +1,396 @@ +# Sharding -- Polynomial Commitments + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + + +- [Introduction](#introduction) +- [Constants](#constants) + - [BLS Field](#bls-field) + - [KZG Trusted setup](#kzg-trusted-setup) +- [Custom types](#custom-types) +- [Helper functions](#helper-functions) + - [`next_power_of_two`](#next_power_of_two) + - [`reverse_bit_order`](#reverse_bit_order) + - [`list_to_reverse_bit_order`](#list_to_reverse_bit_order) +- [Field operations](#field-operations) + - [Generic field operations](#generic-field-operations) + - [`bls_modular_inverse`](#bls_modular_inverse) + - [`roots_of_unity`](#roots_of_unity) + - [Field helper functions](#field-helper-functions) + - [`compute_powers`](#compute_powers) + - [`low_degree_check`](#low_degree_check) + - [`vector_lincomb`](#vector_lincomb) + - [`bytes_to_field_elements`](#bytes_to_field_elements) +- [Polynomial operations](#polynomial-operations) + - [`add_polynomials`](#add_polynomials) + - [`multiply_polynomials`](#multiply_polynomials) + - [`interpolate_polynomial`](#interpolate_polynomial) + - [`evaluate_polynomial_in_evaluation_form`](#evaluate_polynomial_in_evaluation_form) +- [KZG Operations](#kzg-operations) + - [Elliptic curve helper functoins](#elliptic-curve-helper-functoins) + - [`elliptic_curve_lincomb`](#elliptic_curve_lincomb) + - [Hash to field](#hash-to-field) + - [`hash_to_bls_field`](#hash_to_bls_field) + - [KZG operations](#kzg-operations) + - [`verify_kzg_proof`](#verify_kzg_proof) + - [`verify_kzg_multiproof`](#verify_kzg_multiproof) + - [`verify_degree_proof`](#verify_degree_proof) + + + + + +## Introduction + +This document specifies basic polynomial operations and KZG polynomial commitment operations as they are needed for the sharding specification. The implementations are not optimized for performance, but readability. All practical implementations should optimize the polynomial operations, and hints what the best known algorithms for these implementations are are included below. + +## Constants + +### BLS Field + +| Name | Value | Notes | +| - | - | - | +| `BLS_MODULUS` | `0x73eda753299d7d483339d80809a1d80553bda402fffe5bfeffffffff00000001` (curve order of BLS12_381) | +| `PRIMITIVE_ROOT_OF_UNITY` | `7` | Primitive root of unity of the BLS12_381 (inner) BLS_MODULUS | + +### KZG Trusted setup + +| Name | Value | +| - | - | +| `G1_SETUP` | Type `List[G1]`. The G1-side trusted setup `[G, G*s, G*s**2....]`; note that the first point is the generator. | +| `G2_SETUP` | Type `List[G2]`. The G2-side trusted setup `[G, G*s, G*s**2....]` | + +## Custom types + +We define the following Python custom types for type hinting and readability: + +| Name | SSZ equivalent | Description | +| - | - | - | +| `KZGCommitment` | `Bytes48` | A G1 curve point | +| `BLSFieldElement` | `uint256` | A number `x` in the range `0 <= x < BLS_MODULUS` | +| `BLSPolynomialByCoefficients` | `List[BLSFieldElement]` | A polynomial over the BLS field, given in coefficient form | +| `BLSPolynomialByEvaluations` | `List[BLSFieldElement]` | A polynomial over the BLS field, given in evaluation form | + +## Helper functions + +#### `next_power_of_two` + +```python +def next_power_of_two(x: int) -> int: + assert x > 0 + return 2 ** ((x - 1).bit_length()) +``` + +#### `reverse_bit_order` + +```python +def reverse_bit_order(n: int, order: int) -> int: + """ + Reverse the bit order of an integer n + """ + assert is_power_of_two(order) + # Convert n to binary with the same number of bits as "order" - 1, then reverse its bit order + return int(('{:0' + str(order.bit_length() - 1) + 'b}').format(n)[::-1], 2) +``` + +#### `list_to_reverse_bit_order` + +```python +def list_to_reverse_bit_order(l: List[int]) -> List[int]: + """ + Convert a list between normal and reverse bit order. This operation is idempotent. + """ + return [l[reverse_bit_order(i, len(l))] for i in range(len(l))] +``` + +## Field operations + +### Generic field operations + +#### `bls_modular_inverse` + +```python +def bls_modular_inverse(x: BLSFieldElement) -> BLSFieldElement: + """ + Compute the modular inverse of x, i.e. y such that x * y % BLS_MODULUS == 1 and return 1 for x == 0 + """ + lm, hm = 1, 0 + low, high = x % BLS_MODULUS, BLS_MODULUS + while low > 1: + r = high // low + nm, new = hm - lm * r, high - low * r + lm, low, hm, high = nm, new, lm, low + return lm % BLS_MODULUS +``` + +#### `roots_of_unity` + +```python +def roots_of_unity(order: uint64) -> List[BLSFieldElement]: + """ + Compute a list of roots of unity for a given order. + The order must divide the BLS multiplicative group order, i.e. BLS_MODULUS - 1 + """ + assert (BLS_MODULUS - 1) % order == 0 + roots = [] + root_of_unity = pow(PRIMITIVE_ROOT_OF_UNITY, (BLS_MODULUS - 1) // order, BLS_MODULUS) + + current_root_of_unity = 1 + for i in range(SAMPLES_PER_BLOB * FIELD_ELEMENTS_PER_SAMPLE): + roots.append(current_root_of_unity) + current_root_of_unity = current_root_of_unity * root_of_unity % BLS_MODULUS + return roots +``` + +### Field helper functions + +#### `compute_powers` + +```python +def compute_powers(x: BLSFieldElement, n: uint64) -> List[BLSFieldElement]: + current_power = 1 + powers = [] + for _ in range(n): + powers.append(BLSFieldElement(current_power)) + current_power = current_power * int(x) % BLS_MODULUS + return powers +``` + +#### `low_degree_check` + +```python +def low_degree_check(commitments: List[KZGCommitment]): + """ + Checks that the commitments are on a low-degree polynomial. + If there are 2*N commitments, that means they should lie on a polynomial + of degree d = K - N - 1, where K = next_power_of_two(2*N) + (The remaining positions are filled with 0, this is to make FFTs usable) + + For details see here: https://notes.ethereum.org/@dankrad/barycentric_low_degree_check + """ + assert len(commitments) % 2 == 0 + N = len(commitments) // 2 + r = hash_to_bls_field(commitments, 0) + K = next_power_of_two(2 * N) + d = K - N - 1 + r_to_K = pow(r, N, K) + roots = list_to_reverse_bit_order(roots_of_unity(K)) + + # For an efficient implementation, B and Bprime should be precomputed + def B(z): + r = 1 + for w in roots[:d + 1]: + r = r * (z - w) % BLS_MODULUS + return r + + def Bprime(z): + r = 0 + for i in range(d + 1): + m = 1 + for w in roots[:i] + roots[i + 1:d + 1]: + m = m * (z - w) % BLS_MODULUS + r = (r + m) % BLS_MODULUS + return r + + coefs = [] + for i in range(K): + coefs.append( - (r_to_K - 1) * bls_modular_inverse(K * roots[i * (K - 1) % K] * (r - roots[i])) % BLS_MODULUS) + for i in range(d + 1): + coefs[i] = (coefs[i] + B(r) * bls_modular_inverse(Bprime(r) * (r - roots[i]))) % BLS_MODULUS + + assert elliptic_curve_lincomb(commitments, coefs) == bls.inf_G1() +``` + +#### `vector_lincomb` + +```python +def vector_lincomb(vectors: List[List[BLSFieldElement]], scalars: List[BLSFieldElement]) -> List[BLSFieldElement]: + """ + Compute a linear combination of field element vectors. + """ + r = [0]*len(vectors[0]) + for v, a in zip(vectors, scalars): + for i, x in enumerate(v): + r[i] = (r[i] + a * x) % BLS_MODULUS + return [BLSFieldElement(x) for x in r] +``` + +#### `bytes_to_field_elements` + +```python +def bytes_to_field_elements(block: bytes) -> List[BLSFieldElement]: + """ + Slices a block into 31-byte chunks that can fit into field elements. + """ + sliced_block = [block[i:i + 31] for i in range(0, len(bytes), 31)] + return [BLSFieldElement(int.from_bytes(x, "little")) for x in sliced_block] +``` + +## Polynomial operations + +#### `add_polynomials` + +```python +def add_polynomials(a: BLSPolynomialByCoefficients, b: BLSPolynomialByCoefficients) -> BLSPolynomialByCoefficients: + """ + Sum the polynomials ``a`` and ``b`` given by their coefficients. + """ + a, b = (a, b) if len(a) >= len(b) else (b, a) + return [(a[i] + (b[i] if i < len(b) else 0)) % BLS_MODULUS for i in range(len(a))] +``` + +#### `multiply_polynomials` + +```python +def multiply_polynomials(a: BLSPolynomialByCoefficients, b: BLSPolynomialByCoefficients) -> BLSPolynomialByCoefficients: + """ + Multiplies the polynomials `a` and `b` given by their coefficients + """ + r = [0] + for power, coef in enumerate(a): + summand = [0] * power + [coef * x % BLS_MODULUS for x in b] + r = add_polynomials(r, summand) + return r +``` + + +#### `interpolate_polynomial` + +```python +def interpolate_polynomial(xs: List[BLSFieldElement], ys: List[BLSFieldElement]) -> BLSPolynomialByCoefficients: + """ + Lagrange interpolation + """ + assert len(xs) == len(ys) + r = [0] + + for i in range(len(xs)): + summand = [ys[i]] + for j in range(len(ys)): + if j != i: + weight_adjustment = bls_modular_inverse(xs[j] - xs[i]) + summand = multiply_polynomials( + summand, [weight_adjustment, ((BLS_MODULUS - weight_adjustment) * xs[i])] + ) + r = add_polynomials(r, summand) + + return r +``` + +#### `evaluate_polynomial_in_evaluation_form` + +```python +def evaluate_polynomial_in_evaluation_form(poly: BLSPolynomialByEvaluations, x: BLSFieldElement) -> BLSFieldElement: + """ + Evaluates a polynomial (in evaluation form) at an arbitrary point + """ + field_elements_per_blob = SAMPLES_PER_BLOB * FIELD_ELEMENTS_PER_SAMPLE + roots = roots_of_unity(field_elements_per_blob) + + def A(z): + r = 1 + for w in roots: + r = r * (z - w) % BLS_MODULUS + return r + + def Aprime(z): + return field_elements_per_blob * pow(z, field_elements_per_blob - 1, BLS_MODULUS) + + r = 0 + inverses = [bls_modular_inverse(z - x) for z in roots] + for i, x in enumerate(inverses): + r += poly[i] * bls_modular_inverse(Aprime(roots[i])) * x % BLS_MODULUS + r = r * A(x) % BLS_MODULUS + return r +``` + +## KZG Operations + +We are using the KZG10 polynomial commitment scheme (Kate, Zaverucha and Goldberg, 2010: https://www.iacr.org/archive/asiacrypt2010/6477178/6477178.pdf). + +### Elliptic curve helper functoins + +#### `elliptic_curve_lincomb` + +```python +def elliptic_curve_lincomb(points: List[KZGCommitment], scalars: List[BLSFieldElement]) -> KZGCommitment: + """ + BLS multiscalar multiplication. This function can be optimized using Pippenger's algorithm and variants. + This is a non-optimized implementation. + """ + r = bls.inf_G1() + for x, a in zip(points, scalars): + r = r.add(x.mult(a)) + return r +``` + +### Hash to field + +#### `hash_to_bls_field` + +```python +def hash_to_bls_field(x: Container, challenge_number: uint64) -> BLSFieldElement: + """ + This function is used to generate Fiat-Shamir challenges. The output is not uniform over the BLS field. + """ + return ( + (int.from_bytes(hash(hash_tree_root(x) + int.to_bytes(challenge_number, 32, "little")), "little")) + % BLS_MODULUS + ) +``` + +### KZG operations + +#### `verify_kzg_proof` + +```python +def verify_kzg_proof(commitment: KZGCommitment, x: BLSFieldElement, y: BLSFieldElement, proof: KZGCommitment) -> None: + """ + Check that `proof` is a valid KZG proof for the polynomial committed to by `commitment` evaluated + at `x` equals `y`. + """ + zero_poly = G2_SETUP[1].add(G2_SETUP[0].mult(x).neg()) + + assert ( + bls.Pairing(proof, zero_poly) + == bls.Pairing(commitment.add(G1_SETUP[0].mult(y).neg), G2_SETUP[0]) + ) +``` + +#### `verify_kzg_multiproof` + +```python +def verify_kzg_multiproof(commitment: KZGCommitment, + xs: List[BLSFieldElement], + ys: List[BLSFieldElement], + proof: KZGCommitment) -> None: + """ + Verify a KZG multiproof. + """ + zero_poly = elliptic_curve_lincomb(G2_SETUP[:len(xs)], interpolate_polynomial(xs, [0] * len(ys))) + interpolated_poly = elliptic_curve_lincomb(G2_SETUP[:len(xs)], interpolate_polynomial(xs, ys)) + + assert ( + bls.Pairing(proof, zero_poly) + == bls.Pairing(commitment.add(interpolated_poly.neg()), G2_SETUP[0]) + ) +``` + +#### `verify_degree_proof` + +```python +def verify_degree_proof(commitment: KZGCommitment, degree_bound: uint64, proof: KZGCommitment): + """ + Verifies that the commitment is of polynomial degree < degree_bound. + """ + + assert ( + bls.Pairing(proof, G2_SETUP[0]) + == bls.Pairing(commitment, G2_SETUP[-degree_bound]) + ) +``` \ No newline at end of file diff --git a/specs/sharding/validator.md b/specs/sharding/validator.md new file mode 100644 index 000000000..38914095f --- /dev/null +++ b/specs/sharding/validator.md @@ -0,0 +1,141 @@ +# Sharding -- Honest Validator + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + + + - [Introduction](#introduction) + - [Prerequisites](#prerequisites) + - [Constants](#constants) + - [Sample counts](#sample-counts) + - [Helpers](#helpers) + - [`get_validator_row_subnets`](#get_validator_row_subnets) + - [`get_validator_column_subnets`](#get_validator_column_subnets) + - [`reconstruct_polynomial`](#reconstruct_polynomial) + - [Sample verification](#sample-verification) + - [`verify_sample`](#verify_sample) +- [Beacon chain responsibilities](#beacon-chain-responsibilities) + - [Validator assignments](#validator-assignments) + - [Attesting](#attesting) +- [Sample reconstruction](#sample-reconstruction) + - [Minimum online validator requirement](#minimum-online-validator-requirement) + + + + +## Introduction + +This document represents the changes to be made in the code of an "honest validator" to implement executable beacon chain proposal. + +## Prerequisites + +This document is an extension of the [Bellatrix -- Honest Validator](../bellatrix/validator.md) guide. +All behaviors and definitions defined in this document, and documents it extends, carry over unless explicitly noted or overridden. + +All terminology, constants, functions, and protocol mechanics defined in the updated Beacon Chain doc of [Sharding](./beacon-chain.md) are requisite for this document and used throughout. +Please see related Beacon Chain doc before continuing and use them as a reference throughout. + +## Constants + +### Sample counts + +| Name | Value | +| - | - | +| `VALIDATOR_SAMPLE_ROW_COUNT` | `2` | +| `VALIDATOR_SAMPLE_COLUMN_COUNT` | `2` | + +## Helpers + +### `get_validator_row_subnets` + +TODO: Currently the subnets are public (i.e. anyone can derive them.) This is good for a proof of custody with public verifiability, but bad for validator privacy. + +```python +def get_validator_row_subnets(validator: Validator, epoch: Epoch) -> List[uint64]: + return [int.from_bytes(hash_tree_root([validator.pubkey, 0, i])) for i in range(VALIDATOR_SAMPLE_ROW_COUNT)] +``` + +### `get_validator_column_subnets` + +```python +def get_validator_column_subnets(validator: Validator, epoch: Epoch) -> List[uint64]: + return [int.from_bytes(hash_tree_root([validator.pubkey, 1, i])) for i in range(VALIDATOR_SAMPLE_COLUMN_COUNT)] +``` + +### `reconstruct_polynomial` + +```python +def reconstruct_polynomial(samples: List[SignedShardSample]) -> List[SignedShardSample]: + """ + Reconstructs one full row/column from at least 1/2 of the samples + """ + +``` + +## Sample verification + +### `verify_sample` + +```python +def verify_sample(state: BeaconState, block: BeaconBlock, sample: SignedShardSample): + assert sample.row < 2 * get_active_shard_count(state, get_current_epoch(block.slot)) + assert sample.column < 2 * SAMPLES_PER_BLOB + assert block.slot == sample.slot + + # Verify builder signature. + # TODO: We should probably not do this. This should only be done by p2p to verify samples *before* intermediate block is in + # builder = state.validators[signed_block.message.proposer_index] + # signing_root = compute_signing_root(sample, get_domain(state, DOMAIN_SHARD_SAMPLE)) + # assert bls.Verify(sample.builder, signing_root, sample.signature) + + roots_in_rbo = list_to_reverse_bit_order(roots_of_unity(SAMPLES_PER_BLOB * FIELD_ELEMENTS_PER_SAMPLE)) + + # Verify KZG proof + verify_kzg_multiproof(block.body.payload_data.value.sharded_commitments_container.sharded_commitments[sample.row], + roots_in_rbo[sample.column * FIELD_ELEMENTS_PER_SAMPLE:(sample.column + 1) * FIELD_ELEMENTS_PER_SAMPLE] + sample.data, + sample.proof) +``` + +# Beacon chain responsibilities + +## Validator assignments + +### Attesting + +Every attester is assigned `VALIDATOR_SAMPLE_ROW_COUNT` rows and `VALIDATOR_SAMPLE_COLUMN_COUNT` columns of shard samples. As part of their validator duties, they should subscribe to the subnets given by `get_validator_row_subnets` and `get_validator_column_subnets`, for the whole epoch. + +A row or column is *available* for a `slot` if at least half of the total number of samples were received on the subnet and passed `verify_sample`. Otherwise it is called unavailable. + +If a validator is assigned to an attestation at slot `attestation_slot` and had his previous attestation duty at `previous_attestation_slot`, then they should only attest under the following conditions: + + * For all intermediate blocks `block` with `previous_attestation_slot < block.slot <= attestation_slot`: All sample rows and columns assigned to the validator were available. + +If this condition is not fulfilled, then the validator should instead attest to the last block for which the condition holds. + +This leads to the security property that a chain that is not fully available cannot have more than 1/16th of all validators voting for it. TODO: This claim is for an "infinite number" of validators. Compute the concrete security due to sampling bias. + +# Sample reconstruction + +A validator that has received enough samples of a row or column to mark it as available, should reconstruct all samples in that row/column (if they aren't all available already.) The function `reconstruct_polynomial` gives an example implementation for this. + +Once they have run the reconstruction function, they should distribute the samples that they reconstructed on all pubsub that +the local node is subscribed to, if they have not already received that sample on that pubsub. As an example: + + * The validator is subscribed to row `2` and column `5` + * The sample `(row, column) = (2, 5)` is missing in the column `5` pubsub + * After they have reconstruction of row `2`, the validator should send the sample `(2, 5)` on to the row `2` pubsub (if it was missing) as well as the column `5` pubsub. + +TODO: We need to verify the total complexity of doing this and make sure this does not cause too much load on a validator + +## Minimum online validator requirement + +The data availability construction guarantees that reconstruction is possible if 75% of all samples are available. In this case, at least 50% of all rows and 50% of all columns are independently available. In practice, it is likely that some supernodes will centrally collect all samples and fill in any gaps. However, we want to build a system that reliably reconstructs even absent all supernodes. Any row or column with 50% of samples will easily be reconstructed even with only 100s of validators online; so the only question is how we get to 50% of samples for all rows and columns, when some of them might be completely unseeded. + +Each validator will transfer 4 samples between rows and columns where there is overlap. Without loss of generality, look at row 0. Each validator has 1/128 chance of having a sample in this row, and we need 256 samples to reconstruct it. So we expect that we need ~256 * 128 = 32,768 validators to have a fair chance of reconstructing it if it was completely unseeded. + +A more elaborate estimate [here](https://notes.ethereum.org/@dankrad/minimum-reconstruction-validators) needs about 55,000 validators to be online for high safety that each row and column will be reconstructed. \ No newline at end of file From 3d28b37831683a1135d6b8fd4b5f62ced286ded5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 19 Jul 2022 21:55:55 +0800 Subject: [PATCH 074/105] Add `polynomial-commitments.md` to README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fcae73a32..525a345de 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Features are researched and developed in parallel, and then consolidated into se | Code Name or Topic | Specs | Notes | | - | - | - | | Capella (tentative) |
  • Core
    • [Beacon chain changes](specs/capella/beacon-chain.md)
    • [Capella fork](specs/capella/fork.md)
  • Additions
    • [Validator additions](specs/capella/validator.md)
| -| EIP4844 (tentative) |
  • Core
    • [Beacon Chain changes](specs/eip4844/beacon-chain.md)
    • [EIP-4844 fork](specs/eip4844/fork.md)
  • Additions
    • [Honest validator guide changes](specs/eip4844/validator.md)
    • [P2P networking](specs/eip4844/p2p-interface.md)
| +| EIP4844 (tentative) |
  • Core
    • [Beacon Chain changes](specs/eip4844/beacon-chain.md)
    • [EIP-4844 fork](specs/eip4844/fork.md)
    • [Polynomial commitments](specs/eip4844/polynomial-commitments.md)
  • Additions
    • [Honest validator guide changes](specs/eip4844/validator.md)
    • [P2P networking](specs/eip4844/p2p-interface.md)
| | Sharding (outdated) |
  • Core
    • [Beacon Chain changes](specs/sharding/beacon-chain.md)
  • Additions
    • [P2P networking](specs/sharding/p2p-interface.md)
| | Custody Game (outdated) |
  • Core
    • [Beacon Chain changes](specs/custody_game/beacon-chain.md)
  • Additions
    • [Honest validator guide changes](specs/custody_game/validator.md)
| Dependent on sharding | | Data Availability Sampling (outdated) |
  • Core
    • [Core types and functions](specs/das/das-core.md)
    • [Fork choice changes](specs/das/fork-choice.md)
  • Additions
    • [P2P Networking](specs/das/p2p-interface.md)
    • [Sampling process](specs/das/sampling.md)
|
  • Dependent on sharding
  • [Technical explainer](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/B1YJPGkpD)
| From 0f31fcdee4726b9ff4df7dc0afa7e307e15317d6 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 19 Jul 2022 22:08:16 +0800 Subject: [PATCH 075/105] [Refactor] remove useless `spec` param from `compute_committee_indices` --- .../test_process_sync_aggregate.py | 36 +++++++++---------- .../test_process_sync_aggregate_random.py | 2 +- .../eth2spec/test/helpers/light_client.py | 2 +- .../eth2spec/test/helpers/multi_operations.py | 2 +- .../test/helpers/proposer_slashings.py | 2 +- .../eth2spec/test/helpers/sync_committee.py | 8 ++--- .../test/phase0/sanity/test_blocks.py | 2 +- 7 files changed, 25 insertions(+), 29 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/block_processing/sync_aggregate/test_process_sync_aggregate.py b/tests/core/pyspec/eth2spec/test/altair/block_processing/sync_aggregate/test_process_sync_aggregate.py index 8a2deabb1..f593353f9 100644 --- a/tests/core/pyspec/eth2spec/test/altair/block_processing/sync_aggregate/test_process_sync_aggregate.py +++ b/tests/core/pyspec/eth2spec/test/altair/block_processing/sync_aggregate/test_process_sync_aggregate.py @@ -31,7 +31,7 @@ from eth2spec.test.context import ( @spec_state_test @always_bls def test_invalid_signature_bad_domain(spec, state): - committee_indices = compute_committee_indices(spec, state) + committee_indices = compute_committee_indices(state) block = build_empty_block_for_next_slot(spec, state) block.body.sync_aggregate = spec.SyncAggregate( @@ -52,7 +52,7 @@ def test_invalid_signature_bad_domain(spec, state): @spec_state_test @always_bls def test_invalid_signature_missing_participant(spec, state): - committee_indices = compute_committee_indices(spec, state) + committee_indices = compute_committee_indices(state) rng = random.Random(2020) random_participant = rng.choice(committee_indices) @@ -116,7 +116,7 @@ def test_invalid_signature_infinite_signature_with_single_participant(spec, stat @spec_state_test @always_bls def test_invalid_signature_extra_participant(spec, state): - committee_indices = compute_committee_indices(spec, state) + committee_indices = compute_committee_indices(state) rng = random.Random(3030) random_participant = rng.choice(committee_indices) @@ -140,7 +140,7 @@ def test_invalid_signature_extra_participant(spec, state): @with_presets([MINIMAL], reason="to create nonduplicate committee") @spec_state_test def test_sync_committee_rewards_nonduplicate_committee(spec, state): - committee_indices = compute_committee_indices(spec, state) + committee_indices = compute_committee_indices(state) committee_size = len(committee_indices) committee_bits = [True] * committee_size active_validator_count = len(spec.get_active_validator_indices(state, spec.get_current_epoch(state))) @@ -156,7 +156,7 @@ def test_sync_committee_rewards_nonduplicate_committee(spec, state): @with_presets([MAINNET], reason="to create duplicate committee") @spec_state_test def test_sync_committee_rewards_duplicate_committee_no_participation(spec, state): - committee_indices = compute_committee_indices(spec, state) + committee_indices = compute_committee_indices(state) committee_size = len(committee_indices) committee_bits = [False] * committee_size active_validator_count = len(spec.get_active_validator_indices(state, spec.get_current_epoch(state))) @@ -172,7 +172,7 @@ def test_sync_committee_rewards_duplicate_committee_no_participation(spec, state @with_presets([MAINNET], reason="to create duplicate committee") @spec_state_test def test_sync_committee_rewards_duplicate_committee_half_participation(spec, state): - committee_indices = compute_committee_indices(spec, state) + committee_indices = compute_committee_indices(state) committee_size = len(committee_indices) committee_bits = [True] * (committee_size // 2) + [False] * (committee_size // 2) assert len(committee_bits) == committee_size @@ -189,7 +189,7 @@ def test_sync_committee_rewards_duplicate_committee_half_participation(spec, sta @with_presets([MAINNET], reason="to create duplicate committee") @spec_state_test def test_sync_committee_rewards_duplicate_committee_full_participation(spec, state): - committee_indices = compute_committee_indices(spec, state) + committee_indices = compute_committee_indices(state) committee_size = len(committee_indices) committee_bits = [True] * committee_size active_validator_count = len(spec.get_active_validator_indices(state, spec.get_current_epoch(state))) @@ -205,7 +205,7 @@ def test_sync_committee_rewards_duplicate_committee_full_participation(spec, sta @spec_state_test @always_bls def test_sync_committee_rewards_not_full_participants(spec, state): - committee_indices = compute_committee_indices(spec, state) + committee_indices = compute_committee_indices(state) rng = random.Random(1010) committee_bits = [rng.choice([True, False]) for _ in committee_indices] @@ -216,7 +216,7 @@ def test_sync_committee_rewards_not_full_participants(spec, state): @spec_state_test @always_bls def test_sync_committee_rewards_empty_participants(spec, state): - committee_indices = compute_committee_indices(spec, state) + committee_indices = compute_committee_indices(state) committee_bits = [False for _ in committee_indices] yield from run_successful_sync_committee_test(spec, state, committee_indices, committee_bits) @@ -226,7 +226,7 @@ def test_sync_committee_rewards_empty_participants(spec, state): @spec_state_test @always_bls def test_invalid_signature_past_block(spec, state): - committee_indices = compute_committee_indices(spec, state) + committee_indices = compute_committee_indices(state) for _ in range(2): # NOTE: need to transition twice to move beyond the degenerate case at genesis @@ -280,7 +280,7 @@ def test_invalid_signature_previous_committee(spec, state): # Use the previous sync committee to produce the signature. # Ensure that the pubkey sets are different. assert set(old_sync_committee.pubkeys) != set(state.current_sync_committee.pubkeys) - committee_indices = compute_committee_indices(spec, state, old_sync_committee) + committee_indices = compute_committee_indices(state, old_sync_committee) block = build_empty_block_for_next_slot(spec, state) block.body.sync_aggregate = spec.SyncAggregate( @@ -322,7 +322,7 @@ def test_valid_signature_future_committee(spec, state): assert sync_committee != old_current_sync_committee assert sync_committee != old_next_sync_committee - committee_indices = compute_committee_indices(spec, state, sync_committee) + committee_indices = compute_committee_indices(state, sync_committee) block = build_empty_block_for_next_slot(spec, state) block.body.sync_aggregate = spec.SyncAggregate( @@ -344,7 +344,7 @@ def test_valid_signature_future_committee(spec, state): @always_bls @with_presets([MINIMAL], reason="prefer short search to find matching proposer") def test_proposer_in_committee_without_participation(spec, state): - committee_indices = compute_committee_indices(spec, state, state.current_sync_committee) + committee_indices = compute_committee_indices(state, state.current_sync_committee) # NOTE: seem to reliably be getting a matching proposer in the first epoch w/ ``MINIMAL`` preset. for _ in range(spec.SLOTS_PER_EPOCH): @@ -385,7 +385,7 @@ def test_proposer_in_committee_without_participation(spec, state): @always_bls @with_presets([MINIMAL], reason="prefer short search to find matching proposer") def test_proposer_in_committee_with_participation(spec, state): - committee_indices = compute_committee_indices(spec, state, state.current_sync_committee) + committee_indices = compute_committee_indices(state, state.current_sync_committee) participation = [True for _ in committee_indices] # NOTE: seem to reliably be getting a matching proposer in the first epoch w/ ``MINIMAL`` preset. @@ -451,7 +451,7 @@ def test_sync_committee_with_participating_exited_member(spec, state): for _ in range(3): next_epoch_via_block(spec, state) - committee_indices = compute_committee_indices(spec, state) + committee_indices = compute_committee_indices(state) rng = random.Random(1010) exited_index = _exit_validator_from_committee_and_transition_state( @@ -490,7 +490,7 @@ def test_sync_committee_with_nonparticipating_exited_member(spec, state): for _ in range(3): next_epoch_via_block(spec, state) - committee_indices = compute_committee_indices(spec, state) + committee_indices = compute_committee_indices(state) rng = random.Random(1010) exited_index = _exit_validator_from_committee_and_transition_state( @@ -533,7 +533,7 @@ def test_sync_committee_with_participating_withdrawable_member(spec, state): for _ in range(3): next_epoch_via_block(spec, state) - committee_indices = compute_committee_indices(spec, state) + committee_indices = compute_committee_indices(state) rng = random.Random(1010) exited_index = _exit_validator_from_committee_and_transition_state( @@ -572,7 +572,7 @@ def test_sync_committee_with_nonparticipating_withdrawable_member(spec, state): for _ in range(3): next_epoch_via_block(spec, state) - committee_indices = compute_committee_indices(spec, state) + committee_indices = compute_committee_indices(state) rng = random.Random(1010) exited_index = _exit_validator_from_committee_and_transition_state( diff --git a/tests/core/pyspec/eth2spec/test/altair/block_processing/sync_aggregate/test_process_sync_aggregate_random.py b/tests/core/pyspec/eth2spec/test/altair/block_processing/sync_aggregate/test_process_sync_aggregate_random.py index 903df4081..a402e3d54 100644 --- a/tests/core/pyspec/eth2spec/test/altair/block_processing/sync_aggregate/test_process_sync_aggregate_random.py +++ b/tests/core/pyspec/eth2spec/test/altair/block_processing/sync_aggregate/test_process_sync_aggregate_random.py @@ -28,7 +28,7 @@ from eth2spec.test.context import ( def _test_harness_for_randomized_test_case(spec, state, expect_duplicates=False, participation_fn=None): - committee_indices = compute_committee_indices(spec, state) + committee_indices = compute_committee_indices(state) if participation_fn: participating_indices = participation_fn(committee_indices) diff --git a/tests/core/pyspec/eth2spec/test/helpers/light_client.py b/tests/core/pyspec/eth2spec/test/helpers/light_client.py index 144b9719c..8d632b3a1 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/light_client.py +++ b/tests/core/pyspec/eth2spec/test/helpers/light_client.py @@ -39,7 +39,7 @@ def get_sync_aggregate(spec, state, num_participants=None, signature_slot=None): transition_to(spec, signature_state, signature_slot) # Fetch sync committee - committee_indices = compute_committee_indices(spec, signature_state) + committee_indices = compute_committee_indices(signature_state) committee_size = len(committee_indices) # By default, use full participation diff --git a/tests/core/pyspec/eth2spec/test/helpers/multi_operations.py b/tests/core/pyspec/eth2spec/test/helpers/multi_operations.py index c2cc6d98a..44ed0ae89 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/multi_operations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/multi_operations.py @@ -180,7 +180,7 @@ def get_random_voluntary_exits(spec, state, to_be_slashed_indices, rng): def get_random_sync_aggregate(spec, state, slot, block_root=None, fraction_participated=1.0, rng=Random(2099)): - committee_indices = compute_committee_indices(spec, state, state.current_sync_committee) + committee_indices = compute_committee_indices(state, state.current_sync_committee) participant_count = int(len(committee_indices) * fraction_participated) participant_indices = rng.sample(range(len(committee_indices)), participant_count) participants = [ diff --git a/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py b/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py index faa4d4288..517190869 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py +++ b/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -30,7 +30,7 @@ def check_proposer_slashing_effect(spec, pre_state, state, slashed_index, block= # Altair introduces sync committee (SC) reward and penalty sc_reward_for_slashed = sc_penalty_for_slashed = sc_reward_for_proposer = sc_penalty_for_proposer = 0 if is_post_altair(spec) and block is not None: - committee_indices = compute_committee_indices(spec, state, state.current_sync_committee) + committee_indices = compute_committee_indices(state, state.current_sync_committee) committee_bits = block.body.sync_aggregate.sync_committee_bits sc_reward_for_slashed, sc_penalty_for_slashed = compute_sync_committee_participant_reward_and_penalty( spec, diff --git a/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py b/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py index 417802ece..cc05b862b 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py @@ -74,7 +74,7 @@ def compute_sync_committee_proposer_reward(spec, state, committee_indices, commi return spec.Gwei(participant_reward * participant_number) -def compute_committee_indices(spec, state, committee=None): +def compute_committee_indices(state, committee=None): """ Given a ``committee``, calculate and return the related indices """ @@ -129,11 +129,7 @@ def run_sync_committee_processing(spec, state, block, expect_exception=False): if expect_exception: assert pre_state.balances == state.balances else: - committee_indices = compute_committee_indices( - spec, - state, - state.current_sync_committee, - ) + committee_indices = compute_committee_indices(state, state.current_sync_committee) committee_bits = block.body.sync_aggregate.sync_committee_bits validate_sync_committee_rewards( spec, diff --git a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py index 44d663422..a09c02b9a 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py @@ -773,7 +773,7 @@ def test_deposit_top_up(spec, state): # Altair introduces sync committee (sm) reward and penalty sync_committee_reward = sync_committee_penalty = 0 if is_post_altair(spec): - committee_indices = compute_committee_indices(spec, state, state.current_sync_committee) + committee_indices = compute_committee_indices(state, state.current_sync_committee) committee_bits = block.body.sync_aggregate.sync_committee_bits sync_committee_reward, sync_committee_penalty = compute_sync_committee_participant_reward_and_penalty( spec, From 06d9fd8cd31c06eb49036283d04226877b12e21e Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Tue, 3 May 2022 12:15:04 +0200 Subject: [PATCH 076/105] Add `LightClient(Finality|Optimistic)Update` Introduces reduced `LightClientUpdate` structures to allow keeping track of the latest `finalized_header` and `optimistic_header`. This may also help in scheduling the next query for a full `LightClientUpdate` once sync committee finality has been reached. --- specs/altair/light-client/full-node.md | 30 +++++++++ specs/altair/light-client/sync-protocol.md | 71 +++++++++++++++++++++- 2 files changed, 100 insertions(+), 1 deletion(-) diff --git a/specs/altair/light-client/full-node.md b/specs/altair/light-client/full-node.md index a7dd55101..09810a037 100644 --- a/specs/altair/light-client/full-node.md +++ b/specs/altair/light-client/full-node.md @@ -14,6 +14,8 @@ - [Deriving light client data](#deriving-light-client-data) - [`create_light_client_bootstrap`](#create_light_client_bootstrap) - [`create_light_client_update`](#create_light_client_update) + - [`create_light_client_finality_update`](#create_light_client_finality_update) + - [`create_light_client_optimistic_update`](#create_light_client_optimistic_update) @@ -133,3 +135,31 @@ Full nodes SHOULD provide the best derivable `LightClientUpdate` (according to ` - `LightClientUpdate` are assigned to sync committee periods based on their `attested_header.slot` - `LightClientUpdate` are only considered if `compute_sync_committee_period(compute_epoch_at_slot(update.attested_header.slot)) == compute_sync_committee_period(compute_epoch_at_slot(update.signature_slot))` - Only `LightClientUpdate` with `next_sync_committee` as selected by fork choice are provided, regardless of ranking by `is_better_update`. To uniquely identify a non-finalized sync committee fork, all of `period`, `current_sync_committee` and `next_sync_committee` need to be incorporated, as sync committees may reappear over time. + +### `create_light_client_finality_update` + +```python +def create_light_client_finality_update(update: LightClientUpdate) -> LightClientFinalityUpdate: + return LightClientFinalityUpdate( + attested_header=update.attested_header, + finalized_header=update.finalized_header, + finality_branch=update.finality_branch, + sync_aggregate=update.sync_aggregate, + signature_slot=update.signature_slot, + ) +``` + +Full nodes SHOULD provide the `LightClientFinalityUpdate` with the highest `attested_header.slot` (if multiple, highest `signature_slot`) as selected by fork choice, and SHOULD support a push mechanism to deliver new `LightClientFinalityUpdate` whenever `finalized_header` changes. + +### `create_light_client_optimistic_update` + +```python +def create_light_client_optimistic_update(update: LightClientUpdate) -> LightClientOptimisticUpdate: + return LightClientOptimisticUpdate( + attested_header=update.attested_header, + sync_aggregate=update.sync_aggregate, + signature_slot=update.signature_slot, + ) +``` + +Full nodes SHOULD provide the `LightClientOptimisticUpdate` with the highest `attested_header.slot` (if multiple, highest `signature_slot`) as selected by fork choice, and SHOULD support a push mechanism to deliver new `LightClientOptimisticUpdate` whenever `attested_header` changes. diff --git a/specs/altair/light-client/sync-protocol.md b/specs/altair/light-client/sync-protocol.md index 217171db1..08581fd88 100644 --- a/specs/altair/light-client/sync-protocol.md +++ b/specs/altair/light-client/sync-protocol.md @@ -15,6 +15,8 @@ - [Containers](#containers) - [`LightClientBootstrap`](#lightclientbootstrap) - [`LightClientUpdate`](#lightclientupdate) + - [`LightClientFinalityUpdate`](#lightclientfinalityupdate) + - [`LightClientOptimisticUpdate`](#lightclientoptimisticupdate) - [`LightClientStore`](#lightclientstore) - [Helper functions](#helper-functions) - [`is_sync_committee_update`](#is_sync_committee_update) @@ -31,6 +33,8 @@ - [`validate_light_client_update`](#validate_light_client_update) - [`apply_light_client_update`](#apply_light_client_update) - [`process_light_client_update`](#process_light_client_update) + - [`process_light_client_finality_update`](#process_light_client_finality_update) + - [`process_light_client_optimistic_update`](#process_light_client_optimistic_update) @@ -96,6 +100,33 @@ class LightClientUpdate(Container): signature_slot: Slot ``` +### `LightClientFinalityUpdate` + +```python +class LightClientFinalityUpdate(Container): + # The beacon block header that is attested to by the sync committee + attested_header: BeaconBlockHeader + # The finalized beacon block header attested to by Merkle branch + finalized_header: BeaconBlockHeader + finality_branch: Vector[Bytes32, floorlog2(FINALIZED_ROOT_INDEX)] + # Sync committee aggregate signature + sync_aggregate: SyncAggregate + # Slot at which the aggregate signature was created (untrusted) + signature_slot: Slot +``` + +### `LightClientOptimisticUpdate` + +```python +class LightClientOptimisticUpdate(Container): + # The beacon block header that is attested to by the sync committee + attested_header: BeaconBlockHeader + # Sync committee aggregate signature + sync_aggregate: SyncAggregate + # Slot at which the aggregate signature was created (untrusted) + signature_slot: Slot +``` + ### `LightClientStore` ```python @@ -250,7 +281,7 @@ def initialize_light_client_store(trusted_block_root: Root, ## Light client state updates -A light client receives `update` objects of type `LightClientUpdate`. Every `update` triggers `process_light_client_update(store, update, current_slot, genesis_validators_root)` where `current_slot` is the current slot based on a local clock. `process_slot_for_light_client_store` is triggered every time the current slot increments. +A light client receives `update`, `finality_update` and `optimistic_update` objects of type `LightClientUpdate`, `LightClientFinalityUpdate` and `LightClientOptimisticUpdate`. Every `update` triggers `process_light_client_update(store, update, current_slot, genesis_validators_root)` where `current_slot` is the current slot based on a local clock. Likewise, every `finality_update` triggers `process_light_client_finality_update`, and every `optimistic_update` triggers `process_light_client_optimistic_update`. `process_slot_for_light_client_store` is triggered every time the current slot increments. ### `process_slot_for_light_client_store` @@ -420,3 +451,41 @@ def process_light_client_update(store: LightClientStore, apply_light_client_update(store, update) store.best_valid_update = None ``` + +### `process_light_client_finality_update` + +```python +def process_light_client_finality_update(store: LightClientStore, + finality_update: LightClientFinalityUpdate, + current_slot: Slot, + genesis_validators_root: Root) -> None: + update = LightClientUpdate( + attested_header=finality_update.attested_header, + next_sync_committee=SyncCommittee(), + next_sync_committee_branch=[Bytes32() for _ in range(floorlog2(NEXT_SYNC_COMMITTEE_INDEX))], + finalized_header=finality_update.finalized_header, + finality_branch=finality_update.finality_branch, + sync_aggregate=finality_update.sync_aggregate, + signature_slot=finality_update.signature_slot, + ) + process_light_client_update(store, update, current_slot, genesis_validators_root) +``` + +### `process_light_client_optimistic_update` + +```python +def process_light_client_optimistic_update(store: LightClientStore, + optimistic_update: LightClientOptimisticUpdate, + current_slot: Slot, + genesis_validators_root: Root) -> None: + update = LightClientUpdate( + attested_header=optimistic_update.attested_header, + next_sync_committee=SyncCommittee(), + next_sync_committee_branch=[Bytes32() for _ in range(floorlog2(NEXT_SYNC_COMMITTEE_INDEX))], + finalized_header=BeaconBlockHeader(), + finality_branch=[Bytes32() for _ in range(floorlog2(FINALIZED_ROOT_INDEX))], + sync_aggregate=optimistic_update.sync_aggregate, + signature_slot=optimistic_update.signature_slot, + ) + process_light_client_update(store, update, current_slot, genesis_validators_root) +``` From 95019bb9a997e7fd4276d4370cd27f0f1a69fa68 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Wed, 20 Jul 2022 17:50:18 +0200 Subject: [PATCH 077/105] Improve readability of intro text Co-authored-by: Hsiao-Wei Wang --- specs/altair/light-client/sync-protocol.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/specs/altair/light-client/sync-protocol.md b/specs/altair/light-client/sync-protocol.md index 08581fd88..627049a43 100644 --- a/specs/altair/light-client/sync-protocol.md +++ b/specs/altair/light-client/sync-protocol.md @@ -281,7 +281,11 @@ def initialize_light_client_store(trusted_block_root: Root, ## Light client state updates -A light client receives `update`, `finality_update` and `optimistic_update` objects of type `LightClientUpdate`, `LightClientFinalityUpdate` and `LightClientOptimisticUpdate`. Every `update` triggers `process_light_client_update(store, update, current_slot, genesis_validators_root)` where `current_slot` is the current slot based on a local clock. Likewise, every `finality_update` triggers `process_light_client_finality_update`, and every `optimistic_update` triggers `process_light_client_optimistic_update`. `process_slot_for_light_client_store` is triggered every time the current slot increments. +- A light client receives objects of type `LightClientUpdate`, `LightClientFinalityUpdate` and `LightClientOptimisticUpdate`: + - **`update: LightClientUpdate`**: Every `update` triggers `process_light_client_update(store, update, current_slot, genesis_validators_root)` where `current_slot` is the current slot based on a local clock. + - **`finality_update: LightClientFinalityUpdate`**: Every `finality_update` triggers `process_light_client_finality_update(store, finality_update, current_slot, genesis_validators_root)`. + - **`optimistic_update: LightClientOptimisticUpdate`**: Every `optimistic_update` triggers `process_light_client_optimistic_update(store, optimistic_update, current_slot, genesis_validators_root)`. +- `process_slot_for_light_client_store` is triggered every time the current slot increments. ### `process_slot_for_light_client_store` From 9016b926916fb82d0183b43fea4241dcdf09520b Mon Sep 17 00:00:00 2001 From: inphi Date: Thu, 21 Jul 2022 01:58:22 -0400 Subject: [PATCH 078/105] Fix Polynomial container size --- specs/eip4844/validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/eip4844/validator.md b/specs/eip4844/validator.md index 1a45342d0..e673de1e3 100644 --- a/specs/eip4844/validator.md +++ b/specs/eip4844/validator.md @@ -47,7 +47,7 @@ Please see related Beacon Chain doc before continuing and use them as a referenc | Name | SSZ equivalent | Description | | - | - | - | -| `Polynomial` | `List[BLSFieldElement, MAX_BLOBS_PER_BLOCK]` | a polynomial in evaluation form | +| `Polynomial` | `List[BLSFieldElement, FIELD_ELEMENTS_PER_BLOB]` | a polynomial in evaluation form | ## Containers From f9d866eb28db8c5b59dbd166c13d3721ed615323 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Tue, 3 May 2022 12:31:26 +0200 Subject: [PATCH 079/105] Manually trigger `LightClientStore` force updates Replaces `process_slot_for_light_client_store` which force updates the `LightClientStore` automatically based on `finalized_header` age with `try_light_client_store_force_update` which may be manually called based on use case dependent heuristics if light client sync appears stuck. Not all use cases share the same risk profile. --- specs/altair/light-client/sync-protocol.md | 45 +++++++++---------- .../test/altair/light_client/test_sync.py | 10 ++--- tests/formats/light_client/sync.md | 4 +- 3 files changed, 29 insertions(+), 30 deletions(-) diff --git a/specs/altair/light-client/sync-protocol.md b/specs/altair/light-client/sync-protocol.md index 627049a43..058183fc4 100644 --- a/specs/altair/light-client/sync-protocol.md +++ b/specs/altair/light-client/sync-protocol.md @@ -29,9 +29,9 @@ - [Light client initialization](#light-client-initialization) - [`initialize_light_client_store`](#initialize_light_client_store) - [Light client state updates](#light-client-state-updates) - - [`process_slot_for_light_client_store`](#process_slot_for_light_client_store) - [`validate_light_client_update`](#validate_light_client_update) - [`apply_light_client_update`](#apply_light_client_update) + - [`try_light_client_store_force_update`](#try_light_client_store_force_update) - [`process_light_client_update`](#process_light_client_update) - [`process_light_client_finality_update`](#process_light_client_finality_update) - [`process_light_client_optimistic_update`](#process_light_client_optimistic_update) @@ -285,28 +285,7 @@ def initialize_light_client_store(trusted_block_root: Root, - **`update: LightClientUpdate`**: Every `update` triggers `process_light_client_update(store, update, current_slot, genesis_validators_root)` where `current_slot` is the current slot based on a local clock. - **`finality_update: LightClientFinalityUpdate`**: Every `finality_update` triggers `process_light_client_finality_update(store, finality_update, current_slot, genesis_validators_root)`. - **`optimistic_update: LightClientOptimisticUpdate`**: Every `optimistic_update` triggers `process_light_client_optimistic_update(store, optimistic_update, current_slot, genesis_validators_root)`. -- `process_slot_for_light_client_store` is triggered every time the current slot increments. - -### `process_slot_for_light_client_store` - -```python -def process_slot_for_light_client_store(store: LightClientStore, current_slot: Slot) -> None: - if current_slot % UPDATE_TIMEOUT == 0: - store.previous_max_active_participants = store.current_max_active_participants - store.current_max_active_participants = 0 - if ( - current_slot > store.finalized_header.slot + UPDATE_TIMEOUT - and store.best_valid_update is not None - ): - # Forced best update when the update timeout has elapsed. - # Because the apply logic waits for `finalized_header.slot` to indicate sync committee finality, - # the `attested_header` may be treated as `finalized_header` in extended periods of non-finality - # to guarantee progression into later sync committee periods according to `is_better_update`. - if store.best_valid_update.finalized_header.slot <= store.finalized_header.slot: - store.best_valid_update.finalized_header = store.best_valid_update.attested_header - apply_light_client_update(store, store.best_valid_update) - store.best_valid_update = None -``` +- `try_light_client_store_force_update` MAY be called based on use case dependent heuristics if light client sync appears stuck. ### `validate_light_client_update` @@ -399,12 +378,32 @@ def apply_light_client_update(store: LightClientStore, update: LightClientUpdate elif update_finalized_period == store_period + 1: store.current_sync_committee = store.next_sync_committee store.next_sync_committee = update.next_sync_committee + store.previous_max_active_participants = store.current_max_active_participants + store.current_max_active_participants = 0 if update.finalized_header.slot > store.finalized_header.slot: store.finalized_header = update.finalized_header if store.finalized_header.slot > store.optimistic_header.slot: store.optimistic_header = store.finalized_header ``` +### `try_light_client_store_force_update` + +```python +def try_light_client_store_force_update(store: LightClientStore, current_slot: Slot) -> None: + if ( + current_slot > store.finalized_header.slot + UPDATE_TIMEOUT + and store.best_valid_update is not None + ): + # Forced best update when the update timeout has elapsed. + # Because the apply logic waits for `finalized_header.slot` to indicate sync committee finality, + # the `attested_header` may be treated as `finalized_header` in extended periods of non-finality + # to guarantee progression into later sync committee periods according to `is_better_update`. + if store.best_valid_update.finalized_header.slot <= store.finalized_header.slot: + store.best_valid_update.finalized_header = store.best_valid_update.attested_header + apply_light_client_update(store, store.best_valid_update) + store.best_valid_update = None +``` + ### `process_light_client_update` ```python diff --git a/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py b/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py index 9448302e7..ed2d04b4d 100644 --- a/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py +++ b/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py @@ -72,13 +72,13 @@ def get_checks(store): } -def emit_slot(test, spec, state): +def emit_force_update(test, spec, state): current_slot = state.slot - spec.process_slot_for_light_client_store(test.store, current_slot) + spec.try_light_client_store_force_update(test.store, current_slot) yield from [] # Consistently enable `yield from` syntax in calling tests test.steps.append({ - "process_slot": { + "force_update": { "current_slot": int(current_slot), "checks": get_checks(test.store), } @@ -249,7 +249,7 @@ def test_light_client_sync(spec, state): # ``` attested_state = state.copy() next_slots(spec, state, spec.UPDATE_TIMEOUT - 1) - yield from emit_slot(test, spec, state) + yield from emit_force_update(test, spec, state) assert test.store.finalized_header.slot == store_state.slot assert test.store.next_sync_committee == store_state.next_sync_committee assert test.store.best_valid_update is None @@ -293,7 +293,7 @@ def test_light_client_sync(spec, state): assert test.store.next_sync_committee == store_state.next_sync_committee assert test.store.best_valid_update == update assert test.store.optimistic_header.slot == attested_state.slot - yield from emit_slot(test, spec, state) + yield from emit_force_update(test, spec, state) assert test.store.finalized_header.slot == attested_state.slot assert test.store.next_sync_committee == attested_state.next_sync_committee assert test.store.best_valid_update is None diff --git a/tests/formats/light_client/sync.md b/tests/formats/light_client/sync.md index 31ecf6036..aa0d593b6 100644 --- a/tests/formats/light_client/sync.md +++ b/tests/formats/light_client/sync.md @@ -34,9 +34,9 @@ optimistic_header: { } ``` -#### `process_slot` execution step +#### `force_update` execution step -The function `process_slot_for_light_client_store(store, current_slot)` +The function `try_light_client_store_force_update(store, current_slot)` should be executed with the specified parameters: ```yaml From b2c647f2b2c80bd843c387bd79c12385c1c73785 Mon Sep 17 00:00:00 2001 From: Michael de Hoog Date: Thu, 21 Jul 2022 11:11:30 -0500 Subject: [PATCH 080/105] Remove triple m --- specs/eip4844/validator.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/eip4844/validator.md b/specs/eip4844/validator.md index 1a45342d0..8170e9b00 100644 --- a/specs/eip4844/validator.md +++ b/specs/eip4844/validator.md @@ -12,7 +12,7 @@ - [Prerequisites](#prerequisites) - [Custom types](#custom-types) - [Containers](#containers) - - [`BlobsAndCommmitments`](#blobsandcommmitments) + - [`BlobsAndCommitments`](#blobsandcommitments) - [`PolynomialAndCommitment`](#polynomialandcommitment) - [Helpers](#helpers) - [`is_data_available`](#is_data_available) @@ -51,10 +51,10 @@ Please see related Beacon Chain doc before continuing and use them as a referenc ## Containers -### `BlobsAndCommmitments` +### `BlobsAndCommitments` ```python -class BlobsAndCommmitments(Container): +class BlobsAndCommitments(Container): blobs: List[Blob, MAX_BLOBS_PER_BLOCK] kzg_commitments: List[KZGCommitment, MAX_BLOBS_PER_BLOCK] ``` @@ -122,7 +122,7 @@ def compute_aggregated_poly_and_commitment( Return the aggregated polynomial and aggregated KZG commitment. """ # Generate random linear combination challenges - r = hash_to_bls_field(BlobsAndCommmitments(blobs=blobs, kzg_commitments=kzg_commitments)) + r = hash_to_bls_field(BlobsAndCommitments(blobs=blobs, kzg_commitments=kzg_commitments)) r_powers = compute_powers(r, len(kzg_commitments)) # Create aggregated polynomial in evaluation form From 8643e28f12380e25933d84a2a81128a8cfacb88c Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 22 Jul 2022 11:49:07 +0200 Subject: [PATCH 081/105] `try_...` --> `process_...` --- specs/altair/light-client/sync-protocol.md | 8 ++++---- .../pyspec/eth2spec/test/altair/light_client/test_sync.py | 2 +- tests/formats/light_client/sync.md | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/altair/light-client/sync-protocol.md b/specs/altair/light-client/sync-protocol.md index 058183fc4..2fda56fc3 100644 --- a/specs/altair/light-client/sync-protocol.md +++ b/specs/altair/light-client/sync-protocol.md @@ -31,7 +31,7 @@ - [Light client state updates](#light-client-state-updates) - [`validate_light_client_update`](#validate_light_client_update) - [`apply_light_client_update`](#apply_light_client_update) - - [`try_light_client_store_force_update`](#try_light_client_store_force_update) + - [`process_light_client_store_force_update`](#process_light_client_store_force_update) - [`process_light_client_update`](#process_light_client_update) - [`process_light_client_finality_update`](#process_light_client_finality_update) - [`process_light_client_optimistic_update`](#process_light_client_optimistic_update) @@ -285,7 +285,7 @@ def initialize_light_client_store(trusted_block_root: Root, - **`update: LightClientUpdate`**: Every `update` triggers `process_light_client_update(store, update, current_slot, genesis_validators_root)` where `current_slot` is the current slot based on a local clock. - **`finality_update: LightClientFinalityUpdate`**: Every `finality_update` triggers `process_light_client_finality_update(store, finality_update, current_slot, genesis_validators_root)`. - **`optimistic_update: LightClientOptimisticUpdate`**: Every `optimistic_update` triggers `process_light_client_optimistic_update(store, optimistic_update, current_slot, genesis_validators_root)`. -- `try_light_client_store_force_update` MAY be called based on use case dependent heuristics if light client sync appears stuck. +- `process_light_client_store_force_update` MAY be called based on use case dependent heuristics if light client sync appears stuck. ### `validate_light_client_update` @@ -386,10 +386,10 @@ def apply_light_client_update(store: LightClientStore, update: LightClientUpdate store.optimistic_header = store.finalized_header ``` -### `try_light_client_store_force_update` +### `process_light_client_store_force_update` ```python -def try_light_client_store_force_update(store: LightClientStore, current_slot: Slot) -> None: +def process_light_client_store_force_update(store: LightClientStore, current_slot: Slot) -> None: if ( current_slot > store.finalized_header.slot + UPDATE_TIMEOUT and store.best_valid_update is not None diff --git a/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py b/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py index ed2d04b4d..47348717b 100644 --- a/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py +++ b/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py @@ -74,7 +74,7 @@ def get_checks(store): def emit_force_update(test, spec, state): current_slot = state.slot - spec.try_light_client_store_force_update(test.store, current_slot) + spec.process_light_client_store_force_update(test.store, current_slot) yield from [] # Consistently enable `yield from` syntax in calling tests test.steps.append({ diff --git a/tests/formats/light_client/sync.md b/tests/formats/light_client/sync.md index aa0d593b6..ad597fa1d 100644 --- a/tests/formats/light_client/sync.md +++ b/tests/formats/light_client/sync.md @@ -36,7 +36,7 @@ optimistic_header: { #### `force_update` execution step -The function `try_light_client_store_force_update(store, current_slot)` +The function `process_light_client_store_force_update(store, current_slot)` should be executed with the specified parameters: ```yaml From 176a5d227a105d0cf3a2d09a809a1514710d6bf6 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 22 Jul 2022 21:34:09 +0800 Subject: [PATCH 082/105] Move `merkle/single_proof` tests to `light_client/single_merkle_proof` --- .../test_single_merkle_proof.py} | 0 .../eth2spec/test/altair/merkle/__init__.py | 0 tests/formats/light_client/README.md | 1 + .../single_merkle_proof.md} | 0 tests/formats/merkle/README.md | 8 -------- tests/generators/light_client/main.py | 1 + tests/generators/merkle/README.md | 6 ------ tests/generators/merkle/__init__.py | 0 tests/generators/merkle/main.py | 16 ---------------- tests/generators/merkle/requirements.txt | 2 -- 10 files changed, 2 insertions(+), 32 deletions(-) rename tests/core/pyspec/eth2spec/test/altair/{merkle/test_single_proof.py => light_client/test_single_merkle_proof.py} (100%) delete mode 100644 tests/core/pyspec/eth2spec/test/altair/merkle/__init__.py rename tests/formats/{merkle/single_proof.md => light_client/single_merkle_proof.md} (100%) delete mode 100644 tests/formats/merkle/README.md delete mode 100644 tests/generators/merkle/README.md delete mode 100644 tests/generators/merkle/__init__.py delete mode 100644 tests/generators/merkle/main.py delete mode 100644 tests/generators/merkle/requirements.txt diff --git a/tests/core/pyspec/eth2spec/test/altair/merkle/test_single_proof.py b/tests/core/pyspec/eth2spec/test/altair/light_client/test_single_merkle_proof.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/altair/merkle/test_single_proof.py rename to tests/core/pyspec/eth2spec/test/altair/light_client/test_single_merkle_proof.py diff --git a/tests/core/pyspec/eth2spec/test/altair/merkle/__init__.py b/tests/core/pyspec/eth2spec/test/altair/merkle/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/formats/light_client/README.md b/tests/formats/light_client/README.md index 8c7571d4b..505b41601 100644 --- a/tests/formats/light_client/README.md +++ b/tests/formats/light_client/README.md @@ -3,5 +3,6 @@ This series of tests provides reference test vectors for the light client sync protocol spec. Handlers: +- `single_merkle_proof`: see [Single leaf merkle proof test format](./single_merkle_proof.md) - `sync`: see [Light client sync test format](./sync.md) - `update_ranking`: see [`LightClientUpdate` ranking test format](./update_ranking.md) diff --git a/tests/formats/merkle/single_proof.md b/tests/formats/light_client/single_merkle_proof.md similarity index 100% rename from tests/formats/merkle/single_proof.md rename to tests/formats/light_client/single_merkle_proof.md diff --git a/tests/formats/merkle/README.md b/tests/formats/merkle/README.md deleted file mode 100644 index c0f0a205b..000000000 --- a/tests/formats/merkle/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Merkle tests - -This series of tests provides reference test vectors for validating correct -generation and verification of merkle proofs based on static data. - -Handlers: -- `single_proof`: see [Single leaf proof test format](./single_proof.md) -- Different types of merkle proofs may be supported in the future. diff --git a/tests/generators/light_client/main.py b/tests/generators/light_client/main.py index ef6c2cbca..68a0da529 100644 --- a/tests/generators/light_client/main.py +++ b/tests/generators/light_client/main.py @@ -4,6 +4,7 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators if __name__ == "__main__": altair_mods = {key: 'eth2spec.test.altair.light_client.test_' + key for key in [ + 'single_merkle_proof', 'sync', 'update_ranking', ]} diff --git a/tests/generators/merkle/README.md b/tests/generators/merkle/README.md deleted file mode 100644 index a19a67d9e..000000000 --- a/tests/generators/merkle/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Merkle - -The purpose of this test-generator is to provide test-vectors for validating the -correct merkleization of objects and corresponding merkle proofs. - -Test-format documentation can be found [here](../../formats/merkle/README.md). diff --git a/tests/generators/merkle/__init__.py b/tests/generators/merkle/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/generators/merkle/main.py b/tests/generators/merkle/main.py deleted file mode 100644 index 5d07264ef..000000000 --- a/tests/generators/merkle/main.py +++ /dev/null @@ -1,16 +0,0 @@ -from eth2spec.test.helpers.constants import ALTAIR, BELLATRIX -from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators - - -if __name__ == "__main__": - altair_mods = {key: 'eth2spec.test.altair.merkle.test_' + key for key in [ - 'single_proof', - ]} - bellatrix_mods = altair_mods - - all_mods = { - ALTAIR: altair_mods, - BELLATRIX: bellatrix_mods, - } - - run_state_test_generators(runner_name="merkle", all_mods=all_mods) diff --git a/tests/generators/merkle/requirements.txt b/tests/generators/merkle/requirements.txt deleted file mode 100644 index 182248686..000000000 --- a/tests/generators/merkle/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -pytest>=4.4 -../../../[generator] From 78d336e28d9352a94259394bbc24d271332d41b8 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Tue, 3 May 2022 14:21:07 +0200 Subject: [PATCH 083/105] Describe light client sync process Provides documentation about the process that a light client may use to sync a local `LightClientStore`. --- README.md | 2 +- setup.py | 1 + specs/altair/light-client/light-client.md | 37 ++++++++++++++++++++++ specs/altair/light-client/sync-protocol.md | 1 + 4 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 specs/altair/light-client/light-client.md diff --git a/README.md b/README.md index 5e2a0b19f..3a4b2c8c5 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Features are researched and developed in parallel, and then consolidated into se | Seq. | Code Name | Fork Epoch | Specs | | - | - | - | - | | 0 | **Phase0** |`0` |
  • Core
    • [The beacon chain](specs/phase0/beacon-chain.md)
    • [Deposit contract](specs/phase0/deposit-contract.md)
    • [Beacon chain fork choice](specs/phase0/fork-choice.md)
  • Additions
    • [Honest validator guide](specs/phase0/validator.md)
    • [P2P networking](specs/phase0/p2p-interface.md)
    • [Weak subjectivity](specs/phase0/weak-subjectivity.md)
| -| 1 | **Altair** | `74240` |
  • Core
    • [Beacon chain changes](specs/altair/beacon-chain.md)
    • [Altair fork](specs/altair/fork.md)
  • Additions
    • [Light client sync protocol](specs/altair/light-client/sync-protocol.md) ([full node](specs/altair/light-client/full-node.md))
    • [Honest validator guide changes](specs/altair/validator.md)
    • [P2P networking](specs/altair/p2p-interface.md)
| +| 1 | **Altair** | `74240` |
  • Core
    • [Beacon chain changes](specs/altair/beacon-chain.md)
    • [Altair fork](specs/altair/fork.md)
  • Additions
    • [Light client sync protocol](specs/altair/light-client/sync-protocol.md) ([full node](specs/altair/light-client/full-node.md), [light client](specs/altair/light-client/light-client.md))
    • [Honest validator guide changes](specs/altair/validator.md)
    • [P2P networking](specs/altair/p2p-interface.md)
| | 2 | **Bellatrix**
(["The Merge"](https://ethereum.org/en/upgrades/merge/)) | TBD |
  • Core
    • [Beacon Chain changes](specs/bellatrix/beacon-chain.md)
    • [Bellatrix fork](specs/bellatrix/fork.md)
    • [Fork choice changes](specs/bellatrix/fork-choice.md)
  • Additions
    • [Honest validator guide changes](specs/bellatrix/validator.md)
    • [P2P networking](specs/bellatrix/p2p-interface.md)
| ### In-development Specifications diff --git a/setup.py b/setup.py index c8f0cea0c..5a88f7969 100644 --- a/setup.py +++ b/setup.py @@ -934,6 +934,7 @@ class PySpecCommand(Command): if self.spec_fork in (ALTAIR, BELLATRIX, CAPELLA, EIP4844): self.md_doc_paths += """ specs/altair/light-client/full-node.md + specs/altair/light-client/light-client.md specs/altair/light-client/sync-protocol.md specs/altair/beacon-chain.md specs/altair/bls.md diff --git a/specs/altair/light-client/light-client.md b/specs/altair/light-client/light-client.md new file mode 100644 index 000000000..78c02e205 --- /dev/null +++ b/specs/altair/light-client/light-client.md @@ -0,0 +1,37 @@ +# Altair Light Client -- Light Client + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + + +- [Introduction](#introduction) +- [Light client sync process](#light-client-sync-process) + + + + +## Introduction + +This document explains how light clients MAY obtain light client data to sync with the network. + +## Light client sync process + +1. The light client MUST be configured out-of-band with a spec/preset (including fork schedule), with `genesis_state` (including `genesis_time` and `genesis_validators_root`), and with a trusted block root. The trusted block SHOULD be within the weak subjectivity period, and its root SHOULD be from a finalized `Checkpoint`. + +2. The local clock is initialized based on the configured `genesis_time`, and the current fork digest is determined to browse for and connect to relevant light client data providers. + +3. The light client fetches a [`LightClientBootstrap`](./sync-protocol.md#lightclientbootstrap) object for the configured trusted block root. The `bootstrap` object is passed to [`initialize_light_client_store`](./sync-protocol.md#initialize_light_client_store) to obtain a local [`LightClientStore`](./sync-protocol.md#lightclientstore). + +4. The light client tracks the sync committee periods `finalized_period` from `store.finalized_header.slot`, `optimistic_period` from `store.optimistic_header.slot`, and `current_period` from `current_slot` based on the local clock. + + 1. When `finalized_period == optimistic_period` and [`is_next_sync_committee_known`](./sync-protocol.md#is_next_sync_committee_known) indicates `False`, the light client fetches a [`LightClientUpdate`](./sync-protocol.md#lightclientupdate) for `finalized_period`. If `finalized_period == current_period`, this fetch SHOULD be scheduled at a random time before `current_period` advances. + + 2. When `finalized_period + 1 < current_period`, the light client fetches a `LightClientUpdate` for each sync committee period in range `[finalized_period + 1, current_period)` (current period excluded) + + 3. When `finalized_period + 1 >= current_period`, the light client keeps observing [`LightClientFinalityUpdate`](./sync-protocol.md#lightclientfinalityupdate) and [`LightClientOptimisticUpdate`](./sync-protocol.md#lightclientoptimisticupdate). Received objects are passed to [`process_light_client_finality_update`](./sync-protocol.md#process_light_client_finality_update) and [`process_light_client_optimistic_update`](./sync-protocol.md#process_light_client_optimistic_update). This ensures that `finalized_header` and `optimistic_header` reflect the latest blocks. + +5. [`process_light_client_store_force_update`](./sync-protocol.md#process_light_client_store_force_update) MAY be called based on use case dependent heuristics if light client sync appears stuck. If available, falling back to an alternative syncing mechanism to cover the affected sync committee period is preferred. diff --git a/specs/altair/light-client/sync-protocol.md b/specs/altair/light-client/sync-protocol.md index 2fda56fc3..e385b5e44 100644 --- a/specs/altair/light-client/sync-protocol.md +++ b/specs/altair/light-client/sync-protocol.md @@ -51,6 +51,7 @@ uses sync committees introduced in [this beacon chain extension](./beacon-chain. Additional documents describe how the light client sync protocol can be used: - [Full node](./full-node.md) +- [Light client](./light-client.md) ## Constants From 8ced2c3ae10db0029968b569bf46f9081cf2fc04 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 22 Jul 2022 17:46:07 +0200 Subject: [PATCH 084/105] Remove newlines between items for visual consistency --- specs/altair/light-client/light-client.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/specs/altair/light-client/light-client.md b/specs/altair/light-client/light-client.md index 78c02e205..318950437 100644 --- a/specs/altair/light-client/light-client.md +++ b/specs/altair/light-client/light-client.md @@ -21,17 +21,10 @@ This document explains how light clients MAY obtain light client data to sync wi ## Light client sync process 1. The light client MUST be configured out-of-band with a spec/preset (including fork schedule), with `genesis_state` (including `genesis_time` and `genesis_validators_root`), and with a trusted block root. The trusted block SHOULD be within the weak subjectivity period, and its root SHOULD be from a finalized `Checkpoint`. - 2. The local clock is initialized based on the configured `genesis_time`, and the current fork digest is determined to browse for and connect to relevant light client data providers. - 3. The light client fetches a [`LightClientBootstrap`](./sync-protocol.md#lightclientbootstrap) object for the configured trusted block root. The `bootstrap` object is passed to [`initialize_light_client_store`](./sync-protocol.md#initialize_light_client_store) to obtain a local [`LightClientStore`](./sync-protocol.md#lightclientstore). - 4. The light client tracks the sync committee periods `finalized_period` from `store.finalized_header.slot`, `optimistic_period` from `store.optimistic_header.slot`, and `current_period` from `current_slot` based on the local clock. - 1. When `finalized_period == optimistic_period` and [`is_next_sync_committee_known`](./sync-protocol.md#is_next_sync_committee_known) indicates `False`, the light client fetches a [`LightClientUpdate`](./sync-protocol.md#lightclientupdate) for `finalized_period`. If `finalized_period == current_period`, this fetch SHOULD be scheduled at a random time before `current_period` advances. - 2. When `finalized_period + 1 < current_period`, the light client fetches a `LightClientUpdate` for each sync committee period in range `[finalized_period + 1, current_period)` (current period excluded) - 3. When `finalized_period + 1 >= current_period`, the light client keeps observing [`LightClientFinalityUpdate`](./sync-protocol.md#lightclientfinalityupdate) and [`LightClientOptimisticUpdate`](./sync-protocol.md#lightclientoptimisticupdate). Received objects are passed to [`process_light_client_finality_update`](./sync-protocol.md#process_light_client_finality_update) and [`process_light_client_optimistic_update`](./sync-protocol.md#process_light_client_optimistic_update). This ensures that `finalized_header` and `optimistic_header` reflect the latest blocks. - 5. [`process_light_client_store_force_update`](./sync-protocol.md#process_light_client_store_force_update) MAY be called based on use case dependent heuristics if light client sync appears stuck. If available, falling back to an alternative syncing mechanism to cover the affected sync committee period is preferred. From 56363cd94afe707591c9eb56f816019faf1961fb Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Tue, 3 May 2022 15:30:45 +0200 Subject: [PATCH 085/105] Define libp2p protocol for light client sync While the current Altair specs define structures to aid light client development, one missing key aspect is the network protocol definition. Certain implementations have started defining their own REST based APIs, e.g., Lodestar at https://github.com/ChainSafe/lodestar/blob/master/packages/api/src/routes/lightclient.ts While such APIs are useful, REST does not seem to be the ideomatic choice as the sole API available at such a low level for Ethereum. This patch introduces a libp2p based protocol to allow light clients to sync to the latest `BeaconBlockHeader` in a trustless and decentralized manner, building on top of prior work from: - @hwwhww at https://github.com/ethereum/consensus-specs/pull/2267 - @jinfwhuang at https://github.com/ethereum/consensus-specs/pull/2786 - Lodestar's REST API (also has an endpoint to fetch merkle proofs!) --- README.md | 2 +- setup.py | 1 + specs/altair/light-client/p2p-interface.md | 257 +++++++++++++++++++++ specs/altair/light-client/sync-protocol.md | 1 + 4 files changed, 260 insertions(+), 1 deletion(-) create mode 100644 specs/altair/light-client/p2p-interface.md diff --git a/README.md b/README.md index 3a4b2c8c5..56267c0db 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Features are researched and developed in parallel, and then consolidated into se | Seq. | Code Name | Fork Epoch | Specs | | - | - | - | - | | 0 | **Phase0** |`0` |
  • Core
    • [The beacon chain](specs/phase0/beacon-chain.md)
    • [Deposit contract](specs/phase0/deposit-contract.md)
    • [Beacon chain fork choice](specs/phase0/fork-choice.md)
  • Additions
    • [Honest validator guide](specs/phase0/validator.md)
    • [P2P networking](specs/phase0/p2p-interface.md)
    • [Weak subjectivity](specs/phase0/weak-subjectivity.md)
| -| 1 | **Altair** | `74240` |
  • Core
    • [Beacon chain changes](specs/altair/beacon-chain.md)
    • [Altair fork](specs/altair/fork.md)
  • Additions
    • [Light client sync protocol](specs/altair/light-client/sync-protocol.md) ([full node](specs/altair/light-client/full-node.md), [light client](specs/altair/light-client/light-client.md))
    • [Honest validator guide changes](specs/altair/validator.md)
    • [P2P networking](specs/altair/p2p-interface.md)
| +| 1 | **Altair** | `74240` |
  • Core
    • [Beacon chain changes](specs/altair/beacon-chain.md)
    • [Altair fork](specs/altair/fork.md)
  • Additions
    • [Light client sync protocol](specs/altair/light-client/sync-protocol.md) ([full node](specs/altair/light-client/full-node.md), [light client](specs/altair/light-client/light-client.md), [networking](specs/altair/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/altair/validator.md)
    • [P2P networking](specs/altair/p2p-interface.md)
| | 2 | **Bellatrix**
(["The Merge"](https://ethereum.org/en/upgrades/merge/)) | TBD |
  • Core
    • [Beacon Chain changes](specs/bellatrix/beacon-chain.md)
    • [Bellatrix fork](specs/bellatrix/fork.md)
    • [Fork choice changes](specs/bellatrix/fork-choice.md)
  • Additions
    • [Honest validator guide changes](specs/bellatrix/validator.md)
    • [P2P networking](specs/bellatrix/p2p-interface.md)
| ### In-development Specifications diff --git a/setup.py b/setup.py index 5a88f7969..484861b85 100644 --- a/setup.py +++ b/setup.py @@ -935,6 +935,7 @@ class PySpecCommand(Command): self.md_doc_paths += """ specs/altair/light-client/full-node.md specs/altair/light-client/light-client.md + specs/altair/light-client/p2p-interface.md specs/altair/light-client/sync-protocol.md specs/altair/beacon-chain.md specs/altair/bls.md diff --git a/specs/altair/light-client/p2p-interface.md b/specs/altair/light-client/p2p-interface.md new file mode 100644 index 000000000..a624c3f69 --- /dev/null +++ b/specs/altair/light-client/p2p-interface.md @@ -0,0 +1,257 @@ +# Altair Light Client -- Networking + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + + +- [Networking](#networking) + - [Configuration](#configuration) + - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) + - [Topics and messages](#topics-and-messages) + - [Global topics](#global-topics) + - [`light_client_finality_update`](#light_client_finality_update) + - [`light_client_optimistic_update`](#light_client_optimistic_update) + - [The Req/Resp domain](#the-reqresp-domain) + - [Messages](#messages) + - [GetLightClientBootstrap](#getlightclientbootstrap) + - [LightClientUpdatesByRange](#lightclientupdatesbyrange) + - [GetLightClientFinalityUpdate](#getlightclientfinalityupdate) + - [GetLightClientOptimisticUpdate](#getlightclientoptimisticupdate) +- [Light clients](#light-clients) +- [Validator assignments](#validator-assignments) + - [Beacon chain responsibilities](#beacon-chain-responsibilities) + - [Sync committee](#sync-committee) + + + + +## Networking + +This section extends the [networking specification for Altair](../p2p-interface.md) with additional messages, topics and data to the Req-Resp and Gossip domains. + +### Configuration + +| Name | Value | Description | +| - | - | - | +| `MAX_REQUEST_LIGHT_CLIENT_UPDATES` | `2**7` (= 128) | Maximum number of `LightClientUpdate` instances in a single request | + +### The gossip domain: gossipsub + +Gossip meshes are added to allow light clients stay in sync with the network. + +#### Topics and messages + +New global topics are added to provide light clients with the latest updates. + +| name | Message Type | +| - | - | +| `light_client_finality_update` | `LightClientFinalityUpdate` | +| `light_client_optimistic_update` | `LightClientOptimisticUpdate` | + +##### Global topics + +###### `light_client_finality_update` + +This topic is used to propagate the latest `LightClientFinalityUpdate` to light clients, allowing them to keep track of the latest `finalized_header`. + +The following validations MUST pass before forwarding the `finality_update` on the network. +- _[IGNORE]_ No other `finality_update` with a lower or equal `finalized_header.slot` was already forwarded on the network +- _[IGNORE]_ The `finality_update` is received after the block at `signature_slot` was given enough time to propagate through the network -- i.e. validate that one-third of `finality_update.signature_slot` has transpired (`SECONDS_PER_SLOT / INTERVALS_PER_SLOT` seconds after the start of the slot, with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) + +For full nodes, the following validations MUST additionally pass before forwarding the `finality_update` on the network. +- _[IGNORE]_ The received `finality_update` matches the locally computed one exactly (as defined in [`create_light_client_finality_update`](./full-node.md#create_light_client_finality_update)) + +For light clients, the following validations MUST additionally pass before forwarding the `finality_update` on the network. +- _[REJECT]_ The `finality_update` is valid -- i.e. validate that `process_light_client_finality_update` does not indicate errors +- _[IGNORE]_ The `finality_update` advances the `finalized_header` of the local `LightClientStore` -- i.e. validate that processing `finality_update` increases `store.finalized_header.slot` + +Light clients SHOULD call `process_light_client_finality_update` even if the message is ignored. + +###### `light_client_optimistic_update` + +This topic is used to propagate the latest `LightClientOptimisticUpdate` to light clients, allowing them to keep track of the latest `optimistic_header`. + +The following validations MUST pass before forwarding the `optimistic_update` on the network. +- _[IGNORE]_ No other `optimistic_update` with a lower or equal `attested_header.slot` was already forwarded on the network +- _[IGNORE]_ The `optimistic_update` is received after the block at `signature_slot` was given enough time to propagate through the network -- i.e. validate that one-third of `optimistic_update.signature_slot` has transpired (`SECONDS_PER_SLOT / INTERVALS_PER_SLOT` seconds after the start of the slot, with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) + +For full nodes, the following validations MUST additionally pass before forwarding the `optimistic_update` on the network. +- _[IGNORE]_ The received `optimistic_update` matches the locally computed one exactly (as defined in [`create_light_client_optimistic_update`](./full-node.md#create_light_client_optimistic_update)) + +For light clients, the following validations MUST additionally pass before forwarding the `optimistic_update` on the network. +- _[REJECT]_ The `optimistic_update` is valid -- i.e. validate that `process_light_client_optimistic_update` does not indicate errors +- _[IGNORE]_ The `optimistic_update` either matches corresponding fields of the most recently forwarded `LightClientFinalityUpdate` (if any), or it advances the `optimistic_header` of the local `LightClientStore` -- i.e. validate that processing `optimistic_update` increases `store.optimistic_header.slot` + +Light clients SHOULD call `process_light_client_optimistic_update` even if the message is ignored. + +### The Req/Resp domain + +#### Messages + +##### GetLightClientBootstrap + +**Protocol ID:** `/eth2/beacon_chain/req/light_client_bootstrap/1/` + +Request Content: + +``` +( + Root +) +``` + +Response Content: + +``` +( + LightClientBootstrap +) +``` + +Requests the `LightClientBootstrap` structure corresponding to a given post-Altair beacon block root. + +The request MUST be encoded as an SSZ-field. + +Peers SHOULD provide results as defined in [`create_light_client_bootstrap`](./full-node.md#create_light_client_bootstrap). To fulfill a request, the requested block's post state needs to be known. + +When a `LightClientBootstrap` instance cannot be produced for a given block root, peers SHOULD respond with error code `3: ResourceUnavailable`. + +A `ForkDigest`-context based on `compute_fork_version(compute_epoch_at_slot(bootstrap.header.slot))` is used to select the fork namespace of the Response type. + +Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: + +[0]: # (eth2spec: skip) + +| `fork_version` | Response SSZ type | +| ------------------------------- | ------------------------------------ | +| `GENESIS_FORK_VERSION` | n/a | +| `ALTAIR_FORK_VERSION` and later | `altair.LightClientBootstrap` | + +##### LightClientUpdatesByRange + +**Protocol ID:** `/eth2/beacon_chain/req/light_client_updates_by_range/1/` + +Request Content: +``` +( + start_period: uint64 + count: uint64 +) +``` + +Response Content: +``` +( + List[LightClientUpdate, MAX_REQUEST_LIGHT_CLIENT_UPDATES] +) +``` + +Requests the `LightClientUpdate` instances in the sync committee period range `[start_period, start_period + count)`, leading up to the current head sync committee period as selected by fork choice. + +The request MUST be encoded as an SSZ-container. + +The response MUST consist of zero or more `response_chunk`. Each _successful_ `response_chunk` MUST contain a single `LightClientUpdate` payload. + +Peers SHOULD provide results as defined in [`create_light_client_update`](./full-node.md#create_light_client_update). They MUST respond with at least the earliest known result within the requested range, and MUST send results in consecutive order (by period). The response MUST NOT contain more than `min(MAX_REQUEST_LIGHT_CLIENT_UPDATES, count)` results. + +For each `response_chunk`, a `ForkDigest`-context based on `compute_fork_version(compute_epoch_at_slot(update.attested_header.slot))` is used to select the fork namespace of the Response type. Note that this `fork_version` may be different from the one used to verify the `update.sync_aggregate`, which is based on `update.signature_slot`. + +Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: + +[0]: # (eth2spec: skip) + +| `fork_version` | Response chunk SSZ type | +| ------------------------------- | ------------------------------------ | +| `GENESIS_FORK_VERSION` | n/a | +| `ALTAIR_FORK_VERSION` and later | `altair.LightClientUpdate` | + +##### GetLightClientFinalityUpdate + +**Protocol ID:** `/eth2/beacon_chain/req/light_client_finality_update/1/` + +No Request Content. + +Response Content: + +``` +( + LightClientFinalityUpdate +) +``` + +Requests the latest `LightClientFinalityUpdate` known by a peer. + +Peers SHOULD provide results as defined in [`create_light_client_finality_update`](./full-node.md#create_light_client_finality_update). + +When no `LightClientFinalityUpdate` is available, peers SHOULD respond with error code `3: ResourceUnavailable`. + +A `ForkDigest`-context based on `compute_fork_version(compute_epoch_at_slot(finality_update.attested_header.slot))` is used to select the fork namespace of the Response type. Note that this `fork_version` may be different from the one used to verify the `finality_update.sync_aggregate`, which is based on `finality_update.signature_slot`. + +Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: + +[0]: # (eth2spec: skip) + +| `fork_version` | Response SSZ type | +| ------------------------------- | ------------------------------------ | +| `GENESIS_FORK_VERSION` | n/a | +| `ALTAIR_FORK_VERSION` and later | `altair.LightClientFinalityUpdate` | + +##### GetLightClientOptimisticUpdate + +**Protocol ID:** `/eth2/beacon_chain/req/light_client_optimistic_update/1/` + +No Request Content. + +Response Content: + +``` +( + LightClientOptimisticUpdate +) +``` + +Requests the latest `LightClientOptimisticUpdate` known by a peer. + +Peers SHOULD provide results as defined in [`create_light_client_optimistic_update`](./full-node.md#create_light_client_optimistic_update). + +When no `LightClientOptimisticUpdate` is available, peers SHOULD respond with error code `3: ResourceUnavailable`. + +A `ForkDigest`-context based on `compute_fork_version(compute_epoch_at_slot(optimistic_update.attested_header.slot))` is used to select the fork namespace of the Response type. Note that this `fork_version` may be different from the one used to verify the `optimistic_update.sync_aggregate`, which is based on `optimistic_update.signature_slot`. + +Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: + +[0]: # (eth2spec: skip) + +| `fork_version` | Response SSZ type | +| ------------------------------- | ------------------------------------ | +| `GENESIS_FORK_VERSION` | n/a | +| `ALTAIR_FORK_VERSION` and later | `altair.LightClientOptimisticUpdate` | + +## Light clients + +Light clients using libp2p to stay in sync with the network SHOULD subscribe to the [`light_client_finality_update`](#light_client_finality_update) and [`light_client_optimistic_update`](#light_client_optimistic_update) pubsub topics and validate all received messages while the [light client sync process](./light-client.md#light-client-sync-process) supports processing `LightClientFinalityUpdate` and `LightClientOptimistic` structures. + +Light clients MAY also collect historic light client data and make it available to other peers. If they do, they SHOULD advertise supported message endpoints in [the Req/Resp domain](#the-reqresp-domain), and MAY also update the contents of their [`Status`](../../phase0/p2p-interface.md#status) message to reflect the locally available light client data. + +If only limited light client data is locally available, the light client SHOULD use data based on `genesis_block` and `GENESIS_SLOT` in its `Status` message. Hybrid peers that also implement full node functionality MUST only incorporate data based on their full node sync progress into their `Status` message. + +## Validator assignments + +This section extends the [honest validator specification](../validator.md) with additional responsibilities to enable light clients to sync with the network. + +### Beacon chain responsibilities + +All full nodes SHOULD subscribe to and provide stability on the [`light_client_finality_update`](#light_client_finality_update) and [`light_client_optimistic_update`](#light_client_optimistic_update) pubsub topics by validating all received messages. + +### Sync committee + +Whenever fork choice selects a new head block with a sync aggregate participation `>= MIN_SYNC_COMMITTEE_PARTICIPANTS` and a post-Altair parent block, full nodes with at least one validator assigned to the current sync committee at the block's `slot` SHOULD broadcast derived light client data as follows: + +- If `finalized_header.slot` increased, a `LightClientFinalityUpdate` SHOULD be broadcasted to the pubsub topic `light_client_finality_update` if no matching message has not yet been forwarded as part of gossip validation. +- If `attested_header.slot` increased, a `LightClientOptimisticUpdate` SHOULD be broadcasted to the pubsub topic `light_client_optimistic_update` if no matching message has not yet been forwarded as part of gossip validation. + +These messages SHOULD be broadcasted after one-third of `slot` has transpired (`SECONDS_PER_SLOT / INTERVALS_PER_SLOT` seconds after the start of the slot). To ensure that the corresponding block was given enough time to propagate through the network, they SHOULD NOT be sent earlier. Note that this is different from how other messages are handled, e.g., attestations, which may be sent early. diff --git a/specs/altair/light-client/sync-protocol.md b/specs/altair/light-client/sync-protocol.md index e385b5e44..39a0b3d11 100644 --- a/specs/altair/light-client/sync-protocol.md +++ b/specs/altair/light-client/sync-protocol.md @@ -52,6 +52,7 @@ uses sync committees introduced in [this beacon chain extension](./beacon-chain. Additional documents describe how the light client sync protocol can be used: - [Full node](./full-node.md) - [Light client](./light-client.md) +- [Networking](./p2p-interface.md) ## Constants From 7e54cd09b03b8fe047f4faa3af7f39c07668a1a2 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Fri, 22 Jul 2022 13:15:46 -0600 Subject: [PATCH 086/105] Update README.md --- tests/formats/transition/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/formats/transition/README.md b/tests/formats/transition/README.md index 37df65539..41a5bc01d 100644 --- a/tests/formats/transition/README.md +++ b/tests/formats/transition/README.md @@ -8,6 +8,7 @@ Clients should assume forks happen sequentially in the following manner: 0. `phase0` 1. `altair` +2. `bellatrix` For example, if a test case has `post_fork` of `altair`, the test consumer should assume the test begins in `phase0` and use that specification to process the initial state and any blocks up until the fork epoch. After the fork happens, the test consumer should use the specification according to the `altair` fork to process the remaining data. From 52a741f7c6d3bec98e04df3441bc8e7681480877 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Mon, 25 Jul 2022 15:46:29 +0200 Subject: [PATCH 087/105] Improve grammar Co-authored-by: Hsiao-Wei Wang --- specs/altair/light-client/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/altair/light-client/p2p-interface.md b/specs/altair/light-client/p2p-interface.md index a624c3f69..5c2b27b22 100644 --- a/specs/altair/light-client/p2p-interface.md +++ b/specs/altair/light-client/p2p-interface.md @@ -41,7 +41,7 @@ This section extends the [networking specification for Altair](../p2p-interface. ### The gossip domain: gossipsub -Gossip meshes are added to allow light clients stay in sync with the network. +Gossip meshes are added to allow light clients to stay in sync with the network. #### Topics and messages From 2c32fa64739a338a2de621b18b92808ec20a2b73 Mon Sep 17 00:00:00 2001 From: Potuz Date: Tue, 26 Jul 2022 17:34:02 -0300 Subject: [PATCH 088/105] Do not overload index with WithdrawalIndex and ValidatorIndex --- specs/capella/beacon-chain.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/capella/beacon-chain.md b/specs/capella/beacon-chain.md index 06659349a..40592a9bd 100644 --- a/specs/capella/beacon-chain.md +++ b/specs/capella/beacon-chain.md @@ -269,13 +269,13 @@ class BeaconState(Container): #### `withdraw` ```python -def withdraw_balance(state: BeaconState, index: ValidatorIndex, amount: Gwei) -> None: +def withdraw_balance(state: BeaconState, validator_index: ValidatorIndex, amount: Gwei) -> None: # Decrease the validator's balance - decrease_balance(state, index, amount) + decrease_balance(state, validator_index, amount) # Create a corresponding withdrawal receipt withdrawal = Withdrawal( index=state.next_withdrawal_index, - address=ExecutionAddress(state.validators[index].withdrawal_credentials[12:]), + address=ExecutionAddress(state.validators[validator_index].withdrawal_credentials[12:]), amount=amount, ) state.next_withdrawal_index = WithdrawalIndex(state.next_withdrawal_index + 1) From 4893f9efc912bd3fb74e4251ddbe012bc4f92a41 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Wed, 27 Jul 2022 16:10:46 +0600 Subject: [PATCH 089/105] Handling `latestValidHash` in opti-sync --- sync/optimistic.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index c2f06a9be..4c6271059 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -163,6 +163,24 @@ the merge block MUST be treated the same as an `INVALIDATED` block (i.e., it and all its descendants are invalidated and removed from the block tree). +### How to apply `latestValidHash` when payload status is `INVALID` + +Processing the `INVALID` payload status by consensus engine +depends on the value returned in `latestValidHash` parameter. +The general approach is as follows: +1. Consensus engine MUST identify `invalidBlock` as per definition in the table below +2. `invalidBlock` and all its descendants MUST be transitioned from `NOT_VALIDATED` to `INVALIDATED` + +| `latestValidHash` | `invalidBlock` | +|:- |:- | +| Execution block hash | The *child* of a block with `body.execution_payload.block_hash == latestValidHash` in the chain containing a block with payload in question | +| `0x00..00` (all zeroes) | The first block with `body.execution_payload != ExecutionPayload()` in the chain containing a block with payload in question | +| `null` | Block with payload in question | + +When `latestValidHash` is a meaningful execution block hash but consensus engine +can't find a block satisfying `body.execution_payload.block_hash == latestValidHash`, +consensus engine SHOULD behave the same as if `latestValidHash` was `null`. + ### Execution Engine Errors When an execution engine returns an error or fails to respond to a payload From e989ca29dfb5d4d5a6e85c7e16474c69d28f99c1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 28 Jul 2022 00:48:36 +0800 Subject: [PATCH 090/105] Add ToC --- sync/optimistic.md | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 4c6271059..bed41d8dd 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -1,5 +1,37 @@ # Optimistic Sync +## Table of contents + + + + +- [Introduction](#introduction) +- [Constants](#constants) +- [Helpers](#helpers) +- [Mechanisms](#mechanisms) + - [When to optimistically import blocks](#when-to-optimistically-import-blocks) + - [How to optimistically import blocks](#how-to-optimistically-import-blocks) + - [How to apply `latestValidHash` when payload status is `INVALID`](#how-to-apply-latestvalidhash-when-payload-status-is-invalid) + - [Execution Engine Errors](#execution-engine-errors) + - [Assumptions about Execution Engine Behaviour](#assumptions-about-execution-engine-behaviour) + - [Re-Orgs](#re-orgs) +- [Fork Choice](#fork-choice) + - [Fork Choice Poisoning](#fork-choice-poisoning) +- [Checkpoint Sync (Weak Subjectivity Sync)](#checkpoint-sync-weak-subjectivity-sync) +- [Validator assignments](#validator-assignments) + - [Block Production](#block-production) + - [Attesting](#attesting) + - [Participating in Sync Committees](#participating-in-sync-committees) +- [Ethereum Beacon APIs](#ethereum-beacon-apis) +- [Design Decision Rationale](#design-decision-rationale) + - [Why `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`?](#why-safe_slots_to_import_optimistically) + - [Transitioning from VALID -> INVALIDATED or INVALIDATED -> VALID](#transitioning-from-valid---invalidated-or-invalidated---valid) + - [What about Light Clients?](#what-about-light-clients) + - [What if `TERMINAL_BLOCK_HASH` is used?](#what-if-terminal_block_hash-is-used) + + + + ## Introduction In order to provide a syncing execution engine with a partial view of the head @@ -346,7 +378,7 @@ specification since it's only possible with a faulty EE. Such a scenario requires manual intervention. -## What about Light Clients? +### What about Light Clients? An alternative to optimistic sync is to run a light client inside/alongside beacon nodes that mitigates the need for optimistic sync by providing @@ -358,7 +390,7 @@ A notable thing about optimistic sync is that it's *optional*. Should an implementation decide to go the light-client route, then they can just ignore optimistic sync all together. -## What if `TERMINAL_BLOCK_HASH` is used? +### What if `TERMINAL_BLOCK_HASH` is used? If the terminal block hash override is used (i.e., `TERMINAL_BLOCK_HASH != Hash32()`), the [`validate_merge_block`](../specs/bellatrix/fork-choice.md#validate_merge_block) From 29b3afeff3d2dba3b61f1fe9a53732a7d7734e39 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Thu, 28 Jul 2022 16:21:33 +0600 Subject: [PATCH 091/105] Elaborate on why sync optimistically --- sync/optimistic.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index c2f06a9be..6af886814 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -270,6 +270,23 @@ optimistic blocks (and vice-versa). ## Design Decision Rationale +### Why sync optimistically? + +Most of execution engines use state sync as a default sync mechanism on Ethereum Mainnet +because executing blocks from genesis takes several weeks on commodity hardware. + +State sync requires the knowledge of the current head of the chain to eventually converge. +If not constantly fed with the most recent head, state sync won't be able to complete +because the recent state soon becomes unavailable due to state trie pruning. + +Optimistic block import (i.e. import without waiting for a payload to be executed by the engine) +breaks a deadlock between the state sync and processing a beacon block by skipping payload +execution as long as execution engine is syncing. + +Optimistic sync works nicely with execution engines using block execution as a default +sync mechanism (e.g. Erigon). It makes optimistic sync a *generalized* solution for +interaction between consensus and execution engine during the sync process. + ### Why `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`? Nodes can only import an optimistic block if their justified checkpoint is From c51e67a0590d6e88f3d8f513a3cb51c0d6a18a5e Mon Sep 17 00:00:00 2001 From: David Theodore Date: Fri, 29 Jul 2022 00:13:32 -0700 Subject: [PATCH 092/105] fixed typo in on_block() definition --- specs/bellatrix/fork-choice.md | 2 +- specs/phase0/fork-choice.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/bellatrix/fork-choice.md b/specs/bellatrix/fork-choice.md index 60a54da9c..312768e44 100644 --- a/specs/bellatrix/fork-choice.md +++ b/specs/bellatrix/fork-choice.md @@ -165,7 +165,7 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: assert block.parent_root in store.block_states # Make a copy of the state to avoid mutability issues pre_state = copy(store.block_states[block.parent_root]) - # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. + # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past. assert get_current_slot(store) >= block.slot # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index 1593e07fe..661ad613b 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -406,7 +406,7 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: assert block.parent_root in store.block_states # Make a copy of the state to avoid mutability issues pre_state = copy(store.block_states[block.parent_root]) - # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. + # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past. assert get_current_slot(store) >= block.slot # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) From 47ee4308fc9e5e7f44bae7b92787b337c29642d5 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Fri, 29 Jul 2022 17:31:32 +0600 Subject: [PATCH 093/105] Apply suggestions from code review Co-authored-by: Hsiao-Wei Wang --- sync/optimistic.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 6af886814..6d8f67e42 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -272,10 +272,10 @@ optimistic blocks (and vice-versa). ### Why sync optimistically? -Most of execution engines use state sync as a default sync mechanism on Ethereum Mainnet +Most execution engines use state sync as a default sync mechanism on Ethereum Mainnet because executing blocks from genesis takes several weeks on commodity hardware. -State sync requires the knowledge of the current head of the chain to eventually converge. +State sync requires the knowledge of the current head of the chain to converge eventually. If not constantly fed with the most recent head, state sync won't be able to complete because the recent state soon becomes unavailable due to state trie pruning. From 8d70a73cac604aebdf7b23c9c3e067bd3fb9cf3a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 1 Aug 2022 09:26:18 -0600 Subject: [PATCH 094/105] Apply suggestions from code review --- sync/optimistic.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index bed41d8dd..8af450b33 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -197,20 +197,19 @@ removed from the block tree). ### How to apply `latestValidHash` when payload status is `INVALID` -Processing the `INVALID` payload status by consensus engine -depends on the value returned in `latestValidHash` parameter. +Processing an `INVALID` payload status depends on the `latestValidHash` parameter. The general approach is as follows: -1. Consensus engine MUST identify `invalidBlock` as per definition in the table below -2. `invalidBlock` and all its descendants MUST be transitioned from `NOT_VALIDATED` to `INVALIDATED` +1. Consensus engine MUST identify `invalidBlock` as per definition in the table below. +2. `invalidBlock` and all of its descendants MUST be transitioned from `NOT_VALIDATED` to `INVALIDATED`. | `latestValidHash` | `invalidBlock` | |:- |:- | -| Execution block hash | The *child* of a block with `body.execution_payload.block_hash == latestValidHash` in the chain containing a block with payload in question | +| Execution block hash | The *child* of a block with `body.execution_payload.block_hash == latestValidHash` in the chain containing the block with payload in question | | `0x00..00` (all zeroes) | The first block with `body.execution_payload != ExecutionPayload()` in the chain containing a block with payload in question | | `null` | Block with payload in question | When `latestValidHash` is a meaningful execution block hash but consensus engine -can't find a block satisfying `body.execution_payload.block_hash == latestValidHash`, +cannot find a block satisfying `body.execution_payload.block_hash == latestValidHash`, consensus engine SHOULD behave the same as if `latestValidHash` was `null`. ### Execution Engine Errors From 7e360240f986beec8405646a89f71c9bfdfef4ac Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 1 Aug 2022 11:40:37 -0600 Subject: [PATCH 095/105] more execution payload tests and cleanup old ones --- specs/bellatrix/beacon-chain.md | 2 - .../test_process_execution_payload.py | 288 +++++++++++++----- 2 files changed, 206 insertions(+), 84 deletions(-) diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index 983ecda23..9285594ab 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -298,8 +298,6 @@ def slash_validator(state: BeaconState, increase_balance(state, whistleblower_index, Gwei(whistleblower_reward - proposer_reward)) ``` - - ## Beacon chain state transition function ### Execution engine diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py index cd7a74259..9dc701f7d 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py @@ -1,3 +1,6 @@ +import os +from random import Random + from eth2spec.test.helpers.execution_payload import ( build_empty_execution_payload, get_execution_payload_header, @@ -46,14 +49,8 @@ def run_execution_payload_processing(spec, state, execution_payload, valid=True, assert state.latest_execution_payload_header == get_execution_payload_header(spec, execution_payload) -@with_bellatrix_and_later -@spec_state_test -def test_success_first_payload(spec, state): - # pre-state - state = build_state_with_incomplete_transition(spec, state) +def run_success_test(spec, state): next_slot(spec, state) - - # execution payload execution_payload = build_empty_execution_payload(spec, state) yield from run_execution_payload_processing(spec, state, execution_payload) @@ -61,12 +58,25 @@ def test_success_first_payload(spec, state): @with_bellatrix_and_later @spec_state_test -def test_success_regular_payload(spec, state): - # pre-state - state = build_state_with_complete_transition(spec, state) - next_slot(spec, state) +def test_success_first_payload(spec, state): + state = build_state_with_incomplete_transition(spec, state) + assert not spec.is_merge_transition_complete(state) - # execution payload + yield from run_success_test(spec, state) + + +@with_bellatrix_and_later +@spec_state_test +def test_success_regular_payload(spec, state): + state = build_state_with_complete_transition(spec, state) + assert spec.is_merge_transition_complete(state) + + yield from run_success_test(spec, state) + + +def run_gap_slot_test(spec, state): + next_slot(spec, state) + next_slot(spec, state) execution_payload = build_empty_execution_payload(spec, state) yield from run_execution_payload_processing(spec, state, execution_payload) @@ -75,83 +85,66 @@ def test_success_regular_payload(spec, state): @with_bellatrix_and_later @spec_state_test def test_success_first_payload_with_gap_slot(spec, state): - # pre-state state = build_state_with_incomplete_transition(spec, state) - next_slot(spec, state) - next_slot(spec, state) - - # execution payload - execution_payload = build_empty_execution_payload(spec, state) - - yield from run_execution_payload_processing(spec, state, execution_payload) + yield from run_gap_slot_test(spec, state) @with_bellatrix_and_later @spec_state_test def test_success_regular_payload_with_gap_slot(spec, state): - # pre-state state = build_state_with_complete_transition(spec, state) - next_slot(spec, state) - next_slot(spec, state) + yield from run_gap_slot_test(spec, state) - # execution payload + +def run_bad_execution_test(spec, state): + # completely valid payload, but execution itself fails (e.g. block exceeds gas limit) + next_slot(spec, state) execution_payload = build_empty_execution_payload(spec, state) - yield from run_execution_payload_processing(spec, state, execution_payload) + yield from run_execution_payload_processing(spec, state, execution_payload, valid=False, execution_valid=False) @with_bellatrix_and_later @spec_state_test def test_bad_execution_first_payload(spec, state): - # completely valid payload, but execution itself fails (e.g. block exceeds gas limit) - - # pre-state state = build_state_with_incomplete_transition(spec, state) - next_slot(spec, state) - - # execution payload - execution_payload = build_empty_execution_payload(spec, state) - - yield from run_execution_payload_processing(spec, state, execution_payload, valid=False, execution_valid=False) + yield from run_bad_execution_test(spec, state) @with_bellatrix_and_later @spec_state_test def test_bad_execution_regular_payload(spec, state): - # completely valid payload, but execution itself fails (e.g. block exceeds gas limit) - - # pre-state state = build_state_with_complete_transition(spec, state) + yield from run_bad_execution_test(spec, state) + + +@with_bellatrix_and_later +@spec_state_test +def test_bad_parent_hash_first_payload(spec, state): + state = build_state_with_incomplete_transition(spec, state) next_slot(spec, state) - # execution payload execution_payload = build_empty_execution_payload(spec, state) + execution_payload.parent_hash = b'\x55' * 32 - yield from run_execution_payload_processing(spec, state, execution_payload, valid=False, execution_valid=False) + yield from run_execution_payload_processing(spec, state, execution_payload, valid=True) @with_bellatrix_and_later @spec_state_test def test_bad_parent_hash_regular_payload(spec, state): - # pre-state state = build_state_with_complete_transition(spec, state) next_slot(spec, state) - # execution payload execution_payload = build_empty_execution_payload(spec, state) execution_payload.parent_hash = spec.Hash32() yield from run_execution_payload_processing(spec, state, execution_payload, valid=False) -@with_bellatrix_and_later -@spec_state_test -def test_bad_random_first_payload(spec, state): - # pre-state - state = build_state_with_incomplete_transition(spec, state) +def run_bad_prev_randao_test(spec, state): next_slot(spec, state) - # execution payload execution_payload = build_empty_execution_payload(spec, state) execution_payload.prev_randao = b'\x42' * 32 @@ -160,26 +153,21 @@ def test_bad_random_first_payload(spec, state): @with_bellatrix_and_later @spec_state_test -def test_bad_random_regular_payload(spec, state): - # pre-state - state = build_state_with_complete_transition(spec, state) - next_slot(spec, state) - - # execution payload - execution_payload = build_empty_execution_payload(spec, state) - execution_payload.prev_randao = b'\x04' * 32 - - yield from run_execution_payload_processing(spec, state, execution_payload, valid=False) +def test_bad_prev_randao_first_payload(spec, state): + state = build_state_with_incomplete_transition(spec, state) + yield from run_bad_prev_randao_test(spec, state) @with_bellatrix_and_later @spec_state_test -def test_bad_everything_regular_payload(spec, state): - # pre-state +def test_bad_pre_randao_regular_payload(spec, state): state = build_state_with_complete_transition(spec, state) + yield from run_bad_prev_randao_test(spec, state) + + +def run_bad_everything_test(spec, state): next_slot(spec, state) - # execution payload execution_payload = build_empty_execution_payload(spec, state) execution_payload.parent_hash = spec.Hash32() execution_payload.prev_randao = spec.Bytes32() @@ -190,59 +178,195 @@ def test_bad_everything_regular_payload(spec, state): @with_bellatrix_and_later @spec_state_test -def test_bad_timestamp_first_payload(spec, state): - # pre-state +def test_bad_everything_first_payload(spec, state): state = build_state_with_incomplete_transition(spec, state) + yield from run_bad_everything_test(spec, state) + + +@with_bellatrix_and_later +@spec_state_test +def test_bad_everything_regular_payload(spec, state): + state = build_state_with_complete_transition(spec, state) + yield from run_bad_everything_test(spec, state) + + +def run_bad_timestamp_test(spec, state, is_future): next_slot(spec, state) # execution payload execution_payload = build_empty_execution_payload(spec, state) - execution_payload.timestamp = execution_payload.timestamp + 1 + if is_future: + timestamp = execution_payload.timestamp + 1 + else: + timestamp = execution_payload.timestamp - 1 + execution_payload.timestamp = timestamp yield from run_execution_payload_processing(spec, state, execution_payload, valid=False) @with_bellatrix_and_later @spec_state_test -def test_bad_timestamp_regular_payload(spec, state): - # pre-state +def test_future_timestamp_first_payload(spec, state): + state = build_state_with_incomplete_transition(spec, state) + yield from run_bad_timestamp_test(spec, state, is_future=True) + + +@with_bellatrix_and_later +@spec_state_test +def test_future_timestamp_regular_payload(spec, state): state = build_state_with_complete_transition(spec, state) + yield from run_bad_timestamp_test(spec, state, is_future=True) + + +@with_bellatrix_and_later +@spec_state_test +def test_past_timestamp_first_payload(spec, state): + state = build_state_with_incomplete_transition(spec, state) + yield from run_bad_timestamp_test(spec, state, is_future=False) + + +@with_bellatrix_and_later +@spec_state_test +def test_past_timestamp_regular_payload(spec, state): + state = build_state_with_complete_transition(spec, state) + yield from run_bad_timestamp_test(spec, state, is_future=False) + + +def run_non_empty_extra_data_test(spec, state): next_slot(spec, state) - # execution payload execution_payload = build_empty_execution_payload(spec, state) - execution_payload.timestamp = execution_payload.timestamp + 1 + execution_payload.extra_data = b'\x45' * 12 - yield from run_execution_payload_processing(spec, state, execution_payload, valid=False) + yield from run_execution_payload_processing(spec, state, execution_payload) + assert state.latest_execution_payload_header.extra_data == execution_payload.extra_data @with_bellatrix_and_later @spec_state_test def test_non_empty_extra_data_first_payload(spec, state): - # pre-state state = build_state_with_incomplete_transition(spec, state) - next_slot(spec, state) - - # execution payload - execution_payload = build_empty_execution_payload(spec, state) - execution_payload.extra_data = b'\x45' * 12 - - yield from run_execution_payload_processing(spec, state, execution_payload) - - assert state.latest_execution_payload_header.extra_data == execution_payload.extra_data + yield from run_non_empty_extra_data_test(spec, state) @with_bellatrix_and_later @spec_state_test def test_non_empty_extra_data_regular_payload(spec, state): - # pre-state state = build_state_with_complete_transition(spec, state) + yield from run_non_empty_extra_data_test(spec, state) + + +def run_non_empty_transactions_test(spec, state): next_slot(spec, state) - # execution payload execution_payload = build_empty_execution_payload(spec, state) - execution_payload.extra_data = b'\x45' * 12 + num_transactions = 2 + execution_payload.transactions = [ + spec.Transaction(b'\x99' * 128) + for _ in range(num_transactions) + ] yield from run_execution_payload_processing(spec, state, execution_payload) + assert state.latest_execution_payload_header.transactions_root == execution_payload.transactions.hash_tree_root() - assert state.latest_execution_payload_header.extra_data == execution_payload.extra_data + +@with_bellatrix_and_later +@spec_state_test +def test_non_empty_transactions_first_payload(spec, state): + state = build_state_with_incomplete_transition(spec, state) + yield from run_non_empty_extra_data_test(spec, state) + + +@with_bellatrix_and_later +@spec_state_test +def test_non_empty_transactions_regular_payload(spec, state): + state = build_state_with_complete_transition(spec, state) + yield from run_non_empty_extra_data_test(spec, state) + + +def run_zero_length_transaction_test(spec, state): + next_slot(spec, state) + + execution_payload = build_empty_execution_payload(spec, state) + execution_payload.transactions = [spec.Transaction(b'')] + assert len(execution_payload.transactions[0]) == 0 + + yield from run_execution_payload_processing(spec, state, execution_payload) + assert state.latest_execution_payload_header.transactions_root == execution_payload.transactions.hash_tree_root() + + +@with_bellatrix_and_later +@spec_state_test +def test_zero_length_transaction_first_payload(spec, state): + state = build_state_with_incomplete_transition(spec, state) + yield from run_zero_length_transaction_test(spec, state) + + +@with_bellatrix_and_later +@spec_state_test +def test_zero_length_transaction_regular_payload(spec, state): + state = build_state_with_complete_transition(spec, state) + yield from run_zero_length_transaction_test(spec, state) + + +def build_randomized_execution_payload(spec, state, rng): + execution_payload = build_empty_execution_payload(spec, state) + execution_payload.fee_recipient = spec.ExecutionAddress(os.urandom(20)) + execution_payload.state_root = spec.Bytes32(os.urandom(32)) + execution_payload.receipts_root = spec.Bytes32(os.urandom(32)) + execution_payload.logs_bloom = spec.ByteVector[spec.BYTES_PER_LOGS_BLOOM](os.urandom(spec.BYTES_PER_LOGS_BLOOM)) + execution_payload.block_number = rng.randint(0, 10e10) + execution_payload.gas_limit = rng.randint(0, 10e10) + execution_payload.gas_used = rng.randint(0, 10e10) + execution_payload.extra_data = spec.ByteList[spec.MAX_EXTRA_DATA_BYTES]( + os.urandom(rng.randint(0, spec.MAX_EXTRA_DATA_BYTES)) + ) + execution_payload.base_fee_per_gas = rng.randint(0, 2**256 - 1) + execution_payload.block_hash = spec.Hash32(os.urandom(32)) + + num_transactions = rng.randint(0, 100) + execution_payload.transactions = [ + spec.Transaction(os.urandom(rng.randint(0, 1000))) + for _ in range(num_transactions) + ] + + return execution_payload + + +def run_randomized_non_validated_execution_fields_test(spec, state, execution_valid=True, rng=Random(5555)): + next_slot(spec, state) + execution_payload = build_randomized_execution_payload(spec, state, rng) + + yield from run_execution_payload_processing( + spec, state, + execution_payload, + valid=execution_valid, execution_valid=execution_valid + ) + + +@with_bellatrix_and_later +@spec_state_test +def test_randomized_non_validated_execution_fields_first_payload__valid(spec, state): + state = build_state_with_incomplete_transition(spec, state) + yield from run_randomized_non_validated_execution_fields_test(spec, state) + + +@with_bellatrix_and_later +@spec_state_test +def test_randomized_non_validated_execution_fields_regular_payload__valid(spec, state): + state = build_state_with_complete_transition(spec, state) + yield from run_randomized_non_validated_execution_fields_test(spec, state) + + +@with_bellatrix_and_later +@spec_state_test +def test_randomized_non_validated_execution_fields_first_payload__invalid(spec, state): + state = build_state_with_incomplete_transition(spec, state) + yield from run_randomized_non_validated_execution_fields_test(spec, state, execution_valid=False) + + +@with_bellatrix_and_later +@spec_state_test +def test_randomized_non_validated_execution_fields_regular_payload__invalid(spec, state): + state = build_state_with_complete_transition(spec, state) + yield from run_randomized_non_validated_execution_fields_test(spec, state, execution_valid=False) From a3b7c6c5055fccdf529145f63946a75108910001 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Tue, 2 Aug 2022 12:25:15 +0600 Subject: [PATCH 096/105] Apply suggestions from code review Co-authored-by: Danny Ryan --- sync/optimistic.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 6d8f67e42..1511ce859 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -279,9 +279,9 @@ State sync requires the knowledge of the current head of the chain to converge e If not constantly fed with the most recent head, state sync won't be able to complete because the recent state soon becomes unavailable due to state trie pruning. -Optimistic block import (i.e. import without waiting for a payload to be executed by the engine) -breaks a deadlock between the state sync and processing a beacon block by skipping payload -execution as long as execution engine is syncing. +Optimistic block import (i.e. import when the execution engine *cannot* currently validate the payload) +breaks a deadlock between the execution layer sync process and importing beacon blocks +while the execution engine is syncing. Optimistic sync works nicely with execution engines using block execution as a default sync mechanism (e.g. Erigon). It makes optimistic sync a *generalized* solution for From b9476bf0ee529bbbaee5e492888468f249d8c058 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Tue, 2 Aug 2022 20:04:10 +0600 Subject: [PATCH 097/105] Expand on opti-sync in context of block sync --- sync/optimistic.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 1511ce859..b2a30aef6 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -283,9 +283,14 @@ Optimistic block import (i.e. import when the execution engine *cannot* currentl breaks a deadlock between the execution layer sync process and importing beacon blocks while the execution engine is syncing. -Optimistic sync works nicely with execution engines using block execution as a default -sync mechanism (e.g. Erigon). It makes optimistic sync a *generalized* solution for -interaction between consensus and execution engine during the sync process. +Optimistic sync is also an optimal strategy for execution engines using block execution as a default +sync mechanism (e.g. Erigon). Alternatively, a consensus engine may inform the execution engine with a payload +obtained from a checkpoint block, then wait till the execution layer catches up with it and proceed +in lock step after that. This alternative approach would keep user in limbo for several hours and +would increase time of the sync process as batch sync has more opportunities for optimisation than the lock step. + +Aforementioned premises make optimistic sync a *generalized* solution for interaction between consensus and +execution engines during the sync process. ### Why `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`? From 3d785dad7af181b20dd1c9d979ec264f9c9d4124 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 3 Aug 2022 08:24:32 -0600 Subject: [PATCH 098/105] pr feedback from @hwwhww --- .../test_process_execution_payload.py | 21 ++++++++++--------- .../test/helpers/execution_payload.py | 10 +++++++-- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py index 9dc701f7d..90125b69c 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py @@ -1,6 +1,6 @@ -import os from random import Random +from eth2spec.debug.random_value import get_random_bytes_list from eth2spec.test.helpers.execution_payload import ( build_empty_execution_payload, get_execution_payload_header, @@ -60,7 +60,6 @@ def run_success_test(spec, state): @spec_state_test def test_success_first_payload(spec, state): state = build_state_with_incomplete_transition(spec, state) - assert not spec.is_merge_transition_complete(state) yield from run_success_test(spec, state) @@ -69,7 +68,6 @@ def test_success_first_payload(spec, state): @spec_state_test def test_success_regular_payload(spec, state): state = build_state_with_complete_transition(spec, state) - assert spec.is_merge_transition_complete(state) yield from run_success_test(spec, state) @@ -311,22 +309,25 @@ def test_zero_length_transaction_regular_payload(spec, state): def build_randomized_execution_payload(spec, state, rng): execution_payload = build_empty_execution_payload(spec, state) - execution_payload.fee_recipient = spec.ExecutionAddress(os.urandom(20)) - execution_payload.state_root = spec.Bytes32(os.urandom(32)) - execution_payload.receipts_root = spec.Bytes32(os.urandom(32)) - execution_payload.logs_bloom = spec.ByteVector[spec.BYTES_PER_LOGS_BLOOM](os.urandom(spec.BYTES_PER_LOGS_BLOOM)) + execution_payload.fee_recipient = spec.ExecutionAddress(get_random_bytes_list(rng, 20)) + execution_payload.state_root = spec.Bytes32(get_random_bytes_list(rng, 32)) + execution_payload.receipts_root = spec.Bytes32(get_random_bytes_list(rng, 32)) + execution_payload.logs_bloom = spec.ByteVector[spec.BYTES_PER_LOGS_BLOOM]( + get_random_bytes_list(rng, spec.BYTES_PER_LOGS_BLOOM) + ) execution_payload.block_number = rng.randint(0, 10e10) execution_payload.gas_limit = rng.randint(0, 10e10) execution_payload.gas_used = rng.randint(0, 10e10) + extra_data_length = rng.randint(0, spec.MAX_EXTRA_DATA_BYTES) execution_payload.extra_data = spec.ByteList[spec.MAX_EXTRA_DATA_BYTES]( - os.urandom(rng.randint(0, spec.MAX_EXTRA_DATA_BYTES)) + get_random_bytes_list(rng, extra_data_length) ) execution_payload.base_fee_per_gas = rng.randint(0, 2**256 - 1) - execution_payload.block_hash = spec.Hash32(os.urandom(32)) + execution_payload.block_hash = spec.Hash32(get_random_bytes_list(rng, 32)) num_transactions = rng.randint(0, 100) execution_payload.transactions = [ - spec.Transaction(os.urandom(rng.randint(0, 1000))) + spec.Transaction(get_random_bytes_list(rng, rng.randint(0, 1000))) for _ in range(num_transactions) ] diff --git a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py index 5225d4efe..cce4c89ed 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py @@ -61,14 +61,20 @@ def get_execution_payload_header(spec, execution_payload): def build_state_with_incomplete_transition(spec, state): - return build_state_with_execution_payload_header(spec, state, spec.ExecutionPayloadHeader()) + state = build_state_with_execution_payload_header(spec, state, spec.ExecutionPayloadHeader()) + assert not spec.is_merge_transition_complete(state) + + return state def build_state_with_complete_transition(spec, state): pre_state_payload = build_empty_execution_payload(spec, state) payload_header = get_execution_payload_header(spec, pre_state_payload) - return build_state_with_execution_payload_header(spec, state, payload_header) + state = build_state_with_execution_payload_header(spec, state, payload_header) + assert spec.is_merge_transition_complete(state) + + return state def build_state_with_execution_payload_header(spec, state, execution_payload_header): From 9352f127ef1c3cc324749ae8f82d00daea578953 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 9 Aug 2022 07:49:41 -0600 Subject: [PATCH 099/105] minor typo --- sync/optimistic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index b2a30aef6..ea0ed7500 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -285,7 +285,7 @@ while the execution engine is syncing. Optimistic sync is also an optimal strategy for execution engines using block execution as a default sync mechanism (e.g. Erigon). Alternatively, a consensus engine may inform the execution engine with a payload -obtained from a checkpoint block, then wait till the execution layer catches up with it and proceed +obtained from a checkpoint block, then wait until the execution layer catches up with it and proceed in lock step after that. This alternative approach would keep user in limbo for several hours and would increase time of the sync process as batch sync has more opportunities for optimisation than the lock step. From f92312c59df908ece4e429c345ed9e12135098b9 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Aug 2022 22:00:00 +0800 Subject: [PATCH 100/105] Update ToC --- sync/optimistic.md | 1 + 1 file changed, 1 insertion(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index c205773cb..dc69135de 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -24,6 +24,7 @@ - [Participating in Sync Committees](#participating-in-sync-committees) - [Ethereum Beacon APIs](#ethereum-beacon-apis) - [Design Decision Rationale](#design-decision-rationale) + - [Why sync optimistically?](#why-sync-optimistically) - [Why `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`?](#why-safe_slots_to_import_optimistically) - [Transitioning from VALID -> INVALIDATED or INVALIDATED -> VALID](#transitioning-from-valid---invalidated-or-invalidated---valid) - [What about Light Clients?](#what-about-light-clients) From ccc40e198232c7f1398179ddbdb4a03607128cfc Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 15 Aug 2022 07:49:53 -0600 Subject: [PATCH 101/105] a few more bellatrix tests (#2962) * Do not overload index with WithdrawalIndex and ValidatorIndex * a few more bellatrix tests * use function from other PR * fix tests * Update tests/core/pyspec/eth2spec/test/bellatrix/transition/test_transition.py Co-authored-by: Hsiao-Wei Wang * refactor to reuse bellatrix transitio ntests for all transitions Co-authored-by: Potuz Co-authored-by: Hsiao-Wei Wang --- .../test/altair/transition/test_transition.py | 56 +++++++++++++++++++ .../test_process_execution_payload.py | 29 +--------- .../test/bellatrix/sanity/test_blocks.py | 23 +++++++- .../bellatrix/transition/test_transition.py | 35 ------------ .../test/helpers/execution_payload.py | 28 ++++++++++ 5 files changed, 106 insertions(+), 65 deletions(-) delete mode 100644 tests/core/pyspec/eth2spec/test/bellatrix/transition/test_transition.py diff --git a/tests/core/pyspec/eth2spec/test/altair/transition/test_transition.py b/tests/core/pyspec/eth2spec/test/altair/transition/test_transition.py index 32861b866..dcbd8a38b 100644 --- a/tests/core/pyspec/eth2spec/test/altair/transition/test_transition.py +++ b/tests/core/pyspec/eth2spec/test/altair/transition/test_transition.py @@ -3,6 +3,9 @@ from eth2spec.test.context import ( ForkMeta, with_fork_metas, ) +from eth2spec.test.helpers.random import ( + randomize_state, +) from eth2spec.test.helpers.constants import ( ALL_PRE_POST_FORKS, ) @@ -17,9 +20,31 @@ from eth2spec.test.helpers.fork_transition import ( skip_slots, state_transition_across_slots, transition_to_next_epoch_and_append_blocks, + transition_until_fork, ) +@with_fork_metas([ForkMeta(pre_fork_name=pre, post_fork_name=post, fork_epoch=2) for pre, post in ALL_PRE_POST_FORKS]) +def test_simple_transition(state, fork_epoch, spec, post_spec, pre_tag, post_tag): + transition_until_fork(spec, state, fork_epoch) + + # check pre state + assert spec.get_current_epoch(state) < fork_epoch + + yield "pre", state + + # irregular state transition to handle fork: + blocks = [] + state, block = do_fork(state, spec, post_spec, fork_epoch) + blocks.append(post_tag(block)) + + # continue regular state transition with new spec into next epoch + transition_to_next_epoch_and_append_blocks(post_spec, state, post_tag, blocks, only_last_block=True) + + yield "blocks", blocks + yield "post", state + + @with_fork_metas([ForkMeta(pre_fork_name=pre, post_fork_name=post, fork_epoch=2) for pre, post in ALL_PRE_POST_FORKS]) def test_normal_transition(state, fork_epoch, spec, post_spec, pre_tag, post_tag): """ @@ -56,6 +81,37 @@ def test_normal_transition(state, fork_epoch, spec, post_spec, pre_tag, post_tag yield "post", state +@with_fork_metas([ForkMeta(pre_fork_name=pre, post_fork_name=post, fork_epoch=8) for pre, post in ALL_PRE_POST_FORKS]) +def test_transition_randomized_state(state, fork_epoch, spec, post_spec, pre_tag, post_tag): + randomize_state(spec, state) + + transition_until_fork(spec, state, fork_epoch) + + # check pre state + assert spec.get_current_epoch(state) < fork_epoch + + yield "pre", state + + # irregular state transition to handle fork: + blocks = [] + # since there are slashed validators, set with_block=False here + state, _ = do_fork(state, spec, post_spec, fork_epoch, with_block=False) + slashed_indices = [index for index, validator in enumerate(state.validators) if validator.slashed] + + # continue regular state transition with new spec into next epoch + transition_to_next_epoch_and_append_blocks( + post_spec, + state, + post_tag, + blocks, + only_last_block=True, + ignoring_proposers=slashed_indices, + ) + + yield "blocks", blocks + yield "post", state + + @with_fork_metas([ForkMeta(pre_fork_name=pre, post_fork_name=post, fork_epoch=2) for pre, post in ALL_PRE_POST_FORKS]) def test_transition_missing_first_post_block(state, fork_epoch, spec, post_spec, pre_tag, post_tag): """ diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py index 90125b69c..2819b2a24 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py @@ -1,8 +1,8 @@ from random import Random -from eth2spec.debug.random_value import get_random_bytes_list from eth2spec.test.helpers.execution_payload import ( build_empty_execution_payload, + build_randomized_execution_payload, get_execution_payload_header, build_state_with_incomplete_transition, build_state_with_complete_transition, @@ -307,33 +307,6 @@ def test_zero_length_transaction_regular_payload(spec, state): yield from run_zero_length_transaction_test(spec, state) -def build_randomized_execution_payload(spec, state, rng): - execution_payload = build_empty_execution_payload(spec, state) - execution_payload.fee_recipient = spec.ExecutionAddress(get_random_bytes_list(rng, 20)) - execution_payload.state_root = spec.Bytes32(get_random_bytes_list(rng, 32)) - execution_payload.receipts_root = spec.Bytes32(get_random_bytes_list(rng, 32)) - execution_payload.logs_bloom = spec.ByteVector[spec.BYTES_PER_LOGS_BLOOM]( - get_random_bytes_list(rng, spec.BYTES_PER_LOGS_BLOOM) - ) - execution_payload.block_number = rng.randint(0, 10e10) - execution_payload.gas_limit = rng.randint(0, 10e10) - execution_payload.gas_used = rng.randint(0, 10e10) - extra_data_length = rng.randint(0, spec.MAX_EXTRA_DATA_BYTES) - execution_payload.extra_data = spec.ByteList[spec.MAX_EXTRA_DATA_BYTES]( - get_random_bytes_list(rng, extra_data_length) - ) - execution_payload.base_fee_per_gas = rng.randint(0, 2**256 - 1) - execution_payload.block_hash = spec.Hash32(get_random_bytes_list(rng, 32)) - - num_transactions = rng.randint(0, 100) - execution_payload.transactions = [ - spec.Transaction(get_random_bytes_list(rng, rng.randint(0, 1000))) - for _ in range(num_transactions) - ] - - return execution_payload - - def run_randomized_non_validated_execution_fields_test(spec, state, execution_valid=True, rng=Random(5555)): next_slot(spec, state) execution_payload = build_randomized_execution_payload(spec, state, rng) diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/bellatrix/sanity/test_blocks.py index bad922d4f..ef6bb75a9 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/sanity/test_blocks.py @@ -1,9 +1,14 @@ +from random import Random from eth2spec.test.helpers.state import ( - state_transition_and_sign_block + state_transition_and_sign_block, + next_slot, ) from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot ) +from eth2spec.test.helpers.execution_payload import ( + build_randomized_execution_payload +) from eth2spec.test.context import ( with_bellatrix_and_later, spec_state_test ) @@ -22,7 +27,21 @@ def test_empty_block_transition_no_tx(spec, state): yield 'blocks', [signed_block] yield 'post', state -# TODO: tests with EVM, mock or replacement? + +@with_bellatrix_and_later +@spec_state_test +def test_empty_block_transition_randomized_payload(spec, state): + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + next_slot_state = state.copy() + next_slot(spec, next_slot_state) + block.body.execution_payload = build_randomized_execution_payload(spec, next_slot_state, rng=Random(34433)) + + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state @with_bellatrix_and_later diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/transition/test_transition.py b/tests/core/pyspec/eth2spec/test/bellatrix/transition/test_transition.py deleted file mode 100644 index 2382a3758..000000000 --- a/tests/core/pyspec/eth2spec/test/bellatrix/transition/test_transition.py +++ /dev/null @@ -1,35 +0,0 @@ -from eth2spec.test.context import ( - ForkMeta, - with_fork_metas, -) -from eth2spec.test.helpers.constants import ( - AFTER_BELLATRIX_PRE_POST_FORKS, -) -from eth2spec.test.helpers.fork_transition import ( - do_fork, - transition_to_next_epoch_and_append_blocks, - transition_until_fork, -) - - -@with_fork_metas([ - ForkMeta(pre_fork_name=pre, post_fork_name=post, fork_epoch=2) for pre, post in AFTER_BELLATRIX_PRE_POST_FORKS -]) -def test_sample_transition(state, fork_epoch, spec, post_spec, pre_tag, post_tag): - transition_until_fork(spec, state, fork_epoch) - - # check pre state - assert spec.get_current_epoch(state) < fork_epoch - - yield "pre", state - - # irregular state transition to handle fork: - blocks = [] - state, block = do_fork(state, spec, post_spec, fork_epoch) - blocks.append(post_tag(block)) - - # continue regular state transition with new spec into next epoch - transition_to_next_epoch_and_append_blocks(post_spec, state, post_tag, blocks, only_last_block=True) - - yield "blocks", blocks - yield "post", state diff --git a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py index cce4c89ed..83162e1c2 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py @@ -1,3 +1,4 @@ +from eth2spec.debug.random_value import get_random_bytes_list from eth2spec.test.context import is_post_capella @@ -38,6 +39,33 @@ def build_empty_execution_payload(spec, state, randao_mix=None): return payload +def build_randomized_execution_payload(spec, state, rng): + execution_payload = build_empty_execution_payload(spec, state) + execution_payload.fee_recipient = spec.ExecutionAddress(get_random_bytes_list(rng, 20)) + execution_payload.state_root = spec.Bytes32(get_random_bytes_list(rng, 32)) + execution_payload.receipts_root = spec.Bytes32(get_random_bytes_list(rng, 32)) + execution_payload.logs_bloom = spec.ByteVector[spec.BYTES_PER_LOGS_BLOOM]( + get_random_bytes_list(rng, spec.BYTES_PER_LOGS_BLOOM) + ) + execution_payload.block_number = rng.randint(0, 10e10) + execution_payload.gas_limit = rng.randint(0, 10e10) + execution_payload.gas_used = rng.randint(0, 10e10) + extra_data_length = rng.randint(0, spec.MAX_EXTRA_DATA_BYTES) + execution_payload.extra_data = spec.ByteList[spec.MAX_EXTRA_DATA_BYTES]( + get_random_bytes_list(rng, extra_data_length) + ) + execution_payload.base_fee_per_gas = rng.randint(0, 2**256 - 1) + execution_payload.block_hash = spec.Hash32(get_random_bytes_list(rng, 32)) + + num_transactions = rng.randint(0, 100) + execution_payload.transactions = [ + spec.Transaction(get_random_bytes_list(rng, rng.randint(0, 1000))) + for _ in range(num_transactions) + ] + + return execution_payload + + def get_execution_payload_header(spec, execution_payload): payload_header = spec.ExecutionPayloadHeader( parent_hash=execution_payload.parent_hash, From 5d0f4af06385a53beefe814a9c6268a372e9f4e2 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 15 Aug 2022 08:00:14 -0600 Subject: [PATCH 102/105] Merge mainnet ttd and bellatrix values (#2969) * merge mainnet ttd and bellatrix values * Update configs/minimal.yaml Co-authored-by: Paul Harris Co-authored-by: Paul Harris --- configs/mainnet.yaml | 6 +++--- configs/minimal.yaml | 2 +- specs/bellatrix/beacon-chain.md | 2 +- specs/bellatrix/fork.md | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index e86f5b5c6..929d39f8a 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -12,8 +12,8 @@ CONFIG_NAME: 'mainnet' # Transition # --------------------------------------------------------------- -# TBD, 2**256-2**10 is a placeholder -TERMINAL_TOTAL_DIFFICULTY: 115792089237316195423570985008687907853269984665640564039457584007913129638912 +# Estimated on Sept 15, 2022 +TERMINAL_TOTAL_DIFFICULTY: 58750000000000000000000 # By default, don't use these params TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 @@ -43,7 +43,7 @@ ALTAIR_FORK_VERSION: 0x01000000 ALTAIR_FORK_EPOCH: 74240 # Oct 27, 2021, 10:56:23am UTC # Bellatrix BELLATRIX_FORK_VERSION: 0x02000000 -BELLATRIX_FORK_EPOCH: 18446744073709551615 +BELLATRIX_FORK_EPOCH: 144896 # Sept 6, 2022, 11:34:47am UTC # Capella CAPELLA_FORK_VERSION: 0x03000000 CAPELLA_FORK_EPOCH: 18446744073709551615 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 0917cfb57..5dde4b749 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -12,7 +12,7 @@ CONFIG_NAME: 'minimal' # Transition # --------------------------------------------------------------- -# TBD, 2**256-2**10 is a placeholder +# 2**256-2**10 for testing minimal network TERMINAL_TOTAL_DIFFICULTY: 115792089237316195423570985008687907853269984665640564039457584007913129638912 # By default, don't use these params TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index 9285594ab..6d39c1ae8 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -91,7 +91,7 @@ Bellatrix updates a few configuration values to move penalty parameters to their | Name | Value | | - | - | -| `TERMINAL_TOTAL_DIFFICULTY` | **TBD** | +| `TERMINAL_TOTAL_DIFFICULTY` | `58750000000000000000000` (Estimated: Sept 15, 2022)| | `TERMINAL_BLOCK_HASH` | `Hash32()` | | `TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH` | `FAR_FUTURE_EPOCH` | diff --git a/specs/bellatrix/fork.md b/specs/bellatrix/fork.md index 5c0e174da..19700e383 100644 --- a/specs/bellatrix/fork.md +++ b/specs/bellatrix/fork.md @@ -29,7 +29,7 @@ Warning: this configuration is not definitive. | Name | Value | | - | - | | `BELLATRIX_FORK_VERSION` | `Version('0x02000000')` | -| `BELLATRIX_FORK_EPOCH` | `Epoch(18446744073709551615)` **TBD** | +| `BELLATRIX_FORK_EPOCH` | `Epoch(144896)` (Sept 6, 2022, 11:34:47am UTC) | ## Helper functions From 1d0dce75b4f006c75be72e760acb328394e036a1 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Mon, 15 Aug 2022 20:34:51 +0600 Subject: [PATCH 103/105] Opti-sync: extend optimistic node definition (#2955) * Extend optimistic node definition * Update sync/optimistic.md Co-authored-by: terencechain Co-authored-by: terencechain --- sync/optimistic.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index dc69135de..4e03cc6bb 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -122,8 +122,12 @@ def is_optimistic_candidate_block(opt_store: OptimisticStore, current_slot: Slot return False ``` -Let only a node which returns `is_optimistic(opt_store, head) is True` be an *optimistic -node*. Let only a validator on an optimistic node be an *optimistic validator*. +Let a node be an *optimistic node* if its fork choice is in one of the following states: +1. `is_optimistic(opt_store, head) is True` +2. Blocks from every viable (with respect to FFG) branch have transitioned from `NOT_VALIDATED` to `INVALIDATED` +leaving the block tree without viable branches + +Let only a validator on an optimistic node be an *optimistic validator*. When this specification only defines behaviour for an optimistic node/validator, but *not* for the non-optimistic case, assume default From 91b584eb216cee1530a92171d2b7fc32050375c7 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 15 Aug 2022 09:11:41 -0600 Subject: [PATCH 104/105] bump VERSION.txt (#2971) --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index a94c1f6e2..8f890920f 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -1.2.0-rc.1 +1.2.0-rc.2 From d95c45523f0151981f2f2a664adf4b4cd260dc87 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 16 Aug 2022 14:29:56 +0800 Subject: [PATCH 105/105] Fix path: the bellatrix-new tests have been moved to altair file --- .../pyspec/eth2spec/test/bellatrix/transition/__init__.py | 0 tests/generators/transition/main.py | 8 +------- 2 files changed, 1 insertion(+), 7 deletions(-) delete mode 100644 tests/core/pyspec/eth2spec/test/bellatrix/transition/__init__.py diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/transition/__init__.py b/tests/core/pyspec/eth2spec/test/bellatrix/transition/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/generators/transition/main.py b/tests/generators/transition/main.py index 5c3b43b4f..7de7213bd 100644 --- a/tests/generators/transition/main.py +++ b/tests/generators/transition/main.py @@ -16,9 +16,6 @@ from eth2spec.test.altair.transition import ( test_slashing as test_altair_slashing, test_operations as test_altair_operations, ) -from eth2spec.test.bellatrix.transition import ( - test_transition as test_bellatrix_transition, -) def create_provider(tests_src, preset_name: str, pre_fork_name: str, post_fork_name: str) -> gen_typing.TestProvider: @@ -47,10 +44,7 @@ if __name__ == "__main__": test_altair_slashing, test_altair_operations, ) - bellatrix_tests = ( - test_bellatrix_transition, - ) - all_tests = altair_tests + bellatrix_tests + all_tests = altair_tests for transition_test_module in all_tests: for pre_fork, post_fork in ALL_PRE_POST_FORKS: gen_runner.run_generator("transition", [