From 14d4f44c11ff67aae1e12b368331743f66ca85c8 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Wed, 3 Aug 2022 18:19:39 +0600 Subject: [PATCH 01/37] Extend fork_choice test format with on_payload_info --- tests/formats/fork_choice/README.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/formats/fork_choice/README.md b/tests/formats/fork_choice/README.md index 3266ad4c0..fd8308f48 100644 --- a/tests/formats/fork_choice/README.md +++ b/tests/formats/fork_choice/README.md @@ -69,7 +69,7 @@ The file is located in the same folder (see below). After this step, the `store` object may have been updated. -#### `on_merge_block` execution +#### `on_merge_block` execution step Adds `PowBlock` data which is required for executing `on_block(store, block)`. ```yaml @@ -97,6 +97,30 @@ The file is located in the same folder (see below). After this step, the `store` object may have been updated. +#### `on_payload_info` execution step + +Optional step for optimistic sync tests. + +```yaml +{ + block_hash: string, -- Encoded 32-byte value of payload's block hash. + payload_status: { + status: string, -- Enum, "VALID" | "INVALID" | "SYNCING" | "ACCEPTED" | "INVALID_BLOCK_HASH". + latestValidHash: string, -- Encoded 32-byte value of the latest valid block hash, may be `null`. + validationError: string, -- Message providing additional details on the validation error, may be `null`. + } +} +``` + +This step sets the [`payloadStatus`](https://github.com/ethereum/execution-apis/blob/main/src/engine/specification.md#PayloadStatusV1) +value that Execution Layer client mock returns in responses to the following Engine API calls: +* [`engine_newPayloadV1(payload)`](https://github.com/ethereum/execution-apis/blob/main/src/engine/specification.md#engine_newpayloadv1) if `payload.blockHash == payload_info.block_hash` +* [`engine_forkchoiceUpdatedV1(forkchoiceState, ...)`](https://github.com/ethereum/execution-apis/blob/main/src/engine/specification.md#engine_forkchoiceupdatedv1) if `forkchoiceState.headBlockHash == payload_info.block_hash` + +*Note:* Status of a payload must be *initialized* via `on_payload_info` before the corresponding `on_block` execution step. + +*Note:* Status of the same payload may be updated for several times throughout the test. + #### Checks step The checks to verify the current status of `store`. From f8d92c3bd8479ea832ceeaf802c26c44687e66a7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 16 Aug 2022 01:22:19 +0800 Subject: [PATCH 02/37] Add basic test case --- sync/optimistic.md | 4 +- .../eth2spec/test/bellatrix/sync/__init__.py | 0 .../test/bellatrix/sync/test_optimistic.py | 98 +++++++++ .../eth2spec/test/helpers/attestations.py | 6 +- .../eth2spec/test/helpers/fork_choice.py | 74 +++---- .../eth2spec/test/helpers/optimistic_sync.py | 189 ++++++++++++++++++ tests/generators/sync/main.py | 14 ++ tests/generators/sync/requirements.txt | 2 + 8 files changed, 346 insertions(+), 41 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/bellatrix/sync/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/bellatrix/sync/test_optimistic.py create mode 100644 tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py create mode 100644 tests/generators/sync/main.py create mode 100644 tests/generators/sync/requirements.txt diff --git a/sync/optimistic.md b/sync/optimistic.md index 4e03cc6bb..07a509592 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -88,8 +88,8 @@ Let `current_slot: Slot` be `(time - genesis_time) // SECONDS_PER_SLOT` where class OptimisticStore(object): optimistic_roots: Set[Root] head_block_root: Root - blocks: Dict[Root, BeaconBlock] - block_states: Dict[Root, BeaconState] + blocks: Dict[Root, BeaconBlock] = field(default_factory=dict) + block_states: Dict[Root, BeaconState] = field(default_factory=dict) ``` ```python diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/sync/__init__.py b/tests/core/pyspec/eth2spec/test/bellatrix/sync/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/sync/test_optimistic.py b/tests/core/pyspec/eth2spec/test/bellatrix/sync/test_optimistic.py new file mode 100644 index 000000000..899c6b5ab --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/bellatrix/sync/test_optimistic.py @@ -0,0 +1,98 @@ +from eth2spec.test.context import ( + spec_state_test, + with_bellatrix_and_later, +) +from eth2spec.test.helpers.attestations import ( + state_transition_with_full_block, +) +from eth2spec.test.helpers.block import ( + build_empty_block_for_next_slot, +) +from eth2spec.test.helpers.fork_choice import ( + get_genesis_forkchoice_store_and_block, + on_tick_and_append_step, +) +from eth2spec.test.helpers.optimistic_sync import ( + PayloadStatusV1, + PayloadStatusV1Status, + MegaStore, + add_optimistic_block, + get_optimistic_store, +) +from eth2spec.test.helpers.state import ( + next_epoch, + state_transition_and_sign_block, +) + + +@with_bellatrix_and_later +@spec_state_test +def test_from_syncing_to_invalid(spec, state): + test_steps = [] + # Initialization + fc_store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state) + op_store = get_optimistic_store(spec, state, anchor_block) + mega_store = MegaStore(spec, fc_store, op_store) + yield 'anchor_state', state + yield 'anchor_block', anchor_block + + next_epoch(spec, state) + + current_time = ( + (spec.SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY * 10 + state.slot) * spec.config.SECONDS_PER_SLOT + + fc_store.genesis_time + ) + on_tick_and_append_step(spec, fc_store, current_time, test_steps) + + # Block 0 + block_0 = build_empty_block_for_next_slot(spec, state) + block_0.body.execution_payload.block_hash = spec.hash(bytes(f'block_0', 'UTF-8')) + signed_block = state_transition_and_sign_block(spec, state, block_0) + yield from add_optimistic_block(spec, mega_store, signed_block, test_steps, status=PayloadStatusV1Status.VALID) + assert spec.get_head(mega_store.fc_store) == mega_store.opt_store.head_block_root + + state_0 = state.copy() + + # Create VALID chain `a` + signed_blocks_a = [] + for i in range(3): + block = build_empty_block_for_next_slot(spec, state) + block.body.execution_payload.block_hash = spec.hash(bytes(f'chain_a_{i}', 'UTF-8')) + block.body.execution_payload.parent_hash = ( + spec.hash(bytes(f'chain_a_{i - 1}', 'UTF-8')) if i != 0 else block_0.body.execution_payload.block_hash + ) + + signed_block = state_transition_and_sign_block(spec, state, block) + yield from add_optimistic_block(spec, mega_store, signed_block, test_steps, status=PayloadStatusV1Status.VALID) + assert spec.get_head(mega_store.fc_store) == mega_store.opt_store.head_block_root + signed_blocks_a.append(signed_block.copy()) + + # Create SYNCING chain `b` + signed_blocks_b = [] + state = state_0.copy() + for i in range(3): + block = build_empty_block_for_next_slot(spec, state) + block.body.execution_payload.block_hash = spec.hash(bytes(f'chain_b_{i}', 'UTF-8')) + block.body.execution_payload.parent_hash = ( + spec.hash(bytes(f'chain_b_{i - 1}', 'UTF-8')) if i != 0 else block_0.body.execution_payload.block_hash + ) + signed_block = state_transition_with_full_block(spec, state, True, True, block=block) + signed_blocks_b.append(signed_block.copy()) + yield from add_optimistic_block(spec, mega_store, signed_block, test_steps, + status=PayloadStatusV1Status.SYNCING) + assert spec.get_head(mega_store.fc_store) == mega_store.opt_store.head_block_root + + # Now add block 4 to chain `b` with INVALID + block = build_empty_block_for_next_slot(spec, state) + block.body.execution_payload.block_hash = spec.hash(bytes(f'chain_b_3', 'UTF-8')) + block.body.execution_payload.parent_hash = signed_blocks_b[-1].message.body.execution_payload.block_hash + signed_block = state_transition_and_sign_block(spec, state, block) + payload_status = PayloadStatusV1( + status=PayloadStatusV1Status.INVALID, + latest_valid_hash=block_0.body.execution_payload.block_hash, + validation_error="invalid", + ) + yield from add_optimistic_block(spec, mega_store, signed_block, test_steps, payload_status=payload_status) + assert mega_store.opt_store.head_block_root == signed_blocks_a[-1].message.hash_tree_root() + + yield 'steps', test_steps diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 7ba71a969..854dbf59a 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -251,11 +251,13 @@ def state_transition_with_full_block(spec, fill_cur_epoch, fill_prev_epoch, participation_fn=None, - sync_aggregate=None): + sync_aggregate=None, + block=None): """ Build and apply a block with attestions at the calculated `slot_to_attest` of current epoch and/or previous epoch. """ - block = build_empty_block_for_next_slot(spec, state) + if block is None: + block = build_empty_block_for_next_slot(spec, state) if fill_cur_epoch and state.slot >= spec.MIN_ATTESTATION_INCLUSION_DELAY: slot_to_attest = state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 if slot_to_attest >= spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)): diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py index d524060a2..a1d2e6ffc 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py @@ -13,18 +13,8 @@ def get_anchor_root(spec, state): return spec.hash_tree_root(anchor_block_header) -def add_block_to_store(spec, store, signed_block): - pre_state = store.block_states[signed_block.message.parent_root] - block_time = pre_state.genesis_time + signed_block.message.slot * spec.config.SECONDS_PER_SLOT - - if store.time < block_time: - spec.on_tick(store, block_time) - - spec.on_block(store, signed_block) - - def tick_and_add_block(spec, store, signed_block, test_steps, valid=True, - merge_block=False, block_not_found=False): + merge_block=False, block_not_found=False, is_optimistic=False): pre_state = store.block_states[signed_block.message.parent_root] block_time = pre_state.genesis_time + signed_block.message.slot * spec.config.SECONDS_PER_SLOT if merge_block: @@ -37,6 +27,7 @@ def tick_and_add_block(spec, store, signed_block, test_steps, valid=True, spec, store, signed_block, test_steps, valid=valid, block_not_found=block_not_found, + is_optimistic=is_optimistic, ) return post_state @@ -119,25 +110,33 @@ def add_block(spec, signed_block, test_steps, valid=True, - block_not_found=False): + block_not_found=False, + is_optimistic=False): """ Run on_block and on_attestation """ yield get_block_file_name(signed_block), signed_block if not valid: - try: + if is_optimistic: run_on_block(spec, store, signed_block, valid=True) - except (AssertionError, BlockNotFoundException) as e: - if isinstance(e, BlockNotFoundException) and not block_not_found: - assert False test_steps.append({ 'block': get_block_file_name(signed_block), 'valid': False, }) - return else: - assert False + try: + run_on_block(spec, store, signed_block, valid=True) + except (AssertionError, BlockNotFoundException) as e: + if isinstance(e, BlockNotFoundException) and not block_not_found: + assert False + test_steps.append({ + 'block': get_block_file_name(signed_block), + 'valid': False, + }) + return + else: + assert False run_on_block(spec, store, signed_block, valid=True) test_steps.append({'block': get_block_file_name(signed_block)}) @@ -153,25 +152,26 @@ def add_block(spec, block_root = signed_block.message.hash_tree_root() assert store.blocks[block_root] == signed_block.message assert store.block_states[block_root].hash_tree_root() == signed_block.message.state_root - test_steps.append({ - 'checks': { - 'time': int(store.time), - 'head': get_formatted_head_output(spec, store), - 'justified_checkpoint': { - 'epoch': int(store.justified_checkpoint.epoch), - 'root': encode_hex(store.justified_checkpoint.root), - }, - 'finalized_checkpoint': { - 'epoch': int(store.finalized_checkpoint.epoch), - 'root': encode_hex(store.finalized_checkpoint.root), - }, - 'best_justified_checkpoint': { - 'epoch': int(store.best_justified_checkpoint.epoch), - 'root': encode_hex(store.best_justified_checkpoint.root), - }, - 'proposer_boost_root': encode_hex(store.proposer_boost_root), - } - }) + if not is_optimistic: + test_steps.append({ + 'checks': { + 'time': int(store.time), + 'head': get_formatted_head_output(spec, store), + 'justified_checkpoint': { + 'epoch': int(store.justified_checkpoint.epoch), + 'root': encode_hex(store.justified_checkpoint.root), + }, + 'finalized_checkpoint': { + 'epoch': int(store.finalized_checkpoint.epoch), + 'root': encode_hex(store.finalized_checkpoint.root), + }, + 'best_justified_checkpoint': { + 'epoch': int(store.best_justified_checkpoint.epoch), + 'root': encode_hex(store.best_justified_checkpoint.root), + }, + 'proposer_boost_root': encode_hex(store.proposer_boost_root), + } + }) return store.block_states[signed_block.message.hash_tree_root()] diff --git a/tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py b/tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py new file mode 100644 index 000000000..3fc31e5b0 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py @@ -0,0 +1,189 @@ +from dataclasses import dataclass +from enum import Enum +from typing import ( + Dict, + Optional, +) + +from eth_utils import encode_hex + +from eth2spec.utils.ssz.ssz_typing import Bytes32 +from eth2spec.test.helpers.fork_choice import ( + add_block, +) + + +class PayloadStatusV1StatusAlias(Enum): + NOT_VALIDATED = "NOT_VALIDATED" + INVALIDATED = "INVALIDATED" + + +class PayloadStatusV1Status(Enum): + VALID = "VALID" + INVALID = "INVALID" + SYNCING = "SYNCING" + ACCEPTED = "ACCEPTED" + INVALID_BLOCK_HASH = "INVALID_BLOCK_HASH" + + @property + def alias(self) -> PayloadStatusV1StatusAlias: + if self.value in (self.SYNCING.value, self.ACCEPTED.value): + return PayloadStatusV1StatusAlias.NOT_VALIDATED + elif self.value in (self.INVALID.value, self.INVALID_BLOCK_HASH.value): + return PayloadStatusV1StatusAlias.INVALIDATED + + +@dataclass +class PayloadStatusV1: + status: PayloadStatusV1Status = PayloadStatusV1Status.VALID + latest_valid_hash: Optional[Bytes32] = None + validation_error: Optional[str] = None + + @property + def formatted_output(self): + return { + 'status': str(self.status), + 'latest_valid_hash': encode_hex(self.latest_valid_hash) if self.latest_valid_hash is not None else 'null', + 'validation_error': str(self.validation_error) if self.validation_error is not None else 'null' + } + + +class MegaStore(object): + spec = None + fc_store = None + opt_store = None + block_payload_statuses: Dict[Bytes32, PayloadStatusV1] = dict() + + def __init__(self, spec, fc_store, opt_store): + self.spec = spec + self.fc_store = fc_store + self.opt_store = opt_store + + +def get_optimistic_store(spec, anchor_state, anchor_block): + assert anchor_block.state_root == anchor_state.hash_tree_root() + + opt_store = spec.OptimisticStore( + optimistic_roots=set(), + head_block_root=anchor_block.hash_tree_root(), + + ) + anchor_block_root = anchor_block.hash_tree_root() + opt_store.blocks[anchor_block_root] = anchor_block.copy() + opt_store.block_states[anchor_block_root] = anchor_state.copy() + + return opt_store + + +def add_optimistic_block(spec, mega_store, signed_block, test_steps, + payload_status=None, status=PayloadStatusV1Status.SYNCING): + block = signed_block.message + block_root = block.hash_tree_root() + el_block_hash = block.body.execution_payload.block_hash + + if payload_status is None: + payload_status = PayloadStatusV1(status=status) + if payload_status.status == PayloadStatusV1Status.VALID: + payload_status.latest_valid_hash = el_block_hash + + mega_store.block_payload_statuses[block_root] = payload_status + test_steps.append({ + 'payload_status': payload_status.formatted_output, + }) + + # Optimistic sync + + valid = True + + # Case: INVALID + if payload_status.status == PayloadStatusV1Status.INVALID: + # Update parent status to INVALID + assert payload_status.latest_valid_hash is not None + current_block = block + while el_block_hash != payload_status.latest_valid_hash and el_block_hash != spec.Bytes32(): + current_block_root = current_block.hash_tree_root() + assert current_block_root in mega_store.block_payload_statuses + mega_store.block_payload_statuses[current_block_root].status = PayloadStatusV1Status.INVALID + # Get parent + current_block = mega_store.fc_store.blocks[current_block.parent_root] + el_block_hash = current_block.body.execution_payload.block_hash + + if payload_status.status != PayloadStatusV1Status.VALID: + valid = False + + yield from add_block(spec, mega_store.fc_store, signed_block, + valid=valid, + test_steps=test_steps, + is_optimistic=True) + + # Update stores + is_optimistic_candidate = spec.is_optimistic_candidate_block( + mega_store.opt_store, + current_slot=spec.get_current_slot(mega_store.fc_store), + block=signed_block.message, + ) + if is_optimistic_candidate: + mega_store.opt_store.optimistic_roots.add(block_root) + mega_store.opt_store.blocks[block_root] = signed_block.message.copy() + if not is_invalidated(mega_store, block_root): + mega_store.opt_store.block_states[block_root] = mega_store.fc_store.block_states[block_root].copy() + + # Clean up the invalidated blocks + clean_up_store(mega_store) + + # Update head + mega_store.opt_store.head_block_root = get_opt_head_block_root(spec, mega_store) + test_steps.append({ + 'checks': { + 'head': get_formatted_optimistic_head_output(mega_store), + } + }) + + +def get_opt_head_block_root(spec, mega_store): + """ + Copied and modified from fork-choice spec `get_head` function. + """ + store = mega_store.fc_store + + # Get filtered block tree that only includes viable branches + blocks = spec.get_filtered_block_tree(store) + # Execute the LMD-GHOST fork choice + head = store.justified_checkpoint.root + while True: + children = [ + root for root in blocks.keys() + if ( + blocks[root].parent_root == head + and not is_invalidated(mega_store, root) # For optimistic sync + ) + ] + if len(children) == 0: + return head + # Sort by latest attesting balance with ties broken lexicographically + # Ties broken by favoring block with lexicographically higher root + head = max(children, key=lambda root: (spec.get_latest_attesting_balance(store, root), root)) + + +def is_invalidated(mega_store, block_root): + if block_root in mega_store.block_payload_statuses: + return mega_store.block_payload_statuses[block_root].status.alias == PayloadStatusV1StatusAlias.INVALIDATED + else: + return False + + +def get_formatted_optimistic_head_output(mega_store): + head = mega_store.opt_store.head_block_root + slot = mega_store.fc_store.blocks[head].slot + return { + 'slot': int(slot), + 'root': encode_hex(head), + } + + +def clean_up_store(mega_store): + """ + Remove invalidated blocks + """ + # TODO + ... diff --git a/tests/generators/sync/main.py b/tests/generators/sync/main.py new file mode 100644 index 000000000..ad83b78a0 --- /dev/null +++ b/tests/generators/sync/main.py @@ -0,0 +1,14 @@ +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators +from eth2spec.test.helpers.constants import BELLATRIX + + +if __name__ == "__main__": + bellatrix_mods = {key: 'eth2spec.test.bellatrix.sync.test_' + key for key in [ + 'optimistic', + ]} + + all_mods = { + BELLATRIX: bellatrix_mods, + } + + run_state_test_generators(runner_name="sync", all_mods=all_mods) diff --git a/tests/generators/sync/requirements.txt b/tests/generators/sync/requirements.txt new file mode 100644 index 000000000..735f863fa --- /dev/null +++ b/tests/generators/sync/requirements.txt @@ -0,0 +1,2 @@ +pytest>=4.4 +../../../[generator] \ No newline at end of file From b67fb5b049474b38045428c346f96bc1d277af28 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Wed, 24 Aug 2022 23:29:59 +0600 Subject: [PATCH 03/37] Update tests/formats/fork_choice/README.md Co-authored-by: Hsiao-Wei Wang --- tests/formats/fork_choice/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/formats/fork_choice/README.md b/tests/formats/fork_choice/README.md index fd8308f48..f79d436eb 100644 --- a/tests/formats/fork_choice/README.md +++ b/tests/formats/fork_choice/README.md @@ -106,8 +106,8 @@ Optional step for optimistic sync tests. block_hash: string, -- Encoded 32-byte value of payload's block hash. payload_status: { status: string, -- Enum, "VALID" | "INVALID" | "SYNCING" | "ACCEPTED" | "INVALID_BLOCK_HASH". - latestValidHash: string, -- Encoded 32-byte value of the latest valid block hash, may be `null`. - validationError: string, -- Message providing additional details on the validation error, may be `null`. + latest_valid_hash: string, -- Encoded 32-byte value of the latest valid block hash, may be `null`. + validation_error: string, -- Message providing additional details on the validation error, may be `null`. } } ``` From ac717b106af596fc5758b0004b98badefcf18986 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 31 Aug 2022 22:02:47 +0800 Subject: [PATCH 04/37] Address PR feedback from @mkalinin --- .../test/bellatrix/sync/test_optimistic.py | 5 +++-- .../pyspec/eth2spec/test/helpers/fork_choice.py | 6 +++--- .../eth2spec/test/helpers/optimistic_sync.py | 14 ++++++++------ 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/sync/test_optimistic.py b/tests/core/pyspec/eth2spec/test/bellatrix/sync/test_optimistic.py index 899c6b5ab..cf757b583 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/sync/test_optimistic.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/sync/test_optimistic.py @@ -79,7 +79,7 @@ def test_from_syncing_to_invalid(spec, state): signed_block = state_transition_with_full_block(spec, state, True, True, block=block) signed_blocks_b.append(signed_block.copy()) yield from add_optimistic_block(spec, mega_store, signed_block, test_steps, - status=PayloadStatusV1Status.SYNCING) + status=PayloadStatusV1Status.SYNCING, valid=False) assert spec.get_head(mega_store.fc_store) == mega_store.opt_store.head_block_root # Now add block 4 to chain `b` with INVALID @@ -92,7 +92,8 @@ def test_from_syncing_to_invalid(spec, state): latest_valid_hash=block_0.body.execution_payload.block_hash, validation_error="invalid", ) - yield from add_optimistic_block(spec, mega_store, signed_block, test_steps, payload_status=payload_status) + yield from add_optimistic_block(spec, mega_store, signed_block, test_steps, + payload_status=payload_status, valid=False) assert mega_store.opt_store.head_block_root == signed_blocks_a[-1].message.hash_tree_root() yield 'steps', test_steps diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py index a1d2e6ffc..bd8abd95b 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py @@ -137,9 +137,9 @@ def add_block(spec, return else: assert False - - run_on_block(spec, store, signed_block, valid=True) - test_steps.append({'block': get_block_file_name(signed_block)}) + else: + run_on_block(spec, store, signed_block, valid=True) + test_steps.append({'block': get_block_file_name(signed_block)}) # An on_block step implies receiving block's attestations for attestation in signed_block.message.body.attestations: diff --git a/tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py b/tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py index 3fc31e5b0..d88513a8a 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py +++ b/tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py @@ -76,7 +76,14 @@ def get_optimistic_store(spec, anchor_state, anchor_block): def add_optimistic_block(spec, mega_store, signed_block, test_steps, - payload_status=None, status=PayloadStatusV1Status.SYNCING): + payload_status=None, status=PayloadStatusV1Status.SYNCING, + valid=True): + """ + Add a block with optimistic sync logic + + ``valid`` indicates if the given ``signed_block.message.body.execution_payload`` is valid/invalid + from ``notify_new_payload`` method response. + """ block = signed_block.message block_root = block.hash_tree_root() el_block_hash = block.body.execution_payload.block_hash @@ -93,8 +100,6 @@ def add_optimistic_block(spec, mega_store, signed_block, test_steps, # Optimistic sync - valid = True - # Case: INVALID if payload_status.status == PayloadStatusV1Status.INVALID: # Update parent status to INVALID @@ -108,9 +113,6 @@ def add_optimistic_block(spec, mega_store, signed_block, test_steps, current_block = mega_store.fc_store.blocks[current_block.parent_root] el_block_hash = current_block.body.execution_payload.block_hash - if payload_status.status != PayloadStatusV1Status.VALID: - valid = False - yield from add_block(spec, mega_store.fc_store, signed_block, valid=valid, test_steps=test_steps, From 0f8b5ae6bd3fe8a0b36b875a3553d04012bdfac5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 2 Sep 2022 20:41:55 +0800 Subject: [PATCH 05/37] Apply PR feedback from @michaelsproul --- .../pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py | 6 +++++- tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py | 7 ++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index 1d2b03fd2..dccb9313f 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -9,7 +9,6 @@ import sys import json from typing import Iterable, AnyStr, Any, Callable import traceback - from ruamel.yaml import ( YAML, ) @@ -98,6 +97,11 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): yaml = YAML(pure=True) yaml.default_flow_style = None + def _represent_none(self, _): + return self.represent_scalar('tag:yaml.org,2002:null', 'null') + + yaml.representer.add_representer(type(None), _represent_none) + # Spec config is using a YAML subset cfg_yaml = YAML(pure=True) cfg_yaml.default_flow_style = False # Emit separate line for each key diff --git a/tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py b/tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py index d88513a8a..a89c3083e 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py +++ b/tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py @@ -42,9 +42,9 @@ class PayloadStatusV1: @property def formatted_output(self): return { - 'status': str(self.status), - 'latest_valid_hash': encode_hex(self.latest_valid_hash) if self.latest_valid_hash is not None else 'null', - 'validation_error': str(self.validation_error) if self.validation_error is not None else 'null' + 'status': str(self.status.value), + 'latest_valid_hash': encode_hex(self.latest_valid_hash) if self.latest_valid_hash is not None else None, + 'validation_error': str(self.validation_error) if self.validation_error is not None else None } @@ -95,6 +95,7 @@ def add_optimistic_block(spec, mega_store, signed_block, test_steps, mega_store.block_payload_statuses[block_root] = payload_status test_steps.append({ + 'block_hash': encode_hex(el_block_hash), 'payload_status': payload_status.formatted_output, }) From 2e7309125866b6f60961f569bb9c2638a2e11b9e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 8 Sep 2022 22:35:53 +0800 Subject: [PATCH 06/37] Add `get_valid_flag_value` helper to determine the `valid` flag value --- .../test/bellatrix/sync/test_optimistic.py | 4 ++-- .../eth2spec/test/helpers/optimistic_sync.py | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/sync/test_optimistic.py b/tests/core/pyspec/eth2spec/test/bellatrix/sync/test_optimistic.py index cf757b583..4fb8adbaf 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/sync/test_optimistic.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/sync/test_optimistic.py @@ -79,7 +79,7 @@ def test_from_syncing_to_invalid(spec, state): signed_block = state_transition_with_full_block(spec, state, True, True, block=block) signed_blocks_b.append(signed_block.copy()) yield from add_optimistic_block(spec, mega_store, signed_block, test_steps, - status=PayloadStatusV1Status.SYNCING, valid=False) + status=PayloadStatusV1Status.SYNCING) assert spec.get_head(mega_store.fc_store) == mega_store.opt_store.head_block_root # Now add block 4 to chain `b` with INVALID @@ -93,7 +93,7 @@ def test_from_syncing_to_invalid(spec, state): validation_error="invalid", ) yield from add_optimistic_block(spec, mega_store, signed_block, test_steps, - payload_status=payload_status, valid=False) + payload_status=payload_status) assert mega_store.opt_store.head_block_root == signed_blocks_a[-1].message.hash_tree_root() yield 'steps', test_steps diff --git a/tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py b/tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py index a89c3083e..6f42aa9ba 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py +++ b/tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py @@ -75,9 +75,18 @@ def get_optimistic_store(spec, anchor_state, anchor_block): return opt_store +def get_valid_flag_value(status: PayloadStatusV1Status) -> bool: + if status == PayloadStatusV1Status.VALID: + return True + elif status.alias == PayloadStatusV1StatusAlias.NOT_VALIDATED: + return True + else: + # status.alias == PayloadStatusV1StatusAlias.INVALIDATED or other cases + return False + + def add_optimistic_block(spec, mega_store, signed_block, test_steps, - payload_status=None, status=PayloadStatusV1Status.SYNCING, - valid=True): + payload_status=None, status=PayloadStatusV1Status.SYNCING): """ Add a block with optimistic sync logic @@ -99,6 +108,9 @@ def add_optimistic_block(spec, mega_store, signed_block, test_steps, 'payload_status': payload_status.formatted_output, }) + # Set `valid` flag + valid = get_valid_flag_value(payload_status.status) + # Optimistic sync # Case: INVALID From e4407c64e3d88a32ee04a0e03bf4cf950444dcab Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sun, 11 Sep 2022 02:19:01 +0800 Subject: [PATCH 07/37] Add `test_process_deposit::test_key_validate_invalid` --- .../core/pyspec/eth2spec/test/helpers/deposits.py | 15 +++++++++++---- .../block_processing/test_process_deposit.py | 14 ++++++++++++++ tests/core/pyspec/eth2spec/utils/bls.py | 5 +++++ 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/deposits.py b/tests/core/pyspec/eth2spec/test/helpers/deposits.py index dae05c2eb..18929b1d7 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/deposits.py +++ b/tests/core/pyspec/eth2spec/test/helpers/deposits.py @@ -137,14 +137,21 @@ def prepare_random_genesis_deposits(spec, return deposits, root, deposit_data_list -def prepare_state_and_deposit(spec, state, validator_index, amount, withdrawal_credentials=None, signed=False): +def prepare_state_and_deposit(spec, state, validator_index, amount, + pubkey=None, + privkey=None, + withdrawal_credentials=None, + signed=False): """ Prepare the state for the deposit, and create a deposit for the given validator, depositing the given amount. """ deposit_data_list = [] - pubkey = pubkeys[validator_index] - privkey = privkeys[validator_index] + if pubkey is None: + pubkey = pubkeys[validator_index] + + if privkey is None: + privkey = privkeys[validator_index] # insecurely use pubkey as withdrawal key if no credentials provided if withdrawal_credentials is None: @@ -196,7 +203,7 @@ def run_deposit_processing(spec, state, deposit, validator_index, valid=True, ef yield 'post', state - if not effective: + if not effective or not bls.KeyValidate(deposit.data.pubkey): assert len(state.validators) == pre_validator_count assert len(state.balances) == pre_validator_count if validator_index < pre_validator_count: diff --git a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py index df0bd2a17..f1a27f8ec 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py +++ b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py @@ -233,3 +233,17 @@ def test_bad_merkle_proof(spec, state): sign_deposit_data(spec, deposit.data, privkeys[validator_index]) yield from run_deposit_processing(spec, state, deposit, validator_index, valid=False) + + +@with_all_phases +@spec_state_test +def test_key_validate_invalid(spec, state): + validator_index = len(state.validators) + amount = spec.MAX_EFFECTIVE_BALANCE + + # All-zero pubkey would not pass `bls.KeyValidate`, but `process_deposit` would not throw exception. + pubkey = b'\x00' * 48 + + deposit = prepare_state_and_deposit(spec, state, validator_index, amount, pubkey=pubkey, signed=True) + + yield from run_deposit_processing(spec, state, deposit, validator_index) diff --git a/tests/core/pyspec/eth2spec/utils/bls.py b/tests/core/pyspec/eth2spec/utils/bls.py index e33017ade..fd5fd89bf 100644 --- a/tests/core/pyspec/eth2spec/utils/bls.py +++ b/tests/core/pyspec/eth2spec/utils/bls.py @@ -138,3 +138,8 @@ def pairing_check(values): * pairing(p_q_2[1], p_q_2[0], final_exponentiate=False) ) return final_exponentiation == FQ12.one() + + +@only_with_bls(alt_return=True) +def KeyValidate(pubkey): + return py_ecc_bls.KeyValidate(pubkey) From 4d2cfff2d64469c8a6083845c95c6128d7e67d7e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 12 Sep 2022 22:59:29 +0800 Subject: [PATCH 08/37] Add `test_key_validate_invalid_decompression` --- .../block_processing/test_process_deposit.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py index f1a27f8ec..8922032b4 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py +++ b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py @@ -237,7 +237,7 @@ def test_bad_merkle_proof(spec, state): @with_all_phases @spec_state_test -def test_key_validate_invalid(spec, state): +def test_key_validate_invalid_subgroup(spec, state): validator_index = len(state.validators) amount = spec.MAX_EFFECTIVE_BALANCE @@ -247,3 +247,19 @@ def test_key_validate_invalid(spec, state): deposit = prepare_state_and_deposit(spec, state, validator_index, amount, pubkey=pubkey, signed=True) yield from run_deposit_processing(spec, state, deposit, validator_index) + + +@with_all_phases +@spec_state_test +def test_key_validate_invalid_decompression(spec, state): + validator_index = len(state.validators) + amount = spec.MAX_EFFECTIVE_BALANCE + + # `deserialization_fails_infinity_with_true_b_flag` BLS G1 deserialization test case. + # This pubkey would not pass `bls.KeyValidate`, but `process_deposit` would not throw exception. + pubkey_hex = 'c01000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000' + pubkey = bytes.fromhex(pubkey_hex) + + deposit = prepare_state_and_deposit(spec, state, validator_index, amount, pubkey=pubkey, signed=True) + + yield from run_deposit_processing(spec, state, deposit, validator_index) From d70dcd9926a4bbe987f1b4e65c3e05bd029fcfb8 Mon Sep 17 00:00:00 2001 From: Ben Guidarelli Date: Thu, 15 Sep 2022 10:21:06 -0400 Subject: [PATCH 09/37] Fix link to beacon-chain doc --- specs/altair/light-client/sync-protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/altair/light-client/sync-protocol.md b/specs/altair/light-client/sync-protocol.md index 39a0b3d11..ede86f852 100644 --- a/specs/altair/light-client/sync-protocol.md +++ b/specs/altair/light-client/sync-protocol.md @@ -47,7 +47,7 @@ Such environments include resource-constrained devices (e.g. phones for trust-mi and metered VMs (e.g. blockchain VMs for cross-chain bridges). This document suggests a minimal light client design for the beacon chain that -uses sync committees introduced in [this beacon chain extension](./beacon-chain.md). +uses sync committees introduced in [this beacon chain extension](../beacon-chain.md). Additional documents describe how the light client sync protocol can be used: - [Full node](./full-node.md) From 63d284a85c511bcb11cd1d11b67d8a5d8e5f6c96 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 16 Sep 2022 09:55:36 -0600 Subject: [PATCH 10/37] bump version to v1.2.0 --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index 328994069..26aaba0e8 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -1.2.0-rc.3 \ No newline at end of file +1.2.0 From 70f90c5296b9ecc95b6f63b8854d103c5d6a18c7 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 19 Sep 2022 11:05:47 -0600 Subject: [PATCH 11/37] rmove withdrawn_epoch --- specs/capella/beacon-chain.md | 16 +-- specs/capella/fork.md | 16 +-- .../test_process_bls_to_execution_change.py | 16 ++- .../block_processing/test_process_deposit.py | 30 +++++ .../test_process_full_withdrawals.py | 123 +++++++++++++++--- .../eth2spec/test/helpers/capella/fork.py | 5 +- .../pyspec/eth2spec/test/helpers/deposits.py | 15 ++- .../pyspec/eth2spec/test/helpers/genesis.py | 3 - .../eth2spec/test/helpers/withdrawals.py | 17 +++ .../block_processing/test_process_deposit.py | 46 ++++++- 10 files changed, 228 insertions(+), 59 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_deposit.py create mode 100644 tests/core/pyspec/eth2spec/test/helpers/withdrawals.py diff --git a/specs/capella/beacon-chain.md b/specs/capella/beacon-chain.md index 40592a9bd..25588c1e8 100644 --- a/specs/capella/beacon-chain.md +++ b/specs/capella/beacon-chain.md @@ -193,7 +193,6 @@ class Validator(Container): activation_epoch: Epoch exit_epoch: Epoch withdrawable_epoch: Epoch # When validator can withdraw funds - fully_withdrawn_epoch: Epoch # [New in Capella] ``` #### `BeaconBlockBody` @@ -297,13 +296,14 @@ def has_eth1_withdrawal_credential(validator: Validator) -> bool: #### `is_fully_withdrawable_validator` ```python -def is_fully_withdrawable_validator(validator: Validator, epoch: Epoch) -> bool: +def is_fully_withdrawable_validator(validator: Validator, balance: Gwei, epoch: Epoch) -> bool: """ Check if ``validator`` is fully withdrawable. """ return ( has_eth1_withdrawal_credential(validator) - and validator.withdrawable_epoch <= epoch < validator.fully_withdrawn_epoch + and validator.withdrawable_epoch <= epoch + and balance > 0 ) ``` @@ -349,11 +349,11 @@ def process_epoch(state: BeaconState) -> None: ```python def process_full_withdrawals(state: BeaconState) -> None: current_epoch = get_current_epoch(state) - for index, validator in enumerate(state.validators): - if is_fully_withdrawable_validator(validator, current_epoch): - # TODO, consider the zero-balance case - withdraw_balance(state, ValidatorIndex(index), state.balances[index]) - validator.fully_withdrawn_epoch = current_epoch + for index in range(len(state.validators)): + balance = state.balances[index] + validator = state.validators[index] + if is_fully_withdrawable_validator(validator, balance, current_epoch): + withdraw_balance(state, ValidatorIndex(index), balance) ``` #### Partial withdrawals diff --git a/specs/capella/fork.md b/specs/capella/fork.md index c22387ee7..34d373be0 100644 --- a/specs/capella/fork.md +++ b/specs/capella/fork.md @@ -90,7 +90,7 @@ def upgrade_to_capella(pre: bellatrix.BeaconState) -> BeaconState: eth1_data_votes=pre.eth1_data_votes, eth1_deposit_index=pre.eth1_deposit_index, # Registry - validators=[], + validators=pre.validators, balances=pre.balances, # Randomness randao_mixes=pre.randao_mixes, @@ -117,19 +117,5 @@ def upgrade_to_capella(pre: bellatrix.BeaconState) -> BeaconState: next_partial_withdrawal_validator_index=ValidatorIndex(0), ) - for pre_validator in pre.validators: - post_validator = Validator( - pubkey=pre_validator.pubkey, - withdrawal_credentials=pre_validator.withdrawal_credentials, - effective_balance=pre_validator.effective_balance, - slashed=pre_validator.slashed, - activation_eligibility_epoch=pre_validator.activation_eligibility_epoch, - activation_epoch=pre_validator.activation_epoch, - exit_epoch=pre_validator.exit_epoch, - withdrawable_epoch=pre_validator.withdrawable_epoch, - fully_withdrawn_epoch=FAR_FUTURE_EPOCH, - ) - post.validators.append(post_validator) - return post ``` diff --git a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_bls_to_execution_change.py b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_bls_to_execution_change.py index 4b69e04a6..abcbec0f1 100644 --- a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_bls_to_execution_change.py +++ b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_bls_to_execution_change.py @@ -82,7 +82,9 @@ def test_success_not_activated(spec, state): signed_address_change = get_signed_address_change(spec, state) yield from run_bls_to_execution_change_processing(spec, state, signed_address_change) - assert not spec.is_fully_withdrawable_validator(state.validators[validator_index], spec.get_current_epoch(state)) + validator = state.validators[validator_index] + balance = state.balances[validator_index] + assert not spec.is_fully_withdrawable_validator(validator, balance, spec.get_current_epoch(state)) @with_capella_and_later @@ -98,7 +100,9 @@ def test_success_in_activation_queue(spec, state): signed_address_change = get_signed_address_change(spec, state) yield from run_bls_to_execution_change_processing(spec, state, signed_address_change) - assert not spec.is_fully_withdrawable_validator(state.validators[validator_index], spec.get_current_epoch(state)) + validator = state.validators[validator_index] + balance = state.balances[validator_index] + assert not spec.is_fully_withdrawable_validator(validator, balance, spec.get_current_epoch(state)) @with_capella_and_later @@ -126,7 +130,9 @@ def test_success_exited(spec, state): signed_address_change = get_signed_address_change(spec, state, validator_index=validator_index) yield from run_bls_to_execution_change_processing(spec, state, signed_address_change) - assert not spec.is_fully_withdrawable_validator(state.validators[validator_index], spec.get_current_epoch(state)) + validator = state.validators[validator_index] + balance = state.balances[validator_index] + assert not spec.is_fully_withdrawable_validator(validator, balance, spec.get_current_epoch(state)) @with_capella_and_later @@ -142,7 +148,9 @@ def test_success_withdrawable(spec, state): signed_address_change = get_signed_address_change(spec, state, validator_index=validator_index) yield from run_bls_to_execution_change_processing(spec, state, signed_address_change) - assert spec.is_fully_withdrawable_validator(state.validators[validator_index], spec.get_current_epoch(state)) + validator = state.validators[validator_index] + balance = state.balances[validator_index] + assert spec.is_fully_withdrawable_validator(validator, balance, spec.get_current_epoch(state)) @with_capella_and_later diff --git a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_deposit.py b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_deposit.py new file mode 100644 index 000000000..34c58ef39 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_deposit.py @@ -0,0 +1,30 @@ +from eth2spec.test.context import ( + spec_state_test, + with_capella_and_later, +) +from eth2spec.test.helpers.state import next_epoch_via_block +from eth2spec.test.helpers.deposits import ( + prepare_state_and_deposit, + run_deposit_processing, +) +from eth2spec.test.helpers.withdrawals import set_validator_fully_withdrawable + + +@with_capella_and_later +@spec_state_test +def test_success_top_up_to_withdrawn_validator(spec, state): + validator_index = 0 + + # Fully withdraw validator + set_validator_fully_withdrawable(spec, state, validator_index) + assert state.balances[validator_index] > 0 + next_epoch_via_block(spec, state) + assert state.balances[validator_index] == 0 + + # Make a top-up balance to validator + amount = spec.MAX_EFFECTIVE_BALANCE // 4 + deposit = prepare_state_and_deposit(spec, state, validator_index, amount, signed=True) + + yield from run_deposit_processing(spec, state, deposit, validator_index) + + state.balances[validator_index] == amount diff --git a/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_full_withdrawals.py b/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_full_withdrawals.py index 1498666bb..35d2968cb 100644 --- a/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_full_withdrawals.py +++ b/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_full_withdrawals.py @@ -1,27 +1,28 @@ +from random import Random + from eth2spec.test.context import ( with_capella_and_later, spec_state_test, ) -from eth2spec.test.helpers.epoch_processing import run_epoch_processing_with - - -def set_validator_withdrawable(spec, state, index, withdrawable_epoch=None): - if withdrawable_epoch is None: - withdrawable_epoch = spec.get_current_epoch(state) - - validator = state.validators[index] - validator.withdrawable_epoch = withdrawable_epoch - validator.withdrawal_credentials = spec.ETH1_ADDRESS_WITHDRAWAL_PREFIX + validator.withdrawal_credentials[1:] - - assert spec.is_fully_withdrawable_validator(validator, withdrawable_epoch) +from eth2spec.test.helpers.random import ( + randomize_state, +) +from eth2spec.test.helpers.epoch_processing import ( + run_epoch_processing_to, +) +from eth2spec.test.helpers.withdrawals import ( + set_validator_fully_withdrawable, +) def run_process_full_withdrawals(spec, state, num_expected_withdrawals=None): + run_epoch_processing_to(spec, state, 'process_full_withdrawals') + pre_next_withdrawal_index = state.next_withdrawal_index pre_withdrawal_queue = state.withdrawal_queue.copy() to_be_withdrawn_indices = [ index for index, validator in enumerate(state.validators) - if spec.is_fully_withdrawable_validator(validator, spec.get_current_epoch(state)) + if spec.is_fully_withdrawable_validator(validator, state.balances[index], spec.get_current_epoch(state)) ] if num_expected_withdrawals is not None: @@ -29,11 +30,11 @@ def run_process_full_withdrawals(spec, state, num_expected_withdrawals=None): else: num_expected_withdrawals = len(to_be_withdrawn_indices) - yield from run_epoch_processing_with(spec, state, 'process_full_withdrawals') + yield 'pre', state + spec.process_full_withdrawals(state) + yield 'post', state for index in to_be_withdrawn_indices: - validator = state.validators[index] - assert validator.fully_withdrawn_epoch == spec.get_current_epoch(state) assert state.balances[index] == 0 assert len(state.withdrawal_queue) == len(pre_withdrawal_queue) + num_expected_withdrawals @@ -42,13 +43,49 @@ def run_process_full_withdrawals(spec, state, num_expected_withdrawals=None): @with_capella_and_later @spec_state_test -def test_no_withdrawals(spec, state): +def test_no_withdrawable_validators(spec, state): pre_validators = state.validators.copy() yield from run_process_full_withdrawals(spec, state, 0) assert pre_validators == state.validators +@with_capella_and_later +@spec_state_test +def test_withdrawable_epoch_but_0_balance(spec, state): + current_epoch = spec.get_current_epoch(state) + set_validator_fully_withdrawable(spec, state, 0, current_epoch) + + state.validators[0].effective_balance = 10000000000 + state.balances[0] = 0 + + yield from run_process_full_withdrawals(spec, state, 0) + + +@with_capella_and_later +@spec_state_test +def test_withdrawable_epoch_but_0_effective_balance_0_balance(spec, state): + current_epoch = spec.get_current_epoch(state) + set_validator_fully_withdrawable(spec, state, 0, current_epoch) + + state.validators[0].effective_balance = 0 + state.balances[0] = 0 + + yield from run_process_full_withdrawals(spec, state, 0) + + +@with_capella_and_later +@spec_state_test +def test_withdrawable_epoch_but_0_effective_balance_nonzero_balance(spec, state): + current_epoch = spec.get_current_epoch(state) + set_validator_fully_withdrawable(spec, state, 0, current_epoch) + + state.validators[0].effective_balance = 0 + state.balances[0] = 100000000 + + yield from run_process_full_withdrawals(spec, state, 1) + + @with_capella_and_later @spec_state_test def test_no_withdrawals_but_some_next_epoch(spec, state): @@ -56,7 +93,7 @@ def test_no_withdrawals_but_some_next_epoch(spec, state): # Make a few validators withdrawable at the *next* epoch for index in range(3): - set_validator_withdrawable(spec, state, index, current_epoch + 1) + set_validator_fully_withdrawable(spec, state, index, current_epoch + 1) yield from run_process_full_withdrawals(spec, state, 0) @@ -65,7 +102,7 @@ def test_no_withdrawals_but_some_next_epoch(spec, state): @spec_state_test def test_single_withdrawal(spec, state): # Make one validator withdrawable - set_validator_withdrawable(spec, state, 0) + set_validator_fully_withdrawable(spec, state, 0) assert state.next_withdrawal_index == 0 yield from run_process_full_withdrawals(spec, state, 1) @@ -78,7 +115,7 @@ def test_single_withdrawal(spec, state): def test_multi_withdrawal(spec, state): # Make a few validators withdrawable for index in range(3): - set_validator_withdrawable(spec, state, index) + set_validator_fully_withdrawable(spec, state, index) yield from run_process_full_withdrawals(spec, state, 3) @@ -88,6 +125,50 @@ def test_multi_withdrawal(spec, state): def test_all_withdrawal(spec, state): # Make all validators withdrawable for index in range(len(state.validators)): - set_validator_withdrawable(spec, state, index) + set_validator_fully_withdrawable(spec, state, index) yield from run_process_full_withdrawals(spec, state, len(state.validators)) + + +def run_random_full_withdrawals_test(spec, state, rng): + randomize_state(spec, state, rng) + for index in range(len(state.validators)): + # 50% withdrawable + if rng.choice([True, False]): + set_validator_fully_withdrawable(spec, state, index) + validator = state.validators[index] + # 12.5% unset credentials + if rng.randint(0, 7) == 0: + validator.withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX + validator.withdrawal_credentials[1:] + # 12.5% not enough balance + if rng.randint(0, 7) == 0: + state.balances[index] = 0 + # 12.5% not close enough epoch + if rng.randint(0, 7) == 0: + validator.withdrawable_epoch += 1 + + yield from run_process_full_withdrawals(spec, state, None) + + +@with_capella_and_later +@spec_state_test +def test_random_withdrawals_0(spec, state): + yield from run_random_full_withdrawals_test(spec, state, Random(444)) + + +@with_capella_and_later +@spec_state_test +def test_random_withdrawals_1(spec, state): + yield from run_random_full_withdrawals_test(spec, state, Random(420)) + + +@with_capella_and_later +@spec_state_test +def test_random_withdrawals_2(spec, state): + yield from run_random_full_withdrawals_test(spec, state, Random(200)) + + +@with_capella_and_later +@spec_state_test +def test_random_withdrawals_3(spec, state): + yield from run_random_full_withdrawals_test(spec, state, Random(2000000)) diff --git a/tests/core/pyspec/eth2spec/test/helpers/capella/fork.py b/tests/core/pyspec/eth2spec/test/helpers/capella/fork.py index 41975904f..8e0aec9c6 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/capella/fork.py +++ b/tests/core/pyspec/eth2spec/test/helpers/capella/fork.py @@ -16,7 +16,7 @@ def run_fork_test(post_spec, pre_state): # Eth1 'eth1_data', 'eth1_data_votes', 'eth1_deposit_index', # Registry - 'balances', + 'validators', 'balances', # Randomness 'randao_mixes', # Slashings @@ -36,7 +36,7 @@ def run_fork_test(post_spec, pre_state): assert getattr(pre_state, field) == getattr(post_state, field) # Modified fields - modified_fields = ['fork', 'validators'] + modified_fields = ['fork'] for field in modified_fields: assert getattr(pre_state, field) != getattr(post_state, field) @@ -50,7 +50,6 @@ def run_fork_test(post_spec, pre_state): ] for field in stable_validator_fields: assert getattr(pre_validator, field) == getattr(post_validator, field) - assert post_validator.fully_withdrawn_epoch == post_spec.FAR_FUTURE_EPOCH assert pre_state.fork.current_version == post_state.fork.previous_version assert post_state.fork.current_version == post_spec.config.CAPELLA_FORK_VERSION diff --git a/tests/core/pyspec/eth2spec/test/helpers/deposits.py b/tests/core/pyspec/eth2spec/test/helpers/deposits.py index 18929b1d7..4639b22dc 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/deposits.py +++ b/tests/core/pyspec/eth2spec/test/helpers/deposits.py @@ -188,8 +188,12 @@ def run_deposit_processing(spec, state, deposit, validator_index, valid=True, ef """ pre_validator_count = len(state.validators) pre_balance = 0 + is_top_up = False + # is a top-up if validator_index < pre_validator_count: + is_top_up = True pre_balance = get_balance(state, validator_index) + pre_effective_balance = state.validators[validator_index].effective_balance yield 'pre', state yield 'deposit', deposit @@ -219,9 +223,12 @@ def run_deposit_processing(spec, state, deposit, validator_index, valid=True, ef assert len(state.balances) == pre_validator_count + 1 assert get_balance(state, validator_index) == pre_balance + deposit.data.amount - effective = min(spec.MAX_EFFECTIVE_BALANCE, - pre_balance + deposit.data.amount) - effective -= effective % spec.EFFECTIVE_BALANCE_INCREMENT - assert state.validators[validator_index].effective_balance == effective + if is_top_up: + # Top-ups do not change effective balance + assert state.validators[validator_index].effective_balance == pre_effective_balance + else: + effective_balance = min(spec.MAX_EFFECTIVE_BALANCE, deposit.data.amount) + effective_balance -= effective_balance % spec.EFFECTIVE_BALANCE_INCREMENT + assert state.validators[validator_index].effective_balance == effective_balance assert state.eth1_deposit_index == state.eth1_data.deposit_count diff --git a/tests/core/pyspec/eth2spec/test/helpers/genesis.py b/tests/core/pyspec/eth2spec/test/helpers/genesis.py index b7b941125..e497a0ce2 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/genesis.py +++ b/tests/core/pyspec/eth2spec/test/helpers/genesis.py @@ -20,9 +20,6 @@ def build_mock_validator(spec, i: int, balance: int): effective_balance=min(balance - balance % spec.EFFECTIVE_BALANCE_INCREMENT, spec.MAX_EFFECTIVE_BALANCE) ) - if spec.fork in (CAPELLA): - validator.fully_withdrawn_epoch = spec.FAR_FUTURE_EPOCH - return validator diff --git a/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py b/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py new file mode 100644 index 000000000..739f5eb06 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py @@ -0,0 +1,17 @@ +def set_validator_fully_withdrawable(spec, state, index, withdrawable_epoch=None): + if withdrawable_epoch is None: + withdrawable_epoch = spec.get_current_epoch(state) + + validator = state.validators[index] + validator.withdrawable_epoch = withdrawable_epoch + # set exit epoch as well to avoid interactions with other epoch process, e.g. forced ejecions + if validator.exit_epoch > withdrawable_epoch: + validator.exit_epoch = withdrawable_epoch + + if validator.withdrawal_credentials[0:1] == spec.BLS_WITHDRAWAL_PREFIX: + validator.withdrawal_credentials = spec.ETH1_ADDRESS_WITHDRAWAL_PREFIX + validator.withdrawal_credentials[1:] + + if state.balances[index] == 0: + state.balances[index] = 10000000000 + + assert spec.is_fully_withdrawable_validator(validator, state.balances[index], withdrawable_epoch) diff --git a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py index 8922032b4..20d8d7d74 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py +++ b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py @@ -141,13 +141,57 @@ def test_invalid_sig_new_deposit(spec, state): @with_all_phases @spec_state_test -def test_success_top_up(spec, state): +def test_success_top_up__max_effective_balance(spec, state): validator_index = 0 amount = spec.MAX_EFFECTIVE_BALANCE // 4 deposit = prepare_state_and_deposit(spec, state, validator_index, amount, signed=True) + state.balances[validator_index] = spec.MAX_EFFECTIVE_BALANCE + state.validators[validator_index].effective_balance = spec.MAX_EFFECTIVE_BALANCE + yield from run_deposit_processing(spec, state, deposit, validator_index) + assert state.balances[validator_index] == spec.MAX_EFFECTIVE_BALANCE + amount + assert state.validators[validator_index].effective_balance == spec.MAX_EFFECTIVE_BALANCE + + +@with_all_phases +@spec_state_test +def test_success_top_up__less_effective_balance(spec, state): + validator_index = 0 + amount = spec.MAX_EFFECTIVE_BALANCE // 4 + deposit = prepare_state_and_deposit(spec, state, validator_index, amount, signed=True) + + initial_balance = spec.MAX_EFFECTIVE_BALANCE - 1000 + initial_effective_balance = spec.MAX_EFFECTIVE_BALANCE - spec.EFFECTIVE_BALANCE_INCREMENT + state.balances[validator_index] = initial_balance + state.validators[validator_index].effective_balance = initial_effective_balance + + yield from run_deposit_processing(spec, state, deposit, validator_index) + + assert state.balances[validator_index] == initial_balance + amount + # unchanged effective balance + assert state.validators[validator_index].effective_balance == initial_effective_balance + + +@with_all_phases +@spec_state_test +def test_success_top_up__zero_balance(spec, state): + validator_index = 0 + amount = spec.MAX_EFFECTIVE_BALANCE // 4 + deposit = prepare_state_and_deposit(spec, state, validator_index, amount, signed=True) + + initial_balance = 0 + initial_effective_balance = 0 + state.balances[validator_index] = initial_balance + state.validators[validator_index].effective_balance = initial_effective_balance + + yield from run_deposit_processing(spec, state, deposit, validator_index) + + assert state.balances[validator_index] == initial_balance + amount + # unchanged effective balance + assert state.validators[validator_index].effective_balance == initial_effective_balance + @with_all_phases @spec_state_test From b63ed22588854c0822838ff808027e1169e2e124 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Mon, 19 Sep 2022 20:10:48 +0100 Subject: [PATCH 12/37] Fix signature of compute_aggregated_poly_and_commitment --- specs/eip4844/validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/eip4844/validator.md b/specs/eip4844/validator.md index 6c4e893f9..0df13aed4 100644 --- a/specs/eip4844/validator.md +++ b/specs/eip4844/validator.md @@ -117,7 +117,7 @@ def compute_powers(x: BLSFieldElement, n: uint64) -> Sequence[BLSFieldElement]: ```python def compute_aggregated_poly_and_commitment( - blobs: Sequence[BLSFieldElement], + blobs: Sequence[Sequence[BLSFieldElement]], kzg_commitments: Sequence[KZGCommitment]) -> Tuple[Polynomial, KZGCommitment]: """ Return the aggregated polynomial and aggregated KZG commitment. From b35155005bd22682a74bc7e1480feb5d843611be Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Mon, 19 Sep 2022 20:16:19 +0100 Subject: [PATCH 13/37] Rename matrix_lincomb to vector_lincomb and lincomb to g1_lincomb --- specs/eip4844/polynomial-commitments.md | 16 ++++++++-------- specs/eip4844/validator.md | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/specs/eip4844/polynomial-commitments.md b/specs/eip4844/polynomial-commitments.md index f66e3eb2e..6ebd3fd3a 100644 --- a/specs/eip4844/polynomial-commitments.md +++ b/specs/eip4844/polynomial-commitments.md @@ -15,8 +15,8 @@ - [BLS12-381 helpers](#bls12-381-helpers) - [`bls_modular_inverse`](#bls_modular_inverse) - [`div`](#div) - - [`lincomb`](#lincomb) - - [`matrix_lincomb`](#matrix_lincomb) + - [`g1_lincomb`](#g1_lincomb) + - [`vector_lincomb`](#vector_lincomb) - [KZG](#kzg) - [`blob_to_kzg_commitment`](#blob_to_kzg_commitment) - [`verify_kzg_proof`](#verify_kzg_proof) @@ -85,10 +85,10 @@ def div(x: BLSFieldElement, y: BLSFieldElement) -> BLSFieldElement: return (int(x) * int(bls_modular_inverse(y))) % BLS_MODULUS ``` -#### `lincomb` +#### `g1_lincomb` ```python -def lincomb(points: Sequence[KZGCommitment], scalars: Sequence[BLSFieldElement]) -> KZGCommitment: +def g1_lincomb(points: Sequence[KZGCommitment], scalars: Sequence[BLSFieldElement]) -> KZGCommitment: """ BLS multiscalar multiplication. This function can be optimized using Pippenger's algorithm and variants. """ @@ -99,10 +99,10 @@ def lincomb(points: Sequence[KZGCommitment], scalars: Sequence[BLSFieldElement]) return KZGCommitment(bls.G1_to_bytes48(result)) ``` -#### `matrix_lincomb` +#### `vector_lincomb` ```python -def matrix_lincomb(vectors: Sequence[Sequence[BLSFieldElement]], +def vector_lincomb(vectors: Sequence[Sequence[BLSFieldElement]], scalars: Sequence[BLSFieldElement]) -> Sequence[BLSFieldElement]: """ Given a list of ``vectors``, interpret it as a 2D matrix and compute the linear combination @@ -123,7 +123,7 @@ KZG core functions. These are also defined in EIP-4844 execution specs. ```python def blob_to_kzg_commitment(blob: Blob) -> KZGCommitment: - return lincomb(KZG_SETUP_LAGRANGE, blob) + return g1_lincomb(KZG_SETUP_LAGRANGE, blob) ``` #### `verify_kzg_proof` @@ -165,7 +165,7 @@ def compute_kzg_proof(polynomial: Sequence[BLSFieldElement], z: BLSFieldElement) # Calculate quotient polynomial by doing point-by-point division quotient_polynomial = [div(a, b) for a, b in zip(polynomial_shifted, denominator_poly)] - return KZGProof(lincomb(KZG_SETUP_LAGRANGE, quotient_polynomial)) + return KZGProof(g1_lincomb(KZG_SETUP_LAGRANGE, quotient_polynomial)) ``` ### Polynomials diff --git a/specs/eip4844/validator.md b/specs/eip4844/validator.md index 0df13aed4..f624c5157 100644 --- a/specs/eip4844/validator.md +++ b/specs/eip4844/validator.md @@ -127,10 +127,10 @@ def compute_aggregated_poly_and_commitment( r_powers = compute_powers(r, len(kzg_commitments)) # Create aggregated polynomial in evaluation form - aggregated_poly = Polynomial(matrix_lincomb(blobs, r_powers)) + aggregated_poly = Polynomial(vector_lincomb(blobs, r_powers)) # Compute commitment to aggregated polynomial - aggregated_poly_commitment = KZGCommitment(lincomb(kzg_commitments, r_powers)) + aggregated_poly_commitment = KZGCommitment(g1_lincomb(kzg_commitments, r_powers)) return aggregated_poly, aggregated_poly_commitment ``` From 0f8e12e929a86dc7f71f232dc349715c620d8d32 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 20 Sep 2022 18:13:08 +0800 Subject: [PATCH 14/37] Add a redirection README.md for the sync tests --- tests/formats/sync/README.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tests/formats/sync/README.md diff --git a/tests/formats/sync/README.md b/tests/formats/sync/README.md new file mode 100644 index 000000000..ff9f8168c --- /dev/null +++ b/tests/formats/sync/README.md @@ -0,0 +1,3 @@ +# Sync tests + +It re-uses the [fork choice test format](../fork_choice/README.md) to apply the test script. From 91ea9e6959b64640fad2f617c2f006e8eb6d9b04 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 20 Sep 2022 08:18:42 -0600 Subject: [PATCH 15/37] review suggestion from @hwwhww Co-authored-by: Hsiao-Wei Wang --- .../test/capella/block_processing/test_process_deposit.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_deposit.py b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_deposit.py index 34c58ef39..93dd655fc 100644 --- a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_deposit.py +++ b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_deposit.py @@ -28,3 +28,9 @@ def test_success_top_up_to_withdrawn_validator(spec, state): yield from run_deposit_processing(spec, state, deposit, validator_index) state.balances[validator_index] == amount + state.validators[validator_index].effective_balance == 0 + + validator = state.validators[validator_index] + balance = state.balances[validator_index] + current_epoch = spec.get_current_epoch(state) + assert spec.is_fully_withdrawable_validator(validator, balance, current_epoch) From 93b7ae299d78cd0c0dfb1721abfadac9abb0f930 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 21 Sep 2022 01:20:20 +0800 Subject: [PATCH 16/37] Update BLS test format: output `null` for invalid case --- tests/formats/bls/aggregate.md | 4 ++-- tests/formats/bls/eth_aggregate_pubkeys.md | 4 ++-- tests/formats/bls/sign.md | 9 +++++++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/formats/bls/aggregate.md b/tests/formats/bls/aggregate.md index 81ce85fe6..7cdebcf4d 100644 --- a/tests/formats/bls/aggregate.md +++ b/tests/formats/bls/aggregate.md @@ -8,11 +8,11 @@ The test data is declared in a `data.yaml` file: ```yaml input: List[BLS Signature] -- list of input BLS signatures -output: BLS Signature -- expected output, single BLS signature or empty. +output: BLS Signature -- expected output, single BLS signature or `null`. ``` - `BLS Signature` here is encoded as a string: hexadecimal encoding of 96 bytes (192 nibbles), prefixed with `0x`. -- No output value if the input is invalid. +- output value is `null` if the input is invalid. All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. diff --git a/tests/formats/bls/eth_aggregate_pubkeys.md b/tests/formats/bls/eth_aggregate_pubkeys.md index 4f66adec2..2b72c1dca 100644 --- a/tests/formats/bls/eth_aggregate_pubkeys.md +++ b/tests/formats/bls/eth_aggregate_pubkeys.md @@ -8,11 +8,11 @@ The test data is declared in a `data.yaml` file: ```yaml input: List[BLS Pubkey] -- list of input BLS pubkeys -output: BLSPubkey -- expected output, single BLS pubkeys or empty. +output: BLSPubkey -- expected output, single BLS pubkeys or `null`. ``` - `BLS Pubkey` here is encoded as a string: hexadecimal encoding of 48 bytes (96 nibbles), prefixed with `0x`. -- No output value if the input is invalid. +- output value is `null` if the input is invalid. ## Condition diff --git a/tests/formats/bls/sign.md b/tests/formats/bls/sign.md index 93001beee..09e928614 100644 --- a/tests/formats/bls/sign.md +++ b/tests/formats/bls/sign.md @@ -10,7 +10,12 @@ The test data is declared in a `data.yaml` file: input: privkey: bytes32 -- the private key used for signing message: bytes32 -- input message to sign (a hash) -output: BLS Signature -- expected output, single BLS signature or empty. +output: BLS Signature -- expected output, single BLS signature or `null`. ``` -All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. +- All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. +- output value is `null` if the input is invalid. + +## Condition + +The `sign` handler should sign `message` with `privkey`, and the resulting signature should match the expected `output`. From 3bc7ff9f80308e8415abe08ce50c79aab70ba4ce Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Tue, 20 Sep 2022 14:43:38 -0500 Subject: [PATCH 17/37] Fix a few things in Capella specs --- specs/capella/beacon-chain.md | 14 +++++++------- specs/capella/fork-choice.md | 2 +- specs/capella/fork.md | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/specs/capella/beacon-chain.md b/specs/capella/beacon-chain.md index 40592a9bd..9cd1c93ae 100644 --- a/specs/capella/beacon-chain.md +++ b/specs/capella/beacon-chain.md @@ -52,11 +52,11 @@ Capella is a consensus-layer upgrade containing a number of features related to validator withdrawals. Including: -* Automatic withdrawals of `withdrawable` validators +* Automatic withdrawals of `withdrawable` validators. * Partial withdrawals sweep for validators with 0x01 withdrawal - credentials and balances in exceess of `MAX_EFFECTIVE_BALANCE` + credentials and balances in excess of `MAX_EFFECTIVE_BALANCE`. * Operation to change from `BLS_WITHDRAWAL_PREFIX` to - `ETH1_ADDRESS_WITHDRAWAL_PREFIX` versioned withdrawal credentials to enable withdrawals for a validator + `ETH1_ADDRESS_WITHDRAWAL_PREFIX` versioned withdrawal credentials to enable withdrawals for a validator. ## Custom types @@ -64,7 +64,7 @@ We define the following Python custom types for type hinting and readability: | Name | SSZ equivalent | Description | | - | - | - | -| `WithdrawalIndex` | `uint64` | an index of a `Withdrawal`| +| `WithdrawalIndex` | `uint64` | an index of a `Withdrawal` | ## Constants @@ -84,9 +84,9 @@ We define the following Python custom types for type hinting and readability: ### State list lengths -| Name | Value | Unit | Duration | +| Name | Value | Unit | | - | - | :-: | :-: | -| `WITHDRAWAL_QUEUE_LIMIT` | `uint64(2**40)` (= 1,099,511,627,776) | withdrawals enqueued in state| +| `WITHDRAWAL_QUEUE_LIMIT` | `uint64(2**40)` (= 1,099,511,627,776) | withdrawals enqueued in state | ### Max operations per block @@ -289,7 +289,7 @@ def withdraw_balance(state: BeaconState, validator_index: ValidatorIndex, amount ```python def has_eth1_withdrawal_credential(validator: Validator) -> bool: """ - Check if ``validator`` has an 0x01 prefixed "eth1" withdrawal credential + Check if ``validator`` has an 0x01 prefixed "eth1" withdrawal credential. """ return validator.withdrawal_credentials[:1] == ETH1_ADDRESS_WITHDRAWAL_PREFIX ``` diff --git a/specs/capella/fork-choice.md b/specs/capella/fork-choice.md index f7a76275d..0e0a393c3 100644 --- a/specs/capella/fork-choice.md +++ b/specs/capella/fork-choice.md @@ -58,5 +58,5 @@ class PayloadAttributes(object): timestamp: uint64 prev_randao: Bytes32 suggested_fee_recipient: ExecutionAddress - withdrawals: Sequence[Withdrawal] # new in Capella + withdrawals: Sequence[Withdrawal] # [New in Capella] ``` diff --git a/specs/capella/fork.md b/specs/capella/fork.md index c22387ee7..85400d651 100644 --- a/specs/capella/fork.md +++ b/specs/capella/fork.md @@ -65,7 +65,7 @@ an irregular state change is made to upgrade to Capella. The upgrade occurs after the completion of the inner loop of `process_slots` that sets `state.slot` equal to `CAPELLA_FORK_EPOCH * SLOTS_PER_EPOCH`. Care must be taken when transitioning through the fork boundary as implementations will need a modified [state transition function](../phase0/beacon-chain.md#beacon-chain-state-transition-function) that deviates from the Phase 0 document. -In particular, the outer `state_transition` function defined in the Phase 0 document will not expose the precise fork slot to execute the upgrade in the presence of skipped slots at the fork boundary. Instead the logic must be within `process_slots`. +In particular, the outer `state_transition` function defined in the Phase 0 document will not expose the precise fork slot to execute the upgrade in the presence of skipped slots at the fork boundary. Instead, the logic must be within `process_slots`. ```python def upgrade_to_capella(pre: bellatrix.BeaconState) -> BeaconState: From 603e27f459fec98ca55b288739c7c7175d515d7a Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Tue, 20 Sep 2022 15:24:14 -0500 Subject: [PATCH 18/37] Fix state list lengths table --- specs/capella/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/capella/beacon-chain.md b/specs/capella/beacon-chain.md index 9cd1c93ae..36d8984a3 100644 --- a/specs/capella/beacon-chain.md +++ b/specs/capella/beacon-chain.md @@ -85,7 +85,7 @@ We define the following Python custom types for type hinting and readability: ### State list lengths | Name | Value | Unit | -| - | - | :-: | :-: | +| - | - | :-: | | `WITHDRAWAL_QUEUE_LIMIT` | `uint64(2**40)` (= 1,099,511,627,776) | withdrawals enqueued in state | ### Max operations per block From ceee7bfbf1b5b6c6b59cf28ee277803781839bf2 Mon Sep 17 00:00:00 2001 From: inphi Date: Wed, 21 Sep 2022 09:10:37 -0400 Subject: [PATCH 19/37] EIP-4844: Fee market updates to execution --- specs/eip4844/beacon-chain.md | 85 +++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/specs/eip4844/beacon-chain.md b/specs/eip4844/beacon-chain.md index a1385d9e2..4cf953593 100644 --- a/specs/eip4844/beacon-chain.md +++ b/specs/eip4844/beacon-chain.md @@ -19,6 +19,8 @@ - [Containers](#containers) - [Extended containers](#extended-containers) - [`BeaconBlockBody`](#beaconblockbody) + - [`ExecutionPayload`](#executionpayload) + - [`ExecutionPayloadHeader`](#executionpayloadheader) - [Helper functions](#helper-functions) - [Misc](#misc) - [`kzg_commitment_to_versioned_hash`](#kzg_commitment_to_versioned_hash) @@ -26,6 +28,8 @@ - [`verify_kzg_commitments_against_transactions`](#verify_kzg_commitments_against_transactions) - [Beacon chain state transition function](#beacon-chain-state-transition-function) - [Block processing](#block-processing) + - [Execution payload](#execution-payload) + - [`process_execution_payload`](#process_execution_payload) - [Blob KZG commitments](#blob-kzg-commitments) - [Testing](#testing) @@ -96,6 +100,52 @@ class BeaconBlockBody(Container): blob_kzg_commitments: List[KZGCommitment, MAX_BLOBS_PER_BLOCK] # [New in EIP-4844] ``` +#### `ExecutionPayload` + +```python +class ExecutionPayload(Container): + # Execution block header fields + parent_hash: Hash32 + fee_recipient: ExecutionAddress # 'beneficiary' in the yellow paper + state_root: Bytes32 + receipts_root: Bytes32 + logs_bloom: ByteVector[BYTES_PER_LOGS_BLOOM] + prev_randao: Bytes32 # 'difficulty' in the yellow paper + block_number: uint64 # 'number' in the yellow paper + gas_limit: uint64 + gas_used: uint64 + timestamp: uint64 + extra_data: ByteList[MAX_EXTRA_DATA_BYTES] + base_fee_per_gas: uint256 + excess_blobs: uint64 # [New in EIP-4844] + # Extra payload fields + block_hash: Hash32 # Hash of execution block + transactions: List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD] +``` + +#### `ExecutionPayloadHeader` + +```python +class ExecutionPayloadHeader(Container): + # Execution block header fields + parent_hash: Hash32 + fee_recipient: ExecutionAddress + state_root: Bytes32 + receipts_root: Bytes32 + logs_bloom: ByteVector[BYTES_PER_LOGS_BLOOM] + prev_randao: Bytes32 + block_number: uint64 + gas_limit: uint64 + gas_used: uint64 + timestamp: uint64 + extra_data: ByteList[MAX_EXTRA_DATA_BYTES] + base_fee_per_gas: uint256 + excess_blobs: uint64 # [New in EIP-4844] + # Extra payload fields + block_hash: Hash32 # Hash of execution block + transactions_root: Root +``` + ## Helper functions ### Misc @@ -156,6 +206,41 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_blob_kzg_commitments(state, block.body) # [New in EIP-4844] ``` +#### Execution payload + +##### `process_execution_payload` + +```python +def process_execution_payload(state: BeaconState, payload: ExecutionPayload, execution_engine: ExecutionEngine) -> None: + # Verify consistency of the parent hash with respect to the previous execution payload header + if is_merge_transition_complete(state): + assert payload.parent_hash == state.latest_execution_payload_header.block_hash + # Verify prev_randao + assert payload.prev_randao == get_randao_mix(state, get_current_epoch(state)) + # Verify timestamp + assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) + # Verify the execution payload is valid + assert execution_engine.notify_new_payload(payload) + # Cache execution payload header + state.latest_execution_payload_header = ExecutionPayloadHeader( + parent_hash=payload.parent_hash, + fee_recipient=payload.fee_recipient, + state_root=payload.state_root, + receipts_root=payload.receipts_root, + logs_bloom=payload.logs_bloom, + prev_randao=payload.prev_randao, + block_number=payload.block_number, + gas_limit=payload.gas_limit, + gas_used=payload.gas_used, + timestamp=payload.timestamp, + extra_data=payload.extra_data, + base_fee_per_gas=payload.base_fee_per_gas, + excess_blobs=payload.excess_blobs, # [New in EIP-4844] + block_hash=payload.block_hash, + transactions_root=hash_tree_root(payload.transactions), + ) +``` + #### Blob KZG commitments ```python From bfca7f9a96a6847940c73bd2508b18318577de45 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 21 Sep 2022 07:33:05 -0600 Subject: [PATCH 20/37] Apply suggestions from code review Co-authored-by: Alex Stokes --- .../test/capella/block_processing/test_process_deposit.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_deposit.py b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_deposit.py index 93dd655fc..e0603d301 100644 --- a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_deposit.py +++ b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_deposit.py @@ -20,6 +20,9 @@ def test_success_top_up_to_withdrawn_validator(spec, state): assert state.balances[validator_index] > 0 next_epoch_via_block(spec, state) assert state.balances[validator_index] == 0 + assert state.validators[validator_index].effective_balance > 0 + next_epoch_via_block(spec, state) + assert state.validators[validator_index].effective_balance == 0 # Make a top-up balance to validator amount = spec.MAX_EFFECTIVE_BALANCE // 4 @@ -27,8 +30,8 @@ def test_success_top_up_to_withdrawn_validator(spec, state): yield from run_deposit_processing(spec, state, deposit, validator_index) - state.balances[validator_index] == amount - state.validators[validator_index].effective_balance == 0 + assert state.balances[validator_index] == amount + assert state.validators[validator_index].effective_balance == 0 validator = state.validators[validator_index] balance = state.balances[validator_index] From 7066307a0fbd97d149cca63acc3b48ead244f3fd Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 21 Sep 2022 16:36:27 -0500 Subject: [PATCH 21/37] Fix section name for withdraw_balance --- specs/capella/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/capella/beacon-chain.md b/specs/capella/beacon-chain.md index 36d8984a3..86408db88 100644 --- a/specs/capella/beacon-chain.md +++ b/specs/capella/beacon-chain.md @@ -266,7 +266,7 @@ class BeaconState(Container): ### Beacon state mutators -#### `withdraw` +#### `withdraw_balance` ```python def withdraw_balance(state: BeaconState, validator_index: ValidatorIndex, amount: Gwei) -> None: From 5517729859110240ce5fb09754e683b40c32bcb9 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 21 Sep 2022 16:41:33 -0500 Subject: [PATCH 22/37] Fix TOC with check_toc --- specs/capella/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/capella/beacon-chain.md b/specs/capella/beacon-chain.md index 86408db88..ec8cad982 100644 --- a/specs/capella/beacon-chain.md +++ b/specs/capella/beacon-chain.md @@ -29,7 +29,7 @@ - [`BeaconState`](#beaconstate) - [Helpers](#helpers) - [Beacon state mutators](#beacon-state-mutators) - - [`withdraw`](#withdraw) + - [`withdraw_balance`](#withdraw_balance) - [Predicates](#predicates) - [`has_eth1_withdrawal_credential`](#has_eth1_withdrawal_credential) - [`is_fully_withdrawable_validator`](#is_fully_withdrawable_validator) From 1bb863b2a62a13329d477cb0d33269dc6e0c3821 Mon Sep 17 00:00:00 2001 From: terencechain Date: Wed, 21 Sep 2022 14:49:18 -0700 Subject: [PATCH 23/37] eip4844: signed_blobs_header -> signed_blobs_sidecar --- specs/eip4844/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/eip4844/p2p-interface.md b/specs/eip4844/p2p-interface.md index 913bbd752..5c9c46f2c 100644 --- a/specs/eip4844/p2p-interface.md +++ b/specs/eip4844/p2p-interface.md @@ -108,7 +108,7 @@ Alias `sidecar = signed_blobs_sidecar.message`. - _[REJECT]_ the beacon proposer signature, `signed_blobs_sidecar.signature`, is valid -- i.e. - Let `domain = get_domain(state, DOMAIN_BLOBS_SIDECAR, sidecar.beacon_block_slot // SLOTS_PER_EPOCH)` - Let `signing_root = compute_signing_root(sidecar, domain)` - - Verify `bls.Verify(proposer_pubkey, signing_root, signed_blob_header.signature) is True`, + - Verify `bls.Verify(proposer_pubkey, signing_root, signed_blobs_sidecar.signature) is True`, where `proposer_pubkey` is the pubkey of the beacon block proposer of `sidecar.beacon_block_slot` - _[IGNORE]_ The sidecar is the first sidecar with valid signature received for the `(proposer_index, sidecar.beacon_block_slot)` combination, where `proposer_index` is the validator index of the beacon block proposer of `sidecar.beacon_block_slot` From fdb5c7dcf3ac3e051a0abeb7a2d71f99e4c767e7 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Thu, 22 Sep 2022 15:19:22 -0500 Subject: [PATCH 24/37] Add partial withdrawal balance tests --- .../test_process_bls_to_execution_change.py | 2 +- .../test_process_withdrawals.py | 2 +- .../test_process_partial_withdrawals.py | 58 +++++++++++++++++-- 3 files changed, 55 insertions(+), 7 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_bls_to_execution_change.py b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_bls_to_execution_change.py index 4b69e04a6..4ad1bca86 100644 --- a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_bls_to_execution_change.py +++ b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_bls_to_execution_change.py @@ -185,7 +185,7 @@ def test_fail_incorrect_from_bls_pubkey(spec, state): @always_bls def test_fail_bad_signature(spec, state): signed_address_change = get_signed_address_change(spec, state) - # Mutate sigature + # Mutate signature signed_address_change.signature = spec.BLSSignature(b'\x42' * 96) yield from run_bls_to_execution_change_processing(spec, state, signed_address_change, valid=False) diff --git a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_withdrawals.py b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_withdrawals.py index 26ace24b3..4932bc60d 100644 --- a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_withdrawals.py +++ b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_withdrawals.py @@ -235,7 +235,7 @@ def test_fail_many_dequeued_incorrectly(spec, state): if i % 3 == 0: withdrawal.index += 1 elif i % 3 == 1: - withdrawal.address = (i).to_bytes(20, 'big') + withdrawal.address = i.to_bytes(20, 'big') else: withdrawal.amount += 1 diff --git a/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_partial_withdrawals.py b/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_partial_withdrawals.py index bf2e73fa1..431c2f2d3 100644 --- a/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_partial_withdrawals.py +++ b/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_partial_withdrawals.py @@ -10,13 +10,18 @@ from eth2spec.test.helpers.state import next_epoch from eth2spec.test.helpers.random import randomize_state -def set_validator_partially_withdrawable(spec, state, index, rng=random.Random(666)): +def set_eth1_withdrawal_credential_with_balance(spec, state, index, balance): validator = state.validators[index] validator.withdrawal_credentials = spec.ETH1_ADDRESS_WITHDRAWAL_PREFIX + validator.withdrawal_credentials[1:] - validator.effective_balance = spec.MAX_EFFECTIVE_BALANCE - state.balances[index] = spec.MAX_EFFECTIVE_BALANCE + rng.randint(1, 100000000) + validator.effective_balance = min(balance, spec.MAX_EFFECTIVE_BALANCE) + state.balances[index] = balance - assert spec.is_partially_withdrawable_validator(validator, state.balances[index]) + +def set_validator_partially_withdrawable(spec, state, index, rng=random.Random(666)): + balance = spec.MAX_EFFECTIVE_BALANCE + rng.randint(1, 100000000) + set_eth1_withdrawal_credential_with_balance(spec, state, index, balance) + + assert spec.is_partially_withdrawable_validator(state.validators[index], state.balances[index]) def run_process_partial_withdrawals(spec, state, num_expected_withdrawals=None): @@ -62,6 +67,49 @@ def test_success_no_withdrawable(spec, state): assert pre_validators == state.validators +@with_capella_and_later +@spec_state_test +def test_success_no_max_effective_balance(spec, state): + validator_index = len(state.validators) // 2 + # To be partially withdrawable, the validator's effective balance must be maxed out + set_eth1_withdrawal_credential_with_balance(spec, state, validator_index, spec.MAX_EFFECTIVE_BALANCE - 1) + validator = state.validators[validator_index] + + assert validator.effective_balance < spec.MAX_EFFECTIVE_BALANCE + assert not spec.is_partially_withdrawable_validator(validator, state.balances[validator_index]) + + yield from run_process_partial_withdrawals(spec, state, 0) + + +@with_capella_and_later +@spec_state_test +def test_success_no_excess_balance(spec, state): + validator_index = len(state.validators) // 2 + # To be partially withdrawable, the validator needs an excess balance + set_eth1_withdrawal_credential_with_balance(spec, state, validator_index, spec.MAX_EFFECTIVE_BALANCE) + validator = state.validators[validator_index] + + assert validator.effective_balance == spec.MAX_EFFECTIVE_BALANCE + assert not spec.is_partially_withdrawable_validator(validator, state.balances[validator_index]) + + yield from run_process_partial_withdrawals(spec, state, 0) + + +@with_capella_and_later +@spec_state_test +def test_success_excess_balance_but_no_max_effective_balance(spec, state): + validator_index = len(state.validators) // 2 + set_validator_partially_withdrawable(spec, state, validator_index) + validator = state.validators[validator_index] + + # To be partially withdrawable, the validator needs both a maxed out effective balance and an excess balance + validator.effective_balance = spec.MAX_EFFECTIVE_BALANCE - 1 + + assert not spec.is_partially_withdrawable_validator(validator, state.balances[validator_index]) + + yield from run_process_partial_withdrawals(spec, state, 0) + + @with_capella_and_later @spec_state_test def test_success_one_partial_withdrawable(spec, state): @@ -155,7 +203,7 @@ def test_success_max_partial_withdrawable(spec, state): @with_capella_and_later -@with_presets([MINIMAL], reason="not no enough validators with mainnet config") +@with_presets([MINIMAL], reason="not enough validators with mainnet config") @spec_state_test def test_success_max_plus_one_withdrawable(spec, state): # Sanity check that this test works for this state From f4ba8b55ee833cc9e1be014fb2fd06ea682f2d53 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 26 Sep 2022 16:39:16 +0300 Subject: [PATCH 25/37] EIP4844: Implement reverse bit ordering in KZG commitments Co-authored-by: Dankrad Feist Co-authored-by: Hsiao-Wei Wang --- setup.py | 9 +++- specs/eip4844/polynomial-commitments.md | 59 ++++++++++++++++++++++--- 2 files changed, 62 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 72b011f47..34aca0aed 100644 --- a/setup.py +++ b/setup.py @@ -589,9 +589,16 @@ from eth2spec.bellatrix import {preset_name} as bellatrix from eth2spec.utils.ssz.ssz_impl import serialize as ssz_serialize ''' + + @classmethod + def preparations(cls): + return super().preparations() + '\n' + ''' +T = TypeVar('T') # For generic function +''' + @classmethod def sundry_functions(cls) -> str: - return super().sundry_functions() + ''' + return super().sundry_functions() + '\n\n' + ''' # TODO: for mainnet, load pre-generated trusted setup file to reduce building time. # TESTING_FIELD_ELEMENTS_PER_BLOB is hardcoded copy from minimal presets TESTING_FIELD_ELEMENTS_PER_BLOB = 4 diff --git a/specs/eip4844/polynomial-commitments.md b/specs/eip4844/polynomial-commitments.md index 6ebd3fd3a..66a2e9e11 100644 --- a/specs/eip4844/polynomial-commitments.md +++ b/specs/eip4844/polynomial-commitments.md @@ -12,6 +12,10 @@ - [Preset](#preset) - [Trusted setup](#trusted-setup) - [Helper functions](#helper-functions) + - [Bit-reversal permutation](#bit-reversal-permutation) + - [`is_power_of_two`](#is_power_of_two) + - [`reverse_bits`](#reverse_bits) + - [`bit_reversal_permutation`](#bit_reversal_permutation) - [BLS12-381 helpers](#bls12-381-helpers) - [`bls_modular_inverse`](#bls_modular_inverse) - [`div`](#div) @@ -64,6 +68,47 @@ but reusing the `mainnet` settings in public networks is a critical security req ## Helper functions +### Bit-reversal permutation + +All polynomials (which are always given in Lagrange form) should be interpreted as being in +bit-reversal permutation. In practice, clients can implement this by storing the lists +`KZG_SETUP_LAGRANGE` and `ROOTS_OF_UNITY` in bit-reversal permutation, so these functions only +have to be called once at startup. + +#### `is_power_of_two` + +```python +def is_power_of_two(value: int) -> bool: + """ + Check if ``value`` is a power of two integer. + """ + return (value > 0) and (value & (value - 1) == 0) +``` + +#### `reverse_bits` + +```python +def reverse_bits(n: int, order: int) -> int: + """ + Reverse the bit order of an integer n + """ + assert is_power_of_two(order) + # Convert n to binary with the same number of bits as "order" - 1, then reverse its bit order + return int(('{:0' + str(order.bit_length() - 1) + 'b}').format(n)[::-1], 2) +``` + +#### `bit_reversal_permutation` + +```python +def bit_reversal_permutation(l: Sequence[T]) -> Sequence[T]: + """ + Return a copy with bit-reversed permutation. This operation is idempotent. + + The input and output are a sequence of generic type ``T`` objects. + """ + return [l[reverse_bits(i, len(l))] for i in range(len(l))] +``` + ### BLS12-381 helpers #### `bls_modular_inverse` @@ -123,7 +168,7 @@ KZG core functions. These are also defined in EIP-4844 execution specs. ```python def blob_to_kzg_commitment(blob: Blob) -> KZGCommitment: - return g1_lincomb(KZG_SETUP_LAGRANGE, blob) + return g1_lincomb(bit_reversal_permutation(KZG_SETUP_LAGRANGE), blob) ``` #### `verify_kzg_proof` @@ -149,7 +194,9 @@ def verify_kzg_proof(polynomial_kzg: KZGCommitment, ```python def compute_kzg_proof(polynomial: Sequence[BLSFieldElement], z: BLSFieldElement) -> KZGProof: - """Compute KZG proof at point `z` with `polynomial` being in evaluation form""" + """ + Compute KZG proof at point `z` with `polynomial` being in evaluation form + """ # To avoid SSZ overflow/underflow, convert element into int polynomial = [int(i) for i in polynomial] @@ -161,11 +208,11 @@ def compute_kzg_proof(polynomial: Sequence[BLSFieldElement], z: BLSFieldElement) # Make sure we won't divide by zero during division assert z not in ROOTS_OF_UNITY - denominator_poly = [(x - z) % BLS_MODULUS for x in ROOTS_OF_UNITY] + denominator_poly = [(x - z) % BLS_MODULUS for x in bit_reversal_permutation(ROOTS_OF_UNITY)] # Calculate quotient polynomial by doing point-by-point division quotient_polynomial = [div(a, b) for a, b in zip(polynomial_shifted, denominator_poly)] - return KZGProof(g1_lincomb(KZG_SETUP_LAGRANGE, quotient_polynomial)) + return KZGProof(g1_lincomb(bit_reversal_permutation(KZG_SETUP_LAGRANGE), quotient_polynomial)) ``` ### Polynomials @@ -187,9 +234,11 @@ def evaluate_polynomial_in_evaluation_form(polynomial: Sequence[BLSFieldElement] # Make sure we won't divide by zero during division assert z not in ROOTS_OF_UNITY + roots_of_unity_brp = bit_reversal_permutation(ROOTS_OF_UNITY) + result = 0 for i in range(width): - result += div(int(polynomial[i]) * int(ROOTS_OF_UNITY[i]), (z - ROOTS_OF_UNITY[i])) + result += div(int(polynomial[i]) * int(roots_of_unity_brp[i]), (z - roots_of_unity_brp[i])) result = result * (pow(z, width, BLS_MODULUS) - 1) * inverse_width % BLS_MODULUS return result ``` From d197ed1451eb3d441438da5bc6d8b27d9e6c5776 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 26 Sep 2022 18:57:00 +0300 Subject: [PATCH 26/37] EIP4844: Introduce bytes_to_bls_field() helper Improves separation between BLS cryptography and Ethereum SSZ logic. Now the BLS library just implements bytes_to_bls_field(). Then hash_to_bls_field() does the Ethereum SSZ magic and calls bytes_to_bls_field(). --- specs/eip4844/polynomial-commitments.md | 11 +++++++++++ specs/eip4844/validator.md | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/specs/eip4844/polynomial-commitments.md b/specs/eip4844/polynomial-commitments.md index 66a2e9e11..45f942b0f 100644 --- a/specs/eip4844/polynomial-commitments.md +++ b/specs/eip4844/polynomial-commitments.md @@ -17,6 +17,7 @@ - [`reverse_bits`](#reverse_bits) - [`bit_reversal_permutation`](#bit_reversal_permutation) - [BLS12-381 helpers](#bls12-381-helpers) + - [`bytes_to_bls_field`](#bytes_to_bls_field) - [`bls_modular_inverse`](#bls_modular_inverse) - [`div`](#div) - [`g1_lincomb`](#g1_lincomb) @@ -111,6 +112,16 @@ def bit_reversal_permutation(l: Sequence[T]) -> Sequence[T]: ### BLS12-381 helpers +#### `bytes_to_bls_field` + +```python +def bytes_to_bls_field(b: Bytes32) -> BLSFieldElement: + """ + Convert bytes to a BLS field scalar. The output is not uniform over the BLS field. + """ + return int.from_bytes(b, "little") % BLS_MODULUS +``` + #### `bls_modular_inverse` ```python diff --git a/specs/eip4844/validator.md b/specs/eip4844/validator.md index f624c5157..a1faffa25 100644 --- a/specs/eip4844/validator.md +++ b/specs/eip4844/validator.md @@ -96,7 +96,7 @@ def hash_to_bls_field(x: Container) -> BLSFieldElement: Compute 32-byte hash of serialized container and convert it to BLS field. The output is not uniform over the BLS field. """ - return int.from_bytes(hash(ssz_serialize(x)), "little") % BLS_MODULUS + return bytes_to_bls_field(hash(ssz_serialize(x))) ``` ### `compute_powers` From 9ac605cc5ed0602f16b9b322b854f572e2ee1b62 Mon Sep 17 00:00:00 2001 From: Ramana Kumar Date: Tue, 27 Sep 2022 12:13:56 +0100 Subject: [PATCH 27/37] Fix incorrect comment bit-reversal permutation is not idempotent --- specs/eip4844/polynomial-commitments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/eip4844/polynomial-commitments.md b/specs/eip4844/polynomial-commitments.md index 45f942b0f..69cb5e0e6 100644 --- a/specs/eip4844/polynomial-commitments.md +++ b/specs/eip4844/polynomial-commitments.md @@ -103,7 +103,7 @@ def reverse_bits(n: int, order: int) -> int: ```python def bit_reversal_permutation(l: Sequence[T]) -> Sequence[T]: """ - Return a copy with bit-reversed permutation. This operation is idempotent. + Return a copy with bit-reversed permutation. The permutation is an involution (inverts itself). The input and output are a sequence of generic type ``T`` objects. """ From 44424c408038221896780854b93a3c227b2d1201 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 28 Sep 2022 12:22:53 +0800 Subject: [PATCH 28/37] Bump dep packages version and fix lint issues --- Makefile | 2 +- setup.py | 38 ++++++++++------- specs/eip4844/polynomial-commitments.md | 4 +- .../test/bellatrix/sync/test_optimistic.py | 4 +- tests/generators/bls/main.py | 42 +++++++++---------- 5 files changed, 48 insertions(+), 42 deletions(-) diff --git a/Makefile b/Makefile index 725e64de8..5f0706dd9 100644 --- a/Makefile +++ b/Makefile @@ -143,7 +143,7 @@ lint: pyspec lint_generators: pyspec . venv/bin/activate; cd $(TEST_GENERATORS_DIR); \ - flake8 --config $(LINTER_CONFIG_FILE) + flake8 --config $(LINTER_CONFIG_FILE) compile_deposit_contract: @cd $(SOLIDITY_DEPOSIT_CONTRACT_DIR) diff --git a/setup.py b/setup.py index 34aca0aed..6db4aa870 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ from collections import OrderedDict def installPackage(package: str): subprocess.check_call([sys.executable, '-m', 'pip', 'install', package]) -RUAMEL_YAML_VERSION = "ruamel.yaml==0.16.5" +RUAMEL_YAML_VERSION = "ruamel.yaml==0.17.21" try: import ruamel.yaml except ImportError: @@ -78,6 +78,7 @@ class VariableDefinition(NamedTuple): type_name: Optional[str] value: str comment: Optional[str] # e.g. "noqa: E501" + type_hint: Optional[str] # e.g., "Final" class SpecObject(NamedTuple): @@ -152,18 +153,18 @@ def _get_eth2_spec_comment(child: LinkRefDef) -> Optional[str]: return title[len(ETH2_SPEC_COMMENT_PREFIX):].strip() -def _parse_value(name: str, typed_value: str) -> VariableDefinition: +def _parse_value(name: str, typed_value: str, type_hint: Optional[str]=None) -> VariableDefinition: comment = None if name == "BLS12_381_Q": comment = "noqa: E501" typed_value = typed_value.strip() if '(' not in typed_value: - return VariableDefinition(type_name=None, value=typed_value, comment=comment) + return VariableDefinition(type_name=None, value=typed_value, comment=comment, type_hint=type_hint) i = typed_value.index('(') type_name = typed_value[:i] - return VariableDefinition(type_name=type_name, value=typed_value[i+1:-1], comment=comment) + return VariableDefinition(type_name=type_name, value=typed_value[i+1:-1], comment=comment, type_hint=type_hint) def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str]) -> SpecObject: @@ -241,10 +242,13 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str]) -> value_def = _parse_value(name, value) if name in preset: - preset_vars[name] = VariableDefinition(value_def.type_name, preset[name], value_def.comment) + preset_vars[name] = VariableDefinition(value_def.type_name, preset[name], value_def.comment, None) elif name in config: - config_vars[name] = VariableDefinition(value_def.type_name, config[name], value_def.comment) + config_vars[name] = VariableDefinition(value_def.type_name, config[name], value_def.comment, None) else: + if name == 'ENDIANNESS': + # Deal with mypy Literal typing check + value_def = _parse_value(name, value, type_hint='Final') constant_vars[name] = value_def elif isinstance(child, LinkRefDef): @@ -337,7 +341,7 @@ from dataclasses import ( field, ) from typing import ( - Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar, NamedTuple + Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar, NamedTuple, Final ) from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes @@ -703,7 +707,10 @@ def objects_to_spec(preset_name: str, def format_constant(name: str, vardef: VariableDefinition) -> str: if vardef.type_name is None: - out = f'{name} = {vardef.value}' + if vardef.type_hint is None: + out = f'{name} = {vardef.value}' + else: + out = f'{name}: {vardef.type_hint} = {vardef.value}' else: out = f'{name} = {vardef.type_name}({vardef.value})' if vardef.comment is not None: @@ -1115,19 +1122,18 @@ setup( python_requires=">=3.8, <4", extras_require={ "test": ["pytest>=4.4", "pytest-cov", "pytest-xdist"], - "lint": ["flake8==3.7.7", "mypy==0.812", "pylint==2.12.2"], - "generator": ["python-snappy==0.5.4", "filelock"], + "lint": ["flake8==5.0.4", "mypy==0.981", "pylint==2.15.3"], + "generator": ["python-snappy==0.6.1", "filelock"], }, install_requires=[ - "eth-utils>=1.3.0,<2", - "eth-typing>=2.1.0,<3.0.0", - "pycryptodome==3.9.4", - "py_ecc==5.2.0", + "eth-utils>=2.0.0,<3", + "eth-typing>=3.2.0,<4.0.0", + "pycryptodome==3.15.0", + "py_ecc==6.0.0", "milagro_bls_binding==1.9.0", - "dataclasses==0.6", "remerkleable==0.1.24", RUAMEL_YAML_VERSION, - "lru-dict==1.1.6", + "lru-dict==1.1.8", MARKO_VERSION, ] ) diff --git a/specs/eip4844/polynomial-commitments.md b/specs/eip4844/polynomial-commitments.md index 45f942b0f..9d4f641b5 100644 --- a/specs/eip4844/polynomial-commitments.md +++ b/specs/eip4844/polynomial-commitments.md @@ -101,13 +101,13 @@ def reverse_bits(n: int, order: int) -> int: #### `bit_reversal_permutation` ```python -def bit_reversal_permutation(l: Sequence[T]) -> Sequence[T]: +def bit_reversal_permutation(sequence: Sequence[T]) -> Sequence[T]: """ Return a copy with bit-reversed permutation. This operation is idempotent. The input and output are a sequence of generic type ``T`` objects. """ - return [l[reverse_bits(i, len(l))] for i in range(len(l))] + return [sequence[reverse_bits(i, len(sequence))] for i in range(len(sequence))] ``` ### BLS12-381 helpers diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/sync/test_optimistic.py b/tests/core/pyspec/eth2spec/test/bellatrix/sync/test_optimistic.py index 4fb8adbaf..c849ccbd8 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/sync/test_optimistic.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/sync/test_optimistic.py @@ -46,7 +46,7 @@ def test_from_syncing_to_invalid(spec, state): # Block 0 block_0 = build_empty_block_for_next_slot(spec, state) - block_0.body.execution_payload.block_hash = spec.hash(bytes(f'block_0', 'UTF-8')) + block_0.body.execution_payload.block_hash = spec.hash(bytes('block_0', 'UTF-8')) signed_block = state_transition_and_sign_block(spec, state, block_0) yield from add_optimistic_block(spec, mega_store, signed_block, test_steps, status=PayloadStatusV1Status.VALID) assert spec.get_head(mega_store.fc_store) == mega_store.opt_store.head_block_root @@ -84,7 +84,7 @@ def test_from_syncing_to_invalid(spec, state): # Now add block 4 to chain `b` with INVALID block = build_empty_block_for_next_slot(spec, state) - block.body.execution_payload.block_hash = spec.hash(bytes(f'chain_b_3', 'UTF-8')) + block.body.execution_payload.block_hash = spec.hash(bytes('chain_b_3', 'UTF-8')) block.body.execution_payload.parent_hash = signed_blocks_b[-1].message.body.execution_payload.block_hash signed_block = state_transition_and_sign_block(spec, state, block) payload_status = PayloadStatusV1( diff --git a/tests/generators/bls/main.py b/tests/generators/bls/main.py index 60c31d9d9..dca5ae7dd 100644 --- a/tests/generators/bls/main.py +++ b/tests/generators/bls/main.py @@ -89,7 +89,7 @@ def case01_sign(): # Edge case: privkey == 0 expect_exception(bls.Sign, ZERO_PRIVKEY, message) expect_exception(milagro_bls.Sign, ZERO_PRIVKEY_BYTES, message) - yield f'sign_case_zero_privkey', { + yield 'sign_case_zero_privkey', { 'input': { 'privkey': encode_hex(ZERO_PRIVKEY_BYTES), 'message': encode_hex(message), @@ -153,7 +153,7 @@ def case02_verify(): # Invalid pubkey and signature with the point at infinity assert not bls.Verify(G1_POINT_AT_INFINITY, SAMPLE_MESSAGE, G2_POINT_AT_INFINITY) assert not milagro_bls.Verify(G1_POINT_AT_INFINITY, SAMPLE_MESSAGE, G2_POINT_AT_INFINITY) - yield f'verify_infinity_pubkey_and_infinity_signature', { + yield 'verify_infinity_pubkey_and_infinity_signature', { 'input': { 'pubkey': encode_hex(G1_POINT_AT_INFINITY), 'message': encode_hex(SAMPLE_MESSAGE), @@ -177,7 +177,7 @@ def case03_aggregate(): expect_exception(bls.Aggregate, []) # No signatures to aggregate. Follow IETF BLS spec, return `None` to represent INVALID. # https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-04#section-2.8 - yield f'aggregate_na_signatures', { + yield 'aggregate_na_signatures', { 'input': [], 'output': None, } @@ -185,7 +185,7 @@ def case03_aggregate(): # Valid to aggregate G2 point at infinity aggregate_sig = bls.Aggregate([G2_POINT_AT_INFINITY]) assert aggregate_sig == milagro_bls.Aggregate([G2_POINT_AT_INFINITY]) == G2_POINT_AT_INFINITY - yield f'aggregate_infinity_signature', { + yield 'aggregate_infinity_signature', { 'input': [encode_hex(G2_POINT_AT_INFINITY)], 'output': encode_hex(aggregate_sig), } @@ -244,7 +244,7 @@ def case04_fast_aggregate_verify(): # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == Z1_SIGNATURE assert not bls.FastAggregateVerify([], message, G2_POINT_AT_INFINITY) assert not milagro_bls.FastAggregateVerify([], message, G2_POINT_AT_INFINITY) - yield f'fast_aggregate_verify_na_pubkeys_and_infinity_signature', { + yield 'fast_aggregate_verify_na_pubkeys_and_infinity_signature', { 'input': { 'pubkeys': [], 'message': encode_hex(message), @@ -256,7 +256,7 @@ def case04_fast_aggregate_verify(): # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == 0x00... assert not bls.FastAggregateVerify([], message, ZERO_SIGNATURE) assert not milagro_bls.FastAggregateVerify([], message, ZERO_SIGNATURE) - yield f'fast_aggregate_verify_na_pubkeys_and_zero_signature', { + yield 'fast_aggregate_verify_na_pubkeys_and_zero_signature', { 'input': { 'pubkeys': [], 'message': encode_hex(message), @@ -272,7 +272,7 @@ def case04_fast_aggregate_verify(): aggregate_signature = bls.Aggregate(signatures) assert not bls.FastAggregateVerify(pubkeys_with_infinity, SAMPLE_MESSAGE, aggregate_signature) assert not milagro_bls.FastAggregateVerify(pubkeys_with_infinity, SAMPLE_MESSAGE, aggregate_signature) - yield f'fast_aggregate_verify_infinity_pubkey', { + yield 'fast_aggregate_verify_infinity_pubkey', { 'input': { 'pubkeys': [encode_hex(pubkey) for pubkey in pubkeys_with_infinity], 'message': encode_hex(SAMPLE_MESSAGE), @@ -300,7 +300,7 @@ def case05_aggregate_verify(): aggregate_signature = bls.Aggregate(sigs) assert bls.AggregateVerify(pubkeys, messages, aggregate_signature) assert milagro_bls.AggregateVerify(pubkeys, messages, aggregate_signature) - yield f'aggregate_verify_valid', { + yield 'aggregate_verify_valid', { 'input': { 'pubkeys': pubkeys_serial, 'messages': messages_serial, @@ -312,7 +312,7 @@ def case05_aggregate_verify(): tampered_signature = aggregate_signature[:4] + b'\xff\xff\xff\xff' assert not bls.AggregateVerify(pubkey, messages, tampered_signature) assert not milagro_bls.AggregateVerify(pubkeys, messages, tampered_signature) - yield f'aggregate_verify_tampered_signature', { + yield 'aggregate_verify_tampered_signature', { 'input': { 'pubkeys': pubkeys_serial, 'messages': messages_serial, @@ -324,7 +324,7 @@ def case05_aggregate_verify(): # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == Z1_SIGNATURE assert not bls.AggregateVerify([], [], G2_POINT_AT_INFINITY) assert not milagro_bls.AggregateVerify([], [], G2_POINT_AT_INFINITY) - yield f'aggregate_verify_na_pubkeys_and_infinity_signature', { + yield 'aggregate_verify_na_pubkeys_and_infinity_signature', { 'input': { 'pubkeys': [], 'messages': [], @@ -336,7 +336,7 @@ def case05_aggregate_verify(): # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == 0x00... assert not bls.AggregateVerify([], [], ZERO_SIGNATURE) assert not milagro_bls.AggregateVerify([], [], ZERO_SIGNATURE) - yield f'aggregate_verify_na_pubkeys_and_zero_signature', { + yield 'aggregate_verify_na_pubkeys_and_zero_signature', { 'input': { 'pubkeys': [], 'messages': [], @@ -350,7 +350,7 @@ def case05_aggregate_verify(): messages_with_sample = messages + [SAMPLE_MESSAGE] assert not bls.AggregateVerify(pubkeys_with_infinity, messages_with_sample, aggregate_signature) assert not milagro_bls.AggregateVerify(pubkeys_with_infinity, messages_with_sample, aggregate_signature) - yield f'aggregate_verify_infinity_pubkey', { + yield 'aggregate_verify_infinity_pubkey', { 'input': { 'pubkeys': [encode_hex(pubkey) for pubkey in pubkeys_with_infinity], 'messages': [encode_hex(message) for message in messages_with_sample], @@ -375,7 +375,7 @@ def case06_eth_aggregate_pubkeys(): # Valid pubkeys aggregate_pubkey = spec.eth_aggregate_pubkeys(PUBKEYS) assert aggregate_pubkey == milagro_bls._AggregatePKs(PUBKEYS) - yield f'eth_aggregate_pubkeys_valid_pubkeys', { + yield 'eth_aggregate_pubkeys_valid_pubkeys', { 'input': [encode_hex(pubkey) for pubkey in PUBKEYS], 'output': encode_hex(aggregate_pubkey), } @@ -383,7 +383,7 @@ def case06_eth_aggregate_pubkeys(): # Invalid pubkeys -- len(pubkeys) == 0 expect_exception(spec.eth_aggregate_pubkeys, []) expect_exception(milagro_bls._AggregatePKs, []) - yield f'eth_aggregate_pubkeys_empty_list', { + yield 'eth_aggregate_pubkeys_empty_list', { 'input': [], 'output': None, } @@ -391,7 +391,7 @@ def case06_eth_aggregate_pubkeys(): # Invalid pubkeys -- [ZERO_PUBKEY] expect_exception(spec.eth_aggregate_pubkeys, [ZERO_PUBKEY]) expect_exception(milagro_bls._AggregatePKs, [ZERO_PUBKEY]) - yield f'eth_aggregate_pubkeys_zero_pubkey', { + yield 'eth_aggregate_pubkeys_zero_pubkey', { 'input': [encode_hex(ZERO_PUBKEY)], 'output': None, } @@ -399,7 +399,7 @@ def case06_eth_aggregate_pubkeys(): # Invalid pubkeys -- G1 point at infinity expect_exception(spec.eth_aggregate_pubkeys, [G1_POINT_AT_INFINITY]) expect_exception(milagro_bls._AggregatePKs, [G1_POINT_AT_INFINITY]) - yield f'eth_aggregate_pubkeys_infinity_pubkey', { + yield 'eth_aggregate_pubkeys_infinity_pubkey', { 'input': [encode_hex(G1_POINT_AT_INFINITY)], 'output': None, } @@ -408,7 +408,7 @@ def case06_eth_aggregate_pubkeys(): x40_pubkey = b'\x40' + b'\00' * 47 expect_exception(spec.eth_aggregate_pubkeys, [x40_pubkey]) expect_exception(milagro_bls._AggregatePKs, [x40_pubkey]) - yield f'eth_aggregate_pubkeys_x40_pubkey', { + yield 'eth_aggregate_pubkeys_x40_pubkey', { 'input': [encode_hex(x40_pubkey)], 'output': None, } @@ -455,7 +455,7 @@ def case07_eth_fast_aggregate_verify(): tampered_signature = aggregate_signature[:-4] + b'\xff\xff\xff\xff' identifier = f'{pubkeys_serial}_{encode_hex(message)}' assert not spec.eth_fast_aggregate_verify(pubkeys, message, tampered_signature) - yield f'eth_fast_aggregate_verify_tampered_signature_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { + yield 'eth_fast_aggregate_verify_tampered_signature_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkeys': pubkeys_serial, 'message': encode_hex(message), @@ -466,7 +466,7 @@ def case07_eth_fast_aggregate_verify(): # NOTE: Unlike `FastAggregateVerify`, len(pubkeys) == 0 and signature == G2_POINT_AT_INFINITY is VALID assert spec.eth_fast_aggregate_verify([], message, G2_POINT_AT_INFINITY) - yield f'eth_fast_aggregate_verify_na_pubkeys_and_infinity_signature', { + yield 'eth_fast_aggregate_verify_na_pubkeys_and_infinity_signature', { 'input': { 'pubkeys': [], 'message': encode_hex(message), @@ -477,7 +477,7 @@ def case07_eth_fast_aggregate_verify(): # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == 0x00... assert not spec.eth_fast_aggregate_verify([], message, ZERO_SIGNATURE) - yield f'eth_fast_aggregate_verify_na_pubkeys_and_zero_signature', { + yield 'eth_fast_aggregate_verify_na_pubkeys_and_zero_signature', { 'input': { 'pubkeys': [], 'message': encode_hex(message), @@ -492,7 +492,7 @@ def case07_eth_fast_aggregate_verify(): signatures = [bls.Sign(privkey, SAMPLE_MESSAGE) for privkey in PRIVKEYS] aggregate_signature = bls.Aggregate(signatures) assert not spec.eth_fast_aggregate_verify(pubkeys_with_infinity, SAMPLE_MESSAGE, aggregate_signature) - yield f'eth_fast_aggregate_verify_infinity_pubkey', { + yield 'eth_fast_aggregate_verify_infinity_pubkey', { 'input': { 'pubkeys': [encode_hex(pubkey) for pubkey in pubkeys_with_infinity], 'message': encode_hex(SAMPLE_MESSAGE), From 93cb17dc7f14114e382e1657841d22cdc885c0ff Mon Sep 17 00:00:00 2001 From: terence tsao Date: Thu, 29 Sep 2022 07:36:39 -0700 Subject: [PATCH 29/37] Eip4844: various fixes --- specs/eip4844/p2p-interface.md | 13 +++++-------- specs/eip4844/validator.md | 2 +- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/specs/eip4844/p2p-interface.md b/specs/eip4844/p2p-interface.md index 5c9c46f2c..081337dbd 100644 --- a/specs/eip4844/p2p-interface.md +++ b/specs/eip4844/p2p-interface.md @@ -33,10 +33,10 @@ The specification of these changes continues in the same format as the network s ## Configuration -| Name | Value | Description | -|------------------------------------------|-------------------------------|---------------------------------------------------------------------| -| `MAX_REQUEST_BLOBS_SIDECARS` | `2**7` (= 128) | Maximum number of blobs sidecars in a single request | -| `MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS` | `2**13` (= 8192, ~1.2 months) | The minimum epoch range over which a node must serve blobs sidecars | +| Name | Value | Description | +|------------------------------------------|------------------------------------|---------------------------------------------------------------------| +| `MAX_REQUEST_BLOBS_SIDECARS` | `2**7` (= 128) | Maximum number of blobs sidecars in a single request | +| `MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS` | `2**13epoch` (= 8192, ~1.2 months) | The minimum epoch range over which a node must serve blobs sidecars | ## Containers @@ -207,10 +207,7 @@ or disconnected at any time. *Note*: The above requirement implies that nodes that start from a recent weak subjectivity checkpoint MUST backfill the local blobs database to at least epoch `current_epoch - MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS` -to be fully compliant with `BlobsSidecarsByRange` requests. To safely perform such a -backfill of blocks to the recent state, the node MUST validate both (1) the -proposer signatures and (2) that the blocks form a valid chain up to the most -recent block referenced in the weak subjectivity state. +to be fully compliant with `BlobsSidecarsByRange` requests. *Note*: Although clients that bootstrap from a weak subjectivity checkpoint can begin participating in the networking immediately, other peers MAY diff --git a/specs/eip4844/validator.md b/specs/eip4844/validator.md index a1faffa25..c29a7c414 100644 --- a/specs/eip4844/validator.md +++ b/specs/eip4844/validator.md @@ -245,7 +245,7 @@ def get_signed_blobs_sidecar(state: BeaconState, blobs_sidecar: BlobsSidecar, pr return SignedBlobsSidecar(message=blobs_sidecar, signature=signature) ``` -This `signed_blobs_sidecar` is then published to the global `blobs_sidecar` topic as soon as the `beacon_block` is published. +This `signed_blobs_sidecar` is then published to the global `blobs_sidecar` topic as soon as the `signed_beacon_block` is published. After publishing the sidecar peers on the network may request the sidecar through sync-requests, or a local user may be interested. The validator MUST hold on to blobs for `MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS` epochs and serve when capable, From 06c91e4843ea3675b4de7e10e1fd5e662e677741 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Thu, 29 Sep 2022 07:53:52 -0700 Subject: [PATCH 30/37] Try a different format for epoch --- specs/eip4844/p2p-interface.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/eip4844/p2p-interface.md b/specs/eip4844/p2p-interface.md index 081337dbd..7be539b40 100644 --- a/specs/eip4844/p2p-interface.md +++ b/specs/eip4844/p2p-interface.md @@ -33,10 +33,10 @@ The specification of these changes continues in the same format as the network s ## Configuration -| Name | Value | Description | -|------------------------------------------|------------------------------------|---------------------------------------------------------------------| -| `MAX_REQUEST_BLOBS_SIDECARS` | `2**7` (= 128) | Maximum number of blobs sidecars in a single request | -| `MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS` | `2**13epoch` (= 8192, ~1.2 months) | The minimum epoch range over which a node must serve blobs sidecars | +| Name | Value | Description | +|------------------------------------------|-------------------------------------|---------------------------------------------------------------------| +| `MAX_REQUEST_BLOBS_SIDECARS` | `2**7` (= 128) | Maximum number of blobs sidecars in a single request | +| `MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS` | `2**13` (= 8192epochs, ~1.2 months) | The minimum epoch range over which a node must serve blobs sidecars | ## Containers From 779e6e7b7e18781418f11df423e6ad18955a93bd Mon Sep 17 00:00:00 2001 From: terencechain Date: Thu, 29 Sep 2022 13:43:28 -0700 Subject: [PATCH 31/37] Update specs/eip4844/p2p-interface.md Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> --- specs/eip4844/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/eip4844/p2p-interface.md b/specs/eip4844/p2p-interface.md index 7be539b40..eab8815a4 100644 --- a/specs/eip4844/p2p-interface.md +++ b/specs/eip4844/p2p-interface.md @@ -36,7 +36,7 @@ The specification of these changes continues in the same format as the network s | Name | Value | Description | |------------------------------------------|-------------------------------------|---------------------------------------------------------------------| | `MAX_REQUEST_BLOBS_SIDECARS` | `2**7` (= 128) | Maximum number of blobs sidecars in a single request | -| `MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS` | `2**13` (= 8192epochs, ~1.2 months) | The minimum epoch range over which a node must serve blobs sidecars | +| `MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS` | `2**13` (= 8192 epochs, ~1.2 months) | The minimum epoch range over which a node must serve blobs sidecars | ## Containers From 807650e8a165c3930fd5b6604f511ad7f0779373 Mon Sep 17 00:00:00 2001 From: Ramana Kumar Date: Sat, 1 Oct 2022 15:25:47 +0100 Subject: [PATCH 32/37] Fix some type annotations for blobs blobs should not be Sequence[BLSFieldElement], it should be Sequence[Sequence[BLSFieldElement]]. But we can be more specific and use Sequence[Blob]. --- specs/eip4844/validator.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/eip4844/validator.md b/specs/eip4844/validator.md index c29a7c414..e1bcf9564 100644 --- a/specs/eip4844/validator.md +++ b/specs/eip4844/validator.md @@ -117,7 +117,7 @@ def compute_powers(x: BLSFieldElement, n: uint64) -> Sequence[BLSFieldElement]: ```python def compute_aggregated_poly_and_commitment( - blobs: Sequence[Sequence[BLSFieldElement]], + blobs: Sequence[Blob], kzg_commitments: Sequence[KZGCommitment]) -> Tuple[Polynomial, KZGCommitment]: """ Return the aggregated polynomial and aggregated KZG commitment. @@ -167,7 +167,7 @@ def validate_blobs_sidecar(slot: Slot, ### `compute_proof_from_blobs` ```python -def compute_proof_from_blobs(blobs: Sequence[BLSFieldElement]) -> KZGProof: +def compute_proof_from_blobs(blobs: Sequence[Blob]) -> KZGProof: commitments = [blob_to_kzg_commitment(blob) for blob in blobs] aggregated_poly, aggregated_poly_commitment = compute_aggregated_poly_and_commitment(blobs, commitments) x = hash_to_bls_field(PolynomialAndCommitment( @@ -206,7 +206,7 @@ use the `payload_id` to retrieve `blobs` and `blob_kzg_commitments` via `get_blo ```python def validate_blobs_and_kzg_commitments(execution_payload: ExecutionPayload, - blobs: Sequence[BLSFieldElement], + blobs: Sequence[Blob], blob_kzg_commitments: Sequence[KZGCommitment]) -> None: # Optionally sanity-check that the KZG commitments match the versioned hashes in the transactions assert verify_kzg_commitments_against_transactions(execution_payload.transactions, blob_kzg_commitments) From fda2a69ff7b9809cd6b3fdfb7c9a5dc68f01d55e Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Sun, 2 Oct 2022 12:19:55 +0200 Subject: [PATCH 33/37] Typo in sync committee duties description I assume it refers to producing a signature for the last slot of phase0 fork --- specs/altair/validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/altair/validator.md b/specs/altair/validator.md index 626535374..013a51649 100644 --- a/specs/altair/validator.md +++ b/specs/altair/validator.md @@ -146,7 +146,7 @@ This function is a predicate indicating the presence or absence of the validator *Note*: Being assigned to a sync committee for a given `slot` means that the validator produces and broadcasts signatures for `slot - 1` for inclusion in `slot`. This means that when assigned to an `epoch` sync committee signatures must be produced and broadcast for slots on range `[compute_start_slot_at_epoch(epoch) - 1, compute_start_slot_at_epoch(epoch) + SLOTS_PER_EPOCH - 1)` rather than for the range `[compute_start_slot_at_epoch(epoch), compute_start_slot_at_epoch(epoch) + SLOTS_PER_EPOCH)`. -To reduce complexity during the Altair fork, sync committees are not expected to produce signatures for `compute_epoch_at_slot(ALTAIR_FORK_EPOCH) - 1`. +To reduce complexity during the Altair fork, sync committees are not expected to produce signatures for `compute_start_slot_at_epoch(ALTAIR_FORK_EPOCH) - 1`. ```python def compute_sync_committee_period(epoch: Epoch) -> uint64: From 491f14c76f376fe242669a90681943b5a3c484d0 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 19 Sep 2022 11:27:09 -0600 Subject: [PATCH 34/37] some capella sanity tests --- presets/minimal/capella.yaml | 2 +- .../test_process_bls_to_execution_change.py | 29 +--- .../test_process_partial_withdrawals.py | 20 +-- .../test/capella/sanity/test_blocks.py | 135 ++++++++++++++++++ .../test/helpers/bls_to_execution_changes.py | 27 ++++ .../eth2spec/test/helpers/withdrawals.py | 14 ++ 6 files changed, 184 insertions(+), 43 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py create mode 100644 tests/core/pyspec/eth2spec/test/helpers/bls_to_execution_changes.py diff --git a/presets/minimal/capella.yaml b/presets/minimal/capella.yaml index bf78685cd..0476172a1 100644 --- a/presets/minimal/capella.yaml +++ b/presets/minimal/capella.yaml @@ -21,4 +21,4 @@ MAX_BLS_TO_EXECUTION_CHANGES: 16 # Execution # --------------------------------------------------------------- # [customized] Lower than MAX_PARTIAL_WITHDRAWALS_PER_EPOCH so not all processed in one block -MAX_WITHDRAWALS_PER_PAYLOAD: 16 +MAX_WITHDRAWALS_PER_PAYLOAD: 8 diff --git a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_bls_to_execution_change.py b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_bls_to_execution_change.py index bbac7fd36..8ff02489c 100644 --- a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_bls_to_execution_change.py +++ b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_bls_to_execution_change.py @@ -1,5 +1,5 @@ -from eth2spec.utils import bls -from eth2spec.test.helpers.keys import pubkeys, privkeys, pubkey_to_privkey +from eth2spec.test.helpers.keys import pubkeys +from eth2spec.test.helpers.bls_to_execution_changes import get_signed_address_change from eth2spec.test.context import spec_state_test, expect_assertion_error, with_capella_and_later, always_bls @@ -37,31 +37,6 @@ def run_bls_to_execution_change_processing(spec, state, signed_address_change, v yield 'post', state -def get_signed_address_change(spec, state, validator_index=None, withdrawal_pubkey=None): - if validator_index is None: - validator_index = 0 - - if withdrawal_pubkey is None: - key_index = -1 - validator_index - withdrawal_pubkey = pubkeys[key_index] - withdrawal_privkey = privkeys[key_index] - else: - withdrawal_privkey = pubkey_to_privkey[withdrawal_pubkey] - - domain = spec.get_domain(state, spec.DOMAIN_BLS_TO_EXECUTION_CHANGE) - address_change = spec.BLSToExecutionChange( - validator_index=validator_index, - from_bls_pubkey=withdrawal_pubkey, - to_execution_address=b'\x42' * 20, - ) - - signing_root = spec.compute_signing_root(address_change, domain) - return spec.SignedBLSToExecutionChange( - message=address_change, - signature=bls.Sign(withdrawal_privkey, signing_root), - ) - - @with_capella_and_later @spec_state_test def test_success(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_partial_withdrawals.py b/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_partial_withdrawals.py index 431c2f2d3..7569d2862 100644 --- a/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_partial_withdrawals.py +++ b/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_partial_withdrawals.py @@ -8,20 +8,10 @@ from eth2spec.test.context import ( from eth2spec.test.helpers.epoch_processing import run_epoch_processing_to from eth2spec.test.helpers.state import next_epoch from eth2spec.test.helpers.random import randomize_state - - -def set_eth1_withdrawal_credential_with_balance(spec, state, index, balance): - validator = state.validators[index] - validator.withdrawal_credentials = spec.ETH1_ADDRESS_WITHDRAWAL_PREFIX + validator.withdrawal_credentials[1:] - validator.effective_balance = min(balance, spec.MAX_EFFECTIVE_BALANCE) - state.balances[index] = balance - - -def set_validator_partially_withdrawable(spec, state, index, rng=random.Random(666)): - balance = spec.MAX_EFFECTIVE_BALANCE + rng.randint(1, 100000000) - set_eth1_withdrawal_credential_with_balance(spec, state, index, balance) - - assert spec.is_partially_withdrawable_validator(state.validators[index], state.balances[index]) +from eth2spec.test.helpers.withdrawals import ( + set_validator_partially_withdrawable, + set_eth1_withdrawal_credential_with_balance, +) def run_process_partial_withdrawals(spec, state, num_expected_withdrawals=None): @@ -228,7 +218,7 @@ def run_random_partial_withdrawals_test(spec, state, rng): num_partially_withdrawable = rng.randint(0, num_validators - 1) partially_withdrawable_indices = rng.sample(range(num_validators), num_partially_withdrawable) for index in partially_withdrawable_indices: - set_validator_partially_withdrawable(spec, state, index) + set_validator_partially_withdrawable(spec, state, index, excess_balance=rng.randint(1, 1000000000)) # Note: due to the randomness and other epoch processing, some of these set as "partially withdrawable" # may not be partially withdrawable once we get to ``process_partial_withdrawals``, diff --git a/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py new file mode 100644 index 000000000..28c20a2cd --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py @@ -0,0 +1,135 @@ +from eth2spec.test.context import ( + with_capella_and_later, spec_state_test +) + +from eth2spec.test.helpers.state import ( + state_transition_and_sign_block, +) +from eth2spec.test.helpers.block import ( + build_empty_block_for_next_slot, build_empty_block, +) +from eth2spec.test.helpers.bls_to_execution_changes import get_signed_address_change +from eth2spec.test.helpers.withdrawals import ( + set_validator_fully_withdrawable, + set_validator_partially_withdrawable, +) +from eth2spec.test.helpers.voluntary_exits import prepare_signed_exits + + +@with_capella_and_later +@spec_state_test +def test_successful_bls_change(spec, state): + index = 0 + signed_address_change = get_signed_address_change(spec, state, validator_index=index) + pre_credentials = state.validators[index].withdrawal_credentials + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + block.body.bls_to_execution_changes.append(signed_address_change) + + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state + + post_credentials = state.validators[index].withdrawal_credentials + assert pre_credentials != post_credentials + assert post_credentials[:1] == spec.ETH1_ADDRESS_WITHDRAWAL_PREFIX + assert post_credentials[1:12] == b'\x00' * 11 + assert post_credentials[12:] == signed_address_change.message.to_execution_address + + +@with_capella_and_later +@spec_state_test +def test_full_withdrawal_in_epoch_transition(spec, state): + index = 0 + current_epoch = spec.get_current_epoch(state) + set_validator_fully_withdrawable(spec, state, index, current_epoch) + yield 'pre', state + + # trigger epoch transition + block = build_empty_block(spec, state, state.slot + spec.SLOTS_PER_EPOCH) + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state + + assert state.balances[index] == 0 + + +@with_capella_and_later +@spec_state_test +def test_partial_withdrawal_in_epoch_transition(spec, state): + index = state.next_withdrawal_index + set_validator_partially_withdrawable(spec, state, index, excess_balance=1000000000000) + pre_balance = state.balances[index] + pre_withdrawal_queue_len = len(state.withdrawal_queue) + + yield 'pre', state + + # trigger epoch transition + block = build_empty_block(spec, state, state.slot + spec.SLOTS_PER_EPOCH) + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state + + assert state.balances[index] < pre_balance + # Potentially less than due to sync committee penalty + assert state.balances[index] <= spec.MAX_EFFECTIVE_BALANCE + # Withdrawal is processed within the context of the block so queue empty + assert len(state.withdrawal_queue) == pre_withdrawal_queue_len + + +@with_capella_and_later +@spec_state_test +def test_many_partial_withdrawals_in_epoch_transition(spec, state): + assert len(state.validators) > spec.MAX_WITHDRAWALS_PER_PAYLOAD + assert spec.MAX_PARTIAL_WITHDRAWALS_PER_EPOCH > spec.MAX_WITHDRAWALS_PER_PAYLOAD + + for i in range(spec.MAX_WITHDRAWALS_PER_PAYLOAD + 1): + index = (i + state.next_withdrawal_index) % len(state.validators) + set_validator_partially_withdrawable(spec, state, index, excess_balance=1000000000000) + + pre_withdrawal_queue_len = len(state.withdrawal_queue) + + yield 'pre', state + + # trigger epoch transition + block = build_empty_block(spec, state, state.slot + spec.SLOTS_PER_EPOCH) + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state + + # All new partial withdrawals processed except 1 + assert len(state.withdrawal_queue) == pre_withdrawal_queue_len + 1 + + +@with_capella_and_later +@spec_state_test +def test_exit_and_bls_change(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + + index = 0 + signed_address_change = get_signed_address_change(spec, state, validator_index=index) + signed_exit = prepare_signed_exits(spec, state, [index])[0] + + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + block.body.voluntary_exits.append(signed_exit) + block.body.bls_to_execution_changes.append(signed_address_change) + + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state + + validator = state.validators[index] + balance = state.balances[index] + current_epoch = spec.get_current_epoch(state) + assert not spec.is_fully_withdrawable_validator(validator, balance, current_epoch) + assert validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH + assert spec.is_fully_withdrawable_validator(validator, balance, validator.withdrawable_epoch) diff --git a/tests/core/pyspec/eth2spec/test/helpers/bls_to_execution_changes.py b/tests/core/pyspec/eth2spec/test/helpers/bls_to_execution_changes.py new file mode 100644 index 000000000..61c84b515 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/bls_to_execution_changes.py @@ -0,0 +1,27 @@ +from eth2spec.utils import bls +from eth2spec.test.helpers.keys import pubkeys, privkeys, pubkey_to_privkey + + +def get_signed_address_change(spec, state, validator_index=None, withdrawal_pubkey=None): + if validator_index is None: + validator_index = 0 + + if withdrawal_pubkey is None: + key_index = -1 - validator_index + withdrawal_pubkey = pubkeys[key_index] + withdrawal_privkey = privkeys[key_index] + else: + withdrawal_privkey = pubkey_to_privkey[withdrawal_pubkey] + + domain = spec.get_domain(state, spec.DOMAIN_BLS_TO_EXECUTION_CHANGE) + address_change = spec.BLSToExecutionChange( + validator_index=validator_index, + from_bls_pubkey=withdrawal_pubkey, + to_execution_address=b'\x42' * 20, + ) + + signing_root = spec.compute_signing_root(address_change, domain) + return spec.SignedBLSToExecutionChange( + message=address_change, + signature=bls.Sign(withdrawal_privkey, signing_root), + ) diff --git a/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py b/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py index 739f5eb06..526ac0caa 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py +++ b/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py @@ -15,3 +15,17 @@ def set_validator_fully_withdrawable(spec, state, index, withdrawable_epoch=None state.balances[index] = 10000000000 assert spec.is_fully_withdrawable_validator(validator, state.balances[index], withdrawable_epoch) + + +def set_eth1_withdrawal_credential_with_balance(spec, state, index, balance): + validator = state.validators[index] + validator.withdrawal_credentials = spec.ETH1_ADDRESS_WITHDRAWAL_PREFIX + validator.withdrawal_credentials[1:] + validator.effective_balance = min(balance, spec.MAX_EFFECTIVE_BALANCE) + state.balances[index] = balance + + +def set_validator_partially_withdrawable(spec, state, index, excess_balance=1000000000): + set_eth1_withdrawal_credential_with_balance(spec, state, index, spec.MAX_EFFECTIVE_BALANCE + excess_balance) + validator = state.validators[index] + + assert spec.is_partially_withdrawable_validator(validator, state.balances[index]) From 092617ec1e236a485b65f555bbfd99d1bcbb2f45 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Sun, 9 Oct 2022 17:02:50 -0500 Subject: [PATCH 35/37] Remove duplicated definition of `Validator` --- specs/capella/beacon-chain.md | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/specs/capella/beacon-chain.md b/specs/capella/beacon-chain.md index 88c21d3fe..09d568f18 100644 --- a/specs/capella/beacon-chain.md +++ b/specs/capella/beacon-chain.md @@ -24,7 +24,6 @@ - [Extended Containers](#extended-containers) - [`ExecutionPayload`](#executionpayload) - [`ExecutionPayloadHeader`](#executionpayloadheader) - - [`Validator`](#validator) - [`BeaconBlockBody`](#beaconblockbody) - [`BeaconState`](#beaconstate) - [Helpers](#helpers) @@ -180,21 +179,6 @@ class ExecutionPayloadHeader(Container): withdrawals_root: Root # [New in Capella] ``` -#### `Validator` - -```python -class Validator(Container): - pubkey: BLSPubkey - withdrawal_credentials: Bytes32 # Commitment to pubkey for withdrawals - effective_balance: Gwei # Balance at stake - slashed: boolean - # Status epochs - activation_eligibility_epoch: Epoch # When criteria for activation were met - activation_epoch: Epoch - exit_epoch: Epoch - withdrawable_epoch: Epoch # When validator can withdraw funds -``` - #### `BeaconBlockBody` ```python From 24f8ec170b6572b1798af768f7ae5549438e0207 Mon Sep 17 00:00:00 2001 From: ethosdev <79124435+ethosdev@users.noreply.github.com> Date: Fri, 14 Oct 2022 21:05:45 +0000 Subject: [PATCH 36/37] Remove work-in-progress notes in Bellatrix specs (#3033) * Remove the work-in-progress note in Bellatrix spec Bellatrix is done and released. * Remove work-in-progress notes in Bellatrix specs * Remove work-in-progress notes in Bellatrix specs * Remove work-in-progress notes in Bellatrix specs --- specs/bellatrix/beacon-chain.md | 2 -- specs/bellatrix/fork-choice.md | 2 -- specs/bellatrix/fork.md | 2 -- specs/bellatrix/validator.md | 2 -- 4 files changed, 8 deletions(-) diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index 6d39c1ae8..1133cba06 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -1,7 +1,5 @@ # Bellatrix -- The Beacon Chain -**Notice**: This document is a work-in-progress for researchers and implementers. - ## Table of contents diff --git a/specs/bellatrix/fork-choice.md b/specs/bellatrix/fork-choice.md index 312768e44..94d068827 100644 --- a/specs/bellatrix/fork-choice.md +++ b/specs/bellatrix/fork-choice.md @@ -1,7 +1,5 @@ # Bellatrix -- Fork Choice -**Notice**: This document is a work-in-progress for researchers and implementers. - ## Table of contents diff --git a/specs/bellatrix/fork.md b/specs/bellatrix/fork.md index 19700e383..a114e5a5f 100644 --- a/specs/bellatrix/fork.md +++ b/specs/bellatrix/fork.md @@ -1,7 +1,5 @@ # Bellatrix -- Fork Logic -**Notice**: This document is a work-in-progress for researchers and implementers. - ## Table of contents diff --git a/specs/bellatrix/validator.md b/specs/bellatrix/validator.md index c88aa9bab..a176d7534 100644 --- a/specs/bellatrix/validator.md +++ b/specs/bellatrix/validator.md @@ -1,7 +1,5 @@ # Bellatrix -- Honest Validator -**Notice**: This document is a work-in-progress for researchers and implementers. - ## Table of contents From 6181035d5ddef4b668d0fcfee460da9888009dd9 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 15 Oct 2022 12:40:17 +0800 Subject: [PATCH 37/37] Fix typo (#3039) --- .../pyspec/eth2spec/test/phase0/fork_choice/test_ex_ante.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_ex_ante.py b/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_ex_ante.py index d93101156..0a145dfa5 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_ex_ante.py +++ b/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_ex_ante.py @@ -196,7 +196,7 @@ def test_ex_ante_attestations_is_greater_than_proposer_boost_with_boost(spec, st def test_ex_ante_sandwich_without_attestations(spec, state): """ Simple Sandwich test with boost and no attestations. - Obejcts: + Objects: Block A - slot N Block B (parent A) - slot N+1 Block C (parent A) - slot N+2