From dab5a936c4b1fdfae0cc9c9b5c98ac0643097bd7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 28 Apr 2020 23:55:46 +0800 Subject: [PATCH 01/42] wip shard fork choice rule --- setup.py | 1 + specs/phase1/shard-fork-choice.md | 209 ++++++++++++++++++ .../test/fork_choice/test_get_head.py | 30 +-- .../test/fork_choice/test_on_shard_head.py | 97 ++++++++ .../eth2spec/test/helpers/fork_choice.py | 27 +++ 5 files changed, 335 insertions(+), 29 deletions(-) create mode 100644 specs/phase1/shard-fork-choice.md create mode 100644 tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py create mode 100644 tests/core/pyspec/eth2spec/test/helpers/fork_choice.py diff --git a/setup.py b/setup.py index e0d6561dd..316a7d32a 100644 --- a/setup.py +++ b/setup.py @@ -378,6 +378,7 @@ class PySpecCommand(Command): specs/phase1/shard-transition.md specs/phase1/fork-choice.md specs/phase1/phase1-fork.md + specs/phase1/shard-fork-choice.md """ else: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md new file mode 100644 index 000000000..46e467e12 --- /dev/null +++ b/specs/phase1/shard-fork-choice.md @@ -0,0 +1,209 @@ +# Ethereum 2.0 Phase 1 -- Beacon Chain + Shard Chain Fork Choice + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + +- [Introduction](#introduction) +- [Fork choice](#fork-choice) + - [Helpers](#helpers) + - [Extended `Store`](#extended-store) + - [Updated `get_forkchoice_store`](#updated-get_forkchoice_store) + - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance) + - [`get_shard_head`](#get_shard_head) + - [`get_shard_ancestor`](#get_shard_ancestor) + - [`filter_shard_block_tree`](#filter_shard_block_tree) + - [`get_filtered_block_tree`](#get_filtered_block_tree) + - [Handlers](#handlers) + - [`on_shard_block`](#on_shard_block) + + + +## Introduction + +This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase 1. + +## Fork choice + +### Helpers + +#### Extended `Store` + +```python +@dataclass +class Store(object): + time: uint64 + genesis_time: uint64 + justified_checkpoint: Checkpoint + finalized_checkpoint: Checkpoint + best_justified_checkpoint: Checkpoint + blocks: Dict[Root, BeaconBlock] = field(default_factory=dict) + block_states: Dict[Root, BeaconState] = field(default_factory=dict) + checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) + latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) + # shard chain + shard_init_slots: Dict[Shard, Slot] = field(default_factory=dict) + shard_blocks: Dict[Shard, Dict[Root, ShardBlock]] = field(default_factory=dict) + shard_block_states: Dict[Shard, Dict[Root, ShardState]] = field(default_factory=dict) +``` + +#### Updated `get_forkchoice_store` + +```python +def get_forkchoice_store(anchor_state: BeaconState, + shard_init_slots: Dict[Shard, Slot], + anchor_state_shard_blocks: Dict[Shard, Dict[Root, ShardBlock]]) -> Store: + shard_count = len(anchor_state.shard_states) + anchor_block_header = anchor_state.latest_block_header.copy() + if anchor_block_header.state_root == Bytes32(): + anchor_block_header.state_root = hash_tree_root(anchor_state) + anchor_root = hash_tree_root(anchor_block_header) + anchor_epoch = get_current_epoch(anchor_state) + justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + return Store( + time=anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot, + genesis_time=anchor_state.genesis_time, + justified_checkpoint=justified_checkpoint, + finalized_checkpoint=finalized_checkpoint, + best_justified_checkpoint=justified_checkpoint, + blocks={anchor_root: anchor_block_header}, + block_states={anchor_root: anchor_state.copy()}, + checkpoint_states={justified_checkpoint: anchor_state.copy()}, + # shard chain + shard_init_slots=shard_init_slots, + shard_blocks=anchor_state_shard_blocks, + shard_block_states={ + shard: { + anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard] + } + for shard in map(Shard, range(shard_count)) + }, + ) +``` + +#### `get_shard_latest_attesting_balance` + +```python +def get_shard_latest_attesting_balance(store: Store, shard: Shard, root: Root) -> Gwei: + state = store.checkpoint_states[store.justified_checkpoint] + active_indices = get_active_validator_indices(state, get_current_epoch(state)) + return Gwei(sum( + state.validators[i].effective_balance for i in active_indices + if ( + i in store.latest_messages and get_shard_ancestor( + store, shard, store.latest_messages[i].root, store.shard_blocks[shard][root].slot + ) == root + ) + )) +``` + +#### `get_shard_head` + +```python +def get_shard_head(store: Store, shard: Shard) -> Root: + # Get filtered block tree that only includes viable branches + blocks = get_filtered_shard_block_tree(store, shard) + + # Execute the LMD-GHOST fork choice + head_beacon_root = get_head(store) + head_shard_root = store.block_states[head_beacon_root].shard_states[shard].latest_block_root + while True: + children = [ + root for root in blocks.keys() + if blocks[root].shard_parent_root == head_shard_root + ] + if len(children) == 0: + return head_shard_root + # Sort by latest attesting balance with ties broken lexicographically + head_shard_root = max(children, key=lambda root: (get_shard_latest_attesting_balance(store, shard, root), root)) +``` + +#### `get_shard_ancestor` + +```python +def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Root: + block = store.shard_blocks[shard][root] + if block.slot > slot: + return get_shard_ancestor(store, shard, block.shard_parent_root, slot) + elif block.slot == slot: + return root + else: + # root is older than queried slot, thus a skip slot. Return earliest root prior to slot + return root +``` + +#### `filter_shard_block_tree` + +```python +def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks: Dict[Root, ShardBlock]) -> bool: + block = store.shard_blocks[shard][block_root] + children = [ + root for root in store.shard_blocks[shard].keys() + if ( + store.shard_blocks[shard][root].shard_parent_root == block_root + and store.shard_blocks[shard][root].slot != store.shard_init_slots[shard] + ) + ] + + if any(children): + filter_block_tree_result = [filter_shard_block_tree(store, shard, child, blocks) for child in children] + if any(filter_block_tree_result): + blocks[block_root] = block + return True + return False + + return False +``` + +#### `get_filtered_block_tree` + +```python +def get_filtered_shard_block_tree(store: Store, shard: Shard) -> Dict[Root, ShardBlock]: + base_beacon_block_root = get_head(store) + base_shard_block_root = store.block_states[base_beacon_block_root].shard_states[shard].latest_block_root + blocks: Dict[Root, ShardBlock] = {} + filter_shard_block_tree(store, shard, base_shard_block_root, blocks) + return blocks +``` + +### Handlers + +#### `on_shard_block` + +```python +def on_shard_block(store: Store, shard: Shard, signed_shard_block: SignedShardBlock) -> None: + shard_block = signed_shard_block.message + + # 1. Check shard parent exists + assert shard_block.shard_parent_root in store.shard_block_states[shard] + pre_shard_state = store.shard_block_states[shard][shard_block.shard_parent_root] + + # 2. Check beacon parent exists + assert shard_block.beacon_parent_root in store.block_states + beacon_state = store.block_states[shard_block.beacon_parent_root] + + # 3. Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert shard_block.slot > finalized_slot + + # 4. Check block is a descendant of the finalized block at the checkpoint finalized slot + assert ( + shard_block.beacon_parent_root == store.finalized_checkpoint.root + or get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root + ) + + # Add new block to the store + store.shard_blocks[shard][hash_tree_root(shard_block)] = shard_block + + # Check the block is valid and compute the post-state + verify_shard_block_message(beacon_state, pre_shard_state, shard_block, shard_block.slot, shard) + verify_shard_block_signature(beacon_state, signed_shard_block) + post_state = get_post_shard_state(beacon_state, pre_shard_state, shard_block) + # Add new state for this block to the store + store.shard_block_states[shard][hash_tree_root(shard_block)] = post_state +``` diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py index 17d4f644f..e25aad18f 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py @@ -1,41 +1,13 @@ from eth2spec.test.context import with_all_phases, spec_state_test from eth2spec.test.helpers.attestations import get_valid_attestation, next_epoch_with_attestations from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.helpers.fork_choice import add_attestation_to_store, add_block_to_store, get_anchor_root from eth2spec.test.helpers.state import ( next_epoch, state_transition_and_sign_block, ) -def add_block_to_store(spec, store, signed_block): - pre_state = store.block_states[signed_block.message.parent_root] - block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT - - if store.time < block_time: - spec.on_tick(store, block_time) - - spec.on_block(store, signed_block) - - -def add_attestation_to_store(spec, store, attestation): - parent_block = store.blocks[attestation.data.beacon_block_root] - pre_state = store.block_states[spec.hash_tree_root(parent_block)] - block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT - next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT - - if store.time < next_epoch_time: - spec.on_tick(store, next_epoch_time) - - spec.on_attestation(store, attestation) - - -def get_anchor_root(spec, state): - anchor_block_header = state.latest_block_header.copy() - if anchor_block_header.state_root == spec.Bytes32(): - anchor_block_header.state_root = spec.hash_tree_root(state) - return spec.hash_tree_root(anchor_block_header) - - @with_all_phases @spec_state_test def test_genesis(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py new file mode 100644 index 000000000..8e72e214e --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -0,0 +1,97 @@ +from eth2spec.utils.ssz.ssz_impl import hash_tree_root + +from eth2spec.test.context import spec_state_test, with_all_phases_except, PHASE0 +from eth2spec.test.helpers.shard_block import ( + build_attestation_with_shard_transition, + build_shard_block, + build_shard_transitions_till_slot, +) +from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root +from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_block +from eth2spec.test.helpers.block import build_empty_block + + +def run_on_shard_block(spec, store, shard, signed_block, valid=True): + if not valid: + try: + spec.on_shard_block(store, shard, signed_block) + except AssertionError: + return + else: + assert False + + spec.on_shard_block(store, shard, signed_block) + assert store.shard_blocks[shard][hash_tree_root(signed_block.message)] == signed_block.message + + +def run_apply_shard_and_beacon(spec, state, store, shard, committee_index): + store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + + # Create SignedShardBlock + body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE + shard_block = build_shard_block(spec, state, shard, body=body, signed=True) + shard_blocks = [shard_block] + + # Attester creates `attestation` + # Use temporary next state to get ShardTransition of shard block + shard_transitions = build_shard_transitions_till_slot( + spec, + state, + shards=[shard, ], + shard_blocks={shard: shard_blocks}, + target_len_offset_slot=1, + ) + shard_transition = shard_transitions[shard] + attestation = build_attestation_with_shard_transition( + spec, + state, + slot=state.slot, + index=committee_index, + target_len_offset_slot=1, + shard_transition=shard_transition, + ) + + # Propose beacon block at slot + beacon_block = build_empty_block(spec, state, slot=state.slot + 1) + beacon_block.body.attestations = [attestation] + beacon_block.body.shard_transitions = shard_transitions + signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) + + run_on_shard_block(spec, store, shard, shard_block) + add_block_to_store(spec, store, signed_beacon_block) + + assert spec.get_head(store) == beacon_block.hash_tree_root() + assert spec.get_shard_head(store, shard) == shard_block.message.hash_tree_root() + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_basic(spec, state): + spec.PHASE_1_GENESIS_SLOT = 0 # FIXME: remove mocking + state = spec.upgrade_to_phase1(state) + next_slot(spec, state) + + # Initialization + shard_count = len(state.shard_states) + # Genesis shard blocks + anchor_shard_blocks = { + shard: { + state.shard_states[shard].latest_block_root: spec.ShardBlock( + slot=state.slot, + ) + } + for shard in map(spec.Shard, range(shard_count)) + } + shard_init_slots = { + shard: state.slot + for shard in map(spec.Shard, range(shard_count)) + } + store = spec.get_forkchoice_store(state, shard_init_slots, anchor_shard_blocks) + anchor_root = get_anchor_root(spec, state) + assert spec.get_head(store) == anchor_root + + committee_index = spec.CommitteeIndex(0) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + + run_apply_shard_and_beacon(spec, state, store, shard, committee_index) + run_apply_shard_and_beacon(spec, state, store, shard, committee_index) diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py new file mode 100644 index 000000000..04e36ea84 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py @@ -0,0 +1,27 @@ +def get_anchor_root(spec, state): + anchor_block_header = state.latest_block_header.copy() + if anchor_block_header.state_root == spec.Bytes32(): + anchor_block_header.state_root = spec.hash_tree_root(state) + return spec.hash_tree_root(anchor_block_header) + + +def add_block_to_store(spec, store, signed_block): + pre_state = store.block_states[signed_block.message.parent_root] + block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT + + if store.time < block_time: + spec.on_tick(store, block_time) + + spec.on_block(store, signed_block) + + +def add_attestation_to_store(spec, store, attestation): + parent_block = store.blocks[attestation.data.beacon_block_root] + pre_state = store.block_states[spec.hash_tree_root(parent_block)] + block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT + next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT + + if store.time < next_epoch_time: + spec.on_tick(store, next_epoch_time) + + spec.on_attestation(store, attestation) From cddf9cf114744c07d8a1ce455c5cf73ff5ad89db Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 30 Apr 2020 20:06:20 +0800 Subject: [PATCH 02/42] Refactor --- specs/phase1/shard-fork-choice.md | 56 ++++++++++--------- .../test/fork_choice/test_on_shard_head.py | 18 +----- 2 files changed, 31 insertions(+), 43 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 46e467e12..e9026b991 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -35,7 +35,13 @@ This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase ```python @dataclass -class Store(object): +class Store: + + @dataclass + class ShardStore: + blocks: Dict[Root, ShardBlock] = field(default_factory=dict) + block_states: Dict[Root, ShardState] = field(default_factory=dict) + time: uint64 genesis_time: uint64 justified_checkpoint: Checkpoint @@ -46,17 +52,13 @@ class Store(object): checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) # shard chain - shard_init_slots: Dict[Shard, Slot] = field(default_factory=dict) - shard_blocks: Dict[Shard, Dict[Root, ShardBlock]] = field(default_factory=dict) - shard_block_states: Dict[Shard, Dict[Root, ShardState]] = field(default_factory=dict) + shards: Dict[Shard, ShardStore] = field(default_factory=dict) # noqa: F821 ``` #### Updated `get_forkchoice_store` ```python -def get_forkchoice_store(anchor_state: BeaconState, - shard_init_slots: Dict[Shard, Slot], - anchor_state_shard_blocks: Dict[Shard, Dict[Root, ShardBlock]]) -> Store: +def get_forkchoice_store(anchor_state: BeaconState) -> Store: shard_count = len(anchor_state.shard_states) anchor_block_header = anchor_state.latest_block_header.copy() if anchor_block_header.state_root == Bytes32(): @@ -65,6 +67,14 @@ def get_forkchoice_store(anchor_state: BeaconState, anchor_epoch = get_current_epoch(anchor_state) justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + + shard_stores = {} + for shard in map(Shard, range(shard_count)): + shard_stores[shard] = Store.ShardStore( + blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot)}, + block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]}, + ) + return Store( time=anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot, genesis_time=anchor_state.genesis_time, @@ -75,14 +85,7 @@ def get_forkchoice_store(anchor_state: BeaconState, block_states={anchor_root: anchor_state.copy()}, checkpoint_states={justified_checkpoint: anchor_state.copy()}, # shard chain - shard_init_slots=shard_init_slots, - shard_blocks=anchor_state_shard_blocks, - shard_block_states={ - shard: { - anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard] - } - for shard in map(Shard, range(shard_count)) - }, + shards=shard_stores, ) ``` @@ -96,7 +99,7 @@ def get_shard_latest_attesting_balance(store: Store, shard: Shard, root: Root) - state.validators[i].effective_balance for i in active_indices if ( i in store.latest_messages and get_shard_ancestor( - store, shard, store.latest_messages[i].root, store.shard_blocks[shard][root].slot + store, shard, store.latest_messages[i].root, store.shards[shard].blocks[root].slot ) == root ) )) @@ -127,7 +130,7 @@ def get_shard_head(store: Store, shard: Shard) -> Root: ```python def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Root: - block = store.shard_blocks[shard][root] + block = store.shards[shard].blocks[root] if block.slot > slot: return get_shard_ancestor(store, shard, block.shard_parent_root, slot) elif block.slot == slot: @@ -141,13 +144,11 @@ def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Ro ```python def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks: Dict[Root, ShardBlock]) -> bool: - block = store.shard_blocks[shard][block_root] + shard_store = store.shards[shard] + block = shard_store.blocks[block_root] children = [ - root for root in store.shard_blocks[shard].keys() - if ( - store.shard_blocks[shard][root].shard_parent_root == block_root - and store.shard_blocks[shard][root].slot != store.shard_init_slots[shard] - ) + root for root in shard_store.blocks.keys() + if shard_store.blocks[root].shard_parent_root == block_root ] if any(children): @@ -178,10 +179,11 @@ def get_filtered_shard_block_tree(store: Store, shard: Shard) -> Dict[Root, Shar ```python def on_shard_block(store: Store, shard: Shard, signed_shard_block: SignedShardBlock) -> None: shard_block = signed_shard_block.message + shard_store = store.shards[shard] # 1. Check shard parent exists - assert shard_block.shard_parent_root in store.shard_block_states[shard] - pre_shard_state = store.shard_block_states[shard][shard_block.shard_parent_root] + assert shard_block.shard_parent_root in shard_store.block_states + pre_shard_state = shard_store.block_states[shard_block.shard_parent_root] # 2. Check beacon parent exists assert shard_block.beacon_parent_root in store.block_states @@ -198,12 +200,12 @@ def on_shard_block(store: Store, shard: Shard, signed_shard_block: SignedShardBl ) # Add new block to the store - store.shard_blocks[shard][hash_tree_root(shard_block)] = shard_block + shard_store.blocks[hash_tree_root(shard_block)] = shard_block # Check the block is valid and compute the post-state verify_shard_block_message(beacon_state, pre_shard_state, shard_block, shard_block.slot, shard) verify_shard_block_signature(beacon_state, signed_shard_block) post_state = get_post_shard_state(beacon_state, pre_shard_state, shard_block) # Add new state for this block to the store - store.shard_block_states[shard][hash_tree_root(shard_block)] = post_state + shard_store.block_states[hash_tree_root(shard_block)] = post_state ``` diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 8e72e214e..220c510e7 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -21,7 +21,7 @@ def run_on_shard_block(spec, store, shard, signed_block, valid=True): assert False spec.on_shard_block(store, shard, signed_block) - assert store.shard_blocks[shard][hash_tree_root(signed_block.message)] == signed_block.message + assert store.shards[shard].blocks[hash_tree_root(signed_block.message)] == signed_block.message def run_apply_shard_and_beacon(spec, state, store, shard, committee_index): @@ -72,21 +72,7 @@ def test_basic(spec, state): next_slot(spec, state) # Initialization - shard_count = len(state.shard_states) - # Genesis shard blocks - anchor_shard_blocks = { - shard: { - state.shard_states[shard].latest_block_root: spec.ShardBlock( - slot=state.slot, - ) - } - for shard in map(spec.Shard, range(shard_count)) - } - shard_init_slots = { - shard: state.slot - for shard in map(spec.Shard, range(shard_count)) - } - store = spec.get_forkchoice_store(state, shard_init_slots, anchor_shard_blocks) + store = spec.get_forkchoice_store(state) anchor_root = get_anchor_root(spec, state) assert spec.get_head(store) == anchor_root From 8fafb6a9e5b17ee3e93fa0c1b3a8795608f7336b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 1 May 2020 10:25:58 +0800 Subject: [PATCH 03/42] Make `ShardStore` an independent object --- specs/phase1/shard-fork-choice.md | 97 ++++++------------- .../test/fork_choice/test_on_shard_head.py | 20 ++-- 2 files changed, 43 insertions(+), 74 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index e9026b991..4b3f42194 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -31,75 +31,38 @@ This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase ### Helpers -#### Extended `Store` +#### `ShardStore` ```python @dataclass -class Store: - - @dataclass - class ShardStore: - blocks: Dict[Root, ShardBlock] = field(default_factory=dict) - block_states: Dict[Root, ShardState] = field(default_factory=dict) - - time: uint64 - genesis_time: uint64 - justified_checkpoint: Checkpoint - finalized_checkpoint: Checkpoint - best_justified_checkpoint: Checkpoint - blocks: Dict[Root, BeaconBlock] = field(default_factory=dict) - block_states: Dict[Root, BeaconState] = field(default_factory=dict) - checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) - latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) - # shard chain - shards: Dict[Shard, ShardStore] = field(default_factory=dict) # noqa: F821 +class ShardStore: + shard: Shard + blocks: Dict[Root, ShardBlock] = field(default_factory=dict) + block_states: Dict[Root, ShardState] = field(default_factory=dict) ``` -#### Updated `get_forkchoice_store` +#### Updated `get_forkchoice_shard_store` ```python -def get_forkchoice_store(anchor_state: BeaconState) -> Store: - shard_count = len(anchor_state.shard_states) - anchor_block_header = anchor_state.latest_block_header.copy() - if anchor_block_header.state_root == Bytes32(): - anchor_block_header.state_root = hash_tree_root(anchor_state) - anchor_root = hash_tree_root(anchor_block_header) - anchor_epoch = get_current_epoch(anchor_state) - justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) - finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) - - shard_stores = {} - for shard in map(Shard, range(shard_count)): - shard_stores[shard] = Store.ShardStore( - blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot)}, - block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]}, - ) - - return Store( - time=anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot, - genesis_time=anchor_state.genesis_time, - justified_checkpoint=justified_checkpoint, - finalized_checkpoint=finalized_checkpoint, - best_justified_checkpoint=justified_checkpoint, - blocks={anchor_root: anchor_block_header}, - block_states={anchor_root: anchor_state.copy()}, - checkpoint_states={justified_checkpoint: anchor_state.copy()}, - # shard chain - shards=shard_stores, +def get_forkchoice_shard_store(anchor_state: BeaconState, shard: Shard) -> ShardStore: + return ShardStore( + shard=shard, + blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot)}, + block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]}, ) ``` #### `get_shard_latest_attesting_balance` ```python -def get_shard_latest_attesting_balance(store: Store, shard: Shard, root: Root) -> Gwei: +def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, root: Root) -> Gwei: state = store.checkpoint_states[store.justified_checkpoint] active_indices = get_active_validator_indices(state, get_current_epoch(state)) return Gwei(sum( state.validators[i].effective_balance for i in active_indices if ( i in store.latest_messages and get_shard_ancestor( - store, shard, store.latest_messages[i].root, store.shards[shard].blocks[root].slot + store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot ) == root ) )) @@ -108,13 +71,13 @@ def get_shard_latest_attesting_balance(store: Store, shard: Shard, root: Root) - #### `get_shard_head` ```python -def get_shard_head(store: Store, shard: Shard) -> Root: +def get_shard_head(store: Store, shard_store: ShardStore) -> Root: # Get filtered block tree that only includes viable branches - blocks = get_filtered_shard_block_tree(store, shard) + blocks = get_filtered_shard_block_tree(store, shard_store) # Execute the LMD-GHOST fork choice head_beacon_root = get_head(store) - head_shard_root = store.block_states[head_beacon_root].shard_states[shard].latest_block_root + head_shard_root = store.block_states[head_beacon_root].shard_states[shard_store.shard].latest_block_root while True: children = [ root for root in blocks.keys() @@ -123,16 +86,18 @@ def get_shard_head(store: Store, shard: Shard) -> Root: if len(children) == 0: return head_shard_root # Sort by latest attesting balance with ties broken lexicographically - head_shard_root = max(children, key=lambda root: (get_shard_latest_attesting_balance(store, shard, root), root)) + head_shard_root = max( + children, key=lambda root: (get_shard_latest_attesting_balance(store, shard_store, root), root) + ) ``` #### `get_shard_ancestor` ```python -def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Root: - block = store.shards[shard].blocks[root] +def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: Slot) -> Root: + block = shard_store.blocks[root] if block.slot > slot: - return get_shard_ancestor(store, shard, block.shard_parent_root, slot) + return get_shard_ancestor(store, shard_store, block.shard_parent_root, slot) elif block.slot == slot: return root else: @@ -143,8 +108,10 @@ def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Ro #### `filter_shard_block_tree` ```python -def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks: Dict[Root, ShardBlock]) -> bool: - shard_store = store.shards[shard] +def filter_shard_block_tree(store: Store, + shard_store: ShardStore, + block_root: Root, + blocks: Dict[Root, ShardBlock]) -> bool: block = shard_store.blocks[block_root] children = [ root for root in shard_store.blocks.keys() @@ -152,7 +119,7 @@ def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks ] if any(children): - filter_block_tree_result = [filter_shard_block_tree(store, shard, child, blocks) for child in children] + filter_block_tree_result = [filter_shard_block_tree(store, shard_store, child, blocks) for child in children] if any(filter_block_tree_result): blocks[block_root] = block return True @@ -164,11 +131,12 @@ def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks #### `get_filtered_block_tree` ```python -def get_filtered_shard_block_tree(store: Store, shard: Shard) -> Dict[Root, ShardBlock]: +def get_filtered_shard_block_tree(store: Store, shard_store: ShardStore) -> Dict[Root, ShardBlock]: + shard = shard_store.shard base_beacon_block_root = get_head(store) base_shard_block_root = store.block_states[base_beacon_block_root].shard_states[shard].latest_block_root blocks: Dict[Root, ShardBlock] = {} - filter_shard_block_tree(store, shard, base_shard_block_root, blocks) + filter_shard_block_tree(store, shard_store, base_shard_block_root, blocks) return blocks ``` @@ -177,10 +145,9 @@ def get_filtered_shard_block_tree(store: Store, shard: Shard) -> Dict[Root, Shar #### `on_shard_block` ```python -def on_shard_block(store: Store, shard: Shard, signed_shard_block: SignedShardBlock) -> None: +def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: SignedShardBlock) -> None: shard_block = signed_shard_block.message - shard_store = store.shards[shard] - + shard = shard_store.shard # 1. Check shard parent exists assert shard_block.shard_parent_root in shard_store.block_states pre_shard_state = shard_store.block_states[shard_block.shard_parent_root] diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 220c510e7..f4b883f06 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -11,20 +11,21 @@ from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_blo from eth2spec.test.helpers.block import build_empty_block -def run_on_shard_block(spec, store, shard, signed_block, valid=True): +def run_on_shard_block(spec, store, shard_store, signed_block, valid=True): if not valid: try: - spec.on_shard_block(store, shard, signed_block) + spec.on_shard_block(store, shard_store, signed_block) except AssertionError: return else: assert False - spec.on_shard_block(store, shard, signed_block) - assert store.shards[shard].blocks[hash_tree_root(signed_block.message)] == signed_block.message + spec.on_shard_block(store, shard_store, signed_block) + assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message -def run_apply_shard_and_beacon(spec, state, store, shard, committee_index): +def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index): + shard = shard_store.shard store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH # Create SignedShardBlock @@ -57,11 +58,11 @@ def run_apply_shard_and_beacon(spec, state, store, shard, committee_index): beacon_block.body.shard_transitions = shard_transitions signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) - run_on_shard_block(spec, store, shard, shard_block) + run_on_shard_block(spec, store, shard_store, shard_block) add_block_to_store(spec, store, signed_beacon_block) assert spec.get_head(store) == beacon_block.hash_tree_root() - assert spec.get_shard_head(store, shard) == shard_block.message.hash_tree_root() + assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() @with_all_phases_except([PHASE0]) @@ -78,6 +79,7 @@ def test_basic(spec, state): committee_index = spec.CommitteeIndex(0) shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard_store = spec.get_forkchoice_shard_store(state, shard) - run_apply_shard_and_beacon(spec, state, store, shard, committee_index) - run_apply_shard_and_beacon(spec, state, store, shard, committee_index) + run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) + run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) From fca1bbccb948b132fef22a9732c2f0a6aef0b2a4 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 1 May 2020 10:33:23 +0800 Subject: [PATCH 04/42] Remove `get_filtered_shard_block_tree` --- specs/phase1/shard-fork-choice.md | 50 ++++--------------------------- 1 file changed, 5 insertions(+), 45 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 4b3f42194..b026aabfc 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -11,13 +11,11 @@ - [Introduction](#introduction) - [Fork choice](#fork-choice) - [Helpers](#helpers) - - [Extended `Store`](#extended-store) - - [Updated `get_forkchoice_store`](#updated-get_forkchoice_store) + - [`ShardStore`](#shardstore) + - [`get_forkchoice_shard_store`](#get_forkchoice_shard_store) - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance) - [`get_shard_head`](#get_shard_head) - [`get_shard_ancestor`](#get_shard_ancestor) - - [`filter_shard_block_tree`](#filter_shard_block_tree) - - [`get_filtered_block_tree`](#get_filtered_block_tree) - [Handlers](#handlers) - [`on_shard_block`](#on_shard_block) @@ -41,7 +39,7 @@ class ShardStore: block_states: Dict[Root, ShardState] = field(default_factory=dict) ``` -#### Updated `get_forkchoice_shard_store` +#### `get_forkchoice_shard_store` ```python def get_forkchoice_shard_store(anchor_state: BeaconState, shard: Shard) -> ShardStore: @@ -72,16 +70,13 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro ```python def get_shard_head(store: Store, shard_store: ShardStore) -> Root: - # Get filtered block tree that only includes viable branches - blocks = get_filtered_shard_block_tree(store, shard_store) - # Execute the LMD-GHOST fork choice head_beacon_root = get_head(store) head_shard_root = store.block_states[head_beacon_root].shard_states[shard_store.shard].latest_block_root while True: children = [ - root for root in blocks.keys() - if blocks[root].shard_parent_root == head_shard_root + root for root in shard_store.blocks.keys() + if shard_store.blocks[root].shard_parent_root == head_shard_root ] if len(children) == 0: return head_shard_root @@ -105,41 +100,6 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: return root ``` -#### `filter_shard_block_tree` - -```python -def filter_shard_block_tree(store: Store, - shard_store: ShardStore, - block_root: Root, - blocks: Dict[Root, ShardBlock]) -> bool: - block = shard_store.blocks[block_root] - children = [ - root for root in shard_store.blocks.keys() - if shard_store.blocks[root].shard_parent_root == block_root - ] - - if any(children): - filter_block_tree_result = [filter_shard_block_tree(store, shard_store, child, blocks) for child in children] - if any(filter_block_tree_result): - blocks[block_root] = block - return True - return False - - return False -``` - -#### `get_filtered_block_tree` - -```python -def get_filtered_shard_block_tree(store: Store, shard_store: ShardStore) -> Dict[Root, ShardBlock]: - shard = shard_store.shard - base_beacon_block_root = get_head(store) - base_shard_block_root = store.block_states[base_beacon_block_root].shard_states[shard].latest_block_root - blocks: Dict[Root, ShardBlock] = {} - filter_shard_block_tree(store, shard_store, base_shard_block_root, blocks) - return blocks -``` - ### Handlers #### `on_shard_block` From 79b1b4bdbe0c4822d4a2e197a7bf59930ffe69c1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 1 May 2020 10:52:45 +0800 Subject: [PATCH 05/42] Add `(shard, shard_root)` to `LatestMessage` --- specs/phase1/fork-choice.md | 32 ++++++++++++++++++- .../test/fork_choice/test_on_attestation.py | 13 ++++++-- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/specs/phase1/fork-choice.md b/specs/phase1/fork-choice.md index d8bf7fa09..f4e771ddb 100644 --- a/specs/phase1/fork-choice.md +++ b/specs/phase1/fork-choice.md @@ -10,6 +10,9 @@ - [Introduction](#introduction) - [Fork choice](#fork-choice) + - [Helpers](#helpers) + - [Extended `LatestMessage`](#extended-latestmessage) + - [Updated `update_latest_messages`](#updated-update_latest_messages) - [Handlers](#handlers) @@ -25,6 +28,33 @@ Due to the changes in the structure of `IndexedAttestation` in Phase 1, `on_atte The rest of the fork choice remains stable. +### Helpers + +#### Extended `LatestMessage` + +```python +@dataclass(eq=True, frozen=True) +class LatestMessage(object): + epoch: Epoch + root: Root + shard: Shard + shard_root: Root +``` + +#### Updated `update_latest_messages` + +```python +def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIndex], attestation: Attestation) -> None: + target = attestation.data.target + beacon_block_root = attestation.data.beacon_block_root + shard = get_shard(store.block_states[beacon_block_root], attestation) + for i in attesting_indices: + if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: + store.latest_messages[i] = LatestMessage( + epoch=target.epoch, root=beacon_block_root, shard=shard, shard_root=attestation.data.head_shard_root + ) +``` + ### Handlers ```python @@ -49,4 +79,4 @@ def on_attestation(store: Store, attestation: Attestation) -> None: if attestation.aggregation_bits[i] ] update_latest_messages(store, attesting_indices, attestation) -``` \ No newline at end of file +``` diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 360c18ccd..1bce42ca3 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -18,18 +18,25 @@ def run_on_attestation(spec, state, store, attestation, valid=True): if spec.fork == PHASE0: sample_index = indexed_attestation.attesting_indices[0] + latest_message = spec.LatestMessage( + epoch=attestation.data.target.epoch, + root=attestation.data.beacon_block_root, + ) else: attesting_indices = [ index for i, index in enumerate(indexed_attestation.committee) if attestation.aggregation_bits[i] ] sample_index = attesting_indices[0] - assert ( - store.latest_messages[sample_index] == - spec.LatestMessage( + latest_message = spec.LatestMessage( epoch=attestation.data.target.epoch, root=attestation.data.beacon_block_root, + shard=spec.get_shard(state, attestation), + shard_root=attestation.data.head_shard_root, ) + + assert ( + store.latest_messages[sample_index] == latest_message ) From 2dc041807af913f16e54cf44a46fef07f80dbe2e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 29 May 2020 23:50:18 +0800 Subject: [PATCH 06/42] Implement `get_start_shard` --- setup.py | 15 ++++- specs/phase1/beacon-chain.md | 61 ++++++++++++++++++- specs/phase1/phase1-fork.md | 1 + .../test_process_crosslink.py | 2 +- .../test/phase_1/sanity/test_blocks.py | 4 +- 5 files changed, 74 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 4f3f2b872..d0f063b33 100644 --- a/setup.py +++ b/setup.py @@ -140,7 +140,7 @@ SUNDRY_CONSTANTS_FUNCTIONS = ''' def ceillog2(x: uint64) -> int: return (x - 1).bit_length() ''' -SUNDRY_FUNCTIONS = ''' +PHASE0_SUNDRY_FUNCTIONS = ''' # Monkey patch hash cache _hash = hash hash_cache: Dict[bytes, Bytes32] = {} @@ -220,6 +220,13 @@ get_attesting_indices = cache_this( _get_attesting_indices, lru_size=SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT * 3)''' +PHASE1_SUNDRY_FUNCTIONS = ''' +_get_start_shard = get_start_shard +get_start_shard = cache_this( + lambda state, slot: (state.validators.hash_tree_root(), slot), + _get_start_shard, lru_size=SLOTS_PER_EPOCH * 3)''' + + def objects_to_spec(spec_object: SpecObject, imports: str, fork: str) -> str: """ Given all the objects that constitute a spec, combine them into a single pyfile. @@ -250,9 +257,11 @@ def objects_to_spec(spec_object: SpecObject, imports: str, fork: str) -> str: + '\n\n' + CONFIG_LOADER + '\n\n' + ssz_objects_instantiation_spec + '\n\n' + functions_spec - + '\n' + SUNDRY_FUNCTIONS - + '\n' + + '\n' + PHASE0_SUNDRY_FUNCTIONS ) + if fork == 'phase1': + spec += '\n' + PHASE1_SUNDRY_FUNCTIONS + spec += '\n' return spec diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 39a73c0aa..124442b59 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -47,6 +47,7 @@ - [`get_light_client_committee`](#get_light_client_committee) - [`get_shard_proposer_index`](#get_shard_proposer_index) - [`get_indexed_attestation`](#get_indexed_attestation) + - [`get_committee_count_delta`](#get_committee_count_delta) - [`get_start_shard`](#get_start_shard) - [`get_shard`](#get_shard) - [`get_latest_slot_for_shard`](#get_latest_slot_for_shard) @@ -69,6 +70,7 @@ - [Shard transition false positives](#shard-transition-false-positives) - [Light client processing](#light-client-processing) - [Epoch transition](#epoch-transition) + - [Phase 1 final updates](#phase-1-final-updates) - [Custody game updates](#custody-game-updates) - [Online-tracking](#online-tracking) - [Light client committee updates](#light-client-committee-updates) @@ -280,6 +282,7 @@ class BeaconState(Container): current_justified_checkpoint: Checkpoint finalized_checkpoint: Checkpoint # Phase 1 + current_epoch_start_shard: Shard shard_states: List[ShardState, MAX_SHARDS] online_countdown: List[OnlineEpochs, VALIDATOR_REGISTRY_LIMIT] # not a raw byte array, considered its large size. current_light_committee: CompactCommittee @@ -530,18 +533,53 @@ def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation) ) ``` +#### `get_committee_count_delta` + +```python +def get_committee_count_delta(state: BeaconState, start_slot: Slot, stop_slot: Slot) -> uint64: + """ + Return the sum of committee counts between ``[start_slot, stop_slot)``. + """ + committee_sum = 0 + for slot in range(start_slot, stop_slot): + count = get_committee_count_at_slot(state, Slot(slot)) + committee_sum += count + return committee_sum +``` + #### `get_start_shard` ```python def get_start_shard(state: BeaconState, slot: Slot) -> Shard: - # TODO: implement start shard logic - return Shard(0) + """ + Return the start shard at ``slot``. + """ + current_epoch_start_slot = compute_start_slot_at_epoch(get_current_epoch(state)) + active_shard_count = get_active_shard_count(state) + if current_epoch_start_slot == slot: + return state.current_epoch_start_shard + elif current_epoch_start_slot > slot: + # Current epoch or the next epoch lookahead + shard_delta = get_committee_count_delta(state, start_slot=current_epoch_start_slot, stop_slot=slot) + return Shard((state.current_epoch_start_shard + shard_delta) % active_shard_count) + else: + # Previous epoch + shard_delta = get_committee_count_delta(state, start_slot=slot, stop_slot=current_epoch_start_slot) + max_committees_per_epoch = MAX_COMMITTEES_PER_SLOT * SLOTS_PER_EPOCH + return Shard( + # Ensure positive + (state.current_epoch_start_shard + max_committees_per_epoch * active_shard_count - shard_delta) + % active_shard_count + ) ``` #### `get_shard` ```python def get_shard(state: BeaconState, attestation: Attestation) -> Shard: + """ + Return the shard that the given attestation is attesting. + """ return compute_shard_from_committee_index(state, attestation.data.index, attestation.data.slot) ``` @@ -549,6 +587,9 @@ def get_shard(state: BeaconState, attestation: Attestation) -> Shard: ```python def get_latest_slot_for_shard(state: BeaconState, shard: Shard) -> Slot: + """ + Return the latest slot number of the given shard. + """ return state.shard_states[shard].slot ``` @@ -556,7 +597,11 @@ def get_latest_slot_for_shard(state: BeaconState, shard: Shard) -> Slot: ```python def get_offset_slots(state: BeaconState, shard: Shard) -> Sequence[Slot]: - return compute_offset_slots(state.shard_states[shard].slot, state.slot) + """ + Return the offset slots of the given shard. + The offset slot are after the latest slot and before current slot. + """ + return compute_offset_slots(get_latest_slot_for_shard(state, shard), state.slot) ``` ### Predicates @@ -993,9 +1038,19 @@ def process_epoch(state: BeaconState) -> None: process_reveal_deadlines(state) process_slashings(state) process_final_updates(state) + process_phase_1_final_updates(state) +``` + +#### Phase 1 final updates + +```python +def process_phase_1_final_updates(state: BeaconState) -> None: process_custody_final_updates(state) process_online_tracking(state) process_light_client_committee_updates(state) + + # Update current_epoch_start_shard + state.current_epoch_start_shard = get_start_shard(state, state.slot) ``` #### Custody game updates diff --git a/specs/phase1/phase1-fork.md b/specs/phase1/phase1-fork.md index cc7d8f33e..d362ed633 100644 --- a/specs/phase1/phase1-fork.md +++ b/specs/phase1/phase1-fork.md @@ -99,6 +99,7 @@ def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: current_justified_checkpoint=pre.current_justified_checkpoint, finalized_checkpoint=pre.finalized_checkpoint, # Phase 1 + current_epoch_start_shard=Shard(0), shard_states=List[ShardState, MAX_SHARDS]( ShardState( slot=pre.slot, diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py index 1f066b344..af0ff1a90 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py @@ -18,7 +18,7 @@ def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): # At the beginning, let `x = state.slot`, `state.shard_states[shard].slot == x - 1` slot_x = state.slot committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot) assert state.shard_states[shard].slot == slot_x - 1 # Create SignedShardBlock diff --git a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py index 60af35d45..1499d34cd 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py @@ -67,7 +67,7 @@ def test_process_beacon_block_with_normal_shard_transition(spec, state): target_len_offset_slot = 1 committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot) assert state.shard_states[shard].slot == state.slot - 1 pre_gasprice = state.shard_states[shard].gasprice @@ -93,7 +93,7 @@ def test_process_beacon_block_with_empty_proposal_transition(spec, state): target_len_offset_slot = 1 committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot) assert state.shard_states[shard].slot == state.slot - 1 # No new shard block From 870ad8b921705cb658ebb0b88ff4e08bdeb54609 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 30 May 2020 03:56:56 +0800 Subject: [PATCH 07/42] Fix test --- .../pyspec/eth2spec/test/fork_choice/test_on_shard_head.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index f4b883f06..5b4205e91 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -30,6 +30,7 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) # Create SignedShardBlock body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE + target_len_offset_slot = 1 shard_block = build_shard_block(spec, state, shard, body=body, signed=True) shard_blocks = [shard_block] @@ -38,17 +39,15 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) shard_transitions = build_shard_transitions_till_slot( spec, state, - shards=[shard, ], shard_blocks={shard: shard_blocks}, - target_len_offset_slot=1, + on_time_slot=state.slot + target_len_offset_slot, ) shard_transition = shard_transitions[shard] attestation = build_attestation_with_shard_transition( spec, state, - slot=state.slot, index=committee_index, - target_len_offset_slot=1, + on_time_slot=state.slot + target_len_offset_slot, shard_transition=shard_transition, ) From 92db6da50854071eecea138576e9771a5004a596 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 1 Jun 2020 17:56:22 +0800 Subject: [PATCH 08/42] Apply suggestions from Terence Co-authored-by: terence tsao --- specs/phase1/beacon-chain.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 124442b59..1367d029b 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -578,7 +578,7 @@ def get_start_shard(state: BeaconState, slot: Slot) -> Shard: ```python def get_shard(state: BeaconState, attestation: Attestation) -> Shard: """ - Return the shard that the given attestation is attesting. + Return the shard that the given ``attestation`` is attesting. """ return compute_shard_from_committee_index(state, attestation.data.index, attestation.data.slot) ``` @@ -588,7 +588,7 @@ def get_shard(state: BeaconState, attestation: Attestation) -> Shard: ```python def get_latest_slot_for_shard(state: BeaconState, shard: Shard) -> Slot: """ - Return the latest slot number of the given shard. + Return the latest slot number of the given ``shard``. """ return state.shard_states[shard].slot ``` @@ -598,7 +598,7 @@ def get_latest_slot_for_shard(state: BeaconState, shard: Shard) -> Slot: ```python def get_offset_slots(state: BeaconState, shard: Shard) -> Sequence[Slot]: """ - Return the offset slots of the given shard. + Return the offset slots of the given ``shard``. The offset slot are after the latest slot and before current slot. """ return compute_offset_slots(get_latest_slot_for_shard(state, shard), state.slot) From 30f72dd69646d037a06943d026e2d91ef2b9c2c5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 1 Jun 2020 23:15:16 +0800 Subject: [PATCH 09/42] Fix `get_shard` and `compute_shard_from_committee_index` calls --- specs/phase1/beacon-chain.md | 25 ++++++------- .../eth2spec/test/helpers/attestations.py | 37 +++++++++---------- .../eth2spec/test/helpers/shard_block.py | 2 +- .../test/helpers/shard_transitions.py | 15 ++++++++ .../test/phase_0/sanity/test_blocks.py | 20 +++++++--- .../test_process_shard_transition.py | 9 ++--- .../test/phase_1/sanity/test_blocks.py | 4 +- 7 files changed, 67 insertions(+), 45 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 228cd888e..52b6f2afb 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -578,7 +578,7 @@ def get_start_shard(state: BeaconState, slot: Slot) -> Shard: active_shard_count = get_active_shard_count(state) if current_epoch_start_slot == slot: return state.current_epoch_start_shard - elif current_epoch_start_slot > slot: + elif slot > current_epoch_start_slot: # Current epoch or the next epoch lookahead shard_delta = get_committee_count_delta(state, start_slot=current_epoch_start_slot, stop_slot=slot) return Shard((state.current_epoch_start_shard + shard_delta) % active_shard_count) @@ -693,8 +693,7 @@ def is_on_time_attestation(state: BeaconState, """ Check if the given attestation is on-time. """ - # TODO: MIN_ATTESTATION_INCLUSION_DELAY should always be 1 - return attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY == state.slot + return attestation.data.slot == compute_previous_slot(state.slot) ``` #### `is_winning_attestation` @@ -803,20 +802,19 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: else: assert attestation.data.source == state.previous_justified_checkpoint - shard = get_shard(state, attestation) - # Type 1: on-time attestations, the custody bits should be non-empty. if attestation.custody_bits_blocks != []: # Ensure on-time attestation assert is_on_time_attestation(state, attestation) # Correct data root count + shard = get_shard(state, attestation) assert len(attestation.custody_bits_blocks) == len(get_offset_slots(state, shard)) # Correct parent block root assert data.beacon_block_root == get_block_root_at_slot(state, compute_previous_slot(state.slot)) # Type 2: no shard transition, no custody bits else: # Ensure delayed attestation - assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY < state.slot + assert data.slot < compute_previous_slot(state.slot) # Late attestations cannot have a shard transition root assert data.shard_transition_root == Root() @@ -911,9 +909,10 @@ def process_crosslink_for_shard(state: BeaconState, committee_index: CommitteeIndex, shard_transition: ShardTransition, attestations: Sequence[Attestation]) -> Root: - committee = get_beacon_committee(state, state.slot, committee_index) + on_time_attestation_slot = compute_previous_slot(state.slot) + committee = get_beacon_committee(state, on_time_attestation_slot, committee_index) online_indices = get_online_validator_indices(state) - shard = compute_shard_from_committee_index(state, committee_index, state.slot) + shard = compute_shard_from_committee_index(state, committee_index, on_time_attestation_slot) # Loop over all shard transition roots shard_transition_roots = set([a.data.shard_transition_root for a in attestations]) @@ -968,15 +967,15 @@ def process_crosslink_for_shard(state: BeaconState, def process_crosslinks(state: BeaconState, shard_transitions: Sequence[ShardTransition], attestations: Sequence[Attestation]) -> None: - committee_count = get_committee_count_at_slot(state, state.slot) + on_time_attestation_slot = compute_previous_slot(state.slot) + committee_count = get_committee_count_at_slot(state, on_time_attestation_slot) for committee_index in map(CommitteeIndex, range(committee_count)): - shard = compute_shard_from_committee_index(state, committee_index, state.slot) # All attestations in the block for this committee/shard and current slot shard_attestations = [ attestation for attestation in attestations if is_on_time_attestation(state, attestation) and attestation.data.index == committee_index ] - + shard = compute_shard_from_committee_index(state, committee_index, on_time_attestation_slot) winning_root = process_crosslink_for_shard(state, committee_index, shard_transitions[shard], shard_attestations) if winning_root != Root(): # Mark relevant pending attestations as creating a successful crosslink @@ -1083,7 +1082,7 @@ def process_epoch(state: BeaconState) -> None: process_registry_updates(state) process_reveal_deadlines(state) process_slashings(state) - process_final_updates(state) + process_final_updates(state) # phase 0 final updates process_phase_1_final_updates(state) ``` @@ -1096,7 +1095,7 @@ def process_phase_1_final_updates(state: BeaconState) -> None: process_light_client_committee_updates(state) # Update current_epoch_start_shard - state.current_epoch_start_shard = get_start_shard(state, state.slot) + state.current_epoch_start_shard = get_start_shard(state, Slot(state.slot + 1)) ``` #### Custody game updates diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index c533182ef..1372b0654 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -1,8 +1,9 @@ from typing import List from eth2spec.test.context import expect_assertion_error, PHASE0, PHASE1 -from eth2spec.test.helpers.state import state_transition_and_sign_block, next_epoch, next_slot, transition_to +from eth2spec.test.helpers.state import state_transition_and_sign_block, next_epoch, next_slot from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.helpers.shard_transitions import get_shard_transition_of_committee from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.ssz.ssz_typing import Bitlist @@ -81,12 +82,12 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() else: - # No shard transition + # No shard transition -> no shard block shard = spec.get_shard(state, spec.Attestation(data=attestation_data)) if on_time: temp_state = state.copy() next_slot(spec, temp_state) - shard_transition = spec.get_shard_transition(temp_state, shard, []) + shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=[]) lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() @@ -180,7 +181,7 @@ def get_valid_attestation(spec, fill_aggregate_attestation(spec, state, attestation, signed=signed, filter_participant_set=filter_participant_set) if spec.fork == PHASE1 and on_time: - attestation = convert_to_valid_on_time_attestation(spec, state, attestation, signed) + attestation = convert_to_valid_on_time_attestation(spec, state, attestation, signed=signed) return attestation @@ -317,7 +318,19 @@ def next_epoch_with_attestations(spec, committees_per_slot = spec.get_committee_count_at_slot(state, slot_to_attest) if slot_to_attest >= spec.compute_start_slot_at_epoch(spec.get_current_epoch(post_state)): for index in range(committees_per_slot): - cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest, index=index, signed=True) + if spec.fork == PHASE1: + shard = spec.compute_shard_from_committee_index(post_state, index, slot_to_attest) + shard_transition = get_shard_transition_of_committee( + spec, post_state, index, slot=slot_to_attest + ) + block.body.shard_transitions[shard] = shard_transition + else: + shard_transition = None + + cur_attestation = get_valid_attestation( + spec, post_state, slot_to_attest, + shard_transition=shard_transition, index=index, signed=True, on_time=True + ) block.body.attestations.append(cur_attestation) if fill_prev_epoch: @@ -328,9 +341,6 @@ def next_epoch_with_attestations(spec, spec, post_state, slot_to_attest, index=index, signed=True, on_time=False) block.body.attestations.append(prev_attestation) - if spec.fork == PHASE1: - fill_block_shard_transitions_by_attestations(spec, post_state, block) - signed_block = state_transition_and_sign_block(spec, post_state, block) signed_blocks.append(signed_block) @@ -396,14 +406,3 @@ def cached_prepare_state_with_attestations(spec, state): # Put the LRU cache result into the state view, as if we transitioned the original view state.set_backing(_prep_state_cache_dict[key]) - - -def fill_block_shard_transitions_by_attestations(spec, state, block): - block.body.shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS - for attestation in block.body.attestations: - shard = spec.get_shard(state, attestation) - if attestation.data.slot == state.slot: - temp_state = state.copy() - transition_to(spec, temp_state, slot=block.slot) - shard_transition = spec.get_shard_transition(temp_state, shard, []) - block.body.shard_transitions[shard] = shard_transition diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 805b955f7..58efada83 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -70,7 +70,7 @@ def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): return shard_transitions -def build_attestation_with_shard_transition(spec, state, index, on_time_slot, shard_transition=None): +def build_attestation_with_shard_transition(spec, state, index, on_time_slot, shard_transition): temp_state = state.copy() transition_to(spec, temp_state, on_time_slot - 1) attestation = get_valid_on_time_attestation( diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py index 4ac0ddcfb..abb5e7278 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py @@ -1,4 +1,5 @@ from eth2spec.test.context import expect_assertion_error +from eth2spec.test.helpers.state import transition_to def run_shard_transitions_processing(spec, state, shard_transitions, attestations, valid=True): @@ -26,3 +27,17 @@ def run_shard_transitions_processing(spec, state, shard_transitions, attestation # yield post-state yield 'post', state + + +def get_shard_transition_of_committee(spec, state, committee_index, slot=None, shard_blocks=None): + if shard_blocks is None: + shard_blocks = [] + + if slot is None: + slot = state.slot + + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + temp_state = state.copy() + transition_to(spec, temp_state, slot + 1) + shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=shard_blocks) + return shard_transition diff --git a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py index f0cfc462e..1e057e5a9 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py @@ -16,8 +16,9 @@ from eth2spec.test.helpers.attester_slashings import ( get_indexed_attestation_participants, ) from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing, check_proposer_slashing_effect -from eth2spec.test.helpers.attestations import get_valid_attestation, fill_block_shard_transitions_by_attestations +from eth2spec.test.helpers.attestations import get_valid_attestation from eth2spec.test.helpers.deposits import prepare_state_and_deposit +from eth2spec.test.helpers.shard_transitions import get_shard_transition_of_committee from eth2spec.test.context import ( spec_state_test, with_all_phases, expect_assertion_error, always_bls, with_phases, @@ -687,14 +688,23 @@ def test_attestation(spec, state): yield 'pre', state - attestation = get_valid_attestation(spec, state, signed=True, on_time=True) + attestation_block = build_empty_block(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + index = 0 + if spec.fork == PHASE1: + shard = spec.compute_shard_from_committee_index(state, index, state.slot) + shard_transition = get_shard_transition_of_committee(spec, state, index) + attestation_block.body.shard_transitions[shard] = shard_transition + else: + shard_transition = None + + attestation = get_valid_attestation( + spec, state, shard_transition=shard_transition, index=index, signed=True, on_time=True + ) # Add to state via block transition pre_current_attestations_len = len(state.current_epoch_attestations) - attestation_block = build_empty_block(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) attestation_block.body.attestations.append(attestation) - if spec.fork == PHASE1: - fill_block_shard_transitions_by_attestations(spec, state, attestation_block) signed_attestation_block = state_transition_and_sign_block(spec, state, attestation_block) assert len(state.current_epoch_attestations) == pre_current_attestations_len + 1 diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py index ba408cd48..00ffbe0a8 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py @@ -15,11 +15,10 @@ from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): state = transition_to_valid_shard_slot(spec, state) - # At the beginning, let `x = state.slot`, `state.shard_states[shard].slot == x - 1` - slot_x = state.slot + init_slot = state.slot committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot) - assert state.shard_states[shard].slot == slot_x - 1 + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) + assert state.shard_states[shard].slot == state.slot - 1 # Create SignedShardBlock body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE @@ -50,7 +49,7 @@ def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): if valid: # After state transition, - assert state.slot == slot_x + target_len_offset_slot + assert state.slot == init_slot + target_len_offset_slot shard_state = state.shard_states[shard] assert shard_state != pre_shard_state assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1] diff --git a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py index 1499d34cd..0175bd40d 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py @@ -67,7 +67,7 @@ def test_process_beacon_block_with_normal_shard_transition(spec, state): target_len_offset_slot = 1 committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 pre_gasprice = state.shard_states[shard].gasprice @@ -93,7 +93,7 @@ def test_process_beacon_block_with_empty_proposal_transition(spec, state): target_len_offset_slot = 1 committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 # No new shard block From 5f10ac13bf91c181152a5469562dfb9d89383926 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 1 Jun 2020 23:22:59 +0800 Subject: [PATCH 10/42] PR feedback from Terence and Danny: refactor `get_committee_count_delta` --- specs/phase1/beacon-chain.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 52b6f2afb..484b24cf1 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -558,13 +558,9 @@ def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation) ```python def get_committee_count_delta(state: BeaconState, start_slot: Slot, stop_slot: Slot) -> uint64: """ - Return the sum of committee counts between ``[start_slot, stop_slot)``. + Return the sum of committee counts in range ``[start_slot, stop_slot)``. """ - committee_sum = 0 - for slot in range(start_slot, stop_slot): - count = get_committee_count_at_slot(state, Slot(slot)) - committee_sum += count - return committee_sum + return sum(get_committee_count_at_slot(state, Slot(slot)) for slot in range(start_slot, stop_slot)) ``` #### `get_start_shard` From 142ba17451e05b0e6478796fe9bea9eb2de4c488 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 2 Jun 2020 18:08:28 +0800 Subject: [PATCH 11/42] PR review from Danny --- specs/phase1/shard-fork-choice.md | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index b026aabfc..867730203 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -59,7 +59,9 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro return Gwei(sum( state.validators[i].effective_balance for i in active_indices if ( - i in store.latest_messages and get_shard_ancestor( + i in store.latest_messages and + store.latest_messages[i].shard == shard_store.shard and + get_shard_ancestor( store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot ) == root ) @@ -71,12 +73,18 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro ```python def get_shard_head(store: Store, shard_store: ShardStore) -> Root: # Execute the LMD-GHOST fork choice + shard_blocks = shard_store.blocks head_beacon_root = get_head(store) - head_shard_root = store.block_states[head_beacon_root].shard_states[shard_store.shard].latest_block_root + head_shard_state = store.block_states[head_beacon_root].shard_states[shard_store.shard] + head_shard_root = head_shard_state.latest_block_root while True: + # Find the valid child block roots children = [ root for root in shard_store.blocks.keys() - if shard_store.blocks[root].shard_parent_root == head_shard_root + if ( + shard_blocks[root].shard_parent_root == head_shard_root + and shard_blocks[root].slot > head_shard_state.slot + ) ] if len(children) == 0: return head_shard_root @@ -116,14 +124,15 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si assert shard_block.beacon_parent_root in store.block_states beacon_state = store.block_states[shard_block.beacon_parent_root] - # 3. Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - assert shard_block.slot > finalized_slot + # 3. Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) + finalized_beacon_state = store.block_states[store.finalized_checkpoint.root] + finalized_shard_state = finalized_beacon_state.shard_states[shard] + assert shard_block.slot > finalized_shard_state.slot # 4. Check block is a descendant of the finalized block at the checkpoint finalized slot + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert ( - shard_block.beacon_parent_root == store.finalized_checkpoint.root - or get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root + get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root ) # Add new block to the store From 5c5cedd60d1c08583d8627e93db1b1b179f6b7f4 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 3 Jun 2020 22:31:16 +0800 Subject: [PATCH 12/42] Apply PR feedback from Danny and Terence --- specs/phase1/shard-fork-choice.md | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 867730203..3b6bc5ac9 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -6,7 +6,7 @@ -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + - [Introduction](#introduction) - [Fork choice](#fork-choice) @@ -23,7 +23,7 @@ ## Introduction -This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase 1. +This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase 1. It assumes the [beacon chain fork choice spec](./fork-choice.md). ## Fork choice @@ -59,9 +59,9 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro return Gwei(sum( state.validators[i].effective_balance for i in active_indices if ( - i in store.latest_messages and - store.latest_messages[i].shard == shard_store.shard and - get_shard_ancestor( + i in store.latest_messages + and store.latest_messages[i].shard == shard_store.shard + and get_shard_ancestor( store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot ) == root ) @@ -116,20 +116,25 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: SignedShardBlock) -> None: shard_block = signed_shard_block.message shard = shard_store.shard - # 1. Check shard parent exists + + # Check shard + # TODO: check it in networking spec + assert shard_block.shard == shard + + # Check shard parent exists assert shard_block.shard_parent_root in shard_store.block_states pre_shard_state = shard_store.block_states[shard_block.shard_parent_root] - # 2. Check beacon parent exists + # Check beacon parent exists assert shard_block.beacon_parent_root in store.block_states beacon_state = store.block_states[shard_block.beacon_parent_root] - # 3. Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) + # Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) finalized_beacon_state = store.block_states[store.finalized_checkpoint.root] finalized_shard_state = finalized_beacon_state.shard_states[shard] assert shard_block.slot > finalized_shard_state.slot - # 4. Check block is a descendant of the finalized block at the checkpoint finalized slot + # Check block is a descendant of the finalized block at the checkpoint finalized slot finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert ( get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root From 68e934bf1552517346c1b31f1fb9e181b51d82cb Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 3 Jun 2020 23:08:38 +0800 Subject: [PATCH 13/42] Add `get_start_shard` unittests and update minimal config 1. Add unittests for testing `get_start_shard` with better granularity 2. Change `INITIAL_ACTIVE_SHARDS` from `4` to `2` for tight crosslinking --- configs/minimal.yaml | 2 +- .../test/phase_1/unittests/__init__.py | 0 .../phase_1/unittests/test_get_start_shard.py | 71 +++++++++++++++++++ 3 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 tests/core/pyspec/eth2spec/test/phase_1/unittests/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py diff --git a/configs/minimal.yaml b/configs/minimal.yaml index d8e346ffa..bb46294f5 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -165,7 +165,7 @@ PHASE_1_FORK_VERSION: 0x01000001 # [customized] for testing PHASE_1_GENESIS_SLOT: 8 # [customized] reduced for testing -INITIAL_ACTIVE_SHARDS: 4 +INITIAL_ACTIVE_SHARDS: 2 # Phase 1: General diff --git a/tests/core/pyspec/eth2spec/test/phase_1/unittests/__init__.py b/tests/core/pyspec/eth2spec/test/phase_1/unittests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py b/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py new file mode 100644 index 000000000..f9f605d9f --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py @@ -0,0 +1,71 @@ +from eth2spec.test.context import ( + PHASE0, + with_all_phases_except, + spec_state_test, +) +from eth2spec.test.helpers.state import next_epoch + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_get_committee_count_delta(spec, state): + assert spec.get_committee_count_delta(state, 0, 0) == 0 + assert spec.get_committee_count_at_slot(state, 0) != 0 + assert spec.get_committee_count_delta(state, 0, 1) == spec.get_committee_count_at_slot(state, 0) + assert spec.get_committee_count_delta(state, 1, 2) == spec.get_committee_count_at_slot(state, 1) + assert spec.get_committee_count_delta(state, 0, 2) == ( + spec.get_committee_count_at_slot(state, 0) + spec.get_committee_count_at_slot(state, 1) + ) + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_get_start_shard_current_epoch_start(spec, state): + assert state.current_epoch_start_shard == 0 + next_epoch(spec, state) + active_shard_count = spec.get_active_shard_count(state) + assert state.current_epoch_start_shard == ( + spec.get_committee_count_delta(state, 0, spec.SLOTS_PER_EPOCH) % active_shard_count + ) + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + + slot = current_epoch_start_slot + start_shard = spec.get_start_shard(state, slot) + assert start_shard == state.current_epoch_start_shard + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_get_start_shard_next_slot(spec, state): + next_epoch(spec, state) + active_shard_count = spec.get_active_shard_count(state) + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + + slot = current_epoch_start_slot + 1 + start_shard = spec.get_start_shard(state, slot) + + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + expected_start_shard = ( + state.current_epoch_start_shard + + spec.get_committee_count_delta(state, start_slot=current_epoch_start_slot, stop_slot=slot) + ) % active_shard_count + assert start_shard == expected_start_shard + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_get_start_shard_previous_slot(spec, state): + next_epoch(spec, state) + active_shard_count = spec.get_active_shard_count(state) + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + + slot = current_epoch_start_slot - 1 + start_shard = spec.get_start_shard(state, slot) + + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + expected_start_shard = ( + state.current_epoch_start_shard + + spec.MAX_COMMITTEES_PER_SLOT * spec.SLOTS_PER_EPOCH * active_shard_count + - spec.get_committee_count_delta(state, start_slot=slot, stop_slot=current_epoch_start_slot) + ) % active_shard_count + assert start_shard == expected_start_shard From a685be3bbecb18d7bf336af911a638d68e6b64f9 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 00:42:08 +0800 Subject: [PATCH 14/42] PR feedback from Danny Co-authored-by: Danny Ryan --- .../eth2spec/test/phase_1/unittests/test_get_start_shard.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py b/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py index f9f605d9f..27afd4a4e 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py @@ -46,8 +46,8 @@ def test_get_start_shard_next_slot(spec, state): current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) expected_start_shard = ( - state.current_epoch_start_shard + - spec.get_committee_count_delta(state, start_slot=current_epoch_start_slot, stop_slot=slot) + state.current_epoch_start_shard + + spec.get_committee_count_delta(state, start_slot=current_epoch_start_slot, stop_slot=slot) ) % active_shard_count assert start_shard == expected_start_shard From e1981a7bfdeb2baf296cbfdb1f74169b56341883 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 01:00:52 +0800 Subject: [PATCH 15/42] `head_shard_root` -> `shard_head_root` --- specs/phase1/fork-choice.md | 2 +- specs/phase1/shard-fork-choice.md | 8 ++++---- .../eth2spec/test/fork_choice/test_on_attestation.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/phase1/fork-choice.md b/specs/phase1/fork-choice.md index f4e771ddb..3f9fbdbfb 100644 --- a/specs/phase1/fork-choice.md +++ b/specs/phase1/fork-choice.md @@ -51,7 +51,7 @@ def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIn for i in attesting_indices: if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: store.latest_messages[i] = LatestMessage( - epoch=target.epoch, root=beacon_block_root, shard=shard, shard_root=attestation.data.head_shard_root + epoch=target.epoch, root=beacon_block_root, shard=shard, shard_root=attestation.data.shard_head_root ) ``` diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 3b6bc5ac9..fb98893ac 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -76,20 +76,20 @@ def get_shard_head(store: Store, shard_store: ShardStore) -> Root: shard_blocks = shard_store.blocks head_beacon_root = get_head(store) head_shard_state = store.block_states[head_beacon_root].shard_states[shard_store.shard] - head_shard_root = head_shard_state.latest_block_root + shard_head_root = head_shard_state.latest_block_root while True: # Find the valid child block roots children = [ root for root in shard_store.blocks.keys() if ( - shard_blocks[root].shard_parent_root == head_shard_root + shard_blocks[root].shard_parent_root == shard_head_root and shard_blocks[root].slot > head_shard_state.slot ) ] if len(children) == 0: - return head_shard_root + return shard_head_root # Sort by latest attesting balance with ties broken lexicographically - head_shard_root = max( + shard_head_root = max( children, key=lambda root: (get_shard_latest_attesting_balance(store, shard_store, root), root) ) ``` diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 04d2588d9..a5334c5c7 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -32,7 +32,7 @@ def run_on_attestation(spec, state, store, attestation, valid=True): epoch=attestation.data.target.epoch, root=attestation.data.beacon_block_root, shard=spec.get_shard(state, attestation), - shard_root=attestation.data.head_shard_root, + shard_root=attestation.data.shard_head_root, ) assert ( From c0108afe7713ac10542c478b8d07514bdad42c3b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 05:06:04 +0800 Subject: [PATCH 16/42] Use shard_block.slot to get seed for proposer selection --- specs/phase1/beacon-chain.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 484b24cf1..fccb93f55 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -538,7 +538,8 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque ```python def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard) -> ValidatorIndex: committee = get_shard_committee(beacon_state, compute_epoch_at_slot(slot), shard) - r = bytes_to_int(get_seed(beacon_state, get_current_epoch(beacon_state), DOMAIN_SHARD_COMMITTEE)[:8]) + epoch = compute_epoch_at_slot(slot) + r = bytes_to_int(get_seed(beacon_state, epoch, DOMAIN_SHARD_COMMITTEE)[:8]) return committee[r % len(committee)] ``` From d3445217416ad3a8f73dbf45e4669970d063682a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 05:30:13 +0800 Subject: [PATCH 17/42] Bugfix: should set `shard` for empty proposal --- specs/phase1/shard-transition.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index e6221a980..5e8616568 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -221,7 +221,7 @@ def get_proposal_at_slot(beacon_state: BeaconState, validate_signature=validate_signature, ) if len(choices) == 0: - block = ShardBlock(slot=slot) + block = ShardBlock(slot=slot, shard=shard) proposal = SignedShardBlock(message=block) elif len(choices) == 1: proposal = choices[0] From 26aae40941aac98d731655a475e3085e777c0110 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 05:31:53 +0800 Subject: [PATCH 18/42] Use epoch of the shard_block.slot for generating seed --- specs/phase1/beacon-chain.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 484b24cf1..fccb93f55 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -538,7 +538,8 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque ```python def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard) -> ValidatorIndex: committee = get_shard_committee(beacon_state, compute_epoch_at_slot(slot), shard) - r = bytes_to_int(get_seed(beacon_state, get_current_epoch(beacon_state), DOMAIN_SHARD_COMMITTEE)[:8]) + epoch = compute_epoch_at_slot(slot) + r = bytes_to_int(get_seed(beacon_state, epoch, DOMAIN_SHARD_COMMITTEE)[:8]) return committee[r % len(committee)] ``` From 376c83619024caee0dfd5f1c7da552ca78cedc10 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 4 Jun 2020 12:49:22 +1000 Subject: [PATCH 19/42] Clarify proposer slashing gossip conditions --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 9949db66f..767748e89 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -263,7 +263,7 @@ Additional global topics are used to propagate lower frequency validator message - _[IGNORE]_ The voluntary exit is the first valid voluntary exit received for the validator with index `signed_voluntary_exit.message.validator_index`. - _[REJECT]_ All of the conditions within `process_voluntary_exit` pass validation. - `proposer_slashing` - This topic is used solely for propagating proposer slashings to proposers on the network. Proposer slashings are sent in their entirety. The following validations MUST pass before forwarding the `proposer_slashing` on to the network - - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.index`. + - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.header_1.proposer_index`. - _[REJECT]_ All of the conditions within `process_proposer_slashing` pass validation. - `attester_slashing` - This topic is used solely for propagating attester slashings to proposers on the network. Attester slashings are sent in their entirety. Clients who receive an attester slashing on this topic MUST validate the conditions within `process_attester_slashing` before forwarding it across the network. - _[IGNORE]_ At least one index in the intersection of the attesting indices of each attestation has not yet been seen in any prior `attester_slashing` (i.e. `attester_slashed_indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices)`, verify if `any(attester_slashed_indices.difference(prior_seen_attester_slashed_indices))`). From 8afb93f5a36adac5ef312ee3fa4678632df2b6c4 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 11:19:04 +0800 Subject: [PATCH 20/42] Add `shard_block.slot` to seed --- specs/phase1/beacon-chain.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index fccb93f55..4719d2e6f 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -537,9 +537,13 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque ```python def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard) -> ValidatorIndex: - committee = get_shard_committee(beacon_state, compute_epoch_at_slot(slot), shard) + """ + Return the proposer's index of shard block at ``slot``. + """ epoch = compute_epoch_at_slot(slot) - r = bytes_to_int(get_seed(beacon_state, epoch, DOMAIN_SHARD_COMMITTEE)[:8]) + committee = get_shard_committee(beacon_state, epoch, shard) + seed = hash(get_seed(beacon_state, epoch, DOMAIN_SHARD_COMMITTEE) + int_to_bytes(slot, length=8)) + r = bytes_to_int(seed[:8]) return committee[r % len(committee)] ``` From c2b7ff7422e449dc2486261adad6196259015579 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 4 Jun 2020 16:13:49 +1000 Subject: [PATCH 21/42] Re-clarify proposer slashing check Fixes a typo from #1871 --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 767748e89..bdf0919e0 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -263,7 +263,7 @@ Additional global topics are used to propagate lower frequency validator message - _[IGNORE]_ The voluntary exit is the first valid voluntary exit received for the validator with index `signed_voluntary_exit.message.validator_index`. - _[REJECT]_ All of the conditions within `process_voluntary_exit` pass validation. - `proposer_slashing` - This topic is used solely for propagating proposer slashings to proposers on the network. Proposer slashings are sent in their entirety. The following validations MUST pass before forwarding the `proposer_slashing` on to the network - - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.header_1.proposer_index`. + - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.signed_header_1.message.proposer_index`. - _[REJECT]_ All of the conditions within `process_proposer_slashing` pass validation. - `attester_slashing` - This topic is used solely for propagating attester slashings to proposers on the network. Attester slashings are sent in their entirety. Clients who receive an attester slashing on this topic MUST validate the conditions within `process_attester_slashing` before forwarding it across the network. - _[IGNORE]_ At least one index in the intersection of the attesting indices of each attestation has not yet been seen in any prior `attester_slashing` (i.e. `attester_slashed_indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices)`, verify if `any(attester_slashed_indices.difference(prior_seen_attester_slashed_indices))`). From 7e44456be543193b3c4848faead8ea9f7baa1583 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 5 Jun 2020 09:50:49 -0600 Subject: [PATCH 22/42] mod compute_subnet_for_attestation to be usable for lookahead --- specs/phase0/p2p-interface.md | 4 ++-- specs/phase0/validator.md | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index bdf0919e0..6c4cdd2a6 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -275,7 +275,7 @@ Additional global topics are used to propagate lower frequency validator message Attestation subnets are used to propagate unaggregated attestations to subsections of the network. Their `Name`s are: - `beacon_attestation_{subnet_id}` - These topics are used to propagate unaggregated attestations to the subnet `subnet_id` (typically beacon and persistent committees) to be aggregated before being gossiped to `beacon_aggregate_and_proof`. The following validations MUST pass before forwarding the `attestation` on the subnet. - - _[REJECT]_ The attestation is for the correct subnet (i.e. `compute_subnet_for_attestation(state, attestation) == subnet_id`). + - _[REJECT]_ The attestation is for the correct subnet (i.e. `compute_subnet_for_attestation(state, attestation.data.slot, attestation.data.index) == subnet_id`). - _[IGNORE]_ `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` (a client MAY queue future attestations for processing at the appropriate slot). - _[REJECT]_ The attestation is unaggregated -- that is, it has exactly one participating validator (`len(get_attesting_indices(state, attestation.data, attestation.aggregation_bits)) == 1`). - _[IGNORE]_ There has been no other valid attestation seen on an attestation subnet that has an identical `attestation.data.target.epoch` and participating validator index. @@ -286,7 +286,7 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio Attestation broadcasting is grouped into subnets defined by a topic. The number of subnets is defined via `ATTESTATION_SUBNET_COUNT`. The correct subnet for an attestation can be calculated with `compute_subnet_for_attestation`. `beacon_attestation_{subnet_id}` topics, are rotated through throughout the epoch in a similar fashion to rotating through shards in committees in Phase 1. -Unaggregated attestations are sent to the subnet topic, `beacon_attestation_{compute_subnet_for_attestation(state, attestation)}` as `Attestation`s. +Unaggregated attestations are sent to the subnet topic, `beacon_attestation_{compute_subnet_for_attestation(state, attestation.data.slot, attestation.data.index)}` as `Attestation`s. Aggregated attestations are sent to the `beacon_aggregate_and_proof` topic as `AggregateAndProof`s. diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 5e8ddc977..35c52f346 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -199,8 +199,8 @@ The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahe Specifically a validator should: * Call `get_committee_assignment(state, next_epoch, validator_index)` when checking for next epoch assignments. -* Find peers of the pubsub topic `committee_index{committee_index % ATTESTATION_SUBNET_COUNT}_beacon_attestation`. - * If an _insufficient_ number of current peers are subscribed to the topic, the validator must discover new peers on this topic. Via the discovery protocol, find peers with an ENR containing the `attnets` entry such that `ENR["attnets"][committee_index % ATTESTATION_SUBNET_COUNT] == True`. Then validate that the peers are still persisted on the desired topic by requesting `GetMetaData` and checking the resulting `attnets` field. +* Find peers of the pubsub topic `beacon_attestation_{compute_subnet_for_attestation(state, slot, committee_index)}`. + * If an _insufficient_ number of current peers are subscribed to the topic, the validator must discover new peers on this topic. Via the discovery protocol, find peers with an ENR containing the `attnets` entry such that `ENR["attnets"][compute_subnet_for_attestation(state, slot, committee_index)] == True`. Then validate that the peers are still persisted on the desired topic by requesting `GetMetaData` and checking the resulting `attnets` field. * If the validator is assigned to be an aggregator for the slot (see `is_aggregator()`), then subscribe to the topic. *Note*: If the validator is _not_ assigned to be an aggregator, the validator only needs sufficient number of peers on the topic to be able to publish messages. The validator does not need to _subscribe_ and listen to all messages on the topic. @@ -425,18 +425,18 @@ def get_attestation_signature(state: BeaconState, attestation_data: AttestationD #### Broadcast attestation -Finally, the validator broadcasts `attestation` to the associated attestation subnet -- the `beacon_attestation_{compute_subnet_for_attestation(state, attestation)}` pubsub topic. +Finally, the validator broadcasts `attestation` to the associated attestation subnet -- the `beacon_attestation_{compute_subnet_for_attestation(state, attestation.data.slot, attestation.data.committee_index)}` pubsub topic. ```python -def compute_subnet_for_attestation(state: BeaconState, attestation: Attestation) -> uint64: +def compute_subnet_for_attestation(state: BeaconState, slot: Slot, committee_index: CommitteeIndex) -> uint64: """ Compute the correct subnet for an attestation for Phase 0. Note, this mimics expected Phase 1 behavior where attestations will be mapped to their shard subnet. """ - slots_since_epoch_start = attestation.data.slot % SLOTS_PER_EPOCH - committees_since_epoch_start = get_committee_count_at_slot(state, attestation.data.slot) * slots_since_epoch_start + slots_since_epoch_start = slot % SLOTS_PER_EPOCH + committees_since_epoch_start = get_committee_count_at_slot(state, slot) * slots_since_epoch_start - return (committees_since_epoch_start + attestation.data.index) % ATTESTATION_SUBNET_COUNT + return (committees_since_epoch_start + committee_index) % ATTESTATION_SUBNET_COUNT ``` ### Attestation aggregation From c9a53b8039cb2eada12cbb09ca1cd74509356549 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 05:33:01 +0800 Subject: [PATCH 23/42] WIP test case --- .../test/fork_choice/test_on_shard_head.py | 91 ++++++++++++------- .../eth2spec/test/helpers/shard_block.py | 26 ++++-- 2 files changed, 78 insertions(+), 39 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 5b4205e91..1151d18d7 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -5,6 +5,7 @@ from eth2spec.test.helpers.shard_block import ( build_attestation_with_shard_transition, build_shard_block, build_shard_transitions_till_slot, + get_committee_index_of_shard, ) from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_block @@ -24,44 +25,65 @@ def run_on_shard_block(spec, store, shard_store, signed_block, valid=True): assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message -def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index): +def run_apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer): shard = shard_store.shard + committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) + has_shard_committee = committee_index is not None store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH # Create SignedShardBlock - body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE - target_len_offset_slot = 1 - shard_block = build_shard_block(spec, state, shard, body=body, signed=True) - shard_blocks = [shard_block] + # Check offsets + temp_state = state.copy() + next_slot(spec, temp_state) + offset_slots = spec.get_offset_slots(temp_state, shard) + if state.slot in offset_slots: + # Build block + body = b'\x56' * 4 + shard_head_root = spec.get_shard_head(store, shard_store) + shard_parent_state = shard_store.block_states[shard_head_root] + assert shard_parent_state.slot != state.slot + shard_block = build_shard_block( + spec, state, shard, + shard_parent_state=shard_parent_state, slot=state.slot, body=body, signed=True + ) + shard_blocks_buffer.append(shard_block) + run_on_shard_block(spec, store, shard_store, shard_block) + assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() + + beacon_block = build_empty_block(spec, state, slot=state.slot + 1) # Attester creates `attestation` - # Use temporary next state to get ShardTransition of shard block - shard_transitions = build_shard_transitions_till_slot( - spec, - state, - shard_blocks={shard: shard_blocks}, - on_time_slot=state.slot + target_len_offset_slot, - ) - shard_transition = shard_transitions[shard] - attestation = build_attestation_with_shard_transition( - spec, - state, - index=committee_index, - on_time_slot=state.slot + target_len_offset_slot, - shard_transition=shard_transition, - ) + if has_shard_committee and len(shard_blocks_buffer) > 0: + # Use temporary next state to get ShardTransition of shard block + shard_transitions = build_shard_transitions_till_slot( + spec, + state, + shard_blocks={shard: shard_blocks_buffer}, + on_time_slot=state.slot + 1, + ) + shard_transition = shard_transitions[shard] + + attestation = build_attestation_with_shard_transition( + spec, + state, + index=committee_index, + on_time_slot=state.slot + 1, + shard_transition=shard_transition, + ) + assert attestation.data.slot == state.slot + assert spec.get_shard(state, attestation) == shard + beacon_block.body.attestations = [attestation] + beacon_block.body.shard_transitions = shard_transitions - # Propose beacon block at slot - beacon_block = build_empty_block(spec, state, slot=state.slot + 1) - beacon_block.body.attestations = [attestation] - beacon_block.body.shard_transitions = shard_transitions signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) - run_on_shard_block(spec, store, shard_store, shard_block) add_block_to_store(spec, store, signed_beacon_block) - assert spec.get_head(store) == beacon_block.hash_tree_root() - assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() + + if has_shard_committee: + shard_blocks_buffer = [] # clear buffer + + return has_shard_committee, shard_blocks_buffer @with_all_phases_except([PHASE0]) @@ -69,16 +91,19 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) def test_basic(spec, state): spec.PHASE_1_GENESIS_SLOT = 0 # FIXME: remove mocking state = spec.upgrade_to_phase1(state) - next_slot(spec, state) # Initialization store = spec.get_forkchoice_store(state) anchor_root = get_anchor_root(spec, state) assert spec.get_head(store) == anchor_root - committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard = spec.Shard(1) shard_store = spec.get_forkchoice_shard_store(state, shard) - - run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) - run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) + shard_block_count = 2 + shard_blocks_buffer = [] + while shard_block_count > 0: + has_shard_committee, shard_blocks_buffer = run_apply_shard_and_beacon( + spec, state, store, shard_store, shard_blocks_buffer + ) + if has_shard_committee: + shard_block_count -= 1 diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 58efada83..410213edd 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -23,19 +23,24 @@ def build_shard_block(spec, shard, slot=None, body=None, + shard_parent_state=None, signed=False): - shard_state = beacon_state.shard_states[shard] + if shard_parent_state is None: + shard_parent_state = beacon_state.shard_states[shard] + if slot is None: - slot = shard_state.slot + 1 + slot = shard_parent_state.slot + 1 if body is None: body = b'\x56' * 128 - proposer_index = spec.get_shard_proposer_index(beacon_state, slot, shard) + temp_state = beacon_state.copy() + transition_to(spec, temp_state, slot) beacon_state, beacon_parent_root = get_state_and_beacon_parent_root_at_slot(spec, beacon_state, slot) - + assert beacon_state == temp_state + proposer_index = spec.get_shard_proposer_index(temp_state, slot, shard) block = spec.ShardBlock( - shard_parent_root=shard_state.latest_block_root, + shard_parent_root=shard_parent_state.latest_block_root, beacon_parent_root=beacon_parent_root, slot=slot, shard=shard, @@ -59,7 +64,6 @@ def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): for shard, blocks in shard_blocks.items(): offset_slots = spec.get_offset_slots(temp_state, shard) len_offset_slots = len(offset_slots) - assert len_offset_slots == on_time_slot - state.shard_states[shard].slot - 1 shard_transition = spec.get_shard_transition(temp_state, shard, blocks) if len(blocks) > 0: shard_block_root = blocks[-1].message.hash_tree_root() @@ -84,3 +88,13 @@ def build_attestation_with_shard_transition(spec, state, index, on_time_slot, sh if shard_transition is not None: assert attestation.data.shard_transition_root == shard_transition.hash_tree_root() return attestation + + +def get_committee_index_of_shard(spec, state, slot, shard): # Optional[CommitteeIndex] + active_shard_count = spec.get_active_shard_count(state) + committee_count = spec.get_committee_count_at_slot(state, slot) + start_shard = spec.get_start_shard(state, slot) + for committee_index in range(committee_count): + if (start_shard + committee_index) % active_shard_count == shard: + return committee_index + return None From 727353c054f259aa81e7738099fc055722e9734c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 18:39:27 +0800 Subject: [PATCH 24/42] Verify shard_block.slot fits the expected offset_slots --- specs/phase1/shard-fork-choice.md | 9 +++++---- specs/phase1/shard-transition.md | 3 +++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index fb98893ac..b60edd948 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -127,7 +127,6 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si # Check beacon parent exists assert shard_block.beacon_parent_root in store.block_states - beacon_state = store.block_states[shard_block.beacon_parent_root] # Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) finalized_beacon_state = store.block_states[store.finalized_checkpoint.root] @@ -144,9 +143,11 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si shard_store.blocks[hash_tree_root(shard_block)] = shard_block # Check the block is valid and compute the post-state - verify_shard_block_message(beacon_state, pre_shard_state, shard_block, shard_block.slot, shard) - verify_shard_block_signature(beacon_state, signed_shard_block) - post_state = get_post_shard_state(beacon_state, pre_shard_state, shard_block) + beacon_head_root = get_head(store) + beacon_head_state = store.block_states[beacon_head_root] + assert verify_shard_block_message(beacon_head_state, pre_shard_state, shard_block, shard_block.slot, shard) + assert verify_shard_block_signature(beacon_head_state, signed_shard_block) + post_state = get_post_shard_state(beacon_head_state, pre_shard_state, shard_block) # Add new state for this block to the store shard_store.block_states[hash_tree_root(shard_block)] = post_state ``` diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 5e8616568..6c4f652d2 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -50,6 +50,9 @@ def verify_shard_block_message(beacon_state: BeaconState, shard: Shard) -> bool: assert block.shard_parent_root == shard_state.latest_block_root assert block.slot == slot + next_slot = Slot(beacon_state.slot + 1) + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), next_slot) + assert slot in offset_slots assert block.shard == shard assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard) assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE From f8597d296545c6711d1c7cf8fe25d5dbb5b1987b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 20:31:54 +0800 Subject: [PATCH 25/42] Add `get_pendings_shard_blocks` --- specs/phase1/shard-fork-choice.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index b60edd948..5bc2cbe4e 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -16,6 +16,7 @@ - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance) - [`get_shard_head`](#get_shard_head) - [`get_shard_ancestor`](#get_shard_ancestor) + - [`get_pendings_shard_blocks`](#get_pendings_shard_blocks) - [Handlers](#handlers) - [`on_shard_block`](#on_shard_block) @@ -108,6 +109,31 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: return root ``` +#### `get_pendings_shard_blocks` + +```python +def get_pendings_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ShardBlock]: + """ + Return the shard blocks branch that from shard head to beacon head. + """ + shard = shard_store.shard + + beacon_head_root = get_head(store) + beacon_head_state = store.block_states[beacon_head_root] + latest_shard_block_root = beacon_head_state.shard_states[shard].latest_block_root + + shard_head_root = get_shard_head(store, shard_store) + root = shard_head_root + shard_blocks = [] + while root != latest_shard_block_root: + shard_block = shard_store.blocks[root] + shard_blocks.append(shard_block) + root = shard_block.shard_parent_root + + shard_blocks.reverse() + return shard_blocks +``` + ### Handlers #### `on_shard_block` From ab42eee4c04818d58fbcea5ba83d1c5890d0c3b8 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 20:32:31 +0800 Subject: [PATCH 26/42] Update shard fork choice rule to be able to handle mainnet config --- .../test/fork_choice/test_on_shard_head.py | 97 ++++++++++++------- 1 file changed, 62 insertions(+), 35 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 1151d18d7..7e94ddd8e 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -1,6 +1,6 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root -from eth2spec.test.context import spec_state_test, with_all_phases_except, PHASE0 +from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_except, never_bls from eth2spec.test.helpers.shard_block import ( build_attestation_with_shard_transition, build_shard_block, @@ -8,7 +8,7 @@ from eth2spec.test.helpers.shard_block import ( get_committee_index_of_shard, ) from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root -from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_block +from eth2spec.test.helpers.state import state_transition_and_sign_block from eth2spec.test.helpers.block import build_empty_block @@ -25,35 +25,51 @@ def run_on_shard_block(spec, store, shard_store, signed_block, valid=True): assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message -def run_apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer): +def apply_shard_block(spec, store, shard_store, beacon_head_state, shard_blocks_buffer): shard = shard_store.shard - committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) - has_shard_committee = committee_index is not None + body = b'\x56' * 4 + shard_head_root = spec.get_shard_head(store, shard_store) + shard_parent_state = shard_store.block_states[shard_head_root] + assert shard_parent_state.slot != beacon_head_state.slot + shard_block = build_shard_block( + spec, beacon_head_state, shard, + shard_parent_state=shard_parent_state, slot=beacon_head_state.slot, body=body, signed=True + ) + shard_blocks_buffer.append(shard_block) + run_on_shard_block(spec, store, shard_store, shard_block) + assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() + + +def check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer): + pending_shard_blocks = [ + spec.SignedShardBlock(message=b) + for b in spec.get_pendings_shard_blocks(store, shard_store) + ] + assert pending_shard_blocks == shard_blocks_buffer + + +def is_in_offset_sets(spec, beacon_head_state, shard): + offset_slots = spec.compute_offset_slots( + beacon_head_state.shard_states[shard].slot, beacon_head_state.slot + 1 + ) + return beacon_head_state.slot in offset_slots + + +def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer): store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH - # Create SignedShardBlock - # Check offsets - temp_state = state.copy() - next_slot(spec, temp_state) - offset_slots = spec.get_offset_slots(temp_state, shard) - if state.slot in offset_slots: - # Build block - body = b'\x56' * 4 - shard_head_root = spec.get_shard_head(store, shard_store) - shard_parent_state = shard_store.block_states[shard_head_root] - assert shard_parent_state.slot != state.slot - shard_block = build_shard_block( - spec, state, shard, - shard_parent_state=shard_parent_state, slot=state.slot, body=body, signed=True - ) - shard_blocks_buffer.append(shard_block) - run_on_shard_block(spec, store, shard_store, shard_block) - assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() + shard = shard_store.shard + committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) + has_shard_committee = committee_index is not None # has committee of `shard` at this slot + # On beacon blocks at `state.slot + 1` beacon_block = build_empty_block(spec, state, slot=state.slot + 1) - # Attester creates `attestation` + # If next slot has committee of `shard`, add `shard_transtion` to the proposing beacon block if has_shard_committee and len(shard_blocks_buffer) > 0: + # Sanity check `get_pendings_shard_blocks` function + check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer) + # Use temporary next state to get ShardTransition of shard block shard_transitions = build_shard_transitions_till_slot( spec, @@ -62,7 +78,6 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buf on_time_slot=state.slot + 1, ) shard_transition = shard_transitions[shard] - attestation = build_attestation_with_shard_transition( spec, state, @@ -75,35 +90,47 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buf beacon_block.body.attestations = [attestation] beacon_block.body.shard_transitions = shard_transitions - signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) + # Clear buffer + shard_blocks_buffer.clear() + signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) add_block_to_store(spec, store, signed_beacon_block) assert spec.get_head(store) == beacon_block.hash_tree_root() - if has_shard_committee: - shard_blocks_buffer = [] # clear buffer + # On shard block at updated `state.slot` + if is_in_offset_sets(spec, state, shard): + # The created shard block would be appended to `shard_blocks_buffer` + apply_shard_block(spec, store, shard_store, state, shard_blocks_buffer) - return has_shard_committee, shard_blocks_buffer + return has_shard_committee @with_all_phases_except([PHASE0]) @spec_state_test +@never_bls # Set to never_bls for testing `check_pending_shard_blocks` def test_basic(spec, state): - spec.PHASE_1_GENESIS_SLOT = 0 # FIXME: remove mocking + spec.PHASE_1_GENESIS_SLOT = 0 # NOTE: mock genesis slot here state = spec.upgrade_to_phase1(state) + shard = spec.Shard(1) # Initialization store = spec.get_forkchoice_store(state) anchor_root = get_anchor_root(spec, state) assert spec.get_head(store) == anchor_root - shard = spec.Shard(1) shard_store = spec.get_forkchoice_shard_store(state, shard) - shard_block_count = 2 + shard_head_root = spec.get_shard_head(store, shard_store) + assert shard_head_root == state.shard_states[shard].latest_block_root + assert shard_store.block_states[shard_head_root].slot == 1 + assert shard_store.block_states[shard_head_root] == state.shard_states[shard] + + # For mainnet config, it's possible that only one committee of `shard` per epoch. + # we set this counter to test more rounds. + shard_committee_counter = 2 shard_blocks_buffer = [] - while shard_block_count > 0: - has_shard_committee, shard_blocks_buffer = run_apply_shard_and_beacon( + while shard_committee_counter > 0: + has_shard_committee = apply_shard_and_beacon( spec, state, store, shard_store, shard_blocks_buffer ) if has_shard_committee: - shard_block_count -= 1 + shard_committee_counter -= 1 From 6f9c290bfb97240a9f41370b026d066ed8df6917 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 20:40:09 +0800 Subject: [PATCH 27/42] Add TODO flag of latest message --- specs/phase1/shard-fork-choice.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 5bc2cbe4e..a474c8214 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -61,6 +61,8 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro state.validators[i].effective_balance for i in active_indices if ( i in store.latest_messages + # TODO: check the latest message logic: currently, validator's previous vote of another shard + # would be ignored once their newer vote is accepted. Check if it makes sense. and store.latest_messages[i].shard == shard_store.shard and get_shard_ancestor( store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot From a154d0c22b9a3175e74383e4df53e5b09f6df0fd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 21:24:17 +0800 Subject: [PATCH 28/42] Fix typo --- specs/phase1/shard-fork-choice.md | 6 +++--- .../pyspec/eth2spec/test/fork_choice/test_on_shard_head.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index a474c8214..427b72784 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -16,7 +16,7 @@ - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance) - [`get_shard_head`](#get_shard_head) - [`get_shard_ancestor`](#get_shard_ancestor) - - [`get_pendings_shard_blocks`](#get_pendings_shard_blocks) + - [`get_pending_shard_blocks`](#get_pending_shard_blocks) - [Handlers](#handlers) - [`on_shard_block`](#on_shard_block) @@ -111,10 +111,10 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: return root ``` -#### `get_pendings_shard_blocks` +#### `get_pending_shard_blocks` ```python -def get_pendings_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ShardBlock]: +def get_pending_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ShardBlock]: """ Return the shard blocks branch that from shard head to beacon head. """ diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 7e94ddd8e..1ba15968f 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -43,7 +43,7 @@ def apply_shard_block(spec, store, shard_store, beacon_head_state, shard_blocks_ def check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer): pending_shard_blocks = [ spec.SignedShardBlock(message=b) - for b in spec.get_pendings_shard_blocks(store, shard_store) + for b in spec.get_pending_shard_blocks(store, shard_store) ] assert pending_shard_blocks == shard_blocks_buffer @@ -67,7 +67,7 @@ def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer) # If next slot has committee of `shard`, add `shard_transtion` to the proposing beacon block if has_shard_committee and len(shard_blocks_buffer) > 0: - # Sanity check `get_pendings_shard_blocks` function + # Sanity check `get_pending_shard_blocks` function check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer) # Use temporary next state to get ShardTransition of shard block @@ -129,6 +129,7 @@ def test_basic(spec, state): shard_committee_counter = 2 shard_blocks_buffer = [] while shard_committee_counter > 0: + print(f'state.slot', state.slot) has_shard_committee = apply_shard_and_beacon( spec, state, store, shard_store, shard_blocks_buffer ) From 2d4788fe7d032f473fe5d60d732c40505ad8f485 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 5 Jun 2020 16:19:25 +0800 Subject: [PATCH 29/42] Fix `verify_shard_block_message` Add check for `block.beacon_parent_root` per Terence's suggestion Update `get_shard_transition` 1. Disable verification: it will be fix in v-guide 2. Use `on_time_slot` to compute offset_slots Rework tests --- specs/phase1/shard-fork-choice.md | 11 ++-- specs/phase1/shard-transition.md | 66 +++++++++++-------- .../test/fork_choice/test_on_shard_head.py | 19 ++---- .../eth2spec/test/helpers/attestations.py | 2 +- .../eth2spec/test/helpers/shard_block.py | 36 +++------- .../test/helpers/shard_transitions.py | 5 +- .../test_process_shard_transition.py | 31 ++++----- .../test/phase_1/sanity/test_blocks.py | 56 ++++++---------- 8 files changed, 94 insertions(+), 132 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 427b72784..844dbfb86 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -151,10 +151,11 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si # Check shard parent exists assert shard_block.shard_parent_root in shard_store.block_states - pre_shard_state = shard_store.block_states[shard_block.shard_parent_root] + shard_parent_state = shard_store.block_states[shard_block.shard_parent_root] # Check beacon parent exists assert shard_block.beacon_parent_root in store.block_states + beacon_parent_state = store.block_states[shard_block.beacon_parent_root] # Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) finalized_beacon_state = store.block_states[store.finalized_checkpoint.root] @@ -171,11 +172,9 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si shard_store.blocks[hash_tree_root(shard_block)] = shard_block # Check the block is valid and compute the post-state - beacon_head_root = get_head(store) - beacon_head_state = store.block_states[beacon_head_root] - assert verify_shard_block_message(beacon_head_state, pre_shard_state, shard_block, shard_block.slot, shard) - assert verify_shard_block_signature(beacon_head_state, signed_shard_block) - post_state = get_post_shard_state(beacon_head_state, pre_shard_state, shard_block) + assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block, shard_block.slot, shard) + assert verify_shard_block_signature(beacon_parent_state, signed_shard_block) + post_state = get_post_shard_state(beacon_parent_state, shard_parent_state, shard_block) # Add new state for this block to the store shard_store.block_states[hash_tree_root(shard_block)] = post_state ``` diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 6c4f652d2..8d75879f5 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -30,7 +30,7 @@ This document describes the shard transition function and fraud proofs as part o ### Misc ```python -def compute_shard_transition_digest(beacon_state: BeaconState, +def compute_shard_transition_digest(beacon_parent_state: BeaconState, shard_state: ShardState, beacon_parent_root: Root, shard_body_root: Root) -> Bytes32: @@ -44,17 +44,33 @@ def compute_shard_transition_digest(beacon_state: BeaconState, ```python def verify_shard_block_message(beacon_state: BeaconState, - shard_state: ShardState, + shard_parent_state: ShardState, block: ShardBlock, slot: Slot, - shard: Shard) -> bool: - assert block.shard_parent_root == shard_state.latest_block_root - assert block.slot == slot - next_slot = Slot(beacon_state.slot + 1) + shard: Shard, + beacon_parent_slot: Slot=None) -> bool: + # Check `shard_parent_root` field + assert block.shard_parent_root == shard_parent_state.latest_block_root + # Check `beacon_parent_root` field + if beacon_parent_slot is None: + beacon_parent_slot = beacon_state.slot + if beacon_parent_slot == beacon_state.slot: + beacon_parent_block_header = beacon_state.latest_block_header.copy() + if beacon_parent_block_header.state_root == Root(): + beacon_parent_block_header.state_root = hash_tree_root(beacon_state) + beacon_parent_root = hash_tree_root(beacon_parent_block_header) + else: + beacon_parent_root = get_block_root_at_slot(beacon_state, beacon_parent_slot) + assert block.beacon_parent_root == beacon_parent_root + # Check `slot` field + next_slot = Slot(slot + 1) offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), next_slot) assert slot in offset_slots + # Check `shard` field assert block.shard == shard + # Check `proposer_index` field assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard) + # Check `body` field assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE return True ``` @@ -177,7 +193,7 @@ def compute_shard_body_roots(proposals: Sequence[SignedShardBlock]) -> Sequence[ ```python def get_proposal_choices_at_slot(beacon_state: BeaconState, - shard_state: ShardState, + shard_parent_state: ShardState, slot: Slot, shard: Shard, shard_blocks: Sequence[SignedShardBlock], @@ -188,25 +204,16 @@ def get_proposal_choices_at_slot(beacon_state: BeaconState, """ choices = [] shard_blocks_at_slot = [block for block in shard_blocks if block.message.slot == slot] + shard_state = shard_parent_state.copy() for block in shard_blocks_at_slot: - try: - # Verify block message and signature - # TODO these validations should have been checked upon receiving shard blocks. - assert verify_shard_block_message(beacon_state, shard_state, block.message, slot, shard) - if validate_signature: - assert verify_shard_block_signature(beacon_state, block) - - shard_state = get_post_shard_state(beacon_state, shard_state, block.message) - except Exception: - pass # TODO: throw error in the test helper - else: - choices.append(block) + shard_state = get_post_shard_state(beacon_state, shard_state, block.message) + choices.append(block) return choices ``` ```python def get_proposal_at_slot(beacon_state: BeaconState, - shard_state: ShardState, + shard_parent_state: ShardState, slot: Shard, shard: Shard, shard_blocks: Sequence[SignedShardBlock], @@ -217,7 +224,7 @@ def get_proposal_at_slot(beacon_state: BeaconState, """ choices = get_proposal_choices_at_slot( beacon_state=beacon_state, - shard_state=shard_state, + shard_parent_state=shard_parent_state, slot=slot, shard=shard, shard_blocks=shard_blocks, @@ -232,7 +239,7 @@ def get_proposal_at_slot(beacon_state: BeaconState, proposal = get_winning_proposal(beacon_state, choices) # Apply state transition - shard_state = get_post_shard_state(beacon_state, shard_state, proposal.message) + shard_state = get_post_shard_state(beacon_state, shard_parent_state, proposal.message) return proposal, shard_state ``` @@ -242,15 +249,17 @@ def get_shard_state_transition_result( beacon_state: BeaconState, shard: Shard, shard_blocks: Sequence[SignedShardBlock], + on_time_slot: Slot, validate_signature: bool=True, ) -> Tuple[Sequence[SignedShardBlock], Sequence[ShardState], Sequence[Root]]: proposals = [] shard_states = [] shard_state = beacon_state.shard_states[shard] - for slot in get_offset_slots(beacon_state, shard): + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) + for slot in offset_slots: proposal, shard_state = get_proposal_at_slot( beacon_state=beacon_state, - shard_state=shard_state, + shard_parent_state=shard_state, slot=slot, shard=shard, shard_blocks=shard_blocks, @@ -271,9 +280,12 @@ Suppose you are a committee member on shard `shard` at slot `current_slot` and y ```python def get_shard_transition(beacon_state: BeaconState, shard: Shard, - shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition: - offset_slots = get_offset_slots(beacon_state, shard) - proposals, shard_states, shard_data_roots = get_shard_state_transition_result(beacon_state, shard, shard_blocks) + shard_blocks: Sequence[SignedShardBlock], + on_time_slot: Slot) -> ShardTransition: + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) + proposals, shard_states, shard_data_roots = get_shard_state_transition_result( + beacon_state, shard, shard_blocks, on_time_slot + ) shard_block_lengths = [] proposer_signatures = [] diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 1ba15968f..ca79cfd23 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -1,8 +1,8 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_except, never_bls +from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.shard_block import ( - build_attestation_with_shard_transition, build_shard_block, build_shard_transitions_till_slot, get_committee_index_of_shard, @@ -25,15 +25,15 @@ def run_on_shard_block(spec, store, shard_store, signed_block, valid=True): assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message -def apply_shard_block(spec, store, shard_store, beacon_head_state, shard_blocks_buffer): +def apply_shard_block(spec, store, shard_store, beacon_parent_state, shard_blocks_buffer): shard = shard_store.shard body = b'\x56' * 4 shard_head_root = spec.get_shard_head(store, shard_store) shard_parent_state = shard_store.block_states[shard_head_root] - assert shard_parent_state.slot != beacon_head_state.slot + assert shard_parent_state.slot != beacon_parent_state.slot shard_block = build_shard_block( - spec, beacon_head_state, shard, - shard_parent_state=shard_parent_state, slot=beacon_head_state.slot, body=body, signed=True + spec, beacon_parent_state, shard, + shard_parent_state=shard_parent_state, slot=beacon_parent_state.slot, body=body, signed=True ) shard_blocks_buffer.append(shard_block) run_on_shard_block(spec, store, shard_store, shard_block) @@ -62,30 +62,26 @@ def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer) committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) has_shard_committee = committee_index is not None # has committee of `shard` at this slot - # On beacon blocks at `state.slot + 1` beacon_block = build_empty_block(spec, state, slot=state.slot + 1) # If next slot has committee of `shard`, add `shard_transtion` to the proposing beacon block if has_shard_committee and len(shard_blocks_buffer) > 0: # Sanity check `get_pending_shard_blocks` function check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer) - # Use temporary next state to get ShardTransition of shard block shard_transitions = build_shard_transitions_till_slot( spec, state, shard_blocks={shard: shard_blocks_buffer}, - on_time_slot=state.slot + 1, ) shard_transition = shard_transitions[shard] - attestation = build_attestation_with_shard_transition( + attestation = get_valid_on_time_attestation( spec, state, index=committee_index, - on_time_slot=state.slot + 1, shard_transition=shard_transition, + signed=False, ) - assert attestation.data.slot == state.slot assert spec.get_shard(state, attestation) == shard beacon_block.body.attestations = [attestation] beacon_block.body.shard_transitions = shard_transitions @@ -129,7 +125,6 @@ def test_basic(spec, state): shard_committee_counter = 2 shard_blocks_buffer = [] while shard_committee_counter > 0: - print(f'state.slot', state.slot) has_shard_committee = apply_shard_and_beacon( spec, state, store, shard_store, shard_blocks_buffer ) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 1372b0654..106069dd6 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -87,7 +87,7 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t if on_time: temp_state = state.copy() next_slot(spec, temp_state) - shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=[]) + shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=[], on_time_slot=slot + 1) lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 410213edd..0a2ed67d2 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -1,6 +1,4 @@ -from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import get_state_and_beacon_parent_root_at_slot -from eth2spec.test.helpers.state import transition_to from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.bls import only_with_bls @@ -34,11 +32,8 @@ def build_shard_block(spec, if body is None: body = b'\x56' * 128 - temp_state = beacon_state.copy() - transition_to(spec, temp_state, slot) beacon_state, beacon_parent_root = get_state_and_beacon_parent_root_at_slot(spec, beacon_state, slot) - assert beacon_state == temp_state - proposer_index = spec.get_shard_proposer_index(temp_state, slot, shard) + proposer_index = spec.get_shard_proposer_index(beacon_state, slot, shard) block = spec.ShardBlock( shard_parent_root=shard_parent_state.latest_block_root, beacon_parent_root=beacon_parent_root, @@ -57,14 +52,17 @@ def build_shard_block(spec, return signed_block -def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): - temp_state = state.copy() - transition_to(spec, temp_state, on_time_slot) +def build_shard_transitions_till_slot(spec, parent_beacon_state, shard_blocks): shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS + on_time_slot = parent_beacon_state.slot + 1 for shard, blocks in shard_blocks.items(): - offset_slots = spec.get_offset_slots(temp_state, shard) + offset_slots = spec.compute_offset_slots( + spec.get_latest_slot_for_shard(parent_beacon_state, shard), + on_time_slot, + ) len_offset_slots = len(offset_slots) - shard_transition = spec.get_shard_transition(temp_state, shard, blocks) + shard_transition = spec.get_shard_transition(parent_beacon_state, shard, blocks, on_time_slot) + if len(blocks) > 0: shard_block_root = blocks[-1].message.hash_tree_root() assert shard_transition.shard_states[len_offset_slots - 1].latest_block_root == shard_block_root @@ -74,22 +72,6 @@ def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): return shard_transitions -def build_attestation_with_shard_transition(spec, state, index, on_time_slot, shard_transition): - temp_state = state.copy() - transition_to(spec, temp_state, on_time_slot - 1) - attestation = get_valid_on_time_attestation( - spec, - temp_state, - index=index, - shard_transition=shard_transition, - signed=True, - ) - assert attestation.data.slot == temp_state.slot - if shard_transition is not None: - assert attestation.data.shard_transition_root == shard_transition.hash_tree_root() - return attestation - - def get_committee_index_of_shard(spec, state, slot, shard): # Optional[CommitteeIndex] active_shard_count = spec.get_active_shard_count(state) committee_count = spec.get_committee_count_at_slot(state, slot) diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py index abb5e7278..8e62b2f27 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py @@ -1,5 +1,4 @@ from eth2spec.test.context import expect_assertion_error -from eth2spec.test.helpers.state import transition_to def run_shard_transitions_processing(spec, state, shard_transitions, attestations, valid=True): @@ -37,7 +36,5 @@ def get_shard_transition_of_committee(spec, state, committee_index, slot=None, s slot = state.slot shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) - temp_state = state.copy() - transition_to(spec, temp_state, slot + 1) - shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=shard_blocks) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks, on_time_slot=slot + 1) return shard_transition diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py index 00ffbe0a8..dab4973da 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py @@ -2,71 +2,64 @@ from eth2spec.test.context import ( PHASE0, with_all_phases_except, spec_state_test, - always_bls, ) +from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.shard_transitions import run_shard_transitions_processing from eth2spec.test.helpers.shard_block import ( - build_attestation_with_shard_transition, build_shard_block, build_shard_transitions_till_slot, ) -from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot +from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot, next_slot def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): state = transition_to_valid_shard_slot(spec, state) - init_slot = state.slot committee_index = spec.CommitteeIndex(0) shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 + transition_to(spec, state, state.slot + target_len_offset_slot) + assert state.shard_states[shard].slot == state.slot - target_len_offset_slot - 1 # Create SignedShardBlock body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE - shard_block = build_shard_block(spec, state, shard, body=body, signed=True) + shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) shard_blocks = [shard_block] - # Create a shard_transitions that would be included at beacon block `state.slot + target_len_offset_slot` shard_transitions = build_shard_transitions_till_slot( spec, state, shard_blocks={shard: shard_blocks}, - on_time_slot=state.slot + target_len_offset_slot, ) shard_transition = shard_transitions[shard] - # Create an attestation that would be included at beacon block `state.slot + target_len_offset_slot` - attestation = build_attestation_with_shard_transition( + attestation = get_valid_on_time_attestation( spec, state, index=committee_index, - on_time_slot=state.slot + target_len_offset_slot, shard_transition=shard_transition, + signed=False, ) + next_slot(spec, state) pre_gasprice = state.shard_states[shard].gasprice - - transition_to(spec, state, state.slot + target_len_offset_slot) pre_shard_state = state.shard_states[shard] - yield from run_shard_transitions_processing(spec, state, shard_transitions, [attestation], valid=valid) if valid: - # After state transition, - assert state.slot == init_slot + target_len_offset_slot shard_state = state.shard_states[shard] assert shard_state != pre_shard_state assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1] - + assert shard_state.latest_block_root == shard_block.message.hash_tree_root() if target_len_offset_slot == 1: assert shard_state.gasprice > pre_gasprice @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_basic_crosslinks(spec, state): + # NOTE: this test is only for full crosslink (minimal config), not for mainnet yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=1, valid=True) @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_multiple_offset_slots(spec, state): - yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=3, valid=True) + # NOTE: this test is only for full crosslink (minimal config), not for mainnet + yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=2, valid=True) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py index 0175bd40d..828cd19d7 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py @@ -4,37 +4,40 @@ from eth2spec.test.context import ( PHASE0, with_all_phases_except, spec_state_test, - always_bls, ) +from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import build_empty_block from eth2spec.test.helpers.shard_block import ( - build_attestation_with_shard_transition, build_shard_block, build_shard_transitions_till_slot, ) -from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot +from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot, transition_to -def run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index, valid=True): - shard_transitions = build_shard_transitions_till_slot( - spec, state, shard_blocks, on_time_slot=state.slot + target_len_offset_slot - ) +def run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard, valid=True): + transition_to(spec, state, state.slot + target_len_offset_slot) + + body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE + shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) + shard_blocks: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]} + + shard_transitions = build_shard_transitions_till_slot(spec, state, shard_blocks) attestations = [ - build_attestation_with_shard_transition( + get_valid_on_time_attestation( spec, state, - on_time_slot=state.slot + target_len_offset_slot, index=committee_index, shard_transition=shard_transitions[shard], + signed=True, ) for shard in shard_blocks.keys() ] - # Propose beacon block at slot `x + 1` - beacon_block = build_empty_block(spec, state, slot=state.slot + target_len_offset_slot) + beacon_block = build_empty_block(spec, state, slot=state.slot + 1) beacon_block.body.attestations = attestations beacon_block.body.shard_transitions = shard_transitions + pre_gasprice = state.shard_states[shard].gasprice pre_shard_states = state.shard_states.copy() yield 'pre', state.copy() yield 'block', beacon_block @@ -52,17 +55,18 @@ def run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_off assert post_shard_state == shard_transitions[shard].shard_states[ len(shard_transitions[shard].shard_states) - 1 ] - assert beacon_block.slot == shard_transitions[shard].shard_states[0].slot + target_len_offset_slot assert post_shard_state.slot == state.slot - 1 if len(shard_blocks[shard]) == 0: # `latest_block_root` is the same assert post_shard_state.latest_block_root == pre_shard_states[shard].latest_block_root + if target_len_offset_slot == 1 and len(shard_blocks) > 0: + assert post_shard_state.gasprice > pre_gasprice @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_process_beacon_block_with_normal_shard_transition(spec, state): + # NOTE: this test is only for full crosslink (minimal config), not for mainnet state = transition_to_valid_shard_slot(spec, state) target_len_offset_slot = 1 @@ -70,25 +74,13 @@ def test_process_beacon_block_with_normal_shard_transition(spec, state): shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 - pre_gasprice = state.shard_states[shard].gasprice - - # Create SignedShardBlock at slot `shard_state.slot + 1` - body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE - shard_block = build_shard_block(spec, state, shard, body=body, signed=True) - shard_blocks: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]} - - yield from run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index) - - shard_state = state.shard_states[shard] - - if target_len_offset_slot == 1 and len(shard_blocks) > 0: - assert shard_state.gasprice > pre_gasprice + yield from run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard) @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_process_beacon_block_with_empty_proposal_transition(spec, state): + # NOTE: this test is only for full crosslink (minimal config), not for mainnet state = transition_to_valid_shard_slot(spec, state) target_len_offset_slot = 1 @@ -96,12 +88,4 @@ def test_process_beacon_block_with_empty_proposal_transition(spec, state): shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 - # No new shard block - shard_blocks = {} - - pre_gasprice = state.shard_states[shard].gasprice - - yield from run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index) - - if target_len_offset_slot == 1 and len(shard_blocks) > 0: - assert state.shard_states[shard].gasprice > pre_gasprice + yield from run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard) From 2afa315cb3b1d8abcef3af2ef7df9046eb4f8f23 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 5 Jun 2020 21:49:50 +0800 Subject: [PATCH 30/42] clean leftover --- tests/core/pyspec/eth2spec/test/helpers/attestations.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 106069dd6..ef90a71aa 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -85,9 +85,7 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t # No shard transition -> no shard block shard = spec.get_shard(state, spec.Attestation(data=attestation_data)) if on_time: - temp_state = state.copy() - next_slot(spec, temp_state) - shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=[], on_time_slot=slot + 1) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=[], on_time_slot=slot + 1) lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() From a71c0a5ccc4c74f5ae8c3c071aa47506aa5cc03a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 6 Jun 2020 02:35:46 +0800 Subject: [PATCH 31/42] Per #1704 discussion, remove `on_time_slot`: the given `beacon_state` should be transitioned. --- specs/phase1/shard-transition.md | 4 ++-- tests/core/pyspec/eth2spec/test/helpers/attestations.py | 6 ++---- tests/core/pyspec/eth2spec/test/helpers/shard_block.py | 2 +- .../core/pyspec/eth2spec/test/helpers/shard_transitions.py | 7 ++----- 4 files changed, 7 insertions(+), 12 deletions(-) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 8d75879f5..191dbd1aa 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -280,8 +280,8 @@ Suppose you are a committee member on shard `shard` at slot `current_slot` and y ```python def get_shard_transition(beacon_state: BeaconState, shard: Shard, - shard_blocks: Sequence[SignedShardBlock], - on_time_slot: Slot) -> ShardTransition: + shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition: + on_time_slot = Slot(beacon_state.slot + 1) offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) proposals, shard_states, shard_data_roots = get_shard_state_transition_result( beacon_state, shard, shard_blocks, on_time_slot diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index ef90a71aa..1e0560405 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -85,7 +85,7 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t # No shard transition -> no shard block shard = spec.get_shard(state, spec.Attestation(data=attestation_data)) if on_time: - shard_transition = spec.get_shard_transition(state, shard, shard_blocks=[], on_time_slot=slot + 1) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=[]) lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() @@ -318,9 +318,7 @@ def next_epoch_with_attestations(spec, for index in range(committees_per_slot): if spec.fork == PHASE1: shard = spec.compute_shard_from_committee_index(post_state, index, slot_to_attest) - shard_transition = get_shard_transition_of_committee( - spec, post_state, index, slot=slot_to_attest - ) + shard_transition = get_shard_transition_of_committee(spec, post_state, index) block.body.shard_transitions[shard] = shard_transition else: shard_transition = None diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 0a2ed67d2..e63096b92 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -61,7 +61,7 @@ def build_shard_transitions_till_slot(spec, parent_beacon_state, shard_blocks): on_time_slot, ) len_offset_slots = len(offset_slots) - shard_transition = spec.get_shard_transition(parent_beacon_state, shard, blocks, on_time_slot) + shard_transition = spec.get_shard_transition(parent_beacon_state, shard, blocks) if len(blocks) > 0: shard_block_root = blocks[-1].message.hash_tree_root() diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py index 8e62b2f27..d10d1ee7b 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py @@ -28,13 +28,10 @@ def run_shard_transitions_processing(spec, state, shard_transitions, attestation yield 'post', state -def get_shard_transition_of_committee(spec, state, committee_index, slot=None, shard_blocks=None): +def get_shard_transition_of_committee(spec, state, committee_index, shard_blocks=None): if shard_blocks is None: shard_blocks = [] - if slot is None: - slot = state.slot - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) - shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks, on_time_slot=slot + 1) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks) return shard_transition From a4cc189f2b34b9e99d53df32dc24b383d3b4d3bd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 6 Jun 2020 05:19:46 +0800 Subject: [PATCH 32/42] Apply PR feedback from Danny --- specs/phase1/shard-fork-choice.md | 10 +-- specs/phase1/shard-transition.md | 63 ++++--------------- .../test/fork_choice/test_on_shard_head.py | 4 +- 3 files changed, 21 insertions(+), 56 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 844dbfb86..b1fc3080e 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -46,7 +46,7 @@ class ShardStore: def get_forkchoice_shard_store(anchor_state: BeaconState, shard: Shard) -> ShardStore: return ShardStore( shard=shard, - blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot)}, + blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot, shard=shard)}, block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]}, ) ``` @@ -168,13 +168,15 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root ) - # Add new block to the store - shard_store.blocks[hash_tree_root(shard_block)] = shard_block - # Check the block is valid and compute the post-state assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block, shard_block.slot, shard) assert verify_shard_block_signature(beacon_parent_state, signed_shard_block) + post_state = get_post_shard_state(beacon_parent_state, shard_parent_state, shard_block) + + # Add new block to the store + shard_store.blocks[hash_tree_root(shard_block)] = shard_block + # Add new state for this block to the store shard_store.block_states[hash_tree_root(shard_block)] = post_state ``` diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 191dbd1aa..fe3223933 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -47,20 +47,14 @@ def verify_shard_block_message(beacon_state: BeaconState, shard_parent_state: ShardState, block: ShardBlock, slot: Slot, - shard: Shard, - beacon_parent_slot: Slot=None) -> bool: + shard: Shard) -> bool: # Check `shard_parent_root` field assert block.shard_parent_root == shard_parent_state.latest_block_root # Check `beacon_parent_root` field - if beacon_parent_slot is None: - beacon_parent_slot = beacon_state.slot - if beacon_parent_slot == beacon_state.slot: - beacon_parent_block_header = beacon_state.latest_block_header.copy() - if beacon_parent_block_header.state_root == Root(): - beacon_parent_block_header.state_root = hash_tree_root(beacon_state) - beacon_parent_root = hash_tree_root(beacon_parent_block_header) - else: - beacon_parent_root = get_block_root_at_slot(beacon_state, beacon_parent_slot) + beacon_parent_block_header = beacon_state.latest_block_header.copy() + if beacon_parent_block_header.state_root == Root(): + beacon_parent_block_header.state_root = hash_tree_root(beacon_state) + beacon_parent_root = hash_tree_root(beacon_parent_block_header) assert block.beacon_parent_root == beacon_parent_root # Check `slot` field next_slot = Slot(slot + 1) @@ -191,26 +185,6 @@ def compute_shard_body_roots(proposals: Sequence[SignedShardBlock]) -> Sequence[ return [hash_tree_root(proposal.message.body) for proposal in proposals] ``` -```python -def get_proposal_choices_at_slot(beacon_state: BeaconState, - shard_parent_state: ShardState, - slot: Slot, - shard: Shard, - shard_blocks: Sequence[SignedShardBlock], - validate_signature: bool=True) -> Sequence[SignedShardBlock]: - """ - Return the valid shard blocks at the given ``slot``. - Note that this function doesn't change the state. - """ - choices = [] - shard_blocks_at_slot = [block for block in shard_blocks if block.message.slot == slot] - shard_state = shard_parent_state.copy() - for block in shard_blocks_at_slot: - shard_state = get_post_shard_state(beacon_state, shard_state, block.message) - choices.append(block) - return choices -``` - ```python def get_proposal_at_slot(beacon_state: BeaconState, shard_parent_state: ShardState, @@ -222,21 +196,14 @@ def get_proposal_at_slot(beacon_state: BeaconState, Return ``proposal``, ``shard_state`` of the given ``slot``. Note that this function doesn't change the state. """ - choices = get_proposal_choices_at_slot( - beacon_state=beacon_state, - shard_parent_state=shard_parent_state, - slot=slot, - shard=shard, - shard_blocks=shard_blocks, - validate_signature=validate_signature, - ) - if len(choices) == 0: + shard_blocks = [block for block in shard_blocks if block.message.slot == slot] + if len(shard_blocks) == 0: block = ShardBlock(slot=slot, shard=shard) proposal = SignedShardBlock(message=block) - elif len(choices) == 1: - proposal = choices[0] + elif len(shard_blocks) == 1: + proposal = shard_blocks[0] else: - proposal = get_winning_proposal(beacon_state, choices) + proposal = get_winning_proposal(beacon_state, shard_blocks) # Apply state transition shard_state = get_post_shard_state(beacon_state, shard_parent_state, proposal.message) @@ -249,13 +216,12 @@ def get_shard_state_transition_result( beacon_state: BeaconState, shard: Shard, shard_blocks: Sequence[SignedShardBlock], - on_time_slot: Slot, validate_signature: bool=True, ) -> Tuple[Sequence[SignedShardBlock], Sequence[ShardState], Sequence[Root]]: proposals = [] shard_states = [] shard_state = beacon_state.shard_states[shard] - offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), Slot(beacon_state.slot + 1)) for slot in offset_slots: proposal, shard_state = get_proposal_at_slot( beacon_state=beacon_state, @@ -281,11 +247,8 @@ Suppose you are a committee member on shard `shard` at slot `current_slot` and y def get_shard_transition(beacon_state: BeaconState, shard: Shard, shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition: - on_time_slot = Slot(beacon_state.slot + 1) - offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) - proposals, shard_states, shard_data_roots = get_shard_state_transition_result( - beacon_state, shard, shard_blocks, on_time_slot - ) + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), Slot(beacon_state.slot + 1)) + proposals, shard_states, shard_data_roots = get_shard_state_transition_result(beacon_state, shard, shard_blocks) shard_block_lengths = [] proposer_signatures = [] diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index ca79cfd23..1a9042960 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -89,11 +89,11 @@ def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer) # Clear buffer shard_blocks_buffer.clear() - signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) + signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) # transition! add_block_to_store(spec, store, signed_beacon_block) assert spec.get_head(store) == beacon_block.hash_tree_root() - # On shard block at updated `state.slot` + # On shard block at transitioned `state.slot` if is_in_offset_sets(spec, state, shard): # The created shard block would be appended to `shard_blocks_buffer` apply_shard_block(spec, store, shard_store, state, shard_blocks_buffer) From 435505746cd13f336c0bad1c71829df7fd322ca2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 8 Jun 2020 17:12:46 +0800 Subject: [PATCH 33/42] PR feedback from Terence: fix `get_shard_latest_attesting_balance` Co-authored-by: terence tsao --- specs/phase1/shard-fork-choice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 427b72784..5fef81868 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -65,7 +65,7 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro # would be ignored once their newer vote is accepted. Check if it makes sense. and store.latest_messages[i].shard == shard_store.shard and get_shard_ancestor( - store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot + store, shard_store, store.latest_messages[i].shard_root, shard_store.blocks[root].slot ) == root ) )) From 7e67aaeb35c6d8ffc4d8ced61f274161b6c04660 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 8 Jun 2020 18:15:14 +0800 Subject: [PATCH 34/42] Rename `build_shard_transitions_till_slot` to `get_shard_transitions` --- .../pyspec/eth2spec/test/fork_choice/test_on_shard_head.py | 4 ++-- tests/core/pyspec/eth2spec/test/helpers/shard_block.py | 2 +- .../phase_1/block_processing/test_process_shard_transition.py | 4 ++-- tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 1a9042960..24eeaedbe 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -4,7 +4,7 @@ from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_excep from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.shard_block import ( build_shard_block, - build_shard_transitions_till_slot, + get_shard_transitions, get_committee_index_of_shard, ) from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root @@ -69,7 +69,7 @@ def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer) # Sanity check `get_pending_shard_blocks` function check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer) # Use temporary next state to get ShardTransition of shard block - shard_transitions = build_shard_transitions_till_slot( + shard_transitions = get_shard_transitions( spec, state, shard_blocks={shard: shard_blocks_buffer}, diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index e63096b92..f8b4a155f 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -52,7 +52,7 @@ def build_shard_block(spec, return signed_block -def build_shard_transitions_till_slot(spec, parent_beacon_state, shard_blocks): +def get_shard_transitions(spec, parent_beacon_state, shard_blocks): shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS on_time_slot = parent_beacon_state.slot + 1 for shard, blocks in shard_blocks.items(): diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py index dab4973da..e97cc90a8 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py @@ -7,7 +7,7 @@ from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.shard_transitions import run_shard_transitions_processing from eth2spec.test.helpers.shard_block import ( build_shard_block, - build_shard_transitions_till_slot, + get_shard_transitions, ) from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot, next_slot @@ -24,7 +24,7 @@ def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) shard_blocks = [shard_block] - shard_transitions = build_shard_transitions_till_slot( + shard_transitions = get_shard_transitions( spec, state, shard_blocks={shard: shard_blocks}, diff --git a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py index 828cd19d7..33b0beac7 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py @@ -9,7 +9,7 @@ from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import build_empty_block from eth2spec.test.helpers.shard_block import ( build_shard_block, - build_shard_transitions_till_slot, + get_shard_transitions, ) from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot, transition_to @@ -21,7 +21,7 @@ def run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, comm shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) shard_blocks: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]} - shard_transitions = build_shard_transitions_till_slot(spec, state, shard_blocks) + shard_transitions = get_shard_transitions(spec, state, shard_blocks) attestations = [ get_valid_on_time_attestation( spec, From e03a970eaf88e82ade443ec37ac7451f5903966c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 8 Jun 2020 23:49:24 +0800 Subject: [PATCH 35/42] PR feedback from danny: simplify `verify_shard_block_message` params --- specs/phase1/shard-fork-choice.md | 2 +- specs/phase1/shard-transition.md | 19 +++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index b1fc3080e..6c431a68d 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -169,7 +169,7 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si ) # Check the block is valid and compute the post-state - assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block, shard_block.slot, shard) + assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block) assert verify_shard_block_signature(beacon_parent_state, signed_shard_block) post_state = get_post_shard_state(beacon_parent_state, shard_parent_state, shard_block) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index fe3223933..c62b059ee 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -43,27 +43,26 @@ def compute_shard_transition_digest(beacon_parent_state: BeaconState, ### Shard block verification functions ```python -def verify_shard_block_message(beacon_state: BeaconState, +def verify_shard_block_message(beacon_parent_state: BeaconState, shard_parent_state: ShardState, - block: ShardBlock, - slot: Slot, - shard: Shard) -> bool: + block: ShardBlock) -> bool: # Check `shard_parent_root` field assert block.shard_parent_root == shard_parent_state.latest_block_root # Check `beacon_parent_root` field - beacon_parent_block_header = beacon_state.latest_block_header.copy() + beacon_parent_block_header = beacon_parent_state.latest_block_header.copy() if beacon_parent_block_header.state_root == Root(): - beacon_parent_block_header.state_root = hash_tree_root(beacon_state) + beacon_parent_block_header.state_root = hash_tree_root(beacon_parent_state) beacon_parent_root = hash_tree_root(beacon_parent_block_header) assert block.beacon_parent_root == beacon_parent_root # Check `slot` field - next_slot = Slot(slot + 1) - offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), next_slot) - assert slot in offset_slots + shard = block.shard + next_slot = Slot(block.slot + 1) + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_parent_state, shard), next_slot) + assert block.slot in offset_slots # Check `shard` field assert block.shard == shard # Check `proposer_index` field - assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard) + assert block.proposer_index == get_shard_proposer_index(beacon_parent_state, block.slot, shard) # Check `body` field assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE return True From 2d895e9388cd7364448fb357a769a9f03d7a5141 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Jun 2020 00:13:27 +0800 Subject: [PATCH 36/42] PR feedback from danny --- specs/phase1/shard-fork-choice.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 61e4cd36f..0607613e8 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -76,18 +76,18 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro ```python def get_shard_head(store: Store, shard_store: ShardStore) -> Root: # Execute the LMD-GHOST fork choice - shard_blocks = shard_store.blocks - head_beacon_root = get_head(store) - head_shard_state = store.block_states[head_beacon_root].shard_states[shard_store.shard] - shard_head_root = head_shard_state.latest_block_root + beacon_head_root = get_head(store) + shard_head_state = store.block_states[beacon_head_root].shard_states[shard_store.shard] + shard_head_root = shard_head_state.latest_block_root + shard_blocks = { + root: shard_block for root, shard_block in shard_store.blocks.items() + if shard_block.slot > shard_head_state.slot + } while True: # Find the valid child block roots children = [ - root for root in shard_store.blocks.keys() - if ( - shard_blocks[root].shard_parent_root == shard_head_root - and shard_blocks[root].slot > head_shard_state.slot - ) + root for root, shard_block in shard_blocks.items() + if shard_block.shard_parent_root == shard_head_root ] if len(children) == 0: return shard_head_root @@ -107,7 +107,7 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: elif block.slot == slot: return root else: - # root is older than queried slot, thus a skip slot. Return earliest root prior to slot + # root is older than queried slot, thus a skip slot. Return most recent root prior to slot return root ``` @@ -116,7 +116,7 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: ```python def get_pending_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ShardBlock]: """ - Return the shard blocks branch that from shard head to beacon head. + Return the canonical shard block branch that has not yet been crosslinked. """ shard = shard_store.shard From ef11e924fa62d6e18ffc04df09ed6f56b753babb Mon Sep 17 00:00:00 2001 From: terence tsao Date: Tue, 9 Jun 2020 13:34:46 -0700 Subject: [PATCH 37/42] Update phase1 beacon-chain config table --- specs/phase1/beacon-chain.md | 52 ++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 4719d2e6f..6bc1d057a 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -102,23 +102,47 @@ Configuration is not namespaced. Instead it is strictly an extension; ### Misc -| Name | Value | Unit | Duration | -| - | - | - | - | +| Name | Value | +| - | - | | `MAX_SHARDS` | `2**10` (= 1024) | -| `ONLINE_PERIOD` | `OnlineEpochs(2**3)` (= 8) | online epochs | ~51 min | | `LIGHT_CLIENT_COMMITTEE_SIZE` | `2**7` (= 128) | +| `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | + +### Shard block configs +| Name | Value | +| - | - | +| `MAX_SHARD_BLOCK_SIZE` | `2**20` (= 1,048,576) | +| `TARGET_SHARD_BLOCK_SIZE` | `2**18` (= 262,144) | +| `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | +| `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | + +### Gwei values + +| Name | Value | +| - | - | +| `MAX_GASPRICE` | `Gwei(2**14)` (= 16,384) | Gwei | +| `MIN_GASPRICE` | `Gwei(2**3)` (= 8) | Gwei | + +### Initial values + +| Name | Value | +| - | - | +| `NO_SIGNATURE` | `BLSSignature(b'\x00' * 96)` | + +### Time parameters + +| Name | Value | Unit | Duration | +| - | - | :-: | :-: | +| `ONLINE_PERIOD` | `OnlineEpochs(2**3)` (= 8) | online epochs | ~51 mins | | `LIGHT_CLIENT_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | -| `MAX_SHARD_BLOCK_SIZE` | `2**20` (= 1,048,576) | | -| `TARGET_SHARD_BLOCK_SIZE` | `2**18` (= 262,144) | | -| `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | | -| `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | | -| `MAX_GASPRICE` | `Gwei(2**14)` (= 16,384) | Gwei | | -| `MIN_GASPRICE` | `Gwei(2**3)` (= 8) | Gwei | | -| `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | | -| `DOMAIN_SHARD_PROPOSAL` | `DomainType('0x80000000')` | | -| `DOMAIN_SHARD_COMMITTEE` | `DomainType('0x81000000')` | | -| `DOMAIN_LIGHT_CLIENT` | `DomainType('0x82000000')` | | -| `NO_SIGNATURE` | `BLSSignature(b'\x00' * 96)` | | + +### Domain types + +| Name | Value | +| - | - | +| `DOMAIN_SHARD_PROPOSAL` | `DomainType('0x80000000')` | +| `DOMAIN_SHARD_COMMITTEE` | `DomainType('0x81000000')` | +| `DOMAIN_LIGHT_CLIENT` | `DomainType('0x82000000')` | ## Updated containers From 0f6efcd50ddbd6800871f7a4bd781f1d87ac4dec Mon Sep 17 00:00:00 2001 From: terence tsao Date: Tue, 9 Jun 2020 15:38:40 -0700 Subject: [PATCH 38/42] Update toc --- specs/phase1/beacon-chain.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 6bc1d057a..87de9d34e 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -12,6 +12,11 @@ - [Custom types](#custom-types) - [Configuration](#configuration) - [Misc](#misc) + - [Shard block configs](#shard-block-configs) + - [Gwei values](#gwei-values) + - [Initial values](#initial-values) + - [Time parameters](#time-parameters) + - [Domain types](#domain-types) - [Updated containers](#updated-containers) - [Extended `AttestationData`](#extended-attestationdata) - [Extended `Attestation`](#extended-attestation) @@ -109,6 +114,7 @@ Configuration is not namespaced. Instead it is strictly an extension; | `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | ### Shard block configs + | Name | Value | | - | - | | `MAX_SHARD_BLOCK_SIZE` | `2**20` (= 1,048,576) | From 378d24948763af2a4492c2bee33dddf8cea6d94d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 10 Jun 2020 11:02:10 +1000 Subject: [PATCH 39/42] Avoid redundant call to get_ancestor --- specs/phase0/fork-choice.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index b9d8ecd3c..a4d6b6cfa 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -371,15 +371,16 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: # Update finalized checkpoint if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: store.finalized_checkpoint = state.finalized_checkpoint - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - - # Update justified if new justified is later than store justified - # or if store justified is not in chain with finalized checkpoint - if ( - state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch - or get_ancestor(store, store.justified_checkpoint.root, finalized_slot) != store.finalized_checkpoint.root - ): - store.justified_checkpoint = state.current_justified_checkpoint + + if store.justified_checkpoint != state.current_justified_checkpoint: + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + # Update justified if new justified is later than store justified + # or if store justified is not in chain with finalized checkpoint + if ( + state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch + or get_ancestor(store, store.justified_checkpoint.root, finalized_slot) != store.finalized_checkpoint.root + ): + store.justified_checkpoint = state.current_justified_checkpoint ``` #### `on_attestation` From 479c40450dac7770434d2577499da0c5f63140ce Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 10 Jun 2020 18:16:26 +0800 Subject: [PATCH 40/42] Friendly lint fix --- specs/phase0/fork-choice.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index a4d6b6cfa..ad1dc540c 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -371,14 +371,15 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: # Update finalized checkpoint if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: store.finalized_checkpoint = state.finalized_checkpoint - + if store.justified_checkpoint != state.current_justified_checkpoint: finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) # Update justified if new justified is later than store justified # or if store justified is not in chain with finalized checkpoint if ( state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch - or get_ancestor(store, store.justified_checkpoint.root, finalized_slot) != store.finalized_checkpoint.root + or get_ancestor( + store, store.justified_checkpoint.root, finalized_slot) != store.finalized_checkpoint.root ): store.justified_checkpoint = state.current_justified_checkpoint ``` From 1dc6b55617deb1aabd8801ad412740c9d6adc50d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 10 Jun 2020 09:40:34 -0500 Subject: [PATCH 41/42] rearrange fork choice condition for clarity --- specs/phase0/fork-choice.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index ad1dc540c..132477ee8 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -372,15 +372,17 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: store.finalized_checkpoint = state.finalized_checkpoint + # Potentially update justified if different from store if store.justified_checkpoint != state.current_justified_checkpoint: - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) # Update justified if new justified is later than store justified - # or if store justified is not in chain with finalized checkpoint - if ( - state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch - or get_ancestor( - store, store.justified_checkpoint.root, finalized_slot) != store.finalized_checkpoint.root - ): + if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + store.justified_checkpoint = state.current_justified_checkpoint + return + + # Update justified if store justified is not in chain with finalized checkpoint + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + ancestor_at_finalized_slot = get_ancestor(store, store.justified_checkpoint.root, finalized_slot) + if ancestor_at_finalized_slot != store.finalized_checkpoint.root: store.justified_checkpoint = state.current_justified_checkpoint ``` From 993ed5c2c6474af552326582153bebc2f210080d Mon Sep 17 00:00:00 2001 From: Aditya Asgaonkar Date: Fri, 12 Jun 2020 14:37:07 -0700 Subject: [PATCH 42/42] Resetting branch to dev and adding single commit --- specs/phase0/fork-choice.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index 132477ee8..02143d0a4 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -301,7 +301,8 @@ def store_target_checkpoint_state(store: Store, target: Checkpoint) -> None: # Store target checkpoint state if not yet seen if target not in store.checkpoint_states: base_state = store.block_states[target.root].copy() - process_slots(base_state, compute_start_slot_at_epoch(target.epoch)) + if base_state.slot < compute_start_slot_at_epoch(target.epoch): + process_slots(base_state, compute_start_slot_at_epoch(target.epoch)) store.checkpoint_states[target] = base_state ```