diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 5abf6c93c..174694add 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -165,7 +165,7 @@ PHASE_1_FORK_VERSION: 0x01000001 # [customized] for testing PHASE_1_GENESIS_SLOT: 8 # [customized] reduced for testing -INITIAL_ACTIVE_SHARDS: 4 +INITIAL_ACTIVE_SHARDS: 2 # Phase 1: General diff --git a/setup.py b/setup.py index 4f3f2b872..403da206e 100644 --- a/setup.py +++ b/setup.py @@ -140,7 +140,7 @@ SUNDRY_CONSTANTS_FUNCTIONS = ''' def ceillog2(x: uint64) -> int: return (x - 1).bit_length() ''' -SUNDRY_FUNCTIONS = ''' +PHASE0_SUNDRY_FUNCTIONS = ''' # Monkey patch hash cache _hash = hash hash_cache: Dict[bytes, Bytes32] = {} @@ -220,6 +220,13 @@ get_attesting_indices = cache_this( _get_attesting_indices, lru_size=SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT * 3)''' +PHASE1_SUNDRY_FUNCTIONS = ''' +_get_start_shard = get_start_shard +get_start_shard = cache_this( + lambda state, slot: (state.validators.hash_tree_root(), slot), + _get_start_shard, lru_size=SLOTS_PER_EPOCH * 3)''' + + def objects_to_spec(spec_object: SpecObject, imports: str, fork: str) -> str: """ Given all the objects that constitute a spec, combine them into a single pyfile. @@ -250,9 +257,11 @@ def objects_to_spec(spec_object: SpecObject, imports: str, fork: str) -> str: + '\n\n' + CONFIG_LOADER + '\n\n' + ssz_objects_instantiation_spec + '\n\n' + functions_spec - + '\n' + SUNDRY_FUNCTIONS - + '\n' + + '\n' + PHASE0_SUNDRY_FUNCTIONS ) + if fork == 'phase1': + spec += '\n' + PHASE1_SUNDRY_FUNCTIONS + spec += '\n' return spec @@ -385,6 +394,7 @@ class PySpecCommand(Command): specs/phase1/shard-transition.md specs/phase1/fork-choice.md specs/phase1/phase1-fork.md + specs/phase1/shard-fork-choice.md """ else: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index 614c21cc6..02143d0a4 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -372,15 +372,19 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: # Update finalized checkpoint if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: store.finalized_checkpoint = state.finalized_checkpoint - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - # Update justified if new justified is later than store justified - # or if store justified is not in chain with finalized checkpoint - if ( - state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch - or get_ancestor(store, store.justified_checkpoint.root, finalized_slot) != store.finalized_checkpoint.root - ): - store.justified_checkpoint = state.current_justified_checkpoint + # Potentially update justified if different from store + if store.justified_checkpoint != state.current_justified_checkpoint: + # Update justified if new justified is later than store justified + if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + store.justified_checkpoint = state.current_justified_checkpoint + return + + # Update justified if store justified is not in chain with finalized checkpoint + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + ancestor_at_finalized_slot = get_ancestor(store, store.justified_checkpoint.root, finalized_slot) + if ancestor_at_finalized_slot != store.finalized_checkpoint.root: + store.justified_checkpoint = state.current_justified_checkpoint ``` #### `on_attestation` diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 9949db66f..6c4cdd2a6 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -263,7 +263,7 @@ Additional global topics are used to propagate lower frequency validator message - _[IGNORE]_ The voluntary exit is the first valid voluntary exit received for the validator with index `signed_voluntary_exit.message.validator_index`. - _[REJECT]_ All of the conditions within `process_voluntary_exit` pass validation. - `proposer_slashing` - This topic is used solely for propagating proposer slashings to proposers on the network. Proposer slashings are sent in their entirety. The following validations MUST pass before forwarding the `proposer_slashing` on to the network - - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.index`. + - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.signed_header_1.message.proposer_index`. - _[REJECT]_ All of the conditions within `process_proposer_slashing` pass validation. - `attester_slashing` - This topic is used solely for propagating attester slashings to proposers on the network. Attester slashings are sent in their entirety. Clients who receive an attester slashing on this topic MUST validate the conditions within `process_attester_slashing` before forwarding it across the network. - _[IGNORE]_ At least one index in the intersection of the attesting indices of each attestation has not yet been seen in any prior `attester_slashing` (i.e. `attester_slashed_indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices)`, verify if `any(attester_slashed_indices.difference(prior_seen_attester_slashed_indices))`). @@ -275,7 +275,7 @@ Additional global topics are used to propagate lower frequency validator message Attestation subnets are used to propagate unaggregated attestations to subsections of the network. Their `Name`s are: - `beacon_attestation_{subnet_id}` - These topics are used to propagate unaggregated attestations to the subnet `subnet_id` (typically beacon and persistent committees) to be aggregated before being gossiped to `beacon_aggregate_and_proof`. The following validations MUST pass before forwarding the `attestation` on the subnet. - - _[REJECT]_ The attestation is for the correct subnet (i.e. `compute_subnet_for_attestation(state, attestation) == subnet_id`). + - _[REJECT]_ The attestation is for the correct subnet (i.e. `compute_subnet_for_attestation(state, attestation.data.slot, attestation.data.index) == subnet_id`). - _[IGNORE]_ `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` (a client MAY queue future attestations for processing at the appropriate slot). - _[REJECT]_ The attestation is unaggregated -- that is, it has exactly one participating validator (`len(get_attesting_indices(state, attestation.data, attestation.aggregation_bits)) == 1`). - _[IGNORE]_ There has been no other valid attestation seen on an attestation subnet that has an identical `attestation.data.target.epoch` and participating validator index. @@ -286,7 +286,7 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio Attestation broadcasting is grouped into subnets defined by a topic. The number of subnets is defined via `ATTESTATION_SUBNET_COUNT`. The correct subnet for an attestation can be calculated with `compute_subnet_for_attestation`. `beacon_attestation_{subnet_id}` topics, are rotated through throughout the epoch in a similar fashion to rotating through shards in committees in Phase 1. -Unaggregated attestations are sent to the subnet topic, `beacon_attestation_{compute_subnet_for_attestation(state, attestation)}` as `Attestation`s. +Unaggregated attestations are sent to the subnet topic, `beacon_attestation_{compute_subnet_for_attestation(state, attestation.data.slot, attestation.data.index)}` as `Attestation`s. Aggregated attestations are sent to the `beacon_aggregate_and_proof` topic as `AggregateAndProof`s. diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 5e8ddc977..35c52f346 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -199,8 +199,8 @@ The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahe Specifically a validator should: * Call `get_committee_assignment(state, next_epoch, validator_index)` when checking for next epoch assignments. -* Find peers of the pubsub topic `committee_index{committee_index % ATTESTATION_SUBNET_COUNT}_beacon_attestation`. - * If an _insufficient_ number of current peers are subscribed to the topic, the validator must discover new peers on this topic. Via the discovery protocol, find peers with an ENR containing the `attnets` entry such that `ENR["attnets"][committee_index % ATTESTATION_SUBNET_COUNT] == True`. Then validate that the peers are still persisted on the desired topic by requesting `GetMetaData` and checking the resulting `attnets` field. +* Find peers of the pubsub topic `beacon_attestation_{compute_subnet_for_attestation(state, slot, committee_index)}`. + * If an _insufficient_ number of current peers are subscribed to the topic, the validator must discover new peers on this topic. Via the discovery protocol, find peers with an ENR containing the `attnets` entry such that `ENR["attnets"][compute_subnet_for_attestation(state, slot, committee_index)] == True`. Then validate that the peers are still persisted on the desired topic by requesting `GetMetaData` and checking the resulting `attnets` field. * If the validator is assigned to be an aggregator for the slot (see `is_aggregator()`), then subscribe to the topic. *Note*: If the validator is _not_ assigned to be an aggregator, the validator only needs sufficient number of peers on the topic to be able to publish messages. The validator does not need to _subscribe_ and listen to all messages on the topic. @@ -425,18 +425,18 @@ def get_attestation_signature(state: BeaconState, attestation_data: AttestationD #### Broadcast attestation -Finally, the validator broadcasts `attestation` to the associated attestation subnet -- the `beacon_attestation_{compute_subnet_for_attestation(state, attestation)}` pubsub topic. +Finally, the validator broadcasts `attestation` to the associated attestation subnet -- the `beacon_attestation_{compute_subnet_for_attestation(state, attestation.data.slot, attestation.data.committee_index)}` pubsub topic. ```python -def compute_subnet_for_attestation(state: BeaconState, attestation: Attestation) -> uint64: +def compute_subnet_for_attestation(state: BeaconState, slot: Slot, committee_index: CommitteeIndex) -> uint64: """ Compute the correct subnet for an attestation for Phase 0. Note, this mimics expected Phase 1 behavior where attestations will be mapped to their shard subnet. """ - slots_since_epoch_start = attestation.data.slot % SLOTS_PER_EPOCH - committees_since_epoch_start = get_committee_count_at_slot(state, attestation.data.slot) * slots_since_epoch_start + slots_since_epoch_start = slot % SLOTS_PER_EPOCH + committees_since_epoch_start = get_committee_count_at_slot(state, slot) * slots_since_epoch_start - return (committees_since_epoch_start + attestation.data.index) % ATTESTATION_SUBNET_COUNT + return (committees_since_epoch_start + committee_index) % ATTESTATION_SUBNET_COUNT ``` ### Attestation aggregation diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 847f8dce3..87de9d34e 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -12,6 +12,11 @@ - [Custom types](#custom-types) - [Configuration](#configuration) - [Misc](#misc) + - [Shard block configs](#shard-block-configs) + - [Gwei values](#gwei-values) + - [Initial values](#initial-values) + - [Time parameters](#time-parameters) + - [Domain types](#domain-types) - [Updated containers](#updated-containers) - [Extended `AttestationData`](#extended-attestationdata) - [Extended `Attestation`](#extended-attestation) @@ -48,6 +53,7 @@ - [`get_light_client_committee`](#get_light_client_committee) - [`get_shard_proposer_index`](#get_shard_proposer_index) - [`get_indexed_attestation`](#get_indexed_attestation) + - [`get_committee_count_delta`](#get_committee_count_delta) - [`get_start_shard`](#get_start_shard) - [`get_shard`](#get_shard) - [`get_latest_slot_for_shard`](#get_latest_slot_for_shard) @@ -73,6 +79,7 @@ - [New Attester slashing processing](#new-attester-slashing-processing) - [Light client processing](#light-client-processing) - [Epoch transition](#epoch-transition) + - [Phase 1 final updates](#phase-1-final-updates) - [Custody game updates](#custody-game-updates) - [Online-tracking](#online-tracking) - [Light client committee updates](#light-client-committee-updates) @@ -100,23 +107,48 @@ Configuration is not namespaced. Instead it is strictly an extension; ### Misc -| Name | Value | Unit | Duration | -| - | - | - | - | +| Name | Value | +| - | - | | `MAX_SHARDS` | `2**10` (= 1024) | -| `ONLINE_PERIOD` | `OnlineEpochs(2**3)` (= 8) | online epochs | ~51 min | | `LIGHT_CLIENT_COMMITTEE_SIZE` | `2**7` (= 128) | +| `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | + +### Shard block configs + +| Name | Value | +| - | - | +| `MAX_SHARD_BLOCK_SIZE` | `2**20` (= 1,048,576) | +| `TARGET_SHARD_BLOCK_SIZE` | `2**18` (= 262,144) | +| `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | +| `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | + +### Gwei values + +| Name | Value | +| - | - | +| `MAX_GASPRICE` | `Gwei(2**14)` (= 16,384) | Gwei | +| `MIN_GASPRICE` | `Gwei(2**3)` (= 8) | Gwei | + +### Initial values + +| Name | Value | +| - | - | +| `NO_SIGNATURE` | `BLSSignature(b'\x00' * 96)` | + +### Time parameters + +| Name | Value | Unit | Duration | +| - | - | :-: | :-: | +| `ONLINE_PERIOD` | `OnlineEpochs(2**3)` (= 8) | online epochs | ~51 mins | | `LIGHT_CLIENT_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | -| `MAX_SHARD_BLOCK_SIZE` | `2**20` (= 1,048,576) | | -| `TARGET_SHARD_BLOCK_SIZE` | `2**18` (= 262,144) | | -| `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | | -| `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | | -| `MAX_GASPRICE` | `Gwei(2**14)` (= 16,384) | Gwei | | -| `MIN_GASPRICE` | `Gwei(2**3)` (= 8) | Gwei | | -| `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | | -| `DOMAIN_SHARD_PROPOSAL` | `DomainType('0x80000000')` | | -| `DOMAIN_SHARD_COMMITTEE` | `DomainType('0x81000000')` | | -| `DOMAIN_LIGHT_CLIENT` | `DomainType('0x82000000')` | | -| `NO_SIGNATURE` | `BLSSignature(b'\x00' * 96)` | | + +### Domain types + +| Name | Value | +| - | - | +| `DOMAIN_SHARD_PROPOSAL` | `DomainType('0x80000000')` | +| `DOMAIN_SHARD_COMMITTEE` | `DomainType('0x81000000')` | +| `DOMAIN_LIGHT_CLIENT` | `DomainType('0x82000000')` | ## Updated containers @@ -285,6 +317,7 @@ class BeaconState(Container): current_justified_checkpoint: Checkpoint finalized_checkpoint: Checkpoint # Phase 1 + current_epoch_start_shard: Shard shard_states: List[ShardState, MAX_SHARDS] online_countdown: List[OnlineEpochs, VALIDATOR_REGISTRY_LIMIT] # not a raw byte array, considered its large size. current_light_committee: CompactCommittee @@ -534,8 +567,13 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque ```python def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard) -> ValidatorIndex: - committee = get_shard_committee(beacon_state, compute_epoch_at_slot(slot), shard) - r = bytes_to_int(get_seed(beacon_state, get_current_epoch(beacon_state), DOMAIN_SHARD_COMMITTEE)[:8]) + """ + Return the proposer's index of shard block at ``slot``. + """ + epoch = compute_epoch_at_slot(slot) + committee = get_shard_committee(beacon_state, epoch, shard) + seed = hash(get_seed(beacon_state, epoch, DOMAIN_SHARD_COMMITTEE) + int_to_bytes(slot, length=8)) + r = bytes_to_int(seed[:8]) return committee[r % len(committee)] ``` @@ -550,18 +588,49 @@ def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation) ) ``` +#### `get_committee_count_delta` + +```python +def get_committee_count_delta(state: BeaconState, start_slot: Slot, stop_slot: Slot) -> uint64: + """ + Return the sum of committee counts in range ``[start_slot, stop_slot)``. + """ + return sum(get_committee_count_at_slot(state, Slot(slot)) for slot in range(start_slot, stop_slot)) +``` + #### `get_start_shard` ```python def get_start_shard(state: BeaconState, slot: Slot) -> Shard: - # TODO: implement start shard logic - return Shard(0) + """ + Return the start shard at ``slot``. + """ + current_epoch_start_slot = compute_start_slot_at_epoch(get_current_epoch(state)) + active_shard_count = get_active_shard_count(state) + if current_epoch_start_slot == slot: + return state.current_epoch_start_shard + elif slot > current_epoch_start_slot: + # Current epoch or the next epoch lookahead + shard_delta = get_committee_count_delta(state, start_slot=current_epoch_start_slot, stop_slot=slot) + return Shard((state.current_epoch_start_shard + shard_delta) % active_shard_count) + else: + # Previous epoch + shard_delta = get_committee_count_delta(state, start_slot=slot, stop_slot=current_epoch_start_slot) + max_committees_per_epoch = MAX_COMMITTEES_PER_SLOT * SLOTS_PER_EPOCH + return Shard( + # Ensure positive + (state.current_epoch_start_shard + max_committees_per_epoch * active_shard_count - shard_delta) + % active_shard_count + ) ``` #### `get_shard` ```python def get_shard(state: BeaconState, attestation: Attestation) -> Shard: + """ + Return the shard that the given ``attestation`` is attesting. + """ return compute_shard_from_committee_index(state, attestation.data.index, attestation.data.slot) ``` @@ -569,6 +638,9 @@ def get_shard(state: BeaconState, attestation: Attestation) -> Shard: ```python def get_latest_slot_for_shard(state: BeaconState, shard: Shard) -> Slot: + """ + Return the latest slot number of the given ``shard``. + """ return state.shard_states[shard].slot ``` @@ -577,7 +649,8 @@ def get_latest_slot_for_shard(state: BeaconState, shard: Shard) -> Slot: ```python def get_offset_slots(state: BeaconState, shard: Shard) -> Sequence[Slot]: """ - Return the offset slots of the given ``shard`` between that latest included slot and current slot. + Return the offset slots of the given ``shard``. + The offset slot are after the latest slot and before current slot. """ return compute_offset_slots(get_latest_slot_for_shard(state, shard), state.slot) ``` @@ -651,8 +724,7 @@ def is_on_time_attestation(state: BeaconState, """ Check if the given attestation is on-time. """ - # TODO: MIN_ATTESTATION_INCLUSION_DELAY should always be 1 - return attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY == state.slot + return attestation.data.slot == compute_previous_slot(state.slot) ``` #### `is_winning_attestation` @@ -761,20 +833,19 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: else: assert attestation.data.source == state.previous_justified_checkpoint - shard = get_shard(state, attestation) - # Type 1: on-time attestations, the custody bits should be non-empty. if attestation.custody_bits_blocks != []: # Ensure on-time attestation assert is_on_time_attestation(state, attestation) # Correct data root count + shard = get_shard(state, attestation) assert len(attestation.custody_bits_blocks) == len(get_offset_slots(state, shard)) # Correct parent block root assert data.beacon_block_root == get_block_root_at_slot(state, compute_previous_slot(state.slot)) # Type 2: no shard transition, no custody bits else: # Ensure delayed attestation - assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY < state.slot + assert data.slot < compute_previous_slot(state.slot) # Late attestations cannot have a shard transition root assert data.shard_transition_root == Root() @@ -869,9 +940,10 @@ def process_crosslink_for_shard(state: BeaconState, committee_index: CommitteeIndex, shard_transition: ShardTransition, attestations: Sequence[Attestation]) -> Root: - committee = get_beacon_committee(state, state.slot, committee_index) + on_time_attestation_slot = compute_previous_slot(state.slot) + committee = get_beacon_committee(state, on_time_attestation_slot, committee_index) online_indices = get_online_validator_indices(state) - shard = compute_shard_from_committee_index(state, committee_index, state.slot) + shard = compute_shard_from_committee_index(state, committee_index, on_time_attestation_slot) # Loop over all shard transition roots shard_transition_roots = set([a.data.shard_transition_root for a in attestations]) @@ -926,15 +998,15 @@ def process_crosslink_for_shard(state: BeaconState, def process_crosslinks(state: BeaconState, shard_transitions: Sequence[ShardTransition], attestations: Sequence[Attestation]) -> None: - committee_count = get_committee_count_at_slot(state, state.slot) + on_time_attestation_slot = compute_previous_slot(state.slot) + committee_count = get_committee_count_at_slot(state, on_time_attestation_slot) for committee_index in map(CommitteeIndex, range(committee_count)): - shard = compute_shard_from_committee_index(state, committee_index, state.slot) # All attestations in the block for this committee/shard and current slot shard_attestations = [ attestation for attestation in attestations if is_on_time_attestation(state, attestation) and attestation.data.index == committee_index ] - + shard = compute_shard_from_committee_index(state, committee_index, on_time_attestation_slot) winning_root = process_crosslink_for_shard(state, committee_index, shard_transitions[shard], shard_attestations) if winning_root != Root(): # Mark relevant pending attestations as creating a successful crosslink @@ -1041,10 +1113,20 @@ def process_epoch(state: BeaconState) -> None: process_registry_updates(state) process_reveal_deadlines(state) process_slashings(state) - process_final_updates(state) + process_final_updates(state) # phase 0 final updates + process_phase_1_final_updates(state) +``` + +#### Phase 1 final updates + +```python +def process_phase_1_final_updates(state: BeaconState) -> None: process_custody_final_updates(state) process_online_tracking(state) process_light_client_committee_updates(state) + + # Update current_epoch_start_shard + state.current_epoch_start_shard = get_start_shard(state, Slot(state.slot + 1)) ``` #### Custody game updates diff --git a/specs/phase1/fork-choice.md b/specs/phase1/fork-choice.md index d8bf7fa09..3f9fbdbfb 100644 --- a/specs/phase1/fork-choice.md +++ b/specs/phase1/fork-choice.md @@ -10,6 +10,9 @@ - [Introduction](#introduction) - [Fork choice](#fork-choice) + - [Helpers](#helpers) + - [Extended `LatestMessage`](#extended-latestmessage) + - [Updated `update_latest_messages`](#updated-update_latest_messages) - [Handlers](#handlers) @@ -25,6 +28,33 @@ Due to the changes in the structure of `IndexedAttestation` in Phase 1, `on_atte The rest of the fork choice remains stable. +### Helpers + +#### Extended `LatestMessage` + +```python +@dataclass(eq=True, frozen=True) +class LatestMessage(object): + epoch: Epoch + root: Root + shard: Shard + shard_root: Root +``` + +#### Updated `update_latest_messages` + +```python +def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIndex], attestation: Attestation) -> None: + target = attestation.data.target + beacon_block_root = attestation.data.beacon_block_root + shard = get_shard(store.block_states[beacon_block_root], attestation) + for i in attesting_indices: + if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: + store.latest_messages[i] = LatestMessage( + epoch=target.epoch, root=beacon_block_root, shard=shard, shard_root=attestation.data.shard_head_root + ) +``` + ### Handlers ```python @@ -49,4 +79,4 @@ def on_attestation(store: Store, attestation: Attestation) -> None: if attestation.aggregation_bits[i] ] update_latest_messages(store, attesting_indices, attestation) -``` \ No newline at end of file +``` diff --git a/specs/phase1/phase1-fork.md b/specs/phase1/phase1-fork.md index cc7d8f33e..d362ed633 100644 --- a/specs/phase1/phase1-fork.md +++ b/specs/phase1/phase1-fork.md @@ -99,6 +99,7 @@ def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: current_justified_checkpoint=pre.current_justified_checkpoint, finalized_checkpoint=pre.finalized_checkpoint, # Phase 1 + current_epoch_start_shard=Shard(0), shard_states=List[ShardState, MAX_SHARDS]( ShardState( slot=pre.slot, diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md new file mode 100644 index 000000000..0607613e8 --- /dev/null +++ b/specs/phase1/shard-fork-choice.md @@ -0,0 +1,182 @@ +# Ethereum 2.0 Phase 1 -- Beacon Chain + Shard Chain Fork Choice + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + + +- [Introduction](#introduction) +- [Fork choice](#fork-choice) + - [Helpers](#helpers) + - [`ShardStore`](#shardstore) + - [`get_forkchoice_shard_store`](#get_forkchoice_shard_store) + - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance) + - [`get_shard_head`](#get_shard_head) + - [`get_shard_ancestor`](#get_shard_ancestor) + - [`get_pending_shard_blocks`](#get_pending_shard_blocks) + - [Handlers](#handlers) + - [`on_shard_block`](#on_shard_block) + + + +## Introduction + +This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase 1. It assumes the [beacon chain fork choice spec](./fork-choice.md). + +## Fork choice + +### Helpers + +#### `ShardStore` + +```python +@dataclass +class ShardStore: + shard: Shard + blocks: Dict[Root, ShardBlock] = field(default_factory=dict) + block_states: Dict[Root, ShardState] = field(default_factory=dict) +``` + +#### `get_forkchoice_shard_store` + +```python +def get_forkchoice_shard_store(anchor_state: BeaconState, shard: Shard) -> ShardStore: + return ShardStore( + shard=shard, + blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot, shard=shard)}, + block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]}, + ) +``` + +#### `get_shard_latest_attesting_balance` + +```python +def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, root: Root) -> Gwei: + state = store.checkpoint_states[store.justified_checkpoint] + active_indices = get_active_validator_indices(state, get_current_epoch(state)) + return Gwei(sum( + state.validators[i].effective_balance for i in active_indices + if ( + i in store.latest_messages + # TODO: check the latest message logic: currently, validator's previous vote of another shard + # would be ignored once their newer vote is accepted. Check if it makes sense. + and store.latest_messages[i].shard == shard_store.shard + and get_shard_ancestor( + store, shard_store, store.latest_messages[i].shard_root, shard_store.blocks[root].slot + ) == root + ) + )) +``` + +#### `get_shard_head` + +```python +def get_shard_head(store: Store, shard_store: ShardStore) -> Root: + # Execute the LMD-GHOST fork choice + beacon_head_root = get_head(store) + shard_head_state = store.block_states[beacon_head_root].shard_states[shard_store.shard] + shard_head_root = shard_head_state.latest_block_root + shard_blocks = { + root: shard_block for root, shard_block in shard_store.blocks.items() + if shard_block.slot > shard_head_state.slot + } + while True: + # Find the valid child block roots + children = [ + root for root, shard_block in shard_blocks.items() + if shard_block.shard_parent_root == shard_head_root + ] + if len(children) == 0: + return shard_head_root + # Sort by latest attesting balance with ties broken lexicographically + shard_head_root = max( + children, key=lambda root: (get_shard_latest_attesting_balance(store, shard_store, root), root) + ) +``` + +#### `get_shard_ancestor` + +```python +def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: Slot) -> Root: + block = shard_store.blocks[root] + if block.slot > slot: + return get_shard_ancestor(store, shard_store, block.shard_parent_root, slot) + elif block.slot == slot: + return root + else: + # root is older than queried slot, thus a skip slot. Return most recent root prior to slot + return root +``` + +#### `get_pending_shard_blocks` + +```python +def get_pending_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ShardBlock]: + """ + Return the canonical shard block branch that has not yet been crosslinked. + """ + shard = shard_store.shard + + beacon_head_root = get_head(store) + beacon_head_state = store.block_states[beacon_head_root] + latest_shard_block_root = beacon_head_state.shard_states[shard].latest_block_root + + shard_head_root = get_shard_head(store, shard_store) + root = shard_head_root + shard_blocks = [] + while root != latest_shard_block_root: + shard_block = shard_store.blocks[root] + shard_blocks.append(shard_block) + root = shard_block.shard_parent_root + + shard_blocks.reverse() + return shard_blocks +``` + +### Handlers + +#### `on_shard_block` + +```python +def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: SignedShardBlock) -> None: + shard_block = signed_shard_block.message + shard = shard_store.shard + + # Check shard + # TODO: check it in networking spec + assert shard_block.shard == shard + + # Check shard parent exists + assert shard_block.shard_parent_root in shard_store.block_states + shard_parent_state = shard_store.block_states[shard_block.shard_parent_root] + + # Check beacon parent exists + assert shard_block.beacon_parent_root in store.block_states + beacon_parent_state = store.block_states[shard_block.beacon_parent_root] + + # Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) + finalized_beacon_state = store.block_states[store.finalized_checkpoint.root] + finalized_shard_state = finalized_beacon_state.shard_states[shard] + assert shard_block.slot > finalized_shard_state.slot + + # Check block is a descendant of the finalized block at the checkpoint finalized slot + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert ( + get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root + ) + + # Check the block is valid and compute the post-state + assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block) + assert verify_shard_block_signature(beacon_parent_state, signed_shard_block) + + post_state = get_post_shard_state(beacon_parent_state, shard_parent_state, shard_block) + + # Add new block to the store + shard_store.blocks[hash_tree_root(shard_block)] = shard_block + + # Add new state for this block to the store + shard_store.block_states[hash_tree_root(shard_block)] = post_state +``` diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index e6221a980..c62b059ee 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -30,7 +30,7 @@ This document describes the shard transition function and fraud proofs as part o ### Misc ```python -def compute_shard_transition_digest(beacon_state: BeaconState, +def compute_shard_transition_digest(beacon_parent_state: BeaconState, shard_state: ShardState, beacon_parent_root: Root, shard_body_root: Root) -> Bytes32: @@ -43,15 +43,27 @@ def compute_shard_transition_digest(beacon_state: BeaconState, ### Shard block verification functions ```python -def verify_shard_block_message(beacon_state: BeaconState, - shard_state: ShardState, - block: ShardBlock, - slot: Slot, - shard: Shard) -> bool: - assert block.shard_parent_root == shard_state.latest_block_root - assert block.slot == slot +def verify_shard_block_message(beacon_parent_state: BeaconState, + shard_parent_state: ShardState, + block: ShardBlock) -> bool: + # Check `shard_parent_root` field + assert block.shard_parent_root == shard_parent_state.latest_block_root + # Check `beacon_parent_root` field + beacon_parent_block_header = beacon_parent_state.latest_block_header.copy() + if beacon_parent_block_header.state_root == Root(): + beacon_parent_block_header.state_root = hash_tree_root(beacon_parent_state) + beacon_parent_root = hash_tree_root(beacon_parent_block_header) + assert block.beacon_parent_root == beacon_parent_root + # Check `slot` field + shard = block.shard + next_slot = Slot(block.slot + 1) + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_parent_state, shard), next_slot) + assert block.slot in offset_slots + # Check `shard` field assert block.shard == shard - assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard) + # Check `proposer_index` field + assert block.proposer_index == get_shard_proposer_index(beacon_parent_state, block.slot, shard) + # Check `body` field assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE return True ``` @@ -172,38 +184,9 @@ def compute_shard_body_roots(proposals: Sequence[SignedShardBlock]) -> Sequence[ return [hash_tree_root(proposal.message.body) for proposal in proposals] ``` -```python -def get_proposal_choices_at_slot(beacon_state: BeaconState, - shard_state: ShardState, - slot: Slot, - shard: Shard, - shard_blocks: Sequence[SignedShardBlock], - validate_signature: bool=True) -> Sequence[SignedShardBlock]: - """ - Return the valid shard blocks at the given ``slot``. - Note that this function doesn't change the state. - """ - choices = [] - shard_blocks_at_slot = [block for block in shard_blocks if block.message.slot == slot] - for block in shard_blocks_at_slot: - try: - # Verify block message and signature - # TODO these validations should have been checked upon receiving shard blocks. - assert verify_shard_block_message(beacon_state, shard_state, block.message, slot, shard) - if validate_signature: - assert verify_shard_block_signature(beacon_state, block) - - shard_state = get_post_shard_state(beacon_state, shard_state, block.message) - except Exception: - pass # TODO: throw error in the test helper - else: - choices.append(block) - return choices -``` - ```python def get_proposal_at_slot(beacon_state: BeaconState, - shard_state: ShardState, + shard_parent_state: ShardState, slot: Shard, shard: Shard, shard_blocks: Sequence[SignedShardBlock], @@ -212,24 +195,17 @@ def get_proposal_at_slot(beacon_state: BeaconState, Return ``proposal``, ``shard_state`` of the given ``slot``. Note that this function doesn't change the state. """ - choices = get_proposal_choices_at_slot( - beacon_state=beacon_state, - shard_state=shard_state, - slot=slot, - shard=shard, - shard_blocks=shard_blocks, - validate_signature=validate_signature, - ) - if len(choices) == 0: - block = ShardBlock(slot=slot) + shard_blocks = [block for block in shard_blocks if block.message.slot == slot] + if len(shard_blocks) == 0: + block = ShardBlock(slot=slot, shard=shard) proposal = SignedShardBlock(message=block) - elif len(choices) == 1: - proposal = choices[0] + elif len(shard_blocks) == 1: + proposal = shard_blocks[0] else: - proposal = get_winning_proposal(beacon_state, choices) + proposal = get_winning_proposal(beacon_state, shard_blocks) # Apply state transition - shard_state = get_post_shard_state(beacon_state, shard_state, proposal.message) + shard_state = get_post_shard_state(beacon_state, shard_parent_state, proposal.message) return proposal, shard_state ``` @@ -244,10 +220,11 @@ def get_shard_state_transition_result( proposals = [] shard_states = [] shard_state = beacon_state.shard_states[shard] - for slot in get_offset_slots(beacon_state, shard): + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), Slot(beacon_state.slot + 1)) + for slot in offset_slots: proposal, shard_state = get_proposal_at_slot( beacon_state=beacon_state, - shard_state=shard_state, + shard_parent_state=shard_state, slot=slot, shard=shard, shard_blocks=shard_blocks, @@ -269,7 +246,7 @@ Suppose you are a committee member on shard `shard` at slot `current_slot` and y def get_shard_transition(beacon_state: BeaconState, shard: Shard, shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition: - offset_slots = get_offset_slots(beacon_state, shard) + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), Slot(beacon_state.slot + 1)) proposals, shard_states, shard_data_roots = get_shard_state_transition_result(beacon_state, shard, shard_blocks) shard_block_lengths = [] diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py index 4371bffd0..f9a739804 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py @@ -1,41 +1,13 @@ from eth2spec.test.context import with_all_phases, spec_state_test from eth2spec.test.helpers.attestations import get_valid_attestation, next_epoch_with_attestations from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.helpers.fork_choice import add_attestation_to_store, add_block_to_store, get_anchor_root from eth2spec.test.helpers.state import ( next_epoch, state_transition_and_sign_block, ) -def add_block_to_store(spec, store, signed_block): - pre_state = store.block_states[signed_block.message.parent_root] - block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT - - if store.time < block_time: - spec.on_tick(store, block_time) - - spec.on_block(store, signed_block) - - -def add_attestation_to_store(spec, store, attestation): - parent_block = store.blocks[attestation.data.beacon_block_root] - pre_state = store.block_states[spec.hash_tree_root(parent_block)] - block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT - next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT - - if store.time < next_epoch_time: - spec.on_tick(store, next_epoch_time) - - spec.on_attestation(store, attestation) - - -def get_anchor_root(spec, state): - anchor_block_header = state.latest_block_header.copy() - if anchor_block_header.state_root == spec.Bytes32(): - anchor_block_header.state_root = spec.hash_tree_root(state) - return spec.hash_tree_root(anchor_block_header) - - @with_all_phases @spec_state_test def test_genesis(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index b2d33d0aa..a5334c5c7 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -18,18 +18,25 @@ def run_on_attestation(spec, state, store, attestation, valid=True): if spec.fork == PHASE0: sample_index = indexed_attestation.attesting_indices[0] + latest_message = spec.LatestMessage( + epoch=attestation.data.target.epoch, + root=attestation.data.beacon_block_root, + ) else: attesting_indices = [ index for i, index in enumerate(indexed_attestation.committee) if attestation.aggregation_bits[i] ] sample_index = attesting_indices[0] - assert ( - store.latest_messages[sample_index] == - spec.LatestMessage( + latest_message = spec.LatestMessage( epoch=attestation.data.target.epoch, root=attestation.data.beacon_block_root, + shard=spec.get_shard(state, attestation), + shard_root=attestation.data.shard_head_root, ) + + assert ( + store.latest_messages[sample_index] == latest_message ) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py new file mode 100644 index 000000000..24eeaedbe --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -0,0 +1,132 @@ +from eth2spec.utils.ssz.ssz_impl import hash_tree_root + +from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_except, never_bls +from eth2spec.test.helpers.attestations import get_valid_on_time_attestation +from eth2spec.test.helpers.shard_block import ( + build_shard_block, + get_shard_transitions, + get_committee_index_of_shard, +) +from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root +from eth2spec.test.helpers.state import state_transition_and_sign_block +from eth2spec.test.helpers.block import build_empty_block + + +def run_on_shard_block(spec, store, shard_store, signed_block, valid=True): + if not valid: + try: + spec.on_shard_block(store, shard_store, signed_block) + except AssertionError: + return + else: + assert False + + spec.on_shard_block(store, shard_store, signed_block) + assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message + + +def apply_shard_block(spec, store, shard_store, beacon_parent_state, shard_blocks_buffer): + shard = shard_store.shard + body = b'\x56' * 4 + shard_head_root = spec.get_shard_head(store, shard_store) + shard_parent_state = shard_store.block_states[shard_head_root] + assert shard_parent_state.slot != beacon_parent_state.slot + shard_block = build_shard_block( + spec, beacon_parent_state, shard, + shard_parent_state=shard_parent_state, slot=beacon_parent_state.slot, body=body, signed=True + ) + shard_blocks_buffer.append(shard_block) + run_on_shard_block(spec, store, shard_store, shard_block) + assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() + + +def check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer): + pending_shard_blocks = [ + spec.SignedShardBlock(message=b) + for b in spec.get_pending_shard_blocks(store, shard_store) + ] + assert pending_shard_blocks == shard_blocks_buffer + + +def is_in_offset_sets(spec, beacon_head_state, shard): + offset_slots = spec.compute_offset_slots( + beacon_head_state.shard_states[shard].slot, beacon_head_state.slot + 1 + ) + return beacon_head_state.slot in offset_slots + + +def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer): + store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + + shard = shard_store.shard + committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) + has_shard_committee = committee_index is not None # has committee of `shard` at this slot + + beacon_block = build_empty_block(spec, state, slot=state.slot + 1) + + # If next slot has committee of `shard`, add `shard_transtion` to the proposing beacon block + if has_shard_committee and len(shard_blocks_buffer) > 0: + # Sanity check `get_pending_shard_blocks` function + check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer) + # Use temporary next state to get ShardTransition of shard block + shard_transitions = get_shard_transitions( + spec, + state, + shard_blocks={shard: shard_blocks_buffer}, + ) + shard_transition = shard_transitions[shard] + attestation = get_valid_on_time_attestation( + spec, + state, + index=committee_index, + shard_transition=shard_transition, + signed=False, + ) + assert spec.get_shard(state, attestation) == shard + beacon_block.body.attestations = [attestation] + beacon_block.body.shard_transitions = shard_transitions + + # Clear buffer + shard_blocks_buffer.clear() + + signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) # transition! + add_block_to_store(spec, store, signed_beacon_block) + assert spec.get_head(store) == beacon_block.hash_tree_root() + + # On shard block at transitioned `state.slot` + if is_in_offset_sets(spec, state, shard): + # The created shard block would be appended to `shard_blocks_buffer` + apply_shard_block(spec, store, shard_store, state, shard_blocks_buffer) + + return has_shard_committee + + +@with_all_phases_except([PHASE0]) +@spec_state_test +@never_bls # Set to never_bls for testing `check_pending_shard_blocks` +def test_basic(spec, state): + spec.PHASE_1_GENESIS_SLOT = 0 # NOTE: mock genesis slot here + state = spec.upgrade_to_phase1(state) + shard = spec.Shard(1) + + # Initialization + store = spec.get_forkchoice_store(state) + anchor_root = get_anchor_root(spec, state) + assert spec.get_head(store) == anchor_root + + shard_store = spec.get_forkchoice_shard_store(state, shard) + shard_head_root = spec.get_shard_head(store, shard_store) + assert shard_head_root == state.shard_states[shard].latest_block_root + assert shard_store.block_states[shard_head_root].slot == 1 + assert shard_store.block_states[shard_head_root] == state.shard_states[shard] + + # For mainnet config, it's possible that only one committee of `shard` per epoch. + # we set this counter to test more rounds. + shard_committee_counter = 2 + shard_blocks_buffer = [] + while shard_committee_counter > 0: + has_shard_committee = apply_shard_and_beacon( + spec, state, store, shard_store, shard_blocks_buffer + ) + if has_shard_committee: + shard_committee_counter -= 1 diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index c533182ef..1e0560405 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -1,8 +1,9 @@ from typing import List from eth2spec.test.context import expect_assertion_error, PHASE0, PHASE1 -from eth2spec.test.helpers.state import state_transition_and_sign_block, next_epoch, next_slot, transition_to +from eth2spec.test.helpers.state import state_transition_and_sign_block, next_epoch, next_slot from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.helpers.shard_transitions import get_shard_transition_of_committee from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.ssz.ssz_typing import Bitlist @@ -81,12 +82,10 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() else: - # No shard transition + # No shard transition -> no shard block shard = spec.get_shard(state, spec.Attestation(data=attestation_data)) if on_time: - temp_state = state.copy() - next_slot(spec, temp_state) - shard_transition = spec.get_shard_transition(temp_state, shard, []) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=[]) lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() @@ -180,7 +179,7 @@ def get_valid_attestation(spec, fill_aggregate_attestation(spec, state, attestation, signed=signed, filter_participant_set=filter_participant_set) if spec.fork == PHASE1 and on_time: - attestation = convert_to_valid_on_time_attestation(spec, state, attestation, signed) + attestation = convert_to_valid_on_time_attestation(spec, state, attestation, signed=signed) return attestation @@ -317,7 +316,17 @@ def next_epoch_with_attestations(spec, committees_per_slot = spec.get_committee_count_at_slot(state, slot_to_attest) if slot_to_attest >= spec.compute_start_slot_at_epoch(spec.get_current_epoch(post_state)): for index in range(committees_per_slot): - cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest, index=index, signed=True) + if spec.fork == PHASE1: + shard = spec.compute_shard_from_committee_index(post_state, index, slot_to_attest) + shard_transition = get_shard_transition_of_committee(spec, post_state, index) + block.body.shard_transitions[shard] = shard_transition + else: + shard_transition = None + + cur_attestation = get_valid_attestation( + spec, post_state, slot_to_attest, + shard_transition=shard_transition, index=index, signed=True, on_time=True + ) block.body.attestations.append(cur_attestation) if fill_prev_epoch: @@ -328,9 +337,6 @@ def next_epoch_with_attestations(spec, spec, post_state, slot_to_attest, index=index, signed=True, on_time=False) block.body.attestations.append(prev_attestation) - if spec.fork == PHASE1: - fill_block_shard_transitions_by_attestations(spec, post_state, block) - signed_block = state_transition_and_sign_block(spec, post_state, block) signed_blocks.append(signed_block) @@ -396,14 +402,3 @@ def cached_prepare_state_with_attestations(spec, state): # Put the LRU cache result into the state view, as if we transitioned the original view state.set_backing(_prep_state_cache_dict[key]) - - -def fill_block_shard_transitions_by_attestations(spec, state, block): - block.body.shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS - for attestation in block.body.attestations: - shard = spec.get_shard(state, attestation) - if attestation.data.slot == state.slot: - temp_state = state.copy() - transition_to(spec, temp_state, slot=block.slot) - shard_transition = spec.get_shard_transition(temp_state, shard, []) - block.body.shard_transitions[shard] = shard_transition diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py new file mode 100644 index 000000000..04e36ea84 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py @@ -0,0 +1,27 @@ +def get_anchor_root(spec, state): + anchor_block_header = state.latest_block_header.copy() + if anchor_block_header.state_root == spec.Bytes32(): + anchor_block_header.state_root = spec.hash_tree_root(state) + return spec.hash_tree_root(anchor_block_header) + + +def add_block_to_store(spec, store, signed_block): + pre_state = store.block_states[signed_block.message.parent_root] + block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT + + if store.time < block_time: + spec.on_tick(store, block_time) + + spec.on_block(store, signed_block) + + +def add_attestation_to_store(spec, store, attestation): + parent_block = store.blocks[attestation.data.beacon_block_root] + pre_state = store.block_states[spec.hash_tree_root(parent_block)] + block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT + next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT + + if store.time < next_epoch_time: + spec.on_tick(store, next_epoch_time) + + spec.on_attestation(store, attestation) diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 805b955f7..f8b4a155f 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -1,6 +1,4 @@ -from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import get_state_and_beacon_parent_root_at_slot -from eth2spec.test.helpers.state import transition_to from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.bls import only_with_bls @@ -23,19 +21,21 @@ def build_shard_block(spec, shard, slot=None, body=None, + shard_parent_state=None, signed=False): - shard_state = beacon_state.shard_states[shard] + if shard_parent_state is None: + shard_parent_state = beacon_state.shard_states[shard] + if slot is None: - slot = shard_state.slot + 1 + slot = shard_parent_state.slot + 1 if body is None: body = b'\x56' * 128 - proposer_index = spec.get_shard_proposer_index(beacon_state, slot, shard) beacon_state, beacon_parent_root = get_state_and_beacon_parent_root_at_slot(spec, beacon_state, slot) - + proposer_index = spec.get_shard_proposer_index(beacon_state, slot, shard) block = spec.ShardBlock( - shard_parent_root=shard_state.latest_block_root, + shard_parent_root=shard_parent_state.latest_block_root, beacon_parent_root=beacon_parent_root, slot=slot, shard=shard, @@ -52,15 +52,17 @@ def build_shard_block(spec, return signed_block -def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): - temp_state = state.copy() - transition_to(spec, temp_state, on_time_slot) +def get_shard_transitions(spec, parent_beacon_state, shard_blocks): shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS + on_time_slot = parent_beacon_state.slot + 1 for shard, blocks in shard_blocks.items(): - offset_slots = spec.get_offset_slots(temp_state, shard) + offset_slots = spec.compute_offset_slots( + spec.get_latest_slot_for_shard(parent_beacon_state, shard), + on_time_slot, + ) len_offset_slots = len(offset_slots) - assert len_offset_slots == on_time_slot - state.shard_states[shard].slot - 1 - shard_transition = spec.get_shard_transition(temp_state, shard, blocks) + shard_transition = spec.get_shard_transition(parent_beacon_state, shard, blocks) + if len(blocks) > 0: shard_block_root = blocks[-1].message.hash_tree_root() assert shard_transition.shard_states[len_offset_slots - 1].latest_block_root == shard_block_root @@ -70,17 +72,11 @@ def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): return shard_transitions -def build_attestation_with_shard_transition(spec, state, index, on_time_slot, shard_transition=None): - temp_state = state.copy() - transition_to(spec, temp_state, on_time_slot - 1) - attestation = get_valid_on_time_attestation( - spec, - temp_state, - index=index, - shard_transition=shard_transition, - signed=True, - ) - assert attestation.data.slot == temp_state.slot - if shard_transition is not None: - assert attestation.data.shard_transition_root == shard_transition.hash_tree_root() - return attestation +def get_committee_index_of_shard(spec, state, slot, shard): # Optional[CommitteeIndex] + active_shard_count = spec.get_active_shard_count(state) + committee_count = spec.get_committee_count_at_slot(state, slot) + start_shard = spec.get_start_shard(state, slot) + for committee_index in range(committee_count): + if (start_shard + committee_index) % active_shard_count == shard: + return committee_index + return None diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py index 4ac0ddcfb..d10d1ee7b 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py @@ -26,3 +26,12 @@ def run_shard_transitions_processing(spec, state, shard_transitions, attestation # yield post-state yield 'post', state + + +def get_shard_transition_of_committee(spec, state, committee_index, shard_blocks=None): + if shard_blocks is None: + shard_blocks = [] + + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks) + return shard_transition diff --git a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py index f0cfc462e..1e057e5a9 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py @@ -16,8 +16,9 @@ from eth2spec.test.helpers.attester_slashings import ( get_indexed_attestation_participants, ) from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing, check_proposer_slashing_effect -from eth2spec.test.helpers.attestations import get_valid_attestation, fill_block_shard_transitions_by_attestations +from eth2spec.test.helpers.attestations import get_valid_attestation from eth2spec.test.helpers.deposits import prepare_state_and_deposit +from eth2spec.test.helpers.shard_transitions import get_shard_transition_of_committee from eth2spec.test.context import ( spec_state_test, with_all_phases, expect_assertion_error, always_bls, with_phases, @@ -687,14 +688,23 @@ def test_attestation(spec, state): yield 'pre', state - attestation = get_valid_attestation(spec, state, signed=True, on_time=True) + attestation_block = build_empty_block(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + index = 0 + if spec.fork == PHASE1: + shard = spec.compute_shard_from_committee_index(state, index, state.slot) + shard_transition = get_shard_transition_of_committee(spec, state, index) + attestation_block.body.shard_transitions[shard] = shard_transition + else: + shard_transition = None + + attestation = get_valid_attestation( + spec, state, shard_transition=shard_transition, index=index, signed=True, on_time=True + ) # Add to state via block transition pre_current_attestations_len = len(state.current_epoch_attestations) - attestation_block = build_empty_block(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) attestation_block.body.attestations.append(attestation) - if spec.fork == PHASE1: - fill_block_shard_transitions_by_attestations(spec, state, attestation_block) signed_attestation_block = state_transition_and_sign_block(spec, state, attestation_block) assert len(state.current_epoch_attestations) == pre_current_attestations_len + 1 diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py index b4721da5e..e97cc90a8 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py @@ -2,72 +2,64 @@ from eth2spec.test.context import ( PHASE0, with_all_phases_except, spec_state_test, - always_bls, ) +from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.shard_transitions import run_shard_transitions_processing from eth2spec.test.helpers.shard_block import ( - build_attestation_with_shard_transition, build_shard_block, - build_shard_transitions_till_slot, + get_shard_transitions, ) -from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot +from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot, next_slot def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): state = transition_to_valid_shard_slot(spec, state) - # At the beginning, let `x = state.slot`, `state.shard_states[shard].slot == x - 1` - slot_x = state.slot committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) - assert state.shard_states[shard].slot == slot_x - 1 + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) + assert state.shard_states[shard].slot == state.slot - 1 + transition_to(spec, state, state.slot + target_len_offset_slot) + assert state.shard_states[shard].slot == state.slot - target_len_offset_slot - 1 # Create SignedShardBlock body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE - shard_block = build_shard_block(spec, state, shard, body=body, signed=True) + shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) shard_blocks = [shard_block] - # Create a shard_transitions that would be included at beacon block `state.slot + target_len_offset_slot` - shard_transitions = build_shard_transitions_till_slot( + shard_transitions = get_shard_transitions( spec, state, shard_blocks={shard: shard_blocks}, - on_time_slot=state.slot + target_len_offset_slot, ) shard_transition = shard_transitions[shard] - # Create an attestation that would be included at beacon block `state.slot + target_len_offset_slot` - attestation = build_attestation_with_shard_transition( + attestation = get_valid_on_time_attestation( spec, state, index=committee_index, - on_time_slot=state.slot + target_len_offset_slot, shard_transition=shard_transition, + signed=False, ) + next_slot(spec, state) pre_gasprice = state.shard_states[shard].gasprice - - transition_to(spec, state, state.slot + target_len_offset_slot) pre_shard_state = state.shard_states[shard] - yield from run_shard_transitions_processing(spec, state, shard_transitions, [attestation], valid=valid) if valid: - # After state transition, - assert state.slot == slot_x + target_len_offset_slot shard_state = state.shard_states[shard] assert shard_state != pre_shard_state assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1] - + assert shard_state.latest_block_root == shard_block.message.hash_tree_root() if target_len_offset_slot == 1: assert shard_state.gasprice > pre_gasprice @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_basic_crosslinks(spec, state): + # NOTE: this test is only for full crosslink (minimal config), not for mainnet yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=1, valid=True) @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_multiple_offset_slots(spec, state): - yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=3, valid=True) + # NOTE: this test is only for full crosslink (minimal config), not for mainnet + yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=2, valid=True) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py index 60af35d45..33b0beac7 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py @@ -4,37 +4,40 @@ from eth2spec.test.context import ( PHASE0, with_all_phases_except, spec_state_test, - always_bls, ) +from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import build_empty_block from eth2spec.test.helpers.shard_block import ( - build_attestation_with_shard_transition, build_shard_block, - build_shard_transitions_till_slot, + get_shard_transitions, ) -from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot +from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot, transition_to -def run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index, valid=True): - shard_transitions = build_shard_transitions_till_slot( - spec, state, shard_blocks, on_time_slot=state.slot + target_len_offset_slot - ) +def run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard, valid=True): + transition_to(spec, state, state.slot + target_len_offset_slot) + + body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE + shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) + shard_blocks: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]} + + shard_transitions = get_shard_transitions(spec, state, shard_blocks) attestations = [ - build_attestation_with_shard_transition( + get_valid_on_time_attestation( spec, state, - on_time_slot=state.slot + target_len_offset_slot, index=committee_index, shard_transition=shard_transitions[shard], + signed=True, ) for shard in shard_blocks.keys() ] - # Propose beacon block at slot `x + 1` - beacon_block = build_empty_block(spec, state, slot=state.slot + target_len_offset_slot) + beacon_block = build_empty_block(spec, state, slot=state.slot + 1) beacon_block.body.attestations = attestations beacon_block.body.shard_transitions = shard_transitions + pre_gasprice = state.shard_states[shard].gasprice pre_shard_states = state.shard_states.copy() yield 'pre', state.copy() yield 'block', beacon_block @@ -52,56 +55,37 @@ def run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_off assert post_shard_state == shard_transitions[shard].shard_states[ len(shard_transitions[shard].shard_states) - 1 ] - assert beacon_block.slot == shard_transitions[shard].shard_states[0].slot + target_len_offset_slot assert post_shard_state.slot == state.slot - 1 if len(shard_blocks[shard]) == 0: # `latest_block_root` is the same assert post_shard_state.latest_block_root == pre_shard_states[shard].latest_block_root + if target_len_offset_slot == 1 and len(shard_blocks) > 0: + assert post_shard_state.gasprice > pre_gasprice @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_process_beacon_block_with_normal_shard_transition(spec, state): + # NOTE: this test is only for full crosslink (minimal config), not for mainnet state = transition_to_valid_shard_slot(spec, state) target_len_offset_slot = 1 committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 - pre_gasprice = state.shard_states[shard].gasprice - - # Create SignedShardBlock at slot `shard_state.slot + 1` - body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE - shard_block = build_shard_block(spec, state, shard, body=body, signed=True) - shard_blocks: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]} - - yield from run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index) - - shard_state = state.shard_states[shard] - - if target_len_offset_slot == 1 and len(shard_blocks) > 0: - assert shard_state.gasprice > pre_gasprice + yield from run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard) @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_process_beacon_block_with_empty_proposal_transition(spec, state): + # NOTE: this test is only for full crosslink (minimal config), not for mainnet state = transition_to_valid_shard_slot(spec, state) target_len_offset_slot = 1 committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 - # No new shard block - shard_blocks = {} - - pre_gasprice = state.shard_states[shard].gasprice - - yield from run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index) - - if target_len_offset_slot == 1 and len(shard_blocks) > 0: - assert state.shard_states[shard].gasprice > pre_gasprice + yield from run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/unittests/__init__.py b/tests/core/pyspec/eth2spec/test/phase_1/unittests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py b/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py new file mode 100644 index 000000000..27afd4a4e --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py @@ -0,0 +1,71 @@ +from eth2spec.test.context import ( + PHASE0, + with_all_phases_except, + spec_state_test, +) +from eth2spec.test.helpers.state import next_epoch + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_get_committee_count_delta(spec, state): + assert spec.get_committee_count_delta(state, 0, 0) == 0 + assert spec.get_committee_count_at_slot(state, 0) != 0 + assert spec.get_committee_count_delta(state, 0, 1) == spec.get_committee_count_at_slot(state, 0) + assert spec.get_committee_count_delta(state, 1, 2) == spec.get_committee_count_at_slot(state, 1) + assert spec.get_committee_count_delta(state, 0, 2) == ( + spec.get_committee_count_at_slot(state, 0) + spec.get_committee_count_at_slot(state, 1) + ) + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_get_start_shard_current_epoch_start(spec, state): + assert state.current_epoch_start_shard == 0 + next_epoch(spec, state) + active_shard_count = spec.get_active_shard_count(state) + assert state.current_epoch_start_shard == ( + spec.get_committee_count_delta(state, 0, spec.SLOTS_PER_EPOCH) % active_shard_count + ) + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + + slot = current_epoch_start_slot + start_shard = spec.get_start_shard(state, slot) + assert start_shard == state.current_epoch_start_shard + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_get_start_shard_next_slot(spec, state): + next_epoch(spec, state) + active_shard_count = spec.get_active_shard_count(state) + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + + slot = current_epoch_start_slot + 1 + start_shard = spec.get_start_shard(state, slot) + + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + expected_start_shard = ( + state.current_epoch_start_shard + + spec.get_committee_count_delta(state, start_slot=current_epoch_start_slot, stop_slot=slot) + ) % active_shard_count + assert start_shard == expected_start_shard + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_get_start_shard_previous_slot(spec, state): + next_epoch(spec, state) + active_shard_count = spec.get_active_shard_count(state) + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + + slot = current_epoch_start_slot - 1 + start_shard = spec.get_start_shard(state, slot) + + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + expected_start_shard = ( + state.current_epoch_start_shard + + spec.MAX_COMMITTEES_PER_SLOT * spec.SLOTS_PER_EPOCH * active_shard_count + - spec.get_committee_count_delta(state, start_slot=slot, stop_slot=current_epoch_start_slot) + ) % active_shard_count + assert start_shard == expected_start_shard