From d789f3d32d67ce1cd811dfcc403e5fb76bc256d9 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 1 Apr 2020 12:02:56 -0600 Subject: [PATCH 001/165] getting phase 1 val guide in place --- specs/phase1/beacon-chain.md | 34 +++-- specs/phase1/validator.md | 261 +++++++++++++++++++++++++++++++++++ 2 files changed, 285 insertions(+), 10 deletions(-) create mode 100644 specs/phase1/validator.md diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 9c0cd0a6c..135e2d3d3 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -30,6 +30,7 @@ - [`ShardTransition`](#shardtransition) - [`CompactCommittee`](#compactcommittee) - [`AttestationCustodyBitWrapper`](#attestationcustodybitwrapper) + - [`LightClientVote`](#lightclientvote) - [Helper functions](#helper-functions) - [Misc](#misc-1) - [`get_previous_slot`](#get_previous_slot) @@ -211,7 +212,7 @@ class BeaconBlockBody(Container): # Shards shard_transitions: Vector[ShardTransition, MAX_SHARDS] # Light clients - light_client_signature_bitfield: Bitvector[LIGHT_CLIENT_COMMITTEE_SIZE] + light_client_bits: Bitvector[LIGHT_CLIENT_COMMITTEE_SIZE] light_client_signature: BLSSignature ``` @@ -355,6 +356,16 @@ class AttestationCustodyBitWrapper(Container): bit: boolean ``` +### `LightClientVote` + +```python +class LightClientVote(Container): + slot: Slot + block_root: Root + aggregation_bits: Bitvector[LIGHT_CLIENT_COMMITTEE_SIZE] + signature: BLSSignature +``` + ## Helper functions ### Misc @@ -577,7 +588,7 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_randao(state, block.body) process_eth1_data(state, block.body) verify_shard_transition_false_positives(state, block.body) - process_light_client_signatures(state, block.body) + process_light_client_aggregate(state, block.body) process_operations(state, block.body) ``` @@ -862,21 +873,24 @@ def verify_shard_transition_false_positives(state: BeaconState, block_body: Beac #### Light client processing ```python -def process_light_client_signatures(state: BeaconState, block_body: BeaconBlockBody) -> None: +def process_light_client_aggregate(state: BeaconState, block_body: BeaconBlockBody) -> None: committee = get_light_client_committee(state, get_current_epoch(state)) + previous_slot = get_previous_slot(state.slot) + previous_block_root = get_block_root_at_slot(state, previous_slot) + total_reward = Gwei(0) signer_pubkeys = [] for bit_index, participant_index in enumerate(committee): - if block_body.light_client_signature_bitfield[bit_index]: + if block_body.light_client_bits[bit_index]: signer_pubkeys.append(state.validators[participant_index].pubkey) - increase_balance(state, participant_index, get_base_reward(state, participant_index)) - total_reward += get_base_reward(state, participant_index) + if not state.validators[participant_index].slashed: + increase_balance(state, participant_index, get_base_reward(state, participant_index)) + total_reward += get_base_reward(state, participant_index) increase_balance(state, get_beacon_proposer_index(state), Gwei(total_reward // PROPOSER_REWARD_QUOTIENT)) - - slot = get_previous_slot(state.slot) - signing_root = compute_signing_root(get_block_root_at_slot(state, slot), - get_domain(state, DOMAIN_LIGHT_CLIENT, compute_epoch_at_slot(slot))) + + signing_root = compute_signing_root(previous_block_root, + get_domain(state, DOMAIN_LIGHT_CLIENT, compute_epoch_at_slot(previous_slot))) assert bls.FastAggregateVerify(signer_pubkeys, signing_root, signature=block_body.light_client_signature) ``` diff --git a/specs/phase1/validator.md b/specs/phase1/validator.md new file mode 100644 index 000000000..600f05c17 --- /dev/null +++ b/specs/phase1/validator.md @@ -0,0 +1,261 @@ +# Ethereum 2.0 Phase 0 -- Honest Validator + +**Notice**: This document is a work-in-progress for researchers and implementers. This is an accompanying document to [Ethereum 2.0 Phase 1](./), which describes the expected actions of a "validator" participating in the Ethereum 2.0 Phase 1 protocol. + +## Table of contents + + + + + + +- [Introduction](#introduction) +- [Prerequisites](#prerequisites) +- [Constants](#constants) +- [Becoming a validator](#becoming-a-validator) +- [Beacon chain validator assignments](#beacon-chain-validator-assignments) +- [Beacon chain responsibilities](#beacon-chain-responsibilities) + - [Block proposal](#block-proposal) + - [Preparing for a `BeaconBlock`](#preparing-for-a-beaconblock) + - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody) + - [Custody slashings](#custody-slashings) + - [Custody key reveals](#custody-key-reveals) + - [Early derived secret reveals](#early-derived-secret-reveals) + - [Shard transitions](#shard-transitions) + - [Light client fields](#light-client-fields) + - [Packaging into a `SignedBeaconBlock`](#packaging-into-a-signedbeaconblock) + - [Attesting](#attesting) + - [`FullAttestationData`](#fullattestationdata) + - [`FullAttestation`](#fullattestation) + - [Timing](#timing) + - [Attestation data](#attestation-data) + - [Head shard root](#head-shard-root) + - [Shard transition](#shard-transition) + - [Construct attestation](#construct-attestation) + - [Custody bits blocks](#custody-bits-blocks) + - [Signature](#signature) + + + + +## Introduction + +This document represents the expected behavior of an "honest validator" with respect to Phase 1 of the Ethereum 2.0 protocol. This document does not distinguish between a "node" (i.e. the functionality of following and reading the beacon chain) and a "validator client" (i.e. the functionality of actively participating in consensus). The separation of concerns between these (potentially) two pieces of software is left as a design decision that is out of scope. + +A validator is an entity that participates in the consensus of the Ethereum 2.0 protocol. This is an optional role for users in which they can post ETH as collateral and verify and attest to the validity of blocks to seek financial returns in exchange for building and securing the protocol. This is similar to proof-of-work networks in which miners provide collateral in the form of hardware/hash-power to seek returns in exchange for building and securing the protocol. + +## Prerequisites + +This document is an extension of the [Phase 0 -- Validator](../phase0/validator.md). All behaviors and definitions defined in the Phase 0 doc carry over unless explicitly noted or overridden. + +All terminology, constants, functions, and protocol mechanics defined in the [Phase 1 -- The Beacon Chain](./beacon-chain.md) and [Phase 1 -- Custody Game](./custody-game.md) docs are requisite for this document and used throughout. Please see the Phase 1 docs before continuing and use as a reference throughout. + +## Constants + +See constants from [Phase 0 validator guide](../phase0/validator.md#constants). + +## Becoming a validator + +Becoming a validator in Phase 1 is unchanged from Phase 0. See the [Phase 0 validator guide](../phase0/validator.md#becoming-a-validator) for details. + +## Beacon chain validator assignments + +Beacon chain validator assignments to beacon committees and beacon block proposal are unchanged from Phase 0. See the [Phase 0 validator guide](../phase0/validator.md#validator-assignments) for details. + +## Beacon chain responsibilities + +A validator has two primary responsibilities to the beacon chain: [proposing blocks](#block-proposal) and [creating attestations](#attestations-1). Proposals happen infrequently, whereas attestations should be created once per epoch. + +These responsibilities are largely unchanged from Phase 0, but utilize the updated `SignedBeaconBlock`, `BeaconBlock`, `BeaconBlockBody`, `Attestation`, and `AttestationData` definitions found in Phase 1. Below notes only the additional and modified behavior with respect to Phase 0. + +### Block proposal + +#### Preparing for a `BeaconBlock` + +`slot`, `proposer_index`, `parent_root` fields are unchanged. + +#### Constructing the `BeaconBlockBody` + +`randao_reveal`, `eth1_data`, and `graffiti` are unchanged. + +`proposer_slashings`, `deposits`, and `voluntary_exits` are unchanged. + +`attester_slashings` and `attestations` operate exactly as in Phase 0, but with new definitations of `AttesterSlashing` and `Attestation`, along with modified validation conditions found in `process_attester_slashing` and `process_attestation`. + +##### Custody slashings + +Up to `MAX_CUSTODY_SLASHINGS`, [`CustodySlashing`](./custody-game.md#custodyslashing) objects can be included in the `block`. The custody slashings must satisfy the verification conditions found in [custody slashings processing](./custody-game.md#custody-slashings). The validator receives a small "whistleblower" reward for each custody slashing included (THIS IS NOT CURRENTLY THE CASE BUT PROBABLY SHOULD BE). + +##### Custody key reveals + +Up to `MAX_CUSTODY_KEY_REVEALS`, [`CustodyKeyReveal`](./custody-game.md#custodykeyreveal) objects can be included in the `block`. The custody key reveals must satisfy the verification conditions found in [custody key reveal processing](./custody-game.md#custody-key-reveals). The validator receives a small reward for each custody key reveal included. + +##### Early derived secret reveals + +Up to `MAX_EARLY_DERIVED_SECRET_REVEALS`, [`EarlyDerivedSecretReveal`](./custody-game.md#earlyderivedsecretreveal) objects can be included in the `block`. The early derived secret reveals must satisfy the verification conditions found in [early derived secret reveal processing](./custody-game.md#custody-key-reveals). The validator receives a small "whistleblower" reward for each early derived secrete reveal included. + +##### Shard transitions + +Exactly `MAX_SHARDS` [`ShardTransition`](./beacon-chain#shardtransition) objects are included in the block. Default each to an empty `ShardTransition()`. Then for each committee assigned to the slot with an associated `committee_index` and `shard`, set `shard_transitions[shard] = full_transitions[winning_root]` if the committee had enough weight to form a crosslink this slot. + +Specifically: +* Call `shards, winning_roots = get_successful_shard_transitions(state, block.slot, attestations)` +* Let `full_transitions` be a dictionary mapping from the `shard_transition_root`s found in `attestations` to the corresponding full `ShardTransition` +* Then for each `shard` and `winning_root` in `zip(shards, winning_roots)` set `shard_transitions[shard] = full_transitions[winning_root]` + +```python +def get_successful_shard_transitions(state: BeaconState, + slot: Slot, + attestations: Attestation) -> Tuple[Sequence[Shard], Sequence[Root]]: + shards = [] + winning_roots = [] + committee_count = get_committee_count_at_slot(state, slot) + for committee_index in map(CommitteeIndex, range(committee_count)): + shard = compute_shard_from_committee_index(state, committee_index, slot) + # All attestations in the block for this committee/shard and current slot + shard_attestations = [ + attestation for attestation in attestations + if attestation.data.index == committee_index and attestation.data.slot == slot + ] + committee = get_beacon_committee(state, state.slot, committee_index) + + # Loop over all shard transition roots, looking for a winning root + shard_transition_roots = set([a.data.shard_transition_root for a in attestations]) + for shard_transition_root in sorted(shard_transition_roots): + transition_attestations = [a for a in attestations if a.data.shard_transition_root == shard_transition_root] + transition_participants: Set[ValidatorIndex] = set() + for attestation in transition_attestations: + participants = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) + transition_participants = transition_participants.union(participants) + + enough_online_stake = ( + get_total_balance(state, online_indices.intersection(transition_participants)) * 3 >= + get_total_balance(state, online_indices.intersection(committee)) * 2 + ) + if enough_online_stake: + shards.append(shard) + transitions.append(shard_transition_root) + break + + return shards, winning_roots +``` + +##### Light client fields + +First retrieve `best_aggregate` from `get_best_light_client_aggregate` where `aggregates` is a list of valid aggregated `LightClientVote`s for the previous slot. + +Then: +* Set `light_client_bits = best_aggregate.aggregation_bits` +* Set `light_client_signature = best_aggregate.signature` + +```python +def select_best_light_client_aggregate(block: BeaconBlock, + aggregates: Sequence[LightClientVote]) -> LightClientVote: + viable_aggregates = [ + aggregate in aggregates + if aggregate.slot == get_previous_slot(block.slot) and aggregate.beacon_block_root == block.parent_root + ] + + return max( + viable_aggregates, + key=lambda a: len([_ for i in a.aggregation_bits if i == 1]), + default=LightClientVote(), + ) +``` + +#### Packaging into a `SignedBeaconBlock` + +Packaging into a `SignedBeaconBlock` is unchanged from Phase 0. + +### Attesting + +A validator is expected to create, sign, and broadcast an attestation during each epoch. + +Assignments and the core of this duty are unchanged from Phase 0. There are a few additional fields related to the assigned shard chain and custody bit. + +The `Attestation` and `AttestationData` defined in the [Phase 1 Beacon Chain spec]() utilizes `shard_transition_root: Root` rather than a full `ShardTransition`. For the purposes of the validator and p2p layer, a modified `FullAttestationData` and containing `FullAttestation` are used to send the accompanying `ShardTransition` in its entirety. Note that due to the properties of SSZ `hash_tree_root`, the root and signatures of `AttestationData` and `FullAttestationData` are equivalent. + +#### `FullAttestationData` + +```python +class FullAttestationData(Container): + slot: Slot + index: CommitteeIndex + # LMD GHOST vote + beacon_block_root: Root + # FFG vote + source: Checkpoint + target: Checkpoint + # Current-slot shard block root + head_shard_root: Root + # Full shard transition + shard_transition: ShardTransition +``` + +#### `FullAttestation` + +```python +class FullAttestation(Container): + aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] + data: FullAttestationData + custody_bits_blocks: List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_SHARD_BLOCKS_PER_ATTESTATION] + signature: BLSSignature +``` + +#### Timing + +Note the timing of when to create/broadcast is altered from Phase 1. + +A validator should create and broadcast the `attestation` to the associated attestation subnet when either (a) the validator has received a valid `BeaconBlock` from the expected beacon block proposer and a valid `ShardBlock` for the expected shard block porposer for the assigned `slot` or (b) one-half of the `slot` has transpired (`SECONDS_PER_SLOT / 2` seconds after the start of `slot`) -- whichever comes _first_. + +#### Attestation data + +`attestation_data` is constructed in the same manner as Phase 0 but uses `FullAttestationData` with the addition of two fields -- `head_shard_root` and `shard_transition`. + +- Let `head_block` be the result of running the fork choice during the assigned slot. +- Let `head_state` be the state of `head_block` processed through any empty slots up to the assigned slot using `process_slots(state, slot)`. +- Let `head_shard_block` be the result of running the fork choice on the assigned shard chain during the assigned slot. + +##### Head shard root + +Set `attestation_data.head_shard_root = hash_tree_root(head_shard_block)`. + +##### Shard transition + +Set `shard_transition` to the value returned by `get_shard_transition()`. + +```python +def get_shard_transition(state: BeaconState, shard: Shard, shard_blocks: Sequence[ShardBlock]) + latest_shard_slot = get_latest_slot_for_shard(state, shard) + offset_slots = [Slot(latest_shard_slot + x) for x in SHARD_BLOCK_OFFSETS if latest_shard_slot + x <= state.slot] + return ShardTransition() +``` + +#### Construct attestation + +Next, the validator creates `attestation`, a `FullAttestation` as defined above. + +`attestation.data` and `attestation.aggregation_bits` are unchanged from Phase 0. + +##### Custody bits blocks + +- Let `attestation.custody_bits_blocks` be a the value returned by `get_custody_bits_blocks()` + +```python +def get_custody_bits_blocks() -> List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_SHARD_BLOCKS_PER_ATTESTATION]: + pass +``` + +##### Signature + +Set `attestation.signature = attestation_signature` where `attestation_signature` is obtained from: + +```python +def get_attestation_signature(state: BeaconState, + attestation_data: AttestationData, + custody_bits_blocks, + privkey: int) -> List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_SHARD_BLOCKS_PER_ATTESTATION]: + pass +``` + + From 6067c511c5043aa642d175d84b46d10d3e26f55b Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 2 Apr 2020 16:48:02 -0600 Subject: [PATCH 002/165] add light client to phase 1 validator --- specs/phase1/beacon-chain.md | 25 ++--- specs/phase1/validator.md | 202 +++++++++++++++++++++++++++++++++-- 2 files changed, 206 insertions(+), 21 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 135e2d3d3..bca952a92 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -12,6 +12,7 @@ - [Custom types](#custom-types) - [Configuration](#configuration) - [Misc](#misc) + - [Domain types](#domain-types) - [Updated containers](#updated-containers) - [Extended `AttestationData`](#extended-attestationdata) - [Extended `Attestation`](#extended-attestation) @@ -30,7 +31,6 @@ - [`ShardTransition`](#shardtransition) - [`CompactCommittee`](#compactcommittee) - [`AttestationCustodyBitWrapper`](#attestationcustodybitwrapper) - - [`LightClientVote`](#lightclientvote) - [Helper functions](#helper-functions) - [Misc](#misc-1) - [`get_previous_slot`](#get_previous_slot) @@ -105,9 +105,16 @@ Configuration is not namespaced. Instead it is strictly an extension; | `MAX_GASPRICE` | `Gwei(2**14)` (= 16,384) | Gwei | | | `MIN_GASPRICE` | `Gwei(2**3)` (= 8) | Gwei | | | `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | | -| `DOMAIN_SHARD_PROPOSAL` | `DomainType('0x80000000')` | | -| `DOMAIN_SHARD_COMMITTEE` | `DomainType('0x81000000')` | | -| `DOMAIN_LIGHT_CLIENT` | `DomainType('0x82000000')` | | + +### Domain types + +| Name | Value | +| - | - | +| `DOMAIN_SHARD_PROPOSAL` | `DomainType('0x80000000')` | +| `DOMAIN_SHARD_COMMITTEE` | `DomainType('0x81000000')` | +| `DOMAIN_LIGHT_CLIENT` | `DomainType('0x82000000')` | +| `DOMAIN_LIGHT_SELECTION_PROOF` | `DomainType('0x83000000')` | +| `DOMAIN_LIGHT_AGGREGATE_AND_PROOF` | `DomainType('0x84000000')` | ## Updated containers @@ -356,16 +363,6 @@ class AttestationCustodyBitWrapper(Container): bit: boolean ``` -### `LightClientVote` - -```python -class LightClientVote(Container): - slot: Slot - block_root: Root - aggregation_bits: Bitvector[LIGHT_CLIENT_COMMITTEE_SIZE] - signature: BLSSignature -``` - ## Helper functions ### Misc diff --git a/specs/phase1/validator.md b/specs/phase1/validator.md index 600f05c17..acd98ee5d 100644 --- a/specs/phase1/validator.md +++ b/specs/phase1/validator.md @@ -12,6 +12,7 @@ - [Introduction](#introduction) - [Prerequisites](#prerequisites) - [Constants](#constants) + - [Misc](#misc) - [Becoming a validator](#becoming-a-validator) - [Beacon chain validator assignments](#beacon-chain-validator-assignments) - [Beacon chain responsibilities](#beacon-chain-responsibilities) @@ -34,6 +35,20 @@ - [Construct attestation](#construct-attestation) - [Custody bits blocks](#custody-bits-blocks) - [Signature](#signature) + - [Light client committee](#light-client-committee) + - [Preparation](#preparation) + - [Light clent vote](#light-clent-vote) + - [Light client vote data](#light-client-vote-data) + - [`LightClientVoteData`](#lightclientvotedata) + - [Construct vote](#construct-vote) + - [`LightClientVote`](#lightclientvote) + - [Broadcast](#broadcast) + - [Light client vote aggregation](#light-client-vote-aggregation) + - [Aggregation selection](#aggregation-selection) + - [Construct aggregate](#construct-aggregate) + - [Broadcast aggregate](#broadcast-aggregate) + - [`LightAggregateAndProof`](#lightaggregateandproof) + - [`SignedLightAggregateAndProof`](#signedlightaggregateandproof) @@ -54,6 +69,12 @@ All terminology, constants, functions, and protocol mechanics defined in the [Ph See constants from [Phase 0 validator guide](../phase0/validator.md#constants). +### Misc + +| Name | Value | Unit | Duration | +| - | - | :-: | :-: | +| `TARGET_LIGHT_CLIENT_AGGREGATORS_PER_SLOT` | `2**2` (= 8) | validators | | + ## Becoming a validator Becoming a validator in Phase 1 is unchanged from Phase 0. See the [Phase 0 validator guide](../phase0/validator.md#becoming-a-validator) for details. @@ -68,6 +89,8 @@ A validator has two primary responsibilities to the beacon chain: [proposing blo These responsibilities are largely unchanged from Phase 0, but utilize the updated `SignedBeaconBlock`, `BeaconBlock`, `BeaconBlockBody`, `Attestation`, and `AttestationData` definitions found in Phase 1. Below notes only the additional and modified behavior with respect to Phase 0. +Phase 1 adds light client committees and associated responsibilities, discussed [below](#light-client-committee). + ### Block proposal #### Preparing for a `BeaconBlock` @@ -120,9 +143,12 @@ def get_successful_shard_transitions(state: BeaconState, committee = get_beacon_committee(state, state.slot, committee_index) # Loop over all shard transition roots, looking for a winning root - shard_transition_roots = set([a.data.shard_transition_root for a in attestations]) + shard_transition_roots = set([a.data.shard_transition_root for a in shard_attestations]) for shard_transition_root in sorted(shard_transition_roots): - transition_attestations = [a for a in attestations if a.data.shard_transition_root == shard_transition_root] + transition_attestations = [ + a for a in shard_attestations + if a.data.shard_transition_root == shard_transition_root + ] transition_participants: Set[ValidatorIndex] = set() for attestation in transition_attestations: participants = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) @@ -149,10 +175,10 @@ Then: * Set `light_client_signature = best_aggregate.signature` ```python -def select_best_light_client_aggregate(block: BeaconBlock, - aggregates: Sequence[LightClientVote]) -> LightClientVote: +def get_best_light_client_aggregate(block: BeaconBlock, + aggregates: Sequence[LightClientVote]) -> LightClientVote: viable_aggregates = [ - aggregate in aggregates + aggregate for aggregate in aggregates if aggregate.slot == get_previous_slot(block.slot) and aggregate.beacon_block_root == block.parent_root ] @@ -225,7 +251,7 @@ Set `attestation_data.head_shard_root = hash_tree_root(head_shard_block)`. Set `shard_transition` to the value returned by `get_shard_transition()`. ```python -def get_shard_transition(state: BeaconState, shard: Shard, shard_blocks: Sequence[ShardBlock]) +def get_shard_transition(state: BeaconState, shard: Shard, shard_blocks: Sequence[ShardBlock]) -> ShardTransition: latest_shard_slot = get_latest_slot_for_shard(state, shard) offset_slots = [Slot(latest_shard_slot + x) for x in SHARD_BLOCK_OFFSETS if latest_shard_slot + x <= state.slot] return ShardTransition() @@ -254,8 +280,170 @@ Set `attestation.signature = attestation_signature` where `attestation_signature def get_attestation_signature(state: BeaconState, attestation_data: AttestationData, custody_bits_blocks, - privkey: int) -> List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_SHARD_BLOCKS_PER_ATTESTATION]: + privkey: int + ) -> List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_SHARD_BLOCKS_PER_ATTESTATION]: pass ``` +### Light client committee + +In addition to the core beacon chain responsibilities, Phase 1 adds an additional role -- the Light Client Committee -- to aid in light client functionality. + +Validators serve on the light client committee for `LIGHT_CLIENT_COMMITTEE_PERIOD` epochs and the assignment to be on a committee is known `LIGHT_CLIENT_COMMITTEE_PERIOD` epochs in advance. + +#### Preparation + +When `get_current_epoch(state) % LIGHT_CLIENT_COMMITTEE_PERIOD == LIGHT_CLIENT_COMMITTEE_PERIOD - LIGHT_CLIENT_PREPARATION_EPOCHS` each validator must check if they are in the next period light client committee by calling `is_in_next_light_client_committee()`. + +If the validator is in the next light client committee, they must join the `light_client_votes` pubsub topic to begin duties at the start of the next period. + +```python +def is_in_next_light_client_committee(state: BeaconState, index: ValidatorIndex) -> boolean: + period_start_epoch = get_current_epoch(state) + LIGHT_CLIENT_COMMITTEE_PERIOD % get_current_epoch(state) + next_committee = get_light_client_committee(state, period_start_epoch) + return index in next_committee +``` + +#### Light clent vote + +During a period of epochs that the validator is a part of the light client committee (`validator_index in get_light_client_committee(state, epoch)`), the validator creates and broadcasts a `LightClientVote` at each slot. + +A validator should create and broadcast the `light_client_vote` to the `light_client_votes` pubsub topic when either (a) the validator has received a valid block from the expected block proposer for the current `slot` or (b) two-thirds of the `slot` have transpired (`SECONDS_PER_SLOT / 3` seconds after the start of `slot`) -- whichever comes _first_. + +- Let `light_client_committee = get_light_client_committee(state, compute_epoch_at_slot(slot))` + +##### Light client vote data + +First the validator constructs `light_client_vote_data`, a [`LightClientVoteData`](#lightclientvotedata) object. + +* Let `head_block` be the result of running the fork choice during the assigned slot. +* Set `light_client_vote.slot = slot`. +* Set `light_client_vote.beacon_block_root = hash_tree_root(head_block)`. + +###### `LightClientVoteData` + +```python +class LightClientVoteData(Container): + slot: Slot + beacon_block_root: Root +``` + +##### Construct vote + +Then the validator constructs `light_client_vote`, a [`LightClientVote`](#lightclientvote) object. + +* Set `light_client_vote.data = light_client_vote_data`. +* Set `light_client_vote.aggregation_bits` to be a `Bitvector[LIGHT_CLIENT_COMMITTEE_SIZE]`, where the bit of the index of the validator in the `light_client_committee` is set to `0b1` and all other bits are are set to `0b0`. +* Set `light_client_vote.signature = vote_signature` where `vote_signature` is obtained from: + +```python +def get_light_client_vote_signature(state: BeaconState, + light_client_vote_data: LightClientVoteData, + privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_LIGHT_CLIENT, compute_epoch_at_slot(light_client_vote_data.slot)) + signing_root = compute_signing_root(light_client_vote_data, domain) + return bls.Sign(privkey, signing_root) +``` + +###### `LightClientVote` + +```python +class LightClientVote(Container): + data: LightClientVoteData + aggregation_bits: Bitvector[LIGHT_CLIENT_COMMITTEE_SIZE] + signature: BLSSignature +``` + +##### Broadcast + +Finally, the validator broadcasts `light_client_vote` to the `light_client_votes` pubsub topic. + +#### Light client vote aggregation + +Some validators in the light client committee are selected to locally aggregate light client votes with a similar `light_client_vote_data` to their constructed `light_client_vote` for the assigned `slot`. + +#### Aggregation selection + +A validator is selected to aggregate based upon the return value of `is_light_client_aggregator()`. + +```python +def get_light_client_slot_signature(state: BeaconState, slot: Slot, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_LIGHT_SELECTION_PROOF, compute_epoch_at_slot(slot)) + signing_root = compute_signing_root(slot, domain) + return bls.Sign(privkey, signing_root) +``` + +```python +def is_light_client_aggregator(state: BeaconState, slot: Slot, slot_signature: BLSSignature) -> bool: + committee = get_light_client_committee(state, compute_epoch_at_slot(slot)) + modulo = max(1, len(committee) // TARGET_LIGHT_CLIENT_AGGREGATORS_PER_SLOT) + return bytes_to_int(hash(slot_signature)[0:8]) % modulo == 0 +``` + +#### Construct aggregate + +If the validator is selected to aggregate (`is_light_client_aggregator()`), they construct an aggregate light client vote via the following. + +Collect `light_client_votes` seen via gossip during the `slot` that have an equivalent `light_client_vote_data` to that constructed by the validator, and create a `aggregate_light_client_vote: LightClientVote` with the following fields. + +* Set `aggregate_light_client_vote.data = light_client_vote_data` where `light_client_vote_data` is the `LightClientVoteData` object that is the same for each individual light client vote being aggregated. +* Set `aggregate_light_client_vote.aggregation_bits` to be a `Bitvector[LIGHT_CLIENT_COMMITTEE_SIZE]`, where each bit set from each individual light client vote is set to `0b1`. +* Set `aggregate_light_client_vote.signature = aggregate_light_client_signature` where `aggregate_light_client_signature` is obtained from `get_aggregate_light_client_signature`. + +```python +def get_aggregate_light_client_signature(light_client_votes: Sequence[LightClientVote]) -> BLSSignature: + signatures = [light_client_vote.signature for light_client_vote in light_client_votes] + return bls.Aggregate(signatures) +``` + +#### Broadcast aggregate + +If the validator is selected to aggregate (`is_light_client_aggregator`), then they broadcast their best aggregate light client vote as a `SignedLightAggregateAndProof` to the global aggregate light client vote channel (`aggregate_light_client_votes`) two-thirds of the way through the `slot`-that is, `SECONDS_PER_SLOT * 2 / 3` seconds after the start of `slot`. + +Selection proofs are provided in `LightAggregateAndProof` to prove to the gossip channel that the validator has been selected as an aggregator. + +`LightAggregateAndProof` messages are signed by the aggregator and broadcast inside of `SignedLightAggregateAndProof` objects to prevent a class of DoS attacks and message forgeries. + +First, `light_aggregate_and_proof = get_light_aggregate_and_proof(state, validator_index, aggregate_light_client_vote, privkey)` is constructed. + +```python +def get_light_aggregate_and_proof(state: BeaconState, + aggregator_index: ValidatorIndex, + aggregate: Attestation, + privkey: int) -> LightAggregateAndProof: + return LightAggregateAndProof( + aggregator_index=aggregator_index, + aggregate=aggregate, + selection_proof=get_light_client_slot_signature(state, aggregate.data.slot, privkey), + ) +``` + +Then `signed_light_aggregate_and_proof = SignedLightAggregateAndProof(message=light_aggregate_and_proof, signature=signature)` is constructed and broadast. Where `signature` is obtained from: + +```python +def get_light_aggregate_and_proof_signature(state: BeaconState, + aggregate_and_proof: LightAggregateAndProof, + privkey: int) -> BLSSignature: + aggregate = aggregate_and_proof.aggregate + domain = get_domain(state, DOMAIN_CLIENT_AGGREGATE_AND_PROOF, compute_epoch_at_slot(aggregate.data.slot)) + signing_root = compute_signing_root(aggregate_and_proof, domain) + return bls.Sign(privkey, signing_root) +``` + +##### `LightAggregateAndProof` + +```python +class LightAggregateAndProof(Container): + aggregator_index: ValidatorIndex + aggregate: Attestation + selection_proof: BLSSignature +``` + +##### `SignedLightAggregateAndProof` + +```python +class SignedLightAggregateAndProof(Container): + message: LightAggregateAndProof + signature: BLSSignature +``` From d61b2991a0955b8934e9e4a780ed880a1b11b3af Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 2 Apr 2020 16:58:39 -0600 Subject: [PATCH 003/165] fix lint --- setup.py | 4 +++- specs/phase1/validator.md | 28 ++++++++++++++++------------ 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/setup.py b/setup.py index 911eb65b0..28179bfce 100644 --- a/setup.py +++ b/setup.py @@ -108,7 +108,7 @@ SSZObject = TypeVar('SSZObject', bound=View) PHASE1_IMPORTS = '''from eth2spec.phase0 import spec as phase0 from eth2spec.config.config_util import apply_constants_config from typing import ( - Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable + Any, Dict, Set, Sequence, NewType, Tuple, Optional, TypeVar, Callable ) from dataclasses import ( @@ -373,11 +373,13 @@ class PySpecCommand(Command): self.md_doc_paths = """ specs/phase0/beacon-chain.md specs/phase0/fork-choice.md + specs/phase0/validator.md specs/phase1/custody-game.md specs/phase1/beacon-chain.md specs/phase1/fraud-proofs.md specs/phase1/fork-choice.md specs/phase1/phase1-fork.md + specs/phase1/validator.md """ else: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) diff --git a/specs/phase1/validator.md b/specs/phase1/validator.md index acd98ee5d..fd8787adf 100644 --- a/specs/phase1/validator.md +++ b/specs/phase1/validator.md @@ -132,6 +132,7 @@ def get_successful_shard_transitions(state: BeaconState, attestations: Attestation) -> Tuple[Sequence[Shard], Sequence[Root]]: shards = [] winning_roots = [] + online_indices = get_online_validator_indices(state) committee_count = get_committee_count_at_slot(state, slot) for committee_index in map(CommitteeIndex, range(committee_count)): shard = compute_shard_from_committee_index(state, committee_index, slot) @@ -160,7 +161,7 @@ def get_successful_shard_transitions(state: BeaconState, ) if enough_online_stake: shards.append(shard) - transitions.append(shard_transition_root) + winning_roots.append(shard_transition_root) break return shards, winning_roots @@ -184,7 +185,7 @@ def get_best_light_client_aggregate(block: BeaconBlock, return max( viable_aggregates, - key=lambda a: len([_ for i in a.aggregation_bits if i == 1]), + key=lambda a: len([i for i in a.aggregation_bits if i == 1]), default=LightClientVote(), ) ``` @@ -251,9 +252,13 @@ Set `attestation_data.head_shard_root = hash_tree_root(head_shard_block)`. Set `shard_transition` to the value returned by `get_shard_transition()`. ```python -def get_shard_transition(state: BeaconState, shard: Shard, shard_blocks: Sequence[ShardBlock]) -> ShardTransition: +def get_shard_transition(state: BeaconState, + shard: Shard, + shard_blocks: Sequence[ShardBlockWrapper]) -> ShardTransition: + """ latest_shard_slot = get_latest_slot_for_shard(state, shard) offset_slots = [Slot(latest_shard_slot + x) for x in SHARD_BLOCK_OFFSETS if latest_shard_slot + x <= state.slot] + """ return ShardTransition() ``` @@ -279,9 +284,8 @@ Set `attestation.signature = attestation_signature` where `attestation_signature ```python def get_attestation_signature(state: BeaconState, attestation_data: AttestationData, - custody_bits_blocks, - privkey: int - ) -> List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_SHARD_BLOCKS_PER_ATTESTATION]: + cb_blocks: List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_SHARD_BLOCKS_PER_ATTESTATION], + privkey: int) -> BLSSignature: pass ``` @@ -408,9 +412,9 @@ First, `light_aggregate_and_proof = get_light_aggregate_and_proof(state, validat ```python def get_light_aggregate_and_proof(state: BeaconState, - aggregator_index: ValidatorIndex, - aggregate: Attestation, - privkey: int) -> LightAggregateAndProof: + aggregator_index: ValidatorIndex, + aggregate: Attestation, + privkey: int) -> LightAggregateAndProof: return LightAggregateAndProof( aggregator_index=aggregator_index, aggregate=aggregate, @@ -422,10 +426,10 @@ Then `signed_light_aggregate_and_proof = SignedLightAggregateAndProof(message=li ```python def get_light_aggregate_and_proof_signature(state: BeaconState, - aggregate_and_proof: LightAggregateAndProof, - privkey: int) -> BLSSignature: + aggregate_and_proof: LightAggregateAndProof, + privkey: int) -> BLSSignature: aggregate = aggregate_and_proof.aggregate - domain = get_domain(state, DOMAIN_CLIENT_AGGREGATE_AND_PROOF, compute_epoch_at_slot(aggregate.data.slot)) + domain = get_domain(state, DOMAIN_LIGHT_AGGREGATE_AND_PROOF, compute_epoch_at_slot(aggregate.data.slot)) signing_root = compute_signing_root(aggregate_and_proof, domain) return bls.Sign(privkey, signing_root) ``` From f135eff021483a832942373e7af0d82ae99aa58d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 3 Apr 2020 08:33:32 -0600 Subject: [PATCH 004/165] add lookahed for shard subnets for beacon committee in validator guide --- specs/phase1/validator.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/specs/phase1/validator.md b/specs/phase1/validator.md index fd8787adf..a26987a34 100644 --- a/specs/phase1/validator.md +++ b/specs/phase1/validator.md @@ -15,6 +15,7 @@ - [Misc](#misc) - [Becoming a validator](#becoming-a-validator) - [Beacon chain validator assignments](#beacon-chain-validator-assignments) + - [Lookahead](#lookahead) - [Beacon chain responsibilities](#beacon-chain-responsibilities) - [Block proposal](#block-proposal) - [Preparing for a `BeaconBlock`](#preparing-for-a-beaconblock) @@ -83,6 +84,14 @@ Becoming a validator in Phase 1 is unchanged from Phase 0. See the [Phase 0 vali Beacon chain validator assignments to beacon committees and beacon block proposal are unchanged from Phase 0. See the [Phase 0 validator guide](../phase0/validator.md#validator-assignments) for details. +### Lookahead + +Lookahead for beacon committee assignments operates in the same manner as Phase 0, but committee members must join a shard block pubsub topic in addition to the committee attestation topic.o + +Specifically _after_ finding stable peers of attestation subnets (see Phase 0) a validator should: +* Let `shard = compute_shard_from_committee_index(committe_index)` +* Subscribe to the pubsub topic `shard_{shard}_shard_block` (attestation subnet peers should have this topic available). + ## Beacon chain responsibilities A validator has two primary responsibilities to the beacon chain: [proposing blocks](#block-proposal) and [creating attestations](#attestations-1). Proposals happen infrequently, whereas attestations should be created once per epoch. From ca6af0c2e9bfba1667ea7b6a67a03144be7aa23b Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Sun, 5 Apr 2020 14:39:00 +0100 Subject: [PATCH 005/165] 256-bit custody atoms for better alignment with rest of the spec and greater efficiency --- specs/phase1/custody-game.md | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index eb243f8fb..b4033ea4d 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -49,8 +49,9 @@ This document details the beacon chain additions and changes in Phase 1 of Ether | Name | Value | Unit | | - | - | - | -| `BLS12_381_Q` | `4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787` | - | -| `BYTES_PER_CUSTODY_ATOM` | `48` | bytes | +| `CUSTODY_PRIME` | `2 ** 256 - 189` | - | +| `CUSTODY_SECRETS` | `3` | - | +| `BYTES_PER_CUSTODY_ATOM` | `32` | bytes | ## Configuration @@ -175,7 +176,7 @@ def legendre_bit(a: int, q: int) -> int: return 0 ``` -### `custody_atoms` +### `get_custody_atoms` Given one set of data, return the custody atoms: each atom will be combined with one legendre bit. @@ -186,16 +187,28 @@ def get_custody_atoms(bytez: bytes) -> Sequence[bytes]: for i in range(0, len(bytez), BYTES_PER_CUSTODY_ATOM)] ``` +### `get_custody_secrets` + +Extract the custody secrets from the signature + +```python +def get_custody_secrets(key: BLSSignature): + full_G2_element = bls.signature_to_G2(key) + signature = full_G2_element[0].coeffs + signature_bytes = sum(x.to_bytes(48, "little") for x in signature) + secrets = [int.from_bytes(x[i:i+BYTES_PER_CUSTODY_ATOM]) for i in range(0, len(signature_bytes), 32)] + return secrets +``` + ### `compute_custody_bit` ```python def compute_custody_bit(key: BLSSignature, data: bytes) -> bit: - full_G2_element = bls.signature_to_G2(key) - s = full_G2_element[0].coeffs + secrets = get_custody_secrets(key) custody_atoms = get_custody_atoms(data) n = len(custody_atoms) - a = sum(s[i % 2]**i * int.from_bytes(atom, "little") for i, atom in enumerate(custody_atoms) + s[n % 2]**n) - return legendre_bit(a, BLS12_381_Q) + uhf = sum(secrets[i % CUSTORY_SECRETS]**i * int.from_bytes(atom, "little") for i, atom in enumerate(custody_atoms)) + secrets[n % CUSTORY_SECRETS]**n + return legendre_bit(uhf + secrets[0], BLS12_381_Q) ``` ### `get_randao_epoch_for_custody_period` From bf34fdf0239541b713afeabe527c557d54c7c1b8 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Sun, 5 Apr 2020 15:10:09 +0100 Subject: [PATCH 006/165] Fix ToC --- specs/phase1/custody-game.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index b4033ea4d..370972502 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -24,7 +24,8 @@ - [`EarlyDerivedSecretReveal`](#earlyderivedsecretreveal) - [Helpers](#helpers) - [`legendre_bit`](#legendre_bit) - - [`custody_atoms`](#custody_atoms) + - [`get_custody_secrets`](#get_custody_secrets) + - [`get_custody_atoms`](#get_custody_atoms) - [`compute_custody_bit`](#compute_custody_bit) - [`get_randao_epoch_for_custody_period`](#get_randao_epoch_for_custody_period) - [`get_custody_period_for_validator`](#get_custody_period_for_validator) From c3c24b4fc4a1592a9dc6d42fb57694aae59ee383 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Sun, 5 Apr 2020 15:35:11 +0100 Subject: [PATCH 007/165] Fix lint --- specs/phase1/custody-game.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 370972502..c1ed24383 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -193,11 +193,12 @@ def get_custody_atoms(bytez: bytes) -> Sequence[bytes]: Extract the custody secrets from the signature ```python -def get_custody_secrets(key: BLSSignature): +def get_custody_secrets(key: BLSSignature) -> Sequence[int]: full_G2_element = bls.signature_to_G2(key) signature = full_G2_element[0].coeffs - signature_bytes = sum(x.to_bytes(48, "little") for x in signature) - secrets = [int.from_bytes(x[i:i+BYTES_PER_CUSTODY_ATOM]) for i in range(0, len(signature_bytes), 32)] + signature_bytes = b"".join(x.to_bytes(48, "little") for x in signature) + secrets = [int.from_bytes(signature_bytes[i:i + BYTES_PER_CUSTODY_ATOM], "little") + for i in range(0, len(signature_bytes), 32)] return secrets ``` @@ -208,8 +209,9 @@ def compute_custody_bit(key: BLSSignature, data: bytes) -> bit: secrets = get_custody_secrets(key) custody_atoms = get_custody_atoms(data) n = len(custody_atoms) - uhf = sum(secrets[i % CUSTORY_SECRETS]**i * int.from_bytes(atom, "little") for i, atom in enumerate(custody_atoms)) + secrets[n % CUSTORY_SECRETS]**n - return legendre_bit(uhf + secrets[0], BLS12_381_Q) + uhf = (sum(secrets[i % CUSTODY_SECRETS]**i * int.from_bytes(atom, "little") % CUSTODY_PRIME + for i, atom in enumerate(custody_atoms)) + secrets[n % CUSTODY_SECRETS]**n) % CUSTODY_PRIME + return legendre_bit(uhf + secrets[0], CUSTODY_PRIME) ``` ### `get_randao_epoch_for_custody_period` From 907c56dabdeae9020decb76437f7d6b772838078 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Sun, 5 Apr 2020 15:47:59 +0100 Subject: [PATCH 008/165] Fix ToC --- specs/phase1/custody-game.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index c1ed24383..6315955f5 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -24,8 +24,8 @@ - [`EarlyDerivedSecretReveal`](#earlyderivedsecretreveal) - [Helpers](#helpers) - [`legendre_bit`](#legendre_bit) - - [`get_custody_secrets`](#get_custody_secrets) - [`get_custody_atoms`](#get_custody_atoms) + - [`get_custody_secrets`](#get_custody_secrets) - [`compute_custody_bit`](#compute_custody_bit) - [`get_randao_epoch_for_custody_period`](#get_randao_epoch_for_custody_period) - [`get_custody_period_for_validator`](#get_custody_period_for_validator) From ab2ee0e2c2898ccb8398d555e64a8a6d34ebbeec Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Fri, 24 Apr 2020 17:06:27 +0100 Subject: [PATCH 009/165] Restoring chunk challenges and testing --- Makefile | 2 +- configs/minimal.yaml | 4 +- setup.py | 2 +- specs/phase1/beacon-chain.md | 95 +++++ specs/phase1/custody-game.md | 169 ++++++--- .../eth2spec/test/helpers/attestations.py | 15 +- .../pyspec/eth2spec/test/helpers/custody.py | 72 ++-- .../test/helpers/phase1/attestations.py | 6 +- .../test_process_chunk_challenge.py | 238 ++++++++++++ .../test_process_custody_slashing.py | 349 ++++++++++++++++++ .../test_process_final_custody_updates.py | 319 ++++++++++++++++ .../test_process_reveal_deadlines.py | 46 +++ 12 files changed, 1216 insertions(+), 101 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py create mode 100644 tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py create mode 100644 tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_final_custody_updates.py create mode 100644 tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_reveal_deadlines.py diff --git a/Makefile b/Makefile index e8f3d21bc..8cd7daf58 100644 --- a/Makefile +++ b/Makefile @@ -71,7 +71,7 @@ pyspec: # installs the packages to run pyspec tests install_test: - python3 -m venv venv; . venv/bin/activate; pip3 install .[test] .[lint] + python3.8 -m venv venv; . venv/bin/activate; pip3 install .[lint]; pip3 install -e .[test] test: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 9daf428b4..256c2b3fa 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -205,9 +205,9 @@ RANDAO_PENALTY_EPOCHS: 2 # [customized] quicker for testing EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 4096 # 2**11 (= 2,048) epochs -EPOCHS_PER_CUSTODY_PERIOD: 2048 +EPOCHS_PER_CUSTODY_PERIOD: 8 # 2**11 (= 2,048) epochs -CUSTODY_PERIOD_TO_RANDAO_PADDING: 2048 +CUSTODY_PERIOD_TO_RANDAO_PADDING: 8 # 2**7 (= 128) epochs MAX_REVEAL_LATENESS_DECREMENT: 128 diff --git a/setup.py b/setup.py index 911eb65b0..91c29a6cc 100644 --- a/setup.py +++ b/setup.py @@ -121,7 +121,7 @@ from lru import LRU from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( View, boolean, Container, List, Vector, uint64, uint8, bit, - ByteList, Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, + ByteList, ByteVector, Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils import bls diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 596b3818f..18c5f347f 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -109,6 +109,99 @@ Configuration is not namespaced. Instead it is strictly an extension; | `DOMAIN_SHARD_PROPOSAL` | `DomainType('0x80000000')` | | | `DOMAIN_SHARD_COMMITTEE` | `DomainType('0x81000000')` | | | `DOMAIN_LIGHT_CLIENT` | `DomainType('0x82000000')` | | +| `MAX_CUSTODY_CHUNK_CHALLENGE_RECORDS` | `2**20` (= 1,048,576) | +| `BYTES_PER_CUSTODY_CHUNK` | `2**12` | bytes | +| `MAX_CUSTODY_RESPONSE_DEPTH` | `ceillog2(MAX_SHARD_BLOCK_SIZE // BYTES_PER_CUSTODY_CHUNK) | - | - | + + + +## New containers + +### `CustodyChunkChallenge` + +```python +class CustodyChunkChallenge(Container): + responder_index: ValidatorIndex + shard_transition: ShardTransition + attestation: Attestation + data_index: uint64 + chunk_index: uint64 +``` + +### `CustodyChunkChallengeRecord` + +```python +class CustodyChunkChallengeRecord(Container): + challenge_index: uint64 + challenger_index: ValidatorIndex + responder_index: ValidatorIndex + inclusion_epoch: Epoch + data_root: Root + depth: uint64 + chunk_index: uint64 +``` + +#### `CustodyChunkResponse` + +```python +class CustodyChunkResponse(Container): + challenge_index: uint64 + chunk_index: uint64 + chunk: ByteVector[BYTES_PER_CUSTODY_CHUNK] + branch: List[Root, MAX_CUSTODY_RESPONSE_DEPTH] +``` + +#### `CustodySlashing` + +```python +class CustodySlashing(Container): + # Attestation.custody_bits_blocks[data_index][committee.index(malefactor_index)] is the target custody bit to check. + # (Attestation.data.shard_transition_root as ShardTransition).shard_data_roots[data_index] is the root of the data. + data_index: uint64 + malefactor_index: ValidatorIndex + malefactor_secret: BLSSignature + whistleblower_index: ValidatorIndex + shard_transition: ShardTransition + attestation: Attestation + data: ByteList[MAX_SHARD_BLOCK_SIZE] +``` + +#### `SignedCustodySlashing` + +```python +class SignedCustodySlashing(Container): + message: CustodySlashing + signature: BLSSignature +``` + + +#### `CustodyKeyReveal` + +```python +class CustodyKeyReveal(Container): + # Index of the validator whose key is being revealed + revealer_index: ValidatorIndex + # Reveal (masked signature) + reveal: BLSSignature +``` + +#### `EarlyDerivedSecretReveal` + +Represents an early (punishable) reveal of one of the derived secrets, where derived secrets are RANDAO reveals and custody reveals (both are part of the same domain). + +```python +class EarlyDerivedSecretReveal(Container): + # Index of the validator whose key is being revealed + revealed_index: ValidatorIndex + # RANDAO epoch of the key that is being revealed + epoch: Epoch + # Reveal (masked signature) + reveal: BLSSignature + # Index of the validator who revealed (whistleblower) + masker_index: ValidatorIndex + # Mask used to hide the actual reveal signature (prevent reveal from being stolen) + mask: Bytes32 +``` ## Updated containers @@ -285,6 +378,8 @@ class BeaconState(Container): # at RANDAO reveal period % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS exposed_derived_secrets: Vector[List[ValidatorIndex, MAX_EARLY_DERIVED_SECRET_REVEALS * SLOTS_PER_EPOCH], EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] + custody_chunk_challenge_records: List[CustodyChunkChallengeRecord, MAX_CUSTODY_CHUNK_CHALLENGE_RECORDS] + custody_chunk_challenge_index: uint64 ``` ## New containers diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 6315955f5..f069c4cd2 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -65,6 +65,9 @@ This document details the beacon chain additions and changes in Phase 1 of Ether | `EPOCHS_PER_CUSTODY_PERIOD` | `2**11` (= 2,048) | epochs | ~9 days | | `CUSTODY_PERIOD_TO_RANDAO_PADDING` | `2**11` (= 2,048) | epochs | ~9 days | | `MAX_REVEAL_LATENESS_DECREMENT` | `2**7` (= 128) | epochs | ~14 hours | +| `CHUNK_RESPONSE_DEADLINE` | `2**14` (= 16,384) | epochs | ~73 days | +| `MAX_CHUNK_CHALLENGE_DELAY` | `2**11` (= 16,384) | epochs | ~9 days | +| `CUSTODY_RESPONSE_DEADLINE` | `2**14` (= 16,384) | epochs | ~73 days | ### Max operations per block @@ -72,6 +75,7 @@ This document details the beacon chain additions and changes in Phase 1 of Ether | - | - | | `MAX_CUSTODY_KEY_REVEALS` | `2**8` (= 256) | | `MAX_EARLY_DERIVED_SECRET_REVEALS` | `1` | +| `MAX_CUSTODY_CHUNK_CHALLENGES` | `2**2` (= 4) | | `MAX_CUSTODY_SLASHINGS` | `1` | ### Reward and penalty quotients @@ -93,61 +97,20 @@ The following types are defined, mapping into `DomainType` (little endian): ### New Beacon Chain operations -#### `CustodySlashing` - -```python -class CustodySlashing(Container): - # Attestation.custody_bits_blocks[data_index][committee.index(malefactor_index)] is the target custody bit to check. - # (Attestation.data.shard_transition_root as ShardTransition).shard_data_roots[data_index] is the root of the data. - data_index: uint64 - malefactor_index: ValidatorIndex - malefactor_secret: BLSSignature - whistleblower_index: ValidatorIndex - shard_transition: ShardTransition - attestation: Attestation - data: ByteList[MAX_SHARD_BLOCK_SIZE] -``` - -#### `SignedCustodySlashing` - -```python -class SignedCustodySlashing(Container): - message: CustodySlashing - signature: BLSSignature -``` - - -#### `CustodyKeyReveal` - -```python -class CustodyKeyReveal(Container): - # Index of the validator whose key is being revealed - revealer_index: ValidatorIndex - # Reveal (masked signature) - reveal: BLSSignature -``` - -#### `EarlyDerivedSecretReveal` - -Represents an early (punishable) reveal of one of the derived secrets, where derived secrets are RANDAO reveals and custody reveals (both are part of the same domain). - -```python -class EarlyDerivedSecretReveal(Container): - # Index of the validator whose key is being revealed - revealed_index: ValidatorIndex - # RANDAO epoch of the key that is being revealed - epoch: Epoch - # Reveal (masked signature) - reveal: BLSSignature - # Index of the validator who revealed (whistleblower) - masker_index: ValidatorIndex - # Mask used to hide the actual reveal signature (prevent reveal from being stolen) - mask: Bytes32 -``` - - ## Helpers +### `replace_empty_or_append` + +```python +def replace_empty_or_append(list: List, new_element: Any) -> int: + for i in range(len(list)): + if list[i] == empty(typeof(new_element)): + list[i] = new_element + return i + list.append(new_element) + return len(list) - 1 +``` + ### `legendre_bit` Returns the Legendre symbol `(a/q)` normalizes as a bit (i.e. `((a/q) + 1) // 2`). In a production implementation, a well-optimized library (e.g. GMP) should be used for this. @@ -248,6 +211,84 @@ def process_custody_game_operations(state: BeaconState, body: BeaconBlockBody) - for_ops(body.custody_slashings, process_custody_slashing) ``` +#### Chunk challenges + +Verify that `len(block.body.custody_chunk_challenges) <= MAX_CUSTODY_CHUNK_CHALLENGES`. + +For each `challenge` in `block.body.custody_chunk_challenges`, run the following function: + +```python +def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge) -> None: + # Verify the attestation + assert is_valid_indexed_attestation(state, get_indexed_attestation(state, challenge.attestation)) + # Verify it is not too late to challenge + assert (challenge.attestation.data.target.epoch + MAX_CHUNK_CHALLENGE_DELAY + >= get_current_epoch(state)) + responder = state.validators[challenge.responder_index] + assert (responder.exit_epoch == FAR_FUTURE_EPOCH + or responder.exit_epoch + MAX_CHUNK_CHALLENGE_DELAY >= get_current_epoch(state)) + # Verify responder is slashable + assert is_slashable_validator(responder, get_current_epoch(state)) + # Verify the responder participated in the attestation + attesters = get_attesting_indices(state, challenge.attestation.data, challenge.attestation.aggregation_bits) + assert challenge.responder_index in attesters + # Verify shard transition is correctly given + assert hash_tree_root(challenge.shard_transition) == challenge.attestation.data.shard_transition_root + data_root = challenge.shard_transition.shard_data_roots[challenge.data_index] + # Verify the challenge is not a duplicate + for record in state.custody_chunk_challenge_records: + assert ( + record.data_root != challenge.attestation.data.crosslink.data_root or + record.chunk_index != challenge.chunk_index + ) + # Verify depth + transition_chunks = (challenge.shard_transition.shard_block_lengths[challenge.data_index] + BYTES_PER_CUSTODY_CHUNK - 1) // BYTES_PER_CUSTODY_CHUNK + depth = ceillog2(transition_chunks) + assert challenge.chunk_index < transition_chunks + # Add new chunk challenge record + new_record = CustodyChunkChallengeRecord( + challenge_index=state.custody_chunk_challenge_index, + challenger_index=get_beacon_proposer_index(state), + responder_index=challenge.responder_index, + inclusion_epoch=get_current_epoch(state), + data_root=challenge.shard_transition.shard_data_roots[challenge.data_index], + depth=depth, + chunk_index=challenge.chunk_index, + ) + replace_empty_or_append(state.custody_chunk_challenge_records, new_record) + + state.custody_chunk_challenge_index += 1 + # Postpone responder withdrawability + responder.withdrawable_epoch = FAR_FUTURE_EPOCH +``` + +#### Custody chunk response + +```python +def process_chunk_challenge_response(state: BeaconState, + response: CustodyChunkResponse) -> None: + + challenge = next((record for record in state.custody_chunk_challenge_records if record.challenge_index == response.challenge_index), None) + assert(challenge is not None) + + # Verify chunk index + assert response.chunk_index == challenge.chunk_index + # Verify the chunk matches the crosslink data root + assert is_valid_merkle_branch( + leaf=hash_tree_root(response.chunk), + branch=response.branch, + depth=challenge.depth, + index=response.chunk_index, + root=challenge.data_root, + ) + # Clear the challenge + records = state.custody_chunk_challenge_records + records[records.index(challenge)] = CustodyChunkChallengeRecord() + # Reward the proposer + proposer_index = get_beacon_proposer_index(state) + increase_balance(state, proposer_index, get_base_reward(state, proposer_index) // MINOR_REWARD_QUOTIENT) +``` + #### Custody key reveals ```python @@ -431,14 +472,22 @@ Run `process_reveal_deadlines(state)` after `process_registry_updates(state)`: def process_reveal_deadlines(state: BeaconState) -> None: epoch = get_current_epoch(state) for index, validator in enumerate(state.validators): - if get_custody_period_for_validator(ValidatorIndex(index), epoch) > validator.next_custody_secret_to_reveal: - # ------------------ WARNING ----------------------- # - # UNSAFE REMOVAL OF SLASHING TO PRIORITIZE PHASE 0 CI # - # Must find generic way to handle key reveals in tests # - # ---------------------------------------------------- # + if get_custody_period_for_validator(ValidatorIndex(index), epoch) > validator.next_custody_secret_to_reveal + (CUSTODY_RESPONSE_DEADLINE // EPOCHS_PER_CUSTODY_PERIOD): + slash_validator(state, ValidatorIndex(index)) +``` - # slash_validator(state, ValidatorIndex(index)) - pass +Run `process_challenge_deadlines(state)` immediately after `process_reveal_deadlines(state)`: + +```python +# begin insert @process_challenge_deadlines + process_challenge_deadlines(state) +# end insert @process_challenge_deadlines +def process_challenge_deadlines(state: BeaconState) -> None: + for custody_chunk_challenge in state.custody_chunk_challenge_records: + if get_current_epoch(state) > custody_chunk_challenge.inclusion_epoch + CUSTODY_RESPONSE_DEADLINE: + slash_validator(state, custody_chunk_challenge.responder_index, custody_chunk_challenge.challenger_index) + records = state.custody_chunk_challenge_records + records[records.index(custody_chunk_challenge)] = CustodyChunkChallengeRecord() ``` ### Final updates diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 8215f5c5b..325fee260 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -43,7 +43,7 @@ def run_attestation_processing(spec, state, attestation, valid=True): yield 'post', state -def build_attestation_data(spec, state, slot, index): +def build_attestation_data(spec, state, slot, index, shard_transition_root=None): assert state.slot >= slot if slot == state.slot: @@ -72,6 +72,7 @@ def build_attestation_data(spec, state, slot, index): beacon_block_root=block_root, source=spec.Checkpoint(epoch=source_epoch, root=source_root), target=spec.Checkpoint(epoch=spec.compute_epoch_at_slot(slot), root=epoch_boundary_root), + shard_transition_root=shard_transition_root, ) @@ -89,7 +90,7 @@ def convert_to_valid_on_time_attestation(spec, state, attestation, signed=False) return attestation -def get_valid_on_time_attestation(spec, state, slot=None, index=None, signed=False): +def get_valid_on_time_attestation(spec, state, slot=None, index=None, signed=False, shard_transition_root=None): ''' Construct on-time attestation for next slot ''' @@ -98,10 +99,10 @@ def get_valid_on_time_attestation(spec, state, slot=None, index=None, signed=Fal if index is None: index = 0 - return get_valid_attestation(spec, state, slot=slot, index=index, signed=signed, on_time=True) + return get_valid_attestation(spec, state, slot=slot, index=index, signed=signed, on_time=True, shard_transition_root=shard_transition_root) -def get_valid_late_attestation(spec, state, slot=None, index=None, signed=False): +def get_valid_late_attestation(spec, state, slot=None, index=None, signed=False, shard_transition_root=None): ''' Construct on-time attestation for next slot ''' @@ -110,16 +111,16 @@ def get_valid_late_attestation(spec, state, slot=None, index=None, signed=False) if index is None: index = 0 - return get_valid_attestation(spec, state, slot=slot, index=index, signed=signed, on_time=False) + return get_valid_attestation(spec, state, slot=slot, index=index, signed=signed, on_time=False, shard_transition_root=shard_transition_root) -def get_valid_attestation(spec, state, slot=None, index=None, empty=False, signed=False, on_time=True): +def get_valid_attestation(spec, state, slot=None, index=None, empty=False, signed=False, on_time=True, shard_transition_root=None): if slot is None: slot = state.slot if index is None: index = 0 - attestation_data = build_attestation_data(spec, state, slot, index) + attestation_data = build_attestation_data(spec, state, slot, index, shard_transition_root=shard_transition_root) beacon_committee = spec.get_beacon_committee( state, diff --git a/tests/core/pyspec/eth2spec/test/helpers/custody.py b/tests/core/pyspec/eth2spec/test/helpers/custody.py index 7c51675cd..45c29be9a 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/custody.py +++ b/tests/core/pyspec/eth2spec/test/helpers/custody.py @@ -1,8 +1,8 @@ from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls -from eth2spec.utils.ssz.ssz_typing import Bitlist, ByteVector, Bitvector +from eth2spec.utils.ssz.ssz_typing import Bitlist, ByteVector, Bitvector, ByteList from eth2spec.utils.ssz.ssz_impl import hash_tree_root -from eth2spec.utils.merkle_minimal import get_merkle_tree, get_merkle_proof +from eth2spec.utils.merkle_minimal import get_merkle_root, get_merkle_tree, get_merkle_proof from remerkleable.core import pack_bits_to_chunks from remerkleable.tree import subtree_fill_to_contents, get_depth @@ -61,7 +61,7 @@ def bitlist_from_int(max_len, num_bits, n): return Bitlist[max_len](*[(n >> i) & 0b1 for i in range(num_bits)]) -def get_valid_bit_challenge(spec, state, attestation, invalid_custody_bit=False): +def get_valid_custody_slashing(spec, state, attestation, invalid_custody_bit=False): beacon_committee = spec.get_beacon_committee( state, attestation.data.slot, @@ -96,21 +96,39 @@ def get_valid_bit_challenge(spec, state, attestation, invalid_custody_bit=False) ) +def get_valid_chunk_challenge(spec, state, attestation, shard_transition): + shard = spec.compute_shard_from_committee_index(state, attestation.data.index, attestation.data.slot) + crosslink_committee = spec.get_beacon_committee( + state, + attestation.data.slot, + attestation.data.index + ) + responder_index = crosslink_committee[0] + data_index = len(shard_transition.shard_block_lengths) - 1 + + chunk_count = (shard_transition.shard_block_lengths[data_index] + spec.BYTES_PER_CUSTODY_CHUNK - 1) // spec.BYTES_PER_CUSTODY_CHUNK + + return spec.CustodyChunkChallenge( + responder_index=responder_index, + attestation=attestation, + chunk_index=chunk_count - 1, + data_index=data_index, + shard_transition=shard_transition, + ) + + def custody_chunkify(spec, x): chunks = [bytes(x[i:i + spec.BYTES_PER_CUSTODY_CHUNK]) for i in range(0, len(x), spec.BYTES_PER_CUSTODY_CHUNK)] chunks[-1] = chunks[-1].ljust(spec.BYTES_PER_CUSTODY_CHUNK, b"\0") return chunks -def get_valid_custody_response(spec, state, bit_challenge, custody_data, challenge_index, invalid_chunk_bit=False): +def get_valid_custody_chunk_response(spec, state, chunk_challenge, block_length, challenge_index, + invalid_chunk_data=False): + custody_data = get_custody_test_vector(block_length) chunks = custody_chunkify(spec, custody_data) - chunk_index = len(chunks) - 1 - chunk_bit = spec.get_custody_chunk_bit(bit_challenge.responder_key, chunks[chunk_index]) - - while chunk_bit == bit_challenge.chunk_bits[chunk_index] ^ invalid_chunk_bit: - chunk_index -= 1 - chunk_bit = spec.get_custody_chunk_bit(bit_challenge.responder_key, chunks[chunk_index]) + chunk_index = chunk_challenge.chunk_index chunks_hash_tree_roots = [hash_tree_root(ByteVector[spec.BYTES_PER_CUSTODY_CHUNK](chunk)) for chunk in chunks] chunks_hash_tree_roots += [ @@ -120,31 +138,29 @@ def get_valid_custody_response(spec, state, bit_challenge, custody_data, challen data_branch = get_merkle_proof(data_tree, chunk_index) - bitlist_chunk_index = chunk_index // BYTES_PER_CHUNK - print(bitlist_chunk_index) - bitlist_chunk_nodes = pack_bits_to_chunks(bit_challenge.chunk_bits) - bitlist_tree = subtree_fill_to_contents(bitlist_chunk_nodes, get_depth(spec.MAX_CUSTODY_CHUNKS)) - print(bitlist_tree) - bitlist_chunk_branch = None # TODO; extract proof from merkle tree - - bitlist_chunk_index = chunk_index // 256 - - chunk_bits_leaf = Bitvector[256](bit_challenge.chunk_bits[bitlist_chunk_index * 256: - (bitlist_chunk_index + 1) * 256]) - - return spec.CustodyResponse( + return spec.CustodyChunkResponse( challenge_index=challenge_index, chunk_index=chunk_index, chunk=ByteVector[spec.BYTES_PER_CUSTODY_CHUNK](chunks[chunk_index]), - data_branch=data_branch, - chunk_bits_branch=bitlist_chunk_branch, - chunk_bits_leaf=chunk_bits_leaf, + branch=data_branch, ) def get_custody_test_vector(bytelength): - ints = bytelength // 4 - return b"".join(i.to_bytes(4, "little") for i in range(ints)) + ints = bytelength // 4 + 1 + return (b"".join(i.to_bytes(4, "little") for i in range(ints)))[:bytelength] + + +def get_shard_transition(spec, start_slot, block_lengths): + b = [hash_tree_root(ByteVector[x](get_custody_test_vector(x))) for x in block_lengths] + shard_transition = spec.ShardTransition( + start_slot=start_slot, + shard_block_lengths=block_lengths, + shard_data_roots=b, + shard_states=[spec.Root() for x in block_lengths], + proposer_signature_aggregate=spec.BLSSignature(), + ) + return shard_transition def get_custody_merkle_root(data): diff --git a/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py index 0e16e1fac..061553ef9 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py @@ -5,21 +5,23 @@ from eth2spec.test.helpers.keys import privkeys import eth2spec.test.helpers.attestations as phase0_attestations -def get_valid_on_time_attestation(spec, state, index=None, signed=False): +def get_valid_on_time_attestation(spec, state, index=None, signed=False, shard_transition_root=None): ''' Construct on-time attestation for next slot ''' if index is None: index = 0 - attestation = phase0_attestations.get_valid_attestation(spec, state, state.slot, index, False) + attestation = phase0_attestations.get_valid_attestation(spec, state, state.slot, index, False, shard_transition_root=shard_transition_root) shard = spec.get_shard(state, attestation) offset_slots = spec.compute_offset_slots(spec.get_latest_slot_for_shard(state, shard), state.slot + 1) + print(offset_slots) for _ in offset_slots: attestation.custody_bits_blocks.append( Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE]([0 for _ in attestation.aggregation_bits]) ) + print(len(attestation.custody_bits_blocks)) if signed: sign_attestation(spec, state, attestation) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py new file mode 100644 index 000000000..09b25a1ad --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py @@ -0,0 +1,238 @@ +from eth2spec.test.helpers.custody import ( + get_valid_chunk_challenge, + get_valid_custody_chunk_response, + get_custody_test_vector, + get_custody_merkle_root, + get_shard_transition, +) +from eth2spec.test.helpers.attestations import ( + get_valid_on_time_attestation, +) +from eth2spec.test.helpers.state import transition_to +from eth2spec.utils.ssz.ssz_impl import hash_tree_root +from eth2spec.test.context import ( + with_all_phases_except, + spec_state_test, + expect_assertion_error, +) +from eth2spec.test.phase_0.block_processing.test_process_attestation import run_attestation_processing + + +def run_chunk_challenge_processing(spec, state, custody_chunk_challenge, valid=True): + """ + Run ``process_chunk_challenge``, yielding: + - pre-state ('pre') + - CustodyBitChallenge ('custody_chunk_challenge') + - post-state ('post'). + If ``valid == False``, run expecting ``AssertionError`` + """ + yield 'pre', state + yield 'custody_chunk_challenge', custody_chunk_challenge + + if not valid: + expect_assertion_error(lambda: spec.custody_chunk_challenge(state, custody_chunk_challenge)) + yield 'post', None + return + + spec.process_chunk_challenge(state, custody_chunk_challenge) + + assert state.custody_chunk_challenge_records[state.custody_chunk_challenge_index - 1].responder_index == \ + custody_chunk_challenge.responder_index + assert state.custody_chunk_challenge_records[state.custody_chunk_challenge_index - 1].chunk_index == \ + custody_chunk_challenge.chunk_index + + yield 'post', state + + +def run_custody_chunk_response_processing(spec, state, custody_response, valid=True): + """ + Run ``process_chunk_challenge_response``, yielding: + - pre-state ('pre') + - CustodyResponse ('custody_response') + - post-state ('post'). + If ``valid == False``, run expecting ``AssertionError`` + """ + yield 'pre', state + yield 'custody_response', custody_response + + if not valid: + expect_assertion_error(lambda: spec.process_custody_response(state, custody_response)) + yield 'post', None + return + + spec.process_chunk_challenge_response(state, custody_response) + + assert state.custody_chunk_challenge_records[custody_response.challenge_index] == spec.CustodyChunkChallengeRecord() + + yield 'post', state + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_challenge_appended(spec, state): + transition_to(spec, state, state.slot + 1) + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + data_index = 0 + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition_root=hash_tree_root(shard_transition)) + + transition_chunks = (shard_transition.shard_block_lengths[data_index] + spec.BYTES_PER_CUSTODY_CHUNK - 1) // spec.BYTES_PER_CUSTODY_CHUNK + test_vector = get_custody_test_vector(shard_transition.shard_block_lengths[data_index]) + shard_root = get_custody_merkle_root(test_vector) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD) + + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) + + yield from run_chunk_challenge_processing(spec, state, challenge) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_multiple_epochs_custody(spec, state): + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 3) + + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + data_index = 0 + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition_root=hash_tree_root(shard_transition)) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) + + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) + + yield from run_chunk_challenge_processing(spec, state, challenge) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_many_epochs_custody(spec, state): + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 100) + + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + data_index = 0 + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition_root=hash_tree_root(shard_transition)) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) + + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) + + yield from run_chunk_challenge_processing(spec, state, challenge) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_off_chain_attestation(spec, state): + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) + + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + data_index = 0 + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition_root=hash_tree_root(shard_transition)) + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) + + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) + + yield from run_chunk_challenge_processing(spec, state, challenge) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_custody_response(spec, state): + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) + + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + data_index = 0 + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition_root=hash_tree_root(shard_transition)) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) + + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) + + _, _, _ = run_chunk_challenge_processing(spec, state, challenge) + + chunk_challenge_index = state.custody_chunk_challenge_index - 1 + + custody_response = get_valid_custody_chunk_response(spec, state, challenge, 2**15 // 3, chunk_challenge_index) + + yield from run_custody_chunk_response_processing(spec, state, custody_response) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_custody_response_multiple_epochs(spec, state): + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 3) + + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + data_index = 0 + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition_root=hash_tree_root(shard_transition)) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) + + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) + + _, _, _ = run_chunk_challenge_processing(spec, state, challenge) + + chunk_challenge_index = state.custody_chunk_challenge_index - 1 + + custody_response = get_valid_custody_chunk_response(spec, state, challenge, 2**15 // 3, chunk_challenge_index) + + yield from run_custody_chunk_response_processing(spec, state, custody_response) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_custody_response_many_epochs(spec, state): + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 100) + + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + data_index = 0 + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition_root=hash_tree_root(shard_transition)) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) + + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) + + _, _, _ = run_chunk_challenge_processing(spec, state, challenge) + + chunk_challenge_index = state.custody_chunk_challenge_index - 1 + + custody_response = get_valid_custody_chunk_response(spec, state, challenge, 2**15 // 3, chunk_challenge_index) + + yield from run_custody_chunk_response_processing(spec, state, custody_response) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py new file mode 100644 index 000000000..358aa5c0d --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py @@ -0,0 +1,349 @@ +from eth2spec.test.helpers.custody import ( + get_valid_bit_challenge, + get_valid_custody_bit_response, + get_custody_test_vector, + get_custody_merkle_root, +) +from eth2spec.test.helpers.attestations import ( + get_valid_attestation, +) +from eth2spec.utils.ssz.ssz_impl import hash_tree_root +from eth2spec.test.helpers.state import next_epoch, get_balance +from eth2spec.test.helpers.block import apply_empty_block +from eth2spec.test.context import ( + with_all_phases_except, + spec_state_test, + expect_assertion_error, +) +from eth2spec.test.phase_0.block_processing.test_process_attestation import run_attestation_processing + + +def run_bit_challenge_processing(spec, state, custody_bit_challenge, valid=True): + """ + Run ``process_bit_challenge``, yielding: + - pre-state ('pre') + - CustodyBitChallenge ('custody_bit_challenge') + - post-state ('post'). + If ``valid == False``, run expecting ``AssertionError`` + """ + yield 'pre', state + yield 'custody_bit_challenge', custody_bit_challenge + + if not valid: + expect_assertion_error(lambda: spec.process_bit_challenge(state, custody_bit_challenge)) + yield 'post', None + return + + spec.process_bit_challenge(state, custody_bit_challenge) + + assert state.custody_bit_challenge_records[state.custody_bit_challenge_index - 1].chunk_bits_merkle_root == \ + hash_tree_root(custody_bit_challenge.chunk_bits) + assert state.custody_bit_challenge_records[state.custody_bit_challenge_index - 1].challenger_index == \ + custody_bit_challenge.challenger_index + assert state.custody_bit_challenge_records[state.custody_bit_challenge_index - 1].responder_index == \ + custody_bit_challenge.responder_index + + yield 'post', state + + +def run_custody_bit_response_processing(spec, state, custody_response, valid=True): + """ + Run ``process_bit_challenge_response``, yielding: + - pre-state ('pre') + - CustodyResponse ('custody_response') + - post-state ('post'). + If ``valid == False``, run expecting ``AssertionError`` + """ + yield 'pre', state + yield 'custody_response', custody_response + + if not valid: + expect_assertion_error(lambda: spec.process_custody_response(state, custody_response)) + yield 'post', None + return + + challenge = state.custody_bit_challenge_records[custody_response.challenge_index] + pre_slashed_balance = get_balance(state, challenge.challenger_index) + + spec.process_custody_response(state, custody_response) + + slashed_validator = state.validators[challenge.challenger_index] + + assert slashed_validator.slashed + assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH + assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH + + assert get_balance(state, challenge.challenger_index) < pre_slashed_balance + yield 'post', state + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_challenge_appended(spec, state): + state.slot = spec.SLOTS_PER_EPOCH + attestation = get_valid_attestation(spec, state, signed=True) + + test_vector = get_custody_test_vector( + spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) + shard_root = get_custody_merkle_root(test_vector) + attestation.data.crosslink.data_root = shard_root + attestation.custody_bits[0] = 0 + + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + _, _, _ = run_attestation_processing(spec, state, attestation) + + state.slot += spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD + + challenge = get_valid_bit_challenge(spec, state, attestation) + + yield from run_bit_challenge_processing(spec, state, challenge) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_multiple_epochs_custody(spec, state): + state.slot = spec.SLOTS_PER_EPOCH * 3 + attestation = get_valid_attestation(spec, state, signed=True) + + test_vector = get_custody_test_vector( + spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) + shard_root = get_custody_merkle_root(test_vector) + attestation.data.crosslink.data_root = shard_root + attestation.custody_bits[0] = 0 + + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + _, _, _ = run_attestation_processing(spec, state, attestation) + + state.slot += spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1) + + challenge = get_valid_bit_challenge(spec, state, attestation) + + yield from run_bit_challenge_processing(spec, state, challenge) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_many_epochs_custody(spec, state): + state.slot = spec.SLOTS_PER_EPOCH * 100 + attestation = get_valid_attestation(spec, state, signed=True) + + test_vector = get_custody_test_vector( + spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) + shard_root = get_custody_merkle_root(test_vector) + attestation.data.crosslink.data_root = shard_root + attestation.custody_bits[0] = 0 + + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + _, _, _ = run_attestation_processing(spec, state, attestation) + + state.slot += spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1) + + challenge = get_valid_bit_challenge(spec, state, attestation) + + yield from run_bit_challenge_processing(spec, state, challenge) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_off_chain_attestation(spec, state): + state.slot = spec.SLOTS_PER_EPOCH + attestation = get_valid_attestation(spec, state, signed=True) + + test_vector = get_custody_test_vector( + spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) + shard_root = get_custody_merkle_root(test_vector) + attestation.data.crosslink.data_root = shard_root + attestation.custody_bits[0] = 0 + + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + state.slot += spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD + + challenge = get_valid_bit_challenge(spec, state, attestation) + + yield from run_bit_challenge_processing(spec, state, challenge) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_invalid_custody_bit_challenge(spec, state): + state.slot = spec.SLOTS_PER_EPOCH + attestation = get_valid_attestation(spec, state, signed=True) + + test_vector = get_custody_test_vector( + spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) + shard_root = get_custody_merkle_root(test_vector) + attestation.data.crosslink.data_root = shard_root + attestation.custody_bits[0] = 0 + + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + _, _, _ = run_attestation_processing(spec, state, attestation) + + state.slot += spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD + + challenge = get_valid_bit_challenge(spec, state, attestation, invalid_custody_bit=True) + + yield from run_bit_challenge_processing(spec, state, challenge, valid=False) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_max_reveal_lateness_1(spec, state): + next_epoch(spec, state) + apply_empty_block(spec, state) + + attestation = get_valid_attestation(spec, state, signed=True) + + test_vector = get_custody_test_vector( + spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) + shard_root = get_custody_merkle_root(test_vector) + attestation.data.crosslink.data_root = shard_root + attestation.custody_bits[0] = 0 + + next_epoch(spec, state) + apply_empty_block(spec, state) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + challenge = get_valid_bit_challenge(spec, state, attestation) + + responder_index = challenge.responder_index + target_epoch = attestation.data.target.epoch + + state.validators[responder_index].max_reveal_lateness = 3 + + latest_reveal_epoch = spec.get_randao_epoch_for_custody_period( + spec.get_custody_period_for_validator(state, responder_index, target_epoch), + responder_index + ) + 2 * spec.EPOCHS_PER_CUSTODY_PERIOD + state.validators[responder_index].max_reveal_lateness + + while spec.get_current_epoch(state) < latest_reveal_epoch - 2: + next_epoch(spec, state) + apply_empty_block(spec, state) + + yield from run_bit_challenge_processing(spec, state, challenge) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_max_reveal_lateness_2(spec, state): + next_epoch(spec, state) + apply_empty_block(spec, state) + + attestation = get_valid_attestation(spec, state, signed=True) + + test_vector = get_custody_test_vector( + spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) + shard_root = get_custody_merkle_root(test_vector) + attestation.data.crosslink.data_root = shard_root + attestation.custody_bits[0] = 0 + + next_epoch(spec, state) + apply_empty_block(spec, state) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + challenge = get_valid_bit_challenge(spec, state, attestation) + + responder_index = challenge.responder_index + + state.validators[responder_index].max_reveal_lateness = 3 + + for i in range(spec.get_randao_epoch_for_custody_period( + spec.get_custody_period_for_validator(state, responder_index), + responder_index + ) + 2 * spec.EPOCHS_PER_CUSTODY_PERIOD + state.validators[responder_index].max_reveal_lateness - 1): + next_epoch(spec, state) + apply_empty_block(spec, state) + + yield from run_bit_challenge_processing(spec, state, challenge, False) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_custody_response(spec, state): + state.slot = spec.SLOTS_PER_EPOCH + attestation = get_valid_attestation(spec, state, signed=True) + + test_vector = get_custody_test_vector( + spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) + shard_root = get_custody_merkle_root(test_vector) + attestation.data.crosslink.data_root = shard_root + attestation.custody_bits[0] = 0 + + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + _, _, _ = run_attestation_processing(spec, state, attestation) + + state.slot += spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD + + challenge = get_valid_bit_challenge(spec, state, attestation) + + _, _, _ = run_bit_challenge_processing(spec, state, challenge) + + bit_challenge_index = state.custody_bit_challenge_index - 1 + + custody_response = get_valid_custody_bit_response(spec, state, challenge, test_vector, bit_challenge_index) + + yield from run_custody_bit_response_processing(spec, state, custody_response) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_custody_response_multiple_epochs(spec, state): + state.slot = spec.SLOTS_PER_EPOCH * 3 + attestation = get_valid_attestation(spec, state, signed=True) + + test_vector = get_custody_test_vector( + spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) + shard_root = get_custody_merkle_root(test_vector) + attestation.data.crosslink.data_root = shard_root + attestation.custody_bits[0] = 0 + + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + _, _, _ = run_attestation_processing(spec, state, attestation) + + state.slot += spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD + + challenge = get_valid_bit_challenge(spec, state, attestation) + + _, _, _ = run_bit_challenge_processing(spec, state, challenge) + + bit_challenge_index = state.custody_bit_challenge_index - 1 + + custody_response = get_valid_custody_bit_response(spec, state, challenge, test_vector, bit_challenge_index) + + yield from run_custody_bit_response_processing(spec, state, custody_response) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_custody_response_many_epochs(spec, state): + state.slot = spec.SLOTS_PER_EPOCH * 100 + attestation = get_valid_attestation(spec, state, signed=True) + + test_vector = get_custody_test_vector( + spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) + shard_root = get_custody_merkle_root(test_vector) + attestation.data.crosslink.data_root = shard_root + attestation.custody_bits[0] = 0 + + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + _, _, _ = run_attestation_processing(spec, state, attestation) + + state.slot += spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD + + challenge = get_valid_bit_challenge(spec, state, attestation) + + _, _, _ = run_bit_challenge_processing(spec, state, challenge) + + bit_challenge_index = state.custody_bit_challenge_index - 1 + + custody_response = get_valid_custody_bit_response(spec, state, challenge, test_vector, bit_challenge_index) + + yield from run_custody_bit_response_processing(spec, state, custody_response) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_final_custody_updates.py b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_final_custody_updates.py new file mode 100644 index 000000000..d200ed2b3 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_final_custody_updates.py @@ -0,0 +1,319 @@ +from eth2spec.test.helpers.custody import ( + get_valid_bit_challenge, + get_valid_chunk_challenge, + get_valid_custody_bit_response, + get_valid_custody_chunk_response, + get_valid_custody_key_reveal, + get_custody_test_vector, + get_custody_merkle_root, +) +from eth2spec.test.helpers.attestations import ( + get_valid_attestation, +) +from eth2spec.test.helpers.state import next_epoch +from eth2spec.test.helpers.block import apply_empty_block +from eth2spec.test.context import ( + with_all_phases_except, + spec_state_test, +) +from eth2spec.test.phase_0.block_processing.test_process_attestation import run_attestation_processing +from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_with + +from eth2spec.test.phase_1.block_processing.test_process_bit_challenge import ( + run_bit_challenge_processing, + run_custody_bit_response_processing, +) +from eth2spec.test.phase_1.block_processing.test_process_chunk_challenge import ( + run_chunk_challenge_processing, + run_custody_chunk_response_processing, +) +from eth2spec.test.phase_1.block_processing.test_process_custody_key_reveal import run_custody_key_reveal_processing + + +def run_process_final_custody_updates(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_final_custody_updates') + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_validator_withdrawal_delay(spec, state): + spec.initiate_validator_exit(state, 0) + assert state.validators[0].withdrawable_epoch < spec.FAR_FUTURE_EPOCH + + yield from run_process_final_custody_updates(spec, state) + + assert state.validators[0].withdrawable_epoch == spec.FAR_FUTURE_EPOCH + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_validator_withdrawal_reenable_after_custody_reveal(spec, state): + spec.initiate_validator_exit(state, 0) + assert state.validators[0].withdrawable_epoch < spec.FAR_FUTURE_EPOCH + + next_epoch(spec, state) + apply_empty_block(spec, state) + + assert state.validators[0].withdrawable_epoch == spec.FAR_FUTURE_EPOCH + + while spec.get_current_epoch(state) < state.validators[0].exit_epoch: + next_epoch(spec, state) + apply_empty_block(spec, state) + + while (state.validators[0].next_custody_secret_to_reveal + <= spec.get_custody_period_for_validator(state, 0, state.validators[0].exit_epoch - 1)): + custody_key_reveal = get_valid_custody_key_reveal(spec, state, validator_index=0) + _, _, _ = run_custody_key_reveal_processing(spec, state, custody_key_reveal) + + yield from run_process_final_custody_updates(spec, state) + + assert state.validators[0].withdrawable_epoch < spec.FAR_FUTURE_EPOCH + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_validator_withdrawal_suspend_after_bit_challenge(spec, state): + state.slot = spec.SLOTS_PER_EPOCH + + attestation = get_valid_attestation(spec, state, signed=True) + + test_vector = get_custody_test_vector( + spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) + shard_root = get_custody_merkle_root(test_vector) + attestation.data.crosslink.data_root = shard_root + attestation.custody_bits[0] = 0 + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + next_epoch(spec, state) + apply_empty_block(spec, state) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + validator_index = spec.get_crosslink_committee( + state, + attestation.data.target.epoch, + attestation.data.crosslink.shard + )[0] + + spec.initiate_validator_exit(state, validator_index) + assert state.validators[validator_index].withdrawable_epoch < spec.FAR_FUTURE_EPOCH + + next_epoch(spec, state) + apply_empty_block(spec, state) + + assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH + + while spec.get_current_epoch(state) < state.validators[validator_index].exit_epoch: + next_epoch(spec, state) + apply_empty_block(spec, state) + + while (state.validators[validator_index].next_custody_secret_to_reveal + <= spec.get_custody_period_for_validator( + state, + validator_index, + state.validators[validator_index].exit_epoch - 1)): + custody_key_reveal = get_valid_custody_key_reveal(spec, state, validator_index=validator_index) + _, _, _ = run_custody_key_reveal_processing(spec, state, custody_key_reveal) + + next_epoch(spec, state) + apply_empty_block(spec, state) + + challenge = get_valid_bit_challenge(spec, state, attestation) + + _, _, _ = run_bit_challenge_processing(spec, state, challenge) + + yield from run_process_final_custody_updates(spec, state) + + assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_validator_withdrawal_suspend_after_chunk_challenge(spec, state): + state.slot = spec.SLOTS_PER_EPOCH + + attestation = get_valid_attestation(spec, state, signed=True) + + test_vector = get_custody_test_vector( + spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) + shard_root = get_custody_merkle_root(test_vector) + attestation.data.crosslink.data_root = shard_root + attestation.custody_bits[0] = 0 + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + next_epoch(spec, state) + apply_empty_block(spec, state) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + validator_index = spec.get_crosslink_committee( + state, + attestation.data.target.epoch, + attestation.data.crosslink.shard + )[0] + + spec.initiate_validator_exit(state, validator_index) + assert state.validators[validator_index].withdrawable_epoch < spec.FAR_FUTURE_EPOCH + + next_epoch(spec, state) + apply_empty_block(spec, state) + + assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH + + while spec.get_current_epoch(state) < state.validators[validator_index].exit_epoch: + next_epoch(spec, state) + apply_empty_block(spec, state) + + while (state.validators[validator_index].next_custody_secret_to_reveal + <= spec.get_custody_period_for_validator( + state, + validator_index, + state.validators[validator_index].exit_epoch - 1)): + custody_key_reveal = get_valid_custody_key_reveal(spec, state, validator_index=validator_index) + _, _, _ = run_custody_key_reveal_processing(spec, state, custody_key_reveal) + + next_epoch(spec, state) + apply_empty_block(spec, state) + + challenge = get_valid_chunk_challenge(spec, state, attestation) + + _, _, _ = run_chunk_challenge_processing(spec, state, challenge) + + yield from run_process_final_custody_updates(spec, state) + + assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_validator_withdrawal_resume_after_bit_challenge_response(spec, state): + state.slot = spec.SLOTS_PER_EPOCH + + attestation = get_valid_attestation(spec, state, signed=True) + + test_vector = get_custody_test_vector( + spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) + shard_root = get_custody_merkle_root(test_vector) + attestation.data.crosslink.data_root = shard_root + attestation.custody_bits[0] = 0 + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + next_epoch(spec, state) + apply_empty_block(spec, state) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + validator_index = spec.get_crosslink_committee( + state, + attestation.data.target.epoch, + attestation.data.crosslink.shard + )[0] + + spec.initiate_validator_exit(state, validator_index) + assert state.validators[validator_index].withdrawable_epoch < spec.FAR_FUTURE_EPOCH + + next_epoch(spec, state) + apply_empty_block(spec, state) + + assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH + + while spec.get_current_epoch(state) < state.validators[validator_index].exit_epoch: + next_epoch(spec, state) + apply_empty_block(spec, state) + + while (state.validators[validator_index].next_custody_secret_to_reveal + <= spec.get_custody_period_for_validator( + state, + validator_index, + state.validators[validator_index].exit_epoch - 1)): + custody_key_reveal = get_valid_custody_key_reveal(spec, state, validator_index=validator_index) + _, _, _ = run_custody_key_reveal_processing(spec, state, custody_key_reveal) + + next_epoch(spec, state) + apply_empty_block(spec, state) + + challenge = get_valid_bit_challenge(spec, state, attestation) + + _, _, _ = run_bit_challenge_processing(spec, state, challenge) + + next_epoch(spec, state) + apply_empty_block(spec, state) + + assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH + + bit_challenge_index = state.custody_bit_challenge_index - 1 + response = get_valid_custody_bit_response( + spec, + state, + challenge, + test_vector, + bit_challenge_index, + invalid_chunk_bit=False) + + _, _, _ = run_custody_bit_response_processing(spec, state, response) + + yield from run_process_final_custody_updates(spec, state) + + assert state.validators[validator_index].withdrawable_epoch < spec.FAR_FUTURE_EPOCH + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_validator_withdrawal_resume_after_chunk_challenge_response(spec, state): + state.slot = spec.SLOTS_PER_EPOCH + + attestation = get_valid_attestation(spec, state, signed=True) + + test_vector = get_custody_test_vector( + spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) + shard_root = get_custody_merkle_root(test_vector) + attestation.data.crosslink.data_root = shard_root + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + next_epoch(spec, state) + apply_empty_block(spec, state) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + validator_index = spec.get_crosslink_committee( + state, + attestation.data.target.epoch, + attestation.data.crosslink.shard + )[0] + + spec.initiate_validator_exit(state, validator_index) + assert state.validators[validator_index].withdrawable_epoch < spec.FAR_FUTURE_EPOCH + + next_epoch(spec, state) + apply_empty_block(spec, state) + + assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH + + while spec.get_current_epoch(state) < state.validators[validator_index].exit_epoch: + next_epoch(spec, state) + apply_empty_block(spec, state) + + while (state.validators[validator_index].next_custody_secret_to_reveal + <= spec.get_custody_period_for_validator( + state, + validator_index, + state.validators[validator_index].exit_epoch - 1)): + custody_key_reveal = get_valid_custody_key_reveal(spec, state, validator_index=validator_index) + _, _, _ = run_custody_key_reveal_processing(spec, state, custody_key_reveal) + + next_epoch(spec, state) + apply_empty_block(spec, state) + + challenge = get_valid_chunk_challenge(spec, state, attestation) + + _, _, _ = run_chunk_challenge_processing(spec, state, challenge) + + next_epoch(spec, state) + apply_empty_block(spec, state) + + assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH + + chunk_challenge_index = state.custody_chunk_challenge_index - 1 + response = get_valid_custody_chunk_response(spec, state, challenge, test_vector, chunk_challenge_index) + + _, _, _ = run_custody_chunk_response_processing(spec, state, response) + + yield from run_process_final_custody_updates(spec, state) + + assert state.validators[validator_index].withdrawable_epoch < spec.FAR_FUTURE_EPOCH diff --git a/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_reveal_deadlines.py b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_reveal_deadlines.py new file mode 100644 index 000000000..688b046c8 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_reveal_deadlines.py @@ -0,0 +1,46 @@ +from eth2spec.test.helpers.custody import ( + get_valid_custody_key_reveal, +) +from eth2spec.test.helpers.state import next_epoch +from eth2spec.test.context import ( + with_all_phases_except, + spec_state_test, +) +from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_with +from eth2spec.test.phase_1.block_processing.test_process_custody_key_reveal import run_custody_key_reveal_processing + + +def run_process_challenge_deadlines(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_challenge_deadlines') + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_validator_slashed_after_reveal_deadline(spec, state): + assert state.validators[0].slashed == 0 + + state.slot += ((spec.CHUNK_RESPONSE_DEADLINE + spec.EPOCHS_PER_CUSTODY_PERIOD) + * spec.SLOTS_PER_EPOCH) + next_epoch(spec, state) + + yield from run_process_challenge_deadlines(spec, state) + + assert state.validators[0].slashed == 1 + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_validator_not_slashed_after_reveal(spec, state): + state.slot += spec.EPOCHS_PER_CUSTODY_PERIOD * spec.SLOTS_PER_EPOCH + custody_key_reveal = get_valid_custody_key_reveal(spec, state) + + _, _, _ = run_custody_key_reveal_processing(spec, state, custody_key_reveal) + + assert state.validators[0].slashed == 0 + + state.slot += spec.CHUNK_RESPONSE_DEADLINE * spec.SLOTS_PER_EPOCH + next_epoch(spec, state) + + yield from run_process_challenge_deadlines(spec, state) + + assert state.validators[0].slashed == 0 From 2449db1bb6457c0b9e96bd7ccd3c3e3873050a1d Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Mon, 27 Apr 2020 16:08:49 +0100 Subject: [PATCH 010/165] Phase 1 block tests are working --- specs/phase1/beacon-chain.md | 9 +- specs/phase1/custody-game.md | 30 +- .../eth2spec/test/helpers/attestations.py | 48 ++- .../pyspec/eth2spec/test/helpers/custody.py | 81 ++-- .../test_process_chunk_challenge.py | 14 +- .../test_process_custody_key_reveal.py | 16 - .../test_process_custody_slashing.py | 365 +++++------------- 7 files changed, 211 insertions(+), 352 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 18c5f347f..fc208cbaa 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -111,7 +111,7 @@ Configuration is not namespaced. Instead it is strictly an extension; | `DOMAIN_LIGHT_CLIENT` | `DomainType('0x82000000')` | | | `MAX_CUSTODY_CHUNK_CHALLENGE_RECORDS` | `2**20` (= 1,048,576) | | `BYTES_PER_CUSTODY_CHUNK` | `2**12` | bytes | -| `MAX_CUSTODY_RESPONSE_DEPTH` | `ceillog2(MAX_SHARD_BLOCK_SIZE // BYTES_PER_CUSTODY_CHUNK) | - | - | +| `CUSTODY_RESPONSE_DEPTH` | `ceillog2(MAX_SHARD_BLOCK_SIZE // BYTES_PER_CUSTODY_CHUNK) | - | - | @@ -137,7 +137,6 @@ class CustodyChunkChallengeRecord(Container): responder_index: ValidatorIndex inclusion_epoch: Epoch data_root: Root - depth: uint64 chunk_index: uint64 ``` @@ -148,7 +147,7 @@ class CustodyChunkResponse(Container): challenge_index: uint64 chunk_index: uint64 chunk: ByteVector[BYTES_PER_CUSTODY_CHUNK] - branch: List[Root, MAX_CUSTODY_RESPONSE_DEPTH] + branch: Vector[Root, CUSTODY_RESPONSE_DEPTH] ``` #### `CustodySlashing` @@ -281,7 +280,8 @@ class Validator(Container): # (of the particular validator) in which the validator is activated # = get_custody_period_for_validator(...) next_custody_secret_to_reveal: uint64 - max_reveal_lateness: Epoch + # TODO: The max_reveal_lateness doesn't really make sense anymore. + # So how do we incentivise early custody key reveals now? ``` ### Extended `BeaconBlockBody` @@ -428,6 +428,7 @@ class ShardTransition(Container): # Shard block lengths shard_block_lengths: List[uint64, MAX_SHARD_BLOCKS_PER_ATTESTATION] # Shard data roots + # The root is of ByteVector[MAX_] shard_data_roots: List[Bytes32, MAX_SHARD_BLOCKS_PER_ATTESTATION] # Intermediate shard states shard_states: List[ShardState, MAX_SHARD_BLOCKS_PER_ATTESTATION] diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index f069c4cd2..3047fb263 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -17,11 +17,6 @@ - [Reward and penalty quotients](#reward-and-penalty-quotients) - [Signature domain types](#signature-domain-types) - [Data structures](#data-structures) - - [New Beacon Chain operations](#new-beacon-chain-operations) - - [`CustodySlashing`](#custodyslashing) - - [`SignedCustodySlashing`](#signedcustodyslashing) - - [`CustodyKeyReveal`](#custodykeyreveal) - - [`EarlyDerivedSecretReveal`](#earlyderivedsecretreveal) - [Helpers](#helpers) - [`legendre_bit`](#legendre_bit) - [`get_custody_atoms`](#get_custody_atoms) @@ -64,7 +59,6 @@ This document details the beacon chain additions and changes in Phase 1 of Ether | `EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS` | `2**14` (= 16,384) | epochs | ~73 days | | `EPOCHS_PER_CUSTODY_PERIOD` | `2**11` (= 2,048) | epochs | ~9 days | | `CUSTODY_PERIOD_TO_RANDAO_PADDING` | `2**11` (= 2,048) | epochs | ~9 days | -| `MAX_REVEAL_LATENESS_DECREMENT` | `2**7` (= 128) | epochs | ~14 hours | | `CHUNK_RESPONSE_DEADLINE` | `2**14` (= 16,384) | epochs | ~73 days | | `MAX_CHUNK_CHALLENGE_DELAY` | `2**11` (= 16,384) | epochs | ~9 days | | `CUSTODY_RESPONSE_DEADLINE` | `2**14` (= 16,384) | epochs | ~73 days | @@ -168,7 +162,7 @@ def get_custody_secrets(key: BLSSignature) -> Sequence[int]: ### `compute_custody_bit` ```python -def compute_custody_bit(key: BLSSignature, data: bytes) -> bit: +def compute_custody_bit(key: BLSSignature, data: ByteList[MAX_SHARD_BLOCK_SIZE]) -> bit: secrets = get_custody_secrets(key) custody_atoms = get_custody_atoms(data) n = len(custody_atoms) @@ -243,7 +237,6 @@ def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge ) # Verify depth transition_chunks = (challenge.shard_transition.shard_block_lengths[challenge.data_index] + BYTES_PER_CUSTODY_CHUNK - 1) // BYTES_PER_CUSTODY_CHUNK - depth = ceillog2(transition_chunks) assert challenge.chunk_index < transition_chunks # Add new chunk challenge record new_record = CustodyChunkChallengeRecord( @@ -252,7 +245,6 @@ def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge responder_index=challenge.responder_index, inclusion_epoch=get_current_epoch(state), data_root=challenge.shard_transition.shard_data_roots[challenge.data_index], - depth=depth, chunk_index=challenge.chunk_index, ) replace_empty_or_append(state.custody_chunk_challenge_records, new_record) @@ -277,7 +269,7 @@ def process_chunk_challenge_response(state: BeaconState, assert is_valid_merkle_branch( leaf=hash_tree_root(response.chunk), branch=response.branch, - depth=challenge.depth, + depth=CUSTODY_RESPONSE_DEPTH, index=response.chunk_index, root=challenge.data_root, ) @@ -311,18 +303,6 @@ def process_custody_key_reveal(state: BeaconState, reveal: CustodyKeyReveal) -> signing_root = compute_signing_root(epoch_to_sign, domain) assert bls.Verify(revealer.pubkey, signing_root, reveal.reveal) - # Decrement max reveal lateness if response is timely - if epoch_to_sign + EPOCHS_PER_CUSTODY_PERIOD >= get_current_epoch(state): - if revealer.max_reveal_lateness >= MAX_REVEAL_LATENESS_DECREMENT: - revealer.max_reveal_lateness -= MAX_REVEAL_LATENESS_DECREMENT - else: - revealer.max_reveal_lateness = 0 - else: - revealer.max_reveal_lateness = max( - revealer.max_reveal_lateness, - get_current_epoch(state) - epoch_to_sign - EPOCHS_PER_CUSTODY_PERIOD - ) - # Process reveal revealer.next_custody_secret_to_reveal += 1 @@ -417,13 +397,15 @@ def process_custody_slashing(state: BeaconState, signed_custody_slashing: Signed assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) # TODO: custody_slashing.data is not chunked like shard blocks yet, result is lots of padding. + # ??? What does this mean? # TODO: can do a single combined merkle proof of data being attested. # Verify the shard transition is indeed attested by the attestation shard_transition = custody_slashing.shard_transition - assert hash_tree_root(shard_transition) == attestation.shard_transition_root + assert hash_tree_root(shard_transition) == attestation.data.shard_transition_root # Verify that the provided data matches the shard-transition - assert hash_tree_root(custody_slashing.data) == shard_transition.shard_data_roots[custody_slashing.data_index] + assert custody_slashing.data.get_backing().get_left().merkle_root() == shard_transition.shard_data_roots[custody_slashing.data_index] + assert len(custody_slashing.data) == shard_transition.shard_block_lengths[custody_slashing.data_index] # Verify existence and participation of claimed malefactor attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 325fee260..5e536002f 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -6,6 +6,8 @@ from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.ssz.ssz_typing import Bitlist +from eth2spec.utils.ssz.ssz_impl import hash_tree_root +from eth2spec.test.helpers.custody import get_custody_test_vector def run_attestation_processing(spec, state, attestation, valid=True): @@ -72,17 +74,43 @@ def build_attestation_data(spec, state, slot, index, shard_transition_root=None) beacon_block_root=block_root, source=spec.Checkpoint(epoch=source_epoch, root=source_root), target=spec.Checkpoint(epoch=spec.compute_epoch_at_slot(slot), root=epoch_boundary_root), - shard_transition_root=shard_transition_root, + shard_transition_root=shard_transition_root if shard_transition_root else spec.Root(), ) -def convert_to_valid_on_time_attestation(spec, state, attestation, signed=False): +def convert_to_valid_on_time_attestation(spec, state, attestation, signed=False, shard_transition=None, valid_custody_bits=None): shard = spec.get_shard(state, attestation) offset_slots = spec.compute_offset_slots(spec.get_latest_slot_for_shard(state, shard), state.slot + 1) - for offset_slot in offset_slots: + + if valid_custody_bits is not None: + beacon_committee = spec.get_beacon_committee( + state, + attestation.data.slot, + attestation.data.index, + ) + current_epoch = spec.get_current_epoch(state) + custody_secrets = [None for i in beacon_committee] + for i in range(len(beacon_committee)): + validator = state.validators[beacon_committee[i]] + + period = spec.get_custody_period_for_validator(beacon_committee[i], attestation.data.target.epoch) + + epoch_to_sign = spec.get_randao_epoch_for_custody_period(period, beacon_committee[i]) + + domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch_to_sign) + signing_root = spec.compute_signing_root(spec.Epoch(epoch_to_sign), domain) + custody_secrets[i] = bls.Sign(privkeys[beacon_committee[i]], signing_root) + + + for i, offset_slot in enumerate(offset_slots): attestation.custody_bits_blocks.append( Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE]([0 for _ in attestation.aggregation_bits]) ) + if valid_custody_bits is not None: + test_vector = get_custody_test_vector(shard_transition.shard_block_lengths[i]) + for j in range(len(attestation.custody_bits_blocks[i])): + if attestation.aggregation_bits[j]: + attestation.custody_bits_blocks[i][j] = spec.compute_custody_bit(custody_secrets[j], test_vector) ^ (not valid_custody_bits) if signed: sign_attestation(spec, state, attestation) @@ -90,7 +118,7 @@ def convert_to_valid_on_time_attestation(spec, state, attestation, signed=False) return attestation -def get_valid_on_time_attestation(spec, state, slot=None, index=None, signed=False, shard_transition_root=None): +def get_valid_on_time_attestation(spec, state, slot=None, index=None, signed=False, shard_transition=None, valid_custody_bits=None): ''' Construct on-time attestation for next slot ''' @@ -99,10 +127,10 @@ def get_valid_on_time_attestation(spec, state, slot=None, index=None, signed=Fal if index is None: index = 0 - return get_valid_attestation(spec, state, slot=slot, index=index, signed=signed, on_time=True, shard_transition_root=shard_transition_root) + return get_valid_attestation(spec, state, slot=slot, index=index, signed=signed, on_time=True, shard_transition=shard_transition, valid_custody_bits=valid_custody_bits) -def get_valid_late_attestation(spec, state, slot=None, index=None, signed=False, shard_transition_root=None): +def get_valid_late_attestation(spec, state, slot=None, index=None, signed=False, shard_transition=None): ''' Construct on-time attestation for next slot ''' @@ -111,16 +139,16 @@ def get_valid_late_attestation(spec, state, slot=None, index=None, signed=False, if index is None: index = 0 - return get_valid_attestation(spec, state, slot=slot, index=index, signed=signed, on_time=False, shard_transition_root=shard_transition_root) + return get_valid_attestation(spec, state, slot=slot, index=index, signed=signed, on_time=False, shard_transition=shard_transition) -def get_valid_attestation(spec, state, slot=None, index=None, empty=False, signed=False, on_time=True, shard_transition_root=None): +def get_valid_attestation(spec, state, slot=None, index=None, empty=False, signed=False, on_time=True, shard_transition=None, valid_custody_bits=None): if slot is None: slot = state.slot if index is None: index = 0 - attestation_data = build_attestation_data(spec, state, slot, index, shard_transition_root=shard_transition_root) + attestation_data = build_attestation_data(spec, state, slot, index, shard_transition_root=hash_tree_root(shard_transition) if shard_transition else spec.Root()) beacon_committee = spec.get_beacon_committee( state, @@ -140,7 +168,7 @@ def get_valid_attestation(spec, state, slot=None, index=None, empty=False, signe sign_attestation(spec, state, attestation) if spec.fork == 'phase1' and on_time: - attestation = convert_to_valid_on_time_attestation(spec, state, attestation, signed) + attestation = convert_to_valid_on_time_attestation(spec, state, attestation, signed, shard_transition=shard_transition, valid_custody_bits=valid_custody_bits) return attestation diff --git a/tests/core/pyspec/eth2spec/test/helpers/custody.py b/tests/core/pyspec/eth2spec/test/helpers/custody.py index 45c29be9a..f4b2d6a3d 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/custody.py +++ b/tests/core/pyspec/eth2spec/test/helpers/custody.py @@ -1,10 +1,10 @@ from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls -from eth2spec.utils.ssz.ssz_typing import Bitlist, ByteVector, Bitvector, ByteList +from eth2spec.utils.ssz.ssz_typing import Bitlist, ByteVector, Bitvector, ByteList, uint64 from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.merkle_minimal import get_merkle_root, get_merkle_tree, get_merkle_proof from remerkleable.core import pack_bits_to_chunks -from remerkleable.tree import subtree_fill_to_contents, get_depth +from remerkleable.tree import subtree_fill_to_contents, get_depth, Node, Gindex, gindex_bit_iter, Root BYTES_PER_CHUNK = 32 @@ -61,39 +61,45 @@ def bitlist_from_int(max_len, num_bits, n): return Bitlist[max_len](*[(n >> i) & 0b1 for i in range(num_bits)]) -def get_valid_custody_slashing(spec, state, attestation, invalid_custody_bit=False): +def get_valid_custody_slashing(spec, state, attestation, shard_transition, invalid_custody_bit=False): beacon_committee = spec.get_beacon_committee( state, attestation.data.slot, - attestation.data.crosslink.shard, + attestation.data.index, ) - responder_index = beacon_committee[0] - challenger_index = beacon_committee[-1] + malefactor_index = beacon_committee[0] + whistleblower_index = beacon_committee[-1] epoch = spec.get_randao_epoch_for_custody_period(attestation.data.target.epoch, - responder_index) + malefactor_index) # Generate the responder key domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch) signing_root = spec.compute_signing_root(spec.Epoch(epoch), domain) - responder_key = bls.Sign(privkeys[responder_index], signing_root) + malefactor_key = bls.Sign(privkeys[malefactor_index], signing_root) + data_index = 0 + data=ByteList[spec.MAX_SHARD_BLOCK_SIZE](get_custody_test_vector(shard_transition.shard_block_lengths[data_index])) + print(hash_tree_root(data)) + print(data.get_backing().get_left().merkle_root()) - chunk_count = spec.get_custody_chunk_count(attestation.data.crosslink) - - chunk_bits = bitlist_from_int(spec.MAX_CUSTODY_CHUNKS, chunk_count, 0) - - n = 0 - while spec.get_chunk_bits_root(chunk_bits) == attestation.custody_bits[0] ^ invalid_custody_bit: - chunk_bits = bitlist_from_int(spec.MAX_CUSTODY_CHUNKS, chunk_count, n) - n += 1 - - return spec.CustodyBitChallenge( - responder_index=responder_index, + slashing = spec.CustodySlashing( + data_index=data_index, + malefactor_index=malefactor_index, + malefactor_secret=malefactor_key, + whistleblower_index=whistleblower_index, + shard_transition=shard_transition, attestation=attestation, - challenger_index=challenger_index, - responder_key=responder_key, - chunk_bits=chunk_bits, + data=data, ) + slashing_domain = spec.get_domain(state, spec.DOMAIN_CUSTODY_BIT_SLASHING) + slashing_root = spec.compute_signing_root(slashing, domain) + + signed_slashing = spec.SignedCustodySlashing( + message=slashing, + signature=bls.Sign(privkeys[whistleblower_index], slashing_root) + ) + + return signed_slashing def get_valid_chunk_challenge(spec, state, attestation, shard_transition): @@ -123,20 +129,35 @@ def custody_chunkify(spec, x): return chunks +def build_proof(anchor, leaf_index): + if leaf_index <= 1: + return [] # Nothing to prove / invalid index + node = anchor + proof = [] + # Walk down, top to bottom to the leaf + bit_iter, _ = gindex_bit_iter(leaf_index) + for bit in bit_iter: + # Always take the opposite hand for the proof. + # 1 = right as leaf, thus get left + if bit: + proof.append(node.get_left().merkle_root()) + node = node.get_right() + else: + proof.append(node.get_right().merkle_root()) + node = node.get_left() + + return list(reversed(proof)) + + def get_valid_custody_chunk_response(spec, state, chunk_challenge, block_length, challenge_index, invalid_chunk_data=False): custody_data = get_custody_test_vector(block_length) + custody_data_block = ByteList[spec.MAX_SHARD_BLOCK_SIZE](custody_data) chunks = custody_chunkify(spec, custody_data) chunk_index = chunk_challenge.chunk_index - chunks_hash_tree_roots = [hash_tree_root(ByteVector[spec.BYTES_PER_CUSTODY_CHUNK](chunk)) for chunk in chunks] - chunks_hash_tree_roots += [ - hash_tree_root(ByteVector[spec.BYTES_PER_CUSTODY_CHUNK](b"\0" * spec.BYTES_PER_CUSTODY_CHUNK)) - for i in range(2 ** spec.ceillog2(len(chunks)) - len(chunks))] - data_tree = get_merkle_tree(chunks_hash_tree_roots) - - data_branch = get_merkle_proof(data_tree, chunk_index) + data_branch = build_proof(custody_data_block.get_backing().get_left(), chunk_index + 2**spec.CUSTODY_RESPONSE_DEPTH) return spec.CustodyChunkResponse( challenge_index=challenge_index, @@ -152,7 +173,7 @@ def get_custody_test_vector(bytelength): def get_shard_transition(spec, start_slot, block_lengths): - b = [hash_tree_root(ByteVector[x](get_custody_test_vector(x))) for x in block_lengths] + b = [ByteList[spec.MAX_SHARD_BLOCK_SIZE](get_custody_test_vector(x)).get_backing().get_left().merkle_root() for x in block_lengths] shard_transition = spec.ShardTransition( start_slot=start_slot, shard_block_lengths=block_lengths, diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py index 09b25a1ad..d409b468f 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py @@ -75,7 +75,7 @@ def test_challenge_appended(spec, state): offset_slots = spec.get_offset_slots(state, shard) shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) data_index = 0 - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition_root=hash_tree_root(shard_transition)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) transition_chunks = (shard_transition.shard_block_lengths[data_index] + spec.BYTES_PER_CUSTODY_CHUNK - 1) // spec.BYTES_PER_CUSTODY_CHUNK test_vector = get_custody_test_vector(shard_transition.shard_block_lengths[data_index]) @@ -101,7 +101,7 @@ def test_multiple_epochs_custody(spec, state): offset_slots = spec.get_offset_slots(state, shard) shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) data_index = 0 - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition_root=hash_tree_root(shard_transition)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -123,7 +123,7 @@ def test_many_epochs_custody(spec, state): offset_slots = spec.get_offset_slots(state, shard) shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) data_index = 0 - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition_root=hash_tree_root(shard_transition)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -145,7 +145,7 @@ def test_off_chain_attestation(spec, state): offset_slots = spec.get_offset_slots(state, shard) shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) data_index = 0 - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition_root=hash_tree_root(shard_transition)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) @@ -163,7 +163,7 @@ def test_custody_response(spec, state): offset_slots = spec.get_offset_slots(state, shard) shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) data_index = 0 - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition_root=hash_tree_root(shard_transition)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -191,7 +191,7 @@ def test_custody_response_multiple_epochs(spec, state): offset_slots = spec.get_offset_slots(state, shard) shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) data_index = 0 - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition_root=hash_tree_root(shard_transition)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -219,7 +219,7 @@ def test_custody_response_many_epochs(spec, state): offset_slots = spec.get_offset_slots(state, shard) shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) data_index = 0 - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition_root=hash_tree_root(shard_transition)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py index 8c2436d5b..fd16a344c 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py @@ -28,30 +28,14 @@ def run_custody_key_reveal_processing(spec, state, custody_key_reveal, valid=Tru pre_next_custody_secret_to_reveal = \ state.validators[revealer_index].next_custody_secret_to_reveal - pre_reveal_lateness = state.validators[revealer_index].max_reveal_lateness spec.process_custody_key_reveal(state, custody_key_reveal) post_next_custody_secret_to_reveal = \ state.validators[revealer_index].next_custody_secret_to_reveal - post_reveal_lateness = state.validators[revealer_index].max_reveal_lateness assert post_next_custody_secret_to_reveal == pre_next_custody_secret_to_reveal + 1 - if spec.get_current_epoch(state) > spec.get_randao_epoch_for_custody_period( - pre_next_custody_secret_to_reveal, - revealer_index - ) + spec.EPOCHS_PER_CUSTODY_PERIOD: - assert post_reveal_lateness > 0 - if pre_reveal_lateness == 0: - assert post_reveal_lateness == spec.get_current_epoch(state) - spec.get_randao_epoch_for_custody_period( - pre_next_custody_secret_to_reveal, - revealer_index - ) - spec.EPOCHS_PER_CUSTODY_PERIOD - else: - if pre_reveal_lateness > 0: - assert post_reveal_lateness < pre_reveal_lateness - yield 'post', state diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py index 358aa5c0d..706fd8f49 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py @@ -1,14 +1,15 @@ from eth2spec.test.helpers.custody import ( - get_valid_bit_challenge, - get_valid_custody_bit_response, + get_valid_custody_slashing, get_custody_test_vector, get_custody_merkle_root, + get_shard_transition, ) from eth2spec.test.helpers.attestations import ( - get_valid_attestation, + get_valid_on_time_attestation, ) from eth2spec.utils.ssz.ssz_impl import hash_tree_root -from eth2spec.test.helpers.state import next_epoch, get_balance +from eth2spec.utils.ssz.ssz_typing import ByteList +from eth2spec.test.helpers.state import next_epoch, get_balance, transition_to from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.context import ( with_all_phases_except, @@ -18,332 +19,174 @@ from eth2spec.test.context import ( from eth2spec.test.phase_0.block_processing.test_process_attestation import run_attestation_processing -def run_bit_challenge_processing(spec, state, custody_bit_challenge, valid=True): +def run_custody_slashing_processing(spec, state, custody_slashing, valid=True, correct=True): """ Run ``process_bit_challenge``, yielding: - pre-state ('pre') - - CustodyBitChallenge ('custody_bit_challenge') + - CustodySlashing ('custody_slashing') - post-state ('post'). If ``valid == False``, run expecting ``AssertionError`` """ yield 'pre', state - yield 'custody_bit_challenge', custody_bit_challenge + yield 'custody_slashing', custody_slashing if not valid: - expect_assertion_error(lambda: spec.process_bit_challenge(state, custody_bit_challenge)) + expect_assertion_error(lambda: spec.process_custody_slashing(state, custody_slashing)) yield 'post', None return - spec.process_bit_challenge(state, custody_bit_challenge) + if correct: + pre_slashed_balance = get_balance(state, custody_slashing.message.malefactor_index) + else: + pre_slashed_balance = get_balance(state, custody_slashing.message.whistleblower_index) - assert state.custody_bit_challenge_records[state.custody_bit_challenge_index - 1].chunk_bits_merkle_root == \ - hash_tree_root(custody_bit_challenge.chunk_bits) - assert state.custody_bit_challenge_records[state.custody_bit_challenge_index - 1].challenger_index == \ - custody_bit_challenge.challenger_index - assert state.custody_bit_challenge_records[state.custody_bit_challenge_index - 1].responder_index == \ - custody_bit_challenge.responder_index - - yield 'post', state - - -def run_custody_bit_response_processing(spec, state, custody_response, valid=True): - """ - Run ``process_bit_challenge_response``, yielding: - - pre-state ('pre') - - CustodyResponse ('custody_response') - - post-state ('post'). - If ``valid == False``, run expecting ``AssertionError`` - """ - yield 'pre', state - yield 'custody_response', custody_response - - if not valid: - expect_assertion_error(lambda: spec.process_custody_response(state, custody_response)) - yield 'post', None - return - - challenge = state.custody_bit_challenge_records[custody_response.challenge_index] - pre_slashed_balance = get_balance(state, challenge.challenger_index) - - spec.process_custody_response(state, custody_response) - - slashed_validator = state.validators[challenge.challenger_index] + spec.process_custody_slashing(state, custody_slashing) + if correct: + slashed_validator = state.validators[custody_slashing.message.malefactor_index] + assert get_balance(state, custody_slashing.message.malefactor_index) < pre_slashed_balance + else: + slashed_validator = state.validators[custody_slashing.message.whistleblower_index] + assert get_balance(state, custody_slashing.message.whistleblower_index) < pre_slashed_balance + assert slashed_validator.slashed assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH - assert get_balance(state, challenge.challenger_index) < pre_slashed_balance yield 'post', state @with_all_phases_except(['phase0']) @spec_state_test -def test_challenge_appended(spec, state): - state.slot = spec.SLOTS_PER_EPOCH - attestation = get_valid_attestation(spec, state, signed=True) +def test_custody_slashing(spec, state): + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + data_index = 0 + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition, valid_custody_bits=False) - test_vector = get_custody_test_vector( - spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) - shard_root = get_custody_merkle_root(test_vector) - attestation.data.crosslink.data_root = shard_root - attestation.custody_bits[0] = 0 - - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) _, _, _ = run_attestation_processing(spec, state, attestation) - state.slot += spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) - challenge = get_valid_bit_challenge(spec, state, attestation) + data_index = 0 - yield from run_bit_challenge_processing(spec, state, challenge) + slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition) + + yield from run_custody_slashing_processing(spec, state, slashing, correct=True) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_incorrect_custody_slashing(spec, state): + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + data_index = 0 + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition, valid_custody_bits=True) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) + + data_index = 0 + + slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition) + + yield from run_custody_slashing_processing(spec, state, slashing, correct=False) @with_all_phases_except(['phase0']) @spec_state_test def test_multiple_epochs_custody(spec, state): - state.slot = spec.SLOTS_PER_EPOCH * 3 - attestation = get_valid_attestation(spec, state, signed=True) + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 3) + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + data_index = 0 + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition, valid_custody_bits=False) - test_vector = get_custody_test_vector( - spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) - shard_root = get_custody_merkle_root(test_vector) - attestation.data.crosslink.data_root = shard_root - attestation.custody_bits[0] = 0 - - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) _, _, _ = run_attestation_processing(spec, state, attestation) - state.slot += spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1) + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) - challenge = get_valid_bit_challenge(spec, state, attestation) + data_index = 0 - yield from run_bit_challenge_processing(spec, state, challenge) + slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition) + + yield from run_custody_slashing_processing(spec, state, slashing, correct=True) @with_all_phases_except(['phase0']) @spec_state_test def test_many_epochs_custody(spec, state): - state.slot = spec.SLOTS_PER_EPOCH * 100 - attestation = get_valid_attestation(spec, state, signed=True) + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH* 100) + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + data_index = 0 + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition, valid_custody_bits=False) - test_vector = get_custody_test_vector( - spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) - shard_root = get_custody_merkle_root(test_vector) - attestation.data.crosslink.data_root = shard_root - attestation.custody_bits[0] = 0 - - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) _, _, _ = run_attestation_processing(spec, state, attestation) - state.slot += spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1) + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) - challenge = get_valid_bit_challenge(spec, state, attestation) + data_index = 0 - yield from run_bit_challenge_processing(spec, state, challenge) + slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition) + + yield from run_custody_slashing_processing(spec, state, slashing, correct=True) @with_all_phases_except(['phase0']) @spec_state_test def test_off_chain_attestation(spec, state): - state.slot = spec.SLOTS_PER_EPOCH - attestation = get_valid_attestation(spec, state, signed=True) + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + data_index = 0 + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition, valid_custody_bits=False) - test_vector = get_custody_test_vector( - spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) - shard_root = get_custody_merkle_root(test_vector) - attestation.data.crosslink.data_root = shard_root - attestation.custody_bits[0] = 0 + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - state.slot += spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD + data_index = 0 - challenge = get_valid_bit_challenge(spec, state, attestation) + slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition) - yield from run_bit_challenge_processing(spec, state, challenge) + yield from run_custody_slashing_processing(spec, state, slashing, correct=True) @with_all_phases_except(['phase0']) @spec_state_test -def test_invalid_custody_bit_challenge(spec, state): - state.slot = spec.SLOTS_PER_EPOCH - attestation = get_valid_attestation(spec, state, signed=True) +def test_invalid_custody_slashing(spec, state): + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + data_index = 0 + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition, valid_custody_bits=False) - test_vector = get_custody_test_vector( - spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) - shard_root = get_custody_merkle_root(test_vector) - attestation.data.crosslink.data_root = shard_root - attestation.custody_bits[0] = 0 - - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) _, _, _ = run_attestation_processing(spec, state, attestation) - state.slot += spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) - challenge = get_valid_bit_challenge(spec, state, attestation, invalid_custody_bit=True) + data_index = 0 - yield from run_bit_challenge_processing(spec, state, challenge, valid=False) + slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition) + slashing.message.data = ByteList[spec.MAX_SHARD_BLOCK_SIZE]() -@with_all_phases_except(['phase0']) -@spec_state_test -def test_max_reveal_lateness_1(spec, state): - next_epoch(spec, state) - apply_empty_block(spec, state) - - attestation = get_valid_attestation(spec, state, signed=True) - - test_vector = get_custody_test_vector( - spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) - shard_root = get_custody_merkle_root(test_vector) - attestation.data.crosslink.data_root = shard_root - attestation.custody_bits[0] = 0 - - next_epoch(spec, state) - apply_empty_block(spec, state) - - _, _, _ = run_attestation_processing(spec, state, attestation) - - challenge = get_valid_bit_challenge(spec, state, attestation) - - responder_index = challenge.responder_index - target_epoch = attestation.data.target.epoch - - state.validators[responder_index].max_reveal_lateness = 3 - - latest_reveal_epoch = spec.get_randao_epoch_for_custody_period( - spec.get_custody_period_for_validator(state, responder_index, target_epoch), - responder_index - ) + 2 * spec.EPOCHS_PER_CUSTODY_PERIOD + state.validators[responder_index].max_reveal_lateness - - while spec.get_current_epoch(state) < latest_reveal_epoch - 2: - next_epoch(spec, state) - apply_empty_block(spec, state) - - yield from run_bit_challenge_processing(spec, state, challenge) - - -@with_all_phases_except(['phase0']) -@spec_state_test -def test_max_reveal_lateness_2(spec, state): - next_epoch(spec, state) - apply_empty_block(spec, state) - - attestation = get_valid_attestation(spec, state, signed=True) - - test_vector = get_custody_test_vector( - spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) - shard_root = get_custody_merkle_root(test_vector) - attestation.data.crosslink.data_root = shard_root - attestation.custody_bits[0] = 0 - - next_epoch(spec, state) - apply_empty_block(spec, state) - - _, _, _ = run_attestation_processing(spec, state, attestation) - - challenge = get_valid_bit_challenge(spec, state, attestation) - - responder_index = challenge.responder_index - - state.validators[responder_index].max_reveal_lateness = 3 - - for i in range(spec.get_randao_epoch_for_custody_period( - spec.get_custody_period_for_validator(state, responder_index), - responder_index - ) + 2 * spec.EPOCHS_PER_CUSTODY_PERIOD + state.validators[responder_index].max_reveal_lateness - 1): - next_epoch(spec, state) - apply_empty_block(spec, state) - - yield from run_bit_challenge_processing(spec, state, challenge, False) - - -@with_all_phases_except(['phase0']) -@spec_state_test -def test_custody_response(spec, state): - state.slot = spec.SLOTS_PER_EPOCH - attestation = get_valid_attestation(spec, state, signed=True) - - test_vector = get_custody_test_vector( - spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) - shard_root = get_custody_merkle_root(test_vector) - attestation.data.crosslink.data_root = shard_root - attestation.custody_bits[0] = 0 - - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - - _, _, _ = run_attestation_processing(spec, state, attestation) - - state.slot += spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD - - challenge = get_valid_bit_challenge(spec, state, attestation) - - _, _, _ = run_bit_challenge_processing(spec, state, challenge) - - bit_challenge_index = state.custody_bit_challenge_index - 1 - - custody_response = get_valid_custody_bit_response(spec, state, challenge, test_vector, bit_challenge_index) - - yield from run_custody_bit_response_processing(spec, state, custody_response) - - -@with_all_phases_except(['phase0']) -@spec_state_test -def test_custody_response_multiple_epochs(spec, state): - state.slot = spec.SLOTS_PER_EPOCH * 3 - attestation = get_valid_attestation(spec, state, signed=True) - - test_vector = get_custody_test_vector( - spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) - shard_root = get_custody_merkle_root(test_vector) - attestation.data.crosslink.data_root = shard_root - attestation.custody_bits[0] = 0 - - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - - _, _, _ = run_attestation_processing(spec, state, attestation) - - state.slot += spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD - - challenge = get_valid_bit_challenge(spec, state, attestation) - - _, _, _ = run_bit_challenge_processing(spec, state, challenge) - - bit_challenge_index = state.custody_bit_challenge_index - 1 - - custody_response = get_valid_custody_bit_response(spec, state, challenge, test_vector, bit_challenge_index) - - yield from run_custody_bit_response_processing(spec, state, custody_response) - - -@with_all_phases_except(['phase0']) -@spec_state_test -def test_custody_response_many_epochs(spec, state): - state.slot = spec.SLOTS_PER_EPOCH * 100 - attestation = get_valid_attestation(spec, state, signed=True) - - test_vector = get_custody_test_vector( - spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) - shard_root = get_custody_merkle_root(test_vector) - attestation.data.crosslink.data_root = shard_root - attestation.custody_bits[0] = 0 - - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - - _, _, _ = run_attestation_processing(spec, state, attestation) - - state.slot += spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD - - challenge = get_valid_bit_challenge(spec, state, attestation) - - _, _, _ = run_bit_challenge_processing(spec, state, challenge) - - bit_challenge_index = state.custody_bit_challenge_index - 1 - - custody_response = get_valid_custody_bit_response(spec, state, challenge, test_vector, bit_challenge_index) - - yield from run_custody_bit_response_processing(spec, state, custody_response) + yield from run_custody_slashing_processing(spec, state, slashing, valid=False) From 0e2931b9b3490027e11a9a4a7c0851466d533d5d Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Tue, 28 Apr 2020 01:09:20 +0100 Subject: [PATCH 011/165] All tests passed --- configs/minimal.yaml | 2 - specs/phase1/beacon-chain.md | 54 +++++ specs/phase1/custody-game.md | 29 ++- specs/phase1/phase1-fork.md | 2 +- .../pyspec/eth2spec/test/helpers/custody.py | 5 +- .../test_process_final_custody_updates.py | 213 +++--------------- 6 files changed, 121 insertions(+), 184 deletions(-) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 256c2b3fa..5f8b6b8a2 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -208,8 +208,6 @@ EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 4096 EPOCHS_PER_CUSTODY_PERIOD: 8 # 2**11 (= 2,048) epochs CUSTODY_PERIOD_TO_RANDAO_PADDING: 8 -# 2**7 (= 128) epochs -MAX_REVEAL_LATENESS_DECREMENT: 128 # Max operations # 2**8 (= 256) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index fc208cbaa..b6b21b721 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -282,6 +282,7 @@ class Validator(Container): next_custody_secret_to_reveal: uint64 # TODO: The max_reveal_lateness doesn't really make sense anymore. # So how do we incentivise early custody key reveals now? + all_custody_secrets_revealed_epoch: Epoch # to be initialized to FAR_FUTURE_EPOCH ``` ### Extended `BeaconBlockBody` @@ -925,6 +926,59 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: state.previous_epoch_attestations.append(pending_attestation) ``` +##### New deposits + +```python +def process_deposit(state: BeaconState, deposit: Deposit) -> None: + # Verify the Merkle branch + assert is_valid_merkle_branch( + leaf=hash_tree_root(deposit.data), + branch=deposit.proof, + depth=DEPOSIT_CONTRACT_TREE_DEPTH + 1, # Add 1 for the List length mix-in + index=state.eth1_deposit_index, + root=state.eth1_data.deposit_root, + ) + + # Deposits must be processed in order + state.eth1_deposit_index += 1 + + pubkey = deposit.data.pubkey + amount = deposit.data.amount + validator_pubkeys = [v.pubkey for v in state.validators] + if pubkey not in validator_pubkeys: + # Verify the deposit signature (proof of possession) which is not checked by the deposit contract + deposit_message = DepositMessage( + pubkey=deposit.data.pubkey, + withdrawal_credentials=deposit.data.withdrawal_credentials, + amount=deposit.data.amount, + ) + domain = compute_domain(DOMAIN_DEPOSIT) # Fork-agnostic domain since deposits are valid across forks + signing_root = compute_signing_root(deposit_message, domain) + if not bls.Verify(pubkey, signing_root, deposit.data.signature): + return + + # Add validator and balance entries + # TODO: This function is duplicated from phase 1 just because the validator definition + # has changed and we need to initialize it properly. Is there a better solution for + # this? + state.validators.append(Validator( + pubkey=pubkey, + withdrawal_credentials=deposit.data.withdrawal_credentials, + activation_eligibility_epoch=FAR_FUTURE_EPOCH, + activation_epoch=FAR_FUTURE_EPOCH, + exit_epoch=FAR_FUTURE_EPOCH, + withdrawable_epoch=FAR_FUTURE_EPOCH, + effective_balance=min(amount - amount % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE), + next_custody_secret_to_reveal=get_custody_period_for_validator(ValidatorIndex(len(state.validators)), get_current_epoch(state)), + all_custody_secrets_revealed_epoch=FAR_FUTURE_EPOCH, + )) + state.balances.append(amount) + else: + # Increase balance by deposit amount + index = ValidatorIndex(validator_pubkeys.index(pubkey)) + increase_balance(state, index, amount) +``` + ##### New Attester slashing processing ```python diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 3047fb263..f40e936ce 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -293,7 +293,12 @@ def process_custody_key_reveal(state: BeaconState, reveal: CustodyKeyReveal) -> epoch_to_sign = get_randao_epoch_for_custody_period(revealer.next_custody_secret_to_reveal, reveal.revealer_index) custody_reveal_period = get_custody_period_for_validator(reveal.revealer_index, get_current_epoch(state)) - assert revealer.next_custody_secret_to_reveal < custody_reveal_period + # Only past custody periods can be revealed, except after exiting the exit + # period can be revealed + assert (revealer.next_custody_secret_to_reveal < custody_reveal_period + or (revealer.exit_epoch <= get_current_epoch(state) and + revealer.next_custody_secret_to_reveal + <= get_custody_period_for_validator(reveal.revealer_index, revealer.exit_epoch - 1))) # Revealed validator is active or exited, but not withdrawn assert is_slashable_validator(revealer, get_current_epoch(state)) @@ -304,6 +309,10 @@ def process_custody_key_reveal(state: BeaconState, reveal: CustodyKeyReveal) -> assert bls.Verify(revealer.pubkey, signing_root, reveal.reveal) # Process reveal + if (revealer.exit_epoch <= get_current_epoch(state) and + revealer.next_custody_secret_to_reveal + == get_custody_period_for_validator(reveal.revealer_index, revealer.exit_epoch - 1)): + revealer.all_custody_secrets_revealed_epoch = get_current_epoch(state) revealer.next_custody_secret_to_reveal += 1 # Reward Block Proposer @@ -480,4 +489,22 @@ After `process_final_updates(state)`, additional updates are made for the custod def process_custody_final_updates(state: BeaconState) -> None: # Clean up exposed RANDAO key reveals state.exposed_derived_secrets[get_current_epoch(state) % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] = [] + + # Reset withdrawable epochs if challenge records are empty + records = state.custody_chunk_challenge_records + validator_indices_in_records = set( + [record.responder_index for record in records] + ) + for index, validator in enumerate(state.validators): + if validator.exit_epoch != FAR_FUTURE_EPOCH: + if (index in validator_indices_in_records + or validator.all_custody_secrets_revealed_epoch == FAR_FUTURE_EPOCH): + # Delay withdrawable epochs if challenge records are not empty or not all + # custody secrets revealed + validator.withdrawable_epoch = FAR_FUTURE_EPOCH + else: + # Reset withdrawable epochs if challenge records are empty + if validator.withdrawable_epoch == FAR_FUTURE_EPOCH: + validator.withdrawable_epoch = Epoch(validator.all_custody_secrets_revealed_epoch + + MIN_VALIDATOR_WITHDRAWABILITY_DELAY) ``` diff --git a/specs/phase1/phase1-fork.md b/specs/phase1/phase1-fork.md index 173fceeb4..9d5f282f3 100644 --- a/specs/phase1/phase1-fork.md +++ b/specs/phase1/phase1-fork.md @@ -79,7 +79,7 @@ def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: exit_epoch=phase0_validator.exit_epoch, withdrawable_epoch=phase0_validator.withdrawable_epoch, next_custody_secret_to_reveal=get_custody_period_for_validator(ValidatorIndex(i), epoch), - max_reveal_lateness=0, # TODO custody refactor. Outdated? + all_custody_secrets_revealed_epoch=FAR_FUTURE_EPOCH, ) for i, phase0_validator in enumerate(pre.validators) ), balances=pre.balances, diff --git a/tests/core/pyspec/eth2spec/test/helpers/custody.py b/tests/core/pyspec/eth2spec/test/helpers/custody.py index f4b2d6a3d..2d937a2ee 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/custody.py +++ b/tests/core/pyspec/eth2spec/test/helpers/custody.py @@ -37,9 +37,10 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): ) -def get_valid_custody_key_reveal(spec, state, period=None): +def get_valid_custody_key_reveal(spec, state, period=None, validator_index=None): current_epoch = spec.get_current_epoch(state) - revealer_index = spec.get_active_validator_indices(state, current_epoch)[0] + revealer_index = (spec.get_active_validator_indices(state, current_epoch)[0] + if validator_index is None else validator_index) revealer = state.validators[revealer_index] if period is None: diff --git a/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_final_custody_updates.py b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_final_custody_updates.py index d200ed2b3..566d795cf 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_final_custody_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_final_custody_updates.py @@ -1,16 +1,15 @@ from eth2spec.test.helpers.custody import ( - get_valid_bit_challenge, get_valid_chunk_challenge, - get_valid_custody_bit_response, get_valid_custody_chunk_response, get_valid_custody_key_reveal, get_custody_test_vector, get_custody_merkle_root, + get_shard_transition ) from eth2spec.test.helpers.attestations import ( - get_valid_attestation, + get_valid_on_time_attestation, ) -from eth2spec.test.helpers.state import next_epoch +from eth2spec.test.helpers.state import next_epoch, transition_to from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.context import ( with_all_phases_except, @@ -19,10 +18,6 @@ from eth2spec.test.context import ( from eth2spec.test.phase_0.block_processing.test_process_attestation import run_attestation_processing from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_with -from eth2spec.test.phase_1.block_processing.test_process_bit_challenge import ( - run_bit_challenge_processing, - run_custody_bit_response_processing, -) from eth2spec.test.phase_1.block_processing.test_process_chunk_challenge import ( run_chunk_challenge_processing, run_custody_chunk_response_processing, @@ -30,8 +25,8 @@ from eth2spec.test.phase_1.block_processing.test_process_chunk_challenge import from eth2spec.test.phase_1.block_processing.test_process_custody_key_reveal import run_custody_key_reveal_processing -def run_process_final_custody_updates(spec, state): - yield from run_epoch_processing_with(spec, state, 'process_final_custody_updates') +def run_process_custody_final_updates(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_custody_final_updates') @with_all_phases_except(['phase0']) @@ -40,7 +35,7 @@ def test_validator_withdrawal_delay(spec, state): spec.initiate_validator_exit(state, 0) assert state.validators[0].withdrawable_epoch < spec.FAR_FUTURE_EPOCH - yield from run_process_final_custody_updates(spec, state) + yield from run_process_custody_final_updates(spec, state) assert state.validators[0].withdrawable_epoch == spec.FAR_FUTURE_EPOCH @@ -61,100 +56,39 @@ def test_validator_withdrawal_reenable_after_custody_reveal(spec, state): apply_empty_block(spec, state) while (state.validators[0].next_custody_secret_to_reveal - <= spec.get_custody_period_for_validator(state, 0, state.validators[0].exit_epoch - 1)): + <= spec.get_custody_period_for_validator(0, state.validators[0].exit_epoch - 1)): custody_key_reveal = get_valid_custody_key_reveal(spec, state, validator_index=0) _, _, _ = run_custody_key_reveal_processing(spec, state, custody_key_reveal) - yield from run_process_final_custody_updates(spec, state) + yield from run_process_custody_final_updates(spec, state) assert state.validators[0].withdrawable_epoch < spec.FAR_FUTURE_EPOCH -@with_all_phases_except(['phase0']) -@spec_state_test -def test_validator_withdrawal_suspend_after_bit_challenge(spec, state): - state.slot = spec.SLOTS_PER_EPOCH - - attestation = get_valid_attestation(spec, state, signed=True) - - test_vector = get_custody_test_vector( - spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) - shard_root = get_custody_merkle_root(test_vector) - attestation.data.crosslink.data_root = shard_root - attestation.custody_bits[0] = 0 - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - next_epoch(spec, state) - apply_empty_block(spec, state) - - _, _, _ = run_attestation_processing(spec, state, attestation) - - validator_index = spec.get_crosslink_committee( - state, - attestation.data.target.epoch, - attestation.data.crosslink.shard - )[0] - - spec.initiate_validator_exit(state, validator_index) - assert state.validators[validator_index].withdrawable_epoch < spec.FAR_FUTURE_EPOCH - - next_epoch(spec, state) - apply_empty_block(spec, state) - - assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH - - while spec.get_current_epoch(state) < state.validators[validator_index].exit_epoch: - next_epoch(spec, state) - apply_empty_block(spec, state) - - while (state.validators[validator_index].next_custody_secret_to_reveal - <= spec.get_custody_period_for_validator( - state, - validator_index, - state.validators[validator_index].exit_epoch - 1)): - custody_key_reveal = get_valid_custody_key_reveal(spec, state, validator_index=validator_index) - _, _, _ = run_custody_key_reveal_processing(spec, state, custody_key_reveal) - - next_epoch(spec, state) - apply_empty_block(spec, state) - - challenge = get_valid_bit_challenge(spec, state, attestation) - - _, _, _ = run_bit_challenge_processing(spec, state, challenge) - - yield from run_process_final_custody_updates(spec, state) - - assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH - - @with_all_phases_except(['phase0']) @spec_state_test def test_validator_withdrawal_suspend_after_chunk_challenge(spec, state): - state.slot = spec.SLOTS_PER_EPOCH + transition_to(spec, state, state.slot + 1) + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + data_index = 0 + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) - attestation = get_valid_attestation(spec, state, signed=True) - - test_vector = get_custody_test_vector( - spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) - shard_root = get_custody_merkle_root(test_vector) - attestation.data.crosslink.data_root = shard_root - attestation.custody_bits[0] = 0 - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - next_epoch(spec, state) - apply_empty_block(spec, state) + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) _, _, _ = run_attestation_processing(spec, state, attestation) - validator_index = spec.get_crosslink_committee( + validator_index = spec.get_beacon_committee( state, - attestation.data.target.epoch, - attestation.data.crosslink.shard + attestation.data.slot, + attestation.data.index )[0] spec.initiate_validator_exit(state, validator_index) assert state.validators[validator_index].withdrawable_epoch < spec.FAR_FUTURE_EPOCH - next_epoch(spec, state) - apply_empty_block(spec, state) + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH @@ -164,7 +98,6 @@ def test_validator_withdrawal_suspend_after_chunk_challenge(spec, state): while (state.validators[validator_index].next_custody_secret_to_reveal <= spec.get_custody_period_for_validator( - state, validator_index, state.validators[validator_index].exit_epoch - 1)): custody_key_reveal = get_valid_custody_key_reveal(spec, state, validator_index=validator_index) @@ -173,108 +106,33 @@ def test_validator_withdrawal_suspend_after_chunk_challenge(spec, state): next_epoch(spec, state) apply_empty_block(spec, state) - challenge = get_valid_chunk_challenge(spec, state, attestation) + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) _, _, _ = run_chunk_challenge_processing(spec, state, challenge) - yield from run_process_final_custody_updates(spec, state) + yield from run_process_custody_final_updates(spec, state) assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH -@with_all_phases_except(['phase0']) -@spec_state_test -def test_validator_withdrawal_resume_after_bit_challenge_response(spec, state): - state.slot = spec.SLOTS_PER_EPOCH - - attestation = get_valid_attestation(spec, state, signed=True) - - test_vector = get_custody_test_vector( - spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) - shard_root = get_custody_merkle_root(test_vector) - attestation.data.crosslink.data_root = shard_root - attestation.custody_bits[0] = 0 - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - next_epoch(spec, state) - apply_empty_block(spec, state) - - _, _, _ = run_attestation_processing(spec, state, attestation) - - validator_index = spec.get_crosslink_committee( - state, - attestation.data.target.epoch, - attestation.data.crosslink.shard - )[0] - - spec.initiate_validator_exit(state, validator_index) - assert state.validators[validator_index].withdrawable_epoch < spec.FAR_FUTURE_EPOCH - - next_epoch(spec, state) - apply_empty_block(spec, state) - - assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH - - while spec.get_current_epoch(state) < state.validators[validator_index].exit_epoch: - next_epoch(spec, state) - apply_empty_block(spec, state) - - while (state.validators[validator_index].next_custody_secret_to_reveal - <= spec.get_custody_period_for_validator( - state, - validator_index, - state.validators[validator_index].exit_epoch - 1)): - custody_key_reveal = get_valid_custody_key_reveal(spec, state, validator_index=validator_index) - _, _, _ = run_custody_key_reveal_processing(spec, state, custody_key_reveal) - - next_epoch(spec, state) - apply_empty_block(spec, state) - - challenge = get_valid_bit_challenge(spec, state, attestation) - - _, _, _ = run_bit_challenge_processing(spec, state, challenge) - - next_epoch(spec, state) - apply_empty_block(spec, state) - - assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH - - bit_challenge_index = state.custody_bit_challenge_index - 1 - response = get_valid_custody_bit_response( - spec, - state, - challenge, - test_vector, - bit_challenge_index, - invalid_chunk_bit=False) - - _, _, _ = run_custody_bit_response_processing(spec, state, response) - - yield from run_process_final_custody_updates(spec, state) - - assert state.validators[validator_index].withdrawable_epoch < spec.FAR_FUTURE_EPOCH - - @with_all_phases_except(['phase0']) @spec_state_test def test_validator_withdrawal_resume_after_chunk_challenge_response(spec, state): - state.slot = spec.SLOTS_PER_EPOCH + transition_to(spec, state, state.slot + 1) + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + data_index = 0 + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) - attestation = get_valid_attestation(spec, state, signed=True) - - test_vector = get_custody_test_vector( - spec.get_custody_chunk_count(attestation.data.crosslink) * spec.BYTES_PER_CUSTODY_CHUNK) - shard_root = get_custody_merkle_root(test_vector) - attestation.data.crosslink.data_root = shard_root - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - next_epoch(spec, state) - apply_empty_block(spec, state) + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) _, _, _ = run_attestation_processing(spec, state, attestation) - validator_index = spec.get_crosslink_committee( + validator_index = spec.get_beacon_committee( state, - attestation.data.target.epoch, - attestation.data.crosslink.shard + attestation.data.slot, + attestation.data.index )[0] spec.initiate_validator_exit(state, validator_index) @@ -291,7 +149,6 @@ def test_validator_withdrawal_resume_after_chunk_challenge_response(spec, state) while (state.validators[validator_index].next_custody_secret_to_reveal <= spec.get_custody_period_for_validator( - state, validator_index, state.validators[validator_index].exit_epoch - 1)): custody_key_reveal = get_valid_custody_key_reveal(spec, state, validator_index=validator_index) @@ -300,7 +157,7 @@ def test_validator_withdrawal_resume_after_chunk_challenge_response(spec, state) next_epoch(spec, state) apply_empty_block(spec, state) - challenge = get_valid_chunk_challenge(spec, state, attestation) + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) _, _, _ = run_chunk_challenge_processing(spec, state, challenge) @@ -310,10 +167,10 @@ def test_validator_withdrawal_resume_after_chunk_challenge_response(spec, state) assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH chunk_challenge_index = state.custody_chunk_challenge_index - 1 - response = get_valid_custody_chunk_response(spec, state, challenge, test_vector, chunk_challenge_index) + custody_response = get_valid_custody_chunk_response(spec, state, challenge, 2**15 // 3, chunk_challenge_index) - _, _, _ = run_custody_chunk_response_processing(spec, state, response) + _, _, _ = run_custody_chunk_response_processing(spec, state, custody_response) - yield from run_process_final_custody_updates(spec, state) + yield from run_process_custody_final_updates(spec, state) assert state.validators[validator_index].withdrawable_epoch < spec.FAR_FUTURE_EPOCH From d58d7627b78885035dbbd0300e61e14266074da3 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 30 Apr 2020 19:25:18 +0100 Subject: [PATCH 012/165] Fix toc --- specs/phase1/beacon-chain.md | 11 ++++++++++- specs/phase1/custody-game.md | 4 ++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index b6b21b721..593f9ba0f 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -12,6 +12,14 @@ - [Custom types](#custom-types) - [Configuration](#configuration) - [Misc](#misc) +- [New containers](#new-containers) + - [`CustodyChunkChallenge`](#custodychunkchallenge) + - [`CustodyChunkChallengeRecord`](#custodychunkchallengerecord) + - [`CustodyChunkResponse`](#custodychunkresponse) + - [`CustodySlashing`](#custodyslashing) + - [`SignedCustodySlashing`](#signedcustodyslashing) + - [`CustodyKeyReveal`](#custodykeyreveal) + - [`EarlyDerivedSecretReveal`](#earlyderivedsecretreveal) - [Updated containers](#updated-containers) - [Extended `AttestationData`](#extended-attestationdata) - [Extended `Attestation`](#extended-attestation) @@ -23,7 +31,7 @@ - [Extended `BeaconBlock`](#extended-beaconblock) - [Extended `SignedBeaconBlock`](#extended-signedbeaconblock) - [Extended `BeaconState`](#extended-beaconstate) -- [New containers](#new-containers) +- [New containers](#new-containers-1) - [`ShardBlockWrapper`](#shardblockwrapper) - [`ShardSignableHeader`](#shardsignableheader) - [`ShardState`](#shardstate) @@ -61,6 +69,7 @@ - [`process_crosslink_for_shard`](#process_crosslink_for_shard) - [`process_crosslinks`](#process_crosslinks) - [`process_attestation`](#process_attestation) + - [New deposits](#new-deposits) - [New Attester slashing processing](#new-attester-slashing-processing) - [Shard transition false positives](#shard-transition-false-positives) - [Light client processing](#light-client-processing) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index f40e936ce..80ed544fe 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -17,7 +17,9 @@ - [Reward and penalty quotients](#reward-and-penalty-quotients) - [Signature domain types](#signature-domain-types) - [Data structures](#data-structures) + - [New Beacon Chain operations](#new-beacon-chain-operations) - [Helpers](#helpers) + - [`replace_empty_or_append`](#replace_empty_or_append) - [`legendre_bit`](#legendre_bit) - [`get_custody_atoms`](#get_custody_atoms) - [`get_custody_secrets`](#get_custody_secrets) @@ -26,6 +28,8 @@ - [`get_custody_period_for_validator`](#get_custody_period_for_validator) - [Per-block processing](#per-block-processing) - [Custody Game Operations](#custody-game-operations) + - [Chunk challenges](#chunk-challenges) + - [Custody chunk response](#custody-chunk-response) - [Custody key reveals](#custody-key-reveals) - [Early derived secret reveals](#early-derived-secret-reveals) - [Custody Slashings](#custody-slashings) From d30f11a7818eecabcccd6f9126edbf462675eab7 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Fri, 1 May 2020 00:16:00 +0100 Subject: [PATCH 013/165] Fix lint --- Makefile | 2 +- specs/phase1/beacon-chain.md | 5 +- specs/phase1/custody-game.md | 17 ++-- .../eth2spec/test/helpers/attestations.py | 32 ++++--- .../pyspec/eth2spec/test/helpers/custody.py | 32 +++---- .../test/helpers/phase1/attestations.py | 6 +- .../test_process_chunk_challenge.py | 85 ++++++++++++++----- .../test_process_custody_slashing.py | 46 ++++------ 8 files changed, 127 insertions(+), 98 deletions(-) diff --git a/Makefile b/Makefile index 8cd7daf58..18223430b 100644 --- a/Makefile +++ b/Makefile @@ -101,7 +101,7 @@ codespell: lint: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - flake8 --ignore=E252,W504,W503 --max-line-length=120 ./eth2spec \ + flake8 --ignore=E252,W504,W503,E128 --max-line-length=120 ./eth2spec \ && cd ./eth2spec && mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase0 \ && mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase1; diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 593f9ba0f..eda9fdd52 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -291,7 +291,7 @@ class Validator(Container): next_custody_secret_to_reveal: uint64 # TODO: The max_reveal_lateness doesn't really make sense anymore. # So how do we incentivise early custody key reveals now? - all_custody_secrets_revealed_epoch: Epoch # to be initialized to FAR_FUTURE_EPOCH + all_custody_secrets_revealed_epoch: Epoch # to be initialized to FAR_FUTURE_EPOCH ``` ### Extended `BeaconBlockBody` @@ -978,7 +978,8 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: exit_epoch=FAR_FUTURE_EPOCH, withdrawable_epoch=FAR_FUTURE_EPOCH, effective_balance=min(amount - amount % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE), - next_custody_secret_to_reveal=get_custody_period_for_validator(ValidatorIndex(len(state.validators)), get_current_epoch(state)), + next_custody_secret_to_reveal=get_custody_period_for_validator(ValidatorIndex(len(state.validators)), + get_current_epoch(state)), all_custody_secrets_revealed_epoch=FAR_FUTURE_EPOCH, )) state.balances.append(amount) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 80ed544fe..9f5a8909b 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -102,7 +102,8 @@ The following types are defined, mapping into `DomainType` (little endian): ```python def replace_empty_or_append(list: List, new_element: Any) -> int: for i in range(len(list)): - if list[i] == empty(typeof(new_element)): + if list[i] == type(new_element)(): + assert False list[i] = new_element return i list.append(new_element) @@ -236,11 +237,12 @@ def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge # Verify the challenge is not a duplicate for record in state.custody_chunk_challenge_records: assert ( - record.data_root != challenge.attestation.data.crosslink.data_root or + record.data_root != data_root or record.chunk_index != challenge.chunk_index ) # Verify depth - transition_chunks = (challenge.shard_transition.shard_block_lengths[challenge.data_index] + BYTES_PER_CUSTODY_CHUNK - 1) // BYTES_PER_CUSTODY_CHUNK + transition_chunks = (challenge.shard_transition.shard_block_lengths[challenge.data_index] + + BYTES_PER_CUSTODY_CHUNK - 1) // BYTES_PER_CUSTODY_CHUNK assert challenge.chunk_index < transition_chunks # Add new chunk challenge record new_record = CustodyChunkChallengeRecord( @@ -264,7 +266,8 @@ def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge def process_chunk_challenge_response(state: BeaconState, response: CustodyChunkResponse) -> None: - challenge = next((record for record in state.custody_chunk_challenge_records if record.challenge_index == response.challenge_index), None) + challenge = next((record for record in state.custody_chunk_challenge_records if + record.challenge_index == response.challenge_index), None) assert(challenge is not None) # Verify chunk index @@ -417,7 +420,8 @@ def process_custody_slashing(state: BeaconState, signed_custody_slashing: Signed shard_transition = custody_slashing.shard_transition assert hash_tree_root(shard_transition) == attestation.data.shard_transition_root # Verify that the provided data matches the shard-transition - assert custody_slashing.data.get_backing().get_left().merkle_root() == shard_transition.shard_data_roots[custody_slashing.data_index] + assert custody_slashing.data.get_backing().get_left().merkle_root() \ + == shard_transition.shard_data_roots[custody_slashing.data_index] assert len(custody_slashing.data) == shard_transition.shard_block_lengths[custody_slashing.data_index] # Verify existence and participation of claimed malefactor @@ -467,7 +471,8 @@ Run `process_reveal_deadlines(state)` after `process_registry_updates(state)`: def process_reveal_deadlines(state: BeaconState) -> None: epoch = get_current_epoch(state) for index, validator in enumerate(state.validators): - if get_custody_period_for_validator(ValidatorIndex(index), epoch) > validator.next_custody_secret_to_reveal + (CUSTODY_RESPONSE_DEADLINE // EPOCHS_PER_CUSTODY_PERIOD): + if get_custody_period_for_validator(ValidatorIndex(index), epoch) > validator.next_custody_secret_to_reveal \ + + (CUSTODY_RESPONSE_DEADLINE // EPOCHS_PER_CUSTODY_PERIOD): slash_validator(state, ValidatorIndex(index)) ``` diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 5e536002f..7f15c0048 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -78,7 +78,8 @@ def build_attestation_data(spec, state, slot, index, shard_transition_root=None) ) -def convert_to_valid_on_time_attestation(spec, state, attestation, signed=False, shard_transition=None, valid_custody_bits=None): +def convert_to_valid_on_time_attestation(spec, state, attestation, signed=False, shard_transition=None, + valid_custody_bits=None): shard = spec.get_shard(state, attestation) offset_slots = spec.compute_offset_slots(spec.get_latest_slot_for_shard(state, shard), state.slot + 1) @@ -88,20 +89,14 @@ def convert_to_valid_on_time_attestation(spec, state, attestation, signed=False, attestation.data.slot, attestation.data.index, ) - current_epoch = spec.get_current_epoch(state) custody_secrets = [None for i in beacon_committee] for i in range(len(beacon_committee)): - validator = state.validators[beacon_committee[i]] - period = spec.get_custody_period_for_validator(beacon_committee[i], attestation.data.target.epoch) - epoch_to_sign = spec.get_randao_epoch_for_custody_period(period, beacon_committee[i]) - domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch_to_sign) signing_root = spec.compute_signing_root(spec.Epoch(epoch_to_sign), domain) custody_secrets[i] = bls.Sign(privkeys[beacon_committee[i]], signing_root) - for i, offset_slot in enumerate(offset_slots): attestation.custody_bits_blocks.append( Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE]([0 for _ in attestation.aggregation_bits]) @@ -110,7 +105,8 @@ def convert_to_valid_on_time_attestation(spec, state, attestation, signed=False, test_vector = get_custody_test_vector(shard_transition.shard_block_lengths[i]) for j in range(len(attestation.custody_bits_blocks[i])): if attestation.aggregation_bits[j]: - attestation.custody_bits_blocks[i][j] = spec.compute_custody_bit(custody_secrets[j], test_vector) ^ (not valid_custody_bits) + attestation.custody_bits_blocks[i][j] = \ + spec.compute_custody_bit(custody_secrets[j], test_vector) ^ (not valid_custody_bits) if signed: sign_attestation(spec, state, attestation) @@ -118,7 +114,8 @@ def convert_to_valid_on_time_attestation(spec, state, attestation, signed=False, return attestation -def get_valid_on_time_attestation(spec, state, slot=None, index=None, signed=False, shard_transition=None, valid_custody_bits=None): +def get_valid_on_time_attestation(spec, state, slot=None, index=None, signed=False, + shard_transition=None, valid_custody_bits=None): ''' Construct on-time attestation for next slot ''' @@ -127,7 +124,9 @@ def get_valid_on_time_attestation(spec, state, slot=None, index=None, signed=Fal if index is None: index = 0 - return get_valid_attestation(spec, state, slot=slot, index=index, signed=signed, on_time=True, shard_transition=shard_transition, valid_custody_bits=valid_custody_bits) + return get_valid_attestation(spec, state, slot=slot, index=index, + signed=signed, on_time=True, shard_transition=shard_transition, + valid_custody_bits=valid_custody_bits) def get_valid_late_attestation(spec, state, slot=None, index=None, signed=False, shard_transition=None): @@ -139,16 +138,19 @@ def get_valid_late_attestation(spec, state, slot=None, index=None, signed=False, if index is None: index = 0 - return get_valid_attestation(spec, state, slot=slot, index=index, signed=signed, on_time=False, shard_transition=shard_transition) + return get_valid_attestation(spec, state, slot=slot, index=index, + signed=signed, on_time=False, shard_transition=shard_transition) -def get_valid_attestation(spec, state, slot=None, index=None, empty=False, signed=False, on_time=True, shard_transition=None, valid_custody_bits=None): +def get_valid_attestation(spec, state, slot=None, index=None, empty=False, signed=False, on_time=True, + shard_transition=None, valid_custody_bits=None): if slot is None: slot = state.slot if index is None: index = 0 - attestation_data = build_attestation_data(spec, state, slot, index, shard_transition_root=hash_tree_root(shard_transition) if shard_transition else spec.Root()) + attestation_data = build_attestation_data(spec, state, slot, index, + shard_transition_root=hash_tree_root(shard_transition) if shard_transition else spec.Root()) beacon_committee = spec.get_beacon_committee( state, @@ -168,7 +170,9 @@ def get_valid_attestation(spec, state, slot=None, index=None, empty=False, signe sign_attestation(spec, state, attestation) if spec.fork == 'phase1' and on_time: - attestation = convert_to_valid_on_time_attestation(spec, state, attestation, signed, shard_transition=shard_transition, valid_custody_bits=valid_custody_bits) + attestation = convert_to_valid_on_time_attestation(spec, state, attestation, signed, + shard_transition=shard_transition, + valid_custody_bits=valid_custody_bits) return attestation diff --git a/tests/core/pyspec/eth2spec/test/helpers/custody.py b/tests/core/pyspec/eth2spec/test/helpers/custody.py index 2d937a2ee..898cf7731 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/custody.py +++ b/tests/core/pyspec/eth2spec/test/helpers/custody.py @@ -1,10 +1,7 @@ from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls -from eth2spec.utils.ssz.ssz_typing import Bitlist, ByteVector, Bitvector, ByteList, uint64 -from eth2spec.utils.ssz.ssz_impl import hash_tree_root -from eth2spec.utils.merkle_minimal import get_merkle_root, get_merkle_tree, get_merkle_proof -from remerkleable.core import pack_bits_to_chunks -from remerkleable.tree import subtree_fill_to_contents, get_depth, Node, Gindex, gindex_bit_iter, Root +from eth2spec.utils.ssz.ssz_typing import Bitlist, ByteVector, ByteList +from remerkleable.tree import gindex_bit_iter BYTES_PER_CHUNK = 32 @@ -79,9 +76,8 @@ def get_valid_custody_slashing(spec, state, attestation, shard_transition, inval signing_root = spec.compute_signing_root(spec.Epoch(epoch), domain) malefactor_key = bls.Sign(privkeys[malefactor_index], signing_root) data_index = 0 - data=ByteList[spec.MAX_SHARD_BLOCK_SIZE](get_custody_test_vector(shard_transition.shard_block_lengths[data_index])) - print(hash_tree_root(data)) - print(data.get_backing().get_left().merkle_root()) + data = ByteList[spec.MAX_SHARD_BLOCK_SIZE]( + get_custody_test_vector(shard_transition.shard_block_lengths[data_index])) slashing = spec.CustodySlashing( data_index=data_index, @@ -93,7 +89,7 @@ def get_valid_custody_slashing(spec, state, attestation, shard_transition, inval data=data, ) slashing_domain = spec.get_domain(state, spec.DOMAIN_CUSTODY_BIT_SLASHING) - slashing_root = spec.compute_signing_root(slashing, domain) + slashing_root = spec.compute_signing_root(slashing, slashing_domain) signed_slashing = spec.SignedCustodySlashing( message=slashing, @@ -103,22 +99,23 @@ def get_valid_custody_slashing(spec, state, attestation, shard_transition, inval return signed_slashing -def get_valid_chunk_challenge(spec, state, attestation, shard_transition): - shard = spec.compute_shard_from_committee_index(state, attestation.data.index, attestation.data.slot) +def get_valid_chunk_challenge(spec, state, attestation, shard_transition, data_index=None, chunk_index=None): crosslink_committee = spec.get_beacon_committee( state, attestation.data.slot, attestation.data.index ) responder_index = crosslink_committee[0] - data_index = len(shard_transition.shard_block_lengths) - 1 + data_index = len(shard_transition.shard_block_lengths) - 1 if not data_index else data_index - chunk_count = (shard_transition.shard_block_lengths[data_index] + spec.BYTES_PER_CUSTODY_CHUNK - 1) // spec.BYTES_PER_CUSTODY_CHUNK + chunk_count = (shard_transition.shard_block_lengths[data_index] + + spec.BYTES_PER_CUSTODY_CHUNK - 1) // spec.BYTES_PER_CUSTODY_CHUNK + chunk_index = chunk_count - 1 if not chunk_index else chunk_index return spec.CustodyChunkChallenge( responder_index=responder_index, attestation=attestation, - chunk_index=chunk_count - 1, + chunk_index=chunk_index, data_index=data_index, shard_transition=shard_transition, ) @@ -174,7 +171,8 @@ def get_custody_test_vector(bytelength): def get_shard_transition(spec, start_slot, block_lengths): - b = [ByteList[spec.MAX_SHARD_BLOCK_SIZE](get_custody_test_vector(x)).get_backing().get_left().merkle_root() for x in block_lengths] + b = [ByteList[spec.MAX_SHARD_BLOCK_SIZE](get_custody_test_vector(x)) + .get_backing().get_left().merkle_root() for x in block_lengths] shard_transition = spec.ShardTransition( start_slot=start_slot, shard_block_lengths=block_lengths, @@ -183,7 +181,3 @@ def get_shard_transition(spec, start_slot, block_lengths): proposer_signature_aggregate=spec.BLSSignature(), ) return shard_transition - - -def get_custody_merkle_root(data): - return None # get_merkle_tree(chunkify(data))[-1][0] diff --git a/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py index 061553ef9..b22a3ea69 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py @@ -1,3 +1,5 @@ +# TODO: What is this file for??? It seems to be broken! +# The phase0 attestations file already adds the custody bit blocks from eth2spec.utils.ssz.ssz_typing import Bitlist from eth2spec.utils import bls @@ -12,16 +14,14 @@ def get_valid_on_time_attestation(spec, state, index=None, signed=False, shard_t if index is None: index = 0 - attestation = phase0_attestations.get_valid_attestation(spec, state, state.slot, index, False, shard_transition_root=shard_transition_root) + attestation = phase0_attestations.get_valid_attestation(spec, state, state.slot, index, False) shard = spec.get_shard(state, attestation) offset_slots = spec.compute_offset_slots(spec.get_latest_slot_for_shard(state, shard), state.slot + 1) - print(offset_slots) for _ in offset_slots: attestation.custody_bits_blocks.append( Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE]([0 for _ in attestation.aggregation_bits]) ) - print(len(attestation.custody_bits_blocks)) if signed: sign_attestation(spec, state, attestation) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py index d409b468f..c71785518 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py @@ -1,15 +1,12 @@ from eth2spec.test.helpers.custody import ( get_valid_chunk_challenge, get_valid_custody_chunk_response, - get_custody_test_vector, - get_custody_merkle_root, get_shard_transition, ) from eth2spec.test.helpers.attestations import ( get_valid_on_time_attestation, ) from eth2spec.test.helpers.state import transition_to -from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.test.context import ( with_all_phases_except, spec_state_test, @@ -30,7 +27,7 @@ def run_chunk_challenge_processing(spec, state, custody_chunk_challenge, valid=T yield 'custody_chunk_challenge', custody_chunk_challenge if not valid: - expect_assertion_error(lambda: spec.custody_chunk_challenge(state, custody_chunk_challenge)) + expect_assertion_error(lambda: spec.process_chunk_challenge(state, custody_chunk_challenge)) yield 'post', None return @@ -74,12 +71,8 @@ def test_challenge_appended(spec, state): shard = 0 offset_slots = spec.get_offset_slots(state, shard) shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) - data_index = 0 - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) - - transition_chunks = (shard_transition.shard_block_lengths[data_index] + spec.BYTES_PER_CUSTODY_CHUNK - 1) // spec.BYTES_PER_CUSTODY_CHUNK - test_vector = get_custody_test_vector(shard_transition.shard_block_lengths[data_index]) - shard_root = get_custody_merkle_root(test_vector) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -92,6 +85,54 @@ def test_challenge_appended(spec, state): yield from run_chunk_challenge_processing(spec, state, challenge) +@with_all_phases_except(['phase0']) +@spec_state_test +def test_duplicate_challenge(spec, state): + transition_to(spec, state, state.slot + 1) + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD) + + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) + + _, _, _ = run_chunk_challenge_processing(spec, state, challenge) + + yield from run_chunk_challenge_processing(spec, state, challenge, valid=False) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_second_challenge(spec, state): + transition_to(spec, state, state.slot + 1) + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD) + + challenge0 = get_valid_chunk_challenge(spec, state, attestation, shard_transition, chunk_index=0) + + _, _, _ = run_chunk_challenge_processing(spec, state, challenge0) + + challenge1 = get_valid_chunk_challenge(spec, state, attestation, shard_transition, chunk_index=1) + + yield from run_chunk_challenge_processing(spec, state, challenge1) + + @with_all_phases_except(['phase0']) @spec_state_test def test_multiple_epochs_custody(spec, state): @@ -100,8 +141,8 @@ def test_multiple_epochs_custody(spec, state): shard = 0 offset_slots = spec.get_offset_slots(state, shard) shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) - data_index = 0 - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -122,8 +163,8 @@ def test_many_epochs_custody(spec, state): shard = 0 offset_slots = spec.get_offset_slots(state, shard) shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) - data_index = 0 - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -144,8 +185,8 @@ def test_off_chain_attestation(spec, state): shard = 0 offset_slots = spec.get_offset_slots(state, shard) shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) - data_index = 0 - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) @@ -162,8 +203,8 @@ def test_custody_response(spec, state): shard = 0 offset_slots = spec.get_offset_slots(state, shard) shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) - data_index = 0 - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -190,8 +231,8 @@ def test_custody_response_multiple_epochs(spec, state): shard = 0 offset_slots = spec.get_offset_slots(state, shard) shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) - data_index = 0 - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -218,8 +259,8 @@ def test_custody_response_many_epochs(spec, state): shard = 0 offset_slots = spec.get_offset_slots(state, shard) shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) - data_index = 0 - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py index 706fd8f49..087e619f5 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py @@ -1,16 +1,12 @@ from eth2spec.test.helpers.custody import ( get_valid_custody_slashing, - get_custody_test_vector, - get_custody_merkle_root, get_shard_transition, ) from eth2spec.test.helpers.attestations import ( get_valid_on_time_attestation, ) -from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ByteList -from eth2spec.test.helpers.state import next_epoch, get_balance, transition_to -from eth2spec.test.helpers.block import apply_empty_block +from eth2spec.test.helpers.state import get_balance, transition_to from eth2spec.test.context import ( with_all_phases_except, spec_state_test, @@ -48,7 +44,7 @@ def run_custody_slashing_processing(spec, state, custody_slashing, valid=True, c else: slashed_validator = state.validators[custody_slashing.message.whistleblower_index] assert get_balance(state, custody_slashing.message.whistleblower_index) < pre_slashed_balance - + assert slashed_validator.slashed assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH @@ -63,8 +59,8 @@ def test_custody_slashing(spec, state): shard = 0 offset_slots = spec.get_offset_slots(state, shard) shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) - data_index = 0 - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition, valid_custody_bits=False) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition, valid_custody_bits=False) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -72,8 +68,6 @@ def test_custody_slashing(spec, state): transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) - data_index = 0 - slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition) yield from run_custody_slashing_processing(spec, state, slashing, correct=True) @@ -86,8 +80,8 @@ def test_incorrect_custody_slashing(spec, state): shard = 0 offset_slots = spec.get_offset_slots(state, shard) shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) - data_index = 0 - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition, valid_custody_bits=True) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition, valid_custody_bits=True) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -95,8 +89,6 @@ def test_incorrect_custody_slashing(spec, state): transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) - data_index = 0 - slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition) yield from run_custody_slashing_processing(spec, state, slashing, correct=False) @@ -109,8 +101,8 @@ def test_multiple_epochs_custody(spec, state): shard = 0 offset_slots = spec.get_offset_slots(state, shard) shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) - data_index = 0 - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition, valid_custody_bits=False) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition, valid_custody_bits=False) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -118,8 +110,6 @@ def test_multiple_epochs_custody(spec, state): transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) - data_index = 0 - slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition) yield from run_custody_slashing_processing(spec, state, slashing, correct=True) @@ -128,12 +118,12 @@ def test_multiple_epochs_custody(spec, state): @with_all_phases_except(['phase0']) @spec_state_test def test_many_epochs_custody(spec, state): - transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH* 100) + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 100) shard = 0 offset_slots = spec.get_offset_slots(state, shard) shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) - data_index = 0 - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition, valid_custody_bits=False) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition, valid_custody_bits=False) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -141,8 +131,6 @@ def test_many_epochs_custody(spec, state): transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) - data_index = 0 - slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition) yield from run_custody_slashing_processing(spec, state, slashing, correct=True) @@ -155,13 +143,11 @@ def test_off_chain_attestation(spec, state): shard = 0 offset_slots = spec.get_offset_slots(state, shard) shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) - data_index = 0 - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition, valid_custody_bits=False) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition, valid_custody_bits=False) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) - data_index = 0 - slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition) yield from run_custody_slashing_processing(spec, state, slashing, correct=True) @@ -174,8 +160,8 @@ def test_invalid_custody_slashing(spec, state): shard = 0 offset_slots = spec.get_offset_slots(state, shard) shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) - data_index = 0 - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition, valid_custody_bits=False) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition, valid_custody_bits=False) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -183,8 +169,6 @@ def test_invalid_custody_slashing(spec, state): transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) - data_index = 0 - slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition) slashing.message.data = ByteList[spec.MAX_SHARD_BLOCK_SIZE]() From b82496fd11a0d2e8dd7a65700889e365beace13b Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Fri, 1 May 2020 00:19:25 +0100 Subject: [PATCH 014/165] Rename file --- ...pdates.py => test_process_custody_final_updates.py} | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) rename tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/{test_process_final_custody_updates.py => test_process_custody_final_updates.py} (96%) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_final_custody_updates.py b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_custody_final_updates.py similarity index 96% rename from tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_final_custody_updates.py rename to tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_custody_final_updates.py index 566d795cf..396414f2a 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_final_custody_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_custody_final_updates.py @@ -2,8 +2,6 @@ from eth2spec.test.helpers.custody import ( get_valid_chunk_challenge, get_valid_custody_chunk_response, get_valid_custody_key_reveal, - get_custody_test_vector, - get_custody_merkle_root, get_shard_transition ) from eth2spec.test.helpers.attestations import ( @@ -72,8 +70,8 @@ def test_validator_withdrawal_suspend_after_chunk_challenge(spec, state): shard = 0 offset_slots = spec.get_offset_slots(state, shard) shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) - data_index = 0 - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -122,8 +120,8 @@ def test_validator_withdrawal_resume_after_chunk_challenge_response(spec, state) shard = 0 offset_slots = spec.get_offset_slots(state, shard) shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) - data_index = 0 - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) From 964bf42335e9880fcc8eb90056ad53b387e316f3 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Fri, 1 May 2020 00:32:02 +0100 Subject: [PATCH 015/165] Fix type --- specs/phase1/custody-game.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 9f5a8909b..c81c6a17b 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -285,7 +285,7 @@ def process_chunk_challenge_response(state: BeaconState, records[records.index(challenge)] = CustodyChunkChallengeRecord() # Reward the proposer proposer_index = get_beacon_proposer_index(state) - increase_balance(state, proposer_index, get_base_reward(state, proposer_index) // MINOR_REWARD_QUOTIENT) + increase_balance(state, proposer_index, Gwei(get_base_reward(state, proposer_index) // MINOR_REWARD_QUOTIENT)) ``` #### Custody key reveals From f4334d1522bb6f3a34fdb0fe612cf4b7c7b0b760 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 5 May 2020 15:28:12 +0800 Subject: [PATCH 016/165] Delete outdated helper --- .../test/helpers/phase1/attestations.py | 65 ------------------- 1 file changed, 65 deletions(-) delete mode 100644 tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py diff --git a/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py deleted file mode 100644 index b22a3ea69..000000000 --- a/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py +++ /dev/null @@ -1,65 +0,0 @@ -# TODO: What is this file for??? It seems to be broken! -# The phase0 attestations file already adds the custody bit blocks -from eth2spec.utils.ssz.ssz_typing import Bitlist -from eth2spec.utils import bls - -from eth2spec.test.helpers.keys import privkeys -import eth2spec.test.helpers.attestations as phase0_attestations - - -def get_valid_on_time_attestation(spec, state, index=None, signed=False, shard_transition_root=None): - ''' - Construct on-time attestation for next slot - ''' - if index is None: - index = 0 - - attestation = phase0_attestations.get_valid_attestation(spec, state, state.slot, index, False) - shard = spec.get_shard(state, attestation) - offset_slots = spec.compute_offset_slots(spec.get_latest_slot_for_shard(state, shard), state.slot + 1) - - for _ in offset_slots: - attestation.custody_bits_blocks.append( - Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE]([0 for _ in attestation.aggregation_bits]) - ) - - if signed: - sign_attestation(spec, state, attestation) - - return attestation - - -def sign_attestation(spec, state, attestation): - if not any(attestation.custody_bits_blocks): - phase0_attestations.sign_attestation(spec, state, attestation) - return - - committee = spec.get_beacon_committee(state, attestation.data.slot, attestation.data.index) - signatures = [] - for block_index, custody_bits in enumerate(attestation.custody_bits_blocks): - for participant, abit, cbit in zip(committee, attestation.aggregation_bits, custody_bits): - if not abit: - continue - signatures.append(get_attestation_custody_signature( - spec, - state, - attestation.data, - block_index, - cbit, - privkeys[participant] - )) - - attestation.signature = bls.Aggregate(signatures) - - -def get_attestation_custody_signature(spec, state, attestation_data, block_index, bit, privkey): - domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) - signing_root = spec.compute_signing_root( - spec.AttestationCustodyBitWrapper( - attestation_data.hash_tree_root(), - block_index, - bit, - ), - domain, - ) - return bls.Sign(privkey, signing_root) From dab5a936c4b1fdfae0cc9c9b5c98ac0643097bd7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 28 Apr 2020 23:55:46 +0800 Subject: [PATCH 017/165] wip shard fork choice rule --- setup.py | 1 + specs/phase1/shard-fork-choice.md | 209 ++++++++++++++++++ .../test/fork_choice/test_get_head.py | 30 +-- .../test/fork_choice/test_on_shard_head.py | 97 ++++++++ .../eth2spec/test/helpers/fork_choice.py | 27 +++ 5 files changed, 335 insertions(+), 29 deletions(-) create mode 100644 specs/phase1/shard-fork-choice.md create mode 100644 tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py create mode 100644 tests/core/pyspec/eth2spec/test/helpers/fork_choice.py diff --git a/setup.py b/setup.py index e0d6561dd..316a7d32a 100644 --- a/setup.py +++ b/setup.py @@ -378,6 +378,7 @@ class PySpecCommand(Command): specs/phase1/shard-transition.md specs/phase1/fork-choice.md specs/phase1/phase1-fork.md + specs/phase1/shard-fork-choice.md """ else: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md new file mode 100644 index 000000000..46e467e12 --- /dev/null +++ b/specs/phase1/shard-fork-choice.md @@ -0,0 +1,209 @@ +# Ethereum 2.0 Phase 1 -- Beacon Chain + Shard Chain Fork Choice + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + +- [Introduction](#introduction) +- [Fork choice](#fork-choice) + - [Helpers](#helpers) + - [Extended `Store`](#extended-store) + - [Updated `get_forkchoice_store`](#updated-get_forkchoice_store) + - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance) + - [`get_shard_head`](#get_shard_head) + - [`get_shard_ancestor`](#get_shard_ancestor) + - [`filter_shard_block_tree`](#filter_shard_block_tree) + - [`get_filtered_block_tree`](#get_filtered_block_tree) + - [Handlers](#handlers) + - [`on_shard_block`](#on_shard_block) + + + +## Introduction + +This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase 1. + +## Fork choice + +### Helpers + +#### Extended `Store` + +```python +@dataclass +class Store(object): + time: uint64 + genesis_time: uint64 + justified_checkpoint: Checkpoint + finalized_checkpoint: Checkpoint + best_justified_checkpoint: Checkpoint + blocks: Dict[Root, BeaconBlock] = field(default_factory=dict) + block_states: Dict[Root, BeaconState] = field(default_factory=dict) + checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) + latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) + # shard chain + shard_init_slots: Dict[Shard, Slot] = field(default_factory=dict) + shard_blocks: Dict[Shard, Dict[Root, ShardBlock]] = field(default_factory=dict) + shard_block_states: Dict[Shard, Dict[Root, ShardState]] = field(default_factory=dict) +``` + +#### Updated `get_forkchoice_store` + +```python +def get_forkchoice_store(anchor_state: BeaconState, + shard_init_slots: Dict[Shard, Slot], + anchor_state_shard_blocks: Dict[Shard, Dict[Root, ShardBlock]]) -> Store: + shard_count = len(anchor_state.shard_states) + anchor_block_header = anchor_state.latest_block_header.copy() + if anchor_block_header.state_root == Bytes32(): + anchor_block_header.state_root = hash_tree_root(anchor_state) + anchor_root = hash_tree_root(anchor_block_header) + anchor_epoch = get_current_epoch(anchor_state) + justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + return Store( + time=anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot, + genesis_time=anchor_state.genesis_time, + justified_checkpoint=justified_checkpoint, + finalized_checkpoint=finalized_checkpoint, + best_justified_checkpoint=justified_checkpoint, + blocks={anchor_root: anchor_block_header}, + block_states={anchor_root: anchor_state.copy()}, + checkpoint_states={justified_checkpoint: anchor_state.copy()}, + # shard chain + shard_init_slots=shard_init_slots, + shard_blocks=anchor_state_shard_blocks, + shard_block_states={ + shard: { + anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard] + } + for shard in map(Shard, range(shard_count)) + }, + ) +``` + +#### `get_shard_latest_attesting_balance` + +```python +def get_shard_latest_attesting_balance(store: Store, shard: Shard, root: Root) -> Gwei: + state = store.checkpoint_states[store.justified_checkpoint] + active_indices = get_active_validator_indices(state, get_current_epoch(state)) + return Gwei(sum( + state.validators[i].effective_balance for i in active_indices + if ( + i in store.latest_messages and get_shard_ancestor( + store, shard, store.latest_messages[i].root, store.shard_blocks[shard][root].slot + ) == root + ) + )) +``` + +#### `get_shard_head` + +```python +def get_shard_head(store: Store, shard: Shard) -> Root: + # Get filtered block tree that only includes viable branches + blocks = get_filtered_shard_block_tree(store, shard) + + # Execute the LMD-GHOST fork choice + head_beacon_root = get_head(store) + head_shard_root = store.block_states[head_beacon_root].shard_states[shard].latest_block_root + while True: + children = [ + root for root in blocks.keys() + if blocks[root].shard_parent_root == head_shard_root + ] + if len(children) == 0: + return head_shard_root + # Sort by latest attesting balance with ties broken lexicographically + head_shard_root = max(children, key=lambda root: (get_shard_latest_attesting_balance(store, shard, root), root)) +``` + +#### `get_shard_ancestor` + +```python +def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Root: + block = store.shard_blocks[shard][root] + if block.slot > slot: + return get_shard_ancestor(store, shard, block.shard_parent_root, slot) + elif block.slot == slot: + return root + else: + # root is older than queried slot, thus a skip slot. Return earliest root prior to slot + return root +``` + +#### `filter_shard_block_tree` + +```python +def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks: Dict[Root, ShardBlock]) -> bool: + block = store.shard_blocks[shard][block_root] + children = [ + root for root in store.shard_blocks[shard].keys() + if ( + store.shard_blocks[shard][root].shard_parent_root == block_root + and store.shard_blocks[shard][root].slot != store.shard_init_slots[shard] + ) + ] + + if any(children): + filter_block_tree_result = [filter_shard_block_tree(store, shard, child, blocks) for child in children] + if any(filter_block_tree_result): + blocks[block_root] = block + return True + return False + + return False +``` + +#### `get_filtered_block_tree` + +```python +def get_filtered_shard_block_tree(store: Store, shard: Shard) -> Dict[Root, ShardBlock]: + base_beacon_block_root = get_head(store) + base_shard_block_root = store.block_states[base_beacon_block_root].shard_states[shard].latest_block_root + blocks: Dict[Root, ShardBlock] = {} + filter_shard_block_tree(store, shard, base_shard_block_root, blocks) + return blocks +``` + +### Handlers + +#### `on_shard_block` + +```python +def on_shard_block(store: Store, shard: Shard, signed_shard_block: SignedShardBlock) -> None: + shard_block = signed_shard_block.message + + # 1. Check shard parent exists + assert shard_block.shard_parent_root in store.shard_block_states[shard] + pre_shard_state = store.shard_block_states[shard][shard_block.shard_parent_root] + + # 2. Check beacon parent exists + assert shard_block.beacon_parent_root in store.block_states + beacon_state = store.block_states[shard_block.beacon_parent_root] + + # 3. Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert shard_block.slot > finalized_slot + + # 4. Check block is a descendant of the finalized block at the checkpoint finalized slot + assert ( + shard_block.beacon_parent_root == store.finalized_checkpoint.root + or get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root + ) + + # Add new block to the store + store.shard_blocks[shard][hash_tree_root(shard_block)] = shard_block + + # Check the block is valid and compute the post-state + verify_shard_block_message(beacon_state, pre_shard_state, shard_block, shard_block.slot, shard) + verify_shard_block_signature(beacon_state, signed_shard_block) + post_state = get_post_shard_state(beacon_state, pre_shard_state, shard_block) + # Add new state for this block to the store + store.shard_block_states[shard][hash_tree_root(shard_block)] = post_state +``` diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py index 17d4f644f..e25aad18f 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py @@ -1,41 +1,13 @@ from eth2spec.test.context import with_all_phases, spec_state_test from eth2spec.test.helpers.attestations import get_valid_attestation, next_epoch_with_attestations from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.helpers.fork_choice import add_attestation_to_store, add_block_to_store, get_anchor_root from eth2spec.test.helpers.state import ( next_epoch, state_transition_and_sign_block, ) -def add_block_to_store(spec, store, signed_block): - pre_state = store.block_states[signed_block.message.parent_root] - block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT - - if store.time < block_time: - spec.on_tick(store, block_time) - - spec.on_block(store, signed_block) - - -def add_attestation_to_store(spec, store, attestation): - parent_block = store.blocks[attestation.data.beacon_block_root] - pre_state = store.block_states[spec.hash_tree_root(parent_block)] - block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT - next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT - - if store.time < next_epoch_time: - spec.on_tick(store, next_epoch_time) - - spec.on_attestation(store, attestation) - - -def get_anchor_root(spec, state): - anchor_block_header = state.latest_block_header.copy() - if anchor_block_header.state_root == spec.Bytes32(): - anchor_block_header.state_root = spec.hash_tree_root(state) - return spec.hash_tree_root(anchor_block_header) - - @with_all_phases @spec_state_test def test_genesis(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py new file mode 100644 index 000000000..8e72e214e --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -0,0 +1,97 @@ +from eth2spec.utils.ssz.ssz_impl import hash_tree_root + +from eth2spec.test.context import spec_state_test, with_all_phases_except, PHASE0 +from eth2spec.test.helpers.shard_block import ( + build_attestation_with_shard_transition, + build_shard_block, + build_shard_transitions_till_slot, +) +from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root +from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_block +from eth2spec.test.helpers.block import build_empty_block + + +def run_on_shard_block(spec, store, shard, signed_block, valid=True): + if not valid: + try: + spec.on_shard_block(store, shard, signed_block) + except AssertionError: + return + else: + assert False + + spec.on_shard_block(store, shard, signed_block) + assert store.shard_blocks[shard][hash_tree_root(signed_block.message)] == signed_block.message + + +def run_apply_shard_and_beacon(spec, state, store, shard, committee_index): + store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + + # Create SignedShardBlock + body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE + shard_block = build_shard_block(spec, state, shard, body=body, signed=True) + shard_blocks = [shard_block] + + # Attester creates `attestation` + # Use temporary next state to get ShardTransition of shard block + shard_transitions = build_shard_transitions_till_slot( + spec, + state, + shards=[shard, ], + shard_blocks={shard: shard_blocks}, + target_len_offset_slot=1, + ) + shard_transition = shard_transitions[shard] + attestation = build_attestation_with_shard_transition( + spec, + state, + slot=state.slot, + index=committee_index, + target_len_offset_slot=1, + shard_transition=shard_transition, + ) + + # Propose beacon block at slot + beacon_block = build_empty_block(spec, state, slot=state.slot + 1) + beacon_block.body.attestations = [attestation] + beacon_block.body.shard_transitions = shard_transitions + signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) + + run_on_shard_block(spec, store, shard, shard_block) + add_block_to_store(spec, store, signed_beacon_block) + + assert spec.get_head(store) == beacon_block.hash_tree_root() + assert spec.get_shard_head(store, shard) == shard_block.message.hash_tree_root() + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_basic(spec, state): + spec.PHASE_1_GENESIS_SLOT = 0 # FIXME: remove mocking + state = spec.upgrade_to_phase1(state) + next_slot(spec, state) + + # Initialization + shard_count = len(state.shard_states) + # Genesis shard blocks + anchor_shard_blocks = { + shard: { + state.shard_states[shard].latest_block_root: spec.ShardBlock( + slot=state.slot, + ) + } + for shard in map(spec.Shard, range(shard_count)) + } + shard_init_slots = { + shard: state.slot + for shard in map(spec.Shard, range(shard_count)) + } + store = spec.get_forkchoice_store(state, shard_init_slots, anchor_shard_blocks) + anchor_root = get_anchor_root(spec, state) + assert spec.get_head(store) == anchor_root + + committee_index = spec.CommitteeIndex(0) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + + run_apply_shard_and_beacon(spec, state, store, shard, committee_index) + run_apply_shard_and_beacon(spec, state, store, shard, committee_index) diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py new file mode 100644 index 000000000..04e36ea84 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py @@ -0,0 +1,27 @@ +def get_anchor_root(spec, state): + anchor_block_header = state.latest_block_header.copy() + if anchor_block_header.state_root == spec.Bytes32(): + anchor_block_header.state_root = spec.hash_tree_root(state) + return spec.hash_tree_root(anchor_block_header) + + +def add_block_to_store(spec, store, signed_block): + pre_state = store.block_states[signed_block.message.parent_root] + block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT + + if store.time < block_time: + spec.on_tick(store, block_time) + + spec.on_block(store, signed_block) + + +def add_attestation_to_store(spec, store, attestation): + parent_block = store.blocks[attestation.data.beacon_block_root] + pre_state = store.block_states[spec.hash_tree_root(parent_block)] + block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT + next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT + + if store.time < next_epoch_time: + spec.on_tick(store, next_epoch_time) + + spec.on_attestation(store, attestation) From cddf9cf114744c07d8a1ce455c5cf73ff5ad89db Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 30 Apr 2020 20:06:20 +0800 Subject: [PATCH 018/165] Refactor --- specs/phase1/shard-fork-choice.md | 56 ++++++++++--------- .../test/fork_choice/test_on_shard_head.py | 18 +----- 2 files changed, 31 insertions(+), 43 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 46e467e12..e9026b991 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -35,7 +35,13 @@ This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase ```python @dataclass -class Store(object): +class Store: + + @dataclass + class ShardStore: + blocks: Dict[Root, ShardBlock] = field(default_factory=dict) + block_states: Dict[Root, ShardState] = field(default_factory=dict) + time: uint64 genesis_time: uint64 justified_checkpoint: Checkpoint @@ -46,17 +52,13 @@ class Store(object): checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) # shard chain - shard_init_slots: Dict[Shard, Slot] = field(default_factory=dict) - shard_blocks: Dict[Shard, Dict[Root, ShardBlock]] = field(default_factory=dict) - shard_block_states: Dict[Shard, Dict[Root, ShardState]] = field(default_factory=dict) + shards: Dict[Shard, ShardStore] = field(default_factory=dict) # noqa: F821 ``` #### Updated `get_forkchoice_store` ```python -def get_forkchoice_store(anchor_state: BeaconState, - shard_init_slots: Dict[Shard, Slot], - anchor_state_shard_blocks: Dict[Shard, Dict[Root, ShardBlock]]) -> Store: +def get_forkchoice_store(anchor_state: BeaconState) -> Store: shard_count = len(anchor_state.shard_states) anchor_block_header = anchor_state.latest_block_header.copy() if anchor_block_header.state_root == Bytes32(): @@ -65,6 +67,14 @@ def get_forkchoice_store(anchor_state: BeaconState, anchor_epoch = get_current_epoch(anchor_state) justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + + shard_stores = {} + for shard in map(Shard, range(shard_count)): + shard_stores[shard] = Store.ShardStore( + blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot)}, + block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]}, + ) + return Store( time=anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot, genesis_time=anchor_state.genesis_time, @@ -75,14 +85,7 @@ def get_forkchoice_store(anchor_state: BeaconState, block_states={anchor_root: anchor_state.copy()}, checkpoint_states={justified_checkpoint: anchor_state.copy()}, # shard chain - shard_init_slots=shard_init_slots, - shard_blocks=anchor_state_shard_blocks, - shard_block_states={ - shard: { - anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard] - } - for shard in map(Shard, range(shard_count)) - }, + shards=shard_stores, ) ``` @@ -96,7 +99,7 @@ def get_shard_latest_attesting_balance(store: Store, shard: Shard, root: Root) - state.validators[i].effective_balance for i in active_indices if ( i in store.latest_messages and get_shard_ancestor( - store, shard, store.latest_messages[i].root, store.shard_blocks[shard][root].slot + store, shard, store.latest_messages[i].root, store.shards[shard].blocks[root].slot ) == root ) )) @@ -127,7 +130,7 @@ def get_shard_head(store: Store, shard: Shard) -> Root: ```python def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Root: - block = store.shard_blocks[shard][root] + block = store.shards[shard].blocks[root] if block.slot > slot: return get_shard_ancestor(store, shard, block.shard_parent_root, slot) elif block.slot == slot: @@ -141,13 +144,11 @@ def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Ro ```python def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks: Dict[Root, ShardBlock]) -> bool: - block = store.shard_blocks[shard][block_root] + shard_store = store.shards[shard] + block = shard_store.blocks[block_root] children = [ - root for root in store.shard_blocks[shard].keys() - if ( - store.shard_blocks[shard][root].shard_parent_root == block_root - and store.shard_blocks[shard][root].slot != store.shard_init_slots[shard] - ) + root for root in shard_store.blocks.keys() + if shard_store.blocks[root].shard_parent_root == block_root ] if any(children): @@ -178,10 +179,11 @@ def get_filtered_shard_block_tree(store: Store, shard: Shard) -> Dict[Root, Shar ```python def on_shard_block(store: Store, shard: Shard, signed_shard_block: SignedShardBlock) -> None: shard_block = signed_shard_block.message + shard_store = store.shards[shard] # 1. Check shard parent exists - assert shard_block.shard_parent_root in store.shard_block_states[shard] - pre_shard_state = store.shard_block_states[shard][shard_block.shard_parent_root] + assert shard_block.shard_parent_root in shard_store.block_states + pre_shard_state = shard_store.block_states[shard_block.shard_parent_root] # 2. Check beacon parent exists assert shard_block.beacon_parent_root in store.block_states @@ -198,12 +200,12 @@ def on_shard_block(store: Store, shard: Shard, signed_shard_block: SignedShardBl ) # Add new block to the store - store.shard_blocks[shard][hash_tree_root(shard_block)] = shard_block + shard_store.blocks[hash_tree_root(shard_block)] = shard_block # Check the block is valid and compute the post-state verify_shard_block_message(beacon_state, pre_shard_state, shard_block, shard_block.slot, shard) verify_shard_block_signature(beacon_state, signed_shard_block) post_state = get_post_shard_state(beacon_state, pre_shard_state, shard_block) # Add new state for this block to the store - store.shard_block_states[shard][hash_tree_root(shard_block)] = post_state + shard_store.block_states[hash_tree_root(shard_block)] = post_state ``` diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 8e72e214e..220c510e7 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -21,7 +21,7 @@ def run_on_shard_block(spec, store, shard, signed_block, valid=True): assert False spec.on_shard_block(store, shard, signed_block) - assert store.shard_blocks[shard][hash_tree_root(signed_block.message)] == signed_block.message + assert store.shards[shard].blocks[hash_tree_root(signed_block.message)] == signed_block.message def run_apply_shard_and_beacon(spec, state, store, shard, committee_index): @@ -72,21 +72,7 @@ def test_basic(spec, state): next_slot(spec, state) # Initialization - shard_count = len(state.shard_states) - # Genesis shard blocks - anchor_shard_blocks = { - shard: { - state.shard_states[shard].latest_block_root: spec.ShardBlock( - slot=state.slot, - ) - } - for shard in map(spec.Shard, range(shard_count)) - } - shard_init_slots = { - shard: state.slot - for shard in map(spec.Shard, range(shard_count)) - } - store = spec.get_forkchoice_store(state, shard_init_slots, anchor_shard_blocks) + store = spec.get_forkchoice_store(state) anchor_root = get_anchor_root(spec, state) assert spec.get_head(store) == anchor_root From 8fafb6a9e5b17ee3e93fa0c1b3a8795608f7336b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 1 May 2020 10:25:58 +0800 Subject: [PATCH 019/165] Make `ShardStore` an independent object --- specs/phase1/shard-fork-choice.md | 97 ++++++------------- .../test/fork_choice/test_on_shard_head.py | 20 ++-- 2 files changed, 43 insertions(+), 74 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index e9026b991..4b3f42194 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -31,75 +31,38 @@ This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase ### Helpers -#### Extended `Store` +#### `ShardStore` ```python @dataclass -class Store: - - @dataclass - class ShardStore: - blocks: Dict[Root, ShardBlock] = field(default_factory=dict) - block_states: Dict[Root, ShardState] = field(default_factory=dict) - - time: uint64 - genesis_time: uint64 - justified_checkpoint: Checkpoint - finalized_checkpoint: Checkpoint - best_justified_checkpoint: Checkpoint - blocks: Dict[Root, BeaconBlock] = field(default_factory=dict) - block_states: Dict[Root, BeaconState] = field(default_factory=dict) - checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) - latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) - # shard chain - shards: Dict[Shard, ShardStore] = field(default_factory=dict) # noqa: F821 +class ShardStore: + shard: Shard + blocks: Dict[Root, ShardBlock] = field(default_factory=dict) + block_states: Dict[Root, ShardState] = field(default_factory=dict) ``` -#### Updated `get_forkchoice_store` +#### Updated `get_forkchoice_shard_store` ```python -def get_forkchoice_store(anchor_state: BeaconState) -> Store: - shard_count = len(anchor_state.shard_states) - anchor_block_header = anchor_state.latest_block_header.copy() - if anchor_block_header.state_root == Bytes32(): - anchor_block_header.state_root = hash_tree_root(anchor_state) - anchor_root = hash_tree_root(anchor_block_header) - anchor_epoch = get_current_epoch(anchor_state) - justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) - finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) - - shard_stores = {} - for shard in map(Shard, range(shard_count)): - shard_stores[shard] = Store.ShardStore( - blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot)}, - block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]}, - ) - - return Store( - time=anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot, - genesis_time=anchor_state.genesis_time, - justified_checkpoint=justified_checkpoint, - finalized_checkpoint=finalized_checkpoint, - best_justified_checkpoint=justified_checkpoint, - blocks={anchor_root: anchor_block_header}, - block_states={anchor_root: anchor_state.copy()}, - checkpoint_states={justified_checkpoint: anchor_state.copy()}, - # shard chain - shards=shard_stores, +def get_forkchoice_shard_store(anchor_state: BeaconState, shard: Shard) -> ShardStore: + return ShardStore( + shard=shard, + blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot)}, + block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]}, ) ``` #### `get_shard_latest_attesting_balance` ```python -def get_shard_latest_attesting_balance(store: Store, shard: Shard, root: Root) -> Gwei: +def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, root: Root) -> Gwei: state = store.checkpoint_states[store.justified_checkpoint] active_indices = get_active_validator_indices(state, get_current_epoch(state)) return Gwei(sum( state.validators[i].effective_balance for i in active_indices if ( i in store.latest_messages and get_shard_ancestor( - store, shard, store.latest_messages[i].root, store.shards[shard].blocks[root].slot + store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot ) == root ) )) @@ -108,13 +71,13 @@ def get_shard_latest_attesting_balance(store: Store, shard: Shard, root: Root) - #### `get_shard_head` ```python -def get_shard_head(store: Store, shard: Shard) -> Root: +def get_shard_head(store: Store, shard_store: ShardStore) -> Root: # Get filtered block tree that only includes viable branches - blocks = get_filtered_shard_block_tree(store, shard) + blocks = get_filtered_shard_block_tree(store, shard_store) # Execute the LMD-GHOST fork choice head_beacon_root = get_head(store) - head_shard_root = store.block_states[head_beacon_root].shard_states[shard].latest_block_root + head_shard_root = store.block_states[head_beacon_root].shard_states[shard_store.shard].latest_block_root while True: children = [ root for root in blocks.keys() @@ -123,16 +86,18 @@ def get_shard_head(store: Store, shard: Shard) -> Root: if len(children) == 0: return head_shard_root # Sort by latest attesting balance with ties broken lexicographically - head_shard_root = max(children, key=lambda root: (get_shard_latest_attesting_balance(store, shard, root), root)) + head_shard_root = max( + children, key=lambda root: (get_shard_latest_attesting_balance(store, shard_store, root), root) + ) ``` #### `get_shard_ancestor` ```python -def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Root: - block = store.shards[shard].blocks[root] +def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: Slot) -> Root: + block = shard_store.blocks[root] if block.slot > slot: - return get_shard_ancestor(store, shard, block.shard_parent_root, slot) + return get_shard_ancestor(store, shard_store, block.shard_parent_root, slot) elif block.slot == slot: return root else: @@ -143,8 +108,10 @@ def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Ro #### `filter_shard_block_tree` ```python -def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks: Dict[Root, ShardBlock]) -> bool: - shard_store = store.shards[shard] +def filter_shard_block_tree(store: Store, + shard_store: ShardStore, + block_root: Root, + blocks: Dict[Root, ShardBlock]) -> bool: block = shard_store.blocks[block_root] children = [ root for root in shard_store.blocks.keys() @@ -152,7 +119,7 @@ def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks ] if any(children): - filter_block_tree_result = [filter_shard_block_tree(store, shard, child, blocks) for child in children] + filter_block_tree_result = [filter_shard_block_tree(store, shard_store, child, blocks) for child in children] if any(filter_block_tree_result): blocks[block_root] = block return True @@ -164,11 +131,12 @@ def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks #### `get_filtered_block_tree` ```python -def get_filtered_shard_block_tree(store: Store, shard: Shard) -> Dict[Root, ShardBlock]: +def get_filtered_shard_block_tree(store: Store, shard_store: ShardStore) -> Dict[Root, ShardBlock]: + shard = shard_store.shard base_beacon_block_root = get_head(store) base_shard_block_root = store.block_states[base_beacon_block_root].shard_states[shard].latest_block_root blocks: Dict[Root, ShardBlock] = {} - filter_shard_block_tree(store, shard, base_shard_block_root, blocks) + filter_shard_block_tree(store, shard_store, base_shard_block_root, blocks) return blocks ``` @@ -177,10 +145,9 @@ def get_filtered_shard_block_tree(store: Store, shard: Shard) -> Dict[Root, Shar #### `on_shard_block` ```python -def on_shard_block(store: Store, shard: Shard, signed_shard_block: SignedShardBlock) -> None: +def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: SignedShardBlock) -> None: shard_block = signed_shard_block.message - shard_store = store.shards[shard] - + shard = shard_store.shard # 1. Check shard parent exists assert shard_block.shard_parent_root in shard_store.block_states pre_shard_state = shard_store.block_states[shard_block.shard_parent_root] diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 220c510e7..f4b883f06 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -11,20 +11,21 @@ from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_blo from eth2spec.test.helpers.block import build_empty_block -def run_on_shard_block(spec, store, shard, signed_block, valid=True): +def run_on_shard_block(spec, store, shard_store, signed_block, valid=True): if not valid: try: - spec.on_shard_block(store, shard, signed_block) + spec.on_shard_block(store, shard_store, signed_block) except AssertionError: return else: assert False - spec.on_shard_block(store, shard, signed_block) - assert store.shards[shard].blocks[hash_tree_root(signed_block.message)] == signed_block.message + spec.on_shard_block(store, shard_store, signed_block) + assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message -def run_apply_shard_and_beacon(spec, state, store, shard, committee_index): +def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index): + shard = shard_store.shard store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH # Create SignedShardBlock @@ -57,11 +58,11 @@ def run_apply_shard_and_beacon(spec, state, store, shard, committee_index): beacon_block.body.shard_transitions = shard_transitions signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) - run_on_shard_block(spec, store, shard, shard_block) + run_on_shard_block(spec, store, shard_store, shard_block) add_block_to_store(spec, store, signed_beacon_block) assert spec.get_head(store) == beacon_block.hash_tree_root() - assert spec.get_shard_head(store, shard) == shard_block.message.hash_tree_root() + assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() @with_all_phases_except([PHASE0]) @@ -78,6 +79,7 @@ def test_basic(spec, state): committee_index = spec.CommitteeIndex(0) shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard_store = spec.get_forkchoice_shard_store(state, shard) - run_apply_shard_and_beacon(spec, state, store, shard, committee_index) - run_apply_shard_and_beacon(spec, state, store, shard, committee_index) + run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) + run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) From fca1bbccb948b132fef22a9732c2f0a6aef0b2a4 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 1 May 2020 10:33:23 +0800 Subject: [PATCH 020/165] Remove `get_filtered_shard_block_tree` --- specs/phase1/shard-fork-choice.md | 50 ++++--------------------------- 1 file changed, 5 insertions(+), 45 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 4b3f42194..b026aabfc 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -11,13 +11,11 @@ - [Introduction](#introduction) - [Fork choice](#fork-choice) - [Helpers](#helpers) - - [Extended `Store`](#extended-store) - - [Updated `get_forkchoice_store`](#updated-get_forkchoice_store) + - [`ShardStore`](#shardstore) + - [`get_forkchoice_shard_store`](#get_forkchoice_shard_store) - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance) - [`get_shard_head`](#get_shard_head) - [`get_shard_ancestor`](#get_shard_ancestor) - - [`filter_shard_block_tree`](#filter_shard_block_tree) - - [`get_filtered_block_tree`](#get_filtered_block_tree) - [Handlers](#handlers) - [`on_shard_block`](#on_shard_block) @@ -41,7 +39,7 @@ class ShardStore: block_states: Dict[Root, ShardState] = field(default_factory=dict) ``` -#### Updated `get_forkchoice_shard_store` +#### `get_forkchoice_shard_store` ```python def get_forkchoice_shard_store(anchor_state: BeaconState, shard: Shard) -> ShardStore: @@ -72,16 +70,13 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro ```python def get_shard_head(store: Store, shard_store: ShardStore) -> Root: - # Get filtered block tree that only includes viable branches - blocks = get_filtered_shard_block_tree(store, shard_store) - # Execute the LMD-GHOST fork choice head_beacon_root = get_head(store) head_shard_root = store.block_states[head_beacon_root].shard_states[shard_store.shard].latest_block_root while True: children = [ - root for root in blocks.keys() - if blocks[root].shard_parent_root == head_shard_root + root for root in shard_store.blocks.keys() + if shard_store.blocks[root].shard_parent_root == head_shard_root ] if len(children) == 0: return head_shard_root @@ -105,41 +100,6 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: return root ``` -#### `filter_shard_block_tree` - -```python -def filter_shard_block_tree(store: Store, - shard_store: ShardStore, - block_root: Root, - blocks: Dict[Root, ShardBlock]) -> bool: - block = shard_store.blocks[block_root] - children = [ - root for root in shard_store.blocks.keys() - if shard_store.blocks[root].shard_parent_root == block_root - ] - - if any(children): - filter_block_tree_result = [filter_shard_block_tree(store, shard_store, child, blocks) for child in children] - if any(filter_block_tree_result): - blocks[block_root] = block - return True - return False - - return False -``` - -#### `get_filtered_block_tree` - -```python -def get_filtered_shard_block_tree(store: Store, shard_store: ShardStore) -> Dict[Root, ShardBlock]: - shard = shard_store.shard - base_beacon_block_root = get_head(store) - base_shard_block_root = store.block_states[base_beacon_block_root].shard_states[shard].latest_block_root - blocks: Dict[Root, ShardBlock] = {} - filter_shard_block_tree(store, shard_store, base_shard_block_root, blocks) - return blocks -``` - ### Handlers #### `on_shard_block` From 79b1b4bdbe0c4822d4a2e197a7bf59930ffe69c1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 1 May 2020 10:52:45 +0800 Subject: [PATCH 021/165] Add `(shard, shard_root)` to `LatestMessage` --- specs/phase1/fork-choice.md | 32 ++++++++++++++++++- .../test/fork_choice/test_on_attestation.py | 13 ++++++-- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/specs/phase1/fork-choice.md b/specs/phase1/fork-choice.md index d8bf7fa09..f4e771ddb 100644 --- a/specs/phase1/fork-choice.md +++ b/specs/phase1/fork-choice.md @@ -10,6 +10,9 @@ - [Introduction](#introduction) - [Fork choice](#fork-choice) + - [Helpers](#helpers) + - [Extended `LatestMessage`](#extended-latestmessage) + - [Updated `update_latest_messages`](#updated-update_latest_messages) - [Handlers](#handlers) @@ -25,6 +28,33 @@ Due to the changes in the structure of `IndexedAttestation` in Phase 1, `on_atte The rest of the fork choice remains stable. +### Helpers + +#### Extended `LatestMessage` + +```python +@dataclass(eq=True, frozen=True) +class LatestMessage(object): + epoch: Epoch + root: Root + shard: Shard + shard_root: Root +``` + +#### Updated `update_latest_messages` + +```python +def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIndex], attestation: Attestation) -> None: + target = attestation.data.target + beacon_block_root = attestation.data.beacon_block_root + shard = get_shard(store.block_states[beacon_block_root], attestation) + for i in attesting_indices: + if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: + store.latest_messages[i] = LatestMessage( + epoch=target.epoch, root=beacon_block_root, shard=shard, shard_root=attestation.data.head_shard_root + ) +``` + ### Handlers ```python @@ -49,4 +79,4 @@ def on_attestation(store: Store, attestation: Attestation) -> None: if attestation.aggregation_bits[i] ] update_latest_messages(store, attesting_indices, attestation) -``` \ No newline at end of file +``` diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 360c18ccd..1bce42ca3 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -18,18 +18,25 @@ def run_on_attestation(spec, state, store, attestation, valid=True): if spec.fork == PHASE0: sample_index = indexed_attestation.attesting_indices[0] + latest_message = spec.LatestMessage( + epoch=attestation.data.target.epoch, + root=attestation.data.beacon_block_root, + ) else: attesting_indices = [ index for i, index in enumerate(indexed_attestation.committee) if attestation.aggregation_bits[i] ] sample_index = attesting_indices[0] - assert ( - store.latest_messages[sample_index] == - spec.LatestMessage( + latest_message = spec.LatestMessage( epoch=attestation.data.target.epoch, root=attestation.data.beacon_block_root, + shard=spec.get_shard(state, attestation), + shard_root=attestation.data.head_shard_root, ) + + assert ( + store.latest_messages[sample_index] == latest_message ) From b1c2c6e3a220c6074a9189433fa04507fed3e00b Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 11 May 2020 19:18:49 +0200 Subject: [PATCH 022/165] Default BLS to ON, keep CI BLS off for now, add milagro option --- Makefile | 6 ++-- setup.py | 1 + tests/core/pyspec/README.md | 5 +++ tests/core/pyspec/eth2spec/test/conftest.py | 35 ++++++++++++++++++--- tests/core/pyspec/eth2spec/test/context.py | 5 +-- tests/core/pyspec/eth2spec/utils/bls.py | 17 +++++++--- 6 files changed, 55 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index e53aaf8a2..f52ef050d 100644 --- a/Makefile +++ b/Makefile @@ -75,15 +75,15 @@ install_test: test: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python -m pytest -n 4 --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec + python -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec find_test: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python -m pytest -k=$(K) --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec + python -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec citest: pyspec mkdir -p tests/core/pyspec/test-reports/eth2spec; . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python -m pytest -n 4 --junitxml=eth2spec/test_results.xml eth2spec + python -m pytest -n 4 --disable-bls --junitxml=eth2spec/test_results.xml eth2spec open_cov: ((open "$(COV_INDEX_FILE)" || xdg-open "$(COV_INDEX_FILE)") &> /dev/null) & diff --git a/setup.py b/setup.py index 37f3c16ef..16ae6ac5c 100644 --- a/setup.py +++ b/setup.py @@ -502,6 +502,7 @@ setup( "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.9.4", "py_ecc==2.0.0", + "milagro_bls_binding==1.0.2", "dataclasses==0.6", "remerkleable==0.1.13", "ruamel.yaml==0.16.5", diff --git a/tests/core/pyspec/README.md b/tests/core/pyspec/README.md index a9ee80105..f7092dbce 100644 --- a/tests/core/pyspec/README.md +++ b/tests/core/pyspec/README.md @@ -55,6 +55,11 @@ Run the test command from the `tests/core/pyspec` directory: pytest --config=minimal eth2spec ``` +Options: +- `--config`, to change the config. Defaults to `minimal`, can be set to `mainnet`, or other configs from the configs directory. +- `--disable-bls`, to disable BLS (only for tests that can run without) +- `--bls-type`, `milagro` or `py_ecc` (default) + ### How to view code coverage report Run `make open_cov` from the root of the specs repository after running `make test` to open the html code coverage report. diff --git a/tests/core/pyspec/eth2spec/test/conftest.py b/tests/core/pyspec/eth2spec/test/conftest.py index 01187b05f..01c974ae0 100644 --- a/tests/core/pyspec/eth2spec/test/conftest.py +++ b/tests/core/pyspec/eth2spec/test/conftest.py @@ -1,6 +1,6 @@ from eth2spec.config import config_util -from eth2spec.test.context import reload_specs - +from eth2spec.test import context +from eth2spec.utils import bls as bls_utils # We import pytest only when it's present, i.e. when we are running tests. # The test-cases themselves can be generated without installing pytest. @@ -27,7 +27,16 @@ def fixture(*args, **kwargs): def pytest_addoption(parser): parser.addoption( - "--config", action="store", default="minimal", help="config: make the pyspec use the specified configuration" + "--config", action="store", type=str, default="minimal", + help="config: make the pyspec use the specified configuration" + ) + parser.addoption( + "--disable-bls", action="store_true", + help="bls-default: make tests that are not dependent on BLS run without BLS" + ) + parser.addoption( + "--bls-type", action="store", type=str, default="py_ecc", choices=["py_ecc", "milagro"], + help="bls-type: use 'pyecc' or 'milagro' implementation for BLS" ) @@ -36,4 +45,22 @@ def config(request): config_name = request.config.getoption("--config") config_util.prepare_config('../../../configs/', config_name) # now that the presets are loaded, reload the specs to apply them - reload_specs() + context.reload_specs() + + +@fixture(autouse=True) +def bls_default(request): + disable_bls = request.config.getoption("--disable-bls") + if disable_bls: + context.DEFAULT_BLS_ACTIVE = False + + +@fixture(autouse=True) +def bls_type(request): + bls_type = request.config.getoption("--bls-type") + if bls_type == "py_ecc": + bls_utils.bls = bls_utils.py_ecc_bls + elif bls_type == "milagro": + bls_utils.bls = bls_utils.milagro_bls + else: + raise Exception(f"unrecognized bls type: {bls_type}") diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 1a182fd31..1108edcf7 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -147,14 +147,15 @@ def single_phase(fn): return entry -# BLS is turned off by default *for performance purposes during TESTING*. +# BLS is turned on by default, it can be disabled in tests by overriding this, or using `--disable-bls`. +# *This is for performance purposes during TESTING, DO NOT DISABLE IN PRODUCTION*. # The runner of the test can indicate the preferred setting (test generators prefer BLS to be ON). # - Some tests are marked as BLS-requiring, and ignore this setting. # (tests that express differences caused by BLS, e.g. invalid signatures being rejected) # - Some other tests are marked as BLS-ignoring, and ignore this setting. # (tests that are heavily performance impacted / require unsigned state transitions) # - Most tests respect the BLS setting. -DEFAULT_BLS_ACTIVE = False +DEFAULT_BLS_ACTIVE = True def spec_test(fn): diff --git a/tests/core/pyspec/eth2spec/utils/bls.py b/tests/core/pyspec/eth2spec/utils/bls.py index 3b648fac9..44d0a8012 100644 --- a/tests/core/pyspec/eth2spec/utils/bls.py +++ b/tests/core/pyspec/eth2spec/utils/bls.py @@ -1,9 +1,13 @@ -from py_ecc.bls import G2ProofOfPossession as bls +from py_ecc.bls import G2ProofOfPossession as py_ecc_bls from py_ecc.bls.g2_primatives import signature_to_G2 as _signature_to_G2 +import milagro_bls_binding as milagro_bls # noqa: F401 for BLS switching option # Flag to make BLS active or not. Used for testing, do not ignore BLS in production unless you know what you are doing. bls_active = True +# To change bls implementation, default to PyECC for correctness. Milagro is a good faster alternative. +bls = py_ecc_bls + STUB_SIGNATURE = b'\x11' * 96 STUB_PUBKEY = b'\x22' * 48 STUB_COORDINATES = _signature_to_G2(bls.Sign(0, b"")) @@ -30,12 +34,12 @@ def Verify(PK, message, signature): @only_with_bls(alt_return=True) def AggregateVerify(pairs, signature): - return bls.AggregateVerify(pairs, signature) + return bls.AggregateVerify(list(pairs), signature) @only_with_bls(alt_return=True) def FastAggregateVerify(PKs, message, signature): - return bls.FastAggregateVerify(PKs, message, signature) + return bls.FastAggregateVerify(list(PKs), message, signature) @only_with_bls(alt_return=STUB_SIGNATURE) @@ -45,7 +49,10 @@ def Aggregate(signatures): @only_with_bls(alt_return=STUB_SIGNATURE) def Sign(SK, message): - return bls.Sign(SK, message) + if bls == py_ecc_bls: + return bls.Sign(SK, message) + else: + return bls.Sign(SK.to_bytes(48, 'big'), message) @only_with_bls(alt_return=STUB_COORDINATES) @@ -55,4 +62,4 @@ def signature_to_G2(signature): @only_with_bls(alt_return=STUB_PUBKEY) def AggregatePKs(pubkeys): - return bls._AggregatePKs(pubkeys) + return bls._AggregatePKs(list(pubkeys)) From 3851a26a0fa1e8c860956ba4da893608a3126752 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 May 2020 07:33:07 -0600 Subject: [PATCH 023/165] add phase 1 custody objects to custody-game.md --- specs/phase1/beacon-chain.md | 99 +----------------------------------- specs/phase1/custody-game.md | 92 +++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 98 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 278efda28..27a9229dd 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -12,14 +12,6 @@ - [Custom types](#custom-types) - [Configuration](#configuration) - [Misc](#misc) -- [New containers](#new-containers) - - [`CustodyChunkChallenge`](#custodychunkchallenge) - - [`CustodyChunkChallengeRecord`](#custodychunkchallengerecord) - - [`CustodyChunkResponse`](#custodychunkresponse) - - [`CustodySlashing`](#custodyslashing) - - [`SignedCustodySlashing`](#signedcustodyslashing) - - [`CustodyKeyReveal`](#custodykeyreveal) - - [`EarlyDerivedSecretReveal`](#earlyderivedsecretreveal) - [Updated containers](#updated-containers) - [Extended `AttestationData`](#extended-attestationdata) - [Extended `Attestation`](#extended-attestation) @@ -31,7 +23,7 @@ - [Extended `BeaconBlock`](#extended-beaconblock) - [Extended `SignedBeaconBlock`](#extended-signedbeaconblock) - [Extended `BeaconState`](#extended-beaconstate) -- [New containers](#new-containers-1) +- [New containers](#new-containers) - [`ShardBlock`](#shardblock) - [`SignedShardBlock`](#signedshardblock) - [`ShardBlockHeader`](#shardblockheader) @@ -124,95 +116,6 @@ Configuration is not namespaced. Instead it is strictly an extension; | `BYTES_PER_CUSTODY_CHUNK` | `2**12` | bytes | | `CUSTODY_RESPONSE_DEPTH` | `ceillog2(MAX_SHARD_BLOCK_SIZE // BYTES_PER_CUSTODY_CHUNK) | - | - | - - -## New containers - -### `CustodyChunkChallenge` - -```python -class CustodyChunkChallenge(Container): - responder_index: ValidatorIndex - shard_transition: ShardTransition - attestation: Attestation - data_index: uint64 - chunk_index: uint64 -``` - -### `CustodyChunkChallengeRecord` - -```python -class CustodyChunkChallengeRecord(Container): - challenge_index: uint64 - challenger_index: ValidatorIndex - responder_index: ValidatorIndex - inclusion_epoch: Epoch - data_root: Root - chunk_index: uint64 -``` - -#### `CustodyChunkResponse` - -```python -class CustodyChunkResponse(Container): - challenge_index: uint64 - chunk_index: uint64 - chunk: ByteVector[BYTES_PER_CUSTODY_CHUNK] - branch: Vector[Root, CUSTODY_RESPONSE_DEPTH] -``` - -#### `CustodySlashing` - -```python -class CustodySlashing(Container): - # Attestation.custody_bits_blocks[data_index][committee.index(malefactor_index)] is the target custody bit to check. - # (Attestation.data.shard_transition_root as ShardTransition).shard_data_roots[data_index] is the root of the data. - data_index: uint64 - malefactor_index: ValidatorIndex - malefactor_secret: BLSSignature - whistleblower_index: ValidatorIndex - shard_transition: ShardTransition - attestation: Attestation - data: ByteList[MAX_SHARD_BLOCK_SIZE] -``` - -#### `SignedCustodySlashing` - -```python -class SignedCustodySlashing(Container): - message: CustodySlashing - signature: BLSSignature -``` - - -#### `CustodyKeyReveal` - -```python -class CustodyKeyReveal(Container): - # Index of the validator whose key is being revealed - revealer_index: ValidatorIndex - # Reveal (masked signature) - reveal: BLSSignature -``` - -#### `EarlyDerivedSecretReveal` - -Represents an early (punishable) reveal of one of the derived secrets, where derived secrets are RANDAO reveals and custody reveals (both are part of the same domain). - -```python -class EarlyDerivedSecretReveal(Container): - # Index of the validator whose key is being revealed - revealed_index: ValidatorIndex - # RANDAO epoch of the key that is being revealed - epoch: Epoch - # Reveal (masked signature) - reveal: BLSSignature - # Index of the validator who revealed (whistleblower) - masker_index: ValidatorIndex - # Mask used to hide the actual reveal signature (prevent reveal from being stolen) - mask: Bytes32 -``` - ## Updated containers The following containers have updated definitions in Phase 1. diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index c81c6a17b..96a7f5616 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -18,6 +18,13 @@ - [Signature domain types](#signature-domain-types) - [Data structures](#data-structures) - [New Beacon Chain operations](#new-beacon-chain-operations) + - [`CustodyChunkChallenge`](#custodychunkchallenge) + - [`CustodyChunkChallengeRecord`](#custodychunkchallengerecord) + - [`CustodyChunkResponse`](#custodychunkresponse) + - [`CustodySlashing`](#custodyslashing) + - [`SignedCustodySlashing`](#signedcustodyslashing) + - [`CustodyKeyReveal`](#custodykeyreveal) + - [`EarlyDerivedSecretReveal`](#earlyderivedsecretreveal) - [Helpers](#helpers) - [`replace_empty_or_append`](#replace_empty_or_append) - [`legendre_bit`](#legendre_bit) @@ -95,6 +102,91 @@ The following types are defined, mapping into `DomainType` (little endian): ### New Beacon Chain operations +#### `CustodyChunkChallenge` + +```python +class CustodyChunkChallenge(Container): + responder_index: ValidatorIndex + shard_transition: ShardTransition + attestation: Attestation + data_index: uint64 + chunk_index: uint64 +``` + +#### `CustodyChunkChallengeRecord` + +```python +class CustodyChunkChallengeRecord(Container): + challenge_index: uint64 + challenger_index: ValidatorIndex + responder_index: ValidatorIndex + inclusion_epoch: Epoch + data_root: Root + chunk_index: uint64 +``` + +#### `CustodyChunkResponse` + +```python +class CustodyChunkResponse(Container): + challenge_index: uint64 + chunk_index: uint64 + chunk: ByteVector[BYTES_PER_CUSTODY_CHUNK] + branch: Vector[Root, CUSTODY_RESPONSE_DEPTH] +``` + +#### `CustodySlashing` + +```python +class CustodySlashing(Container): + # Attestation.custody_bits_blocks[data_index][committee.index(malefactor_index)] is the target custody bit to check. + # (Attestation.data.shard_transition_root as ShardTransition).shard_data_roots[data_index] is the root of the data. + data_index: uint64 + malefactor_index: ValidatorIndex + malefactor_secret: BLSSignature + whistleblower_index: ValidatorIndex + shard_transition: ShardTransition + attestation: Attestation + data: ByteList[MAX_SHARD_BLOCK_SIZE] +``` + +#### `SignedCustodySlashing` + +```python +class SignedCustodySlashing(Container): + message: CustodySlashing + signature: BLSSignature +``` + +#### `CustodyKeyReveal` + +```python +class CustodyKeyReveal(Container): + # Index of the validator whose key is being revealed + revealer_index: ValidatorIndex + # Reveal (masked signature) + reveal: BLSSignature +``` + +#### `EarlyDerivedSecretReveal` + +Represents an early (punishable) reveal of one of the derived secrets, where derived secrets are RANDAO reveals and custody reveals (both are part of the same domain). + +```python +class EarlyDerivedSecretReveal(Container): + # Index of the validator whose key is being revealed + revealed_index: ValidatorIndex + # RANDAO epoch of the key that is being revealed + epoch: Epoch + # Reveal (masked signature) + reveal: BLSSignature + # Index of the validator who revealed (whistleblower) + masker_index: ValidatorIndex + # Mask used to hide the actual reveal signature (prevent reveal from being stolen) + mask: Bytes32 +``` + + ## Helpers ### `replace_empty_or_append` From 3f0e58a8ed52253d84af70a1e041d64648dbe19b Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 May 2020 07:50:05 -0600 Subject: [PATCH 024/165] add chunk challenge and response to block and operations --- configs/mainnet.yaml | 2 ++ configs/minimal.yaml | 2 ++ specs/phase1/beacon-chain.md | 4 +++- specs/phase1/custody-game.md | 7 +++---- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index f5f38de5e..6475d491e 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -214,6 +214,8 @@ MAX_REVEAL_LATENESS_DECREMENT: 128 # 2**8 (= 256) MAX_CUSTODY_KEY_REVEALS: 256 MAX_EARLY_DERIVED_SECRET_REVEALS: 1 +MAX_CUSTODY_CHUNK_CHALLENGES: 4 +MAX_CUSTODY_CHUNK_CHALLENGE_RESP: 16 MAX_CUSTODY_SLASHINGS: 1 # Reward and penalty quotients diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 1b5aac5f6..89e2a7eae 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -215,6 +215,8 @@ CUSTODY_PERIOD_TO_RANDAO_PADDING: 8 # 2**8 (= 256) MAX_CUSTODY_KEY_REVEALS: 256 MAX_EARLY_DERIVED_SECRET_REVEALS: 1 +MAX_CUSTODY_CHUNK_CHALLENGES: 2 +MAX_CUSTODY_CHUNK_CHALLENGE_RESP: 8 MAX_CUSTODY_SLASHINGS: 1 # Reward and penalty quotients diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 27a9229dd..e48c99c4a 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -215,9 +215,11 @@ class BeaconBlockBody(Container): deposits: List[Deposit, MAX_DEPOSITS] voluntary_exits: List[SignedVoluntaryExit, MAX_VOLUNTARY_EXITS] # Custody game - custody_slashings: List[SignedCustodySlashing, MAX_CUSTODY_SLASHINGS] + chunk_challenges: List[CustodyChunkResponse, MAX_CUSTODY_CHUNK_CHALLENGES] + chunk_challenge_responses: List[CustodyChunkResponse, MAX_CUSTODY_CHUNK_CHALLENGE_RESPONSES] custody_key_reveals: List[CustodyKeyReveal, MAX_CUSTODY_KEY_REVEALS] early_derived_secret_reveals: List[EarlyDerivedSecretReveal, MAX_EARLY_DERIVED_SECRET_REVEALS] + custody_slashings: List[SignedCustodySlashing, MAX_CUSTODY_SLASHINGS] # Shards shard_transitions: Vector[ShardTransition, MAX_SHARDS] # Light clients diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 96a7f5616..08893ef53 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -81,6 +81,7 @@ This document details the beacon chain additions and changes in Phase 1 of Ether | `MAX_CUSTODY_KEY_REVEALS` | `2**8` (= 256) | | `MAX_EARLY_DERIVED_SECRET_REVEALS` | `1` | | `MAX_CUSTODY_CHUNK_CHALLENGES` | `2**2` (= 4) | +| `MAX_CUSTODY_CHUNK_CHALLENGE_RESPONSES` | `2**4` (= 16) | | `MAX_CUSTODY_SLASHINGS` | `1` | ### Reward and penalty quotients @@ -297,6 +298,8 @@ def process_custody_game_operations(state: BeaconState, body: BeaconBlockBody) - for operation in operations: fn(state, operation) + for_ops(body.chunk_challenges, process_chunk_challenge) + for_ops(body.chunk_challenge_responses, process_chunk_challenge) for_ops(body.custody_key_reveals, process_custody_key_reveal) for_ops(body.early_derived_secret_reveals, process_early_derived_secret_reveal) for_ops(body.custody_slashings, process_custody_slashing) @@ -304,10 +307,6 @@ def process_custody_game_operations(state: BeaconState, body: BeaconBlockBody) - #### Chunk challenges -Verify that `len(block.body.custody_chunk_challenges) <= MAX_CUSTODY_CHUNK_CHALLENGES`. - -For each `challenge` in `block.body.custody_chunk_challenges`, run the following function: - ```python def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge) -> None: # Verify the attestation From 7fc9dbf297e4900aa4345443722e3c184790a557 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 May 2020 07:52:26 -0600 Subject: [PATCH 025/165] clarify comment for ShardTransition.shard_data_roots --- specs/phase1/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index e48c99c4a..0f5e26a8c 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -352,7 +352,7 @@ class ShardTransition(Container): # Shard block lengths shard_block_lengths: List[uint64, MAX_SHARD_BLOCKS_PER_ATTESTATION] # Shard data roots - # The root is of ByteVector[MAX_] + # The root is of ByteList[MAX_SHARD_BLOCK_SIZE] shard_data_roots: List[Bytes32, MAX_SHARD_BLOCKS_PER_ATTESTATION] # Intermediate shard states shard_states: List[ShardState, MAX_SHARD_BLOCKS_PER_ATTESTATION] From 1623086088e6f0496566ab7d50d16a8c78cdebf0 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 May 2020 08:14:04 -0600 Subject: [PATCH 026/165] make get_validator_from_deposit for better code reuse across phase 0 and 1 --- specs/phase0/beacon-chain.md | 26 +++++++++----- specs/phase1/beacon-chain.md | 66 ++++++++++-------------------------- 2 files changed, 35 insertions(+), 57 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 3ef4081ce..ada599c9d 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1632,6 +1632,22 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: ##### Deposits +```python +def get_validator_from_deposit(state: BeaconState, deposit: Deposit) -> Validator: + amount = deposit.data.amount + effective_balance = min(amount - amount % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) + + return Validator( + pubkey=deposit.data.pubkey, + withdrawal_credentials=deposit.data.withdrawal_credentials, + activation_eligibility_epoch=FAR_FUTURE_EPOCH, + activation_epoch=FAR_FUTURE_EPOCH, + exit_epoch=FAR_FUTURE_EPOCH, + withdrawable_epoch=FAR_FUTURE_EPOCH, + effective_balance=effective_balance, + ) +``` + ```python def process_deposit(state: BeaconState, deposit: Deposit) -> None: # Verify the Merkle branch @@ -1662,15 +1678,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: return # Add validator and balance entries - state.validators.append(Validator( - pubkey=pubkey, - withdrawal_credentials=deposit.data.withdrawal_credentials, - activation_eligibility_epoch=FAR_FUTURE_EPOCH, - activation_epoch=FAR_FUTURE_EPOCH, - exit_epoch=FAR_FUTURE_EPOCH, - withdrawable_epoch=FAR_FUTURE_EPOCH, - effective_balance=min(amount - amount % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE), - )) + state.validators.append(get_validator_from_deposit(state, deposit)) state.balances.append(amount) else: # Increase balance by deposit amount diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 0f5e26a8c..32e9c925b 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -883,58 +883,28 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: state.previous_epoch_attestations.append(pending_attestation) ``` -##### New deposits +##### New default validator for deposits ```python -def process_deposit(state: BeaconState, deposit: Deposit) -> None: - # Verify the Merkle branch - assert is_valid_merkle_branch( - leaf=hash_tree_root(deposit.data), - branch=deposit.proof, - depth=DEPOSIT_CONTRACT_TREE_DEPTH + 1, # Add 1 for the List length mix-in - index=state.eth1_deposit_index, - root=state.eth1_data.deposit_root, +def get_validator_from_deposit(state: BeaconState, deposit: Deposit) -> Validator: + amount = deposit.data.amount + effective_balance = min(amount - amount % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) + next_custody_secret_to_reveal = get_custody_period_for_validator( + ValidatorIndex(len(state.validators)), + get_current_epoch(state), ) - # Deposits must be processed in order - state.eth1_deposit_index += 1 - - pubkey = deposit.data.pubkey - amount = deposit.data.amount - validator_pubkeys = [v.pubkey for v in state.validators] - if pubkey not in validator_pubkeys: - # Verify the deposit signature (proof of possession) which is not checked by the deposit contract - deposit_message = DepositMessage( - pubkey=deposit.data.pubkey, - withdrawal_credentials=deposit.data.withdrawal_credentials, - amount=deposit.data.amount, - ) - domain = compute_domain(DOMAIN_DEPOSIT) # Fork-agnostic domain since deposits are valid across forks - signing_root = compute_signing_root(deposit_message, domain) - if not bls.Verify(pubkey, signing_root, deposit.data.signature): - return - - # Add validator and balance entries - # TODO: This function is duplicated from phase 1 just because the validator definition - # has changed and we need to initialize it properly. Is there a better solution for - # this? - state.validators.append(Validator( - pubkey=pubkey, - withdrawal_credentials=deposit.data.withdrawal_credentials, - activation_eligibility_epoch=FAR_FUTURE_EPOCH, - activation_epoch=FAR_FUTURE_EPOCH, - exit_epoch=FAR_FUTURE_EPOCH, - withdrawable_epoch=FAR_FUTURE_EPOCH, - effective_balance=min(amount - amount % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE), - next_custody_secret_to_reveal=get_custody_period_for_validator(ValidatorIndex(len(state.validators)), - get_current_epoch(state)), - all_custody_secrets_revealed_epoch=FAR_FUTURE_EPOCH, - )) - state.balances.append(amount) - else: - # Increase balance by deposit amount - index = ValidatorIndex(validator_pubkeys.index(pubkey)) - increase_balance(state, index, amount) + return Validator( + pubkey=deposit.data.pubkey, + withdrawal_credentials=deposit.data.withdrawal_credentials, + activation_eligibility_epoch=FAR_FUTURE_EPOCH, + activation_epoch=FAR_FUTURE_EPOCH, + exit_epoch=FAR_FUTURE_EPOCH, + withdrawable_epoch=FAR_FUTURE_EPOCH, + effective_balance=effective_balance, + next_custody_secret_to_reveal=next_custody_secret_to_reveal, + all_custody_secrets_revealed_epoch=FAR_FUTURE_EPOCH, + ) ``` ##### New Attester slashing processing From 78947548e6b5e31e7601f3519fa801515c4e84a0 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 May 2020 08:31:01 -0600 Subject: [PATCH 027/165] add process_challenge_deadlines to process_epoch --- specs/phase1/beacon-chain.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 32e9c925b..9ca4b1e39 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -995,6 +995,7 @@ def process_epoch(state: BeaconState) -> None: process_rewards_and_penalties(state) process_registry_updates(state) process_reveal_deadlines(state) + process_challenge_deadlines(state) process_slashings(state) process_final_updates(state) process_custody_final_updates(state) From de03ebb143136caa1a01e2ba8b465aa2c6e224b1 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 May 2020 08:31:16 -0600 Subject: [PATCH 028/165] many custody game formatting cleanups --- specs/phase1/beacon-chain.md | 2 +- specs/phase1/custody-game.md | 67 ++++++++++++++++-------------------- 2 files changed, 31 insertions(+), 38 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 9ca4b1e39..d53113038 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -63,7 +63,7 @@ - [`process_crosslink_for_shard`](#process_crosslink_for_shard) - [`process_crosslinks`](#process_crosslinks) - [`process_attestation`](#process_attestation) - - [New deposits](#new-deposits) + - [New default validator for deposits](#new-default-validator-for-deposits) - [New Attester slashing processing](#new-attester-slashing-processing) - [Shard transition false positives](#shard-transition-false-positives) - [Light client processing](#light-client-processing) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 08893ef53..f4f45ce1e 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -71,7 +71,7 @@ This document details the beacon chain additions and changes in Phase 1 of Ether | `EPOCHS_PER_CUSTODY_PERIOD` | `2**11` (= 2,048) | epochs | ~9 days | | `CUSTODY_PERIOD_TO_RANDAO_PADDING` | `2**11` (= 2,048) | epochs | ~9 days | | `CHUNK_RESPONSE_DEADLINE` | `2**14` (= 16,384) | epochs | ~73 days | -| `MAX_CHUNK_CHALLENGE_DELAY` | `2**11` (= 16,384) | epochs | ~9 days | +| `MAX_CHUNK_CHALLENGE_DELAY` | `2**11` (= 2,048) | epochs | ~9 days | | `CUSTODY_RESPONSE_DEADLINE` | `2**14` (= 16,384) | epochs | ~73 days | ### Max operations per block @@ -311,12 +311,13 @@ def process_custody_game_operations(state: BeaconState, body: BeaconBlockBody) - def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge) -> None: # Verify the attestation assert is_valid_indexed_attestation(state, get_indexed_attestation(state, challenge.attestation)) - # Verify it is not too late to challenge - assert (challenge.attestation.data.target.epoch + MAX_CHUNK_CHALLENGE_DELAY - >= get_current_epoch(state)) + # Verify it is not too late to challenge the attestation + max_attestation_challenge_epoch = challenge.attestation.data.target.epoch + MAX_CHUNK_CHALLENGE_DELAY + assert get_current_epoch(state) <= max_attestation_challenge_epoch + # Verify it is not too late to challenge the responder responder = state.validators[challenge.responder_index] - assert (responder.exit_epoch == FAR_FUTURE_EPOCH - or responder.exit_epoch + MAX_CHUNK_CHALLENGE_DELAY >= get_current_epoch(state)) + if responder.exit_epoch < FAR_FUTURE_EPOCH: + assert get_current_epoch(state) <= responder.exit_epoch + MAX_CHUNK_CHALLENGE_DELAY # Verify responder is slashable assert is_slashable_validator(responder, get_current_epoch(state)) # Verify the responder participated in the attestation @@ -332,8 +333,8 @@ def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge record.chunk_index != challenge.chunk_index ) # Verify depth - transition_chunks = (challenge.shard_transition.shard_block_lengths[challenge.data_index] - + BYTES_PER_CUSTODY_CHUNK - 1) // BYTES_PER_CUSTODY_CHUNK + shard_block_length = challenge.shard_transition.shard_block_lengths[challenge.data_index] + transition_chunks = (shard_block_length + BYTES_PER_CUSTODY_CHUNK - 1) // BYTES_PER_CUSTODY_CHUNK assert challenge.chunk_index < transition_chunks # Add new chunk challenge record new_record = CustodyChunkChallengeRecord( @@ -357,10 +358,13 @@ def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge def process_chunk_challenge_response(state: BeaconState, response: CustodyChunkResponse) -> None: - challenge = next((record for record in state.custody_chunk_challenge_records if - record.challenge_index == response.challenge_index), None) - assert(challenge is not None) - + # Get matching challenge (if any) from records + matching_challenges = [ + record for record in state.custody_chunk_challenge_records + if record.challenge_index == response.challenge_index + ] + assert len(matching_challenges) > 0 + challenge = matching_challenges[0] # Verify chunk index assert response.chunk_index == challenge.chunk_index # Verify the chunk matches the crosslink data root @@ -391,12 +395,14 @@ def process_custody_key_reveal(state: BeaconState, reveal: CustodyKeyReveal) -> epoch_to_sign = get_randao_epoch_for_custody_period(revealer.next_custody_secret_to_reveal, reveal.revealer_index) custody_reveal_period = get_custody_period_for_validator(reveal.revealer_index, get_current_epoch(state)) - # Only past custody periods can be revealed, except after exiting the exit - # period can be revealed - assert (revealer.next_custody_secret_to_reveal < custody_reveal_period - or (revealer.exit_epoch <= get_current_epoch(state) and - revealer.next_custody_secret_to_reveal - <= get_custody_period_for_validator(reveal.revealer_index, revealer.exit_epoch - 1))) + # Only past custody periods can be revealed, except after exiting the exit period can be revealed + is_past_reveal = revealer.next_custody_secret_to_reveal < custody_reveal_period + is_exited = revealer.exit_epoch <= get_current_epoch(state) + is_exit_period_reveal = ( + revealer.next_custody_secret_to_reveal + == get_custody_period_for_validator(reveal.revealer_index, revealer.exit_epoch - 1) + ) + assert is_past_reveal or (is_exited and is_exit_period_reveal) # Revealed validator is active or exited, but not withdrawn assert is_slashable_validator(revealer, get_current_epoch(state)) @@ -407,9 +413,7 @@ def process_custody_key_reveal(state: BeaconState, reveal: CustodyKeyReveal) -> assert bls.Verify(revealer.pubkey, signing_root, reveal.reveal) # Process reveal - if (revealer.exit_epoch <= get_current_epoch(state) and - revealer.next_custody_secret_to_reveal - == get_custody_period_for_validator(reveal.revealer_index, revealer.exit_epoch - 1)): + if is_exited and is_exit_period_reveal: revealer.all_custody_secrets_revealed_epoch = get_current_epoch(state) revealer.next_custody_secret_to_reveal += 1 @@ -556,23 +560,16 @@ def process_custody_slashing(state: BeaconState, signed_custody_slashing: Signed ### Handling of reveal deadlines -Run `process_reveal_deadlines(state)` after `process_registry_updates(state)`: - ```python def process_reveal_deadlines(state: BeaconState) -> None: epoch = get_current_epoch(state) for index, validator in enumerate(state.validators): - if get_custody_period_for_validator(ValidatorIndex(index), epoch) > validator.next_custody_secret_to_reveal \ - + (CUSTODY_RESPONSE_DEADLINE // EPOCHS_PER_CUSTODY_PERIOD): + deadline = validator.next_custody_secret_to_reveal + (CUSTODY_RESPONSE_DEADLINE // EPOCHS_PER_CUSTODY_PERIOD) + if get_custody_period_for_validator(ValidatorIndex(index), epoch) > deadline: slash_validator(state, ValidatorIndex(index)) ``` -Run `process_challenge_deadlines(state)` immediately after `process_reveal_deadlines(state)`: - ```python -# begin insert @process_challenge_deadlines - process_challenge_deadlines(state) -# end insert @process_challenge_deadlines def process_challenge_deadlines(state: BeaconState) -> None: for custody_chunk_challenge in state.custody_chunk_challenge_records: if get_current_epoch(state) > custody_chunk_challenge.inclusion_epoch + CUSTODY_RESPONSE_DEADLINE: @@ -583,8 +580,6 @@ def process_challenge_deadlines(state: BeaconState) -> None: ### Final updates -After `process_final_updates(state)`, additional updates are made for the custody game: - ```python def process_custody_final_updates(state: BeaconState) -> None: # Clean up exposed RANDAO key reveals @@ -592,13 +587,11 @@ def process_custody_final_updates(state: BeaconState) -> None: # Reset withdrawable epochs if challenge records are empty records = state.custody_chunk_challenge_records - validator_indices_in_records = set( - [record.responder_index for record in records] - ) + validator_indices_in_records = set([record.responder_index for record in records]) for index, validator in enumerate(state.validators): if validator.exit_epoch != FAR_FUTURE_EPOCH: - if (index in validator_indices_in_records - or validator.all_custody_secrets_revealed_epoch == FAR_FUTURE_EPOCH): + all_secrets_are_revealed = validator.all_custody_secrets_revealed_epoch == FAR_FUTURE_EPOCH + if index in validator_indices_in_records or all_secrets_are_revealed: # Delay withdrawable epochs if challenge records are not empty or not all # custody secrets revealed validator.withdrawable_epoch = FAR_FUTURE_EPOCH From 85e78223dd361bbdf6e3e567bdf6860820b008ce Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 May 2020 16:51:46 -0600 Subject: [PATCH 029/165] ensure when performing optimally that you don't lose money during a leak --- specs/phase0/beacon-chain.md | 31 ++++++++--- .../pyspec/eth2spec/test/helpers/rewards.py | 34 +++++++++++++ .../test_process_rewards_and_penalties.py | 51 +++++++++++++++++++ .../test/phase_0/rewards/test_leak.py | 36 +------------ 4 files changed, 110 insertions(+), 42 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 142cf3b02..7bf4cd06e 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1354,6 +1354,19 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: return Gwei(effective_balance * BASE_REWARD_FACTOR // integer_squareroot(total_balance) // BASE_REWARDS_PER_EPOCH) ``` + +```python +def get_finality_delay(state: BeaconState) -> uint64: + return get_previous_epoch(state) - state.finalized_checkpoint.epoch +``` + + +```python +def in_inactivity_leak(state: BeaconState) -> bool: + return get_finality_delay(state) > MIN_EPOCHS_TO_INACTIVITY_PENALTY +``` + + ```python def get_eligible_validator_indices(state: BeaconState) -> Sequence[ValidatorIndex]: previous_epoch = get_previous_epoch(state) @@ -1378,8 +1391,11 @@ def get_attestation_component_deltas(state: BeaconState, for index in get_eligible_validator_indices(state): if index in unslashed_attesting_indices: increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from balance totals to avoid uint64 overflow - reward_numerator = get_base_reward(state, index) * (attesting_balance // increment) - rewards[index] += reward_numerator // (total_balance // increment) + if in_inactivity_leak(state): + rewards[index] += get_base_reward(state, index) + else: + reward_numerator = get_base_reward(state, index) * (attesting_balance // increment) + rewards[index] += reward_numerator // (total_balance // increment) else: penalties[index] += get_base_reward(state, index) return rewards, penalties @@ -1428,7 +1444,10 @@ def get_inclusion_delay_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequ ], key=lambda a: a.inclusion_delay) proposer_reward = Gwei(get_base_reward(state, index) // PROPOSER_REWARD_QUOTIENT) rewards[attestation.proposer_index] += proposer_reward - max_attester_reward = get_base_reward(state, index) - proposer_reward + if in_inactivity_leak(state): + max_attester_reward = get_base_reward(state, index) + else: + max_attester_reward = get_base_reward(state, index) - proposer_reward rewards[index] += Gwei(max_attester_reward // attestation.inclusion_delay) # No penalties associated with inclusion delay @@ -1442,16 +1461,14 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S Return inactivity reward/penalty deltas for each validator. """ penalties = [Gwei(0) for _ in range(len(state.validators))] - finality_delay = get_previous_epoch(state) - state.finalized_checkpoint.epoch - - if finality_delay > MIN_EPOCHS_TO_INACTIVITY_PENALTY: + if in_inactivity_leak(state): matching_target_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) matching_target_attesting_indices = get_unslashed_attesting_indices(state, matching_target_attestations) for index in get_eligible_validator_indices(state): penalties[index] += Gwei(BASE_REWARDS_PER_EPOCH * get_base_reward(state, index)) if index not in matching_target_attesting_indices: effective_balance = state.validators[index].effective_balance - penalties[index] += Gwei(effective_balance * finality_delay // INACTIVITY_PENALTY_QUOTIENT) + penalties[index] += Gwei(effective_balance * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) # No rewards associated with inactivity penalties rewards = [Gwei(0) for _ in range(len(state.validators))] diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index d62fee6ce..034a79fd4 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -1,4 +1,5 @@ from random import Random +from lru import LRU from eth2spec.phase0 import spec as spec_phase0 from eth2spec.test.helpers.attestations import cached_prepare_state_with_attestations @@ -170,6 +171,39 @@ def run_get_inactivity_penalty_deltas(spec, state): assert penalties[index] == 0 +def transition_state_to_leak(spec, state, epochs=None): + if epochs is None: + epochs = spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + assert epochs >= spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + + for _ in range(epochs): + next_epoch(spec, state) + + +_cache_dict = LRU(size=10) + + +def leaking(epochs=None): + + def deco(fn): + def entry(*args, spec, state, **kw): + # If the pre-state is not already known in the LRU, then take it, + # transition it to leak, and put it in the LRU. + # The input state is likely already cached, so the hash-tree-root does not affect speed. + key = (state.hash_tree_root(), spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY, spec.SLOTS_PER_EPOCH, epochs) + global _cache_dict + if key not in _cache_dict: + transition_state_to_leak(spec, state, epochs=epochs) + _cache_dict[key] = state.get_backing() # cache the tree structure, not the view wrapping it. + + # Take an entry out of the LRU. + # No copy is necessary, as we wrap the immutable backing with a new view. + state = spec.BeaconState(backing=_cache_dict[key]) + return fn(*args, spec=spec, state=state, **kw) + return entry + return deco + + def set_some_new_deposits(spec, state, rng): num_validators = len(state.validators) # Set ~1/10 to just recently deposited diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py index eff286448..52d3b3c06 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py @@ -14,6 +14,7 @@ from eth2spec.test.helpers.attestations import ( get_valid_attestation, prepare_state_with_attestations, ) +from eth2spec.test.helpers.rewards import leaking from eth2spec.test.helpers.attester_slashings import get_indexed_attestation_participants from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_with from random import Random @@ -80,6 +81,56 @@ def test_full_attestations(spec, state): assert state.balances[index] < pre_state.balances[index] +@with_all_phases +@spec_state_test +@leaking() +def test_full_attestations_with_leak(spec, state): + attestations = prepare_state_with_attestations(spec, state) + + proposer_indices = [a.proposer_index for a in state.previous_epoch_attestations] + pre_state = state.copy() + + yield from run_process_rewards_and_penalties(spec, state) + + attesting_indices = spec.get_unslashed_attesting_indices(state, attestations) + assert len(attesting_indices) == len(pre_state.validators) + for index in range(len(pre_state.validators)): + # Proposers can still make money during a leak + if index in proposer_indices: + assert state.balances[index] > pre_state.balances[index] + # If not proposer but participated optimally, should have exactly neutral balance + elif index in attesting_indices: + assert state.balances[index] == pre_state.balances[index] + else: + assert state.balances[index] < pre_state.balances[index] + + +@with_all_phases +@spec_state_test +@leaking() +def test_partial_attestations_with_leak(spec, state): + attestations = prepare_state_with_attestations(spec, state) + + attestations = attestations[:len(attestations) // 2] + state.previous_epoch_attestations = state.previous_epoch_attestations[:len(attestations)] + proposer_indices = [a.proposer_index for a in state.previous_epoch_attestations] + pre_state = state.copy() + + yield from run_process_rewards_and_penalties(spec, state) + + attesting_indices = spec.get_unslashed_attesting_indices(state, attestations) + assert len(attesting_indices) < len(pre_state.validators) + for index in range(len(pre_state.validators)): + # Proposers can still make money during a leak + if index in proposer_indices and index in attesting_indices: + assert state.balances[index] > pre_state.balances[index] + # If not proposer but participated optimally, should have exactly neutral balance + elif index in attesting_indices: + assert state.balances[index] == pre_state.balances[index] + else: + assert state.balances[index] < pre_state.balances[index] + + @with_all_phases @spec_state_test def test_full_attestations_random_incorrect_fields(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py index 4e75079c0..b0f9767b2 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py @@ -1,40 +1,6 @@ from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.state import next_epoch +from eth2spec.test.helpers.rewards import leaking import eth2spec.test.helpers.rewards as rewards_helpers -from lru import LRU - - -def transition_state_to_leak(spec, state, epochs=None): - if epochs is None: - epochs = spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY - assert epochs >= spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY - - for _ in range(epochs): - next_epoch(spec, state) - - -_cache_dict = LRU(size=10) - - -def leaking(epochs=None): - - def deco(fn): - def entry(*args, spec, state, **kw): - # If the pre-state is not already known in the LRU, then take it, - # transition it to leak, and put it in the LRU. - # The input state is likely already cached, so the hash-tree-root does not affect speed. - key = (state.hash_tree_root(), spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY, spec.SLOTS_PER_EPOCH, epochs) - global _cache_dict - if key not in _cache_dict: - transition_state_to_leak(spec, state, epochs=epochs) - _cache_dict[key] = state.get_backing() # cache the tree structure, not the view wrapping it. - - # Take an entry out of the LRU. - # No copy is necessary, as we wrap the immutable backing with a new view. - state = spec.BeaconState(backing=_cache_dict[key]) - return fn(*args, spec=spec, state=state, **kw) - return entry - return deco @with_all_phases From 95c3295eeba373e94b9340b8b0fe8c804858ee08 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 May 2020 17:12:53 -0600 Subject: [PATCH 030/165] move proposer negation to inactivity_penalty deltas --- specs/phase0/beacon-chain.md | 19 ++++++++++++------- .../pyspec/eth2spec/test/helpers/rewards.py | 3 ++- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 7bf4cd06e..b0a9724fa 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1355,6 +1355,12 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: ``` +```python +def get_proposer_reward(state: BeaconState, attesting_index: ValidatorIndex) -> Gwei: + return Gwei(get_base_reward(state, attesting_index) // PROPOSER_REWARD_QUOTIENT) +``` + + ```python def get_finality_delay(state: BeaconState) -> uint64: return get_previous_epoch(state) - state.finalized_checkpoint.epoch @@ -1392,6 +1398,7 @@ def get_attestation_component_deltas(state: BeaconState, if index in unslashed_attesting_indices: increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from balance totals to avoid uint64 overflow if in_inactivity_leak(state): + # Full base reward will be cancelled out by inactivity penalty deltas rewards[index] += get_base_reward(state, index) else: reward_numerator = get_base_reward(state, index) * (attesting_balance // increment) @@ -1442,12 +1449,8 @@ def get_inclusion_delay_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequ a for a in matching_source_attestations if index in get_attesting_indices(state, a.data, a.aggregation_bits) ], key=lambda a: a.inclusion_delay) - proposer_reward = Gwei(get_base_reward(state, index) // PROPOSER_REWARD_QUOTIENT) - rewards[attestation.proposer_index] += proposer_reward - if in_inactivity_leak(state): - max_attester_reward = get_base_reward(state, index) - else: - max_attester_reward = get_base_reward(state, index) - proposer_reward + rewards[attestation.proposer_index] += get_proposer_reward(state, index) + max_attester_reward = get_base_reward(state, index) - get_proposer_reward(state, index) rewards[index] += Gwei(max_attester_reward // attestation.inclusion_delay) # No penalties associated with inclusion delay @@ -1465,7 +1468,9 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S matching_target_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) matching_target_attesting_indices = get_unslashed_attesting_indices(state, matching_target_attestations) for index in get_eligible_validator_indices(state): - penalties[index] += Gwei(BASE_REWARDS_PER_EPOCH * get_base_reward(state, index)) + # If validator is performing optimally this cancels all rewards for a neutral balance + base_reward = get_base_reward(state, index) + penalties[index] += Gwei(BASE_REWARDS_PER_EPOCH * base_reward - get_proposer_reward(state, index)) if index not in matching_target_attesting_indices: effective_balance = state.validators[index].effective_balance penalties[index] += Gwei(effective_balance * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 034a79fd4..801434e79 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -160,7 +160,8 @@ def run_get_inactivity_penalty_deltas(spec, state): continue if finality_delay > spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY: - base_penalty = spec.BASE_REWARDS_PER_EPOCH * spec.get_base_reward(state, index) + base_reward = spec.get_base_reward(state, index) + base_penalty = spec.BASE_REWARDS_PER_EPOCH * base_reward - spec.get_proposer_reward(state, index) if not has_enough_for_reward(spec, state, index): assert penalties[index] == 0 elif index in matching_attesting_indices: From 4c5d2c25b3809273777894c1cadd0069314b4835 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 20 May 2020 14:37:15 +0800 Subject: [PATCH 031/165] Bump remerkleable to 0.1.15 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5f0dce763..879ca4d38 100644 --- a/setup.py +++ b/setup.py @@ -503,7 +503,7 @@ setup( "pycryptodome==3.9.4", "py_ecc==4.0.0", "dataclasses==0.6", - "remerkleable==0.1.13", + "remerkleable==0.1.15", "ruamel.yaml==0.16.5", "lru-dict==1.1.6" ] From 796e372c5e03b3e0c0a589d3e90a5309f360455e Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 20 May 2020 15:30:44 +0200 Subject: [PATCH 032/165] remerkleable 0.1.16 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 879ca4d38..da56175af 100644 --- a/setup.py +++ b/setup.py @@ -503,7 +503,7 @@ setup( "pycryptodome==3.9.4", "py_ecc==4.0.0", "dataclasses==0.6", - "remerkleable==0.1.15", + "remerkleable==0.1.16", "ruamel.yaml==0.16.5", "lru-dict==1.1.6" ] From 943e51aef18bb025fa34b2caeb7810f9d77e9890 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 20 May 2020 10:11:47 -0600 Subject: [PATCH 033/165] hww feedback for finality rewards fix --- specs/phase0/beacon-chain.md | 9 +- .../pyspec/eth2spec/test/helpers/rewards.py | 3 +- .../test_process_rewards_and_penalties.py | 116 +++++++----------- 3 files changed, 51 insertions(+), 77 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index b0a9724fa..839af5f7b 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1368,7 +1368,7 @@ def get_finality_delay(state: BeaconState) -> uint64: ```python -def in_inactivity_leak(state: BeaconState) -> bool: +def is_in_inactivity_leak(state: BeaconState) -> bool: return get_finality_delay(state) > MIN_EPOCHS_TO_INACTIVITY_PENALTY ``` @@ -1397,8 +1397,9 @@ def get_attestation_component_deltas(state: BeaconState, for index in get_eligible_validator_indices(state): if index in unslashed_attesting_indices: increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from balance totals to avoid uint64 overflow - if in_inactivity_leak(state): - # Full base reward will be cancelled out by inactivity penalty deltas + if is_in_inactivity_leak(state): + # Since full base reward will be canceled out by inactivity penalty deltas, + # optimal participation receives full base reward compensation here. rewards[index] += get_base_reward(state, index) else: reward_numerator = get_base_reward(state, index) * (attesting_balance // increment) @@ -1464,7 +1465,7 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S Return inactivity reward/penalty deltas for each validator. """ penalties = [Gwei(0) for _ in range(len(state.validators))] - if in_inactivity_leak(state): + if is_in_inactivity_leak(state): matching_target_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) matching_target_attesting_indices = get_unslashed_attesting_indices(state, matching_target_attestations) for index in get_eligible_validator_indices(state): diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 801434e79..c11ba1ec1 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -151,7 +151,6 @@ def run_get_inactivity_penalty_deltas(spec, state): matching_attestations = spec.get_matching_target_attestations(state, spec.get_previous_epoch(state)) matching_attesting_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) - finality_delay = spec.get_previous_epoch(state) - state.finalized_checkpoint.epoch eligible_indices = spec.get_eligible_validator_indices(state) for index in range(len(state.validators)): assert rewards[index] == 0 @@ -159,7 +158,7 @@ def run_get_inactivity_penalty_deltas(spec, state): assert penalties[index] == 0 continue - if finality_delay > spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY: + if spec.is_in_inactivity_leak(state): base_reward = spec.get_base_reward(state, index) base_penalty = spec.BASE_REWARDS_PER_EPOCH * base_reward - spec.get_proposer_reward(state, index) if not has_enough_for_reward(spec, state, index): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py index 52d3b3c06..bafefcad6 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py @@ -63,74 +63,6 @@ def test_genesis_epoch_full_attestations_no_rewards(spec, state): assert state.balances[index] == pre_state.balances[index] -@with_all_phases -@spec_state_test -def test_full_attestations(spec, state): - attestations = prepare_state_with_attestations(spec, state) - - pre_state = state.copy() - - yield from run_process_rewards_and_penalties(spec, state) - - attesting_indices = spec.get_unslashed_attesting_indices(state, attestations) - assert len(attesting_indices) == len(pre_state.validators) - for index in range(len(pre_state.validators)): - if index in attesting_indices: - assert state.balances[index] > pre_state.balances[index] - else: - assert state.balances[index] < pre_state.balances[index] - - -@with_all_phases -@spec_state_test -@leaking() -def test_full_attestations_with_leak(spec, state): - attestations = prepare_state_with_attestations(spec, state) - - proposer_indices = [a.proposer_index for a in state.previous_epoch_attestations] - pre_state = state.copy() - - yield from run_process_rewards_and_penalties(spec, state) - - attesting_indices = spec.get_unslashed_attesting_indices(state, attestations) - assert len(attesting_indices) == len(pre_state.validators) - for index in range(len(pre_state.validators)): - # Proposers can still make money during a leak - if index in proposer_indices: - assert state.balances[index] > pre_state.balances[index] - # If not proposer but participated optimally, should have exactly neutral balance - elif index in attesting_indices: - assert state.balances[index] == pre_state.balances[index] - else: - assert state.balances[index] < pre_state.balances[index] - - -@with_all_phases -@spec_state_test -@leaking() -def test_partial_attestations_with_leak(spec, state): - attestations = prepare_state_with_attestations(spec, state) - - attestations = attestations[:len(attestations) // 2] - state.previous_epoch_attestations = state.previous_epoch_attestations[:len(attestations)] - proposer_indices = [a.proposer_index for a in state.previous_epoch_attestations] - pre_state = state.copy() - - yield from run_process_rewards_and_penalties(spec, state) - - attesting_indices = spec.get_unslashed_attesting_indices(state, attestations) - assert len(attesting_indices) < len(pre_state.validators) - for index in range(len(pre_state.validators)): - # Proposers can still make money during a leak - if index in proposer_indices and index in attesting_indices: - assert state.balances[index] > pre_state.balances[index] - # If not proposer but participated optimally, should have exactly neutral balance - elif index in attesting_indices: - assert state.balances[index] == pre_state.balances[index] - else: - assert state.balances[index] < pre_state.balances[index] - - @with_all_phases @spec_state_test def test_full_attestations_random_incorrect_fields(spec, state): @@ -224,6 +156,7 @@ def run_with_participation(spec, state, participation_fn): return att_participants attestations = prepare_state_with_attestations(spec, state, participation_fn=participation_tracker) + proposer_indices = [a.proposer_index for a in state.previous_epoch_attestations] pre_state = state.copy() @@ -233,10 +166,20 @@ def run_with_participation(spec, state, participation_fn): assert len(attesting_indices) == len(participated) for index in range(len(pre_state.validators)): - if index in participated: - assert state.balances[index] > pre_state.balances[index] + if spec.is_in_inactivity_leak(state): + # Proposers can still make money during a leak + if index in proposer_indices and index in participated: + assert state.balances[index] > pre_state.balances[index] + # If not proposer but participated optimally, should have exactly neutral balance + elif index in attesting_indices: + assert state.balances[index] == pre_state.balances[index] + else: + assert state.balances[index] < pre_state.balances[index] else: - assert state.balances[index] < pre_state.balances[index] + if index in participated: + assert state.balances[index] > pre_state.balances[index] + else: + assert state.balances[index] < pre_state.balances[index] @with_all_phases @@ -246,6 +189,14 @@ def test_almost_empty_attestations(spec, state): yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, 1)) +@with_all_phases +@spec_state_test +@leaking() +def test_almost_empty_attestations_with_leak(spec, state): + rng = Random(1234) + yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, 1)) + + @with_all_phases @spec_state_test def test_random_fill_attestations(spec, state): @@ -253,6 +204,14 @@ def test_random_fill_attestations(spec, state): yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, len(comm) // 3)) +@with_all_phases +@spec_state_test +@leaking() +def test_random_fill_attestations_with_leak(spec, state): + rng = Random(4567) + yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, len(comm) // 3)) + + @with_all_phases @spec_state_test def test_almost_full_attestations(spec, state): @@ -260,12 +219,27 @@ def test_almost_full_attestations(spec, state): yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, len(comm) - 1)) +@with_all_phases +@spec_state_test +@leaking() +def test_almost_full_attestations_with_leak(spec, state): + rng = Random(8901) + yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, len(comm) - 1)) + + @with_all_phases @spec_state_test def test_full_attestation_participation(spec, state): yield from run_with_participation(spec, state, lambda slot, comm_index, comm: comm) +@with_all_phases +@spec_state_test +@leaking() +def test_full_attestation_participation_with_leak(spec, state): + yield from run_with_participation(spec, state, lambda slot, comm_index, comm: comm) + + @with_all_phases @spec_state_test def test_duplicate_attestation(spec, state): From c9f21f1f43d44b1cce67363aa8606eaf66e60684 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 20 May 2020 10:44:08 -0600 Subject: [PATCH 034/165] clarify that eth1 blocks must be at a safe fllow distance before being considered for genesis --- specs/phase0/beacon-chain.md | 5 ++++- specs/phase0/validator.md | 2 -- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 142cf3b02..0e4e2b7a0 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -183,6 +183,7 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | | - | - | +| `ETH1_FOLLOW_DISTANCE` | `2**10` (= 1,024) | | `MAX_COMMITTEES_PER_SLOT` | `2**6` (= 64) | | `TARGET_COMMITTEE_SIZE` | `2**7` (= 128) | | `MAX_VALIDATORS_PER_COMMITTEE` | `2**11` (= 2,048) | @@ -219,6 +220,7 @@ The following values are (non-configurable) constants used throughout the specif | - | - | :-: | :-: | | `MIN_GENESIS_DELAY` | `86400` | seconds | 1 day | | `SECONDS_PER_SLOT` | `12` | seconds | 12 seconds | +| `SECONDS_PER_ETH1_BLOCK` | `14` | seconds | | | `MIN_ATTESTATION_INCLUSION_DELAY` | `2**0` (= 1) | slots | 12 seconds | | `SLOTS_PER_EPOCH` | `2**5` (= 32) | slots | 6.4 minutes | | `MIN_SEED_LOOKAHEAD` | `2**0` (= 1) | epochs | 6.4 minutes | @@ -229,7 +231,6 @@ The following values are (non-configurable) constants used throughout the specif | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours | | `SHARD_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | - ### State list lengths | Name | Value | Unit | Duration | @@ -1136,6 +1137,8 @@ Before the Ethereum 2.0 genesis has been triggered, and for every Ethereum 1.0 b - `eth1_timestamp` is the Unix timestamp corresponding to `eth1_block_hash` - `deposits` is the sequence of all deposits, ordered chronologically, up to (and including) the block with hash `eth1_block_hash` +Eth1 blocks must only be considered once they are at least `SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE` seconds old (i.e. `eth1_timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE <= current_unix_time`). Due to this constraint, if `MIN_GENESIS_DELAY < SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE`, then the `genesis_time` can happen before the time/state is first known. Values should be configured to avoid this case. + ```python def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32, eth1_timestamp: uint64, diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index adf23c840..80e9d48c5 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -85,11 +85,9 @@ All terminology, constants, functions, and protocol mechanics defined in the [Ph | Name | Value | Unit | Duration | | - | - | :-: | :-: | -| `ETH1_FOLLOW_DISTANCE` | `2**10` (= 1,024) | blocks | ~4 hours | | `TARGET_AGGREGATORS_PER_COMMITTEE` | `2**4` (= 16) | validators | | | `RANDOM_SUBNETS_PER_VALIDATOR` | `2**0` (= 1) | subnets | | | `EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | epochs | ~27 hours | -| `SECONDS_PER_ETH1_BLOCK` | `14` | seconds | | | `ATTESTATION_SUBNET_COUNT` | `64` | The number of attestation subnets used in the gossipsub protocol. | ## Becoming a validator From f72d14a7477d388e8e441b5ef7b2ebc1220d93c7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 21 May 2020 01:35:40 +0800 Subject: [PATCH 035/165] Bump `milagro_bls_binding` to 1.2.0 Also verify it in BLS test generator --- setup.py | 3 +-- tests/core/pyspec/eth2spec/utils/bls.py | 5 +++-- tests/generators/bls/main.py | 29 ++++++++++++++++++++++--- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index cc5e39865..792ef42fa 100644 --- a/setup.py +++ b/setup.py @@ -503,8 +503,7 @@ setup( "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.9.4", "py_ecc==4.0.0", - "milagro_bls_binding==1.0.2", - "py_ecc==4.0.0", + "milagro_bls_binding==1.2.0", "dataclasses==0.6", "remerkleable==0.1.13", "ruamel.yaml==0.16.5", diff --git a/tests/core/pyspec/eth2spec/utils/bls.py b/tests/core/pyspec/eth2spec/utils/bls.py index 320e3bc93..778b23da7 100644 --- a/tests/core/pyspec/eth2spec/utils/bls.py +++ b/tests/core/pyspec/eth2spec/utils/bls.py @@ -10,7 +10,8 @@ bls = py_ecc_bls STUB_SIGNATURE = b'\x11' * 96 STUB_PUBKEY = b'\x22' * 48 -STUB_COORDINATES = _signature_to_G2(bls.Sign(0, b"")) +Z2_SIGNATURE = b'\xc0' + b'\x00' * 95 +STUB_COORDINATES = _signature_to_G2(Z2_SIGNATURE) def only_with_bls(alt_return=None): @@ -67,7 +68,7 @@ def Sign(SK, message): if bls == py_ecc_bls: return bls.Sign(SK, message) else: - return bls.Sign(SK.to_bytes(48, 'big'), message) + return bls.Sign(SK.to_bytes(32, 'big'), message) @only_with_bls(alt_return=STUB_COORDINATES) diff --git a/tests/generators/bls/main.py b/tests/generators/bls/main.py index 8c6589b36..9e10b4044 100644 --- a/tests/generators/bls/main.py +++ b/tests/generators/bls/main.py @@ -2,18 +2,22 @@ BLS test vectors generator """ +from hashlib import sha256 from typing import Tuple, Iterable, Any, Callable, Dict from eth_utils import ( encode_hex, int_to_big_endian, ) -from gen_base import gen_runner, gen_typing +import milagro_bls_binding as milagro_bls from eth2spec.utils import bls -from hashlib import sha256 - from eth2spec.test.context import PHASE0 +from gen_base import gen_runner, gen_typing + + +def to_bytes(i): + return i.to_bytes(32, "big") def hash(x): @@ -70,8 +74,15 @@ def case02_verify(): # Valid signature signature = bls.Sign(privkey, message) pubkey = bls.SkToPk(privkey) + + assert milagro_bls.SkToPk(to_bytes(privkey)) == pubkey + assert milagro_bls.Sign(to_bytes(privkey), message) == signature + identifier = f'{encode_hex(pubkey)}_{encode_hex(message)}' + assert bls.Verify(pubkey, message, signature) + assert milagro_bls.Verify(pubkey, message, signature) + yield f'verify_valid_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkey': encode_hex(pubkey), @@ -85,6 +96,7 @@ def case02_verify(): wrong_pubkey = bls.SkToPk(PRIVKEYS[(i + 1) % len(PRIVKEYS)]) identifier = f'{encode_hex(wrong_pubkey)}_{encode_hex(message)}' assert not bls.Verify(wrong_pubkey, message, signature) + assert not milagro_bls.Verify(wrong_pubkey, message, signature) yield f'verify_wrong_pubkey_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkey': encode_hex(wrong_pubkey), @@ -98,6 +110,7 @@ def case02_verify(): tampered_signature = signature[:-4] + b'\xFF\xFF\xFF\xFF' identifier = f'{encode_hex(pubkey)}_{encode_hex(message)}' assert not bls.Verify(pubkey, message, tampered_signature) + assert not milagro_bls.Verify(pubkey, message, tampered_signature) yield f'verify_tampered_signature_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkey': encode_hex(pubkey), @@ -109,6 +122,7 @@ def case02_verify(): # Valid pubkey and signature with the point at infinity assert bls.Verify(Z1_PUBKEY, message, Z2_SIGNATURE) + assert milagro_bls.Verify(Z1_PUBKEY, message, Z2_SIGNATURE) yield f'verify_infinity_pubkey_and_infinity_signature', { 'input': { 'pubkey': encode_hex(Z1_PUBKEY), @@ -152,6 +166,7 @@ def case04_fast_aggregate_verify(): # Valid signature identifier = f'{pubkeys_serial}_{encode_hex(message)}' assert bls.FastAggregateVerify(pubkeys, message, aggregate_signature) + assert milagro_bls.FastAggregateVerify(pubkeys, message, aggregate_signature) yield f'fast_aggregate_verify_valid_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkeys': pubkeys_serial, @@ -166,6 +181,7 @@ def case04_fast_aggregate_verify(): pubkeys_extra_serial = [encode_hex(pubkey) for pubkey in pubkeys_extra] identifier = f'{pubkeys_extra_serial}_{encode_hex(message)}' assert not bls.FastAggregateVerify(pubkeys_extra, message, aggregate_signature) + assert not milagro_bls.FastAggregateVerify(pubkeys_extra, message, aggregate_signature) yield f'fast_aggregate_verify_extra_pubkey_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkeys': pubkeys_extra_serial, @@ -179,6 +195,7 @@ def case04_fast_aggregate_verify(): tampered_signature = aggregate_signature[:-4] + b'\xff\xff\xff\xff' identifier = f'{pubkeys_serial}_{encode_hex(message)}' assert not bls.FastAggregateVerify(pubkeys, message, tampered_signature) + assert not milagro_bls.FastAggregateVerify(pubkeys, message, tampered_signature) yield f'fast_aggregate_verify_tampered_signature_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkeys': pubkeys_serial, @@ -190,6 +207,7 @@ def case04_fast_aggregate_verify(): # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == Z1_SIGNATURE assert not bls.FastAggregateVerify([], message, Z2_SIGNATURE) + assert not milagro_bls.FastAggregateVerify([], message, Z2_SIGNATURE) yield f'fast_aggregate_verify_na_pubkeys_and_infinity_signature', { 'input': { 'pubkeys': [], @@ -201,6 +219,7 @@ def case04_fast_aggregate_verify(): # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == 0x00... assert not bls.FastAggregateVerify([], message, NO_SIGNATURE) + assert not milagro_bls.FastAggregateVerify([], message, NO_SIGNATURE) yield f'fast_aggregate_verify_na_pubkeys_and_na_signature', { 'input': { 'pubkeys': [], @@ -228,6 +247,7 @@ def case05_aggregate_verify(): aggregate_signature = bls.Aggregate(sigs) assert bls.AggregateVerify(pubkeys, messages, aggregate_signature) + assert milagro_bls.AggregateVerify(pubkeys, messages, aggregate_signature) yield f'aggregate_verify_valid', { 'input': { 'pubkeys': pubkeys_serial, @@ -239,6 +259,7 @@ def case05_aggregate_verify(): tampered_signature = aggregate_signature[:4] + b'\xff\xff\xff\xff' assert not bls.AggregateVerify(pubkey, messages, tampered_signature) + assert not milagro_bls.AggregateVerify(pubkeys, messages, tampered_signature) yield f'aggregate_verify_tampered_signature', { 'input': { 'pubkeys': pubkeys_serial, @@ -250,6 +271,7 @@ def case05_aggregate_verify(): # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == Z1_SIGNATURE assert not bls.AggregateVerify([], [], Z2_SIGNATURE) + assert not milagro_bls.AggregateVerify([], [], Z2_SIGNATURE) yield f'aggregate_verify_na_pubkeys_and_infinity_signature', { 'input': { 'pubkeys': [], @@ -261,6 +283,7 @@ def case05_aggregate_verify(): # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == 0x00... assert not bls.AggregateVerify([], [], NO_SIGNATURE) + assert not milagro_bls.AggregateVerify([], [], NO_SIGNATURE) yield f'aggregate_verify_na_pubkeys_and_na_signature', { 'input': { 'pubkeys': [], From db1a90d2eee58bd48985d6d39b766686aa638852 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 21 May 2020 02:05:22 +0800 Subject: [PATCH 036/165] `test_success_surround` changes the signing data of attestation, so it should be never_bls --- .../phase_0/block_processing/test_process_attester_slashing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index 11ead6033..8d7638f51 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -1,6 +1,6 @@ from eth2spec.test.context import ( PHASE0, PHASE1, - spec_state_test, expect_assertion_error, always_bls, with_all_phases, with_phases + spec_state_test, expect_assertion_error, always_bls, never_bls, with_all_phases, with_phases ) from eth2spec.test.helpers.attestations import sign_indexed_attestation from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing, \ @@ -89,6 +89,7 @@ def test_success_double(spec, state): @with_all_phases @spec_state_test +@never_bls def test_success_surround(spec, state): next_epoch_via_block(spec, state) From 4ac2fc7eff6ee437cc8cb796ca35e0cea2984dff Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 20 May 2020 12:28:08 -0600 Subject: [PATCH 037/165] add missing column description fo SECONDS_PER_ETH1_BLOCK Co-authored-by: Diederik Loerakker --- specs/phase0/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 0e4e2b7a0..419c88e65 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -220,7 +220,7 @@ The following values are (non-configurable) constants used throughout the specif | - | - | :-: | :-: | | `MIN_GENESIS_DELAY` | `86400` | seconds | 1 day | | `SECONDS_PER_SLOT` | `12` | seconds | 12 seconds | -| `SECONDS_PER_ETH1_BLOCK` | `14` | seconds | | +| `SECONDS_PER_ETH1_BLOCK` | `14` | seconds | 14 seconds | | `MIN_ATTESTATION_INCLUSION_DELAY` | `2**0` (= 1) | slots | 12 seconds | | `SLOTS_PER_EPOCH` | `2**5` (= 32) | slots | 6.4 minutes | | `MIN_SEED_LOOKAHEAD` | `2**0` (= 1) | epochs | 6.4 minutes | From 607e23949c6e489c2f399bbbbaab7247cd8fc1be Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Tue, 19 May 2020 07:51:53 +0200 Subject: [PATCH 038/165] require blocks to be ordered consecutively in block range request Per the spec, if I request range 5-10, it is permissible for a client to answer with block 7, 9 - even if the blocks 5, 6 and 8 exist. Because blocks 7 and 9 cannot be validated as they arrive in such a request, it seems better to close this gap - this update adds the spec language that forbids well-behaving clients from answering this way. --- specs/phase0/p2p-interface.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 0e8699555..fc457c58a 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -542,12 +542,14 @@ The response MUST consist of zero or more `response_chunk`. Each _successful_ `r Clients MUST keep a record of signed blocks seen since the since the start of the weak subjectivity period and MUST support serving requests of blocks up to their own `head_block_root`. -Clients MUST respond with at least one block, if they have it and it exists in the range. Clients MAY limit the number of blocks in the response. +Clients MUST respond with at least the first block that exists in the range, if they have it. + +The following blocks MUST be ordered consecutively, with each `parent_root` matching the `hash_tree_root` of the previous block. + +Clients MAY limit the number of blocks in the response. The response MUST contain no more than `count` blocks. -Clients MUST order blocks by increasing slot number. - Clients MUST respond with blocks from their view of the current fork choice -- that is, blocks from the single chain defined by the current head. Of note, blocks from slots before the finalization MUST lead to the finalized block reported in the `Status` handshake. Clients MUST respond with blocks that are consistent from a single chain within the context of the request. After the initial block, clients MAY stop in the process of responding if their fork choice changes the view of the chain in the context of the request. From a29cbebc0e314892865667b23db8ff75f76a3983 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Wed, 20 May 2020 09:51:47 +0200 Subject: [PATCH 039/165] cover `step` parameter in stricter range request --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index fc457c58a..787c94c34 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -544,7 +544,7 @@ Clients MUST keep a record of signed blocks seen since the since the start of th Clients MUST respond with at least the first block that exists in the range, if they have it. -The following blocks MUST be ordered consecutively, with each `parent_root` matching the `hash_tree_root` of the previous block. +The following blocks, where they exist, MUST be send in consecutive order without gaps, as selected by `step`. In particular, when `step == 1`, each `parent_root` MUST match the `hash_tree_root` of the preceding block. Clients MAY limit the number of blocks in the response. From 59a43142c2f9181b05006638019a24dfb5550a35 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 20 May 2020 20:39:52 +0200 Subject: [PATCH 040/165] Rebased on latest BlocksByRange spec, fix conflicts, clarify single chain, even with higher step --- specs/phase0/p2p-interface.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 787c94c34..e536c239a 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -544,7 +544,7 @@ Clients MUST keep a record of signed blocks seen since the since the start of th Clients MUST respond with at least the first block that exists in the range, if they have it. -The following blocks, where they exist, MUST be send in consecutive order without gaps, as selected by `step`. In particular, when `step == 1`, each `parent_root` MUST match the `hash_tree_root` of the preceding block. +The following blocks, where they exist, MUST be send in consecutive order. Clients MAY limit the number of blocks in the response. @@ -552,7 +552,10 @@ The response MUST contain no more than `count` blocks. Clients MUST respond with blocks from their view of the current fork choice -- that is, blocks from the single chain defined by the current head. Of note, blocks from slots before the finalization MUST lead to the finalized block reported in the `Status` handshake. -Clients MUST respond with blocks that are consistent from a single chain within the context of the request. After the initial block, clients MAY stop in the process of responding if their fork choice changes the view of the chain in the context of the request. +Clients MUST respond with blocks that are consistent from a single chain within the context of the request. +This applies to any `step` value. In particular when `step == 1`, each `parent_root` MUST match the `hash_tree_root` of the preceding block. + +After the initial block, clients MAY stop in the process of responding if their fork choice changes the view of the chain in the context of the request. #### BeaconBlocksByRoot From 522e34efcfca10f0c8f53e7757cdd172e119b5e7 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 20 May 2020 20:46:08 +0200 Subject: [PATCH 041/165] Fix markdown, use multiple lines for change-control, and add step >= 1 case --- specs/phase0/p2p-interface.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index e536c239a..118be8839 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -532,7 +532,10 @@ Response Content: ) ``` -Requests beacon blocks in the slot range `[start_slot, start_slot + count * step)`, leading up to the current head block as selected by fork choice. `step` defines the slot increment between blocks. For example, requesting blocks starting at `start_slot` 2 with a step value of 2 would return the blocks at slots [2, 4, 6, …]. In cases where a slot is empty for a given slot number, no block is returned. For example, if slot 4 were empty in the previous example, the returned array would contain [2, 6, …]. +Requests beacon blocks in the slot range `[start_slot, start_slot + count * step)`, leading up to the current head block as selected by fork choice. +`step` defines the slot increment between blocks. For example, requesting blocks starting at `start_slot` 2 with a step value of 2 would return the blocks at slots [2, 4, 6, …]. +In cases where a slot is empty for a given slot number, no block is returned. For example, if slot 4 were empty in the previous example, the returned array would contain [2, 6, …]. +A request MUST NOT have a 0 slot increment, i.e. `step >= 1`. `BeaconBlocksByRange` is primarily used to sync historical blocks. From 763d74bbf559e6ed73d50180a617f33ddcfac3b2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 21 May 2020 02:51:38 +0800 Subject: [PATCH 042/165] Just learned bls was disabled by default; fixing the tests --- .../test/validator/test_validator_unittest.py | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py index 5bb246ed5..f21be8cd3 100644 --- a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py +++ b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import spec_state_test, never_bls, with_all_phases +from eth2spec.test.context import spec_state_test, always_bls, with_all_phases from eth2spec.test.helpers.attestations import build_attestation_data from eth2spec.test.helpers.block import build_empty_block from eth2spec.test.helpers.deposits import prepare_state_and_deposit @@ -8,9 +8,11 @@ from eth2spec.utils import bls from eth2spec.utils.ssz.ssz_typing import Bitlist -def run_get_signature_test(spec, state, obj, domain, get_signature_fn, privkey, pubkey): +def run_get_signature_test(spec, state, obj, domain, get_signature_fn, privkey, pubkey, signing_ssz_object=None): + if signing_ssz_object is None: + signing_ssz_object = obj signature = get_signature_fn(state, obj, privkey) - signing_root = spec.compute_signing_root(obj, domain) + signing_root = spec.compute_signing_root(signing_ssz_object, domain) assert bls.Verify(pubkey, signing_root, signature) @@ -55,7 +57,6 @@ def get_mock_aggregate(spec): @with_all_phases @spec_state_test -@never_bls def test_check_if_validator_active(spec, state): active_validator_index = len(state.validators) - 1 assert spec.check_if_validator_active(state, active_validator_index) @@ -73,7 +74,6 @@ def test_check_if_validator_active(spec, state): @with_all_phases @spec_state_test -@never_bls def test_get_committee_assignment_current_epoch(spec, state): epoch = spec.get_current_epoch(state) validator_index = len(state.validators) - 1 @@ -82,7 +82,6 @@ def test_get_committee_assignment_current_epoch(spec, state): @with_all_phases @spec_state_test -@never_bls def test_get_committee_assignment_next_epoch(spec, state): epoch = spec.get_current_epoch(state) + 1 validator_index = len(state.validators) - 1 @@ -91,7 +90,6 @@ def test_get_committee_assignment_next_epoch(spec, state): @with_all_phases @spec_state_test -@never_bls def test_get_committee_assignment_out_bound_epoch(spec, state): epoch = spec.get_current_epoch(state) + 2 validator_index = len(state.validators) - 1 @@ -100,7 +98,6 @@ def test_get_committee_assignment_out_bound_epoch(spec, state): @with_all_phases @spec_state_test -@never_bls def test_is_proposer(spec, state): proposer_index = spec.get_beacon_proposer_index(state) assert spec.is_proposer(state, proposer_index) @@ -132,6 +129,7 @@ def test_get_epoch_signature(spec, state): get_signature_fn=spec.get_epoch_signature, privkey=privkey, pubkey=pubkey, + signing_ssz_object=spec.compute_epoch_at_slot(block.slot), ) @@ -256,6 +254,7 @@ def test_compute_new_state_root(spec, state): @with_all_phases @spec_state_test +@always_bls def test_get_block_signature(spec, state): privkey = privkeys[0] pubkey = pubkeys[0] @@ -277,6 +276,7 @@ def test_get_block_signature(spec, state): @with_all_phases @spec_state_test +@always_bls def test_get_attestation_signature(spec, state): privkey = privkeys[0] pubkey = pubkeys[0] @@ -298,6 +298,7 @@ def test_get_attestation_signature(spec, state): @with_all_phases @spec_state_test +@always_bls def test_get_slot_signature(spec, state): privkey = privkeys[0] pubkey = pubkeys[0] @@ -316,6 +317,7 @@ def test_get_slot_signature(spec, state): @with_all_phases @spec_state_test +@always_bls def test_is_aggregator(spec, state): # TODO: we can test the probabilistic result against `TARGET_AGGREGATORS_PER_COMMITTEE` # if we have more validators and larger committeee size @@ -334,9 +336,10 @@ def test_is_aggregator(spec, state): @with_all_phases @spec_state_test +@always_bls def test_get_aggregate_signature(spec, state): attestations = [] - pubkeys = [] + attesting_pubkeys = [] slot = state.slot committee_index = 0 attestation_data = build_attestation_data(spec, state, slot=slot, index=committee_index) @@ -348,24 +351,26 @@ def test_get_aggregate_signature(spec, state): committee_size = len(beacon_committee) aggregation_bits = Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE](*([0] * committee_size)) for i, validator_index in enumerate(beacon_committee): - bits = aggregation_bits + bits = aggregation_bits.copy() bits[i] = True attestations.append( spec.Attestation( data=attestation_data, aggregation_bits=bits, + signature=spec.get_attestation_signature(state, attestation_data, privkeys[validator_index]), ) ) - pubkeys.append(state.validators[validator_index].pubkey) - pubkey = bls.AggregatePKs(pubkeys) + attesting_pubkeys.append(state.validators[validator_index].pubkey) + assert len(attestations) > 0 signature = spec.get_aggregate_signature(attestations) domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) signing_root = spec.compute_signing_root(attestation_data, domain) - assert bls.Verify(pubkey, signing_root, signature) + assert bls.FastAggregateVerify(attesting_pubkeys, signing_root, signature) @with_all_phases @spec_state_test +@always_bls def test_get_aggregate_and_proof(spec, state): privkey = privkeys[0] aggregator_index = spec.ValidatorIndex(10) @@ -378,6 +383,7 @@ def test_get_aggregate_and_proof(spec, state): @with_all_phases @spec_state_test +@always_bls def test_get_aggregate_and_proof_signature(spec, state): privkey = privkeys[0] pubkey = pubkeys[0] From d92efdf071920f0717f4609e30e8549cdd270720 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 21 May 2020 03:02:02 +0800 Subject: [PATCH 043/165] Should have signed the attestions in `test_filtered_block_tree` test --- tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py index 17d4f644f..4371bffd0 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py @@ -183,7 +183,7 @@ def test_filtered_block_tree(spec, state): for i in range(spec.SLOTS_PER_EPOCH): slot = rogue_block.slot + i for index in range(spec.get_committee_count_at_slot(non_viable_state, slot)): - attestation = get_valid_attestation(spec, non_viable_state, rogue_block.slot + i, index) + attestation = get_valid_attestation(spec, non_viable_state, slot, index, signed=True) attestations.append(attestation) # tick time forward to be able to include up to the latest attestation From 96f785e84be320229ab921476dc83108105505e3 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 20 May 2020 13:23:59 -0600 Subject: [PATCH 044/165] ensure only forward progress with eth1data voting --- setup.py | 5 +- specs/phase0/validator.md | 14 ++++-- .../test/validator/test_validator_unittest.py | 46 +++++++++++++++++-- 3 files changed, 57 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index da56175af..6d7d47767 100644 --- a/setup.py +++ b/setup.py @@ -150,7 +150,10 @@ def get_eth1_data(block: Eth1Block) -> Eth1Data: """ A stub function return mocking Eth1Data. """ - return Eth1Data(block_hash=hash_tree_root(block)) + return Eth1Data( + deposit_root=block.deposit_root, + deposit_count=block.deposit_count, + block_hash=hash_tree_root(block)) def hash(x: bytes) -> Bytes32: # type: ignore diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 80e9d48c5..5e8ddc977 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -252,11 +252,13 @@ The `block.body.eth1_data` field is for block proposers to vote on recent Eth1 d ###### `Eth1Block` -Let `Eth1Block` be an abstract object representing Eth1 blocks with the `timestamp` field available. +Let `Eth1Block` be an abstract object representing Eth1 blocks with the `timestamp` and depost contract data available. ```python class Eth1Block(Container): timestamp: uint64 + deposit_root: Root + deposit_count: uint64 # All other eth1 block fields ``` @@ -289,8 +291,14 @@ def is_candidate_block(block: Eth1Block, period_start: uint64) -> bool: def get_eth1_vote(state: BeaconState, eth1_chain: Sequence[Eth1Block]) -> Eth1Data: period_start = voting_period_start_time(state) # `eth1_chain` abstractly represents all blocks in the eth1 chain sorted by ascending block height - votes_to_consider = [get_eth1_data(block) for block in eth1_chain if - is_candidate_block(block, period_start)] + votes_to_consider = [ + get_eth1_data(block) for block in eth1_chain + if ( + is_candidate_block(block, period_start) + # Ensure cannot move back to earlier deposit contract states + and get_eth1_data(block).deposit_count >= state.eth1_data.deposit_count + ) + ] # Valid votes already cast during this period valid_votes = [vote for vote in state.eth1_data_votes if vote in votes_to_consider] diff --git a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py index 5bb246ed5..9ef499313 100644 --- a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py +++ b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py @@ -190,8 +190,14 @@ def test_get_eth1_vote_consensus_vote(spec, state): assert votes_length >= 3 # We need to have the majority vote state.eth1_data_votes = () - block_1 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE - 1) - block_2 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE) + block_1 = spec.Eth1Block( + timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE - 1, + deposit_count=state.eth1_data.deposit_count, + ) + block_2 = spec.Eth1Block( + timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE, + deposit_count=state.eth1_data.deposit_count, + ) eth1_chain = [block_1, block_2] eth1_data_votes = [] @@ -218,8 +224,14 @@ def test_get_eth1_vote_tie(spec, state): assert votes_length > 0 and votes_length % 2 == 0 state.eth1_data_votes = () - block_1 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE - 1) - block_2 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE) + block_1 = spec.Eth1Block( + timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE - 1, + deposit_count=state.eth1_data.deposit_count, + ) + block_2 = spec.Eth1Block( + timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE, + deposit_count=state.eth1_data.deposit_count, + ) eth1_chain = [block_1, block_2] eth1_data_votes = [] # Half votes are for block_1, another half votes are for block_2 @@ -237,6 +249,32 @@ def test_get_eth1_vote_tie(spec, state): assert eth1_data.block_hash == eth1_chain[0].hash_tree_root() +@with_all_phases +@spec_state_test +def test_get_eth1_vote_chain_in_past(spec, state): + min_new_period_epochs = get_min_new_period_epochs(spec) + for _ in range(min_new_period_epochs + 1): + next_epoch(spec, state) + + period_start = spec.voting_period_start_time(state) + votes_length = spec.get_current_epoch(state) % spec.EPOCHS_PER_ETH1_VOTING_PERIOD + assert votes_length > 0 and votes_length % 2 == 0 + + state.eth1_data_votes = () + block_1 = spec.Eth1Block( + timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE, + deposit_count=state.eth1_data.deposit_count - 1, # Chain prior to current eth1data + ) + eth1_chain = [block_1] + eth1_data_votes = [] + + state.eth1_data_votes = eth1_data_votes + eth1_data = spec.get_eth1_vote(state, eth1_chain) + + # Should be default vote + assert eth1_data == state.eth1_data + + @with_all_phases @spec_state_test def test_compute_new_state_root(spec, state): From 61336a9c0b254b8dd3ffdaa7a0e7a0d480c0251a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 20 May 2020 14:03:16 -0600 Subject: [PATCH 045/165] add deposit_root to validator unit tets --- .../eth2spec/test/validator/test_validator_unittest.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py index 9ef499313..f79fd3fad 100644 --- a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py +++ b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py @@ -193,10 +193,12 @@ def test_get_eth1_vote_consensus_vote(spec, state): block_1 = spec.Eth1Block( timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE - 1, deposit_count=state.eth1_data.deposit_count, + deposit_root=b'\x04' * 32, ) block_2 = spec.Eth1Block( timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE, - deposit_count=state.eth1_data.deposit_count, + deposit_count=state.eth1_data.deposit_count + 1, + deposit_root=b'\x05' * 32, ) eth1_chain = [block_1, block_2] eth1_data_votes = [] @@ -227,10 +229,12 @@ def test_get_eth1_vote_tie(spec, state): block_1 = spec.Eth1Block( timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE - 1, deposit_count=state.eth1_data.deposit_count, + deposit_root=b'\x04' * 32, ) block_2 = spec.Eth1Block( timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE, - deposit_count=state.eth1_data.deposit_count, + deposit_count=state.eth1_data.deposit_count + 1, + deposit_root=b'\x05' * 32, ) eth1_chain = [block_1, block_2] eth1_data_votes = [] @@ -264,6 +268,7 @@ def test_get_eth1_vote_chain_in_past(spec, state): block_1 = spec.Eth1Block( timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE, deposit_count=state.eth1_data.deposit_count - 1, # Chain prior to current eth1data + deposit_root=b'\x42' * 32, ) eth1_chain = [block_1] eth1_data_votes = [] From 87005c6f10cd67f919c1b286dbfcc767172f416d Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 20 May 2020 22:32:47 +0200 Subject: [PATCH 046/165] milagro bls 1.3 with improved error handling --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 792ef42fa..c88edfe56 100644 --- a/setup.py +++ b/setup.py @@ -503,7 +503,7 @@ setup( "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.9.4", "py_ecc==4.0.0", - "milagro_bls_binding==1.2.0", + "milagro_bls_binding==1.3.0", "dataclasses==0.6", "remerkleable==0.1.13", "ruamel.yaml==0.16.5", From 56309342c05d324e359591b902abc355fba82fb1 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 21 May 2020 15:33:47 +0200 Subject: [PATCH 047/165] Use `Bytes32` for `error_message` `ErrorMessage.error_message` is a leftover from an older version of SSZ that was able to encode unbounded lists. This is no longer the case - all collection types now have a fixed upper bound on length. In general, the `error_message`, just like the `graffitti` field, should not be interpreted in any particular way except for debugging and vanity - as such, using the same type, a `Bytes32`, seems reasonable. An alternative would be `List[byte, 256]` which maybe could be "reasonably backwards compatible" with whatever clients are are doing now - depending on how they are dealing with this field type that no longer exists in the SSZ spec :) It would however be the only place where `List[uintN, N]` is used in the current spec. As an exercise, this could be considered a security issue since it's essentially unbounded and undefined behaviour. --- specs/phase0/p2p-interface.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 118be8839..84db361c9 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -391,11 +391,11 @@ The `ErrorMessage` schema is: ``` ( - error_message: String + error_message: Bytes32 ) ``` -*Note*: The String type is encoded as UTF-8 bytes without NULL terminator when SSZ-encoded. As the `ErrorMessage` is not an SSZ-container, only the UTF-8 bytes will be sent when SSZ-encoded. +*Note*: By convention, the `error_message` is a sequence of bytes that can be interpreted as a UTF-8 string up to 32 bytes - a 0 byte shortens the string in this interpretation. Clients MUST treat as valid any bytes. ### Encoding strategies From 5e8457e62d1272e36de0f55ce3e83543258cddf1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 23 May 2020 21:53:55 +0800 Subject: [PATCH 048/165] Fix phase1 on-time sign_indexed_attestation --- tests/core/pyspec/eth2spec/test/helpers/attestations.py | 5 ++++- .../block_processing/test_process_attester_slashing.py | 3 +-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 79f752411..4d246c8c6 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -211,7 +211,10 @@ def sign_indexed_attestation(spec, state, indexed_attestation): indexed_attestation.attestation.aggregation_bits, ) data = indexed_attestation.attestation.data - indexed_attestation.attestation.signature = sign_aggregate_attestation(spec, state, data, participants) + if any(indexed_attestation.attestation.custody_bits_blocks): + sign_on_time_attestation(spec, state, indexed_attestation.attestation) + else: + indexed_attestation.attestation.signature = sign_aggregate_attestation(spec, state, data, participants) def sign_on_time_attestation(spec, state, attestation): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index 8d7638f51..11ead6033 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -1,6 +1,6 @@ from eth2spec.test.context import ( PHASE0, PHASE1, - spec_state_test, expect_assertion_error, always_bls, never_bls, with_all_phases, with_phases + spec_state_test, expect_assertion_error, always_bls, with_all_phases, with_phases ) from eth2spec.test.helpers.attestations import sign_indexed_attestation from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing, \ @@ -89,7 +89,6 @@ def test_success_double(spec, state): @with_all_phases @spec_state_test -@never_bls def test_success_surround(spec, state): next_epoch_via_block(spec, state) From ce1d22d71c3758b77d3b2e3a0b00efc9d5c5d4cd Mon Sep 17 00:00:00 2001 From: terence tsao Date: Sat, 23 May 2020 15:22:49 -0700 Subject: [PATCH 049/165] Use helper `compute_previous_slot` --- specs/phase1/beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 3bb01e262..d14e3bce4 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -801,7 +801,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr # Save updated state state.shard_states[shard] = transition.shard_states[len(transition.shard_states) - 1] - state.shard_states[shard].slot = state.slot - 1 + state.shard_states[shard].slot = compute_previous_slot(state.slot) ``` ###### `process_crosslink_for_shard` @@ -953,7 +953,7 @@ def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSla def verify_shard_transition_false_positives(state: BeaconState, block_body: BeaconBlockBody) -> None: # Verify that a `shard_transition` in a block is empty if an attestation was not processed for it for shard in range(get_active_shard_count(state)): - if state.shard_states[shard].slot != state.slot - 1: + if state.shard_states[shard].slot != compute_previous_slot(state.slot): assert block_body.shard_transitions[shard] == ShardTransition() ``` From d3c26d6b8b16f130a340568b3caf93a31ae2203e Mon Sep 17 00:00:00 2001 From: ericsson Date: Mon, 25 May 2020 18:45:38 +0300 Subject: [PATCH 050/165] `compute_shard_transition_digest` expects `Root` as a fourth parameter --- specs/phase1/shard-transition.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 55b867faa..da3aa648c 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -81,7 +81,7 @@ def shard_state_transition(beacon_state: BeaconState, beacon_state, shard_state, block.beacon_parent_root, - block.body, + hash_tree_root(block.body), ) shard_state.gasprice = compute_updated_gasprice(prev_gasprice, len(block.body)) shard_state.slot = block.slot From 75787d92a8645751338350ea7d95cf27cb8119fa Mon Sep 17 00:00:00 2001 From: Ali Atiia <42751398+aliatiia@users.noreply.github.com> Date: Mon, 25 May 2020 16:45:44 -0400 Subject: [PATCH 051/165] broken link broken link to custory-game.md --- specs/phase1/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 3bb01e262..321b828cc 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -997,7 +997,7 @@ def process_epoch(state: BeaconState) -> None: #### Custody game updates -`process_reveal_deadlines` and `process_custody_final_updates` are defined in [the Custody Game spec](./1_custody-game.md), +`process_reveal_deadlines` and `process_custody_final_updates` are defined in [the Custody Game spec](./custody-game.md), #### Online-tracking From 75633cfcf1139858d002fe01c68d5f37d22bc28b Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 26 May 2020 11:00:02 -0700 Subject: [PATCH 052/165] Update link so gossipsub encodings match the correct section --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 118be8839..a994b0c66 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -203,7 +203,7 @@ Topics are plain UTF-8 strings and are encoded on the wire as determined by prot - `current_fork_version` is the fork version of the epoch of the message to be sent on the topic - `genesis_validators_root` is the static `Root` found in `state.genesis_validators_root` - `Name` - see table below -- `Encoding` - the encoding strategy describes a specific representation of bytes that will be transmitted over the wire. See the [Encodings](#Encoding-strategies) section for further details. +- `Encoding` - the encoding strategy describes a specific representation of bytes that will be transmitted over the wire. See the [Encodings](#Encodings) section for further details. *Note*: `ForkDigestValue` is composed of values that are not known until the genesis block/state are available. Due to this, clients SHOULD NOT subscribe to gossipsub topics until these genesis values are known. From c437578280a3be99c5f2e9c61ad39ff14dbbdaa7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 28 May 2020 21:32:27 +0800 Subject: [PATCH 053/165] Add `shard` field to `ShardBlock` --- specs/phase1/beacon-chain.md | 5 ++++- specs/phase1/shard-transition.md | 1 + tests/core/pyspec/eth2spec/test/helpers/shard_block.py | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 0ae4bae2b..39a73c0aa 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -302,6 +302,7 @@ class ShardBlock(Container): shard_parent_root: Root beacon_parent_root: Root slot: Slot + shard: Shard proposer_index: ValidatorIndex body: ByteList[MAX_SHARD_BLOCK_SIZE] ``` @@ -321,6 +322,7 @@ class ShardBlockHeader(Container): shard_parent_root: Root beacon_parent_root: Root slot: Slot + shard: Shard proposer_index: ValidatorIndex body_root: Root ``` @@ -781,8 +783,9 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr header = ShardBlockHeader( shard_parent_root=shard_parent_root, beacon_parent_root=get_block_root_at_slot(state, offset_slots[i]), - proposer_index=proposal_index, slot=offset_slots[i], + shard=shard, + proposer_index=proposal_index, body_root=transition.shard_data_roots[i] ) shard_parent_root = hash_tree_root(header) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index da3aa648c..53402bb2a 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -50,6 +50,7 @@ def verify_shard_block_message(beacon_state: BeaconState, shard: Shard) -> bool: assert block.shard_parent_root == shard_state.latest_block_root assert block.slot == slot + assert block.shard == shard assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard) assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE return True diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index ef65d2427..805b955f7 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -38,6 +38,7 @@ def build_shard_block(spec, shard_parent_root=shard_state.latest_block_root, beacon_parent_root=beacon_parent_root, slot=slot, + shard=shard, proposer_index=proposer_index, body=body, ) From ca489630325aebffca2a425fe1a5b69573140e6e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 28 May 2020 21:38:11 +0800 Subject: [PATCH 054/165] Rename `head_shard_root` to `shard_head_root` --- specs/phase1/beacon-chain.md | 4 ++-- tests/core/pyspec/eth2spec/test/helpers/attestations.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 0ae4bae2b..bc5a6385c 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -130,7 +130,7 @@ class AttestationData(Container): source: Checkpoint target: Checkpoint # Current-slot shard block root - head_shard_root: Root + shard_head_root: Root # Shard transition root shard_transition_root: Root ``` @@ -823,7 +823,7 @@ def process_crosslink_for_shard(state: BeaconState, for attestation in transition_attestations: participants = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) transition_participants = transition_participants.union(participants) - assert attestation.data.head_shard_root == shard_transition.shard_data_roots[ + assert attestation.data.shard_head_root == shard_transition.shard_data_roots[ len(shard_transition.shard_data_roots) - 1 ] diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 4d246c8c6..c533182ef 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -78,7 +78,7 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t if spec.fork == PHASE1: if shard_transition is not None: lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 - attestation_data.head_shard_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] + attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() else: # No shard transition @@ -88,10 +88,10 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t next_slot(spec, temp_state) shard_transition = spec.get_shard_transition(temp_state, shard, []) lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 - attestation_data.head_shard_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] + attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() else: - attestation_data.head_shard_root = state.shard_states[shard].transition_digest + attestation_data.shard_head_root = state.shard_states[shard].transition_digest attestation_data.shard_transition_root = spec.Root() return attestation_data From 8c9bbc48d8720ed621ba54ab7ae12696ca5279e8 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 28 May 2020 21:49:36 +0800 Subject: [PATCH 055/165] Rework `is_shard_attestation` Change it to `is_on_time_attestation` so that it could be reused in `validate_attestation`. --- specs/phase1/beacon-chain.md | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index bc5a6385c..4f4713f70 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -53,7 +53,7 @@ - [`get_offset_slots`](#get_offset_slots) - [Predicates](#predicates) - [Updated `is_valid_indexed_attestation`](#updated-is_valid_indexed_attestation) - - [`is_shard_attestation`](#is_shard_attestation) + - [`is_on_time_attestation`](#is_on_time_attestation) - [`is_winning_attestation`](#is_winning_attestation) - [`optional_aggregate_verify`](#optional_aggregate_verify) - [`optional_fast_aggregate_verify`](#optional_fast_aggregate_verify) @@ -602,20 +602,16 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe return bls.AggregateVerify(all_pubkeys, all_signing_roots, signature=attestation.signature) ``` -#### `is_shard_attestation` +#### `is_on_time_attestation` ```python -def is_shard_attestation(state: BeaconState, - attestation: Attestation, - committee_index: CommitteeIndex) -> bool: - if not ( - attestation.data.index == committee_index - and attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY == state.slot # Must be on-time attestation - # TODO: MIN_ATTESTATION_INCLUSION_DELAY should always be 1 - ): - return False - - return True +def is_on_time_attestation(state: BeaconState, + attestation: Attestation) -> bool: + """ + Check if the given attestation is on-time. + """ + # TODO: MIN_ATTESTATION_INCLUSION_DELAY should always be 1 + return attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY == state.slot ``` #### `is_winning_attestation` @@ -730,7 +726,7 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: # Type 1: on-time attestations, the custody bits should be non-empty. if attestation.custody_bits_blocks != []: # Ensure on-time attestation - assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY == state.slot + assert is_on_time_attestation(state, attestation) # Correct data root count assert len(attestation.custody_bits_blocks) == len(get_offset_slots(state, shard)) # Correct parent block root @@ -875,7 +871,7 @@ def process_crosslinks(state: BeaconState, shard_transition = shard_transitions[shard] shard_attestations = [ attestation for attestation in attestations - if is_shard_attestation(state, attestation, committee_index) + if is_on_time_attestation(state, attestation) and attestation.data.index == committee_index ] winning_root = process_crosslink_for_shard(state, committee_index, shard_transition, shard_attestations) From 19262888e4997659f1a6c015562284213526d0fa Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 28 May 2020 21:55:49 +0800 Subject: [PATCH 056/165] Rename `verify_shard_transition_false_positives` to `verify_empty_shard_transition` --- specs/phase1/beacon-chain.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 4f4713f70..e536ad6ef 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -66,7 +66,7 @@ - [`process_crosslinks`](#process_crosslinks) - [`process_attestation`](#process_attestation) - [New Attester slashing processing](#new-attester-slashing-processing) - - [Shard transition false positives](#shard-transition-false-positives) + - [Verify empty shard transition](#verify-empty-shard-transition) - [Light client processing](#light-client-processing) - [Epoch transition](#epoch-transition) - [Custody game updates](#custody-game-updates) @@ -671,7 +671,7 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_eth1_data(state, block.body) process_light_client_signatures(state, block.body) process_operations(state, block.body) - verify_shard_transition_false_positives(state, block.body) + verify_empty_shard_transition(state, block.body) ``` #### Operations @@ -943,11 +943,13 @@ def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSla assert slashed_any ``` -#### Shard transition false positives +#### Verify empty shard transition ```python -def verify_shard_transition_false_positives(state: BeaconState, block_body: BeaconBlockBody) -> None: - # Verify that a `shard_transition` in a block is empty if an attestation was not processed for it +def verify_empty_shard_transition(state: BeaconState, block_body: BeaconBlockBody) -> None: + """ + Verify that a `shard_transition` in a block is empty if an attestation was not processed for it. + """ for shard in range(get_active_shard_count(state)): if state.shard_states[shard].slot != compute_previous_slot(state.slot): assert block_body.shard_transitions[shard] == ShardTransition() From 7509ecb742173e0df6768d18fdc4d07686ae5184 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 28 May 2020 22:39:52 +0800 Subject: [PATCH 057/165] Add comments, minor refactoring --- specs/phase1/beacon-chain.md | 75 ++++++++++++++++++-------------- specs/phase1/shard-transition.md | 6 +-- 2 files changed, 44 insertions(+), 37 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 0ae4bae2b..89dd6aaac 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -64,7 +64,7 @@ - [`apply_shard_transition`](#apply_shard_transition) - [`process_crosslink_for_shard`](#process_crosslink_for_shard) - [`process_crosslinks`](#process_crosslinks) - - [`process_attestation`](#process_attestation) + - [Updated `process_attestation`](#updated-process_attestation) - [New Attester slashing processing](#new-attester-slashing-processing) - [Shard transition false positives](#shard-transition-false-positives) - [Light client processing](#light-client-processing) @@ -153,6 +153,7 @@ class PendingAttestation(Container): data: AttestationData inclusion_delay: Slot proposer_index: ValidatorIndex + # Phase 1 crosslink_success: boolean ``` @@ -445,13 +446,13 @@ def compute_offset_slots(start_slot: Slot, end_slot: Slot) -> Sequence[Slot]: #### `compute_updated_gasprice` ```python -def compute_updated_gasprice(prev_gasprice: Gwei, length: uint8) -> Gwei: - if length > TARGET_SHARD_BLOCK_SIZE: - delta = (prev_gasprice * (length - TARGET_SHARD_BLOCK_SIZE) +def compute_updated_gasprice(prev_gasprice: Gwei, shard_block_length: uint8) -> Gwei: + if shard_block_length > TARGET_SHARD_BLOCK_SIZE: + delta = (prev_gasprice * (shard_block_length - TARGET_SHARD_BLOCK_SIZE) // TARGET_SHARD_BLOCK_SIZE // GASPRICE_ADJUSTMENT_COEFFICIENT) return min(prev_gasprice + delta, MAX_GASPRICE) else: - delta = (prev_gasprice * (TARGET_SHARD_BLOCK_SIZE - length) + delta = (prev_gasprice * (TARGET_SHARD_BLOCK_SIZE - shard_block_length) // TARGET_SHARD_BLOCK_SIZE // GASPRICE_ADJUSTMENT_COEFFICIENT) return max(prev_gasprice, MIN_GASPRICE + delta) - delta ``` @@ -477,9 +478,12 @@ def get_online_validator_indices(state: BeaconState) -> Set[ValidatorIndex]: ```python def get_shard_committee(beacon_state: BeaconState, epoch: Epoch, shard: Shard) -> Sequence[ValidatorIndex]: + """ + Return the shard committee of the given ``epoch`` of the given ``shard``. + """ source_epoch = epoch - epoch % SHARD_COMMITTEE_PERIOD if source_epoch >= SHARD_COMMITTEE_PERIOD: - source_epoch -= SHARD_COMMITTEE_PERIOD + source_epoch -= SHARD_COMMITTEE_PERIOD # `SHARD_COMMITTEE_PERIOD` epochs lookahead active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) seed = get_seed(beacon_state, source_epoch, DOMAIN_SHARD_COMMITTEE) active_shard_count = get_active_shard_count(beacon_state) @@ -495,9 +499,12 @@ def get_shard_committee(beacon_state: BeaconState, epoch: Epoch, shard: Shard) - ```python def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: + """ + Return the light client committee that no more than ``TARGET_COMMITTEE_SIZE`` validators. + """ source_epoch = epoch - epoch % LIGHT_CLIENT_COMMITTEE_PERIOD if source_epoch >= LIGHT_CLIENT_COMMITTEE_PERIOD: - source_epoch -= LIGHT_CLIENT_COMMITTEE_PERIOD + source_epoch -= LIGHT_CLIENT_COMMITTEE_PERIOD # `LIGHT_CLIENT_COMMITTEE_PERIOD` epochs lookahead active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) seed = get_seed(beacon_state, source_epoch, DOMAIN_LIGHT_CLIENT) return compute_committee( @@ -554,7 +561,10 @@ def get_latest_slot_for_shard(state: BeaconState, shard: Shard) -> Slot: ```python def get_offset_slots(state: BeaconState, shard: Shard) -> Sequence[Slot]: - return compute_offset_slots(state.shard_states[shard].slot, state.slot) + """ + Return the offset slots of the given ``shard`` between that latest included slot and current slot. + """ + return compute_offset_slots(get_latest_slot_for_shard(state, shard), state.slot) ``` ### Predicates @@ -569,36 +579,38 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe Check if ``indexed_attestation`` has valid indices and signature. """ # Verify aggregate signature - all_pubkeys = [] - all_signing_roots = [] attestation = indexed_attestation.attestation - domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch) aggregation_bits = attestation.aggregation_bits if not any(aggregation_bits) or len(aggregation_bits) != len(indexed_attestation.committee): return False + domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch) + all_pubkeys = [] + all_signing_roots = [] if len(attestation.custody_bits_blocks) == 0: # fall back on phase0 behavior if there is no shard data. - for participant, abit in zip(indexed_attestation.committee, aggregation_bits): - if abit: + for participant, aggregation_bit in zip(indexed_attestation.committee, aggregation_bits): + if aggregation_bit: all_pubkeys.append(state.validators[participant].pubkey) signing_root = compute_signing_root(indexed_attestation.attestation.data, domain) return bls.FastAggregateVerify(all_pubkeys, signing_root, signature=attestation.signature) else: - for i, custody_bits in enumerate(attestation.custody_bits_blocks): + for block_index, custody_bits in enumerate(attestation.custody_bits_blocks): assert len(custody_bits) == len(indexed_attestation.committee) - for participant, abit, cbit in zip(indexed_attestation.committee, aggregation_bits, custody_bits): - if abit: + for participant, aggregation_bit, custody_bit in zip( + indexed_attestation.committee, aggregation_bits, custody_bits + ): + if aggregation_bit: all_pubkeys.append(state.validators[participant].pubkey) # Note: only 2N distinct message hashes attestation_wrapper = AttestationCustodyBitWrapper( attestation_data_root=hash_tree_root(attestation.data), - block_index=i, - bit=cbit + block_index=block_index, + bit=custody_bit ) all_signing_roots.append(compute_signing_root(attestation_wrapper, domain)) else: - assert not cbit + assert not custody_bit return bls.AggregateVerify(all_pubkeys, all_signing_roots, signature=attestation.signature) ``` @@ -767,22 +779,22 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr proposers = [] prev_gasprice = state.shard_states[shard].gasprice shard_parent_root = state.shard_states[shard].latest_block_root - for i in range(len(offset_slots)): + for i, offset_slot in enumerate(offset_slots): shard_block_length = transition.shard_block_lengths[i] shard_state = transition.shard_states[i] # Verify correct calculation of gas prices and slots assert shard_state.gasprice == compute_updated_gasprice(prev_gasprice, shard_block_length) - assert shard_state.slot == offset_slots[i] + assert shard_state.slot == offset_slot # Collect the non-empty proposals result is_empty_proposal = shard_block_length == 0 if not is_empty_proposal: - proposal_index = get_shard_proposer_index(state, offset_slots[i], shard) + proposal_index = get_shard_proposer_index(state, offset_slot, shard) # Reconstruct shard headers header = ShardBlockHeader( shard_parent_root=shard_parent_root, - beacon_parent_root=get_block_root_at_slot(state, offset_slots[i]), + beacon_parent_root=get_block_root_at_slot(state, offset_slot), proposer_index=proposal_index, - slot=offset_slots[i], + slot=offset_slot, body_root=transition.shard_data_roots[i] ) shard_parent_root = hash_tree_root(header) @@ -872,13 +884,12 @@ def process_crosslinks(state: BeaconState, for committee_index in map(CommitteeIndex, range(committee_count)): shard = compute_shard_from_committee_index(state, committee_index, state.slot) # All attestations in the block for this committee/shard and current slot - shard_transition = shard_transitions[shard] shard_attestations = [ attestation for attestation in attestations if is_shard_attestation(state, attestation, committee_index) ] - winning_root = process_crosslink_for_shard(state, committee_index, shard_transition, shard_attestations) + winning_root = process_crosslink_for_shard(state, committee_index, shard_transitions[shard], shard_attestations) if winning_root != Root(): # Mark relevant pending attestations as creating a successful crosslink for pending_attestation in state.current_epoch_attestations: @@ -886,7 +897,7 @@ def process_crosslinks(state: BeaconState, pending_attestation.crosslink_success = True ``` -###### `process_attestation` +###### Updated `process_attestation` ```python def process_attestation(state: BeaconState, attestation: Attestation) -> None: @@ -910,11 +921,9 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: ```python def get_indices_from_committee( committee: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE], - bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE]) -> List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE]: + bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE]) -> Sequence[ValidatorIndex]: assert len(bits) == len(committee) - return List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE]( - [validator_index for i, validator_index in enumerate(committee) if bits[i]] - ) + return ([validator_index for i, validator_index in enumerate(committee) if bits[i]]) ``` ```python @@ -1018,7 +1027,9 @@ def process_online_tracking(state: BeaconState) -> None: ```python def process_light_client_committee_updates(state: BeaconState) -> None: - # Update light client committees + """ + Update light client committees. + """ if get_current_epoch(state) % LIGHT_CLIENT_COMMITTEE_PERIOD == 0: state.current_light_committee = state.next_light_committee new_committee = get_light_client_committee(state, get_current_epoch(state) + LIGHT_CLIENT_COMMITTEE_PERIOD) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index da3aa648c..1a8277066 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -268,12 +268,8 @@ def get_shard_transition(beacon_state: BeaconState, shard: Shard, shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition: offset_slots = get_offset_slots(beacon_state, shard) - start_slot = offset_slots[0] proposals, shard_states, shard_data_roots = get_shard_state_transition_result(beacon_state, shard, shard_blocks) - assert len(proposals) > 0 - assert len(shard_data_roots) > 0 - shard_block_lengths = [] proposer_signatures = [] for proposal in proposals: @@ -287,7 +283,7 @@ def get_shard_transition(beacon_state: BeaconState, proposer_signature_aggregate = NO_SIGNATURE return ShardTransition( - start_slot=start_slot, + start_slot=offset_slots[0], shard_block_lengths=shard_block_lengths, shard_data_roots=shard_data_roots, shard_states=shard_states, From 8ae7f5b6fa46c6423798ba56dee3c7192db8b232 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 29 May 2020 01:24:17 +0800 Subject: [PATCH 058/165] Refactor `is_valid_indexed_attestation`: extract `verify_attestation_custody` --- specs/phase1/beacon-chain.md | 57 +++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 89dd6aaac..ee282f203 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -52,6 +52,7 @@ - [`get_latest_slot_for_shard`](#get_latest_slot_for_shard) - [`get_offset_slots`](#get_offset_slots) - [Predicates](#predicates) + - [`verify_attestation_custody`](#verify_attestation_custody) - [Updated `is_valid_indexed_attestation`](#updated-is_valid_indexed_attestation) - [`is_shard_attestation`](#is_shard_attestation) - [`is_winning_attestation`](#is_winning_attestation) @@ -414,7 +415,7 @@ def unpack_compact_validator(compact_validator: uint64) -> Tuple[ValidatorIndex, ```python def committee_to_compact_committee(state: BeaconState, committee: Sequence[ValidatorIndex]) -> CompactCommittee: """ - Given a state and a list of validator indices, outputs the CompactCommittee representing them. + Given a state and a list of validator indices, outputs the ``CompactCommittee`` representing them. """ validators = [state.validators[i] for i in committee] compact_validators = [ @@ -569,6 +570,37 @@ def get_offset_slots(state: BeaconState, shard: Shard) -> Sequence[Slot]: ### Predicates +#### `verify_attestation_custody` + +```python +def verify_attestation_custody(state: BeaconState, indexed_attestation: IndexedAttestation) -> bool: + """ + Check if ``indexed_attestation`` has valid signature against non-empty custody bits. + """ + attestation = indexed_attestation.attestation + aggregation_bits = attestation.aggregation_bits + domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch) + all_pubkeys = [] + all_signing_roots = [] + for block_index, custody_bits in enumerate(attestation.custody_bits_blocks): + assert len(custody_bits) == len(indexed_attestation.committee) + for participant, aggregation_bit, custody_bit in zip( + indexed_attestation.committee, aggregation_bits, custody_bits + ): + if aggregation_bit: + all_pubkeys.append(state.validators[participant].pubkey) + # Note: only 2N distinct message hashes + attestation_wrapper = AttestationCustodyBitWrapper( + attestation_data_root=hash_tree_root(attestation.data), + block_index=block_index, + bit=custody_bit, + ) + all_signing_roots.append(compute_signing_root(attestation_wrapper, domain)) + else: + assert not custody_bit + return bls.AggregateVerify(all_pubkeys, all_signing_roots, signature=attestation.signature) +``` + #### Updated `is_valid_indexed_attestation` Note that this replaces the Phase 0 `is_valid_indexed_attestation`. @@ -584,34 +616,17 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe if not any(aggregation_bits) or len(aggregation_bits) != len(indexed_attestation.committee): return False - domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch) - all_pubkeys = [] - all_signing_roots = [] if len(attestation.custody_bits_blocks) == 0: # fall back on phase0 behavior if there is no shard data. + domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch) + all_pubkeys = [] for participant, aggregation_bit in zip(indexed_attestation.committee, aggregation_bits): if aggregation_bit: all_pubkeys.append(state.validators[participant].pubkey) signing_root = compute_signing_root(indexed_attestation.attestation.data, domain) return bls.FastAggregateVerify(all_pubkeys, signing_root, signature=attestation.signature) else: - for block_index, custody_bits in enumerate(attestation.custody_bits_blocks): - assert len(custody_bits) == len(indexed_attestation.committee) - for participant, aggregation_bit, custody_bit in zip( - indexed_attestation.committee, aggregation_bits, custody_bits - ): - if aggregation_bit: - all_pubkeys.append(state.validators[participant].pubkey) - # Note: only 2N distinct message hashes - attestation_wrapper = AttestationCustodyBitWrapper( - attestation_data_root=hash_tree_root(attestation.data), - block_index=block_index, - bit=custody_bit - ) - all_signing_roots.append(compute_signing_root(attestation_wrapper, domain)) - else: - assert not custody_bit - return bls.AggregateVerify(all_pubkeys, all_signing_roots, signature=attestation.signature) + return verify_attestation_custody(state, indexed_attestation) ``` #### `is_shard_attestation` From bd9f983eeae34b11ebb84df2d000ab7f22ab3d34 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 29 May 2020 02:20:38 +0800 Subject: [PATCH 059/165] Minor fix --- specs/phase1/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index ee282f203..ce47ef585 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -938,7 +938,7 @@ def get_indices_from_committee( committee: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE], bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE]) -> Sequence[ValidatorIndex]: assert len(bits) == len(committee) - return ([validator_index for i, validator_index in enumerate(committee) if bits[i]]) + return [validator_index for i, validator_index in enumerate(committee) if bits[i]] ``` ```python From b16e6d7a8623d6d90c90ea0fb64cc29a993952b5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 29 May 2020 12:58:19 +0800 Subject: [PATCH 060/165] PR feedback from Danny Co-authored-by: Danny Ryan --- specs/phase1/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index e536ad6ef..f087247f4 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -948,7 +948,7 @@ def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSla ```python def verify_empty_shard_transition(state: BeaconState, block_body: BeaconBlockBody) -> None: """ - Verify that a `shard_transition` in a block is empty if an attestation was not processed for it. + Verify that ``shard_transitions`` are empty if a crosslink was not formed for the associated shard in this slot. """ for shard in range(get_active_shard_count(state)): if state.shard_states[shard].slot != compute_previous_slot(state.slot): From 2dc041807af913f16e54cf44a46fef07f80dbe2e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 29 May 2020 23:50:18 +0800 Subject: [PATCH 061/165] Implement `get_start_shard` --- setup.py | 15 ++++- specs/phase1/beacon-chain.md | 61 ++++++++++++++++++- specs/phase1/phase1-fork.md | 1 + .../test_process_crosslink.py | 2 +- .../test/phase_1/sanity/test_blocks.py | 4 +- 5 files changed, 74 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 4f3f2b872..d0f063b33 100644 --- a/setup.py +++ b/setup.py @@ -140,7 +140,7 @@ SUNDRY_CONSTANTS_FUNCTIONS = ''' def ceillog2(x: uint64) -> int: return (x - 1).bit_length() ''' -SUNDRY_FUNCTIONS = ''' +PHASE0_SUNDRY_FUNCTIONS = ''' # Monkey patch hash cache _hash = hash hash_cache: Dict[bytes, Bytes32] = {} @@ -220,6 +220,13 @@ get_attesting_indices = cache_this( _get_attesting_indices, lru_size=SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT * 3)''' +PHASE1_SUNDRY_FUNCTIONS = ''' +_get_start_shard = get_start_shard +get_start_shard = cache_this( + lambda state, slot: (state.validators.hash_tree_root(), slot), + _get_start_shard, lru_size=SLOTS_PER_EPOCH * 3)''' + + def objects_to_spec(spec_object: SpecObject, imports: str, fork: str) -> str: """ Given all the objects that constitute a spec, combine them into a single pyfile. @@ -250,9 +257,11 @@ def objects_to_spec(spec_object: SpecObject, imports: str, fork: str) -> str: + '\n\n' + CONFIG_LOADER + '\n\n' + ssz_objects_instantiation_spec + '\n\n' + functions_spec - + '\n' + SUNDRY_FUNCTIONS - + '\n' + + '\n' + PHASE0_SUNDRY_FUNCTIONS ) + if fork == 'phase1': + spec += '\n' + PHASE1_SUNDRY_FUNCTIONS + spec += '\n' return spec diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 39a73c0aa..124442b59 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -47,6 +47,7 @@ - [`get_light_client_committee`](#get_light_client_committee) - [`get_shard_proposer_index`](#get_shard_proposer_index) - [`get_indexed_attestation`](#get_indexed_attestation) + - [`get_committee_count_delta`](#get_committee_count_delta) - [`get_start_shard`](#get_start_shard) - [`get_shard`](#get_shard) - [`get_latest_slot_for_shard`](#get_latest_slot_for_shard) @@ -69,6 +70,7 @@ - [Shard transition false positives](#shard-transition-false-positives) - [Light client processing](#light-client-processing) - [Epoch transition](#epoch-transition) + - [Phase 1 final updates](#phase-1-final-updates) - [Custody game updates](#custody-game-updates) - [Online-tracking](#online-tracking) - [Light client committee updates](#light-client-committee-updates) @@ -280,6 +282,7 @@ class BeaconState(Container): current_justified_checkpoint: Checkpoint finalized_checkpoint: Checkpoint # Phase 1 + current_epoch_start_shard: Shard shard_states: List[ShardState, MAX_SHARDS] online_countdown: List[OnlineEpochs, VALIDATOR_REGISTRY_LIMIT] # not a raw byte array, considered its large size. current_light_committee: CompactCommittee @@ -530,18 +533,53 @@ def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation) ) ``` +#### `get_committee_count_delta` + +```python +def get_committee_count_delta(state: BeaconState, start_slot: Slot, stop_slot: Slot) -> uint64: + """ + Return the sum of committee counts between ``[start_slot, stop_slot)``. + """ + committee_sum = 0 + for slot in range(start_slot, stop_slot): + count = get_committee_count_at_slot(state, Slot(slot)) + committee_sum += count + return committee_sum +``` + #### `get_start_shard` ```python def get_start_shard(state: BeaconState, slot: Slot) -> Shard: - # TODO: implement start shard logic - return Shard(0) + """ + Return the start shard at ``slot``. + """ + current_epoch_start_slot = compute_start_slot_at_epoch(get_current_epoch(state)) + active_shard_count = get_active_shard_count(state) + if current_epoch_start_slot == slot: + return state.current_epoch_start_shard + elif current_epoch_start_slot > slot: + # Current epoch or the next epoch lookahead + shard_delta = get_committee_count_delta(state, start_slot=current_epoch_start_slot, stop_slot=slot) + return Shard((state.current_epoch_start_shard + shard_delta) % active_shard_count) + else: + # Previous epoch + shard_delta = get_committee_count_delta(state, start_slot=slot, stop_slot=current_epoch_start_slot) + max_committees_per_epoch = MAX_COMMITTEES_PER_SLOT * SLOTS_PER_EPOCH + return Shard( + # Ensure positive + (state.current_epoch_start_shard + max_committees_per_epoch * active_shard_count - shard_delta) + % active_shard_count + ) ``` #### `get_shard` ```python def get_shard(state: BeaconState, attestation: Attestation) -> Shard: + """ + Return the shard that the given attestation is attesting. + """ return compute_shard_from_committee_index(state, attestation.data.index, attestation.data.slot) ``` @@ -549,6 +587,9 @@ def get_shard(state: BeaconState, attestation: Attestation) -> Shard: ```python def get_latest_slot_for_shard(state: BeaconState, shard: Shard) -> Slot: + """ + Return the latest slot number of the given shard. + """ return state.shard_states[shard].slot ``` @@ -556,7 +597,11 @@ def get_latest_slot_for_shard(state: BeaconState, shard: Shard) -> Slot: ```python def get_offset_slots(state: BeaconState, shard: Shard) -> Sequence[Slot]: - return compute_offset_slots(state.shard_states[shard].slot, state.slot) + """ + Return the offset slots of the given shard. + The offset slot are after the latest slot and before current slot. + """ + return compute_offset_slots(get_latest_slot_for_shard(state, shard), state.slot) ``` ### Predicates @@ -993,9 +1038,19 @@ def process_epoch(state: BeaconState) -> None: process_reveal_deadlines(state) process_slashings(state) process_final_updates(state) + process_phase_1_final_updates(state) +``` + +#### Phase 1 final updates + +```python +def process_phase_1_final_updates(state: BeaconState) -> None: process_custody_final_updates(state) process_online_tracking(state) process_light_client_committee_updates(state) + + # Update current_epoch_start_shard + state.current_epoch_start_shard = get_start_shard(state, state.slot) ``` #### Custody game updates diff --git a/specs/phase1/phase1-fork.md b/specs/phase1/phase1-fork.md index cc7d8f33e..d362ed633 100644 --- a/specs/phase1/phase1-fork.md +++ b/specs/phase1/phase1-fork.md @@ -99,6 +99,7 @@ def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: current_justified_checkpoint=pre.current_justified_checkpoint, finalized_checkpoint=pre.finalized_checkpoint, # Phase 1 + current_epoch_start_shard=Shard(0), shard_states=List[ShardState, MAX_SHARDS]( ShardState( slot=pre.slot, diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py index 1f066b344..af0ff1a90 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py @@ -18,7 +18,7 @@ def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): # At the beginning, let `x = state.slot`, `state.shard_states[shard].slot == x - 1` slot_x = state.slot committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot) assert state.shard_states[shard].slot == slot_x - 1 # Create SignedShardBlock diff --git a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py index 60af35d45..1499d34cd 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py @@ -67,7 +67,7 @@ def test_process_beacon_block_with_normal_shard_transition(spec, state): target_len_offset_slot = 1 committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot) assert state.shard_states[shard].slot == state.slot - 1 pre_gasprice = state.shard_states[shard].gasprice @@ -93,7 +93,7 @@ def test_process_beacon_block_with_empty_proposal_transition(spec, state): target_len_offset_slot = 1 committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot) assert state.shard_states[shard].slot == state.slot - 1 # No new shard block From f70224b84e0e2eebf7e89ff36fc218ea217d3569 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 30 May 2020 03:09:42 +0800 Subject: [PATCH 062/165] Extract `compute_committee_source_epoch` --- specs/phase1/beacon-chain.md | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index ce47ef585..054868847 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -40,6 +40,7 @@ - [`compute_shard_from_committee_index`](#compute_shard_from_committee_index) - [`compute_offset_slots`](#compute_offset_slots) - [`compute_updated_gasprice`](#compute_updated_gasprice) + - [`compute_committee_source_epoch`](#compute_committee_source_epoch) - [Beacon state accessors](#beacon-state-accessors) - [`get_active_shard_count`](#get_active_shard_count) - [`get_online_validator_indices`](#get_online_validator_indices) @@ -458,6 +459,19 @@ def compute_updated_gasprice(prev_gasprice: Gwei, shard_block_length: uint8) -> return max(prev_gasprice, MIN_GASPRICE + delta) - delta ``` +#### `compute_committee_source_epoch` + +```python +def compute_committee_source_epoch(epoch: Epoch, period: uint64) -> Epoch: + """ + Return the source epoch for computing the committee. + """ + source_epoch = epoch - epoch % period + if source_epoch >= period: + source_epoch -= period # `period` epochs lookahead + return source_epoch +``` + ### Beacon state accessors #### `get_active_shard_count` @@ -482,9 +496,7 @@ def get_shard_committee(beacon_state: BeaconState, epoch: Epoch, shard: Shard) - """ Return the shard committee of the given ``epoch`` of the given ``shard``. """ - source_epoch = epoch - epoch % SHARD_COMMITTEE_PERIOD - if source_epoch >= SHARD_COMMITTEE_PERIOD: - source_epoch -= SHARD_COMMITTEE_PERIOD # `SHARD_COMMITTEE_PERIOD` epochs lookahead + source_epoch = compute_committee_source_epoch(epoch, SHARD_COMMITTEE_PERIOD) active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) seed = get_seed(beacon_state, source_epoch, DOMAIN_SHARD_COMMITTEE) active_shard_count = get_active_shard_count(beacon_state) @@ -503,9 +515,7 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque """ Return the light client committee that no more than ``TARGET_COMMITTEE_SIZE`` validators. """ - source_epoch = epoch - epoch % LIGHT_CLIENT_COMMITTEE_PERIOD - if source_epoch >= LIGHT_CLIENT_COMMITTEE_PERIOD: - source_epoch -= LIGHT_CLIENT_COMMITTEE_PERIOD # `LIGHT_CLIENT_COMMITTEE_PERIOD` epochs lookahead + source_epoch = compute_committee_source_epoch(epoch, LIGHT_CLIENT_COMMITTEE_PERIOD) active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) seed = get_seed(beacon_state, source_epoch, DOMAIN_LIGHT_CLIENT) return compute_committee( From cceeab265769cb9a39362fe81de42436b74e3f90 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 29 May 2020 03:14:43 +0800 Subject: [PATCH 063/165] Combine `process_crosslinks` and `verify_empty_shard_transition` into `process_shard_transitions` --- specs/phase1/beacon-chain.md | 24 +++++++++++++++---- .../{crosslinks.py => shard_transitions.py} | 8 +++---- ...nk.py => test_process_shard_transition.py} | 4 ++-- tests/generators/README.md | 2 +- 4 files changed, 26 insertions(+), 12 deletions(-) rename tests/core/pyspec/eth2spec/test/helpers/{crosslinks.py => shard_transitions.py} (65%) rename tests/core/pyspec/eth2spec/test/phase_1/block_processing/{test_process_crosslink.py => test_process_shard_transition.py} (93%) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index f087247f4..853160292 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -64,6 +64,7 @@ - [`apply_shard_transition`](#apply_shard_transition) - [`process_crosslink_for_shard`](#process_crosslink_for_shard) - [`process_crosslinks`](#process_crosslinks) + - [`process_shard_transitions`](#process_shard_transitions) - [`process_attestation`](#process_attestation) - [New Attester slashing processing](#new-attester-slashing-processing) - [Verify empty shard transition](#verify-empty-shard-transition) @@ -671,7 +672,6 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_eth1_data(state, block.body) process_light_client_signatures(state, block.body) process_operations(state, block.body) - verify_empty_shard_transition(state, block.body) ``` #### Operations @@ -695,7 +695,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: # See custody game spec. process_custody_game_operations(state, body) - process_crosslinks(state, body.shard_transitions, body.attestations) + process_shard_transitions(state, body.shard_transitions, body.attestations) # TODO process_operations(body.shard_receipt_proofs, process_shard_receipt_proofs) ``` @@ -882,6 +882,18 @@ def process_crosslinks(state: BeaconState, pending_attestation.crosslink_success = True ``` +###### `process_shard_transitions` + +```python +def process_shard_transitions(state: BeaconState, + shard_transitions: Sequence[ShardTransition], + attestations: Sequence[Attestation]) -> None: + # Process crosslinks + process_crosslinks(state, shard_transitions, attestations) + # Verify the empty proposal shard states + assert verify_empty_shard_transition(state, shard_transitions) +``` + ###### `process_attestation` ```python @@ -893,7 +905,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: data=attestation.data, inclusion_delay=state.slot - attestation.data.slot, proposer_index=get_beacon_proposer_index(state), - crosslink_success=False, # To be filled in during process_crosslinks + crosslink_success=False, # To be filled in during process_shard_transitions ) if attestation.data.target.epoch == get_current_epoch(state): state.current_epoch_attestations.append(pending_attestation) @@ -946,13 +958,15 @@ def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSla #### Verify empty shard transition ```python -def verify_empty_shard_transition(state: BeaconState, block_body: BeaconBlockBody) -> None: +def verify_empty_shard_transition(state: BeaconState, shard_transitions: Sequence[ShardTransition]) -> bool: """ Verify that ``shard_transitions`` are empty if a crosslink was not formed for the associated shard in this slot. """ for shard in range(get_active_shard_count(state)): if state.shard_states[shard].slot != compute_previous_slot(state.slot): - assert block_body.shard_transitions[shard] == ShardTransition() + if shard_transitions[shard] != ShardTransition(): + return False + return True ``` #### Light client processing diff --git a/tests/core/pyspec/eth2spec/test/helpers/crosslinks.py b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py similarity index 65% rename from tests/core/pyspec/eth2spec/test/helpers/crosslinks.py rename to tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py index ea5da89d9..4ac0ddcfb 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/crosslinks.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py @@ -1,9 +1,9 @@ from eth2spec.test.context import expect_assertion_error -def run_crosslinks_processing(spec, state, shard_transitions, attestations, valid=True): +def run_shard_transitions_processing(spec, state, shard_transitions, attestations, valid=True): """ - Run ``process_attestation``, yielding: + Run ``process_shard_transitions``, yielding: - pre-state ('pre') - shard_transitions ('shard_transitions') - attestations ('attestations') @@ -17,12 +17,12 @@ def run_crosslinks_processing(spec, state, shard_transitions, attestations, vali # If the attestation is invalid, processing is aborted, and there is no post-state. if not valid: - expect_assertion_error(lambda: spec.process_crosslinks(state, shard_transitions, attestations)) + expect_assertion_error(lambda: spec.process_shard_transitions(state, shard_transitions, attestations)) yield 'post', None return # process crosslinks - spec.process_crosslinks(state, shard_transitions, attestations) + spec.process_shard_transitions(state, shard_transitions, attestations) # yield post-state yield 'post', state diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py similarity index 93% rename from tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py rename to tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py index 1f066b344..b4721da5e 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py @@ -4,7 +4,7 @@ from eth2spec.test.context import ( spec_state_test, always_bls, ) -from eth2spec.test.helpers.crosslinks import run_crosslinks_processing +from eth2spec.test.helpers.shard_transitions import run_shard_transitions_processing from eth2spec.test.helpers.shard_block import ( build_attestation_with_shard_transition, build_shard_block, @@ -46,7 +46,7 @@ def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): transition_to(spec, state, state.slot + target_len_offset_slot) pre_shard_state = state.shard_states[shard] - yield from run_crosslinks_processing(spec, state, shard_transitions, [attestation], valid=valid) + yield from run_shard_transitions_processing(spec, state, shard_transitions, [attestation], valid=valid) if valid: # After state transition, diff --git a/tests/generators/README.md b/tests/generators/README.md index 77a50606b..9446551fb 100644 --- a/tests/generators/README.md +++ b/tests/generators/README.md @@ -184,7 +184,7 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin if __name__ == "__main__": gen_runner.run_generator("epoch_processing", [ - create_provider('crosslinks', test_process_crosslinks, 'minimal'), + create_provider('final_updates', test_process_final_updates, 'minimal'), ... ]) From 327deb40b276ecf1e8c180d2f2c33b4f5196483c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 29 May 2020 03:20:36 +0800 Subject: [PATCH 064/165] Adjust function blocks --- specs/phase1/beacon-chain.md | 73 +++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 853160292..d939d1c9f 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -61,13 +61,14 @@ - [Operations](#operations) - [New Attestation processing](#new-attestation-processing) - [`validate_attestation`](#validate_attestation) + - [Updated `process_attestation`](#updated-process_attestation) + - [Shard transition processing](#shard-transition-processing) - [`apply_shard_transition`](#apply_shard_transition) - [`process_crosslink_for_shard`](#process_crosslink_for_shard) - [`process_crosslinks`](#process_crosslinks) + - [`verify_empty_shard_transition`](#verify_empty_shard_transition) - [`process_shard_transitions`](#process_shard_transitions) - - [`process_attestation`](#process_attestation) - [New Attester slashing processing](#new-attester-slashing-processing) - - [Verify empty shard transition](#verify-empty-shard-transition) - [Light client processing](#light-client-processing) - [Epoch transition](#epoch-transition) - [Custody game updates](#custody-game-updates) @@ -742,6 +743,27 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) ``` +###### Updated `process_attestation` + +```python +def process_attestation(state: BeaconState, attestation: Attestation) -> None: + validate_attestation(state, attestation) + # Store pending attestation for epoch processing + pending_attestation = PendingAttestation( + aggregation_bits=attestation.aggregation_bits, + data=attestation.data, + inclusion_delay=state.slot - attestation.data.slot, + proposer_index=get_beacon_proposer_index(state), + crosslink_success=False, # To be filled in during process_shard_transitions + ) + if attestation.data.target.epoch == get_current_epoch(state): + state.current_epoch_attestations.append(pending_attestation) + else: + state.previous_epoch_attestations.append(pending_attestation) +``` + +##### Shard transition processing + ###### `apply_shard_transition` ```python @@ -882,6 +904,20 @@ def process_crosslinks(state: BeaconState, pending_attestation.crosslink_success = True ``` +###### `verify_empty_shard_transition` + +```python +def verify_empty_shard_transition(state: BeaconState, shard_transitions: Sequence[ShardTransition]) -> bool: + """ + Verify that a `shard_transition` in a block is empty if an attestation was not processed for it. + """ + for shard in range(get_active_shard_count(state)): + if state.shard_states[shard].slot != compute_previous_slot(state.slot): + if shard_transitions[shard] != ShardTransition(): + return False + return True +``` + ###### `process_shard_transitions` ```python @@ -894,25 +930,6 @@ def process_shard_transitions(state: BeaconState, assert verify_empty_shard_transition(state, shard_transitions) ``` -###### `process_attestation` - -```python -def process_attestation(state: BeaconState, attestation: Attestation) -> None: - validate_attestation(state, attestation) - # Store pending attestation for epoch processing - pending_attestation = PendingAttestation( - aggregation_bits=attestation.aggregation_bits, - data=attestation.data, - inclusion_delay=state.slot - attestation.data.slot, - proposer_index=get_beacon_proposer_index(state), - crosslink_success=False, # To be filled in during process_shard_transitions - ) - if attestation.data.target.epoch == get_current_epoch(state): - state.current_epoch_attestations.append(pending_attestation) - else: - state.previous_epoch_attestations.append(pending_attestation) -``` - ##### New Attester slashing processing ```python @@ -955,20 +972,6 @@ def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSla assert slashed_any ``` -#### Verify empty shard transition - -```python -def verify_empty_shard_transition(state: BeaconState, shard_transitions: Sequence[ShardTransition]) -> bool: - """ - Verify that ``shard_transitions`` are empty if a crosslink was not formed for the associated shard in this slot. - """ - for shard in range(get_active_shard_count(state)): - if state.shard_states[shard].slot != compute_previous_slot(state.slot): - if shard_transitions[shard] != ShardTransition(): - return False - return True -``` - #### Light client processing ```python From 870ad8b921705cb658ebb0b88ff4e08bdeb54609 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 30 May 2020 03:56:56 +0800 Subject: [PATCH 065/165] Fix test --- .../pyspec/eth2spec/test/fork_choice/test_on_shard_head.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index f4b883f06..5b4205e91 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -30,6 +30,7 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) # Create SignedShardBlock body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE + target_len_offset_slot = 1 shard_block = build_shard_block(spec, state, shard, body=body, signed=True) shard_blocks = [shard_block] @@ -38,17 +39,15 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) shard_transitions = build_shard_transitions_till_slot( spec, state, - shards=[shard, ], shard_blocks={shard: shard_blocks}, - target_len_offset_slot=1, + on_time_slot=state.slot + target_len_offset_slot, ) shard_transition = shard_transitions[shard] attestation = build_attestation_with_shard_transition( spec, state, - slot=state.slot, index=committee_index, - target_len_offset_slot=1, + on_time_slot=state.slot + target_len_offset_slot, shard_transition=shard_transition, ) From 437b2ebb90d3f26b2fb92d23792c560f38208603 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Fri, 29 May 2020 18:16:06 -0700 Subject: [PATCH 066/165] Update fork choice spec comment for clarity I think this change more clearly specifies the intended behavior. Given the terseness of the spec's code representation, I think we should aim for as much clarity as possible. --- specs/phase0/fork-choice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index 4ed2733e1..baf7b7b7b 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -150,7 +150,7 @@ def get_ancestor(store: Store, root: Root, slot: Slot) -> Root: elif block.slot == slot: return root else: - # root is older than queried slot, thus a skip slot. Return earliest root prior to slot + # root is older than queried slot, thus a skip slot. Return most recent root prior to slot return root ``` From 33e115f8c65407357852f1605399bfaf79f933cb Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Sat, 30 May 2020 20:59:12 +0200 Subject: [PATCH 067/165] Clean up remaining `[]` List syntax ..and use List[byte, 256] for error message --- specs/phase0/p2p-interface.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 84db361c9..a77f1851a 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -150,6 +150,7 @@ This section outlines constants that are used in this spec. | Name | Value | Description | |---|---|---| | `GOSSIP_MAX_SIZE` | `2**20` (= 1048576, 1 MiB) | The maximum allowed size of uncompressed gossip messages. | +| `MAX_REQUEST_BLOCKS` | `2**10` (= 1024) | Maximum number of blocks in a single request | | `MAX_CHUNK_SIZE` | `2**20` (1048576, 1 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | | `TTFB_TIMEOUT` | `5s` | The maximum time to wait for first byte of request response (time-to-first-byte). | | `RESP_TIMEOUT` | `10s` | The maximum time for complete response transfer. | @@ -391,11 +392,11 @@ The `ErrorMessage` schema is: ``` ( - error_message: Bytes32 + error_message: List[byte, 256] ) ``` -*Note*: By convention, the `error_message` is a sequence of bytes that can be interpreted as a UTF-8 string up to 32 bytes - a 0 byte shortens the string in this interpretation. Clients MUST treat as valid any bytes. +*Note*: By convention, the `error_message` is a sequence of bytes that MAY be interpreted as a UTF-8 string (for debugging purposes). Clients MUST treat as valid any byte sequences. ### Encoding strategies @@ -443,9 +444,9 @@ In case of an invalid input (header or payload), a reader MUST: All messages that contain only a single field MUST be encoded directly as the type of that field and MUST NOT be encoded as an SSZ container. -Responses that are SSZ-lists (for example `[]SignedBeaconBlock`) send their +Responses that are SSZ-lists (for example `List[SignedBeaconBlock, ...]`) send their constituents individually as `response_chunk`s. For example, the -`[]SignedBeaconBlock` response type sends zero or more `response_chunk`s. Each _successful_ `response_chunk` contains a single `SignedBeaconBlock` payload. +`List[SignedBeaconBlock, ...]` response type sends zero or more `response_chunk`s. Each _successful_ `response_chunk` contains a single `SignedBeaconBlock` payload. ### Messages @@ -528,7 +529,7 @@ Request Content: Response Content: ``` ( - []SignedBeaconBlock + List[SignedBeaconBlock, MAX_REQUEST_BLOCKS] ) ``` @@ -545,7 +546,7 @@ The response MUST consist of zero or more `response_chunk`. Each _successful_ `r Clients MUST keep a record of signed blocks seen since the since the start of the weak subjectivity period and MUST support serving requests of blocks up to their own `head_block_root`. -Clients MUST respond with at least the first block that exists in the range, if they have it. +Clients MUST respond with at least the first block that exists in the range, if they have it, and no more than `MAX_REQUEST_BLOCKS` blocks. The following blocks, where they exist, MUST be send in consecutive order. @@ -568,7 +569,7 @@ Request Content: ``` ( - []Root + List[Root, MAX_REQUEST_BLOCKS] ) ``` @@ -576,12 +577,14 @@ Response Content: ``` ( - []SignedBeaconBlock + List[SignedBeaconBlock, MAX_REQUEST_BLOCKS] ) ``` Requests blocks by block root (= `hash_tree_root(SignedBeaconBlock.message)`). The response is a list of `SignedBeaconBlock` whose length is less than or equal to the number of requested blocks. It may be less in the case that the responding peer is missing blocks. +No more than `MAX_REQUEST_BLOCKS` may be requested at a time. + `BeaconBlocksByRoot` is primarily used to recover recent blocks (e.g. when receiving a block or attestation whose parent is unknown). The request MUST be encoded as an SSZ-field. From 6317bd68aa7476d82064daf70d7dbd71236ae3e1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 1 Jun 2020 17:47:47 +0800 Subject: [PATCH 068/165] PR feedback from Danny Co-authored-by: Danny Ryan --- specs/phase1/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 6c792432f..308af4ee3 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -515,7 +515,7 @@ def get_shard_committee(beacon_state: BeaconState, epoch: Epoch, shard: Shard) - ```python def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: """ - Return the light client committee that no more than ``TARGET_COMMITTEE_SIZE`` validators. + Return the light client committee of no more than ``TARGET_COMMITTEE_SIZE`` validators. """ source_epoch = compute_committee_source_epoch(epoch, LIGHT_CLIENT_COMMITTEE_PERIOD) active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) From 92db6da50854071eecea138576e9771a5004a596 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 1 Jun 2020 17:56:22 +0800 Subject: [PATCH 069/165] Apply suggestions from Terence Co-authored-by: terence tsao --- specs/phase1/beacon-chain.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 124442b59..1367d029b 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -578,7 +578,7 @@ def get_start_shard(state: BeaconState, slot: Slot) -> Shard: ```python def get_shard(state: BeaconState, attestation: Attestation) -> Shard: """ - Return the shard that the given attestation is attesting. + Return the shard that the given ``attestation`` is attesting. """ return compute_shard_from_committee_index(state, attestation.data.index, attestation.data.slot) ``` @@ -588,7 +588,7 @@ def get_shard(state: BeaconState, attestation: Attestation) -> Shard: ```python def get_latest_slot_for_shard(state: BeaconState, shard: Shard) -> Slot: """ - Return the latest slot number of the given shard. + Return the latest slot number of the given ``shard``. """ return state.shard_states[shard].slot ``` @@ -598,7 +598,7 @@ def get_latest_slot_for_shard(state: BeaconState, shard: Shard) -> Slot: ```python def get_offset_slots(state: BeaconState, shard: Shard) -> Sequence[Slot]: """ - Return the offset slots of the given shard. + Return the offset slots of the given ``shard``. The offset slot are after the latest slot and before current slot. """ return compute_offset_slots(get_latest_slot_for_shard(state, shard), state.slot) From 30f72dd69646d037a06943d026e2d91ef2b9c2c5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 1 Jun 2020 23:15:16 +0800 Subject: [PATCH 070/165] Fix `get_shard` and `compute_shard_from_committee_index` calls --- specs/phase1/beacon-chain.md | 25 ++++++------- .../eth2spec/test/helpers/attestations.py | 37 +++++++++---------- .../eth2spec/test/helpers/shard_block.py | 2 +- .../test/helpers/shard_transitions.py | 15 ++++++++ .../test/phase_0/sanity/test_blocks.py | 20 +++++++--- .../test_process_shard_transition.py | 9 ++--- .../test/phase_1/sanity/test_blocks.py | 4 +- 7 files changed, 67 insertions(+), 45 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 228cd888e..52b6f2afb 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -578,7 +578,7 @@ def get_start_shard(state: BeaconState, slot: Slot) -> Shard: active_shard_count = get_active_shard_count(state) if current_epoch_start_slot == slot: return state.current_epoch_start_shard - elif current_epoch_start_slot > slot: + elif slot > current_epoch_start_slot: # Current epoch or the next epoch lookahead shard_delta = get_committee_count_delta(state, start_slot=current_epoch_start_slot, stop_slot=slot) return Shard((state.current_epoch_start_shard + shard_delta) % active_shard_count) @@ -693,8 +693,7 @@ def is_on_time_attestation(state: BeaconState, """ Check if the given attestation is on-time. """ - # TODO: MIN_ATTESTATION_INCLUSION_DELAY should always be 1 - return attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY == state.slot + return attestation.data.slot == compute_previous_slot(state.slot) ``` #### `is_winning_attestation` @@ -803,20 +802,19 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: else: assert attestation.data.source == state.previous_justified_checkpoint - shard = get_shard(state, attestation) - # Type 1: on-time attestations, the custody bits should be non-empty. if attestation.custody_bits_blocks != []: # Ensure on-time attestation assert is_on_time_attestation(state, attestation) # Correct data root count + shard = get_shard(state, attestation) assert len(attestation.custody_bits_blocks) == len(get_offset_slots(state, shard)) # Correct parent block root assert data.beacon_block_root == get_block_root_at_slot(state, compute_previous_slot(state.slot)) # Type 2: no shard transition, no custody bits else: # Ensure delayed attestation - assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY < state.slot + assert data.slot < compute_previous_slot(state.slot) # Late attestations cannot have a shard transition root assert data.shard_transition_root == Root() @@ -911,9 +909,10 @@ def process_crosslink_for_shard(state: BeaconState, committee_index: CommitteeIndex, shard_transition: ShardTransition, attestations: Sequence[Attestation]) -> Root: - committee = get_beacon_committee(state, state.slot, committee_index) + on_time_attestation_slot = compute_previous_slot(state.slot) + committee = get_beacon_committee(state, on_time_attestation_slot, committee_index) online_indices = get_online_validator_indices(state) - shard = compute_shard_from_committee_index(state, committee_index, state.slot) + shard = compute_shard_from_committee_index(state, committee_index, on_time_attestation_slot) # Loop over all shard transition roots shard_transition_roots = set([a.data.shard_transition_root for a in attestations]) @@ -968,15 +967,15 @@ def process_crosslink_for_shard(state: BeaconState, def process_crosslinks(state: BeaconState, shard_transitions: Sequence[ShardTransition], attestations: Sequence[Attestation]) -> None: - committee_count = get_committee_count_at_slot(state, state.slot) + on_time_attestation_slot = compute_previous_slot(state.slot) + committee_count = get_committee_count_at_slot(state, on_time_attestation_slot) for committee_index in map(CommitteeIndex, range(committee_count)): - shard = compute_shard_from_committee_index(state, committee_index, state.slot) # All attestations in the block for this committee/shard and current slot shard_attestations = [ attestation for attestation in attestations if is_on_time_attestation(state, attestation) and attestation.data.index == committee_index ] - + shard = compute_shard_from_committee_index(state, committee_index, on_time_attestation_slot) winning_root = process_crosslink_for_shard(state, committee_index, shard_transitions[shard], shard_attestations) if winning_root != Root(): # Mark relevant pending attestations as creating a successful crosslink @@ -1083,7 +1082,7 @@ def process_epoch(state: BeaconState) -> None: process_registry_updates(state) process_reveal_deadlines(state) process_slashings(state) - process_final_updates(state) + process_final_updates(state) # phase 0 final updates process_phase_1_final_updates(state) ``` @@ -1096,7 +1095,7 @@ def process_phase_1_final_updates(state: BeaconState) -> None: process_light_client_committee_updates(state) # Update current_epoch_start_shard - state.current_epoch_start_shard = get_start_shard(state, state.slot) + state.current_epoch_start_shard = get_start_shard(state, Slot(state.slot + 1)) ``` #### Custody game updates diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index c533182ef..1372b0654 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -1,8 +1,9 @@ from typing import List from eth2spec.test.context import expect_assertion_error, PHASE0, PHASE1 -from eth2spec.test.helpers.state import state_transition_and_sign_block, next_epoch, next_slot, transition_to +from eth2spec.test.helpers.state import state_transition_and_sign_block, next_epoch, next_slot from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.helpers.shard_transitions import get_shard_transition_of_committee from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.ssz.ssz_typing import Bitlist @@ -81,12 +82,12 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() else: - # No shard transition + # No shard transition -> no shard block shard = spec.get_shard(state, spec.Attestation(data=attestation_data)) if on_time: temp_state = state.copy() next_slot(spec, temp_state) - shard_transition = spec.get_shard_transition(temp_state, shard, []) + shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=[]) lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() @@ -180,7 +181,7 @@ def get_valid_attestation(spec, fill_aggregate_attestation(spec, state, attestation, signed=signed, filter_participant_set=filter_participant_set) if spec.fork == PHASE1 and on_time: - attestation = convert_to_valid_on_time_attestation(spec, state, attestation, signed) + attestation = convert_to_valid_on_time_attestation(spec, state, attestation, signed=signed) return attestation @@ -317,7 +318,19 @@ def next_epoch_with_attestations(spec, committees_per_slot = spec.get_committee_count_at_slot(state, slot_to_attest) if slot_to_attest >= spec.compute_start_slot_at_epoch(spec.get_current_epoch(post_state)): for index in range(committees_per_slot): - cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest, index=index, signed=True) + if spec.fork == PHASE1: + shard = spec.compute_shard_from_committee_index(post_state, index, slot_to_attest) + shard_transition = get_shard_transition_of_committee( + spec, post_state, index, slot=slot_to_attest + ) + block.body.shard_transitions[shard] = shard_transition + else: + shard_transition = None + + cur_attestation = get_valid_attestation( + spec, post_state, slot_to_attest, + shard_transition=shard_transition, index=index, signed=True, on_time=True + ) block.body.attestations.append(cur_attestation) if fill_prev_epoch: @@ -328,9 +341,6 @@ def next_epoch_with_attestations(spec, spec, post_state, slot_to_attest, index=index, signed=True, on_time=False) block.body.attestations.append(prev_attestation) - if spec.fork == PHASE1: - fill_block_shard_transitions_by_attestations(spec, post_state, block) - signed_block = state_transition_and_sign_block(spec, post_state, block) signed_blocks.append(signed_block) @@ -396,14 +406,3 @@ def cached_prepare_state_with_attestations(spec, state): # Put the LRU cache result into the state view, as if we transitioned the original view state.set_backing(_prep_state_cache_dict[key]) - - -def fill_block_shard_transitions_by_attestations(spec, state, block): - block.body.shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS - for attestation in block.body.attestations: - shard = spec.get_shard(state, attestation) - if attestation.data.slot == state.slot: - temp_state = state.copy() - transition_to(spec, temp_state, slot=block.slot) - shard_transition = spec.get_shard_transition(temp_state, shard, []) - block.body.shard_transitions[shard] = shard_transition diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 805b955f7..58efada83 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -70,7 +70,7 @@ def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): return shard_transitions -def build_attestation_with_shard_transition(spec, state, index, on_time_slot, shard_transition=None): +def build_attestation_with_shard_transition(spec, state, index, on_time_slot, shard_transition): temp_state = state.copy() transition_to(spec, temp_state, on_time_slot - 1) attestation = get_valid_on_time_attestation( diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py index 4ac0ddcfb..abb5e7278 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py @@ -1,4 +1,5 @@ from eth2spec.test.context import expect_assertion_error +from eth2spec.test.helpers.state import transition_to def run_shard_transitions_processing(spec, state, shard_transitions, attestations, valid=True): @@ -26,3 +27,17 @@ def run_shard_transitions_processing(spec, state, shard_transitions, attestation # yield post-state yield 'post', state + + +def get_shard_transition_of_committee(spec, state, committee_index, slot=None, shard_blocks=None): + if shard_blocks is None: + shard_blocks = [] + + if slot is None: + slot = state.slot + + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + temp_state = state.copy() + transition_to(spec, temp_state, slot + 1) + shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=shard_blocks) + return shard_transition diff --git a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py index f0cfc462e..1e057e5a9 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py @@ -16,8 +16,9 @@ from eth2spec.test.helpers.attester_slashings import ( get_indexed_attestation_participants, ) from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing, check_proposer_slashing_effect -from eth2spec.test.helpers.attestations import get_valid_attestation, fill_block_shard_transitions_by_attestations +from eth2spec.test.helpers.attestations import get_valid_attestation from eth2spec.test.helpers.deposits import prepare_state_and_deposit +from eth2spec.test.helpers.shard_transitions import get_shard_transition_of_committee from eth2spec.test.context import ( spec_state_test, with_all_phases, expect_assertion_error, always_bls, with_phases, @@ -687,14 +688,23 @@ def test_attestation(spec, state): yield 'pre', state - attestation = get_valid_attestation(spec, state, signed=True, on_time=True) + attestation_block = build_empty_block(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + index = 0 + if spec.fork == PHASE1: + shard = spec.compute_shard_from_committee_index(state, index, state.slot) + shard_transition = get_shard_transition_of_committee(spec, state, index) + attestation_block.body.shard_transitions[shard] = shard_transition + else: + shard_transition = None + + attestation = get_valid_attestation( + spec, state, shard_transition=shard_transition, index=index, signed=True, on_time=True + ) # Add to state via block transition pre_current_attestations_len = len(state.current_epoch_attestations) - attestation_block = build_empty_block(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) attestation_block.body.attestations.append(attestation) - if spec.fork == PHASE1: - fill_block_shard_transitions_by_attestations(spec, state, attestation_block) signed_attestation_block = state_transition_and_sign_block(spec, state, attestation_block) assert len(state.current_epoch_attestations) == pre_current_attestations_len + 1 diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py index ba408cd48..00ffbe0a8 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py @@ -15,11 +15,10 @@ from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): state = transition_to_valid_shard_slot(spec, state) - # At the beginning, let `x = state.slot`, `state.shard_states[shard].slot == x - 1` - slot_x = state.slot + init_slot = state.slot committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot) - assert state.shard_states[shard].slot == slot_x - 1 + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) + assert state.shard_states[shard].slot == state.slot - 1 # Create SignedShardBlock body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE @@ -50,7 +49,7 @@ def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): if valid: # After state transition, - assert state.slot == slot_x + target_len_offset_slot + assert state.slot == init_slot + target_len_offset_slot shard_state = state.shard_states[shard] assert shard_state != pre_shard_state assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1] diff --git a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py index 1499d34cd..0175bd40d 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py @@ -67,7 +67,7 @@ def test_process_beacon_block_with_normal_shard_transition(spec, state): target_len_offset_slot = 1 committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 pre_gasprice = state.shard_states[shard].gasprice @@ -93,7 +93,7 @@ def test_process_beacon_block_with_empty_proposal_transition(spec, state): target_len_offset_slot = 1 committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 # No new shard block From 5f10ac13bf91c181152a5469562dfb9d89383926 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 1 Jun 2020 23:22:59 +0800 Subject: [PATCH 071/165] PR feedback from Terence and Danny: refactor `get_committee_count_delta` --- specs/phase1/beacon-chain.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 52b6f2afb..484b24cf1 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -558,13 +558,9 @@ def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation) ```python def get_committee_count_delta(state: BeaconState, start_slot: Slot, stop_slot: Slot) -> uint64: """ - Return the sum of committee counts between ``[start_slot, stop_slot)``. + Return the sum of committee counts in range ``[start_slot, stop_slot)``. """ - committee_sum = 0 - for slot in range(start_slot, stop_slot): - count = get_committee_count_at_slot(state, Slot(slot)) - committee_sum += count - return committee_sum + return sum(get_committee_count_at_slot(state, Slot(slot)) for slot in range(start_slot, stop_slot)) ``` #### `get_start_shard` From 2a218520a1338aca2601cd90ffe2d31f64475495 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Mon, 1 Jun 2020 14:31:45 -0700 Subject: [PATCH 072/165] Tidying up `shard_state_transition` --- specs/phase1/shard-transition.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index ce6d289bd..f9de34b31 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -71,21 +71,22 @@ def verify_shard_block_signature(beacon_state: BeaconState, def shard_state_transition(beacon_state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: - # Update shard state + """ + Update ``shard_state`` with shard ``block`` and ``beacon_state`. + """ + shard_state.slot = block.slot prev_gasprice = shard_state.gasprice - if len(block.body) == 0: - latest_block_root = shard_state.latest_block_root - else: - latest_block_root = hash_tree_root(block) - + shard_state.gasprice = compute_updated_gasprice(prev_gasprice, len(block.body)) shard_state.transition_digest = compute_shard_transition_digest( beacon_state, shard_state, block.beacon_parent_root, hash_tree_root(block.body), ) - shard_state.gasprice = compute_updated_gasprice(prev_gasprice, len(block.body)) - shard_state.slot = block.slot + if len(block.body) == 0: + latest_block_root = shard_state.latest_block_root + else: + latest_block_root = hash_tree_root(block) shard_state.latest_block_root = latest_block_root ``` From c6aac1650600407a832fa9646362873ebb785d72 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 2 Jun 2020 17:16:25 +1000 Subject: [PATCH 073/165] Reword fork choice comment --- specs/phase0/fork-choice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index baf7b7b7b..3f1c20c0a 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -285,7 +285,7 @@ def validate_on_attestation(store: Store, attestation: Attestation) -> None: # Attestations must not be for blocks in the future. If not, the attestation should not be considered assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot - # FFG and LMD vote must be consistent with each other + # LMD vote must be consistent with FFG target target_slot = compute_start_slot_at_epoch(target.epoch) assert target.root == get_ancestor(store, attestation.data.beacon_block_root, target_slot) From 2a125e8497c37b8685ba5478be254105df2542d3 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 2 Jun 2020 17:22:33 +1000 Subject: [PATCH 074/165] Update fork-choice.md --- specs/phase0/fork-choice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index 3f1c20c0a..b9d8ecd3c 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -285,7 +285,7 @@ def validate_on_attestation(store: Store, attestation: Attestation) -> None: # Attestations must not be for blocks in the future. If not, the attestation should not be considered assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot - # LMD vote must be consistent with FFG target + # LMD vote must be consistent with FFG vote target target_slot = compute_start_slot_at_epoch(target.epoch) assert target.root == get_ancestor(store, attestation.data.beacon_block_root, target_slot) From 142ba17451e05b0e6478796fe9bea9eb2de4c488 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 2 Jun 2020 18:08:28 +0800 Subject: [PATCH 075/165] PR review from Danny --- specs/phase1/shard-fork-choice.md | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index b026aabfc..867730203 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -59,7 +59,9 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro return Gwei(sum( state.validators[i].effective_balance for i in active_indices if ( - i in store.latest_messages and get_shard_ancestor( + i in store.latest_messages and + store.latest_messages[i].shard == shard_store.shard and + get_shard_ancestor( store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot ) == root ) @@ -71,12 +73,18 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro ```python def get_shard_head(store: Store, shard_store: ShardStore) -> Root: # Execute the LMD-GHOST fork choice + shard_blocks = shard_store.blocks head_beacon_root = get_head(store) - head_shard_root = store.block_states[head_beacon_root].shard_states[shard_store.shard].latest_block_root + head_shard_state = store.block_states[head_beacon_root].shard_states[shard_store.shard] + head_shard_root = head_shard_state.latest_block_root while True: + # Find the valid child block roots children = [ root for root in shard_store.blocks.keys() - if shard_store.blocks[root].shard_parent_root == head_shard_root + if ( + shard_blocks[root].shard_parent_root == head_shard_root + and shard_blocks[root].slot > head_shard_state.slot + ) ] if len(children) == 0: return head_shard_root @@ -116,14 +124,15 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si assert shard_block.beacon_parent_root in store.block_states beacon_state = store.block_states[shard_block.beacon_parent_root] - # 3. Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - assert shard_block.slot > finalized_slot + # 3. Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) + finalized_beacon_state = store.block_states[store.finalized_checkpoint.root] + finalized_shard_state = finalized_beacon_state.shard_states[shard] + assert shard_block.slot > finalized_shard_state.slot # 4. Check block is a descendant of the finalized block at the checkpoint finalized slot + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert ( - shard_block.beacon_parent_root == store.finalized_checkpoint.root - or get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root + get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root ) # Add new block to the store From 24427947b17060623047fca2fbc0b49d8b636fe7 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Tue, 2 Jun 2020 08:09:43 -0700 Subject: [PATCH 076/165] Moved transition_digest to last --- specs/phase1/shard-transition.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index f9de34b31..e6221a980 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -77,17 +77,17 @@ def shard_state_transition(beacon_state: BeaconState, shard_state.slot = block.slot prev_gasprice = shard_state.gasprice shard_state.gasprice = compute_updated_gasprice(prev_gasprice, len(block.body)) + if len(block.body) == 0: + latest_block_root = shard_state.latest_block_root + else: + latest_block_root = hash_tree_root(block) + shard_state.latest_block_root = latest_block_root shard_state.transition_digest = compute_shard_transition_digest( beacon_state, shard_state, block.beacon_parent_root, hash_tree_root(block.body), ) - if len(block.body) == 0: - latest_block_root = shard_state.latest_block_root - else: - latest_block_root = hash_tree_root(block) - shard_state.latest_block_root = latest_block_root ``` We have a pure function `get_post_shard_state` for describing the fraud proof verification and honest validator behavior. From 34412130c5a8df907680649adbae58ae45f3c0bc Mon Sep 17 00:00:00 2001 From: Raw Pong Ghmoa <58883403+q9f@users.noreply.github.com> Date: Mon, 1 Jun 2020 13:59:03 +0200 Subject: [PATCH 077/165] phase0: enable a tunable genesis time --- specs/phase0/beacon-chain.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 1be32da44..e4de2d38b 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -218,7 +218,7 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | Unit | Duration | | - | - | :-: | :-: | -| `MIN_GENESIS_DELAY` | `86400` | seconds | 1 day | +| `GENESIS_DELAY` | `172800` | seconds | 2 days | | `SECONDS_PER_SLOT` | `12` | seconds | 12 seconds | | `SECONDS_PER_ETH1_BLOCK` | `14` | seconds | 14 seconds | | `MIN_ATTESTATION_INCLUSION_DELAY` | `2**0` (= 1) | slots | 12 seconds | @@ -1137,7 +1137,7 @@ Before the Ethereum 2.0 genesis has been triggered, and for every Ethereum 1.0 b - `eth1_timestamp` is the Unix timestamp corresponding to `eth1_block_hash` - `deposits` is the sequence of all deposits, ordered chronologically, up to (and including) the block with hash `eth1_block_hash` -Eth1 blocks must only be considered once they are at least `SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE` seconds old (i.e. `eth1_timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE <= current_unix_time`). Due to this constraint, if `MIN_GENESIS_DELAY < SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE`, then the `genesis_time` can happen before the time/state is first known. Values should be configured to avoid this case. +Eth1 blocks must only be considered once they are at least `SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE` seconds old (i.e. `eth1_timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE <= current_unix_time`). Due to this constraint, if `GENESIS_DELAY < SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE`, then the `genesis_time` can happen before the time/state is first known. Values should be configured to avoid this case. ```python def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32, @@ -1149,7 +1149,7 @@ def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32, epoch=GENESIS_EPOCH, ) state = BeaconState( - genesis_time=eth1_timestamp - eth1_timestamp % MIN_GENESIS_DELAY + 2 * MIN_GENESIS_DELAY, + genesis_time=eth1_timestamp + GENESIS_DELAY, fork=fork, eth1_data=Eth1Data(block_hash=eth1_block_hash, deposit_count=len(deposits)), latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), From b3f2d81ad5063f728c4620ed5a9435b5dfbe8853 Mon Sep 17 00:00:00 2001 From: Raw Pong Ghmoa <58883403+q9f@users.noreply.github.com> Date: Mon, 1 Jun 2020 14:02:29 +0200 Subject: [PATCH 078/165] reflect changes in mainnet.yaml --- configs/mainnet.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 42845c235..3631b7045 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -76,8 +76,8 @@ BLS_WITHDRAWAL_PREFIX: 0x00 # Time parameters # --------------------------------------------------------------- -# 86400 seconds (1 day) -MIN_GENESIS_DELAY: 86400 +# 172800 seconds (2 days) +GENESIS_DELAY: 172800 # 12 seconds SECONDS_PER_SLOT: 12 # 2**0 (= 1) slots 12 seconds From d5ed78e974aa97e22c4695d25f82165a5cc46310 Mon Sep 17 00:00:00 2001 From: Raw Pong Ghmoa <58883403+q9f@users.noreply.github.com> Date: Mon, 1 Jun 2020 14:03:07 +0200 Subject: [PATCH 079/165] reflect changes in minimal.yaml --- configs/minimal.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index d8e346ffa..5abf6c93c 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -77,7 +77,7 @@ BLS_WITHDRAWAL_PREFIX: 0x00 # Time parameters # --------------------------------------------------------------- # [customized] Faster to spin up testnets, but does not give validator reasonable warning time for genesis -MIN_GENESIS_DELAY: 300 +GENESIS_DELAY: 300 # [customized] Faster for testing purposes SECONDS_PER_SLOT: 6 # 2**0 (= 1) slots 6 seconds From 671fae6efe83e0fe7e0e1256d2c0199daf7c42ed Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 2 Jun 2020 11:09:42 -0600 Subject: [PATCH 080/165] change note about genesis delay in p2p spec to match new GENESIS_DELAY config value; fix tests --- specs/phase0/p2p-interface.md | 2 +- .../core/pyspec/eth2spec/test/genesis/test_initialization.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index a994b0c66..e47fafed0 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -1052,7 +1052,7 @@ discv5 uses ENRs and we will presumably need to: Although client software might very well be running locally prior to the solidification of the eth2 genesis state and block, clients cannot form valid ENRs prior to this point. ENRs contain `fork_digest` which utilizes the `genesis_validators_root` for a cleaner separation between chains so prior to knowing genesis, we cannot use `fork_digest` to cleanly find peers on our intended chain. Once genesis data is known, we can then form ENRs and safely find peers. -When using an eth1 deposit contract for deposits, `fork_digest` will be known at least `MIN_GENESIS_DELAY` (24 hours in mainnet configuration) before `genesis_time`, providing ample time to find peers and form initial connections and gossip subnets prior to genesis. +When using an eth1 deposit contract for deposits, `fork_digest` will be known `GENESIS_DELAY` (48hours in mainnet configuration) before `genesis_time`, providing ample time to find peers and form initial connections and gossip subnets prior to genesis. ## Compression/Encoding diff --git a/tests/core/pyspec/eth2spec/test/genesis/test_initialization.py b/tests/core/pyspec/eth2spec/test/genesis/test_initialization.py index 882821337..faade2d17 100644 --- a/tests/core/pyspec/eth2spec/test/genesis/test_initialization.py +++ b/tests/core/pyspec/eth2spec/test/genesis/test_initialization.py @@ -21,7 +21,7 @@ def test_initialize_beacon_state_from_eth1(spec): # initialize beacon_state state = spec.initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits) - assert state.genesis_time == eth1_timestamp - eth1_timestamp % spec.MIN_GENESIS_DELAY + 2 * spec.MIN_GENESIS_DELAY + assert state.genesis_time == eth1_timestamp + spec.GENESIS_DELAY assert len(state.validators) == deposit_count assert state.eth1_data.deposit_root == deposit_root assert state.eth1_data.deposit_count == deposit_count @@ -57,7 +57,7 @@ def test_initialize_beacon_state_some_small_balances(spec): # initialize beacon_state state = spec.initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits) - assert state.genesis_time == eth1_timestamp - eth1_timestamp % spec.MIN_GENESIS_DELAY + 2 * spec.MIN_GENESIS_DELAY + assert state.genesis_time == eth1_timestamp + spec.GENESIS_DELAY assert len(state.validators) == small_deposit_count assert state.eth1_data.deposit_root == deposit_root assert state.eth1_data.deposit_count == len(deposits) From 06dfff022bf05f63e703e1f0d1031d530edf4fb4 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 2 Jun 2020 11:21:00 -0600 Subject: [PATCH 081/165] add comment about default of 0x00 for genesis finalized checkpoint in p2p spec --- specs/phase0/p2p-interface.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index a994b0c66..c7d61d8f1 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -468,9 +468,9 @@ The fields are, as seen by the client at the time of sending the message: - `fork_digest`: The node's `ForkDigest` (`compute_fork_digest(current_fork_version, genesis_validators_root)`) where - `current_fork_version` is the fork version at the node's current epoch defined by the wall-clock time (not necessarily the epoch to which the node is sync) - `genesis_validators_root` is the static `Root` found in `state.genesis_validators_root` -- `finalized_root`: `state.finalized_checkpoint.root` for the state corresponding to the head block. +- `finalized_root`: `state.finalized_checkpoint.root` for the state corresponding to the head block (Note this defaults to `Root(b'\x00' * 32)` for the genesis finalized checkpoint). - `finalized_epoch`: `state.finalized_checkpoint.epoch` for the state corresponding to the head block. -- `head_root`: The hash_tree_root root of the current head block. +- `head_root`: The `hash_tree_root` root of the current head block. - `head_slot`: The slot of the block corresponding to the `head_root`. The dialing client MUST send a `Status` request upon connection. From cf7b9993b5ea1124ddd87579dc27a78ebf092e92 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 2 Jun 2020 13:51:43 -0600 Subject: [PATCH 082/165] clarify `head_root` is for a `BeaconBlock` Co-authored-by: Diederik Loerakker --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index c7d61d8f1..e98070b6f 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -470,7 +470,7 @@ The fields are, as seen by the client at the time of sending the message: - `genesis_validators_root` is the static `Root` found in `state.genesis_validators_root` - `finalized_root`: `state.finalized_checkpoint.root` for the state corresponding to the head block (Note this defaults to `Root(b'\x00' * 32)` for the genesis finalized checkpoint). - `finalized_epoch`: `state.finalized_checkpoint.epoch` for the state corresponding to the head block. -- `head_root`: The `hash_tree_root` root of the current head block. +- `head_root`: The `hash_tree_root` root of the current head block (`BeaconBlock`). - `head_slot`: The slot of the block corresponding to the `head_root`. The dialing client MUST send a `Status` request upon connection. From 314dea97a5018740c5166e4884777c936f5d5150 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 2 Jun 2020 17:26:26 -0600 Subject: [PATCH 083/165] bump VERSION.txt to 0.12.1 --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index d33c3a212..aac2dacab 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -0.12.0 \ No newline at end of file +0.12.1 \ No newline at end of file From 5c5cedd60d1c08583d8627e93db1b1b179f6b7f4 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 3 Jun 2020 22:31:16 +0800 Subject: [PATCH 084/165] Apply PR feedback from Danny and Terence --- specs/phase1/shard-fork-choice.md | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 867730203..3b6bc5ac9 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -6,7 +6,7 @@ -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + - [Introduction](#introduction) - [Fork choice](#fork-choice) @@ -23,7 +23,7 @@ ## Introduction -This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase 1. +This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase 1. It assumes the [beacon chain fork choice spec](./fork-choice.md). ## Fork choice @@ -59,9 +59,9 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro return Gwei(sum( state.validators[i].effective_balance for i in active_indices if ( - i in store.latest_messages and - store.latest_messages[i].shard == shard_store.shard and - get_shard_ancestor( + i in store.latest_messages + and store.latest_messages[i].shard == shard_store.shard + and get_shard_ancestor( store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot ) == root ) @@ -116,20 +116,25 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: SignedShardBlock) -> None: shard_block = signed_shard_block.message shard = shard_store.shard - # 1. Check shard parent exists + + # Check shard + # TODO: check it in networking spec + assert shard_block.shard == shard + + # Check shard parent exists assert shard_block.shard_parent_root in shard_store.block_states pre_shard_state = shard_store.block_states[shard_block.shard_parent_root] - # 2. Check beacon parent exists + # Check beacon parent exists assert shard_block.beacon_parent_root in store.block_states beacon_state = store.block_states[shard_block.beacon_parent_root] - # 3. Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) + # Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) finalized_beacon_state = store.block_states[store.finalized_checkpoint.root] finalized_shard_state = finalized_beacon_state.shard_states[shard] assert shard_block.slot > finalized_shard_state.slot - # 4. Check block is a descendant of the finalized block at the checkpoint finalized slot + # Check block is a descendant of the finalized block at the checkpoint finalized slot finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert ( get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root From 68e934bf1552517346c1b31f1fb9e181b51d82cb Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 3 Jun 2020 23:08:38 +0800 Subject: [PATCH 085/165] Add `get_start_shard` unittests and update minimal config 1. Add unittests for testing `get_start_shard` with better granularity 2. Change `INITIAL_ACTIVE_SHARDS` from `4` to `2` for tight crosslinking --- configs/minimal.yaml | 2 +- .../test/phase_1/unittests/__init__.py | 0 .../phase_1/unittests/test_get_start_shard.py | 71 +++++++++++++++++++ 3 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 tests/core/pyspec/eth2spec/test/phase_1/unittests/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py diff --git a/configs/minimal.yaml b/configs/minimal.yaml index d8e346ffa..bb46294f5 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -165,7 +165,7 @@ PHASE_1_FORK_VERSION: 0x01000001 # [customized] for testing PHASE_1_GENESIS_SLOT: 8 # [customized] reduced for testing -INITIAL_ACTIVE_SHARDS: 4 +INITIAL_ACTIVE_SHARDS: 2 # Phase 1: General diff --git a/tests/core/pyspec/eth2spec/test/phase_1/unittests/__init__.py b/tests/core/pyspec/eth2spec/test/phase_1/unittests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py b/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py new file mode 100644 index 000000000..f9f605d9f --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py @@ -0,0 +1,71 @@ +from eth2spec.test.context import ( + PHASE0, + with_all_phases_except, + spec_state_test, +) +from eth2spec.test.helpers.state import next_epoch + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_get_committee_count_delta(spec, state): + assert spec.get_committee_count_delta(state, 0, 0) == 0 + assert spec.get_committee_count_at_slot(state, 0) != 0 + assert spec.get_committee_count_delta(state, 0, 1) == spec.get_committee_count_at_slot(state, 0) + assert spec.get_committee_count_delta(state, 1, 2) == spec.get_committee_count_at_slot(state, 1) + assert spec.get_committee_count_delta(state, 0, 2) == ( + spec.get_committee_count_at_slot(state, 0) + spec.get_committee_count_at_slot(state, 1) + ) + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_get_start_shard_current_epoch_start(spec, state): + assert state.current_epoch_start_shard == 0 + next_epoch(spec, state) + active_shard_count = spec.get_active_shard_count(state) + assert state.current_epoch_start_shard == ( + spec.get_committee_count_delta(state, 0, spec.SLOTS_PER_EPOCH) % active_shard_count + ) + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + + slot = current_epoch_start_slot + start_shard = spec.get_start_shard(state, slot) + assert start_shard == state.current_epoch_start_shard + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_get_start_shard_next_slot(spec, state): + next_epoch(spec, state) + active_shard_count = spec.get_active_shard_count(state) + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + + slot = current_epoch_start_slot + 1 + start_shard = spec.get_start_shard(state, slot) + + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + expected_start_shard = ( + state.current_epoch_start_shard + + spec.get_committee_count_delta(state, start_slot=current_epoch_start_slot, stop_slot=slot) + ) % active_shard_count + assert start_shard == expected_start_shard + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_get_start_shard_previous_slot(spec, state): + next_epoch(spec, state) + active_shard_count = spec.get_active_shard_count(state) + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + + slot = current_epoch_start_slot - 1 + start_shard = spec.get_start_shard(state, slot) + + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + expected_start_shard = ( + state.current_epoch_start_shard + + spec.MAX_COMMITTEES_PER_SLOT * spec.SLOTS_PER_EPOCH * active_shard_count + - spec.get_committee_count_delta(state, start_slot=slot, stop_slot=current_epoch_start_slot) + ) % active_shard_count + assert start_shard == expected_start_shard From a685be3bbecb18d7bf336af911a638d68e6b64f9 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 00:42:08 +0800 Subject: [PATCH 086/165] PR feedback from Danny Co-authored-by: Danny Ryan --- .../eth2spec/test/phase_1/unittests/test_get_start_shard.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py b/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py index f9f605d9f..27afd4a4e 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py @@ -46,8 +46,8 @@ def test_get_start_shard_next_slot(spec, state): current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) expected_start_shard = ( - state.current_epoch_start_shard + - spec.get_committee_count_delta(state, start_slot=current_epoch_start_slot, stop_slot=slot) + state.current_epoch_start_shard + + spec.get_committee_count_delta(state, start_slot=current_epoch_start_slot, stop_slot=slot) ) % active_shard_count assert start_shard == expected_start_shard From e1981a7bfdeb2baf296cbfdb1f74169b56341883 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 01:00:52 +0800 Subject: [PATCH 087/165] `head_shard_root` -> `shard_head_root` --- specs/phase1/fork-choice.md | 2 +- specs/phase1/shard-fork-choice.md | 8 ++++---- .../eth2spec/test/fork_choice/test_on_attestation.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/phase1/fork-choice.md b/specs/phase1/fork-choice.md index f4e771ddb..3f9fbdbfb 100644 --- a/specs/phase1/fork-choice.md +++ b/specs/phase1/fork-choice.md @@ -51,7 +51,7 @@ def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIn for i in attesting_indices: if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: store.latest_messages[i] = LatestMessage( - epoch=target.epoch, root=beacon_block_root, shard=shard, shard_root=attestation.data.head_shard_root + epoch=target.epoch, root=beacon_block_root, shard=shard, shard_root=attestation.data.shard_head_root ) ``` diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 3b6bc5ac9..fb98893ac 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -76,20 +76,20 @@ def get_shard_head(store: Store, shard_store: ShardStore) -> Root: shard_blocks = shard_store.blocks head_beacon_root = get_head(store) head_shard_state = store.block_states[head_beacon_root].shard_states[shard_store.shard] - head_shard_root = head_shard_state.latest_block_root + shard_head_root = head_shard_state.latest_block_root while True: # Find the valid child block roots children = [ root for root in shard_store.blocks.keys() if ( - shard_blocks[root].shard_parent_root == head_shard_root + shard_blocks[root].shard_parent_root == shard_head_root and shard_blocks[root].slot > head_shard_state.slot ) ] if len(children) == 0: - return head_shard_root + return shard_head_root # Sort by latest attesting balance with ties broken lexicographically - head_shard_root = max( + shard_head_root = max( children, key=lambda root: (get_shard_latest_attesting_balance(store, shard_store, root), root) ) ``` diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 04d2588d9..a5334c5c7 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -32,7 +32,7 @@ def run_on_attestation(spec, state, store, attestation, valid=True): epoch=attestation.data.target.epoch, root=attestation.data.beacon_block_root, shard=spec.get_shard(state, attestation), - shard_root=attestation.data.head_shard_root, + shard_root=attestation.data.shard_head_root, ) assert ( From 74204f795d72a4b0322cac5e2ee5a4ea7ebc257a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 3 Jun 2020 12:16:39 -0600 Subject: [PATCH 088/165] udpate validator guide to work with all updated phase 1 constructions --- specs/phase1/beacon-chain.md | 3 + specs/phase1/shard-transition.md | 138 ------------------ specs/phase1/validator.md | 84 +++++++++-- .../test/validator/test_validator_unittest.py | 47 ++++-- 4 files changed, 112 insertions(+), 160 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 0af51a815..1fabe0370 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -854,6 +854,9 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr shard_parent_root = hash_tree_root(header) headers.append(header) proposers.append(proposal_index) + else: + # Must have a stub for `shard_data_root` if empty slot + assert transition.shard_data_roots[i] == Root() prev_gasprice = shard_state.gasprice diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index e6221a980..ec764f7b2 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -154,141 +154,3 @@ def generate_custody_bit(subkey: BLSPubkey, block: ShardBlock) -> bool: # TODO ... ``` - -## Honest committee member behavior - -### Helper functions - -```python -def get_winning_proposal(beacon_state: BeaconState, proposals: Sequence[SignedShardBlock]) -> SignedShardBlock: - # TODO: Let `winning_proposal` be the proposal with the largest number of total attestations from slots in - # `state.shard_next_slots[shard]....slot-1` supporting it or any of its descendants, breaking ties by choosing - # the first proposal locally seen. Do `proposals.append(winning_proposal)`. - return proposals[-1] # stub -``` - -```python -def compute_shard_body_roots(proposals: Sequence[SignedShardBlock]) -> Sequence[Root]: - return [hash_tree_root(proposal.message.body) for proposal in proposals] -``` - -```python -def get_proposal_choices_at_slot(beacon_state: BeaconState, - shard_state: ShardState, - slot: Slot, - shard: Shard, - shard_blocks: Sequence[SignedShardBlock], - validate_signature: bool=True) -> Sequence[SignedShardBlock]: - """ - Return the valid shard blocks at the given ``slot``. - Note that this function doesn't change the state. - """ - choices = [] - shard_blocks_at_slot = [block for block in shard_blocks if block.message.slot == slot] - for block in shard_blocks_at_slot: - try: - # Verify block message and signature - # TODO these validations should have been checked upon receiving shard blocks. - assert verify_shard_block_message(beacon_state, shard_state, block.message, slot, shard) - if validate_signature: - assert verify_shard_block_signature(beacon_state, block) - - shard_state = get_post_shard_state(beacon_state, shard_state, block.message) - except Exception: - pass # TODO: throw error in the test helper - else: - choices.append(block) - return choices -``` - -```python -def get_proposal_at_slot(beacon_state: BeaconState, - shard_state: ShardState, - slot: Shard, - shard: Shard, - shard_blocks: Sequence[SignedShardBlock], - validate_signature: bool=True) -> Tuple[SignedShardBlock, ShardState]: - """ - Return ``proposal``, ``shard_state`` of the given ``slot``. - Note that this function doesn't change the state. - """ - choices = get_proposal_choices_at_slot( - beacon_state=beacon_state, - shard_state=shard_state, - slot=slot, - shard=shard, - shard_blocks=shard_blocks, - validate_signature=validate_signature, - ) - if len(choices) == 0: - block = ShardBlock(slot=slot) - proposal = SignedShardBlock(message=block) - elif len(choices) == 1: - proposal = choices[0] - else: - proposal = get_winning_proposal(beacon_state, choices) - - # Apply state transition - shard_state = get_post_shard_state(beacon_state, shard_state, proposal.message) - - return proposal, shard_state -``` - -```python -def get_shard_state_transition_result( - beacon_state: BeaconState, - shard: Shard, - shard_blocks: Sequence[SignedShardBlock], - validate_signature: bool=True, -) -> Tuple[Sequence[SignedShardBlock], Sequence[ShardState], Sequence[Root]]: - proposals = [] - shard_states = [] - shard_state = beacon_state.shard_states[shard] - for slot in get_offset_slots(beacon_state, shard): - proposal, shard_state = get_proposal_at_slot( - beacon_state=beacon_state, - shard_state=shard_state, - slot=slot, - shard=shard, - shard_blocks=shard_blocks, - validate_signature=validate_signature, - ) - shard_states.append(shard_state) - proposals.append(proposal) - - shard_data_roots = compute_shard_body_roots(proposals) - - return proposals, shard_states, shard_data_roots -``` - -### Make attestations - -Suppose you are a committee member on shard `shard` at slot `current_slot` and you have received shard blocks `shard_blocks` since the latest successful crosslink for `shard` into the beacon chain. Let `beacon_state` be the head beacon state you are building on, and let `QUARTER_PERIOD = SECONDS_PER_SLOT // 4`. `2 * QUARTER_PERIOD` seconds into slot `current_slot`, run `get_shard_transition(beacon_state, shard, shard_blocks)` to get `shard_transition`. - -```python -def get_shard_transition(beacon_state: BeaconState, - shard: Shard, - shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition: - offset_slots = get_offset_slots(beacon_state, shard) - proposals, shard_states, shard_data_roots = get_shard_state_transition_result(beacon_state, shard, shard_blocks) - - shard_block_lengths = [] - proposer_signatures = [] - for proposal in proposals: - shard_block_lengths.append(len(proposal.message.body)) - if proposal.signature != NO_SIGNATURE: - proposer_signatures.append(proposal.signature) - - if len(proposer_signatures) > 0: - proposer_signature_aggregate = bls.Aggregate(proposer_signatures) - else: - proposer_signature_aggregate = NO_SIGNATURE - - return ShardTransition( - start_slot=offset_slots[0], - shard_block_lengths=shard_block_lengths, - shard_data_roots=shard_data_roots, - shard_states=shard_states, - proposer_signature_aggregate=proposer_signature_aggregate, - ) -``` diff --git a/specs/phase1/validator.md b/specs/phase1/validator.md index a26987a34..c5d1cd868 100644 --- a/specs/phase1/validator.md +++ b/specs/phase1/validator.md @@ -189,7 +189,7 @@ def get_best_light_client_aggregate(block: BeaconBlock, aggregates: Sequence[LightClientVote]) -> LightClientVote: viable_aggregates = [ aggregate for aggregate in aggregates - if aggregate.slot == get_previous_slot(block.slot) and aggregate.beacon_block_root == block.parent_root + if aggregate.slot == compute_previous_slot(block.slot) and aggregate.beacon_block_root == block.parent_root ] return max( @@ -242,7 +242,7 @@ class FullAttestation(Container): Note the timing of when to create/broadcast is altered from Phase 1. -A validator should create and broadcast the `attestation` to the associated attestation subnet when either (a) the validator has received a valid `BeaconBlock` from the expected beacon block proposer and a valid `ShardBlock` for the expected shard block porposer for the assigned `slot` or (b) one-half of the `slot` has transpired (`SECONDS_PER_SLOT / 2` seconds after the start of `slot`) -- whichever comes _first_. +A validator should create and broadcast the `attestation` to the associated attestation subnet when either (a) the validator has received a valid `BeaconBlock` from the expected beacon block proposer and a valid `ShardBlock` for the expected shard block proposer for the assigned `slot` or (b) one-half of the `slot` has transpired (`SECONDS_PER_SLOT / 2` seconds after the start of `slot`) -- whichever comes _first_. #### Attestation data @@ -251,6 +251,9 @@ A validator should create and broadcast the `attestation` to the associated atte - Let `head_block` be the result of running the fork choice during the assigned slot. - Let `head_state` be the state of `head_block` processed through any empty slots up to the assigned slot using `process_slots(state, slot)`. - Let `head_shard_block` be the result of running the fork choice on the assigned shard chain during the assigned slot. +- Let `shard_blocks` be the shard blocks in the chain starting immediately _after_ the most recent crosslink (`head_state.shard_transitions[shard].latest_block_root`) up to the `head_shard_block`. + +*Note*: We assume that the fork choice only follows branches with valid `offset_slots` with respect to the most recent beacon state shard transition for the queried shard. ##### Head shard root @@ -258,17 +261,57 @@ Set `attestation_data.head_shard_root = hash_tree_root(head_shard_block)`. ##### Shard transition -Set `shard_transition` to the value returned by `get_shard_transition()`. +Set `shard_transition` to the value returned by `get_shard_transition(head_state, shard, shard_blocks)`. ```python -def get_shard_transition(state: BeaconState, +def get_shard_state_transition_result( + beacon_state: BeaconState, + shard: Shard, + shard_blocks: Sequence[SignedShardBlock], + validate_signature: bool=True, +) -> Tuple[Sequence[ShardState], Sequence[Root], Sequence[uint64]]: + shard_states = [] + shard_data_roots = [] + shard_block_lengths = [] + + shard_state = beacon_state.shard_states[shard] + shard_block_slots = [shard_block.message.slot for shard_block in shard_blocks] + for slot in get_offset_slots(beacon_state, shard): + if slot in shard_block_slots: + shard_block = shard_blocks[shard_block_slots.index(slot)] + shard_data_roots.append(hash_tree_root(shard_block.message.body)) + else: + shard_block = SignedShardBlock(message=ShardBlock(slot=slot)) + shard_data_roots.append(Root()) + shard_state = get_post_shard_state(beacon_state, shard_state, shard_block.message) + shard_states.append(shard_state) + shard_block_lengths.append(len(shard_block.message.body)) + + return shard_states, shard_data_roots, shard_block_lengths +``` + +```python +def get_shard_transition(beacon_state: BeaconState, shard: Shard, - shard_blocks: Sequence[ShardBlockWrapper]) -> ShardTransition: - """ - latest_shard_slot = get_latest_slot_for_shard(state, shard) - offset_slots = [Slot(latest_shard_slot + x) for x in SHARD_BLOCK_OFFSETS if latest_shard_slot + x <= state.slot] - """ - return ShardTransition() + shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition: + offset_slots = get_offset_slots(beacon_state, shard) + shard_states, shard_data_roots, shard_block_lengths = ( + get_shard_state_transition_result(beacon_state, shard, shard_blocks) + ) + + if len(shard_blocks) > 0: + proposer_signatures = [shard_block.signature for shard_block in shard_blocks] + proposer_signature_aggregate = bls.Aggregate(proposer_signatures) + else: + proposer_signature_aggregate = NO_SIGNATURE + + return ShardTransition( + start_slot=offset_slots[0], + shard_block_lengths=shard_block_lengths, + shard_data_roots=shard_data_roots, + shard_states=shard_states, + proposer_signature_aggregate=proposer_signature_aggregate, + ) ``` #### Construct attestation @@ -292,10 +335,25 @@ Set `attestation.signature = attestation_signature` where `attestation_signature ```python def get_attestation_signature(state: BeaconState, - attestation_data: AttestationData, - cb_blocks: List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_SHARD_BLOCKS_PER_ATTESTATION], + attestation: Attestation, privkey: int) -> BLSSignature: - pass + domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch) + attestation_data_root = hash_tree_root(attestation.data) + index_in_committee = attestation.aggregation_bits.index(True) + signatures = [] + for block_index, custody_bits in enumerate(attestation.custody_bits_blocks): + custody_bit = custody_bits[index_in_committee] + signing_root = compute_signing_root( + AttestationCustodyBitWrapper( + attestation_data_root=attestation_data_root, + block_index=block_index, + bit=custody_bit, + ), + domain, + ) + signatures.append(bls.Sign(privkey, signing_root)) + + return bls.Aggregate(signatures) ``` ### Light client committee diff --git a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py index 1dfa0e4d0..26affd579 100644 --- a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py +++ b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py @@ -1,5 +1,11 @@ -from eth2spec.test.context import spec_state_test, always_bls, with_all_phases -from eth2spec.test.helpers.attestations import build_attestation_data +from random import Random + +from eth2spec.test.context import ( + spec_state_test, + always_bls, with_phases, with_all_phases, with_all_phases_except, + PHASE0, +) +from eth2spec.test.helpers.attestations import build_attestation_data, get_valid_attestation from eth2spec.test.helpers.block import build_empty_block from eth2spec.test.helpers.deposits import prepare_state_and_deposit from eth2spec.test.helpers.keys import privkeys, pubkeys @@ -317,18 +323,19 @@ def test_get_block_signature(spec, state): # Attesting -@with_all_phases +@with_phases([PHASE0]) @spec_state_test @always_bls -def test_get_attestation_signature(spec, state): +def test_get_attestation_signature_phase0(spec, state): privkey = privkeys[0] pubkey = pubkeys[0] - attestation_data = spec.AttestationData(slot=10) - domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) + attestation = get_valid_attestation(spec, state, signed=False) + domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch) + run_get_signature_test( spec=spec, state=state, - obj=attestation_data, + obj=attestation.data, domain=domain, get_signature_fn=spec.get_attestation_signature, privkey=privkey, @@ -336,6 +343,28 @@ def test_get_attestation_signature(spec, state): ) +@with_all_phases_except([PHASE0]) +@spec_state_test +@always_bls +def test_get_attestation_signature_phase1plus(spec, state): + privkey = privkeys[0] + + def single_participant(comm): + rng = Random(1100) + return rng.sample(comm, 1) + + attestation = get_valid_attestation(spec, state, filter_participant_set=single_participant, signed=False) + indexed_attestation = spec.get_indexed_attestation(state, attestation) + + assert indexed_attestation.attestation.aggregation_bits.count(True) == 1 + + # Cannot use normal `run_get_signature_test` due to complex signature type + index_in_committee = indexed_attestation.attestation.aggregation_bits.index(True) + privkey = privkeys[indexed_attestation.committee[index_in_committee]] + attestation.signature = spec.get_attestation_signature(state, attestation, privkey) + assert spec.verify_attestation_custody(state, spec.get_indexed_attestation(state, attestation)) + + # Attestation aggregation @@ -363,7 +392,7 @@ def test_get_slot_signature(spec, state): @always_bls def test_is_aggregator(spec, state): # TODO: we can test the probabilistic result against `TARGET_AGGREGATORS_PER_COMMITTEE` - # if we have more validators and larger committeee size + # if we have more validators and larger committee size slot = state.slot committee_index = 0 has_aggregator = False @@ -377,7 +406,7 @@ def test_is_aggregator(spec, state): assert has_aggregator -@with_all_phases +@with_phases([PHASE0]) @spec_state_test @always_bls def test_get_aggregate_signature(spec, state): From c0108afe7713ac10542c478b8d07514bdad42c3b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 05:06:04 +0800 Subject: [PATCH 089/165] Use shard_block.slot to get seed for proposer selection --- specs/phase1/beacon-chain.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 484b24cf1..fccb93f55 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -538,7 +538,8 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque ```python def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard) -> ValidatorIndex: committee = get_shard_committee(beacon_state, compute_epoch_at_slot(slot), shard) - r = bytes_to_int(get_seed(beacon_state, get_current_epoch(beacon_state), DOMAIN_SHARD_COMMITTEE)[:8]) + epoch = compute_epoch_at_slot(slot) + r = bytes_to_int(get_seed(beacon_state, epoch, DOMAIN_SHARD_COMMITTEE)[:8]) return committee[r % len(committee)] ``` From d3445217416ad3a8f73dbf45e4669970d063682a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 05:30:13 +0800 Subject: [PATCH 090/165] Bugfix: should set `shard` for empty proposal --- specs/phase1/shard-transition.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index e6221a980..5e8616568 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -221,7 +221,7 @@ def get_proposal_at_slot(beacon_state: BeaconState, validate_signature=validate_signature, ) if len(choices) == 0: - block = ShardBlock(slot=slot) + block = ShardBlock(slot=slot, shard=shard) proposal = SignedShardBlock(message=block) elif len(choices) == 1: proposal = choices[0] From 26aae40941aac98d731655a475e3085e777c0110 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 05:31:53 +0800 Subject: [PATCH 091/165] Use epoch of the shard_block.slot for generating seed --- specs/phase1/beacon-chain.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 484b24cf1..fccb93f55 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -538,7 +538,8 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque ```python def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard) -> ValidatorIndex: committee = get_shard_committee(beacon_state, compute_epoch_at_slot(slot), shard) - r = bytes_to_int(get_seed(beacon_state, get_current_epoch(beacon_state), DOMAIN_SHARD_COMMITTEE)[:8]) + epoch = compute_epoch_at_slot(slot) + r = bytes_to_int(get_seed(beacon_state, epoch, DOMAIN_SHARD_COMMITTEE)[:8]) return committee[r % len(committee)] ``` From 376c83619024caee0dfd5f1c7da552ca78cedc10 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 4 Jun 2020 12:49:22 +1000 Subject: [PATCH 092/165] Clarify proposer slashing gossip conditions --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 9949db66f..767748e89 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -263,7 +263,7 @@ Additional global topics are used to propagate lower frequency validator message - _[IGNORE]_ The voluntary exit is the first valid voluntary exit received for the validator with index `signed_voluntary_exit.message.validator_index`. - _[REJECT]_ All of the conditions within `process_voluntary_exit` pass validation. - `proposer_slashing` - This topic is used solely for propagating proposer slashings to proposers on the network. Proposer slashings are sent in their entirety. The following validations MUST pass before forwarding the `proposer_slashing` on to the network - - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.index`. + - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.header_1.proposer_index`. - _[REJECT]_ All of the conditions within `process_proposer_slashing` pass validation. - `attester_slashing` - This topic is used solely for propagating attester slashings to proposers on the network. Attester slashings are sent in their entirety. Clients who receive an attester slashing on this topic MUST validate the conditions within `process_attester_slashing` before forwarding it across the network. - _[IGNORE]_ At least one index in the intersection of the attesting indices of each attestation has not yet been seen in any prior `attester_slashing` (i.e. `attester_slashed_indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices)`, verify if `any(attester_slashed_indices.difference(prior_seen_attester_slashed_indices))`). From 8afb93f5a36adac5ef312ee3fa4678632df2b6c4 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 11:19:04 +0800 Subject: [PATCH 093/165] Add `shard_block.slot` to seed --- specs/phase1/beacon-chain.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index fccb93f55..4719d2e6f 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -537,9 +537,13 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque ```python def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard) -> ValidatorIndex: - committee = get_shard_committee(beacon_state, compute_epoch_at_slot(slot), shard) + """ + Return the proposer's index of shard block at ``slot``. + """ epoch = compute_epoch_at_slot(slot) - r = bytes_to_int(get_seed(beacon_state, epoch, DOMAIN_SHARD_COMMITTEE)[:8]) + committee = get_shard_committee(beacon_state, epoch, shard) + seed = hash(get_seed(beacon_state, epoch, DOMAIN_SHARD_COMMITTEE) + int_to_bytes(slot, length=8)) + r = bytes_to_int(seed[:8]) return committee[r % len(committee)] ``` From c2b7ff7422e449dc2486261adad6196259015579 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 4 Jun 2020 16:13:49 +1000 Subject: [PATCH 094/165] Re-clarify proposer slashing check Fixes a typo from #1871 --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 767748e89..bdf0919e0 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -263,7 +263,7 @@ Additional global topics are used to propagate lower frequency validator message - _[IGNORE]_ The voluntary exit is the first valid voluntary exit received for the validator with index `signed_voluntary_exit.message.validator_index`. - _[REJECT]_ All of the conditions within `process_voluntary_exit` pass validation. - `proposer_slashing` - This topic is used solely for propagating proposer slashings to proposers on the network. Proposer slashings are sent in their entirety. The following validations MUST pass before forwarding the `proposer_slashing` on to the network - - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.header_1.proposer_index`. + - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.signed_header_1.message.proposer_index`. - _[REJECT]_ All of the conditions within `process_proposer_slashing` pass validation. - `attester_slashing` - This topic is used solely for propagating attester slashings to proposers on the network. Attester slashings are sent in their entirety. Clients who receive an attester slashing on this topic MUST validate the conditions within `process_attester_slashing` before forwarding it across the network. - _[IGNORE]_ At least one index in the intersection of the attesting indices of each attestation has not yet been seen in any prior `attester_slashing` (i.e. `attester_slashed_indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices)`, verify if `any(attester_slashed_indices.difference(prior_seen_attester_slashed_indices))`). From 52de25048a20e79ee618396af17f4277c6347f2f Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 5 Jun 2020 09:35:09 -0600 Subject: [PATCH 095/165] fix lihth client refs in val guide --- specs/phase1/validator.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/phase1/validator.md b/specs/phase1/validator.md index c5d1cd868..243c21014 100644 --- a/specs/phase1/validator.md +++ b/specs/phase1/validator.md @@ -480,7 +480,7 @@ First, `light_aggregate_and_proof = get_light_aggregate_and_proof(state, validat ```python def get_light_aggregate_and_proof(state: BeaconState, aggregator_index: ValidatorIndex, - aggregate: Attestation, + aggregate: LightClientVote, privkey: int) -> LightAggregateAndProof: return LightAggregateAndProof( aggregator_index=aggregator_index, @@ -506,7 +506,7 @@ def get_light_aggregate_and_proof_signature(state: BeaconState, ```python class LightAggregateAndProof(Container): aggregator_index: ValidatorIndex - aggregate: Attestation + aggregate: LightClientVote selection_proof: BLSSignature ``` From 7e44456be543193b3c4848faead8ea9f7baa1583 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 5 Jun 2020 09:50:49 -0600 Subject: [PATCH 096/165] mod compute_subnet_for_attestation to be usable for lookahead --- specs/phase0/p2p-interface.md | 4 ++-- specs/phase0/validator.md | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index bdf0919e0..6c4cdd2a6 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -275,7 +275,7 @@ Additional global topics are used to propagate lower frequency validator message Attestation subnets are used to propagate unaggregated attestations to subsections of the network. Their `Name`s are: - `beacon_attestation_{subnet_id}` - These topics are used to propagate unaggregated attestations to the subnet `subnet_id` (typically beacon and persistent committees) to be aggregated before being gossiped to `beacon_aggregate_and_proof`. The following validations MUST pass before forwarding the `attestation` on the subnet. - - _[REJECT]_ The attestation is for the correct subnet (i.e. `compute_subnet_for_attestation(state, attestation) == subnet_id`). + - _[REJECT]_ The attestation is for the correct subnet (i.e. `compute_subnet_for_attestation(state, attestation.data.slot, attestation.data.index) == subnet_id`). - _[IGNORE]_ `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` (a client MAY queue future attestations for processing at the appropriate slot). - _[REJECT]_ The attestation is unaggregated -- that is, it has exactly one participating validator (`len(get_attesting_indices(state, attestation.data, attestation.aggregation_bits)) == 1`). - _[IGNORE]_ There has been no other valid attestation seen on an attestation subnet that has an identical `attestation.data.target.epoch` and participating validator index. @@ -286,7 +286,7 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio Attestation broadcasting is grouped into subnets defined by a topic. The number of subnets is defined via `ATTESTATION_SUBNET_COUNT`. The correct subnet for an attestation can be calculated with `compute_subnet_for_attestation`. `beacon_attestation_{subnet_id}` topics, are rotated through throughout the epoch in a similar fashion to rotating through shards in committees in Phase 1. -Unaggregated attestations are sent to the subnet topic, `beacon_attestation_{compute_subnet_for_attestation(state, attestation)}` as `Attestation`s. +Unaggregated attestations are sent to the subnet topic, `beacon_attestation_{compute_subnet_for_attestation(state, attestation.data.slot, attestation.data.index)}` as `Attestation`s. Aggregated attestations are sent to the `beacon_aggregate_and_proof` topic as `AggregateAndProof`s. diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 5e8ddc977..35c52f346 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -199,8 +199,8 @@ The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahe Specifically a validator should: * Call `get_committee_assignment(state, next_epoch, validator_index)` when checking for next epoch assignments. -* Find peers of the pubsub topic `committee_index{committee_index % ATTESTATION_SUBNET_COUNT}_beacon_attestation`. - * If an _insufficient_ number of current peers are subscribed to the topic, the validator must discover new peers on this topic. Via the discovery protocol, find peers with an ENR containing the `attnets` entry such that `ENR["attnets"][committee_index % ATTESTATION_SUBNET_COUNT] == True`. Then validate that the peers are still persisted on the desired topic by requesting `GetMetaData` and checking the resulting `attnets` field. +* Find peers of the pubsub topic `beacon_attestation_{compute_subnet_for_attestation(state, slot, committee_index)}`. + * If an _insufficient_ number of current peers are subscribed to the topic, the validator must discover new peers on this topic. Via the discovery protocol, find peers with an ENR containing the `attnets` entry such that `ENR["attnets"][compute_subnet_for_attestation(state, slot, committee_index)] == True`. Then validate that the peers are still persisted on the desired topic by requesting `GetMetaData` and checking the resulting `attnets` field. * If the validator is assigned to be an aggregator for the slot (see `is_aggregator()`), then subscribe to the topic. *Note*: If the validator is _not_ assigned to be an aggregator, the validator only needs sufficient number of peers on the topic to be able to publish messages. The validator does not need to _subscribe_ and listen to all messages on the topic. @@ -425,18 +425,18 @@ def get_attestation_signature(state: BeaconState, attestation_data: AttestationD #### Broadcast attestation -Finally, the validator broadcasts `attestation` to the associated attestation subnet -- the `beacon_attestation_{compute_subnet_for_attestation(state, attestation)}` pubsub topic. +Finally, the validator broadcasts `attestation` to the associated attestation subnet -- the `beacon_attestation_{compute_subnet_for_attestation(state, attestation.data.slot, attestation.data.committee_index)}` pubsub topic. ```python -def compute_subnet_for_attestation(state: BeaconState, attestation: Attestation) -> uint64: +def compute_subnet_for_attestation(state: BeaconState, slot: Slot, committee_index: CommitteeIndex) -> uint64: """ Compute the correct subnet for an attestation for Phase 0. Note, this mimics expected Phase 1 behavior where attestations will be mapped to their shard subnet. """ - slots_since_epoch_start = attestation.data.slot % SLOTS_PER_EPOCH - committees_since_epoch_start = get_committee_count_at_slot(state, attestation.data.slot) * slots_since_epoch_start + slots_since_epoch_start = slot % SLOTS_PER_EPOCH + committees_since_epoch_start = get_committee_count_at_slot(state, slot) * slots_since_epoch_start - return (committees_since_epoch_start + attestation.data.index) % ATTESTATION_SUBNET_COUNT + return (committees_since_epoch_start + committee_index) % ATTESTATION_SUBNET_COUNT ``` ### Attestation aggregation From 8e5c98ef3c661aa0b10c5ab4e6b050bb455c0957 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 5 Jun 2020 12:01:53 -0600 Subject: [PATCH 097/165] PR feedback --- specs/phase1/validator.md | 43 +++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/specs/phase1/validator.md b/specs/phase1/validator.md index 243c21014..21f9c600a 100644 --- a/specs/phase1/validator.md +++ b/specs/phase1/validator.md @@ -131,14 +131,16 @@ Up to `MAX_EARLY_DERIVED_SECRET_REVEALS`, [`EarlyDerivedSecretReveal`](./custody Exactly `MAX_SHARDS` [`ShardTransition`](./beacon-chain#shardtransition) objects are included in the block. Default each to an empty `ShardTransition()`. Then for each committee assigned to the slot with an associated `committee_index` and `shard`, set `shard_transitions[shard] = full_transitions[winning_root]` if the committee had enough weight to form a crosslink this slot. Specifically: -* Call `shards, winning_roots = get_successful_shard_transitions(state, block.slot, attestations)` +* Call `shards, winning_roots = get_shard_winning_roots(state, block.slot, attestations)` * Let `full_transitions` be a dictionary mapping from the `shard_transition_root`s found in `attestations` to the corresponding full `ShardTransition` * Then for each `shard` and `winning_root` in `zip(shards, winning_roots)` set `shard_transitions[shard] = full_transitions[winning_root]` +*Note*: The `state` passed into `get_shard_winning_roots` must be transitioned into the epoch of `block.slot` to run accurately due to the internal use of `get_online_validator_indices`. + ```python -def get_successful_shard_transitions(state: BeaconState, - slot: Slot, - attestations: Attestation) -> Tuple[Sequence[Shard], Sequence[Root]]: +def get_shard_winning_roots(state: BeaconState, + slot: Slot, + attestations: sequence[Attestation]) -> Tuple[Sequence[Shard], Sequence[Root]]: shards = [] winning_roots = [] online_indices = get_online_validator_indices(state) @@ -223,7 +225,7 @@ class FullAttestationData(Container): source: Checkpoint target: Checkpoint # Current-slot shard block root - head_shard_root: Root + shard_head_root: Root # Full shard transition shard_transition: ShardTransition ``` @@ -246,57 +248,64 @@ A validator should create and broadcast the `attestation` to the associated atte #### Attestation data -`attestation_data` is constructed in the same manner as Phase 0 but uses `FullAttestationData` with the addition of two fields -- `head_shard_root` and `shard_transition`. +`attestation_data` is constructed in the same manner as Phase 0 but uses `FullAttestationData` with the addition of two fields -- `shard_head_root` and `shard_transition`. - Let `head_block` be the result of running the fork choice during the assigned slot. - Let `head_state` be the state of `head_block` processed through any empty slots up to the assigned slot using `process_slots(state, slot)`. -- Let `head_shard_block` be the result of running the fork choice on the assigned shard chain during the assigned slot. -- Let `shard_blocks` be the shard blocks in the chain starting immediately _after_ the most recent crosslink (`head_state.shard_transitions[shard].latest_block_root`) up to the `head_shard_block`. +- Let `shard_head_block` be the result of running the fork choice on the assigned shard chain during the assigned slot. +- Let `shard_blocks` be the shard blocks in the chain starting immediately _after_ the most recent crosslink (`head_state.shard_transitions[shard].latest_block_root`) up to the `shard_head_block`. *Note*: We assume that the fork choice only follows branches with valid `offset_slots` with respect to the most recent beacon state shard transition for the queried shard. ##### Head shard root -Set `attestation_data.head_shard_root = hash_tree_root(head_shard_block)`. +Set `attestation_data.shard_head_root = hash_tree_root(shard_head_block)`. ##### Shard transition Set `shard_transition` to the value returned by `get_shard_transition(head_state, shard, shard_blocks)`. ```python -def get_shard_state_transition_result( +def get_shard_transition_fields( beacon_state: BeaconState, shard: Shard, shard_blocks: Sequence[SignedShardBlock], validate_signature: bool=True, -) -> Tuple[Sequence[ShardState], Sequence[Root], Sequence[uint64]]: +) -> Tuple[Sequence[uint64], Sequence[Root], Sequence[ShardState]]: shard_states = [] shard_data_roots = [] shard_block_lengths = [] shard_state = beacon_state.shard_states[shard] shard_block_slots = [shard_block.message.slot for shard_block in shard_blocks] - for slot in get_offset_slots(beacon_state, shard): + offset_slots = compute_offset_slots( + get_latest_slot_for_shard(beacon_state, shard), + beacon_state.slot + 1, + ) + for slot in offset_slots: if slot in shard_block_slots: shard_block = shard_blocks[shard_block_slots.index(slot)] shard_data_roots.append(hash_tree_root(shard_block.message.body)) else: - shard_block = SignedShardBlock(message=ShardBlock(slot=slot)) + shard_block = SignedShardBlock(message=ShardBlock(slot=slot, shard=shard)) shard_data_roots.append(Root()) shard_state = get_post_shard_state(beacon_state, shard_state, shard_block.message) shard_states.append(shard_state) shard_block_lengths.append(len(shard_block.message.body)) - return shard_states, shard_data_roots, shard_block_lengths + return shard_block_lengths, shard_data_roots, shard_states ``` ```python def get_shard_transition(beacon_state: BeaconState, shard: Shard, shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition: - offset_slots = get_offset_slots(beacon_state, shard) - shard_states, shard_data_roots, shard_block_lengths = ( - get_shard_state_transition_result(beacon_state, shard, shard_blocks) + offset_slots = compute_offset_slots( + get_latest_slot_for_shard(beacon_state, shard), + beacon_state.slot + 1, + ) + shard_block_lengths, shard_data_roots, shard_states = ( + get_shard_transition_fields(beacon_state, shard, shard_blocks) ) if len(shard_blocks) > 0: From 8a9ccc4f3448d5c06d0e98af62bb5086f9d1a0d8 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 5 Jun 2020 12:07:59 -0600 Subject: [PATCH 098/165] clarify attestations as coming from block.body when getting winning roots in validator guide --- specs/phase1/validator.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/phase1/validator.md b/specs/phase1/validator.md index 21f9c600a..935f34efb 100644 --- a/specs/phase1/validator.md +++ b/specs/phase1/validator.md @@ -74,7 +74,7 @@ See constants from [Phase 0 validator guide](../phase0/validator.md#constants). | Name | Value | Unit | Duration | | - | - | :-: | :-: | -| `TARGET_LIGHT_CLIENT_AGGREGATORS_PER_SLOT` | `2**2` (= 8) | validators | | +| `TARGET_LIGHT_CLIENT_AGGREGATORS_PER_SLOT` | `2**3` (= 8) | validators | | ## Becoming a validator @@ -131,7 +131,7 @@ Up to `MAX_EARLY_DERIVED_SECRET_REVEALS`, [`EarlyDerivedSecretReveal`](./custody Exactly `MAX_SHARDS` [`ShardTransition`](./beacon-chain#shardtransition) objects are included in the block. Default each to an empty `ShardTransition()`. Then for each committee assigned to the slot with an associated `committee_index` and `shard`, set `shard_transitions[shard] = full_transitions[winning_root]` if the committee had enough weight to form a crosslink this slot. Specifically: -* Call `shards, winning_roots = get_shard_winning_roots(state, block.slot, attestations)` +* Call `shards, winning_roots = get_shard_winning_roots(state, block.slot, block.body.attestations)` * Let `full_transitions` be a dictionary mapping from the `shard_transition_root`s found in `attestations` to the corresponding full `ShardTransition` * Then for each `shard` and `winning_root` in `zip(shards, winning_roots)` set `shard_transitions[shard] = full_transitions[winning_root]` @@ -140,7 +140,7 @@ Specifically: ```python def get_shard_winning_roots(state: BeaconState, slot: Slot, - attestations: sequence[Attestation]) -> Tuple[Sequence[Shard], Sequence[Root]]: + attestations: Sequence[Attestation]) -> Tuple[Sequence[Shard], Sequence[Root]]: shards = [] winning_roots = [] online_indices = get_online_validator_indices(state) @@ -280,7 +280,7 @@ def get_shard_transition_fields( shard_block_slots = [shard_block.message.slot for shard_block in shard_blocks] offset_slots = compute_offset_slots( get_latest_slot_for_shard(beacon_state, shard), - beacon_state.slot + 1, + Slot(beacon_state.slot + 1), ) for slot in offset_slots: if slot in shard_block_slots: @@ -302,7 +302,7 @@ def get_shard_transition(beacon_state: BeaconState, shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition: offset_slots = compute_offset_slots( get_latest_slot_for_shard(beacon_state, shard), - beacon_state.slot + 1, + Slot(beacon_state.slot + 1), ) shard_block_lengths, shard_data_roots, shard_states = ( get_shard_transition_fields(beacon_state, shard, shard_blocks) From c9a53b8039cb2eada12cbb09ca1cd74509356549 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 05:33:01 +0800 Subject: [PATCH 099/165] WIP test case --- .../test/fork_choice/test_on_shard_head.py | 91 ++++++++++++------- .../eth2spec/test/helpers/shard_block.py | 26 ++++-- 2 files changed, 78 insertions(+), 39 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 5b4205e91..1151d18d7 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -5,6 +5,7 @@ from eth2spec.test.helpers.shard_block import ( build_attestation_with_shard_transition, build_shard_block, build_shard_transitions_till_slot, + get_committee_index_of_shard, ) from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_block @@ -24,44 +25,65 @@ def run_on_shard_block(spec, store, shard_store, signed_block, valid=True): assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message -def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index): +def run_apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer): shard = shard_store.shard + committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) + has_shard_committee = committee_index is not None store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH # Create SignedShardBlock - body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE - target_len_offset_slot = 1 - shard_block = build_shard_block(spec, state, shard, body=body, signed=True) - shard_blocks = [shard_block] + # Check offsets + temp_state = state.copy() + next_slot(spec, temp_state) + offset_slots = spec.get_offset_slots(temp_state, shard) + if state.slot in offset_slots: + # Build block + body = b'\x56' * 4 + shard_head_root = spec.get_shard_head(store, shard_store) + shard_parent_state = shard_store.block_states[shard_head_root] + assert shard_parent_state.slot != state.slot + shard_block = build_shard_block( + spec, state, shard, + shard_parent_state=shard_parent_state, slot=state.slot, body=body, signed=True + ) + shard_blocks_buffer.append(shard_block) + run_on_shard_block(spec, store, shard_store, shard_block) + assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() + + beacon_block = build_empty_block(spec, state, slot=state.slot + 1) # Attester creates `attestation` - # Use temporary next state to get ShardTransition of shard block - shard_transitions = build_shard_transitions_till_slot( - spec, - state, - shard_blocks={shard: shard_blocks}, - on_time_slot=state.slot + target_len_offset_slot, - ) - shard_transition = shard_transitions[shard] - attestation = build_attestation_with_shard_transition( - spec, - state, - index=committee_index, - on_time_slot=state.slot + target_len_offset_slot, - shard_transition=shard_transition, - ) + if has_shard_committee and len(shard_blocks_buffer) > 0: + # Use temporary next state to get ShardTransition of shard block + shard_transitions = build_shard_transitions_till_slot( + spec, + state, + shard_blocks={shard: shard_blocks_buffer}, + on_time_slot=state.slot + 1, + ) + shard_transition = shard_transitions[shard] + + attestation = build_attestation_with_shard_transition( + spec, + state, + index=committee_index, + on_time_slot=state.slot + 1, + shard_transition=shard_transition, + ) + assert attestation.data.slot == state.slot + assert spec.get_shard(state, attestation) == shard + beacon_block.body.attestations = [attestation] + beacon_block.body.shard_transitions = shard_transitions - # Propose beacon block at slot - beacon_block = build_empty_block(spec, state, slot=state.slot + 1) - beacon_block.body.attestations = [attestation] - beacon_block.body.shard_transitions = shard_transitions signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) - run_on_shard_block(spec, store, shard_store, shard_block) add_block_to_store(spec, store, signed_beacon_block) - assert spec.get_head(store) == beacon_block.hash_tree_root() - assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() + + if has_shard_committee: + shard_blocks_buffer = [] # clear buffer + + return has_shard_committee, shard_blocks_buffer @with_all_phases_except([PHASE0]) @@ -69,16 +91,19 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) def test_basic(spec, state): spec.PHASE_1_GENESIS_SLOT = 0 # FIXME: remove mocking state = spec.upgrade_to_phase1(state) - next_slot(spec, state) # Initialization store = spec.get_forkchoice_store(state) anchor_root = get_anchor_root(spec, state) assert spec.get_head(store) == anchor_root - committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard = spec.Shard(1) shard_store = spec.get_forkchoice_shard_store(state, shard) - - run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) - run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) + shard_block_count = 2 + shard_blocks_buffer = [] + while shard_block_count > 0: + has_shard_committee, shard_blocks_buffer = run_apply_shard_and_beacon( + spec, state, store, shard_store, shard_blocks_buffer + ) + if has_shard_committee: + shard_block_count -= 1 diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 58efada83..410213edd 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -23,19 +23,24 @@ def build_shard_block(spec, shard, slot=None, body=None, + shard_parent_state=None, signed=False): - shard_state = beacon_state.shard_states[shard] + if shard_parent_state is None: + shard_parent_state = beacon_state.shard_states[shard] + if slot is None: - slot = shard_state.slot + 1 + slot = shard_parent_state.slot + 1 if body is None: body = b'\x56' * 128 - proposer_index = spec.get_shard_proposer_index(beacon_state, slot, shard) + temp_state = beacon_state.copy() + transition_to(spec, temp_state, slot) beacon_state, beacon_parent_root = get_state_and_beacon_parent_root_at_slot(spec, beacon_state, slot) - + assert beacon_state == temp_state + proposer_index = spec.get_shard_proposer_index(temp_state, slot, shard) block = spec.ShardBlock( - shard_parent_root=shard_state.latest_block_root, + shard_parent_root=shard_parent_state.latest_block_root, beacon_parent_root=beacon_parent_root, slot=slot, shard=shard, @@ -59,7 +64,6 @@ def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): for shard, blocks in shard_blocks.items(): offset_slots = spec.get_offset_slots(temp_state, shard) len_offset_slots = len(offset_slots) - assert len_offset_slots == on_time_slot - state.shard_states[shard].slot - 1 shard_transition = spec.get_shard_transition(temp_state, shard, blocks) if len(blocks) > 0: shard_block_root = blocks[-1].message.hash_tree_root() @@ -84,3 +88,13 @@ def build_attestation_with_shard_transition(spec, state, index, on_time_slot, sh if shard_transition is not None: assert attestation.data.shard_transition_root == shard_transition.hash_tree_root() return attestation + + +def get_committee_index_of_shard(spec, state, slot, shard): # Optional[CommitteeIndex] + active_shard_count = spec.get_active_shard_count(state) + committee_count = spec.get_committee_count_at_slot(state, slot) + start_shard = spec.get_start_shard(state, slot) + for committee_index in range(committee_count): + if (start_shard + committee_index) % active_shard_count == shard: + return committee_index + return None From 727353c054f259aa81e7738099fc055722e9734c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 18:39:27 +0800 Subject: [PATCH 100/165] Verify shard_block.slot fits the expected offset_slots --- specs/phase1/shard-fork-choice.md | 9 +++++---- specs/phase1/shard-transition.md | 3 +++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index fb98893ac..b60edd948 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -127,7 +127,6 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si # Check beacon parent exists assert shard_block.beacon_parent_root in store.block_states - beacon_state = store.block_states[shard_block.beacon_parent_root] # Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) finalized_beacon_state = store.block_states[store.finalized_checkpoint.root] @@ -144,9 +143,11 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si shard_store.blocks[hash_tree_root(shard_block)] = shard_block # Check the block is valid and compute the post-state - verify_shard_block_message(beacon_state, pre_shard_state, shard_block, shard_block.slot, shard) - verify_shard_block_signature(beacon_state, signed_shard_block) - post_state = get_post_shard_state(beacon_state, pre_shard_state, shard_block) + beacon_head_root = get_head(store) + beacon_head_state = store.block_states[beacon_head_root] + assert verify_shard_block_message(beacon_head_state, pre_shard_state, shard_block, shard_block.slot, shard) + assert verify_shard_block_signature(beacon_head_state, signed_shard_block) + post_state = get_post_shard_state(beacon_head_state, pre_shard_state, shard_block) # Add new state for this block to the store shard_store.block_states[hash_tree_root(shard_block)] = post_state ``` diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 5e8616568..6c4f652d2 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -50,6 +50,9 @@ def verify_shard_block_message(beacon_state: BeaconState, shard: Shard) -> bool: assert block.shard_parent_root == shard_state.latest_block_root assert block.slot == slot + next_slot = Slot(beacon_state.slot + 1) + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), next_slot) + assert slot in offset_slots assert block.shard == shard assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard) assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE From f8597d296545c6711d1c7cf8fe25d5dbb5b1987b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 20:31:54 +0800 Subject: [PATCH 101/165] Add `get_pendings_shard_blocks` --- specs/phase1/shard-fork-choice.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index b60edd948..5bc2cbe4e 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -16,6 +16,7 @@ - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance) - [`get_shard_head`](#get_shard_head) - [`get_shard_ancestor`](#get_shard_ancestor) + - [`get_pendings_shard_blocks`](#get_pendings_shard_blocks) - [Handlers](#handlers) - [`on_shard_block`](#on_shard_block) @@ -108,6 +109,31 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: return root ``` +#### `get_pendings_shard_blocks` + +```python +def get_pendings_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ShardBlock]: + """ + Return the shard blocks branch that from shard head to beacon head. + """ + shard = shard_store.shard + + beacon_head_root = get_head(store) + beacon_head_state = store.block_states[beacon_head_root] + latest_shard_block_root = beacon_head_state.shard_states[shard].latest_block_root + + shard_head_root = get_shard_head(store, shard_store) + root = shard_head_root + shard_blocks = [] + while root != latest_shard_block_root: + shard_block = shard_store.blocks[root] + shard_blocks.append(shard_block) + root = shard_block.shard_parent_root + + shard_blocks.reverse() + return shard_blocks +``` + ### Handlers #### `on_shard_block` From ab42eee4c04818d58fbcea5ba83d1c5890d0c3b8 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 20:32:31 +0800 Subject: [PATCH 102/165] Update shard fork choice rule to be able to handle mainnet config --- .../test/fork_choice/test_on_shard_head.py | 97 ++++++++++++------- 1 file changed, 62 insertions(+), 35 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 1151d18d7..7e94ddd8e 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -1,6 +1,6 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root -from eth2spec.test.context import spec_state_test, with_all_phases_except, PHASE0 +from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_except, never_bls from eth2spec.test.helpers.shard_block import ( build_attestation_with_shard_transition, build_shard_block, @@ -8,7 +8,7 @@ from eth2spec.test.helpers.shard_block import ( get_committee_index_of_shard, ) from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root -from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_block +from eth2spec.test.helpers.state import state_transition_and_sign_block from eth2spec.test.helpers.block import build_empty_block @@ -25,35 +25,51 @@ def run_on_shard_block(spec, store, shard_store, signed_block, valid=True): assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message -def run_apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer): +def apply_shard_block(spec, store, shard_store, beacon_head_state, shard_blocks_buffer): shard = shard_store.shard - committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) - has_shard_committee = committee_index is not None + body = b'\x56' * 4 + shard_head_root = spec.get_shard_head(store, shard_store) + shard_parent_state = shard_store.block_states[shard_head_root] + assert shard_parent_state.slot != beacon_head_state.slot + shard_block = build_shard_block( + spec, beacon_head_state, shard, + shard_parent_state=shard_parent_state, slot=beacon_head_state.slot, body=body, signed=True + ) + shard_blocks_buffer.append(shard_block) + run_on_shard_block(spec, store, shard_store, shard_block) + assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() + + +def check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer): + pending_shard_blocks = [ + spec.SignedShardBlock(message=b) + for b in spec.get_pendings_shard_blocks(store, shard_store) + ] + assert pending_shard_blocks == shard_blocks_buffer + + +def is_in_offset_sets(spec, beacon_head_state, shard): + offset_slots = spec.compute_offset_slots( + beacon_head_state.shard_states[shard].slot, beacon_head_state.slot + 1 + ) + return beacon_head_state.slot in offset_slots + + +def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer): store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH - # Create SignedShardBlock - # Check offsets - temp_state = state.copy() - next_slot(spec, temp_state) - offset_slots = spec.get_offset_slots(temp_state, shard) - if state.slot in offset_slots: - # Build block - body = b'\x56' * 4 - shard_head_root = spec.get_shard_head(store, shard_store) - shard_parent_state = shard_store.block_states[shard_head_root] - assert shard_parent_state.slot != state.slot - shard_block = build_shard_block( - spec, state, shard, - shard_parent_state=shard_parent_state, slot=state.slot, body=body, signed=True - ) - shard_blocks_buffer.append(shard_block) - run_on_shard_block(spec, store, shard_store, shard_block) - assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() + shard = shard_store.shard + committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) + has_shard_committee = committee_index is not None # has committee of `shard` at this slot + # On beacon blocks at `state.slot + 1` beacon_block = build_empty_block(spec, state, slot=state.slot + 1) - # Attester creates `attestation` + # If next slot has committee of `shard`, add `shard_transtion` to the proposing beacon block if has_shard_committee and len(shard_blocks_buffer) > 0: + # Sanity check `get_pendings_shard_blocks` function + check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer) + # Use temporary next state to get ShardTransition of shard block shard_transitions = build_shard_transitions_till_slot( spec, @@ -62,7 +78,6 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buf on_time_slot=state.slot + 1, ) shard_transition = shard_transitions[shard] - attestation = build_attestation_with_shard_transition( spec, state, @@ -75,35 +90,47 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buf beacon_block.body.attestations = [attestation] beacon_block.body.shard_transitions = shard_transitions - signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) + # Clear buffer + shard_blocks_buffer.clear() + signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) add_block_to_store(spec, store, signed_beacon_block) assert spec.get_head(store) == beacon_block.hash_tree_root() - if has_shard_committee: - shard_blocks_buffer = [] # clear buffer + # On shard block at updated `state.slot` + if is_in_offset_sets(spec, state, shard): + # The created shard block would be appended to `shard_blocks_buffer` + apply_shard_block(spec, store, shard_store, state, shard_blocks_buffer) - return has_shard_committee, shard_blocks_buffer + return has_shard_committee @with_all_phases_except([PHASE0]) @spec_state_test +@never_bls # Set to never_bls for testing `check_pending_shard_blocks` def test_basic(spec, state): - spec.PHASE_1_GENESIS_SLOT = 0 # FIXME: remove mocking + spec.PHASE_1_GENESIS_SLOT = 0 # NOTE: mock genesis slot here state = spec.upgrade_to_phase1(state) + shard = spec.Shard(1) # Initialization store = spec.get_forkchoice_store(state) anchor_root = get_anchor_root(spec, state) assert spec.get_head(store) == anchor_root - shard = spec.Shard(1) shard_store = spec.get_forkchoice_shard_store(state, shard) - shard_block_count = 2 + shard_head_root = spec.get_shard_head(store, shard_store) + assert shard_head_root == state.shard_states[shard].latest_block_root + assert shard_store.block_states[shard_head_root].slot == 1 + assert shard_store.block_states[shard_head_root] == state.shard_states[shard] + + # For mainnet config, it's possible that only one committee of `shard` per epoch. + # we set this counter to test more rounds. + shard_committee_counter = 2 shard_blocks_buffer = [] - while shard_block_count > 0: - has_shard_committee, shard_blocks_buffer = run_apply_shard_and_beacon( + while shard_committee_counter > 0: + has_shard_committee = apply_shard_and_beacon( spec, state, store, shard_store, shard_blocks_buffer ) if has_shard_committee: - shard_block_count -= 1 + shard_committee_counter -= 1 From 6f9c290bfb97240a9f41370b026d066ed8df6917 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 20:40:09 +0800 Subject: [PATCH 103/165] Add TODO flag of latest message --- specs/phase1/shard-fork-choice.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 5bc2cbe4e..a474c8214 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -61,6 +61,8 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro state.validators[i].effective_balance for i in active_indices if ( i in store.latest_messages + # TODO: check the latest message logic: currently, validator's previous vote of another shard + # would be ignored once their newer vote is accepted. Check if it makes sense. and store.latest_messages[i].shard == shard_store.shard and get_shard_ancestor( store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot From a154d0c22b9a3175e74383e4df53e5b09f6df0fd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 21:24:17 +0800 Subject: [PATCH 104/165] Fix typo --- specs/phase1/shard-fork-choice.md | 6 +++--- .../pyspec/eth2spec/test/fork_choice/test_on_shard_head.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index a474c8214..427b72784 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -16,7 +16,7 @@ - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance) - [`get_shard_head`](#get_shard_head) - [`get_shard_ancestor`](#get_shard_ancestor) - - [`get_pendings_shard_blocks`](#get_pendings_shard_blocks) + - [`get_pending_shard_blocks`](#get_pending_shard_blocks) - [Handlers](#handlers) - [`on_shard_block`](#on_shard_block) @@ -111,10 +111,10 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: return root ``` -#### `get_pendings_shard_blocks` +#### `get_pending_shard_blocks` ```python -def get_pendings_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ShardBlock]: +def get_pending_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ShardBlock]: """ Return the shard blocks branch that from shard head to beacon head. """ diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 7e94ddd8e..1ba15968f 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -43,7 +43,7 @@ def apply_shard_block(spec, store, shard_store, beacon_head_state, shard_blocks_ def check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer): pending_shard_blocks = [ spec.SignedShardBlock(message=b) - for b in spec.get_pendings_shard_blocks(store, shard_store) + for b in spec.get_pending_shard_blocks(store, shard_store) ] assert pending_shard_blocks == shard_blocks_buffer @@ -67,7 +67,7 @@ def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer) # If next slot has committee of `shard`, add `shard_transtion` to the proposing beacon block if has_shard_committee and len(shard_blocks_buffer) > 0: - # Sanity check `get_pendings_shard_blocks` function + # Sanity check `get_pending_shard_blocks` function check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer) # Use temporary next state to get ShardTransition of shard block @@ -129,6 +129,7 @@ def test_basic(spec, state): shard_committee_counter = 2 shard_blocks_buffer = [] while shard_committee_counter > 0: + print(f'state.slot', state.slot) has_shard_committee = apply_shard_and_beacon( spec, state, store, shard_store, shard_blocks_buffer ) From 2d4788fe7d032f473fe5d60d732c40505ad8f485 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 5 Jun 2020 16:19:25 +0800 Subject: [PATCH 105/165] Fix `verify_shard_block_message` Add check for `block.beacon_parent_root` per Terence's suggestion Update `get_shard_transition` 1. Disable verification: it will be fix in v-guide 2. Use `on_time_slot` to compute offset_slots Rework tests --- specs/phase1/shard-fork-choice.md | 11 ++-- specs/phase1/shard-transition.md | 66 +++++++++++-------- .../test/fork_choice/test_on_shard_head.py | 19 ++---- .../eth2spec/test/helpers/attestations.py | 2 +- .../eth2spec/test/helpers/shard_block.py | 36 +++------- .../test/helpers/shard_transitions.py | 5 +- .../test_process_shard_transition.py | 31 ++++----- .../test/phase_1/sanity/test_blocks.py | 56 ++++++---------- 8 files changed, 94 insertions(+), 132 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 427b72784..844dbfb86 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -151,10 +151,11 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si # Check shard parent exists assert shard_block.shard_parent_root in shard_store.block_states - pre_shard_state = shard_store.block_states[shard_block.shard_parent_root] + shard_parent_state = shard_store.block_states[shard_block.shard_parent_root] # Check beacon parent exists assert shard_block.beacon_parent_root in store.block_states + beacon_parent_state = store.block_states[shard_block.beacon_parent_root] # Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) finalized_beacon_state = store.block_states[store.finalized_checkpoint.root] @@ -171,11 +172,9 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si shard_store.blocks[hash_tree_root(shard_block)] = shard_block # Check the block is valid and compute the post-state - beacon_head_root = get_head(store) - beacon_head_state = store.block_states[beacon_head_root] - assert verify_shard_block_message(beacon_head_state, pre_shard_state, shard_block, shard_block.slot, shard) - assert verify_shard_block_signature(beacon_head_state, signed_shard_block) - post_state = get_post_shard_state(beacon_head_state, pre_shard_state, shard_block) + assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block, shard_block.slot, shard) + assert verify_shard_block_signature(beacon_parent_state, signed_shard_block) + post_state = get_post_shard_state(beacon_parent_state, shard_parent_state, shard_block) # Add new state for this block to the store shard_store.block_states[hash_tree_root(shard_block)] = post_state ``` diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 6c4f652d2..8d75879f5 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -30,7 +30,7 @@ This document describes the shard transition function and fraud proofs as part o ### Misc ```python -def compute_shard_transition_digest(beacon_state: BeaconState, +def compute_shard_transition_digest(beacon_parent_state: BeaconState, shard_state: ShardState, beacon_parent_root: Root, shard_body_root: Root) -> Bytes32: @@ -44,17 +44,33 @@ def compute_shard_transition_digest(beacon_state: BeaconState, ```python def verify_shard_block_message(beacon_state: BeaconState, - shard_state: ShardState, + shard_parent_state: ShardState, block: ShardBlock, slot: Slot, - shard: Shard) -> bool: - assert block.shard_parent_root == shard_state.latest_block_root - assert block.slot == slot - next_slot = Slot(beacon_state.slot + 1) + shard: Shard, + beacon_parent_slot: Slot=None) -> bool: + # Check `shard_parent_root` field + assert block.shard_parent_root == shard_parent_state.latest_block_root + # Check `beacon_parent_root` field + if beacon_parent_slot is None: + beacon_parent_slot = beacon_state.slot + if beacon_parent_slot == beacon_state.slot: + beacon_parent_block_header = beacon_state.latest_block_header.copy() + if beacon_parent_block_header.state_root == Root(): + beacon_parent_block_header.state_root = hash_tree_root(beacon_state) + beacon_parent_root = hash_tree_root(beacon_parent_block_header) + else: + beacon_parent_root = get_block_root_at_slot(beacon_state, beacon_parent_slot) + assert block.beacon_parent_root == beacon_parent_root + # Check `slot` field + next_slot = Slot(slot + 1) offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), next_slot) assert slot in offset_slots + # Check `shard` field assert block.shard == shard + # Check `proposer_index` field assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard) + # Check `body` field assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE return True ``` @@ -177,7 +193,7 @@ def compute_shard_body_roots(proposals: Sequence[SignedShardBlock]) -> Sequence[ ```python def get_proposal_choices_at_slot(beacon_state: BeaconState, - shard_state: ShardState, + shard_parent_state: ShardState, slot: Slot, shard: Shard, shard_blocks: Sequence[SignedShardBlock], @@ -188,25 +204,16 @@ def get_proposal_choices_at_slot(beacon_state: BeaconState, """ choices = [] shard_blocks_at_slot = [block for block in shard_blocks if block.message.slot == slot] + shard_state = shard_parent_state.copy() for block in shard_blocks_at_slot: - try: - # Verify block message and signature - # TODO these validations should have been checked upon receiving shard blocks. - assert verify_shard_block_message(beacon_state, shard_state, block.message, slot, shard) - if validate_signature: - assert verify_shard_block_signature(beacon_state, block) - - shard_state = get_post_shard_state(beacon_state, shard_state, block.message) - except Exception: - pass # TODO: throw error in the test helper - else: - choices.append(block) + shard_state = get_post_shard_state(beacon_state, shard_state, block.message) + choices.append(block) return choices ``` ```python def get_proposal_at_slot(beacon_state: BeaconState, - shard_state: ShardState, + shard_parent_state: ShardState, slot: Shard, shard: Shard, shard_blocks: Sequence[SignedShardBlock], @@ -217,7 +224,7 @@ def get_proposal_at_slot(beacon_state: BeaconState, """ choices = get_proposal_choices_at_slot( beacon_state=beacon_state, - shard_state=shard_state, + shard_parent_state=shard_parent_state, slot=slot, shard=shard, shard_blocks=shard_blocks, @@ -232,7 +239,7 @@ def get_proposal_at_slot(beacon_state: BeaconState, proposal = get_winning_proposal(beacon_state, choices) # Apply state transition - shard_state = get_post_shard_state(beacon_state, shard_state, proposal.message) + shard_state = get_post_shard_state(beacon_state, shard_parent_state, proposal.message) return proposal, shard_state ``` @@ -242,15 +249,17 @@ def get_shard_state_transition_result( beacon_state: BeaconState, shard: Shard, shard_blocks: Sequence[SignedShardBlock], + on_time_slot: Slot, validate_signature: bool=True, ) -> Tuple[Sequence[SignedShardBlock], Sequence[ShardState], Sequence[Root]]: proposals = [] shard_states = [] shard_state = beacon_state.shard_states[shard] - for slot in get_offset_slots(beacon_state, shard): + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) + for slot in offset_slots: proposal, shard_state = get_proposal_at_slot( beacon_state=beacon_state, - shard_state=shard_state, + shard_parent_state=shard_state, slot=slot, shard=shard, shard_blocks=shard_blocks, @@ -271,9 +280,12 @@ Suppose you are a committee member on shard `shard` at slot `current_slot` and y ```python def get_shard_transition(beacon_state: BeaconState, shard: Shard, - shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition: - offset_slots = get_offset_slots(beacon_state, shard) - proposals, shard_states, shard_data_roots = get_shard_state_transition_result(beacon_state, shard, shard_blocks) + shard_blocks: Sequence[SignedShardBlock], + on_time_slot: Slot) -> ShardTransition: + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) + proposals, shard_states, shard_data_roots = get_shard_state_transition_result( + beacon_state, shard, shard_blocks, on_time_slot + ) shard_block_lengths = [] proposer_signatures = [] diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 1ba15968f..ca79cfd23 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -1,8 +1,8 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_except, never_bls +from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.shard_block import ( - build_attestation_with_shard_transition, build_shard_block, build_shard_transitions_till_slot, get_committee_index_of_shard, @@ -25,15 +25,15 @@ def run_on_shard_block(spec, store, shard_store, signed_block, valid=True): assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message -def apply_shard_block(spec, store, shard_store, beacon_head_state, shard_blocks_buffer): +def apply_shard_block(spec, store, shard_store, beacon_parent_state, shard_blocks_buffer): shard = shard_store.shard body = b'\x56' * 4 shard_head_root = spec.get_shard_head(store, shard_store) shard_parent_state = shard_store.block_states[shard_head_root] - assert shard_parent_state.slot != beacon_head_state.slot + assert shard_parent_state.slot != beacon_parent_state.slot shard_block = build_shard_block( - spec, beacon_head_state, shard, - shard_parent_state=shard_parent_state, slot=beacon_head_state.slot, body=body, signed=True + spec, beacon_parent_state, shard, + shard_parent_state=shard_parent_state, slot=beacon_parent_state.slot, body=body, signed=True ) shard_blocks_buffer.append(shard_block) run_on_shard_block(spec, store, shard_store, shard_block) @@ -62,30 +62,26 @@ def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer) committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) has_shard_committee = committee_index is not None # has committee of `shard` at this slot - # On beacon blocks at `state.slot + 1` beacon_block = build_empty_block(spec, state, slot=state.slot + 1) # If next slot has committee of `shard`, add `shard_transtion` to the proposing beacon block if has_shard_committee and len(shard_blocks_buffer) > 0: # Sanity check `get_pending_shard_blocks` function check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer) - # Use temporary next state to get ShardTransition of shard block shard_transitions = build_shard_transitions_till_slot( spec, state, shard_blocks={shard: shard_blocks_buffer}, - on_time_slot=state.slot + 1, ) shard_transition = shard_transitions[shard] - attestation = build_attestation_with_shard_transition( + attestation = get_valid_on_time_attestation( spec, state, index=committee_index, - on_time_slot=state.slot + 1, shard_transition=shard_transition, + signed=False, ) - assert attestation.data.slot == state.slot assert spec.get_shard(state, attestation) == shard beacon_block.body.attestations = [attestation] beacon_block.body.shard_transitions = shard_transitions @@ -129,7 +125,6 @@ def test_basic(spec, state): shard_committee_counter = 2 shard_blocks_buffer = [] while shard_committee_counter > 0: - print(f'state.slot', state.slot) has_shard_committee = apply_shard_and_beacon( spec, state, store, shard_store, shard_blocks_buffer ) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 1372b0654..106069dd6 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -87,7 +87,7 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t if on_time: temp_state = state.copy() next_slot(spec, temp_state) - shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=[]) + shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=[], on_time_slot=slot + 1) lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 410213edd..0a2ed67d2 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -1,6 +1,4 @@ -from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import get_state_and_beacon_parent_root_at_slot -from eth2spec.test.helpers.state import transition_to from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.bls import only_with_bls @@ -34,11 +32,8 @@ def build_shard_block(spec, if body is None: body = b'\x56' * 128 - temp_state = beacon_state.copy() - transition_to(spec, temp_state, slot) beacon_state, beacon_parent_root = get_state_and_beacon_parent_root_at_slot(spec, beacon_state, slot) - assert beacon_state == temp_state - proposer_index = spec.get_shard_proposer_index(temp_state, slot, shard) + proposer_index = spec.get_shard_proposer_index(beacon_state, slot, shard) block = spec.ShardBlock( shard_parent_root=shard_parent_state.latest_block_root, beacon_parent_root=beacon_parent_root, @@ -57,14 +52,17 @@ def build_shard_block(spec, return signed_block -def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): - temp_state = state.copy() - transition_to(spec, temp_state, on_time_slot) +def build_shard_transitions_till_slot(spec, parent_beacon_state, shard_blocks): shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS + on_time_slot = parent_beacon_state.slot + 1 for shard, blocks in shard_blocks.items(): - offset_slots = spec.get_offset_slots(temp_state, shard) + offset_slots = spec.compute_offset_slots( + spec.get_latest_slot_for_shard(parent_beacon_state, shard), + on_time_slot, + ) len_offset_slots = len(offset_slots) - shard_transition = spec.get_shard_transition(temp_state, shard, blocks) + shard_transition = spec.get_shard_transition(parent_beacon_state, shard, blocks, on_time_slot) + if len(blocks) > 0: shard_block_root = blocks[-1].message.hash_tree_root() assert shard_transition.shard_states[len_offset_slots - 1].latest_block_root == shard_block_root @@ -74,22 +72,6 @@ def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): return shard_transitions -def build_attestation_with_shard_transition(spec, state, index, on_time_slot, shard_transition): - temp_state = state.copy() - transition_to(spec, temp_state, on_time_slot - 1) - attestation = get_valid_on_time_attestation( - spec, - temp_state, - index=index, - shard_transition=shard_transition, - signed=True, - ) - assert attestation.data.slot == temp_state.slot - if shard_transition is not None: - assert attestation.data.shard_transition_root == shard_transition.hash_tree_root() - return attestation - - def get_committee_index_of_shard(spec, state, slot, shard): # Optional[CommitteeIndex] active_shard_count = spec.get_active_shard_count(state) committee_count = spec.get_committee_count_at_slot(state, slot) diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py index abb5e7278..8e62b2f27 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py @@ -1,5 +1,4 @@ from eth2spec.test.context import expect_assertion_error -from eth2spec.test.helpers.state import transition_to def run_shard_transitions_processing(spec, state, shard_transitions, attestations, valid=True): @@ -37,7 +36,5 @@ def get_shard_transition_of_committee(spec, state, committee_index, slot=None, s slot = state.slot shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) - temp_state = state.copy() - transition_to(spec, temp_state, slot + 1) - shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=shard_blocks) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks, on_time_slot=slot + 1) return shard_transition diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py index 00ffbe0a8..dab4973da 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py @@ -2,71 +2,64 @@ from eth2spec.test.context import ( PHASE0, with_all_phases_except, spec_state_test, - always_bls, ) +from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.shard_transitions import run_shard_transitions_processing from eth2spec.test.helpers.shard_block import ( - build_attestation_with_shard_transition, build_shard_block, build_shard_transitions_till_slot, ) -from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot +from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot, next_slot def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): state = transition_to_valid_shard_slot(spec, state) - init_slot = state.slot committee_index = spec.CommitteeIndex(0) shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 + transition_to(spec, state, state.slot + target_len_offset_slot) + assert state.shard_states[shard].slot == state.slot - target_len_offset_slot - 1 # Create SignedShardBlock body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE - shard_block = build_shard_block(spec, state, shard, body=body, signed=True) + shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) shard_blocks = [shard_block] - # Create a shard_transitions that would be included at beacon block `state.slot + target_len_offset_slot` shard_transitions = build_shard_transitions_till_slot( spec, state, shard_blocks={shard: shard_blocks}, - on_time_slot=state.slot + target_len_offset_slot, ) shard_transition = shard_transitions[shard] - # Create an attestation that would be included at beacon block `state.slot + target_len_offset_slot` - attestation = build_attestation_with_shard_transition( + attestation = get_valid_on_time_attestation( spec, state, index=committee_index, - on_time_slot=state.slot + target_len_offset_slot, shard_transition=shard_transition, + signed=False, ) + next_slot(spec, state) pre_gasprice = state.shard_states[shard].gasprice - - transition_to(spec, state, state.slot + target_len_offset_slot) pre_shard_state = state.shard_states[shard] - yield from run_shard_transitions_processing(spec, state, shard_transitions, [attestation], valid=valid) if valid: - # After state transition, - assert state.slot == init_slot + target_len_offset_slot shard_state = state.shard_states[shard] assert shard_state != pre_shard_state assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1] - + assert shard_state.latest_block_root == shard_block.message.hash_tree_root() if target_len_offset_slot == 1: assert shard_state.gasprice > pre_gasprice @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_basic_crosslinks(spec, state): + # NOTE: this test is only for full crosslink (minimal config), not for mainnet yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=1, valid=True) @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_multiple_offset_slots(spec, state): - yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=3, valid=True) + # NOTE: this test is only for full crosslink (minimal config), not for mainnet + yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=2, valid=True) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py index 0175bd40d..828cd19d7 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py @@ -4,37 +4,40 @@ from eth2spec.test.context import ( PHASE0, with_all_phases_except, spec_state_test, - always_bls, ) +from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import build_empty_block from eth2spec.test.helpers.shard_block import ( - build_attestation_with_shard_transition, build_shard_block, build_shard_transitions_till_slot, ) -from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot +from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot, transition_to -def run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index, valid=True): - shard_transitions = build_shard_transitions_till_slot( - spec, state, shard_blocks, on_time_slot=state.slot + target_len_offset_slot - ) +def run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard, valid=True): + transition_to(spec, state, state.slot + target_len_offset_slot) + + body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE + shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) + shard_blocks: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]} + + shard_transitions = build_shard_transitions_till_slot(spec, state, shard_blocks) attestations = [ - build_attestation_with_shard_transition( + get_valid_on_time_attestation( spec, state, - on_time_slot=state.slot + target_len_offset_slot, index=committee_index, shard_transition=shard_transitions[shard], + signed=True, ) for shard in shard_blocks.keys() ] - # Propose beacon block at slot `x + 1` - beacon_block = build_empty_block(spec, state, slot=state.slot + target_len_offset_slot) + beacon_block = build_empty_block(spec, state, slot=state.slot + 1) beacon_block.body.attestations = attestations beacon_block.body.shard_transitions = shard_transitions + pre_gasprice = state.shard_states[shard].gasprice pre_shard_states = state.shard_states.copy() yield 'pre', state.copy() yield 'block', beacon_block @@ -52,17 +55,18 @@ def run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_off assert post_shard_state == shard_transitions[shard].shard_states[ len(shard_transitions[shard].shard_states) - 1 ] - assert beacon_block.slot == shard_transitions[shard].shard_states[0].slot + target_len_offset_slot assert post_shard_state.slot == state.slot - 1 if len(shard_blocks[shard]) == 0: # `latest_block_root` is the same assert post_shard_state.latest_block_root == pre_shard_states[shard].latest_block_root + if target_len_offset_slot == 1 and len(shard_blocks) > 0: + assert post_shard_state.gasprice > pre_gasprice @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_process_beacon_block_with_normal_shard_transition(spec, state): + # NOTE: this test is only for full crosslink (minimal config), not for mainnet state = transition_to_valid_shard_slot(spec, state) target_len_offset_slot = 1 @@ -70,25 +74,13 @@ def test_process_beacon_block_with_normal_shard_transition(spec, state): shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 - pre_gasprice = state.shard_states[shard].gasprice - - # Create SignedShardBlock at slot `shard_state.slot + 1` - body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE - shard_block = build_shard_block(spec, state, shard, body=body, signed=True) - shard_blocks: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]} - - yield from run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index) - - shard_state = state.shard_states[shard] - - if target_len_offset_slot == 1 and len(shard_blocks) > 0: - assert shard_state.gasprice > pre_gasprice + yield from run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard) @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_process_beacon_block_with_empty_proposal_transition(spec, state): + # NOTE: this test is only for full crosslink (minimal config), not for mainnet state = transition_to_valid_shard_slot(spec, state) target_len_offset_slot = 1 @@ -96,12 +88,4 @@ def test_process_beacon_block_with_empty_proposal_transition(spec, state): shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 - # No new shard block - shard_blocks = {} - - pre_gasprice = state.shard_states[shard].gasprice - - yield from run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index) - - if target_len_offset_slot == 1 and len(shard_blocks) > 0: - assert state.shard_states[shard].gasprice > pre_gasprice + yield from run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard) From 2afa315cb3b1d8abcef3af2ef7df9046eb4f8f23 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 5 Jun 2020 21:49:50 +0800 Subject: [PATCH 106/165] clean leftover --- tests/core/pyspec/eth2spec/test/helpers/attestations.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 106069dd6..ef90a71aa 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -85,9 +85,7 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t # No shard transition -> no shard block shard = spec.get_shard(state, spec.Attestation(data=attestation_data)) if on_time: - temp_state = state.copy() - next_slot(spec, temp_state) - shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=[], on_time_slot=slot + 1) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=[], on_time_slot=slot + 1) lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() From a71c0a5ccc4c74f5ae8c3c071aa47506aa5cc03a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 6 Jun 2020 02:35:46 +0800 Subject: [PATCH 107/165] Per #1704 discussion, remove `on_time_slot`: the given `beacon_state` should be transitioned. --- specs/phase1/shard-transition.md | 4 ++-- tests/core/pyspec/eth2spec/test/helpers/attestations.py | 6 ++---- tests/core/pyspec/eth2spec/test/helpers/shard_block.py | 2 +- .../core/pyspec/eth2spec/test/helpers/shard_transitions.py | 7 ++----- 4 files changed, 7 insertions(+), 12 deletions(-) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 8d75879f5..191dbd1aa 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -280,8 +280,8 @@ Suppose you are a committee member on shard `shard` at slot `current_slot` and y ```python def get_shard_transition(beacon_state: BeaconState, shard: Shard, - shard_blocks: Sequence[SignedShardBlock], - on_time_slot: Slot) -> ShardTransition: + shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition: + on_time_slot = Slot(beacon_state.slot + 1) offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) proposals, shard_states, shard_data_roots = get_shard_state_transition_result( beacon_state, shard, shard_blocks, on_time_slot diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index ef90a71aa..1e0560405 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -85,7 +85,7 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t # No shard transition -> no shard block shard = spec.get_shard(state, spec.Attestation(data=attestation_data)) if on_time: - shard_transition = spec.get_shard_transition(state, shard, shard_blocks=[], on_time_slot=slot + 1) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=[]) lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() @@ -318,9 +318,7 @@ def next_epoch_with_attestations(spec, for index in range(committees_per_slot): if spec.fork == PHASE1: shard = spec.compute_shard_from_committee_index(post_state, index, slot_to_attest) - shard_transition = get_shard_transition_of_committee( - spec, post_state, index, slot=slot_to_attest - ) + shard_transition = get_shard_transition_of_committee(spec, post_state, index) block.body.shard_transitions[shard] = shard_transition else: shard_transition = None diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 0a2ed67d2..e63096b92 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -61,7 +61,7 @@ def build_shard_transitions_till_slot(spec, parent_beacon_state, shard_blocks): on_time_slot, ) len_offset_slots = len(offset_slots) - shard_transition = spec.get_shard_transition(parent_beacon_state, shard, blocks, on_time_slot) + shard_transition = spec.get_shard_transition(parent_beacon_state, shard, blocks) if len(blocks) > 0: shard_block_root = blocks[-1].message.hash_tree_root() diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py index 8e62b2f27..d10d1ee7b 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py @@ -28,13 +28,10 @@ def run_shard_transitions_processing(spec, state, shard_transitions, attestation yield 'post', state -def get_shard_transition_of_committee(spec, state, committee_index, slot=None, shard_blocks=None): +def get_shard_transition_of_committee(spec, state, committee_index, shard_blocks=None): if shard_blocks is None: shard_blocks = [] - if slot is None: - slot = state.slot - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) - shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks, on_time_slot=slot + 1) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks) return shard_transition From 7f680dfca4ad4c5085859ee4cb380fb7971ee5d9 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 5 Jun 2020 14:06:39 -0600 Subject: [PATCH 108/165] fix tests --- .../pyspec/eth2spec/test/helpers/attestations.py | 4 +--- .../pyspec/eth2spec/test/helpers/shard_block.py | 13 ++++--------- .../eth2spec/test/helpers/shard_transitions.py | 5 +---- .../test_process_shard_transition.py | 14 +++++++++----- 4 files changed, 15 insertions(+), 21 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 1372b0654..48930b95d 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -85,9 +85,7 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t # No shard transition -> no shard block shard = spec.get_shard(state, spec.Attestation(data=attestation_data)) if on_time: - temp_state = state.copy() - next_slot(spec, temp_state) - shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=[]) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=[]) lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 58efada83..20e9ca3b5 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -1,6 +1,5 @@ from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import get_state_and_beacon_parent_root_at_slot -from eth2spec.test.helpers.state import transition_to from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.bls import only_with_bls @@ -53,14 +52,13 @@ def build_shard_block(spec, def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): - temp_state = state.copy() - transition_to(spec, temp_state, on_time_slot) shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS for shard, blocks in shard_blocks.items(): - offset_slots = spec.get_offset_slots(temp_state, shard) + offset_slots = spec.compute_offset_slots(spec.get_latest_slot_for_shard(state, shard), on_time_slot) len_offset_slots = len(offset_slots) + # TODO this is actually unsafe for long offset_slots assert len_offset_slots == on_time_slot - state.shard_states[shard].slot - 1 - shard_transition = spec.get_shard_transition(temp_state, shard, blocks) + shard_transition = spec.get_shard_transition(state, shard, blocks) if len(blocks) > 0: shard_block_root = blocks[-1].message.hash_tree_root() assert shard_transition.shard_states[len_offset_slots - 1].latest_block_root == shard_block_root @@ -71,16 +69,13 @@ def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): def build_attestation_with_shard_transition(spec, state, index, on_time_slot, shard_transition): - temp_state = state.copy() - transition_to(spec, temp_state, on_time_slot - 1) attestation = get_valid_on_time_attestation( spec, - temp_state, + state, index=index, shard_transition=shard_transition, signed=True, ) - assert attestation.data.slot == temp_state.slot if shard_transition is not None: assert attestation.data.shard_transition_root == shard_transition.hash_tree_root() return attestation diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py index abb5e7278..544d88523 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py @@ -1,5 +1,4 @@ from eth2spec.test.context import expect_assertion_error -from eth2spec.test.helpers.state import transition_to def run_shard_transitions_processing(spec, state, shard_transitions, attestations, valid=True): @@ -37,7 +36,5 @@ def get_shard_transition_of_committee(spec, state, committee_index, slot=None, s slot = state.slot shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) - temp_state = state.copy() - transition_to(spec, temp_state, slot + 1) - shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=shard_blocks) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks) return shard_transition diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py index 00ffbe0a8..959eaa2d8 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py @@ -17,19 +17,23 @@ def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): state = transition_to_valid_shard_slot(spec, state) init_slot = state.slot committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) - assert state.shard_states[shard].slot == state.slot - 1 + shard_slot = state.slot + target_len_offset_slot - 1 + shard = spec.compute_shard_from_committee_index(state, committee_index, shard_slot) + assert state.shard_states[shard].slot == init_slot - 1 # Create SignedShardBlock body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE shard_block = build_shard_block(spec, state, shard, body=body, signed=True) shard_blocks = [shard_block] + + # Transition state latest shard slot + transition_to(spec, state, shard_slot) # Create a shard_transitions that would be included at beacon block `state.slot + target_len_offset_slot` shard_transitions = build_shard_transitions_till_slot( spec, state, shard_blocks={shard: shard_blocks}, - on_time_slot=state.slot + target_len_offset_slot, + on_time_slot=init_slot + target_len_offset_slot, ) shard_transition = shard_transitions[shard] # Create an attestation that would be included at beacon block `state.slot + target_len_offset_slot` @@ -37,12 +41,12 @@ def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): spec, state, index=committee_index, - on_time_slot=state.slot + target_len_offset_slot, + on_time_slot=init_slot + target_len_offset_slot, shard_transition=shard_transition, ) pre_gasprice = state.shard_states[shard].gasprice - transition_to(spec, state, state.slot + target_len_offset_slot) + transition_to(spec, state, init_slot + target_len_offset_slot) pre_shard_state = state.shard_states[shard] yield from run_shard_transitions_processing(spec, state, shard_transitions, [attestation], valid=valid) From a4cc189f2b34b9e99d53df32dc24b383d3b4d3bd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 6 Jun 2020 05:19:46 +0800 Subject: [PATCH 109/165] Apply PR feedback from Danny --- specs/phase1/shard-fork-choice.md | 10 +-- specs/phase1/shard-transition.md | 63 ++++--------------- .../test/fork_choice/test_on_shard_head.py | 4 +- 3 files changed, 21 insertions(+), 56 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 844dbfb86..b1fc3080e 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -46,7 +46,7 @@ class ShardStore: def get_forkchoice_shard_store(anchor_state: BeaconState, shard: Shard) -> ShardStore: return ShardStore( shard=shard, - blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot)}, + blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot, shard=shard)}, block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]}, ) ``` @@ -168,13 +168,15 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root ) - # Add new block to the store - shard_store.blocks[hash_tree_root(shard_block)] = shard_block - # Check the block is valid and compute the post-state assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block, shard_block.slot, shard) assert verify_shard_block_signature(beacon_parent_state, signed_shard_block) + post_state = get_post_shard_state(beacon_parent_state, shard_parent_state, shard_block) + + # Add new block to the store + shard_store.blocks[hash_tree_root(shard_block)] = shard_block + # Add new state for this block to the store shard_store.block_states[hash_tree_root(shard_block)] = post_state ``` diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 191dbd1aa..fe3223933 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -47,20 +47,14 @@ def verify_shard_block_message(beacon_state: BeaconState, shard_parent_state: ShardState, block: ShardBlock, slot: Slot, - shard: Shard, - beacon_parent_slot: Slot=None) -> bool: + shard: Shard) -> bool: # Check `shard_parent_root` field assert block.shard_parent_root == shard_parent_state.latest_block_root # Check `beacon_parent_root` field - if beacon_parent_slot is None: - beacon_parent_slot = beacon_state.slot - if beacon_parent_slot == beacon_state.slot: - beacon_parent_block_header = beacon_state.latest_block_header.copy() - if beacon_parent_block_header.state_root == Root(): - beacon_parent_block_header.state_root = hash_tree_root(beacon_state) - beacon_parent_root = hash_tree_root(beacon_parent_block_header) - else: - beacon_parent_root = get_block_root_at_slot(beacon_state, beacon_parent_slot) + beacon_parent_block_header = beacon_state.latest_block_header.copy() + if beacon_parent_block_header.state_root == Root(): + beacon_parent_block_header.state_root = hash_tree_root(beacon_state) + beacon_parent_root = hash_tree_root(beacon_parent_block_header) assert block.beacon_parent_root == beacon_parent_root # Check `slot` field next_slot = Slot(slot + 1) @@ -191,26 +185,6 @@ def compute_shard_body_roots(proposals: Sequence[SignedShardBlock]) -> Sequence[ return [hash_tree_root(proposal.message.body) for proposal in proposals] ``` -```python -def get_proposal_choices_at_slot(beacon_state: BeaconState, - shard_parent_state: ShardState, - slot: Slot, - shard: Shard, - shard_blocks: Sequence[SignedShardBlock], - validate_signature: bool=True) -> Sequence[SignedShardBlock]: - """ - Return the valid shard blocks at the given ``slot``. - Note that this function doesn't change the state. - """ - choices = [] - shard_blocks_at_slot = [block for block in shard_blocks if block.message.slot == slot] - shard_state = shard_parent_state.copy() - for block in shard_blocks_at_slot: - shard_state = get_post_shard_state(beacon_state, shard_state, block.message) - choices.append(block) - return choices -``` - ```python def get_proposal_at_slot(beacon_state: BeaconState, shard_parent_state: ShardState, @@ -222,21 +196,14 @@ def get_proposal_at_slot(beacon_state: BeaconState, Return ``proposal``, ``shard_state`` of the given ``slot``. Note that this function doesn't change the state. """ - choices = get_proposal_choices_at_slot( - beacon_state=beacon_state, - shard_parent_state=shard_parent_state, - slot=slot, - shard=shard, - shard_blocks=shard_blocks, - validate_signature=validate_signature, - ) - if len(choices) == 0: + shard_blocks = [block for block in shard_blocks if block.message.slot == slot] + if len(shard_blocks) == 0: block = ShardBlock(slot=slot, shard=shard) proposal = SignedShardBlock(message=block) - elif len(choices) == 1: - proposal = choices[0] + elif len(shard_blocks) == 1: + proposal = shard_blocks[0] else: - proposal = get_winning_proposal(beacon_state, choices) + proposal = get_winning_proposal(beacon_state, shard_blocks) # Apply state transition shard_state = get_post_shard_state(beacon_state, shard_parent_state, proposal.message) @@ -249,13 +216,12 @@ def get_shard_state_transition_result( beacon_state: BeaconState, shard: Shard, shard_blocks: Sequence[SignedShardBlock], - on_time_slot: Slot, validate_signature: bool=True, ) -> Tuple[Sequence[SignedShardBlock], Sequence[ShardState], Sequence[Root]]: proposals = [] shard_states = [] shard_state = beacon_state.shard_states[shard] - offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), Slot(beacon_state.slot + 1)) for slot in offset_slots: proposal, shard_state = get_proposal_at_slot( beacon_state=beacon_state, @@ -281,11 +247,8 @@ Suppose you are a committee member on shard `shard` at slot `current_slot` and y def get_shard_transition(beacon_state: BeaconState, shard: Shard, shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition: - on_time_slot = Slot(beacon_state.slot + 1) - offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) - proposals, shard_states, shard_data_roots = get_shard_state_transition_result( - beacon_state, shard, shard_blocks, on_time_slot - ) + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), Slot(beacon_state.slot + 1)) + proposals, shard_states, shard_data_roots = get_shard_state_transition_result(beacon_state, shard, shard_blocks) shard_block_lengths = [] proposer_signatures = [] diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index ca79cfd23..1a9042960 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -89,11 +89,11 @@ def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer) # Clear buffer shard_blocks_buffer.clear() - signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) + signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) # transition! add_block_to_store(spec, store, signed_beacon_block) assert spec.get_head(store) == beacon_block.hash_tree_root() - # On shard block at updated `state.slot` + # On shard block at transitioned `state.slot` if is_in_offset_sets(spec, state, shard): # The created shard block would be appended to `shard_blocks_buffer` apply_shard_block(spec, store, shard_store, state, shard_blocks_buffer) From ffeecfbca5b5994b4f34899b9fa3931526d49c80 Mon Sep 17 00:00:00 2001 From: lsankar4033 Date: Fri, 5 Jun 2020 20:26:11 -0700 Subject: [PATCH 110/165] Add test for compute_fork_digest --- .../test/validator/test_validator_unittest.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py index 1dfa0e4d0..0ac1e7b46 100644 --- a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py +++ b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py @@ -314,6 +314,19 @@ def test_get_block_signature(spec, state): ) +@with_all_phases +@spec_state_test +def test_compute_fork_digest(spec, state): + actual_fork_digest = spec.compute_fork_digest(state.fork.current_version, state.genesis_validators_root) + + expected_fork_data_root = spec.hash_tree_root( + spec.ForkData(current_version=state.fork.current_version, + genesis_validators_root=state.genesis_validators_root)) + expected_fork_digest = spec.ForkDigest(expected_fork_data_root[:4]) + + assert actual_fork_digest == expected_fork_digest + + # Attesting From 435505746cd13f336c0bad1c71829df7fd322ca2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 8 Jun 2020 17:12:46 +0800 Subject: [PATCH 111/165] PR feedback from Terence: fix `get_shard_latest_attesting_balance` Co-authored-by: terence tsao --- specs/phase1/shard-fork-choice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 427b72784..5fef81868 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -65,7 +65,7 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro # would be ignored once their newer vote is accepted. Check if it makes sense. and store.latest_messages[i].shard == shard_store.shard and get_shard_ancestor( - store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot + store, shard_store, store.latest_messages[i].shard_root, shard_store.blocks[root].slot ) == root ) )) From 7e67aaeb35c6d8ffc4d8ced61f274161b6c04660 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 8 Jun 2020 18:15:14 +0800 Subject: [PATCH 112/165] Rename `build_shard_transitions_till_slot` to `get_shard_transitions` --- .../pyspec/eth2spec/test/fork_choice/test_on_shard_head.py | 4 ++-- tests/core/pyspec/eth2spec/test/helpers/shard_block.py | 2 +- .../phase_1/block_processing/test_process_shard_transition.py | 4 ++-- tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 1a9042960..24eeaedbe 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -4,7 +4,7 @@ from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_excep from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.shard_block import ( build_shard_block, - build_shard_transitions_till_slot, + get_shard_transitions, get_committee_index_of_shard, ) from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root @@ -69,7 +69,7 @@ def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer) # Sanity check `get_pending_shard_blocks` function check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer) # Use temporary next state to get ShardTransition of shard block - shard_transitions = build_shard_transitions_till_slot( + shard_transitions = get_shard_transitions( spec, state, shard_blocks={shard: shard_blocks_buffer}, diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index e63096b92..f8b4a155f 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -52,7 +52,7 @@ def build_shard_block(spec, return signed_block -def build_shard_transitions_till_slot(spec, parent_beacon_state, shard_blocks): +def get_shard_transitions(spec, parent_beacon_state, shard_blocks): shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS on_time_slot = parent_beacon_state.slot + 1 for shard, blocks in shard_blocks.items(): diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py index dab4973da..e97cc90a8 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py @@ -7,7 +7,7 @@ from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.shard_transitions import run_shard_transitions_processing from eth2spec.test.helpers.shard_block import ( build_shard_block, - build_shard_transitions_till_slot, + get_shard_transitions, ) from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot, next_slot @@ -24,7 +24,7 @@ def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) shard_blocks = [shard_block] - shard_transitions = build_shard_transitions_till_slot( + shard_transitions = get_shard_transitions( spec, state, shard_blocks={shard: shard_blocks}, diff --git a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py index 828cd19d7..33b0beac7 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py @@ -9,7 +9,7 @@ from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import build_empty_block from eth2spec.test.helpers.shard_block import ( build_shard_block, - build_shard_transitions_till_slot, + get_shard_transitions, ) from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot, transition_to @@ -21,7 +21,7 @@ def run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, comm shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) shard_blocks: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]} - shard_transitions = build_shard_transitions_till_slot(spec, state, shard_blocks) + shard_transitions = get_shard_transitions(spec, state, shard_blocks) attestations = [ get_valid_on_time_attestation( spec, From e03a970eaf88e82ade443ec37ac7451f5903966c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 8 Jun 2020 23:49:24 +0800 Subject: [PATCH 113/165] PR feedback from danny: simplify `verify_shard_block_message` params --- specs/phase1/shard-fork-choice.md | 2 +- specs/phase1/shard-transition.md | 19 +++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index b1fc3080e..6c431a68d 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -169,7 +169,7 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si ) # Check the block is valid and compute the post-state - assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block, shard_block.slot, shard) + assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block) assert verify_shard_block_signature(beacon_parent_state, signed_shard_block) post_state = get_post_shard_state(beacon_parent_state, shard_parent_state, shard_block) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index fe3223933..c62b059ee 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -43,27 +43,26 @@ def compute_shard_transition_digest(beacon_parent_state: BeaconState, ### Shard block verification functions ```python -def verify_shard_block_message(beacon_state: BeaconState, +def verify_shard_block_message(beacon_parent_state: BeaconState, shard_parent_state: ShardState, - block: ShardBlock, - slot: Slot, - shard: Shard) -> bool: + block: ShardBlock) -> bool: # Check `shard_parent_root` field assert block.shard_parent_root == shard_parent_state.latest_block_root # Check `beacon_parent_root` field - beacon_parent_block_header = beacon_state.latest_block_header.copy() + beacon_parent_block_header = beacon_parent_state.latest_block_header.copy() if beacon_parent_block_header.state_root == Root(): - beacon_parent_block_header.state_root = hash_tree_root(beacon_state) + beacon_parent_block_header.state_root = hash_tree_root(beacon_parent_state) beacon_parent_root = hash_tree_root(beacon_parent_block_header) assert block.beacon_parent_root == beacon_parent_root # Check `slot` field - next_slot = Slot(slot + 1) - offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), next_slot) - assert slot in offset_slots + shard = block.shard + next_slot = Slot(block.slot + 1) + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_parent_state, shard), next_slot) + assert block.slot in offset_slots # Check `shard` field assert block.shard == shard # Check `proposer_index` field - assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard) + assert block.proposer_index == get_shard_proposer_index(beacon_parent_state, block.slot, shard) # Check `body` field assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE return True From 2d895e9388cd7364448fb357a769a9f03d7a5141 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Jun 2020 00:13:27 +0800 Subject: [PATCH 114/165] PR feedback from danny --- specs/phase1/shard-fork-choice.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 61e4cd36f..0607613e8 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -76,18 +76,18 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro ```python def get_shard_head(store: Store, shard_store: ShardStore) -> Root: # Execute the LMD-GHOST fork choice - shard_blocks = shard_store.blocks - head_beacon_root = get_head(store) - head_shard_state = store.block_states[head_beacon_root].shard_states[shard_store.shard] - shard_head_root = head_shard_state.latest_block_root + beacon_head_root = get_head(store) + shard_head_state = store.block_states[beacon_head_root].shard_states[shard_store.shard] + shard_head_root = shard_head_state.latest_block_root + shard_blocks = { + root: shard_block for root, shard_block in shard_store.blocks.items() + if shard_block.slot > shard_head_state.slot + } while True: # Find the valid child block roots children = [ - root for root in shard_store.blocks.keys() - if ( - shard_blocks[root].shard_parent_root == shard_head_root - and shard_blocks[root].slot > head_shard_state.slot - ) + root for root, shard_block in shard_blocks.items() + if shard_block.shard_parent_root == shard_head_root ] if len(children) == 0: return shard_head_root @@ -107,7 +107,7 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: elif block.slot == slot: return root else: - # root is older than queried slot, thus a skip slot. Return earliest root prior to slot + # root is older than queried slot, thus a skip slot. Return most recent root prior to slot return root ``` @@ -116,7 +116,7 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: ```python def get_pending_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ShardBlock]: """ - Return the shard blocks branch that from shard head to beacon head. + Return the canonical shard block branch that has not yet been crosslinked. """ shard = shard_store.shard From dacf86a5c0ab75bd966f9a3639607427cd876dd3 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 10 Jun 2020 02:43:59 +0800 Subject: [PATCH 115/165] Remove `transition_digest` --- specs/phase1/beacon-chain.md | 1 - specs/phase1/phase1-fork.md | 1 - specs/phase1/shard-fork-choice.md | 2 +- specs/phase1/shard-transition.md | 34 ++++++------------------------- specs/phase1/validator.md | 2 +- 5 files changed, 8 insertions(+), 32 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index d559ebcbd..8ee6a1e83 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -349,7 +349,6 @@ class ShardBlockHeader(Container): class ShardState(Container): slot: Slot gasprice: Gwei - transition_digest: Bytes32 latest_block_root: Root ``` diff --git a/specs/phase1/phase1-fork.md b/specs/phase1/phase1-fork.md index d362ed633..7a7423e10 100644 --- a/specs/phase1/phase1-fork.md +++ b/specs/phase1/phase1-fork.md @@ -104,7 +104,6 @@ def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: ShardState( slot=pre.slot, gasprice=MIN_GASPRICE, - transition_digest=Root(), latest_block_root=Root(), ) for i in range(INITIAL_ACTIVE_SHARDS) ), diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 0607613e8..411ad9b6b 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -172,7 +172,7 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block) assert verify_shard_block_signature(beacon_parent_state, signed_shard_block) - post_state = get_post_shard_state(beacon_parent_state, shard_parent_state, shard_block) + post_state = get_post_shard_state(shard_parent_state, shard_block) # Add new block to the store shard_store.blocks[hash_tree_root(shard_block)] = shard_block diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index ea19b813d..24c39aa3d 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -10,7 +10,6 @@ - [Introduction](#introduction) - [Helper functions](#helper-functions) - - [Misc](#misc) - [Shard block verification functions](#shard-block-verification-functions) - [Shard state transition](#shard-state-transition) - [Fraud proofs](#fraud-proofs) @@ -24,19 +23,6 @@ This document describes the shard transition function and fraud proofs as part o ## Helper functions -### Misc - -```python -def compute_shard_transition_digest(beacon_parent_state: BeaconState, - shard_state: ShardState, - beacon_parent_root: Root, - shard_body_root: Root) -> Bytes32: - # TODO: use SSZ hash tree root - return hash( - hash_tree_root(shard_state) + beacon_parent_root + shard_body_root - ) -``` - ### Shard block verification functions ```python @@ -77,11 +63,10 @@ def verify_shard_block_signature(beacon_state: BeaconState, ## Shard state transition ```python -def shard_state_transition(beacon_state: BeaconState, - shard_state: ShardState, +def shard_state_transition(shard_state: ShardState, block: ShardBlock) -> None: """ - Update ``shard_state`` with shard ``block`` and ``beacon_state`. + Update ``shard_state`` with shard ``block``. """ shard_state.slot = block.slot prev_gasprice = shard_state.gasprice @@ -91,25 +76,18 @@ def shard_state_transition(beacon_state: BeaconState, else: latest_block_root = hash_tree_root(block) shard_state.latest_block_root = latest_block_root - shard_state.transition_digest = compute_shard_transition_digest( - beacon_state, - shard_state, - block.beacon_parent_root, - hash_tree_root(block.body), - ) ``` We have a pure function `get_post_shard_state` for describing the fraud proof verification and honest validator behavior. ```python -def get_post_shard_state(beacon_state: BeaconState, - shard_state: ShardState, +def get_post_shard_state(shard_state: ShardState, block: ShardBlock) -> ShardState: """ A pure function that returns a new post ShardState instead of modifying the given `shard_state`. """ post_state = shard_state.copy() - shard_state_transition(beacon_state, post_state, block) + shard_state_transition(post_state, block) return post_state ``` @@ -151,8 +129,8 @@ def is_valid_fraud_proof(beacon_state: BeaconState, else: shard_state = transition.shard_states[offset_index - 1] # Not doing the actual state updates here. - shard_state = get_post_shard_state(beacon_state, shard_state, block) - if shard_state.transition_digest != transition.shard_states[offset_index].transition_digest: + shard_state = get_post_shard_state(shard_state, block) + if shard_state != transition.shard_states[offset_index]: return True return False diff --git a/specs/phase1/validator.md b/specs/phase1/validator.md index 935f34efb..403a6efb3 100644 --- a/specs/phase1/validator.md +++ b/specs/phase1/validator.md @@ -289,7 +289,7 @@ def get_shard_transition_fields( else: shard_block = SignedShardBlock(message=ShardBlock(slot=slot, shard=shard)) shard_data_roots.append(Root()) - shard_state = get_post_shard_state(beacon_state, shard_state, shard_block.message) + shard_state = get_post_shard_state(shard_state, shard_block.message) shard_states.append(shard_state) shard_block_lengths.append(len(shard_block.message.body)) From f0f7bda2eaf19d40c280958a2c2d6c3fc6644c9b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 10 Jun 2020 02:44:09 +0800 Subject: [PATCH 116/165] Fix tests --- tests/core/pyspec/eth2spec/test/helpers/attestations.py | 2 +- tests/core/pyspec/eth2spec/test/helpers/shard_block.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 1e0560405..2bfe63bd1 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -90,7 +90,7 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() else: - attestation_data.shard_head_root = state.shard_states[shard].transition_digest + attestation_data.shard_head_root = state.shard_states[shard].latest_block_root attestation_data.shard_transition_root = spec.Root() return attestation_data diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 15443e386..ddf66f6c2 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -62,8 +62,6 @@ def get_shard_transitions(spec, parent_beacon_state, shard_blocks): on_time_slot, ) len_offset_slots = len(offset_slots) - # TODO this is actually unsafe for long offset_slots - assert len_offset_slots == on_time_slot - parent_beacon_state.shard_states[shard].slot - 1 shard_transition = spec.get_shard_transition(parent_beacon_state, shard, blocks) if len(blocks) > 0: From ef11e924fa62d6e18ffc04df09ed6f56b753babb Mon Sep 17 00:00:00 2001 From: terence tsao Date: Tue, 9 Jun 2020 13:34:46 -0700 Subject: [PATCH 117/165] Update phase1 beacon-chain config table --- specs/phase1/beacon-chain.md | 52 ++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 4719d2e6f..6bc1d057a 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -102,23 +102,47 @@ Configuration is not namespaced. Instead it is strictly an extension; ### Misc -| Name | Value | Unit | Duration | -| - | - | - | - | +| Name | Value | +| - | - | | `MAX_SHARDS` | `2**10` (= 1024) | -| `ONLINE_PERIOD` | `OnlineEpochs(2**3)` (= 8) | online epochs | ~51 min | | `LIGHT_CLIENT_COMMITTEE_SIZE` | `2**7` (= 128) | +| `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | + +### Shard block configs +| Name | Value | +| - | - | +| `MAX_SHARD_BLOCK_SIZE` | `2**20` (= 1,048,576) | +| `TARGET_SHARD_BLOCK_SIZE` | `2**18` (= 262,144) | +| `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | +| `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | + +### Gwei values + +| Name | Value | +| - | - | +| `MAX_GASPRICE` | `Gwei(2**14)` (= 16,384) | Gwei | +| `MIN_GASPRICE` | `Gwei(2**3)` (= 8) | Gwei | + +### Initial values + +| Name | Value | +| - | - | +| `NO_SIGNATURE` | `BLSSignature(b'\x00' * 96)` | + +### Time parameters + +| Name | Value | Unit | Duration | +| - | - | :-: | :-: | +| `ONLINE_PERIOD` | `OnlineEpochs(2**3)` (= 8) | online epochs | ~51 mins | | `LIGHT_CLIENT_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | -| `MAX_SHARD_BLOCK_SIZE` | `2**20` (= 1,048,576) | | -| `TARGET_SHARD_BLOCK_SIZE` | `2**18` (= 262,144) | | -| `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | | -| `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | | -| `MAX_GASPRICE` | `Gwei(2**14)` (= 16,384) | Gwei | | -| `MIN_GASPRICE` | `Gwei(2**3)` (= 8) | Gwei | | -| `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | | -| `DOMAIN_SHARD_PROPOSAL` | `DomainType('0x80000000')` | | -| `DOMAIN_SHARD_COMMITTEE` | `DomainType('0x81000000')` | | -| `DOMAIN_LIGHT_CLIENT` | `DomainType('0x82000000')` | | -| `NO_SIGNATURE` | `BLSSignature(b'\x00' * 96)` | | + +### Domain types + +| Name | Value | +| - | - | +| `DOMAIN_SHARD_PROPOSAL` | `DomainType('0x80000000')` | +| `DOMAIN_SHARD_COMMITTEE` | `DomainType('0x81000000')` | +| `DOMAIN_LIGHT_CLIENT` | `DomainType('0x82000000')` | ## Updated containers From 0f6efcd50ddbd6800871f7a4bd781f1d87ac4dec Mon Sep 17 00:00:00 2001 From: terence tsao Date: Tue, 9 Jun 2020 15:38:40 -0700 Subject: [PATCH 118/165] Update toc --- specs/phase1/beacon-chain.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 6bc1d057a..87de9d34e 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -12,6 +12,11 @@ - [Custom types](#custom-types) - [Configuration](#configuration) - [Misc](#misc) + - [Shard block configs](#shard-block-configs) + - [Gwei values](#gwei-values) + - [Initial values](#initial-values) + - [Time parameters](#time-parameters) + - [Domain types](#domain-types) - [Updated containers](#updated-containers) - [Extended `AttestationData`](#extended-attestationdata) - [Extended `Attestation`](#extended-attestation) @@ -109,6 +114,7 @@ Configuration is not namespaced. Instead it is strictly an extension; | `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | ### Shard block configs + | Name | Value | | - | - | | `MAX_SHARD_BLOCK_SIZE` | `2**20` (= 1,048,576) | From 378d24948763af2a4492c2bee33dddf8cea6d94d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 10 Jun 2020 11:02:10 +1000 Subject: [PATCH 119/165] Avoid redundant call to get_ancestor --- specs/phase0/fork-choice.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index b9d8ecd3c..a4d6b6cfa 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -371,15 +371,16 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: # Update finalized checkpoint if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: store.finalized_checkpoint = state.finalized_checkpoint - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - - # Update justified if new justified is later than store justified - # or if store justified is not in chain with finalized checkpoint - if ( - state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch - or get_ancestor(store, store.justified_checkpoint.root, finalized_slot) != store.finalized_checkpoint.root - ): - store.justified_checkpoint = state.current_justified_checkpoint + + if store.justified_checkpoint != state.current_justified_checkpoint: + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + # Update justified if new justified is later than store justified + # or if store justified is not in chain with finalized checkpoint + if ( + state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch + or get_ancestor(store, store.justified_checkpoint.root, finalized_slot) != store.finalized_checkpoint.root + ): + store.justified_checkpoint = state.current_justified_checkpoint ``` #### `on_attestation` From 29d968bb2e4054c408a11e2a2e1dd562ac60aa5c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 10 Jun 2020 15:09:40 +1000 Subject: [PATCH 120/165] Use parent_root for finalized chain check --- specs/phase0/fork-choice.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index b9d8ecd3c..95142f80c 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -347,17 +347,17 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: pre_state = store.block_states[block.parent_root].copy() # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. assert get_current_slot(store) >= block.slot - # Add new block to the store - store.blocks[hash_tree_root(block)] = block # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert block.slot > finalized_slot # Check block is a descendant of the finalized block at the checkpoint finalized slot - assert get_ancestor(store, hash_tree_root(block), finalized_slot) == store.finalized_checkpoint.root + assert get_ancestor(store, block.parent_root, finalized_slot) == store.finalized_checkpoint.root # Check the block is valid and compute the post-state state = state_transition(pre_state, signed_block, True) + # Add new block to the store + store.blocks[hash_tree_root(block)] = block # Add new state for this block to the store store.block_states[hash_tree_root(block)] = state From 479c40450dac7770434d2577499da0c5f63140ce Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 10 Jun 2020 18:16:26 +0800 Subject: [PATCH 121/165] Friendly lint fix --- specs/phase0/fork-choice.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index a4d6b6cfa..ad1dc540c 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -371,14 +371,15 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: # Update finalized checkpoint if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: store.finalized_checkpoint = state.finalized_checkpoint - + if store.justified_checkpoint != state.current_justified_checkpoint: finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) # Update justified if new justified is later than store justified # or if store justified is not in chain with finalized checkpoint if ( state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch - or get_ancestor(store, store.justified_checkpoint.root, finalized_slot) != store.finalized_checkpoint.root + or get_ancestor( + store, store.justified_checkpoint.root, finalized_slot) != store.finalized_checkpoint.root ): store.justified_checkpoint = state.current_justified_checkpoint ``` From 1dc6b55617deb1aabd8801ad412740c9d6adc50d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 10 Jun 2020 09:40:34 -0500 Subject: [PATCH 122/165] rearrange fork choice condition for clarity --- specs/phase0/fork-choice.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index ad1dc540c..132477ee8 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -372,15 +372,17 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: store.finalized_checkpoint = state.finalized_checkpoint + # Potentially update justified if different from store if store.justified_checkpoint != state.current_justified_checkpoint: - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) # Update justified if new justified is later than store justified - # or if store justified is not in chain with finalized checkpoint - if ( - state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch - or get_ancestor( - store, store.justified_checkpoint.root, finalized_slot) != store.finalized_checkpoint.root - ): + if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + store.justified_checkpoint = state.current_justified_checkpoint + return + + # Update justified if store justified is not in chain with finalized checkpoint + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + ancestor_at_finalized_slot = get_ancestor(store, store.justified_checkpoint.root, finalized_slot) + if ancestor_at_finalized_slot != store.finalized_checkpoint.root: store.justified_checkpoint = state.current_justified_checkpoint ``` From e46d5effe474e77b3cbc40b43121ee41893d17e1 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Wed, 10 Jun 2020 17:20:42 +0100 Subject: [PATCH 123/165] Add test for slashing after failing to respond to custody chunk challenge --- configs/mainnet.yaml | 2 + configs/minimal.yaml | 2 + specs/phase1/beacon-chain.md | 2 +- .../test_process_challenge_deadlines.py | 76 +++++++++++++++++++ 4 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_challenge_deadlines.py diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 6ba8085a5..2e15e84fd 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -207,6 +207,8 @@ EPOCHS_PER_CUSTODY_PERIOD: 2048 CUSTODY_PERIOD_TO_RANDAO_PADDING: 2048 # 2**7 (= 128) epochs, ~14 hours MAX_REVEAL_LATENESS_DECREMENT: 128 +# 2**14 (= 16,384) epochs +CUSTODY_RESPONSE_DEADLINE: 16384 # Max operations # 2**8 (= 256) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 87509a442..d99aa164a 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -208,6 +208,8 @@ EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 4096 EPOCHS_PER_CUSTODY_PERIOD: 8 # 2**11 (= 2,048) epochs CUSTODY_PERIOD_TO_RANDAO_PADDING: 8 +# 2**14 (= 16,384) epochs +CUSTODY_RESPONSE_DEADLINE: 32 # Max operations # 2**8 (= 256) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 9d55cde3e..e9ba4d88d 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -1137,7 +1137,7 @@ def process_phase_1_final_updates(state: BeaconState) -> None: #### Custody game updates -`process_reveal_deadlines` and `process_custody_final_updates` are defined in [the Custody Game spec](./custody-game.md), +`process_reveal_deadlines`, `process_challenge_deadlines` and `process_custody_final_updates` are defined in [the Custody Game spec](./custody-game.md), #### Online-tracking diff --git a/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_challenge_deadlines.py b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_challenge_deadlines.py new file mode 100644 index 000000000..2e60d167c --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_challenge_deadlines.py @@ -0,0 +1,76 @@ +from eth2spec.test.helpers.custody import ( + get_valid_chunk_challenge, + get_shard_transition, + get_valid_custody_key_reveal, +) +from eth2spec.test.helpers.attestations import ( + get_valid_on_time_attestation, +) +from eth2spec.test.helpers.state import next_epoch_via_block, transition_to +from eth2spec.test.context import ( + with_all_phases_except, + spec_state_test, +) +from eth2spec.test.phase_0.block_processing.test_process_attestation import run_attestation_processing +from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_with + +from eth2spec.test.phase_1.block_processing.test_process_chunk_challenge import ( + run_chunk_challenge_processing, +) +from eth2spec.test.phase_1.block_processing.test_process_custody_key_reveal import run_custody_key_reveal_processing + + +def run_process_challenge_deadlines(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_challenge_deadlines') + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_validator_slashed_after_chunk_challenge(spec, state): + transition_to(spec, state, state.slot + 1) + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + validator_index = spec.get_beacon_committee( + state, + attestation.data.slot, + attestation.data.index + )[0] + + spec.initiate_validator_exit(state, validator_index) + assert state.validators[validator_index].withdrawable_epoch < spec.FAR_FUTURE_EPOCH + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) + + assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH + + while spec.get_current_epoch(state) < state.validators[validator_index].exit_epoch: + next_epoch_via_block(spec, state) + while (state.validators[validator_index].next_custody_secret_to_reveal + <= spec.get_custody_period_for_validator( + validator_index, + state.validators[validator_index].exit_epoch - 1)): + custody_key_reveal = get_valid_custody_key_reveal(spec, state, validator_index=validator_index) + _, _, _ = run_custody_key_reveal_processing(spec, state, custody_key_reveal) + + next_epoch_via_block(spec, state) + + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) + + _, _, _ = run_chunk_challenge_processing(spec, state, challenge) + + assert state.validators[validator_index].slashed == 0 + + transition_to(spec, state, state.slot + (spec.CUSTODY_RESPONSE_DEADLINE + + spec.EPOCHS_PER_CUSTODY_PERIOD) * spec.SLOTS_PER_EPOCH) + + yield from run_process_challenge_deadlines(spec, state) + + assert state.validators[validator_index].slashed == 1 From 4b8f132957e6e6b5f197b0a9fafe8064fd32f072 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 10 Jun 2020 11:32:02 -0500 Subject: [PATCH 124/165] pr feedback --- specs/phase1/validator.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/specs/phase1/validator.md b/specs/phase1/validator.md index 935f34efb..9c6198989 100644 --- a/specs/phase1/validator.md +++ b/specs/phase1/validator.md @@ -75,6 +75,7 @@ See constants from [Phase 0 validator guide](../phase0/validator.md#constants). | Name | Value | Unit | Duration | | - | - | :-: | :-: | | `TARGET_LIGHT_CLIENT_AGGREGATORS_PER_SLOT` | `2**3` (= 8) | validators | | +| `LIGHT_CLIENT_PREPARATION_EPOCHS` | `2**2` (= 4) | epochs | | ## Becoming a validator @@ -86,7 +87,7 @@ Beacon chain validator assignments to beacon committees and beacon block proposa ### Lookahead -Lookahead for beacon committee assignments operates in the same manner as Phase 0, but committee members must join a shard block pubsub topic in addition to the committee attestation topic.o +Lookahead for beacon committee assignments operates in the same manner as Phase 0, but committee members must join a shard block pubsub topic in addition to the committee attestation topic. Specifically _after_ finding stable peers of attestation subnets (see Phase 0) a validator should: * Let `shard = compute_shard_from_committee_index(committe_index)` @@ -196,7 +197,8 @@ def get_best_light_client_aggregate(block: BeaconBlock, return max( viable_aggregates, - key=lambda a: len([i for i in a.aggregation_bits if i == 1]), + # Ties broken by lexicographically by hash_tree_root + key=lambda a: (len([i for i in a.aggregation_bits if i == 1]), hash_tree_root(a)), default=LightClientVote(), ) ``` @@ -378,9 +380,10 @@ When `get_current_epoch(state) % LIGHT_CLIENT_COMMITTEE_PERIOD == LIGHT_CLIENT_C If the validator is in the next light client committee, they must join the `light_client_votes` pubsub topic to begin duties at the start of the next period. ```python -def is_in_next_light_client_committee(state: BeaconState, index: ValidatorIndex) -> boolean: - period_start_epoch = get_current_epoch(state) + LIGHT_CLIENT_COMMITTEE_PERIOD % get_current_epoch(state) - next_committee = get_light_client_committee(state, period_start_epoch) +def is_in_next_light_client_committee(state: BeaconState, index: ValidatorIndex) -> bool: + current_source_epoch = compute_committee_source_epoch(get_current_epoch(state), LIGHT_CLIENT_COMMITTEE_PERIOD) + next_source_epoch = current_source_epoch + LIGHT_CLIENT_COMMITTEE_PERIOD + next_committee = get_light_client_committee(state, next_source_epoch) return index in next_committee ``` From 6502cc1149921c9a27201e5021b341d00030b37e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 11 Jun 2020 00:40:09 +0800 Subject: [PATCH 125/165] Separate config files by phases --- configs/README.md | 2 +- configs/{mainnet.yaml => mainnet_phase0.yaml} | 67 ----------------- configs/mainnet_phase1.yaml | 69 ++++++++++++++++++ configs/{minimal.yaml => minimal_phase0.yaml} | 70 ------------------ configs/minimal_phase1.yaml | 71 +++++++++++++++++++ .../pyspec/eth2spec/config/config_util.py | 22 ++++-- 6 files changed, 157 insertions(+), 144 deletions(-) rename configs/{mainnet.yaml => mainnet_phase0.yaml} (69%) create mode 100644 configs/mainnet_phase1.yaml rename configs/{minimal.yaml => minimal_phase0.yaml} (69%) create mode 100644 configs/minimal_phase1.yaml diff --git a/configs/README.md b/configs/README.md index 4ca54e014..be3c60e6f 100644 --- a/configs/README.md +++ b/configs/README.md @@ -32,4 +32,4 @@ Each preset is a key-value mapping. Presets may contain comments to describe the values. -See [`mainnet.yaml`](./mainnet.yaml) for a complete example. +See [`mainnet_phase0.yaml`](./mainnet_phase0.yaml) for a complete example. diff --git a/configs/mainnet.yaml b/configs/mainnet_phase0.yaml similarity index 69% rename from configs/mainnet.yaml rename to configs/mainnet_phase0.yaml index 3631b7045..39cfddf77 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet_phase0.yaml @@ -151,70 +151,3 @@ DOMAIN_DEPOSIT: 0x03000000 DOMAIN_VOLUNTARY_EXIT: 0x04000000 DOMAIN_SELECTION_PROOF: 0x05000000 DOMAIN_AGGREGATE_AND_PROOF: 0x06000000 -# Phase 1 -DOMAIN_SHARD_PROPOSAL: 0x80000000 -DOMAIN_SHARD_COMMITTEE: 0x81000000 -DOMAIN_LIGHT_CLIENT: 0x82000000 -DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 - - -# Phase 1: Upgrade from Phase 0 -# --------------------------------------------------------------- -PHASE_1_FORK_VERSION: 0x01000000 -# [STUB] -PHASE_1_GENESIS_SLOT: 32 -INITIAL_ACTIVE_SHARDS: 64 - -# Phase 1: General -# --------------------------------------------------------------- -# 2**10` (= 1024) -MAX_SHARDS: 1024 -# 2**3 (= 8) | online epochs | ~51 min -ONLINE_PERIOD: 8 -# 2**7 (= 128) -LIGHT_CLIENT_COMMITTEE_SIZE: 128 -# 2**8 (= 256) | epochs | ~27 hours -LIGHT_CLIENT_COMMITTEE_PERIOD: 256 -# 2**18 (= 262,144) -SHARD_BLOCK_CHUNK_SIZE: 262144 -# 2**2 (= 4) -MAX_SHARD_BLOCK_CHUNKS: 4 -# 3 * 2**16` (= 196,608) -TARGET_SHARD_BLOCK_SIZE: 196608 -# Note: MAX_SHARD_BLOCKS_PER_ATTESTATION is derived from the list length. -SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] -# len(SHARD_BLOCK_OFFSETS) -MAX_SHARD_BLOCKS_PER_ATTESTATION: 12 -# 2**14 (= 16,384) Gwei -MAX_GASPRICE: 16384 -# 2**3 (= 8) Gwei -MIN_GASPRICE: 8 -# 2**3 (= 8) -GASPRICE_ADJUSTMENT_COEFFICIENT: 8 - - -# Phase 1: Custody Game -# --------------------------------------------------------------- - -# Time parameters -# 2**1 (= 2) epochs, 12.8 minutes -RANDAO_PENALTY_EPOCHS: 2 -# 2**14 (= 16,384) epochs ~73 days -EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 16384 -# 2**11 (= 2,048) epochs, ~9 days -EPOCHS_PER_CUSTODY_PERIOD: 2048 -# 2**11 (= 2,048) epochs, ~9 days -CUSTODY_PERIOD_TO_RANDAO_PADDING: 2048 -# 2**7 (= 128) epochs, ~14 hours -MAX_REVEAL_LATENESS_DECREMENT: 128 - -# Max operations -# 2**8 (= 256) -MAX_CUSTODY_KEY_REVEALS: 256 -MAX_EARLY_DERIVED_SECRET_REVEALS: 1 -MAX_CUSTODY_SLASHINGS: 1 - -# Reward and penalty quotients -EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE: 2 -# 2**8 (= 256) -MINOR_REWARD_QUOTIENT: 256 diff --git a/configs/mainnet_phase1.yaml b/configs/mainnet_phase1.yaml new file mode 100644 index 000000000..f85c538c8 --- /dev/null +++ b/configs/mainnet_phase1.yaml @@ -0,0 +1,69 @@ +# Mainnet preset - phase 1 + + +# Phase 1: Upgrade from Phase 0 +# --------------------------------------------------------------- +PHASE_1_FORK_VERSION: 0x01000000 +# [STUB] +PHASE_1_GENESIS_SLOT: 32 +INITIAL_ACTIVE_SHARDS: 64 + +# Phase 1: General +# --------------------------------------------------------------- +# Phase 1 domain +DOMAIN_SHARD_PROPOSAL: 0x80000000 +DOMAIN_SHARD_COMMITTEE: 0x81000000 +DOMAIN_LIGHT_CLIENT: 0x82000000 +DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 + +# 2**10` (= 1024) +MAX_SHARDS: 1024 +# 2**3 (= 8) | online epochs | ~51 min +ONLINE_PERIOD: 8 +# 2**7 (= 128) +LIGHT_CLIENT_COMMITTEE_SIZE: 128 +# 2**8 (= 256) | epochs | ~27 hours +LIGHT_CLIENT_COMMITTEE_PERIOD: 256 +# 2**18 (= 262,144) +SHARD_BLOCK_CHUNK_SIZE: 262144 +# 2**2 (= 4) +MAX_SHARD_BLOCK_CHUNKS: 4 +# 3 * 2**16` (= 196,608) +TARGET_SHARD_BLOCK_SIZE: 196608 +# Note: MAX_SHARD_BLOCKS_PER_ATTESTATION is derived from the list length. +SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] +# len(SHARD_BLOCK_OFFSETS) +MAX_SHARD_BLOCKS_PER_ATTESTATION: 12 +# 2**14 (= 16,384) Gwei +MAX_GASPRICE: 16384 +# 2**3 (= 8) Gwei +MIN_GASPRICE: 8 +# 2**3 (= 8) +GASPRICE_ADJUSTMENT_COEFFICIENT: 8 + + +# Phase 1: Custody Game +# --------------------------------------------------------------- + +# Time parameters +# 2**1 (= 2) epochs, 12.8 minutes +RANDAO_PENALTY_EPOCHS: 2 +# 2**14 (= 16,384) epochs ~73 days +EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 16384 +# 2**11 (= 2,048) epochs, ~9 days +EPOCHS_PER_CUSTODY_PERIOD: 2048 +# 2**11 (= 2,048) epochs, ~9 days +CUSTODY_PERIOD_TO_RANDAO_PADDING: 2048 +# 2**7 (= 128) epochs, ~14 hours +MAX_REVEAL_LATENESS_DECREMENT: 128 + +# Max operations +# 2**8 (= 256) +MAX_CUSTODY_KEY_REVEALS: 256 +MAX_EARLY_DERIVED_SECRET_REVEALS: 1 +MAX_CUSTODY_SLASHINGS: 1 + +# Reward and penalty quotients +EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE: 2 +# 2**8 (= 256) +MINOR_REWARD_QUOTIENT: 256 diff --git a/configs/minimal.yaml b/configs/minimal_phase0.yaml similarity index 69% rename from configs/minimal.yaml rename to configs/minimal_phase0.yaml index 174694add..524d0d5f9 100644 --- a/configs/minimal.yaml +++ b/configs/minimal_phase0.yaml @@ -151,73 +151,3 @@ DOMAIN_DEPOSIT: 0x03000000 DOMAIN_VOLUNTARY_EXIT: 0x04000000 DOMAIN_SELECTION_PROOF: 0x05000000 DOMAIN_AGGREGATE_AND_PROOF: 0x06000000 -# Phase 1 -DOMAIN_SHARD_PROPOSAL: 0x80000000 -DOMAIN_SHARD_COMMITTEE: 0x81000000 -DOMAIN_LIGHT_CLIENT: 0x82000000 -DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 - - -# Phase 1: Upgrade from Phase 0 -# --------------------------------------------------------------- -# [customized] for testnet distinction -PHASE_1_FORK_VERSION: 0x01000001 -# [customized] for testing -PHASE_1_GENESIS_SLOT: 8 -# [customized] reduced for testing -INITIAL_ACTIVE_SHARDS: 2 - - -# Phase 1: General -# --------------------------------------------------------------- -# [customized] reduced for testing -MAX_SHARDS: 8 -# 2**3 (= 8) | online epochs -ONLINE_PERIOD: 8 -# 2**7 (= 128) -LIGHT_CLIENT_COMMITTEE_SIZE: 128 -# 2**8 (= 256) | epochs -LIGHT_CLIENT_COMMITTEE_PERIOD: 256 -# 2**18 (= 262,144) -SHARD_BLOCK_CHUNK_SIZE: 262144 -# 2**2 (= 4) -MAX_SHARD_BLOCK_CHUNKS: 4 -# 3 * 2**16` (= 196,608) -TARGET_SHARD_BLOCK_SIZE: 196608 -# Note: MAX_SHARD_BLOCKS_PER_ATTESTATION is derived from the list length. -SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] -# len(SHARD_BLOCK_OFFSETS) -MAX_SHARD_BLOCKS_PER_ATTESTATION: 12 -# 2**14 (= 16,384) Gwei -MAX_GASPRICE: 16384 -# 2**3 (= 8) Gwei -MIN_GASPRICE: 8 -# 2**3 (= 8) -GASPRICE_ADJUSTMENT_COEFFICIENT: 8 - - -# Phase 1: Custody Game -# --------------------------------------------------------------- - -# Time parameters -# 2**1 (= 2) epochs -RANDAO_PENALTY_EPOCHS: 2 -# [customized] quicker for testing -EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 4096 -# 2**11 (= 2,048) epochs -EPOCHS_PER_CUSTODY_PERIOD: 2048 -# 2**11 (= 2,048) epochs -CUSTODY_PERIOD_TO_RANDAO_PADDING: 2048 -# 2**7 (= 128) epochs -MAX_REVEAL_LATENESS_DECREMENT: 128 - -# Max operations -# 2**8 (= 256) -MAX_CUSTODY_KEY_REVEALS: 256 -MAX_EARLY_DERIVED_SECRET_REVEALS: 1 -MAX_CUSTODY_SLASHINGS: 1 - -# Reward and penalty quotients -EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE: 2 -# 2**8 (= 256) -MINOR_REWARD_QUOTIENT: 256 diff --git a/configs/minimal_phase1.yaml b/configs/minimal_phase1.yaml new file mode 100644 index 000000000..a7066ed92 --- /dev/null +++ b/configs/minimal_phase1.yaml @@ -0,0 +1,71 @@ +# Minimal preset - phase 1 + + +# Phase 1: Upgrade from Phase 0 +# --------------------------------------------------------------- +# [customized] for testnet distinction +PHASE_1_FORK_VERSION: 0x01000001 +# [customized] for testing +PHASE_1_GENESIS_SLOT: 8 +# [customized] reduced for testing +INITIAL_ACTIVE_SHARDS: 2 + +# Phase 1: General +# --------------------------------------------------------------- +# Phase 1 domain +DOMAIN_SHARD_PROPOSAL: 0x80000000 +DOMAIN_SHARD_COMMITTEE: 0x81000000 +DOMAIN_LIGHT_CLIENT: 0x82000000 +DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 + +# [customized] reduced for testing +MAX_SHARDS: 8 +# 2**3 (= 8) | online epochs +ONLINE_PERIOD: 8 +# 2**7 (= 128) +LIGHT_CLIENT_COMMITTEE_SIZE: 128 +# 2**8 (= 256) | epochs +LIGHT_CLIENT_COMMITTEE_PERIOD: 256 +# 2**18 (= 262,144) +SHARD_BLOCK_CHUNK_SIZE: 262144 +# 2**2 (= 4) +MAX_SHARD_BLOCK_CHUNKS: 4 +# 3 * 2**16` (= 196,608) +TARGET_SHARD_BLOCK_SIZE: 196608 +# Note: MAX_SHARD_BLOCKS_PER_ATTESTATION is derived from the list length. +SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] +# len(SHARD_BLOCK_OFFSETS) +MAX_SHARD_BLOCKS_PER_ATTESTATION: 12 +# 2**14 (= 16,384) Gwei +MAX_GASPRICE: 16384 +# 2**3 (= 8) Gwei +MIN_GASPRICE: 8 +# 2**3 (= 8) +GASPRICE_ADJUSTMENT_COEFFICIENT: 8 + + +# Phase 1: Custody Game +# --------------------------------------------------------------- + +# Time parameters +# 2**1 (= 2) epochs +RANDAO_PENALTY_EPOCHS: 2 +# [customized] quicker for testing +EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 4096 +# 2**11 (= 2,048) epochs +EPOCHS_PER_CUSTODY_PERIOD: 2048 +# 2**11 (= 2,048) epochs +CUSTODY_PERIOD_TO_RANDAO_PADDING: 2048 +# 2**7 (= 128) epochs +MAX_REVEAL_LATENESS_DECREMENT: 128 + +# Max operations +# 2**8 (= 256) +MAX_CUSTODY_KEY_REVEALS: 256 +MAX_EARLY_DERIVED_SECRET_REVEALS: 1 +MAX_CUSTODY_SLASHINGS: 1 + +# Reward and penalty quotients +EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE: 2 +# 2**8 (= 256) +MINOR_REWARD_QUOTIENT: 256 diff --git a/tests/core/pyspec/eth2spec/config/config_util.py b/tests/core/pyspec/eth2spec/config/config_util.py index c43c1521b..9a6f3c648 100644 --- a/tests/core/pyspec/eth2spec/config/config_util.py +++ b/tests/core/pyspec/eth2spec/config/config_util.py @@ -1,8 +1,10 @@ -from ruamel.yaml import YAML -from pathlib import Path +import os from os.path import join +from pathlib import Path from typing import Dict, Any +from ruamel.yaml import YAML + config: Dict[str, Any] = {} @@ -35,11 +37,19 @@ def load_config_file(configs_dir: str, presets_name: str) -> Dict[str, Any]: :param presets_name: The name of the presets. (lowercase snake_case) :return: Dictionary, mapping of constant-name -> constant-value """ - path = Path(join(configs_dir, presets_name + '.yaml')) - yaml = YAML(typ='base') - loaded = yaml.load(path) + _, _, config_files = next(os.walk(configs_dir)) + config_files.sort() + loaded_config = {} + for config_file_name in config_files: + if config_file_name.startswith(presets_name): + yaml = YAML(typ='base') + path = Path(join(configs_dir, config_file_name)) + loaded = yaml.load(path) + loaded_config.update(loaded) + assert loaded_config != {} + out: Dict[str, Any] = dict() - for k, v in loaded.items(): + for k, v in loaded_config.items(): if isinstance(v, list): # Clean up integer values. YAML parser renders lists of ints as list of str out[k] = [int(item) if item.isdigit() else item for item in v] From 85ec791935a046af228f4590eef9efbbb4095046 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 11 Jun 2020 00:58:40 +0800 Subject: [PATCH 126/165] Sync with beacon-chain spec --- configs/mainnet_phase1.yaml | 50 +++++++++++++++------------- configs/minimal_phase1.yaml | 65 ++++++++++++++++++++----------------- 2 files changed, 63 insertions(+), 52 deletions(-) diff --git a/configs/mainnet_phase1.yaml b/configs/mainnet_phase1.yaml index f85c538c8..f9bd744b0 100644 --- a/configs/mainnet_phase1.yaml +++ b/configs/mainnet_phase1.yaml @@ -1,45 +1,51 @@ # Mainnet preset - phase 1 -# Phase 1: Upgrade from Phase 0 +# # phase1-fork # --------------------------------------------------------------- PHASE_1_FORK_VERSION: 0x01000000 # [STUB] PHASE_1_GENESIS_SLOT: 32 INITIAL_ACTIVE_SHARDS: 64 -# Phase 1: General -# --------------------------------------------------------------- -# Phase 1 domain -DOMAIN_SHARD_PROPOSAL: 0x80000000 -DOMAIN_SHARD_COMMITTEE: 0x81000000 -DOMAIN_LIGHT_CLIENT: 0x82000000 -DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 -# 2**10` (= 1024) -MAX_SHARDS: 1024 -# 2**3 (= 8) | online epochs | ~51 min -ONLINE_PERIOD: 8 +# beacon-chain +# --------------------------------------------------------------- +# Misc +# 2**10 (= 1,024) +MAX_SHARDS: 8 # 2**7 (= 128) LIGHT_CLIENT_COMMITTEE_SIZE: 128 -# 2**8 (= 256) | epochs | ~27 hours -LIGHT_CLIENT_COMMITTEE_PERIOD: 256 -# 2**18 (= 262,144) -SHARD_BLOCK_CHUNK_SIZE: 262144 -# 2**2 (= 4) -MAX_SHARD_BLOCK_CHUNKS: 4 -# 3 * 2**16` (= 196,608) -TARGET_SHARD_BLOCK_SIZE: 196608 +# 2**3 (= 8) +GASPRICE_ADJUSTMENT_COEFFICIENT: 8 + +# Shard block configs +# 2**20 (= 1048,576) bytes +MAX_SHARD_BLOCK_SIZE: 1048576 +# 2**18 (= 262,144) bytes +TARGET_SHARD_BLOCK_SIZE: 262144 # Note: MAX_SHARD_BLOCKS_PER_ATTESTATION is derived from the list length. SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] # len(SHARD_BLOCK_OFFSETS) MAX_SHARD_BLOCKS_PER_ATTESTATION: 12 + +# Gwei values # 2**14 (= 16,384) Gwei MAX_GASPRICE: 16384 # 2**3 (= 8) Gwei MIN_GASPRICE: 8 -# 2**3 (= 8) -GASPRICE_ADJUSTMENT_COEFFICIENT: 8 + +# Time parameters +# 2**3 (= 8) | online epochs +ONLINE_PERIOD: 8 +# 2**8 (= 256) | epochs +LIGHT_CLIENT_COMMITTEE_PERIOD: 256 + +# Domain types +DOMAIN_SHARD_PROPOSAL: 0x80000000 +DOMAIN_SHARD_COMMITTEE: 0x81000000 +DOMAIN_LIGHT_CLIENT: 0x82000000 +DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 # Phase 1: Custody Game diff --git a/configs/minimal_phase1.yaml b/configs/minimal_phase1.yaml index a7066ed92..556ff2618 100644 --- a/configs/minimal_phase1.yaml +++ b/configs/minimal_phase1.yaml @@ -1,7 +1,7 @@ # Minimal preset - phase 1 -# Phase 1: Upgrade from Phase 0 +# phase1-fork # --------------------------------------------------------------- # [customized] for testnet distinction PHASE_1_FORK_VERSION: 0x01000001 @@ -10,43 +10,48 @@ PHASE_1_GENESIS_SLOT: 8 # [customized] reduced for testing INITIAL_ACTIVE_SHARDS: 2 -# Phase 1: General + +# beacon-chain # --------------------------------------------------------------- -# Phase 1 domain +# Misc +# [customized] reduced for testing +MAX_SHARDS: 8 +# 2**7 (= 128) +LIGHT_CLIENT_COMMITTEE_SIZE: 128 +# 2**3 (= 8) +GASPRICE_ADJUSTMENT_COEFFICIENT: 8 + +# Shard block configs +# 2**20 (= 1048,576) bytes +MAX_SHARD_BLOCK_SIZE: 1048576 +# 2**18 (= 262,144) bytes +TARGET_SHARD_BLOCK_SIZE: 262144 +# Note: MAX_SHARD_BLOCKS_PER_ATTESTATION is derived from the list length. +SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] +# len(SHARD_BLOCK_OFFSETS) +MAX_SHARD_BLOCKS_PER_ATTESTATION: 12 + +# Gwei values +# 2**14 (= 16,384) Gwei +MAX_GASPRICE: 16384 +# 2**3 (= 8) Gwei +MIN_GASPRICE: 8 + +# Time parameters +# 2**3 (= 8) | online epochs +ONLINE_PERIOD: 8 +# 2**8 (= 256) | epochs +LIGHT_CLIENT_COMMITTEE_PERIOD: 256 + +# Domain types DOMAIN_SHARD_PROPOSAL: 0x80000000 DOMAIN_SHARD_COMMITTEE: 0x81000000 DOMAIN_LIGHT_CLIENT: 0x82000000 DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 -# [customized] reduced for testing -MAX_SHARDS: 8 -# 2**3 (= 8) | online epochs -ONLINE_PERIOD: 8 -# 2**7 (= 128) -LIGHT_CLIENT_COMMITTEE_SIZE: 128 -# 2**8 (= 256) | epochs -LIGHT_CLIENT_COMMITTEE_PERIOD: 256 -# 2**18 (= 262,144) -SHARD_BLOCK_CHUNK_SIZE: 262144 -# 2**2 (= 4) -MAX_SHARD_BLOCK_CHUNKS: 4 -# 3 * 2**16` (= 196,608) -TARGET_SHARD_BLOCK_SIZE: 196608 -# Note: MAX_SHARD_BLOCKS_PER_ATTESTATION is derived from the list length. -SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] -# len(SHARD_BLOCK_OFFSETS) -MAX_SHARD_BLOCKS_PER_ATTESTATION: 12 -# 2**14 (= 16,384) Gwei -MAX_GASPRICE: 16384 -# 2**3 (= 8) Gwei -MIN_GASPRICE: 8 -# 2**3 (= 8) -GASPRICE_ADJUSTMENT_COEFFICIENT: 8 - -# Phase 1: Custody Game +# custody-game # --------------------------------------------------------------- - # Time parameters # 2**1 (= 2) epochs RANDAO_PENALTY_EPOCHS: 2 From a1a75a38fe67c763f6d524ae6ce05fbb325186ad Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 11 Jun 2020 11:51:18 +1000 Subject: [PATCH 127/165] Tidy, add comment --- specs/phase0/fork-choice.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index b9d8ecd3c..3cd45f1c1 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -343,8 +343,9 @@ def on_tick(store: Store, time: uint64) -> None: def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: block = signed_block.message # Make a copy of the state to avoid mutability issues - assert block.parent_root in store.block_states pre_state = store.block_states[block.parent_root].copy() + # Parent block must be known + assert block.parent_root in store.block_states # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. assert get_current_slot(store) >= block.slot # Add new block to the store From aa75fb0b693f3402c539ad0ad0f53be6c2de7869 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 12 Jun 2020 00:47:26 +0800 Subject: [PATCH 128/165] Fix `MAX_SHARDS` and move config files to `configs/{network}/{fork}` --- .../{mainnet_phase0.yaml => mainnet/phase0.yaml} | 0 .../{mainnet_phase1.yaml => mainnet/phase1.yaml} | 7 +++---- .../{minimal_phase0.yaml => minimal/phase0.yaml} | 0 .../{minimal_phase1.yaml => minimal/phase1.yaml} | 0 tests/core/pyspec/eth2spec/config/config_util.py | 13 ++++++------- 5 files changed, 9 insertions(+), 11 deletions(-) rename configs/{mainnet_phase0.yaml => mainnet/phase0.yaml} (100%) rename configs/{mainnet_phase1.yaml => mainnet/phase1.yaml} (97%) rename configs/{minimal_phase0.yaml => minimal/phase0.yaml} (100%) rename configs/{minimal_phase1.yaml => minimal/phase1.yaml} (100%) diff --git a/configs/mainnet_phase0.yaml b/configs/mainnet/phase0.yaml similarity index 100% rename from configs/mainnet_phase0.yaml rename to configs/mainnet/phase0.yaml diff --git a/configs/mainnet_phase1.yaml b/configs/mainnet/phase1.yaml similarity index 97% rename from configs/mainnet_phase1.yaml rename to configs/mainnet/phase1.yaml index f9bd744b0..cddbb8dfe 100644 --- a/configs/mainnet_phase1.yaml +++ b/configs/mainnet/phase1.yaml @@ -1,7 +1,7 @@ # Mainnet preset - phase 1 -# # phase1-fork +# phase1-fork # --------------------------------------------------------------- PHASE_1_FORK_VERSION: 0x01000000 # [STUB] @@ -13,7 +13,7 @@ INITIAL_ACTIVE_SHARDS: 64 # --------------------------------------------------------------- # Misc # 2**10 (= 1,024) -MAX_SHARDS: 8 +MAX_SHARDS: 1024 # 2**7 (= 128) LIGHT_CLIENT_COMMITTEE_SIZE: 128 # 2**3 (= 8) @@ -48,9 +48,8 @@ DOMAIN_LIGHT_CLIENT: 0x82000000 DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 -# Phase 1: Custody Game +# custody-game # --------------------------------------------------------------- - # Time parameters # 2**1 (= 2) epochs, 12.8 minutes RANDAO_PENALTY_EPOCHS: 2 diff --git a/configs/minimal_phase0.yaml b/configs/minimal/phase0.yaml similarity index 100% rename from configs/minimal_phase0.yaml rename to configs/minimal/phase0.yaml diff --git a/configs/minimal_phase1.yaml b/configs/minimal/phase1.yaml similarity index 100% rename from configs/minimal_phase1.yaml rename to configs/minimal/phase1.yaml diff --git a/tests/core/pyspec/eth2spec/config/config_util.py b/tests/core/pyspec/eth2spec/config/config_util.py index 9a6f3c648..1977e5b05 100644 --- a/tests/core/pyspec/eth2spec/config/config_util.py +++ b/tests/core/pyspec/eth2spec/config/config_util.py @@ -1,5 +1,4 @@ import os -from os.path import join from pathlib import Path from typing import Dict, Any @@ -37,15 +36,15 @@ def load_config_file(configs_dir: str, presets_name: str) -> Dict[str, Any]: :param presets_name: The name of the presets. (lowercase snake_case) :return: Dictionary, mapping of constant-name -> constant-value """ - _, _, config_files = next(os.walk(configs_dir)) + present_dir = Path(configs_dir) / presets_name + _, _, config_files = next(os.walk(present_dir)) config_files.sort() loaded_config = {} for config_file_name in config_files: - if config_file_name.startswith(presets_name): - yaml = YAML(typ='base') - path = Path(join(configs_dir, config_file_name)) - loaded = yaml.load(path) - loaded_config.update(loaded) + yaml = YAML(typ='base') + path = present_dir / config_file_name + loaded = yaml.load(path) + loaded_config.update(loaded) assert loaded_config != {} out: Dict[str, Any] = dict() From bcfaa1b635954dc9cdf47d02a73aa1b5dbb38eb9 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Fri, 12 Jun 2020 11:07:44 +0100 Subject: [PATCH 129/165] Fix tests --- Makefile | 2 +- .../test_process_chunk_challenge.py | 4 ++-- .../test_process_custody_key_reveal.py | 14 -------------- .../test_process_custody_slashing.py | 2 +- .../test_process_reveal_deadlines.py | 15 +++++++++------ 5 files changed, 13 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 148daaa48..419500ebb 100644 --- a/Makefile +++ b/Makefile @@ -101,7 +101,7 @@ codespell: lint: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - flake8 --ignore=E252,W504,W503,E128 --max-line-length=120 ./eth2spec \ + flake8 --ignore=E252,W504,W503,E128,C901 --max-line-length=120 ./eth2spec \ && cd ./eth2spec && mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase0 \ && mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase1; diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py index c71785518..97fe0c222 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py @@ -158,7 +158,7 @@ def test_multiple_epochs_custody(spec, state): @with_all_phases_except(['phase0']) @spec_state_test def test_many_epochs_custody(spec, state): - transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 100) + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 20) shard = 0 offset_slots = spec.get_offset_slots(state, shard) @@ -254,7 +254,7 @@ def test_custody_response_multiple_epochs(spec, state): @with_all_phases_except(['phase0']) @spec_state_test def test_custody_response_many_epochs(spec, state): - transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 100) + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 20) shard = 0 offset_slots = spec.get_offset_slots(state, shard) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py index fd16a344c..cb96c97e1 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py @@ -87,17 +87,3 @@ def test_double_reveal(spec, state): _, _, _ = run_custody_key_reveal_processing(spec, state, custody_key_reveal) yield from run_custody_key_reveal_processing(spec, state, custody_key_reveal, False) - - -@with_all_phases_except([PHASE0]) -@spec_state_test -@always_bls -def test_max_decrement(spec, state): - state.slot += spec.EPOCHS_PER_CUSTODY_PERIOD * spec.SLOTS_PER_EPOCH * 3 + 150 - custody_key_reveal = get_valid_custody_key_reveal(spec, state) - - _, _, _ = run_custody_key_reveal_processing(spec, state, custody_key_reveal) - - custody_key_reveal2 = get_valid_custody_key_reveal(spec, state) - - yield from run_custody_key_reveal_processing(spec, state, custody_key_reveal2) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py index 087e619f5..997e5ac92 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py @@ -118,7 +118,7 @@ def test_multiple_epochs_custody(spec, state): @with_all_phases_except(['phase0']) @spec_state_test def test_many_epochs_custody(spec, state): - transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 100) + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 20) shard = 0 offset_slots = spec.get_offset_slots(state, shard) shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_reveal_deadlines.py b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_reveal_deadlines.py index 688b046c8..31d45aa41 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_reveal_deadlines.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_reveal_deadlines.py @@ -1,7 +1,7 @@ from eth2spec.test.helpers.custody import ( get_valid_custody_key_reveal, ) -from eth2spec.test.helpers.state import next_epoch +from eth2spec.test.helpers.state import transition_to from eth2spec.test.context import ( with_all_phases_except, spec_state_test, @@ -19,9 +19,12 @@ def run_process_challenge_deadlines(spec, state): def test_validator_slashed_after_reveal_deadline(spec, state): assert state.validators[0].slashed == 0 - state.slot += ((spec.CHUNK_RESPONSE_DEADLINE + spec.EPOCHS_PER_CUSTODY_PERIOD) - * spec.SLOTS_PER_EPOCH) - next_epoch(spec, state) + transition_to(spec, state, spec.get_randao_epoch_for_custody_period(0, 0) * spec.SLOTS_PER_EPOCH) + + transition_to(spec, state, state.slot + ((spec.CUSTODY_RESPONSE_DEADLINE) + * spec.SLOTS_PER_EPOCH)) + + state.validators[0].slashed = 0 yield from run_process_challenge_deadlines(spec, state) @@ -38,8 +41,8 @@ def test_validator_not_slashed_after_reveal(spec, state): assert state.validators[0].slashed == 0 - state.slot += spec.CHUNK_RESPONSE_DEADLINE * spec.SLOTS_PER_EPOCH - next_epoch(spec, state) + transition_to(spec, state, state.slot + ((spec.CUSTODY_RESPONSE_DEADLINE) + * spec.SLOTS_PER_EPOCH)) yield from run_process_challenge_deadlines(spec, state) From 7c6280aa8ef716262fc93a6fd4ba4bfb47a45636 Mon Sep 17 00:00:00 2001 From: dankrad Date: Fri, 12 Jun 2020 11:16:19 +0100 Subject: [PATCH 130/165] Update specs/phase1/custody-game.md Co-authored-by: Hsiao-Wei Wang --- specs/phase1/custody-game.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index d05476af3..3875f28e5 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -264,8 +264,12 @@ def compute_custody_bit(key: BLSSignature, data: ByteList[MAX_SHARD_BLOCK_SIZE]) secrets = get_custody_secrets(key) custody_atoms = get_custody_atoms(data) n = len(custody_atoms) - uhf = (sum(secrets[i % CUSTODY_SECRETS]**i * int.from_bytes(atom, "little") % CUSTODY_PRIME - for i, atom in enumerate(custody_atoms)) + secrets[n % CUSTODY_SECRETS]**n) % CUSTODY_PRIME + uhf = ( + sum( + secrets[i % CUSTODY_SECRETS]**i * int.from_bytes(atom, "little") % CUSTODY_PRIME + for i, atom in enumerate(custody_atoms) + ) + secrets[n % CUSTODY_SECRETS]**n + ) % CUSTODY_PRIME return legendre_bit(uhf + secrets[0], CUSTODY_PRIME) ``` From 31654d8bf459b0af62e1766129f9843a41d07149 Mon Sep 17 00:00:00 2001 From: dankrad Date: Fri, 12 Jun 2020 11:17:44 +0100 Subject: [PATCH 131/165] Update specs/phase1/custody-game.md Co-authored-by: Hsiao-Wei Wang --- specs/phase1/custody-game.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 3875f28e5..84780846b 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -252,7 +252,7 @@ def get_custody_secrets(key: BLSSignature) -> Sequence[int]: full_G2_element = bls.signature_to_G2(key) signature = full_G2_element[0].coeffs signature_bytes = b"".join(x.to_bytes(48, "little") for x in signature) - secrets = [int.from_bytes(signature_bytes[i:i + BYTES_PER_CUSTODY_ATOM], "little") + secrets = [int.from_bytes(signature_bytes[i:i + BYTES_PER_CUSTODY_ATOM], "little") for i in range(0, len(signature_bytes), 32)] return secrets ``` From d41b6a5775b42fb56588acb2cfda77c2ba34cafe Mon Sep 17 00:00:00 2001 From: dankrad Date: Fri, 12 Jun 2020 11:18:07 +0100 Subject: [PATCH 132/165] Update specs/phase1/custody-game.md Co-authored-by: Hsiao-Wei Wang --- specs/phase1/custody-game.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 84780846b..24d64b0ee 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -380,8 +380,8 @@ def process_chunk_challenge_response(state: BeaconState, root=challenge.data_root, ) # Clear the challenge - records = state.custody_chunk_challenge_records - records[records.index(challenge)] = CustodyChunkChallengeRecord() + index_in_records = state.custody_chunk_challenge_records.index(challenge) + state.custody_chunk_challenge_records[index_in_records] = CustodyChunkChallengeRecord() # Reward the proposer proposer_index = get_beacon_proposer_index(state) increase_balance(state, proposer_index, Gwei(get_base_reward(state, proposer_index) // MINOR_REWARD_QUOTIENT)) From 0e8bba2ce326ba15468d9d192fb727cac020d8db Mon Sep 17 00:00:00 2001 From: dankrad Date: Fri, 12 Jun 2020 11:28:30 +0100 Subject: [PATCH 133/165] Update specs/phase1/custody-game.md Co-authored-by: Hsiao-Wei Wang --- specs/phase1/custody-game.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 24d64b0ee..5ee38be8b 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -519,8 +519,10 @@ def process_custody_slashing(state: BeaconState, signed_custody_slashing: Signed shard_transition = custody_slashing.shard_transition assert hash_tree_root(shard_transition) == attestation.data.shard_transition_root # Verify that the provided data matches the shard-transition - assert custody_slashing.data.get_backing().get_left().merkle_root() \ + assert ( + custody_slashing.data.get_backing().get_left().merkle_root() == shard_transition.shard_data_roots[custody_slashing.data_index] + ) assert len(custody_slashing.data) == shard_transition.shard_block_lengths[custody_slashing.data_index] # Verify existence and participation of claimed malefactor From 7bf491d49dff596f66a5eaea395f3f85bb22a26f Mon Sep 17 00:00:00 2001 From: dankrad Date: Fri, 12 Jun 2020 11:28:49 +0100 Subject: [PATCH 134/165] Update specs/phase1/custody-game.md Co-authored-by: Hsiao-Wei Wang --- specs/phase1/custody-game.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 5ee38be8b..d4ce4640a 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -580,8 +580,8 @@ def process_challenge_deadlines(state: BeaconState) -> None: for custody_chunk_challenge in state.custody_chunk_challenge_records: if get_current_epoch(state) > custody_chunk_challenge.inclusion_epoch + CUSTODY_RESPONSE_DEADLINE: slash_validator(state, custody_chunk_challenge.responder_index, custody_chunk_challenge.challenger_index) - records = state.custody_chunk_challenge_records - records[records.index(custody_chunk_challenge)] = CustodyChunkChallengeRecord() + index_in_records = records.index(custody_chunk_challenge) + state.custody_chunk_challenge_records[index_in_records] = CustodyChunkChallengeRecord() ``` ### Final updates From 65c3417f90229c136f797a4592db3d56c7841197 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Fri, 12 Jun 2020 11:53:00 +0100 Subject: [PATCH 135/165] Fix replace_empty_or_append, remove assert False & test --- specs/phase1/custody-game.md | 13 +++++------ .../test_process_chunk_challenge.py | 23 +++++++++++++++++++ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index d4ce4640a..8185b6baa 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -193,14 +193,13 @@ class EarlyDerivedSecretReveal(Container): ### `replace_empty_or_append` ```python -def replace_empty_or_append(list: List, new_element: Any) -> int: - for i in range(len(list)): - if list[i] == type(new_element)(): - assert False - list[i] = new_element +def replace_empty_or_append(l: List, new_element: Any) -> int: + for i in range(len(l)): + if l[i] == type(new_element)(): + l[i] = new_element return i - list.append(new_element) - return len(list) - 1 + l.append(new_element) + return len(l) - 1 ``` ### `legendre_bit` diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py index 97fe0c222..94b1c3b11 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py @@ -85,6 +85,29 @@ def test_challenge_appended(spec, state): yield from run_chunk_challenge_processing(spec, state, challenge) +@with_all_phases_except(['phase0']) +@spec_state_test +def test_challenge_empty_element_replaced(spec, state): + transition_to(spec, state, state.slot + 1) + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD) + + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) + + state.custody_chunk_challenge_records.append(spec.CustodyChunkChallengeRecord()) + + yield from run_chunk_challenge_processing(spec, state, challenge) + + @with_all_phases_except(['phase0']) @spec_state_test def test_duplicate_challenge(spec, state): From 59b35afcd92e87a4c7b309269fe68e993380521a Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Fri, 12 Jun 2020 12:04:30 +0100 Subject: [PATCH 136/165] Refactor universal hash function --- specs/phase1/custody-game.md | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 8185b6baa..1f3ad4378 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -256,19 +256,26 @@ def get_custody_secrets(key: BLSSignature) -> Sequence[int]: return secrets ``` +### `universal_hash_function` + +```python +def universal_hash_function(data_chunks: Sequence[bytes], secrets: Sequence[int]) -> int: + n = len(data_chunks) + return ( + sum( + secrets[i % CUSTODY_SECRETS]**i * int.from_bytes(atom, "little") % CUSTODY_PRIME + for i, atom in enumerate(data_chunks) + ) + secrets[n % CUSTODY_SECRETS]**n + ) % CUSTODY_PRIME +``` + ### `compute_custody_bit` ```python def compute_custody_bit(key: BLSSignature, data: ByteList[MAX_SHARD_BLOCK_SIZE]) -> bit: - secrets = get_custody_secrets(key) custody_atoms = get_custody_atoms(data) - n = len(custody_atoms) - uhf = ( - sum( - secrets[i % CUSTODY_SECRETS]**i * int.from_bytes(atom, "little") % CUSTODY_PRIME - for i, atom in enumerate(custody_atoms) - ) + secrets[n % CUSTODY_SECRETS]**n - ) % CUSTODY_PRIME + secrets = get_custody_secrets(key) + uhf = universal_hash_function(custody_atoms, secrets) return legendre_bit(uhf + secrets[0], CUSTODY_PRIME) ``` @@ -579,7 +586,7 @@ def process_challenge_deadlines(state: BeaconState) -> None: for custody_chunk_challenge in state.custody_chunk_challenge_records: if get_current_epoch(state) > custody_chunk_challenge.inclusion_epoch + CUSTODY_RESPONSE_DEADLINE: slash_validator(state, custody_chunk_challenge.responder_index, custody_chunk_challenge.challenger_index) - index_in_records = records.index(custody_chunk_challenge) + index_in_records = state.custody_chunk_challenge_records.index(custody_chunk_challenge) state.custody_chunk_challenge_records[index_in_records] = CustodyChunkChallengeRecord() ``` From 398fc833b8151e6ac808b4a0d0d37459583c9c61 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Fri, 12 Jun 2020 12:07:31 +0100 Subject: [PATCH 137/165] Fix TOC --- specs/phase1/custody-game.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 1f3ad4378..bab202cf1 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -30,6 +30,7 @@ - [`legendre_bit`](#legendre_bit) - [`get_custody_atoms`](#get_custody_atoms) - [`get_custody_secrets`](#get_custody_secrets) + - [`universal_hash_function`](#universal_hash_function) - [`compute_custody_bit`](#compute_custody_bit) - [`get_randao_epoch_for_custody_period`](#get_randao_epoch_for_custody_period) - [`get_custody_period_for_validator`](#get_custody_period_for_validator) From 04fb9926e819535d32d5c22ed8616c4a0b8a25e9 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Fri, 12 Jun 2020 17:16:08 +0100 Subject: [PATCH 138/165] Remove custody bits from phase 1 and tests --- specs/phase1/beacon-chain.md | 131 +----------------- specs/phase1/custody-game.md | 18 +-- specs/phase1/fork-choice.md | 34 ----- .../test/fork_choice/test_on_attestation.py | 7 +- .../eth2spec/test/helpers/attestations.py | 82 +---------- .../test/helpers/attester_slashings.py | 24 +--- .../test_process_attester_slashing.py | 7 +- .../test_process_attestation.py | 17 +-- 8 files changed, 27 insertions(+), 293 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 61e30e102..04c28f5ab 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -52,15 +52,12 @@ - [`get_shard_committee`](#get_shard_committee) - [`get_light_client_committee`](#get_light_client_committee) - [`get_shard_proposer_index`](#get_shard_proposer_index) - - [`get_indexed_attestation`](#get_indexed_attestation) - [`get_committee_count_delta`](#get_committee_count_delta) - [`get_start_shard`](#get_start_shard) - [`get_shard`](#get_shard) - [`get_latest_slot_for_shard`](#get_latest_slot_for_shard) - [`get_offset_slots`](#get_offset_slots) - [Predicates](#predicates) - - [`verify_attestation_custody`](#verify_attestation_custody) - - [Updated `is_valid_indexed_attestation`](#updated-is_valid_indexed_attestation) - [`is_on_time_attestation`](#is_on_time_attestation) - [`is_winning_attestation`](#is_winning_attestation) - [`optional_aggregate_verify`](#optional_aggregate_verify) @@ -77,7 +74,6 @@ - [`verify_empty_shard_transition`](#verify_empty_shard_transition) - [`process_shard_transitions`](#process_shard_transitions) - [New default validator for deposits](#new-default-validator-for-deposits) - - [New Attester slashing processing](#new-attester-slashing-processing) - [Light client processing](#light-client-processing) - [Epoch transition](#epoch-transition) - [Phase 1 final updates](#phase-1-final-updates) @@ -192,7 +188,6 @@ class AttestationData(Container): class Attestation(Container): aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] data: AttestationData - custody_bits_blocks: List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_SHARD_BLOCKS_PER_ATTESTATION] signature: BLSSignature ``` @@ -212,8 +207,9 @@ class PendingAttestation(Container): ```python class IndexedAttestation(Container): - committee: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE] - attestation: Attestation + attesting_indices: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE] + data: AttestationData + signature: BLSSignature ``` ### Extended `AttesterSlashing` @@ -599,17 +595,6 @@ def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard return committee[r % len(committee)] ``` -#### `get_indexed_attestation` - -```python -def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation) -> IndexedAttestation: - committee = get_beacon_committee(beacon_state, attestation.data.slot, attestation.data.index) - return IndexedAttestation( - committee=committee, - attestation=attestation, - ) -``` - #### `get_committee_count_delta` ```python @@ -679,65 +664,6 @@ def get_offset_slots(state: BeaconState, shard: Shard) -> Sequence[Slot]: ### Predicates -#### `verify_attestation_custody` - -```python -def verify_attestation_custody(state: BeaconState, indexed_attestation: IndexedAttestation) -> bool: - """ - Check if ``indexed_attestation`` has valid signature against non-empty custody bits. - """ - attestation = indexed_attestation.attestation - aggregation_bits = attestation.aggregation_bits - domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch) - all_pubkeys = [] - all_signing_roots = [] - for block_index, custody_bits in enumerate(attestation.custody_bits_blocks): - assert len(custody_bits) == len(indexed_attestation.committee) - for participant, aggregation_bit, custody_bit in zip( - indexed_attestation.committee, aggregation_bits, custody_bits - ): - if aggregation_bit: - all_pubkeys.append(state.validators[participant].pubkey) - # Note: only 2N distinct message hashes - attestation_wrapper = AttestationCustodyBitWrapper( - attestation_data_root=hash_tree_root(attestation.data), - block_index=block_index, - bit=custody_bit, - ) - all_signing_roots.append(compute_signing_root(attestation_wrapper, domain)) - else: - assert not custody_bit - return bls.AggregateVerify(all_pubkeys, all_signing_roots, signature=attestation.signature) -``` - -#### Updated `is_valid_indexed_attestation` - -Note that this replaces the Phase 0 `is_valid_indexed_attestation`. - -```python -def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> bool: - """ - Check if ``indexed_attestation`` has valid indices and signature. - """ - # Verify aggregate signature - attestation = indexed_attestation.attestation - aggregation_bits = attestation.aggregation_bits - if not any(aggregation_bits) or len(aggregation_bits) != len(indexed_attestation.committee): - return False - - if len(attestation.custody_bits_blocks) == 0: - # fall back on phase0 behavior if there is no shard data. - domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch) - all_pubkeys = [] - for participant, aggregation_bit in zip(indexed_attestation.committee, aggregation_bits): - if aggregation_bit: - all_pubkeys.append(state.validators[participant].pubkey) - signing_root = compute_signing_root(indexed_attestation.attestation.data, domain) - return bls.FastAggregateVerify(all_pubkeys, signing_root, signature=attestation.signature) - else: - return verify_attestation_custody(state, indexed_attestation) -``` - #### `is_on_time_attestation` ```python @@ -855,16 +781,11 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: else: assert attestation.data.source == state.previous_justified_checkpoint - # Type 1: on-time attestations, the custody bits should be non-empty. - if attestation.custody_bits_blocks != []: - # Ensure on-time attestation - assert is_on_time_attestation(state, attestation) - # Correct data root count - shard = get_shard(state, attestation) - assert len(attestation.custody_bits_blocks) == len(get_offset_slots(state, shard)) + # Type 1: on-time attestations + if is_on_time_attestation(state, attestation): # Correct parent block root assert data.beacon_block_root == get_block_root_at_slot(state, compute_previous_slot(state.slot)) - # Type 2: no shard transition, no custody bits + # Type 2: no shard transition else: # Ensure delayed attestation assert data.slot < compute_previous_slot(state.slot) @@ -1087,46 +1008,6 @@ def get_validator_from_deposit(state: BeaconState, deposit: Deposit) -> Validato ) ``` -##### New Attester slashing processing - -```python -def get_indices_from_committee( - committee: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE], - bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE]) -> Sequence[ValidatorIndex]: - assert len(bits) == len(committee) - return [validator_index for i, validator_index in enumerate(committee) if bits[i]] -``` - -```python -def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSlashing) -> None: - indexed_attestation_1 = attester_slashing.attestation_1 - indexed_attestation_2 = attester_slashing.attestation_2 - - assert is_slashable_attestation_data( - indexed_attestation_1.attestation.data, - indexed_attestation_2.attestation.data, - ) - assert is_valid_indexed_attestation(state, indexed_attestation_1) - assert is_valid_indexed_attestation(state, indexed_attestation_2) - - indices_1 = get_indices_from_committee( - indexed_attestation_1.committee, - indexed_attestation_1.attestation.aggregation_bits, - ) - indices_2 = get_indices_from_committee( - indexed_attestation_2.committee, - indexed_attestation_2.attestation.aggregation_bits, - ) - - slashed_any = False - indices = set(indices_1).intersection(indices_2) - for index in sorted(indices): - if is_slashable_validator(state.validators[index], get_current_epoch(state)): - slash_validator(state, index) - slashed_any = True - assert slashed_any -``` - #### Light client processing ```python diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index bab202cf1..ca0e9c508 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -60,6 +60,7 @@ This document details the beacon chain additions and changes in Phase 1 of Ether | `CUSTODY_PRIME` | `2 ** 256 - 189` | - | | `CUSTODY_SECRETS` | `3` | - | | `BYTES_PER_CUSTODY_ATOM` | `32` | bytes | +| `CUSTODY_PROBABILITY_EXPONENT` | `10` | - | ## Configuration @@ -141,7 +142,6 @@ class CustodyChunkResponse(Container): ```python class CustodySlashing(Container): - # Attestation.custody_bits_blocks[data_index][committee.index(malefactor_index)] is the target custody bit to check. # (Attestation.data.shard_transition_root as ShardTransition).shard_data_roots[data_index] is the root of the data. data_index: uint64 malefactor_index: ValidatorIndex @@ -277,7 +277,8 @@ def compute_custody_bit(key: BLSSignature, data: ByteList[MAX_SHARD_BLOCK_SIZE]) custody_atoms = get_custody_atoms(data) secrets = get_custody_secrets(key) uhf = universal_hash_function(custody_atoms, secrets) - return legendre_bit(uhf + secrets[0], CUSTODY_PRIME) + legendre_bits = [legendre_bit(uhf + secrets[0], CUSTODY_PRIME) for i in range(CUSTODY_PROBABILITY_EXPONENT)] + return all(legendre_bits) ``` ### `get_randao_epoch_for_custody_period` @@ -518,9 +519,6 @@ def process_custody_slashing(state: BeaconState, signed_custody_slashing: Signed # Verify the attestation assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) - # TODO: custody_slashing.data is not chunked like shard blocks yet, result is lots of padding. - # ??? What does this mean? - # TODO: can do a single combined merkle proof of data being attested. # Verify the shard transition is indeed attested by the attestation shard_transition = custody_slashing.shard_transition @@ -545,18 +543,14 @@ def process_custody_slashing(state: BeaconState, signed_custody_slashing: Signed signing_root = compute_signing_root(epoch_to_sign, domain) assert bls.Verify(malefactor.pubkey, signing_root, custody_slashing.malefactor_secret) - # Get the custody bit - custody_bits = attestation.custody_bits_blocks[custody_slashing.data_index] - committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index) - claimed_custody_bit = custody_bits[committee.index(custody_slashing.malefactor_index)] - # Compute the custody bit computed_custody_bit = compute_custody_bit(custody_slashing.malefactor_secret, custody_slashing.data) - + # Verify the claim - if claimed_custody_bit != computed_custody_bit: + if computed_custody_bit == 1: # Slash the malefactor, reward the other committee members slash_validator(state, custody_slashing.malefactor_index) + committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index) others_count = len(committee) - 1 whistleblower_reward = Gwei(malefactor.effective_balance // WHISTLEBLOWER_REWARD_QUOTIENT // others_count) for attester_index in attesters: diff --git a/specs/phase1/fork-choice.md b/specs/phase1/fork-choice.md index 3f9fbdbfb..f92640ebb 100644 --- a/specs/phase1/fork-choice.md +++ b/specs/phase1/fork-choice.md @@ -9,11 +9,9 @@ - [Introduction](#introduction) -- [Fork choice](#fork-choice) - [Helpers](#helpers) - [Extended `LatestMessage`](#extended-latestmessage) - [Updated `update_latest_messages`](#updated-update_latest_messages) - - [Handlers](#handlers) @@ -22,12 +20,6 @@ This document is the beacon chain fork choice spec for part of Ethereum 2.0 Phase 1. -## Fork choice - -Due to the changes in the structure of `IndexedAttestation` in Phase 1, `on_attestation` must be re-specified to handle this. The bulk of `on_attestation` has been moved out into a few helpers to reduce code duplication where possible. - -The rest of the fork choice remains stable. - ### Helpers #### Extended `LatestMessage` @@ -54,29 +46,3 @@ def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIn epoch=target.epoch, root=beacon_block_root, shard=shard, shard_root=attestation.data.shard_head_root ) ``` - -### Handlers - -```python -def on_attestation(store: Store, attestation: Attestation) -> None: - """ - Run ``on_attestation`` upon receiving a new ``attestation`` from either within a block or directly on the wire. - - An ``attestation`` that is asserted as invalid may be valid at a later time, - consider scheduling it for later processing in such case. - """ - validate_on_attestation(store, attestation) - store_target_checkpoint_state(store, attestation.data.target) - - # Get state at the `target` to fully validate attestation - target_state = store.checkpoint_states[attestation.data.target] - indexed_attestation = get_indexed_attestation(target_state, attestation) - assert is_valid_indexed_attestation(target_state, indexed_attestation) - - # Update latest messages for attesting indices - attesting_indices = [ - index for i, index in enumerate(indexed_attestation.committee) - if attestation.aggregation_bits[i] - ] - update_latest_messages(store, attesting_indices, attestation) -``` diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index a5334c5c7..2c50544d8 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -16,18 +16,13 @@ def run_on_attestation(spec, state, store, attestation, valid=True): indexed_attestation = spec.get_indexed_attestation(state, attestation) spec.on_attestation(store, attestation) + sample_index = indexed_attestation.attesting_indices[0] if spec.fork == PHASE0: - sample_index = indexed_attestation.attesting_indices[0] latest_message = spec.LatestMessage( epoch=attestation.data.target.epoch, root=attestation.data.beacon_block_root, ) else: - attesting_indices = [ - index for i, index in enumerate(indexed_attestation.committee) - if attestation.aggregation_bits[i] - ] - sample_index = attesting_indices[0] latest_message = spec.LatestMessage( epoch=attestation.data.target.epoch, root=attestation.data.beacon_block_root, diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 17ef717bc..b9c91eaa0 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -2,14 +2,13 @@ from lru import LRU from typing import List -from eth2spec.test.context import expect_assertion_error, PHASE0, PHASE1 +from eth2spec.test.context import expect_assertion_error, PHASE1 from eth2spec.test.helpers.state import state_transition_and_sign_block, next_epoch, next_slot from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.shard_transitions import get_shard_transition_of_committee from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.ssz.ssz_typing import Bitlist -from eth2spec.test.helpers.custody import get_custody_test_vector def run_attestation_processing(spec, state, attestation, valid=True): @@ -98,35 +97,7 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t def convert_to_valid_on_time_attestation(spec, state, attestation, shard_transition, - signed=False, valid_custody_bits=None): - shard = spec.get_shard(state, attestation) - offset_slots = spec.compute_offset_slots(spec.get_latest_slot_for_shard(state, shard), state.slot + 1) - - if valid_custody_bits is not None: - beacon_committee = spec.get_beacon_committee( - state, - attestation.data.slot, - attestation.data.index, - ) - custody_secrets = [None for i in beacon_committee] - for i in range(len(beacon_committee)): - period = spec.get_custody_period_for_validator(beacon_committee[i], attestation.data.target.epoch) - epoch_to_sign = spec.get_randao_epoch_for_custody_period(period, beacon_committee[i]) - domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch_to_sign) - signing_root = spec.compute_signing_root(spec.Epoch(epoch_to_sign), domain) - custody_secrets[i] = bls.Sign(privkeys[beacon_committee[i]], signing_root) - - for i in range(len(offset_slots)): - attestation.custody_bits_blocks.append( - Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE]([0 for _ in attestation.aggregation_bits]) - ) - if valid_custody_bits is not None: - test_vector = get_custody_test_vector(shard_transition.shard_block_lengths[i]) - for j in range(len(attestation.custody_bits_blocks[i])): - if attestation.aggregation_bits[j]: - attestation.custody_bits_blocks[i][j] = \ - spec.compute_custody_bit(custody_secrets[j], test_vector) ^ (not valid_custody_bits) - + signed=False): if signed: sign_attestation(spec, state, attestation) @@ -134,7 +105,7 @@ def convert_to_valid_on_time_attestation(spec, state, attestation, shard_transit def get_valid_on_time_attestation(spec, state, slot=None, index=None, - shard_transition=None, valid_custody_bits=None, signed=False): + shard_transition=None, signed=False): ''' Construct on-time attestation for next slot ''' @@ -149,7 +120,6 @@ def get_valid_on_time_attestation(spec, state, slot=None, index=None, slot=slot, index=index, shard_transition=shard_transition, - valid_custody_bits=valid_custody_bits, signed=signed, on_time=True, ) @@ -174,7 +144,6 @@ def get_valid_attestation(spec, index=None, filter_participant_set=None, shard_transition=None, - valid_custody_bits=None, signed=False, on_time=True): # If filter_participant_set filters everything, the attestation has 0 participants, and cannot be signed. @@ -207,7 +176,6 @@ def get_valid_attestation(spec, attestation = convert_to_valid_on_time_attestation( spec, state, attestation, shard_transition, - valid_custody_bits=valid_custody_bits, signed=signed, ) @@ -230,43 +198,9 @@ def sign_aggregate_attestation(spec, state, attestation_data, participants: List def sign_indexed_attestation(spec, state, indexed_attestation): - if spec.fork == PHASE0: - participants = indexed_attestation.attesting_indices - data = indexed_attestation.data - indexed_attestation.signature = sign_aggregate_attestation(spec, state, data, participants) - else: - participants = spec.get_indices_from_committee( - indexed_attestation.committee, - indexed_attestation.attestation.aggregation_bits, - ) - data = indexed_attestation.attestation.data - if any(indexed_attestation.attestation.custody_bits_blocks): - sign_on_time_attestation(spec, state, indexed_attestation.attestation) - else: - indexed_attestation.attestation.signature = sign_aggregate_attestation(spec, state, data, participants) - - -def sign_on_time_attestation(spec, state, attestation): - if not any(attestation.custody_bits_blocks): - sign_attestation(spec, state, attestation) - return - - committee = spec.get_beacon_committee(state, attestation.data.slot, attestation.data.index) - signatures = [] - for block_index, custody_bits in enumerate(attestation.custody_bits_blocks): - for participant, abit, cbit in zip(committee, attestation.aggregation_bits, custody_bits): - if not abit: - continue - signatures.append(get_attestation_custody_signature( - spec, - state, - attestation.data, - block_index, - cbit, - privkeys[participant] - )) - - attestation.signature = bls.Aggregate(signatures) + participants = indexed_attestation.attesting_indices + data = indexed_attestation.data + indexed_attestation.signature = sign_aggregate_attestation(spec, state, data, participants) def get_attestation_custody_signature(spec, state, attestation_data, block_index, bit, privkey): @@ -283,10 +217,6 @@ def get_attestation_custody_signature(spec, state, attestation_data, block_index def sign_attestation(spec, state, attestation): - if spec.fork == PHASE1 and any(attestation.custody_bits_blocks): - sign_on_time_attestation(spec, state, attestation) - return - participants = spec.get_attesting_indices( state, attestation.data, diff --git a/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py b/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py index e743ca8ff..43763895f 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py @@ -1,4 +1,3 @@ -from eth2spec.test.context import PHASE1 from eth2spec.test.helpers.attestations import get_valid_attestation, sign_attestation, sign_indexed_attestation @@ -41,34 +40,19 @@ def get_indexed_attestation_participants(spec, indexed_att): """ Wrapper around index-attestation to return the list of participant indices, regardless of spec phase. """ - if spec.fork == PHASE1: - return list(spec.get_indices_from_committee( - indexed_att.committee, - indexed_att.attestation.aggregation_bits, - )) - else: - return list(indexed_att.attesting_indices) + return list(indexed_att.attesting_indices) def set_indexed_attestation_participants(spec, indexed_att, participants): """ Wrapper around index-attestation to return the list of participant indices, regardless of spec phase. """ - if spec.fork == PHASE1: - indexed_att.attestation.aggregation_bits = [bool(i in participants) for i in indexed_att.committee] - else: - indexed_att.attesting_indices = participants + indexed_att.attesting_indices = participants def get_attestation_1_data(spec, att_slashing): - if spec.fork == PHASE1: - return att_slashing.attestation_1.attestation.data - else: - return att_slashing.attestation_1.data + return att_slashing.attestation_1.data def get_attestation_2_data(spec, att_slashing): - if spec.fork == PHASE1: - return att_slashing.attestation_2.attestation.data - else: - return att_slashing.attestation_2.data + return att_slashing.attestation_2.data diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index 11ead6033..063514498 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -1,5 +1,5 @@ from eth2spec.test.context import ( - PHASE0, PHASE1, + PHASE0, spec_state_test, expect_assertion_error, always_bls, with_all_phases, with_phases ) from eth2spec.test.helpers.attestations import sign_indexed_attestation @@ -162,10 +162,7 @@ def test_same_data(spec, state): indexed_att_1 = attester_slashing.attestation_1 att_2_data = get_attestation_2_data(spec, attester_slashing) - if spec.fork == PHASE1: - indexed_att_1.attestation.data = att_2_data - else: - indexed_att_1.data = att_2_data + indexed_att_1.data = att_2_data sign_indexed_attestation(spec, state, attester_slashing.attestation_1) yield from run_attester_slashing_processing(spec, state, attester_slashing, False) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_attestation.py index ed4328327..34ff28412 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_attestation.py @@ -25,22 +25,9 @@ def test_on_time_success(spec, state): @with_all_phases_except(['phase0']) @spec_state_test @always_bls -def test_on_time_empty_custody_bits_blocks(spec, state): +def test_late_success(spec, state): attestation = get_valid_late_attestation(spec, state, signed=True) - assert not any(attestation.custody_bits_blocks) - - transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) - - yield from run_attestation_processing(spec, state, attestation, False) - - -@with_all_phases_except(['phase0']) -@spec_state_test -@always_bls -def test_late_with_custody_bits_blocks(spec, state): - attestation = get_valid_on_time_attestation(spec, state, signed=True) - transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY + 1) - yield from run_attestation_processing(spec, state, attestation, False) + yield from run_attestation_processing(spec, state, attestation) From f857dbfac246b29bdc92bc80c2ab861f4de114bd Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Fri, 12 Jun 2020 22:47:45 +0100 Subject: [PATCH 139/165] Custody tests --- .../pyspec/eth2spec/test/helpers/custody.py | 46 +++++--- .../test_process_custody_slashing.py | 103 ++++++++++++++---- 2 files changed, 111 insertions(+), 38 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/custody.py b/tests/core/pyspec/eth2spec/test/helpers/custody.py index 898cf7731..edd7ac19f 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/custody.py +++ b/tests/core/pyspec/eth2spec/test/helpers/custody.py @@ -59,7 +59,7 @@ def bitlist_from_int(max_len, num_bits, n): return Bitlist[max_len](*[(n >> i) & 0b1 for i in range(num_bits)]) -def get_valid_custody_slashing(spec, state, attestation, shard_transition, invalid_custody_bit=False): +def get_valid_custody_slashing(spec, state, attestation, shard_transition, custody_secret, data, data_index=0): beacon_committee = spec.get_beacon_committee( state, attestation.data.slot, @@ -68,21 +68,10 @@ def get_valid_custody_slashing(spec, state, attestation, shard_transition, inval malefactor_index = beacon_committee[0] whistleblower_index = beacon_committee[-1] - epoch = spec.get_randao_epoch_for_custody_period(attestation.data.target.epoch, - malefactor_index) - - # Generate the responder key - domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch) - signing_root = spec.compute_signing_root(spec.Epoch(epoch), domain) - malefactor_key = bls.Sign(privkeys[malefactor_index], signing_root) - data_index = 0 - data = ByteList[spec.MAX_SHARD_BLOCK_SIZE]( - get_custody_test_vector(shard_transition.shard_block_lengths[data_index])) - slashing = spec.CustodySlashing( data_index=data_index, malefactor_index=malefactor_index, - malefactor_secret=malefactor_key, + malefactor_secret=custody_secret, whistleblower_index=whistleblower_index, shard_transition=shard_transition, attestation=attestation, @@ -165,9 +154,9 @@ def get_valid_custody_chunk_response(spec, state, chunk_challenge, block_length, ) -def get_custody_test_vector(bytelength): +def get_custody_test_vector(bytelength, offset=0): ints = bytelength // 4 + 1 - return (b"".join(i.to_bytes(4, "little") for i in range(ints)))[:bytelength] + return (b"".join((i + offset).to_bytes(4, "little") for i in range(ints)))[:bytelength] def get_shard_transition(spec, start_slot, block_lengths): @@ -181,3 +170,30 @@ def get_shard_transition(spec, start_slot, block_lengths): proposer_signature_aggregate=spec.BLSSignature(), ) return shard_transition + + +def get_custody_secret(spec, state, validator_index, epoch=None): + period = spec.get_custody_period_for_validator(validator_index, epoch if epoch is not None + else spec.get_current_epoch(state)) + epoch_to_sign = spec.get_randao_epoch_for_custody_period(period, validator_index) + domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch_to_sign) + signing_root = spec.compute_signing_root(spec.Epoch(epoch_to_sign), domain) + return bls.Sign(privkeys[validator_index], signing_root) + + +def get_custody_slashable_test_vector(spec, custody_secret, length, slashable=True): + test_vector = get_custody_test_vector(length) + offset = 0 + while spec.compute_custody_bit(custody_secret, test_vector) != slashable: + offset += 1 + test_vector = test_vector = get_custody_test_vector(length, offset) + return test_vector + + +def get_custody_slashable_shard_transition(spec, start_slot, block_lengths, custody_secret, slashable=True): + shard_transition = get_shard_transition(spec, start_slot, block_lengths) + slashable_test_vector = get_custody_slashable_test_vector(spec, custody_secret, + block_lengths[0], slashable=slashable) + shard_transition.shard_data_roots[0] = ByteList[spec.MAX_SHARD_BLOCK_SIZE](slashable_test_vector) \ + .get_backing().get_left().merkle_root() + return shard_transition, slashable_test_vector diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py index 997e5ac92..c40aa1787 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py @@ -1,6 +1,7 @@ from eth2spec.test.helpers.custody import ( get_valid_custody_slashing, - get_shard_transition, + get_custody_secret, + get_custody_slashable_shard_transition, ) from eth2spec.test.helpers.attestations import ( get_valid_on_time_attestation, @@ -58,9 +59,18 @@ def test_custody_slashing(spec, state): transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) shard = 0 offset_slots = spec.get_offset_slots(state, shard) - shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + + validator_index = spec.get_beacon_committee( + state, + state.slot, + shard, + )[0] + custody_secret = get_custody_secret(spec, state, validator_index) + shard_transition, slashable_test_vector = get_custody_slashable_shard_transition(spec, state.slot, + [2**15 // 3] * len(offset_slots), custody_secret) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, - shard_transition=shard_transition, valid_custody_bits=False) + shard_transition=shard_transition) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -68,8 +78,8 @@ def test_custody_slashing(spec, state): transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) - slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition) - + slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition, + custody_secret, slashable_test_vector) yield from run_custody_slashing_processing(spec, state, slashing, correct=True) @@ -79,9 +89,18 @@ def test_incorrect_custody_slashing(spec, state): transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) shard = 0 offset_slots = spec.get_offset_slots(state, shard) - shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + + validator_index = spec.get_beacon_committee( + state, + state.slot, + shard, + )[0] + custody_secret = get_custody_secret(spec, state, validator_index) + shard_transition, slashable_test_vector = get_custody_slashable_shard_transition(spec, state.slot, + [2**15 // 3] * len(offset_slots), custody_secret, slashable=False) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, - shard_transition=shard_transition, valid_custody_bits=True) + shard_transition=shard_transition) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -89,7 +108,8 @@ def test_incorrect_custody_slashing(spec, state): transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) - slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition) + slashing = get_valid_custody_slashing(spec, state, attestation, + shard_transition, custody_secret, slashable_test_vector) yield from run_custody_slashing_processing(spec, state, slashing, correct=False) @@ -100,9 +120,18 @@ def test_multiple_epochs_custody(spec, state): transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 3) shard = 0 offset_slots = spec.get_offset_slots(state, shard) - shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + + validator_index = spec.get_beacon_committee( + state, + state.slot, + shard, + )[0] + custody_secret = get_custody_secret(spec, state, validator_index) + shard_transition, slashable_test_vector = get_custody_slashable_shard_transition(spec, state.slot, + [2**15 // 3] * len(offset_slots), custody_secret) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, - shard_transition=shard_transition, valid_custody_bits=False) + shard_transition=shard_transition) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -110,8 +139,8 @@ def test_multiple_epochs_custody(spec, state): transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) - slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition) - + slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition, + custody_secret, slashable_test_vector) yield from run_custody_slashing_processing(spec, state, slashing, correct=True) @@ -121,9 +150,18 @@ def test_many_epochs_custody(spec, state): transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 20) shard = 0 offset_slots = spec.get_offset_slots(state, shard) - shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + + validator_index = spec.get_beacon_committee( + state, + state.slot, + shard, + )[0] + custody_secret = get_custody_secret(spec, state, validator_index) + shard_transition, slashable_test_vector = get_custody_slashable_shard_transition(spec, state.slot, + [2**15 // 3] * len(offset_slots), custody_secret) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, - shard_transition=shard_transition, valid_custody_bits=False) + shard_transition=shard_transition) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -131,8 +169,8 @@ def test_many_epochs_custody(spec, state): transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) - slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition) - + slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition, + custody_secret, slashable_test_vector) yield from run_custody_slashing_processing(spec, state, slashing, correct=True) @@ -142,14 +180,23 @@ def test_off_chain_attestation(spec, state): transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) shard = 0 offset_slots = spec.get_offset_slots(state, shard) - shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + + validator_index = spec.get_beacon_committee( + state, + state.slot, + shard, + )[0] + custody_secret = get_custody_secret(spec, state, validator_index) + shard_transition, slashable_test_vector = get_custody_slashable_shard_transition(spec, state.slot, + [2**15 // 3] * len(offset_slots), custody_secret) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, - shard_transition=shard_transition, valid_custody_bits=False) + shard_transition=shard_transition) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) - slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition) - + slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition, + custody_secret, slashable_test_vector) yield from run_custody_slashing_processing(spec, state, slashing, correct=True) @@ -159,9 +206,18 @@ def test_invalid_custody_slashing(spec, state): transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) shard = 0 offset_slots = spec.get_offset_slots(state, shard) - shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + + validator_index = spec.get_beacon_committee( + state, + state.slot, + shard, + )[0] + custody_secret = get_custody_secret(spec, state, validator_index) + shard_transition, slashable_test_vector = get_custody_slashable_shard_transition(spec, state.slot, + [2**15 // 3] * len(offset_slots), custody_secret) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, - shard_transition=shard_transition, valid_custody_bits=False) + shard_transition=shard_transition) transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -169,7 +225,8 @@ def test_invalid_custody_slashing(spec, state): transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) - slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition) + slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition, + custody_secret, slashable_test_vector) slashing.message.data = ByteList[spec.MAX_SHARD_BLOCK_SIZE]() From 42a9f1afdf12faaac235e775d5db7e86720aa1c9 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Fri, 12 Jun 2020 23:44:36 +0100 Subject: [PATCH 140/165] Fix Legendre bit computations --- specs/phase1/custody-game.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index ca0e9c508..5cf5da743 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -277,7 +277,7 @@ def compute_custody_bit(key: BLSSignature, data: ByteList[MAX_SHARD_BLOCK_SIZE]) custody_atoms = get_custody_atoms(data) secrets = get_custody_secrets(key) uhf = universal_hash_function(custody_atoms, secrets) - legendre_bits = [legendre_bit(uhf + secrets[0], CUSTODY_PRIME) for i in range(CUSTODY_PROBABILITY_EXPONENT)] + legendre_bits = [legendre_bit(uhf + secrets[0] + i, CUSTODY_PRIME) for i in range(CUSTODY_PROBABILITY_EXPONENT)] return all(legendre_bits) ``` From 7ad1bb508d6f339b8def65e2dac85eac3614ca89 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 13 Jun 2020 16:04:16 +1000 Subject: [PATCH 141/165] Ensure parent is checked before store lookup --- specs/phase0/fork-choice.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index 3cd45f1c1..d9cf87330 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -342,10 +342,10 @@ def on_tick(store: Store, time: uint64) -> None: ```python def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: block = signed_block.message - # Make a copy of the state to avoid mutability issues - pre_state = store.block_states[block.parent_root].copy() # Parent block must be known assert block.parent_root in store.block_states + # Make a copy of the state to avoid mutability issues + pre_state = store.block_states[block.parent_root].copy() # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. assert get_current_slot(store) >= block.slot # Add new block to the store From f6d7dac30c6d29c3b6252daf6255daf7c217e046 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Sat, 13 Jun 2020 15:15:37 +0100 Subject: [PATCH 142/165] Change to 2**14 epoch (73 day) custody periods as per #1888 --- configs/mainnet.yaml | 17 ++++++------ configs/minimal.yaml | 19 +++++++------ specs/phase1/custody-game.md | 11 ++++---- .../test_process_custody_slashing.py | 2 +- .../test_process_challenge_deadlines.py | 27 +++---------------- .../test_process_reveal_deadlines.py | 13 ++++----- 6 files changed, 37 insertions(+), 52 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 2e15e84fd..d2ab3cab6 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -196,19 +196,20 @@ GASPRICE_ADJUSTMENT_COEFFICIENT: 8 # Phase 1: Custody Game # --------------------------------------------------------------- +# 1/1024 chance of custody bit 1 +CUSTODY_PROBABILITY_EXPONENT: 10 + # Time parameters # 2**1 (= 2) epochs, 12.8 minutes RANDAO_PENALTY_EPOCHS: 2 -# 2**14 (= 16,384) epochs ~73 days -EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 16384 -# 2**11 (= 2,048) epochs, ~9 days -EPOCHS_PER_CUSTODY_PERIOD: 2048 +# 2**15 (= 32,768) epochs, ~146 days +EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 32768 +# 2**14 (= 16,384) epochs ~73 days +EPOCHS_PER_CUSTODY_PERIOD: 16384 # 2**11 (= 2,048) epochs, ~9 days CUSTODY_PERIOD_TO_RANDAO_PADDING: 2048 -# 2**7 (= 128) epochs, ~14 hours -MAX_REVEAL_LATENESS_DECREMENT: 128 -# 2**14 (= 16,384) epochs -CUSTODY_RESPONSE_DEADLINE: 16384 +# 2**15 (= 32,768) epochs, ~146 days +MAX_CHUNK_CHALLENGE_DELAY: 32768 # Max operations # 2**8 (= 256) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index d99aa164a..3fe160bbe 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -199,17 +199,20 @@ GASPRICE_ADJUSTMENT_COEFFICIENT: 8 # Phase 1: Custody Game # --------------------------------------------------------------- +# 1/1024 chance of custody bit 1 [customized for faster testing] +CUSTODY_PROBABILITY_EXPONENT: 2 + # Time parameters # 2**1 (= 2) epochs RANDAO_PENALTY_EPOCHS: 2 -# [customized] quicker for testing -EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 4096 -# 2**11 (= 2,048) epochs -EPOCHS_PER_CUSTODY_PERIOD: 8 -# 2**11 (= 2,048) epochs -CUSTODY_PERIOD_TO_RANDAO_PADDING: 8 -# 2**14 (= 16,384) epochs -CUSTODY_RESPONSE_DEADLINE: 32 +# 2**15 (= 32,768) epochs, ~146 days [customized for faster testing] +EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 128 +# 2**14 (= 16,384) epochs ~73 days [customized for faster testing] +EPOCHS_PER_CUSTODY_PERIOD: 64 +# 2**11 (= 2,048) epochs, ~9 days [customized for faster testing] +CUSTODY_PERIOD_TO_RANDAO_PADDING: 16 +# 2**15 (= 32,768) epochs, ~146 days +MAX_CHUNK_CHALLENGE_DELAY: 128 # Max operations # 2**8 (= 256) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 5cf5da743..30119f3e3 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -69,12 +69,11 @@ This document details the beacon chain additions and changes in Phase 1 of Ether | Name | Value | Unit | Duration | | - | - | :-: | :-: | | `RANDAO_PENALTY_EPOCHS` | `2**1` (= 2) | epochs | 12.8 minutes | -| `EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS` | `2**14` (= 16,384) | epochs | ~73 days | -| `EPOCHS_PER_CUSTODY_PERIOD` | `2**11` (= 2,048) | epochs | ~9 days | +| `EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS` | `2**15` (= 32,768) | epochs | ~146 days | +| `EPOCHS_PER_CUSTODY_PERIOD` | `2**14` (= 16,384) | epochs | ~73 days | | `CUSTODY_PERIOD_TO_RANDAO_PADDING` | `2**11` (= 2,048) | epochs | ~9 days | | `CHUNK_RESPONSE_DEADLINE` | `2**14` (= 16,384) | epochs | ~73 days | -| `MAX_CHUNK_CHALLENGE_DELAY` | `2**11` (= 2,048) | epochs | ~9 days | -| `CUSTODY_RESPONSE_DEADLINE` | `2**14` (= 16,384) | epochs | ~73 days | +| `MAX_CHUNK_CHALLENGE_DELAY` | `2**15` (= 32,768) | epochs | ~146 days | ### Max operations per block @@ -571,7 +570,7 @@ def process_custody_slashing(state: BeaconState, signed_custody_slashing: Signed def process_reveal_deadlines(state: BeaconState) -> None: epoch = get_current_epoch(state) for index, validator in enumerate(state.validators): - deadline = validator.next_custody_secret_to_reveal + (CUSTODY_RESPONSE_DEADLINE // EPOCHS_PER_CUSTODY_PERIOD) + deadline = validator.next_custody_secret_to_reveal + 1 if get_custody_period_for_validator(ValidatorIndex(index), epoch) > deadline: slash_validator(state, ValidatorIndex(index)) ``` @@ -579,7 +578,7 @@ def process_reveal_deadlines(state: BeaconState) -> None: ```python def process_challenge_deadlines(state: BeaconState) -> None: for custody_chunk_challenge in state.custody_chunk_challenge_records: - if get_current_epoch(state) > custody_chunk_challenge.inclusion_epoch + CUSTODY_RESPONSE_DEADLINE: + if get_current_epoch(state) > custody_chunk_challenge.inclusion_epoch + EPOCHS_PER_CUSTODY_PERIOD: slash_validator(state, custody_chunk_challenge.responder_index, custody_chunk_challenge.challenger_index) index_in_records = state.custody_chunk_challenge_records.index(custody_chunk_challenge) state.custody_chunk_challenge_records[index_in_records] = CustodyChunkChallengeRecord() diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py index c40aa1787..d1649484e 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py @@ -147,7 +147,7 @@ def test_multiple_epochs_custody(spec, state): @with_all_phases_except(['phase0']) @spec_state_test def test_many_epochs_custody(spec, state): - transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 20) + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 10) shard = 0 offset_slots = spec.get_offset_slots(state, shard) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_challenge_deadlines.py b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_challenge_deadlines.py index 2e60d167c..6590d8af9 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_challenge_deadlines.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_challenge_deadlines.py @@ -1,12 +1,11 @@ from eth2spec.test.helpers.custody import ( get_valid_chunk_challenge, get_shard_transition, - get_valid_custody_key_reveal, ) from eth2spec.test.helpers.attestations import ( get_valid_on_time_attestation, ) -from eth2spec.test.helpers.state import next_epoch_via_block, transition_to +from eth2spec.test.helpers.state import transition_to from eth2spec.test.context import ( with_all_phases_except, spec_state_test, @@ -17,7 +16,6 @@ from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_ep from eth2spec.test.phase_1.block_processing.test_process_chunk_challenge import ( run_chunk_challenge_processing, ) -from eth2spec.test.phase_1.block_processing.test_process_custody_key_reveal import run_custody_key_reveal_processing def run_process_challenge_deadlines(spec, state): @@ -44,32 +42,15 @@ def test_validator_slashed_after_chunk_challenge(spec, state): attestation.data.index )[0] - spec.initiate_validator_exit(state, validator_index) - assert state.validators[validator_index].withdrawable_epoch < spec.FAR_FUTURE_EPOCH - - transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) - - assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH - - while spec.get_current_epoch(state) < state.validators[validator_index].exit_epoch: - next_epoch_via_block(spec, state) - while (state.validators[validator_index].next_custody_secret_to_reveal - <= spec.get_custody_period_for_validator( - validator_index, - state.validators[validator_index].exit_epoch - 1)): - custody_key_reveal = get_valid_custody_key_reveal(spec, state, validator_index=validator_index) - _, _, _ = run_custody_key_reveal_processing(spec, state, custody_key_reveal) - - next_epoch_via_block(spec, state) - challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) _, _, _ = run_chunk_challenge_processing(spec, state, challenge) assert state.validators[validator_index].slashed == 0 - transition_to(spec, state, state.slot + (spec.CUSTODY_RESPONSE_DEADLINE + - spec.EPOCHS_PER_CUSTODY_PERIOD) * spec.SLOTS_PER_EPOCH) + transition_to(spec, state, state.slot + spec.MAX_CHUNK_CHALLENGE_DELAY * spec.SLOTS_PER_EPOCH) + + state.validators[validator_index].slashed = 0 yield from run_process_challenge_deadlines(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_reveal_deadlines.py b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_reveal_deadlines.py index 31d45aa41..9cc0069b9 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_reveal_deadlines.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_reveal_deadlines.py @@ -18,11 +18,13 @@ def run_process_challenge_deadlines(spec, state): @spec_state_test def test_validator_slashed_after_reveal_deadline(spec, state): assert state.validators[0].slashed == 0 - transition_to(spec, state, spec.get_randao_epoch_for_custody_period(0, 0) * spec.SLOTS_PER_EPOCH) - transition_to(spec, state, state.slot + ((spec.CUSTODY_RESPONSE_DEADLINE) - * spec.SLOTS_PER_EPOCH)) + # Need to run at least one reveal so that not all validators are slashed (otherwise spec fails to find proposers) + custody_key_reveal = get_valid_custody_key_reveal(spec, state, validator_index=1) + _, _, _ = run_custody_key_reveal_processing(spec, state, custody_key_reveal) + + transition_to(spec, state, state.slot + spec.EPOCHS_PER_CUSTODY_PERIOD * spec.SLOTS_PER_EPOCH) state.validators[0].slashed = 0 @@ -34,15 +36,14 @@ def test_validator_slashed_after_reveal_deadline(spec, state): @with_all_phases_except(['phase0']) @spec_state_test def test_validator_not_slashed_after_reveal(spec, state): - state.slot += spec.EPOCHS_PER_CUSTODY_PERIOD * spec.SLOTS_PER_EPOCH + transition_to(spec, state, spec.EPOCHS_PER_CUSTODY_PERIOD * spec.SLOTS_PER_EPOCH) custody_key_reveal = get_valid_custody_key_reveal(spec, state) _, _, _ = run_custody_key_reveal_processing(spec, state, custody_key_reveal) assert state.validators[0].slashed == 0 - transition_to(spec, state, state.slot + ((spec.CUSTODY_RESPONSE_DEADLINE) - * spec.SLOTS_PER_EPOCH)) + transition_to(spec, state, state.slot + spec.EPOCHS_PER_CUSTODY_PERIOD * spec.SLOTS_PER_EPOCH) yield from run_process_challenge_deadlines(spec, state) From fdb6f158676585d5b10944b62f8eb8bc4daa7784 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sat, 13 Jun 2020 15:59:04 -0500 Subject: [PATCH 143/165] unhandled exceptions do not modify forkchoice store --- specs/phase0/fork-choice.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index 95142f80c..ecbabc6f6 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -44,9 +44,11 @@ This document is the beacon chain fork choice spec, part of Ethereum 2.0 Phase 0 The head block root associated with a `store` is defined as `get_head(store)`. At genesis, let `store = get_forkchoice_store(genesis_state)` and update `store` by running: -- `on_tick(time)` whenever `time > store.time` where `time` is the current Unix time -- `on_block(block)` whenever a block `block: SignedBeaconBlock` is received -- `on_attestation(attestation)` whenever an attestation `attestation` is received +- `on_tick(store, time)` whenever `time > store.time` where `time` is the current Unix time +- `on_block(store, block)` whenever a block `block: SignedBeaconBlock` is received +- `on_attestation(store, attestation)` whenever an attestation `attestation` is received + +Any of the above handlers that trigger an unhandled exception (e.g. a failed assert or an out-of-range list access) are considered invalid. Invalid calls to handlers must not modify `store`. *Notes*: From 8697b30eea5c1353c79919b1940ed158cc69de8a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 Jun 2020 15:27:37 +0800 Subject: [PATCH 144/165] Fix configuration --- configs/mainnet/phase1.yaml | 8 ++++++++ configs/minimal/phase1.yaml | 14 +++++++++++--- specs/phase1/beacon-chain.md | 34 ++++++++++++++-------------------- 3 files changed, 33 insertions(+), 23 deletions(-) diff --git a/configs/mainnet/phase1.yaml b/configs/mainnet/phase1.yaml index 399192113..c3a4f93b9 100644 --- a/configs/mainnet/phase1.yaml +++ b/configs/mainnet/phase1.yaml @@ -28,6 +28,10 @@ TARGET_SHARD_BLOCK_SIZE: 262144 SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] # len(SHARD_BLOCK_OFFSETS) MAX_SHARD_BLOCKS_PER_ATTESTATION: 12 +# 2**12 (= 4,096) +BYTES_PER_CUSTODY_CHUNK: 4096 +# ceillog2(MAX_SHARD_BLOCK_SIZE // BYTES_PER_CUSTODY_CHUNK) +CUSTODY_RESPONSE_DEPTH: 8 # Gwei values # 2**14 (= 16,384) Gwei @@ -41,6 +45,10 @@ ONLINE_PERIOD: 8 # 2**8 (= 256) | epochs LIGHT_CLIENT_COMMITTEE_PERIOD: 256 +# Max operations per block +# 2**20 (= 1,048,576) +MAX_CUSTODY_CHUNK_CHALLENGE_RECORDS: 1048576 + # Domain types DOMAIN_SHARD_PROPOSAL: 0x80000000 DOMAIN_SHARD_COMMITTEE: 0x81000000 diff --git a/configs/minimal/phase1.yaml b/configs/minimal/phase1.yaml index 68e60aa87..5e570d646 100644 --- a/configs/minimal/phase1.yaml +++ b/configs/minimal/phase1.yaml @@ -30,6 +30,10 @@ TARGET_SHARD_BLOCK_SIZE: 262144 SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] # len(SHARD_BLOCK_OFFSETS) MAX_SHARD_BLOCKS_PER_ATTESTATION: 12 +# 2**12 (= 4,096) +BYTES_PER_CUSTODY_CHUNK: 4096 +# ceillog2(MAX_SHARD_BLOCK_SIZE // BYTES_PER_CUSTODY_CHUNK) +CUSTODY_RESPONSE_DEPTH: 8 # Gwei values # 2**14 (= 16,384) Gwei @@ -43,6 +47,10 @@ ONLINE_PERIOD: 8 # 2**8 (= 256) | epochs LIGHT_CLIENT_COMMITTEE_PERIOD: 256 +# Max operations per block +# 2**20 (= 1,048,576) +MAX_CUSTODY_CHUNK_CHALLENGE_RECORDS: 1048576 + # Domain types DOMAIN_SHARD_PROPOSAL: 0x80000000 DOMAIN_SHARD_COMMITTEE: 0x81000000 @@ -58,11 +66,11 @@ DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 RANDAO_PENALTY_EPOCHS: 2 # [customized] quicker for testing EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 4096 -# [customized] quickerquicker for testing +# [customized] quicker for testing EPOCHS_PER_CUSTODY_PERIOD: 8 -# [customized] quickerquicker for testing +# [customized] quicker for testing CUSTODY_PERIOD_TO_RANDAO_PADDING: 8 -# [customized] quickerquicker for testing +# [customized] quicker for testing CUSTODY_RESPONSE_DEADLINE: 32 # Max operations diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 61e30e102..dbfbde897 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -16,6 +16,7 @@ - [Gwei values](#gwei-values) - [Initial values](#initial-values) - [Time parameters](#time-parameters) + - [Max operations per block](#max-operations-per-block) - [Domain types](#domain-types) - [Updated containers](#updated-containers) - [Extended `AttestationData`](#extended-attestationdata) @@ -116,12 +117,14 @@ Configuration is not namespaced. Instead it is strictly an extension; ### Shard block configs -| Name | Value | -| - | - | -| `MAX_SHARD_BLOCK_SIZE` | `2**20` (= 1,048,576) | -| `TARGET_SHARD_BLOCK_SIZE` | `2**18` (= 262,144) | -| `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | -| `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | +| Name | Value | Unit | +| - | - | - | +| `MAX_SHARD_BLOCK_SIZE` | `2**20` (= 1,048,576) | bytes | +| `TARGET_SHARD_BLOCK_SIZE` | `2**18` (= 262,144) | bytes | +| `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | - | +| `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | - | +| `BYTES_PER_CUSTODY_CHUNK` | `2**12` (= 4,096) | bytes | +| `CUSTODY_RESPONSE_DEPTH` | `ceillog2(MAX_SHARD_BLOCK_SIZE // BYTES_PER_CUSTODY_CHUNK)` | - | ### Gwei values @@ -142,20 +145,11 @@ Configuration is not namespaced. Instead it is strictly an extension; | - | - | :-: | :-: | | `ONLINE_PERIOD` | `OnlineEpochs(2**3)` (= 8) | online epochs | ~51 mins | | `LIGHT_CLIENT_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | -| `MAX_SHARD_BLOCK_SIZE` | `2**20` (= 1,048,576) | bytes | - | -| `TARGET_SHARD_BLOCK_SIZE` | `2**18` (= 262,144) | bytes | - | -| `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | list of slot offsets | - | -| `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | - | - | -| `MAX_GASPRICE` | `Gwei(2**14)` (= 16,384) | Gwei | | -| `MIN_GASPRICE` | `Gwei(2**3)` (= 8) | Gwei | | -| `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | | | -| `DOMAIN_SHARD_PROPOSAL` | `DomainType('0x80000000')` | | | -| `DOMAIN_SHARD_COMMITTEE` | `DomainType('0x81000000')` | | | -| `DOMAIN_LIGHT_CLIENT` | `DomainType('0x82000000')` | | | -| `MAX_CUSTODY_CHUNK_CHALLENGE_RECORDS` | `2**20` (= 1,048,576) | | | -| `BYTES_PER_CUSTODY_CHUNK` | `2**12` | bytes | | -| `CUSTODY_RESPONSE_DEPTH` | `ceillog2(MAX_SHARD_BLOCK_SIZE // BYTES_PER_CUSTODY_CHUNK)` | - | - | -| `NO_SIGNATURE` | `BLSSignature(b'\x00' * 96)` | | | + +### Max operations per block +| Name | Value | +| - | - | +| `MAX_CUSTODY_CHUNK_CHALLENGE_RECORDS` | `2**20` (= 1,048,576) | ### Domain types From f62125eaa685b6413dbf99ad60b9c5a86fb82afe Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 15 Jun 2020 06:54:48 -0600 Subject: [PATCH 145/165] add phase 1 on-time aggregation --- specs/phase1/validator.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/specs/phase1/validator.md b/specs/phase1/validator.md index 9c6198989..3048cf8b7 100644 --- a/specs/phase1/validator.md +++ b/specs/phase1/validator.md @@ -367,6 +367,39 @@ def get_attestation_signature(state: BeaconState, return bls.Aggregate(signatures) ``` +### Attestation Aggregation + +Some validators are selected to locally aggregate attestations with a similar `attestation_data` to their constructed `attestation` for the assigned `slot`. + +Aggregation selection and the core of this duty are largely unchanged from Phase 0. Any additional components or changes are noted. + +#### Broadcast aggregate + +Note the timing of when to broadcast aggregates is altered in Phase 1+. + +If the validator is selected to aggregate (`is_aggregator`), then they broadcast their best aggregate as a `SignedAggregateAndProof` to the global aggregate channel (`beacon_aggregate_and_proof`) three-fourths of the way through the `slot`-that is, `SECONDS_PER_SLOT * 3 / 4` seconds after the start of `slot`. + +##### `AggregateAndProof` + +`AggregateAndProof` is unchanged other than the contained `Attestation`. + +```python +class AggregateAndProof(Container): + aggregator_index: ValidatorIndex + aggregate: Attestation + selection_proof: BLSSignature +``` + +##### `SignedAggregateAndProof` + +`AggregateAndProof` is unchanged other than the contained `AggregateAndProof`. + +```python +class SignedAggregateAndProof(Container): + message: AggregateAndProof + signature: BLSSignature +``` + ### Light client committee In addition to the core beacon chain responsibilities, Phase 1 adds an additional role -- the Light Client Committee -- to aid in light client functionality. From 458d3434979c218fbc2044d8c13e2c5574e5b140 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 Jun 2020 21:15:11 +0800 Subject: [PATCH 146/165] Fix configs and put domain types to the same table --- configs/mainnet/phase1.yaml | 3 ++- configs/minimal/phase1.yaml | 3 ++- specs/phase1/beacon-chain.md | 13 +++---------- specs/phase1/custody-game.md | 9 --------- specs/phase1/validator.md | 6 +++++- 5 files changed, 12 insertions(+), 22 deletions(-) diff --git a/configs/mainnet/phase1.yaml b/configs/mainnet/phase1.yaml index cddbb8dfe..08cf2317d 100644 --- a/configs/mainnet/phase1.yaml +++ b/configs/mainnet/phase1.yaml @@ -46,7 +46,8 @@ DOMAIN_SHARD_PROPOSAL: 0x80000000 DOMAIN_SHARD_COMMITTEE: 0x81000000 DOMAIN_LIGHT_CLIENT: 0x82000000 DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 - +DOMAIN_LIGHT_SELECTION_PROOF: 0x84000000 +DOMAIN_LIGHT_AGGREGATE_AND_PROOF: 0x85000000 # custody-game # --------------------------------------------------------------- diff --git a/configs/minimal/phase1.yaml b/configs/minimal/phase1.yaml index 556ff2618..c519a7427 100644 --- a/configs/minimal/phase1.yaml +++ b/configs/minimal/phase1.yaml @@ -48,7 +48,8 @@ DOMAIN_SHARD_PROPOSAL: 0x80000000 DOMAIN_SHARD_COMMITTEE: 0x81000000 DOMAIN_LIGHT_CLIENT: 0x82000000 DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 - +DOMAIN_LIGHT_SELECTION_PROOF: 0x84000000 +DOMAIN_LIGHT_AGGREGATE_AND_PROOF: 0x85000000 # custody-game # --------------------------------------------------------------- diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 77cc015c8..dcb02bfb4 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -141,14 +141,6 @@ Configuration is not namespaced. Instead it is strictly an extension; | - | - | :-: | :-: | | `ONLINE_PERIOD` | `OnlineEpochs(2**3)` (= 8) | online epochs | ~51 mins | | `LIGHT_CLIENT_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | -| `MAX_SHARD_BLOCK_SIZE` | `2**20` (= 1,048,576) | | -| `TARGET_SHARD_BLOCK_SIZE` | `2**18` (= 262,144) | | -| `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | | -| `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | | -| `MAX_GASPRICE` | `Gwei(2**14)` (= 16,384) | Gwei | | -| `MIN_GASPRICE` | `Gwei(2**3)` (= 8) | Gwei | | -| `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | | -| `NO_SIGNATURE` | `BLSSignature(b'\x00' * 96)` | | ### Domain types @@ -157,8 +149,9 @@ Configuration is not namespaced. Instead it is strictly an extension; | `DOMAIN_SHARD_PROPOSAL` | `DomainType('0x80000000')` | | `DOMAIN_SHARD_COMMITTEE` | `DomainType('0x81000000')` | | `DOMAIN_LIGHT_CLIENT` | `DomainType('0x82000000')` | -| `DOMAIN_LIGHT_SELECTION_PROOF` | `DomainType('0x83000000')` | -| `DOMAIN_LIGHT_AGGREGATE_AND_PROOF` | `DomainType('0x84000000')` | +| `DOMAIN_CUSTODY_BIT_SLASHING` | `DomainType(0x83000000)` | +| `DOMAIN_LIGHT_SELECTION_PROOF` | `DomainType('0x84000000')` | +| `DOMAIN_LIGHT_AGGREGATE_AND_PROOF` | `DomainType('0x85000000')` | ## Updated containers diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 5f5acd84f..9f89c336d 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -15,7 +15,6 @@ - [Time parameters](#time-parameters) - [Max operations per block](#max-operations-per-block) - [Reward and penalty quotients](#reward-and-penalty-quotients) - - [Signature domain types](#signature-domain-types) - [Data structures](#data-structures) - [New Beacon Chain operations](#new-beacon-chain-operations) - [`CustodySlashing`](#custodyslashing) @@ -79,14 +78,6 @@ This document details the beacon chain additions and changes in Phase 1 of Ether | `EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE` | `2**1` (= 2) | | `MINOR_REWARD_QUOTIENT` | `2**8` (= 256) | -### Signature domain types - -The following types are defined, mapping into `DomainType` (little endian): - -| Name | Value | -| - | - | -| `DOMAIN_CUSTODY_BIT_SLASHING` | `DomainType('0x83000000')` | - ## Data structures ### New Beacon Chain operations diff --git a/specs/phase1/validator.md b/specs/phase1/validator.md index 3048cf8b7..65510cc14 100644 --- a/specs/phase1/validator.md +++ b/specs/phase1/validator.md @@ -36,6 +36,10 @@ - [Construct attestation](#construct-attestation) - [Custody bits blocks](#custody-bits-blocks) - [Signature](#signature) + - [Attestation Aggregation](#attestation-aggregation) + - [Broadcast aggregate](#broadcast-aggregate) + - [`AggregateAndProof`](#aggregateandproof) + - [`SignedAggregateAndProof`](#signedaggregateandproof) - [Light client committee](#light-client-committee) - [Preparation](#preparation) - [Light clent vote](#light-clent-vote) @@ -47,7 +51,7 @@ - [Light client vote aggregation](#light-client-vote-aggregation) - [Aggregation selection](#aggregation-selection) - [Construct aggregate](#construct-aggregate) - - [Broadcast aggregate](#broadcast-aggregate) + - [Broadcast aggregate](#broadcast-aggregate-1) - [`LightAggregateAndProof`](#lightaggregateandproof) - [`SignedLightAggregateAndProof`](#signedlightaggregateandproof) From 50ac8ebb0ccebb0ced41e0471498c422db816856 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 Jun 2020 21:25:40 +0800 Subject: [PATCH 147/165] Fix DOMAIN_CUSTODY_BIT_SLASHING --- specs/phase1/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index dcb02bfb4..436404a05 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -149,7 +149,7 @@ Configuration is not namespaced. Instead it is strictly an extension; | `DOMAIN_SHARD_PROPOSAL` | `DomainType('0x80000000')` | | `DOMAIN_SHARD_COMMITTEE` | `DomainType('0x81000000')` | | `DOMAIN_LIGHT_CLIENT` | `DomainType('0x82000000')` | -| `DOMAIN_CUSTODY_BIT_SLASHING` | `DomainType(0x83000000)` | +| `DOMAIN_CUSTODY_BIT_SLASHING` | `DomainType('0x83000000')` | | `DOMAIN_LIGHT_SELECTION_PROOF` | `DomainType('0x84000000')` | | `DOMAIN_LIGHT_AGGREGATE_AND_PROOF` | `DomainType('0x85000000')` | From 97d50b381bed78e17b3c2f7510182fa38d5de6c1 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 15 Jun 2020 08:11:15 -0600 Subject: [PATCH 148/165] minor fixes in config comments --- configs/mainnet.yaml | 2 +- configs/minimal.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index d2ab3cab6..60af0fbbc 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -204,7 +204,7 @@ CUSTODY_PROBABILITY_EXPONENT: 10 RANDAO_PENALTY_EPOCHS: 2 # 2**15 (= 32,768) epochs, ~146 days EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 32768 -# 2**14 (= 16,384) epochs ~73 days +# 2**14 (= 16,384) epochs ~73 days EPOCHS_PER_CUSTODY_PERIOD: 16384 # 2**11 (= 2,048) epochs, ~9 days CUSTODY_PERIOD_TO_RANDAO_PADDING: 2048 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 3fe160bbe..dedb04f59 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -199,19 +199,19 @@ GASPRICE_ADJUSTMENT_COEFFICIENT: 8 # Phase 1: Custody Game # --------------------------------------------------------------- -# 1/1024 chance of custody bit 1 [customized for faster testing] +# 1/4 chance of custody bit 1 [customized for faster testing] CUSTODY_PROBABILITY_EXPONENT: 2 # Time parameters # 2**1 (= 2) epochs RANDAO_PENALTY_EPOCHS: 2 -# 2**15 (= 32,768) epochs, ~146 days [customized for faster testing] +# 2**7 (= 128) epochs, ~13.7 hours [customized for faster testing] EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 128 -# 2**14 (= 16,384) epochs ~73 days [customized for faster testing] +# 2**6 (= 64) epochs ~6.8 hours [customized for faster testing] EPOCHS_PER_CUSTODY_PERIOD: 64 -# 2**11 (= 2,048) epochs, ~9 days [customized for faster testing] +# 2**4 (= 16) epochs, ~1.7 hours [customized for faster testing] CUSTODY_PERIOD_TO_RANDAO_PADDING: 16 -# 2**15 (= 32,768) epochs, ~146 days +# 2**7 (= 128) epochs, ~13.7 hours [customize for faster testing] MAX_CHUNK_CHALLENGE_DELAY: 128 # Max operations From 01a69288b6dfa94f28f51c525e6869e41eb19164 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 15 Jun 2020 15:33:05 -0600 Subject: [PATCH 149/165] custody 0.01 testing cleanup --- .../eth2spec/test/helpers/attestations.py | 18 +- .../pyspec/eth2spec/test/helpers/custody.py | 4 +- .../test_process_custody_slashing.py | 190 +++++------------- 3 files changed, 50 insertions(+), 162 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index b9c91eaa0..07e227022 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -96,16 +96,7 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t return attestation_data -def convert_to_valid_on_time_attestation(spec, state, attestation, shard_transition, - signed=False): - if signed: - sign_attestation(spec, state, attestation) - - return attestation - - -def get_valid_on_time_attestation(spec, state, slot=None, index=None, - shard_transition=None, signed=False): +def get_valid_on_time_attestation(spec, state, slot=None, index=None, shard_transition=None, signed=False): ''' Construct on-time attestation for next slot ''' @@ -172,13 +163,6 @@ def get_valid_attestation(spec, # fill the attestation with (optionally filtered) participants, and optionally sign it fill_aggregate_attestation(spec, state, attestation, signed=signed, filter_participant_set=filter_participant_set) - if spec.fork == PHASE1 and on_time: - attestation = convert_to_valid_on_time_attestation( - spec, state, attestation, - shard_transition, - signed=signed, - ) - return attestation diff --git a/tests/core/pyspec/eth2spec/test/helpers/custody.py b/tests/core/pyspec/eth2spec/test/helpers/custody.py index edd7ac19f..4f91e6dc5 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/custody.py +++ b/tests/core/pyspec/eth2spec/test/helpers/custody.py @@ -194,6 +194,6 @@ def get_custody_slashable_shard_transition(spec, start_slot, block_lengths, cust shard_transition = get_shard_transition(spec, start_slot, block_lengths) slashable_test_vector = get_custody_slashable_test_vector(spec, custody_secret, block_lengths[0], slashable=slashable) - shard_transition.shard_data_roots[0] = ByteList[spec.MAX_SHARD_BLOCK_SIZE](slashable_test_vector) \ - .get_backing().get_left().merkle_root() + block_data = ByteList[spec.MAX_SHARD_BLOCK_SIZE](slashable_test_vector) + shard_transition.shard_data_roots[0] = block_data.get_backing().get_left().merkle_root() return shard_transition, slashable_test_vector diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py index d1649484e..ec0bac82d 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py @@ -53,21 +53,36 @@ def run_custody_slashing_processing(spec, state, custody_slashing, valid=True, c yield 'post', state -@with_all_phases_except(['phase0']) -@spec_state_test -def test_custody_slashing(spec, state): - transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) - shard = 0 - offset_slots = spec.get_offset_slots(state, shard) +def run_standard_custody_slashing_test(spec, + state, + shard_lateness=None, + shard=None, + validator_index=None, + block_lengths=None, + slashing_message_data=None, + correct=True, + valid=True): + if shard_lateness is None: + shard_lateness = spec.SLOTS_PER_EPOCH + transition_to(spec, state, state.slot + shard_lateness) + + if shard is None: + shard = 0 + if validator_index is None: + validator_index = spec.get_beacon_committee(state, state.slot, shard)[0] + + offset_slots = spec.get_offset_slots(state, shard) + if block_lengths is None: + block_lengths = [2**15 // 3] * len(offset_slots) - validator_index = spec.get_beacon_committee( - state, - state.slot, - shard, - )[0] custody_secret = get_custody_secret(spec, state, validator_index) - shard_transition, slashable_test_vector = get_custody_slashable_shard_transition(spec, state.slot, - [2**15 // 3] * len(offset_slots), custody_secret) + shard_transition, slashable_test_vector = get_custody_slashable_shard_transition( + spec, + state.slot, + block_lengths, + custody_secret, + slashable=correct, + ) attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) @@ -80,154 +95,43 @@ def test_custody_slashing(spec, state): slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition, custody_secret, slashable_test_vector) - yield from run_custody_slashing_processing(spec, state, slashing, correct=True) + + if slashing_message_data is not None: + slashing.message.data = slashing_message_data + + yield from run_custody_slashing_processing(spec, state, slashing, valid=valid, correct=correct) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_custody_slashing(spec, state): + yield from run_standard_custody_slashing_test(spec, state) @with_all_phases_except(['phase0']) @spec_state_test def test_incorrect_custody_slashing(spec, state): - transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) - shard = 0 - offset_slots = spec.get_offset_slots(state, shard) - - validator_index = spec.get_beacon_committee( - state, - state.slot, - shard, - )[0] - custody_secret = get_custody_secret(spec, state, validator_index) - shard_transition, slashable_test_vector = get_custody_slashable_shard_transition(spec, state.slot, - [2**15 // 3] * len(offset_slots), custody_secret, slashable=False) - - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, - shard_transition=shard_transition) - - transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) - - _, _, _ = run_attestation_processing(spec, state, attestation) - - transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) - - slashing = get_valid_custody_slashing(spec, state, attestation, - shard_transition, custody_secret, slashable_test_vector) - - yield from run_custody_slashing_processing(spec, state, slashing, correct=False) + yield from run_standard_custody_slashing_test(spec, state, correct=False) @with_all_phases_except(['phase0']) @spec_state_test def test_multiple_epochs_custody(spec, state): - transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 3) - shard = 0 - offset_slots = spec.get_offset_slots(state, shard) - - validator_index = spec.get_beacon_committee( - state, - state.slot, - shard, - )[0] - custody_secret = get_custody_secret(spec, state, validator_index) - shard_transition, slashable_test_vector = get_custody_slashable_shard_transition(spec, state.slot, - [2**15 // 3] * len(offset_slots), custody_secret) - - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, - shard_transition=shard_transition) - - transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) - - _, _, _ = run_attestation_processing(spec, state, attestation) - - transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) - - slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition, - custody_secret, slashable_test_vector) - yield from run_custody_slashing_processing(spec, state, slashing, correct=True) + yield from run_standard_custody_slashing_test(spec, state, shard_lateness=spec.SLOTS_PER_EPOCH * 3) @with_all_phases_except(['phase0']) @spec_state_test def test_many_epochs_custody(spec, state): - transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 10) - shard = 0 - offset_slots = spec.get_offset_slots(state, shard) - - validator_index = spec.get_beacon_committee( - state, - state.slot, - shard, - )[0] - custody_secret = get_custody_secret(spec, state, validator_index) - shard_transition, slashable_test_vector = get_custody_slashable_shard_transition(spec, state.slot, - [2**15 // 3] * len(offset_slots), custody_secret) - - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, - shard_transition=shard_transition) - - transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) - - _, _, _ = run_attestation_processing(spec, state, attestation) - - transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) - - slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition, - custody_secret, slashable_test_vector) - yield from run_custody_slashing_processing(spec, state, slashing, correct=True) - - -@with_all_phases_except(['phase0']) -@spec_state_test -def test_off_chain_attestation(spec, state): - transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) - shard = 0 - offset_slots = spec.get_offset_slots(state, shard) - - validator_index = spec.get_beacon_committee( - state, - state.slot, - shard, - )[0] - custody_secret = get_custody_secret(spec, state, validator_index) - shard_transition, slashable_test_vector = get_custody_slashable_shard_transition(spec, state.slot, - [2**15 // 3] * len(offset_slots), custody_secret) - - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, - shard_transition=shard_transition) - - transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) - - slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition, - custody_secret, slashable_test_vector) - yield from run_custody_slashing_processing(spec, state, slashing, correct=True) + yield from run_standard_custody_slashing_test(spec, state, shard_lateness=spec.SLOTS_PER_EPOCH * 10) @with_all_phases_except(['phase0']) @spec_state_test def test_invalid_custody_slashing(spec, state): - transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) - shard = 0 - offset_slots = spec.get_offset_slots(state, shard) - - validator_index = spec.get_beacon_committee( + yield from run_standard_custody_slashing_test( + spec, state, - state.slot, - shard, - )[0] - custody_secret = get_custody_secret(spec, state, validator_index) - shard_transition, slashable_test_vector = get_custody_slashable_shard_transition(spec, state.slot, - [2**15 // 3] * len(offset_slots), custody_secret) - - attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, - shard_transition=shard_transition) - - transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) - - _, _, _ = run_attestation_processing(spec, state, attestation) - - transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) - - slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition, - custody_secret, slashable_test_vector) - - slashing.message.data = ByteList[spec.MAX_SHARD_BLOCK_SIZE]() - - yield from run_custody_slashing_processing(spec, state, slashing, valid=False) + slashing_message_data=ByteList[spec.MAX_SHARD_BLOCK_SIZE](), + valid=False, + ) From 55e17fb1f7d01db05b2135c5e302900655e64cb3 Mon Sep 17 00:00:00 2001 From: lsankar4033 Date: Mon, 15 Jun 2020 18:42:15 -0700 Subject: [PATCH 150/165] Add compute_subnet_for_attestation unittest --- .../test/validator/test_validator_unittest.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py index 0ac1e7b46..09ceebad4 100644 --- a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py +++ b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py @@ -349,6 +349,22 @@ def test_get_attestation_signature(spec, state): ) +@with_all_phases +@spec_state_test +def test_compute_subnet_for_attestation(spec, state): + for committee_idx in range(spec.MAX_COMMITTEES_PER_SLOT): + for slot in range(state.slot, state.slot + spec.SLOTS_PER_EPOCH): + actual_subnet_id = spec.compute_subnet_for_attestation(state, slot, committee_idx) + + slots_since_epoch_start = slot % spec.SLOTS_PER_EPOCH + committees_since_epoch_start = spec.get_committee_count_at_slot( + state, slot) * slots_since_epoch_start + expected_subnet_id = (committees_since_epoch_start + + committee_idx) % spec.ATTESTATION_SUBNET_COUNT + + assert actual_subnet_id == expected_subnet_id + + # Attestation aggregation From 113563176ae2e94ff01d473b310b012338bad79b Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Tue, 16 Jun 2020 14:36:12 +0100 Subject: [PATCH 151/165] Updated .gitignore --- .gitignore | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.gitignore b/.gitignore index bcd96f885..ecbeb9a3c 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,13 @@ tests/core/pyspec/test-reports tests/core/pyspec/eth2spec/test_results.xml *.egg-info + +# flake8 config +tox.ini + +# VS code files +.vscode +*.code-workspace + +# npm (for doctoc) +package-lock.json \ No newline at end of file From 2dee432536bf32df86e657464613f6df44ccffb6 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Tue, 16 Jun 2020 14:43:34 +0100 Subject: [PATCH 152/165] Refactor getting Merkle root of data part of ByteList --- tests/core/pyspec/eth2spec/test/helpers/custody.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/custody.py b/tests/core/pyspec/eth2spec/test/helpers/custody.py index 4f91e6dc5..1993e566d 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/custody.py +++ b/tests/core/pyspec/eth2spec/test/helpers/custody.py @@ -136,6 +136,12 @@ def build_proof(anchor, leaf_index): return list(reversed(proof)) +def get_block_data_merkle_root(data_as_bytelist): + # To get the Merkle root of the block data, we need the Merkle root without the length Mixing + # The below implements this in the Remerkleable framework + return data_as_bytelist.get_backing().get_left().merkle_root() + + def get_valid_custody_chunk_response(spec, state, chunk_challenge, block_length, challenge_index, invalid_chunk_data=False): custody_data = get_custody_test_vector(block_length) @@ -160,8 +166,8 @@ def get_custody_test_vector(bytelength, offset=0): def get_shard_transition(spec, start_slot, block_lengths): - b = [ByteList[spec.MAX_SHARD_BLOCK_SIZE](get_custody_test_vector(x)) - .get_backing().get_left().merkle_root() for x in block_lengths] + b = [get_block_data_merkle_root(ByteList[spec.MAX_SHARD_BLOCK_SIZE](get_custody_test_vector(x))) + for x in block_lengths] shard_transition = spec.ShardTransition( start_slot=start_slot, shard_block_lengths=block_lengths, @@ -195,5 +201,5 @@ def get_custody_slashable_shard_transition(spec, start_slot, block_lengths, cust slashable_test_vector = get_custody_slashable_test_vector(spec, custody_secret, block_lengths[0], slashable=slashable) block_data = ByteList[spec.MAX_SHARD_BLOCK_SIZE](slashable_test_vector) - shard_transition.shard_data_roots[0] = block_data.get_backing().get_left().merkle_root() + shard_transition.shard_data_roots[0] = get_block_data_merkle_root(block_data) return shard_transition, slashable_test_vector From a4c2950c4a8bfb28dad81b18db3526d72c9637ca Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Tue, 16 Jun 2020 14:48:00 +0100 Subject: [PATCH 153/165] Phase 1 validator guide stub. This is to start collecting important details with correct validator operation. --- specs/phase1/validator.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 specs/phase1/validator.md diff --git a/specs/phase1/validator.md b/specs/phase1/validator.md new file mode 100644 index 000000000..310be3129 --- /dev/null +++ b/specs/phase1/validator.md @@ -0,0 +1,39 @@ +# Ethereum 2.0 Phase 1 -- Updates to honest validator + +**Notice**: This document is a work-in-progress for researchers and implementers. This is so far only a skeleton that describes non-obvious pitfalls so that they won't be forgotten when the full version of the document is prepared + +## Table of contents + + + + + + +- [Introduction](#introduction) +- [How to avoid slashing](#how-to-avoid-slashing) + - [Custody slashing](#custody-slashing) + + + + +## Introduction + +This is an update to the [Phase 0 -- Honest validator](../phase0/validator.md) honest validator guide. This will only describe the differences in phase 1. All behaviours in phase 0 remain valid + +## How to avoid slashing + +### Custody slashing + +To avoid custody slashings, the attester must never sign any shard transition for which the custody bit is one. The custody bit is computed using the custody secret: + +```python +def get_custody_secret(spec, state, validator_index, epoch=None): + period = spec.get_custody_period_for_validator(validator_index, epoch if epoch is not None + else spec.get_current_epoch(state)) + epoch_to_sign = spec.get_randao_epoch_for_custody_period(period, validator_index) + domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch_to_sign) + signing_root = spec.compute_signing_root(spec.Epoch(epoch_to_sign), domain) + return bls.Sign(privkeys[validator_index], signing_root) +``` + +Note that the valid custody secret is always the one for the **attestation target epoch**, not to be confused with the epoch in which the shard block was generated. While they are the same most of the time, getting this wrong at custody epoch boundaries would result in a custody slashing. From 8186594dfecb74ae006a57cdd928195ffe1752b2 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Tue, 16 Jun 2020 14:57:06 +0100 Subject: [PATCH 154/165] Rename to get_sample_shard_transition --- .../pyspec/eth2spec/test/helpers/custody.py | 4 ++-- .../test_process_chunk_challenge.py | 22 +++++++++---------- .../test_process_challenge_deadlines.py | 4 ++-- .../test_process_custody_final_updates.py | 6 ++--- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/custody.py b/tests/core/pyspec/eth2spec/test/helpers/custody.py index 1993e566d..f63a07099 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/custody.py +++ b/tests/core/pyspec/eth2spec/test/helpers/custody.py @@ -165,7 +165,7 @@ def get_custody_test_vector(bytelength, offset=0): return (b"".join((i + offset).to_bytes(4, "little") for i in range(ints)))[:bytelength] -def get_shard_transition(spec, start_slot, block_lengths): +def get_sample_shard_transition(spec, start_slot, block_lengths): b = [get_block_data_merkle_root(ByteList[spec.MAX_SHARD_BLOCK_SIZE](get_custody_test_vector(x))) for x in block_lengths] shard_transition = spec.ShardTransition( @@ -197,7 +197,7 @@ def get_custody_slashable_test_vector(spec, custody_secret, length, slashable=Tr def get_custody_slashable_shard_transition(spec, start_slot, block_lengths, custody_secret, slashable=True): - shard_transition = get_shard_transition(spec, start_slot, block_lengths) + shard_transition = get_sample_shard_transition(spec, start_slot, block_lengths) slashable_test_vector = get_custody_slashable_test_vector(spec, custody_secret, block_lengths[0], slashable=slashable) block_data = ByteList[spec.MAX_SHARD_BLOCK_SIZE](slashable_test_vector) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py index 94b1c3b11..bdb4325fd 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py @@ -1,7 +1,7 @@ from eth2spec.test.helpers.custody import ( get_valid_chunk_challenge, get_valid_custody_chunk_response, - get_shard_transition, + get_sample_shard_transition, ) from eth2spec.test.helpers.attestations import ( get_valid_on_time_attestation, @@ -70,7 +70,7 @@ def test_challenge_appended(spec, state): transition_to(spec, state, state.slot + 1) shard = 0 offset_slots = spec.get_offset_slots(state, shard) - shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) @@ -91,7 +91,7 @@ def test_challenge_empty_element_replaced(spec, state): transition_to(spec, state, state.slot + 1) shard = 0 offset_slots = spec.get_offset_slots(state, shard) - shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) @@ -114,7 +114,7 @@ def test_duplicate_challenge(spec, state): transition_to(spec, state, state.slot + 1) shard = 0 offset_slots = spec.get_offset_slots(state, shard) - shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) @@ -137,7 +137,7 @@ def test_second_challenge(spec, state): transition_to(spec, state, state.slot + 1) shard = 0 offset_slots = spec.get_offset_slots(state, shard) - shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) @@ -163,7 +163,7 @@ def test_multiple_epochs_custody(spec, state): shard = 0 offset_slots = spec.get_offset_slots(state, shard) - shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) @@ -185,7 +185,7 @@ def test_many_epochs_custody(spec, state): shard = 0 offset_slots = spec.get_offset_slots(state, shard) - shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) @@ -207,7 +207,7 @@ def test_off_chain_attestation(spec, state): shard = 0 offset_slots = spec.get_offset_slots(state, shard) - shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) @@ -225,7 +225,7 @@ def test_custody_response(spec, state): shard = 0 offset_slots = spec.get_offset_slots(state, shard) - shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) @@ -253,7 +253,7 @@ def test_custody_response_multiple_epochs(spec, state): shard = 0 offset_slots = spec.get_offset_slots(state, shard) - shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) @@ -281,7 +281,7 @@ def test_custody_response_many_epochs(spec, state): shard = 0 offset_slots = spec.get_offset_slots(state, shard) - shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_challenge_deadlines.py b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_challenge_deadlines.py index 6590d8af9..f3675732a 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_challenge_deadlines.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_challenge_deadlines.py @@ -1,6 +1,6 @@ from eth2spec.test.helpers.custody import ( get_valid_chunk_challenge, - get_shard_transition, + get_sample_shard_transition, ) from eth2spec.test.helpers.attestations import ( get_valid_on_time_attestation, @@ -28,7 +28,7 @@ def test_validator_slashed_after_chunk_challenge(spec, state): transition_to(spec, state, state.slot + 1) shard = 0 offset_slots = spec.get_offset_slots(state, shard) - shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_custody_final_updates.py b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_custody_final_updates.py index b9afe03bf..6ca6c8c99 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_custody_final_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_custody_final_updates.py @@ -2,7 +2,7 @@ from eth2spec.test.helpers.custody import ( get_valid_chunk_challenge, get_valid_custody_chunk_response, get_valid_custody_key_reveal, - get_shard_transition + get_sample_shard_transition ) from eth2spec.test.helpers.attestations import ( get_valid_on_time_attestation, @@ -66,7 +66,7 @@ def test_validator_withdrawal_suspend_after_chunk_challenge(spec, state): transition_to(spec, state, state.slot + 1) shard = 0 offset_slots = spec.get_offset_slots(state, shard) - shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) @@ -114,7 +114,7 @@ def test_validator_withdrawal_resume_after_chunk_challenge_response(spec, state) transition_to(spec, state, state.slot + 1) shard = 0 offset_slots = spec.get_offset_slots(state, shard) - shard_transition = get_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, shard_transition=shard_transition) From 4b8b3f2cbc62c49d8ab9d989164aa0fe079b1213 Mon Sep 17 00:00:00 2001 From: dankrad Date: Tue, 16 Jun 2020 15:05:01 +0100 Subject: [PATCH 155/165] Update specs/phase1/custody-game.md Co-authored-by: Hsiao-Wei Wang --- specs/phase1/custody-game.md | 1 - 1 file changed, 1 deletion(-) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 4cf047405..9b20dea55 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -368,7 +368,6 @@ def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge ```python def process_chunk_challenge_response(state: BeaconState, response: CustodyChunkResponse) -> None: - # Get matching challenge (if any) from records matching_challenges = [ record for record in state.custody_chunk_challenge_records From 4bc849bcc2c0deb4e1bba67264d75e614bf5e146 Mon Sep 17 00:00:00 2001 From: dankrad Date: Tue, 16 Jun 2020 15:05:12 +0100 Subject: [PATCH 156/165] Update specs/phase1/custody-game.md Co-authored-by: Hsiao-Wei Wang --- specs/phase1/custody-game.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 9b20dea55..8920becda 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -373,7 +373,7 @@ def process_chunk_challenge_response(state: BeaconState, record for record in state.custody_chunk_challenge_records if record.challenge_index == response.challenge_index ] - assert len(matching_challenges) > 0 + assert len(matching_challenges) == 1 challenge = matching_challenges[0] # Verify chunk index assert response.chunk_index == challenge.chunk_index From df1a9325347dee7991127653346498677e9eecdf Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Tue, 16 Jun 2020 15:16:20 +0100 Subject: [PATCH 157/165] Rename misleading variable all_secrets_are_revealed --- specs/phase1/custody-game.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 8920becda..94d739ba9 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -595,13 +595,13 @@ def process_custody_final_updates(state: BeaconState) -> None: validator_indices_in_records = set([record.responder_index for record in records]) for index, validator in enumerate(state.validators): if validator.exit_epoch != FAR_FUTURE_EPOCH: - all_secrets_are_revealed = validator.all_custody_secrets_revealed_epoch == FAR_FUTURE_EPOCH - if index in validator_indices_in_records or all_secrets_are_revealed: + not_all_secrets_are_revealed = validator.all_custody_secrets_revealed_epoch == FAR_FUTURE_EPOCH + if index in validator_indices_in_records or not_all_secrets_are_revealed: # Delay withdrawable epochs if challenge records are not empty or not all # custody secrets revealed validator.withdrawable_epoch = FAR_FUTURE_EPOCH else: - # Reset withdrawable epochs if challenge records are empty + # Reset withdrawable epochs if challenge records are empty and all secrets are revealed if validator.withdrawable_epoch == FAR_FUTURE_EPOCH: validator.withdrawable_epoch = Epoch(validator.all_custody_secrets_revealed_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY) From 5e5a951c6f85c2fd886f3707eb013b1e70fdb9eb Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 16 Jun 2020 23:49:41 +0800 Subject: [PATCH 158/165] Fix conflicts --- configs/mainnet/phase0.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/configs/mainnet/phase0.yaml b/configs/mainnet/phase0.yaml index 7677e939b..39cfddf77 100644 --- a/configs/mainnet/phase0.yaml +++ b/configs/mainnet/phase0.yaml @@ -151,9 +151,3 @@ DOMAIN_DEPOSIT: 0x03000000 DOMAIN_VOLUNTARY_EXIT: 0x04000000 DOMAIN_SELECTION_PROOF: 0x05000000 DOMAIN_AGGREGATE_AND_PROOF: 0x06000000 -<<<<<<< HEAD:configs/mainnet.yaml -# Phase 1 -DOMAIN_SHARD_PROPOSAL: 0x80000000 -DOMAIN_SHARD_COMMITTEE: 0x81000000 -DOMAIN_LIGHT_CLIENT: 0x82000000 -DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 From 58935c19d6e8590cf69be6e6e6246e946668e044 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Tue, 16 Jun 2020 17:23:09 +0100 Subject: [PATCH 159/165] Move constant --- specs/phase1/beacon-chain.md | 5 ----- specs/phase1/custody-game.md | 1 + 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index a3f9ec4e9..59e0bb6da 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -142,11 +142,6 @@ Configuration is not namespaced. Instead it is strictly an extension; | `ONLINE_PERIOD` | `OnlineEpochs(2**3)` (= 8) | online epochs | ~51 mins | | `LIGHT_CLIENT_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | -### Max operations per block -| Name | Value | -| - | - | -| `MAX_CUSTODY_CHUNK_CHALLENGE_RECORDS` | `2**20` (= 1,048,576) | - ### Domain types | Name | Value | diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 94d739ba9..3d4645068 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -79,6 +79,7 @@ This document details the beacon chain additions and changes in Phase 1 of Ether | Name | Value | | - | - | +| `MAX_CUSTODY_CHUNK_CHALLENGE_RECORDS` | `2**20` (= 1,048,576) | | `MAX_CUSTODY_KEY_REVEALS` | `2**8` (= 256) | | `MAX_EARLY_DERIVED_SECRET_REVEALS` | `2**0` (= 1) | | `MAX_CUSTODY_CHUNK_CHALLENGES` | `2**2` (= 4) | From e6e694fad995affe9a79cd92cf6e81e0e4874617 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Tue, 16 Jun 2020 17:35:33 +0100 Subject: [PATCH 160/165] Fix toc --- specs/phase1/beacon-chain.md | 1 - 1 file changed, 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 59e0bb6da..3206a50de 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -16,7 +16,6 @@ - [Gwei values](#gwei-values) - [Initial values](#initial-values) - [Time parameters](#time-parameters) - - [Max operations per block](#max-operations-per-block) - [Domain types](#domain-types) - [Updated containers](#updated-containers) - [Extended `AttestationData`](#extended-attestationdata) From c761c437d4cb4a647a5c65b704731c86f0883385 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 16 Jun 2020 12:11:47 -0600 Subject: [PATCH 161/165] revert e128 and c901 in lint --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 419500ebb..da87f7912 100644 --- a/Makefile +++ b/Makefile @@ -101,7 +101,7 @@ codespell: lint: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - flake8 --ignore=E252,W504,W503,E128,C901 --max-line-length=120 ./eth2spec \ + flake8 --ignore=E252,W504,W503 --max-line-length=120 ./eth2spec \ && cd ./eth2spec && mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase0 \ && mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase1; From 2e0950560b7759024d2cf7239b112698d8b147a5 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 16 Jun 2020 16:03:36 -0600 Subject: [PATCH 162/165] PR feedback --- specs/phase1/shard-fork-choice.md | 2 +- specs/phase1/validator.md | 17 ++++++++--------- .../pyspec/eth2spec/test/helpers/shard_block.py | 2 -- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 0607613e8..e051054fe 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -124,7 +124,7 @@ def get_pending_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ beacon_head_state = store.block_states[beacon_head_root] latest_shard_block_root = beacon_head_state.shard_states[shard].latest_block_root - shard_head_root = get_shard_head(store, shard_store) + shard_head_root = tget_shard_head(store, shard_store) root = shard_head_root shard_blocks = [] while root != latest_shard_block_root: diff --git a/specs/phase1/validator.md b/specs/phase1/validator.md index 65510cc14..6db20e82e 100644 --- a/specs/phase1/validator.md +++ b/specs/phase1/validator.md @@ -95,7 +95,7 @@ Lookahead for beacon committee assignments operates in the same manner as Phase Specifically _after_ finding stable peers of attestation subnets (see Phase 0) a validator should: * Let `shard = compute_shard_from_committee_index(committe_index)` -* Subscribe to the pubsub topic `shard_{shard}_shard_block` (attestation subnet peers should have this topic available). +* Subscribe to the pubsub topic `shard_{shard}_block` (attestation subnet peers should have this topic available). ## Beacon chain responsibilities @@ -140,22 +140,21 @@ Specifically: * Let `full_transitions` be a dictionary mapping from the `shard_transition_root`s found in `attestations` to the corresponding full `ShardTransition` * Then for each `shard` and `winning_root` in `zip(shards, winning_roots)` set `shard_transitions[shard] = full_transitions[winning_root]` -*Note*: The `state` passed into `get_shard_winning_roots` must be transitioned into the epoch of `block.slot` to run accurately due to the internal use of `get_online_validator_indices`. +*Note*: The `state` passed into `get_shard_winning_roots` must be transitioned the slot of `block.slot` to run accurately due to the internal use of `get_online_validator_indices` and `is_on_time_attestation`. ```python def get_shard_winning_roots(state: BeaconState, - slot: Slot, attestations: Sequence[Attestation]) -> Tuple[Sequence[Shard], Sequence[Root]]: shards = [] winning_roots = [] online_indices = get_online_validator_indices(state) - committee_count = get_committee_count_at_slot(state, slot) + committee_count = get_committee_count_at_slot(state, state.slot) for committee_index in map(CommitteeIndex, range(committee_count)): - shard = compute_shard_from_committee_index(state, committee_index, slot) - # All attestations in the block for this committee/shard and current slot + shard = compute_shard_from_committee_index(state, committee_index, state.slot) + # All attestations in the block for this committee/shard and are "on time" shard_attestations = [ attestation for attestation in attestations - if attestation.data.index == committee_index and attestation.data.slot == slot + if is_on_time_attestation(state, attestation) and attestation.data.index == committee_index ] committee = get_beacon_committee(state, state.slot, committee_index) @@ -259,7 +258,7 @@ A validator should create and broadcast the `attestation` to the associated atte - Let `head_block` be the result of running the fork choice during the assigned slot. - Let `head_state` be the state of `head_block` processed through any empty slots up to the assigned slot using `process_slots(state, slot)`. - Let `shard_head_block` be the result of running the fork choice on the assigned shard chain during the assigned slot. -- Let `shard_blocks` be the shard blocks in the chain starting immediately _after_ the most recent crosslink (`head_state.shard_transitions[shard].latest_block_root`) up to the `shard_head_block`. +- Let `shard_blocks` be the shard blocks in the chain starting immediately _after_ the most recent crosslink (`head_state.shard_transitions[shard].latest_block_root`) up to the `shard_head_block` (i.e. the value of the shard fork choice store of `get_pending_shard_blocks(store, shard_store)`). *Note*: We assume that the fork choice only follows branches with valid `offset_slots` with respect to the most recent beacon state shard transition for the queried shard. @@ -420,7 +419,7 @@ If the validator is in the next light client committee, they must join the `ligh def is_in_next_light_client_committee(state: BeaconState, index: ValidatorIndex) -> bool: current_source_epoch = compute_committee_source_epoch(get_current_epoch(state), LIGHT_CLIENT_COMMITTEE_PERIOD) next_source_epoch = current_source_epoch + LIGHT_CLIENT_COMMITTEE_PERIOD - next_committee = get_light_client_committee(state, next_source_epoch) + next_committee = get_light_client_committee(state, get_current_epoch(state) + LIGHT_CLIENT_COMMITTEE_PERIOD) return index in next_committee ``` diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 15443e386..ddf66f6c2 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -62,8 +62,6 @@ def get_shard_transitions(spec, parent_beacon_state, shard_blocks): on_time_slot, ) len_offset_slots = len(offset_slots) - # TODO this is actually unsafe for long offset_slots - assert len_offset_slots == on_time_slot - parent_beacon_state.shard_states[shard].slot - 1 shard_transition = spec.get_shard_transition(parent_beacon_state, shard, blocks) if len(blocks) > 0: From fbf10a0db3155ae413248a9241104c4755fd7db4 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 16 Jun 2020 22:29:28 -0600 Subject: [PATCH 163/165] fix tests --- specs/phase1/validator.md | 43 +------------------ .../test/validator/test_validator_unittest.py | 28 +----------- 2 files changed, 4 insertions(+), 67 deletions(-) diff --git a/specs/phase1/validator.md b/specs/phase1/validator.md index aef8fbb99..558a1b3bf 100644 --- a/specs/phase1/validator.md +++ b/specs/phase1/validator.md @@ -34,8 +34,6 @@ - [Head shard root](#head-shard-root) - [Shard transition](#shard-transition) - [Construct attestation](#construct-attestation) - - [Custody bits blocks](#custody-bits-blocks) - - [Signature](#signature) - [Attestation Aggregation](#attestation-aggregation) - [Broadcast aggregate](#broadcast-aggregate) - [`AggregateAndProof`](#aggregateandproof) @@ -216,7 +214,7 @@ Packaging into a `SignedBeaconBlock` is unchanged from Phase 0. A validator is expected to create, sign, and broadcast an attestation during each epoch. -Assignments and the core of this duty are unchanged from Phase 0. There are a few additional fields related to the assigned shard chain and custody bit. +Assignments and the core of this duty are unchanged from Phase 0. There are a few additional fields related to the assigned shard chain. The `Attestation` and `AttestationData` defined in the [Phase 1 Beacon Chain spec]() utilizes `shard_transition_root: Root` rather than a full `ShardTransition`. For the purposes of the validator and p2p layer, a modified `FullAttestationData` and containing `FullAttestation` are used to send the accompanying `ShardTransition` in its entirety. Note that due to the properties of SSZ `hash_tree_root`, the root and signatures of `AttestationData` and `FullAttestationData` are equivalent. @@ -243,7 +241,6 @@ class FullAttestationData(Container): class FullAttestation(Container): aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] data: FullAttestationData - custody_bits_blocks: List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_SHARD_BLOCKS_PER_ATTESTATION] signature: BLSSignature ``` @@ -334,43 +331,7 @@ def get_shard_transition(beacon_state: BeaconState, Next, the validator creates `attestation`, a `FullAttestation` as defined above. -`attestation.data` and `attestation.aggregation_bits` are unchanged from Phase 0. - -##### Custody bits blocks - -- Let `attestation.custody_bits_blocks` be a the value returned by `get_custody_bits_blocks()` - -```python -def get_custody_bits_blocks() -> List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_SHARD_BLOCKS_PER_ATTESTATION]: - pass -``` - -##### Signature - -Set `attestation.signature = attestation_signature` where `attestation_signature` is obtained from: - -```python -def get_attestation_signature(state: BeaconState, - attestation: Attestation, - privkey: int) -> BLSSignature: - domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch) - attestation_data_root = hash_tree_root(attestation.data) - index_in_committee = attestation.aggregation_bits.index(True) - signatures = [] - for block_index, custody_bits in enumerate(attestation.custody_bits_blocks): - custody_bit = custody_bits[index_in_committee] - signing_root = compute_signing_root( - AttestationCustodyBitWrapper( - attestation_data_root=attestation_data_root, - block_index=block_index, - bit=custody_bit, - ), - domain, - ) - signatures.append(bls.Sign(privkey, signing_root)) - - return bls.Aggregate(signatures) -``` +`attestation.data`, `attestation.aggregation_bits`, and `attestation.signature` are unchanged from Phase 0. But safety/validity in signing the message is premised upon calculation of the "custody bit" [TODO]. ### Attestation Aggregation diff --git a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py index 26affd579..8edd4fd2a 100644 --- a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py +++ b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py @@ -1,8 +1,6 @@ -from random import Random - from eth2spec.test.context import ( spec_state_test, - always_bls, with_phases, with_all_phases, with_all_phases_except, + always_bls, with_phases, with_all_phases, PHASE0, ) from eth2spec.test.helpers.attestations import build_attestation_data, get_valid_attestation @@ -323,7 +321,7 @@ def test_get_block_signature(spec, state): # Attesting -@with_phases([PHASE0]) +@with_all_phases @spec_state_test @always_bls def test_get_attestation_signature_phase0(spec, state): @@ -343,28 +341,6 @@ def test_get_attestation_signature_phase0(spec, state): ) -@with_all_phases_except([PHASE0]) -@spec_state_test -@always_bls -def test_get_attestation_signature_phase1plus(spec, state): - privkey = privkeys[0] - - def single_participant(comm): - rng = Random(1100) - return rng.sample(comm, 1) - - attestation = get_valid_attestation(spec, state, filter_participant_set=single_participant, signed=False) - indexed_attestation = spec.get_indexed_attestation(state, attestation) - - assert indexed_attestation.attestation.aggregation_bits.count(True) == 1 - - # Cannot use normal `run_get_signature_test` due to complex signature type - index_in_committee = indexed_attestation.attestation.aggregation_bits.index(True) - privkey = privkeys[indexed_attestation.committee[index_in_committee]] - attestation.signature = spec.get_attestation_signature(state, attestation, privkey) - assert spec.verify_attestation_custody(state, spec.get_indexed_attestation(state, attestation)) - - # Attestation aggregation From adced70c5425adee09ddf9b09cf2731ed5c6ccdd Mon Sep 17 00:00:00 2001 From: ericsson Date: Thu, 18 Jun 2020 02:04:16 +0300 Subject: [PATCH 164/165] use `aggregate.data` instead of hust `data` --- specs/phase1/validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/validator.md b/specs/phase1/validator.md index 558a1b3bf..15596edc6 100644 --- a/specs/phase1/validator.md +++ b/specs/phase1/validator.md @@ -195,7 +195,7 @@ def get_best_light_client_aggregate(block: BeaconBlock, aggregates: Sequence[LightClientVote]) -> LightClientVote: viable_aggregates = [ aggregate for aggregate in aggregates - if aggregate.slot == compute_previous_slot(block.slot) and aggregate.beacon_block_root == block.parent_root + if aggregate.data.slot == compute_previous_slot(block.slot) and aggregate.data.beacon_block_root == block.parent_root ] return max( From e479e964808e771f2aeb7ede747aa137845f119a Mon Sep 17 00:00:00 2001 From: ericsson Date: Thu, 18 Jun 2020 02:46:06 +0300 Subject: [PATCH 165/165] lint fix --- specs/phase1/validator.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/specs/phase1/validator.md b/specs/phase1/validator.md index 15596edc6..ea69f0389 100644 --- a/specs/phase1/validator.md +++ b/specs/phase1/validator.md @@ -195,7 +195,10 @@ def get_best_light_client_aggregate(block: BeaconBlock, aggregates: Sequence[LightClientVote]) -> LightClientVote: viable_aggregates = [ aggregate for aggregate in aggregates - if aggregate.data.slot == compute_previous_slot(block.slot) and aggregate.data.beacon_block_root == block.parent_root + if ( + aggregate.data.slot == compute_previous_slot(block.slot) + and aggregate.data.beacon_block_root == block.parent_root + ) ] return max(