From 1192158848c622011f7b3fe758f076ac90e40457 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 4 Jan 2021 11:17:30 -0700 Subject: [PATCH] minor cleanups/fixes to shard data avail PR --- specs/phase1/beacon-chain.md | 252 +++++++++++++++++++---------------- 1 file changed, 138 insertions(+), 114 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index e59696806..8c18c682d 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -23,20 +23,24 @@ - [New containers](#new-containers) - [`DataCommitment`](#datacommitment) - [`ShardHeader`](#shardheader) + - [`SignedShardHeader`](#signedshardheader) - [`PendingShardHeader`](#pendingshardheader) - [Helper functions](#helper-functions) - [Misc](#misc-1) + - [`next_power_of_two`](#next_power_of_two) + - [`reverse_bit_order`](#reverse_bit_order) - [`compute_previous_slot`](#compute_previous_slot) - - [`compute_shard_from_committee_index`](#compute_shard_from_committee_index) - [`compute_updated_gasprice`](#compute_updated_gasprice) - [`compute_committee_source_epoch`](#compute_committee_source_epoch) - [Beacon state accessors](#beacon-state-accessors) - [Updated `get_committee_count_per_slot`](#updated-get_committee_count_per_slot) - [`get_active_shard_count`](#get_active_shard_count) - [`get_shard_committee`](#get_shard_committee) + - [`compute_proposer_index`](#compute_proposer_index) - [`get_shard_proposer_index`](#get_shard_proposer_index) - [`get_start_shard`](#get_start_shard) - - [Predicates](#predicates) + - [`compute_shard_from_committee_index`](#compute_shard_from_committee_index) + - [`compute_committee_index_from_shard`](#compute_committee_index_from_shard) - [Block processing](#block-processing) - [Operations](#operations) - [New Attestation processing](#new-attestation-processing) @@ -139,7 +143,7 @@ class AttestationData(Container): ```python class BeaconBlock(phase0.BeaconBlock): - shard_headers: List[Signed[ShardHeader], MAX_SHARD_HEADERS] + shard_headers: List[SignedShardHeader, MAX_SHARD_HEADERS] ``` ### `BeaconState` @@ -184,6 +188,14 @@ class ShardHeader(Container): degree_proof: BLSCommitment ``` +### `SignedShardHeader` + +```python +class SignedShardHeader(Container): + message: ShardHeader + signature: BLSSignature +``` + ### `PendingShardHeader` ```python @@ -193,8 +205,7 @@ class PendingShardHeader(Container): shard: Shard # KZG10 commitment to the data commitment: DataCommitment - # hash_tree_root of the ShardHeader (stored so that attestations - # can be checked against it) + # hash_tree_root of the ShardHeader (stored so that attestations can be checked against it) root: Root # Who voted for the header votes: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] @@ -307,10 +318,14 @@ def get_shard_committee(beacon_state: BeaconState, epoch: Epoch, shard: Shard) - #### `compute_proposer_index` -Updated version to get a proposer index that will only allow proposers with a certain minimum balance, ensuring that the balance is always sufficient to cover gas costs. +Updated version to get a proposer index that will only allow proposers with a certain minimum balance, +ensuring that the balance is always sufficient to cover gas costs. ```python -def compute_proposer_index(state: BeaconState, indices: Sequence[ValidatorIndex], seed: Bytes32, min_effective_balance: GWei = GWei(0)) -> ValidatorIndex: +def compute_proposer_index(beacon_state: BeaconState, + indices: Sequence[ValidatorIndex], + seed: Bytes32, + min_effective_balance: GWei = GWei(0)) -> ValidatorIndex: """ Return from ``indices`` a random index sampled by effective balance. """ @@ -321,8 +336,10 @@ def compute_proposer_index(state: BeaconState, indices: Sequence[ValidatorIndex] while True: candidate_index = indices[compute_shuffled_index(i % total, total, seed)] random_byte = hash(seed + uint_to_bytes(uint64(i // 32)))[i % 32] - effective_balance = state.validators[candidate_index].effective_balance - if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte and effective_balance > min_effective_balance: + effective_balance = beacon_state.validators[candidate_index].effective_balance + if effective_balance <= min_effective_balance: + continue + if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: return candidate_index i += 1 ``` @@ -336,11 +353,18 @@ def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard """ epoch = compute_epoch_at_slot(slot) committee = get_shard_committee(beacon_state, epoch, shard) - seed = hash(get_seed(state, epoch, DOMAIN_BEACON_PROPOSER) + uint_to_bytes(state.slot)) - EFFECTIVE_BALANCE_MAX_DOWNWARD_DEVIATION = EFFECTIVE_BALANCE_INCREMENT - EFFECTIVE_BALANCE_INCREMENT * HYSTERESIS_DOWNWARD_MULTIPLIER // HYSTERESIS_QUOTIENT + seed = hash(get_seed(beacon_state, epoch, DOMAIN_BEACON_PROPOSER) + uint_to_bytes(beacon_state.slot)) - return compute_proposer_index(state, committee, seed, - state.shard_gasprice * MAX_SAMPLES_PER_BLOCK // TARGET_SAMPLES_PER_BLOCK + EFFECTIVE_BALANCE_MAX_DOWNWARD_DEVIATION) + # Proposer must have sufficient balance to pay for worst case fee burn + EFFECTIVE_BALANCE_MAX_DOWNWARD_DEVIATION = ( + (EFFECTIVE_BALANCE_INCREMENT - EFFECTIVE_BALANCE_INCREMENT) + * HYSTERESIS_DOWNWARD_MULTIPLIER // HYSTERESIS_QUOTIENT + ) + min_effective_balance = ( + beacon_state.shard_gasprice * MAX_SAMPLES_PER_BLOCK // TARGET_SAMPLES_PER_BLOCK + + EFFECTIVE_BALANCE_MAX_DOWNWARD_DEVIATION + ) + return compute_proposer_index(beacon_state, committee, seed, min_effective_balance) ``` #### `get_start_shard` @@ -358,7 +382,6 @@ def get_start_shard(state: BeaconState, slot: Slot) -> Shard: committee_count = get_committee_count_per_slot(state, compute_epoch_at_slot(Slot(_slot))) active_shard_count = get_active_shard_count(state, compute_epoch_at_slot(Slot(_slot))) shard = (shard + committee_count) % active_shard_count - return Shard(shard) elif slot < current_epoch_start_slot: # Previous epoch for _slot in list(range(slot, current_epoch_start_slot))[::-1]: @@ -410,20 +433,16 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: for_ops(body.proposer_slashings, process_proposer_slashing) for_ops(body.attester_slashings, process_attester_slashing) + # Limit is dynamic based on active shard count + assert len(body.shard_headers) <= MAX_SHARD_HEADERS_PER_SHARD * get_active_shard_count(state, get_current_epoch(state)) + for_ops(body.shard_headers, process_shard_header) # New attestation processing for_ops(body.attestations, process_attestation) for_ops(body.deposits, process_deposit) for_ops(body.voluntary_exits, process_voluntary_exit) - # Limit is dynamic based on active shard count - assert len(body.shard_headers) <= MAX_SHARD_HEADERS_PER_SHARD * get_active_shard_count(state, get_current_epoch(state)) - for_ops(body.shard_headers, process_shard_header) # See custody game spec. process_custody_game_operations(state, body) - - process_shard_transitions(state, body.shard_transitions, body.attestations) - - # TODO process_operations(body.shard_receipt_proofs, process_shard_receipt_proofs) ``` ### New Attestation processing @@ -441,11 +460,11 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: ```python def update_pending_votes(state: BeaconState, attestation: Attestation) -> None: - if compute_epoch_at_slot(slot) == get_current_epoch(state): + # Find and update the PendingShardHeader object, invalid block if pending header not in state + if compute_epoch_at_slot(attestation.data.slot) == get_current_epoch(state): pending_headers = state.current_epoch_pending_shard_headers else: pending_headers = state.previous_epoch_pending_shard_headers - # Create or update the PendingShardHeader object pending_header = None for header in pending_headers: if header.root == attestation.data.shard_header_root: @@ -457,10 +476,8 @@ def update_pending_votes(state: BeaconState, attestation.data.slot, attestation.data.index, ) - pending_header.votes = bitwise_or( - pending_header.votes, - attestation.aggregation_bits - ) + for i in range(len(pending_header.votes)): + pending_header.votes[i] = pending_header.votes[i] or attestation.aggregation_bits[i] # Check if the PendingShardHeader is eligible for expedited confirmation # Requirement 1: nothing else confirmed @@ -468,47 +485,49 @@ def update_pending_votes(state: BeaconState, c for c in pending_headers if (c.slot, c.shard) == (pending_header.slot, pending_header.shard) ] - if True not in [c.confirmed for c in all_candidates]: - # Requirement 2: >= 2/3 of balance attesting - participants = get_attesting_indices(state, attestation.data, pending_commitment.votes) - participants_balance = get_total_balance(state, participants) - full_committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index) - full_committee_balance = get_total_balance(state, full_committee) - if participants_balance * 3 > full_committee_balance * 2: - pending_header.confirmed = True + if True in [c.confirmed for c in all_candidates]: + return + + # Requirement 2: >= 2/3 of balance attesting + participants = get_attesting_indices(state, attestation.data, pending_commitment.votes) + participants_balance = get_total_balance(state, participants) + full_committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index) + full_committee_balance = get_total_balance(state, full_committee) + if participants_balance * 3 >= full_committee_balance * 2: + pending_header.confirmed = True ``` #### `process_shard_header` ```python def process_shard_header(state: BeaconState, - signed_header: Signed[ShardHeader]) -> None: + signed_header: SignedShardHeader) -> None: header = signed_header.message header_root = hash_tree_root(header) + assert compute_epoch_at_slot(header.slot) in [get_previous_epoch(state), get_current_epoch(state)] + # Verify signature signer_index = get_shard_proposer_index(state, header.slot, header.shard) - assert bls.Verify( - state.validators[signer_index].pubkey, - compute_signing_root(header, get_domain(state, DOMAIN_SHARD_HEADER)), - signed_header.signature - ) + signing_root = compute_signing_root(header, get_domain(state, DOMAIN_SHARD_HEADER)) + assert bls.Verify(state.validators[signer_index].pubkey, signing_root, signed_header.signature) + # Verify the length by verifying the degree. if header.commitment.length == 0: assert header.degree_proof == G1_SETUP[0] assert ( - bls.Pairing(header.degree_proof, G2_SETUP[0]) == - bls.Pairing(header.commitment.point, G2_SETUP[-header.commitment.length])) + bls.Pairing(header.degree_proof, G2_SETUP[0]) + == bls.Pairing(header.commitment.point, G2_SETUP[-header.commitment.length])) ) + # Get the correct pending header list if compute_epoch_at_slot(header.slot) == get_current_epoch(state): pending_headers = state.current_epoch_pending_shard_headers else: - assert compute_epoch_at_slot(header.slot) == get_previous_epoch(state): pending_headers = state.previous_epoch_pending_shard_headers - + # Check that this header is not yet in the pending list - for pending_header in pending_headers: - assert header_root != pending_header.root + assert header_root not in [pending_header.root for pending_header in pending_headers] + # Include it in the pending list index = compute_committee_index_from_shard(state, header.slot, header.shard) committee_length = len(get_beacon_committee(state, header.slot, index)) @@ -518,7 +537,7 @@ def process_shard_header(state: BeaconState, commitment=header.commitment, root=header_root, votes=Bitlist[MAX_VALIDATORS_PER_COMMITTEE]([0] * committee_length), - confirmed=False + confirmed=False, )) ``` @@ -565,102 +584,107 @@ def process_epoch(state: BeaconState) -> None: ```python -def process_pending_headers(state: BeaconState): - for slot in range(SLOTS_PER_EPOCH): +def process_pending_headers(state: BeaconState) -> None: + # Pending header processing applies to the previous epoch. + # Skip if `GENESIS_EPOCH` because no prior epoch to process. + if get_current_epoch(state) == GENESIS_EPOCH: + return + + previous_epoch_start_slot = compute_start_slot_at_epoch(get_previous_epoch(state)) + for slot in range(previous_epoch_start_slot, previous_epoch_start_slot + SLOTS_PER_EPOCH): for shard in range(get_active_shard_count(state)): # Pending headers for this (slot, shard) combo candidates = [ - c for c in state.previous_epoch_pending_shard_headers if - (c.slot, c.shard) == (slot, shard) + c for c in state.previous_epoch_pending_shard_headers + if (c.slot, c.shard) == (slot, shard) ] # The entire committee (and its balance) full_committee = get_beacon_committee(state, slot, shard) full_committee_balance = get_total_balance(state, full_committee) - if True not in [c.confirmed for c in candidates]: - # The set of voters who voted for each header - # (and their total balances) - voting_sets = [ - [v for i, v in enumerate(full_committee) if c.votes[i]] - for c in candidates - ] - voting_balances = [ - get_total_balance(state, voters) - for voters in voting_sets - ] - # Get the index with the most total balance voting for them. - # NOTE: if two choices get exactly the same voting balance, - # the candidate earlier in the list wins - if max(voting_balances) > 0: - winning_index = voting_balances.index(max(voting_balances)) - else: - # If no votes, zero wins - winning_index = [c.root for c in candidates].index(Root()) - candidates[winning_index].confirmed = True - for slot in range(SLOTS_PER_EPOCH): + # If any candidates already confirmed, skip + if True in [c.confirmed for c in candidates]: + continue + + # The set of voters who voted for each header (and their total balances) + voting_sets = [ + [v for i, v in enumerate(full_committee) if c.votes[i]] + for c in candidates + ] + voting_balances = [ + get_total_balance(state, voters) + for voters in voting_sets + ] + # Get the index with the most total balance voting for them. + # NOTE: if two choices get exactly the same voting balance, + # the candidate earlier in the list wins + if max(voting_balances) > 0: + winning_index = voting_balances.index(max(voting_balances)) + else: + # If no votes, zero wins + winning_index = [c.root for c in candidates].index(Root()) + candidates[winning_index].confirmed = True + for slot_index in range(SLOTS_PER_EPOCH): for shard in range(SHARD_COUNT): - state.grandparent_epoch_confirmed_commitments[shard][slot] = DataCommitment() - for c in state.previous_epoch_pending_shard_headers: - if c.confirmed: - state.grandparent_epoch_confirmed_commitments[c.shard][c.slot % SLOTS_PER_EPOCH] = c.commitment -``` + state.grandparent_epoch_confirmed_commitments[shard][slot_index] = DataCommitment() + confirmed_headers = [candidate in state.previous_epoch_pending_shard_headers if candidate.confirmed] + for header in confirmed_headers: + state.grandparent_epoch_confirmed_commitments[c.shard][c.slot % SLOTS_PER_EPOCH] = c.commitment +``` ```python def charge_confirmed_header_fees(state: BeaconState) -> None: new_gasprice = state.shard_gasprice - adjustment_quotient = get_active_shard_count(state, get_current_epoch(state)) * SLOTS_PER_EPOCH * GASPRICE_ADJUSTMENT_COEFFICIENT - for slot in range(SLOTS_PER_EPOCH): + adjustment_quotient = ( + get_active_shard_count(state, get_current_epoch(state)) + * SLOTS_PER_EPOCH * GASPRICE_ADJUSTMENT_COEFFICIENT + ) + previous_epoch_start_slot = compute_start_slot_at_epoch(get_previous_epoch(state)) + for slot in range(previous_epoch_start_slot, previous_epoch_start_slot + SLOTS_PER_EPOCH): for shard in range(SHARD_COUNT): confirmed_candidates = [ - c for c in state.previous_epoch_pending_shard_headers if - (c.slot, c.shard, c.confirmed) == (slot, shard, True) + c for c in state.previous_epoch_pending_shard_headers + if (c.slot, c.shard, c.confirmed) == (slot, shard, True) ] - if confirmed_candidates: - candidate = confirmed_candidates[0] - # Charge EIP 1559 fee - proposer = get_shard_proposer(state, slot, shard) - fee = ( - (state.shard_gasprice * candidates[i].commitment.length) // - TARGET_SAMPLES_PER_BLOCK - ) - decrease_balance(state, proposer, fee) - new_gasprice = compute_updated_gasprice( - new_gasprice, - candidates[i].commitment.length, - adjustment_quotient - ) + if not any(confirmed_candidates): + continue + candidate = confirmed_candidates[0] + + # Charge EIP 1559 fee + proposer = get_shard_proposer(state, slot, shard) + fee = ( + (state.shard_gasprice * candidate.commitment.length) + // TARGET_SAMPLES_PER_BLOCK + ) + decrease_balance(state, proposer, fee) + + # Track updated gas price + new_gasprice = compute_updated_gasprice( + new_gasprice, + candidate.commitment.length, + adjustment_quotient, + ) state.shard_gasprice = new_gasprice ``` ```python def reset_pending_headers(state: BeaconState): state.previous_epoch_pending_shard_headers = state.current_epoch_pending_shard_headers - shards = [ - compute_shard_from_committee_index(state, slot, index) - for i in range() - state, - attestation.data.index, - attestation.data.slot - ) state.current_epoch_pending_shard_headers = [] - # Add dummy "empty" PendingAttestations - # (default to vote for if no shard header available) - for slot in range(SLOTS_IN_EPOCH): - for index in range(get_committee_count_per_slot(get_current_epoch(state))): + # Add dummy "empty" PendingShardHeader (default vote for if no shard header available) + next_epoch = get_current_epoch(state) + 1 + next_epoch_start_slot = compute_start_slot_at_epoch(next_epoch) + for slot in range(next_epoch_start_slot, next_epoch_start_slot + SLOTS_IN_EPOCH): + for index in range(get_committee_count_per_slot(next_epoch) shard = compute_shard_from_committee_index(state, slot, index) - committee_length = len(get_beacon_committee( - state, - header.slot, - header.shard - )) + committee_length = len(get_beacon_committee(state, slot, shard)) state.current_epoch_pending_shard_headers.append(PendingShardHeader( slot=slot, shard=shard, commitment=DataCommitment(), root=Root(), votes=Bitlist[MAX_VALIDATORS_PER_COMMITTEE]([0] * committee_length), - confirmed=False + confirmed=False, )) - ``` #### Custody game updates