diff --git a/Makefile b/Makefile index fb93908cc..bfbc28070 100644 --- a/Makefile +++ b/Makefile @@ -89,7 +89,7 @@ $(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS) python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $(SPEC_DIR)/validator/0_beacon-chain-validator.md $@ $(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS) - python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $(SPEC_DIR)/light_client/merkle_proofs.md $@ + python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $(SPEC_DIR)/light_client/merkle_proofs.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $(SPEC_DIR)/core/1_beacon-chain-misc.md $@ CURRENT_DIR = ${CURDIR} diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 15b749b9d..be787ca3c 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -141,5 +141,5 @@ SHARD_SLOTS_PER_BEACON_SLOT: 2 EPOCHS_PER_SHARD_PERIOD: 4 # PHASE_1_FORK_EPOCH >= EPOCHS_PER_SHARD_PERIOD * 2 PHASE_1_FORK_EPOCH: 8 -# PHASE_1_FORK_SLOT = PHASE_1_FORK_EPOCH * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH -PHASE_1_FORK_SLOT: 128 +# PHASE_1_FORK_SLOT = PHASE_1_FORK_EPOCH * SLOTS_PER_EPOCH +PHASE_1_FORK_SLOT: 64 diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 83f9a2145..9df365124 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -289,18 +289,20 @@ def build_phase0_spec(phase0_sourcefile: str, fork_choice_sourcefile: str, return spec -def build_phase1_spec(phase0_sourcefile: str, - fork_choice_sourcefile: str, +def build_phase1_spec(phase0_beacon_sourcefile: str, + phase0_fork_choice_sourcefile: str, + merkle_proofs_sourcefile: str, phase1_custody_sourcefile: str, phase1_shard_sourcefile: str, - merkle_proofs_sourcefile: str, + phase1_beacon_misc_sourcefile: str, outfile: str=None) -> Optional[str]: all_sourcefiles = ( - phase0_sourcefile, - fork_choice_sourcefile, + phase0_beacon_sourcefile, + phase0_fork_choice_sourcefile, + merkle_proofs_sourcefile, phase1_custody_sourcefile, phase1_shard_sourcefile, - merkle_proofs_sourcefile, + phase1_beacon_misc_sourcefile, ) all_spescs = [get_spec(spec) for spec in all_sourcefiles] for spec in all_spescs: @@ -327,10 +329,11 @@ If building phase 0: If building phase 1: 1st argument is input /core/0_beacon-chain.md 2nd argument is input /core/0_fork-choice.md - 3rd argument is input /core/1_custody-game.md - 4th argument is input /core/1_shard-data-chains.md - 5th argument is input /light_client/merkle_proofs.md - 6th argument is output spec.py + 3rd argument is input /light_client/merkle_proofs.md + 4th argument is input /core/1_custody-game.md + 5th argument is input /core/1_shard-data-chains.md + 6th argument is input /core/1_beacon-chain-misc.md + 7th argument is output spec.py ''' parser = ArgumentParser(description=description) parser.add_argument("-p", "--phase", dest="phase", type=int, default=0, help="Build for phase #") @@ -343,14 +346,14 @@ If building phase 1: else: print(" Phase 0 requires spec, forkchoice, and v-guide inputs as well as an output file.") elif args.phase == 1: - if len(args.files) == 6: + if len(args.files) == 7: build_phase1_spec(*args.files) else: print( " Phase 1 requires input files as well as an output file:\n" "\t core/phase_0: (0_beacon-chain.md, 0_fork-choice.md)\n" - "\t core/phase_1: (1_custody-game.md, 1_shard-data-chains.md)\n" "\t light_client: (merkle_proofs.md)\n" + "\t core/phase_1: (1_custody-game.md, 1_shard-data-chains.md, 1_beacon-chain-misc.md)\n" "\t and output.py" ) else: diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 4970a3b00..96e76c9ce 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1196,6 +1196,7 @@ def process_epoch(state: BeaconState) -> None: # @process_reveal_deadlines # @process_challenge_deadlines process_slashings(state) + # @update_period_committee process_final_updates(state) # @after_process_final_updates ``` @@ -1549,6 +1550,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: (body.deposits, process_deposit), (body.voluntary_exits, process_voluntary_exit), (body.transfers, process_transfer), + # @process_shard_receipt_proofs ): for operation in operations: function(state, operation) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md new file mode 100644 index 000000000..5bb0f6da0 --- /dev/null +++ b/specs/core/1_beacon-chain-misc.md @@ -0,0 +1,238 @@ +# Phase 1 miscellaneous beacon chain changes + +## Table of contents + + + +- [Phase 1 miscellaneous beacon chain changes](#phase-1-miscellaneous-beacon-chain-changes) + - [Table of contents](#table-of-contents) + - [Configuration](#configuration) + - [Containers](#containers) + - [`CompactCommittee`](#compactcommittee) + - [`ShardReceiptProof`](#shardreceiptproof) + - [Helper functions](#helper-functions) + - [`pack_compact_validator`](#pack_compact_validator) + - [`unpack_compact_validator`](#unpack_compact_validator) + - [`committee_to_compact_committee`](#committee_to_compact_committee) + - [`verify_merkle_proof`](#verify_merkle_proof) + - [`compute_historical_state_generalized_index`](#compute_historical_state_generalized_index) + - [`get_generalized_index_of_crosslink_header`](#get_generalized_index_of_crosslink_header) + - [`process_shard_receipt_proof`](#process_shard_receipt_proof) + - [Changes](#changes) + - [Phase 0 container updates](#phase-0-container-updates) + - [`BeaconState`](#beaconstate) + - [`BeaconBlockBody`](#beaconblockbody) + - [Persistent committees](#persistent-committees) + - [Shard receipt processing](#shard-receipt-processing) + + + +## Configuration + +| Name | Value | Unit | Duration +| - | - | - | - | +| `MAX_SHARD_RECEIPT_PROOFS` | `2**0` (= 1) | - | - | +| `PERIOD_COMMITTEE_ROOT_LENGTH` | `2**8` (= 256) | periods | ~9 months | +| `MINOR_REWARD_QUOTIENT` | `2**8` (=256) | - | - | + +## Containers + +#### `CompactCommittee` + +```python +class CompactCommittee(Container): + pubkeys: List[BLSPubkey, MAX_VALIDATORS_PER_COMMITTEE] + compact_validators: List[uint64, MAX_VALIDATORS_PER_COMMITTEE] +``` + +#### `ShardReceiptProof` + +```python +class ShardReceiptProof(Container): + shard: Shard + proof: List[Hash, PLACEHOLDER] + receipt: List[ShardReceiptDelta, PLACEHOLDER] +``` + +## Helper functions + +#### `pack_compact_validator` + +```python +def pack_compact_validator(index: int, slashed: bool, balance_in_increments: int) -> int: + """ + Creates a compact validator object representing index, slashed status, and compressed balance. + Takes as input balance-in-increments (// EFFECTIVE_BALANCE_INCREMENT) to preserve symmetry with + the unpacking function. + """ + return (index << 16) + (slashed << 15) + balance_in_increments +``` + +#### `unpack_compact_validator` + +```python +def unpack_compact_validator(compact_validator: int) -> Tuple[int, bool, int]: + """ + Returns validator index, slashed, balance // EFFECTIVE_BALANCE_INCREMENT + """ + return compact_validator >> 16, bool((compact_validator >> 15) % 2), compact_validator & (2**15 - 1) +``` + +#### `committee_to_compact_committee` + +```python +def committee_to_compact_committee(state: BeaconState, committee: Sequence[ValidatorIndex]) -> CompactCommittee: + """ + Given a state and a list of validator indices, outputs the CompactCommittee representing them. + """ + validators = [state.validators[i] for i in committee] + compact_validators = [ + pack_compact_validator(i, v.slashed, v.effective_balance // EFFECTIVE_BALANCE_INCREMENT) + for i, v in zip(committee, validators) + ] + pubkeys = [v.pubkey for v in validators] + return CompactCommittee(pubkeys=pubkeys, compact_validators=compact_validators) +``` + +#### `verify_merkle_proof` + +```python +def verify_merkle_proof(leaf: Hash, proof: Sequence[Hash], index: GeneralizedIndex, root: Hash) -> bool: + assert len(proof) == get_generalized_index_length(index) + for i, h in enumerate(proof): + if get_generalized_index_bit(index, i): + leaf = hash(h + leaf) + else: + leaf = hash(leaf + h) + return leaf == root +``` + +#### `compute_historical_state_generalized_index` + +```python +def compute_historical_state_generalized_index(earlier: ShardSlot, later: ShardSlot) -> GeneralizedIndex: + """ + Computes the generalized index of the state root of slot `frm` based on the state root of slot `to`. + Relies on the `history_acc` in the `ShardState`, where `history_acc[i]` maintains the most recent 2**i'th + slot state. Works by tracing a `log(later-earlier)` step path from `later` to `earlier` through intermediate + blocks at the next available multiples of descending powers of two. + """ + o = GeneralizedIndex(1) + for i in range(HISTORY_ACCUMULATOR_VECTOR - 1, -1, -1): + if (later - 1) & 2**i > (earlier - 1) & 2**i: + later = later - ((later - 1) % 2**i) - 1 + o = concat_generalized_indices(o, GeneralizedIndex(get_generalized_index(ShardState, ['history_acc', i]))) + return o +``` + +#### `get_generalized_index_of_crosslink_header` + +```python +def get_generalized_index_of_crosslink_header(index: int) -> GeneralizedIndex: + """ + Gets the generalized index for the root of the index'th header in a crosslink. + """ + MAX_CROSSLINK_SIZE = ( + SHARD_BLOCK_SIZE_LIMIT * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK + ) + assert MAX_CROSSLINK_SIZE == get_previous_power_of_two(MAX_CROSSLINK_SIZE) + return GeneralizedIndex(MAX_CROSSLINK_SIZE // SHARD_HEADER_SIZE + index) +``` + +#### `process_shard_receipt_proof` + +```python +def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptProof) -> None: + """ + Processes a ShardReceipt object. + """ + SHARD_SLOTS_PER_EPOCH = SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH + receipt_slot = ( + state.next_shard_receipt_period[receipt_proof.shard] * + SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD + ) + first_slot_in_last_crosslink = state.current_crosslinks[receipt_proof.shard].start_epoch * SHARD_SLOTS_PER_EPOCH + gindex = concat_generalized_indices( + get_generalized_index_of_crosslink_header(0), + GeneralizedIndex(get_generalized_index(ShardBlockHeader, 'state_root')), + compute_historical_state_generalized_index(receipt_slot, first_slot_in_last_crosslink), + GeneralizedIndex(get_generalized_index(ShardState, 'receipt_root')) + ) + assert verify_merkle_proof( + leaf=hash_tree_root(receipt_proof.receipt), + proof=receipt_proof.proof, + index=gindex, + root=state.current_crosslinks[receipt_proof.shard].data_root + ) + for delta in receipt_proof.receipt: + if get_current_epoch(state) < state.validators[delta.index].withdrawable_epoch: + increase_amount = ( + state.validators[delta.index].effective_balance * delta.reward_coefficient // REWARD_COEFFICIENT_BASE + ) + increase_balance(state, delta.index, increase_amount) + decrease_balance(state, delta.index, delta.block_fee) + state.next_shard_receipt_period[receipt_proof.shard] += 1 + proposer_index = get_beacon_proposer_index(state) + increase_balance(state, proposer_index, Gwei(get_base_reward(state, proposer_index) // MINOR_REWARD_QUOTIENT)) +``` + +## Changes + +### Phase 0 container updates + +Add the following fields to the end of the specified container objects. + +#### `BeaconState` + +```python +class BeaconState(Container): + # Period committees + period_committee_roots: Vector[Hash, PERIOD_COMMITTEE_ROOT_LENGTH] + next_shard_receipt_period: Vector[uint64, SHARD_COUNT] +``` + +`period_committee_roots` values are initialized to `Bytes32()` (empty bytes value). +`next_shard_receipt_period` values are initialized to `compute_epoch_of_slot(PHASE_1_FORK_SLOT) // EPOCHS_PER_SHARD_PERIOD`. + +#### `BeaconBlockBody` + +```python +class BeaconBlockBody(Container): + shard_receipt_proofs: List[ShardReceiptProof, MAX_SHARD_RECEIPT_PROOFS] +``` + +`shard_receipt_proofs` is initialized to `[]`. + +### Persistent committees + +Run `update_period_committee` immediately before `process_final_updates`: + +```python +# begin insert @update_period_committee + update_period_committee(state) +# end insert @update_period_committee +def update_period_committee(state: BeaconState) -> None: + """ + Updates period committee roots at boundary blocks. + """ + if (get_current_epoch(state) + 1) % EPOCHS_PER_SHARD_PERIOD == 0: + period = (get_current_epoch(state) + 1) // EPOCHS_PER_SHARD_PERIOD + committees = Vector[CompactCommittee, SHARD_COUNT]([ + committee_to_compact_committee( + state, + get_period_committee(state, Epoch(get_current_epoch(state) + 1), Shard(shard)), + ) + for shard in range(SHARD_COUNT) + ]) + state.period_committee_roots[period % PERIOD_COMMITTEE_ROOT_LENGTH] = hash_tree_root(committees) +``` + +### Shard receipt processing + +Run `process_shard_receipt_proof` on each `ShardReceiptProof` during block processing. + +```python +# begin insert @process_shard_receipt_proofs + (body.shard_receipt_proofs, process_shard_receipt_proof), +# end insert @process_shard_receipt_proofs +``` diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 8e1532f17..3dc549816 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -73,10 +73,10 @@ We define the following Python custom types for type hinting and readability: ### Initial values -| Name | Value | -| - | - | -| `PHASE_1_FORK_EPOCH` | **TBD** | -| `PHASE_1_FORK_SLOT` | **TBD** | +| Name | Value | Unit | +| - | - | - | +| `PHASE_1_FORK_EPOCH` | **TBD** | Epoch | +| `PHASE_1_FORK_SLOT` | **TBD** | Slot | ### Time parameters @@ -359,7 +359,7 @@ def get_default_shard_state(beacon_state: BeaconState, shard: Shard) -> ShardSta return ShardState( basefee=1, shard=shard, - slot=PHASE_1_FORK_SLOT, + slot=PHASE_1_FORK_SLOT * SHARD_SLOTS_PER_BEACON_SLOT, earlier_committee_rewards=[REWARD_COEFFICIENT_BASE for _ in range(len(earlier_committee))], later_committee_rewards=[REWARD_COEFFICIENT_BASE for _ in range(len(later_committee))], earlier_committee_fees=[Gwei(0) for _ in range(len(earlier_committee))], diff --git a/specs/light_client/merkle_proofs.md b/specs/light_client/merkle_proofs.md index ce7dc647c..bbd03d379 100644 --- a/specs/light_client/merkle_proofs.md +++ b/specs/light_client/merkle_proofs.md @@ -152,7 +152,7 @@ def get_item_position(typ: SSZType, index_or_variable_name: Union[int, SSZVariab ``` ```python -def get_generalized_index(typ: SSZType, path: Sequence[Union[int, SSZVariableName]]) -> Optional[GeneralizedIndex]: +def get_generalized_index(typ: SSZType, path: Sequence[Union[int, SSZVariableName]]) -> GeneralizedIndex: """ Converts a path (eg. `[7, "foo", 3]` for `x[7].foo[3]`, `[12, "bar", "__len__"]` for `len(x[12].bar)`) into the generalized index representing its position in the Merkle tree. @@ -162,10 +162,8 @@ def get_generalized_index(typ: SSZType, path: Sequence[Union[int, SSZVariableNam assert not issubclass(typ, BasicValue) # If we descend to a basic type, the path cannot continue further if p == '__len__': typ = uint64 - if issubclass(typ, (List, Bytes)): - root = GeneralizedIndex(root * 2 + 1) - else: - return None + assert issubclass(typ, (List, Bytes)) + root = GeneralizedIndex(root * 2 + 1) else: pos, _, _ = get_item_position(typ, p) base_index = (GeneralizedIndex(2) if issubclass(typ, (List, Bytes)) else GeneralizedIndex(1)) @@ -181,7 +179,7 @@ _Usage note: functions outside this section should manipulate generalized indice #### `concat_generalized_indices` ```python -def concat_generalized_indices(indices: Sequence[GeneralizedIndex]) -> GeneralizedIndex: +def concat_generalized_indices(*indices: GeneralizedIndex) -> GeneralizedIndex: """ Given generalized indices i1 for A -> B, i2 for B -> C .... i_n for Y -> Z, returns the generalized index for A -> Z. diff --git a/test_libs/pyspec/eth2spec/test/merkle_proofs/test_merkle_proofs.py b/test_libs/pyspec/eth2spec/test/merkle_proofs/test_merkle_proofs.py index 91c861de3..62a2f6379 100644 --- a/test_libs/pyspec/eth2spec/test/merkle_proofs/test_merkle_proofs.py +++ b/test_libs/pyspec/eth2spec/test/merkle_proofs/test_merkle_proofs.py @@ -1,10 +1,10 @@ - import re from eth_utils import ( to_tuple, ) from eth2spec.test.context import ( + expect_assertion_error, spec_state_test, with_all_phases_except, ) @@ -89,10 +89,14 @@ generalized_index_cases = [ @spec_state_test def test_get_generalized_index(spec, state): for typ, path, generalized_index in generalized_index_cases: - assert spec.get_generalized_index( - typ=typ, - path=path, - ) == generalized_index + if generalized_index is not None: + assert spec.get_generalized_index( + typ=typ, + path=path, + ) == generalized_index + else: + expect_assertion_error(lambda: spec.get_generalized_index(typ=typ, path=path)) + yield 'typ', typ yield 'path', path yield 'generalized_index', generalized_index