From 3dc7430ae98d8f540dae3bbb4e9a2dcb31a2dbf2 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sat, 27 Jul 2019 08:17:06 -0400 Subject: [PATCH 01/89] Starting on phase 1 misc beacon changes --- specs/core/1_beacon_chain_misc.md | 75 +++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 specs/core/1_beacon_chain_misc.md diff --git a/specs/core/1_beacon_chain_misc.md b/specs/core/1_beacon_chain_misc.md new file mode 100644 index 000000000..09e3c2c61 --- /dev/null +++ b/specs/core/1_beacon_chain_misc.md @@ -0,0 +1,75 @@ +# Phase 1 miscellaneous beacon chain changes + +## Table of contents + + + +- [Helpers](#helpers) + - [pack_compact_validator](#pack_compact_validator) + - [unpack_compact_validator](#unpack_compact_validator) + - [committee_to_compact_committee](#committee_to_compact_committee) +- [Changes](#changes) + - [Persistent committees](#persistent-committees) + + + +## Helpers + +#### `pack_compact_validator` + +```python +def pack_compact_validator(index: uint64, slashed: bool, balance_in_increments: uint64) -> uint64: + """ + Creates a compact validator object representing index, slashed status, and compressed balance. + Takes as input balance-in-increments (// EFFECTIVE_BALANCE_INCREMENT) to preserve symmetry with + the unpacking function. + """ + return (index << 16) + (slashed << 15) + balance_in_increments +``` + +### `unpack_compact_validator` + +```python +def unpack_compact_validator(compact_validator: uint64) -> Tuple[uint64, bool, uint64]: + """ + Returns validator index, slashed, balance // EFFECTIVE_BALANCE_INCREMENT + """ + return compact_validator >> 16, (compact_validator >> 15) % 2, compact_validator & (2**15 - 1) +``` + +#### `committee_to_compact_committee` + +```python +def committee_to_compact_committee(state: BeaconState, committee: Sequence[ValidatorIndex]) -> CompactCommittee: + validators = [state.validators[i] for i in committee] + compact_validators = [ + pack_compact_validator(i, v.slashed, v.effective_balance // EFFECTIVE_BALANCE_INCREMENT) + for i, v in zip(committee, validators) + ] + pubkeys = [v.pubkey for v in validators] + return CompactCommittee(pubkeys=pubkeys, compact_validators=compact_validators) +``` + +## Changes + +### Persistent committees + +Add to the beacon state the following fields: + +* `previous_persistent_committee_root: Hash` +* `current_persistent_committee_root: Hash` +* `next_persistent_committee_root: Hash` + +Process the following function before `process_final_updates`: + +```python +def update_persistent_committee(state: BeaconState): + if (get_current_epoch(state) + 1) % EPOCHS_PER_SHARD_PERIOD == 0: + state.previous_persistent_committee_root = state.current_persistent_committee_root + state.current_persistent_committee_root = state.next_persistent_committee_root + committees = Vector[CompactCommittee, SHARD_COUNT]([ + committee_to_compact_committee(state, get_period_committee(state, get_current_epoch(state) + 1, i)) + for i in range(SHARD_COUNT) + ]) + state.next_persistent_committee_root = hash_tree_root(committees) +``` From fe9fe8a1518ca890df861210f163c82be6c22852 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 30 Jul 2019 10:01:34 -0400 Subject: [PATCH 02/89] [WIP] add receipt processing to phase 1 beacon chain --- specs/core/1_beacon_chain_misc.md | 60 +++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/specs/core/1_beacon_chain_misc.md b/specs/core/1_beacon_chain_misc.md index 09e3c2c61..b5b76a9a9 100644 --- a/specs/core/1_beacon_chain_misc.md +++ b/specs/core/1_beacon_chain_misc.md @@ -50,6 +50,65 @@ def committee_to_compact_committee(state: BeaconState, committee: Sequence[Valid return CompactCommittee(pubkeys=pubkeys, compact_validators=compact_validators) ``` +#### `get_previous_power_of_2` + +```python +def get_previous_power_of_2(x: int) -> int: + return x if x <= 2 else 2 * get_previous_power_of_2(x // 2) +``` + + +#### `concat_generalized_indices` + +```python +def concat_generalized_indices(*indices: Sequence[GeneralizedIndex]) -> GeneralizedIndex: + o = GeneralizedIndex(1) + for i in indices: + o = o * get_previous_power_of_2(i) + i + return o +``` + +#### `compute_historical_state_generalized_index` + +```python +def compute_historical_state_generalized_index(frm: ShardSlot, to: ShardSlot) -> GeneralizedIndex: + o = GeneralizedIndex(1) + for i in range(63, -1, -1): + if (to-1) & 2**i > (frm-1) & 2**i: + to = to - ((to-1) % 2**i) - 1 + o = concat_generalized_indices(o, get_generalized_index(ShardState, 'history_acc', i)) + return o +``` + +#### `get_generalized_index_of_crosslink_header` + +```python +def get_generalized_index_of_crosslink_header(index: int) -> GeneralizedIndex: + MAX_CROSSLINK_SIZE = SHARD_BLOCK_SIZE_LIMIT * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK + assert MAX_CROSSLINK_SIZE == get_previous_power_of_2(MAX_CROSSLINK_SIZE) + return GeneralizedIndex(MAX_CROSSLINK_SIZE // SHARD_HEADER_SIZE + index) +``` + +#### `process_shard_receipt` + +```python +def process_shard_receipt(state: BeaconState, shard: Shard, proof: List[Hash, PLACEHOLDER], receipt: List[ShardReceiptDelta, PLACEHOLDER]): + receipt_slot = state.next_shard_receipt_period[shard] * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD + first_slot_in_last_crosslink = state.current_crosslinks[shard].start_epoch * SLOTS_PER_EPOCH + gindex = concat_generalized_indices( + get_generalized_index_of_crosslink_header(0), + get_generalized_index(ShardBlockHeader, 'state_root') + compute_historical_state_generalized_index(receipt_slot, first_slot_in_last_crosslink) + get_generalized_index(ShardState, 'receipt_root') + ) + assert verify_merkle_proof( + leaf=hash_tree_root(receipt), + proof=proof, + index=gindex, + root=state.current_crosslinks[shard].data_root + ) +``` + ## Changes ### Persistent committees @@ -59,6 +118,7 @@ Add to the beacon state the following fields: * `previous_persistent_committee_root: Hash` * `current_persistent_committee_root: Hash` * `next_persistent_committee_root: Hash` +* `next_shard_receipt_period: Vector[uint, SHARD_COUNT]`, values initialized to `PHASE_1_FORK_SLOT // SLOTS_PER_EPOCH // EPOCHS_PER_SHARD_PERIOD` Process the following function before `process_final_updates`: From caadc0d2349b464588b9c57a168bb58e913d2b12 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 30 Jul 2019 10:15:18 -0400 Subject: [PATCH 03/89] Update 1_beacon_chain_misc.md --- specs/core/1_beacon_chain_misc.md | 36 +++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/specs/core/1_beacon_chain_misc.md b/specs/core/1_beacon_chain_misc.md index b5b76a9a9..9e81acadf 100644 --- a/specs/core/1_beacon_chain_misc.md +++ b/specs/core/1_beacon_chain_misc.md @@ -13,6 +13,17 @@ +## Classes + +#### `ShardReceiptProof` + +```python +class ShardReceiptProof(Container): + shard: Shard + proof: List[Hash, PLACEHOLDER] + receipt: List[ShardReceiptDelta, PLACEHOLDER] +``` + ## Helpers #### `pack_compact_validator` @@ -27,7 +38,7 @@ def pack_compact_validator(index: uint64, slashed: bool, balance_in_increments: return (index << 16) + (slashed << 15) + balance_in_increments ``` -### `unpack_compact_validator` +#### `unpack_compact_validator` ```python def unpack_compact_validator(compact_validator: uint64) -> Tuple[uint64, bool, uint64]: @@ -92,9 +103,9 @@ def get_generalized_index_of_crosslink_header(index: int) -> GeneralizedIndex: #### `process_shard_receipt` ```python -def process_shard_receipt(state: BeaconState, shard: Shard, proof: List[Hash, PLACEHOLDER], receipt: List[ShardReceiptDelta, PLACEHOLDER]): - receipt_slot = state.next_shard_receipt_period[shard] * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD - first_slot_in_last_crosslink = state.current_crosslinks[shard].start_epoch * SLOTS_PER_EPOCH +def process_shard_receipt(state: BeaconState, receipt_proof: ShardReceiptProof): + receipt_slot = state.next_shard_receipt_period[receipt_proof.shard] * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD + first_slot_in_last_crosslink = state.current_crosslinks[receipt_proof.shard].start_epoch * SLOTS_PER_EPOCH gindex = concat_generalized_indices( get_generalized_index_of_crosslink_header(0), get_generalized_index(ShardBlockHeader, 'state_root') @@ -102,11 +113,16 @@ def process_shard_receipt(state: BeaconState, shard: Shard, proof: List[Hash, PL get_generalized_index(ShardState, 'receipt_root') ) assert verify_merkle_proof( - leaf=hash_tree_root(receipt), - proof=proof, + leaf=hash_tree_root(receipt_proof.receipt), + proof=receipt_proof.proof, index=gindex, root=state.current_crosslinks[shard].data_root ) + for delta in receipt_proof.receipt: + increase_balance(state, delta.index, state.validators[delta.index].effective_balance * delta.reward_coefficient // REWARD_COEFFICIENT_BASE) + decrease_balance(state, delta.index, delta.block_fee) + state.next_shard_receipt_period[receipt_proof.shard] += 1 + increase_balance(state, get_beacon_proposer_index(state), MICRO_REWARD) ``` ## Changes @@ -133,3 +149,11 @@ def update_persistent_committee(state: BeaconState): ]) state.next_persistent_committee_root = hash_tree_root(committees) ``` + +### Shard receipt processing + +Add to the beacon block body the following object: + +* `shard_receipts: List[ShardReceipt, MAX_SHARD_RECEIPTS]` + +Use `process_shard_receipt` to process each receipt. From 3f20aca65ee4759f72c865cfc6a5338a48189bf9 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 30 Jul 2019 11:55:58 -0400 Subject: [PATCH 04/89] Added comments and ToC --- specs/core/1_beacon_chain_misc.md | 49 +++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/specs/core/1_beacon_chain_misc.md b/specs/core/1_beacon_chain_misc.md index 9e81acadf..ceb514878 100644 --- a/specs/core/1_beacon_chain_misc.md +++ b/specs/core/1_beacon_chain_misc.md @@ -4,12 +4,21 @@ +- [Classes](#classes) + - [ShardReceiptProof](#shardreceiptproof) - [Helpers](#helpers) - [pack_compact_validator](#pack_compact_validator) - [unpack_compact_validator](#unpack_compact_validator) - [committee_to_compact_committee](#committee_to_compact_committee) + - [get_previous_power_of_2](#get_previous_power_of_2) + - [verify_merkle_proof](#verify_merkle_proof) + - [concat_generalized_indices](#concat_generalized_indices) + - [compute_historical_state_generalized_index](#compute_historical_state_generalized_index) + - [get_generalized_index_of_crosslink_header](#get_generalized_index_of_crosslink_header) + - [process_shard_receipt](#process_shard_receipt) - [Changes](#changes) - [Persistent committees](#persistent-committees) + - [Shard receipt processing](#shard-receipt-processing) @@ -52,6 +61,9 @@ def unpack_compact_validator(compact_validator: uint64) -> Tuple[uint64, bool, u ```python def committee_to_compact_committee(state: BeaconState, committee: Sequence[ValidatorIndex]) -> CompactCommittee: + """ + Given a state and a list of validator indices, outputs the CompactCommittee representing them. + """ validators = [state.validators[i] for i in committee] compact_validators = [ pack_compact_validator(i, v.slashed, v.effective_balance // EFFECTIVE_BALANCE_INCREMENT) @@ -68,11 +80,27 @@ def get_previous_power_of_2(x: int) -> int: return x if x <= 2 else 2 * get_previous_power_of_2(x // 2) ``` +#### `verify_merkle_proof` + +```python +def verify_merkle_proof(leaf: Hash, proof: Sequence[Hash], index: GeneralizedIndex, root: Hash) -> bool: + assert len(proof) == log2(index) + for i, h in enumerate(proof): + if index & 2**i: + leaf = hash(h + leaf) + else: + leaf = hash(leaf + h) + return leaf == root +``` #### `concat_generalized_indices` ```python def concat_generalized_indices(*indices: Sequence[GeneralizedIndex]) -> GeneralizedIndex: + """ + Given generalized indices i1 for A -> B, i2 for B -> C .... i_n for Y -> Z, returns + the generalized index for A -> Z. + """ o = GeneralizedIndex(1) for i in indices: o = o * get_previous_power_of_2(i) + i @@ -82,11 +110,17 @@ def concat_generalized_indices(*indices: Sequence[GeneralizedIndex]) -> Generali #### `compute_historical_state_generalized_index` ```python -def compute_historical_state_generalized_index(frm: ShardSlot, to: ShardSlot) -> GeneralizedIndex: +def compute_historical_state_generalized_index(earlier: ShardSlot, later: ShardSlot) -> GeneralizedIndex: + """ + Computes the generalized index of the state root of slot `frm` based on the state root of slot `to`. + Relies on the `history_acc` in the `ShardState`, where `history_acc[i]` maintains the most recent 2**i'th + slot state. Works by tracing a `log(later-earlier)` step path from `later` to `earlier` through intermediate + blocks at the next available multiples of descending powers of two. + """ o = GeneralizedIndex(1) for i in range(63, -1, -1): - if (to-1) & 2**i > (frm-1) & 2**i: - to = to - ((to-1) % 2**i) - 1 + if (later-1) & 2**i > (earlier-1) & 2**i: + later = later - ((later-1) % 2**i) - 1 o = concat_generalized_indices(o, get_generalized_index(ShardState, 'history_acc', i)) return o ``` @@ -95,6 +129,9 @@ def compute_historical_state_generalized_index(frm: ShardSlot, to: ShardSlot) -> ```python def get_generalized_index_of_crosslink_header(index: int) -> GeneralizedIndex: + """ + Gets the generalized index for the root of the index'th header in a crosslink. + """ MAX_CROSSLINK_SIZE = SHARD_BLOCK_SIZE_LIMIT * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK assert MAX_CROSSLINK_SIZE == get_previous_power_of_2(MAX_CROSSLINK_SIZE) return GeneralizedIndex(MAX_CROSSLINK_SIZE // SHARD_HEADER_SIZE + index) @@ -104,6 +141,9 @@ def get_generalized_index_of_crosslink_header(index: int) -> GeneralizedIndex: ```python def process_shard_receipt(state: BeaconState, receipt_proof: ShardReceiptProof): + """ + Processes a ShardReceipt object. + """ receipt_slot = state.next_shard_receipt_period[receipt_proof.shard] * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD first_slot_in_last_crosslink = state.current_crosslinks[receipt_proof.shard].start_epoch * SLOTS_PER_EPOCH gindex = concat_generalized_indices( @@ -140,6 +180,9 @@ Process the following function before `process_final_updates`: ```python def update_persistent_committee(state: BeaconState): + """ + Updates persistent committee roots at boundary blocks. + """ if (get_current_epoch(state) + 1) % EPOCHS_PER_SHARD_PERIOD == 0: state.previous_persistent_committee_root = state.current_persistent_committee_root state.current_persistent_committee_root = state.next_persistent_committee_root From 5dad213457636a9a6d17002820a3559c6015076b Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 30 Jul 2019 11:58:33 -0400 Subject: [PATCH 05/89] Fixed up ToC --- specs/core/1_beacon_chain_misc.md | 32 ++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/specs/core/1_beacon_chain_misc.md b/specs/core/1_beacon_chain_misc.md index ceb514878..3798d1d40 100644 --- a/specs/core/1_beacon_chain_misc.md +++ b/specs/core/1_beacon_chain_misc.md @@ -4,21 +4,23 @@ -- [Classes](#classes) - - [ShardReceiptProof](#shardreceiptproof) -- [Helpers](#helpers) - - [pack_compact_validator](#pack_compact_validator) - - [unpack_compact_validator](#unpack_compact_validator) - - [committee_to_compact_committee](#committee_to_compact_committee) - - [get_previous_power_of_2](#get_previous_power_of_2) - - [verify_merkle_proof](#verify_merkle_proof) - - [concat_generalized_indices](#concat_generalized_indices) - - [compute_historical_state_generalized_index](#compute_historical_state_generalized_index) - - [get_generalized_index_of_crosslink_header](#get_generalized_index_of_crosslink_header) - - [process_shard_receipt](#process_shard_receipt) -- [Changes](#changes) - - [Persistent committees](#persistent-committees) - - [Shard receipt processing](#shard-receipt-processing) +- [Phase 1 miscellaneous beacon chain changes](#phase-1-miscellaneous-beacon-chain-changes) + - [Table of contents](#table-of-contents) + - [Classes](#classes) + - [ShardReceiptProof](#shardreceiptproof) + - [Helpers](#helpers) + - [pack_compact_validator](#pack_compact_validator) + - [unpack_compact_validator](#unpack_compact_validator) + - [committee_to_compact_committee](#committee_to_compact_committee) + - [get_previous_power_of_2](#get_previous_power_of_2) + - [verify_merkle_proof](#verify_merkle_proof) + - [concat_generalized_indices](#concat_generalized_indices) + - [compute_historical_state_generalized_index](#compute_historical_state_generalized_index) + - [get_generalized_index_of_crosslink_header](#get_generalized_index_of_crosslink_header) + - [process_shard_receipt](#process_shard_receipt) + - [Changes](#changes) + - [Persistent committees](#persistent-committees) + - [Shard receipt processing](#shard-receipt-processing) From fab37e747a44d79f75b08d869b35fda972421bba Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 30 Jul 2019 11:59:12 -0400 Subject: [PATCH 06/89] Fixed position of Classes --- specs/core/1_beacon_chain_misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon_chain_misc.md b/specs/core/1_beacon_chain_misc.md index 3798d1d40..673f99ce4 100644 --- a/specs/core/1_beacon_chain_misc.md +++ b/specs/core/1_beacon_chain_misc.md @@ -6,7 +6,7 @@ - [Phase 1 miscellaneous beacon chain changes](#phase-1-miscellaneous-beacon-chain-changes) - [Table of contents](#table-of-contents) - - [Classes](#classes) + - [Classes](#classes) - [ShardReceiptProof](#shardreceiptproof) - [Helpers](#helpers) - [pack_compact_validator](#pack_compact_validator) From b9fddfe3103a77fb7502af40f2c0755aea65431d Mon Sep 17 00:00:00 2001 From: vbuterin Date: Thu, 1 Aug 2019 10:58:15 -0400 Subject: [PATCH 07/89] Made code work with #1186 --- specs/core/1_beacon_chain_misc.md | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/specs/core/1_beacon_chain_misc.md b/specs/core/1_beacon_chain_misc.md index 673f99ce4..4aa1796b6 100644 --- a/specs/core/1_beacon_chain_misc.md +++ b/specs/core/1_beacon_chain_misc.md @@ -86,29 +86,15 @@ def get_previous_power_of_2(x: int) -> int: ```python def verify_merkle_proof(leaf: Hash, proof: Sequence[Hash], index: GeneralizedIndex, root: Hash) -> bool: - assert len(proof) == log2(index) + assert len(proof) == get_generalized_index_length(index) for i, h in enumerate(proof): - if index & 2**i: + if get_generalized_index_bit(index, i): leaf = hash(h + leaf) else: leaf = hash(leaf + h) return leaf == root ``` -#### `concat_generalized_indices` - -```python -def concat_generalized_indices(*indices: Sequence[GeneralizedIndex]) -> GeneralizedIndex: - """ - Given generalized indices i1 for A -> B, i2 for B -> C .... i_n for Y -> Z, returns - the generalized index for A -> Z. - """ - o = GeneralizedIndex(1) - for i in indices: - o = o * get_previous_power_of_2(i) + i - return o -``` - #### `compute_historical_state_generalized_index` ```python From a273d9e09d27238f9617394ef3e3572915af0b39 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 1 Aug 2019 09:13:09 -0600 Subject: [PATCH 08/89] minor rename of beacon chain misc to conform to other files --- specs/core/{1_beacon_chain_misc.md => 1_beacon-chain-misc.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename specs/core/{1_beacon_chain_misc.md => 1_beacon-chain-misc.md} (100%) diff --git a/specs/core/1_beacon_chain_misc.md b/specs/core/1_beacon-chain-misc.md similarity index 100% rename from specs/core/1_beacon_chain_misc.md rename to specs/core/1_beacon-chain-misc.md From 2ae7323183fca93557fee81ed8bb5ad290585d2a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 1 Aug 2019 09:42:39 -0600 Subject: [PATCH 09/89] use codeblock syntax for misc beacon updates --- specs/core/0_beacon-chain.md | 5 +++++ specs/core/1_beacon-chain-misc.md | 34 +++++++++++++++++++++++-------- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f0169f1d2..16d8048aa 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -480,6 +480,7 @@ class BeaconBlockBody(Container): deposits: List[Deposit, MAX_DEPOSITS] voluntary_exits: List[VoluntaryExit, MAX_VOLUNTARY_EXITS] transfers: List[Transfer, MAX_TRANSFERS] + # @shard_receipts ``` #### `BeaconBlock` @@ -533,6 +534,8 @@ class BeaconState(Container): previous_justified_checkpoint: Checkpoint # Previous epoch snapshot current_justified_checkpoint: Checkpoint finalized_checkpoint: Checkpoint + + # @persistent_committee_fields ``` ## Helper functions @@ -1237,6 +1240,7 @@ def process_epoch(state: BeaconState) -> None: # @process_reveal_deadlines # @process_challenge_deadlines process_slashings(state) + # @update_persistent_committee process_final_updates(state) # @after_process_final_updates ``` @@ -1598,6 +1602,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: (body.deposits, process_deposit), (body.voluntary_exits, process_voluntary_exit), (body.transfers, process_transfer), + # @process_shard_receipts ): for operation in operations: function(state, operation) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 4aa1796b6..f816c6259 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -159,15 +159,23 @@ def process_shard_receipt(state: BeaconState, receipt_proof: ShardReceiptProof): Add to the beacon state the following fields: -* `previous_persistent_committee_root: Hash` -* `current_persistent_committee_root: Hash` -* `next_persistent_committee_root: Hash` -* `next_shard_receipt_period: Vector[uint, SHARD_COUNT]`, values initialized to `PHASE_1_FORK_SLOT // SLOTS_PER_EPOCH // EPOCHS_PER_SHARD_PERIOD` +```python +# begin insert @persistent_committee_fields + previous_persistent_committee_root: Hash + current_persistent_committee_root: Hash + next_persistent_committee_root: Hash + next_shard_receipt_period: Vector[uint, SHARD_COUNT] +# end insert @persistent_committee_fields +``` +`next_shard_receipt_period` values initialized to `PHASE_1_FORK_SLOT // SLOTS_PER_EPOCH // EPOCHS_PER_SHARD_PERIOD` -Process the following function before `process_final_updates`: +Run `update_persistent_committee` immediately before `process_final_updates`: ```python -def update_persistent_committee(state: BeaconState): +# begin insert @update_persistent_committee + update_persistent_committee(state) +# end insert @update_persistent_committee +def update_persistent_committee(state: BeaconState) -> None: """ Updates persistent committee roots at boundary blocks. """ @@ -183,8 +191,18 @@ def update_persistent_committee(state: BeaconState): ### Shard receipt processing -Add to the beacon block body the following object: +Add the `shard_receipts` operation to `BeaconBlockBody`: -* `shard_receipts: List[ShardReceipt, MAX_SHARD_RECEIPTS]` +```python +# begin insert @shard_receipts + shard_receipts: List[ShardReceipt, MAX_SHARD_RECEIPTS] +# end insert @shard_receipts +``` Use `process_shard_receipt` to process each receipt. + +```python +# begin insert @process_shard_receipts + (body.shard_receipts, process_shard_receipts), +# end insert @process_shard_receipts +``` From 6560bc42d2fc877cddf044c0a993efde74ee7b72 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 1 Aug 2019 13:39:32 -0600 Subject: [PATCH 10/89] Apply suggestions from code review Co-Authored-By: Carl Beekhuizen --- specs/core/1_beacon-chain-misc.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index f816c6259..d1ea48dba 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -30,9 +30,9 @@ ```python class ShardReceiptProof(Container): - shard: Shard - proof: List[Hash, PLACEHOLDER] - receipt: List[ShardReceiptDelta, PLACEHOLDER] + shard: Shard + proof: List[Hash, PLACEHOLDER] + receipt: List[ShardReceiptDelta, PLACEHOLDER] ``` ## Helpers @@ -107,9 +107,9 @@ def compute_historical_state_generalized_index(earlier: ShardSlot, later: ShardS """ o = GeneralizedIndex(1) for i in range(63, -1, -1): - if (later-1) & 2**i > (earlier-1) & 2**i: - later = later - ((later-1) % 2**i) - 1 - o = concat_generalized_indices(o, get_generalized_index(ShardState, 'history_acc', i)) + if (later - 1) & 2**i > (earlier - 1) & 2**i: + later = later - ((later - 1) % 2**i) - 1 + o = concat_generalized_indices(o, get_generalized_index(ShardState, 'history_acc', i)) return o ``` @@ -136,8 +136,8 @@ def process_shard_receipt(state: BeaconState, receipt_proof: ShardReceiptProof): first_slot_in_last_crosslink = state.current_crosslinks[receipt_proof.shard].start_epoch * SLOTS_PER_EPOCH gindex = concat_generalized_indices( get_generalized_index_of_crosslink_header(0), - get_generalized_index(ShardBlockHeader, 'state_root') - compute_historical_state_generalized_index(receipt_slot, first_slot_in_last_crosslink) + get_generalized_index(ShardBlockHeader, 'state_root'), + compute_historical_state_generalized_index(receipt_slot, first_slot_in_last_crosslink), get_generalized_index(ShardState, 'receipt_root') ) assert verify_merkle_proof( From 0fa4491c412040ff3feaa9c2c76c5c07d5f63c69 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 1 Aug 2019 13:45:07 -0600 Subject: [PATCH 11/89] lint --- specs/core/0_beacon-chain.md | 1 - 1 file changed, 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 16d8048aa..5ddceebc1 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -534,7 +534,6 @@ class BeaconState(Container): previous_justified_checkpoint: Checkpoint # Previous epoch snapshot current_justified_checkpoint: Checkpoint finalized_checkpoint: Checkpoint - # @persistent_committee_fields ``` From 7132778a6993d12175c8870c933bcc7f53240848 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 6 Aug 2019 10:14:45 -0400 Subject: [PATCH 12/89] Added compact committee class --- specs/core/1_beacon-chain-misc.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index d1ea48dba..a8768d8b5 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -7,6 +7,7 @@ - [Phase 1 miscellaneous beacon chain changes](#phase-1-miscellaneous-beacon-chain-changes) - [Table of contents](#table-of-contents) - [Classes](#classes) + - [CompactCommittee](#compactcommittee) - [ShardReceiptProof](#shardreceiptproof) - [Helpers](#helpers) - [pack_compact_validator](#pack_compact_validator) @@ -26,6 +27,14 @@ ## Classes +#### `CompactCommittee` + +```python +class CompactCommittee(Container): + pubkeys: List[BLSPubkey, MAX_VALIDATORS_PER_COMMITTEE] + compact_validators: List[uint64, MAX_VALIDATORS_PER_COMMITTEE] +``` + #### `ShardReceiptProof` ```python From 36a22830526fed478a143c85c6cb2e4277b957b2 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 7 Aug 2019 12:13:02 -0400 Subject: [PATCH 13/89] Shard receipts cannot penalize withdrawn validators --- specs/core/1_beacon-chain-misc.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index a8768d8b5..c1d6283ca 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -156,8 +156,9 @@ def process_shard_receipt(state: BeaconState, receipt_proof: ShardReceiptProof): root=state.current_crosslinks[shard].data_root ) for delta in receipt_proof.receipt: - increase_balance(state, delta.index, state.validators[delta.index].effective_balance * delta.reward_coefficient // REWARD_COEFFICIENT_BASE) - decrease_balance(state, delta.index, delta.block_fee) + if get_current_epoch(state) < state.validators[delta.index].withdrawable_epoch: + increase_balance(state, delta.index, state.validators[delta.index].effective_balance * delta.reward_coefficient // REWARD_COEFFICIENT_BASE) + decrease_balance(state, delta.index, delta.block_fee) state.next_shard_receipt_period[receipt_proof.shard] += 1 increase_balance(state, get_beacon_proposer_index(state), MICRO_REWARD) ``` From c5acddc071fd64603a0ed5ca3442b25442fb203b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sun, 11 Aug 2019 22:47:23 +0800 Subject: [PATCH 14/89] Enable it in CI --- Makefile | 2 +- scripts/build_spec.py | 27 +++++++++++++++------------ 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 318056689..3f6cf9816 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ $(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS) python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $(SPEC_DIR)/validator/0_beacon-chain-validator.md $@ $(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS) - python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $(SPEC_DIR)/core/0_fork-choice.md $@ + python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/1_beacon-chain-misc.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $(SPEC_DIR)/core/0_fork-choice.md $@ CURRENT_DIR = ${CURDIR} diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 96866cc8a..5c353ede0 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -263,16 +263,18 @@ def build_phase0_spec(phase0_sourcefile: str, fork_choice_sourcefile: str, def build_phase1_spec(phase0_sourcefile: str, + phase1_phase1_shard_misc_source_file: str, phase1_custody_sourcefile: str, phase1_shard_sourcefile: str, fork_choice_sourcefile: str, outfile: str=None) -> Optional[str]: phase0_spec = get_spec(phase0_sourcefile) + phase1_shard_misc = get_spec(phase1_phase1_shard_misc_source_file) phase1_custody = get_spec(phase1_custody_sourcefile) phase1_shard_data = get_spec(phase1_shard_sourcefile) fork_choice_spec = get_spec(fork_choice_sourcefile) spec_objects = phase0_spec - for value in [phase1_custody, phase1_shard_data, fork_choice_spec]: + for value in [phase1_shard_misc, phase1_custody, phase1_shard_data, fork_choice_spec]: spec_objects = combine_spec_objects(spec_objects, value) spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS) if outfile is not None: @@ -285,17 +287,18 @@ if __name__ == '__main__': description = ''' Build the specs from the md docs. If building phase 0: - 1st argument is input spec.md - 2nd argument is input fork_choice.md - 3rd argument is input validator_guide.md + 1st argument is input 0_beacon-chain.md + 2nd argument is input 0_fork-choice.md + 3rd argument is input 0_beacon-chain-validator.md 4th argument is output spec.py If building phase 1: - 1st argument is input spec_phase0.md - 2nd argument is input spec_phase1_custody.md - 3rd argument is input spec_phase1_shard_data.md - 4th argument is input fork_choice.md - 5th argument is output spec.py + 1st argument is input 0_beacon-chain.md + 2nd argument is input 1_shard-chain-misc.md + 3rd argument is input 1_custody-game.md + 4th argument is input 1_shard-data-chains.md + 5th argument is input 0_fork-choice.md + 6th argument is output spec.py ''' parser = ArgumentParser(description=description) parser.add_argument("-p", "--phase", dest="phase", type=int, default=0, help="Build for phase #") @@ -308,10 +311,10 @@ If building phase 1: else: print(" Phase 0 requires spec, forkchoice, and v-guide inputs as well as an output file.") elif args.phase == 1: - if len(args.files) == 5: + if len(args.files) == 6: build_phase1_spec(*args.files) else: - print(" Phase 1 requires 4 input files as well as an output file: " - + "(phase0.md and phase1.md, phase1.md, fork_choice.md, output.py)") + print(" Phase 1 requires 6 input files as well as an output file: " + + "(0_fork-choice.md, 0_beacon-chain.md and 1_shard-chain-misc.md, 1_custody-game.md, 1_shard-data-chains.md, output.py)") else: print("Invalid phase: {0}".format(args.phase)) From e4e6c4d8eef1839394a4a6aba183c4cf4fd77d5d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 12 Aug 2019 00:45:33 +0800 Subject: [PATCH 15/89] Fix the order of build spec --- Makefile | 2 +- scripts/build_spec.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 3f6cf9816..f2d953f9a 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ $(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS) python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $(SPEC_DIR)/validator/0_beacon-chain-validator.md $@ $(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS) - python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/1_beacon-chain-misc.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $(SPEC_DIR)/core/0_fork-choice.md $@ + python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $(SPEC_DIR)/core/1_beacon-chain-misc.md $(SPEC_DIR)/core/0_fork-choice.md $@ CURRENT_DIR = ${CURDIR} diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 6f0f160ac..44e73224f 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -269,12 +269,12 @@ def build_phase1_spec(phase0_sourcefile: str, fork_choice_sourcefile: str, outfile: str=None) -> Optional[str]: phase0_spec = get_spec(phase0_sourcefile) - phase1_shard_misc = get_spec(phase1_phase1_shard_misc_source_file) phase1_custody = get_spec(phase1_custody_sourcefile) phase1_shard_data = get_spec(phase1_shard_sourcefile) + phase1_shard_misc = get_spec(phase1_phase1_shard_misc_source_file) fork_choice_spec = get_spec(fork_choice_sourcefile) spec_objects = phase0_spec - for value in [phase1_shard_misc, phase1_custody, phase1_shard_data, fork_choice_spec]: + for value in [phase1_custody, phase1_shard_data, phase1_shard_misc, fork_choice_spec]: spec_objects = combine_spec_objects(spec_objects, value) spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS) if outfile is not None: @@ -294,9 +294,9 @@ If building phase 0: If building phase 1: 1st argument is input 0_beacon-chain.md - 2nd argument is input 1_shard-chain-misc.md - 3rd argument is input 1_custody-game.md - 4th argument is input 1_shard-data-chains.md + 2nd argument is input 1_custody-game.md + 3rd argument is input 1_shard-data-chains.md + 4th argument is input 1_shard-chain-misc.md 5th argument is input 0_fork-choice.md 6th argument is output spec.py ''' @@ -315,6 +315,6 @@ If building phase 1: build_phase1_spec(*args.files) else: print(" Phase 1 requires 6 input files as well as an output file: " - + "(0_fork-choice.md, 0_beacon-chain.md and 1_shard-chain-misc.md, 1_custody-game.md, 1_shard-data-chains.md, output.py)") + + "(0_fork-choice.md, 0_beacon-chain.md and 1_custody-game.md, 1_shard-data-chains.md, 1_shard-chain-misc.md output.py)") else: print("Invalid phase: {0}".format(args.phase)) From cb3e0f2146387920089309bc059dc6c91306531d Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 13 Aug 2019 06:12:51 -0400 Subject: [PATCH 16/89] Update specs/core/1_beacon-chain-misc.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index c1d6283ca..e82eb2a30 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -49,7 +49,7 @@ class ShardReceiptProof(Container): #### `pack_compact_validator` ```python -def pack_compact_validator(index: uint64, slashed: bool, balance_in_increments: uint64) -> uint64: +def pack_compact_validator(index: int, slashed: bool, balance_in_increments: int) -> int: """ Creates a compact validator object representing index, slashed status, and compressed balance. Takes as input balance-in-increments (// EFFECTIVE_BALANCE_INCREMENT) to preserve symmetry with From f9849ca562c2e4827ab2d6af3d2eecbfc582a8fe Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 13 Aug 2019 06:14:51 -0400 Subject: [PATCH 17/89] Update specs/core/1_beacon-chain-misc.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index e82eb2a30..8c1cee900 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -61,7 +61,7 @@ def pack_compact_validator(index: int, slashed: bool, balance_in_increments: int #### `unpack_compact_validator` ```python -def unpack_compact_validator(compact_validator: uint64) -> Tuple[uint64, bool, uint64]: +def unpack_compact_validator(compact_validator: int) -> Tuple[int, bool, int]: """ Returns validator index, slashed, balance // EFFECTIVE_BALANCE_INCREMENT """ From 0cf454547b8b15bf88a73f466a5fbf6065ba13f4 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 14 Aug 2019 23:27:25 +0200 Subject: [PATCH 18/89] Update specs/core/1_beacon-chain-misc.md --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 8c1cee900..56f9a223f 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -205,7 +205,7 @@ Add the `shard_receipts` operation to `BeaconBlockBody`: ```python # begin insert @shard_receipts - shard_receipts: List[ShardReceipt, MAX_SHARD_RECEIPTS] + shard_receipts: List[ShardReceiptProof, MAX_SHARD_RECEIPTS] # end insert @shard_receipts ``` From 4f92e7fe3024bc1b545c96ac48ea7fd55cf71cb2 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 14 Aug 2019 23:31:48 +0200 Subject: [PATCH 19/89] Update specs/core/1_beacon-chain-misc.md --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 56f9a223f..8524ad51b 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -213,6 +213,6 @@ Use `process_shard_receipt` to process each receipt. ```python # begin insert @process_shard_receipts - (body.shard_receipts, process_shard_receipts), + (body.shard_receipt_proofs, process_shard_receipt_proofs), # end insert @process_shard_receipts ``` From 49a008df605faf3ba94350b0d2a7a04fadb089b3 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 14 Aug 2019 23:31:55 +0200 Subject: [PATCH 20/89] Update specs/core/1_beacon-chain-misc.md --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 8524ad51b..3bb1fa201 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -209,7 +209,7 @@ Add the `shard_receipts` operation to `BeaconBlockBody`: # end insert @shard_receipts ``` -Use `process_shard_receipt` to process each receipt. +Use `process_shard_receipt_proof` to process each receipt. ```python # begin insert @process_shard_receipts From d424863fc0bd2166ad02707679ad80739da8a859 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 14 Aug 2019 23:32:01 +0200 Subject: [PATCH 21/89] Update specs/core/1_beacon-chain-misc.md --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 3bb1fa201..3b32f39e6 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -205,7 +205,7 @@ Add the `shard_receipts` operation to `BeaconBlockBody`: ```python # begin insert @shard_receipts - shard_receipts: List[ShardReceiptProof, MAX_SHARD_RECEIPTS] + shard_receipt_proofs: List[ShardReceiptProof, MAX_SHARD_RECEIPTS] # end insert @shard_receipts ``` From fd24308d19b36a4343185047e233a23fdaacd62b Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 14 Aug 2019 23:32:10 +0200 Subject: [PATCH 22/89] Update specs/core/1_beacon-chain-misc.md --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 3b32f39e6..be376a6ae 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -201,7 +201,7 @@ def update_persistent_committee(state: BeaconState) -> None: ### Shard receipt processing -Add the `shard_receipts` operation to `BeaconBlockBody`: +Add the `shard_receipt_proofs` operation to `BeaconBlockBody`: ```python # begin insert @shard_receipts From 8255091e409968844d24e8e3ceed37c2aee3e729 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 14 Aug 2019 23:35:26 +0200 Subject: [PATCH 23/89] ShardReceipt -> ShardReceiptProof --- specs/core/1_beacon-chain-misc.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index be376a6ae..701b38db0 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -18,7 +18,7 @@ - [concat_generalized_indices](#concat_generalized_indices) - [compute_historical_state_generalized_index](#compute_historical_state_generalized_index) - [get_generalized_index_of_crosslink_header](#get_generalized_index_of_crosslink_header) - - [process_shard_receipt](#process_shard_receipt) + - [process_shard_receipt_proof](#process_shard_receipt_proof) - [Changes](#changes) - [Persistent committees](#persistent-committees) - [Shard receipt processing](#shard-receipt-processing) @@ -134,10 +134,10 @@ def get_generalized_index_of_crosslink_header(index: int) -> GeneralizedIndex: return GeneralizedIndex(MAX_CROSSLINK_SIZE // SHARD_HEADER_SIZE + index) ``` -#### `process_shard_receipt` +#### `process_shard_receipt_proof` ```python -def process_shard_receipt(state: BeaconState, receipt_proof: ShardReceiptProof): +def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptProof): """ Processes a ShardReceipt object. """ From e4a18f6fa416e854c5a5337df7b5da8ff812301d Mon Sep 17 00:00:00 2001 From: vbuterin Date: Thu, 15 Aug 2019 12:51:11 +0200 Subject: [PATCH 24/89] Made persistent committee roots a vector --- specs/core/1_beacon-chain-misc.md | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 701b38db0..0c23e41b7 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -6,6 +6,7 @@ - [Phase 1 miscellaneous beacon chain changes](#phase-1-miscellaneous-beacon-chain-changes) - [Table of contents](#table-of-contents) + - [Configuration](#configuration) - [Classes](#classes) - [CompactCommittee](#compactcommittee) - [ShardReceiptProof](#shardreceiptproof) @@ -25,6 +26,13 @@ +## Configuration + +| Name | Value | Unit | Duration +| - | - | - | - | +| `MAX_SHARD_RECEIPT_PROOFS` | `2**0` (= 1) | - | - | +| `PERSISTENT_COMMITTEE_ROOT_LENGTH` | `2**8` (= 256) | periods | ~9 months | + ## Classes #### `CompactCommittee` @@ -171,9 +179,7 @@ Add to the beacon state the following fields: ```python # begin insert @persistent_committee_fields - previous_persistent_committee_root: Hash - current_persistent_committee_root: Hash - next_persistent_committee_root: Hash + persistent_committee_roots: Vector[Hash, PERSISTENT_COMMITTEE_ROOT_LENGTH] next_shard_receipt_period: Vector[uint, SHARD_COUNT] # end insert @persistent_committee_fields ``` @@ -190,13 +196,12 @@ def update_persistent_committee(state: BeaconState) -> None: Updates persistent committee roots at boundary blocks. """ if (get_current_epoch(state) + 1) % EPOCHS_PER_SHARD_PERIOD == 0: - state.previous_persistent_committee_root = state.current_persistent_committee_root - state.current_persistent_committee_root = state.next_persistent_committee_root + period = (get_current_epoch(state) + 1) // EPOCHS_PER_SHARD_PERIOD committees = Vector[CompactCommittee, SHARD_COUNT]([ committee_to_compact_committee(state, get_period_committee(state, get_current_epoch(state) + 1, i)) for i in range(SHARD_COUNT) ]) - state.next_persistent_committee_root = hash_tree_root(committees) + state.persistent_committee_roots[period % PERSISTENT_COMMITTEE_ROOT_LENGTH] = hash_tree_root(committees) ``` ### Shard receipt processing @@ -205,7 +210,7 @@ Add the `shard_receipt_proofs` operation to `BeaconBlockBody`: ```python # begin insert @shard_receipts - shard_receipt_proofs: List[ShardReceiptProof, MAX_SHARD_RECEIPTS] + shard_receipt_proofs: List[ShardReceiptProof, MAX_SHARD_RECEIPT_PROOFS] # end insert @shard_receipts ``` From df6e531d7401a9388c3a69d92bf542a18c7532d5 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Tue, 20 Aug 2019 11:37:27 +0200 Subject: [PATCH 25/89] Improve beacon proposer selection logic --- specs/core/0_beacon-chain.md | 14 +++++--------- .../eth2spec/test/helpers/proposer_slashings.py | 1 - 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 7ed3226e1..f27d016cf 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -934,15 +934,12 @@ def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: Return the beacon proposer index at the current slot. """ epoch = get_current_epoch(state) - committees_per_slot = get_committee_count(state, epoch) // SLOTS_PER_EPOCH - offset = committees_per_slot * (state.slot % SLOTS_PER_EPOCH) - shard = Shard((get_start_shard(state, epoch) + offset) % SHARD_COUNT) - first_committee = get_crosslink_committee(state, epoch, shard) + indices = get_active_validator_indices(state, epoch) + seed = hash(get_seed(state, epoch) + int_to_bytes(state.slot, length=8)) MAX_RANDOM_BYTE = 2**8 - 1 - seed = get_seed(state, epoch) i = 0 while True: - candidate_index = first_committee[(epoch + i) % len(first_committee)] + candidate_index = indices[compute_shuffled_index(ValidatorIndex(i % len(indices)), len(indices), seed)] random_byte = hash(seed + int_to_bytes(i // 32, length=8))[i % 32] effective_balance = state.validators[candidate_index].effective_balance if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: @@ -1608,9 +1605,8 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: ```python def process_proposer_slashing(state: BeaconState, proposer_slashing: ProposerSlashing) -> None: proposer = state.validators[proposer_slashing.proposer_index] - # Verify that the epoch is the same - assert (compute_epoch_of_slot(proposer_slashing.header_1.slot) - == compute_epoch_of_slot(proposer_slashing.header_2.slot)) + # Verify slots match + assert proposer_slashing.header_1.slot == proposer_slashing.header_2.slot # But the headers are different assert proposer_slashing.header_1 != proposer_slashing.header_2 # Check proposer is slashable diff --git a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py index d5b7f7b7f..ce53c5931 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -18,7 +18,6 @@ def get_valid_proposer_slashing(spec, state, signed_1=False, signed_2=False): ) header_2 = deepcopy(header_1) header_2.parent_root = b'\x99' * 32 - header_2.slot = slot + 1 if signed_1: sign_block_header(spec, state, header_1, privkey) From c1f2e92ad1af9e1bd88c6f85c7087beebfcbe7b7 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 23 Aug 2019 12:33:25 +0200 Subject: [PATCH 26/89] Update specs/core/1_beacon-chain-misc.md Co-Authored-By: Danny Ryan --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 0c23e41b7..4539be6b0 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -180,7 +180,7 @@ Add to the beacon state the following fields: ```python # begin insert @persistent_committee_fields persistent_committee_roots: Vector[Hash, PERSISTENT_COMMITTEE_ROOT_LENGTH] - next_shard_receipt_period: Vector[uint, SHARD_COUNT] + next_shard_receipt_period: Vector[uint64, SHARD_COUNT] # end insert @persistent_committee_fields ``` `next_shard_receipt_period` values initialized to `PHASE_1_FORK_SLOT // SLOTS_PER_EPOCH // EPOCHS_PER_SHARD_PERIOD` From 520c872d21dd14797685b1011681287cfb224de8 Mon Sep 17 00:00:00 2001 From: Matt Garnett <14004106+c-o-l-o-r@users.noreply.github.com> Date: Thu, 8 Aug 2019 09:59:06 -0400 Subject: [PATCH 27/89] Improve readability of SSZ type defaults --- specs/simple-serialize.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 119022248..bd3c34864 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -62,7 +62,7 @@ * **bitlist**: ordered variable-length collection of `boolean` values, limited to `N` bits * notation `Bitlist[N]` * **union**: union type containing one of the given subtypes - * notation `Union[type_1, type_2, ...]`, e.g. `union[null, uint64]` + * notation `Union[type_0, type_1, ...]`, e.g. `union[null, uint64]` ### Variable-size and fixed-size @@ -78,8 +78,18 @@ For convenience we alias: * `null`: `{}` ### Default values +Assuming a helper function `default(type)` which returns the default value for `type`, we can recursively define the default value for all types. -The default value of a type upon initialization is recursively defined using `0` for `uintN`, `False` for `boolean` and the elements of `Bitvector`, and `[]` for lists and `Bitlist`. Unions default to the first type in the union (with type index zero), which is `null` if present in the union. +| Type | Default Value | +| ---- | ------------- | +| `uintN` | `0` | +| `boolean` | `False` | +| `Container` | `[default(type) for type in container]` | +| `Vector[type, N]` | `[default(type)] * N` | +| `Bitvector[boolean, N]` | `[False] * N` | +| `List[type, N]` | `[]` | +| `Bitlist[boolean, N]` | `[]` | +| `Union[type_0, type_1, ...]` | `default(type_0)` | #### `is_zero` From 4f462bc88bc6b36f55b6427ed3d4e4f3315bf61e Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 7 Aug 2019 13:33:41 +0200 Subject: [PATCH 28/89] make nil-count randomization work for lists --- test_libs/pyspec/eth2spec/debug/random_value.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index 9a7d47239..ce1522c32 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -94,6 +94,8 @@ def get_random_ssz_object(rng: Random, length = 1 elif mode == RandomizationMode.mode_max_count: length = max_list_length + elif mode == RandomizationMode.mode_nil_count: + length = 0 if typ.length < length: # SSZ imposes a hard limit on lists, we can't put in more than that length = typ.length From a8ac4d6c6fe53fe122871997e2003795304e30db Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 1 Aug 2019 12:44:41 +0200 Subject: [PATCH 29/89] Update libp2p networking spec --- specs/networking/libp2p-standardization.md | 158 ----- specs/networking/messaging.md | 45 -- specs/networking/node-identification.md | 31 - specs/networking/p2p-interface.md | 712 +++++++++++++++++++++ specs/networking/rpc-interface.md | 283 -------- 5 files changed, 712 insertions(+), 517 deletions(-) delete mode 100644 specs/networking/libp2p-standardization.md delete mode 100644 specs/networking/messaging.md delete mode 100644 specs/networking/node-identification.md create mode 100644 specs/networking/p2p-interface.md delete mode 100644 specs/networking/rpc-interface.md diff --git a/specs/networking/libp2p-standardization.md b/specs/networking/libp2p-standardization.md deleted file mode 100644 index d1ba07e65..000000000 --- a/specs/networking/libp2p-standardization.md +++ /dev/null @@ -1,158 +0,0 @@ -ETH 2.0 Networking Spec - Libp2p standard protocols -=== - -# Abstract - -Ethereum 2.0 clients plan to use the libp2p protocol networking stack for -mainnet release. This document aims to standardize the libp2p client protocols, -configuration and messaging formats. - -# Libp2p Components - -## Transport - -This section details the libp2p transport layer that underlies the -[protocols](#protocols) that are listed in this document. - -Libp2p allows composition of multiple transports. Eth2.0 clients should support -TCP/IP and optionally websockets. Websockets are useful for implementations -running in the browser and therefore native clients would ideally support these implementations -by supporting websockets. - -An ideal libp2p transport would therefore support both TCP/IP and websockets. - -*Note: There is active development in libp2p to facilitate the -[QUIC](https://github.com/libp2p/go-libp2p-quic-transport) transport, which may -be adopted in the future* - -### Encryption - -Libp2p currently offers [Secio](https://github.com/libp2p/specs/pull/106) which -can upgrade a transport which will then encrypt all future communication. Secio -generates a symmetric ephemeral key which peers use to encrypt their -communication. It can support a range of ciphers and currently supports key -derivation for elliptic curve-based public keys. - -Current defaults are: -- Key agreement: `ECDH-P256` (also supports `ECDH-P384`) -- Cipher: `AES-128` (also supports `AES-256`, `TwofishCTR`) -- Digests: `SHA256` (also supports `SHA512`) - -*Note: Secio is being deprecated in favour of [TLS -1.3](https://github.com/libp2p/specs/blob/master/tls/tls.md). It is our -intention to transition to use TLS 1.3 for encryption between nodes, rather -than Secio.* - - -## Protocols - -This section lists the necessary libp2p protocols required by Ethereum 2.0 -running a libp2p network stack. - -## Multistream-select - -#### Protocol id: `/multistream/1.0.0` - -Clients running libp2p should support the -[multistream-select](https://github.com/multiformats/multistream-select/) -protocol which allows clients to negotiate libp2p protocols establish streams -per protocol. - -## Multiplexing - -Libp2p allows clients to compose multiple multiplexing methods. Clients should -support [mplex](https://github.com/libp2p/specs/tree/master/mplex) and -optionally [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md) -(these can be composed). - -**Mplex protocol id: `/mplex/6.7.0`** - -**Yamux protocol id: `/yamux/1.0.0`** - -## Gossipsub - -#### Protocol id: `/eth/serenity/gossipsub/1.0.0` - -*Note: Parameters listed here are subject to a large-scale network feasibility -study* - -The [Gossipsub](https://github.com/libp2p/specs/tree/master/pubsub/gossipsub) -protocol is used for block and attestation propagation across the -network. - -### Configuration Parameters - -Gossipsub has a number of internal configuration parameters which directly -effect the network performance. Clients can implement independently, however -we aim to standardize these across clients to optimize the gossip network for -propagation times and message duplication. Current network-related defaults are: - -``` -( - // The target number of peers in the overlay mesh network (D in the libp2p specs). - mesh_size: 6 - // The minimum number of peers in the mesh network before adding more (D_lo in the libp2p specs). - mesh_lo: 4 - // The maximum number of peers in the mesh network before removing some (D_high in the libp2p sepcs). - mesh_high: 12 - // The number of peers to gossip to during a heartbeat (D_lazy in the libp2p sepcs). - gossip_lazy: 6 // defaults to `mesh_size` - // Time to live for fanout peers (seconds). - fanout_ttl: 60 - // The number of heartbeats to gossip about. - gossip_history: 3 - // Time between each heartbeat (seconds). - heartbeat_interval: 1 -) -``` - -### Topics - -*The Go and Js implementations use string topics - This is likely to be -updated to topic hashes in later versions - https://github.com/libp2p/rust-libp2p/issues/473* - -For Eth2.0 clients, topics are sent as `SHA2-256` hashes of the topic string. - -There are two main topics used to propagate attestations and beacon blocks to -all nodes on the network. - -- The `beacon_block` topic - This topic is used solely for propagating new - beacon blocks to all nodes on the networks. -- The `beacon_attestation` topic - This topic is used to propagate - aggregated attestations to subscribing nodes (typically block proposers) to - be included into future blocks. Attestations are aggregated in their - respective subnets before publishing on this topic. - -Shards are grouped into their own subnets (defined by a shard topic). The -number of shard subnets is defined via `SHARD_SUBNET_COUNT` and the shard -`shard_number % SHARD_SUBNET_COUNT` is assigned to the topic: -`shard{shard_number % SHARD_SUBNET_COUNT}_attestation`. - -### Messages - -*Note: The message format here is Eth2.0-specific* - -Each Gossipsub -[Message](https://github.com/libp2p/go-libp2p-pubsub/blob/master/pb/rpc.proto#L17-L24) -has a maximum size of 512KB (estimated from expected largest uncompressed block -size). - -The `data` field of a Gossipsub `Message` is an SSZ-encoded object. For the `beacon_block` topic, -this is a `beacon_block`. For the `beacon_attestation` topic, this is -an `attestation`. - -## Eth-2 RPC - -#### Protocol Id: `/eth/serenity/beacon/rpc/1` - -The [RPC Interface](./rpc-interface.md) is specified in this repository. - -## Discovery - -Discovery Version 5 -([discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md)) -will be used for discovery. This protocol uses a UDP transport and specifies -its own encryption, ip-discovery and topic advertisement. Therefore, it has no -need to establish streams through `multistream-select`, rather, act -as a standalone implementation that feeds discovered peers/topics (ENR-records) as -`multiaddrs` into the libp2p service. diff --git a/specs/networking/messaging.md b/specs/networking/messaging.md deleted file mode 100644 index d7cb5bb5b..000000000 --- a/specs/networking/messaging.md +++ /dev/null @@ -1,45 +0,0 @@ -# Eth 2.0 Networking Spec - Messaging - -## Abstract - -This specification describes how individual Ethereum 2.0 messages are represented on the wire. - -The key words “MUST”, “MUST NOT”, “REQUIRED”, “SHALL”, “SHALL”, NOT", “SHOULD”, “SHOULD NOT”, “RECOMMENDED”, “MAY”, and “OPTIONAL” in this document are to be interpreted as described in [RFC 2119](https://tools.ietf.org/html/rfc2119). - -## Motivation - -This specification seeks to define a messaging protocol that is flexible enough to be changed easily as the Eth 2.0 specification evolves. - -Note that while `libp2p` is the chosen networking stack for Ethereum 2.0, as of this writing some clients do not have workable `libp2p` implementations. To allow those clients to communicate, we define a message envelope that includes the body's compression, encoding, and body length. Once `libp2p` is available across all implementations, this message envelope will be removed because `libp2p` will negotiate the values defined in the envelope upfront. - -## Specification - -### Message structure - -An Eth 2.0 message consists of an envelope that defines the message's compression, encoding, and length followed by the body itself. - -Visually, a message looks like this: - -``` -+--------------------------+ -| compression nibble | -+--------------------------+ -| encoding nibble | -+--------------------------+ -| body length (uint64) | -+--------------------------+ -| | -| body | -| | -+--------------------------+ -``` - -Clients MUST ignore messages with malformed bodies. The compression/encoding nibbles MUST be one of the following values: - -### Compression nibble values - -- `0x0`: no compression - -### Encoding nibble values - -- `0x1`: SSZ diff --git a/specs/networking/node-identification.md b/specs/networking/node-identification.md deleted file mode 100644 index 32ec4dfad..000000000 --- a/specs/networking/node-identification.md +++ /dev/null @@ -1,31 +0,0 @@ -# Eth 2.0 Networking Spec - Node Identification - -## Abstract - -This specification describes how Ethereum 2.0 nodes identify and address each other on the network. - -The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL", NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [RFC 2119](https://tools.ietf.org/html/rfc2119). - -## Specification - -Clients use Ethereum Node Records (as described in [EIP-778](http://eips.ethereum.org/EIPS/eip-778)) to discover one another. Each ENR includes, among other things, the following keys: - -- The node's IP. -- The node's TCP port. -- The node's public key. - -For clients to be addressable, their ENR responses MUST contain all of the above keys. Client MUST verify the signature of any received ENRs, and disconnect from peers whose ENR signatures are invalid. Each node's public key MUST be unique. - -The keys above are enough to construct a [multiaddr](https://github.com/multiformats/multiaddr) for use with the rest of the `libp2p` stack. - -It is RECOMMENDED that clients set their TCP port to the default of `9000`. - -### Peer ID generation - -The `libp2p` networking stack identifies peers via a "peer ID." Simply put, a node's Peer ID is the SHA2-256 `multihash` of the node's public key struct (serialized in protobuf, refer to the [Peer ID spec](https://github.com/libp2p/specs/pull/100)). `go-libp2p-crypto` contains the canonical implementation of how to hash `secp256k1` keys for use as a peer ID. - -## See also - -- [multiaddr](https://github.com/multiformats/multiaddr) -- [multihash](https://multiformats.io/multihash/) -- [go-libp2p-crypto](https://github.com/libp2p/go-libp2p-crypto) diff --git a/specs/networking/p2p-interface.md b/specs/networking/p2p-interface.md new file mode 100644 index 000000000..72f5c0fd6 --- /dev/null +++ b/specs/networking/p2p-interface.md @@ -0,0 +1,712 @@ +# Overview + +This document contains the network specification for Ethereum 2.0 clients. + +It consists of four main sections: + +1. A specification of the network fundamentals detailing the two network configurations: interoperability test network, and mainnet launch. +2. A specification of the three network interaction _domains_ of ETH2.0: (a) the gossip domain, (b) the discovery domain, \(c\) the Req/Resp domain. +3. The rationale and further explanation for the design choices made in the previous two sections. +4. An analysis of the maturity/state of the libp2p features required by this spec across the languages in which ETH 2.0 clients are being developed. + +## Table of Contents +[TOC] + +# Network Fundamentals + +This section outlines the specification for the networking stack in Ethereum 2.0 clients. + +Sections that have differing parameters for mainnet launch and interoperability testing are split into subsections. Sections that are not split have the same parameters for interoperability testing as mainnet launch. + +## Transport + +Even though libp2p is a multi-transport stack (designed to listen on multiple simultaneous transports and endpoints transparently), we hereby define a profile for basic interoperability. + +#### Interop + +All implementations MUST support the TCP libp2p transport, and it MUST be enabled for both dialing and listening (i.e. outbound and inbound connections). + +The libp2p TCP transport supports listening on IPv4 and IPv6 addresses (and on multiple simultaneously). Clients SHOULD allow the operator to configure the listen IP addresses and ports, including the addressing schemes (IPv4, IPv6). + +To facilitate connectivity, and avert possible IPv6 routability/support issues, clients participating in the interoperability testnet MUST expose at least ONE IPv4 endpoint. + +All listening endpoints must be publicly dialable, and thus not rely on libp2p circuit relay, AutoNAT or AutoRelay facilities. + +Nodes operating behind a NAT, or otherwise undialable by default (e.g. container runtime, firewall, etc.), MUST have their infrastructure configured to enable inbound traffic on the announced public listening endpoint. + +#### Mainnet + +All requirements from the interoperability testnet apply, except for the IPv4 addressing scheme requirement. + +At this stage, clients are licensed to drop IPv4 support if they wish to do so, cognizant of the potential disadvantages in terms of Internet-wide routability/support. Clients MAY choose to listen only on IPv6, but MUST retain capability to dial both IPv4 and IPv6 addresses. + +Usage of circuit relay, AutoNAT or AutoRelay will be specifically re-examined closer to the time. + +## Encryption and identification + +#### Interop + +[SecIO](https://github.com/libp2p/specs/tree/master/secio) with `secp256k1` identities will be used for initial interoperability testing. + +The following SecIO parameters MUST be supported by all stacks: + +- Key agreement: ECDH-P256. +- Cipher: AES-128. +- Digest: SHA256. + +#### Mainnet + +[Noise Framework](http://www.noiseprotocol.org/) handshakes will be used for mainnet. libp2p Noise support [is in the process of being standardised](https://github.com/libp2p/specs/issues/195) in the libp2p project. + +Noise support will presumably include IX, IK and XX handshake patterns, and may rely on Curve25519 keys, ChaCha20 and Poly1305 ciphers, and SHA256 as a hash function. These aspects are being actively debated in the referenced issue [ETH 2.0 implementers are welcome to comment and contribute to the discussion.] + +## Protocol Negotiation + +#### Interop + +Connection-level and stream-level (see the rationale section below for explanations) protocol negotiation MUST be conducted using [multistream-select v1.0](https://github.com/multiformats/multistream-select/). Its protocol ID is: `/multistream/1.0.0`. + +#### Mainnet + +Clients MUST support [multistream-select 1.0](https://github.com/multiformats/multistream-select/) and MAY support [multiselect 2.0](https://github.com/libp2p/specs/pull/95). Depending on the number of clients that have implementations for multiselect 2.0 by mainnet, [multistream-select 1.0](https://github.com/multiformats/multistream-select/) may be phased out. + +## Multiplexing + +During connection bootstrapping, libp2p dynamically negotiates a mutually supported multiplexing method to conduct parallel conversations. This applies to transports that are natively incapable of multiplexing (e.g. TCP, WebSockets, WebRTC), and is omitted for capable transports (e.g. QUIC). + +Two multiplexers are commonplace in libp2p implementations: [mplex](https://github.com/libp2p/specs/tree/master/mplex) and [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md). Their protocol IDs are, respectively: `/mplex/6.7.0` and `/yamux/1.0.0`. + +Clients MUST support [mplex](https://github.com/libp2p/specs/tree/master/mplex) and MAY support [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md). If both are supported by the client, yamux must take precedence during negotiation. See the Rationale section of this document for tradeoffs. + +# ETH2 network interaction domains + +## Constants + +This section outlines constants that are used in this spec. + +- `RQRP_MAX_SIZE`: The max size of uncompressed req/resp messages that clients will allow. + Value: TBD +- `GOSSIP_MAX_SIZE`: The max size of uncompressed gossip messages + Value: 1MB (estimated from expected largest uncompressed block size). +- `SHARD_SUBNET_COUNT`: The number of shard subnets used in the gossipsub protocol. + Value: TBD + +## The gossip domain: gossipsub + +Clients MUST support the [gossipsub](https://github.com/libp2p/specs/tree/master/pubsub/gossipsub) libp2p protocol. + +**Protocol ID:** `/meshsub/1.0.0` + +**Gossipsub Parameters** + +*Note: Parameters listed here are subject to a large-scale network feasibility study.* + +The following gossipsub parameters will be used: + +- `D` (topic stable mesh target count): 6 +- `D_low` (topic stable mesh low watermark): 4 +- `D_high` (topic stable mesh high watermark): 12 +- `D_lazy` (gossip target): 6 +- `fanout_ttl` (ttl for fanout maps for topics we are not subscribed to but have published to, seconds): 60 +- `gossip_advertise` (number of windows to gossip about): 3 +- `gossip_history` (number of heartbeat intervals to retain message IDs): 5 +- `heartbeat_interval` (frequency of heartbeat, seconds): 1 + +### Topics + +Topics are plain UTF-8 strings, and are encoded on the wire as determined by protobuf (gossipsub messages are enveloped in protobuf messages). + +Topic strings have form: `/eth2/TopicName/TopicEncoding`. This defines both the type of data being sent on the topic and how the data field of the message is encoded. (Further details can be found in [Messages](#Messages)). + +There are two main topics used to propagate attestations and beacon blocks to all nodes on the network. Their `TopicName`'s are: + +- `beacon_block` - This topic is used solely for propagating new beacon blocks to all nodes on the networks. Blocks are sent in their entirety. Clients who receive a block on this topic MUST validate the block proposer signature before forwarding it across the network. +- `beacon_attestation` - This topic is used to propagate aggregated attestations (in their entirety) to subscribing nodes (typically block proposers) to be included in future blocks. Similarly to beacon blocks, clients will be expected to perform some sort of validation before forwarding, but the precise mechanism is still TBD. + +Additional topics are used to propagate lower frequency validator messages. Their `TopicName`’s are: + +- `voluntary_exit` - This topic is used solely for propagating voluntary validator exits to proposers on the network. Voluntary exits are sent in their entirety. Clients who receive a voluntary exit on this topic MUST validate the conditions within `process_voluntary_exit` before forwarding it across the network. +- `proposer_slashing` - This topic is used solely for propagating proposer slashings to proposers on the network. Proposer slashings are sent in their entirety. Clients who receive a proposer slashing on this topic MUST validate the conditions within `process_proposer_slashing` before forwarding it across the network. +- `attester_slashing` - This topic is used solely for propagating attester slashings to proposers on the network. Attester slashings are sent in their entirety. Clients who receive an attester slashing on this topic MUST validate the conditions within `process_attester_slashing` before forwarding it across the network. + +#### Interop + +Unaggregated attestations from all shards are sent to the `beacon_attestation` topic. + +#### Mainnet + +Shards are grouped into their own subnets (defined by a shard topic). The number of shard subnets is defined via `SHARD_SUBNET_COUNT` and the shard `shard_number % SHARD_SUBNET_COUNT` is assigned to the topic: `shard{shard_number % SHARD_SUBNET_COUNT}_beacon_attestation`. Unaggregated attestations are sent to the subnet topic. Aggregated attestations are sent to the `beacon_attestation` topic. + +### Messages + +Each gossipsub [message](https://github.com/libp2p/go-libp2p-pubsub/blob/master/pb/rpc.proto#L17-L24) has a maximum size of `GOSSIP_MAX_SIZE`. + +Clients MUST reject (fail validation) messages that are over this size limit. Likewise, clients MUST NOT emit or propagate messages larger than this limit. + +The payload is carried in the `data` field of a gossipsub message, and varies depending on the topic: + + +| Topic | Message Type | +|------------------------------|-------------------| +| beacon_block | BeaconBlock | +| beacon_attestation | Attestation | +| shard{N}\_beacon_attestation | Attestation | +| voluntary_exit | VoluntaryExit | +| proposer_slashing | ProposerSlashing | +| attester_slashing | AttesterSlashing | + +Clients MUST reject (fail validation) messages containing an incorrect type, or invalid payload. + +When processing incoming gossip, clients MAY descore or disconnect peers who fail to observe these constraints. + +### Encodings + +Topics are post-fixed with an encoding. Encodings define how the payload of a gossipsub message is encoded. + +#### Interop + +- `ssz` - All objects are SSZ-encoded. Example: The beacon block topic string is: `/beacon_block/ssz` and the data field of a gossipsub message is an ssz-encoded `BeaconBlock`. + +#### Mainnet + +- `ssz_snappy` - All objects are ssz-encoded and then compressed with snappy. Example: The beacon attestation topic string is: `/beacon_attestation/ssz_snappy` and the data field of a gossipsub message is an `Attestation` that has been ssz-encoded then compressed with snappy. + +Implementations MUST use a single encoding. Changing an encoding will require coordination between participating implementations. + +## The discovery domain: discv5 + +Discovery Version 5 ([discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md)) is used for peer discovery, both in the interoperability testnet and mainnet. + +`discv5` is a standalone protocol, running on UDP on a dedicated port, meant for peer discovery only. `discv5` supports self-certified, flexible peer records (ENRs) and topic-based advertisement, both of which are (or will be) requirements in this context. + +### Integration into libp2p stacks + +`discv5` SHOULD be integrated into the client’s libp2p stack by implementing an adaptor to make it conform to the [service discovery](https://github.com/libp2p/go-libp2p-core/blob/master/discovery/discovery.go) and [peer routing](https://github.com/libp2p/go-libp2p-core/blob/master/routing/routing.go#L36-L44) abstractions and interfaces (go-libp2p links provided). + +Inputs to operations include peer IDs (when locating a specific peer), or capabilities (when searching for peers with a specific capability), and the outputs will be multiaddrs converted from the ENR records returned by the discv5 backend. + +This integration enables the libp2p stack to subsequently form connections and streams with discovered peers. + +### ENR structure + +The Ethereum Node Record (ENR) for an Ethereum 2.0 client MUST contain the following entries (exclusive of the sequence number and signature, which MUST be present in an ENR): + +- The compressed secp256k1 publickey, 33 bytes (`secp256k1` field). +- An IPv4 address (`ip` field) and/or IPv6 address (`ip6` field). +- A TCP port (`tcp` field) representing the local libp2p listening port. +- A UDP port (`udp` field) representing the local discv5 listening port. + +Specifications of these parameters can be found in the [ENR Specification](http://eips.ethereum.org/EIPS/eip-778). + +#### Interop + +In the interoperability testnet, all peers will support all capabilities defined in this document (gossip, full Req/Resp suite, discovery protocol), therefore the ENR record does not need to carry ETH2 capability information, as it would be superfluous. + +Nonetheless, ENRs MUST carry a generic `eth2` key with nil value, denoting that the peer is indeed a ETH2 peer, in order to eschew connecting to ETH1 peers. + +#### Mainnet + +On mainnet, ENRs MUST include a structure enumerating the capabilities offered by the peer in an efficient manner. The concrete solution is currently undefined. Proposals include using namespaced bloom filters mapping capabilities to specific protocol IDs supported under that capability. + +### Topic advertisement + +#### Interop + +This feature will not be used in the interoperability testnet. + +#### Mainnet + +In mainnet, we plan to use discv5’s topic advertisement feature as a rendezvous facility for peers on shards (thus subscribing to the relevant gossipsub topics). + +## The Req/Resp domain + +### Protocol identification + +Each message type is segregated into its own libp2p protocol ID, which is a case-sensitive UTF-8 string of the form: + +``` +/ProtocolPrefix/MessageName/SchemaVersion/Encoding +``` + +With: + +- `ProtocolPrefix` - messages are grouped into families identified by a shared libp2p protocol name prefix. In this case, we use `/eth2/beacon_chain/req`. +- `MessageName` - each request is identified by a name consisting of English alphabet, digits and underscores (`_`). +- `SchemaVersion` - an ordinal version number (e.g. 1, 2, 3…) Each schema is versioned to facilitate backward and forward-compatibility when possible. +- `Encoding` - while the schema defines the data types in more abstract terms, the encoding strategy describes a specific representation of bytes that will be transmitted over the wire. See the [Encodings](#Encoding-strategies) section, for further details. + +This protocol segregation allows libp2p `multistream-select 1.0` / `multiselect 2.0` to handle the request type, version and encoding negotiation before establishing the underlying streams. + +### Req/Resp interaction + +We use ONE stream PER request/response interaction. Streams are closed when the interaction finishes, whether in success or in error. + +Request/response messages MUST adhere to the encoding specified in the protocol name, and follow this structure (relaxed BNF grammar): + +``` +request ::= | +response ::= | | +result ::= “0” | “1” | “2” | [“128” ... ”255”] +``` + +The encoding-dependent header may carry metadata or assertions such as the encoded payload length, for integrity and attack proofing purposes. It is not strictly necessary to length-prefix payloads, because req/resp streams are single-use, and stream closures implicitly delimit the boundaries, but certain encodings like SSZ do, for added security. + +`encoded-payload` has a maximum byte size of `RQRP_MAX_SIZE`. + +Clients MUST ensure the payload size is less than or equal to `RQRP_MAX_SIZE`, if not, they SHOULD reset the stream immediately. Clients tracking peer reputation MAY decrement the score of the misbehaving peer under this circumstance. + +#### Requesting side + +Once a new stream with the protocol ID for the request type has been negotiated, the full request message should be sent immediately. It should be encoded according to the encoding strategy. + +The requester MUST close the write side of the stream once it finishes writing the request message - at this point, the stream will be half-closed. + +The requester MUST wait a maximum of **5 seconds** for the first response byte to arrive (time to first byte – or TTFB – timeout). On that happening, the requester will allow further **10 seconds** to receive the full response. + +If any of these timeouts fire, the requester SHOULD reset the stream and deem the req/resp operation to have failed. + +#### Responding side + +Once a new stream with the protocol ID for the request type has been negotiated, the responder must process the incoming request message according to the encoding strategy, until EOF (denoting stream half-closure by the requester). + +The responder MUST: + +1. Use the encoding strategy to read the optional header. +2. If there are any length assertions for length `N`, it should read exactly `N` bytes from the stream, at which point an EOF should arise (no more bytes). Should this is not the case, it should be treated as a failure. +3. Deserialize the expected type, and process the request. +4. Write the response (result, optional header, payload). +5. Close their write side of the stream. At this point, the stream will be fully closed. + +If steps (1), (2) or (3) fail due to invalid, malformed or inconsistent data, the responder MUST respond in error. Clients tracking peer reputation MAY record such failures, as well as unexpected events, e.g. early stream resets. + +The entire request should be read in no more than **5 seconds**. Upon a timeout, the responder SHOULD reset the stream. + +The responder SHOULD send a response promptly, starting with a **single-byte** response code which determines the contents of the response (`result` particle in the BNF grammar above). + +It can have one of the following values: + +- 0: **Success** -- a normal response follows, with contents matching the expected message schema and encoding specified in the request. +- 1: **InvalidRequest** -- the contents of the request are semantically invalid, or the payload is malformed, or could not be understood. The response payload adheres to the ErrorMessage schema (described below). +- 2: **ServerError** -- the responder encountered an error while processing the request. The response payload adheres to the ErrorMessage schema (described below). + +Clients MAY use response codes above `128` to indicate alternative, erroneous request-specific responses. + +The range `[3, 127]` is RESERVED for future usages, and should be treated as error if not recognised expressly. + +The `ErrorMessage` schema is: + +``` +( + error_message: String +) +``` + +*Note that the String type is encoded as UTF-8 bytes when SSZ-encoded.* + +A response therefore has the form: +``` + +--------+--------+--------+--------+--------+--------+ + | result | header (opt) | encoded_response | + +--------+--------+--------+--------+--------+--------+ +``` +Here `result` represents the 1-byte response code. + +### Encoding strategies + +The token of the negotiated protocol ID specifies the type of encoding to be used for the req/resp interaction. Two values are possible at this time: + +- `ssz`: the contents are [SSZ](https://github.com/ethereum/eth2.0-specs/blob/192442be51a8a6907d6401dffbf5c73cb220b760/specs/networking/libp2p-standardization.md#ssz-encoding) encoded. This encoding type MUST be supported by all clients. +- `ssz_snappy`: the contents are SSZ encoded, and subsequently compressed with [Snappy](https://github.com/google/snappy). MAY be supported in the interoperability testnet; and MUST be supported in mainnet. + +#### SSZ encoding strategy (with or without Snappy) + +The [SimpleSerialize (SSZ) specification](https://github.com/ethereum/eth2.0-specs/blob/192442be51a8a6907d6401dffbf5c73cb220b760/specs/simple-serialize.md) outlines how objects are SSZ-encoded. If the Snappy variant is selected, we feed the serialised form to the Snappy compressor on encoding. The inverse happens on decoding. + +**Encoding-dependent header:** Req/Resp protocols using the `ssz` or `ssz_snappy` encoding strategies MUST prefix all encoded and compressed (if applicable) payloads with an unsigned [protobuf varint](https://developers.google.com/protocol-buffers/docs/encoding#varints). + +Note that parameters defined as `[]VariableName` are SSZ-encoded containerless vectors. + +### Messages + +#### Hello + +**Protocol ID:** ``/eth2/beacon_chain/req/hello/1/`` + +**Content**: +``` +( + fork_version: bytes4 + finalized_root: bytes32 + finalized_epoch: uint64 + head_root: bytes32 + head_slot: uint64 +) +``` +The fields are: + +- `fork_version`: The beacon_state `Fork` version +- `finalized_root`: The latest finalized root the node knows about +- `finalized_epoch`: The latest finalized epoch the node knows about +- `head_root`: The block hash tree root corresponding to the head of the chain as seen by the sending node +- `head_slot`: The slot corresponding to the `head_root`. + +Clients exchange hello messages upon connection, forming a two-phase handshake. The first message the initiating client sends MUST be the hello message. In response, the receiving client MUST respond with its own hello message. + +Clients SHOULD immediately disconnect from one another following the handshake above under the following conditions: + +1. If `fork_version` doesn’t match the local fork version, since the client’s chain is on another fork. `fork_version` can also be used to segregate testnets. +2. If the (`finalized_root`, `finalized_epoch`) shared by the peer is not in the client's chain at the expected epoch. For example, if Peer 1 sends (root, epoch) of (A, 5) and Peer 2 sends (B, 3) but Peer 1 has root C at epoch 3, then Peer 1 would disconnect because it knows that their chains are irreparably disjoint. + +Once the handshake completes, the client with the lower `finalized_epoch` or `head_slot` (if the clients have equal `finalized_epoch`s) SHOULD request beacon blocks from its counterparty via the `BeaconBlocks` request. + +#### Goodbye + +**Protocol ID:** ``/eth2/beacon_chain/req/goodbye/1/`` + +**Content:** +``` +( + reason: uint64 +) +``` +Client MAY send goodbye messages upon disconnection. The reason field MAY be one of the following values: + +- 1: Client shut down. +- 2: Irrelevant network. +- 3: Fault/error. + +Clients MAY use reason codes above `128` to indicate alternative, erroneous request-specific responses. + +The range `[4, 127]` is RESERVED for future usage. + +#### BeaconBlocks + +**Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks/1/` + +Request Content +``` +( + head_block_root: HashTreeRoot + start_slot: uint64 + count: uint64 + step: uint64 +) +``` + +Response Content: +``` +( + blocks: []BeaconBlock +) +``` + +Requests count beacon blocks from the peer starting from `start_slot` on the chain defined by `head_block_root`. The response MUST contain no more than count blocks. step defines the slot increment between blocks. For example, requesting blocks starting at `start_slot` 2 with a step value of 2 would return the blocks at [2, 4, 6, …]. In cases where a slot is empty for a given slot number, no block is returned. For example, if slot 4 were empty in the previous example, the returned array would contain [2, 6, …]. A step value of 1 returns all blocks on the range `[start_slot, start_slot + count)`. + +`BeaconBlocks` is primarily used to sync historical blocks. + +Clients MUST support requesting blocks since the start of the weak subjectivity period and up to the given `head_block_root`. + +Clients MUST support `head_block_root` values since the latest finalized epoch. + +#### RecentBeaconBlocks + +**Protocol ID:** `/eth2/beacon_chain/req/recent_beacon_blocks/1/` + +Request Content: + +``` +( + block_roots: []HashTreeRoot +) +``` + +Response Content: + +``` +( + blocks: []BeaconBlock +) +``` + +Requests blocks by their block roots. The response is a list of `BeaconBlock` with the same length as the request. Blocks are returned in order of the request and any missing/unknown blocks are left empty (SSZ null `BeaconBlock`). + +`RecentBeaconBlocks` is primarily used to recover recent blocks, for example when receiving a block or attestation whose parent is unknown. + +Clients MUST support requesting blocks since the latest finalized epoch. + +# Design Decision Rationale + +## Transport + +### Why are we defining specific transports? + +libp2p peers can listen on multiple transports concurrently, and these can change over time. multiaddrs not only encode the address, but also the transport to be used to dial. + +Due to this dynamic nature, agreeing on specific transports like TCP, QUIC or WebSockets on paper becomes irrelevant. + +However, it is useful to define a minimum baseline for interoperability purposes. + +### Can clients support other transports/handshakes than the ones mandated by the spec? + +Clients may support other transports such as libp2p QUIC, WebSockets, and WebRTC transports, if available in the language of choice. While interoperability shall not be harmed by lack of such support, the advantages are desirable: + +- better latency, performance and other QoS characteristics (QUIC). +- paving the way for interfacing with future light clients (WebSockets, WebRTC). + +The libp2p QUIC transport inherently relies on TLS 1.3 per requirement in section 7 of the [QUIC protocol specification](https://tools.ietf.org/html/draft-ietf-quic-transport-22#section-7), and the accompanying [QUIC-TLS document](https://tools.ietf.org/html/draft-ietf-quic-tls-22). + +The usage of one handshake procedure or the other shall be transparent to the ETH 2.0 application layer, once the libp2p Host/Node object has been configured appropriately. + +### What are advantages of using TCP/QUIC/Websockets? + +TCP is a reliable, ordered, full-duplex, congestion controlled network protocol that powers much of the Internet as we know it today. HTTP/1.1 and HTTP/2 run atop TCP. + +QUIC is a new protocol that’s in the final stages of specification by the IETF QUIC WG. It emerged from Google’s SPDY experiment. The QUIC transport is undoubtedly promising. It’s UDP based yet reliable, ordered, reduces latency vs. TCP, is multiplexed, natively secure (TLS 1.3), offers stream-level and connection-level congestion control (thus removing head-of-line blocking), 0-RTT connection establishment, and endpoint migration, amongst other features. UDP also has better NAT traversal properties than TCP -- something we desperately pursue in peer-to-peer networks. + +QUIC is being adopted as the underlying protocol for HTTP/3. This has the potential to award us censorship resistance via deep packet inspection for free. Provided that we use the same port numbers and encryption mechanisms as HTTP/3, our traffic may be indistinguishable from standard web traffic, and we may only become subject to standard IP-based firewall filtering -- something we can counteract via other mechanisms. + +WebSockets and/or WebRTC transports are necessary for interaction with browsers, and will become increasingly important as we incorporate browser-based light clients to the ETH2 network. + +### Why do we not just support a single transport? + +Networks evolve. Hardcoding design decisions leads to ossification, preventing the evolution of networks alongside the state of the art. Introducing changes on an ossified protocol is very costly, and sometimes, downright impracticable without causing undesirable breakage. + +Modelling for upgradeability and dynamic transport selection from the get-go lays the foundation for a future-proof stack. + +Clients can adopt new transports without breaking old ones; and the multi-transport ability enables constrained and sandboxed environments (e.g. browsers, embedded devices) to interact with the network as first-class citizens via suitable/native transports (e.g. WSS), without the need for proxying or trust delegation to servers. + +### Why are we not using QUIC for mainnet from the start? + +The QUIC standard is still not finalised (at working draft 22 at the time of writing), and not all mainstream runtimes/languages have mature, standard, and/or fully-interoperable [QUIC support](https://github.com/quicwg/base-drafts/wiki/Implementations). One remarkable example is node.js, where the QUIC implementation is [in early development](https://github.com/nodejs/quic). + +## Multiplexing + +### Why are we using mplex/yamux? + +[Yamux](https://github.com/hashicorp/yamux/blob/master/spec.md) is a multiplexer invented by Hashicorp that supports stream-level congestion control. Implementations exist in a limited set of languages, and it’s not a trivial piece to develop. + +Conscious of that, the libp2p community conceptualised [mplex](https://github.com/libp2p/specs/blob/master/mplex/README.md) as a simple, minimal multiplexer for usage with libp2p. It does not support stream-level congestion control, and is subject to head-of-line blocking. + +Overlay multiplexers are not necessary with QUIC, as the protocol provides native multiplexing, but they need to be layered atop TCP, WebSockets, and other transports that lack such support. + +## Protocol Negotiation + +### When is multiselect 2.0 due and why are we using it for mainnet? + +multiselect 2.0 is currently being conceptualised. Debate started [on this issue](https://github.com/libp2p/specs/pull/95), but it got overloaded – as it tends to happen with large conceptual OSS discussions that touch the heart and core of a system. + +In the following weeks (August 2019), there will be a renewed initiative to first define the requirements, constraints, assumptions and features, in order to lock in basic consensus upfront, to subsequently build on that consensus by submitting a specification for implementation. + +We plan to use multiselect 2.0 for mainnet because it will: + +1. Reduce round trips during connection bootstrapping and stream protocol negotiation. +2. Enable efficient one-stream-per-request interaction patterns. +3. Leverage *push data* mechanisms of underlying protocols to expedite negotiation. +4. Provide the building blocks for enhanced censorship resistance. + +### What is the difference between connection-level and stream-level protocol negotiation? + +All libp2p connections must be authenticated, encrypted, and multiplexed. Connections using network transports unsupportive of native authentication/encryption and multiplexing (e.g. TCP) need to undergo protocol negotiation to agree on a mutually supported: + +1. authentication/encryption mechanism (such as SecIO, TLS 1.3, Noise). +2. overlay multiplexer (such as mplex, Yamux, spdystream). + +In this specification, we refer to these two as *connection-level negotiations*. Transports supporting those features natively (such as QUIC) omit those negotiations. + +After successfully selecting a multiplexer, all subsequent I/O happens over *streams*. When opening streams, peers pin a protocol to that stream, by conducting *stream-level protocol negotiation*. + +At present, multistream-select 1.0 is used for both types of negotiation, but multiselect 2.0 will use dedicated mechanisms for connection bootstrapping process and stream protocol negotiation. + +## Encryption + +### Why are we using SecIO for interop? Why not for mainnet? + +SecIO has been the default encryption layer for libp2p for years. It is used in IPFS and Filecoin. And although it will be superseded shortly, it is proven to work at scale. + +SecIO is the common denominator across the various language libraries at this stage. It is widely implemented. That’s why we have chosen to use it for initial interop to minimize overhead in getting to a basic interoperability testnet. + +We won’t be using it for mainnet because, amongst other things, it requires several round trips to be sound, and doesn’t support early data (0-RTT data), a mechanism that multiselect 2.0 will leverage to reduce round trips during connection bootstrapping. + +SecIO is not considered secure for the purposes of this spec. + +## Why are we using Noise/TLS 1.3 for mainnet? + +Copied from the Noise Protocol Framework website: + +> Noise is a framework for building crypto protocols. Noise protocols support mutual and optional authentication, identity hiding, forward secrecy, zero round-trip encryption, and other advanced features. + +Noise in itself does not specify a single handshake procedure, but provides a framework to build secure handshakes based on Diffie-Hellman key agreement with a variety of tradeoffs and guarantees. + +Noise handshakes are lightweight and simple to understand, and are used in major cryptographic-centric projects like WireGuard, I2P, Lightning. [Various](https://www.wireguard.com/papers/kobeissi-bhargavan-noise-explorer-2018.pdf) [studies](https://eprint.iacr.org/2019/436.pdf) have assessed the stated security goals of several Noise handshakes with positive results. + +On the other hand, TLS 1.3 is the newest, simplified iteration of TLS. Old, insecure, obsolete ciphers and algorithms have been removed, adopting Ed25519 as the sole ECDH key agreement function. Handshakes are faster, 1-RTT data is supported, and session resumption is a reality, amongst other features. + +Note that [TLS 1.3 is a prerequisite of the QUIC transport](https://tools.ietf.org/html/draft-ietf-quic-transport-22#section-7), although an experiment exists to integrate Noise as the QUIC crypto layer: [nQUIC](https://eprint.iacr.org/2019/028). + +### Why are we using encryption at all? + +Transport level encryption secures message exchange and provides properties that are useful for privacy, safety, and censorship resistance. These properties are derived from the following security guarantees that apply to the entire communication between two peers: + +- Peer authentication: the peer I’m talking to is really who they claim to be, and who I expect them to be. +- Confidentiality: no observer can eavesdrop on the content of our messages. +- Integrity: the data has not been tampered with by a third-party while in transit. +- Non-repudiation: the originating peer cannot dispute that they sent the message. +- Depending on the chosen algorithms and mechanisms (e.g. continuous HMAC), we may obtain additional guarantees, such as non-replayability (this byte could’ve only been sent *now;* e.g. by using continuous HMACs), or perfect forward secrecy (in the case that a peer key is compromised, the content of a past conversation will not be compromised). + +Note that transport-level encryption is not exclusive of application-level encryption or cryptography. Transport-level encryption secures the communication itself, while application-level cryptography is necessary for the application’s use cases (e.g. signatures, randomness, etc.) + +### Will mainnnet networking be untested when it launches? + +Before launching mainnet, the testnet will be switched over to mainnet networking parameters, including Noise handshakes, and other new protocols. This gives us an opportunity to drill coordinated network upgrades and verifying that there are no significant upgradeability gaps. + + +## Gossipsub + +### Why are we using a pub/sub algorithm for block and attestation propagation? + +Pubsub is a technique to broadcast/disseminate data across a network rapidly. Such data is packaged in fire-and-forget messages that do not require a response from every recipient. Peers subscribed to a topic participate in the propagation of messages in that topic. + +The alternative is to maintain a fully connected mesh (all peers connected to each other 1:1), which scales poorly (O(n^2)). + +### Why are we using topics to segregate encodings, yet only support one encoding? + +For future extensibility with almost zero overhead now (besides the extra bytes in the topic name). + +### How do we upgrade gossip channels (e.g. changes in encoding, compression)? + +Such upgrades lead to fragmentation, so they’ll need to be carried out in a coordinated manner most likely during a hard fork. + +### Why are the topics strings and not hashes? + +Topics names have a hierarchical structure. In the future, gossipsub may support wildcard subscriptions (e.g. subscribe to all children topics under a root prefix). Using hashes as topic names would preclude us from leveraging such features going forward. No security guarantees are lost as a result of choosing plaintext topic names, since the domain is finite anyway. + +### Why are there `SHARD_SUBNET_COUNT` subnets, and why is this not defined? + +Depending on the number of validators, it may be more efficient to group shard subnets and might provide better stability for the gossipsub channel. The exact grouping will be dependent on more involved network tests. This constant allows for more flexibility in setting up the network topology for attestation aggregation (as aggregation should happen on each subnet). + +### Why are we sending entire objects in the pubsub and not just hashes? + +Entire objects should be sent to get the greatest propagation speeds. If only hashes are sent, then block and attestation propagation is dependent on recursive requests from each peer. In a hash-only scenario, peers could receive hashes without knowing who to download the actual contents from. Sending entire objects ensures that they get propagated through the entire network. + +### Should clients gossip blocks if they *cannot* validate the proposer signature due to not yet being synced, not knowing the head block, etc? + +The prohibition of unverified-block-gossiping extends to nodes that cannot verify a signature due to not being fully synced to ensure that such (amplified) DOS attacks are not possible. + +### How are we going to discover peers in a gossipsub topic? + +Via discv5 topics. ENRs should not be used for this purpose, as they store identity, location and capability info, not volatile advertisements. + +In the interoperability testnet, all peers will be subscribed to all global beacon chain topics, so discovering peers in specific shard topics will be unnecessary. + +## Req/Resp + +### Why segregate requests into dedicated protocol IDs? + +Requests are segregated by protocol ID to: + +1. Leverage protocol routing in libp2p, such that the libp2p stack will route the incoming stream to the appropriate handler. This allows each the handler function for each request type to be self-contained. For an analogy, think about how you attach HTTP handlers to a REST API server. +2. Version requests independently. In a coarser-grained umbrella protocol, the entire protocol would have to be versioned even if just one field in a single message changed. +3. Enable clients to select the individual requests/versions they support. It would no longer be a strict requirement to support all requests, and clients, in principle, could support a subset of equests and variety of versions. +4. Enable flexibility and agility for clients adopting spec changes that impact the request, by signalling to peers exactly which subset of new/old requests they support. +5. Enable clients to explicitly choose backwards compatibility at the request granularity. Without this, clients would be forced to support entire versions of the coarser request protocol. +6. Parallelise RFCs (or ETH2 EIPs). By decoupling requests from one another, each RFC that affects the request protocol can be deployed/tested/debated independently without relying on a synchronisation point to version the general top-level protocol. + 1. This has the benefit that clients can explicitly choose which RFCs to deploy without buying into all other RFCs that may be included in that top-level version. + 2. Affording this level of granularity with a top-level protocol would imply creating as many variants (e.g. /protocol/43-{a,b,c,d,...}) as the cartesian product of RFCs inflight, O(n^2). +7. Allow us to simplify the payload of requests. Request-id’s and method-ids no longer need to be sent. The encoding/request type and version can all be handled by the framework. + +CAVEAT: the protocol negotiation component in the current version of libp2p is called multistream-select 1.0. It is somewhat naïve and introduces overhead on every request when negotiating streams, although implementation-specific optimizations are possible to save this cost. Multiselect 2.0 will remove this overhead by memoizing previously selected protocols, and modelling shared protocol tables. Fortunately this req/resp protocol is not the expected network bottleneck in the protocol so the additional overhead is not expected to hinder interop testing. More info is to be released from the libp2p community in the coming weeks. + +### Why are messages length-prefixed with a protobuf varint in the SSZ encoding? + +In stream-oriented protocols, we need to delimit messages from one another, so that the reader knows where one message ends and the next one starts. Length-prefixing is an effective solution. Alternatively, one could set a delimiter char/string, but this can readily cause ambiguity if the message itself may contain the delimiter. It also introduces another set of edge cases to model for, thus causing unnecessary complexity, especially if messages are to be compressed (and thus mutated beyond our control). + +That said, in our case, streams are single-use. libp2p streams are full-duplex, and each party is responsible for closing their write side (like in TCP). We therefore use stream closure to mark the end of a request. + +Nevertheless, messages are still length-prefixed to prevent DOS attacks where malicious actors send large amounts of data disguised as a request. A length prefix allows clients to set a maximum limit, and once that limit is read, the client can cease reading and disconnect the stream. This allows a client to determine the exact length of the packet being sent, and it capacitates it to reset the stream early if the other party expresses they intend to send too much data. + +[Protobuf varint](https://developers.google.com/protocol-buffers/docs/encoding#varints) is an efficient technique to encode variable-length ints. Instead of reserving a fixed-size field of as many bytes as necessary to convey the maximum possible value, this field is elastic in exchange for 1-bit overhead per byte. + +### Why do we version protocol strings with ordinals instead of semver? + +Using semver for network protocols is confusing. It is never clear what a change in a field, even if backwards compatible on deserialisation, actually implies. Network protocol agreement should be explicit. Imagine two peers: + +- Peer A supporting v1.1.1 of protocol X. +- Peer B supporting v1.1.2 of protocol X. + +These two peers should never speak to each other because the results can be unpredictable. This is an oversimplification: imagine the same problem with a set of 10 possible versions. We now have 10^2 (100) possible outcomes that peers need to model for. The resulting complexity is unwieldy. + +For this reason, we rely on negotiation of explicit, verbatim protocols. In the above case, peer B would provide backwards compatibility by supporting and advertising both v1.1.1 and v1.1.2 of the protocol. + +Therefore, semver would be relegated to convey expectations at the human level, and it wouldn't do a good job there either, because it's unclear if "backwards-compatibility" and "breaking change" apply only to wire schema level, to behaviour, etc. + +For this reason, we remove semver out of the picture and replace it with ordinals that require explicit agreement and do not mandate a specific policy for changes. + +### Why is it called Req/Resp and not RPC? + +Req/Resp is used to avoid confusion with JSON-RPC and similar user-client interaction mechanisms. + +## Discovery + +### Why are we using discv5 and not libp2p Kademlia DHT? + +discv5 is a standalone protocol, running on UDP on a dedicated port, meant for peer and service discovery only. discv5 supports self-certified, flexible peer records (ENRs) and topic-based advertisement, both of which are, or will be, requirements in this context. + +On the other hand, libp2p Kademlia DHT is a fully-fledged DHT protocol/implementation with content routing and storage capabilities, both of which are irrelevant in this context. + +We assume that ETH1 nodes will evolve to support discv5. By sharing the discovery network between ETH1 and ETH2, we benefit from the additive effect on network size that enhances resilience and resistance against certain attacks, to which smaller networks are more vulnerable. It should also assist light clients of both networks find nodes with specific capabilities. + +discv5 is in the process of being audited. + +### What is the difference between an ENR and a multiaddr, and why are we using ENRs? + +Ethereum Node Records are self-certified node records. Nodes craft and disseminate ENRs for themselves, proving authorship via a cryptographic signature. ENRs are sequentially indexed, enabling conflicts to be resolved. + +ENRs are key-value records with string-indexed ASCII keys. They can store arbitrary information, but EIP-778 specifies a pre-defined dictionary, including IPv4 and IPv6 addresses, secp256k1 public keys, etc. + +Comparing ENRs and multiaddrs is like comparing apples and bananas. ENRs are self-certified containers of identity, addresses, and metadata about a node. Multiaddrs are address strings with the peculiarity that they’re self-describing, composable and future-proof. An ENR can contain multiaddrs, and multiaddrs can be derived securely from the fields of an authenticated ENR. + +discv5 uses ENRs and we will presumably need to: + +1. Add `multiaddr` to the dictionary, so that nodes can advertise their multiaddr under a reserved namespace in ENRs. – and/or – +2. Define a bi-directional conversion function between multiaddrs and the corresponding denormalized fields in an ENR (ip, ip6, tcp, tcp6, etc.), for compatibility with nodes that do not support multiaddr natively (e.g. ETH1 nodes). + +## Compression/Encoding + +### Why are we using SSZ for encoding? + +SSZ is used at the consensus layer and all implementations should have support for ssz encoding/decoding requiring no further dependencies to be added to client implementations. This is a natural choice for serializing objects to be sent across the wire. The actual data in most protocols will be further compressed for efficiency. + +SSZ has well defined schema’s for consensus objects (typically sent across the wire) reducing any serialization schema data that needs to be sent. It also has defined all required types that are required for this network specification. + +### Why are we compressing, and at which layers? + +We compress on the wire to achieve smaller payloads per-message, which, in aggregate, result in higher efficiency, better utilisation of available bandwidth, and overall reduction in network-wide traffic overhead. + +At this time, libp2p does not have an out-of-the-box compression feature that can be dynamically negotiated and layered atop connections and streams, but this will be raised in the libp2p community for consideration. + +This is a non-trivial feature because the behaviour of network IO loops, kernel buffers, chunking, packet fragmentation, amongst others, need to be taken into account. libp2p streams are unbounded streams, whereas compression algorithms work best on bounded byte streams of which we have some prior knowledge. + +Compression tends not to be a one-size-fits-all problem. Lots of variables need careful evaluation, and generic approaches/choices lead to poor size shavings, which may even be counterproductive when factoring in the CPU and memory tradeoff. + +For all these reasons, generically negotiating compression algorithms may be treated as a research problem at the libp2p community, one we’re happy to tackle in the medium-term. + +At this stage, the wisest choice is to consider libp2p a messenger of bytes, and to make application layer participate in compressing those bytes. This looks different depending on the interaction layer: + +- Gossip domain: since gossipsub has a framing protocol and exposes an API, we compress the payload (when dictated by the encoding token in the topic name) prior to publishing the message via the API. No length prefixing is necessary because protobuf takes care of bounding the field in the serialised form. +- Req/Resp domain: since we define custom protocols that operate on byte streams, implementers are encouraged to encapsulate the encoding and compression logic behind MessageReader and MessageWriter components/strategies that can be layered on top of the raw byte streams. + +### Why are using Snappy for compression? + +Snappy is used in Ethereum 1.0. It is well maintained by Google, has good benchmarks and can calculate the size of the uncompressed object without inflating it in memory. This prevents DOS vectors where large uncompressed data is sent. + +### Can I get access to unencrypted bytes on the wire for debugging purposes? + +Yes, you can add loggers in your libp2p protocol handlers to log incoming and outgoing messages. It is recommended to use programming design patterns to encapsulate the logging logic cleanly. + +If your libp2p library relies on frameworks/runtimes such as Netty (jvm) or Node.js (javascript), you can use logging facilities in those frameworks/runtimes to enable message tracing. + +For specific ad-hoc testing scenarios, you can use the [plaintext/2.0.0 secure channel](https://github.com/libp2p/specs/blob/master/plaintext/README.md) (which is essentially no-op encryption or message authentication), in combination with tcpdump or Wireshark to inspect the wire. + +# libp2p Implementations Matrix + +This section will soon contain a matrix showing the maturity/state of the libp2p features required by this spec across the languages in which ETH 2.0 clients are being developed. diff --git a/specs/networking/rpc-interface.md b/specs/networking/rpc-interface.md deleted file mode 100644 index be154075c..000000000 --- a/specs/networking/rpc-interface.md +++ /dev/null @@ -1,283 +0,0 @@ -# Eth 2.0 Networking Spec - RPC Interface - -## Abstract - -The Ethereum 2.0 networking stack uses two modes of communication: a broadcast protocol that gossips information to interested parties via GossipSub, and an RPC protocol that retrieves information from specific clients. This specification defines the RPC protocol. - -The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL", NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [RFC 2119](https://tools.ietf.org/html/rfc2119). - -## Dependencies - -This specification assumes familiarity with the [Messaging](./messaging.md), [Node Identification](./node-identification.md), and [Beacon Chain](../core/0_beacon-chain.md) specifications. - -# Specification - -## Message schemas - -Message body schemas are notated like this: - -``` -( - field_name_1: type - field_name_2: type -) -``` - -Embedded types are serialized as SSZ Containers unless otherwise noted. - -All referenced data structures can be found in the [Beacon Chain](../core/0_beacon-chain.md#data-structures) specification. - -## `libp2p` protocol names - -A "Protocol ID" in `libp2p` parlance refers to a human-readable identifier `libp2p` uses in order to identify sub-protocols and stream messages of different types over the same connection. Peers exchange supported protocol IDs via the `Identify` protocol upon connection. When opening a new stream, peers pin a particular protocol ID to it, and the stream remains contextualized thereafter. Since messages are sent inside a stream, they do not need to bear the protocol ID. - -## RPC-over-`libp2p` - -To facilitate RPC-over-`libp2p`, a single protocol name is used: `/eth/serenity/beacon/rpc/1`. The version number in the protocol name is neither backwards or forwards compatible, and will be incremented whenever changes to the below structures are required. - -Remote method calls are wrapped in a "request" structure: - -``` -( - id: uint64 - method_id: uint16 - body: (message_body...) -) -``` - -and their corresponding responses are wrapped in a "response" structure: - -``` -( - id: uint64 - response_code: uint16 - result: bytes -) -``` - -A union type is used to determine the contents of the `body` field in the request structure. Each "body" entry in the RPC calls below corresponds to one subtype in the `body` type union. - -The details of the RPC-Over-`libp2p` protocol are similar to [JSON-RPC 2.0](https://www.jsonrpc.org/specification). Specifically: - -1. The `id` member is REQUIRED. -2. The `id` member in the response MUST be the same as the value of the `id` in the request. -3. The `id` member MUST be unique within the context of a single connection. Monotonically increasing `id`s are RECOMMENDED. -4. The `method_id` member is REQUIRED. -5. The `result` member is REQUIRED on success. -6. The `result` member is OPTIONAL on errors, and MAY contain additional information about the error. -7. `response_code` MUST be `0` on success. - -Structuring RPC requests in this manner allows multiple calls and responses to be multiplexed over the same stream without switching. Note that this implies that responses MAY arrive in a different order than requests. - -The "method ID" fields in the below messages refer to the `method` field in the request structure above. - -The first 1,000 values in `response_code` are reserved for system use. The following response codes are predefined: - -1. `0`: No error. -2. `10`: Parse error. -2. `20`: Invalid request. -3. `30`: Method not found. -4. `40`: Server error. - -### Alternative for non-`libp2p` clients - -Since some clients are waiting for `libp2p` implementations in their respective languages. As such, they MAY listen for raw TCP messages on port `9000`. To distinguish RPC messages from other messages on that port, a byte prefix of `ETH` (`0x455448`) MUST be prepended to all messages. This option will be removed once `libp2p` is ready in all supported languages. - -## Messages - -### Hello - -**Method ID:** `0` - -**Body**: - -``` -( - network_id: uint8 - chain_id: uint64 - finalized_root: bytes32 - finalized_epoch: uint64 - best_root: bytes32 - best_slot: uint64 -) -``` - -Clients exchange `hello` messages upon connection, forming a two-phase handshake. The first message the initiating client sends MUST be the `hello` message. In response, the receiving client MUST respond with its own `hello` message. - -Clients SHOULD immediately disconnect from one another following the handshake above under the following conditions: - -1. If `network_id` belongs to a different chain, since the client definitionally cannot sync with this client. -2. If the `finalized_root` shared by the peer is not in the client's chain at the expected epoch. For example, if Peer 1 in the diagram below has `(root, epoch)` of `(A, 5)` and Peer 2 has `(B, 3)`, Peer 1 would disconnect because it knows that `B` is not the root in their chain at epoch 3: - -``` - Root A - - +---+ - |xxx| +----+ Epoch 5 - +-+-+ - ^ - | - +-+-+ - | | +----+ Epoch 4 - +-+-+ -Root B ^ - | -+---+ +-+-+ -|xxx+<---+--->+ | +----+ Epoch 3 -+---+ | +---+ - | - +-+-+ - | | +-----------+ Epoch 2 - +-+-+ - ^ - | - +-+-+ - | | +-----------+ Epoch 1 - +---+ -``` - -Once the handshake completes, the client with the higher `finalized_epoch` or `best_slot` (if the clients have equal `finalized_epoch`s) SHOULD request beacon block roots from its counterparty via `beacon_block_roots` (i.e. RPC method `10`). - -### Goodbye - -**Method ID:** `1` - -**Body:** - -``` -( - reason: uint64 -) -``` - -Client MAY send `goodbye` messages upon disconnection. The reason field MAY be one of the following values: - -- `1`: Client shut down. -- `2`: Irrelevant network. -- `3`: Fault/error. - -Clients MAY define custom goodbye reasons as long as the value is larger than `1000`. - -### Get status - -**Method ID:** `2` - -**Request body:** - -``` -( - sha: bytes32 - user_agent: bytes - timestamp: uint64 -) -``` - -**Response body:** - -``` -( - sha: bytes32 - user_agent: bytes - timestamp: uint64 -) -``` - -Returns metadata about the remote node. - -### Request beacon block roots - -**Method ID:** `10` - -**Request body** - -``` -( - start_slot: uint64 - count: uint64 -) -``` - -**Response body:** - -``` -# BlockRootSlot -( - block_root: bytes32 - slot: uint64 -) - -( - roots: []BlockRootSlot -) -``` - -Requests a list of block roots and slots from the peer. The `count` parameter MUST be less than or equal to `32768`. The slots MUST be returned in ascending slot order. - -### Beacon block headers - -**Method ID:** `11` - -**Request body** - -``` -( - start_root: HashTreeRoot - start_slot: uint64 - max_headers: uint64 - skip_slots: uint64 -) -``` - -**Response body:** - -``` -( - headers: []BeaconBlockHeader -) -``` - -Requests beacon block headers from the peer starting from `(start_root, start_slot)`. The response MUST contain no more than `max_headers` headers. `skip_slots` defines the maximum number of slots to skip between blocks. For example, requesting blocks starting at slots `2` a `skip_slots` value of `1` would return the blocks at `[2, 4, 6, 8, 10]`. In cases where a slot is empty for a given slot number, the closest previous block MUST be returned. For example, if slot `4` were empty in the previous example, the returned array would contain `[2, 3, 6, 8, 10]`. If slot three were further empty, the array would contain `[2, 6, 8, 10]`—i.e. duplicate blocks MUST be collapsed. A `skip_slots` value of `0` returns all blocks. - -The function of the `skip_slots` parameter helps facilitate light client sync - for example, in [#459](https://github.com/ethereum/eth2.0-specs/issues/459) - and allows clients to balance the peers from whom they request headers. Clients could, for instance, request every 10th block from a set of peers where each peer has a different starting block in order to populate block data. - -### Beacon block bodies - -**Method ID:** `12` - -**Request body:** - -``` -( - block_roots: []HashTreeRoot -) -``` - -**Response body:** - -``` -( - block_bodies: []BeaconBlockBody -) -``` - -Requests the `block_bodies` associated with the provided `block_roots` from the peer. Responses MUST return `block_roots` in the order provided in the request. If the receiver does not have a particular `block_root`, it must return a zero-value `block_body` (i.e. a `block_body` container with all zero fields). - -### Beacon chain state - -*Note*: This section is preliminary, pending the definition of the data structures to be transferred over the wire during fast sync operations. - -**Method ID:** `13` - -**Request body:** - -``` -( - hashes: []HashTreeRoot -) -``` - -**Response body:** TBD - -Requests contain the hashes of Merkle tree nodes that when merkleized yield the block's `state_root`. - -The response will contain the values that, when hashed, yield the hashes inside the request body. From 4bae9b4bd18ccf350a5a64dc66926e64cceee517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 1 Aug 2019 15:47:11 +0100 Subject: [PATCH 30/89] add table of contents; amend heading level. --- specs/networking/p2p-interface.md | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/specs/networking/p2p-interface.md b/specs/networking/p2p-interface.md index 72f5c0fd6..3b5da6eb0 100644 --- a/specs/networking/p2p-interface.md +++ b/specs/networking/p2p-interface.md @@ -10,7 +10,33 @@ It consists of four main sections: 4. An analysis of the maturity/state of the libp2p features required by this spec across the languages in which ETH 2.0 clients are being developed. ## Table of Contents -[TOC] + + + + + +- [Network Fundamentals](#network-fundamentals) + - [Transport](#transport) + - [Encryption and identification](#encryption-and-identification) + - [Protocol Negotiation](#protocol-negotiation) + - [Multiplexing](#multiplexing) +- [ETH2 network interaction domains](#eth2-network-interaction-domains) + - [Constants](#constants) + - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) + - [The discovery domain: discv5](#the-discovery-domain-discv5) + - [The Req/Resp domain](#the-reqresp-domain) +- [Design Decision Rationale](#design-decision-rationale) + - [Transport](#transport-1) + - [Multiplexing](#multiplexing-1) + - [Protocol Negotiation](#protocol-negotiation-1) + - [Encryption](#encryption) + - [Gossipsub](#gossipsub) + - [Req/Resp](#reqresp) + - [Discovery](#discovery) + - [Compression/Encoding](#compressionencoding) +- [libp2p Implementations Matrix](#libp2p-implementations-matrix) + + # Network Fundamentals @@ -529,7 +555,7 @@ We won’t be using it for mainnet because, amongst other things, it requires se SecIO is not considered secure for the purposes of this spec. -## Why are we using Noise/TLS 1.3 for mainnet? +### Why are we using Noise/TLS 1.3 for mainnet? Copied from the Noise Protocol Framework website: From aa2859299f56f37568299eba93f951da2b7ad9d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 1 Aug 2019 15:56:53 +0100 Subject: [PATCH 31/89] gossip domain: clarify why we use plaintext topic names. --- specs/networking/p2p-interface.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/specs/networking/p2p-interface.md b/specs/networking/p2p-interface.md index 3b5da6eb0..4480773b3 100644 --- a/specs/networking/p2p-interface.md +++ b/specs/networking/p2p-interface.md @@ -604,7 +604,11 @@ Such upgrades lead to fragmentation, so they’ll need to be carried out in a co ### Why are the topics strings and not hashes? -Topics names have a hierarchical structure. In the future, gossipsub may support wildcard subscriptions (e.g. subscribe to all children topics under a root prefix). Using hashes as topic names would preclude us from leveraging such features going forward. No security guarantees are lost as a result of choosing plaintext topic names, since the domain is finite anyway. +Topics names have a hierarchical structure. In the future, gossipsub may support wildcard subscriptions (e.g. subscribe to all children topics under a root prefix) by way of prefix matching. Enforcing hashes for topic names would preclude us from leveraging such features going forward. + +No security or privacy guarantees are lost as a result of choosing plaintext topic names, since the domain is finite anyway, and calculating a digest's preimage would be trivial. + +Furthermore, the ETH2 topic names are shorter their digest equivalents (asuming SHA-256 hash), so hashing topics would bloat messages unnecessarily. ### Why are there `SHARD_SUBNET_COUNT` subnets, and why is this not defined? From e81629df6344e83bd93267bf7c5132a1abe97979 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 1 Aug 2019 16:04:59 +0100 Subject: [PATCH 32/89] document doctoc command for posterity. --- specs/networking/p2p-interface.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/networking/p2p-interface.md b/specs/networking/p2p-interface.md index 4480773b3..20fc803c8 100644 --- a/specs/networking/p2p-interface.md +++ b/specs/networking/p2p-interface.md @@ -11,6 +11,7 @@ It consists of four main sections: ## Table of Contents + From 26d342e918c0e401e2103fb720ee90e1f884da1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 1 Aug 2019 16:57:04 +0100 Subject: [PATCH 33/89] fmt. --- specs/networking/p2p-interface.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/networking/p2p-interface.md b/specs/networking/p2p-interface.md index 20fc803c8..66b1fa694 100644 --- a/specs/networking/p2p-interface.md +++ b/specs/networking/p2p-interface.md @@ -313,8 +313,8 @@ The responder SHOULD send a response promptly, starting with a **single-byte** r It can have one of the following values: - 0: **Success** -- a normal response follows, with contents matching the expected message schema and encoding specified in the request. -- 1: **InvalidRequest** -- the contents of the request are semantically invalid, or the payload is malformed, or could not be understood. The response payload adheres to the ErrorMessage schema (described below). -- 2: **ServerError** -- the responder encountered an error while processing the request. The response payload adheres to the ErrorMessage schema (described below). +- 1: **InvalidRequest** -- the contents of the request are semantically invalid, or the payload is malformed, or could not be understood. The response payload adheres to the `ErrorMessage` schema (described below). +- 2: **ServerError** -- the responder encountered an error while processing the request. The response payload adheres to the `ErrorMessage` schema (described below). Clients MAY use response codes above `128` to indicate alternative, erroneous request-specific responses. From e147684907f2b614c6e8db99f5b60e92dace3714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Fri, 2 Aug 2019 12:35:50 +0100 Subject: [PATCH 34/89] apply editorial suggestions. Co-Authored-By: Hsiao-Wei Wang Co-Authored-By: Preston Van Loon --- specs/networking/p2p-interface.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/networking/p2p-interface.md b/specs/networking/p2p-interface.md index 66b1fa694..fa87635e0 100644 --- a/specs/networking/p2p-interface.md +++ b/specs/networking/p2p-interface.md @@ -79,13 +79,13 @@ The following SecIO parameters MUST be supported by all stacks: - Key agreement: ECDH-P256. - Cipher: AES-128. -- Digest: SHA256. +- Digest: SHA-256. #### Mainnet [Noise Framework](http://www.noiseprotocol.org/) handshakes will be used for mainnet. libp2p Noise support [is in the process of being standardised](https://github.com/libp2p/specs/issues/195) in the libp2p project. -Noise support will presumably include IX, IK and XX handshake patterns, and may rely on Curve25519 keys, ChaCha20 and Poly1305 ciphers, and SHA256 as a hash function. These aspects are being actively debated in the referenced issue [ETH 2.0 implementers are welcome to comment and contribute to the discussion.] +Noise support will presumably include IX, IK and XX handshake patterns, and may rely on Curve25519 keys, ChaCha20 and Poly1305 ciphers, and SHA-256 as a hash function. These aspects are being actively debated in the referenced issue [ETH 2.0 implementers are welcome to comment and contribute to the discussion.] ## Protocol Negotiation @@ -427,7 +427,7 @@ Response Content: ) ``` -Requests count beacon blocks from the peer starting from `start_slot` on the chain defined by `head_block_root`. The response MUST contain no more than count blocks. step defines the slot increment between blocks. For example, requesting blocks starting at `start_slot` 2 with a step value of 2 would return the blocks at [2, 4, 6, …]. In cases where a slot is empty for a given slot number, no block is returned. For example, if slot 4 were empty in the previous example, the returned array would contain [2, 6, …]. A step value of 1 returns all blocks on the range `[start_slot, start_slot + count)`. +Requests count beacon blocks from the peer starting from `start_slot` on the chain defined by `head_block_root`. The response MUST contain no more than count blocks. `step` defines the slot increment between blocks. For example, requesting blocks starting at `start_slot` 2 with a step value of 2 would return the blocks at [2, 4, 6, …]. In cases where a slot is empty for a given slot number, no block is returned. For example, if slot 4 were empty in the previous example, the returned array would contain [2, 6, …]. A step value of 1 returns all blocks on the range `[start_slot, start_slot + count)`. `BeaconBlocks` is primarily used to sync historical blocks. @@ -609,7 +609,7 @@ Topics names have a hierarchical structure. In the future, gossipsub may support No security or privacy guarantees are lost as a result of choosing plaintext topic names, since the domain is finite anyway, and calculating a digest's preimage would be trivial. -Furthermore, the ETH2 topic names are shorter their digest equivalents (asuming SHA-256 hash), so hashing topics would bloat messages unnecessarily. +Furthermore, the ETH2 topic names are shorter their digest equivalents (assuming SHA-256 hash), so hashing topics would bloat messages unnecessarily. ### Why are there `SHARD_SUBNET_COUNT` subnets, and why is this not defined? From d0e1f29bebc71fb43e05b642959521dcea59f5dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Fri, 2 Aug 2019 12:38:59 +0100 Subject: [PATCH 35/89] apply more editorial suggestions. Co-Authored-By: Hsiao-Wei Wang --- specs/networking/p2p-interface.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/networking/p2p-interface.md b/specs/networking/p2p-interface.md index fa87635e0..2661ecdb0 100644 --- a/specs/networking/p2p-interface.md +++ b/specs/networking/p2p-interface.md @@ -229,7 +229,7 @@ Specifications of these parameters can be found in the [ENR Specification](http: In the interoperability testnet, all peers will support all capabilities defined in this document (gossip, full Req/Resp suite, discovery protocol), therefore the ENR record does not need to carry ETH2 capability information, as it would be superfluous. -Nonetheless, ENRs MUST carry a generic `eth2` key with nil value, denoting that the peer is indeed a ETH2 peer, in order to eschew connecting to ETH1 peers. +Nonetheless, ENRs MUST carry a generic `eth2` key with nil value, denoting that the peer is indeed an ETH2 peer, in order to eschew connecting to ETH1 peers. #### Mainnet @@ -609,7 +609,7 @@ Topics names have a hierarchical structure. In the future, gossipsub may support No security or privacy guarantees are lost as a result of choosing plaintext topic names, since the domain is finite anyway, and calculating a digest's preimage would be trivial. -Furthermore, the ETH2 topic names are shorter their digest equivalents (assuming SHA-256 hash), so hashing topics would bloat messages unnecessarily. +Furthermore, the ETH2 topic names are shorter than their digest equivalents (assuming SHA-256 hash), so hashing topics would bloat messages unnecessarily. ### Why are there `SHARD_SUBNET_COUNT` subnets, and why is this not defined? @@ -637,7 +637,7 @@ Requests are segregated by protocol ID to: 1. Leverage protocol routing in libp2p, such that the libp2p stack will route the incoming stream to the appropriate handler. This allows each the handler function for each request type to be self-contained. For an analogy, think about how you attach HTTP handlers to a REST API server. 2. Version requests independently. In a coarser-grained umbrella protocol, the entire protocol would have to be versioned even if just one field in a single message changed. -3. Enable clients to select the individual requests/versions they support. It would no longer be a strict requirement to support all requests, and clients, in principle, could support a subset of equests and variety of versions. +3. Enable clients to select the individual requests/versions they support. It would no longer be a strict requirement to support all requests, and clients, in principle, could support a subset of requests and variety of versions. 4. Enable flexibility and agility for clients adopting spec changes that impact the request, by signalling to peers exactly which subset of new/old requests they support. 5. Enable clients to explicitly choose backwards compatibility at the request granularity. Without this, clients would be forced to support entire versions of the coarser request protocol. 6. Parallelise RFCs (or ETH2 EIPs). By decoupling requests from one another, each RFC that affects the request protocol can be deployed/tested/debated independently without relying on a synchronisation point to version the general top-level protocol. From 26ea675f9c1fda09c8165ace2a32c876ed9dd905 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Fri, 2 Aug 2019 21:12:40 +0200 Subject: [PATCH 36/89] Updates * constants -> configurations * constant name updates * initial validation requirement for attestations * allow aggregated attestations to be published * move discv5 down a bit * additional rationale --- specs/networking/p2p-interface.md | 143 +++++++++++++++++------------- 1 file changed, 80 insertions(+), 63 deletions(-) diff --git a/specs/networking/p2p-interface.md b/specs/networking/p2p-interface.md index 2661ecdb0..ed2047190 100644 --- a/specs/networking/p2p-interface.md +++ b/specs/networking/p2p-interface.md @@ -22,10 +22,10 @@ It consists of four main sections: - [Protocol Negotiation](#protocol-negotiation) - [Multiplexing](#multiplexing) - [ETH2 network interaction domains](#eth2-network-interaction-domains) - - [Constants](#constants) + - [Configuration](#configuration) - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) - - [The discovery domain: discv5](#the-discovery-domain-discv5) - [The Req/Resp domain](#the-reqresp-domain) + - [The discovery domain: discv5](#the-discovery-domain-discv5) - [Design Decision Rationale](#design-decision-rationale) - [Transport](#transport-1) - [Multiplexing](#multiplexing-1) @@ -89,6 +89,8 @@ Noise support will presumably include IX, IK and XX handshake patterns, and may ## Protocol Negotiation +Clients MUST use exact equality when negotiating protocol versions to use and MAY use the version to give priority to higher version numbers. + #### Interop Connection-level and stream-level (see the rationale section below for explanations) protocol negotiation MUST be conducted using [multistream-select v1.0](https://github.com/multiformats/multistream-select/). Its protocol ID is: `/multistream/1.0.0`. @@ -107,16 +109,15 @@ Clients MUST support [mplex](https://github.com/libp2p/specs/tree/master/mplex) # ETH2 network interaction domains -## Constants +## Configuration This section outlines constants that are used in this spec. -- `RQRP_MAX_SIZE`: The max size of uncompressed req/resp messages that clients will allow. - Value: TBD -- `GOSSIP_MAX_SIZE`: The max size of uncompressed gossip messages - Value: 1MB (estimated from expected largest uncompressed block size). -- `SHARD_SUBNET_COUNT`: The number of shard subnets used in the gossipsub protocol. - Value: TBD +| `REQ_RESP_MAX_SIZE` | `TODO` | The max size of uncompressed req/resp messages that clients will allow. | +| `GOSSIP_MAX_SIZE` | `2**20` (= 1048576, 1 MiB) | The max size of uncompressed gossip messages | +| `SHARD_SUBNET_COUNT` | `TODO` | The number of shard subnets used in the gossipsub protocol. | +| `TTFB_TIMEOUT` | `5s` | Maximum time to wait for first byte of request response (time-to-first-byte) | +| `RESP_TIMEOUT` | `10s` | Maximum time for complete response transfer | ## The gossip domain: gossipsub @@ -128,7 +129,7 @@ Clients MUST support the [gossipsub](https://github.com/libp2p/specs/tree/master *Note: Parameters listed here are subject to a large-scale network feasibility study.* -The following gossipsub parameters will be used: +The following gossipsub [parameters](https://github.com/libp2p/specs/tree/master/pubsub/gossipsub#meshsub-an-overlay-mesh-router) will be used: - `D` (topic stable mesh target count): 6 - `D_low` (topic stable mesh low watermark): 4 @@ -147,8 +148,8 @@ Topic strings have form: `/eth2/TopicName/TopicEncoding`. This defines both the There are two main topics used to propagate attestations and beacon blocks to all nodes on the network. Their `TopicName`'s are: -- `beacon_block` - This topic is used solely for propagating new beacon blocks to all nodes on the networks. Blocks are sent in their entirety. Clients who receive a block on this topic MUST validate the block proposer signature before forwarding it across the network. -- `beacon_attestation` - This topic is used to propagate aggregated attestations (in their entirety) to subscribing nodes (typically block proposers) to be included in future blocks. Similarly to beacon blocks, clients will be expected to perform some sort of validation before forwarding, but the precise mechanism is still TBD. +- `beacon_block` - This topic is used solely for propagating new beacon blocks to all nodes on the networks. Blocks are sent in their entirety. Clients MUST validate the block proposer signature before forwarding it across the network. +- `beacon_attestation` - This topic is used to propagate aggregated attestations (in their entirety) to subscribing nodes (typically block proposers) to be included in future blocks. Clients MUST validate that the block being voted for passes validation before forwarding the attestation on the network (TODO: [additional validations](https://github.com/ethereum/eth2.0-specs/issues/1332)). Additional topics are used to propagate lower frequency validator messages. Their `TopicName`’s are: @@ -158,12 +159,14 @@ Additional topics are used to propagate lower frequency validator messages. Thei #### Interop -Unaggregated attestations from all shards are sent to the `beacon_attestation` topic. +Unaggregated and aggregated attestations from all shards are sent to the `beacon_attestation` topic. Clients are not required to publish aggregate attestations but must be able to process them. #### Mainnet Shards are grouped into their own subnets (defined by a shard topic). The number of shard subnets is defined via `SHARD_SUBNET_COUNT` and the shard `shard_number % SHARD_SUBNET_COUNT` is assigned to the topic: `shard{shard_number % SHARD_SUBNET_COUNT}_beacon_attestation`. Unaggregated attestations are sent to the subnet topic. Aggregated attestations are sent to the `beacon_attestation` topic. +TODO: [aggregation strategy](https://github.com/ethereum/eth2.0-specs/issues/1331) + ### Messages Each gossipsub [message](https://github.com/libp2p/go-libp2p-pubsub/blob/master/pb/rpc.proto#L17-L24) has a maximum size of `GOSSIP_MAX_SIZE`. @@ -200,51 +203,6 @@ Topics are post-fixed with an encoding. Encodings define how the payload of a go Implementations MUST use a single encoding. Changing an encoding will require coordination between participating implementations. -## The discovery domain: discv5 - -Discovery Version 5 ([discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md)) is used for peer discovery, both in the interoperability testnet and mainnet. - -`discv5` is a standalone protocol, running on UDP on a dedicated port, meant for peer discovery only. `discv5` supports self-certified, flexible peer records (ENRs) and topic-based advertisement, both of which are (or will be) requirements in this context. - -### Integration into libp2p stacks - -`discv5` SHOULD be integrated into the client’s libp2p stack by implementing an adaptor to make it conform to the [service discovery](https://github.com/libp2p/go-libp2p-core/blob/master/discovery/discovery.go) and [peer routing](https://github.com/libp2p/go-libp2p-core/blob/master/routing/routing.go#L36-L44) abstractions and interfaces (go-libp2p links provided). - -Inputs to operations include peer IDs (when locating a specific peer), or capabilities (when searching for peers with a specific capability), and the outputs will be multiaddrs converted from the ENR records returned by the discv5 backend. - -This integration enables the libp2p stack to subsequently form connections and streams with discovered peers. - -### ENR structure - -The Ethereum Node Record (ENR) for an Ethereum 2.0 client MUST contain the following entries (exclusive of the sequence number and signature, which MUST be present in an ENR): - -- The compressed secp256k1 publickey, 33 bytes (`secp256k1` field). -- An IPv4 address (`ip` field) and/or IPv6 address (`ip6` field). -- A TCP port (`tcp` field) representing the local libp2p listening port. -- A UDP port (`udp` field) representing the local discv5 listening port. - -Specifications of these parameters can be found in the [ENR Specification](http://eips.ethereum.org/EIPS/eip-778). - -#### Interop - -In the interoperability testnet, all peers will support all capabilities defined in this document (gossip, full Req/Resp suite, discovery protocol), therefore the ENR record does not need to carry ETH2 capability information, as it would be superfluous. - -Nonetheless, ENRs MUST carry a generic `eth2` key with nil value, denoting that the peer is indeed an ETH2 peer, in order to eschew connecting to ETH1 peers. - -#### Mainnet - -On mainnet, ENRs MUST include a structure enumerating the capabilities offered by the peer in an efficient manner. The concrete solution is currently undefined. Proposals include using namespaced bloom filters mapping capabilities to specific protocol IDs supported under that capability. - -### Topic advertisement - -#### Interop - -This feature will not be used in the interoperability testnet. - -#### Mainnet - -In mainnet, we plan to use discv5’s topic advertisement feature as a rendezvous facility for peers on shards (thus subscribing to the relevant gossipsub topics). - ## The Req/Resp domain ### Protocol identification @@ -288,7 +246,7 @@ Once a new stream with the protocol ID for the request type has been negotiated, The requester MUST close the write side of the stream once it finishes writing the request message - at this point, the stream will be half-closed. -The requester MUST wait a maximum of **5 seconds** for the first response byte to arrive (time to first byte – or TTFB – timeout). On that happening, the requester will allow further **10 seconds** to receive the full response. +The requester MUST wait a maximum of `TTFB_TIMEOUT` for the first response byte to arrive (time to first byte – or TTFB – timeout). On that happening, the requester will allow further `RESP_TIMEOUT` to receive the full response. If any of these timeouts fire, the requester SHOULD reset the stream and deem the req/resp operation to have failed. @@ -306,11 +264,11 @@ The responder MUST: If steps (1), (2) or (3) fail due to invalid, malformed or inconsistent data, the responder MUST respond in error. Clients tracking peer reputation MAY record such failures, as well as unexpected events, e.g. early stream resets. -The entire request should be read in no more than **5 seconds**. Upon a timeout, the responder SHOULD reset the stream. +The entire request should be read in no more than `RESP_TIMEOUT`. Upon a timeout, the responder SHOULD reset the stream. The responder SHOULD send a response promptly, starting with a **single-byte** response code which determines the contents of the response (`result` particle in the BNF grammar above). -It can have one of the following values: +It can have one of the following values, encoded as a single unsigned byte: - 0: **Success** -- a normal response follows, with contents matching the expected message schema and encoding specified in the request. - 1: **InvalidRequest** -- the contents of the request are semantically invalid, or the payload is malformed, or could not be understood. The response payload adheres to the `ErrorMessage` schema (described below). @@ -461,6 +419,53 @@ Requests blocks by their block roots. The response is a list of `BeaconBlock` wi Clients MUST support requesting blocks since the latest finalized epoch. +## The discovery domain: discv5 + +Discovery Version 5 ([discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md)) is used for peer discovery, both in the interoperability testnet and mainnet. + +`discv5` is a standalone protocol, running on UDP on a dedicated port, meant for peer discovery only. `discv5` supports self-certified, flexible peer records (ENRs) and topic-based advertisement, both of which are (or will be) requirements in this context. + +:warning: Under construction. :warning: + +### Integration into libp2p stacks + +`discv5` SHOULD be integrated into the client’s libp2p stack by implementing an adaptor to make it conform to the [service discovery](https://github.com/libp2p/go-libp2p-core/blob/master/discovery/discovery.go) and [peer routing](https://github.com/libp2p/go-libp2p-core/blob/master/routing/routing.go#L36-L44) abstractions and interfaces (go-libp2p links provided). + +Inputs to operations include peer IDs (when locating a specific peer), or capabilities (when searching for peers with a specific capability), and the outputs will be multiaddrs converted from the ENR records returned by the discv5 backend. + +This integration enables the libp2p stack to subsequently form connections and streams with discovered peers. + +### ENR structure + +The Ethereum Node Record (ENR) for an Ethereum 2.0 client MUST contain the following entries (exclusive of the sequence number and signature, which MUST be present in an ENR): + +- The compressed secp256k1 publickey, 33 bytes (`secp256k1` field). +- An IPv4 address (`ip` field) and/or IPv6 address (`ip6` field). +- A TCP port (`tcp` field) representing the local libp2p listening port. +- A UDP port (`udp` field) representing the local discv5 listening port. + +Specifications of these parameters can be found in the [ENR Specification](http://eips.ethereum.org/EIPS/eip-778). + +#### Interop + +In the interoperability testnet, all peers will support all capabilities defined in this document (gossip, full Req/Resp suite, discovery protocol), therefore the ENR record does not need to carry ETH2 capability information, as it would be superfluous. + +Nonetheless, ENRs MUST carry a generic `eth2` key with nil value, denoting that the peer is indeed an ETH2 peer, in order to eschew connecting to ETH1 peers. + +#### Mainnet + +On mainnet, ENRs MUST include a structure enumerating the capabilities offered by the peer in an efficient manner. The concrete solution is currently undefined. Proposals include using namespaced bloom filters mapping capabilities to specific protocol IDs supported under that capability. + +### Topic advertisement + +#### Interop + +This feature will not be used in the interoperability testnet. + +#### Mainnet + +In mainnet, we plan to use discv5’s topic advertisement feature as a rendezvous facility for peers on shards (thus subscribing to the relevant gossipsub topics). + # Design Decision Rationale ## Transport @@ -601,7 +606,19 @@ For future extensibility with almost zero overhead now (besides the extra bytes ### How do we upgrade gossip channels (e.g. changes in encoding, compression)? -Such upgrades lead to fragmentation, so they’ll need to be carried out in a coordinated manner most likely during a hard fork. +Changing gossipsub / broadcasts requires a coordinated upgrade where all clients start publishing to the new topic together, for example during a hard fork. + +One can envision a two-phase deployment as well where clients start listening to the new topic in a first phase then start publishing some time later, letting the traffic naturally move over to the new topic. + +### Why must all clients use the same gossip topic instead of one negotiated between each peer pair? + +Supporting multiple topics / encodings would require the presence of relayers to translate between encodings and topics so as to avoid network fragmentation where participants have diverging views on the gossiped state, making the protocol more complicated and fragile. + +Gossip protocols typically remember what messages they've seen for a finite period of time based on message identity - if you publish the same message again after that time has passed, it will be re-broadcast - adding a relay delay also makes this scenario more likely. + +One can imagine that in a complicated upgrade scenario, we might have peers publishing the same message on two topics/encodings, but the price here is pretty high in terms of overhead - both computational and networking, so we'd rather avoid that. + +It is permitted for clients to publish data on alternative topics as long as they also publish on the network-wide mandatory topic. ### Why are the topics strings and not hashes? @@ -625,7 +642,7 @@ The prohibition of unverified-block-gossiping extends to nodes that cannot verif ### How are we going to discover peers in a gossipsub topic? -Via discv5 topics. ENRs should not be used for this purpose, as they store identity, location and capability info, not volatile advertisements. +Via discv5 topics. ENRs should not be used for this purpose, as they store identity, location and capability info, not volatile [advertisements](#topic-advertisement). In the interoperability testnet, all peers will be subscribed to all global beacon chain topics, so discovering peers in specific shard topics will be unnecessary. From f0ce20b9225a59a38e1c10fb78df683002ce2254 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Sat, 3 Aug 2019 09:27:49 +0200 Subject: [PATCH 37/89] cleanups --- specs/networking/p2p-interface.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/specs/networking/p2p-interface.md b/specs/networking/p2p-interface.md index ed2047190..eaa767216 100644 --- a/specs/networking/p2p-interface.md +++ b/specs/networking/p2p-interface.md @@ -51,9 +51,7 @@ Even though libp2p is a multi-transport stack (designed to listen on multiple si #### Interop -All implementations MUST support the TCP libp2p transport, and it MUST be enabled for both dialing and listening (i.e. outbound and inbound connections). - -The libp2p TCP transport supports listening on IPv4 and IPv6 addresses (and on multiple simultaneously). Clients SHOULD allow the operator to configure the listen IP addresses and ports, including the addressing schemes (IPv4, IPv6). +All implementations MUST support the TCP libp2p transport, and it MUST be enabled for both dialing and listening (i.e. outbound and inbound connections). The libp2p TCP transport supports listening on IPv4 and IPv6 addresses (and on multiple simultaneously). To facilitate connectivity, and avert possible IPv6 routability/support issues, clients participating in the interoperability testnet MUST expose at least ONE IPv4 endpoint. @@ -236,9 +234,9 @@ result ::= “0” | “1” | “2” | [“128” ... ”255”] The encoding-dependent header may carry metadata or assertions such as the encoded payload length, for integrity and attack proofing purposes. It is not strictly necessary to length-prefix payloads, because req/resp streams are single-use, and stream closures implicitly delimit the boundaries, but certain encodings like SSZ do, for added security. -`encoded-payload` has a maximum byte size of `RQRP_MAX_SIZE`. +`encoded-payload` has a maximum byte size of `REQ_RESP_MAX_SIZE`. -Clients MUST ensure the payload size is less than or equal to `RQRP_MAX_SIZE`, if not, they SHOULD reset the stream immediately. Clients tracking peer reputation MAY decrement the score of the misbehaving peer under this circumstance. +Clients MUST ensure the payload size is less than or equal to `REQ_RESP_MAX_SIZE`, if not, they SHOULD reset the stream immediately. Clients tracking peer reputation MAY decrement the score of the misbehaving peer under this circumstance. #### Requesting side @@ -286,7 +284,7 @@ The `ErrorMessage` schema is: ) ``` -*Note that the String type is encoded as UTF-8 bytes when SSZ-encoded.* +*Note that the String type is encoded as UTF-8 bytes without NULL terminator when SSZ-encoded.* A response therefore has the form: ``` From 94d2368970569990506c89b787780043c1aef314 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Sun, 4 Aug 2019 20:56:41 +0200 Subject: [PATCH 38/89] discuss length-prefixing pro/con, consider for removal, add link --- specs/networking/p2p-interface.md | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/specs/networking/p2p-interface.md b/specs/networking/p2p-interface.md index eaa767216..6f79b5d49 100644 --- a/specs/networking/p2p-interface.md +++ b/specs/networking/p2p-interface.md @@ -664,11 +664,23 @@ CAVEAT: the protocol negotiation component in the current version of libp2p is c ### Why are messages length-prefixed with a protobuf varint in the SSZ encoding? -In stream-oriented protocols, we need to delimit messages from one another, so that the reader knows where one message ends and the next one starts. Length-prefixing is an effective solution. Alternatively, one could set a delimiter char/string, but this can readily cause ambiguity if the message itself may contain the delimiter. It also introduces another set of edge cases to model for, thus causing unnecessary complexity, especially if messages are to be compressed (and thus mutated beyond our control). +We are using single-use streams where each stream is closed at the end of the message - thus libp2p transparently handles message delimiting in the underlying stream. libp2p streams are full-duplex, and each party is responsible for closing their write side (like in TCP). We can therefore use stream closure to mark the end of the request and response independently. -That said, in our case, streams are single-use. libp2p streams are full-duplex, and each party is responsible for closing their write side (like in TCP). We therefore use stream closure to mark the end of a request. +Nevertheless, messages are still length-prefixed - this is now being considered for removal. -Nevertheless, messages are still length-prefixed to prevent DOS attacks where malicious actors send large amounts of data disguised as a request. A length prefix allows clients to set a maximum limit, and once that limit is read, the client can cease reading and disconnect the stream. This allows a client to determine the exact length of the packet being sent, and it capacitates it to reset the stream early if the other party expresses they intend to send too much data. +Advantages of length-prefixing include: + +* Reader can prepare a correctly sized buffer before reading message +* Alignment with protocols like gRPC over HTTP/2 that prefix with length +* Sanity checking of stream closure / message length + +Disadvantages include: + +* Redundant methods of message delimiting - both stream end marker and length prefix +* Harder to stream as length must be known up-front +* Additional code path required to verify length + +In some protocols, adding a length prefix serves as a form of DoS protection against very long messages, allowing the client to abort if an overlong message is about to be sent. In this protocol, we are globally limiting message sizes using `REQ_RESP_MAX_SIZE`, thus an the length prefix does not afford any additional protection. [Protobuf varint](https://developers.google.com/protocol-buffers/docs/encoding#varints) is an efficient technique to encode variable-length ints. Instead of reserving a fixed-size field of as many bytes as necessary to convey the maximum possible value, this field is elastic in exchange for 1-bit overhead per byte. @@ -728,7 +740,7 @@ SSZ has well defined schema’s for consensus objects (typically sent across the We compress on the wire to achieve smaller payloads per-message, which, in aggregate, result in higher efficiency, better utilisation of available bandwidth, and overall reduction in network-wide traffic overhead. -At this time, libp2p does not have an out-of-the-box compression feature that can be dynamically negotiated and layered atop connections and streams, but this will be raised in the libp2p community for consideration. +At this time, libp2p does not have an out-of-the-box compression feature that can be dynamically negotiated and layered atop connections and streams, but is [being considered](https://github.com/libp2p/libp2p/issues/81). This is a non-trivial feature because the behaviour of network IO loops, kernel buffers, chunking, packet fragmentation, amongst others, need to be taken into account. libp2p streams are unbounded streams, whereas compression algorithms work best on bounded byte streams of which we have some prior knowledge. From dee70df3a9756f83d2424f8c5ca1e154eb1b7a64 Mon Sep 17 00:00:00 2001 From: JSON <49416440+JSON@users.noreply.github.com> Date: Fri, 9 Aug 2019 00:30:05 -0500 Subject: [PATCH 39/89] doc standardization for networking spec (#1338) * Update p2p-interface.md * Update p2p-interface.md * Update p2p-interface.md * Update specs/networking/p2p-interface.md Co-Authored-By: Hsiao-Wei Wang * Update specs/networking/p2p-interface.md Co-Authored-By: Hsiao-Wei Wang * Update specs/networking/p2p-interface.md Co-Authored-By: Hsiao-Wei Wang * Update specs/networking/p2p-interface.md Co-Authored-By: Hsiao-Wei Wang * Update specs/networking/p2p-interface.md Co-Authored-By: Hsiao-Wei Wang * Update specs/networking/p2p-interface.md Co-Authored-By: Hsiao-Wei Wang * Update specs/networking/p2p-interface.md Co-Authored-By: Hsiao-Wei Wang * Update specs/networking/p2p-interface.md Co-Authored-By: Hsiao-Wei Wang * Update specs/networking/p2p-interface.md Co-Authored-By: Hsiao-Wei Wang * Update specs/networking/p2p-interface.md Co-Authored-By: Hsiao-Wei Wang --- specs/networking/p2p-interface.md | 229 +++++++++++++++--------------- 1 file changed, 115 insertions(+), 114 deletions(-) diff --git a/specs/networking/p2p-interface.md b/specs/networking/p2p-interface.md index 6f79b5d49..53e203ca6 100644 --- a/specs/networking/p2p-interface.md +++ b/specs/networking/p2p-interface.md @@ -1,45 +1,45 @@ -# Overview +# Ethereum 2.0 networking specification -This document contains the network specification for Ethereum 2.0 clients. +This document contains the networking specification for Ethereum 2.0 clients. It consists of four main sections: -1. A specification of the network fundamentals detailing the two network configurations: interoperability test network, and mainnet launch. -2. A specification of the three network interaction _domains_ of ETH2.0: (a) the gossip domain, (b) the discovery domain, \(c\) the Req/Resp domain. +1. A specification of the network fundamentals detailing the two network configurations: interoperability test network and mainnet launch. +2. A specification of the three network interaction *domains* of Eth 2.0: (a) the gossip domain, (b) the discovery domain, and (c) the Req/Resp domain. 3. The rationale and further explanation for the design choices made in the previous two sections. -4. An analysis of the maturity/state of the libp2p features required by this spec across the languages in which ETH 2.0 clients are being developed. +4. An analysis of the maturity/state of the libp2p features required by this spec across the languages in which Eth 2.0 clients are being developed. -## Table of Contents +## Table of contents -- [Network Fundamentals](#network-fundamentals) +- [Network fundamentals](#network-fundamentals) - [Transport](#transport) - [Encryption and identification](#encryption-and-identification) - - [Protocol Negotiation](#protocol-negotiation) + - [Protocol negotiation](#protocol-negotiation) - [Multiplexing](#multiplexing) -- [ETH2 network interaction domains](#eth2-network-interaction-domains) +- [Eth 2.0 network interaction domains](#eth-20-network-interaction-domains) - [Configuration](#configuration) - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) - [The Req/Resp domain](#the-reqresp-domain) - [The discovery domain: discv5](#the-discovery-domain-discv5) -- [Design Decision Rationale](#design-decision-rationale) +- [Design decision rationale](#design-decision-rationale) - [Transport](#transport-1) - [Multiplexing](#multiplexing-1) - - [Protocol Negotiation](#protocol-negotiation-1) + - [Protocol negotiation](#protocol-negotiation-1) - [Encryption](#encryption) - [Gossipsub](#gossipsub) - [Req/Resp](#reqresp) - [Discovery](#discovery) - [Compression/Encoding](#compressionencoding) -- [libp2p Implementations Matrix](#libp2p-implementations-matrix) +- [libp2p implementations matrix](#libp2p-implementations-matrix) -# Network Fundamentals +# Network fundamentals This section outlines the specification for the networking stack in Ethereum 2.0 clients. @@ -53,9 +53,9 @@ Even though libp2p is a multi-transport stack (designed to listen on multiple si All implementations MUST support the TCP libp2p transport, and it MUST be enabled for both dialing and listening (i.e. outbound and inbound connections). The libp2p TCP transport supports listening on IPv4 and IPv6 addresses (and on multiple simultaneously). -To facilitate connectivity, and avert possible IPv6 routability/support issues, clients participating in the interoperability testnet MUST expose at least ONE IPv4 endpoint. +To facilitate connectivity and avert possible IPv6 routability/support issues, clients participating in the interoperability testnet MUST expose at least ONE IPv4 endpoint. -All listening endpoints must be publicly dialable, and thus not rely on libp2p circuit relay, AutoNAT or AutoRelay facilities. +All listening endpoints must be publicly dialable, and thus not rely on libp2p circuit relay, AutoNAT, or AutoRelay facilities. Nodes operating behind a NAT, or otherwise undialable by default (e.g. container runtime, firewall, etc.), MUST have their infrastructure configured to enable inbound traffic on the announced public listening endpoint. @@ -65,7 +65,7 @@ All requirements from the interoperability testnet apply, except for the IPv4 ad At this stage, clients are licensed to drop IPv4 support if they wish to do so, cognizant of the potential disadvantages in terms of Internet-wide routability/support. Clients MAY choose to listen only on IPv6, but MUST retain capability to dial both IPv4 and IPv6 addresses. -Usage of circuit relay, AutoNAT or AutoRelay will be specifically re-examined closer to the time. +Usage of circuit relay, AutoNAT, or AutoRelay will be specifically re-examined closer to the time. ## Encryption and identification @@ -81,9 +81,9 @@ The following SecIO parameters MUST be supported by all stacks: #### Mainnet -[Noise Framework](http://www.noiseprotocol.org/) handshakes will be used for mainnet. libp2p Noise support [is in the process of being standardised](https://github.com/libp2p/specs/issues/195) in the libp2p project. +[Noise Framework](http://www.noiseprotocol.org/) handshakes will be used for mainnet. libp2p Noise support [is in the process of being standardized](https://github.com/libp2p/specs/issues/195) in the libp2p project. -Noise support will presumably include IX, IK and XX handshake patterns, and may rely on Curve25519 keys, ChaCha20 and Poly1305 ciphers, and SHA-256 as a hash function. These aspects are being actively debated in the referenced issue [ETH 2.0 implementers are welcome to comment and contribute to the discussion.] +Noise support will presumably include IX, IK, and XX handshake patterns, and may rely on Curve25519 keys, ChaCha20 and Poly1305 ciphers, and SHA-256 as a hash function. These aspects are being actively debated in the referenced issue (Eth 2.0 implementers are welcome to comment and contribute to the discussion). ## Protocol Negotiation @@ -91,7 +91,7 @@ Clients MUST use exact equality when negotiating protocol versions to use and MA #### Interop -Connection-level and stream-level (see the rationale section below for explanations) protocol negotiation MUST be conducted using [multistream-select v1.0](https://github.com/multiformats/multistream-select/). Its protocol ID is: `/multistream/1.0.0`. +Connection-level and stream-level (see the [Rationale](#design-decision-rationale) section below for explanations) protocol negotiation MUST be conducted using [multistream-select v1.0](https://github.com/multiformats/multistream-select/). Its protocol ID is: `/multistream/1.0.0`. #### Mainnet @@ -103,19 +103,21 @@ During connection bootstrapping, libp2p dynamically negotiates a mutually suppor Two multiplexers are commonplace in libp2p implementations: [mplex](https://github.com/libp2p/specs/tree/master/mplex) and [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md). Their protocol IDs are, respectively: `/mplex/6.7.0` and `/yamux/1.0.0`. -Clients MUST support [mplex](https://github.com/libp2p/specs/tree/master/mplex) and MAY support [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md). If both are supported by the client, yamux must take precedence during negotiation. See the Rationale section of this document for tradeoffs. +Clients MUST support [mplex](https://github.com/libp2p/specs/tree/master/mplex) and MAY support [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md). If both are supported by the client, yamux must take precedence during negotiation. See the [Rationale](#design-decision-rationale) section below for tradeoffs. -# ETH2 network interaction domains +# Eth 2.0 network interaction domains ## Configuration This section outlines constants that are used in this spec. -| `REQ_RESP_MAX_SIZE` | `TODO` | The max size of uncompressed req/resp messages that clients will allow. | -| `GOSSIP_MAX_SIZE` | `2**20` (= 1048576, 1 MiB) | The max size of uncompressed gossip messages | +| Name | Value | Description | +|---|---|---| +| `REQ_RESP_MAX_SIZE` | `TODO` | The maximum size of uncompressed req/resp messages that clients will allow. | +| `GOSSIP_MAX_SIZE` | `2**20` (= 1048576, 1 MiB) | The maximum size of uncompressed gossip messages. | | `SHARD_SUBNET_COUNT` | `TODO` | The number of shard subnets used in the gossipsub protocol. | -| `TTFB_TIMEOUT` | `5s` | Maximum time to wait for first byte of request response (time-to-first-byte) | -| `RESP_TIMEOUT` | `10s` | Maximum time for complete response transfer | +| `TTFB_TIMEOUT` | `5s` | The maximum time to wait for first byte of request response (time-to-first-byte). | +| `RESP_TIMEOUT` | `10s` | The maximum time for complete response transfer. | ## The gossip domain: gossipsub @@ -125,7 +127,7 @@ Clients MUST support the [gossipsub](https://github.com/libp2p/specs/tree/master **Gossipsub Parameters** -*Note: Parameters listed here are subject to a large-scale network feasibility study.* +*Note*: Parameters listed here are subject to a large-scale network feasibility study. The following gossipsub [parameters](https://github.com/libp2p/specs/tree/master/pubsub/gossipsub#meshsub-an-overlay-mesh-router) will be used: @@ -140,16 +142,16 @@ The following gossipsub [parameters](https://github.com/libp2p/specs/tree/master ### Topics -Topics are plain UTF-8 strings, and are encoded on the wire as determined by protobuf (gossipsub messages are enveloped in protobuf messages). +Topics are plain UTF-8 strings and are encoded on the wire as determined by protobuf (gossipsub messages are enveloped in protobuf messages). Topic strings have form: `/eth2/TopicName/TopicEncoding`. This defines both the type of data being sent on the topic and how the data field of the message is encoded. (Further details can be found in [Messages](#Messages)). -There are two main topics used to propagate attestations and beacon blocks to all nodes on the network. Their `TopicName`'s are: +There are two main topics used to propagate attestations and beacon blocks to all nodes on the network. Their `TopicName`s are: - `beacon_block` - This topic is used solely for propagating new beacon blocks to all nodes on the networks. Blocks are sent in their entirety. Clients MUST validate the block proposer signature before forwarding it across the network. - `beacon_attestation` - This topic is used to propagate aggregated attestations (in their entirety) to subscribing nodes (typically block proposers) to be included in future blocks. Clients MUST validate that the block being voted for passes validation before forwarding the attestation on the network (TODO: [additional validations](https://github.com/ethereum/eth2.0-specs/issues/1332)). -Additional topics are used to propagate lower frequency validator messages. Their `TopicName`’s are: +Additional topics are used to propagate lower frequency validator messages. Their `TopicName`s are: - `voluntary_exit` - This topic is used solely for propagating voluntary validator exits to proposers on the network. Voluntary exits are sent in their entirety. Clients who receive a voluntary exit on this topic MUST validate the conditions within `process_voluntary_exit` before forwarding it across the network. - `proposer_slashing` - This topic is used solely for propagating proposer slashings to proposers on the network. Proposer slashings are sent in their entirety. Clients who receive a proposer slashing on this topic MUST validate the conditions within `process_proposer_slashing` before forwarding it across the network. @@ -193,11 +195,11 @@ Topics are post-fixed with an encoding. Encodings define how the payload of a go #### Interop -- `ssz` - All objects are SSZ-encoded. Example: The beacon block topic string is: `/beacon_block/ssz` and the data field of a gossipsub message is an ssz-encoded `BeaconBlock`. +- `ssz` - All objects are [SSZ-encoded](#ssz-encoding). Example: The beacon block topic string is `/beacon_block/ssz`, and the data field of a gossipsub message is an ssz-encoded `BeaconBlock`. #### Mainnet -- `ssz_snappy` - All objects are ssz-encoded and then compressed with snappy. Example: The beacon attestation topic string is: `/beacon_attestation/ssz_snappy` and the data field of a gossipsub message is an `Attestation` that has been ssz-encoded then compressed with snappy. +- `ssz_snappy` - All objects are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy). Example: The beacon attestation topic string is `/beacon_attestation/ssz_snappy`, and the data field of a gossipsub message is an `Attestation` that has been SSZ-encoded and then compressed with Snappy. Implementations MUST use a single encoding. Changing an encoding will require coordination between participating implementations. @@ -215,16 +217,16 @@ With: - `ProtocolPrefix` - messages are grouped into families identified by a shared libp2p protocol name prefix. In this case, we use `/eth2/beacon_chain/req`. - `MessageName` - each request is identified by a name consisting of English alphabet, digits and underscores (`_`). -- `SchemaVersion` - an ordinal version number (e.g. 1, 2, 3…) Each schema is versioned to facilitate backward and forward-compatibility when possible. -- `Encoding` - while the schema defines the data types in more abstract terms, the encoding strategy describes a specific representation of bytes that will be transmitted over the wire. See the [Encodings](#Encoding-strategies) section, for further details. +- `SchemaVersion` - an ordinal version number (e.g. 1, 2, 3…). Each schema is versioned to facilitate backward and forward-compatibility when possible. +- `Encoding` - while the schema defines the data types in more abstract terms, the encoding strategy describes a specific representation of bytes that will be transmitted over the wire. See the [Encodings](#Encoding-strategies) section for further details. -This protocol segregation allows libp2p `multistream-select 1.0` / `multiselect 2.0` to handle the request type, version and encoding negotiation before establishing the underlying streams. +This protocol segregation allows libp2p `multistream-select 1.0` / `multiselect 2.0` to handle the request type, version, and encoding negotiation before establishing the underlying streams. ### Req/Resp interaction We use ONE stream PER request/response interaction. Streams are closed when the interaction finishes, whether in success or in error. -Request/response messages MUST adhere to the encoding specified in the protocol name, and follow this structure (relaxed BNF grammar): +Request/response messages MUST adhere to the encoding specified in the protocol name and follow this structure (relaxed BNF grammar): ``` request ::= | @@ -232,19 +234,19 @@ response ::= | | result ::= “0” | “1” | “2” | [“128” ... ”255”] ``` -The encoding-dependent header may carry metadata or assertions such as the encoded payload length, for integrity and attack proofing purposes. It is not strictly necessary to length-prefix payloads, because req/resp streams are single-use, and stream closures implicitly delimit the boundaries, but certain encodings like SSZ do, for added security. +The encoding-dependent header may carry metadata or assertions such as the encoded payload length, for integrity and attack proofing purposes. Because req/resp streams are single-use and stream closures implicitly delimit the boundaries, it is not strictly necessary to length-prefix payloads; however, certain encodings like SSZ do, for added security. `encoded-payload` has a maximum byte size of `REQ_RESP_MAX_SIZE`. -Clients MUST ensure the payload size is less than or equal to `REQ_RESP_MAX_SIZE`, if not, they SHOULD reset the stream immediately. Clients tracking peer reputation MAY decrement the score of the misbehaving peer under this circumstance. +Clients MUST ensure the payload size is less than or equal to `REQ_RESP_MAX_SIZE`; if not, they SHOULD reset the stream immediately. Clients tracking peer reputation MAY decrement the score of the misbehaving peer under this circumstance. #### Requesting side Once a new stream with the protocol ID for the request type has been negotiated, the full request message should be sent immediately. It should be encoded according to the encoding strategy. -The requester MUST close the write side of the stream once it finishes writing the request message - at this point, the stream will be half-closed. +The requester MUST close the write side of the stream once it finishes writing the request message—at this point, the stream will be half-closed. -The requester MUST wait a maximum of `TTFB_TIMEOUT` for the first response byte to arrive (time to first byte – or TTFB – timeout). On that happening, the requester will allow further `RESP_TIMEOUT` to receive the full response. +The requester MUST wait a maximum of `TTFB_TIMEOUT` for the first response byte to arrive (time to first byte—or TTFB—timeout). On that happening, the requester will allow further `RESP_TIMEOUT` to receive the full response. If any of these timeouts fire, the requester SHOULD reset the stream and deem the req/resp operation to have failed. @@ -255,12 +257,12 @@ Once a new stream with the protocol ID for the request type has been negotiated, The responder MUST: 1. Use the encoding strategy to read the optional header. -2. If there are any length assertions for length `N`, it should read exactly `N` bytes from the stream, at which point an EOF should arise (no more bytes). Should this is not the case, it should be treated as a failure. +2. If there are any length assertions for length `N`, it should read exactly `N` bytes from the stream, at which point an EOF should arise (no more bytes). Should this not be the case, it should be treated as a failure. 3. Deserialize the expected type, and process the request. 4. Write the response (result, optional header, payload). 5. Close their write side of the stream. At this point, the stream will be fully closed. -If steps (1), (2) or (3) fail due to invalid, malformed or inconsistent data, the responder MUST respond in error. Clients tracking peer reputation MAY record such failures, as well as unexpected events, e.g. early stream resets. +If steps (1), (2), or (3) fail due to invalid, malformed, or inconsistent data, the responder MUST respond in error. Clients tracking peer reputation MAY record such failures, as well as unexpected events, e.g. early stream resets. The entire request should be read in no more than `RESP_TIMEOUT`. Upon a timeout, the responder SHOULD reset the stream. @@ -274,7 +276,7 @@ It can have one of the following values, encoded as a single unsigned byte: Clients MAY use response codes above `128` to indicate alternative, erroneous request-specific responses. -The range `[3, 127]` is RESERVED for future usages, and should be treated as error if not recognised expressly. +The range `[3, 127]` is RESERVED for future usages, and should be treated as error if not recognized expressly. The `ErrorMessage` schema is: @@ -284,7 +286,7 @@ The `ErrorMessage` schema is: ) ``` -*Note that the String type is encoded as UTF-8 bytes without NULL terminator when SSZ-encoded.* +*Note*: The String type is encoded as UTF-8 bytes without NULL terminator when SSZ-encoded. A response therefore has the form: ``` @@ -292,22 +294,22 @@ A response therefore has the form: | result | header (opt) | encoded_response | +--------+--------+--------+--------+--------+--------+ ``` -Here `result` represents the 1-byte response code. +Here, `result` represents the 1-byte response code. ### Encoding strategies The token of the negotiated protocol ID specifies the type of encoding to be used for the req/resp interaction. Two values are possible at this time: -- `ssz`: the contents are [SSZ](https://github.com/ethereum/eth2.0-specs/blob/192442be51a8a6907d6401dffbf5c73cb220b760/specs/networking/libp2p-standardization.md#ssz-encoding) encoded. This encoding type MUST be supported by all clients. -- `ssz_snappy`: the contents are SSZ encoded, and subsequently compressed with [Snappy](https://github.com/google/snappy). MAY be supported in the interoperability testnet; and MUST be supported in mainnet. +- `ssz`: The contents are [SSZ-encoded](#ssz-encoding). This encoding type MUST be supported by all clients. +- `ssz_snappy`: The contents are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy). MAY be supported in the interoperability testnet; MUST be supported in mainnet. -#### SSZ encoding strategy (with or without Snappy) +#### SSZ-encoding strategy (with or without Snappy) -The [SimpleSerialize (SSZ) specification](https://github.com/ethereum/eth2.0-specs/blob/192442be51a8a6907d6401dffbf5c73cb220b760/specs/simple-serialize.md) outlines how objects are SSZ-encoded. If the Snappy variant is selected, we feed the serialised form to the Snappy compressor on encoding. The inverse happens on decoding. +The [SimpleSerialize (SSZ) specification](../simple-serialize.md) outlines how objects are SSZ-encoded. If the Snappy variant is selected, we feed the serialized form to the Snappy compressor on encoding. The inverse happens on decoding. **Encoding-dependent header:** Req/Resp protocols using the `ssz` or `ssz_snappy` encoding strategies MUST prefix all encoded and compressed (if applicable) payloads with an unsigned [protobuf varint](https://developers.google.com/protocol-buffers/docs/encoding#varints). -Note that parameters defined as `[]VariableName` are SSZ-encoded containerless vectors. +*Note*: Parameters defined as `[]VariableName` are SSZ-encoded containerless vectors. ### Messages @@ -327,10 +329,10 @@ Note that parameters defined as `[]VariableName` are SSZ-encoded containerless v ``` The fields are: -- `fork_version`: The beacon_state `Fork` version -- `finalized_root`: The latest finalized root the node knows about -- `finalized_epoch`: The latest finalized epoch the node knows about -- `head_root`: The block hash tree root corresponding to the head of the chain as seen by the sending node +- `fork_version`: The beacon_state `Fork` version. +- `finalized_root`: The latest finalized root the node knows about. +- `finalized_epoch`: The latest finalized epoch the node knows about. +- `head_root`: The block hash tree root corresponding to the head of the chain as seen by the sending node. - `head_slot`: The slot corresponding to the `head_root`. Clients exchange hello messages upon connection, forming a two-phase handshake. The first message the initiating client sends MUST be the hello message. In response, the receiving client MUST respond with its own hello message. @@ -413,7 +415,7 @@ Response Content: Requests blocks by their block roots. The response is a list of `BeaconBlock` with the same length as the request. Blocks are returned in order of the request and any missing/unknown blocks are left empty (SSZ null `BeaconBlock`). -`RecentBeaconBlocks` is primarily used to recover recent blocks, for example when receiving a block or attestation whose parent is unknown. +`RecentBeaconBlocks` is primarily used to recover recent blocks (ex. when receiving a block or attestation whose parent is unknown). Clients MUST support requesting blocks since the latest finalized epoch. @@ -446,9 +448,9 @@ Specifications of these parameters can be found in the [ENR Specification](http: #### Interop -In the interoperability testnet, all peers will support all capabilities defined in this document (gossip, full Req/Resp suite, discovery protocol), therefore the ENR record does not need to carry ETH2 capability information, as it would be superfluous. +In the interoperability testnet, all peers will support all capabilities defined in this document (gossip, full Req/Resp suite, discovery protocol), therefore the ENR record does not need to carry Eth 2.0 capability information, as it would be superfluous. -Nonetheless, ENRs MUST carry a generic `eth2` key with nil value, denoting that the peer is indeed an ETH2 peer, in order to eschew connecting to ETH1 peers. +Nonetheless, ENRs MUST carry a generic `eth2` key with nil value, denoting that the peer is indeed an Eth 2.0 peer, in order to eschew connecting to Eth 1.0 peers. #### Mainnet @@ -464,15 +466,15 @@ This feature will not be used in the interoperability testnet. In mainnet, we plan to use discv5’s topic advertisement feature as a rendezvous facility for peers on shards (thus subscribing to the relevant gossipsub topics). -# Design Decision Rationale +# Design decision rationale ## Transport ### Why are we defining specific transports? -libp2p peers can listen on multiple transports concurrently, and these can change over time. multiaddrs not only encode the address, but also the transport to be used to dial. +libp2p peers can listen on multiple transports concurrently, and these can change over time. Multiaddrs encode not only the address but also the transport to be used to dial. -Due to this dynamic nature, agreeing on specific transports like TCP, QUIC or WebSockets on paper becomes irrelevant. +Due to this dynamic nature, agreeing on specific transports like TCP, QUIC, or WebSockets on paper becomes irrelevant. However, it is useful to define a minimum baseline for interoperability purposes. @@ -480,34 +482,34 @@ However, it is useful to define a minimum baseline for interoperability purposes Clients may support other transports such as libp2p QUIC, WebSockets, and WebRTC transports, if available in the language of choice. While interoperability shall not be harmed by lack of such support, the advantages are desirable: -- better latency, performance and other QoS characteristics (QUIC). -- paving the way for interfacing with future light clients (WebSockets, WebRTC). +- Better latency, performance, and other QoS characteristics (QUIC). +- Paving the way for interfacing with future light clients (WebSockets, WebRTC). -The libp2p QUIC transport inherently relies on TLS 1.3 per requirement in section 7 of the [QUIC protocol specification](https://tools.ietf.org/html/draft-ietf-quic-transport-22#section-7), and the accompanying [QUIC-TLS document](https://tools.ietf.org/html/draft-ietf-quic-tls-22). +The libp2p QUIC transport inherently relies on TLS 1.3 per requirement in section 7 of the [QUIC protocol specification](https://tools.ietf.org/html/draft-ietf-quic-transport-22#section-7) and the accompanying [QUIC-TLS document](https://tools.ietf.org/html/draft-ietf-quic-tls-22). -The usage of one handshake procedure or the other shall be transparent to the ETH 2.0 application layer, once the libp2p Host/Node object has been configured appropriately. +The usage of one handshake procedure or the other shall be transparent to the Eth 2.0 application layer, once the libp2p Host/Node object has been configured appropriately. -### What are advantages of using TCP/QUIC/Websockets? +### What are the advantages of using TCP/QUIC/Websockets? -TCP is a reliable, ordered, full-duplex, congestion controlled network protocol that powers much of the Internet as we know it today. HTTP/1.1 and HTTP/2 run atop TCP. +TCP is a reliable, ordered, full-duplex, congestion-controlled network protocol that powers much of the Internet as we know it today. HTTP/1.1 and HTTP/2 run atop TCP. -QUIC is a new protocol that’s in the final stages of specification by the IETF QUIC WG. It emerged from Google’s SPDY experiment. The QUIC transport is undoubtedly promising. It’s UDP based yet reliable, ordered, reduces latency vs. TCP, is multiplexed, natively secure (TLS 1.3), offers stream-level and connection-level congestion control (thus removing head-of-line blocking), 0-RTT connection establishment, and endpoint migration, amongst other features. UDP also has better NAT traversal properties than TCP -- something we desperately pursue in peer-to-peer networks. +QUIC is a new protocol that’s in the final stages of specification by the IETF QUIC WG. It emerged from Google’s SPDY experiment. The QUIC transport is undoubtedly promising. It’s UDP-based yet reliable, ordered, multiplexed, natively secure (TLS 1.3), reduces latency vs. TCP, and offers stream-level and connection-level congestion control (thus removing head-of-line blocking), 0-RTT connection establishment, and endpoint migration, amongst other features. UDP also has better NAT traversal properties than TCP—something we desperately pursue in peer-to-peer networks. -QUIC is being adopted as the underlying protocol for HTTP/3. This has the potential to award us censorship resistance via deep packet inspection for free. Provided that we use the same port numbers and encryption mechanisms as HTTP/3, our traffic may be indistinguishable from standard web traffic, and we may only become subject to standard IP-based firewall filtering -- something we can counteract via other mechanisms. +QUIC is being adopted as the underlying protocol for HTTP/3. This has the potential to award us censorship resistance via deep packet inspection for free. Provided that we use the same port numbers and encryption mechanisms as HTTP/3, our traffic may be indistinguishable from standard web traffic, and we may only become subject to standard IP-based firewall filtering—something we can counteract via other mechanisms. -WebSockets and/or WebRTC transports are necessary for interaction with browsers, and will become increasingly important as we incorporate browser-based light clients to the ETH2 network. +WebSockets and/or WebRTC transports are necessary for interaction with browsers, and will become increasingly important as we incorporate browser-based light clients to the Eth 2.0 network. ### Why do we not just support a single transport? Networks evolve. Hardcoding design decisions leads to ossification, preventing the evolution of networks alongside the state of the art. Introducing changes on an ossified protocol is very costly, and sometimes, downright impracticable without causing undesirable breakage. -Modelling for upgradeability and dynamic transport selection from the get-go lays the foundation for a future-proof stack. +Modeling for upgradeability and dynamic transport selection from the get-go lays the foundation for a future-proof stack. -Clients can adopt new transports without breaking old ones; and the multi-transport ability enables constrained and sandboxed environments (e.g. browsers, embedded devices) to interact with the network as first-class citizens via suitable/native transports (e.g. WSS), without the need for proxying or trust delegation to servers. +Clients can adopt new transports without breaking old ones, and the multi-transport ability enables constrained and sandboxed environments (e.g. browsers, embedded devices) to interact with the network as first-class citizens via suitable/native transports (e.g. WSS), without the need for proxying or trust delegation to servers. ### Why are we not using QUIC for mainnet from the start? -The QUIC standard is still not finalised (at working draft 22 at the time of writing), and not all mainstream runtimes/languages have mature, standard, and/or fully-interoperable [QUIC support](https://github.com/quicwg/base-drafts/wiki/Implementations). One remarkable example is node.js, where the QUIC implementation is [in early development](https://github.com/nodejs/quic). +The QUIC standard is still not finalized (at working draft 22 at the time of writing), and not all mainstream runtimes/languages have mature, standard, and/or fully-interoperable [QUIC support](https://github.com/quicwg/base-drafts/wiki/Implementations). One remarkable example is node.js, where the QUIC implementation is [in early development](https://github.com/nodejs/quic). ## Multiplexing @@ -515,17 +517,17 @@ The QUIC standard is still not finalised (at working draft 22 at the time of wri [Yamux](https://github.com/hashicorp/yamux/blob/master/spec.md) is a multiplexer invented by Hashicorp that supports stream-level congestion control. Implementations exist in a limited set of languages, and it’s not a trivial piece to develop. -Conscious of that, the libp2p community conceptualised [mplex](https://github.com/libp2p/specs/blob/master/mplex/README.md) as a simple, minimal multiplexer for usage with libp2p. It does not support stream-level congestion control, and is subject to head-of-line blocking. +Conscious of that, the libp2p community conceptualized [mplex](https://github.com/libp2p/specs/blob/master/mplex/README.md) as a simple, minimal multiplexer for usage with libp2p. It does not support stream-level congestion control and is subject to head-of-line blocking. -Overlay multiplexers are not necessary with QUIC, as the protocol provides native multiplexing, but they need to be layered atop TCP, WebSockets, and other transports that lack such support. +Overlay multiplexers are not necessary with QUIC since the protocol provides native multiplexing, but they need to be layered atop TCP, WebSockets, and other transports that lack such support. -## Protocol Negotiation +## Protocol negotiation ### When is multiselect 2.0 due and why are we using it for mainnet? -multiselect 2.0 is currently being conceptualised. Debate started [on this issue](https://github.com/libp2p/specs/pull/95), but it got overloaded – as it tends to happen with large conceptual OSS discussions that touch the heart and core of a system. +multiselect 2.0 is currently being conceptualized. The debate started [on this issue](https://github.com/libp2p/specs/pull/95), but it got overloaded—as it tends to happen with large conceptual OSS discussions that touch the heart and core of a system. -In the following weeks (August 2019), there will be a renewed initiative to first define the requirements, constraints, assumptions and features, in order to lock in basic consensus upfront, to subsequently build on that consensus by submitting a specification for implementation. +In the following weeks (August 2019), there will be a renewed initiative to first define the requirements, constraints, assumptions, and features, in order to lock in basic consensus upfront and subsequently build on that consensus by submitting a specification for implementation. We plan to use multiselect 2.0 for mainnet because it will: @@ -561,35 +563,34 @@ SecIO is not considered secure for the purposes of this spec. ### Why are we using Noise/TLS 1.3 for mainnet? -Copied from the Noise Protocol Framework website: +Copied from the Noise Protocol Framework [website](http://www.noiseprotocol.org): > Noise is a framework for building crypto protocols. Noise protocols support mutual and optional authentication, identity hiding, forward secrecy, zero round-trip encryption, and other advanced features. Noise in itself does not specify a single handshake procedure, but provides a framework to build secure handshakes based on Diffie-Hellman key agreement with a variety of tradeoffs and guarantees. -Noise handshakes are lightweight and simple to understand, and are used in major cryptographic-centric projects like WireGuard, I2P, Lightning. [Various](https://www.wireguard.com/papers/kobeissi-bhargavan-noise-explorer-2018.pdf) [studies](https://eprint.iacr.org/2019/436.pdf) have assessed the stated security goals of several Noise handshakes with positive results. +Noise handshakes are lightweight and simple to understand, and are used in major cryptographic-centric projects like WireGuard, I2P, and Lightning. [Various](https://www.wireguard.com/papers/kobeissi-bhargavan-noise-explorer-2018.pdf) [studies](https://eprint.iacr.org/2019/436.pdf) have assessed the stated security goals of several Noise handshakes with positive results. On the other hand, TLS 1.3 is the newest, simplified iteration of TLS. Old, insecure, obsolete ciphers and algorithms have been removed, adopting Ed25519 as the sole ECDH key agreement function. Handshakes are faster, 1-RTT data is supported, and session resumption is a reality, amongst other features. -Note that [TLS 1.3 is a prerequisite of the QUIC transport](https://tools.ietf.org/html/draft-ietf-quic-transport-22#section-7), although an experiment exists to integrate Noise as the QUIC crypto layer: [nQUIC](https://eprint.iacr.org/2019/028). +*Note*: [TLS 1.3 is a prerequisite of the QUIC transport](https://tools.ietf.org/html/draft-ietf-quic-transport-22#section-7), although an experiment exists to integrate Noise as the QUIC crypto layer: [nQUIC](https://eprint.iacr.org/2019/028). ### Why are we using encryption at all? Transport level encryption secures message exchange and provides properties that are useful for privacy, safety, and censorship resistance. These properties are derived from the following security guarantees that apply to the entire communication between two peers: -- Peer authentication: the peer I’m talking to is really who they claim to be, and who I expect them to be. +- Peer authentication: the peer I’m talking to is really who they claim to be and who I expect them to be. - Confidentiality: no observer can eavesdrop on the content of our messages. - Integrity: the data has not been tampered with by a third-party while in transit. - Non-repudiation: the originating peer cannot dispute that they sent the message. - Depending on the chosen algorithms and mechanisms (e.g. continuous HMAC), we may obtain additional guarantees, such as non-replayability (this byte could’ve only been sent *now;* e.g. by using continuous HMACs), or perfect forward secrecy (in the case that a peer key is compromised, the content of a past conversation will not be compromised). -Note that transport-level encryption is not exclusive of application-level encryption or cryptography. Transport-level encryption secures the communication itself, while application-level cryptography is necessary for the application’s use cases (e.g. signatures, randomness, etc.) +Note that transport-level encryption is not exclusive of application-level encryption or cryptography. Transport-level encryption secures the communication itself, while application-level cryptography is necessary for the application’s use cases (e.g. signatures, randomness, etc.). ### Will mainnnet networking be untested when it launches? Before launching mainnet, the testnet will be switched over to mainnet networking parameters, including Noise handshakes, and other new protocols. This gives us an opportunity to drill coordinated network upgrades and verifying that there are no significant upgradeability gaps. - ## Gossipsub ### Why are we using a pub/sub algorithm for block and attestation propagation? @@ -604,27 +605,27 @@ For future extensibility with almost zero overhead now (besides the extra bytes ### How do we upgrade gossip channels (e.g. changes in encoding, compression)? -Changing gossipsub / broadcasts requires a coordinated upgrade where all clients start publishing to the new topic together, for example during a hard fork. +Changing gossipsub/broadcasts requires a coordinated upgrade where all clients start publishing to the new topic together, for example during a hard fork. -One can envision a two-phase deployment as well where clients start listening to the new topic in a first phase then start publishing some time later, letting the traffic naturally move over to the new topic. +One can envision a two-phase deployment as well where clients start listening to the new topic in the first phase then start publishing some time later, letting the traffic naturally move over to the new topic. ### Why must all clients use the same gossip topic instead of one negotiated between each peer pair? -Supporting multiple topics / encodings would require the presence of relayers to translate between encodings and topics so as to avoid network fragmentation where participants have diverging views on the gossiped state, making the protocol more complicated and fragile. +Supporting multiple topics/encodings would require the presence of relayers to translate between encodings and topics so as to avoid network fragmentation where participants have diverging views on the gossiped state, making the protocol more complicated and fragile. -Gossip protocols typically remember what messages they've seen for a finite period of time based on message identity - if you publish the same message again after that time has passed, it will be re-broadcast - adding a relay delay also makes this scenario more likely. +Gossip protocols typically remember what messages they've seen for a finite period of time-based on message identity—if you publish the same message again after that time has passed, it will be re-broadcast—adding a relay delay also makes this scenario more likely. -One can imagine that in a complicated upgrade scenario, we might have peers publishing the same message on two topics/encodings, but the price here is pretty high in terms of overhead - both computational and networking, so we'd rather avoid that. +One can imagine that in a complicated upgrade scenario, we might have peers publishing the same message on two topics/encodings, but the price here is pretty high in terms of overhead—both computational and networking—so we'd rather avoid that. It is permitted for clients to publish data on alternative topics as long as they also publish on the network-wide mandatory topic. ### Why are the topics strings and not hashes? -Topics names have a hierarchical structure. In the future, gossipsub may support wildcard subscriptions (e.g. subscribe to all children topics under a root prefix) by way of prefix matching. Enforcing hashes for topic names would preclude us from leveraging such features going forward. +Topic names have a hierarchical structure. In the future, gossipsub may support wildcard subscriptions (e.g. subscribe to all children topics under a root prefix) by way of prefix matching. Enforcing hashes for topic names would preclude us from leveraging such features going forward. No security or privacy guarantees are lost as a result of choosing plaintext topic names, since the domain is finite anyway, and calculating a digest's preimage would be trivial. -Furthermore, the ETH2 topic names are shorter than their digest equivalents (assuming SHA-256 hash), so hashing topics would bloat messages unnecessarily. +Furthermore, the Eth 2.0 topic names are shorter than their digest equivalents (assuming SHA-256 hash), so hashing topics would bloat messages unnecessarily. ### Why are there `SHARD_SUBNET_COUNT` subnets, and why is this not defined? @@ -640,7 +641,7 @@ The prohibition of unverified-block-gossiping extends to nodes that cannot verif ### How are we going to discover peers in a gossipsub topic? -Via discv5 topics. ENRs should not be used for this purpose, as they store identity, location and capability info, not volatile [advertisements](#topic-advertisement). +Via discv5 topics. ENRs should not be used for this purpose, as they store identity, location, and capability information, not volatile [advertisements](#topic-advertisement). In the interoperability testnet, all peers will be subscribed to all global beacon chain topics, so discovering peers in specific shard topics will be unnecessary. @@ -650,23 +651,23 @@ In the interoperability testnet, all peers will be subscribed to all global beac Requests are segregated by protocol ID to: -1. Leverage protocol routing in libp2p, such that the libp2p stack will route the incoming stream to the appropriate handler. This allows each the handler function for each request type to be self-contained. For an analogy, think about how you attach HTTP handlers to a REST API server. +1. Leverage protocol routing in libp2p, such that the libp2p stack will route the incoming stream to the appropriate handler. This allows the handler function for each request type to be self-contained. For an analogy, think about how you attach HTTP handlers to a REST API server. 2. Version requests independently. In a coarser-grained umbrella protocol, the entire protocol would have to be versioned even if just one field in a single message changed. 3. Enable clients to select the individual requests/versions they support. It would no longer be a strict requirement to support all requests, and clients, in principle, could support a subset of requests and variety of versions. 4. Enable flexibility and agility for clients adopting spec changes that impact the request, by signalling to peers exactly which subset of new/old requests they support. 5. Enable clients to explicitly choose backwards compatibility at the request granularity. Without this, clients would be forced to support entire versions of the coarser request protocol. -6. Parallelise RFCs (or ETH2 EIPs). By decoupling requests from one another, each RFC that affects the request protocol can be deployed/tested/debated independently without relying on a synchronisation point to version the general top-level protocol. +6. Parallelise RFCs (or Eth 2.0 EIPs). By decoupling requests from one another, each RFC that affects the request protocol can be deployed/tested/debated independently without relying on a synchronization point to version the general top-level protocol. 1. This has the benefit that clients can explicitly choose which RFCs to deploy without buying into all other RFCs that may be included in that top-level version. 2. Affording this level of granularity with a top-level protocol would imply creating as many variants (e.g. /protocol/43-{a,b,c,d,...}) as the cartesian product of RFCs inflight, O(n^2). 7. Allow us to simplify the payload of requests. Request-id’s and method-ids no longer need to be sent. The encoding/request type and version can all be handled by the framework. -CAVEAT: the protocol negotiation component in the current version of libp2p is called multistream-select 1.0. It is somewhat naïve and introduces overhead on every request when negotiating streams, although implementation-specific optimizations are possible to save this cost. Multiselect 2.0 will remove this overhead by memoizing previously selected protocols, and modelling shared protocol tables. Fortunately this req/resp protocol is not the expected network bottleneck in the protocol so the additional overhead is not expected to hinder interop testing. More info is to be released from the libp2p community in the coming weeks. +**Caveat**: The protocol negotiation component in the current version of libp2p is called multistream-select 1.0. It is somewhat naïve and introduces overhead on every request when negotiating streams, although implementation-specific optimizations are possible to save this cost. Multiselect 2.0 will remove this overhead by memoizing previously selected protocols, and modeling shared protocol tables. Fortunately, this req/resp protocol is not the expected network bottleneck in the protocol so the additional overhead is not expected to hinder interop testing. More info is to be released from the libp2p community in the coming weeks. -### Why are messages length-prefixed with a protobuf varint in the SSZ encoding? +### Why are messages length-prefixed with a protobuf varint in the SSZ-encoding? -We are using single-use streams where each stream is closed at the end of the message - thus libp2p transparently handles message delimiting in the underlying stream. libp2p streams are full-duplex, and each party is responsible for closing their write side (like in TCP). We can therefore use stream closure to mark the end of the request and response independently. +We are using single-use streams where each stream is closed at the end of the message. Thus, libp2p transparently handles message delimiting in the underlying stream. libp2p streams are full-duplex, and each party is responsible for closing their write side (like in TCP). We can therefore use stream closure to mark the end of the request and response independently. -Nevertheless, messages are still length-prefixed - this is now being considered for removal. +Nevertheless, messages are still length-prefixed—this is now being considered for removal. Advantages of length-prefixing include: @@ -676,17 +677,17 @@ Advantages of length-prefixing include: Disadvantages include: -* Redundant methods of message delimiting - both stream end marker and length prefix +* Redundant methods of message delimiting—both stream end marker and length prefix * Harder to stream as length must be known up-front * Additional code path required to verify length -In some protocols, adding a length prefix serves as a form of DoS protection against very long messages, allowing the client to abort if an overlong message is about to be sent. In this protocol, we are globally limiting message sizes using `REQ_RESP_MAX_SIZE`, thus an the length prefix does not afford any additional protection. +In some protocols, adding a length prefix serves as a form of DoS protection against very long messages, allowing the client to abort if an overlong message is about to be sent. In this protocol, we are globally limiting message sizes using `REQ_RESP_MAX_SIZE`, thus the length prefix does not afford any additional protection. [Protobuf varint](https://developers.google.com/protocol-buffers/docs/encoding#varints) is an efficient technique to encode variable-length ints. Instead of reserving a fixed-size field of as many bytes as necessary to convey the maximum possible value, this field is elastic in exchange for 1-bit overhead per byte. ### Why do we version protocol strings with ordinals instead of semver? -Using semver for network protocols is confusing. It is never clear what a change in a field, even if backwards compatible on deserialisation, actually implies. Network protocol agreement should be explicit. Imagine two peers: +Using semver for network protocols is confusing. It is never clear what a change in a field, even if backwards compatible on deserialization, actually implies. Network protocol agreement should be explicit. Imagine two peers: - Peer A supporting v1.1.1 of protocol X. - Peer B supporting v1.1.2 of protocol X. @@ -695,9 +696,9 @@ These two peers should never speak to each other because the results can be unpr For this reason, we rely on negotiation of explicit, verbatim protocols. In the above case, peer B would provide backwards compatibility by supporting and advertising both v1.1.1 and v1.1.2 of the protocol. -Therefore, semver would be relegated to convey expectations at the human level, and it wouldn't do a good job there either, because it's unclear if "backwards-compatibility" and "breaking change" apply only to wire schema level, to behaviour, etc. +Therefore, semver would be relegated to convey expectations at the human level, and it wouldn't do a good job there either, because it's unclear if "backwards compatibility" and "breaking change" apply only to wire schema level, to behavior, etc. -For this reason, we remove semver out of the picture and replace it with ordinals that require explicit agreement and do not mandate a specific policy for changes. +For this reason, we remove and replace semver with ordinals that require explicit agreement and do not mandate a specific policy for changes. ### Why is it called Req/Resp and not RPC? @@ -711,7 +712,7 @@ discv5 is a standalone protocol, running on UDP on a dedicated port, meant for p On the other hand, libp2p Kademlia DHT is a fully-fledged DHT protocol/implementation with content routing and storage capabilities, both of which are irrelevant in this context. -We assume that ETH1 nodes will evolve to support discv5. By sharing the discovery network between ETH1 and ETH2, we benefit from the additive effect on network size that enhances resilience and resistance against certain attacks, to which smaller networks are more vulnerable. It should also assist light clients of both networks find nodes with specific capabilities. +We assume that Eth 1.0 nodes will evolve to support discv5. By sharing the discovery network between Eth 1.0 and 2.0, we benefit from the additive effect on network size that enhances resilience and resistance against certain attacks, to which smaller networks are more vulnerable. It should also help light clients of both networks find nodes with specific capabilities. discv5 is in the process of being audited. @@ -721,41 +722,41 @@ Ethereum Node Records are self-certified node records. Nodes craft and dissemina ENRs are key-value records with string-indexed ASCII keys. They can store arbitrary information, but EIP-778 specifies a pre-defined dictionary, including IPv4 and IPv6 addresses, secp256k1 public keys, etc. -Comparing ENRs and multiaddrs is like comparing apples and bananas. ENRs are self-certified containers of identity, addresses, and metadata about a node. Multiaddrs are address strings with the peculiarity that they’re self-describing, composable and future-proof. An ENR can contain multiaddrs, and multiaddrs can be derived securely from the fields of an authenticated ENR. +Comparing ENRs and multiaddrs is like comparing apples and oranges. ENRs are self-certified containers of identity, addresses, and metadata about a node. Multiaddrs are address strings with the peculiarity that they’re self-describing, composable and future-proof. An ENR can contain multiaddrs, and multiaddrs can be derived securely from the fields of an authenticated ENR. discv5 uses ENRs and we will presumably need to: 1. Add `multiaddr` to the dictionary, so that nodes can advertise their multiaddr under a reserved namespace in ENRs. – and/or – -2. Define a bi-directional conversion function between multiaddrs and the corresponding denormalized fields in an ENR (ip, ip6, tcp, tcp6, etc.), for compatibility with nodes that do not support multiaddr natively (e.g. ETH1 nodes). +2. Define a bi-directional conversion function between multiaddrs and the corresponding denormalized fields in an ENR (ip, ip6, tcp, tcp6, etc.), for compatibility with nodes that do not support multiaddr natively (e.g. Eth 1.0 nodes). ## Compression/Encoding ### Why are we using SSZ for encoding? -SSZ is used at the consensus layer and all implementations should have support for ssz encoding/decoding requiring no further dependencies to be added to client implementations. This is a natural choice for serializing objects to be sent across the wire. The actual data in most protocols will be further compressed for efficiency. +SSZ is used at the consensus layer, and all implementations should have support for SSZ-encoding/decoding, requiring no further dependencies to be added to client implementations. This is a natural choice for serializing objects to be sent across the wire. The actual data in most protocols will be further compressed for efficiency. -SSZ has well defined schema’s for consensus objects (typically sent across the wire) reducing any serialization schema data that needs to be sent. It also has defined all required types that are required for this network specification. +SSZ has well-defined schemas for consensus objects (typically sent across the wire) reducing any serialization schema data that needs to be sent. It also has defined all required types that are required for this network specification. ### Why are we compressing, and at which layers? -We compress on the wire to achieve smaller payloads per-message, which, in aggregate, result in higher efficiency, better utilisation of available bandwidth, and overall reduction in network-wide traffic overhead. +We compress on the wire to achieve smaller payloads per-message, which, in aggregate, result in higher efficiency, better utilization of available bandwidth, and overall reduction in network-wide traffic overhead. -At this time, libp2p does not have an out-of-the-box compression feature that can be dynamically negotiated and layered atop connections and streams, but is [being considered](https://github.com/libp2p/libp2p/issues/81). +At this time, libp2p does not have an out-of-the-box compression feature that can be dynamically negotiated and layered atop connections and streams, but it is [being considered](https://github.com/libp2p/libp2p/issues/81). -This is a non-trivial feature because the behaviour of network IO loops, kernel buffers, chunking, packet fragmentation, amongst others, need to be taken into account. libp2p streams are unbounded streams, whereas compression algorithms work best on bounded byte streams of which we have some prior knowledge. +This is a non-trivial feature because the behavior of network IO loops, kernel buffers, chunking, and packet fragmentation, amongst others, need to be taken into account. libp2p streams are unbounded streams, whereas compression algorithms work best on bounded byte streams of which we have some prior knowledge. -Compression tends not to be a one-size-fits-all problem. Lots of variables need careful evaluation, and generic approaches/choices lead to poor size shavings, which may even be counterproductive when factoring in the CPU and memory tradeoff. +Compression tends not to be a one-size-fits-all problem. A lot of variables need careful evaluation, and generic approaches/choices lead to poor size shavings, which may even be counterproductive when factoring in the CPU and memory tradeoff. For all these reasons, generically negotiating compression algorithms may be treated as a research problem at the libp2p community, one we’re happy to tackle in the medium-term. At this stage, the wisest choice is to consider libp2p a messenger of bytes, and to make application layer participate in compressing those bytes. This looks different depending on the interaction layer: -- Gossip domain: since gossipsub has a framing protocol and exposes an API, we compress the payload (when dictated by the encoding token in the topic name) prior to publishing the message via the API. No length prefixing is necessary because protobuf takes care of bounding the field in the serialised form. +- Gossip domain: since gossipsub has a framing protocol and exposes an API, we compress the payload (when dictated by the encoding token in the topic name) prior to publishing the message via the API. No length prefixing is necessary because protobuf takes care of bounding the field in the serialized form. - Req/Resp domain: since we define custom protocols that operate on byte streams, implementers are encouraged to encapsulate the encoding and compression logic behind MessageReader and MessageWriter components/strategies that can be layered on top of the raw byte streams. ### Why are using Snappy for compression? -Snappy is used in Ethereum 1.0. It is well maintained by Google, has good benchmarks and can calculate the size of the uncompressed object without inflating it in memory. This prevents DOS vectors where large uncompressed data is sent. +Snappy is used in Ethereum 1.0. It is well maintained by Google, has good benchmarks, and can calculate the size of the uncompressed object without inflating it in memory. This prevents DOS vectors where large uncompressed data is sent. ### Can I get access to unencrypted bytes on the wire for debugging purposes? @@ -765,6 +766,6 @@ If your libp2p library relies on frameworks/runtimes such as Netty (jvm) or Node For specific ad-hoc testing scenarios, you can use the [plaintext/2.0.0 secure channel](https://github.com/libp2p/specs/blob/master/plaintext/README.md) (which is essentially no-op encryption or message authentication), in combination with tcpdump or Wireshark to inspect the wire. -# libp2p Implementations Matrix +# libp2p implementations matrix -This section will soon contain a matrix showing the maturity/state of the libp2p features required by this spec across the languages in which ETH 2.0 clients are being developed. +This section will soon contain a matrix showing the maturity/state of the libp2p features required by this spec across the languages in which Eth 2.0 clients are being developed. From 194f292348490c27cd14b6469b5a7cc0c40b0861 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Fri, 9 Aug 2019 12:16:07 +1000 Subject: [PATCH 40/89] Minor corrections and clarifications to the network specification --- specs/networking/p2p-interface.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/specs/networking/p2p-interface.md b/specs/networking/p2p-interface.md index 53e203ca6..874c51d12 100644 --- a/specs/networking/p2p-interface.md +++ b/specs/networking/p2p-interface.md @@ -114,6 +114,7 @@ This section outlines constants that are used in this spec. | Name | Value | Description | |---|---|---| | `REQ_RESP_MAX_SIZE` | `TODO` | The maximum size of uncompressed req/resp messages that clients will allow. | +| `SSZ_MAX_LIST_SIZE` | `TODO` | The maximum size of SSZ-encoded variable lists. | | `GOSSIP_MAX_SIZE` | `2**20` (= 1048576, 1 MiB) | The maximum size of uncompressed gossip messages. | | `SHARD_SUBNET_COUNT` | `TODO` | The number of shard subnets used in the gossipsub protocol. | | `TTFB_TIMEOUT` | `5s` | The maximum time to wait for first byte of request response (time-to-first-byte). | @@ -195,11 +196,11 @@ Topics are post-fixed with an encoding. Encodings define how the payload of a go #### Interop -- `ssz` - All objects are [SSZ-encoded](#ssz-encoding). Example: The beacon block topic string is `/beacon_block/ssz`, and the data field of a gossipsub message is an ssz-encoded `BeaconBlock`. +- `ssz` - All objects are [SSZ-encoded](#ssz-encoding). Example: The beacon block topic string is `/eth2/beacon_block/ssz`, and the data field of a gossipsub message is an ssz-encoded `BeaconBlock`. #### Mainnet -- `ssz_snappy` - All objects are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy). Example: The beacon attestation topic string is `/beacon_attestation/ssz_snappy`, and the data field of a gossipsub message is an `Attestation` that has been SSZ-encoded and then compressed with Snappy. +- `ssz_snappy` - All objects are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy). Example: The beacon attestation topic string is `/eth2/beacon_attestation/ssz_snappy`, and the data field of a gossipsub message is an `Attestation` that has been SSZ-encoded and then compressed with Snappy. Implementations MUST use a single encoding. Changing an encoding will require coordination between participating implementations. @@ -286,7 +287,7 @@ The `ErrorMessage` schema is: ) ``` -*Note*: The String type is encoded as UTF-8 bytes without NULL terminator when SSZ-encoded. +*Note*: The String type is encoded as UTF-8 bytes without NULL terminator when SSZ-encoded. As the `ErrorMessage` is not an SSZ-container, only the UTF-8 bytes will be sent when SSZ-encoded. A response therefore has the form: ``` @@ -300,7 +301,7 @@ Here, `result` represents the 1-byte response code. The token of the negotiated protocol ID specifies the type of encoding to be used for the req/resp interaction. Two values are possible at this time: -- `ssz`: The contents are [SSZ-encoded](#ssz-encoding). This encoding type MUST be supported by all clients. +- `ssz`: the contents are [SSZ-encoded](#ssz-encoding). This encoding type MUST be supported by all clients. For objects containing a single field, only the field is SSZ-encoded not a container with a single field. For example, the `BeaconBlocks` response would be an SSZ-encoded list of `BeaconBlock`s. All SSZ-Lists in the Req/Resp domain will have a max-list size of `SSZ_MAX_LIST_SIZE`. - `ssz_snappy`: The contents are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy). MAY be supported in the interoperability testnet; MUST be supported in mainnet. #### SSZ-encoding strategy (with or without Snappy) From f62aedec31c2778e26fb20cfa1e72661b2583d64 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 22 Aug 2019 15:04:38 -0600 Subject: [PATCH 41/89] minor formatting --- specs/networking/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/networking/p2p-interface.md b/specs/networking/p2p-interface.md index 874c51d12..8589ddfc7 100644 --- a/specs/networking/p2p-interface.md +++ b/specs/networking/p2p-interface.md @@ -301,7 +301,7 @@ Here, `result` represents the 1-byte response code. The token of the negotiated protocol ID specifies the type of encoding to be used for the req/resp interaction. Two values are possible at this time: -- `ssz`: the contents are [SSZ-encoded](#ssz-encoding). This encoding type MUST be supported by all clients. For objects containing a single field, only the field is SSZ-encoded not a container with a single field. For example, the `BeaconBlocks` response would be an SSZ-encoded list of `BeaconBlock`s. All SSZ-Lists in the Req/Resp domain will have a max-list size of `SSZ_MAX_LIST_SIZE`. +- `ssz`: the contents are [SSZ-encoded](#ssz-encoding). This encoding type MUST be supported by all clients. For objects containing a single field, only the field is SSZ-encoded not a container with a single field. For example, the `BeaconBlocks` response would be an SSZ-encoded list of `BeaconBlock`s. All SSZ-Lists in the Req/Resp domain will have a maximum list size of `SSZ_MAX_LIST_SIZE`. - `ssz_snappy`: The contents are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy). MAY be supported in the interoperability testnet; MUST be supported in mainnet. #### SSZ-encoding strategy (with or without Snappy) From 4283c8cb0372667e4f5dfdcb5323d5ff10cefbbb Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 22 Aug 2019 23:57:11 +0200 Subject: [PATCH 42/89] Fix ssz-generic bitvector tests: those invalid cases of higher length than type describes, but same byte size, should have at least 1 bit set in the overflow to be invalid --- test_generators/ssz_generic/ssz_bitvector.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/test_generators/ssz_generic/ssz_bitvector.py b/test_generators/ssz_generic/ssz_bitvector.py index 2b04577e8..3ad95cb25 100644 --- a/test_generators/ssz_generic/ssz_bitvector.py +++ b/test_generators/ssz_generic/ssz_bitvector.py @@ -5,11 +5,19 @@ from random import Random from eth2spec.debug.random_value import RandomizationMode, get_random_ssz_object -def bitvector_case_fn(rng: Random, mode: RandomizationMode, size: int): - return get_random_ssz_object(rng, Bitvector[size], +def bitvector_case_fn(rng: Random, mode: RandomizationMode, size: int, invalid_making_pos: int): + bits = get_random_ssz_object(rng, Bitvector[size], max_bytes_length=(size + 7) // 8, max_list_length=size, mode=mode, chaos=False) + if invalid_making_pos is not None and invalid_making_pos <= size: + already_invalid = False + for i in range(invalid_making_pos, size): + if bits[i]: + already_invalid = True + if not already_invalid: + bits[invalid_making_pos] = True + return bits def valid_cases(): @@ -23,8 +31,12 @@ def invalid_cases(): # zero length bitvecors are illegal yield 'bitvec_0', invalid_test_case(lambda: b'') rng = Random(1234) + # Create a vector with test_size bits, but make the type typ_size instead, + # which is invalid when used with the given type size + # (and a bit set just after typ_size bits if necessary to avoid the valid 0 padding-but-same-last-byte case) for (typ_size, test_size) in [(1, 2), (2, 3), (3, 4), (4, 5), (5, 6), (8, 9), (9, 8), (16, 8), (32, 33), (512, 513)]: for mode in [RandomizationMode.mode_random, RandomizationMode.mode_zero, RandomizationMode.mode_max]: yield f'bitvec_{typ_size}_{mode.to_name()}_{test_size}', \ - invalid_test_case(lambda: serialize(bitvector_case_fn(rng, mode, test_size))) + invalid_test_case(lambda: serialize(bitvector_case_fn(rng, mode, test_size, + invalid_making_pos=typ_size))) From 0547d0ce672a8982dfad6ea69975878a03a5a259 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 23 Aug 2019 08:39:10 -0600 Subject: [PATCH 43/89] add note on local aggregation for interop --- specs/networking/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/networking/p2p-interface.md b/specs/networking/p2p-interface.md index 8589ddfc7..03858d74e 100644 --- a/specs/networking/p2p-interface.md +++ b/specs/networking/p2p-interface.md @@ -160,7 +160,7 @@ Additional topics are used to propagate lower frequency validator messages. Thei #### Interop -Unaggregated and aggregated attestations from all shards are sent to the `beacon_attestation` topic. Clients are not required to publish aggregate attestations but must be able to process them. +Unaggregated and aggregated attestations from all shards are sent to the `beacon_attestation` topic. Clients are not required to publish aggregate attestations but must be able to process them. All validating clients SHOULD to perform local attestation aggregation to prepare for block proposing. #### Mainnet From 3d97160098ab2c6d494dc4ce4d1a9bcbb429df6d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 23 Aug 2019 10:09:37 -0600 Subject: [PATCH 44/89] Update specs/networking/p2p-interface.md Co-Authored-By: Diederik Loerakker --- specs/networking/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/networking/p2p-interface.md b/specs/networking/p2p-interface.md index 03858d74e..c9bab8406 100644 --- a/specs/networking/p2p-interface.md +++ b/specs/networking/p2p-interface.md @@ -160,7 +160,7 @@ Additional topics are used to propagate lower frequency validator messages. Thei #### Interop -Unaggregated and aggregated attestations from all shards are sent to the `beacon_attestation` topic. Clients are not required to publish aggregate attestations but must be able to process them. All validating clients SHOULD to perform local attestation aggregation to prepare for block proposing. +Unaggregated and aggregated attestations from all shards are sent to the `beacon_attestation` topic. Clients are not required to publish aggregate attestations but must be able to process them. All validating clients SHOULD try to perform local attestation aggregation to prepare for block proposing. #### Mainnet From db34ee17b63d2856f15f9182a803eeca5495c9f1 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 23 Aug 2019 11:31:20 -0600 Subject: [PATCH 45/89] fix minor testing bug --- test_generators/ssz_generic/ssz_bitvector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/ssz_generic/ssz_bitvector.py b/test_generators/ssz_generic/ssz_bitvector.py index 3ad95cb25..d84614068 100644 --- a/test_generators/ssz_generic/ssz_bitvector.py +++ b/test_generators/ssz_generic/ssz_bitvector.py @@ -5,7 +5,7 @@ from random import Random from eth2spec.debug.random_value import RandomizationMode, get_random_ssz_object -def bitvector_case_fn(rng: Random, mode: RandomizationMode, size: int, invalid_making_pos: int): +def bitvector_case_fn(rng: Random, mode: RandomizationMode, size: int, invalid_making_pos: int=None): bits = get_random_ssz_object(rng, Bitvector[size], max_bytes_length=(size + 7) // 8, max_list_length=size, From 1392c931d0f208ab3f156ea56e3cb667644706de Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 23 Aug 2019 12:00:01 -0600 Subject: [PATCH 46/89] add GeneralizedIndex to custom types --- specs/core/1_beacon-chain-misc.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 4539be6b0..b6f23a00e 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -6,6 +6,7 @@ - [Phase 1 miscellaneous beacon chain changes](#phase-1-miscellaneous-beacon-chain-changes) - [Table of contents](#table-of-contents) + - [Custom types](#custom-types) - [Configuration](#configuration) - [Classes](#classes) - [CompactCommittee](#compactcommittee) @@ -26,6 +27,14 @@ +## Custom types + +We define the following Python custom types for type hinting and readability: + +| Name | SSZ equivalent | Description | +| - | - | - | +| `GeneralizedIndex` | `uint64` | a generalized index into an SSZ merkle tree | + ## Configuration | Name | Value | Unit | Duration From 56954ec5089eaaa08423a453c6a44db3d8ce1457 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 23 Aug 2019 12:16:57 -0600 Subject: [PATCH 47/89] fix adding fields to phase 1 ssz objects --- specs/core/0_beacon-chain.md | 2 -- specs/core/1_beacon-chain-misc.md | 37 ++++++++++++++++++------------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 5ddceebc1..90c179107 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -480,7 +480,6 @@ class BeaconBlockBody(Container): deposits: List[Deposit, MAX_DEPOSITS] voluntary_exits: List[VoluntaryExit, MAX_VOLUNTARY_EXITS] transfers: List[Transfer, MAX_TRANSFERS] - # @shard_receipts ``` #### `BeaconBlock` @@ -534,7 +533,6 @@ class BeaconState(Container): previous_justified_checkpoint: Checkpoint # Previous epoch snapshot current_justified_checkpoint: Checkpoint finalized_checkpoint: Checkpoint - # @persistent_committee_fields ``` ## Helper functions diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index b6f23a00e..65c4cdbbc 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -182,17 +182,32 @@ def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptP ## Changes -### Persistent committees +### Phase 0 container updates -Add to the beacon state the following fields: +Add the following fields to the end of the specified container objects. + +#### `BeaconState` ```python -# begin insert @persistent_committee_fields +class BeaconState(Container): + # Persistent committees persistent_committee_roots: Vector[Hash, PERSISTENT_COMMITTEE_ROOT_LENGTH] next_shard_receipt_period: Vector[uint64, SHARD_COUNT] -# end insert @persistent_committee_fields ``` -`next_shard_receipt_period` values initialized to `PHASE_1_FORK_SLOT // SLOTS_PER_EPOCH // EPOCHS_PER_SHARD_PERIOD` + +`persistent_committee_roots` values are initialized to `Bytes32()` (empty bytes value). +`next_shard_receipt_period` values are initialized to `PHASE_1_FORK_SLOT // SLOTS_PER_EPOCH // EPOCHS_PER_SHARD_PERIOD`. + +#### `BeaconBlockBody` + +```python +class BeaconBlockBody(Container): + shard_receipt_proofs: List[ShardReceiptProof, MAX_SHARD_RECEIPT_PROOFS] +``` + +`shard_receipt_proofs` is initialized to `[]`. + +### Persistent committees Run `update_persistent_committee` immediately before `process_final_updates`: @@ -215,18 +230,10 @@ def update_persistent_committee(state: BeaconState) -> None: ### Shard receipt processing -Add the `shard_receipt_proofs` operation to `BeaconBlockBody`: - -```python -# begin insert @shard_receipts - shard_receipt_proofs: List[ShardReceiptProof, MAX_SHARD_RECEIPT_PROOFS] -# end insert @shard_receipts -``` - -Use `process_shard_receipt_proof` to process each receipt. +Run `process_shard_receipt_proof` on each `ShardReceiptProof` during block processing. ```python # begin insert @process_shard_receipts - (body.shard_receipt_proofs, process_shard_receipt_proofs), + (body.shard_receipt_proofs, process_shard_receipt_proof), # end insert @process_shard_receipts ``` From bcdbf7dfc7881ede422f8e29eadaeb50caea1dd6 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 24 Aug 2019 02:44:22 +0800 Subject: [PATCH 48/89] Fix some flake8 errors --- specs/core/1_beacon-chain-misc.md | 53 +++++++++++++++---------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 65c4cdbbc..b4a8d6dbe 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -6,43 +6,37 @@ - [Phase 1 miscellaneous beacon chain changes](#phase-1-miscellaneous-beacon-chain-changes) - [Table of contents](#table-of-contents) - - [Custom types](#custom-types) - [Configuration](#configuration) - - [Classes](#classes) - - [CompactCommittee](#compactcommittee) - - [ShardReceiptProof](#shardreceiptproof) - - [Helpers](#helpers) - - [pack_compact_validator](#pack_compact_validator) - - [unpack_compact_validator](#unpack_compact_validator) - - [committee_to_compact_committee](#committee_to_compact_committee) - - [get_previous_power_of_2](#get_previous_power_of_2) - - [verify_merkle_proof](#verify_merkle_proof) - - [concat_generalized_indices](#concat_generalized_indices) - - [compute_historical_state_generalized_index](#compute_historical_state_generalized_index) - - [get_generalized_index_of_crosslink_header](#get_generalized_index_of_crosslink_header) - - [process_shard_receipt_proof](#process_shard_receipt_proof) + - [Containers](#containers) + - [`CompactCommittee`](#compactcommittee) + - [`ShardReceiptProof`](#shardreceiptproof) + - [Helper functions](#helper-functions) + - [`pack_compact_validator`](#pack_compact_validator) + - [`unpack_compact_validator`](#unpack_compact_validator) + - [`committee_to_compact_committee`](#committee_to_compact_committee) + - [`get_previous_power_of_2`](#get_previous_power_of_2) + - [`verify_merkle_proof`](#verify_merkle_proof) + - [`compute_historical_state_generalized_index`](#compute_historical_state_generalized_index) + - [`get_generalized_index_of_crosslink_header`](#get_generalized_index_of_crosslink_header) + - [`process_shard_receipt_proof`](#process_shard_receipt_proof) - [Changes](#changes) + - [Phase 0 container updates](#phase-0-container-updates) + - [`BeaconState`](#beaconstate) + - [`BeaconBlockBody`](#beaconblockbody) - [Persistent committees](#persistent-committees) - [Shard receipt processing](#shard-receipt-processing) -## Custom types - -We define the following Python custom types for type hinting and readability: - -| Name | SSZ equivalent | Description | -| - | - | - | -| `GeneralizedIndex` | `uint64` | a generalized index into an SSZ merkle tree | - ## Configuration | Name | Value | Unit | Duration | - | - | - | - | | `MAX_SHARD_RECEIPT_PROOFS` | `2**0` (= 1) | - | - | | `PERSISTENT_COMMITTEE_ROOT_LENGTH` | `2**8` (= 256) | periods | ~9 months | +| `MICRO_REWARD` | `Gwei(2**0)` (=1) | Gwei | - | -## Classes +## Containers #### `CompactCommittee` @@ -61,7 +55,7 @@ class ShardReceiptProof(Container): receipt: List[ShardReceiptDelta, PLACEHOLDER] ``` -## Helpers +## Helper functions #### `pack_compact_validator` @@ -146,7 +140,9 @@ def get_generalized_index_of_crosslink_header(index: int) -> GeneralizedIndex: """ Gets the generalized index for the root of the index'th header in a crosslink. """ - MAX_CROSSLINK_SIZE = SHARD_BLOCK_SIZE_LIMIT * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK + MAX_CROSSLINK_SIZE = ( + SHARD_BLOCK_SIZE_LIMIT * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK + ) assert MAX_CROSSLINK_SIZE == get_previous_power_of_2(MAX_CROSSLINK_SIZE) return GeneralizedIndex(MAX_CROSSLINK_SIZE // SHARD_HEADER_SIZE + index) ``` @@ -170,11 +166,14 @@ def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptP leaf=hash_tree_root(receipt_proof.receipt), proof=receipt_proof.proof, index=gindex, - root=state.current_crosslinks[shard].data_root + root=state.current_crosslinks[receipt_proof.shard].data_root ) for delta in receipt_proof.receipt: if get_current_epoch(state) < state.validators[delta.index].withdrawable_epoch: - increase_balance(state, delta.index, state.validators[delta.index].effective_balance * delta.reward_coefficient // REWARD_COEFFICIENT_BASE) + increase_amount = ( + state.validators[delta.index].effective_balance * delta.reward_coefficient // REWARD_COEFFICIENT_BASE + ) + increase_balance(state, delta.index, increase_amount) decrease_balance(state, delta.index, delta.block_fee) state.next_shard_receipt_period[receipt_proof.shard] += 1 increase_balance(state, get_beacon_proposer_index(state), MICRO_REWARD) From 17043891ff39af47cca4c4867c284b8ba14735e5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 24 Aug 2019 03:03:00 +0800 Subject: [PATCH 49/89] Fix some mypy errors --- specs/core/1_beacon-chain-misc.md | 13 ++++++++----- specs/light_client/merkle_proofs.md | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index b4a8d6dbe..21b754156 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -76,7 +76,7 @@ def unpack_compact_validator(compact_validator: int) -> Tuple[int, bool, int]: """ Returns validator index, slashed, balance // EFFECTIVE_BALANCE_INCREMENT """ - return compact_validator >> 16, (compact_validator >> 15) % 2, compact_validator & (2**15 - 1) + return compact_validator >> 16, bool((compact_validator >> 15) % 2), compact_validator & (2**15 - 1) ``` #### `committee_to_compact_committee` @@ -129,7 +129,7 @@ def compute_historical_state_generalized_index(earlier: ShardSlot, later: ShardS for i in range(63, -1, -1): if (later - 1) & 2**i > (earlier - 1) & 2**i: later = later - ((later - 1) % 2**i) - 1 - o = concat_generalized_indices(o, get_generalized_index(ShardState, 'history_acc', i)) + o = concat_generalized_indices(o, get_generalized_index(ShardState, ['history_acc', i])) return o ``` @@ -150,7 +150,7 @@ def get_generalized_index_of_crosslink_header(index: int) -> GeneralizedIndex: #### `process_shard_receipt_proof` ```python -def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptProof): +def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptProof) -> None: """ Processes a ShardReceipt object. """ @@ -221,8 +221,11 @@ def update_persistent_committee(state: BeaconState) -> None: if (get_current_epoch(state) + 1) % EPOCHS_PER_SHARD_PERIOD == 0: period = (get_current_epoch(state) + 1) // EPOCHS_PER_SHARD_PERIOD committees = Vector[CompactCommittee, SHARD_COUNT]([ - committee_to_compact_committee(state, get_period_committee(state, get_current_epoch(state) + 1, i)) - for i in range(SHARD_COUNT) + committee_to_compact_committee( + state, + get_period_committee(state, Epoch(get_current_epoch(state) + 1), Shard(shard)), + ) + for shard in range(SHARD_COUNT) ]) state.persistent_committee_roots[period % PERSISTENT_COMMITTEE_ROOT_LENGTH] = hash_tree_root(committees) ``` diff --git a/specs/light_client/merkle_proofs.md b/specs/light_client/merkle_proofs.md index ce7dc647c..344f365b0 100644 --- a/specs/light_client/merkle_proofs.md +++ b/specs/light_client/merkle_proofs.md @@ -181,7 +181,7 @@ _Usage note: functions outside this section should manipulate generalized indice #### `concat_generalized_indices` ```python -def concat_generalized_indices(indices: Sequence[GeneralizedIndex]) -> GeneralizedIndex: +def concat_generalized_indices(*indices: GeneralizedIndex) -> GeneralizedIndex: """ Given generalized indices i1 for A -> B, i2 for B -> C .... i_n for Y -> Z, returns the generalized index for A -> Z. From 6923bdc46a1d8785092dcee7f61a65df81e2041f Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 23 Aug 2019 14:31:26 -0600 Subject: [PATCH 50/89] remove Optional None from get_generalized_index. instead throw --- specs/core/1_beacon-chain-misc.md | 6 +++--- specs/light_client/merkle_proofs.md | 8 +++----- .../test/merkle_proofs/test_merkle_proofs.py | 14 +++++++++----- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 21b754156..bd7130f2c 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -129,7 +129,7 @@ def compute_historical_state_generalized_index(earlier: ShardSlot, later: ShardS for i in range(63, -1, -1): if (later - 1) & 2**i > (earlier - 1) & 2**i: later = later - ((later - 1) % 2**i) - 1 - o = concat_generalized_indices(o, get_generalized_index(ShardState, ['history_acc', i])) + o = concat_generalized_indices(o, GeneralizedIndex(get_generalized_index(ShardState, ['history_acc', i]))) return o ``` @@ -158,9 +158,9 @@ def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptP first_slot_in_last_crosslink = state.current_crosslinks[receipt_proof.shard].start_epoch * SLOTS_PER_EPOCH gindex = concat_generalized_indices( get_generalized_index_of_crosslink_header(0), - get_generalized_index(ShardBlockHeader, 'state_root'), + GeneralizedIndex(get_generalized_index(ShardBlockHeader, 'state_root')), compute_historical_state_generalized_index(receipt_slot, first_slot_in_last_crosslink), - get_generalized_index(ShardState, 'receipt_root') + GeneralizedIndex(get_generalized_index(ShardState, 'receipt_root')) ) assert verify_merkle_proof( leaf=hash_tree_root(receipt_proof.receipt), diff --git a/specs/light_client/merkle_proofs.md b/specs/light_client/merkle_proofs.md index 344f365b0..bbd03d379 100644 --- a/specs/light_client/merkle_proofs.md +++ b/specs/light_client/merkle_proofs.md @@ -152,7 +152,7 @@ def get_item_position(typ: SSZType, index_or_variable_name: Union[int, SSZVariab ``` ```python -def get_generalized_index(typ: SSZType, path: Sequence[Union[int, SSZVariableName]]) -> Optional[GeneralizedIndex]: +def get_generalized_index(typ: SSZType, path: Sequence[Union[int, SSZVariableName]]) -> GeneralizedIndex: """ Converts a path (eg. `[7, "foo", 3]` for `x[7].foo[3]`, `[12, "bar", "__len__"]` for `len(x[12].bar)`) into the generalized index representing its position in the Merkle tree. @@ -162,10 +162,8 @@ def get_generalized_index(typ: SSZType, path: Sequence[Union[int, SSZVariableNam assert not issubclass(typ, BasicValue) # If we descend to a basic type, the path cannot continue further if p == '__len__': typ = uint64 - if issubclass(typ, (List, Bytes)): - root = GeneralizedIndex(root * 2 + 1) - else: - return None + assert issubclass(typ, (List, Bytes)) + root = GeneralizedIndex(root * 2 + 1) else: pos, _, _ = get_item_position(typ, p) base_index = (GeneralizedIndex(2) if issubclass(typ, (List, Bytes)) else GeneralizedIndex(1)) diff --git a/test_libs/pyspec/eth2spec/test/merkle_proofs/test_merkle_proofs.py b/test_libs/pyspec/eth2spec/test/merkle_proofs/test_merkle_proofs.py index 91c861de3..62a2f6379 100644 --- a/test_libs/pyspec/eth2spec/test/merkle_proofs/test_merkle_proofs.py +++ b/test_libs/pyspec/eth2spec/test/merkle_proofs/test_merkle_proofs.py @@ -1,10 +1,10 @@ - import re from eth_utils import ( to_tuple, ) from eth2spec.test.context import ( + expect_assertion_error, spec_state_test, with_all_phases_except, ) @@ -89,10 +89,14 @@ generalized_index_cases = [ @spec_state_test def test_get_generalized_index(spec, state): for typ, path, generalized_index in generalized_index_cases: - assert spec.get_generalized_index( - typ=typ, - path=path, - ) == generalized_index + if generalized_index is not None: + assert spec.get_generalized_index( + typ=typ, + path=path, + ) == generalized_index + else: + expect_assertion_error(lambda: spec.get_generalized_index(typ=typ, path=path)) + yield 'typ', typ yield 'path', path yield 'generalized_index', generalized_index From b6d854de097cdfd7b63124a45b0f2b7070a56db9 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sat, 24 Aug 2019 13:50:57 +0200 Subject: [PATCH 51/89] Fix ToC --- specs/core/1_beacon-chain-misc.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index bd7130f2c..720de4096 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -8,17 +8,17 @@ - [Table of contents](#table-of-contents) - [Configuration](#configuration) - [Containers](#containers) - - [`CompactCommittee`](#compactcommittee) - - [`ShardReceiptProof`](#shardreceiptproof) + - [`CompactCommittee`](#compactcommittee) + - [`ShardReceiptProof`](#shardreceiptproof) - [Helper functions](#helper-functions) - - [`pack_compact_validator`](#pack_compact_validator) - - [`unpack_compact_validator`](#unpack_compact_validator) - - [`committee_to_compact_committee`](#committee_to_compact_committee) - - [`get_previous_power_of_2`](#get_previous_power_of_2) - - [`verify_merkle_proof`](#verify_merkle_proof) - - [`compute_historical_state_generalized_index`](#compute_historical_state_generalized_index) - - [`get_generalized_index_of_crosslink_header`](#get_generalized_index_of_crosslink_header) - - [`process_shard_receipt_proof`](#process_shard_receipt_proof) + - [`pack_compact_validator`](#pack_compact_validator) + - [`unpack_compact_validator`](#unpack_compact_validator) + - [`committee_to_compact_committee`](#committee_to_compact_committee) + - [`get_previous_power_of_2`](#get_previous_power_of_2) + - [`verify_merkle_proof`](#verify_merkle_proof) + - [`compute_historical_state_generalized_index`](#compute_historical_state_generalized_index) + - [`get_generalized_index_of_crosslink_header`](#get_generalized_index_of_crosslink_header) + - [`process_shard_receipt_proof`](#process_shard_receipt_proof) - [Changes](#changes) - [Phase 0 container updates](#phase-0-container-updates) - [`BeaconState`](#beaconstate) From fb59160e6a32d437c032a33c00124a74de2327fb Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sat, 24 Aug 2019 13:54:48 +0200 Subject: [PATCH 52/89] Persistent -> period, process_shard_receipt: add _proofs --- specs/core/1_beacon-chain-misc.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 720de4096..a105fc8a0 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -33,7 +33,7 @@ | Name | Value | Unit | Duration | - | - | - | - | | `MAX_SHARD_RECEIPT_PROOFS` | `2**0` (= 1) | - | - | -| `PERSISTENT_COMMITTEE_ROOT_LENGTH` | `2**8` (= 256) | periods | ~9 months | +| `PERIOD_COMMITTEE_ROOT_LENGTH` | `2**8` (= 256) | periods | ~9 months | | `MICRO_REWARD` | `Gwei(2**0)` (=1) | Gwei | - | ## Containers @@ -189,12 +189,12 @@ Add the following fields to the end of the specified container objects. ```python class BeaconState(Container): - # Persistent committees - persistent_committee_roots: Vector[Hash, PERSISTENT_COMMITTEE_ROOT_LENGTH] + # Period committees + period_committee_roots: Vector[Hash, PERIOD_COMMITTEE_ROOT_LENGTH] next_shard_receipt_period: Vector[uint64, SHARD_COUNT] ``` -`persistent_committee_roots` values are initialized to `Bytes32()` (empty bytes value). +`period_committee_roots` values are initialized to `Bytes32()` (empty bytes value). `next_shard_receipt_period` values are initialized to `PHASE_1_FORK_SLOT // SLOTS_PER_EPOCH // EPOCHS_PER_SHARD_PERIOD`. #### `BeaconBlockBody` @@ -208,15 +208,15 @@ class BeaconBlockBody(Container): ### Persistent committees -Run `update_persistent_committee` immediately before `process_final_updates`: +Run `update_period_committee` immediately before `process_final_updates`: ```python -# begin insert @update_persistent_committee - update_persistent_committee(state) -# end insert @update_persistent_committee -def update_persistent_committee(state: BeaconState) -> None: +# begin insert @update_period_committee + update_period_committee(state) +# end insert @update_period_committee +def update_period_committee(state: BeaconState) -> None: """ - Updates persistent committee roots at boundary blocks. + Updates period committee roots at boundary blocks. """ if (get_current_epoch(state) + 1) % EPOCHS_PER_SHARD_PERIOD == 0: period = (get_current_epoch(state) + 1) // EPOCHS_PER_SHARD_PERIOD @@ -227,7 +227,7 @@ def update_persistent_committee(state: BeaconState) -> None: ) for shard in range(SHARD_COUNT) ]) - state.persistent_committee_roots[period % PERSISTENT_COMMITTEE_ROOT_LENGTH] = hash_tree_root(committees) + state.period_committee_roots[period % PERIOD_COMMITTEE_ROOT_LENGTH] = hash_tree_root(committees) ``` ### Shard receipt processing @@ -235,7 +235,7 @@ def update_persistent_committee(state: BeaconState) -> None: Run `process_shard_receipt_proof` on each `ShardReceiptProof` during block processing. ```python -# begin insert @process_shard_receipts +# begin insert @process_shard_receipt_proofs (body.shard_receipt_proofs, process_shard_receipt_proof), -# end insert @process_shard_receipts +# end insert @process_shard_receipt_proofs ``` From 7175ac55935a8460b24a0105ac3b39510fe26ba1 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sat, 24 Aug 2019 13:55:18 +0200 Subject: [PATCH 53/89] Update specs/core/0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 320bb8fd8..a26b4ae88 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1550,7 +1550,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: (body.deposits, process_deposit), (body.voluntary_exits, process_voluntary_exit), (body.transfers, process_transfer), - # @process_shard_receipts + # @process_shard_receipt_proofs ): for operation in operations: function(state, operation) From a509c68c302916cb75b7063065a9d8d59cbb4bb3 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sat, 24 Aug 2019 13:55:35 +0200 Subject: [PATCH 54/89] Update specs/core/0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a26b4ae88..102525d25 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1196,7 +1196,7 @@ def process_epoch(state: BeaconState) -> None: # @process_reveal_deadlines # @process_challenge_deadlines process_slashings(state) - # @update_persistent_committee + # @update_period_committee process_final_updates(state) # @after_process_final_updates ``` From 178dd23314f5db4ad15992bf761115df0f515ba3 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 24 Aug 2019 20:26:35 +0800 Subject: [PATCH 55/89] `MINOR_REWARD_QUOTIENT` for rewarding the proposer for including shard receipt proof Co-Authored-By: vbuterin --- specs/core/1_beacon-chain-misc.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index a105fc8a0..d07f1c217 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -34,7 +34,7 @@ | - | - | - | - | | `MAX_SHARD_RECEIPT_PROOFS` | `2**0` (= 1) | - | - | | `PERIOD_COMMITTEE_ROOT_LENGTH` | `2**8` (= 256) | periods | ~9 months | -| `MICRO_REWARD` | `Gwei(2**0)` (=1) | Gwei | - | +| `MINOR_REWARD_QUOTIENT` | `2**8` (=256) | - | - | ## Containers @@ -176,7 +176,8 @@ def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptP increase_balance(state, delta.index, increase_amount) decrease_balance(state, delta.index, delta.block_fee) state.next_shard_receipt_period[receipt_proof.shard] += 1 - increase_balance(state, get_beacon_proposer_index(state), MICRO_REWARD) + proposer_index = get_beacon_proposer_index(state) + increase_balance(state, proposer_index, Gwei(get_base_reward(state, proposer_index) // MINOR_REWARD_QUOTIENT)) ``` ## Changes From 01af8e6297770f0599ac26efb49e89d6803efcc3 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sun, 25 Aug 2019 17:19:19 +0800 Subject: [PATCH 56/89] Use `get_previous_power_of_two` from merkle proofs spec --- specs/core/1_beacon-chain-misc.md | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index d07f1c217..fcd004b95 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -14,7 +14,6 @@ - [`pack_compact_validator`](#pack_compact_validator) - [`unpack_compact_validator`](#unpack_compact_validator) - [`committee_to_compact_committee`](#committee_to_compact_committee) - - [`get_previous_power_of_2`](#get_previous_power_of_2) - [`verify_merkle_proof`](#verify_merkle_proof) - [`compute_historical_state_generalized_index`](#compute_historical_state_generalized_index) - [`get_generalized_index_of_crosslink_header`](#get_generalized_index_of_crosslink_header) @@ -95,13 +94,6 @@ def committee_to_compact_committee(state: BeaconState, committee: Sequence[Valid return CompactCommittee(pubkeys=pubkeys, compact_validators=compact_validators) ``` -#### `get_previous_power_of_2` - -```python -def get_previous_power_of_2(x: int) -> int: - return x if x <= 2 else 2 * get_previous_power_of_2(x // 2) -``` - #### `verify_merkle_proof` ```python @@ -143,7 +135,7 @@ def get_generalized_index_of_crosslink_header(index: int) -> GeneralizedIndex: MAX_CROSSLINK_SIZE = ( SHARD_BLOCK_SIZE_LIMIT * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK ) - assert MAX_CROSSLINK_SIZE == get_previous_power_of_2(MAX_CROSSLINK_SIZE) + assert MAX_CROSSLINK_SIZE == get_previous_power_of_two(MAX_CROSSLINK_SIZE) return GeneralizedIndex(MAX_CROSSLINK_SIZE // SHARD_HEADER_SIZE + index) ``` From 9b3cb306e3ce5dca1e3d3df1584ca2c29d2ff9b6 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Mon, 26 Aug 2019 10:09:13 +0200 Subject: [PATCH 57/89] Update specs/core/1_beacon-chain-misc.md Co-Authored-By: John Adler --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index fcd004b95..ee63eca64 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -118,7 +118,7 @@ def compute_historical_state_generalized_index(earlier: ShardSlot, later: ShardS blocks at the next available multiples of descending powers of two. """ o = GeneralizedIndex(1) - for i in range(63, -1, -1): + for i in range(HISTORY_ACCUMULATOR_VECTOR - 1, -1, -1): if (later - 1) & 2**i > (earlier - 1) & 2**i: later = later - ((later - 1) % 2**i) - 1 o = concat_generalized_indices(o, GeneralizedIndex(get_generalized_index(ShardState, ['history_acc', i]))) From f1caa85aaf260fbd35774efff8eb057100ac3875 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Mon, 26 Aug 2019 10:09:43 +0200 Subject: [PATCH 58/89] Update specs/core/1_beacon-chain-misc.md Co-Authored-By: John Adler --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index ee63eca64..9e9bfddb9 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -147,7 +147,7 @@ def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptP Processes a ShardReceipt object. """ receipt_slot = state.next_shard_receipt_period[receipt_proof.shard] * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD - first_slot_in_last_crosslink = state.current_crosslinks[receipt_proof.shard].start_epoch * SLOTS_PER_EPOCH + first_slot_in_last_crosslink = state.current_crosslinks[receipt_proof.shard].start_epoch * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH gindex = concat_generalized_indices( get_generalized_index_of_crosslink_header(0), GeneralizedIndex(get_generalized_index(ShardBlockHeader, 'state_root')), From ffdc36920e9acc2472a9ba7657b13219ab77279e Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 26 Aug 2019 08:55:50 -0600 Subject: [PATCH 59/89] lint --- specs/core/1_beacon-chain-misc.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 9e9bfddb9..c23b018f0 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -146,8 +146,9 @@ def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptP """ Processes a ShardReceipt object. """ + SHARD_SLOTS_PER_EPOCH = SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH receipt_slot = state.next_shard_receipt_period[receipt_proof.shard] * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD - first_slot_in_last_crosslink = state.current_crosslinks[receipt_proof.shard].start_epoch * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH + first_slot_in_last_crosslink = state.current_crosslinks[receipt_proof.shard].start_epoch * SHARD_SLOTS_PER_EPOCH gindex = concat_generalized_indices( get_generalized_index_of_crosslink_header(0), GeneralizedIndex(get_generalized_index(ShardBlockHeader, 'state_root')), From 79e34a00e82518bdef23b265a15b5f7f9ef1c797 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Mon, 26 Aug 2019 17:37:18 -0700 Subject: [PATCH 60/89] Update sync_protocol.md --- specs/light_client/sync_protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index 944abf8c1..feef1fcea 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -86,7 +86,7 @@ def unpack_compact_validator(compact_validator: CompactValidator) -> Tuple[Valid """ return ( ValidatorIndex(compact_validator >> 16), - (compact_validator >> 15) % 2, + (compact_validator >> 15) % 2 == 0, uint64(compact_validator & (2**15 - 1)), ) ``` From ab4820ced6ed85effc871de712064cbb2a531ca3 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 27 Aug 2019 13:13:47 +0200 Subject: [PATCH 61/89] Update specs/core/1_beacon-chain-misc.md Co-Authored-By: John Adler --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index c23b018f0..41d0de233 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -147,7 +147,7 @@ def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptP Processes a ShardReceipt object. """ SHARD_SLOTS_PER_EPOCH = SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH - receipt_slot = state.next_shard_receipt_period[receipt_proof.shard] * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD + receipt_slot = state.next_shard_receipt_period[receipt_proof.shard] * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD first_slot_in_last_crosslink = state.current_crosslinks[receipt_proof.shard].start_epoch * SHARD_SLOTS_PER_EPOCH gindex = concat_generalized_indices( get_generalized_index_of_crosslink_header(0), From 0b38ff0fe26f7291f771be9b65b6bf9353ccbf7e Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 27 Aug 2019 13:17:38 +0200 Subject: [PATCH 62/89] Update specs/core/1_beacon-chain-misc.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 41d0de233..ddf06a1f0 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -189,7 +189,7 @@ class BeaconState(Container): ``` `period_committee_roots` values are initialized to `Bytes32()` (empty bytes value). -`next_shard_receipt_period` values are initialized to `PHASE_1_FORK_SLOT // SLOTS_PER_EPOCH // EPOCHS_PER_SHARD_PERIOD`. +`next_shard_receipt_period` values are initialized to `compute_epoch_of_shard_slot(PHASE_1_FORK_SLOT) // EPOCHS_PER_SHARD_PERIOD`. #### `BeaconBlockBody` From 0f2e814c63238f14339dd1a30723ced8336cf63e Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 27 Aug 2019 13:19:04 +0200 Subject: [PATCH 63/89] Shard slot -> slot for PHASE_1_FORK_SLOT --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index ddf06a1f0..35a58e53b 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -189,7 +189,7 @@ class BeaconState(Container): ``` `period_committee_roots` values are initialized to `Bytes32()` (empty bytes value). -`next_shard_receipt_period` values are initialized to `compute_epoch_of_shard_slot(PHASE_1_FORK_SLOT) // EPOCHS_PER_SHARD_PERIOD`. +`next_shard_receipt_period` values are initialized to `compute_epoch_of_slot(PHASE_1_FORK_SLOT) // EPOCHS_PER_SHARD_PERIOD`. #### `BeaconBlockBody` From 17702e6d88515435a2a7c017d3d026d47acdbd8d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 27 Aug 2019 20:10:39 +0800 Subject: [PATCH 64/89] Shard slot -> slot for PHASE_1_FORK_SLOT part2 --- configs/minimal.yaml | 4 ++-- specs/core/1_shard-data-chains.md | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 15b749b9d..be787ca3c 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -141,5 +141,5 @@ SHARD_SLOTS_PER_BEACON_SLOT: 2 EPOCHS_PER_SHARD_PERIOD: 4 # PHASE_1_FORK_EPOCH >= EPOCHS_PER_SHARD_PERIOD * 2 PHASE_1_FORK_EPOCH: 8 -# PHASE_1_FORK_SLOT = PHASE_1_FORK_EPOCH * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH -PHASE_1_FORK_SLOT: 128 +# PHASE_1_FORK_SLOT = PHASE_1_FORK_EPOCH * SLOTS_PER_EPOCH +PHASE_1_FORK_SLOT: 64 diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 8e1532f17..3dc549816 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -73,10 +73,10 @@ We define the following Python custom types for type hinting and readability: ### Initial values -| Name | Value | -| - | - | -| `PHASE_1_FORK_EPOCH` | **TBD** | -| `PHASE_1_FORK_SLOT` | **TBD** | +| Name | Value | Unit | +| - | - | - | +| `PHASE_1_FORK_EPOCH` | **TBD** | Epoch | +| `PHASE_1_FORK_SLOT` | **TBD** | Slot | ### Time parameters @@ -359,7 +359,7 @@ def get_default_shard_state(beacon_state: BeaconState, shard: Shard) -> ShardSta return ShardState( basefee=1, shard=shard, - slot=PHASE_1_FORK_SLOT, + slot=PHASE_1_FORK_SLOT * SHARD_SLOTS_PER_BEACON_SLOT, earlier_committee_rewards=[REWARD_COEFFICIENT_BASE for _ in range(len(earlier_committee))], later_committee_rewards=[REWARD_COEFFICIENT_BASE for _ in range(len(later_committee))], earlier_committee_fees=[Gwei(0) for _ in range(len(earlier_committee))], From 979fa38ae8a7e2ebdba82a993baa0e1cde3fb639 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 27 Aug 2019 20:10:55 +0800 Subject: [PATCH 65/89] fix linter error --- specs/core/1_beacon-chain-misc.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 35a58e53b..5bb0f6da0 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -147,7 +147,10 @@ def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptP Processes a ShardReceipt object. """ SHARD_SLOTS_PER_EPOCH = SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH - receipt_slot = state.next_shard_receipt_period[receipt_proof.shard] * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD + receipt_slot = ( + state.next_shard_receipt_period[receipt_proof.shard] * + SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD + ) first_slot_in_last_crosslink = state.current_crosslinks[receipt_proof.shard].start_epoch * SHARD_SLOTS_PER_EPOCH gindex = concat_generalized_indices( get_generalized_index_of_crosslink_header(0), From d0b4dc2b0164c626df766b25ad6ecddd57de9a18 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 27 Aug 2019 09:16:02 -0600 Subject: [PATCH 66/89] remove bad length checks from process attestation; ensure committee count and committee size not equal --- specs/core/0_beacon-chain.md | 5 ----- test_libs/pyspec/eth2spec/test/context.py | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 01a6ca352..4970a3b00 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1615,11 +1615,6 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: proposer_index=get_beacon_proposer_index(state), ) - # Check bitlist lengths - committee_size = get_committee_count(state, attestation.data.target.epoch) - assert len(attestation.aggregation_bits) == committee_size - assert len(attestation.custody_bits) == committee_size - if data.target.epoch == get_current_epoch(state): assert data.source == state.current_justified_checkpoint parent_crosslink = state.current_crosslinks[data.crosslink.shard] diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index 5cc42c510..9f7fc41d4 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -10,7 +10,7 @@ from .utils import vector_test, with_meta_tags def with_state(fn): def entry(*args, **kw): try: - kw['state'] = create_genesis_state(spec=kw['spec'], num_validators=spec_phase0.SLOTS_PER_EPOCH * 8) + kw['state'] = create_genesis_state(spec=kw['spec'], num_validators=spec_phase0.SLOTS_PER_EPOCH * 9) except KeyError: raise TypeError('Spec decorator must come within state decorator to inject spec into state.') return fn(*args, **kw) From 82faaf101d50f33206094a99e60a3efa86662f9b Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 27 Aug 2019 09:38:20 -0600 Subject: [PATCH 67/89] fix tests --- test_libs/pyspec/eth2spec/test/context.py | 2 +- .../block_processing/test_process_bit_challenge.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index 9f7fc41d4..80edaba9b 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -10,7 +10,7 @@ from .utils import vector_test, with_meta_tags def with_state(fn): def entry(*args, **kw): try: - kw['state'] = create_genesis_state(spec=kw['spec'], num_validators=spec_phase0.SLOTS_PER_EPOCH * 9) + kw['state'] = create_genesis_state(spec=kw['spec'], num_validators=spec_phase0.SLOTS_PER_EPOCH * 10) except KeyError: raise TypeError('Spec decorator must come within state decorator to inject spec into state.') return fn(*args, **kw) diff --git a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_bit_challenge.py b/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_bit_challenge.py index e4880555a..ae6ff258c 100644 --- a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_bit_challenge.py +++ b/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_bit_challenge.py @@ -212,13 +212,16 @@ def test_max_reveal_lateness_1(spec, state): challenge = get_valid_bit_challenge(spec, state, attestation) responder_index = challenge.responder_index + target_epoch = attestation.data.target.epoch state.validators[responder_index].max_reveal_lateness = 3 - for i in range(spec.get_randao_epoch_for_custody_period( - spec.get_custody_period_for_validator(state, responder_index), + latest_reveal_epoch = spec.get_randao_epoch_for_custody_period( + spec.get_custody_period_for_validator(state, responder_index, target_epoch), responder_index - ) + 2 * spec.EPOCHS_PER_CUSTODY_PERIOD + state.validators[responder_index].max_reveal_lateness - 2): + ) + 2 * spec.EPOCHS_PER_CUSTODY_PERIOD + state.validators[responder_index].max_reveal_lateness + + while spec.get_current_epoch(state) < latest_reveal_epoch - 2: next_epoch(spec, state) apply_empty_block(spec, state) From 78a0e15e9c5f0087ae51e482a05c7e78c844bce0 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 27 Aug 2019 10:36:25 -0600 Subject: [PATCH 68/89] explicitly cast to bool --- specs/light_client/sync_protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index feef1fcea..207e0e63e 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -86,7 +86,7 @@ def unpack_compact_validator(compact_validator: CompactValidator) -> Tuple[Valid """ return ( ValidatorIndex(compact_validator >> 16), - (compact_validator >> 15) % 2 == 0, + bool((compact_validator >> 15) % 2), uint64(compact_validator & (2**15 - 1)), ) ``` From c8c47e39865567e4f9eb7a180bc32e99192f34e8 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Wed, 28 Aug 2019 10:22:34 +0100 Subject: [PATCH 69/89] Abstract away compute_proposer_index for phase 1 --- specs/core/0_beacon-chain.md | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f27d016cf..7398da312 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -68,6 +68,7 @@ - [`is_valid_merkle_branch`](#is_valid_merkle_branch) - [Misc](#misc-1) - [`compute_shuffled_index`](#compute_shuffled_index) + - [`compute_proposer_index`](#compute_proposer_index) - [`compute_committee`](#compute_committee) - [`compute_epoch_of_slot`](#compute_epoch_of_slot) - [`compute_start_slot_of_epoch`](#compute_start_slot_of_epoch) @@ -729,6 +730,25 @@ def compute_shuffled_index(index: ValidatorIndex, index_count: uint64, seed: Has return ValidatorIndex(index) ``` +#### `compute_proposer_index` + +```python +def compute_proposer_index(state: BeaconState, indices: Sequence[ValidatorIndex], seed: Hash) -> ValidatorIndex: + """ + Return from ``indices`` a random index sampled by effective balance. + """ + assert len(indices) > 0 + MAX_RANDOM_BYTE = 2**8 - 1 + i = 0 + while True: + candidate_index = indices[compute_shuffled_index(ValidatorIndex(i % len(indices)), len(indices), seed)] + random_byte = hash(seed + int_to_bytes(i // 32, length=8))[i % 32] + effective_balance = state.validators[candidate_index].effective_balance + if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: + return ValidatorIndex(candidate_index) + i += 1 +``` + #### `compute_committee` ```python @@ -934,17 +954,9 @@ def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: Return the beacon proposer index at the current slot. """ epoch = get_current_epoch(state) - indices = get_active_validator_indices(state, epoch) seed = hash(get_seed(state, epoch) + int_to_bytes(state.slot, length=8)) - MAX_RANDOM_BYTE = 2**8 - 1 - i = 0 - while True: - candidate_index = indices[compute_shuffled_index(ValidatorIndex(i % len(indices)), len(indices), seed)] - random_byte = hash(seed + int_to_bytes(i // 32, length=8))[i % 32] - effective_balance = state.validators[candidate_index].effective_balance - if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: - return ValidatorIndex(candidate_index) - i += 1 + indices = get_active_validator_indices(state, epoch) + return compute_proposer_index(state, indices, seed) ``` #### `get_attestation_data_slot` From 9a712ead6875c7c70edf081a71c95b10cc90b4b0 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Sun, 1 Sep 2019 10:24:46 -0700 Subject: [PATCH 70/89] Update merkle_proofs.md --- specs/light_client/merkle_proofs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/light_client/merkle_proofs.md b/specs/light_client/merkle_proofs.md index ce7dc647c..e42de9f37 100644 --- a/specs/light_client/merkle_proofs.md +++ b/specs/light_client/merkle_proofs.md @@ -62,7 +62,7 @@ Note that the generalized index has the convenient property that the two childre def merkle_tree(leaves: Sequence[Hash]) -> Sequence[Hash]: padded_length = get_next_power_of_two(len(leaves)) o = [Hash()] * padded_length + list(leaves) + [Hash()] * (padded_length - len(leaves)) - for i in range(len(leaves) - 1, 0, -1): + for i in range(len(padded_length) - 1, 0, -1): o[i] = hash(o[i * 2] + o[i * 2 + 1]) return o ``` From 84965be2517a028056a16580d6663e830e481558 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Sun, 1 Sep 2019 11:07:44 -0700 Subject: [PATCH 71/89] Update merkle_proofs.md --- specs/light_client/merkle_proofs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/light_client/merkle_proofs.md b/specs/light_client/merkle_proofs.md index e42de9f37..092f7142e 100644 --- a/specs/light_client/merkle_proofs.md +++ b/specs/light_client/merkle_proofs.md @@ -62,7 +62,7 @@ Note that the generalized index has the convenient property that the two childre def merkle_tree(leaves: Sequence[Hash]) -> Sequence[Hash]: padded_length = get_next_power_of_two(len(leaves)) o = [Hash()] * padded_length + list(leaves) + [Hash()] * (padded_length - len(leaves)) - for i in range(len(padded_length) - 1, 0, -1): + for i in range(padded_length - 1, 0, -1): o[i] = hash(o[i * 2] + o[i * 2 + 1]) return o ``` From ab2fac66967387d9a49b46775f839f7305637517 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 3 Sep 2019 07:14:04 +0100 Subject: [PATCH 72/89] Added misc beacon chain updates to ToC --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 6fbd38aa4..0de6cd9ef 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ Core specifications for Eth 2.0 client validation can be found in [specs/core](s ### Phase 1 * [Custody Game](specs/core/1_custody-game.md) * [Shard Data Chains](specs/core/1_shard-data-chains.md) +* [Misc beacon chain updates](specs/core/1_beacon-chain-misc.md) ### Phase 2 From d1fe8f16fd4449cbf766d4ef2484b0891c04be58 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 3 Sep 2019 18:59:18 +0100 Subject: [PATCH 73/89] Deposit contract fixes (#1362) --- .../contracts/validator_registration.json | 2 +- .../contracts/validator_registration.v.py | 18 ++- deposit_contract/requirements-testing.txt | 2 +- .../tests/contracts/test_deposit.py | 112 +++++++++++++----- specs/core/0_deposit-contract.md | 4 +- specs/validator/0_beacon-chain-validator.md | 2 +- 6 files changed, 99 insertions(+), 41 deletions(-) diff --git a/deposit_contract/contracts/validator_registration.json b/deposit_contract/contracts/validator_registration.json index 3a6bfb2d8..fbf20e74c 100644 --- a/deposit_contract/contracts/validator_registration.json +++ b/deposit_contract/contracts/validator_registration.json @@ -1 +1 @@ -{"abi": [{"name": "DepositEvent", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}, {"type": "bytes", "name": "index", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "get_hash_tree_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 91674}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 10433}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}], "constant": false, "payable": true, "type": "function", "gas": 1334417}], "bytecode": "0x740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009857600080fd5b6101406000601f818352015b600061014051602081106100b757600080fd5b600260c052602060c020015460208261016001015260208101905061014051602081106100e357600080fd5b600260c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012157600080fd5b60c0519050606051600161014051018060405190131561014057600080fd5b809190121561014e57600080fd5b6020811061015b57600080fd5b600260c052602060c02001555b81516001018083528114156100a4575b50506112f956600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052600015610277575b6101605261014052600061018052610140516101a0526101c060006008818352015b61018051600860008112156100da578060000360020a82046100e1565b8060020a82025b905090506101805260ff6101a051166101e052610180516101e0516101805101101561010c57600080fd5b6101e0516101805101610180526101a0517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff86000811215610155578060000360020a820461015c565b8060020a82025b905090506101a0525b81516001018083528114156100bd575b50506018600860208206610200016020828401111561019357600080fd5b60208061022082610180600060046015f15050818152809050905090508051602001806102c0828460006004600a8704601201f16101d057600080fd5b50506102c05160206001820306601f82010390506103206102c0516008818352015b826103205111156102025761021e565b6000610320516102e001535b81516001018083528114156101f2575b50505060206102a05260406102c0510160206001820306601f8201039050610280525b6000610280511115156102535761026f565b602061028051036102a001516020610280510361028052610241565b610160515650005b63863a311b600051141561050857341561029057600080fd5b6000610140526101405161016052600154610180526101a060006020818352015b60016001610180511614156103325760006101a051602081106102d357600080fd5b600060c052602060c02001546020826102400101526020810190506101605160208261024001015260208101905080610240526102409050602060c0825160208401600060025af161032457600080fd5b60c0519050610160526103a0565b6000610160516020826101c00101526020810190506101a0516020811061035857600080fd5b600260c052602060c02001546020826101c0010152602081019050806101c0526101c09050602060c0825160208401600060025af161039657600080fd5b60c0519050610160525b61018060026103ae57600080fd5b60028151048152505b81516001018083528114156102b1575b505060006101605160208261044001015260208101905061014051610160516101805163806732896102c0526001546102e0526102e0516006580161009b565b506103405260006103a0525b6103405160206001820306601f82010390506103a0511015156104355761044e565b6103a05161036001526103a0516020016103a052610413565b61018052610160526101405261034060088060208461044001018260208501600060046012f150508051820191505060006018602082066103c0016020828401111561049957600080fd5b6020806103e082610140600060046015f150508181528090509050905060188060208461044001018260208501600060046014f150508051820191505080610440526104409050602060c0825160208401600060025af16104f957600080fd5b60c051905060005260206000f3005b63621fd130600051141561061a57341561052157600080fd5b63806732896101405260015461016052610160516006580161009b565b506101c0526000610220525b6101c05160206001820306601f82010390506102205110151561056c57610585565b610220516101e00152610220516020016102205261054a565b6101c0805160200180610280828460006004600a8704601201f16105a857600080fd5b50506102805160206001820306601f82010390506102e0610280516008818352015b826102e05111156105da576105f6565b60006102e0516102a001535b81516001018083528114156105ca575b5050506020610260526040610280510160206001820306601f8201039050610260f3005b63c47e300d600051141561117457606060046101403760506004356004016101a037603060043560040135111561065057600080fd5b604060243560040161022037602060243560040135111561067057600080fd5b608060443560040161028037606060443560040135111561069057600080fd5b63ffffffff600154106106a257600080fd5b633b9aca0061034052610340516106b857600080fd5b61034051340461032052633b9aca006103205110156106d657600080fd5b60306101a051146106e657600080fd5b602061022051146106f657600080fd5b6060610280511461070657600080fd5b6101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e05161030051610320516103405161036051610380516103a05163806732896103c052610320516103e0526103e0516006580161009b565b506104405260006104a0525b6104405160206001820306601f82010390506104a051101515610796576107af565b6104a05161046001526104a0516020016104a052610774565b6103a05261038052610360526103405261032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a052610440805160200180610360828460006004600a8704601201f161081657600080fd5b50506101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e05161030051610320516103405161036051610380516103a0516103c0516103e05161040051610420516104405161046051610480516104a05163806732896104c0526001546104e0526104e0516006580161009b565b506105405260006105a0525b6105405160206001820306601f82010390506105a0511015156108c7576108e0565b6105a05161056001526105a0516020016105a0526108a5565b6104a05261048052610460526104405261042052610400526103e0526103c0526103a05261038052610360526103405261032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a0526105408051602001806105c0828460006004600a8704601201f161096757600080fd5b505060a06106405261064051610680526101a08051602001806106405161068001828460006004600a8704601201f161099f57600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516040818352015b83610620511015156109dd576109fa565b6000610620516020850101535b81516001018083528114156109cc575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106a0526102208051602001806106405161068001828460006004600a8704601201f1610a5157600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610a8f57610aac565b6000610620516020850101535b8151600101808352811415610a7e575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106c0526103608051602001806106405161068001828460006004600a8704601201f1610b0357600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610b4157610b5e565b6000610620516020850101535b8151600101808352811415610b30575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106e0526102808051602001806106405161068001828460006004600a8704601201f1610bb557600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516060818352015b8361062051101515610bf357610c10565b6000610620516020850101535b8151600101808352811415610be2575b50505050602061064051610680015160206001820306601f82010390506106405101016106405261064051610700526105c08051602001806106405161068001828460006004600a8704601201f1610c6757600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610ca557610cc2565b6000610620516020850101535b8151600101808352811415610c94575b50505050602061064051610680015160206001820306601f8201039050610640510101610640527f649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c561064051610680a160006107205260006101a06030806020846107e001018260208501600060046016f150508051820191505060006010602082066107600160208284011115610d5957600080fd5b60208061078082610720600060046015f15050818152809050905090506010806020846107e001018260208501600060046013f1505080518201915050806107e0526107e09050602060c0825160208401600060025af1610db957600080fd5b60c0519050610740526000600060406020820661088001610280518284011115610de257600080fd5b6060806108a0826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af1610e2257600080fd5b60c0519050602082610a800101526020810190506000604060206020820661094001610280518284011115610e5657600080fd5b606080610960826020602088068803016102800160006004601bf1505081815280905090509050602080602084610a0001018260208501600060046015f150508051820191505061072051602082610a0001015260208101905080610a0052610a009050602060c0825160208401600060025af1610ed357600080fd5b60c0519050602082610a8001015260208101905080610a8052610a809050602060c0825160208401600060025af1610f0a57600080fd5b60c0519050610860526000600061074051602082610b20010152602081019050610220602080602084610b2001018260208501600060046015f150508051820191505080610b2052610b209050602060c0825160208401600060025af1610f7057600080fd5b60c0519050602082610ca00101526020810190506000610360600880602084610c2001018260208501600060046012f15050805182019150506000601860208206610ba00160208284011115610fc557600080fd5b602080610bc082610720600060046015f1505081815280905090509050601880602084610c2001018260208501600060046014f150508051820191505061086051602082610c2001015260208101905080610c2052610c209050602060c0825160208401600060025af161103857600080fd5b60c0519050602082610ca001015260208101905080610ca052610ca09050602060c0825160208401600060025af161106f57600080fd5b60c0519050610b0052600180546001825401101561108c57600080fd5b6001815401815550600154610d2052610d4060006020818352015b60016001610d20511614156110dc57610b0051610d4051602081106110cb57600080fd5b600060c052602060c0200155611170565b6000610d4051602081106110ef57600080fd5b600060c052602060c0200154602082610d60010152602081019050610b0051602082610d6001015260208101905080610d6052610d609050602060c0825160208401600060025af161114057600080fd5b60c0519050610b0052610d20600261115757600080fd5b60028151048152505b81516001018083528114156110a7575b5050005b60006000fd5b61017f6112f90361017f60003961017f6112f9036000f3"} \ No newline at end of file +{"abi": [{"name": "DepositEvent", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}, {"type": "bytes", "name": "index", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "get_deposit_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 95389}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 17683}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}, {"type": "bytes32", "name": "deposit_data_root"}], "constant": false, "payable": true, "type": "function", "gas": 1754607}], "bytecode": "0x740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009857600080fd5b6101406000601f818352015b600061014051602081106100b757600080fd5b600260c052602060c020015460208261016001015260208101905061014051602081106100e357600080fd5b600260c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012157600080fd5b60c0519050606051600161014051018060405190131561014057600080fd5b809190121561014e57600080fd5b6020811061015b57600080fd5b600260c052602060c02001555b81516001018083528114156100a4575b50506111d656600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052600015610265575b6101605261014052600061018052610140516101a0526101c060006008818352015b61018051600860008112156100da578060000360020a82046100e1565b8060020a82025b905090506101805260ff6101a051166101e052610180516101e0516101805101101561010c57600080fd5b6101e0516101805101610180526101a0517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff86000811215610155578060000360020a820461015c565b8060020a82025b905090506101a0525b81516001018083528114156100bd575b50506018600860208206610200016020828401111561019357600080fd5b60208061022082610180600060046015f15050818152809050905090508051602001806102c0828460006004600a8704601201f16101d057600080fd5b50506103206102c0516020818352015b60206103205111156101f15761020d565b6000610320516102e001535b81516001018083528114156101e0575b505060206102a05260406102c0510160206001820306601f8201039050610280525b6000610280511115156102415761025d565b602061028051036102a00151602061028051036102805261022f565b610160515650005b63c5f2892f60005114156104f757341561027e57600080fd5b6000610140526101405161016052600154610180526101a060006020818352015b60016001610180511614156103205760006101a051602081106102c157600080fd5b600060c052602060c02001546020826102400101526020810190506101605160208261024001015260208101905080610240526102409050602060c0825160208401600060025af161031257600080fd5b60c05190506101605261038e565b6000610160516020826101c00101526020810190506101a0516020811061034657600080fd5b600260c052602060c02001546020826101c0010152602081019050806101c0526101c09050602060c0825160208401600060025af161038457600080fd5b60c0519050610160525b610180600261039c57600080fd5b60028151048152505b815160010180835281141561029f575b505060006101605160208261046001015260208101905061014051610160516101805163806732896102e05260015461030052610300516006580161009b565b506103605260006103c0525b6103605160206001820306601f82010390506103c0511015156104235761043c565b6103c05161038001526103c0516020016103c052610401565b61018052610160526101405261036060088060208461046001018260208501600060046012f150508051820191505060006018602082066103e0016020828401111561048757600080fd5b60208061040082610140600060046015f150508181528090509050905060188060208461046001018260208501600060046014f150508051820191505080610460526104609050602060c0825160208401600060025af16104e757600080fd5b60c051905060005260206000f350005b63621fd13060005114156105f857341561051057600080fd5b63806732896101405260015461016052610160516006580161009b565b506101c0526000610220525b6101c05160206001820306601f82010390506102205110151561055b57610574565b610220516101e001526102205160200161022052610539565b6101c0805160200180610280828460006004600a8704601201f161059757600080fd5b50506102e0610280516020818352015b60206102e05111156105b8576105d4565b60006102e0516102a001535b81516001018083528114156105a7575b50506020610260526040610280510160206001820306601f8201039050610260f350005b6322895118600051141561105157605060043560040161014037603060043560040135111561062657600080fd5b60406024356004016101c037602060243560040135111561064657600080fd5b608060443560040161022037606060443560040135111561066657600080fd5b63ffffffff6001541061067857600080fd5b633b9aca006102e0526102e05161068e57600080fd5b6102e05134046102c052633b9aca006102c05110156106ac57600080fd5b603061014051146106bc57600080fd5b60206101c051146106cc57600080fd5b606061022051146106dc57600080fd5b610140610360525b61036051516020610360510161036052610360610360511015610706576106e4565b6380673289610380526102c0516103a0526103a0516006580161009b565b50610400526000610460525b6104005160206001820306601f8201039050610460511015156107525761076b565b6104605161042001526104605160200161046052610730565b610340610360525b610360515260206103605103610360526101406103605110151561079657610773565b610400805160200180610300828460006004600a8704601201f16107b957600080fd5b5050610140610480525b610480515160206104805101610480526104806104805110156107e5576107c3565b63806732896104a0526001546104c0526104c0516006580161009b565b50610520526000610580525b6105205160206001820306601f82010390506105805110151561083057610849565b610580516105400152610580516020016105805261080e565b610460610480525b610480515260206104805103610480526101406104805110151561087457610851565b6105208051602001806105a0828460006004600a8704601201f161089757600080fd5b505060a06106205261062051610660526101408051602001806106205161066001828460006004600a8704601201f16108cf57600080fd5b5050610600610620516106600151610240818352015b6102406106005111156108f757610918565b600061060051610620516106800101535b81516001018083528114156108e5575b5050602061062051610660015160206001820306601f82010390506106205101016106205261062051610680526101c08051602001806106205161066001828460006004600a8704601201f161096d57600080fd5b5050610600610620516106600151610240818352015b610240610600511115610995576109b6565b600061060051610620516106800101535b8151600101808352811415610983575b5050602061062051610660015160206001820306601f820103905061062051010161062052610620516106a0526103008051602001806106205161066001828460006004600a8704601201f1610a0b57600080fd5b5050610600610620516106600151610240818352015b610240610600511115610a3357610a54565b600061060051610620516106800101535b8151600101808352811415610a21575b5050602061062051610660015160206001820306601f820103905061062051010161062052610620516106c0526102208051602001806106205161066001828460006004600a8704601201f1610aa957600080fd5b5050610600610620516106600151610240818352015b610240610600511115610ad157610af2565b600061060051610620516106800101535b8151600101808352811415610abf575b5050602061062051610660015160206001820306601f820103905061062051010161062052610620516106e0526105a08051602001806106205161066001828460006004600a8704601201f1610b4757600080fd5b5050610600610620516106600151610240818352015b610240610600511115610b6f57610b90565b600061060051610620516106800101535b8151600101808352811415610b5d575b5050602061062051610660015160206001820306601f8201039050610620510101610620527f649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c561062051610660a160006107005260006101406030806020846107c001018260208501600060046016f150508051820191505060006010602082066107400160208284011115610c2557600080fd5b60208061076082610700600060046015f15050818152809050905090506010806020846107c001018260208501600060046013f1505080518201915050806107c0526107c09050602060c0825160208401600060025af1610c8557600080fd5b60c0519050610720526000600060406020820661086001610220518284011115610cae57600080fd5b606080610880826020602088068803016102200160006004601bf1505081815280905090509050602060c0825160208401600060025af1610cee57600080fd5b60c0519050602082610a600101526020810190506000604060206020820661092001610220518284011115610d2257600080fd5b606080610940826020602088068803016102200160006004601bf15050818152809050905090506020806020846109e001018260208501600060046015f1505080518201915050610700516020826109e0010152602081019050806109e0526109e09050602060c0825160208401600060025af1610d9f57600080fd5b60c0519050602082610a6001015260208101905080610a6052610a609050602060c0825160208401600060025af1610dd657600080fd5b60c0519050610840526000600061072051602082610b000101526020810190506101c0602080602084610b0001018260208501600060046015f150508051820191505080610b0052610b009050602060c0825160208401600060025af1610e3c57600080fd5b60c0519050602082610c800101526020810190506000610300600880602084610c0001018260208501600060046012f15050805182019150506000601860208206610b800160208284011115610e9157600080fd5b602080610ba082610700600060046015f1505081815280905090509050601880602084610c0001018260208501600060046014f150508051820191505061084051602082610c0001015260208101905080610c0052610c009050602060c0825160208401600060025af1610f0457600080fd5b60c0519050602082610c8001015260208101905080610c8052610c809050602060c0825160208401600060025af1610f3b57600080fd5b60c0519050610ae052606435610ae05114610f5557600080fd5b6001805460018254011015610f6957600080fd5b6001815401815550600154610d0052610d2060006020818352015b60016001610d0051161415610fb957610ae051610d205160208110610fa857600080fd5b600060c052602060c020015561104d565b6000610d205160208110610fcc57600080fd5b600060c052602060c0200154602082610d40010152602081019050610ae051602082610d4001015260208101905080610d4052610d409050602060c0825160208401600060025af161101d57600080fd5b60c0519050610ae052610d00600261103457600080fd5b60028151048152505b8151600101808352811415610f84575b5050005b60006000fd5b61017f6111d60361017f60003961017f6111d6036000f3"} \ No newline at end of file diff --git a/deposit_contract/contracts/validator_registration.v.py b/deposit_contract/contracts/validator_registration.v.py index bad619b07..6ee27db7a 100644 --- a/deposit_contract/contracts/validator_registration.v.py +++ b/deposit_contract/contracts/validator_registration.v.py @@ -1,10 +1,11 @@ +# Vyper target 0.1.0b12 MIN_DEPOSIT_AMOUNT: constant(uint256) = 1000000000 # Gwei DEPOSIT_CONTRACT_TREE_DEPTH: constant(uint256) = 32 MAX_DEPOSIT_COUNT: constant(uint256) = 4294967295 # 2**DEPOSIT_CONTRACT_TREE_DEPTH - 1 PUBKEY_LENGTH: constant(uint256) = 48 # bytes WITHDRAWAL_CREDENTIALS_LENGTH: constant(uint256) = 32 # bytes -AMOUNT_LENGTH: constant(uint256) = 8 # bytes SIGNATURE_LENGTH: constant(uint256) = 96 # bytes +AMOUNT_LENGTH: constant(uint256) = 8 # bytes DepositEvent: event({ pubkey: bytes[48], @@ -42,7 +43,7 @@ def to_little_endian_64(value: uint256) -> bytes[8]: @public @constant -def get_hash_tree_root() -> bytes32: +def get_deposit_root() -> bytes32: zero_bytes32: bytes32 = 0x0000000000000000000000000000000000000000000000000000000000000000 node: bytes32 = zero_bytes32 size: uint256 = self.deposit_count @@ -65,13 +66,16 @@ def get_deposit_count() -> bytes[8]: @public def deposit(pubkey: bytes[PUBKEY_LENGTH], withdrawal_credentials: bytes[WITHDRAWAL_CREDENTIALS_LENGTH], - signature: bytes[SIGNATURE_LENGTH]): + signature: bytes[SIGNATURE_LENGTH], + deposit_data_root: bytes32): # Avoid overflowing the Merkle tree (and prevent edge case in computing `self.branch`) assert self.deposit_count < MAX_DEPOSIT_COUNT - # Validate deposit data + # Check deposit amount deposit_amount: uint256 = msg.value / as_wei_value(1, "gwei") assert deposit_amount >= MIN_DEPOSIT_AMOUNT + + # Length checks to facilitate formal verification (see https://github.com/ethereum/eth2.0-specs/pull/1362/files#r320361859) assert len(pubkey) == PUBKEY_LENGTH assert len(withdrawal_credentials) == WITHDRAWAL_CREDENTIALS_LENGTH assert len(signature) == SIGNATURE_LENGTH @@ -80,7 +84,7 @@ def deposit(pubkey: bytes[PUBKEY_LENGTH], amount: bytes[8] = self.to_little_endian_64(deposit_amount) log.DepositEvent(pubkey, withdrawal_credentials, amount, signature, self.to_little_endian_64(self.deposit_count)) - # Compute `DepositData` hash tree root + # Compute deposit data root (`DepositData` hash tree root) zero_bytes32: bytes32 = 0x0000000000000000000000000000000000000000000000000000000000000000 pubkey_root: bytes32 = sha256(concat(pubkey, slice(zero_bytes32, start=0, len=64 - PUBKEY_LENGTH))) signature_root: bytes32 = sha256(concat( @@ -91,8 +95,10 @@ def deposit(pubkey: bytes[PUBKEY_LENGTH], sha256(concat(pubkey_root, withdrawal_credentials)), sha256(concat(amount, slice(zero_bytes32, start=0, len=32 - AMOUNT_LENGTH), signature_root)), )) + # Verify computed and expected deposit data roots match + assert node == deposit_data_root - # Add `DepositData` hash tree root to Merkle tree (update a single `branch` node) + # Add deposit data root to Merkle tree (update a single `branch` node) self.deposit_count += 1 size: uint256 = self.deposit_count for height in range(DEPOSIT_CONTRACT_TREE_DEPTH): diff --git a/deposit_contract/requirements-testing.txt b/deposit_contract/requirements-testing.txt index 280d7e527..0b3d9d22c 100644 --- a/deposit_contract/requirements-testing.txt +++ b/deposit_contract/requirements-testing.txt @@ -1,5 +1,5 @@ eth-tester[py-evm]==0.1.0b39 -vyper==0.1.0b10 +vyper==0.1.0b12 web3==5.0.0b2 pytest==3.6.1 ../test_libs/pyspec diff --git a/deposit_contract/tests/contracts/test_deposit.py b/deposit_contract/tests/contracts/test_deposit.py index 1c96d074e..01586d070 100644 --- a/deposit_contract/tests/contracts/test_deposit.py +++ b/deposit_contract/tests/contracts/test_deposit.py @@ -6,7 +6,6 @@ import pytest import eth_utils from tests.contracts.conftest import ( - DEPOSIT_CONTRACT_TREE_DEPTH, FULL_DEPOSIT_AMOUNT, MIN_DEPOSIT_AMOUNT, ) @@ -14,29 +13,42 @@ from tests.contracts.conftest import ( from eth2spec.phase0.spec import ( DepositData, ) -from eth2spec.utils.hash_function import hash from eth2spec.utils.ssz.ssz_typing import List from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, ) +SAMPLE_PUBKEY = b'\x11' * 48 +SAMPLE_WITHDRAWAL_CREDENTIALS = b'\x22' * 32 +SAMPLE_VALID_SIGNATURE = b'\x33' * 96 + + @pytest.fixture -def deposit_input(): +def deposit_input(amount): """ pubkey: bytes[48] withdrawal_credentials: bytes[32] signature: bytes[96] + deposit_data_root: bytes[32] """ return ( - b'\x11' * 48, - b'\x22' * 32, - b'\x33' * 96, + SAMPLE_PUBKEY, + SAMPLE_WITHDRAWAL_CREDENTIALS, + SAMPLE_VALID_SIGNATURE, + hash_tree_root( + DepositData( + pubkey=SAMPLE_PUBKEY, + withdrawal_credentials=SAMPLE_WITHDRAWAL_CREDENTIALS, + amount=amount, + signature=SAMPLE_VALID_SIGNATURE, + ), + ) ) @pytest.mark.parametrize( - 'success,deposit_amount', + ('success', 'amount'), [ (True, FULL_DEPOSIT_AMOUNT), (True, MIN_DEPOSIT_AMOUNT), @@ -47,18 +59,24 @@ def deposit_input(): def test_deposit_amount(registration_contract, w3, success, - deposit_amount, + amount, assert_tx_failed, deposit_input): call = registration_contract.functions.deposit(*deposit_input) if success: - assert call.transact({"value": deposit_amount * eth_utils.denoms.gwei}) + assert call.transact({"value": amount * eth_utils.denoms.gwei}) else: assert_tx_failed( - lambda: call.transact({"value": deposit_amount * eth_utils.denoms.gwei}) + lambda: call.transact({"value": amount * eth_utils.denoms.gwei}) ) +@pytest.mark.parametrize( + 'amount', + [ + (FULL_DEPOSIT_AMOUNT) + ] +) @pytest.mark.parametrize( 'invalid_pubkey,invalid_withdrawal_credentials,invalid_signature,success', [ @@ -71,38 +89,62 @@ def test_deposit_amount(registration_contract, def test_deposit_inputs(registration_contract, w3, assert_tx_failed, - deposit_input, + amount, invalid_pubkey, invalid_withdrawal_credentials, invalid_signature, success): - pubkey = deposit_input[0][2:] if invalid_pubkey else deposit_input[0] - if invalid_withdrawal_credentials: # this one is different to satisfy linter - withdrawal_credentials = deposit_input[1][2:] - else: - withdrawal_credentials = deposit_input[1] - signature = deposit_input[2][2:] if invalid_signature else deposit_input[2] + pubkey = SAMPLE_PUBKEY[2:] if invalid_pubkey else SAMPLE_PUBKEY + withdrawal_credentials = ( + SAMPLE_WITHDRAWAL_CREDENTIALS[2:] if invalid_withdrawal_credentials + else SAMPLE_WITHDRAWAL_CREDENTIALS + ) + signature = SAMPLE_VALID_SIGNATURE[2:] if invalid_signature else SAMPLE_VALID_SIGNATURE call = registration_contract.functions.deposit( pubkey, withdrawal_credentials, signature, + hash_tree_root( + DepositData( + pubkey=SAMPLE_PUBKEY if invalid_pubkey else pubkey, + withdrawal_credentials=( + SAMPLE_WITHDRAWAL_CREDENTIALS if invalid_withdrawal_credentials + else withdrawal_credentials + ), + amount=amount, + signature=SAMPLE_VALID_SIGNATURE if invalid_signature else signature, + ), + ) ) if success: - assert call.transact({"value": FULL_DEPOSIT_AMOUNT * eth_utils.denoms.gwei}) + assert call.transact({"value": amount * eth_utils.denoms.gwei}) else: assert_tx_failed( - lambda: call.transact({"value": FULL_DEPOSIT_AMOUNT * eth_utils.denoms.gwei}) + lambda: call.transact({"value": amount * eth_utils.denoms.gwei}) ) -def test_deposit_event_log(registration_contract, a0, w3, deposit_input): +def test_deposit_event_log(registration_contract, a0, w3): log_filter = registration_contract.events.DepositEvent.createFilter( fromBlock='latest', ) - deposit_amount_list = [randint(MIN_DEPOSIT_AMOUNT, FULL_DEPOSIT_AMOUNT * 2) for _ in range(3)] + for i in range(3): + deposit_input = ( + SAMPLE_PUBKEY, + SAMPLE_WITHDRAWAL_CREDENTIALS, + SAMPLE_VALID_SIGNATURE, + hash_tree_root( + DepositData( + pubkey=SAMPLE_PUBKEY, + withdrawal_credentials=SAMPLE_WITHDRAWAL_CREDENTIALS, + amount=deposit_amount_list[i], + signature=SAMPLE_VALID_SIGNATURE, + ), + ) + ) registration_contract.functions.deposit( *deposit_input, ).transact({"value": deposit_amount_list[i] * eth_utils.denoms.gwei}) @@ -118,7 +160,7 @@ def test_deposit_event_log(registration_contract, a0, w3, deposit_input): assert log['index'] == i.to_bytes(8, 'little') -def test_deposit_tree(registration_contract, w3, assert_tx_failed, deposit_input): +def test_deposit_tree(registration_contract, w3, assert_tx_failed): log_filter = registration_contract.events.DepositEvent.createFilter( fromBlock='latest', ) @@ -126,6 +168,20 @@ def test_deposit_tree(registration_contract, w3, assert_tx_failed, deposit_input deposit_amount_list = [randint(MIN_DEPOSIT_AMOUNT, FULL_DEPOSIT_AMOUNT * 2) for _ in range(10)] deposit_data_list = [] for i in range(0, 10): + deposit_data = DepositData( + pubkey=SAMPLE_PUBKEY, + withdrawal_credentials=SAMPLE_WITHDRAWAL_CREDENTIALS, + amount=deposit_amount_list[i], + signature=SAMPLE_VALID_SIGNATURE, + ) + deposit_input = ( + SAMPLE_PUBKEY, + SAMPLE_WITHDRAWAL_CREDENTIALS, + SAMPLE_VALID_SIGNATURE, + hash_tree_root(deposit_data), + ) + deposit_data_list.append(deposit_data) + tx_hash = registration_contract.functions.deposit( *deposit_input, ).transact({"value": deposit_amount_list[i] * eth_utils.denoms.gwei}) @@ -138,12 +194,8 @@ def test_deposit_tree(registration_contract, w3, assert_tx_failed, deposit_input assert log["index"] == i.to_bytes(8, 'little') - deposit_data_list.append(DepositData( - pubkey=deposit_input[0], - withdrawal_credentials=deposit_input[1], - amount=deposit_amount_list[i], - signature=deposit_input[2], - )) - + # Check deposit count and root + count = len(deposit_data_list).to_bytes(8, 'little') + assert count == registration_contract.functions.get_deposit_count().call() root = hash_tree_root(List[DepositData, 2**32](*deposit_data_list)) - assert root == registration_contract.functions.get_hash_tree_root().call() + assert root == registration_contract.functions.get_deposit_root().call() diff --git a/specs/core/0_deposit-contract.md b/specs/core/0_deposit-contract.md index ade1006a0..06962594e 100644 --- a/specs/core/0_deposit-contract.md +++ b/specs/core/0_deposit-contract.md @@ -34,11 +34,11 @@ This document represents the specification for the beacon chain deposit contract ## Ethereum 1.0 deposit contract -The initial deployment phases of Ethereum 2.0 are implemented without consensus changes to Ethereum 1.0. A deposit contract at address `DEPOSIT_CONTRACT_ADDRESS` is added to Ethereum 1.0 for deposits of ETH to the beacon chain. Validator balances will be withdrawable to the shards in Phase 2 (i.e. when the EVM 2.0 is deployed and the shards have state). +The initial deployment phases of Ethereum 2.0 are implemented without consensus changes to Ethereum 1.0. A deposit contract at address `DEPOSIT_CONTRACT_ADDRESS` is added to Ethereum 1.0 for deposits of ETH to the beacon chain. Validator balances will be withdrawable to the shards in Phase 2. ### `deposit` function -The deposit contract has a public `deposit` function to make deposits. It takes as arguments `pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96]` corresponding to a [`DepositData`](./0_beacon-chain.md#depositdata) object. +The deposit contract has a public `deposit` function to make deposits. It takes as arguments `pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96], deposit_data_root: bytes32`. The first three arguments populate a [`DepositData`](./0_beacon-chain.md#depositdata) object, and `deposit_data_root` is the expected `DepositData` root as a protection against malformatted calldata. #### Deposit amount diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index ef5ad4415..3764e7df1 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -218,7 +218,7 @@ def get_epoch_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> ##### Eth1 Data -The `block.eth1_data` field is for block proposers to vote on recent Eth 1.0 data. This recent data contains an Eth 1.0 block hash as well as the associated deposit root (as calculated by the `get_hash_tree_root()` method of the deposit contract) and deposit count after execution of the corresponding Eth 1.0 block. If over half of the block proposers in the current Eth 1.0 voting period vote for the same `eth1_data` then `state.eth1_data` updates at the end of the voting period. Each deposit in `block.body.deposits` must verify against `state.eth1_data.eth1_deposit_root`. +The `block.eth1_data` field is for block proposers to vote on recent Eth 1.0 data. This recent data contains an Eth 1.0 block hash as well as the associated deposit root (as calculated by the `get_deposit_root()` method of the deposit contract) and deposit count after execution of the corresponding Eth 1.0 block. If over half of the block proposers in the current Eth 1.0 voting period vote for the same `eth1_data` then `state.eth1_data` updates at the end of the voting period. Each deposit in `block.body.deposits` must verify against `state.eth1_data.eth1_deposit_root`. Let `get_eth1_data(distance: uint64) -> Eth1Data` be the (subjective) function that returns the Eth 1.0 data at distance `distance` relative to the Eth 1.0 head at the start of the current Eth 1.0 voting period. Let `previous_eth1_distance` be the distance relative to the Eth 1.0 block corresponding to `state.eth1_data.block_hash` at the start of the current Eth 1.0 voting period. An honest block proposer sets `block.eth1_data = get_eth1_vote(state, previous_eth1_distance)` where: From dbcce177691c3466bc964254b186bcf87e98b5f7 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Wed, 4 Sep 2019 09:00:59 -0700 Subject: [PATCH 74/89] Update sync_protocol.md --- specs/light_client/sync_protocol.md | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index 207e0e63e..3425c0f54 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -13,9 +13,8 @@ - [Constants](#constants) - [Containers](#containers) - [`LightClientUpdate`](#lightclientupdate) - - [Helpers](#helpers) - [`LightClientMemory`](#lightclientmemory) - - [`unpack_compact_validator`](#unpack_compact_validator) + - [Helpers](#helpers) - [`get_persistent_committee_pubkeys_and_balances`](#get_persistent_committee_pubkeys_and_balances) - [Light client state updates](#light-client-state-updates) - [Data overhead](#data-overhead) @@ -62,12 +61,9 @@ class LightClientUpdate(container): committee_branch: Vector[Hash, PERSISTENT_COMMITTEE_ROOT_IN_BEACON_STATE_DEPTH + log_2(SHARD_COUNT)] ``` -## Helpers - ### `LightClientMemory` ```python -@dataclass class LightClientMemory(object): shard: Shard # Randomly initialized and retained forever header: BeaconBlockHeader # Beacon header which is not expected to revert @@ -77,19 +73,7 @@ class LightClientMemory(object): next_committee: CompactCommittee ``` -### `unpack_compact_validator` - -```python -def unpack_compact_validator(compact_validator: CompactValidator) -> Tuple[ValidatorIndex, bool, uint64]: - """ - Return the index, slashed, effective_balance // EFFECTIVE_BALANCE_INCREMENT of ``compact_validator``. - """ - return ( - ValidatorIndex(compact_validator >> 16), - bool((compact_validator >> 15) % 2), - uint64(compact_validator & (2**15 - 1)), - ) -``` +## Helpers ### `get_persistent_committee_pubkeys_and_balances` From f1065faf9cd711fd6b2b03fd58b4b50fc201a4b8 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Thu, 5 Sep 2019 06:54:20 -0700 Subject: [PATCH 75/89] Update sync_protocol.md --- specs/light_client/sync_protocol.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index 3425c0f54..18810d48d 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -61,10 +61,13 @@ class LightClientUpdate(container): committee_branch: Vector[Hash, PERSISTENT_COMMITTEE_ROOT_IN_BEACON_STATE_DEPTH + log_2(SHARD_COUNT)] ``` +## Helpers + ### `LightClientMemory` ```python class LightClientMemory(object): + @dataclass shard: Shard # Randomly initialized and retained forever header: BeaconBlockHeader # Beacon header which is not expected to revert # Persistent committees corresponding to the beacon header @@ -73,8 +76,6 @@ class LightClientMemory(object): next_committee: CompactCommittee ``` -## Helpers - ### `get_persistent_committee_pubkeys_and_balances` ```python From 1e74cf5f0d2866d1811b5abb8548452c3db5be26 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Thu, 5 Sep 2019 06:55:40 -0700 Subject: [PATCH 76/89] Update sync_protocol.md --- specs/light_client/sync_protocol.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index 18810d48d..c3b035270 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -13,8 +13,8 @@ - [Constants](#constants) - [Containers](#containers) - [`LightClientUpdate`](#lightclientupdate) - - [`LightClientMemory`](#lightclientmemory) - [Helpers](#helpers) + - [`LightClientMemory`](#lightclientmemory) - [`get_persistent_committee_pubkeys_and_balances`](#get_persistent_committee_pubkeys_and_balances) - [Light client state updates](#light-client-state-updates) - [Data overhead](#data-overhead) @@ -66,8 +66,8 @@ class LightClientUpdate(container): ### `LightClientMemory` ```python +@dataclass class LightClientMemory(object): - @dataclass shard: Shard # Randomly initialized and retained forever header: BeaconBlockHeader # Beacon header which is not expected to revert # Persistent committees corresponding to the beacon header From c06ffc4924bbc80d4864c7370e05eacc02003544 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Fri, 6 Sep 2019 17:07:45 -0400 Subject: [PATCH 77/89] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0de6cd9ef..8fb2bc591 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ See the [Eth 2.0 Phase 2 Wiki](https://hackmd.io/UzysWse1Th240HELswKqVA?view) fo * [General test format](specs/test_formats/README.md) * [Merkle proof formats](specs/light_client/merkle_proofs.md) * [Light client syncing protocol](specs/light_client/sync_protocol.md) -* [Beacon node API for validator](specs/validator/0_beacon-node-validator-api.md) +* [Beacon node API for validator](https://github.com/ethereum/eth2.0-APIs/blob/master/apis/validator/beacon-node-validator-api.md) ## Additional specifications for client implementers From 1d71ae9af4251b2bf3e6b1200b52104da771a8d8 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Sat, 7 Sep 2019 09:28:45 -0400 Subject: [PATCH 78/89] Update README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 8fb2bc591..fdfbd1c5e 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,6 @@ See the [Eth 2.0 Phase 2 Wiki](https://hackmd.io/UzysWse1Th240HELswKqVA?view) fo * [General test format](specs/test_formats/README.md) * [Merkle proof formats](specs/light_client/merkle_proofs.md) * [Light client syncing protocol](specs/light_client/sync_protocol.md) -* [Beacon node API for validator](https://github.com/ethereum/eth2.0-APIs/blob/master/apis/validator/beacon-node-validator-api.md) - ## Additional specifications for client implementers From d66e6ca0b8f055591d42ba3d09918013dbee7448 Mon Sep 17 00:00:00 2001 From: Gregory Markou <16929357+GregTheGreek@users.noreply.github.com> Date: Sun, 15 Sep 2019 17:12:36 -0400 Subject: [PATCH 79/89] Update 1_custody-game.md Fix table --- specs/core/1_custody-game.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index d789a6855..f03ffada6 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -77,7 +77,8 @@ This document details the beacon chain additions and changes in Phase 1 of Ether ## Constants ### Misc - +| Name | Value | +| - | - | | `BLS12_381_Q` | `4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787` | | `MINOR_REWARD_QUOTIENT` | `2**8` (= 256) | From 6d9dd04b5024dd21fe68e8bef40798a9cd68968d Mon Sep 17 00:00:00 2001 From: Gregory Markou <16929357+GregTheGreek@users.noreply.github.com> Date: Sun, 15 Sep 2019 21:03:50 -0400 Subject: [PATCH 80/89] Update 1_custody-game.md --- specs/core/1_custody-game.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index f03ffada6..3a7668caf 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -340,7 +340,7 @@ def legendre_bit(a: int, q: int) -> int: Given one proof of custody chunk, returns the proof of custody subchunks of the correct sizes. ```python -def custody_subchunkify(bytez: bytes) -> list: +def custody_subchunkify(bytez: bytes) -> list[int]: bytez += b'\x00' * (-len(bytez) % BYTES_PER_CUSTODY_SUBCHUNK) return [bytez[i:i + BYTES_PER_CUSTODY_SUBCHUNK] for i in range(0, len(bytez), BYTES_PER_CUSTODY_SUBCHUNK)] From 970ae2fad1c82e250e8a1369126e6e4df310ed47 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 16 Sep 2019 09:16:44 -0500 Subject: [PATCH 81/89] update validator doc with modified beacon proposer selection --- specs/validator/0_beacon-chain-validator.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index ef5ad4415..ddf5b5f23 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -160,7 +160,7 @@ def get_committee_assignment(state: BeaconState, return None ``` -A validator can use the following function to see if they are supposed to propose during their assigned committee slot. This function can only be run with a `state` of the slot in question. Proposer selection is only stable within the context of the current epoch. +A validator can use the following function to see if they are supposed to propose during a slot. This function can only be run with a `state` of the slot in question. Proposer selection is only stable within the context of the current epoch. ```python def is_proposer(state: BeaconState, @@ -170,6 +170,8 @@ def is_proposer(state: BeaconState, *Note*: To see if a validator is assigned to propose during the slot, the beacon state must be in the epoch in question. At the epoch boundaries, the validator must run an epoch transition into the epoch to successfully check the proposal assignment of the first slot. +*Note*: `BeaconBlock` proposal is distinct from crosslink committee assignment, and in a given epoch each responsbility might occur at different a different slot. + ### Lookahead The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing, which must be checked during the epoch in question. From 794a2407a197ad98a9ee87158a71ec10a3b52f54 Mon Sep 17 00:00:00 2001 From: Gregory Markou <16929357+GregTheGreek@users.noreply.github.com> Date: Mon, 16 Sep 2019 11:19:51 -0400 Subject: [PATCH 82/89] Update specs/core/1_custody-game.md Co-Authored-By: Danny Ryan --- specs/core/1_custody-game.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 3a7668caf..64807d80d 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -340,7 +340,7 @@ def legendre_bit(a: int, q: int) -> int: Given one proof of custody chunk, returns the proof of custody subchunks of the correct sizes. ```python -def custody_subchunkify(bytez: bytes) -> list[int]: +def custody_subchunkify(bytez: bytes) -> list[bytes]: bytez += b'\x00' * (-len(bytez) % BYTES_PER_CUSTODY_SUBCHUNK) return [bytez[i:i + BYTES_PER_CUSTODY_SUBCHUNK] for i in range(0, len(bytez), BYTES_PER_CUSTODY_SUBCHUNK)] From 834b81122237afd977cd35a9e9b2e172ebafc733 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 16 Sep 2019 13:26:27 -0600 Subject: [PATCH 83/89] fix custody_subchunkify return type --- specs/core/1_custody-game.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 64807d80d..0eea43dc0 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -340,7 +340,7 @@ def legendre_bit(a: int, q: int) -> int: Given one proof of custody chunk, returns the proof of custody subchunks of the correct sizes. ```python -def custody_subchunkify(bytez: bytes) -> list[bytes]: +def custody_subchunkify(bytez: bytes) -> Sequence[bytes]: bytez += b'\x00' * (-len(bytez) % BYTES_PER_CUSTODY_SUBCHUNK) return [bytez[i:i + BYTES_PER_CUSTODY_SUBCHUNK] for i in range(0, len(bytez), BYTES_PER_CUSTODY_SUBCHUNK)] From 2e5389978402736121819fd6dfe0d9031da47b6a Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 21 Sep 2019 02:59:41 +0300 Subject: [PATCH 84/89] typo --- specs/validator/0_beacon-chain-validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index ddf5b5f23..a9736de70 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -170,7 +170,7 @@ def is_proposer(state: BeaconState, *Note*: To see if a validator is assigned to propose during the slot, the beacon state must be in the epoch in question. At the epoch boundaries, the validator must run an epoch transition into the epoch to successfully check the proposal assignment of the first slot. -*Note*: `BeaconBlock` proposal is distinct from crosslink committee assignment, and in a given epoch each responsbility might occur at different a different slot. +*Note*: `BeaconBlock` proposal is distinct from crosslink committee assignment, and in a given epoch each responsability might occur at different a different slot. ### Lookahead From 2ec363d614ea0d597824c45e9bd28cad15663ae3 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 23 Sep 2019 12:42:29 -0600 Subject: [PATCH 85/89] typo --- specs/validator/0_beacon-chain-validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index a9736de70..a25dc969b 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -170,7 +170,7 @@ def is_proposer(state: BeaconState, *Note*: To see if a validator is assigned to propose during the slot, the beacon state must be in the epoch in question. At the epoch boundaries, the validator must run an epoch transition into the epoch to successfully check the proposal assignment of the first slot. -*Note*: `BeaconBlock` proposal is distinct from crosslink committee assignment, and in a given epoch each responsability might occur at different a different slot. +*Note*: `BeaconBlock` proposal is distinct from crosslink committee assignment, and in a given epoch each responsibility might occur at different a different slot. ### Lookahead From 47a818c705335d166b5a0d99bdefff9f3d1f042a Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sun, 22 Sep 2019 20:27:42 +0100 Subject: [PATCH 86/89] Add domain_type to get_seed --- configs/mainnet.yaml | 4 ++-- configs/minimal.yaml | 4 ++-- specs/core/0_beacon-chain.md | 20 +++++++++---------- specs/core/1_shard-data-chains.md | 5 +++-- specs/validator/0_beacon-chain-validator.md | 2 +- .../eth2spec/test/helpers/attestations.py | 2 +- 6 files changed, 19 insertions(+), 18 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index d4e69dab5..86fa8dcc5 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -123,8 +123,8 @@ MAX_TRANSFERS: 0 # Signature domains # --------------------------------------------------------------- DOMAIN_BEACON_PROPOSER: 0x00000000 -DOMAIN_RANDAO: 0x01000000 -DOMAIN_ATTESTATION: 0x02000000 +DOMAIN_BEACON_ATTESTER: 0x01000000 +DOMAIN_RANDAO: 0x02000000 DOMAIN_DEPOSIT: 0x03000000 DOMAIN_VOLUNTARY_EXIT: 0x04000000 DOMAIN_TRANSFER: 0x05000000 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index be787ca3c..8c6a9e11f 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -125,8 +125,8 @@ MAX_TRANSFERS: 0 # Signature domains # --------------------------------------------------------------- DOMAIN_BEACON_PROPOSER: 0x00000000 -DOMAIN_RANDAO: 0x01000000 -DOMAIN_ATTESTATION: 0x02000000 +DOMAIN_BEACON_ATTESTER: 0x01000000 +DOMAIN_RANDAO: 0x02000000 DOMAIN_DEPOSIT: 0x03000000 DOMAIN_VOLUNTARY_EXIT: 0x04000000 DOMAIN_TRANSFER: 0x05000000 diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 5bdfc7a5b..cfdf24e71 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -19,7 +19,7 @@ - [State list lengths](#state-list-lengths) - [Rewards and penalties](#rewards-and-penalties) - [Max operations per block](#max-operations-per-block) - - [Signature domain types](#signature-domain-types) + - [Domain types](#domain-types) - [Containers](#containers) - [Misc dependencies](#misc-dependencies) - [`Fork`](#fork) @@ -148,7 +148,7 @@ We define the following Python custom types for type hinting and readability: | `Gwei` | `uint64` | an amount in Gwei | | `Hash` | `Bytes32` | a hash | | `Version` | `Bytes4` | a fork version number | -| `DomainType` | `Bytes4` | a signature domain type | +| `DomainType` | `Bytes4` | a domain type | | `Domain` | `Bytes8` | a signature domain | | `BLSPubkey` | `Bytes48` | a BLS12-381 public key | | `BLSSignature` | `Bytes96` | a BLS12-381 signature | @@ -250,15 +250,15 @@ The following values are (non-configurable) constants used throughout the specif | `MAX_VOLUNTARY_EXITS` | `2**4` (= 16) | | `MAX_TRANSFERS` | `0` | -### Signature domain types +### Domain types The following types are defined, mapping into `DomainType` (little endian): | Name | Value | | - | - | | `DOMAIN_BEACON_PROPOSER` | `0` | -| `DOMAIN_RANDAO` | `1` | -| `DOMAIN_ATTESTATION` | `2` | +| `DOMAIN_BEACON_ATTESTER` | `1` | +| `DOMAIN_RANDAO` | `2` | | `DOMAIN_DEPOSIT` | `3` | | `DOMAIN_VOLUNTARY_EXIT` | `4` | | `DOMAIN_TRANSFER` | `5` | @@ -671,7 +671,7 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b1)), ], signature=indexed_attestation.signature, - domain=get_domain(state, DOMAIN_ATTESTATION, indexed_attestation.data.target.epoch), + domain=get_domain(state, DOMAIN_BEACON_ATTESTER, indexed_attestation.data.target.epoch), ): return False return True @@ -870,12 +870,12 @@ def get_validator_churn_limit(state: BeaconState) -> uint64: #### `get_seed` ```python -def get_seed(state: BeaconState, epoch: Epoch) -> Hash: +def get_seed(state: BeaconState, epoch: Epoch, domain_type: DomainType) -> Hash: """ Return the seed at ``epoch``. """ mix = get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD - 1)) # Avoid underflow - return hash(mix + int_to_bytes(epoch, length=32)) + return hash(domain_type + mix + int_to_bytes(epoch, length=32)) ``` #### `get_committee_count` @@ -901,7 +901,7 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> S """ return compute_committee( indices=get_active_validator_indices(state, epoch), - seed=get_seed(state, epoch), + seed=get_seed(state, epoch, DOMAIN_BEACON_ATTESTER), index=(shard + SHARD_COUNT - get_start_shard(state, epoch)) % SHARD_COUNT, count=get_committee_count(state, epoch), ) @@ -941,7 +941,7 @@ def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: Return the beacon proposer index at the current slot. """ epoch = get_current_epoch(state) - seed = hash(get_seed(state, epoch) + int_to_bytes(state.slot, length=8)) + seed = hash(get_seed(state, epoch, DOMAIN_BEACON_PROPOSER) + int_to_bytes(state.slot, length=8)) indices = get_active_validator_indices(state, epoch) return compute_proposer_index(state, indices, seed) ``` diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 3dc549816..a6e9c997f 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -225,7 +225,7 @@ def get_period_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> Sequ """ full_committee = compute_committee( indices=get_active_validator_indices(state, epoch), - seed=get_seed(state, epoch), + seed=get_seed(state, epoch, DOMAIN_SHARD_ATTESTER), index=shard, count=SHARD_COUNT, ) @@ -270,7 +270,8 @@ def get_shard_block_proposer_index(state: BeaconState, return None MAX_RANDOM_BYTE = 2**8 - 1 - seed = hash(get_seed(state, current_epoch) + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8)) + seed = get_seed(state, current_epoch, DOMAIN_SHARD_PROPOSER) + seed = hash(seed + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8)) i = 0 while True: candidate_index = active_indices[(slot + i) % len(active_indices)] diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index fc91fd2e7..29723f391 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -345,7 +345,7 @@ def get_signed_attestation_data(state: BeaconState, attestation: IndexedAttestat custody_bit=0b0, ) - domain = get_domain(state, DOMAIN_ATTESTATION, attestation.data.target.epoch) + domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch) return bls_sign(privkey, hash_tree_root(attestation_data_and_custody_bit), domain) ``` diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 868517018..23d1a8f8f 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -122,7 +122,7 @@ def get_attestation_signature(spec, state, attestation_data, privkey, custody_bi privkey=privkey, domain=spec.get_domain( state=state, - domain_type=spec.DOMAIN_ATTESTATION, + domain_type=spec.DOMAIN_BEACON_ATTESTER, message_epoch=attestation_data.target.epoch, ) ) From a18312559507f27cc0092fc55bfee07bfa2d5334 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 22 Sep 2019 21:04:48 +0100 Subject: [PATCH 87/89] cleanups to get_seed 1) Put `domain_type` and `epoch` upfront. This pattern can be reused for signature domains. 2) Change `int_to_bytes(epoch, length=32)` to `int_to_bytes(epoch, length=8)` to match `uint64` length. --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index cfdf24e71..4f20898a3 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -875,7 +875,7 @@ def get_seed(state: BeaconState, epoch: Epoch, domain_type: DomainType) -> Hash: Return the seed at ``epoch``. """ mix = get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD - 1)) # Avoid underflow - return hash(domain_type + mix + int_to_bytes(epoch, length=32)) + return hash(domain_type + int_to_bytes(epoch, length=8) + mix) ``` #### `get_committee_count` From e7db58cb7e402a943c698f5d0a235ed21abea558 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Mon, 23 Sep 2019 19:07:10 +0100 Subject: [PATCH 88/89] Rename ACTIVATION_EXIT_DELAY to MAX_SEED_LOOKAHEAD for phase 1 --- configs/mainnet.yaml | 2 +- configs/minimal.yaml | 2 +- specs/core/0_beacon-chain.md | 4 ++-- specs/core/1_custody-game.md | 8 ++++---- specs/validator/0_beacon-chain-validator.md | 2 +- .../epoch_processing/test_process_registry_updates.py | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 86fa8dcc5..c11f1e54c 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -60,7 +60,7 @@ SLOTS_PER_EPOCH: 64 # 2**0 (= 1) epochs 6.4 minutes MIN_SEED_LOOKAHEAD: 1 # 2**2 (= 4) epochs 25.6 minutes -ACTIVATION_EXIT_DELAY: 4 +MAX_SEED_LOOKAHEAD: 4 # 2**10 (= 1,024) slots ~1.7 hours SLOTS_PER_ETH1_VOTING_PERIOD: 1024 # 2**13 (= 8,192) slots ~13 hours diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 8c6a9e11f..4c32eae4d 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -59,7 +59,7 @@ SLOTS_PER_EPOCH: 8 # 2**0 (= 1) epochs MIN_SEED_LOOKAHEAD: 1 # 2**2 (= 4) epochs -ACTIVATION_EXIT_DELAY: 4 +MAX_SEED_LOOKAHEAD: 4 # [customized] higher frequency new deposits from eth1 for testing SLOTS_PER_ETH1_VOTING_PERIOD: 16 # [customized] smaller state diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 4f20898a3..681d82457 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -210,7 +210,7 @@ The following values are (non-configurable) constants used throughout the specif | `MIN_ATTESTATION_INCLUSION_DELAY` | `2**0` (= 1) | slots | 6 seconds | | `SLOTS_PER_EPOCH` | `2**6` (= 64) | slots | 6.4 minutes | | `MIN_SEED_LOOKAHEAD` | `2**0` (= 1) | epochs | 6.4 minutes | -| `ACTIVATION_EXIT_DELAY` | `2**2` (= 4) | epochs | 25.6 minutes | +| `MAX_SEED_LOOKAHEAD` | `2**2` (= 4) | epochs | 25.6 minutes | | `SLOTS_PER_ETH1_VOTING_PERIOD` | `2**10` (= 1,024) | slots | ~1.7 hours | | `SLOTS_PER_HISTORICAL_ROOT` | `2**13` (= 8,192) | slots | ~13 hours | | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours | @@ -779,7 +779,7 @@ def compute_activation_exit_epoch(epoch: Epoch) -> Epoch: """ Return the epoch during which validator activations and exits initiated in ``epoch`` take effect. """ - return Epoch(epoch + 1 + ACTIVATION_EXIT_DELAY) + return Epoch(epoch + 1 + MAX_SEED_LOOKAHEAD) ``` #### `compute_domain` diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 0eea43dc0..158d575e2 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -282,7 +282,7 @@ def ceillog2(x: uint64) -> int: ### `is_valid_merkle_branch_with_mixin` ```python -def is_valid_merkle_branch_with_mixin(leaf: Hash, +def is_valid_merkle_branch_with_mixin(leaf: Hash, branch: Sequence[Hash], depth: uint64, index: uint64, @@ -315,7 +315,7 @@ def legendre_bit(a: int, q: int) -> int: if a >= q: return legendre_bit(a % q, q) if a == 0: - return 0 + return 0 assert(q > a > 0 and q % 2 == 1) t = 1 n = q @@ -602,7 +602,7 @@ def process_bit_challenge(state: BeaconState, challenge: CustodyBitChallenge) -> # Verify attestation is eligible for challenging responder = state.validators[challenge.responder_index] assert get_current_epoch(state) <= get_randao_epoch_for_custody_period( - get_custody_period_for_validator(state, challenge.responder_index, epoch), + get_custody_period_for_validator(state, challenge.responder_index, epoch), challenge.responder_index ) + 2 * EPOCHS_PER_CUSTODY_PERIOD + responder.max_reveal_lateness @@ -673,7 +673,7 @@ def process_chunk_challenge_response(state: BeaconState, # Verify bit challenge data is null assert response.chunk_bits_branch == [] and response.chunk_bits_leaf == Hash() # Verify minimum delay - assert get_current_epoch(state) >= challenge.inclusion_epoch + ACTIVATION_EXIT_DELAY + assert get_current_epoch(state) >= challenge.inclusion_epoch + MAX_SEED_LOOKAHEAD # Verify the chunk matches the crosslink data root assert is_valid_merkle_branch( leaf=hash_tree_root(response.chunk), diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 29723f391..8a9cf1b5d 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -114,7 +114,7 @@ Once a validator has been processed and added to the beacon state's `validators` ### Activation -In normal operation, the validator is quickly activated, at which point the validator is added to the shuffling and begins validation after an additional `ACTIVATION_EXIT_DELAY` epochs (25.6 minutes). +In normal operation, the validator is quickly activated, at which point the validator is added to the shuffling and begins validation after an additional `MAX_SEED_LOOKAHEAD` epochs (25.6 minutes). The function [`is_active_validator`](../core/0_beacon-chain.md#is_active_validator) can be used to check if a validator is active during a given epoch. Usage is as follows: diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py index ab6a74a70..bfd992ffa 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py @@ -21,7 +21,7 @@ def test_activation(spec, state): index = 0 mock_deposit(spec, state, index) - for _ in range(spec.ACTIVATION_EXIT_DELAY + 1): + for _ in range(spec.MAX_SEED_LOOKAHEAD + 1): next_epoch(spec, state) yield from run_process_registry_updates(spec, state) @@ -73,7 +73,7 @@ def test_ejection(spec, state): # Mock an ejection state.validators[index].effective_balance = spec.EJECTION_BALANCE - for _ in range(spec.ACTIVATION_EXIT_DELAY + 1): + for _ in range(spec.MAX_SEED_LOOKAHEAD + 1): next_epoch(spec, state) yield from run_process_registry_updates(spec, state) From 25efbe74589895bcd63d908e5686520b5bbaf699 Mon Sep 17 00:00:00 2001 From: Justin Date: Mon, 23 Sep 2019 20:58:29 +0100 Subject: [PATCH 89/89] Rename seed to epoch_seed as per Danny --- specs/core/1_shard-data-chains.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index a6e9c997f..f24c6f9c3 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -270,8 +270,8 @@ def get_shard_block_proposer_index(state: BeaconState, return None MAX_RANDOM_BYTE = 2**8 - 1 - seed = get_seed(state, current_epoch, DOMAIN_SHARD_PROPOSER) - seed = hash(seed + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8)) + epoch_seed = get_seed(state, current_epoch, DOMAIN_SHARD_PROPOSER) + seed = hash(epoch_seed + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8)) i = 0 while True: candidate_index = active_indices[(slot + i) % len(active_indices)]