From 22d44969f75af3a9dfc76aeafb584c4e40599247 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 16 Apr 2019 14:54:17 +1000 Subject: [PATCH 01/84] Add initial libp2p standardization --- specs/networking/libp2p-standardization.md | 131 +++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 specs/networking/libp2p-standardization.md diff --git a/specs/networking/libp2p-standardization.md b/specs/networking/libp2p-standardization.md new file mode 100644 index 000000000..ddefca02a --- /dev/null +++ b/specs/networking/libp2p-standardization.md @@ -0,0 +1,131 @@ +ETH 2.0 Networking Spec - Libp2p standard protocols +=== + +# Abstract + +Ethereum 2.0 clients plan to use the libp2p protocol networking stack for +mainnet release. This document aims to standardize the libp2p client protocols, +configuration and messaging formats. + +# Libp2p Protocols + +## Gossipsub + +#### Protocol id: `/meshsub/1.0.0` + +*Note: Parameters listed here are subject to a large-scale network feasibility +study* + +The [Gossipsub](https://github.com/libp2p/specs/tree/master/pubsub/gossipsub) +protocol will be used for block and attestation propagation across the +network. + +### Configuration Parameters + +Gossipsub has a number of internal configuration parameters which directly +effect the network performance. Clients can implement independently, however +we aim to standardize these across clients to optimize the gossip network for +propagation times and message duplication. Current network-related defaults are: + +``` +( + // The target number of peers in the overlay mesh network (D in the libp2p specs). + mesh_size: 6 + // The minimum number of peers in the mesh network before adding more (D_lo in the libp2p specs). + mesh_lo: 4 + // The maximum number of peers in the mesh network before removing some (D_high in the libp2p sepcs). + mesh_high: 12 + // The number of peers to gossip to during a heartbeat (D_lazy in the libp2p sepcs). + gossip_lazy: 6 // defaults to `mesh_size` + // Time to live for fanout peers (seconds). + fanout_ttl: 60 + // The number of heartbeats to gossip about. + gossip_history: 3 + // Time between each heartbeat (seconds). + heartbeat_interval: 1 +) +``` + +### Topics + +*The Go and Js implementations use string topics - This is likely to be +updated to topic hashes in later versions - https://github.com/libp2p/rust-libp2p/issues/473* + +For Eth2.0 clients, topics will be sent as `SHA2-256` hashes of the topic string. + +There is one dedicated topic for propagating beacon blocks and aggregated +attestations across the network. This topic will have the string +`beacon_chain`. Each shard will have it's own topic allowing relevant parties +to subscribe to in order to receive local shard attestations. The shard topics are +prefixed with `shard` followed by the number of the shard. For example, +messages relating to shard 10, will have the topic string `shard10`. + +### Messages + +Messages sent across gossipsub are fixed-size length-prefixed byte arrays. +Each message has a maximum size of 512KB (estimated from expected largest uncompressed +block size). + +The byte array is prefixed with a unsigned 64 bit length number encoded as an +`unsigned varint` (https://github.com/multiformats/unsigned-varint). Gossipsub messages therefore take the form: +``` ++--------------------------+ +| message length | ++--------------------------+ +| | +| body (<1M) | +| | ++--------------------------+ +``` + +The body of the message is an SSZ-encoded object representing either a +beacon block or attestation. The type of objected is determined via a prefixed +nibble. Currently there are two objects that are sent across the gossip +network. They are (with their corresponding nibble specification): + +- `0x1`: Beacon block +- `0x2`: Attestation + +The body therefore takes the form: +``` ++--------------------------+ +| type nibble | ++--------------------------+ +| | +| SSZ-encoded object | +| | ++--------------------------+ +``` + +## Eth-2 RPC + +#### Protocol Id: `/eth/serenity/beacon/rpc/1` + +The [RPC Interface](./rpc-interface.md) is specified in this repository. + + +## Identify + +#### Protocol Id: `/ipfs/id/1.0.0` (to be updated to `/p2p/id/1.0.0`) + +The Identify protocol (defined in go - [identify-go](https://github.com/ipfs/go-ipfs/blob/master/core/commands/id.go) and rust [rust-identify](https://github.com/libp2p/rust-libp2p/blob/master/protocols/identify/src/lib.rs)) +allows a node A to query another node B which information B knows about A. This also includes the addresses B is listening on. + +This protocol allows nodes to discover addresses of other nodes to be added to +peer discovery. It further allows nodes to determine the capabilities of all it's connected +peers. + +### Configuration Parameters + +The protocol has two configurable parameters, which can be used to identify the +type of connecting node. Suggested format: +``` + version: `/eth/serenity/1.0.0` + user_agent: +``` + +## Discovery + +#### Protocol Id: `/eth/serenity/disc/1.0.0` + +The discovery protocol to be determined. From f7ef9a1ba5e7b863538ab48097060f98ec536412 Mon Sep 17 00:00:00 2001 From: Matthew Slipper Date: Sat, 20 Apr 2019 17:26:07 -0700 Subject: [PATCH 02/84] Don't use SSZ in RPC request/response wrappers --- specs/networking/rpc-interface.md | 42 +++++++++++++++++-------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/specs/networking/rpc-interface.md b/specs/networking/rpc-interface.md index 5d408b5a0..ca6008a40 100644 --- a/specs/networking/rpc-interface.md +++ b/specs/networking/rpc-interface.md @@ -39,32 +39,36 @@ To facilitate RPC-over-`libp2p`, a single protocol name is used: `/eth/serenity/ Remote method calls are wrapped in a "request" structure: ``` -( - id: uint64 - method_id: uint16 - body: Request -) + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | ++ id (uint64) + +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| method_id (uint16) | body_len (uint32) | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | body (body_len bytes) | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ``` and their corresponding responses are wrapped in a "response" structure: ``` -( - id: uint64 - response_code: uint16 - result: bytes -) + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | ++ id (uint64) + +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| response_code (uint16) | result_len (uint32) | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | result (result_len bytes) | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ``` -If an error occurs, a variant of the response structure is returned: - -``` -( - id: uint64 - response_code: uint16 - result: bytes -) -``` +Note that the above structures are NOT encoded as SSZ but rather as sequences of bytes according to the packet diagrams above. This is because SSZ does not support structures without an explicit schema. Since the `body` and `result` fields depend on the value of `method_id` and `response_code`, a schema for the above structure cannot be known beforehand. The details of the RPC-Over-`libp2p` protocol are similar to [JSON-RPC 2.0](https://www.jsonrpc.org/specification). Specifically: From b83a7c4a23c8b27df64e92efcd05eb201b3244e2 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 24 Apr 2019 16:44:22 +1000 Subject: [PATCH 03/84] Add @prestonvanloon and @djrtwo's comments for muliple beacon topics --- specs/networking/libp2p-standardization.md | 42 +++++++++------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/specs/networking/libp2p-standardization.md b/specs/networking/libp2p-standardization.md index ddefca02a..126ed53b4 100644 --- a/specs/networking/libp2p-standardization.md +++ b/specs/networking/libp2p-standardization.md @@ -53,12 +53,20 @@ updated to topic hashes in later versions - https://github.com/libp2p/rust-libp2 For Eth2.0 clients, topics will be sent as `SHA2-256` hashes of the topic string. -There is one dedicated topic for propagating beacon blocks and aggregated -attestations across the network. This topic will have the string -`beacon_chain`. Each shard will have it's own topic allowing relevant parties -to subscribe to in order to receive local shard attestations. The shard topics are -prefixed with `shard` followed by the number of the shard. For example, -messages relating to shard 10, will have the topic string `shard10`. +There are two main topics used to propagate attestations and beacon blocks to +all nodes on the network. + +- The `beacon_block` topic - This topic is used solely for propagating new + beacon blocks to all nodes on the networks. +- The `beacon_attestation` topic - This topic is used to for propagate + aggregated attestations to subscribing nodes (typically block proposers) to + be included into future blocks. Attestations will be aggregated in their + respective subnets before publishing on this topic. + +Shards will be grouped into their own subnets (defined by a shard topic). The +number of shard subnets will be defined via `SHARD_SUBNET_COUNT` and the shard +`shard_number % SHARD_SUBNET_COUNT` will be assigned to the topic: +`shard{shard_number % SHARD_SUBNET_COUNT}`. ### Messages @@ -78,24 +86,9 @@ The byte array is prefixed with a unsigned 64 bit length number encoded as an +--------------------------+ ``` -The body of the message is an SSZ-encoded object representing either a -beacon block or attestation. The type of objected is determined via a prefixed -nibble. Currently there are two objects that are sent across the gossip -network. They are (with their corresponding nibble specification): - -- `0x1`: Beacon block -- `0x2`: Attestation - -The body therefore takes the form: -``` -+--------------------------+ -| type nibble | -+--------------------------+ -| | -| SSZ-encoded object | -| | -+--------------------------+ -``` +The body of the message is an SSZ-encoded object. For the `beacon_block` topic, +this will be a `beacon_block`. For the `beacon_attestation` topic, this will be +an `attestation`. ## Eth-2 RPC @@ -103,7 +96,6 @@ The body therefore takes the form: The [RPC Interface](./rpc-interface.md) is specified in this repository. - ## Identify #### Protocol Id: `/ipfs/id/1.0.0` (to be updated to `/p2p/id/1.0.0`) From 8b316c6db4bcbd96416a1a12e76f57a626f03df6 Mon Sep 17 00:00:00 2001 From: Justin Date: Wed, 1 May 2019 12:04:27 +0100 Subject: [PATCH 04/84] Start moving state_transition.py to state transitition spec The state transition spec should be reasonably self-contained, limiting the amount of "magic" outside of it. This PR is a first step in this direction, specifically for operation processing. --- specs/core/0_beacon-chain.md | 44 ++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 55791e25f..bfb51e99a 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1691,12 +1691,28 @@ def process_eth1_data(state: BeaconState, block: BeaconBlock) -> None: #### Operations +The sub-sections below define helper functions, one per operation type. The full processing of operations is done by running `process_operations(state, block.body)`. + +```python +def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: + assert len(body.deposits) == min(MAX_DEPOSITS, state.latest_eth1_data.deposit_count - state.deposit_index) + assert len(body.transfers) == len(set(body.transfers)) + + for operations, max_operations, function in { + (body.proposer_slashings, MAX_PROPOSER_SLASHINGS, process_proposer_slashing), + (body.attester_slashings, MAX_ATTESTER_SLASHINGS, attester_slashings), + (body.attestations, MAX_ATTESTATIONS, process_attestation), + (body.deposits, MAX_DEPOSITS, process_deposit), + (body.voluntary_exits, MAX_VOLUNTARY_EXITS, process_voluntary_exit), + (body.transfers, MAX_TRANSFERS, process_transfer), + }: + assert len(operations) <= max_operations + for operation in operations: + function(state, operation) +``` + ##### Proposer slashings -Verify that `len(block.body.proposer_slashings) <= MAX_PROPOSER_SLASHINGS`. - -For each `proposer_slashing` in `block.body.proposer_slashings`, run the following function: - ```python def process_proposer_slashing(state: BeaconState, proposer_slashing: ProposerSlashing) -> None: @@ -1721,10 +1737,6 @@ def process_proposer_slashing(state: BeaconState, ##### Attester slashings -Verify that `len(block.body.attester_slashings) <= MAX_ATTESTER_SLASHINGS`. - -For each `attester_slashing` in `block.body.attester_slashings`, run the following function: - ```python def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSlashing) -> None: @@ -1759,10 +1771,6 @@ def process_attester_slashing(state: BeaconState, ##### Attestations -Verify that `len(block.body.attestations) <= MAX_ATTESTATIONS`. - -For each `attestation` in `block.body.attestations`, run the following function: - ```python def process_attestation(state: BeaconState, attestation: Attestation) -> None: """ @@ -1801,10 +1809,6 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: ##### Deposits -Verify that `len(block.body.deposits) == min(MAX_DEPOSITS, state.latest_eth1_data.deposit_count - state.deposit_index)`. - -For each `deposit` in `block.body.deposits`, run the following function: - ```python def process_deposit(state: BeaconState, deposit: Deposit) -> None: """ @@ -1851,10 +1855,6 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: ##### Voluntary exits -Verify that `len(block.body.voluntary_exits) <= MAX_VOLUNTARY_EXITS`. - -For each `exit` in `block.body.voluntary_exits`, run the following function: - ```python def process_voluntary_exit(state: BeaconState, exit: VoluntaryExit) -> None: """ @@ -1879,10 +1879,6 @@ def process_voluntary_exit(state: BeaconState, exit: VoluntaryExit) -> None: ##### Transfers -Verify that `len(block.body.transfers) <= MAX_TRANSFERS` and that all transfers are distinct. - -For each `transfer` in `block.body.transfers`, run the following function: - ```python def process_transfer(state: BeaconState, transfer: Transfer) -> None: """ From 591a2b47c8af28a43f5349435749d7f60f9894d4 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Wed, 1 May 2019 12:08:15 +0100 Subject: [PATCH 05/84] Simplify state_transition.py --- specs/core/0_beacon-chain.md | 6 +- .../eth2spec/phase0/state_transition.py | 69 +------------------ 2 files changed, 5 insertions(+), 70 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index bfb51e99a..e57fce037 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1698,14 +1698,14 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: assert len(body.deposits) == min(MAX_DEPOSITS, state.latest_eth1_data.deposit_count - state.deposit_index) assert len(body.transfers) == len(set(body.transfers)) - for operations, max_operations, function in { + for operations, max_operations, function in ( (body.proposer_slashings, MAX_PROPOSER_SLASHINGS, process_proposer_slashing), - (body.attester_slashings, MAX_ATTESTER_SLASHINGS, attester_slashings), + (body.attester_slashings, MAX_ATTESTER_SLASHINGS, process_attester_slashing), (body.attestations, MAX_ATTESTATIONS, process_attestation), (body.deposits, MAX_DEPOSITS, process_deposit), (body.voluntary_exits, MAX_VOLUNTARY_EXITS, process_voluntary_exit), (body.transfers, MAX_TRANSFERS, process_transfer), - }: + ): assert len(operations) <= max_operations for operation in operations: function(state, operation) diff --git a/test_libs/pyspec/eth2spec/phase0/state_transition.py b/test_libs/pyspec/eth2spec/phase0/state_transition.py index 1bef358d4..2aa0a38a6 100644 --- a/test_libs/pyspec/eth2spec/phase0/state_transition.py +++ b/test_libs/pyspec/eth2spec/phase0/state_transition.py @@ -14,76 +14,11 @@ from .spec import ( ) -def expected_deposit_count(state: BeaconState) -> int: - return min( - spec.MAX_DEPOSITS, - state.latest_eth1_data.deposit_count - state.deposit_index - ) - - -def process_operation_type(state: BeaconState, - operations: List[Any], - max_operations: int, - tx_fn: Callable[[BeaconState, Any], None]) -> None: - assert len(operations) <= max_operations - for operation in operations: - tx_fn(state, operation) - - -def process_operations(state: BeaconState, block: BeaconBlock) -> None: - process_operation_type( - state, - block.body.proposer_slashings, - spec.MAX_PROPOSER_SLASHINGS, - spec.process_proposer_slashing, - ) - - process_operation_type( - state, - block.body.attester_slashings, - spec.MAX_ATTESTER_SLASHINGS, - spec.process_attester_slashing, - ) - - process_operation_type( - state, - block.body.attestations, - spec.MAX_ATTESTATIONS, - spec.process_attestation, - ) - - assert len(block.body.deposits) == expected_deposit_count(state) - process_operation_type( - state, - block.body.deposits, - spec.MAX_DEPOSITS, - spec.process_deposit, - ) - - process_operation_type( - state, - block.body.voluntary_exits, - spec.MAX_VOLUNTARY_EXITS, - spec.process_voluntary_exit, - ) - - assert len(block.body.transfers) == len(set(block.body.transfers)) - process_operation_type( - state, - block.body.transfers, - spec.MAX_TRANSFERS, - spec.process_transfer, - ) - - -def process_block(state: BeaconState, - block: BeaconBlock, - verify_state_root: bool=False) -> None: +def process_block(state: BeaconState, block: BeaconBlock, verify_state_root) -> None: spec.process_block_header(state, block) spec.process_randao(state, block) spec.process_eth1_data(state, block) - - process_operations(state, block) + spec.process_operations(state, block.body) if verify_state_root: spec.verify_block_state_root(state, block) From 5df79d7565fd4fc49fe4367cb40afa7b121d5718 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Wed, 1 May 2019 13:14:10 +0100 Subject: [PATCH 06/84] Get rid of state_transition.py --- specs/core/0_beacon-chain.md | 81 ++++++++----------- .../eth2spec/phase0/state_transition.py | 47 ----------- .../test_process_attestation.py | 4 +- .../test_process_block_header.py | 2 +- .../test_process_crosslinks.py | 4 +- test_libs/pyspec/tests/helpers.py | 8 +- test_libs/pyspec/tests/test_finality.py | 5 +- test_libs/pyspec/tests/test_sanity.py | 6 +- 8 files changed, 44 insertions(+), 113 deletions(-) delete mode 100644 test_libs/pyspec/eth2spec/phase0/state_transition.py diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index e57fce037..50e390326 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -105,7 +105,6 @@ - [Registry updates](#registry-updates) - [Slashings](#slashings) - [Final updates](#final-updates) - - [Per-slot processing](#per-slot-processing) - [Per-block processing](#per-block-processing) - [Block header](#block-header) - [RANDAO](#randao) @@ -1264,27 +1263,27 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], ## Beacon chain state transition function -We now define the state transition function. At a high level, the state transition is made up of four parts: +The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. -1. State caching, which happens at the start of every slot. -2. The per-epoch transitions, which happens at the start of the first slot of every epoch. -3. The per-slot transitions, which happens at every slot. -4. The per-block transitions, which happens at every block. +```python +def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconState: + assert state.slot < block.slot + while state.slot < block.slot: + # State caching at the start of every slot + cache_state(state) + # Per-epoch processing at the start of the first slot of every epoch + if (state.slot + 1) % SLOTS_PER_EPOCH == 0: + process_epoch_transition(state) + # Slot incrementing + state.slot += 1 + # Block processing at every block + process_block(state, block) +``` -Transition section notes: -* The state caching caches the state root of the previous slot and updates block and state roots records. -* The per-epoch transitions focus on the [validator](#dfn-validator) registry, including adjusting balances and activating and exiting [validators](#dfn-validator), as well as processing crosslinks and managing block justification/finalization. -* The per-slot transitions focus on the slot counter. -* The per-block transitions generally focus on verifying aggregate signatures and saving temporary records relating to the per-block activity in the `BeaconState`. - -Beacon blocks that trigger unhandled Python exceptions (e.g. out-of-range list accesses) and failed `assert`s during the state transition are considered invalid. - -Note: If there are skipped slots between a block and its parent block, run the steps in the [state-root](#state-caching), [per-epoch](#per-epoch-processing), and [per-slot](#per-slot-processing) sections once for each skipped slot and then once for the slot containing the new block. +Note: Beacon blocks that trigger unhandled Python exceptions (e.g. out-of-range list accesses) and failed `assert`s during the state transition are considered invalid. ### State caching -At every `slot > GENESIS_SLOT` run the following function: - ```python def cache_state(state: BeaconState) -> None: # Cache latest known state root (for previous slot) @@ -1302,12 +1301,18 @@ def cache_state(state: BeaconState) -> None: ### Per-epoch processing -The steps below happen when `state.slot > GENESIS_SLOT and (state.slot + 1) % SLOTS_PER_EPOCH == 0`. +```python +def process_epoch_transition(state: BeaconState) -> None: + process_justification_and_finalization(state) + process_crosslinks(state) + process_rewards_and_penalties(state) + process_registry_updates(state) + process_slashings(state) + process_final_updates(state) +``` #### Helper functions -We define epoch transition helper functions: - ```python def get_total_active_balance(state: BeaconState) -> Gwei: return get_total_balance(state, get_active_validator_indices(state, get_current_epoch(state))) @@ -1387,8 +1392,6 @@ def get_earliest_attestation(state: BeaconState, attestations: List[PendingAttes #### Justification and finalization -Run the following function: - ```python def process_justification_and_finalization(state: BeaconState) -> None: if get_current_epoch(state) <= GENESIS_EPOCH + 1: @@ -1436,8 +1439,6 @@ def process_justification_and_finalization(state: BeaconState) -> None: #### Crosslinks -Run the following function: - ```python def process_crosslinks(state: BeaconState) -> None: state.previous_crosslinks = [c for c in state.current_crosslinks] @@ -1453,8 +1454,6 @@ def process_crosslinks(state: BeaconState) -> None: #### Rewards and penalties -First, we define additional helpers: - ```python def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: adjusted_quotient = integer_squareroot(get_total_active_balance(state)) // BASE_REWARD_QUOTIENT @@ -1526,8 +1525,6 @@ def get_crosslink_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: return [rewards, penalties] ``` -Run the following function: - ```python def process_rewards_and_penalties(state: BeaconState) -> None: if get_current_epoch(state) == GENESIS_EPOCH: @@ -1542,8 +1539,6 @@ def process_rewards_and_penalties(state: BeaconState) -> None: #### Registry updates -Run the following function: - ```python def process_registry_updates(state: BeaconState) -> None: # Process activation eligibility and ejections @@ -1568,8 +1563,6 @@ def process_registry_updates(state: BeaconState) -> None: #### Slashings -Run the following function: - ```python def process_slashings(state: BeaconState) -> None: current_epoch = get_current_epoch(state) @@ -1592,8 +1585,6 @@ def process_slashings(state: BeaconState) -> None: #### Final updates -Run the following function: - ```python def process_final_updates(state: BeaconState) -> None: current_epoch = get_current_epoch(state) @@ -1632,18 +1623,16 @@ def process_final_updates(state: BeaconState) -> None: state.current_epoch_attestations = [] ``` -### Per-slot processing - -At every `slot > GENESIS_SLOT` run the following function: - -```python -def advance_slot(state: BeaconState) -> None: - state.slot += 1 -``` - ### Per-block processing -For every `block` except the genesis block, run `process_block_header(state, block)`, `process_randao(state, block)` and `process_eth1_data(state, block)`. +```python +def process_block(state: BeaconState, block: BeaconBlock) -> None: + process_block_header(state, block) + process_randao(state, block) + process_eth1_data(state, block) + process_operations(state, block.body) + # verify_block_state_root(state, block) +``` #### Block header @@ -1691,8 +1680,6 @@ def process_eth1_data(state: BeaconState, block: BeaconBlock) -> None: #### Operations -The sub-sections below define helper functions, one per operation type. The full processing of operations is done by running `process_operations(state, block.body)`. - ```python def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: assert len(body.deposits) == min(MAX_DEPOSITS, state.latest_eth1_data.deposit_count - state.deposit_index) @@ -1913,8 +1900,6 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: #### State root verification -Verify the block's `state_root` by running the following function: - ```python def verify_block_state_root(state: BeaconState, block: BeaconBlock) -> None: assert block.state_root == hash_tree_root(state) diff --git a/test_libs/pyspec/eth2spec/phase0/state_transition.py b/test_libs/pyspec/eth2spec/phase0/state_transition.py deleted file mode 100644 index 2aa0a38a6..000000000 --- a/test_libs/pyspec/eth2spec/phase0/state_transition.py +++ /dev/null @@ -1,47 +0,0 @@ -from . import spec - - -from typing import ( - Any, - Callable, - List -) - -from .spec import ( - BeaconState, - BeaconBlock, - Slot, -) - - -def process_block(state: BeaconState, block: BeaconBlock, verify_state_root) -> None: - spec.process_block_header(state, block) - spec.process_randao(state, block) - spec.process_eth1_data(state, block) - spec.process_operations(state, block.body) - if verify_state_root: - spec.verify_block_state_root(state, block) - - -def process_epoch_transition(state: BeaconState) -> None: - spec.process_justification_and_finalization(state) - spec.process_crosslinks(state) - spec.process_rewards_and_penalties(state) - spec.process_registry_updates(state) - spec.process_slashings(state) - spec.process_final_updates(state) - - -def state_transition_to(state: BeaconState, up_to: Slot) -> BeaconState: - while state.slot < up_to: - spec.cache_state(state) - if (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0: - process_epoch_transition(state) - spec.advance_slot(state) - - -def state_transition(state: BeaconState, - block: BeaconBlock, - verify_state_root: bool=False) -> BeaconState: - state_transition_to(state, block.slot) - process_block(state, block, verify_state_root) diff --git a/test_libs/pyspec/tests/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/block_processing/test_process_attestation.py index 1be60c860..105d1e0a4 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attestation.py @@ -3,13 +3,11 @@ import pytest import eth2spec.phase0.spec as spec -from eth2spec.phase0.state_transition import ( - state_transition, -) from eth2spec.phase0.spec import ( get_current_epoch, process_attestation, slot_to_epoch, + state_transition, ) from tests.helpers import ( build_empty_block_for_next_slot, diff --git a/test_libs/pyspec/tests/block_processing/test_process_block_header.py b/test_libs/pyspec/tests/block_processing/test_process_block_header.py index b35b0a9c1..6fd6e674e 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/tests/block_processing/test_process_block_header.py @@ -5,10 +5,10 @@ import pytest from eth2spec.phase0.spec import ( get_beacon_proposer_index, cache_state, - advance_slot, process_block_header, ) from tests.helpers import ( + advance_slot, build_empty_block_for_next_slot, next_slot, ) diff --git a/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py index fe694724a..60e0dec53 100644 --- a/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py @@ -3,13 +3,11 @@ import pytest import eth2spec.phase0.spec as spec -from eth2spec.phase0.state_transition import ( - state_transition, -) from eth2spec.phase0.spec import ( cache_state, get_crosslink_deltas, process_crosslinks, + state_transition, ) from tests.helpers import ( add_attestation_to_state, diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index 63e4cd710..2cb272ff3 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -2,9 +2,6 @@ from copy import deepcopy from py_ecc import bls -from eth2spec.phase0.state_transition import ( - state_transition, -) import eth2spec.phase0.spec as spec from eth2spec.utils.minimal_ssz import signing_root from eth2spec.phase0.spec import ( @@ -38,6 +35,7 @@ from eth2spec.phase0.spec import ( get_shard_delta, hash_tree_root, slot_to_epoch, + state_transition, verify_merkle_branch, hash, ) @@ -53,6 +51,10 @@ pubkeys = [bls.privtopub(privkey) for privkey in privkeys] pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} +def advance_slot(state) -> None: + state.slot += 1 + + def get_balance(state, index): return state.balances[index] diff --git a/test_libs/pyspec/tests/test_finality.py b/test_libs/pyspec/tests/test_finality.py index ca048c2b2..816dfd6bd 100644 --- a/test_libs/pyspec/tests/test_finality.py +++ b/test_libs/pyspec/tests/test_finality.py @@ -4,9 +4,6 @@ import pytest import eth2spec.phase0.spec as spec -from eth2spec.phase0.state_transition import ( - state_transition, -) from .helpers import ( build_empty_block_for_next_slot, fill_aggregate_attestation, @@ -67,7 +64,7 @@ def next_epoch_with_attestations(state, fill_aggregate_attestation(post_state, prev_attestation) block.body.attestations.append(prev_attestation) - state_transition(post_state, block) + spec.state_transition(post_state, block) blocks.append(block) return state, blocks, post_state diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index b7d31f122..4d826fc13 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -20,13 +20,10 @@ from eth2spec.phase0.spec import ( get_state_root, get_current_epoch, get_domain, - advance_slot, cache_state, verify_merkle_branch, - hash, -) -from eth2spec.phase0.state_transition import ( state_transition, + hash, ) from eth2spec.utils.merkle_minimal import ( calc_merkle_tree_from_leaves, @@ -34,6 +31,7 @@ from eth2spec.utils.merkle_minimal import ( get_merkle_root, ) from .helpers import ( + advance_slot, get_balance, build_deposit_data, build_empty_block_for_next_slot, From 2e63a9b5a0322e0e87f72baee3a279999dbdddc7 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Wed, 1 May 2019 13:29:03 +0100 Subject: [PATCH 07/84] clean up --- specs/core/0_beacon-chain.md | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 50e390326..8f69213bf 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -97,7 +97,7 @@ - [On genesis](#on-genesis) - [Beacon chain state transition function](#beacon-chain-state-transition-function) - [State caching](#state-caching) - - [Per-epoch processing](#per-epoch-processing) + - [Epoch processing](#epoch-processing) - [Helper functions](#helper-functions-1) - [Justification and finalization](#justification-and-finalization) - [Crosslinks](#crosslinks) @@ -105,7 +105,7 @@ - [Registry updates](#registry-updates) - [Slashings](#slashings) - [Final updates](#final-updates) - - [Per-block processing](#per-block-processing) + - [Block processing](#block-processing) - [Block header](#block-header) - [RANDAO](#randao) - [Eth1 data](#eth1-data) @@ -1263,7 +1263,7 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], ## Beacon chain state transition function -The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. +The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. State transitions that trigger an unhandled exception (e.g. failed `assert`s and out-of-range list accesses) are considered invalid. ```python def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconState: @@ -1271,17 +1271,15 @@ def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconState: while state.slot < block.slot: # State caching at the start of every slot cache_state(state) - # Per-epoch processing at the start of the first slot of every epoch + # Epoch processing at the start of the first slot of every epoch if (state.slot + 1) % SLOTS_PER_EPOCH == 0: - process_epoch_transition(state) + process_epoch(state) # Slot incrementing state.slot += 1 # Block processing at every block process_block(state, block) ``` -Note: Beacon blocks that trigger unhandled Python exceptions (e.g. out-of-range list accesses) and failed `assert`s during the state transition are considered invalid. - ### State caching ```python @@ -1299,10 +1297,10 @@ def cache_state(state: BeaconState) -> None: state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = latest_block_root ``` -### Per-epoch processing +### Epoch processing ```python -def process_epoch_transition(state: BeaconState) -> None: +def process_epoch(state: BeaconState) -> None: process_justification_and_finalization(state) process_crosslinks(state) process_rewards_and_penalties(state) @@ -1623,7 +1621,7 @@ def process_final_updates(state: BeaconState) -> None: state.current_epoch_attestations = [] ``` -### Per-block processing +### Block processing ```python def process_block(state: BeaconState, block: BeaconBlock) -> None: From 9b24d06b2cbfa494a8169eebc3f97f32b9ab3399 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Wed, 1 May 2019 14:16:55 +0100 Subject: [PATCH 08/84] Cleanup --- specs/core/0_beacon-chain.md | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8f69213bf..153e42cf6 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1263,18 +1263,19 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], ## Beacon chain state transition function -The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. State transitions that trigger an unhandled exception (e.g. failed `assert`s and out-of-range list accesses) are considered invalid. +The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. State transitions that trigger an unhandled exception (e.g. a failed `assert` or an out-of-range list access) are considered invalid. ```python def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconState: assert state.slot < block.slot + # Slot processing (including slots with no blocks) while state.slot < block.slot: # State caching at the start of every slot cache_state(state) # Epoch processing at the start of the first slot of every epoch if (state.slot + 1) % SLOTS_PER_EPOCH == 0: process_epoch(state) - # Slot incrementing + # Increment slot number state.slot += 1 # Block processing at every block process_block(state, block) @@ -1284,17 +1285,15 @@ def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconState: ```python def cache_state(state: BeaconState) -> None: - # Cache latest known state root (for previous slot) - latest_state_root = hash_tree_root(state) - state.latest_state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = latest_state_root + # Cache state and block roots of previous slot + previous_state_root = hash_tree_root(state) + state.latest_state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_state_root + previous_block_root = signing_root(state.latest_block_header) + state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_block_root - # Store latest known state root (for previous slot) in latest_block_header if it is empty + # Cache previous state root in latest_block_header, if empty if state.latest_block_header.state_root == ZERO_HASH: - state.latest_block_header.state_root = latest_state_root - - # Cache latest known block root (for previous slot) - latest_block_root = signing_root(state.latest_block_header) - state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = latest_block_root + state.latest_block_header.state_root = previous_state_root ``` ### Epoch processing From 7980cf3ef79ee4f7ee418de5c8169907280169a2 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Wed, 1 May 2019 15:07:55 +0100 Subject: [PATCH 09/84] Fix --- specs/core/0_beacon-chain.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 153e42cf6..4c1498ccf 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1285,15 +1285,18 @@ def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconState: ```python def cache_state(state: BeaconState) -> None: - # Cache state and block roots of previous slot + # Cache state root of previous slot previous_state_root = hash_tree_root(state) state.latest_state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_state_root - previous_block_root = signing_root(state.latest_block_header) - state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_block_root # Cache previous state root in latest_block_header, if empty if state.latest_block_header.state_root == ZERO_HASH: state.latest_block_header.state_root = previous_state_root + + # Cache block root of previous slot + previous_block_root = signing_root(state.latest_block_header) + state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_block_root + ``` ### Epoch processing From bbca108a80effe0e167132f88dd1a6899bb86c70 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Thu, 2 May 2019 16:34:47 +1000 Subject: [PATCH 10/84] Add Transport and lower-level libp2p specifications --- specs/networking/libp2p-standardization.md | 79 ++++++++++++++++++++-- 1 file changed, 74 insertions(+), 5 deletions(-) diff --git a/specs/networking/libp2p-standardization.md b/specs/networking/libp2p-standardization.md index 126ed53b4..c5add549a 100644 --- a/specs/networking/libp2p-standardization.md +++ b/specs/networking/libp2p-standardization.md @@ -9,6 +9,57 @@ configuration and messaging formats. # Libp2p Protocols +## Transport + +This section details the libp2p transport layer that underlies the +[protocols](#protocols) that are listed in this document. + +Libp2p allows composition of multiple transports. Eth2.0 clients should support +TCP/IP and optionally websockets. Websockets are useful for implementations +running in the browser and therefore native clients would ideally support these implementations +by supporting websockets. + +An ideal libp2p transport would therefore be TCP/IP with a fallback to +websockets. + +### Encryption + +Libp2p currently offers [Secio](https://github.com/libp2p/specs/pull/106) which +can upgrade a transport which will then encrypt all future communication. Secio +generates a symmetric ephemeral key which peers use to encrypt their +communication. It can support a range of ciphers and currently supports key +derivation for elliptic curve-based public keys. + +Current defaults are: +- Key agreement: `ECDH-P256` (also supports `ECDH-P384`) +- Cipher: `AES-128` (also supports `AES-256`, `TwofishCTR`) +- Digests: `SHA256` (also supports `SHA512`) + + +## Protocols + +This section lists the necessary libp2p protocols required by Ethereum 2.0 +running a libp2p network stack. + +## Multistream-select + +#### Protocol id: `/multistream/1.0.0` + +Clients running libp2p should support the [multistream-select](https://github.com/multiformats/multistream-select/) +protocol which allows clients to negotiate libp2p protocols establish streams +per protocol. + +## Multiplexing + +Libp2p allows clients to compose multiple multiplexing methods. Clients should +support [mplex](https://github.com/libp2p/specs/tree/master/mplex) and +optionally [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md) +(these can be composed). + +**Mplex protocol id: `/mplex/6.7.0`** + +**Yamux protocol id: `/yamux/1.0.0`** + ## Gossipsub #### Protocol id: `/meshsub/1.0.0` @@ -70,11 +121,14 @@ number of shard subnets will be defined via `SHARD_SUBNET_COUNT` and the shard ### Messages -Messages sent across gossipsub are fixed-size length-prefixed byte arrays. -Each message has a maximum size of 512KB (estimated from expected largest uncompressed -block size). +#### Libp2p Specification -The byte array is prefixed with a unsigned 64 bit length number encoded as an +*This section simply outlines the data sent across the wire as specified by +libp2p - this section is aimed at gossipsub implementers to standardize their implementation of this protocol* + +Libp2p raw gossipsub messages are sent across the wire as fixed-size length-prefixed byte arrays. + +The byte array is prefixed with an unsigned 64 bit length number encoded as an `unsigned varint` (https://github.com/multiformats/unsigned-varint). Gossipsub messages therefore take the form: ``` +--------------------------+ @@ -86,7 +140,17 @@ The byte array is prefixed with a unsigned 64 bit length number encoded as an +--------------------------+ ``` -The body of the message is an SSZ-encoded object. For the `beacon_block` topic, +The body represents a protobuf-encoded [Message](https://github.com/libp2p/go-libp2p-pubsub/blob/master/pb/rpc.proto#L17-L24). + +In the following section we discuss the data being sent in the `data` field of +the protobuf gossipsub `Message`. + +#### Eth2.0 Specifics + +Each message has a maximum size of 512KB (estimated from expected largest uncompressed +block size). + +The `data` that is sent in a Gossipsub message is an SSZ-encoded object. For the `beacon_block` topic, this will be a `beacon_block`. For the `beacon_attestation` topic, this will be an `attestation`. @@ -100,6 +164,9 @@ The [RPC Interface](./rpc-interface.md) is specified in this repository. #### Protocol Id: `/ipfs/id/1.0.0` (to be updated to `/p2p/id/1.0.0`) +*To be updated to incorporate discv5* + + The Identify protocol (defined in go - [identify-go](https://github.com/ipfs/go-ipfs/blob/master/core/commands/id.go) and rust [rust-identify](https://github.com/libp2p/rust-libp2p/blob/master/protocols/identify/src/lib.rs)) allows a node A to query another node B which information B knows about A. This also includes the addresses B is listening on. @@ -120,4 +187,6 @@ type of connecting node. Suggested format: #### Protocol Id: `/eth/serenity/disc/1.0.0` +*To be updated to incorporate discv5* + The discovery protocol to be determined. From 78181834ab6dbbfdc58a295d71d1309ba5829957 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 2 May 2019 16:38:23 +1000 Subject: [PATCH 11/84] Update specs/networking/libp2p-standardization.md Co-Authored-By: AgeManning --- specs/networking/libp2p-standardization.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/networking/libp2p-standardization.md b/specs/networking/libp2p-standardization.md index c5add549a..5fe26f435 100644 --- a/specs/networking/libp2p-standardization.md +++ b/specs/networking/libp2p-standardization.md @@ -109,7 +109,7 @@ all nodes on the network. - The `beacon_block` topic - This topic is used solely for propagating new beacon blocks to all nodes on the networks. -- The `beacon_attestation` topic - This topic is used to for propagate +- The `beacon_attestation` topic - This topic is used to propagate aggregated attestations to subscribing nodes (typically block proposers) to be included into future blocks. Attestations will be aggregated in their respective subnets before publishing on this topic. From 9f2cdd9c7807265edcb259dd91d37a9ca51988ca Mon Sep 17 00:00:00 2001 From: Justin Date: Thu, 2 May 2019 11:07:25 +0100 Subject: [PATCH 12/84] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 4c1498ccf..86caf7402 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1266,19 +1266,22 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. State transitions that trigger an unhandled exception (e.g. a failed `assert` or an out-of-range list access) are considered invalid. ```python -def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconState: +def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconBlock: + # Block must post-date the state assert state.slot < block.slot - # Slot processing (including slots with no blocks) + # Process slots (including those with no blocks) since block while state.slot < block.slot: - # State caching at the start of every slot + # Cache state at the start of every slot cache_state(state) - # Epoch processing at the start of the first slot of every epoch + # Process epoch at the start of the first slot of every epoch if (state.slot + 1) % SLOTS_PER_EPOCH == 0: process_epoch(state) # Increment slot number state.slot += 1 - # Block processing at every block + # Process block process_block(state, block) + # Return post-state + return state ``` ### State caching From 66b152f79ecad957b38ecccc16ce2f131afd10cc Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 3 May 2019 05:07:11 -0500 Subject: [PATCH 13/84] Allow multiple bit challenges, and recover withdrawability Resolves #864 items 4, 7, 14 --- specs/core/1_custody-game.md | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index d56526611..0de84ff06 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -146,7 +146,7 @@ This document details the beacon chain additions and changes in Phase 1 of Ether 'challenge_index': 'uint64', 'challenger_index': ValidatorIndex, 'responder_index': ValidatorIndex, - 'deadline': Epoch, + 'inclusion_epoch': Epoch, 'crosslink_data_root': Hash, 'depth': 'uint64', 'chunk_index': 'uint64', @@ -160,7 +160,7 @@ This document details the beacon chain additions and changes in Phase 1 of Ether 'challenge_index': 'uint64', 'challenger_index': ValidatorIndex, 'responder_index': ValidatorIndex, - 'deadline': Epoch, + 'inclusion_epoch': Epoch, 'crosslink_data_root': Hash, 'chunk_count': 'uint64', 'chunk_bits_merkle_root': Hash, @@ -485,7 +485,7 @@ def process_chunk_challenge(state: BeaconState, challenge_index=state.custody_challenge_index, challenger_index=get_beacon_proposer_index(state), responder_index=challenge.responder_index - deadline=get_current_epoch(state) + CUSTODY_RESPONSE_DEADLINE, + inclusion_epoch=get_current_epoch(state), crosslink_data_root=challenge.attestation.data.crosslink_data_root, depth=depth, chunk_index=challenge.chunk_index, @@ -528,10 +528,9 @@ def process_bit_challenge(state: BeaconState, attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield) assert challenge.responder_index in attesters - # A validator can be the challenger or responder for at most one challenge at a time + # A validator can be the challenger for at most one challenge at a time for record in state.custody_bit_challenge_records: assert record.challenger_index != challenge.challenger_index - assert record.responder_index != challenge.responder_index # Verify the responder is a valid custody key epoch_to_sign = get_randao_epoch_for_custody_period( @@ -563,7 +562,7 @@ def process_bit_challenge(state: BeaconState, challenge_index=state.custody_challenge_index, challenger_index=challenge.challenger_index, responder_index=challenge.responder_index, - deadline=get_current_epoch(state) + CUSTODY_RESPONSE_DEADLINE, + inclusion_epoch=get_current_epoch(state), crosslink_data_root=challenge.attestation.data.crosslink_data_root, chunk_count=chunk_count, chunk_bits_merkle_root=merkle_root(pad_to_power_of_2((challenge.chunk_bits))), @@ -604,6 +603,8 @@ def process_chunk_challenge_response(state: BeaconState, assert response.chunk_index == challenge.chunk_index # Verify bit challenge data is null assert response.chunk_bits_branch == [] and response.chunk_bits_leaf == ZERO_HASH + # Verify minimum delay + assert get_current_epoch(state) >= challenge.inclusion_epoch + ACTIVATION_EXIT_DELAY # Verify the chunk matches the crosslink data root assert verify_merkle_branch( leaf=hash_tree_root(response.chunk), @@ -626,6 +627,9 @@ def process_bit_challenge_response(state: BeaconState, challenge: CustodyBitChallengeRecord) -> None: # Verify chunk index assert response.chunk_index < challenge.chunk_count + # Verify responder has not been slashed + responder = state.validator_registry[record.responder_index] + assert not responder.slashed # Verify the chunk matches the crosslink data root assert verify_merkle_branch( leaf=hash_tree_root(response.chunk), @@ -671,13 +675,13 @@ Run `process_challenge_deadlines(state)` immediately after `process_reveal_deadl ```python def process_challenge_deadlines(state: BeaconState) -> None: for challenge in state.custody_chunk_challenge_records: - if get_current_epoch(state) > challenge.deadline: + if get_current_epoch(state) > challenge.inclusion_epoch + CUSTODY_RESPONSE_DEADLINE: slash_validator(state, challenge.responder_index, challenge.challenger_index) records = state.custody_chunk_challenge_records records[records.index(challenge)] = CustodyChunkChallengeRecord() for challenge in state.custody_bit_challenge_records: - if get_current_epoch(state) > challenge.deadline: + if get_current_epoch(state) > challenge.inclusion_epoch + CUSTODY_RESPONSE_DEADLINE: slash_validator(state, challenge.responder_index, challenge.challenger_index) records = state.custody_bit_challenge_records records[records.index(challenge)] = CustodyBitChallengeRecord() @@ -688,6 +692,18 @@ Append this to `process_final_updates(state)`: ```python # Clean up exposed RANDAO key reveals state.exposed_derived_secrets[current_epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] = [] + # Reset withdrawable epochs if challenge records are empty + for index, validator in enumerate(state.validator_registry): + eligible = True + for records in (state.custody_chunk_challenge_records, state.bit_challenge_records): + for filter_func in (lambda rec: rec.challenger_index == index, lambda rec: rec.responder_index == index): + if len(list(filter(filter_func, records))) > 0: + eligible = False + if eligible: + if validator.exit_epoch == FAR_FUTURE_EPOCH: + validator.withdrawable_epoch = FAR_FUTURE_EPOCH + else: + validator.withdrawable_epoch = validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY ``` In `process_penalties_and_exits`, change the definition of `eligible` to the following (note that it is not a pure function because `state` is declared in the surrounding scope): From 2ccd357f0eab881eaf7f6bd39e09987467731c91 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 3 May 2019 21:05:54 +0800 Subject: [PATCH 14/84] Update specs/core/1_custody-game.md Co-Authored-By: vbuterin --- specs/core/1_custody-game.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 0de84ff06..307e18573 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -628,7 +628,7 @@ def process_bit_challenge_response(state: BeaconState, # Verify chunk index assert response.chunk_index < challenge.chunk_count # Verify responder has not been slashed - responder = state.validator_registry[record.responder_index] + responder = state.validator_registry[challenge.responder_index] assert not responder.slashed # Verify the chunk matches the crosslink data root assert verify_merkle_branch( From 197a7200efaf1bf9024910e3c41563e3b0c96f77 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Fri, 3 May 2019 21:21:42 +0100 Subject: [PATCH 15/84] Add HW and Danny comments --- specs/core/0_beacon-chain.md | 61 +++++++++---------- .../test_process_block_header.py | 4 +- .../test_process_crosslinks.py | 4 +- test_libs/pyspec/tests/test_sanity.py | 4 +- 4 files changed, 36 insertions(+), 37 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 86caf7402..b4b44e628 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -96,7 +96,6 @@ - [`slash_validator`](#slash_validator) - [On genesis](#on-genesis) - [Beacon chain state transition function](#beacon-chain-state-transition-function) - - [State caching](#state-caching) - [Epoch processing](#epoch-processing) - [Helper functions](#helper-functions-1) - [Justification and finalization](#justification-and-finalization) @@ -1267,36 +1266,36 @@ The post-state corresponding to a pre-state `state` and a block `block` is defin ```python def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconBlock: - # Block must post-date the state - assert state.slot < block.slot # Process slots (including those with no blocks) since block - while state.slot < block.slot: - # Cache state at the start of every slot - cache_state(state) - # Process epoch at the start of the first slot of every epoch - if (state.slot + 1) % SLOTS_PER_EPOCH == 0: - process_epoch(state) - # Increment slot number - state.slot += 1 + process_slots(state, block.slot) # Process block process_block(state, block) # Return post-state return state ``` -### State caching +```python +def process_slots(state: BeaconState, slot: Slot) -> None: + assert state.slot < slot + while state.slot < slot: + process_slot(state) + # Process epoch on the last slot of every epoch + if (state.slot + 1) % SLOTS_PER_EPOCH == 0: + process_epoch(state) + state.slot += 1 +``` ```python -def cache_state(state: BeaconState) -> None: - # Cache state root of previous slot +def process_slot(state: BeaconState) -> None: + # Cache state root previous_state_root = hash_tree_root(state) state.latest_state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_state_root - # Cache previous state root in latest_block_header, if empty + # Cache latest block header state root if state.latest_block_header.state_root == ZERO_HASH: state.latest_block_header.state_root = previous_state_root - # Cache block root of previous slot + # Cache block root previous_block_root = signing_root(state.latest_block_header) state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_block_root @@ -1631,10 +1630,10 @@ def process_final_updates(state: BeaconState) -> None: ```python def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) - process_randao(state, block) - process_eth1_data(state, block) + process_randao(state, block.body) + process_eth1_data(state, block.body) process_operations(state, block.body) - # verify_block_state_root(state, block) + # process_state_root(state, block) ``` #### Block header @@ -1661,31 +1660,33 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: #### RANDAO ```python -def process_randao(state: BeaconState, block: BeaconBlock) -> None: +def process_randao(state: BeaconState, body: BeaconBlockBody) -> None: proposer = state.validator_registry[get_beacon_proposer_index(state)] # Verify that the provided randao value is valid - assert bls_verify(proposer.pubkey, hash_tree_root(get_current_epoch(state)), block.body.randao_reveal, get_domain(state, DOMAIN_RANDAO)) + assert bls_verify(proposer.pubkey, hash_tree_root(get_current_epoch(state)), body.randao_reveal, get_domain(state, DOMAIN_RANDAO)) # Mix it in state.latest_randao_mixes[get_current_epoch(state) % LATEST_RANDAO_MIXES_LENGTH] = ( xor(get_randao_mix(state, get_current_epoch(state)), - hash(block.body.randao_reveal)) + hash(body.randao_reveal)) ) ``` #### Eth1 data ```python -def process_eth1_data(state: BeaconState, block: BeaconBlock) -> None: - state.eth1_data_votes.append(block.body.eth1_data) - if state.eth1_data_votes.count(block.body.eth1_data) * 2 > SLOTS_PER_ETH1_VOTING_PERIOD: - state.latest_eth1_data = block.body.eth1_data +def process_eth1_data(state: BeaconState, body: BeaconBlockBody) -> None: + state.eth1_data_votes.append(body.eth1_data) + if state.eth1_data_votes.count(body.eth1_data) * 2 > SLOTS_PER_ETH1_VOTING_PERIOD: + state.latest_eth1_data = body.eth1_data ``` #### Operations ```python def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: + # Verify that outstanding deposits are processed up to the maximum number of deposits assert len(body.deposits) == min(MAX_DEPOSITS, state.latest_eth1_data.deposit_count - state.deposit_index) + # Verify that there are no duplicate transfers assert len(body.transfers) == len(set(body.transfers)) for operations, max_operations, function in ( @@ -1704,8 +1705,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: ##### Proposer slashings ```python -def process_proposer_slashing(state: BeaconState, - proposer_slashing: ProposerSlashing) -> None: +def process_proposer_slashing(state: BeaconState, proposer_slashing: ProposerSlashing) -> None: """ Process ``ProposerSlashing`` operation. Note that this function mutates ``state``. @@ -1728,8 +1728,7 @@ def process_proposer_slashing(state: BeaconState, ##### Attester slashings ```python -def process_attester_slashing(state: BeaconState, - attester_slashing: AttesterSlashing) -> None: +def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSlashing) -> None: """ Process ``AttesterSlashing`` operation. Note that this function mutates ``state``. @@ -1904,6 +1903,6 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: #### State root verification ```python -def verify_block_state_root(state: BeaconState, block: BeaconBlock) -> None: +def process_state_root(state: BeaconState, block: BeaconBlock) -> None: assert block.state_root == hash_tree_root(state) ``` diff --git a/test_libs/pyspec/tests/block_processing/test_process_block_header.py b/test_libs/pyspec/tests/block_processing/test_process_block_header.py index 6fd6e674e..149bab514 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/tests/block_processing/test_process_block_header.py @@ -4,7 +4,7 @@ import pytest from eth2spec.phase0.spec import ( get_beacon_proposer_index, - cache_state, + process_slot, process_block_header, ) from tests.helpers import ( @@ -18,7 +18,7 @@ pytestmark = pytest.mark.header def prepare_state_for_header_processing(state): - cache_state(state) + process_slot(state) advance_slot(state) diff --git a/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py index 60e0dec53..4e3d4b5d8 100644 --- a/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py @@ -4,7 +4,7 @@ import pytest import eth2spec.phase0.spec as spec from eth2spec.phase0.spec import ( - cache_state, + process_slot, get_crosslink_deltas, process_crosslinks, state_transition, @@ -33,7 +33,7 @@ def run_process_crosslinks(state, valid=True): state_transition(state, block) # cache state before epoch transition - cache_state(state) + process_slot(state) post_state = deepcopy(state) process_crosslinks(post_state) diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index 4d826fc13..7fff6fb55 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -20,7 +20,7 @@ from eth2spec.phase0.spec import ( get_state_root, get_current_epoch, get_domain, - cache_state, + process_slot, verify_merkle_branch, state_transition, hash, @@ -51,7 +51,7 @@ pytestmark = pytest.mark.sanity def test_slot_transition(state): test_state = deepcopy(state) - cache_state(test_state) + process_slot(test_state) advance_slot(test_state) assert test_state.slot == state.slot + 1 assert get_state_root(test_state, state.slot) == state.hash_tree_root() From 4c1073fa2f898de7ffe26fd8aa197d38c25a8f35 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 3 May 2019 22:22:19 +0100 Subject: [PATCH 16/84] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b4b44e628..8d999c65c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1279,7 +1279,7 @@ def process_slots(state: BeaconState, slot: Slot) -> None: assert state.slot < slot while state.slot < slot: process_slot(state) - # Process epoch on the last slot of every epoch + # Process epoch on the first slot of the next epoch if (state.slot + 1) % SLOTS_PER_EPOCH == 0: process_epoch(state) state.slot += 1 From c7fea5ff38b260fccbe9416d5e0fd769884e5343 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 6 May 2019 12:28:16 +1000 Subject: [PATCH 17/84] Update libp2p-standardization based on latest comments --- specs/networking/libp2p-standardization.md | 65 ++++++++-------------- 1 file changed, 24 insertions(+), 41 deletions(-) diff --git a/specs/networking/libp2p-standardization.md b/specs/networking/libp2p-standardization.md index c5add549a..dec337f39 100644 --- a/specs/networking/libp2p-standardization.md +++ b/specs/networking/libp2p-standardization.md @@ -7,7 +7,7 @@ Ethereum 2.0 clients plan to use the libp2p protocol networking stack for mainnet release. This document aims to standardize the libp2p client protocols, configuration and messaging formats. -# Libp2p Protocols +# Libp2p Components ## Transport @@ -19,8 +19,11 @@ TCP/IP and optionally websockets. Websockets are useful for implementations running in the browser and therefore native clients would ideally support these implementations by supporting websockets. -An ideal libp2p transport would therefore be TCP/IP with a fallback to -websockets. +An ideal libp2p transport would therefore support both TCP/IP and websockets. + +*Note: There is active development in libp2p to facilitate the +[QUIC](https://github.com/libp2p/go-libp2p-quic-transport) transport, which may +be adopted in the future* ### Encryption @@ -35,6 +38,11 @@ Current defaults are: - Cipher: `AES-128` (also supports `AES-256`, `TwofishCTR`) - Digests: `SHA256` (also supports `SHA512`) +*Note: Secio is being deprecated in favour of [TLS +1.3](https://github.com/libp2p/specs/blob/master/tls/tls.md). It is our +intention to transition to use TLS 1.3 for encryption between nodes, rather +than Secio.* + ## Protocols @@ -45,7 +53,8 @@ running a libp2p network stack. #### Protocol id: `/multistream/1.0.0` -Clients running libp2p should support the [multistream-select](https://github.com/multiformats/multistream-select/) +Clients running libp2p should support the +[multistream-select](https://github.com/multiformats/multistream-select/) protocol which allows clients to negotiate libp2p protocols establish streams per protocol. @@ -62,7 +71,7 @@ optionally [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md) ## Gossipsub -#### Protocol id: `/meshsub/1.0.0` +#### Protocol id: `/eth/serenity/gossipsub/1.0.0` *Note: Parameters listed here are subject to a large-scale network feasibility study* @@ -121,36 +130,14 @@ number of shard subnets will be defined via `SHARD_SUBNET_COUNT` and the shard ### Messages -#### Libp2p Specification +*Note: The message format here is Eth2.0-specific* -*This section simply outlines the data sent across the wire as specified by -libp2p - this section is aimed at gossipsub implementers to standardize their implementation of this protocol* +Each Gossipsub +[Message](https://github.com/libp2p/go-libp2p-pubsub/blob/master/pb/rpc.proto#L17-L24) +has a maximum size of 512KB (estimated from expected largest uncompressed block +size). -Libp2p raw gossipsub messages are sent across the wire as fixed-size length-prefixed byte arrays. - -The byte array is prefixed with an unsigned 64 bit length number encoded as an -`unsigned varint` (https://github.com/multiformats/unsigned-varint). Gossipsub messages therefore take the form: -``` -+--------------------------+ -| message length | -+--------------------------+ -| | -| body (<1M) | -| | -+--------------------------+ -``` - -The body represents a protobuf-encoded [Message](https://github.com/libp2p/go-libp2p-pubsub/blob/master/pb/rpc.proto#L17-L24). - -In the following section we discuss the data being sent in the `data` field of -the protobuf gossipsub `Message`. - -#### Eth2.0 Specifics - -Each message has a maximum size of 512KB (estimated from expected largest uncompressed -block size). - -The `data` that is sent in a Gossipsub message is an SSZ-encoded object. For the `beacon_block` topic, +The `data` field of a Gossipsub `Message` is an SSZ-encoded object. For the `beacon_block` topic, this will be a `beacon_block`. For the `beacon_attestation` topic, this will be an `attestation`. @@ -162,10 +149,10 @@ The [RPC Interface](./rpc-interface.md) is specified in this repository. ## Identify -#### Protocol Id: `/ipfs/id/1.0.0` (to be updated to `/p2p/id/1.0.0`) - -*To be updated to incorporate discv5* +**Note: This protocol is a placeholder and will be updated once the discv5 +discovery protocol is added to this document** +#### Protocol Id: `/eth/serentiy/id/1.0.0` The Identify protocol (defined in go - [identify-go](https://github.com/ipfs/go-ipfs/blob/master/core/commands/id.go) and rust [rust-identify](https://github.com/libp2p/rust-libp2p/blob/master/protocols/identify/src/lib.rs)) allows a node A to query another node B which information B knows about A. This also includes the addresses B is listening on. @@ -185,8 +172,4 @@ type of connecting node. Suggested format: ## Discovery -#### Protocol Id: `/eth/serenity/disc/1.0.0` - -*To be updated to incorporate discv5* - -The discovery protocol to be determined. +**To be updated to incorporate discv5** From 3a309155aa51288103b6d4f09343b68389b9dca7 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 May 2019 22:06:00 +0200 Subject: [PATCH 18/84] fix deposit domain: forks are ignored for deposit validity, deposits are always accepted, if coming from the correct contract(s). --- specs/core/0_beacon-chain.md | 6 ++++-- specs/validator/0_beacon-chain-validator.md | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index dd2d3d1a6..cee97f9f5 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -966,7 +966,8 @@ def get_domain(state: BeaconState, """ epoch = get_current_epoch(state) if message_epoch is None else message_epoch fork_version = state.fork.previous_version if epoch < state.fork.epoch else state.fork.current_version - return bytes_to_int(fork_version + int_to_bytes4(domain_type)) + # fork version is on the big-endian side: when signing using only the type (e.g. deposits), the type can be passed directly. + return bytes_to_int(int_to_bytes4(domain_type) + fork_version) ``` ### `get_bitfield_bit` @@ -1766,7 +1767,8 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: validator_pubkeys = [v.pubkey for v in state.validator_registry] if pubkey not in validator_pubkeys: # Verify the deposit signature (proof of possession) - if not bls_verify(pubkey, signing_root(deposit.data), deposit.data.signature, get_domain(state, DOMAIN_DEPOSIT)): + # Note: deposits are valid regardless of fork version, hence the type is passed directly as domain. + if not bls_verify(pubkey, signing_root(deposit.data), deposit.data.signature, DOMAIN_DEPOSIT): return # Add validator and balance entries diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 600ed0839..a74fbb80f 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -103,7 +103,7 @@ To submit a deposit: * Pack the validator's [initialization parameters](#initialization) into `deposit_data`, a [`DepositData`](../core/0_beacon-chain.md#depositdata) SSZ object. * Let `amount` be the amount in Gwei to be deposited by the validator where `MIN_DEPOSIT_AMOUNT <= amount <= MAX_EFFECTIVE_BALANCE`. * Set `deposit_data.amount = amount`. -* Let `signature` be the result of `bls_sign` of the `signing_root(deposit_data)` with `domain=DOMAIN_DEPOSIT`. +* Let `signature` be the result of `bls_sign` of the `signing_root(deposit_data)` with `domain=DOMAIN_DEPOSIT`. (Deposits are valid regardless of fork version, hence the type is passed directly as domain.) * Send a transaction on the Ethereum 1.0 chain to `DEPOSIT_CONTRACT_ADDRESS` executing `def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96])` along with a deposit of `amount` Gwei. *Note*: Deposits made for the same `pubkey` are treated as for the same validator. A singular `Validator` will be added to `state.validator_registry` with each additional deposit amount added to the validator's balance. A validator can only be activated when total deposits for the validator pubkey meet or exceed `MAX_EFFECTIVE_BALANCE`. From f371daeb204c1ede12fcb1bf6e5b15263f1bc504 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 7 May 2019 09:01:07 +0100 Subject: [PATCH 19/84] Update specs/core/0_beacon-chain.md Co-Authored-By: JustinDrake --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8d999c65c..aaa85e425 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1265,7 +1265,7 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. State transitions that trigger an unhandled exception (e.g. a failed `assert` or an out-of-range list access) are considered invalid. ```python -def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconBlock: +def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconState: # Process slots (including those with no blocks) since block process_slots(state, block.slot) # Process block From 50009ea85bd1fd96b81c3f3b6fb3f39b58662050 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Tue, 7 May 2019 10:12:33 +0100 Subject: [PATCH 20/84] Implement HW's exception-handling suggestion --- scripts/phase0/build_spec.py | 5 +++-- specs/core/0_beacon-chain.md | 17 +++++++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index 54adfdde7..a1f0209bb 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -5,7 +5,8 @@ import function_puller def build_phase0_spec(sourcefile, outfile): code_lines = [] code_lines.append(""" - + +import copy from typing import ( Any, Callable, @@ -88,7 +89,7 @@ def apply_constants_preset(preset: Dict[str, Any]): # Deal with derived constants global_vars['GENESIS_EPOCH'] = slot_to_epoch(GENESIS_SLOT) - + # Initialize SSZ types again, to account for changed lengths init_SSZ_types() """) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index aaa85e425..84d780ad6 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1262,16 +1262,21 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], ## Beacon chain state transition function -The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. State transitions that trigger an unhandled exception (e.g. a failed `assert` or an out-of-range list access) are considered invalid. +The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. ```python def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconState: - # Process slots (including those with no blocks) since block - process_slots(state, block.slot) - # Process block - process_block(state, block) - # Return post-state + pre_state = copy.deepcopy(state) + try: + # Process slots (including those with no blocks) since block + process_slots(state, block.slot) + # Process block + process_block(state, block) + except Exception: + # State is not advanced on exceptions (e.g. a failed `assert` or an out-of-range list access) + return pre_state return state + ``` ```python From bff71b6e90dc87a1e2e85b631ec5e3593a922722 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 7 May 2019 11:08:14 -0600 Subject: [PATCH 21/84] change some language to be more declarative rather than about the future --- specs/networking/libp2p-standardization.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/specs/networking/libp2p-standardization.md b/specs/networking/libp2p-standardization.md index 67bf74d1d..0c65734a5 100644 --- a/specs/networking/libp2p-standardization.md +++ b/specs/networking/libp2p-standardization.md @@ -77,7 +77,7 @@ optionally [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md) study* The [Gossipsub](https://github.com/libp2p/specs/tree/master/pubsub/gossipsub) -protocol will be used for block and attestation propagation across the +protocol is used for block and attestation propagation across the network. ### Configuration Parameters @@ -111,7 +111,7 @@ propagation times and message duplication. Current network-related defaults are: *The Go and Js implementations use string topics - This is likely to be updated to topic hashes in later versions - https://github.com/libp2p/rust-libp2p/issues/473* -For Eth2.0 clients, topics will be sent as `SHA2-256` hashes of the topic string. +For Eth2.0 clients, topics are sent as `SHA2-256` hashes of the topic string. There are two main topics used to propagate attestations and beacon blocks to all nodes on the network. @@ -120,12 +120,12 @@ all nodes on the network. beacon blocks to all nodes on the networks. - The `beacon_attestation` topic - This topic is used to propagate aggregated attestations to subscribing nodes (typically block proposers) to - be included into future blocks. Attestations will be aggregated in their + be included into future blocks. Attestations are aggregated in their respective subnets before publishing on this topic. -Shards will be grouped into their own subnets (defined by a shard topic). The -number of shard subnets will be defined via `SHARD_SUBNET_COUNT` and the shard -`shard_number % SHARD_SUBNET_COUNT` will be assigned to the topic: +Shards are grouped into their own subnets (defined by a shard topic). The +number of shard subnets is defined via `SHARD_SUBNET_COUNT` and the shard +`shard_number % SHARD_SUBNET_COUNT` is assigned to the topic: `shard{shard_number % SHARD_SUBNET_COUNT}`. ### Messages @@ -138,7 +138,7 @@ has a maximum size of 512KB (estimated from expected largest uncompressed block size). The `data` field of a Gossipsub `Message` is an SSZ-encoded object. For the `beacon_block` topic, -this will be a `beacon_block`. For the `beacon_attestation` topic, this will be +this is a `beacon_block`. For the `beacon_attestation` topic, this is an `attestation`. ## Eth-2 RPC From 3c87754deee2bd21521579c7abb90d37484c0ebc Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 8 May 2019 10:28:08 +1000 Subject: [PATCH 22/84] Rename shard topics to explicitly state --- specs/networking/libp2p-standardization.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/networking/libp2p-standardization.md b/specs/networking/libp2p-standardization.md index 0c65734a5..1f728fa54 100644 --- a/specs/networking/libp2p-standardization.md +++ b/specs/networking/libp2p-standardization.md @@ -126,7 +126,7 @@ all nodes on the network. Shards are grouped into their own subnets (defined by a shard topic). The number of shard subnets is defined via `SHARD_SUBNET_COUNT` and the shard `shard_number % SHARD_SUBNET_COUNT` is assigned to the topic: -`shard{shard_number % SHARD_SUBNET_COUNT}`. +`shard{shard_number % SHARD_SUBNET_COUNT}_attestation`. ### Messages From c37157ead1a97fd3fb2fccdda4f0b67ea258a1ea Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Wed, 8 May 2019 19:15:23 +0100 Subject: [PATCH 23/84] Revert exception handling --- specs/core/0_beacon-chain.md | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 3470459e1..bf210c60c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1216,19 +1216,15 @@ Let `genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state))`. ## Beacon chain state transition function -The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. +The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. State transitions that trigger an unhandled excpetion (e.g. a failed `assert` or an out-of-range list access) are considered invalid. ```python def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconState: - pre_state = copy.deepcopy(state) - try: - # Process slots (including those with no blocks) since block - process_slots(state, block.slot) - # Process block - process_block(state, block) - except Exception: - # State is not advanced on exceptions (e.g. a failed `assert` or an out-of-range list access) - return pre_state + # Process slots (including those with no blocks) since block + process_slots(state, block.slot) + # Process block + process_block(state, block) + # Return post-state return state ``` From 8da4b8173e38bec3f46820beab731e173384c689 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 8 May 2019 14:49:53 -0600 Subject: [PATCH 24/84] remove unnecessary import of copy --- scripts/phase0/build_spec.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index ef7c055d2..f7587bad5 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -6,7 +6,6 @@ def build_phase0_spec(sourcefile, outfile): code_lines = [] code_lines.append(""" -import copy from typing import ( Any, Dict, From 4db4d879301b130eba3667d8f455de8eb07e7c80 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 9 May 2019 14:57:36 +0800 Subject: [PATCH 25/84] Refactor `process_final_updates` --- specs/core/1_custody-game.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 1b6b1d2e4..e03e54ed0 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -683,13 +683,12 @@ Append this to `process_final_updates(state)`: # Clean up exposed RANDAO key reveals state.exposed_derived_secrets[current_epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] = [] # Reset withdrawable epochs if challenge records are empty + records = state.custody_chunk_challenge_records + state.bit_challenge_records + validator_indices_in_records = set( + [record.challenger_index for record in records] + [record.responder_index for record in records] + ) for index, validator in enumerate(state.validator_registry): - eligible = True - for records in (state.custody_chunk_challenge_records, state.bit_challenge_records): - for filter_func in (lambda rec: rec.challenger_index == index, lambda rec: rec.responder_index == index): - if len(list(filter(filter_func, records))) > 0: - eligible = False - if eligible: + if index not in validator_indices_in_records: if validator.exit_epoch == FAR_FUTURE_EPOCH: validator.withdrawable_epoch = FAR_FUTURE_EPOCH else: From ba1949b2bc59be426bca8db1f0cd5a9eb0c2c376 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Sun, 12 May 2019 13:46:17 -0700 Subject: [PATCH 26/84] Update 1_custody-game.md --- specs/core/1_custody-game.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index a030c577b..429e8e035 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -104,6 +104,8 @@ This document details the beacon chain additions and changes in Phase 1 of Ether ### Reward and penalty quotients +| Name | Value | +| - | - | | `EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE` | `2**1` (= 2) | ### Signature domains From feb3b5ea0d021192c9b6759c20f10d2e21d6f711 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 13 May 2019 10:55:08 +1000 Subject: [PATCH 27/84] Correct typo --- specs/networking/libp2p-standardization.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/networking/libp2p-standardization.md b/specs/networking/libp2p-standardization.md index 1f728fa54..d9e856586 100644 --- a/specs/networking/libp2p-standardization.md +++ b/specs/networking/libp2p-standardization.md @@ -152,7 +152,7 @@ The [RPC Interface](./rpc-interface.md) is specified in this repository. **Note: This protocol is a placeholder and will be updated once the discv5 discovery protocol is added to this document** -#### Protocol Id: `/eth/serentiy/id/1.0.0` +#### Protocol Id: `/eth/serenity/id/1.0.0` The Identify protocol (defined in go - [identify-go](https://github.com/ipfs/go-ipfs/blob/master/core/commands/id.go) and rust [rust-identify](https://github.com/libp2p/rust-libp2p/blob/master/protocols/identify/src/lib.rs)) allows a node A to query another node B which information B knows about A. This also includes the addresses B is listening on. From 7bb85a69ed5759bbf64a73bac111865fc53b2412 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 13 May 2019 16:34:30 -0400 Subject: [PATCH 28/84] add process_slots usage to validator guide --- specs/validator/0_beacon-chain-validator.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 49290b432..ef6e8bf25 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -238,7 +238,7 @@ A validator should create and broadcast the attestation halfway through the `slo First the validator should construct `attestation_data`, an [`AttestationData`](../core/0_beacon-chain.md#attestationdata) object based upon the state at the assigned slot. * Let `head_block` be the result of running the fork choice during the assigned slot. -* Let `head_state` be the state of `head_block` processed through any empty slots up to the assigned slot. +* Let `head_state` be the state of `head_block` processed through any empty slots up to the assigned slot using `process_slots(state, slot)`. ##### LMD GHOST vote @@ -360,7 +360,7 @@ def is_proposer_at_slot(state: BeaconState, return get_beacon_proposer_index(state) == validator_index ``` -*Note*: To see if a validator is assigned to proposer during the slot, the validator must run an empty slot transition from the previous state to the current slot. +*Note*: To see if a validator is assigned to proposer during the slot, the validator must run an empty slot transition from the previous state to the current slot using `process_slots(state, current_slot)`. ### Lookahead From a0a2aa90de166d840ba69dcc487079c7b26571f7 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 13 May 2019 16:40:45 -0400 Subject: [PATCH 29/84] lint --- specs/core/0_beacon-chain.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 0d0e37679..524a40b17 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1253,7 +1253,6 @@ def process_slot(state: BeaconState) -> None: # Cache block root previous_block_root = signing_root(state.latest_block_header) state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_block_root - ``` ### Epoch processing @@ -1610,7 +1609,12 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: def process_randao(state: BeaconState, body: BeaconBlockBody) -> None: proposer = state.validator_registry[get_beacon_proposer_index(state)] # Verify that the provided randao value is valid - assert bls_verify(proposer.pubkey, hash_tree_root(get_current_epoch(state)), body.randao_reveal, get_domain(state, DOMAIN_RANDAO)) + assert bls_verify( + proposer.pubkey, + hash_tree_root(get_current_epoch(state)), + body.randao_reveal, + get_domain(state, DOMAIN_RANDAO), + ) # Mix it in state.latest_randao_mixes[get_current_epoch(state) % LATEST_RANDAO_MIXES_LENGTH] = ( xor(get_randao_mix(state, get_current_epoch(state)), From 5ba90d68e194310816f22f36caa8a7cc3d95cf52 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 13 May 2019 16:53:28 -0400 Subject: [PATCH 30/84] add flag for validate state root --- specs/core/0_beacon-chain.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 524a40b17..a9c4f8811 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1220,11 +1220,14 @@ Let `genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state))`. The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. State transitions that trigger an unhandled excpetion (e.g. a failed `assert` or an out-of-range list access) are considered invalid. ```python -def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconState: +def state_transition(state: BeaconState, block: BeaconBlock, validate_state_root: bool=False) -> BeaconState: # Process slots (including those with no blocks) since block process_slots(state, block.slot) # Process block process_block(state, block) + # Validate state root if received from external source + if validate_state_root: + process_state_root(state, block) # Return post-state return state ``` @@ -1579,7 +1582,6 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_randao(state, block.body) process_eth1_data(state, block.body) process_operations(state, block.body) - # process_state_root(state, block) ``` #### Block header From c2942c00c6555cd03aa0595b9ee947e5eb06e6f1 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 13 May 2019 17:02:20 -0400 Subject: [PATCH 31/84] lint requires install_test in ci --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fd7708f8d..2bfda20ff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -102,10 +102,12 @@ workflows: test_spec: jobs: - checkout_specs - - lint - install_test: requires: - checkout_specs - test: requires: - install_test + - lint: + requires: + - install_test From f830f83a1d6ae1b6728bfd357da6cbef0d4a5107 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 14 May 2019 10:32:20 +0800 Subject: [PATCH 32/84] Update config.yml --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2bfda20ff..439afc9a9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -102,12 +102,12 @@ workflows: test_spec: jobs: - checkout_specs + - lint: + requires: + - checkout_specs - install_test: requires: - checkout_specs - test: requires: - install_test - - lint: - requires: - - install_test From c60635d2c9c829865189b4856ef4748cff7a3d9d Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 14 May 2019 06:15:03 +0100 Subject: [PATCH 33/84] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a9c4f8811..ee37d9217 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -117,7 +117,6 @@ - [Deposits](#deposits) - [Voluntary exits](#voluntary-exits) - [Transfers](#transfers) - - [State root verification](#state-root-verification) @@ -373,6 +372,7 @@ The types are defined topologically to aid in facilitating an executable version 'signature': 'bytes96', } ``` + #### `Validator` ```python @@ -577,9 +577,7 @@ The types are defined topologically to aid in facilitating an executable version 'latest_block_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], 'latest_state_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], 'latest_active_index_roots': ['bytes32', LATEST_ACTIVE_INDEX_ROOTS_LENGTH], - # Balances slashed at every withdrawal period 'latest_slashed_balances': ['uint64', LATEST_SLASHED_EXIT_LENGTH], - # `latest_block_header.state_root == ZERO_HASH` temporarily 'latest_block_header': BeaconBlockHeader, 'historical_roots': ['bytes32'], @@ -1225,9 +1223,9 @@ def state_transition(state: BeaconState, block: BeaconBlock, validate_state_root process_slots(state, block.slot) # Process block process_block(state, block) - # Validate state root if received from external source + # Validate state root (`validate_state_root == True` in production) if validate_state_root: - process_state_root(state, block) + assert block.state_root == hash_tree_root(state) # Return post-state return state ``` @@ -1838,10 +1836,3 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: assert not (0 < state.balances[transfer.sender] < MIN_DEPOSIT_AMOUNT) assert not (0 < state.balances[transfer.recipient] < MIN_DEPOSIT_AMOUNT) ``` - -#### State root verification - -```python -def process_state_root(state: BeaconState, block: BeaconBlock) -> None: - assert block.state_root == hash_tree_root(state) -``` From 5e7b173b22e0d550092ac967f1dc84d2c794e935 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 14 May 2019 17:29:11 -0400 Subject: [PATCH 34/84] fix up validator guide crosslink committee instructions --- specs/validator/0_beacon-chain-validator.md | 119 +++++++++----------- 1 file changed, 53 insertions(+), 66 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index ef6e8bf25..67a09c797 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -20,6 +20,8 @@ - [Process deposit](#process-deposit) - [Validator index](#validator-index) - [Activation](#activation) + - [Validator assignments](#validator-assignments) + - [Lookahead](#lookahead) - [Beacon chain responsibilities](#beacon-chain-responsibilities) - [Block proposal](#block-proposal) - [Block header](#block-header) @@ -45,8 +47,6 @@ - [Aggregation bitfield](#aggregation-bitfield) - [Custody bitfield](#custody-bitfield) - [Aggregate signature](#aggregate-signature) - - [Validator assignments](#validator-assignments) - - [Lookahead](#lookahead) - [How to avoid slashing](#how-to-avoid-slashing) - [Proposer slashing](#proposer-slashing) - [Attester slashing](#attester-slashing) @@ -127,13 +127,61 @@ Once a validator is activated, the validator is assigned [responsibilities](#bea *Note*: There is a maximum validator churn per finalized epoch so the delay until activation is variable depending upon finality, total active validator balance, and the number of validators in the queue to be activated. +## Validator assignments + +A validator can get committee assignments for a given epoch using the following helper via `get_committee_assignment(state, epoch, validator_index)` where `epoch <= next_epoch`. + +```python +def get_committee_assignment( + state: BeaconState, + epoch: Epoch, + validator_index: ValidatorIndex) -> Tuple[List[ValidatorIndex], Shard, Slot]: + """ + Return the committee assignment in the ``epoch`` for ``validator_index``. + ``assignment`` returned is a tuple of the following form: + * ``assignment[0]`` is the list of validators in the committee + * ``assignment[1]`` is the shard to which the committee is assigned + * ``assignment[2]`` is the slot at which the committee is assigned + """ + next_epoch = get_current_epoch(state) + 1 + assert epoch <= next_epoch + + committees_per_slot = get_epoch_committee_count(state, epoch) // SLOTS_PER_EPOCH + epoch_start_slot = get_epoch_start_slot(epoch) + for slot in range(epoch_start_slot, epoch_start_slot + SLOTS_PER_EPOCH) + offset = committees_per_slot * (slot % SLOTS_PER_EPOCH) + for i in range(committees_per_slot): + shard = (get_epoch_start_shard(state, epoch) + i) % SHARD_COUNT + committee = get_crosslink_committee(state, epoch, shard) + if validator_index in committee: + return committee, shard, slot +``` + +A validator can use the following function to see if they are supposed to propose during their assigned committee slot. This function can only be run during the slot in question. Proposer selection is only stable within the context of the current epoch. + +```python +def is_proposer(state: BeaconState, + validator_index: ValidatorIndex) -> bool: + return get_beacon_proposer_index(state) == validator_index +``` + +*Note*: To see if a validator is assigned to proposer during the slot, the validator must run an empty slot transition from the previous state to the current slot using `process_slots(state, current_slot)`. + +### Lookahead + +The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing, which must checked during the slot in question. + +`get_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (`current_epoch + 1`). A validator should plan for future assignments which involves noting at which future slot one will have to attest and also which shard one should begin syncing (in Phase 1+). + +Specifically, a validator should call `get_committee_assignment(state, next_epoch, validator_index)` when checking for next epoch assignments. + ## Beacon chain responsibilities A validator has two primary responsibilities to the beacon chain: [proposing blocks](#block-proposal) and [creating attestations](#attestations-1). Proposals happen infrequently, whereas attestations should be created once per epoch. ### Block proposal -A validator is expected to propose a [`BeaconBlock`](../core/0_beacon-chain.md#beaconblock) at the beginning of any slot during which `get_beacon_proposer_index(state)` returns the validator's `validator_index`. To propose, the validator selects the `BeaconBlock`, `parent`, that in their view of the fork choice is the head of the chain during `slot - 1`. The validator is to create, sign, and broadcast a `block` that is a child of `parent` and that executes a valid [beacon chain state transition](../core/0_beacon-chain.md#beacon-chain-state-transition-function). +A validator is expected to propose a [`BeaconBlock`](../core/0_beacon-chain.md#beaconblock) at the beginning of any slot during which `is_proposer(state, validator_index)` returns `True`. To propose, the validator selects the `BeaconBlock`, `parent`, that in their view of the fork choice is the head of the chain during `slot - 1`. The validator is to create, sign, and broadcast a `block` that is a child of `parent` and that executes a valid [beacon chain state transition](../core/0_beacon-chain.md#beacon-chain-state-transition-function). There is one proposer per slot, so if there are N active validators any individual validator will on average be assigned to propose once per N slots (e.g. at 312500 validators = 10 million ETH, that's once per ~3 weeks). @@ -229,7 +277,7 @@ Up to `MAX_VOLUNTARY_EXITS` [`VoluntaryExit`](../core/0_beacon-chain.md#voluntar ### Attestations -A validator is expected to create, sign, and broadcast an attestation during each epoch. The slot during which the validator performs this role is any slot at which `get_crosslink_committees_at_slot(state, slot)` contains a committee that contains `validator_index`. +A validator is expected to create, sign, and broadcast an attestation during each epoch. The committee, assigned shard, and assigned slot for which the validator performs this role during an epoch is defined by `get_committee_assignment(state, epoch, validator_index)`. A validator should create and broadcast the attestation halfway through the `slot` during which the validator is assigned ― that is, `SECONDS_PER_SLOT * 0.5` seconds after the start of `slot`. @@ -259,7 +307,7 @@ Set `attestation_data.beacon_block_root = signing_root(head_block)`. Construct `attestation_data.crosslink` via the following -* Set `attestation_data.crosslink.shard = shard` where `shard` is the shard associated with the validator's committee defined by `get_crosslink_committees_at_slot`. +* Set `attestation_data.crosslink.shard = shard` where `shard` is the shard associated with the validator's committee. * Set `attestation_data.crosslink.epoch = min(attestation_data.target_epoch, head_state.current_crosslinks[shard].epoch + MAX_EPOCHS_PER_CROSSLINK)`. * Set `attestation_data.crosslink.parent_root = hash_tree_root(head_state.current_crosslinks[shard])`. * Set `attestation_data.crosslink.data_root = ZERO_HASH`. *Note*: This is a stub for Phase 0. @@ -310,67 +358,6 @@ signed_attestation_data = bls_sign( ) ``` -## Validator assignments - -A validator can get the current, previous, and next epoch committee assignments using the following helper via `get_committee_assignment(state, epoch, validator_index)` where `previous_epoch <= epoch <= next_epoch`. - -```python -def get_committee_assignment( - state: BeaconState, - epoch: Epoch, - validator_index: ValidatorIndex) -> Tuple[List[ValidatorIndex], Shard, Slot]: - """ - Return the committee assignment in the ``epoch`` for ``validator_index``. - ``assignment`` returned is a tuple of the following form: - * ``assignment[0]`` is the list of validators in the committee - * ``assignment[1]`` is the shard to which the committee is assigned - * ``assignment[2]`` is the slot at which the committee is assigned - """ - previous_epoch = get_previous_epoch(state) - next_epoch = get_current_epoch(state) + 1 - assert previous_epoch <= epoch <= next_epoch - - epoch_start_slot = get_epoch_start_slot(epoch) - for slot in range(epoch_start_slot, epoch_start_slot + SLOTS_PER_EPOCH): - crosslink_committees = get_crosslink_committees_at_slot( - state, - slot, - ) - selected_committees = [ - committee # Tuple[List[ValidatorIndex], Shard] - for committee in crosslink_committees - if validator_index in committee[0] - ] - if len(selected_committees) > 0: - validators = selected_committees[0][0] - shard = selected_committees[0][1] - - assignment = (validators, shard, slot) - return assignment -``` - -A validator can use the following function to see if they are supposed to propose during their assigned committee slot. This function can only be run during the slot in question. Proposer selection is only stable within the context of the current epoch. - -```python -def is_proposer_at_slot(state: BeaconState, - slot: Slot, - validator_index: ValidatorIndex) -> bool: - assert state.slot == slot - - return get_beacon_proposer_index(state) == validator_index -``` - -*Note*: To see if a validator is assigned to proposer during the slot, the validator must run an empty slot transition from the previous state to the current slot using `process_slots(state, current_slot)`. - - -### Lookahead - -The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing, which must checked during the slot in question. - -`get_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (`current_epoch + 1`). A validator should plan for future assignments which involves noting at which future slot one will have to attest and also which shard one should begin syncing (in Phase 1+). - -Specifically, a validator should call `get_committee_assignment(state, next_epoch, validator_index)` when checking for next epoch assignments. - ## How to avoid slashing "Slashing" is the burning of some amount of validator funds and immediate ejection from the active validator set. In Phase 0, there are two ways in which funds can be slashed -- [proposer slashing](#proposer-slashing) and [attester slashing](#attester-slashing). Although being slashed has serious repercussions, it is simple enough to avoid being slashed all together by remaining _consistent_ with respect to the messages a validator has previously signed. From 01efe52eb0d7523994b96ccb6e44b85929b0ad42 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 14 May 2019 17:32:44 -0400 Subject: [PATCH 35/84] fix start shard --- specs/validator/0_beacon-chain-validator.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 67a09c797..3b498bfd4 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -149,9 +149,9 @@ def get_committee_assignment( committees_per_slot = get_epoch_committee_count(state, epoch) // SLOTS_PER_EPOCH epoch_start_slot = get_epoch_start_slot(epoch) for slot in range(epoch_start_slot, epoch_start_slot + SLOTS_PER_EPOCH) - offset = committees_per_slot * (slot % SLOTS_PER_EPOCH) + slot_start_shard = get_epoch_start_shard(state, epoch) + committees_per_slot * (slot % SLOTS_PER_EPOCH) for i in range(committees_per_slot): - shard = (get_epoch_start_shard(state, epoch) + i) % SHARD_COUNT + shard = (slot_start_shard + i) % SHARD_COUNT committee = get_crosslink_committee(state, epoch, shard) if validator_index in committee: return committee, shard, slot From bc95906e4f706460f9e431178da45ae8d1244b41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20=C5=A0kvorc?= Date: Wed, 15 May 2019 10:40:42 +0200 Subject: [PATCH 36/84] Typo fix --- specs/validator/0_beacon-chain-validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index ef6e8bf25..ae0e02a92 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -365,7 +365,7 @@ def is_proposer_at_slot(state: BeaconState, ### Lookahead -The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing, which must checked during the slot in question. +The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing, which must be checked during the slot in question. `get_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (`current_epoch + 1`). A validator should plan for future assignments which involves noting at which future slot one will have to attest and also which shard one should begin syncing (in Phase 1+). From 2885f853c28d2442b4507b80d24471292a41ddb2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 May 2019 16:34:07 +0800 Subject: [PATCH 37/84] clean up lint --- .circleci/config.yml | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 439afc9a9..3fe6643b7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -96,7 +96,7 @@ jobs: reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run linter - command: make install_lint && make pyspec && make lint + command: make install_lint && make lint workflows: version: 2.1 test_spec: diff --git a/Makefile b/Makefile index 8cc889f21..a6b379b71 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ citest: $(PY_SPEC_ALL_TARGETS) install_lint: cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install flake8==3.5.0 -lint: +lint: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; \ flake8 --max-line-length=120 ./eth2spec; From b0747703a77c36d821b6bf049aa8abc8e1603603 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 May 2019 17:23:26 +0800 Subject: [PATCH 38/84] Make CI job `lint` require `test` --- .circleci/config.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3fe6643b7..d2b284e25 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,7 +52,7 @@ jobs: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} paths: - ~/specs-repo - install_test: + install_env: docker: - image: circleci/python:3.6 working_directory: ~/specs-repo @@ -64,7 +64,7 @@ jobs: reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Install pyspec requirements - command: make install_test + command: make install_test && make install_lint - save_cached_venv: venv_name: v1-pyspec reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' @@ -96,18 +96,18 @@ jobs: reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run linter - command: make install_lint && make lint + command: make lint workflows: version: 2.1 test_spec: jobs: - checkout_specs - - lint: - requires: - - checkout_specs - - install_test: + - install_env: requires: - checkout_specs - test: requires: - - install_test + - install_env + - lint: + requires: + - test From 1b3dfa67813e435761a71a3cc1c5e552efc043dd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 May 2019 17:34:49 +0800 Subject: [PATCH 39/84] kick the cache --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d2b284e25..b9f09d9cc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,13 +60,13 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec + venv_name: v1-pyspec-02 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Install pyspec requirements command: make install_test && make install_lint - save_cached_venv: - venv_name: v1-pyspec + venv_name: v1-pyspec-02 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' venv_path: ./test_libs/pyspec/venv test: @@ -77,7 +77,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec + venv_name: v1-pyspec-02 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run py-tests @@ -92,7 +92,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec + venv_name: v1-pyspec-02 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run linter From 32f76641e34afb703355ab8b8ac93fafa0d59478 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 May 2019 17:41:48 +0800 Subject: [PATCH 40/84] Add build_pyspec job --- .circleci/config.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b9f09d9cc..a0700a004 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,6 +69,19 @@ jobs: venv_name: v1-pyspec-02 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' venv_path: ./test_libs/pyspec/venv + build_pyspec: + docker: + - image: circleci/python:3.6 + working_directory: ~/specs-repo + steps: + - restore_cache: + key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} + - restore_cached_venv: + venv_name: v1-pyspec-02 + reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' + - run: + name: Build pyspec + command: make pyspec test: docker: - image: circleci/python:3.6 @@ -105,9 +118,13 @@ workflows: - install_env: requires: - checkout_specs + - build_pyspec: + requires: + - checkout_specs - test: requires: - install_env + - build_pyspec - lint: requires: - - test + - build_pyspec From ee4fdf5d06c909b4d7bc6f1dff3bce5a253b54e7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 May 2019 17:45:09 +0800 Subject: [PATCH 41/84] Kick the cache again --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a0700a004..972f0e351 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,13 +60,13 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec-02 + venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Install pyspec requirements command: make install_test && make install_lint - save_cached_venv: - venv_name: v1-pyspec-02 + venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' venv_path: ./test_libs/pyspec/venv build_pyspec: @@ -77,7 +77,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec-02 + venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Build pyspec @@ -90,7 +90,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec-02 + venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run py-tests @@ -105,7 +105,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec-02 + venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run linter From be4c792fc301b7ee254b61466720e40e155b1601 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 May 2019 18:20:20 +0800 Subject: [PATCH 42/84] Change it back to serial workflow --- .circleci/config.yml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 972f0e351..f3c5f6a81 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,19 +69,6 @@ jobs: venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' venv_path: ./test_libs/pyspec/venv - build_pyspec: - docker: - - image: circleci/python:3.6 - working_directory: ~/specs-repo - steps: - - restore_cache: - key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - - restore_cached_venv: - venv_name: v1-pyspec-03 - reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - - run: - name: Build pyspec - command: make pyspec test: docker: - image: circleci/python:3.6 @@ -118,13 +105,9 @@ workflows: - install_env: requires: - checkout_specs - - build_pyspec: - requires: - - checkout_specs - test: requires: - install_env - - build_pyspec - lint: requires: - - build_pyspec + - test From 7f6896cca3931c0e4a6b352bfd0ba3fa3dca3212 Mon Sep 17 00:00:00 2001 From: Matthew Slipper Date: Thu, 16 May 2019 16:43:10 -0700 Subject: [PATCH 43/84] Update to use union type --- specs/networking/rpc-interface.md | 34 ++++++++++--------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/specs/networking/rpc-interface.md b/specs/networking/rpc-interface.md index ca6008a40..48ee9333b 100644 --- a/specs/networking/rpc-interface.md +++ b/specs/networking/rpc-interface.md @@ -39,36 +39,24 @@ To facilitate RPC-over-`libp2p`, a single protocol name is used: `/eth/serenity/ Remote method calls are wrapped in a "request" structure: ``` - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | -+ id (uint64) + -| | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| method_id (uint16) | body_len (uint32) | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | body (body_len bytes) | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +( + id: uint64 + method_id: uint16 + body: (message_body...) +) ``` and their corresponding responses are wrapped in a "response" structure: ``` - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | -+ id (uint64) + -| | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| response_code (uint16) | result_len (uint32) | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | result (result_len bytes) | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +( + id: uint64 + response_code: uint16 + result: bytes +) ``` -Note that the above structures are NOT encoded as SSZ but rather as sequences of bytes according to the packet diagrams above. This is because SSZ does not support structures without an explicit schema. Since the `body` and `result` fields depend on the value of `method_id` and `response_code`, a schema for the above structure cannot be known beforehand. +A union type is used to determine the contents of the `body` field in the request structure. Each "body" entry in the RPC calls below corresponds to one subtype in the `body` type union. The details of the RPC-Over-`libp2p` protocol are similar to [JSON-RPC 2.0](https://www.jsonrpc.org/specification). Specifically: From 05f9dc7baa356402a5a52b2bc0f910340ce75c52 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 17 May 2019 05:59:01 -0400 Subject: [PATCH 44/84] Fix #1090 Avoid signed integer --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ee37d9217..aa8cf57ca 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -887,7 +887,7 @@ def get_shuffled_index(index: ValidatorIndex, index_count: int, seed: Bytes32) - # See the 'generalized domain' algorithm on page 3 for round in range(SHUFFLE_ROUND_COUNT): pivot = bytes_to_int(hash(seed + int_to_bytes(round, length=1))[0:8]) % index_count - flip = (pivot - index) % index_count + flip = (pivot + index_count - index) % index_count position = max(index, flip) source = hash(seed + int_to_bytes(round, length=1) + int_to_bytes(position // 256, length=4)) byte = source[(position % 256) // 8] From f19188816b2bac23b41222181be1bde31fdc79d1 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 17 May 2019 06:07:38 -0400 Subject: [PATCH 45/84] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index aa8cf57ca..622eabaf8 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1362,19 +1362,19 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process finalizations bitfield = state.justification_bitfield # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source - if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_epoch == current_epoch - 3: + if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_epoch + 3 == current_epoch: state.finalized_epoch = old_previous_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source - if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_epoch == current_epoch - 2: + if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_epoch + 2 == current_epoch: state.finalized_epoch = old_previous_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source - if (bitfield >> 0) % 8 == 0b111 and old_current_justified_epoch == current_epoch - 2: + if (bitfield >> 0) % 8 == 0b111 and old_current_justified_epoch + 2 == current_epoch: state.finalized_epoch = old_current_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source - if (bitfield >> 0) % 4 == 0b11 and old_current_justified_epoch == current_epoch - 1: + if (bitfield >> 0) % 4 == 0b11 and old_current_justified_epoch + 1 == current_epoch: state.finalized_epoch = old_current_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) ``` From 694b31b934d13ffbddf17574fa7f04b824d67c98 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 17 May 2019 06:11:39 -0400 Subject: [PATCH 46/84] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 622eabaf8..b75f14153 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -647,7 +647,7 @@ def get_previous_epoch(state: BeaconState) -> Epoch: Return the current epoch if it's genesis epoch. """ current_epoch = get_current_epoch(state) - return (current_epoch - 1) if current_epoch > GENESIS_EPOCH else current_epoch + return GENESIS_EPOCH if current_epoch == GENESIS_EPOCH else current_epoch - 1 ``` ### `get_current_epoch` From 37aca60fae778268841a3e64253b85bb616b61ed Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 17 May 2019 10:55:07 -0400 Subject: [PATCH 47/84] pr feedback --- specs/validator/0_beacon-chain-validator.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 5c92c9953..46eb55173 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -165,11 +165,11 @@ def is_proposer(state: BeaconState, return get_beacon_proposer_index(state) == validator_index ``` -*Note*: To see if a validator is assigned to proposer during the slot, the validator must run an empty slot transition from the previous state to the current slot using `process_slots(state, current_slot)`. +*Note*: To see if a validator is assigned to propose during the slot, the beacon state must be in the epoch in question. At the epoch boundaries, the validator must run an epoch transition into the epoch to successfully check the proposal assignment of the first slot. ### Lookahead -The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing, which must be checked during the slot in question. +The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing, which must be checked during the epoch in question. `get_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (`current_epoch + 1`). A validator should plan for future assignments which involves noting at which future slot one will have to attest and also which shard one should begin syncing (in Phase 1+). @@ -181,7 +181,7 @@ A validator has two primary responsibilities to the beacon chain: [proposing blo ### Block proposal -A validator is expected to propose a [`BeaconBlock`](../core/0_beacon-chain.md#beaconblock) at the beginning of any slot during which `is_proposer(state, validator_index)` returns `True`. To propose, the validator selects the `BeaconBlock`, `parent`, that in their view of the fork choice is the head of the chain during `slot - 1`. The validator is to create, sign, and broadcast a `block` that is a child of `parent` and that executes a valid [beacon chain state transition](../core/0_beacon-chain.md#beacon-chain-state-transition-function). +A validator is expected to propose a [`BeaconBlock`](../core/0_beacon-chain.md#beaconblock) at the beginning of any slot during which `is_proposer(state, validator_index)` returns `True`. To propose, the validator selects the `BeaconBlock`, `parent`, that in their view of the fork choice is the head of the chain during `slot - 1`. The validator creates, signs, and broadcasts a `block` that is a child of `parent` that satisfies a valid [beacon chain state transition](../core/0_beacon-chain.md#beacon-chain-state-transition-function). There is one proposer per slot, so if there are N active validators any individual validator will on average be assigned to propose once per N slots (e.g. at 312500 validators = 10 million ETH, that's once per ~3 weeks). From acb7444184b533de6c72fc65be1b0f7c506414c3 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 17 May 2019 10:58:02 -0400 Subject: [PATCH 48/84] pr feedback --- specs/validator/0_beacon-chain-validator.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 46eb55173..302deeae1 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -157,7 +157,7 @@ def get_committee_assignment( return committee, shard, slot ``` -A validator can use the following function to see if they are supposed to propose during their assigned committee slot. This function can only be run during the slot in question. Proposer selection is only stable within the context of the current epoch. +A validator can use the following function to see if they are supposed to propose during their assigned committee slot. This function can only be run with a `state` of the slot in question. Proposer selection is only stable within the context of the current epoch. ```python def is_proposer(state: BeaconState, @@ -171,7 +171,7 @@ def is_proposer(state: BeaconState, The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing, which must be checked during the epoch in question. -`get_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (`current_epoch + 1`). A validator should plan for future assignments which involves noting at which future slot one will have to attest and also which shard one should begin syncing (in Phase 1+). +`get_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (`current_epoch + 1`). A validator should plan for future assignments by noting at which future slot they will have to attest and also which shard they should begin syncing (in Phase 1+). Specifically, a validator should call `get_committee_assignment(state, next_epoch, validator_index)` when checking for next epoch assignments. From e1d973d5467e25fe496cc67ba3800aab096d1c54 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 17 May 2019 11:01:18 -0400 Subject: [PATCH 49/84] Update specs/validator/0_beacon-chain-validator.md Co-Authored-By: Hsiao-Wei Wang --- specs/validator/0_beacon-chain-validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 302deeae1..9e4fa7960 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -148,7 +148,7 @@ def get_committee_assignment( committees_per_slot = get_epoch_committee_count(state, epoch) // SLOTS_PER_EPOCH epoch_start_slot = get_epoch_start_slot(epoch) - for slot in range(epoch_start_slot, epoch_start_slot + SLOTS_PER_EPOCH) + for slot in range(epoch_start_slot, epoch_start_slot + SLOTS_PER_EPOCH): slot_start_shard = get_epoch_start_shard(state, epoch) + committees_per_slot * (slot % SLOTS_PER_EPOCH) for i in range(committees_per_slot): shard = (slot_start_shard + i) % SHARD_COUNT From 174e1e4dbef7605f1e4131f66d3fd0f52276b6a4 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 17 May 2019 11:04:05 -0400 Subject: [PATCH 50/84] pr feedback --- specs/validator/0_beacon-chain-validator.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 302deeae1..0940d592a 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -149,7 +149,8 @@ def get_committee_assignment( committees_per_slot = get_epoch_committee_count(state, epoch) // SLOTS_PER_EPOCH epoch_start_slot = get_epoch_start_slot(epoch) for slot in range(epoch_start_slot, epoch_start_slot + SLOTS_PER_EPOCH) - slot_start_shard = get_epoch_start_shard(state, epoch) + committees_per_slot * (slot % SLOTS_PER_EPOCH) + offset = committees_per_slot * (slot % SLOTS_PER_EPOCH) + slot_start_shard = (get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT for i in range(committees_per_slot): shard = (slot_start_shard + i) % SHARD_COUNT committee = get_crosslink_committee(state, epoch, shard) From 24edca3456166c11ebce912d3397c1618b79a3dc Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 17 May 2019 13:52:23 -0400 Subject: [PATCH 51/84] Fix to make Danny and hww happy --- specs/core/1_custody-game.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index e03e54ed0..e28b30794 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -689,9 +689,7 @@ Append this to `process_final_updates(state)`: ) for index, validator in enumerate(state.validator_registry): if index not in validator_indices_in_records: - if validator.exit_epoch == FAR_FUTURE_EPOCH: - validator.withdrawable_epoch = FAR_FUTURE_EPOCH - else: + if validator.exit_epoch != FAR_FUTURE_EPOCH and validator.withdrawable_epoch == FAR_FUTURE_EPOCH: validator.withdrawable_epoch = validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY ``` From 85c16544566f82be57c5905ccf3b7b48e2888d1d Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sun, 19 May 2019 09:33:01 -0400 Subject: [PATCH 52/84] Crosslinks store start and end epoch Solves #1034 --- specs/core/0_beacon-chain.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b75f14153..643985e85 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -281,8 +281,9 @@ The types are defined topologically to aid in facilitating an executable version { # Shard number 'shard': 'uint64', - # Epoch number - 'epoch': 'uint64', + # Crosslinking data from epochs [start....end-1] + 'start_epoch': 'uint64', + 'end_epoch': 'uint64', # Root of the previous crosslink 'parent_root': 'bytes32', # Root of the crosslinked shard data since the previous crosslink @@ -1728,7 +1729,8 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: # Check FFG data, crosslink data, and signature assert ffg_data == (data.source_epoch, data.source_root, data.target_epoch) - assert data.crosslink.epoch == min(data.target_epoch, parent_crosslink.epoch + MAX_EPOCHS_PER_CROSSLINK) + assert data.crosslink.start_epoch == parent_crosslink.end_epoch + assert data.crosslink.end_epoch == min(data.target_epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK) assert data.crosslink.parent_root == hash_tree_root(parent_crosslink) assert data.crosslink.data_root == ZERO_HASH # [to be removed in phase 1] validate_indexed_attestation(state, convert_to_indexed(state, attestation)) From a2108741e804046b1ba721676482091a27f0e0fb Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 19 May 2019 15:47:59 -0600 Subject: [PATCH 53/84] fix tests with new starT_epoch and end_epoch in Crosslink --- test_libs/pyspec/eth2spec/utils/bls_stub.py | 2 +- .../tests/block_processing/test_process_attestation.py | 2 +- test_libs/pyspec/tests/helpers.py | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/bls_stub.py b/test_libs/pyspec/eth2spec/utils/bls_stub.py index 108c4ef71..ae97de175 100644 --- a/test_libs/pyspec/eth2spec/utils/bls_stub.py +++ b/test_libs/pyspec/eth2spec/utils/bls_stub.py @@ -9,4 +9,4 @@ def bls_verify_multiple(pubkeys, message_hashes, signature, domain): def bls_aggregate_pubkeys(pubkeys): - return b'\x42' * 96 + return b'\x42' * 48 diff --git a/test_libs/pyspec/tests/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/block_processing/test_process_attestation.py index 708d68dca..6851561e9 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attestation.py @@ -124,7 +124,7 @@ def test_bad_previous_crosslink(state): for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): next_slot(state) - state.current_crosslinks[attestation.data.crosslink.shard].epoch += 10 + state.current_crosslinks[attestation.data.crosslink.shard].end_epoch += 10 pre_state, post_state = run_attestation_processing(state, attestation, False) diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index 5ddb2dc15..7af210f85 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -177,6 +177,7 @@ def build_attestation_data(state, slot, shard): justified_block_root = state.current_justified_root crosslinks = state.current_crosslinks if slot_to_epoch(slot) == get_current_epoch(state) else state.previous_crosslinks + parent_crosslink = crosslinks[shard] return AttestationData( beacon_block_root=block_root, source_epoch=justified_epoch, @@ -185,9 +186,10 @@ def build_attestation_data(state, slot, shard): target_root=epoch_boundary_root, crosslink=Crosslink( shard=shard, - epoch=min(slot_to_epoch(slot), crosslinks[shard].epoch + MAX_EPOCHS_PER_CROSSLINK), + start_epoch=parent_crosslink.end_epoch, + end_epoch=min(slot_to_epoch(slot), parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK), data_root=spec.ZERO_HASH, - parent_root=hash_tree_root(crosslinks[shard]), + parent_root=hash_tree_root(parent_crosslink), ), ) From 62f8d19ffc2f80763e7dfef5a66a19782080d5c4 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 19 May 2019 16:06:10 -0600 Subject: [PATCH 54/84] add some attestation crosslink tests --- .../test_process_attestation.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/test_libs/pyspec/tests/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/block_processing/test_process_attestation.py index 6851561e9..97eddb902 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attestation.py @@ -64,6 +64,22 @@ def test_success_prevous_epoch(state): return pre_state, attestation, post_state +def test_success_since_max_epochs_per_crosslink(state): + for _ in range(spec.MAX_EPOCHS_PER_CROSSLINK + 2): + next_epoch(state) + + attestation = get_valid_attestation(state) + data = attestation.data + assert data.crosslink.end_epoch - data.crosslink.start_epoch == spec.MAX_EPOCHS_PER_CROSSLINK + + for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): + next_slot(state) + + pre_state, post_state = run_attestation_processing(state, attestation) + + return pre_state, attestation, post_state + + def test_before_inclusion_delay(state): attestation = get_valid_attestation(state) # do not increment slot to allow for inclusion delay @@ -131,6 +147,32 @@ def test_bad_previous_crosslink(state): return pre_state, attestation, post_state +def test_bad_crosslink_start_epoch(state): + next_epoch(state) + attestation = get_valid_attestation(state) + for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): + next_slot(state) + + attestation.data.crosslink.start_epoch += 1 + + pre_state, post_state = run_attestation_processing(state, attestation, False) + + return pre_state, attestation, post_state + + +def test_bad_crosslink_end_epoch(state): + next_epoch(state) + attestation = get_valid_attestation(state) + for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): + next_slot(state) + + attestation.data.crosslink.end_epoch += 1 + + pre_state, post_state = run_attestation_processing(state, attestation, False) + + return pre_state, attestation, post_state + + def test_non_empty_custody_bitfield(state): attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY From 4c5e0548833bd3b4d92c22806afce48a1b4c1ab3 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 19 May 2019 16:11:39 -0600 Subject: [PATCH 55/84] fix previous crosslink root test --- .../pyspec/tests/block_processing/test_process_attestation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/tests/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/block_processing/test_process_attestation.py index 97eddb902..763178717 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attestation.py @@ -140,7 +140,7 @@ def test_bad_previous_crosslink(state): for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): next_slot(state) - state.current_crosslinks[attestation.data.crosslink.shard].end_epoch += 10 + attestation.data.crosslink.parent_root = b'\x27' * 32 pre_state, post_state = run_attestation_processing(state, attestation, False) From c14452bcf445761926da99a1259f0a792268bdff Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sun, 19 May 2019 19:44:12 -0400 Subject: [PATCH 56/84] Updated get_custody_chunk_count Co-requisite with #1097 --- specs/core/1_custody-game.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 0e8fcf6e9..7965e25f9 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -255,8 +255,8 @@ The `empty` function accepts and SSZ type as input and returns an object of that ```python def get_custody_chunk_count(attestation: Attestation) -> int: - crosslink_start_epoch = attestation.data.latest_crosslink.epoch - crosslink_end_epoch = slot_to_epoch(attestation.data.slot) + crosslink_start_epoch = attestation.data.latest_crosslink.start_epoch + crosslink_end_epoch = attestation.data.latest_crosslink.end_epoch crosslink_crosslink_length = min(MAX_EPOCHS_PER_CROSSLINK, end_epoch - start_epoch) chunks_per_epoch = 2 * BYTES_PER_SHARD_BLOCK * SLOTS_PER_EPOCH // BYTES_PER_CUSTODY_CHUNK return crosslink_crosslink_length * chunks_per_epoch From a68aa82b894153eca18f290d306623bf4118357b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 20 May 2019 11:39:13 +0800 Subject: [PATCH 57/84] Update validator guide --- specs/validator/0_beacon-chain-validator.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 0940d592a..d05f25ef2 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -306,10 +306,12 @@ Set `attestation_data.beacon_block_root = signing_root(head_block)`. ##### Crosslink vote -Construct `attestation_data.crosslink` via the following +Construct `attestation_data.crosslink` via the following. * Set `attestation_data.crosslink.shard = shard` where `shard` is the shard associated with the validator's committee. -* Set `attestation_data.crosslink.epoch = min(attestation_data.target_epoch, head_state.current_crosslinks[shard].epoch + MAX_EPOCHS_PER_CROSSLINK)`. +* Let `parent_crosslink = head_state.current_crosslinks[shard]`. +* Set `attestation_data.crosslink.start_epoch = parent_crosslink.end_epoch`. +* Set `attestation_data.crosslink.end_epoch = min(attestation_data.target_epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK)`. * Set `attestation_data.crosslink.parent_root = hash_tree_root(head_state.current_crosslinks[shard])`. * Set `attestation_data.crosslink.data_root = ZERO_HASH`. *Note*: This is a stub for Phase 0. From 2018dd83f5a324472623db15a159be2882ffe8fa Mon Sep 17 00:00:00 2001 From: Justin Date: Mon, 20 May 2019 09:29:09 +0100 Subject: [PATCH 58/84] Update 1_custody-game.md --- specs/core/1_custody-game.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 7965e25f9..f91fe81de 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -254,12 +254,10 @@ The `empty` function accepts and SSZ type as input and returns an object of that ### `get_crosslink_chunk_count` ```python -def get_custody_chunk_count(attestation: Attestation) -> int: - crosslink_start_epoch = attestation.data.latest_crosslink.start_epoch - crosslink_end_epoch = attestation.data.latest_crosslink.end_epoch - crosslink_crosslink_length = min(MAX_EPOCHS_PER_CROSSLINK, end_epoch - start_epoch) +def get_custody_chunk_count(crosslink: Crosslink) -> int: + crosslink_length = min(MAX_EPOCHS_PER_CROSSLINK, crosslink.end_epoch - crosslink.start_epoch) chunks_per_epoch = 2 * BYTES_PER_SHARD_BLOCK * SLOTS_PER_EPOCH // BYTES_PER_CUSTODY_CHUNK - return crosslink_crosslink_length * chunks_per_epoch + return crosslink_length * chunks_per_epoch ``` ### `get_custody_chunk_bit` @@ -470,7 +468,7 @@ def process_chunk_challenge(state: BeaconState, record.chunk_index != challenge.chunk_index ) # Verify depth - depth = math.log2(next_power_of_two(get_custody_chunk_count(challenge.attestation))) + depth = math.log2(next_power_of_two(get_custody_chunk_count(challenge.attestation.data.crosslink))) assert challenge.chunk_index < 2**depth # Add new chunk challenge record new_record = CustodyChunkChallengeRecord( @@ -544,7 +542,7 @@ def process_bit_challenge(state: BeaconState, ) # Verify the chunk count - chunk_count = get_custody_chunk_count(challenge.attestation) + chunk_count = get_custody_chunk_count(challenge.attestation.data.crosslink) assert verify_bitfield(challenge.chunk_bits, chunk_count) # Verify the first bit of the hash of the chunk bits does not equal the custody bit custody_bit = get_bitfield_bit(attestation.custody_bitfield, attesters.index(responder_index)) From 83123a33da792397101bdee263d03a25ee281e0b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 20 May 2019 17:16:20 +0800 Subject: [PATCH 59/84] Set genesis_state.latest_block_header with `body_root` of empty BeaconBlockBody (#1098) --- specs/core/0_beacon-chain.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b75f14153..b34bbe7d1 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1189,7 +1189,11 @@ Let `genesis_state = get_genesis_beacon_state(genesis_deposits, eth2genesis.gene ```python def get_genesis_beacon_state(deposits: List[Deposit], genesis_time: int, genesis_eth1_data: Eth1Data) -> BeaconState: - state = BeaconState(genesis_time=genesis_time, latest_eth1_data=genesis_eth1_data) + state = BeaconState( + genesis_time=genesis_time, + latest_eth1_data=genesis_eth1_data, + latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), + ) # Process genesis deposits for deposit in deposits: From 90a3f56e0feedd491e0c835ab33114e4d8a6d660 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 21 May 2019 09:13:57 +0100 Subject: [PATCH 60/84] Edit BLS spec warning Fix #898. --- specs/bls_signature.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/specs/bls_signature.md b/specs/bls_signature.md index d119c4499..3fe1bcc0e 100644 --- a/specs/bls_signature.md +++ b/specs/bls_signature.md @@ -1,6 +1,8 @@ # BLS signature verification -**Warning: This document is pending academic review and should not yet be considered secure.** +**Notice**: This document is a placeholder to facilitate the emergence of cross-client testnets. Substantive changes are postponed until [BLS standardisation](https://github.com/pairingwg/bls_standard) is finalized. + +**Warning**: The constructions in this document should not be considered secure. In particular, the `hash_to_G2` function is known to be unsecure. ## Table of contents From 847fcf52cc5228cb32998feffb05511b2312d7b8 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 21 May 2019 11:30:38 -0600 Subject: [PATCH 61/84] utilize bls_domain directly for deposits --- specs/core/0_beacon-chain.md | 20 ++++++++++++++++---- specs/validator/0_beacon-chain-validator.md | 2 +- test_libs/pyspec/tests/helpers.py | 5 +++-- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f1986989b..79038eea8 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -174,6 +174,7 @@ These configurations are updated for releases, but may be out of sync during `de | Name | Value | | - | - | | `DEPOSIT_CONTRACT_TREE_DEPTH` | `2**5` (= 32) | +| `DEPOSIT_FORK_VERSION` | `b'\x00' * 4` | ### Gwei values @@ -629,6 +630,16 @@ The `hash` function is SHA256. `def signing_root(object: SSZContainer) -> Bytes32` is a function defined in the [SimpleSerialize spec](../simple-serialize.md#self-signed-containers) to compute signing messages. +### `bls_domain` + +```python +def bls_domain(domain_type: int, fork_version: bytes) -> int: + """ + Return the bls domain given by the ``domain_type`` and 4 byte ``fork_version``.. + """ + return bytes_to_int(int_to_bytes(domain_type, length=4) + fork_version) +``` + ### `slot_to_epoch` ```python @@ -968,8 +979,7 @@ def get_domain(state: BeaconState, """ epoch = get_current_epoch(state) if message_epoch is None else message_epoch fork_version = state.fork.previous_version if epoch < state.fork.epoch else state.fork.current_version - # fork version is on the big-endian side: when signing using only the type (e.g. deposits), the type can be passed directly. - return bytes_to_int(int_to_bytes(domain_type, length=4) + fork_version) + return bls_domain(domain_type, fork_version) ``` ### `get_bitfield_bit` @@ -1766,8 +1776,10 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: validator_pubkeys = [v.pubkey for v in state.validator_registry] if pubkey not in validator_pubkeys: # Verify the deposit signature (proof of possession) - # Note: deposits are valid regardless of fork version, hence the type is passed directly as domain. - if not bls_verify(pubkey, signing_root(deposit.data), deposit.data.signature, DOMAIN_DEPOSIT): + # Note: deposits are valid across forks, hence the deposit domain is retrieved directly from `bls_domain` + if not bls_verify( + pubkey, signing_root(deposit.data), deposit.data.signature, bls_domain(DOMAIN_DEPOSIT, DEPOSIT_FORK_VERSION) + ): return # Add validator and balance entries diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index c2fe927c0..49bb4fc3a 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -98,7 +98,7 @@ To submit a deposit: * Pack the validator's [initialization parameters](#initialization) into `deposit_data`, a [`DepositData`](../core/0_beacon-chain.md#depositdata) SSZ object. * Let `amount` be the amount in Gwei to be deposited by the validator where `MIN_DEPOSIT_AMOUNT <= amount <= MAX_EFFECTIVE_BALANCE`. * Set `deposit_data.amount = amount`. -* Let `signature` be the result of `bls_sign` of the `signing_root(deposit_data)` with `domain=DOMAIN_DEPOSIT`. (Deposits are valid regardless of fork version, hence the type is passed directly as domain.) +* Let `signature` be the result of `bls_sign` of the `signing_root(deposit_data)` with `domain=bls_domain(DOMAIN_DEPOSIT, DEPOSIT_FORK_VERSION)`. (Deposits are valid regardless of fork version, hence the static fork version being directly passed into `bls_domain`). * Send a transaction on the Ethereum 1.0 chain to `DEPOSIT_CONTRACT_ADDRESS` executing `def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96])` along with a deposit of `amount` Gwei. *Note*: Deposits made for the same `pubkey` are treated as for the same validator. A singular `Validator` will be added to `state.validator_registry` with each additional deposit amount added to the validator's balance. A validator can only be activated when total deposits for the validator pubkey meet or exceed `MAX_EFFECTIVE_BALANCE`. diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index 7af210f85..eebbd2daf 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -24,6 +24,7 @@ from eth2spec.phase0.spec import ( VoluntaryExit, # functions convert_to_indexed, + bls_domain, get_active_validator_indices, get_attesting_indices, get_block_root, @@ -144,9 +145,9 @@ def build_deposit_data(state, pubkey, privkey, amount): signature = bls.sign( message_hash=signing_root(deposit_data), privkey=privkey, - domain=get_domain( - state, + domain=bls_domain( spec.DOMAIN_DEPOSIT, + spec.DEPOSIT_FORK_VERSION, ) ) deposit_data.signature = signature From b075a7a0ab3e85b40716c3bf23cbaae76753fbb3 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 21 May 2019 11:33:52 -0600 Subject: [PATCH 62/84] add bls_domain to toc --- specs/core/0_beacon-chain.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 79038eea8..9f965c1d7 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -51,6 +51,7 @@ - [`hash`](#hash) - [`hash_tree_root`](#hash_tree_root) - [`signing_root`](#signing_root) + - [`bls_domain`](#bls_domain) - [`slot_to_epoch`](#slot_to_epoch) - [`get_previous_epoch`](#get_previous_epoch) - [`get_current_epoch`](#get_current_epoch) From 6b5f4b44eae5d3b1ba7b403eed8954aff5c96097 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 22 May 2019 01:35:24 +0200 Subject: [PATCH 63/84] avoid zero constant for deposits fork-version, just default to it --- specs/core/0_beacon-chain.md | 7 +++---- specs/validator/0_beacon-chain-validator.md | 2 +- test_libs/pyspec/tests/helpers.py | 1 - 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9f965c1d7..c14228dc1 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -175,7 +175,6 @@ These configurations are updated for releases, but may be out of sync during `de | Name | Value | | - | - | | `DEPOSIT_CONTRACT_TREE_DEPTH` | `2**5` (= 32) | -| `DEPOSIT_FORK_VERSION` | `b'\x00' * 4` | ### Gwei values @@ -634,9 +633,9 @@ The `hash` function is SHA256. ### `bls_domain` ```python -def bls_domain(domain_type: int, fork_version: bytes) -> int: +def bls_domain(domain_type: int, fork_version=b'\x00\x00\x00\x00') -> int: """ - Return the bls domain given by the ``domain_type`` and 4 byte ``fork_version``.. + Return the bls domain given by the ``domain_type`` and optional 4 byte ``fork_version`` (defaults to zero). """ return bytes_to_int(int_to_bytes(domain_type, length=4) + fork_version) ``` @@ -1779,7 +1778,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: # Verify the deposit signature (proof of possession) # Note: deposits are valid across forks, hence the deposit domain is retrieved directly from `bls_domain` if not bls_verify( - pubkey, signing_root(deposit.data), deposit.data.signature, bls_domain(DOMAIN_DEPOSIT, DEPOSIT_FORK_VERSION) + pubkey, signing_root(deposit.data), deposit.data.signature, bls_domain(DOMAIN_DEPOSIT) ): return diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 49bb4fc3a..f8272d446 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -98,7 +98,7 @@ To submit a deposit: * Pack the validator's [initialization parameters](#initialization) into `deposit_data`, a [`DepositData`](../core/0_beacon-chain.md#depositdata) SSZ object. * Let `amount` be the amount in Gwei to be deposited by the validator where `MIN_DEPOSIT_AMOUNT <= amount <= MAX_EFFECTIVE_BALANCE`. * Set `deposit_data.amount = amount`. -* Let `signature` be the result of `bls_sign` of the `signing_root(deposit_data)` with `domain=bls_domain(DOMAIN_DEPOSIT, DEPOSIT_FORK_VERSION)`. (Deposits are valid regardless of fork version, hence the static fork version being directly passed into `bls_domain`). +* Let `signature` be the result of `bls_sign` of the `signing_root(deposit_data)` with `domain=bls_domain(DOMAIN_DEPOSIT)`. (Deposits are valid regardless of fork version, `bls_domain` will default to zeroes there). * Send a transaction on the Ethereum 1.0 chain to `DEPOSIT_CONTRACT_ADDRESS` executing `def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96])` along with a deposit of `amount` Gwei. *Note*: Deposits made for the same `pubkey` are treated as for the same validator. A singular `Validator` will be added to `state.validator_registry` with each additional deposit amount added to the validator's balance. A validator can only be activated when total deposits for the validator pubkey meet or exceed `MAX_EFFECTIVE_BALANCE`. diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index eebbd2daf..06ca8a1d5 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -147,7 +147,6 @@ def build_deposit_data(state, pubkey, privkey, amount): privkey=privkey, domain=bls_domain( spec.DOMAIN_DEPOSIT, - spec.DEPOSIT_FORK_VERSION, ) ) deposit_data.signature = signature From 2c79e584c2b797efd4a19b1ed85c0f16dbc28d1a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 21 May 2019 22:42:47 -0600 Subject: [PATCH 64/84] minor lint --- test_libs/pyspec/tests/helpers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index 06ca8a1d5..a4849bfbb 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -145,9 +145,7 @@ def build_deposit_data(state, pubkey, privkey, amount): signature = bls.sign( message_hash=signing_root(deposit_data), privkey=privkey, - domain=bls_domain( - spec.DOMAIN_DEPOSIT, - ) + domain=bls_domain(spec.DOMAIN_DEPOSIT), ) deposit_data.signature = signature return deposit_data From c13421a9a7d13585b0558e1c4a8a0221d97d3c2d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 22 May 2019 16:52:44 -0400 Subject: [PATCH 65/84] type hinting for fork version Co-Authored-By: Hsiao-Wei Wang --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index c14228dc1..60f774e9a 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -633,7 +633,7 @@ The `hash` function is SHA256. ### `bls_domain` ```python -def bls_domain(domain_type: int, fork_version=b'\x00\x00\x00\x00') -> int: +def bls_domain(domain_type: int, fork_version: bytes=b'\x00\x00\x00\x00') -> int: """ Return the bls domain given by the ``domain_type`` and optional 4 byte ``fork_version`` (defaults to zero). """ From ae6d30fd62e4c4f20086388dcbf8f100265932b1 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Thu, 23 May 2019 11:48:04 +1000 Subject: [PATCH 66/84] Update with discv5 --- specs/networking/libp2p-standardization.md | 31 +++++----------------- 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/specs/networking/libp2p-standardization.md b/specs/networking/libp2p-standardization.md index d9e856586..b6a46db1b 100644 --- a/specs/networking/libp2p-standardization.md +++ b/specs/networking/libp2p-standardization.md @@ -147,29 +147,12 @@ an `attestation`. The [RPC Interface](./rpc-interface.md) is specified in this repository. -## Identify - -**Note: This protocol is a placeholder and will be updated once the discv5 -discovery protocol is added to this document** - -#### Protocol Id: `/eth/serenity/id/1.0.0` - -The Identify protocol (defined in go - [identify-go](https://github.com/ipfs/go-ipfs/blob/master/core/commands/id.go) and rust [rust-identify](https://github.com/libp2p/rust-libp2p/blob/master/protocols/identify/src/lib.rs)) -allows a node A to query another node B which information B knows about A. This also includes the addresses B is listening on. - -This protocol allows nodes to discover addresses of other nodes to be added to -peer discovery. It further allows nodes to determine the capabilities of all it's connected -peers. - -### Configuration Parameters - -The protocol has two configurable parameters, which can be used to identify the -type of connecting node. Suggested format: -``` - version: `/eth/serenity/1.0.0` - user_agent: -``` - ## Discovery -**To be updated to incorporate discv5** +Discovery Version 5 +([discv5])(https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md) +will be used for discovery. This protocol uses a UDP transport and specifies +its own encryption, ip-discovery and topic advertisement. Therefore, it has no +need to establish establish streams through `multistream-select`, rather, act +as a standalone implementation that feeds discovered peers/topics (ENR-records) as +`multiaddrs` into the libp2p service. From 650c4244bd8266fd35bc683f60e5eb54ae7b6243 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 24 May 2019 01:34:39 +0800 Subject: [PATCH 67/84] Things are different after rebasing #1035 --- specs/core/1_custody-game.md | 38 ++++++++++++------------------------ 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index f91fe81de..82bb352dd 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -38,6 +38,16 @@ - [`get_chunk_bits_root`](#get_chunk_bits_root) - [`get_randao_epoch_for_custody_period`](#get_randao_epoch_for_custody_period) - [`get_validators_custody_reveal_period`](#get_validators_custody_reveal_period) + - [`replace_empty_or_append`](#replace_empty_or_append) + - [Per-block processing](#per-block-processing) + - [Operations](#operations) + - [Custody key reveals](#custody-key-reveals) + - [Early derived secret reveals](#early-derived-secret-reveals) + - [Chunk challenges](#chunk-challenges) + - [Bit challenges](#bit-challenges) + - [Custody responses](#custody-responses) + - [Per-epoch processing](#per-epoch-processing) + - [Handling of custody-related deadlines](#handling-of-custody-related-deadlines) @@ -289,7 +299,7 @@ def get_randao_epoch_for_custody_period(period: int, validator_index: ValidatorI ### `get_validators_custody_reveal_period` - ```python +```python def get_validators_custody_reveal_period(state: BeaconState, validator_index: ValidatorIndex, epoch: Epoch=None) -> int: @@ -370,7 +380,7 @@ def process_custody_key_reveal(state: BeaconState, increase_balance(state, proposer_index, base_reward(state, index) // MINOR_REWARD_QUOTIENT) ``` -##### Early derived secret reveals +#### Early derived secret reveals Verify that `len(block.body.early_derived_secret_reveals) <= MAX_EARLY_DERIVED_SECRET_REVEALS`. @@ -689,28 +699,6 @@ Append this to `process_final_updates(state)`: ) for index, validator in enumerate(state.validator_registry): if index not in validator_indices_in_records: - if validator.exit_epoch != FAR_FUTURE_EPOCH and validator.withdrawable_epoch == FAR_FUTURE_EPOCH: + if validator.exit_epoch != FAR_FUTURE_EPOCH: validator.withdrawable_epoch = validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY ``` - -In `process_penalties_and_exits`, change the definition of `eligible` to the following (note that it is not a pure function because `state` is declared in the surrounding scope): - -```python -def eligible(state: BeaconState, index: ValidatorIndex) -> bool: - validator = state.validator_registry[index] - # Cannot exit if there are still open chunk challenges - if len([record for record in state.custody_chunk_challenge_records if record.responder_index == index]) > 0: - return False - # Cannot exit if there are still open bit challenges - if len([record for record in state.custody_bit_challenge_records if record.responder_index == index]) > 0: - return False - # Cannot exit if you have not revealed all of your custody keys - elif validator.next_custody_reveal_period <= get_validators_custody_reveal_period(state, index, validator.exit_epoch): - return False - # Cannot exit if you already have - elif validator.withdrawable_epoch < FAR_FUTURE_EPOCH: - return False - # Return minimum time - else: - return current_epoch >= validator.exit_epoch + MIN_VALIDATOR_WITHDRAWAL_EPOCHS -``` From 0f7abfa4ce85316be09ee7657c425b0d6fcfb155 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 24 May 2019 02:54:41 +0800 Subject: [PATCH 68/84] Update specs/core/1_custody-game.md Co-Authored-By: Carl Beekhuizen --- specs/core/1_custody-game.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 82bb352dd..568879b17 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -699,6 +699,6 @@ Append this to `process_final_updates(state)`: ) for index, validator in enumerate(state.validator_registry): if index not in validator_indices_in_records: - if validator.exit_epoch != FAR_FUTURE_EPOCH: + if validator.exit_epoch != FAR_FUTURE_EPOCH and validator.withdrawable_epoch == FAR_FUTURE_EPOCH: validator.withdrawable_epoch = validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY ``` From 32f97a1c8025206c3c033a159f7e82dff3a5030c Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 23 May 2019 15:12:29 -0700 Subject: [PATCH 69/84] update version used in test generators to get SHA-256 hash --- test_generators/bls/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/bls/requirements.txt b/test_generators/bls/requirements.txt index 8a933d41c..5eebde29f 100644 --- a/test_generators/bls/requirements.txt +++ b/test_generators/bls/requirements.txt @@ -1,3 +1,3 @@ -py-ecc==1.6.0 +py-ecc==1.7.0 eth-utils==1.4.1 ../../test_libs/gen_helpers From 2cd188358bf982439fa29b87cdb7e29492ab9bbc Mon Sep 17 00:00:00 2001 From: Age Manning Date: Fri, 24 May 2019 21:24:35 +1000 Subject: [PATCH 70/84] Fix typo in libp2p-standardization --- specs/networking/libp2p-standardization.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/networking/libp2p-standardization.md b/specs/networking/libp2p-standardization.md index b6a46db1b..d1ba07e65 100644 --- a/specs/networking/libp2p-standardization.md +++ b/specs/networking/libp2p-standardization.md @@ -150,9 +150,9 @@ The [RPC Interface](./rpc-interface.md) is specified in this repository. ## Discovery Discovery Version 5 -([discv5])(https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md) +([discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md)) will be used for discovery. This protocol uses a UDP transport and specifies its own encryption, ip-discovery and topic advertisement. Therefore, it has no -need to establish establish streams through `multistream-select`, rather, act +need to establish streams through `multistream-select`, rather, act as a standalone implementation that feeds discovered peers/topics (ENR-records) as `multiaddrs` into the libp2p service. From 28b76bcd4c84a93a51d87207a5a825496809c4c6 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 24 May 2019 11:59:22 -0600 Subject: [PATCH 71/84] a number of minor fixes in phsae 1 --- specs/core/1_shard-data-chains.md | 48 +++++++++++++++---------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index e1092b404..1d1186247 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -215,7 +215,7 @@ def get_shard_header(block: ShardBlock) -> ShardBlockHeader: def verify_shard_attestation_signature(state: BeaconState, attestation: ShardAttestation) -> None: data = attestation.data - persistent_committee = get_persistent_committee(state, data.crosslink.shard, data.slot) + persistent_committee = get_persistent_committee(state, data.shard, data.slot) assert verify_bitfield(attestation.aggregation_bitfield, len(persistent_committee)) pubkeys = [] for i, index in enumerate(persistent_committee): @@ -225,7 +225,7 @@ def verify_shard_attestation_signature(state: BeaconState, pubkeys.append(validator.pubkey) assert bls_verify( pubkey=bls_aggregate_pubkeys(pubkeys), - message_hash=data.crosslink.shard_block_root, + message_hash=data.shard_block_root, signature=attestation.aggregate_signature, domain=get_domain(state, slot_to_epoch(data.slot), DOMAIN_SHARD_ATTESTER) ) @@ -280,22 +280,22 @@ def is_valid_shard_block(beacon_blocks: List[BeaconBlock], return True # Check slot number - assert block.slot >= PHASE_1_GENESIS_SLOT + assert candidate.slot >= PHASE_1_GENESIS_SLOT assert unix_time >= beacon_state.genesis_time + (block.slot - GENESIS_SLOT) * SECONDS_PER_SLOT # Check shard number - assert block.shard <= SHARD_COUNT + assert candidate.shard <= SHARD_COUNT # Check beacon block - beacon_block = beacon_blocks[block.slot] - assert block.beacon_block_root == signing_root(beacon_block) - assert beacon_block.slot <= block.slot: + beacon_block = beacon_blocks[candidate.slot] + assert candidate.beacon_block_root == signing_root(beacon_block) + assert beacon_block.slot <= candidate.slot: # Check state root - assert block.state_root == ZERO_HASH # [to be removed in phase 2] + assert candidate.state_root == ZERO_HASH # [to be removed in phase 2] # Check parent block - if block.slot == PHASE_1_GENESIS_SLOT: + if candidate.slot == PHASE_1_GENESIS_SLOT: assert candidate.parent_root == ZERO_HASH else: parent_block = next( @@ -303,26 +303,26 @@ def is_valid_shard_block(beacon_blocks: List[BeaconBlock], signing_root(block) == candidate.parent_root , None) assert parent_block != None - assert parent_block.shard == block.shard - assert parent_block.slot < block.slot + assert parent_block.shard == candidate.shard + assert parent_block.slot < candidate.slot assert signing_root(beacon_blocks[parent_block.slot]) == parent_block.beacon_chain_root # Check attestations - assert len(block.attestations) <= MAX_SHARD_ATTESTIONS - for _, attestation in enumerate(block.attestations): - assert max(GENESIS_SHARD_SLOT, block.slot - SLOTS_PER_EPOCH) <= attestation.data.slot - assert attestation.data.slot <= block.slot - MIN_ATTESTATION_INCLUSION_DELAY - assert attestation.data.crosslink.shard == block.shard + assert len(candidate.attestations) <= MAX_SHARD_ATTESTIONS + for _, attestation in enumerate(candidate.attestations): + assert max(GENESIS_SHARD_SLOT, candidate.slot - SLOTS_PER_EPOCH) <= attestation.data.slot + assert attestation.data.slot <= candidate.slot - MIN_ATTESTATION_INCLUSION_DELAY + assert attestation.data.crosslink.shard == candidate.shard verify_shard_attestation_signature(beacon_state, attestation) # Check signature - proposer_index = get_shard_proposer_index(beacon_state, block.shard, block.slot) + proposer_index = get_shard_proposer_index(beacon_state, candidate.shard, candidate.slot) assert proposer_index is not None assert bls_verify( pubkey=validators[proposer_index].pubkey, message_hash=signing_root(block), - signature=block.signature, - domain=get_domain(beacon_state, slot_to_epoch(block.slot), DOMAIN_SHARD_PROPOSER) + signature=candidate.signature, + domain=get_domain(beacon_state, slot_to_epoch(candidate.slot), DOMAIN_SHARD_PROPOSER) ) return True @@ -339,18 +339,18 @@ Let: ```python def is_valid_shard_attestation(valid_shard_blocks: List[ShardBlock], beacon_state: BeaconState, - candidate: Attestation) -> bool: + candidate: ShardAttestation) -> bool: # Check shard block shard_block = next( block for block in valid_shard_blocks if - signing_root(block) == candidate.attestation.data.crosslink.shard_block_root + signing_root(block) == candidate.data.shard_block_root , None) assert shard_block != None - assert shard_block.slot == attestation.data.slot - assert shard_block.shard == attestation.data.crosslink.shard + assert shard_block.slot == candidate.data.slot + assert shard_block.shard == candidate.data.shard # Check signature - verify_shard_attestation_signature(beacon_state, attestation) + verify_shard_attestation_signature(beacon_state, candidate) return True ``` From a82a6f9a1a72a701aa9669ea6bf9141e49f3c769 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 27 May 2019 17:17:20 +1000 Subject: [PATCH 72/84] Sort attester slashing indices to avoid arbitrary ordering Closes #1125 --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 60f774e9a..0678de9d2 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1707,7 +1707,7 @@ def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSla slashed_any = False attesting_indices_1 = attestation_1.custody_bit_0_indices + attestation_1.custody_bit_1_indices attesting_indices_2 = attestation_2.custody_bit_0_indices + attestation_2.custody_bit_1_indices - for index in set(attesting_indices_1).intersection(attesting_indices_2): + for index in sorted(set(attesting_indices_1).intersection(attesting_indices_2)): if is_slashable_validator(state.validator_registry[index], get_current_epoch(state)): slash_validator(state, index) slashed_any = True From d63b553a2da3f5293d7202183c18cfdf28816b7f Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 21:45:47 +0200 Subject: [PATCH 73/84] efficiency bugfix in bytes encoding, improve typing doc + bugfix --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 4 ++-- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 655cd9da4..46d842e16 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -14,7 +14,7 @@ def is_basic_type(typ): def serialize_basic(value, typ): if is_uint(typ): return value.to_bytes(uint_byte_size(typ), 'little') - if issubclass(typ, bool): + if is_bool_type(typ): if value: return b'\x01' else: @@ -39,7 +39,7 @@ def serialize(obj, typ): if is_basic_type(typ): return serialize_basic(obj, typ) elif is_list_type(typ) or is_vector_type(typ): - return encode_series(list(obj), [read_elem_typ(typ)]*len(obj)) + return encode_series(obj, [read_elem_typ(typ)]*len(obj)) elif is_container_typ(typ): return encode_series(obj.get_field_values(), typ.get_field_types()) else: diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 1d8aed0de..93a272c8c 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -376,12 +376,20 @@ def infer_input_type(fn): return fn(obj, typ) return infer_helper +def is_bool_type(typ): + return issubclass(typ, bool) def is_list_type(typ): + """ + Checks if the given type is a kind of list. Can be bytes. + """ return (hasattr(typ, '_name') and typ._name == 'List') or typ == bytes def is_vector_type(typ): - return issubclass(typ, Vector) + """ + Checks if the given type is a kind of vector. Can be BytesN. + """ + return issubclass(typ, Vector) or issubclass(typ, BytesN) def is_container_typ(typ): return issubclass(typ, Container) From 87b3466eae90ebe557899083ca50e3babc29bb6f Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 21:46:14 +0200 Subject: [PATCH 74/84] update encoder and decoder for reading from parsed data --- test_libs/pyspec/eth2spec/debug/decode.py | 45 +++++++++++++---------- test_libs/pyspec/eth2spec/debug/encode.py | 25 +++++++------ 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/decode.py b/test_libs/pyspec/eth2spec/debug/decode.py index e9aa8bc2b..261692bed 100644 --- a/test_libs/pyspec/eth2spec/debug/decode.py +++ b/test_libs/pyspec/eth2spec/debug/decode.py @@ -1,28 +1,35 @@ -from eth2spec.utils.minimal_ssz import hash_tree_root +from eth2spec.utils.ssz.ssz_impl import hash_tree_root +from eth2spec.utils.ssz.ssz_typing import * -def decode(json, typ): - if isinstance(typ, str) and typ[:4] == 'uint': - return json - elif typ == 'bool': - assert json in (True, False) - return json - elif isinstance(typ, list): - return [decode(element, typ[0]) for element in json] - elif isinstance(typ, str) and typ[:4] == 'byte': - return bytes.fromhex(json[2:]) - elif hasattr(typ, 'fields'): +def decode(data, typ): + if is_uint(typ): + return data + elif is_bool_type(typ): + assert data in (True, False) + return data + elif issubclass(typ, list): + elem_typ = read_list_elem_typ(typ) + return [decode(element, elem_typ) for element in data] + elif issubclass(typ, Vector): + elem_typ = read_vector_elem_typ(typ) + return Vector(decode(element, elem_typ) for element in data) + elif issubclass(typ, bytes): + return bytes.fromhex(data[2:]) + elif issubclass(typ, BytesN): + return BytesN(bytes.fromhex(data[2:])) + elif is_container_typ(typ): temp = {} - for field, subtype in typ.fields.items(): - temp[field] = decode(json[field], subtype) - if field + "_hash_tree_root" in json: - assert(json[field + "_hash_tree_root"][2:] == + for field, subtype in typ.get_fields(): + temp[field] = decode(data[field], subtype) + if field + "_hash_tree_root" in data: + assert(data[field + "_hash_tree_root"][2:] == hash_tree_root(temp[field], subtype).hex()) ret = typ(**temp) - if "hash_tree_root" in json: - assert(json["hash_tree_root"][2:] == + if "hash_tree_root" in data: + assert(data["hash_tree_root"][2:] == hash_tree_root(ret, typ).hex()) return ret else: - print(json, typ) + print(data, typ) raise Exception("Type not recognized") diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index b38e5fe98..3c0658c8f 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -1,24 +1,27 @@ -from eth2spec.utils.minimal_ssz import hash_tree_root +from eth2spec.utils.ssz.ssz_impl import hash_tree_root +from eth2spec.utils.ssz.ssz_typing import * def encode(value, typ, include_hash_tree_roots=False): - if isinstance(typ, str) and typ[:4] == 'uint': - if typ[4:] == '128' or typ[4:] == '256': + if is_uint(typ): + if issubclass(typ, uint) and typ.byte_len > 8: return str(value) return value - elif typ == 'bool': + elif is_bool_type(typ): assert value in (True, False) return value - elif isinstance(typ, list): - return [encode(element, typ[0], include_hash_tree_roots) for element in value] - elif isinstance(typ, str) and typ[:4] == 'byte': + elif issubclass(typ, list) or issubclass(typ, Vector): + elem_typ = read_elem_typ(typ) + return [encode(element, elem_typ, include_hash_tree_roots) for element in value] + elif issubclass(typ, bytes): return '0x' + value.hex() - elif hasattr(typ, 'fields'): + elif is_container_typ(typ): ret = {} - for field, subtype in typ.fields.items(): - ret[field] = encode(getattr(value, field), subtype, include_hash_tree_roots) + for field, subtype in typ.get_fields(): + field_value = getattr(value, field) + ret[field] = encode(field_value, subtype, include_hash_tree_roots) if include_hash_tree_roots: - ret[field + "_hash_tree_root"] = '0x' + hash_tree_root(getattr(value, field), subtype).hex() + ret[field + "_hash_tree_root"] = '0x' + hash_tree_root(field_value, subtype).hex() if include_hash_tree_roots: ret["hash_tree_root"] = '0x' + hash_tree_root(value, typ).hex() return ret From c68944bd53cbe8759c233a188f3e58943012e84c Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 22:18:34 +0200 Subject: [PATCH 75/84] separate type (direct) from kinds (alias incl) --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 26 +++++++-------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 33 +++++++++++++++---- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 46d842e16..c3cc579bd 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -8,11 +8,11 @@ BYTES_PER_LENGTH_OFFSET = 4 def is_basic_type(typ): - return is_uint(typ) or typ == bool + return is_uint_type(typ) or is_bool_type(typ) def serialize_basic(value, typ): - if is_uint(typ): + if is_uint_type(typ): return value.to_bytes(uint_byte_size(typ), 'little') if is_bool_type(typ): if value: @@ -24,11 +24,11 @@ def serialize_basic(value, typ): def is_fixed_size(typ): if is_basic_type(typ): return True - elif is_list_type(typ): + elif is_list_kind(typ): return False - elif is_vector_type(typ): - return is_fixed_size(read_vector_elem_typ(typ)) - elif is_container_typ(typ): + elif is_vector_kind(typ): + return is_fixed_size(read_vector_elem_type(typ)) + elif is_container_type(typ): return all(is_fixed_size(t) for t in typ.get_field_types()) else: raise Exception("Type not supported: {}".format(typ)) @@ -38,9 +38,9 @@ def is_fixed_size(typ): def serialize(obj, typ): if is_basic_type(typ): return serialize_basic(obj, typ) - elif is_list_type(typ) or is_vector_type(typ): - return encode_series(obj, [read_elem_typ(typ)]*len(obj)) - elif is_container_typ(typ): + elif is_list_kind(typ) or is_vector_kind(typ): + return encode_series(obj, [read_elem_type(typ)]*len(obj)) + elif is_container_type(typ): return encode_series(obj.get_field_values(), typ.get_field_types()) else: raise Exception("Type not supported: {}".format(typ)) @@ -103,15 +103,15 @@ def mix_in_length(root, length): def hash_tree_root(obj, typ): if is_basic_type(typ): return merkleize_chunks(chunkify(serialize_basic(obj, typ))) - elif is_list_type(typ) or is_vector_type(typ): - subtype = read_elem_typ(typ) + elif is_list_kind(typ) or is_vector_kind(typ): + subtype = read_elem_type(typ) if is_basic_type(subtype): leaves = chunkify(pack(obj, subtype)) else: leaves = [hash_tree_root(elem, subtype) for elem in obj] leaf_root = merkleize_chunks(leaves) return mix_in_length(leaf_root, len(obj)) if is_list_type(typ) else leaf_root - elif is_container_typ(typ): + elif is_container_type(typ): leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields()] return merkleize_chunks(chunkify(b''.join(leaves))) else: @@ -120,7 +120,7 @@ def hash_tree_root(obj, typ): @infer_input_type def signing_root(obj, typ): - assert is_container_typ(typ) + assert is_container_type(typ) leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields()[:-1]] return merkleize_chunks(chunkify(b''.join(leaves))) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 93a272c8c..d16c66abb 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -61,7 +61,7 @@ class uint256(uint): return super().__new__(cls, value) -def is_uint(typ): +def is_uint_type(typ): # All integers are uint in the scope of the spec here. # Since we default to uint64. Bounds can be checked elsewhere. return issubclass(typ, int) @@ -380,29 +380,48 @@ def is_bool_type(typ): return issubclass(typ, bool) def is_list_type(typ): + """ + Checks if the given type is a list. + """ + return (hasattr(typ, '_name') and typ._name == 'List') + +def is_bytes_type(typ): + # Do not accept subclasses of bytes here, to avoid confusion with BytesN + return typ == bytes + +def is_list_kind(typ): """ Checks if the given type is a kind of list. Can be bytes. """ - return (hasattr(typ, '_name') and typ._name == 'List') or typ == bytes + return is_list_type(typ) or is_bytes_type(typ) def is_vector_type(typ): + """ + Checks if the given type is a vector. + """ + return issubclass(typ, Vector) + +def is_bytesn_type(typ): + return issubclass(typ, BytesN) + +def is_vector_kind(typ): """ Checks if the given type is a kind of vector. Can be BytesN. """ - return issubclass(typ, Vector) or issubclass(typ, BytesN) + return is_vector_type(typ) or is_bytesn_type(typ) -def is_container_typ(typ): +def is_container_type(typ): return issubclass(typ, Container) -def read_list_elem_typ(list_typ: Type[List[T]]) -> T: +def read_list_elem_type(list_typ: Type[List[T]]) -> T: if list_typ.__args__ is None or len(list_typ.__args__) != 1: raise TypeError("Supplied list-type is invalid, no element type found.") return list_typ.__args__[0] -def read_vector_elem_typ(vector_typ: Type[Vector[T, L]]) -> T: +def read_vector_elem_type(vector_typ: Type[Vector[T, L]]) -> T: return vector_typ.elem_type -def read_elem_typ(typ): +def read_elem_type(typ): if typ == bytes: return byte elif is_list_type(typ): From 0f79ed709bd4846299fb4379ed22d1de47f94e02 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 22:19:18 +0200 Subject: [PATCH 76/84] update yaml encoder/decoder and obj randomizer for typed ssz usage --- test_libs/pyspec/eth2spec/debug/decode.py | 16 +-- test_libs/pyspec/eth2spec/debug/encode.py | 11 +- .../pyspec/eth2spec/debug/random_value.py | 112 +++++++++--------- 3 files changed, 68 insertions(+), 71 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/decode.py b/test_libs/pyspec/eth2spec/debug/decode.py index 261692bed..c9657dc28 100644 --- a/test_libs/pyspec/eth2spec/debug/decode.py +++ b/test_libs/pyspec/eth2spec/debug/decode.py @@ -3,22 +3,22 @@ from eth2spec.utils.ssz.ssz_typing import * def decode(data, typ): - if is_uint(typ): + if is_uint_type(typ): return data elif is_bool_type(typ): assert data in (True, False) return data - elif issubclass(typ, list): - elem_typ = read_list_elem_typ(typ) + elif is_list_type(typ): + elem_typ = read_list_elem_type(typ) return [decode(element, elem_typ) for element in data] - elif issubclass(typ, Vector): - elem_typ = read_vector_elem_typ(typ) + elif is_vector_type(typ): + elem_typ = read_vector_elem_type(typ) return Vector(decode(element, elem_typ) for element in data) - elif issubclass(typ, bytes): + elif is_bytes_type(typ): return bytes.fromhex(data[2:]) - elif issubclass(typ, BytesN): + elif is_bytesn_type(typ): return BytesN(bytes.fromhex(data[2:])) - elif is_container_typ(typ): + elif is_container_type(typ): temp = {} for field, subtype in typ.get_fields(): temp[field] = decode(data[field], subtype) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index 3c0658c8f..832203e35 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -3,19 +3,20 @@ from eth2spec.utils.ssz.ssz_typing import * def encode(value, typ, include_hash_tree_roots=False): - if is_uint(typ): + if is_uint_type(typ): + # Larger uints are boxed and the class declares their byte length if issubclass(typ, uint) and typ.byte_len > 8: return str(value) return value elif is_bool_type(typ): assert value in (True, False) return value - elif issubclass(typ, list) or issubclass(typ, Vector): - elem_typ = read_elem_typ(typ) + elif is_list_type(typ) or is_vector_type(typ): + elem_typ = read_elem_type(typ) return [encode(element, elem_typ, include_hash_tree_roots) for element in value] - elif issubclass(typ, bytes): + elif issubclass(typ, bytes): # both bytes and BytesN return '0x' + value.hex() - elif is_container_typ(typ): + elif is_container_type(typ): ret = {} for field, subtype in typ.get_fields(): field_value = getattr(value, field) diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index f28181943..5abd73086 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -2,10 +2,11 @@ from random import Random from typing import Any from enum import Enum +from eth2spec.utils.ssz.ssz_typing import * +from eth2spec.utils.ssz.ssz_impl import is_basic_type -UINT_SIZES = [8, 16, 32, 64, 128, 256] - -basic_types = ["uint%d" % v for v in UINT_SIZES] + ['bool', 'byte'] +# in bytes +UINT_SIZES = [1, 2, 4, 8, 16, 32] random_mode_names = ["random", "zero", "max", "nil", "one", "lengthy"] @@ -49,60 +50,61 @@ def get_random_ssz_object(rng: Random, """ if chaos: mode = rng.choice(list(RandomizationMode)) - if isinstance(typ, str): - # Bytes array - if typ == 'bytes': - if mode == RandomizationMode.mode_nil_count: - return b'' - if mode == RandomizationMode.mode_max_count: - return get_random_bytes_list(rng, max_bytes_length) - if mode == RandomizationMode.mode_one_count: - return get_random_bytes_list(rng, 1) - if mode == RandomizationMode.mode_zero: - return b'\x00' - if mode == RandomizationMode.mode_max: - return b'\xff' - return get_random_bytes_list(rng, rng.randint(0, max_bytes_length)) - elif typ[:5] == 'bytes' and len(typ) > 5: - length = int(typ[5:]) - # Sanity, don't generate absurdly big random values - # If a client is aiming to performance-test, they should create a benchmark suite. - assert length <= max_bytes_length - if mode == RandomizationMode.mode_zero: - return b'\x00' * length - if mode == RandomizationMode.mode_max: - return b'\xff' * length - return get_random_bytes_list(rng, length) - # Basic types - else: - if mode == RandomizationMode.mode_zero: - return get_min_basic_value(typ) - if mode == RandomizationMode.mode_max: - return get_max_basic_value(typ) - return get_random_basic_value(rng, typ) + # Bytes array + if is_bytes_type(typ): + if mode == RandomizationMode.mode_nil_count: + return b'' + if mode == RandomizationMode.mode_max_count: + return get_random_bytes_list(rng, max_bytes_length) + if mode == RandomizationMode.mode_one_count: + return get_random_bytes_list(rng, 1) + if mode == RandomizationMode.mode_zero: + return b'\x00' + if mode == RandomizationMode.mode_max: + return b'\xff' + return get_random_bytes_list(rng, rng.randint(0, max_bytes_length)) + elif is_bytesn_type(typ): + length = typ.length + # Sanity, don't generate absurdly big random values + # If a client is aiming to performance-test, they should create a benchmark suite. + assert length <= max_bytes_length + if mode == RandomizationMode.mode_zero: + return b'\x00' * length + if mode == RandomizationMode.mode_max: + return b'\xff' * length + return get_random_bytes_list(rng, length) + # Basic types + elif is_basic_type(typ): + if mode == RandomizationMode.mode_zero: + return get_min_basic_value(typ) + if mode == RandomizationMode.mode_max: + return get_max_basic_value(typ) + return get_random_basic_value(rng, typ) # Vector: - elif isinstance(typ, list) and len(typ) == 2: + elif is_vector_type(typ): + elem_typ = read_vector_elem_type(typ) return [ - get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode, chaos) - for _ in range(typ[1]) + get_random_ssz_object(rng, elem_typ, max_bytes_length, max_list_length, mode, chaos) + for _ in range(typ.length) ] # List: - elif isinstance(typ, list) and len(typ) == 1: + elif is_list_type(typ): + elem_typ = read_list_elem_type(typ) length = rng.randint(0, max_list_length) if mode == RandomizationMode.mode_one_count: length = 1 if mode == RandomizationMode.mode_max_count: length = max_list_length return [ - get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode, chaos) + get_random_ssz_object(rng, elem_typ, max_bytes_length, max_list_length, mode, chaos) for _ in range(length) ] # Container: - elif hasattr(typ, 'fields'): + elif is_container_type(typ): return typ(**{ field: get_random_ssz_object(rng, subtype, max_bytes_length, max_list_length, mode, chaos) - for field, subtype in typ.fields.items() + for field, subtype in typ.get_fields() }) else: print(typ) @@ -114,39 +116,33 @@ def get_random_bytes_list(rng: Random, length: int) -> bytes: def get_random_basic_value(rng: Random, typ: str) -> Any: - if typ == 'bool': + if is_bool_type(typ): return rng.choice((True, False)) - if typ[:4] == 'uint': - size = int(typ[4:]) + if is_uint_type(typ): + size = uint_byte_size(typ) assert size in UINT_SIZES - return rng.randint(0, 2**size - 1) - if typ == 'byte': - return rng.randint(0, 8) + return rng.randint(0, 256**size - 1) else: raise ValueError("Not a basic type") def get_min_basic_value(typ: str) -> Any: - if typ == 'bool': + if is_bool_type(typ): return False - if typ[:4] == 'uint': - size = int(typ[4:]) + if is_uint_type(typ): + size = uint_byte_size(typ) assert size in UINT_SIZES return 0 - if typ == 'byte': - return 0x00 else: raise ValueError("Not a basic type") def get_max_basic_value(typ: str) -> Any: - if typ == 'bool': + if is_bool_type(typ): return True - if typ[:4] == 'uint': - size = int(typ[4:]) + if is_uint_type(typ): + size = uint_byte_size(typ) assert size in UINT_SIZES - return 2**size - 1 - if typ == 'byte': - return 0xff + return 256**size - 1 else: raise ValueError("Not a basic type") From d023d2d20f06474f7322d113049e3365a733a74a Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 23:40:05 +0200 Subject: [PATCH 77/84] lots of bugfixes --- test_libs/pyspec/eth2spec/debug/encode.py | 4 +- .../pyspec/eth2spec/debug/random_value.py | 6 +- .../pyspec/eth2spec/utils/merkle_minimal.py | 7 +- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 5 +- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 90 ++++++++++++++----- 5 files changed, 81 insertions(+), 31 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index 832203e35..a3c3c1189 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -4,6 +4,8 @@ from eth2spec.utils.ssz.ssz_typing import * def encode(value, typ, include_hash_tree_roots=False): if is_uint_type(typ): + if hasattr(typ, '__supertype__'): + typ = typ.__supertype__ # Larger uints are boxed and the class declares their byte length if issubclass(typ, uint) and typ.byte_len > 8: return str(value) @@ -14,7 +16,7 @@ def encode(value, typ, include_hash_tree_roots=False): elif is_list_type(typ) or is_vector_type(typ): elem_typ = read_elem_type(typ) return [encode(element, elem_typ, include_hash_tree_roots) for element in value] - elif issubclass(typ, bytes): # both bytes and BytesN + elif isinstance(typ, type) and issubclass(typ, bytes): # both bytes and BytesN return '0x' + value.hex() elif is_container_type(typ): ret = {} diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index 5abd73086..ab9c4c885 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -115,7 +115,7 @@ def get_random_bytes_list(rng: Random, length: int) -> bytes: return bytes(rng.getrandbits(8) for _ in range(length)) -def get_random_basic_value(rng: Random, typ: str) -> Any: +def get_random_basic_value(rng: Random, typ) -> Any: if is_bool_type(typ): return rng.choice((True, False)) if is_uint_type(typ): @@ -126,7 +126,7 @@ def get_random_basic_value(rng: Random, typ: str) -> Any: raise ValueError("Not a basic type") -def get_min_basic_value(typ: str) -> Any: +def get_min_basic_value(typ) -> Any: if is_bool_type(typ): return False if is_uint_type(typ): @@ -137,7 +137,7 @@ def get_min_basic_value(typ: str) -> Any: raise ValueError("Not a basic type") -def get_max_basic_value(typ: str) -> Any: +def get_max_basic_value(typ) -> Any: if is_bool_type(typ): return True if is_uint_type(typ): diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py index e3e5d35d8..420f0b5f1 100644 --- a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py @@ -34,10 +34,13 @@ def get_merkle_proof(tree, item_index): def next_power_of_two(v: int) -> int: """ - Get the next power of 2. (for 64 bit range ints) + Get the next power of 2. (for 64 bit range ints). + 0 is a special case, to have non-empty defaults. Examples: - 0 -> 0, 1 -> 1, 2 -> 2, 3 -> 4, 32 -> 32, 33 -> 64 + 0 -> 1, 1 -> 1, 2 -> 2, 3 -> 4, 32 -> 32, 33 -> 64 """ + if v == 0: + return 1 # effectively fill the bitstring (1 less, do not want to with ones, then increment for next power of 2. v -= 1 v |= v >> (1 << 0) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index c3cc579bd..826714c96 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -112,7 +112,7 @@ def hash_tree_root(obj, typ): leaf_root = merkleize_chunks(leaves) return mix_in_length(leaf_root, len(obj)) if is_list_type(typ) else leaf_root elif is_container_type(typ): - leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields()] + leaves = [hash_tree_root(field_value, field_typ) for field_value, field_typ in obj.get_typed_values()] return merkleize_chunks(chunkify(b''.join(leaves))) else: raise Exception("Type not supported: obj {} type {}".format(obj, typ)) @@ -121,6 +121,7 @@ def hash_tree_root(obj, typ): @infer_input_type def signing_root(obj, typ): assert is_container_type(typ) - leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields()[:-1]] + # ignore last field + leaves = [hash_tree_root(field_value, field_typ) for field_value, field_typ in obj.get_typed_values()[:-1]] return merkleize_chunks(chunkify(b''.join(leaves))) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index d16c66abb..0121ded9e 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,6 +1,6 @@ -from typing import List, Iterable, Type, NewType -from typing import Union from inspect import isclass +from typing import List, Iterable, TypeVar, Type, NewType +from typing import Union # SSZ integers @@ -64,17 +64,25 @@ class uint256(uint): def is_uint_type(typ): # All integers are uint in the scope of the spec here. # Since we default to uint64. Bounds can be checked elsewhere. - return issubclass(typ, int) + + # However, some are wrapped in a NewType + if hasattr(typ, '__supertype__'): + # get the type that the NewType is wrapping + typ = typ.__supertype__ + + return isinstance(typ, type) and issubclass(typ, int) def uint_byte_size(typ): - if issubclass(typ, uint): - return typ.byte_len - elif issubclass(typ, int): - # Default to uint64 - return 8 - else: - raise TypeError("Type %s is not an uint (or int-default uint64) type" % typ) + if hasattr(typ, '__supertype__'): + typ = typ.__supertype__ + if isinstance(typ, type): + if issubclass(typ, uint): + return typ.byte_len + elif issubclass(typ, int): + # Default to uint64 + return 8 + raise TypeError("Type %s is not an uint (or int-default uint64) type" % typ) # SSZ Container base class @@ -86,7 +94,7 @@ class Container(object): def __init__(self, **kwargs): cls = self.__class__ - for f, t in cls.get_fields().items(): + for f, t in cls.get_fields(): if f not in kwargs: setattr(self, f, get_zero_value(t)) else: @@ -117,7 +125,10 @@ class Container(object): @classmethod def get_fields(cls): - return dict(cls.__annotations__).items() + return list(dict(cls.__annotations__).items()) + + def get_typed_values(self): + return list(zip(self.get_field_values(), self.get_field_types())) @classmethod def get_field_names(cls): @@ -134,6 +145,9 @@ class Container(object): def _is_vector_instance_of(a, b): + # Other must not be a BytesN + if issubclass(b, bytes): + return False if not hasattr(b, 'elem_type') or not hasattr(b, 'length'): # Vector (b) is not an instance of Vector[X, Y] (a) return False @@ -146,6 +160,9 @@ def _is_vector_instance_of(a, b): def _is_equal_vector_type(a, b): + # Other must not be a BytesN + if issubclass(b, bytes): + return False if not hasattr(a, 'elem_type') or not hasattr(a, 'length'): if not hasattr(b, 'elem_type') or not hasattr(b, 'length'): # Vector == Vector @@ -237,6 +254,9 @@ class Vector(metaclass=VectorMeta): def _is_bytes_n_instance_of(a, b): + # Other has to be a Bytes derivative class to be a BytesN + if not issubclass(b, bytes): + return False if not hasattr(b, 'length'): # BytesN (b) is not an instance of BytesN[X] (a) return False @@ -249,6 +269,9 @@ def _is_bytes_n_instance_of(a, b): def _is_equal_bytes_n_type(a, b): + # Other has to be a Bytes derivative class to be a BytesN + if not issubclass(b, bytes): + return False if not hasattr(a, 'length'): if not hasattr(b, 'length'): # BytesN == BytesN @@ -267,7 +290,7 @@ class BytesNMeta(type): out = type.__new__(cls, class_name, parents, attrs) if 'length' in attrs: setattr(out, 'length', attrs['length']) - out._name = 'Vector' + out._name = 'BytesN' out.elem_type = byte return out @@ -318,7 +341,7 @@ class BytesN(bytes, metaclass=BytesNMeta): else: bytesval = b'\x00' * cls.length if len(bytesval) != cls.length: - raise TypeError("bytesN[%d] cannot be initialized with value of %d bytes" % (cls.length, len(bytesval))) + raise TypeError("BytesN[%d] cannot be initialized with value of %d bytes" % (cls.length, len(bytesval))) return super().__new__(cls, bytesval) def serialize(self): @@ -334,7 +357,7 @@ class BytesN(bytes, metaclass=BytesNMeta): # ----------------------------- def get_zero_value(typ): - if is_uint(typ): + if is_uint_type(typ): return 0 if issubclass(typ, bool): return False @@ -354,7 +377,7 @@ def get_zero_value(typ): # ----------------------------- def infer_type(obj): - if is_uint(obj.__class__): + if is_uint_type(obj.__class__): return obj.__class__ elif isinstance(obj, int): return uint64 @@ -370,39 +393,50 @@ def infer_input_type(fn): """ Decorator to run infer_type on the obj if typ argument is None """ + def infer_helper(obj, typ=None): if typ is None: typ = infer_type(obj) return fn(obj, typ) + return infer_helper + def is_bool_type(typ): - return issubclass(typ, bool) + if hasattr(typ, '__supertype__'): + typ = typ.__supertype__ + return isinstance(typ, type) and issubclass(typ, bool) + def is_list_type(typ): """ Checks if the given type is a list. """ - return (hasattr(typ, '_name') and typ._name == 'List') + return hasattr(typ, '_name') and typ._name == 'List' + def is_bytes_type(typ): # Do not accept subclasses of bytes here, to avoid confusion with BytesN return typ == bytes + def is_list_kind(typ): """ Checks if the given type is a kind of list. Can be bytes. """ return is_list_type(typ) or is_bytes_type(typ) + def is_vector_type(typ): """ Checks if the given type is a vector. """ - return issubclass(typ, Vector) + return isinstance(typ, type) and issubclass(typ, Vector) + def is_bytesn_type(typ): - return issubclass(typ, BytesN) + return isinstance(typ, type) and issubclass(typ, BytesN) + def is_vector_kind(typ): """ @@ -410,23 +444,33 @@ def is_vector_kind(typ): """ return is_vector_type(typ) or is_bytesn_type(typ) + def is_container_type(typ): - return issubclass(typ, Container) + return isinstance(typ, type) and issubclass(typ, Container) + + +T = TypeVar('T') +L = TypeVar('L') + def read_list_elem_type(list_typ: Type[List[T]]) -> T: if list_typ.__args__ is None or len(list_typ.__args__) != 1: raise TypeError("Supplied list-type is invalid, no element type found.") return list_typ.__args__[0] + def read_vector_elem_type(vector_typ: Type[Vector[T, L]]) -> T: return vector_typ.elem_type + def read_elem_type(typ): if typ == bytes: return byte elif is_list_type(typ): - return read_list_elem_typ(typ) + return read_list_elem_type(typ) elif is_vector_type(typ): - return read_vector_elem_typ(typ) + return read_vector_elem_type(typ) + elif issubclass(typ, bytes): + return byte else: raise TypeError("Unexpected type: {}".format(typ)) From b4c4df6a09e6feff9f1bdc1595f9703e836bf233 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 23:41:29 +0200 Subject: [PATCH 78/84] bugfix in sss_types global building in script --- scripts/phase0/function_puller.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index 0047d5321..d7765a4a4 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -70,10 +70,10 @@ def get_spec(file_name: str) -> List[str]: code_lines.append(' ' + type_line) code_lines.append('\n') for (ssz_type_name, _) in type_defs: - code_lines.append(f' global_vars["{ssz_type_name}"] = {ssz_type_name},') + code_lines.append(f' global_vars["{ssz_type_name}"] = {ssz_type_name}') code_lines.append(' global_vars["ssz_types"] = [') for (ssz_type_name, _) in type_defs: - code_lines.append(f' {ssz_type_name},') + code_lines.append(f' "{ssz_type_name}",') code_lines.append(' ]') code_lines.append('\n') code_lines.append('def get_ssz_type_by_name(name: str) -> Container:') From 54b14b5ac305ecb3fcc25086b6a63c81dafc61a9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 23:41:48 +0200 Subject: [PATCH 79/84] update ssz-static generator --- test_generators/ssz_static/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/ssz_static/main.py b/test_generators/ssz_static/main.py index 1234294db..abb167613 100644 --- a/test_generators/ssz_static/main.py +++ b/test_generators/ssz_static/main.py @@ -2,7 +2,7 @@ from random import Random from eth2spec.debug import random_value, encode from eth2spec.phase0 import spec -from eth2spec.utils.minimal_ssz import ( +from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, signing_root, serialize, From c99fa52d9fdfe5f4bc2fe6b043f8106a0feeaf80 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 23:56:17 +0200 Subject: [PATCH 80/84] fix dev branch build script, missing quotes --- scripts/phase0/function_puller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index 750f19590..e54df3ef0 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -66,7 +66,7 @@ def get_spec(file_name: str) -> List[str]: code_lines.append('\n') code_lines.append('ssz_types = [\n') for (ssz_type_name, _) in type_defs: - code_lines.append(f' {ssz_type_name},\n') + code_lines.append(f' "{ssz_type_name}",\n') code_lines.append(']') code_lines.append('\n') code_lines.append('def get_ssz_type_by_name(name: str) -> SSZType:') From 9e61cc2ed3124c2118b820543f5424d680153854 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 28 May 2019 00:45:13 +0200 Subject: [PATCH 81/84] bugfix: typing edge-case: python bool is subclass of int. --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 0121ded9e..326dcb748 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -70,7 +70,7 @@ def is_uint_type(typ): # get the type that the NewType is wrapping typ = typ.__supertype__ - return isinstance(typ, type) and issubclass(typ, int) + return isinstance(typ, type) and issubclass(typ, int) and not issubclass(typ, bool) def uint_byte_size(typ): From 5e28adf5561172cde56b5af09e8de481ff9fbd0c Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 28 May 2019 00:51:27 +0200 Subject: [PATCH 82/84] bugfix: don't forget about var-length bytes getting a length mixin --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 826714c96..96eaff481 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -110,7 +110,7 @@ def hash_tree_root(obj, typ): else: leaves = [hash_tree_root(elem, subtype) for elem in obj] leaf_root = merkleize_chunks(leaves) - return mix_in_length(leaf_root, len(obj)) if is_list_type(typ) else leaf_root + return mix_in_length(leaf_root, len(obj)) if is_list_kind(typ) else leaf_root elif is_container_type(typ): leaves = [hash_tree_root(field_value, field_typ) for field_value, field_typ in obj.get_typed_values()] return merkleize_chunks(chunkify(b''.join(leaves))) From 00ffaf4d71e6f26667f37616418618cef7fc0559 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 28 May 2019 12:37:41 +0800 Subject: [PATCH 83/84] Fix ssz path --- test_generators/operations/deposits.py | 2 +- test_libs/pyspec/tests/helpers.py | 2 +- test_libs/pyspec/tests/test_sanity.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test_generators/operations/deposits.py b/test_generators/operations/deposits.py index 075ccbd5b..297a27859 100644 --- a/test_generators/operations/deposits.py +++ b/test_generators/operations/deposits.py @@ -5,7 +5,7 @@ from eth_utils import ( from gen_base import gen_suite, gen_typing from preset_loader import loader from eth2spec.debug.encode import encode -from eth2spec.utils.minimal_ssz import signing_root +from eth2spec.utils.ssz.ssz_impl import signing_root from eth2spec.utils.merkle_minimal import get_merkle_root, calc_merkle_tree_from_leaves, get_merkle_proof from typing import List, Tuple diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index a4849bfbb..ca40bf1d8 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -3,7 +3,7 @@ from copy import deepcopy from py_ecc import bls import eth2spec.phase0.spec as spec -from eth2spec.utils.minimal_ssz import signing_root +from eth2spec.utils.ssz.ssz_impl import signing_root from eth2spec.phase0.spec import ( # constants ZERO_HASH, diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index 1c05e6b53..b4d7a8e8b 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -5,7 +5,7 @@ import pytest from py_ecc import bls import eth2spec.phase0.spec as spec -from eth2spec.utils.minimal_ssz import signing_root +from eth2spec.utils.ssz.ssz_impl import signing_root from eth2spec.phase0.spec import ( # constants ZERO_HASH, From f0ceefc36d8d54670be1b8e48a0b5128d0d6208b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 28 May 2019 13:57:42 +0800 Subject: [PATCH 84/84] Make `test_libs/pyspec/tests` pass 1. Use `typing_inspect` library to check if it's `typing.List[Any]`. Use it in `is_list_type`. 2. Add `__hash__`, `__eq__`, `__repr__` methods to some classes. --- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 56 +++++++++++++------ test_libs/pyspec/requirements.txt | 1 + 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 326dcb748..3662c52b5 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,6 +1,7 @@ from inspect import isclass from typing import List, Iterable, TypeVar, Type, NewType from typing import Union +from typing_inspect import get_origin # SSZ integers @@ -119,6 +120,18 @@ class Container(object): def __repr__(self): return {field: getattr(self, field) for field in self.get_field_names()} + def __str__(self): + output = [] + for field in self.get_field_names(): + output.append(f'{field}: {getattr(self, field)}') + return "\n".join(output) + + def __eq__(self, other): + return self.hash_tree_root() == other.hash_tree_root() + + def __hash__(self): + return hash(self.hash_tree_root()) + @classmethod def get_fields_dict(cls): return dict(cls.__annotations__) @@ -203,6 +216,9 @@ class VectorMeta(type): def __ne__(self, other): return not _is_equal_vector_type(self, other) + def __hash__(self): + return hash(self.__class__) + class Vector(metaclass=VectorMeta): @@ -252,6 +268,9 @@ class Vector(metaclass=VectorMeta): def __len__(self): return len(self.items) + def __eq__(self, other): + return self.hash_tree_root() == other.hash_tree_root() + def _is_bytes_n_instance_of(a, b): # Other has to be a Bytes derivative class to be a BytesN @@ -309,6 +328,9 @@ class BytesNMeta(type): def __ne__(self, other): return not _is_equal_bytes_n_type(self, other) + def __hash__(self): + return hash(self.__class__) + def parse_bytes(val): if val is None: @@ -355,23 +377,25 @@ class BytesN(bytes, metaclass=BytesNMeta): # SSZ Defaults # ----------------------------- - def get_zero_value(typ): + result = None if is_uint_type(typ): - return 0 - if issubclass(typ, bool): - return False - if issubclass(typ, list): - return [] - if issubclass(typ, Vector): - return typ() - if issubclass(typ, BytesN): - return typ() - if issubclass(typ, bytes): - return b'' - if issubclass(typ, Container): - return typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}), - + result = 0 + elif is_list_type(typ): + result = [] + elif issubclass(typ, bool): + result = False + elif issubclass(typ, Vector): + result = typ() + elif issubclass(typ, BytesN): + result = typ() + elif issubclass(typ, bytes): + result = b'' + elif issubclass(typ, Container): + result = typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}) + else: + return Exception("Type not supported: {}".format(typ)) + return result # Type helpers # ----------------------------- @@ -412,7 +436,7 @@ def is_list_type(typ): """ Checks if the given type is a list. """ - return hasattr(typ, '_name') and typ._name == 'List' + return get_origin(typ) is List or get_origin(typ) is list def is_bytes_type(typ): diff --git a/test_libs/pyspec/requirements.txt b/test_libs/pyspec/requirements.txt index 78d41708d..3b38930bd 100644 --- a/test_libs/pyspec/requirements.txt +++ b/test_libs/pyspec/requirements.txt @@ -2,3 +2,4 @@ eth-utils>=1.3.0,<2 eth-typing>=2.1.0,<3.0.0 pycryptodome==3.7.3 py_ecc>=1.6.0 +typing_inspect==0.4.0