diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 45d5b6894..af446d575 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -116,8 +116,6 @@ MAX_ATTESTATIONS: 128 MAX_DEPOSITS: 16 # 2**4 (= 16) MAX_VOLUNTARY_EXITS: 16 -# Originally 2**4 (= 16), disabled for now. -MAX_TRANSFERS: 0 # Signature domains @@ -127,7 +125,6 @@ DOMAIN_BEACON_ATTESTER: 0x01000000 DOMAIN_RANDAO: 0x02000000 DOMAIN_DEPOSIT: 0x03000000 DOMAIN_VOLUNTARY_EXIT: 0x04000000 -DOMAIN_TRANSFER: 0x05000000 DOMAIN_CUSTODY_BIT_CHALLENGE: 0x06000000 DOMAIN_SHARD_PROPOSER: 0x80000000 DOMAIN_SHARD_ATTESTER: 0x81000000 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 75b93f66a..53599e83a 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -118,8 +118,6 @@ MAX_ATTESTATIONS: 128 MAX_DEPOSITS: 16 # 2**4 (= 16) MAX_VOLUNTARY_EXITS: 16 -# Originally 2**4 (= 16), disabled for now. -MAX_TRANSFERS: 0 # Signature domains @@ -129,7 +127,6 @@ DOMAIN_BEACON_ATTESTER: 0x01000000 DOMAIN_RANDAO: 0x02000000 DOMAIN_DEPOSIT: 0x03000000 DOMAIN_VOLUNTARY_EXIT: 0x04000000 -DOMAIN_TRANSFER: 0x05000000 DOMAIN_CUSTODY_BIT_CHALLENGE: 0x06000000 DOMAIN_SHARD_PROPOSER: 0x80000000 DOMAIN_SHARD_ATTESTER: 0x81000000 diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8d8fd2c15..732cec31e 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -39,7 +39,6 @@ - [`Attestation`](#attestation) - [`Deposit`](#deposit) - [`VoluntaryExit`](#voluntaryexit) - - [`Transfer`](#transfer) - [Beacon blocks](#beacon-blocks) - [`BeaconBlockBody`](#beaconblockbody) - [`BeaconBlock`](#beaconblock) @@ -81,6 +80,7 @@ - [`get_active_validator_indices`](#get_active_validator_indices) - [`get_validator_churn_limit`](#get_validator_churn_limit) - [`get_seed`](#get_seed) + - [`get_committee_count_at_slot`](#get_committee_count_at_slot) - [`get_beacon_committee`](#get_beacon_committee) - [`get_beacon_proposer_index`](#get_beacon_proposer_index) - [`get_total_balance`](#get_total_balance) @@ -114,7 +114,6 @@ - [Attestations](#attestations) - [Deposits](#deposits) - [Voluntary exits](#voluntary-exits) - - [Transfers](#transfers) @@ -241,7 +240,6 @@ The following values are (non-configurable) constants used throughout the specif | `MAX_ATTESTATIONS` | `2**7` (= 128) | | `MAX_DEPOSITS` | `2**4` (= 16) | | `MAX_VOLUNTARY_EXITS` | `2**4` (= 16) | -| `MAX_TRANSFERS` | `0` | ### Domain types @@ -254,7 +252,6 @@ The following types are defined, mapping into `DomainType` (little endian): | `DOMAIN_RANDAO` | `2` | | `DOMAIN_DEPOSIT` | `3` | | `DOMAIN_VOLUNTARY_EXIT` | `4` | -| `DOMAIN_TRANSFER` | `5` | ## Containers @@ -288,14 +285,14 @@ class Checkpoint(Container): ```python class Validator(Container): pubkey: BLSPubkey - withdrawal_credentials: Hash # Commitment to pubkey for withdrawals and transfers + withdrawal_credentials: Hash # Commitment to pubkey for withdrawals effective_balance: Gwei # Balance at stake slashed: boolean # Status epochs activation_eligibility_epoch: Epoch # When criteria for activation were met activation_epoch: Epoch exit_epoch: Epoch - withdrawable_epoch: Epoch # When validator can withdraw or transfer funds + withdrawable_epoch: Epoch # When validator can withdraw funds ``` #### `AttestationData` @@ -423,19 +420,6 @@ class VoluntaryExit(Container): signature: BLSSignature ``` -#### `Transfer` - -```python -class Transfer(Container): - sender: ValidatorIndex - recipient: ValidatorIndex - amount: Gwei - fee: Gwei - slot: Slot # Slot at which transfer must be processed - pubkey: BLSPubkey # Withdrawal pubkey - signature: BLSSignature # Signature checked against withdrawal pubkey -``` - ### Beacon blocks #### `BeaconBlockBody` @@ -451,7 +435,6 @@ class BeaconBlockBody(Container): attestations: List[Attestation, MAX_ATTESTATIONS] deposits: List[Deposit, MAX_DEPOSITS] voluntary_exits: List[VoluntaryExit, MAX_VOLUNTARY_EXITS] - transfers: List[Transfer, MAX_TRANSFERS] ``` #### `BeaconBlock` @@ -1053,6 +1036,7 @@ def initialize_beacon_state_from_eth1(eth1_block_hash: Hash, genesis_time=eth1_timestamp - eth1_timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY, eth1_data=Eth1Data(block_hash=eth1_block_hash, deposit_count=len(deposits)), latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), + randao_mixes=[eth1_block_hash] * EPOCHS_PER_HISTORICAL_VECTOR, # Seed RANDAO with Eth1 entropy ) # Process deposits @@ -1274,9 +1258,7 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence rewards[attestation.proposer_index] += proposer_reward max_attester_reward = get_base_reward(state, index) - proposer_reward rewards[index] += Gwei( - max_attester_reward - * (SLOTS_PER_EPOCH + MIN_ATTESTATION_INCLUSION_DELAY - attestation.inclusion_delay) - // SLOTS_PER_EPOCH + max_attester_reward // attestation.inclusion_delay ) # Inactivity penalty @@ -1435,8 +1417,6 @@ def process_eth1_data(state: BeaconState, body: BeaconBlockBody) -> None: def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: # Verify that outstanding deposits are processed up to the maximum number of deposits assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) - # Verify that there are no duplicate transfers - assert len(body.transfers) == len(set(body.transfers)) for operations, function in ( (body.proposer_slashings, process_proposer_slashing), @@ -1444,7 +1424,6 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: (body.attestations, process_attestation), (body.deposits, process_deposit), (body.voluntary_exits, process_voluntary_exit), - (body.transfers, process_transfer), # @process_shard_receipt_proofs ): for operation in operations: @@ -1583,33 +1562,3 @@ def process_voluntary_exit(state: BeaconState, exit: VoluntaryExit) -> None: # Initiate exit initiate_validator_exit(state, exit.validator_index) ``` - -##### Transfers - -```python -def process_transfer(state: BeaconState, transfer: Transfer) -> None: - # Verify the balance the covers amount and fee (with overflow protection) - assert state.balances[transfer.sender] >= max(transfer.amount + transfer.fee, transfer.amount, transfer.fee) - # A transfer is valid in only one slot - assert state.slot == transfer.slot - # Sender must satisfy at least one of the following: - assert ( - # 1) Never have been eligible for activation - state.validators[transfer.sender].activation_eligibility_epoch == FAR_FUTURE_EPOCH or - # 2) Be withdrawable - get_current_epoch(state) >= state.validators[transfer.sender].withdrawable_epoch or - # 3) Have a balance of at least MAX_EFFECTIVE_BALANCE after the transfer - state.balances[transfer.sender] >= transfer.amount + transfer.fee + MAX_EFFECTIVE_BALANCE - ) - # Verify that the pubkey is valid - assert state.validators[transfer.sender].withdrawal_credentials == BLS_WITHDRAWAL_PREFIX + hash(transfer.pubkey)[1:] - # Verify that the signature is valid - assert bls_verify(transfer.pubkey, signing_root(transfer), transfer.signature, get_domain(state, DOMAIN_TRANSFER)) - # Process the transfer - decrease_balance(state, transfer.sender, transfer.amount + transfer.fee) - increase_balance(state, transfer.recipient, transfer.amount) - increase_balance(state, get_beacon_proposer_index(state), transfer.fee) - # Verify balances are not dust - assert not (0 < state.balances[transfer.sender] < MIN_DEPOSIT_AMOUNT) - assert not (0 < state.balances[transfer.recipient] < MIN_DEPOSIT_AMOUNT) -``` diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 6a8d06d4a..3fdafe3ea 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -9,6 +9,7 @@ - [Configuration](#configuration) - [Containers](#containers) - [`CompactCommittee`](#compactcommittee) + - [`ShardReceiptDelta`](#shardreceiptdelta) - [`ShardReceiptProof`](#shardreceiptproof) - [Helper functions](#helper-functions) - [`pack_compact_validator`](#pack_compact_validator) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 224dfd5f2..67e12a08c 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -336,7 +336,7 @@ def legendre_bit(a: int, q: int) -> int: return 0 ``` -### ```custody_subchunkify``` +### `custody_subchunkify` Given one proof of custody chunk, returns the proof of custody subchunks of the correct sizes. diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 6edc5daba..d9c88e72b 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -18,10 +18,11 @@ - [Rewards and penalties](#rewards-and-penalties) - [Signature domain types](#signature-domain-types) - [Containers](#containers) + - [`Crosslink`](#crosslink) - [`ShardBlock`](#shardblock) - [`ShardBlockHeader`](#shardblockheader) - [`ShardState`](#shardstate) - - [`ShardAttestationData`](#ShardAttestationData) + - [`ShardAttestationData`](#shardattestationdata) - [Helper functions](#helper-functions) - [Misc](#misc-1) - [`compute_epoch_of_shard_slot`](#compute_epoch_of_shard_slot) diff --git a/specs/light_client/merkle_proofs.md b/specs/light_client/merkle_proofs.md index b920f50b1..906e7d241 100644 --- a/specs/light_client/merkle_proofs.md +++ b/specs/light_client/merkle_proofs.md @@ -258,6 +258,18 @@ def get_branch_indices(tree_index: GeneralizedIndex) -> Sequence[GeneralizedInde return o[:-1] ``` +```python +def get_path_indices(tree_index: GeneralizedIndex) -> Sequence[GeneralizedIndex]: + """ + Get the generalized indices of the chunks along the path from the chunk with the + given tree index to the root. + """ + o = [tree_index] + while o[-1] > 1: + o.append(generalized_index_parent(o[-1])) + return o[:-1] +``` + ```python def get_helper_indices(indices: Sequence[GeneralizedIndex]) -> Sequence[GeneralizedIndex]: """ @@ -265,40 +277,39 @@ def get_helper_indices(indices: Sequence[GeneralizedIndex]) -> Sequence[Generali generalized indices. Note that the decreasing order is chosen deliberately to ensure equivalence to the order of hashes in a regular single-item Merkle proof in the single-item case. """ - all_indices: Set[GeneralizedIndex] = set() + all_helper_indices: Set[GeneralizedIndex] = set() + all_path_indices: Set[GeneralizedIndex] = set() for index in indices: - all_indices = all_indices.union(set(list(get_branch_indices(index)) + [index])) + all_helper_indices = all_helper_indices.union(set(get_branch_indices(index))) + all_path_indices = all_path_indices.union(set(get_path_indices(index))) - return sorted([ - x for x in all_indices if ( - not ( - generalized_index_child(x, False) in all_indices and - generalized_index_child(x, True) in all_indices - ) and not (x in indices) - ) - ], reverse=True) + return sorted(all_helper_indices.difference(all_path_indices), reverse=True) ``` Now we provide the Merkle proof verification functions. First, for single item proofs: ```python -def verify_merkle_proof(leaf: Hash, proof: Sequence[Hash], index: GeneralizedIndex, root: Hash) -> bool: +def calculate_merkle_root(leaf: Hash, proof: Sequence[Hash], index: GeneralizedIndex) -> Hash: assert len(proof) == get_generalized_index_length(index) for i, h in enumerate(proof): if get_generalized_index_bit(index, i): leaf = hash(h + leaf) else: leaf = hash(leaf + h) - return leaf == root + return leaf +``` + +```python +def verify_merkle_proof(leaf: Hash, proof: Sequence[Hash], index: GeneralizedIndex, root: Hash) -> bool: + return calculate_merkle_root(leaf, proof, index) == root ``` Now for multi-item proofs: ```python -def verify_merkle_multiproof(leaves: Sequence[Hash], - proof: Sequence[Hash], - indices: Sequence[GeneralizedIndex], - root: Hash) -> bool: +def calculate_multi_merkle_root(leaves: Sequence[Hash], + proof: Sequence[Hash], + indices: Sequence[GeneralizedIndex]) -> Hash: assert len(leaves) == len(indices) helper_indices = get_helper_indices(indices) assert len(proof) == len(helper_indices) @@ -317,7 +328,15 @@ def verify_merkle_multiproof(leaves: Sequence[Hash], ) keys.append(GeneralizedIndex(k // 2)) pos += 1 - return objects[GeneralizedIndex(1)] == root + return objects[GeneralizedIndex(1)] ``` -Note that the single-item proof is a special case of a multi-item proof; a valid single-item proof verifies correctly when put into the multi-item verification function (making the natural trivial changes to input arguments, `index -> [index]` and `leaf -> [leaf]`). +```python +def verify_merkle_multiproof(leaves: Sequence[Hash], + proof: Sequence[Hash], + indices: Sequence[GeneralizedIndex], + root: Hash) -> bool: + return calculate_multi_merkle_root(leaves, proof, indices) == root +``` + +Note that the single-item proof is a special case of a multi-item proof; a valid single-item proof verifies correctly when put into the multi-item verification function (making the natural trivial changes to input arguments, `index -> [index]` and `leaf -> [leaf]`). Note also that `calculate_merkle_root` and `calculate_multi_merkle_root` can be used independently to compute the new Merkle root of a proof with leaves updated. diff --git a/specs/networking/p2p-interface.md b/specs/networking/p2p-interface.md index 3f02757f9..50a24a83a 100644 --- a/specs/networking/p2p-interface.md +++ b/specs/networking/p2p-interface.md @@ -113,9 +113,8 @@ This section outlines constants that are used in this spec. | Name | Value | Description | |---|---|---| -| `REQ_RESP_MAX_SIZE` | `TODO` | The maximum size of uncompressed req/resp messages that clients will allow. | -| `SSZ_MAX_LIST_SIZE` | `TODO` | The maximum size of SSZ-encoded variable lists. | -| `GOSSIP_MAX_SIZE` | `2**20` (= 1048576, 1 MiB) | The maximum size of uncompressed gossip messages. | +| `GOSSIP_MAX_SIZE` | `2**20` (= 1048576, 1 MiB) | The maximum allowed size of uncompressed gossip messages. | +| `MAX_CHUNK_SIZE` | `2**20` (1048576, 1 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | | `ATTESTATION_SUBNET_COUNT` | `64` | The number of shard subnets used in the gossipsub protocol. | | `TTFB_TIMEOUT` | `5s` | The maximum time to wait for first byte of request response (time-to-first-byte). | | `RESP_TIMEOUT` | `10s` | The maximum time for complete response transfer. | @@ -253,23 +252,24 @@ Request/response messages MUST adhere to the encoding specified in the protocol ``` request ::= | -response ::= | | +response ::= + +response_chunk ::= | | result ::= “0” | “1” | “2” | [“128” ... ”255”] ``` The encoding-dependent header may carry metadata or assertions such as the encoded payload length, for integrity and attack proofing purposes. Because req/resp streams are single-use and stream closures implicitly delimit the boundaries, it is not strictly necessary to length-prefix payloads; however, certain encodings like SSZ do, for added security. -`encoded-payload` has a maximum byte size of `REQ_RESP_MAX_SIZE`. +A `response` is formed by one or more `response_chunk`s. The exact request determines whether a response consists of a single `response_chunk` or possibly many. Responses that consist of a single SSZ-list (such as `BlocksByRange` and `BlocksByRoot`) send each list item as a `response_chunk`. All other response types (non-Lists) send a single `response_chunk`. The encoded-payload of a `response_chunk` has a maximum uncompressed byte size of `MAX_CHUNK_SIZE`. -Clients MUST ensure the payload size is less than or equal to `REQ_RESP_MAX_SIZE`; if not, they SHOULD reset the stream immediately. Clients tracking peer reputation MAY decrement the score of the misbehaving peer under this circumstance. +Clients MUST ensure the each encoded payload of a `response_chunk` is less than or equal to `MAX_CHUNK_SIZE`; if not, they SHOULD reset the stream immediately. Clients tracking peer reputation MAY decrement the score of the misbehaving peer under this circumstance. #### Requesting side -Once a new stream with the protocol ID for the request type has been negotiated, the full request message should be sent immediately. It should be encoded according to the encoding strategy. +Once a new stream with the protocol ID for the request type has been negotiated, the full request message SHOULD be sent immediately. The request MUST be encoded according to the encoding strategy. -The requester MUST close the write side of the stream once it finishes writing the request message—at this point, the stream will be half-closed. +The requester MUST close the write side of the stream once it finishes writing the request message. At this point, the stream will be half-closed. -The requester MUST wait a maximum of `TTFB_TIMEOUT` for the first response byte to arrive (time to first byte—or TTFB—timeout). On that happening, the requester will allow further `RESP_TIMEOUT` to receive the full response. +The requester MUST wait a maximum of `TTFB_TIMEOUT` for the first response byte to arrive (time to first byte—or TTFB—timeout). On that happening, the requester allows a further `RESP_TIMEOUT` for each subsequent `response_chunk` received. For responses consisting of potentially many `response_chunk`s (an SSZ-list) the requester SHOULD read from the stream until either; a) An error result is received in one of the chunks, b) The responder closes the stream, c) More than `MAX_CHUNK_SIZE` bytes have been read for a single `response_chunk` payload or d) More than the maximum number of requested chunks are read. For requests consisting of a single `response_chunk` and a length-prefix, the requester should read the exact number of bytes defined by the length-prefix before closing the stream. If any of these timeouts fire, the requester SHOULD reset the stream and deem the req/resp operation to have failed. @@ -282,16 +282,16 @@ The responder MUST: 1. Use the encoding strategy to read the optional header. 2. If there are any length assertions for length `N`, it should read exactly `N` bytes from the stream, at which point an EOF should arise (no more bytes). Should this not be the case, it should be treated as a failure. 3. Deserialize the expected type, and process the request. -4. Write the response (result, optional header, payload). +4. Write the response which may consist of one or more `response_chunk`s (result, optional header, payload). 5. Close their write side of the stream. At this point, the stream will be fully closed. If steps (1), (2), or (3) fail due to invalid, malformed, or inconsistent data, the responder MUST respond in error. Clients tracking peer reputation MAY record such failures, as well as unexpected events, e.g. early stream resets. The entire request should be read in no more than `RESP_TIMEOUT`. Upon a timeout, the responder SHOULD reset the stream. -The responder SHOULD send a response promptly, starting with a **single-byte** response code which determines the contents of the response (`result` particle in the BNF grammar above). +The responder SHOULD send a `response_chunk` promptly. Chunks start with a **single-byte** response code which determines the contents of the `response_chunk` (`result` particle in the BNF grammar above). For multiple chunks, only the last chunk is allowed to have a non-zero error code (i.e. The chunk stream is terminated once an error occurs). -It can have one of the following values, encoded as a single unsigned byte: +The response code can have one of the following values, encoded as a single unsigned byte: - 0: **Success** -- a normal response follows, with contents matching the expected message schema and encoding specified in the request. - 1: **InvalidRequest** -- the contents of the request are semantically invalid, or the payload is malformed, or could not be understood. The response payload adheres to the `ErrorMessage` schema (described below). @@ -311,7 +311,7 @@ The `ErrorMessage` schema is: *Note*: The String type is encoded as UTF-8 bytes without NULL terminator when SSZ-encoded. As the `ErrorMessage` is not an SSZ-container, only the UTF-8 bytes will be sent when SSZ-encoded. -A response therefore has the form: +A response therefore has the form of one or more `response_chunk`s, each structured as follows: ``` +--------+--------+--------+--------+--------+--------+ | result | header (opt) | encoded_response | @@ -323,7 +323,7 @@ Here, `result` represents the 1-byte response code. The token of the negotiated protocol ID specifies the type of encoding to be used for the req/resp interaction. Two values are possible at this time: -- `ssz`: the contents are [SSZ-encoded](#ssz-encoding). This encoding type MUST be supported by all clients. For objects containing a single field, only the field is SSZ-encoded not a container with a single field. For example, the `BeaconBlocks` response would be an SSZ-encoded list of `BeaconBlock`s. All SSZ-Lists in the Req/Resp domain will have a maximum list size of `SSZ_MAX_LIST_SIZE`. +- `ssz`: the contents are [SSZ-encoded](../simple-serialize.md). This encoding type MUST be supported by all clients. For objects containing a single field, only the field is SSZ-encoded not a container with a single field. For example, the `BeaconBlocksByRoot` request is an SSZ-encoded list of `HashTreeRoots`'s. - `ssz_snappy`: The contents are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy). MAY be supported in the interoperability testnet; MUST be supported in mainnet. #### SSZ-encoding strategy (with or without Snappy) @@ -332,18 +332,22 @@ The [SimpleSerialize (SSZ) specification](../simple-serialize.md) outlines how o **Encoding-dependent header:** Req/Resp protocols using the `ssz` or `ssz_snappy` encoding strategies MUST prefix all encoded and compressed (if applicable) payloads with an unsigned [protobuf varint](https://developers.google.com/protocol-buffers/docs/encoding#varints). -*Note*: Parameters defined as `[]VariableName` are SSZ-encoded containerless vectors. +All messages that contain only a single field MUST be encoded directly as the type of that field and MUST NOT be encoded as an SSZ container. + +Responses that are SSZ-lists (for example `[]BeaconBlocks`) send their +constituents individually as `response_chunk`s. For example, the +`[]BeaconBlocks` response type sends one or more `response_chunk`s. Each _successful_ `response_chunk` contains a single `BeaconBlock` payload. ### Messages -#### Hello +#### Status -**Protocol ID:** ``/eth2/beacon_chain/req/hello/1/`` +**Protocol ID:** ``/eth2/beacon_chain/req/status/1/`` -**Content**: +Request, Response Content: ``` ( - fork_version: bytes4 + head_fork_version: bytes4 finalized_root: bytes32 finalized_epoch: uint64 head_root: bytes32 @@ -352,29 +356,33 @@ The [SimpleSerialize (SSZ) specification](../simple-serialize.md) outlines how o ``` The fields are: -- `fork_version`: The beacon_state `Fork` version. +- `head_fork_version`: The beacon_state `Fork` version. - `finalized_root`: The latest finalized root the node knows about. - `finalized_epoch`: The latest finalized epoch the node knows about. - `head_root`: The block hash tree root corresponding to the head of the chain as seen by the sending node. - `head_slot`: The slot corresponding to the `head_root`. -Clients exchange hello messages upon connection, forming a two-phase handshake. The first message the initiating client sends MUST be the hello message. In response, the receiving client MUST respond with its own hello message. +The dialing client MUST send a `Status` request upon connection. + +The request/response MUST be encoded as an SSZ-container. + +The response MUST consist of a single `response_chunk`. Clients SHOULD immediately disconnect from one another following the handshake above under the following conditions: -1. If `fork_version` doesn’t match the local fork version, since the client’s chain is on another fork. `fork_version` can also be used to segregate testnets. +1. If `head_fork_version` does not match the expected fork version at the epoch of the `head_slot`, since the client’s chain is on another fork. `head_fork_version` can also be used to segregate testnets. 2. If the (`finalized_root`, `finalized_epoch`) shared by the peer is not in the client's chain at the expected epoch. For example, if Peer 1 sends (root, epoch) of (A, 5) and Peer 2 sends (B, 3) but Peer 1 has root C at epoch 3, then Peer 1 would disconnect because it knows that their chains are irreparably disjoint. -Once the handshake completes, the client with the lower `finalized_epoch` or `head_slot` (if the clients have equal `finalized_epoch`s) SHOULD request beacon blocks from its counterparty via the `BeaconBlocks` request. +Once the handshake completes, the client with the lower `finalized_epoch` or `head_slot` (if the clients have equal `finalized_epoch`s) SHOULD request beacon blocks from its counterparty via the `BeaconBlocksByRange` request. #### Goodbye **Protocol ID:** ``/eth2/beacon_chain/req/goodbye/1/`` -**Content:** +Request, Response Content: ``` ( - reason: uint64 + uint64 ) ``` Client MAY send goodbye messages upon disconnection. The reason field MAY be one of the following values: @@ -387,11 +395,15 @@ Clients MAY use reason codes above `128` to indicate alternative, erroneous requ The range `[4, 127]` is RESERVED for future usage. -#### BeaconBlocks +The request/response MUST be encoded as a single SSZ-field. -**Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks/1/` +The response MUST consist of a single `response_chunk`. -Request Content +#### BeaconBlocksByRange + +**Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks_by_range/1/` + +Request Content: ``` ( head_block_root: HashTreeRoot @@ -404,27 +416,35 @@ Request Content Response Content: ``` ( - blocks: []BeaconBlock + []BeaconBlock ) ``` Requests count beacon blocks from the peer starting from `start_slot` on the chain defined by `head_block_root`. The response MUST contain no more than count blocks. `step` defines the slot increment between blocks. For example, requesting blocks starting at `start_slot` 2 with a step value of 2 would return the blocks at [2, 4, 6, …]. In cases where a slot is empty for a given slot number, no block is returned. For example, if slot 4 were empty in the previous example, the returned array would contain [2, 6, …]. A step value of 1 returns all blocks on the range `[start_slot, start_slot + count)`. -`BeaconBlocks` is primarily used to sync historical blocks. +The request MUST be encoded as an SSZ-container. + +The response MUST consist of at least one `response_chunk` and MAY consist of many. Each _successful_ `response_chunk` MUST contain a single `BeaconBlock` payload. + +`BeaconBlocksByRange` is primarily used to sync historical blocks. Clients MUST support requesting blocks since the start of the weak subjectivity period and up to the given `head_block_root`. Clients MUST support `head_block_root` values since the latest finalized epoch. -#### RecentBeaconBlocks +Clients MUST respond with at least one block, if they have it. -**Protocol ID:** `/eth2/beacon_chain/req/recent_beacon_blocks/1/` +Clients MUST order blocks by increasing slot number. + +#### BeaconBlocksByRoot + +**Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks_by_root/1/` Request Content: ``` ( - block_roots: []HashTreeRoot + []HashTreeRoot ) ``` @@ -432,16 +452,22 @@ Response Content: ``` ( - blocks: []BeaconBlock + []BeaconBlock ) ``` -Requests blocks by their block roots. The response is a list of `BeaconBlock` with the same length as the request. Blocks are returned in order of the request and any missing/unknown blocks are left empty (SSZ null `BeaconBlock`). +Requests blocks by their block roots. The response is a list of `BeaconBlock` whose length is less than or equal to the number of requested blocks. It may be less in the case that the responding peer is missing blocks. -`RecentBeaconBlocks` is primarily used to recover recent blocks (ex. when receiving a block or attestation whose parent is unknown). +`BeaconBlocksByRoot` is primarily used to recover recent blocks (e.g. when receiving a block or attestation whose parent is unknown). + +The request MUST be encoded as an SSZ-field. + +The response MUST consist of at least one `response_chunk` and MAY consist of many. Each _successful_ `response_chunk` MUST contain a single `BeaconBlock` payload. Clients MUST support requesting blocks since the latest finalized epoch. +Clients MUST respond with at least one block, if they have it. + ## The discovery domain: discv5 Discovery Version 5 ([discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md)) is used for peer discovery, both in the interoperability testnet and mainnet. @@ -716,7 +742,7 @@ Disadvantages include: * Harder to stream as length must be known up-front * Additional code path required to verify length -In some protocols, adding a length prefix serves as a form of DoS protection against very long messages, allowing the client to abort if an overlong message is about to be sent. In this protocol, we are globally limiting message sizes using `REQ_RESP_MAX_SIZE`, thus the length prefix does not afford any additional protection. +In some protocols, adding a length prefix serves as a form of DoS protection against very long messages, allowing the client to abort if an overlong message is about to be sent. In this protocol, we are globally limiting message sizes using `MAX_CHUNK_SIZE`, thus the length prefix does not afford any additional protection. [Protobuf varint](https://developers.google.com/protocol-buffers/docs/encoding#varints) is an efficient technique to encode variable-length ints. Instead of reserving a fixed-size field of as many bytes as necessary to convey the maximum possible value, this field is elastic in exchange for 1-bit overhead per byte. diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 5e17962d1..cb6134793 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -139,7 +139,7 @@ return bytes(array) ### `Bitlist[N]` -Note that from the offset coding, the length (in bytes) of the bitlist is known. An additional leading `1` bit is added so that the length in bits will also be known. +Note that from the offset coding, the length (in bytes) of the bitlist is known. An additional `1` bit is added to the end, at index `e` where `e` is the length of the bitlist (not the limit), so that the length in bits will also be known. ```python array = [0] * ((len(value) // 8) + 1) @@ -181,7 +181,15 @@ return serialized_type_index + serialized_bytes ## Deserialization -Because serialization is an injective function (i.e. two distinct objects of the same type will serialize to different values) any bytestring has at most one object it could deserialize to. Efficient algorithms for computing this object can be found in [the implementations](#implementations). +Because serialization is an injective function (i.e. two distinct objects of the same type will serialize to different values) any bytestring has at most one object it could deserialize to. + +Deserialization can be implemented using a recursive algorithm. The deserialization of basic objects is easy, and from there we can find a simple recursive algorithm for all fixed-size objects. For variable-size objects we have to do one of the following depending on what kind of object it is: + +* Vector/list of a variable-size object: The serialized data will start with offsets of all the serialized objects (`BYTES_PER_LENGTH_OFFSET` bytes each). + * Using the first offset, we can compute the length of the list (divide by `BYTES_PER_LENGTH_OFFSET`), as it gives us the total number of bytes in the offset data. + * The size of each object in the vector/list can be inferred from the difference of two offsets. To get the size of the last object, the total number of bytes has to be known (it is not generally possible to deserialize an SSZ object of unknown length) +* Containers follow the same principles as vectors, with the difference that there may be fixed-size objects in a container as well. This means the `fixed_parts` data will contain offsets as well as fixed-size objects. +* In the case of bitlists, the length in bits cannot be uniquely inferred from the number of bytes in the object. Because of this, they have a bit at the end that is always set. This bit has to be used to infer the size of the bitlist in bits. Note that deserialization requires hardening against invalid inputs. A non-exhaustive list: @@ -189,6 +197,8 @@ Note that deserialization requires hardening against invalid inputs. A non-exhau - Scope: Extra unused bytes, not aligned with element size. - More elements than a list limit allows. Part of enforcing consensus. +Efficient algorithms for computing this object can be found in [the implementations](#implementations). + ## Merkleization We first define helper functions: @@ -247,4 +257,5 @@ We similarly define "summary types" and "expansion types". For example, [`Beacon | Go | Prysm | Prysmatic Labs | [https://github.com/prysmaticlabs/go-ssz](https://github.com/prysmaticlabs/go-ssz) | | Swift | Yeeth | Dean Eigenmann | [https://github.com/yeeth/SimpleSerialize.swift](https://github.com/yeeth/SimpleSerialize.swift) | | C# | | Jordan Andrews | [https://github.com/codingupastorm/csharp-ssz](https://github.com/codingupastorm/csharp-ssz) | +| C# | Cortex | Sly Gryphon | [https://www.nuget.org/packages/Cortex.SimpleSerialize](https://www.nuget.org/packages/Cortex.SimpleSerialize) | | C++ | | Jiyun Kim | [https://github.com/NAKsir-melody/cpp_ssz](https://github.com/NAKsir-melody/cpp_ssz) | diff --git a/specs/test_formats/operations/README.md b/specs/test_formats/operations/README.md index be009486d..f1ec0429a 100644 --- a/specs/test_formats/operations/README.md +++ b/specs/test_formats/operations/README.md @@ -46,7 +46,6 @@ Operations: | `block_header` | `Block` | **`block`** | `process_block_header(state, block)` | | `deposit` | `Deposit` | `deposit` | `process_deposit(state, deposit)` | | `proposer_slashing` | `ProposerSlashing` | `proposer_slashing` | `process_proposer_slashing(state, proposer_slashing)` | -| `transfer` | `Transfer` | `transfer` | `process_transfer(state, transfer)` | | `voluntary_exit` | `VoluntaryExit` | `voluntary_exit` | `process_voluntary_exit(state, voluntary_exit)` | Note that `block_header` is not strictly an operation (and is a full `Block`), but processed in the same manner, and hence included here. diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index c6623dd09..13decee05 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -108,10 +108,11 @@ In Phase 0, all incoming validator deposits originate from the Ethereum 1.0 proo To submit a deposit: - Pack the validator's [initialization parameters](#initialization) into `deposit_data`, a [`DepositData`](../core/0_beacon-chain.md#depositdata) SSZ object. -- Let `amount` be the amount in Gwei to be deposited by the validator where `MIN_DEPOSIT_AMOUNT <= amount <= MAX_EFFECTIVE_BALANCE`. +- Let `amount` be the amount in Gwei to be deposited by the validator where `amount >= MIN_DEPOSIT_AMOUNT`. - Set `deposit_data.amount = amount`. - Let `signature` be the result of `bls_sign` of the `signing_root(deposit_data)` with `domain=compute_domain(DOMAIN_DEPOSIT)`. (Deposits are valid regardless of fork version, `compute_domain` will default to zeroes there). -- Send a transaction on the Ethereum 1.0 chain to `DEPOSIT_CONTRACT_ADDRESS` executing `def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96])` along with a deposit of `amount` Gwei. +- Let `deposit_data_root` be `hash_tree_root(deposit_data)`. +- Send a transaction on the Ethereum 1.0 chain to `DEPOSIT_CONTRACT_ADDRESS` executing `def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96], deposit_data_root: bytes32)` along with a deposit of `amount` Gwei. *Note*: Deposits made for the same `pubkey` are treated as for the same validator. A singular `Validator` will be added to `state.validators` with each additional deposit amount added to the validator's balance. A validator can only be activated when total deposits for the validator pubkey meet or exceed `MAX_EFFECTIVE_BALANCE`. diff --git a/test_generators/bls/main.py b/test_generators/bls/main.py index a74397e77..587b3adc0 100644 --- a/test_generators/bls/main.py +++ b/test_generators/bls/main.py @@ -11,6 +11,12 @@ from eth_utils import ( from gen_base import gen_runner, gen_typing from py_ecc import bls +from hashlib import sha256 + + +def hash(x): + return sha256(x).digest() + F2Q_COEFF_LEN = 48 G2_COMPRESSED_Z_LEN = 48 @@ -122,7 +128,8 @@ def case04_sign_messages(): for message in MESSAGES: for domain in DOMAINS: sig = bls.sign(message, privkey, domain) - yield f'sign_msg_{int_to_hex(privkey)}_{encode_hex(message)}_{encode_hex(domain)}', { + full_name = f'{int_to_hex(privkey)}_{encode_hex(message)}_{encode_hex(domain)}' + yield f'sign_msg_case_{(hash(bytes(full_name, "utf-8"))[:8]).hex()}', { 'input': { 'privkey': int_to_hex(privkey), 'message': encode_hex(message), diff --git a/test_generators/epoch_processing/main.py b/test_generators/epoch_processing/main.py index 52581d8c3..b6e3d6c04 100644 --- a/test_generators/epoch_processing/main.py +++ b/test_generators/epoch_processing/main.py @@ -6,6 +6,7 @@ from eth2spec.test.phase_0.epoch_processing import ( test_process_final_updates, test_process_justification_and_finalization, test_process_registry_updates, + test_process_rewards_and_penalties, test_process_slashings ) from gen_base import gen_runner, gen_typing @@ -40,6 +41,9 @@ if __name__ == "__main__": create_provider('justification_and_finalization', test_process_justification_and_finalization, 'mainnet'), create_provider('registry_updates', test_process_registry_updates, 'minimal'), create_provider('registry_updates', test_process_registry_updates, 'mainnet'), + create_provider('rewards_and_penalties', test_process_rewards_and_penalties, 'minimal'), + # runs full epochs filled with data, with uncached ssz. Disabled for now. + # create_provider('rewards_and_penalties', test_process_rewards_and_penalties, 'mainnet'), create_provider('slashings', test_process_slashings, 'minimal'), create_provider('slashings', test_process_slashings, 'mainnet'), ]) diff --git a/test_generators/operations/main.py b/test_generators/operations/main.py index 995a626b4..de3eb7cf1 100644 --- a/test_generators/operations/main.py +++ b/test_generators/operations/main.py @@ -6,7 +6,6 @@ from eth2spec.test.phase_0.block_processing import ( test_process_block_header, test_process_deposit, test_process_proposer_slashing, - test_process_transfer, test_process_voluntary_exit, ) @@ -48,10 +47,6 @@ if __name__ == "__main__": create_provider('deposit', test_process_deposit, 'mainnet'), create_provider('proposer_slashing', test_process_proposer_slashing, 'minimal'), create_provider('proposer_slashing', test_process_proposer_slashing, 'mainnet'), - create_provider('transfer', test_process_transfer, 'minimal'), - # Disabled, due to the high amount of different transfer tests, this produces a shocking size of tests. - # Unnecessarily, as transfer are disabled currently, so not a priority. - # create_provider('transfer', test_process_transfer, 'mainnet'), create_provider('voluntary_exit', test_process_voluntary_exit, 'minimal'), create_provider('voluntary_exit', test_process_voluntary_exit, 'mainnet'), ]) diff --git a/test_libs/config_helpers/requirements.txt b/test_libs/config_helpers/requirements.txt index f2f208c3f..6c7334268 100644 --- a/test_libs/config_helpers/requirements.txt +++ b/test_libs/config_helpers/requirements.txt @@ -1 +1 @@ -ruamel.yaml==0.15.96 +ruamel.yaml==0.16.5 diff --git a/test_libs/config_helpers/setup.py b/test_libs/config_helpers/setup.py index 9f0ea0641..3f893f3d4 100644 --- a/test_libs/config_helpers/setup.py +++ b/test_libs/config_helpers/setup.py @@ -4,6 +4,6 @@ setup( name='config_helpers', packages=['preset_loader'], install_requires=[ - "ruamel.yaml==0.15.96" + "ruamel.yaml==0.16.5" ] ) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index 3bb8f1153..b06d2984d 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -6,15 +6,67 @@ from .helpers.genesis import create_genesis_state from .utils import vector_test, with_meta_tags +from typing import Any, Callable, Sequence -def with_state(fn): - def entry(*args, **kw): - try: - kw['state'] = create_genesis_state(spec=kw['spec'], num_validators=spec_phase0.SLOTS_PER_EPOCH * 10) - except KeyError: - raise TypeError('Spec decorator must come within state decorator to inject spec into state.') - return fn(*args, **kw) - return entry + +def with_custom_state(balances_fn: Callable[[Any], Sequence[int]], + threshold_fn: Callable[[Any], int]): + def deco(fn): + def entry(*args, **kw): + try: + spec = kw['spec'] + + balances = balances_fn(spec) + activation_threshold = threshold_fn(spec) + + kw['state'] = create_genesis_state(spec=spec, validator_balances=balances, + activation_threshold=activation_threshold) + except KeyError: + raise TypeError('Spec decorator must come within state decorator to inject spec into state.') + return fn(*args, **kw) + return entry + return deco + + +def default_activation_threshold(spec): + """ + Helper method to use the default balance activation threshold for state creation for tests. + Usage: `@with_custom_state(threshold_fn=default_activation_threshold, ...)` + """ + return spec.MAX_EFFECTIVE_BALANCE + + +def default_balances(spec): + """ + Helper method to create a series of default balances. + Usage: `@with_custom_state(balances_fn=default_balances, ...)` + """ + num_validators = spec.SLOTS_PER_EPOCH * 8 + return [spec.MAX_EFFECTIVE_BALANCE] * num_validators + + +with_state = with_custom_state(default_balances, default_activation_threshold) + + +def low_balances(spec): + """ + Helper method to create a series of low balances. + Usage: `@with_custom_state(balances_fn=low_balances, ...)` + """ + num_validators = spec.SLOTS_PER_EPOCH * 8 + # Technically the balances cannot be this low starting from genesis, but it is useful for testing + low_balance = 18 * 10 ** 9 + return [low_balance] * num_validators + + +def misc_balances(spec): + """ + Helper method to create a series of balances that includes some misc. balances. + Usage: `@with_custom_state(balances_fn=misc_balances, ...)` + """ + num_validators = spec.SLOTS_PER_EPOCH * 8 + num_misc_validators = spec.SLOTS_PER_EPOCH + return [spec.MAX_EFFECTIVE_BALANCE] * num_validators + [spec.MIN_DEPOSIT_AMOUNT] * num_misc_validators # BLS is turned off by default *for performance purposes during TESTING*. diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_initialization.py b/test_libs/pyspec/eth2spec/test/genesis/test_initialization.py index 2ff57be74..462065bb9 100644 --- a/test_libs/pyspec/eth2spec/test/genesis/test_initialization.py +++ b/test_libs/pyspec/eth2spec/test/genesis/test_initialization.py @@ -8,7 +8,7 @@ from eth2spec.test.helpers.deposits import ( @spec_test def test_initialize_beacon_state_from_eth1(spec): deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT - deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True) + deposits, deposit_root, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True) eth1_block_hash = b'\x12' * 32 eth1_timestamp = spec.MIN_GENESIS_TIME @@ -25,6 +25,43 @@ def test_initialize_beacon_state_from_eth1(spec): assert state.eth1_data.deposit_root == deposit_root assert state.eth1_data.deposit_count == deposit_count assert state.eth1_data.block_hash == eth1_block_hash + assert spec.get_total_active_balance(state) == deposit_count * spec.MAX_EFFECTIVE_BALANCE + + # yield state + yield 'state', state + + +@with_phases(['phase0']) +@spec_test +def test_initialize_beacon_state_some_small_balances(spec): + main_deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT + main_deposits, _, deposit_data_list = prepare_genesis_deposits(spec, main_deposit_count, + spec.MAX_EFFECTIVE_BALANCE, signed=True) + # For deposits above, and for another deposit_count, add a balance of EFFECTIVE_BALANCE_INCREMENT + small_deposit_count = main_deposit_count * 2 + small_deposits, deposit_root, _ = prepare_genesis_deposits(spec, small_deposit_count, + spec.MIN_DEPOSIT_AMOUNT, + signed=True, + deposit_data_list=deposit_data_list) + deposits = main_deposits + small_deposits + + eth1_block_hash = b'\x12' * 32 + eth1_timestamp = spec.MIN_GENESIS_TIME + + yield 'eth1_block_hash', eth1_block_hash + yield 'eth1_timestamp', eth1_timestamp + yield 'deposits', deposits + + # initialize beacon_state + state = spec.initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits) + + assert state.genesis_time == eth1_timestamp - eth1_timestamp % spec.SECONDS_PER_DAY + 2 * spec.SECONDS_PER_DAY + assert len(state.validators) == small_deposit_count + assert state.eth1_data.deposit_root == deposit_root + assert state.eth1_data.deposit_count == len(deposits) + assert state.eth1_data.block_hash == eth1_block_hash + # only main deposits participate to the active balance + assert spec.get_total_active_balance(state) == main_deposit_count * spec.MAX_EFFECTIVE_BALANCE # yield state yield 'state', state diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_validity.py b/test_libs/pyspec/eth2spec/test/genesis/test_validity.py index 07ad3a73c..a003938e7 100644 --- a/test_libs/pyspec/eth2spec/test/genesis/test_validity.py +++ b/test_libs/pyspec/eth2spec/test/genesis/test_validity.py @@ -6,7 +6,7 @@ from eth2spec.test.helpers.deposits import ( def create_valid_beacon_state(spec): deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT - deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True) + deposits, _, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True) eth1_block_hash = b'\x12' * 32 eth1_timestamp = spec.MIN_GENESIS_TIME @@ -65,7 +65,7 @@ def test_is_valid_genesis_state_true_more_balance(spec): @spec_test def test_is_valid_genesis_state_true_one_more_validator(spec): deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT + 1 - deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True) + deposits, _, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True) eth1_block_hash = b'\x12' * 32 eth1_timestamp = spec.MIN_GENESIS_TIME @@ -78,7 +78,7 @@ def test_is_valid_genesis_state_true_one_more_validator(spec): @spec_test def test_is_valid_genesis_state_false_not_enough_validator(spec): deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT - 1 - deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True) + deposits, _, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True) eth1_block_hash = b'\x12' * 32 eth1_timestamp = spec.MIN_GENESIS_TIME diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index f3454fe4c..d36e62ace 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -114,7 +114,8 @@ def get_attestation_signature(spec, state, attestation_data, privkey, custody_bi ) -def fill_aggregate_attestation(spec, state, attestation): +def fill_aggregate_attestation(spec, state, attestation, signed=False): + beacon_committee = spec.get_beacon_committee( state, attestation.data.slot, @@ -123,11 +124,15 @@ def fill_aggregate_attestation(spec, state, attestation): for i in range(len(beacon_committee)): attestation.aggregation_bits[i] = True + if signed: + sign_attestation(spec, state, attestation) -def add_attestation_to_state(spec, state, attestation, slot): + +def add_attestations_to_state(spec, state, attestations, slot): block = build_empty_block_for_next_slot(spec, state) block.slot = slot - block.body.attestations.append(attestation) + for attestation in attestations: + block.body.attestations.append(attestation) spec.process_slots(state, block.slot) sign_block(spec, state, block) spec.state_transition(state, block) diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index 89574c977..9f91338e7 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -55,8 +55,9 @@ def build_deposit(spec, return deposit, root, deposit_data_list -def prepare_genesis_deposits(spec, genesis_validator_count, amount, signed=False): - deposit_data_list = [] +def prepare_genesis_deposits(spec, genesis_validator_count, amount, signed=False, deposit_data_list=None): + if deposit_data_list is None: + deposit_data_list = [] genesis_deposits = [] for validator_index in range(genesis_validator_count): pubkey = pubkeys[validator_index] @@ -75,7 +76,7 @@ def prepare_genesis_deposits(spec, genesis_validator_count, amount, signed=False ) genesis_deposits.append(deposit) - return genesis_deposits, root + return genesis_deposits, root, deposit_data_list def prepare_state_and_deposit(spec, state, validator_index, amount, withdrawal_credentials=None, signed=False): diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index 9e3c77b7b..d96b2ceb2 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -1,3 +1,4 @@ +import copy from eth2spec.test.helpers.keys import pubkeys @@ -16,28 +17,30 @@ def build_mock_validator(spec, i: int, balance: int): ) -def create_genesis_state(spec, num_validators): +def create_genesis_state(spec, validator_balances, activation_threshold): deposit_root = b'\x42' * 32 + eth1_block_hash = b'\xda' * 32 state = spec.BeaconState( genesis_time=0, - eth1_deposit_index=num_validators, + eth1_deposit_index=len(validator_balances), eth1_data=spec.Eth1Data( deposit_root=deposit_root, - deposit_count=num_validators, - block_hash=spec.Hash(), + deposit_count=len(validator_balances), + block_hash=eth1_block_hash, ), latest_block_header=spec.BeaconBlockHeader(body_root=spec.hash_tree_root(spec.BeaconBlockBody())), + randao_mixes=[eth1_block_hash] * spec.EPOCHS_PER_HISTORICAL_VECTOR, ) # We "hack" in the initial validators, # as it is much faster than creating and processing genesis deposits for every single test case. - state.balances = [spec.MAX_EFFECTIVE_BALANCE] * num_validators - state.validators = [build_mock_validator(spec, i, state.balances[i]) for i in range(num_validators)] + state.balances = copy.deepcopy(validator_balances) + state.validators = [build_mock_validator(spec, i, state.balances[i]) for i in range(len(validator_balances))] # Process genesis activations for validator in state.validators: - if validator.effective_balance >= spec.MAX_EFFECTIVE_BALANCE: + if validator.effective_balance >= activation_threshold: validator.activation_eligibility_epoch = spec.GENESIS_EPOCH validator.activation_epoch = spec.GENESIS_EPOCH diff --git a/test_libs/pyspec/eth2spec/test/helpers/transfers.py b/test_libs/pyspec/eth2spec/test/helpers/transfers.py deleted file mode 100644 index 3d3b0f4e3..000000000 --- a/test_libs/pyspec/eth2spec/test/helpers/transfers.py +++ /dev/null @@ -1,53 +0,0 @@ -from eth2spec.test.helpers.keys import pubkeys, privkeys -from eth2spec.test.helpers.state import get_balance -from eth2spec.utils.bls import bls_sign -from eth2spec.utils.ssz.ssz_impl import signing_root - - -def get_valid_transfer(spec, state, slot=None, sender_index=None, - recipient_index=None, amount=None, fee=None, signed=False): - if slot is None: - slot = state.slot - current_epoch = spec.get_current_epoch(state) - if sender_index is None: - sender_index = spec.get_active_validator_indices(state, current_epoch)[-1] - if recipient_index is None: - recipient_index = spec.get_active_validator_indices(state, current_epoch)[0] - transfer_pubkey = pubkeys[-1] - transfer_privkey = privkeys[-1] - - if fee is None: - fee = get_balance(state, sender_index) // 32 - if amount is None: - amount = get_balance(state, sender_index) - fee - - transfer = spec.Transfer( - sender=sender_index, - recipient=recipient_index, - amount=amount, - fee=fee, - slot=slot, - pubkey=transfer_pubkey, - ) - if signed: - sign_transfer(spec, state, transfer, transfer_privkey) - - # ensure withdrawal_credentials reproducible - state.validators[transfer.sender].withdrawal_credentials = ( - spec.BLS_WITHDRAWAL_PREFIX + spec.hash(transfer.pubkey)[1:] - ) - - return transfer - - -def sign_transfer(spec, state, transfer, privkey): - transfer.signature = bls_sign( - message_hash=signing_root(transfer), - privkey=privkey, - domain=spec.get_domain( - state=state, - domain_type=spec.DOMAIN_TRANSFER, - message_epoch=spec.get_current_epoch(state), - ) - ) - return transfer diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 13faed3f4..04fa880e0 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -3,6 +3,9 @@ from eth2spec.test.context import ( expect_assertion_error, always_bls, never_bls, with_all_phases, with_phases, + spec_test, + low_balances, + with_custom_state, ) from eth2spec.test.helpers.attestations import ( get_valid_attestation, @@ -60,6 +63,17 @@ def test_success(spec, state): yield from run_attestation_processing(spec, state, attestation) +@with_all_phases +@spec_test +@with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +def test_success_multi_proposer_index_iterations(spec, state): + state.slot += spec.SLOTS_PER_EPOCH * 2 + attestation = get_valid_attestation(spec, state, signed=True) + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + yield from run_attestation_processing(spec, state, attestation) + + @with_all_phases @spec_state_test def test_success_previous_epoch(spec, state): diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py deleted file mode 100644 index 1b839562e..000000000 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py +++ /dev/null @@ -1,368 +0,0 @@ -from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases -from eth2spec.test.helpers.state import next_epoch -from eth2spec.test.helpers.block import apply_empty_block -from eth2spec.test.helpers.transfers import get_valid_transfer, sign_transfer - - -def run_transfer_processing(spec, state, transfer, valid=True): - """ - Run ``process_transfer``, yielding: - - pre-state ('pre') - - transfer ('transfer') - - post-state ('post'). - If ``valid == False``, run expecting ``AssertionError`` - """ - - yield 'pre', state - yield 'transfer', transfer - - if not valid: - expect_assertion_error(lambda: spec.process_transfer(state, transfer)) - yield 'post', None - return - - proposer_index = spec.get_beacon_proposer_index(state) - pre_transfer_sender_balance = state.balances[transfer.sender] - pre_transfer_recipient_balance = state.balances[transfer.recipient] - pre_transfer_proposer_balance = state.balances[proposer_index] - - spec.process_transfer(state, transfer) - yield 'post', state - - sender_balance = state.balances[transfer.sender] - recipient_balance = state.balances[transfer.recipient] - assert sender_balance == pre_transfer_sender_balance - transfer.amount - transfer.fee - assert recipient_balance == pre_transfer_recipient_balance + transfer.amount - assert state.balances[proposer_index] == pre_transfer_proposer_balance + transfer.fee - - -@with_all_phases -@spec_state_test -def test_success_non_activated(spec, state): - transfer = get_valid_transfer(spec, state, signed=True) - # un-activate so validator can transfer - state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - - yield from run_transfer_processing(spec, state, transfer) - - -@with_all_phases -@spec_state_test -def test_success_withdrawable(spec, state): - next_epoch(spec, state) - apply_empty_block(spec, state) - - transfer = get_valid_transfer(spec, state, signed=True) - - # withdrawable_epoch in past so can transfer - state.validators[transfer.sender].withdrawable_epoch = spec.get_current_epoch(state) - 1 - - yield from run_transfer_processing(spec, state, transfer) - - -@with_all_phases -@spec_state_test -def test_success_active_above_max_effective(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 - transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) - - yield from run_transfer_processing(spec, state, transfer) - - -@with_all_phases -@spec_state_test -def test_success_active_above_max_effective_fee(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 - transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=0, fee=1, signed=True) - - yield from run_transfer_processing(spec, state, transfer) - - -@with_all_phases -@spec_state_test -@always_bls -def test_invalid_signature(spec, state): - transfer = get_valid_transfer(spec, state) - # un-activate so validator can transfer - state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - - yield from run_transfer_processing(spec, state, transfer, False) - - -@with_all_phases -@spec_state_test -def test_active_but_transfer_past_effective_balance(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - amount = spec.MAX_EFFECTIVE_BALANCE // 32 - state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE - transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=amount, fee=0, signed=True) - - yield from run_transfer_processing(spec, state, transfer, False) - - -@with_all_phases -@spec_state_test -def test_incorrect_slot(spec, state): - transfer = get_valid_transfer(spec, state, slot=state.slot + 1, signed=True) - # un-activate so validator can transfer - state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - - yield from run_transfer_processing(spec, state, transfer, False) - - -@with_all_phases -@spec_state_test -def test_transfer_clean(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - state.balances[sender_index] = spec.MIN_DEPOSIT_AMOUNT - transfer = get_valid_transfer(spec, state, sender_index=sender_index, - amount=spec.MIN_DEPOSIT_AMOUNT, fee=0, signed=True) - - # un-activate so validator can transfer - state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - - yield from run_transfer_processing(spec, state, transfer) - - -@with_all_phases -@spec_state_test -def test_transfer_clean_split_to_fee(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - state.balances[sender_index] = spec.MIN_DEPOSIT_AMOUNT - transfer = get_valid_transfer(spec, state, sender_index=sender_index, - amount=spec.MIN_DEPOSIT_AMOUNT // 2, fee=spec.MIN_DEPOSIT_AMOUNT // 2, signed=True) - - # un-activate so validator can transfer - state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - - yield from run_transfer_processing(spec, state, transfer) - - -@with_all_phases -@spec_state_test -def test_insufficient_balance_for_fee(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - state.balances[sender_index] = spec.MIN_DEPOSIT_AMOUNT - transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=0, fee=1, signed=True) - - # un-activate so validator can transfer - state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - - yield from run_transfer_processing(spec, state, transfer, False) - - -@with_all_phases -@spec_state_test -def test_insufficient_balance_for_fee_result_full(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - transfer = get_valid_transfer(spec, state, sender_index=sender_index, - amount=0, fee=state.balances[sender_index] + 1, signed=True) - - # un-activate so validator can transfer - state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - - yield from run_transfer_processing(spec, state, transfer, False) - - -@with_all_phases -@spec_state_test -def test_insufficient_balance_for_amount_result_dust(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - state.balances[sender_index] = spec.MIN_DEPOSIT_AMOUNT - transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) - - # un-activate so validator can transfer - state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - - yield from run_transfer_processing(spec, state, transfer, False) - - -@with_all_phases -@spec_state_test -def test_insufficient_balance_for_amount_result_full(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - transfer = get_valid_transfer(spec, state, sender_index=sender_index, - amount=state.balances[sender_index] + 1, fee=0, signed=True) - - # un-activate so validator can transfer - state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - - yield from run_transfer_processing(spec, state, transfer, False) - - -@with_all_phases -@spec_state_test -def test_insufficient_balance_for_combined_result_dust(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - # Enough to pay fee without dust, and amount without dust, but not both. - state.balances[sender_index] = spec.MIN_DEPOSIT_AMOUNT + 1 - transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=1, signed=True) - - # un-activate so validator can transfer - state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - - yield from run_transfer_processing(spec, state, transfer, False) - - -@with_all_phases -@spec_state_test -def test_insufficient_balance_for_combined_result_full(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - # Enough to pay fee fully without dust left, and amount fully without dust left, but not both. - state.balances[sender_index] = spec.MIN_DEPOSIT_AMOUNT * 2 + 1 - transfer = get_valid_transfer(spec, state, sender_index=sender_index, - amount=spec.MIN_DEPOSIT_AMOUNT + 1, - fee=spec.MIN_DEPOSIT_AMOUNT + 1, signed=True) - - # un-activate so validator can transfer - state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - - yield from run_transfer_processing(spec, state, transfer, False) - - -@with_all_phases -@spec_state_test -def test_insufficient_balance_for_combined_big_amount(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - # Enough to pay fee fully without dust left, and amount fully without dust left, but not both. - # Try to create a dust balance (off by 1) with combination of fee and amount. - state.balances[sender_index] = spec.MIN_DEPOSIT_AMOUNT * 2 + 1 - transfer = get_valid_transfer(spec, state, sender_index=sender_index, - amount=spec.MIN_DEPOSIT_AMOUNT + 1, fee=1, signed=True) - - # un-activate so validator can transfer - state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - - yield from run_transfer_processing(spec, state, transfer, False) - - -@with_all_phases -@spec_state_test -def test_insufficient_balance_for_combined_big_fee(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - # Enough to pay fee fully without dust left, and amount fully without dust left, but not both. - # Try to create a dust balance (off by 1) with combination of fee and amount. - state.balances[sender_index] = spec.MIN_DEPOSIT_AMOUNT * 2 + 1 - transfer = get_valid_transfer(spec, state, sender_index=sender_index, - amount=1, fee=spec.MIN_DEPOSIT_AMOUNT + 1, signed=True) - - # un-activate so validator can transfer - state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - - yield from run_transfer_processing(spec, state, transfer, False) - - -@with_all_phases -@spec_state_test -def test_insufficient_balance_off_by_1_fee(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - # Enough to pay fee fully without dust left, and amount fully without dust left, but not both. - # Try to print money by using the full balance as amount, plus 1 for fee. - transfer = get_valid_transfer(spec, state, sender_index=sender_index, - amount=state.balances[sender_index], fee=1, signed=True) - - # un-activate so validator can transfer - state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - - yield from run_transfer_processing(spec, state, transfer, False) - - -@with_all_phases -@spec_state_test -def test_insufficient_balance_off_by_1_amount(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - # Enough to pay fee fully without dust left, and amount fully without dust left, but not both. - # Try to print money by using the full balance as fee, plus 1 for amount. - transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, - fee=state.balances[sender_index], signed=True) - - # un-activate so validator can transfer - state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - - yield from run_transfer_processing(spec, state, transfer, False) - - -@with_all_phases -@spec_state_test -def test_insufficient_balance_duplicate_as_fee_and_amount(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - # Enough to pay fee fully without dust left, and amount fully without dust left, but not both. - # Try to print money by using the full balance, twice. - transfer = get_valid_transfer(spec, state, sender_index=sender_index, - amount=state.balances[sender_index], - fee=state.balances[sender_index], signed=True) - - # un-activate so validator can transfer - state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - - yield from run_transfer_processing(spec, state, transfer, False) - - -@with_all_phases -@spec_state_test -def test_no_dust_sender(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - balance = state.balances[sender_index] - transfer = get_valid_transfer( - spec, - state, - sender_index=sender_index, - amount=balance - spec.MIN_DEPOSIT_AMOUNT + 1, - fee=0, - signed=True, - ) - - # un-activate so validator can transfer - state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - - yield from run_transfer_processing(spec, state, transfer, False) - - -@with_all_phases -@spec_state_test -def test_no_dust_recipient(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 - transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) - state.balances[transfer.recipient] = 0 - - # un-activate so validator can transfer - state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - - yield from run_transfer_processing(spec, state, transfer, False) - - -@with_all_phases -@spec_state_test -def test_non_existent_sender(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0) - transfer.sender = len(state.validators) - sign_transfer(spec, state, transfer, 42) # mostly valid signature, but sender won't exist, use bogus key. - - yield from run_transfer_processing(spec, state, transfer, False) - - -@with_all_phases -@spec_state_test -def test_non_existent_recipient(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 - transfer = get_valid_transfer(spec, state, sender_index=sender_index, - recipient_index=len(state.validators), amount=1, fee=0, signed=True) - - yield from run_transfer_processing(spec, state, transfer, False) - - -@with_all_phases -@spec_state_test -def test_invalid_pubkey(spec, state): - transfer = get_valid_transfer(spec, state, signed=True) - state.validators[transfer.sender].withdrawal_credentials = spec.Hash() - - # un-activate so validator can transfer - state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - - yield from run_transfer_processing(spec, state, transfer, False) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 002d3b169..917c06e3d 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -8,7 +8,7 @@ def run_process_just_and_fin(spec, state): yield from run_epoch_processing_with(spec, state, 'process_justification_and_finalization') -def add_mock_attestations(spec, state, epoch, source, target, sufficient_support=False): +def add_mock_attestations(spec, state, epoch, source, target, sufficient_support=False, messed_up_target=False): # we must be at the end of the epoch assert (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0 @@ -61,6 +61,8 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support ), inclusion_delay=1, )) + if messed_up_target: + attestations[len(attestations) - 1].data.target.root = b'\x99' * 32 def get_checkpoints(spec, epoch): @@ -190,7 +192,7 @@ def finalize_on_123(spec, state, epoch, sufficient_support): assert state.finalized_checkpoint == old_finalized # no new finalized -def finalize_on_12(spec, state, epoch, sufficient_support): +def finalize_on_12(spec, state, epoch, sufficient_support, messed_up_target): assert epoch > 2 state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch @@ -212,13 +214,14 @@ def finalize_on_12(spec, state, epoch, sufficient_support): epoch=epoch - 1, source=c2, target=c1, - sufficient_support=sufficient_support) + sufficient_support=sufficient_support, + messed_up_target=messed_up_target) # process! yield from run_process_just_and_fin(spec, state) assert state.previous_justified_checkpoint == c2 # changed to old current - if sufficient_support: + if sufficient_support and not messed_up_target: assert state.current_justified_checkpoint == c1 # changed to 1st latest assert state.finalized_checkpoint == c2 # finalized previous justified epoch else: @@ -265,10 +268,16 @@ def test_123_poor_support(spec, state): @with_all_phases @spec_state_test def test_12_ok_support(spec, state): - yield from finalize_on_12(spec, state, 3, True) + yield from finalize_on_12(spec, state, 3, True, False) + + +@with_all_phases +@spec_state_test +def test_12_ok_support_messed_target(spec, state): + yield from finalize_on_12(spec, state, 3, True, True) @with_all_phases @spec_state_test def test_12_poor_support(spec, state): - yield from finalize_on_12(spec, state, 3, False) + yield from finalize_on_12(spec, state, 3, False, False) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py new file mode 100644 index 000000000..99ba9d284 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py @@ -0,0 +1,206 @@ +from copy import deepcopy + +from eth2spec.test.context import spec_state_test, with_all_phases, spec_test, \ + misc_balances, with_custom_state, default_activation_threshold +from eth2spec.test.helpers.state import ( + next_epoch, + next_slot, +) +from eth2spec.test.helpers.attestations import ( + add_attestations_to_state, + get_valid_attestation, +) +from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_with + + +def run_process_rewards_and_penalties(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_rewards_and_penalties') + + +@with_all_phases +@spec_state_test +def test_genesis_epoch_no_attestations_no_penalties(spec, state): + pre_state = deepcopy(state) + + assert spec.compute_epoch_at_slot(state.slot) == spec.GENESIS_EPOCH + + yield from run_process_rewards_and_penalties(spec, state) + + for index in range(len(pre_state.validators)): + assert state.balances[index] == pre_state.balances[index] + + +@with_all_phases +@spec_state_test +def test_genesis_epoch_full_attestations_no_rewards(spec, state): + attestations = [] + for slot in range(spec.SLOTS_PER_EPOCH - 1): + # create an attestation for each slot + if slot < spec.SLOTS_PER_EPOCH: + attestation = get_valid_attestation(spec, state, signed=True) + attestations.append(attestation) + # fill each created slot in state after inclusion delay + if slot - spec.MIN_ATTESTATION_INCLUSION_DELAY >= 0: + include_att = attestations[slot - spec.MIN_ATTESTATION_INCLUSION_DELAY] + add_attestations_to_state(spec, state, [include_att], state.slot) + next_slot(spec, state) + + # ensure has not cross the epoch boundary + assert spec.compute_epoch_at_slot(state.slot) == spec.GENESIS_EPOCH + + pre_state = deepcopy(state) + + yield from run_process_rewards_and_penalties(spec, state) + + for index in range(len(pre_state.validators)): + assert state.balances[index] == pre_state.balances[index] + + +def prepare_state_with_full_attestations(spec, state): + attestations = [] + for slot in range(spec.SLOTS_PER_EPOCH + spec.MIN_ATTESTATION_INCLUSION_DELAY): + # create an attestation for each slot in epoch + if slot < spec.SLOTS_PER_EPOCH: + attestation = get_valid_attestation(spec, state, signed=True) + attestations.append(attestation) + # fill each created slot in state after inclusion delay + if slot - spec.MIN_ATTESTATION_INCLUSION_DELAY >= 0: + include_att = attestations[slot - spec.MIN_ATTESTATION_INCLUSION_DELAY] + add_attestations_to_state(spec, state, [include_att], state.slot) + next_slot(spec, state) + + assert spec.compute_epoch_at_slot(state.slot) == spec.GENESIS_EPOCH + 1 + assert len(state.previous_epoch_attestations) == spec.SLOTS_PER_EPOCH + + return attestations + + +@with_all_phases +@spec_state_test +def test_full_attestations(spec, state): + attestations = prepare_state_with_full_attestations(spec, state) + + pre_state = deepcopy(state) + + yield from run_process_rewards_and_penalties(spec, state) + + attesting_indices = spec.get_unslashed_attesting_indices(state, attestations) + assert len(attesting_indices) > 0 + for index in range(len(pre_state.validators)): + if index in attesting_indices: + assert state.balances[index] > pre_state.balances[index] + else: + assert state.balances[index] < pre_state.balances[index] + + +@with_all_phases +@spec_test +@with_custom_state(balances_fn=misc_balances, threshold_fn=default_activation_threshold) +def test_full_attestations_misc_balances(spec, state): + attestations = prepare_state_with_full_attestations(spec, state) + + pre_state = deepcopy(state) + + yield from run_process_rewards_and_penalties(spec, state) + + attesting_indices = spec.get_unslashed_attesting_indices(state, attestations) + assert len(attesting_indices) > 0 + assert len(attesting_indices) != len(pre_state.validators) + for index in range(len(pre_state.validators)): + if index in attesting_indices: + assert state.balances[index] > pre_state.balances[index] + elif spec.is_active_validator(pre_state.validators[index], spec.compute_epoch_at_slot(state.slot)): + assert state.balances[index] < pre_state.balances[index] + else: + assert state.balances[index] == pre_state.balances[index] + + +@with_all_phases +@spec_state_test +def test_no_attestations_all_penalties(spec, state): + next_epoch(spec, state) + pre_state = deepcopy(state) + + assert spec.compute_epoch_at_slot(state.slot) == spec.GENESIS_EPOCH + 1 + + yield from run_process_rewards_and_penalties(spec, state) + + for index in range(len(pre_state.validators)): + assert state.balances[index] < pre_state.balances[index] + + +@with_all_phases +@spec_state_test +def test_duplicate_attestation(spec, state): + """ + Although duplicate attestations can be included on-chain, they should only + be rewarded for once. + This test addresses this issue found at Interop + https://github.com/djrtwo/interop-test-cases/tree/master/tests/prysm_16_duplicate_attestation_rewards + """ + attestation = get_valid_attestation(spec, state, signed=True) + + indexed_attestation = spec.get_indexed_attestation(state, attestation) + participants = indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices + + assert len(participants) > 0 + + single_state = deepcopy(state) + dup_state = deepcopy(state) + + inclusion_slot = state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY + add_attestations_to_state(spec, single_state, [attestation], inclusion_slot) + add_attestations_to_state(spec, dup_state, [attestation, attestation], inclusion_slot) + + next_epoch(spec, single_state) + next_epoch(spec, dup_state) + + # Run non-duplicate inclusion rewards for comparision. Do not yield test vectors + for _ in run_process_rewards_and_penalties(spec, single_state): + pass + + # Output duplicate inclusion to test vectors + yield from run_process_rewards_and_penalties(spec, dup_state) + + for index in participants: + assert state.balances[index] < single_state.balances[index] + assert single_state.balances[index] == dup_state.balances[index] + + +@with_all_phases +@spec_state_test +# Case when some eligible attestations are slashed. Modifies attesting_balance and consequently rewards/penalties. +def test_attestations_some_slashed(spec, state): + attestations = [] + for slot in range(spec.SLOTS_PER_EPOCH + spec.MIN_ATTESTATION_INCLUSION_DELAY): + # create an attestation for each slot in epoch + if slot < spec.SLOTS_PER_EPOCH: + attestation = get_valid_attestation(spec, state, signed=True) + attestations.append(attestation) + # fill each created slot in state after inclusion delay + if slot - spec.MIN_ATTESTATION_INCLUSION_DELAY >= 0: + include_att = attestations[slot - spec.MIN_ATTESTATION_INCLUSION_DELAY] + add_attestations_to_state(spec, state, [include_att], state.slot) + next_slot(spec, state) + + attesting_indices_before_slashings = list(spec.get_unslashed_attesting_indices(state, attestations)) + + # Slash maximum amount of validators allowed per epoch. + for i in range(spec.MIN_PER_EPOCH_CHURN_LIMIT): + spec.slash_validator(state, attesting_indices_before_slashings[i]) + + assert spec.compute_epoch_at_slot(state.slot) == spec.GENESIS_EPOCH + 1 + assert len(state.previous_epoch_attestations) == spec.SLOTS_PER_EPOCH + + pre_state = deepcopy(state) + + yield from run_process_rewards_and_penalties(spec, state) + + attesting_indices = spec.get_unslashed_attesting_indices(state, attestations) + assert len(attesting_indices) > 0 + assert len(attesting_indices_before_slashings) - len(attesting_indices) == spec.MIN_PER_EPOCH_CHURN_LIMIT + for index in range(len(pre_state.validators)): + if index in attesting_indices: + assert state.balances[index] > pre_state.balances[index] + else: + assert state.balances[index] < pre_state.balances[index] diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 886f9bf6a..bffb1c1fc 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -378,40 +378,6 @@ def test_voluntary_exit(spec, state): assert state.validators[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH -# @with_all_phases -# @spec_state_test -# def test_transfer(spec, state): - # overwrite default 0 to test - # spec.MAX_TRANSFERS = 1 - - # sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - # amount = get_balance(state, sender_index) - - # transfer = get_valid_transfer(spec, state, state.slot + 1, sender_index, amount, signed=True) - # recipient_index = transfer.recipient - # pre_transfer_recipient_balance = get_balance(state, recipient_index) - - # un-activate so validator can transfer - # state.validators[sender_index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - - # yield 'pre', state - - # Add to state via block transition - # block = build_empty_block_for_next_slot(spec, state) - # block.body.transfers.append(transfer) - # sign_block(spec, state, block) - - # state_transition_and_sign_block(spec, state, block) - - # yield 'blocks', [block] - # yield 'post', state - - # sender_balance = get_balance(state, sender_index) - # recipient_balance = get_balance(state, recipient_index) - # assert sender_balance == 0 - # assert recipient_balance == pre_transfer_recipient_balance + amount - - @with_all_phases @spec_state_test def test_balance_driven_status_transitions(spec, state): @@ -509,6 +475,8 @@ def test_eth1_data_votes_no_consensus(spec, state): if spec.SLOTS_PER_ETH1_VOTING_PERIOD > 16: return + pre_eth1_hash = state.eth1_data.block_hash + offset_block = build_empty_block(spec, state, slot=spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1) sign_block(spec, state, offset_block) state_transition_and_sign_block(spec, state, offset_block) @@ -528,7 +496,7 @@ def test_eth1_data_votes_no_consensus(spec, state): blocks.append(block) assert len(state.eth1_data_votes) == spec.SLOTS_PER_ETH1_VOTING_PERIOD - assert state.eth1_data.block_hash == b'\x00' * 32 + assert state.eth1_data.block_hash == pre_eth1_hash yield 'blocks', blocks yield 'post', state