From c10e59bdf7b2fad73d967061dcdc0519ef1fa06f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 8 Apr 2020 10:32:16 +0800 Subject: [PATCH 01/70] Fix `INACTIVITY_PENALTY_QUOTIENT` The amount of inactivity penalty was adjusted to half since we were applying penalty for missing FFG target and source. But now we only apply it for missing target, so `INACTIVITY_PENALTY_QUOTIENT` should be `2**24`. --- configs/mainnet.yaml | 4 ++-- configs/minimal.yaml | 4 ++-- specs/phase0/beacon-chain.md | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 6d71cfa47..3d2de75f0 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -122,8 +122,8 @@ BASE_REWARD_FACTOR: 64 WHISTLEBLOWER_REWARD_QUOTIENT: 512 # 2**3 (= 8) PROPOSER_REWARD_QUOTIENT: 8 -# 2**25 (= 33,554,432) -INACTIVITY_PENALTY_QUOTIENT: 33554432 +# 2**24 (= 16,777,216) +INACTIVITY_PENALTY_QUOTIENT: 16777216 # 2**5 (= 32) MIN_SLASHING_PENALTY_QUOTIENT: 32 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 9daf428b4..b39a4fc01 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -122,8 +122,8 @@ BASE_REWARD_FACTOR: 64 WHISTLEBLOWER_REWARD_QUOTIENT: 512 # 2**3 (= 8) PROPOSER_REWARD_QUOTIENT: 8 -# 2**25 (= 33,554,432) -INACTIVITY_PENALTY_QUOTIENT: 33554432 +# 2**24 (= 16,777,216) +INACTIVITY_PENALTY_QUOTIENT: 16777216 # 2**5 (= 32) MIN_SLASHING_PENALTY_QUOTIENT: 32 diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 23fa5ceee..c841d2dbf 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -242,10 +242,10 @@ The following values are (non-configurable) constants used throughout the specif | `BASE_REWARD_FACTOR` | `2**6` (= 64) | | `WHISTLEBLOWER_REWARD_QUOTIENT` | `2**9` (= 512) | | `PROPOSER_REWARD_QUOTIENT` | `2**3` (= 8) | -| `INACTIVITY_PENALTY_QUOTIENT` | `2**25` (= 33,554,432) | +| `INACTIVITY_PENALTY_QUOTIENT` | `2**24` (= 16,777,216) | | `MIN_SLASHING_PENALTY_QUOTIENT` | `2**5` (= 32) | -- The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**12 epochs` (about 18 days) is the time it takes the inactivity penalty to reduce the balance of non-participating validators to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline validators after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)`; so after `INVERSE_SQRT_E_DROP_TIME` epochs, it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. +- The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**12` epochs (about 18 days) is the time it takes the inactivity penalty to reduce the balance of non-participating validators to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline validators after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)`; so after `INVERSE_SQRT_E_DROP_TIME` epochs, it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. ### Max operations per block From 890c27d091bb74947ba3fdd6b86464fddf9cc1ef Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 9 Apr 2020 17:33:14 +0800 Subject: [PATCH 02/70] The input parameter `index` in `compute_shuffled_index` is the position of the given list, not `ValidatorIndex` --- specs/phase0/beacon-chain.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 23fa5ceee..8a51b5e8b 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -722,9 +722,9 @@ def is_valid_merkle_branch(leaf: Bytes32, branch: Sequence[Bytes32], depth: uint #### `compute_shuffled_index` ```python -def compute_shuffled_index(index: ValidatorIndex, index_count: uint64, seed: Bytes32) -> ValidatorIndex: +def compute_shuffled_index(index: uint64, index_count: uint64, seed: Bytes32) -> uint64: """ - Return the shuffled validator index corresponding to ``seed`` (and ``index_count``). + Return the shuffled index corresponding to ``seed`` (and ``index_count``). """ assert index < index_count @@ -732,14 +732,14 @@ def compute_shuffled_index(index: ValidatorIndex, index_count: uint64, seed: Byt # See the 'generalized domain' algorithm on page 3 for current_round in range(SHUFFLE_ROUND_COUNT): pivot = bytes_to_int(hash(seed + int_to_bytes(current_round, length=1))[0:8]) % index_count - flip = ValidatorIndex((pivot + index_count - index) % index_count) + flip = (pivot + index_count - index) % index_count position = max(index, flip) source = hash(seed + int_to_bytes(current_round, length=1) + int_to_bytes(position // 256, length=4)) byte = source[(position % 256) // 8] bit = (byte >> (position % 8)) % 2 index = flip if bit else index - return ValidatorIndex(index) + return index ``` #### `compute_proposer_index` @@ -753,11 +753,11 @@ def compute_proposer_index(state: BeaconState, indices: Sequence[ValidatorIndex] MAX_RANDOM_BYTE = 2**8 - 1 i = 0 while True: - candidate_index = indices[compute_shuffled_index(ValidatorIndex(i % len(indices)), len(indices), seed)] + candidate_index = indices[compute_shuffled_index(i % len(indices), len(indices), seed)] random_byte = hash(seed + int_to_bytes(i // 32, length=8))[i % 32] effective_balance = state.validators[candidate_index].effective_balance if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: - return ValidatorIndex(candidate_index) + return candidate_index i += 1 ``` From b2f6325db339e630feb9a459099bc8f0c5f4ce49 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 9 Apr 2020 17:48:12 +0800 Subject: [PATCH 03/70] Fix `compute_committee` --- specs/phase0/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 8a51b5e8b..82257947d 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -773,7 +773,7 @@ def compute_committee(indices: Sequence[ValidatorIndex], """ start = (len(indices) * index) // count end = (len(indices) * (index + 1)) // count - return [indices[compute_shuffled_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)] + return [indices[compute_shuffled_index(i, len(indices), seed)] for i in range(start, end)] ``` #### `compute_epoch_at_slot` From 6fdee7547502acc8f872031cc8d6f3afb502b83c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 17 Apr 2020 17:27:57 +0800 Subject: [PATCH 04/70] Fix phase0 types --- setup.py | 10 +++++++--- specs/phase0/beacon-chain.md | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index d1c62fb72..d78161b29 100644 --- a/setup.py +++ b/setup.py @@ -108,7 +108,7 @@ SSZObject = TypeVar('SSZObject', bound=View) PHASE1_IMPORTS = '''from eth2spec.phase0 import spec as phase0 from eth2spec.config.config_util import apply_constants_config from typing import ( - Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable + Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable, Optional ) from dataclasses import ( @@ -146,8 +146,11 @@ _hash = hash hash_cache: Dict[bytes, Bytes32] = {} -def get_eth1_data(distance: uint64) -> Bytes32: - return hash(distance) +def get_eth1_data(block: Eth1Block) -> Eth1Data: + """ + A stub function return mocking Eth1Data. + """ + return Eth1Data(block_hash=hash_tree_root(block)) def hash(x: bytes) -> Bytes32: # type: ignore @@ -373,6 +376,7 @@ class PySpecCommand(Command): self.md_doc_paths = """ specs/phase0/beacon-chain.md specs/phase0/fork-choice.md + specs/phase0/validator.md specs/phase1/custody-game.md specs/phase1/beacon-chain.md specs/phase1/fraud-proofs.md diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 2ee1d46bc..5c84e41f7 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1125,7 +1125,7 @@ def slash_validator(state: BeaconState, whistleblower_reward = Gwei(validator.effective_balance // WHISTLEBLOWER_REWARD_QUOTIENT) proposer_reward = Gwei(whistleblower_reward // PROPOSER_REWARD_QUOTIENT) increase_balance(state, proposer_index, proposer_reward) - increase_balance(state, whistleblower_index, whistleblower_reward - proposer_reward) + increase_balance(state, whistleblower_index, Gwei(whistleblower_reward - proposer_reward)) ``` ## Genesis @@ -1229,7 +1229,7 @@ def process_slots(state: BeaconState, slot: Slot) -> None: # Process epoch on the start slot of the next epoch if (state.slot + 1) % SLOTS_PER_EPOCH == 0: process_epoch(state) - state.slot += Slot(1) + state.slot = Slot(state.slot + 1) ``` ```python From cafd98b9e85550acc091c797f1deb1027b802b9a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 17 Apr 2020 18:15:46 +0800 Subject: [PATCH 05/70] Fix utils.hash_function typing --- .../pyspec/eth2spec/utils/hash_function.py | 23 +++++-------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/tests/core/pyspec/eth2spec/utils/hash_function.py b/tests/core/pyspec/eth2spec/utils/hash_function.py index 2c9b5a579..627f9b990 100644 --- a/tests/core/pyspec/eth2spec/utils/hash_function.py +++ b/tests/core/pyspec/eth2spec/utils/hash_function.py @@ -1,28 +1,17 @@ from hashlib import sha256 +from typing import Dict, Union ZERO_BYTES32 = b'\x00' * 32 -def _hash(x): +def _hash(x: Union[bytes, bytearray, memoryview]) -> bytes: return sha256(x).digest() -# Minimal collection of (key, value) pairs, for fast hash-retrieval, to save on repetitive computation cost. -# Key = the hash input -# Value = the hash output -hash_cache = [] +hash_cache: Dict[bytes, bytes] = {} -def add_zero_hashes_to_cache(): - zerohashes = [(None, ZERO_BYTES32)] - for layer in range(1, 32): - k = zerohashes[layer - 1][1] + zerohashes[layer - 1][1] - zerohashes.append((k, _hash(k))) - hash_cache.extend(zerohashes[1:]) - - -def hash(x): - for (k, h) in hash_cache: - if x == k: - return h +def hash(x: bytes) -> bytes: + if x in hash_cache: + return hash_cache[x] return _hash(x) From 3575b18cd4b714427f876f646570a76df66f5f11 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 17 Apr 2020 18:18:22 +0800 Subject: [PATCH 06/70] Fix `config_util.py` typing --- tests/core/pyspec/eth2spec/config/config_util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/config/config_util.py b/tests/core/pyspec/eth2spec/config/config_util.py index 4c5768a29..c43c1521b 100644 --- a/tests/core/pyspec/eth2spec/config/config_util.py +++ b/tests/core/pyspec/eth2spec/config/config_util.py @@ -24,12 +24,12 @@ def apply_constants_config(spec_globals: Dict[str, Any], warn_if_unknown: bool = # Load presets from a file, and then prepares the global config setting. This does not apply the config. # To apply the config, reload the spec module (it will re-initialize with the config taken from here). -def prepare_config(configs_path, config_name): +def prepare_config(configs_path: str, config_name: str) -> None: global config config = load_config_file(configs_path, config_name) -def load_config_file(configs_dir, presets_name) -> Dict[str, Any]: +def load_config_file(configs_dir: str, presets_name: str) -> Dict[str, Any]: """ Loads the given preset :param presets_name: The name of the presets. (lowercase snake_case) @@ -38,7 +38,7 @@ def load_config_file(configs_dir, presets_name) -> Dict[str, Any]: path = Path(join(configs_dir, presets_name + '.yaml')) yaml = YAML(typ='base') loaded = yaml.load(path) - out = dict() + out: Dict[str, Any] = dict() for k, v in loaded.items(): if isinstance(v, list): # Clean up integer values. YAML parser renders lists of ints as list of str From 5929aac799e8647b6905dd829948efcdfa5849c9 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 21 Apr 2020 17:56:27 +0100 Subject: [PATCH 07/70] Cosmetic changes from #1737 --- specs/phase0/beacon-chain.md | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 5c84e41f7..111655285 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -35,7 +35,7 @@ - [`DepositMessage`](#depositmessage) - [`DepositData`](#depositdata) - [`BeaconBlockHeader`](#beaconblockheader) - - [`SigningRoot`](#signingroot) + - [`SigningData`](#signingdata) - [Beacon operations](#beacon-operations) - [`ProposerSlashing`](#proposerslashing) - [`AttesterSlashing`](#attesterslashing) @@ -191,7 +191,6 @@ The following values are (non-configurable) constants used throughout the specif | `HYSTERESIS_DOWNWARD_MULTIPLIER` | `1` | | `HYSTERESIS_UPWARD_MULTIPLIER` | `5` | - - For the safety of committees, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](http://web.archive.org/web/20190504131341/https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) ### Gwei values @@ -269,7 +268,6 @@ The following values are (non-configurable) constants used throughout the specif | `DOMAIN_SELECTION_PROOF` | `DomainType('0x05000000')` | | `DOMAIN_AGGREGATE_AND_PROOF` | `DomainType('0x06000000')` | - ## Containers The following types are [SimpleSerialize (SSZ)](../../ssz/simple-serialize.md) containers. @@ -399,10 +397,10 @@ class BeaconBlockHeader(Container): body_root: Root ``` -#### `SigningRoot` +#### `SigningData` ```python -class SigningRoot(Container): +class SigningData(Container): object_root: Root domain: Domain ``` @@ -852,13 +850,12 @@ def compute_domain(domain_type: DomainType, fork_version: Version=None, genesis_ ```python def compute_signing_root(ssz_object: SSZObject, domain: Domain) -> Root: """ - Return the signing root of an object by calculating the root of the object-domain tree. + Return the signing root for the corresponding signing data. """ - domain_wrapped_object = SigningRoot( + return hash_tree_root(SigningData( object_root=hash_tree_root(ssz_object), domain=domain, - ) - return hash_tree_root(domain_wrapped_object) + )) ``` ### Beacon state accessors From 9bbac0d2ccbde7e82afcee760b9fb0a47e16244b Mon Sep 17 00:00:00 2001 From: Aditya Asgaonkar Date: Tue, 21 Apr 2020 18:50:02 -0700 Subject: [PATCH 08/70] Added consistency check for FFG & LMD vote in validate_on_atttestation(), fixes #1636, fixes #1456, fixes #1408 --- specs/phase0/fork-choice.md | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index c42609be0..f844f735b 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -22,6 +22,7 @@ - [`get_latest_attesting_balance`](#get_latest_attesting_balance) - [`filter_block_tree`](#filter_block_tree) - [`get_filtered_block_tree`](#get_filtered_block_tree) + - [`is_descendant_block`](#is_descendant_block) - [`get_head`](#get_head) - [`should_update_justified_checkpoint`](#should_update_justified_checkpoint) - [`on_attestation` helpers](#on_attestation-helpers) @@ -162,7 +163,7 @@ def get_latest_attesting_balance(store: Store, root: Root) -> Gwei: active_indices = get_active_validator_indices(state, get_current_epoch(state)) return Gwei(sum( state.validators[i].effective_balance for i in active_indices - if (i in store.latest_messages + if (i in store.latest_messages and get_ancestor(store, store.latest_messages[i].root, store.blocks[root].slot) == root) )) ``` @@ -220,6 +221,28 @@ def get_filtered_block_tree(store: Store) -> Dict[Root, BeaconBlock]: return blocks ``` +#### `is_descendant_block` + +```python +def is_descendant_block(store: Store, base_root: Root, descendant_root: Root) -> bool: + """ + Checks if the block with root ``descendant_root`` is a descendant of the block with root ``base_root`` + """ + descendants = [base_root] + + # Traverse the descendants block tree and check if ``descendant_root`` is encountered + while(descendants): + if descendants[0] == descendant_root: + return True + descendants.extend([ + root for root in store.blocks.keys() + if store.blocks[root].parent_root == descendants[0] + ]) + descendants = descendants[1:] + + return False +``` + #### `get_head` ```python @@ -286,6 +309,9 @@ def validate_on_attestation(store: Store, attestation: Attestation) -> None: # Attestations must not be for blocks in the future. If not, the attestation should not be considered assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot + # FFG and LMD vote must be consistent with each other + assert is_descendant_block(store, target.root, attestation.data.beacon_block_root) + # Attestations can only affect the fork choice of subsequent slots. # Delay consideration in the fork choice until their slot is in the past. assert get_current_slot(store) >= attestation.data.slot + 1 From 9acea519382c0fa8752328b106423890f8d1f01c Mon Sep 17 00:00:00 2001 From: Aditya Asgaonkar Date: Sat, 25 Apr 2020 14:17:28 -0700 Subject: [PATCH 09/70] Simplified by re-using get_ancestor() --- specs/phase0/fork-choice.md | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index f844f735b..42d0cea11 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -22,7 +22,6 @@ - [`get_latest_attesting_balance`](#get_latest_attesting_balance) - [`filter_block_tree`](#filter_block_tree) - [`get_filtered_block_tree`](#get_filtered_block_tree) - - [`is_descendant_block`](#is_descendant_block) - [`get_head`](#get_head) - [`should_update_justified_checkpoint`](#should_update_justified_checkpoint) - [`on_attestation` helpers](#on_attestation-helpers) @@ -221,28 +220,6 @@ def get_filtered_block_tree(store: Store) -> Dict[Root, BeaconBlock]: return blocks ``` -#### `is_descendant_block` - -```python -def is_descendant_block(store: Store, base_root: Root, descendant_root: Root) -> bool: - """ - Checks if the block with root ``descendant_root`` is a descendant of the block with root ``base_root`` - """ - descendants = [base_root] - - # Traverse the descendants block tree and check if ``descendant_root`` is encountered - while(descendants): - if descendants[0] == descendant_root: - return True - descendants.extend([ - root for root in store.blocks.keys() - if store.blocks[root].parent_root == descendants[0] - ]) - descendants = descendants[1:] - - return False -``` - #### `get_head` ```python @@ -310,7 +287,8 @@ def validate_on_attestation(store: Store, attestation: Attestation) -> None: assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot # FFG and LMD vote must be consistent with each other - assert is_descendant_block(store, target.root, attestation.data.beacon_block_root) + target_slot = compute_start_slot_at_epoch(target.epoch) + assert target.root == get_ancestor(store, attestation.data.beacon_block_root, target_slot) # Attestations can only affect the fork choice of subsequent slots. # Delay consideration in the fork choice until their slot is in the past. From 4d980aec71f93f2f788fc8ea76bec14521d53abc Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 23 Apr 2020 13:23:31 +0800 Subject: [PATCH 10/70] Fix validator guide 1. Avoid negative computation in `is_candidate_block` 2. Fix `get_block_signature`: avoid extra casting; it's simpler to use BeaconBlock instead of BeaconHeader --- specs/phase0/validator.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index bc7510403..4576c4b90 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -281,8 +281,8 @@ def voting_period_start_time(state: BeaconState) -> uint64: ```python def is_candidate_block(block: Eth1Block, period_start: uint64) -> bool: return ( - block.timestamp <= period_start - SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE - and block.timestamp >= period_start - SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE * 2 + block.timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE <= period_start + and block.timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE * 2 >= period_start ) ``` @@ -350,9 +350,9 @@ def compute_new_state_root(state: BeaconState, block: BeaconBlock) -> Root: `signed_block = SignedBeaconBlock(message=block, signature=block_signature)`, where `block_signature` is obtained from: ```python -def get_block_signature(state: BeaconState, header: BeaconBlockHeader, privkey: int) -> BLSSignature: - domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(header.slot)) - signing_root = compute_signing_root(header, domain) +def get_block_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(block.slot)) + signing_root = compute_signing_root(block, domain) return bls.Sign(privkey, signing_root) ``` From 303d7d5adb35e51a49c570e15256763c136de8a1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 23 Apr 2020 13:27:00 +0800 Subject: [PATCH 11/70] Add validator guide tests 1. "Becoming a validator" 2. "Validator assignments" 3. "Beacon chain responsibilities: Block proposal" --- .../test/validator/test_validator_unittest.py | 182 ++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py diff --git a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py new file mode 100644 index 000000000..019221eee --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py @@ -0,0 +1,182 @@ +from eth2spec.test.context import spec_state_test, never_bls, with_all_phases +from eth2spec.test.helpers.block import build_empty_block +from eth2spec.test.helpers.deposits import prepare_state_and_deposit +from eth2spec.test.helpers.keys import privkeys, pubkeys +from eth2spec.test.helpers.state import next_epoch +from eth2spec.utils import bls + + +def run_is_candidate_block(spec, eth1_block, period_start, success): + result = spec.is_candidate_block(eth1_block, period_start) + if success: + assert result + else: + assert not result + + +def get_min_new_period_epochs(spec): + return int( + spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE * 2 # to seconds + / spec.SECONDS_PER_SLOT / spec.SLOTS_PER_EPOCH + ) + + +# +# Becoming a validator +# + + +@with_all_phases +@spec_state_test +@never_bls +def test_check_if_validator_active(spec, state): + active_validator_index = len(state.validators) - 1 + assert spec.check_if_validator_active(state, active_validator_index) + new_validator_index = len(state.validators) + amount = spec.MAX_EFFECTIVE_BALANCE + deposit = prepare_state_and_deposit(spec, state, new_validator_index, amount, signed=True) + spec.process_deposit(state, deposit) + assert not spec.check_if_validator_active(state, new_validator_index) + + +# +# Validator assignments +# + + +@with_all_phases +@spec_state_test +@never_bls +def test_get_committee_assignment(spec, state): + epoch = spec.get_current_epoch(state) + validator_index = len(state.validators) - 1 + assignment = spec.get_committee_assignment(state, epoch, validator_index) + committee, committee_index, slot = assignment + assert spec.compute_epoch_at_slot(slot) == epoch + assert committee == spec.get_beacon_committee(state, slot, committee_index) + assert committee_index < spec.get_committee_count_at_slot(state, slot) + + +@with_all_phases +@spec_state_test +@never_bls +def test_is_proposer(spec, state): + proposer_index = spec.get_beacon_proposer_index(state) + assert spec.is_proposer(state, proposer_index) + + proposer_index = proposer_index + 1 % len(state.validators) + assert not spec.is_proposer(state, proposer_index) + + +# +# Beacon chain responsibilities +# + + +# Block proposal + + +@with_all_phases +@spec_state_test +def test_get_epoch_signature(spec, state): + block = spec.BeaconBlock() + privkey = privkeys[0] + pubkey = pubkeys[0] + signature = spec.get_epoch_signature(state, block, privkey) + domain = spec.get_domain(state, spec.DOMAIN_RANDAO, spec.compute_epoch_at_slot(block.slot)) + signing_root = spec.compute_signing_root(spec.compute_epoch_at_slot(block.slot), domain) + assert bls.Verify(pubkey, signing_root, signature) + + +@with_all_phases +@spec_state_test +def test_is_candidate_block(spec, state): + period_start = spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE * 2 + 1000 + run_is_candidate_block( + spec, + spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE), + period_start, + success=True, + ) + run_is_candidate_block( + spec, + spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE + 1), + period_start, + success=False, + ) + run_is_candidate_block( + spec, + spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE * 2), + period_start, + success=True, + ) + run_is_candidate_block( + spec, + spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE * 2 - 1), + period_start, + success=False, + ) + + +@with_all_phases +@spec_state_test +def test_get_eth1_data_default_vote(spec, state): + min_new_period_epochs = get_min_new_period_epochs(spec) + for _ in range(min_new_period_epochs): + next_epoch(spec, state) + + state.eth1_data_votes = () + eth1_chain = [] + eth1_data = spec.get_eth1_vote(state, eth1_chain) + assert eth1_data == state.eth1_data + + +@with_all_phases +@spec_state_test +def test_get_eth1_data_consensus_vote(spec, state): + min_new_period_epochs = get_min_new_period_epochs(spec) + for _ in range(min_new_period_epochs): + next_epoch(spec, state) + + period_start = spec.voting_period_start_time(state) + votes_length = spec.get_current_epoch(state) % spec.EPOCHS_PER_ETH1_VOTING_PERIOD + state.eth1_data_votes = () + eth1_chain = [] + eth1_data_votes = [] + block = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE) + for i in range(votes_length): + eth1_chain.append(block) + eth1_data_votes.append(spec.get_eth1_data(block)) + + state.eth1_data_votes = eth1_data_votes + eth1_data = spec.get_eth1_vote(state, eth1_chain) + print(state.eth1_data_votes) + assert eth1_data.block_hash == block.hash_tree_root() + + +@with_all_phases +@spec_state_test +def test_compute_new_state_root(spec, state): + pre_state = state.copy() + post_state = state.copy() + block = build_empty_block(spec, state, state.slot + 1) + state_root = spec.compute_new_state_root(state, block) + + assert state_root != pre_state.hash_tree_root() + + # dumb verification + spec.process_slots(post_state, block.slot) + spec.process_block(post_state, block) + assert state_root == post_state.hash_tree_root() + + +@with_all_phases +@spec_state_test +def test_get_block_signature(spec, state): + privkey = privkeys[0] + pubkey = pubkeys[0] + block = build_empty_block(spec, state) + signature = spec.get_block_signature(state, block, privkey) + domain = spec.get_domain(state, spec.DOMAIN_BEACON_PROPOSER, spec.compute_epoch_at_slot(block.slot)) + signing_root = spec.compute_signing_root(block, domain) + assert bls.Verify(pubkey, signing_root, signature) From bdae27e317b44c1ab031a0cbdb5b4d61d0fe05a7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 23 Apr 2020 15:08:36 +0800 Subject: [PATCH 12/70] Add bls.AggregatePKs helper --- tests/core/pyspec/eth2spec/utils/bls.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/core/pyspec/eth2spec/utils/bls.py b/tests/core/pyspec/eth2spec/utils/bls.py index 83371ac62..3b648fac9 100644 --- a/tests/core/pyspec/eth2spec/utils/bls.py +++ b/tests/core/pyspec/eth2spec/utils/bls.py @@ -51,3 +51,8 @@ def Sign(SK, message): @only_with_bls(alt_return=STUB_COORDINATES) def signature_to_G2(signature): return _signature_to_G2(signature) + + +@only_with_bls(alt_return=STUB_PUBKEY) +def AggregatePKs(pubkeys): + return bls._AggregatePKs(pubkeys) From 8adc15e83de9ab6c0ac10b93aa739df2be989704 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 23 Apr 2020 15:09:23 +0800 Subject: [PATCH 13/70] Add validator guide tests 1. "Beacon chain responsibilities: Attesting" 2. "Beacon chain responsibilities: Attestation aggregation" --- .../test/validator/test_validator_unittest.py | 118 +++++++++++++++++- 1 file changed, 116 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py index 019221eee..4f6697f2f 100644 --- a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py +++ b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py @@ -1,9 +1,11 @@ from eth2spec.test.context import spec_state_test, never_bls, with_all_phases +from eth2spec.test.helpers.attestations import build_attestation_data from eth2spec.test.helpers.block import build_empty_block from eth2spec.test.helpers.deposits import prepare_state_and_deposit from eth2spec.test.helpers.keys import privkeys, pubkeys from eth2spec.test.helpers.state import next_epoch from eth2spec.utils import bls +from eth2spec.utils.ssz.ssz_typing import Bitlist def run_is_candidate_block(spec, eth1_block, period_start, success): @@ -21,6 +23,14 @@ def get_min_new_period_epochs(spec): ) +def get_mock_aggregate(spec): + return spec.Attestation( + data=spec.AttestationData( + slot=10, + ) + ) + + # # Becoming a validator # @@ -47,7 +57,7 @@ def test_check_if_validator_active(spec, state): @with_all_phases @spec_state_test @never_bls -def test_get_committee_assignment(spec, state): +def test_get_committee_assignment_current_epoch(spec, state): epoch = spec.get_current_epoch(state) validator_index = len(state.validators) - 1 assignment = spec.get_committee_assignment(state, epoch, validator_index) @@ -150,7 +160,6 @@ def test_get_eth1_data_consensus_vote(spec, state): state.eth1_data_votes = eth1_data_votes eth1_data = spec.get_eth1_vote(state, eth1_chain) - print(state.eth1_data_votes) assert eth1_data.block_hash == block.hash_tree_root() @@ -180,3 +189,108 @@ def test_get_block_signature(spec, state): domain = spec.get_domain(state, spec.DOMAIN_BEACON_PROPOSER, spec.compute_epoch_at_slot(block.slot)) signing_root = spec.compute_signing_root(block, domain) assert bls.Verify(pubkey, signing_root, signature) + + +# Attesting + + +@with_all_phases +@spec_state_test +def test_get_attestation_signature(spec, state): + privkey = privkeys[0] + pubkey = pubkeys[0] + attestation_data = spec.AttestationData(slot=10) + signature = spec.get_attestation_signature(state, attestation_data, privkey) + domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) + signing_root = spec.compute_signing_root(attestation_data, domain) + assert bls.Verify(pubkey, signing_root, signature) + + +# Attestation aggregation + + +@with_all_phases +@spec_state_test +def test_get_slot_signature(spec, state): + privkey = privkeys[0] + pubkey = pubkeys[0] + slot = 10 + signature = spec.get_slot_signature(state, spec.Slot(slot), privkey) + domain = spec.get_domain(state, spec.DOMAIN_SELECTION_PROOF, spec.compute_epoch_at_slot(slot)) + signing_root = spec.compute_signing_root(spec.Slot(slot), domain) + assert bls.Verify(pubkey, signing_root, signature) + + +@with_all_phases +@spec_state_test +def test_is_aggregator(spec, state): + # TODO: we can test the probabilistic result against `TARGET_AGGREGATORS_PER_COMMITTEE` + # if we have more validators and larger committeee size + slot = state.slot + committee_index = 0 + has_aggregator = False + beacon_committee = spec.get_beacon_committee(state, slot, committee_index) + for validator_index in beacon_committee: + privkey = privkeys[validator_index] + slot_signature = spec.get_slot_signature(state, slot, privkey) + if spec.is_aggregator(state, slot, committee_index, slot_signature): + has_aggregator = True + break + assert has_aggregator + + +@with_all_phases +@spec_state_test +def test_get_aggregate_signature(spec, state): + attestations = [] + pubkeys = [] + slot = state.slot + committee_index = 0 + attestation_data = build_attestation_data(spec, state, slot=slot, index=committee_index) + beacon_committee = spec.get_beacon_committee( + state, + attestation_data.slot, + attestation_data.index, + ) + committee_size = len(beacon_committee) + aggregation_bits = Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE](*([0] * committee_size)) + for i, validator_index in enumerate(beacon_committee): + bits = aggregation_bits + bits[i] = True + attestations.append( + spec.Attestation( + data=attestation_data, + aggregation_bits=bits, + ) + ) + pubkeys.append(state.validators[validator_index].pubkey) + pubkey = bls.AggregatePKs(pubkeys) + signature = spec.get_aggregate_signature(attestations) + domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) + signing_root = spec.compute_signing_root(attestation_data, domain) + assert bls.Verify(pubkey, signing_root, signature) + + +@with_all_phases +@spec_state_test +def test_get_aggregate_and_proof(spec, state): + privkey = privkeys[0] + aggregator_index = spec.ValidatorIndex(10) + aggregate = get_mock_aggregate(spec) + aggregate_and_proof = spec.get_aggregate_and_proof(state, aggregator_index, aggregate, privkey) + assert aggregate_and_proof.aggregator_index == aggregator_index + assert aggregate_and_proof.aggregate == aggregate + assert aggregate_and_proof.selection_proof == spec.get_slot_signature(state, aggregate.data.slot, privkey) + + +@with_all_phases +@spec_state_test +def test_get_aggregate_and_proof_signature(spec, state): + privkey = privkeys[0] + pubkey = pubkeys[0] + aggregate = get_mock_aggregate(spec) + aggregate_and_proof = spec.get_aggregate_and_proof(state, spec.ValidatorIndex(1), aggregate, privkey) + signature = spec.get_aggregate_and_proof_signature(state, aggregate_and_proof, privkey) + domain = spec.get_domain(state, spec.DOMAIN_AGGREGATE_AND_PROOF, spec.compute_epoch_at_slot(aggregate.data.slot)) + signing_root = spec.compute_signing_root(aggregate_and_proof, domain) + assert bls.Verify(pubkey, signing_root, signature) From 70bd73d2b5f4b8c3f19a9397fd983c23fd1eecc6 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 27 Apr 2020 20:47:36 +0800 Subject: [PATCH 14/70] Apply PR feedback from @djrtwo Fix get_eth1_vote test cases --- .../test/validator/test_validator_unittest.py | 164 ++++++++++++++---- 1 file changed, 131 insertions(+), 33 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py index 4f6697f2f..a655cb486 100644 --- a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py +++ b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py @@ -8,12 +8,29 @@ from eth2spec.utils import bls from eth2spec.utils.ssz.ssz_typing import Bitlist -def run_is_candidate_block(spec, eth1_block, period_start, success): - result = spec.is_candidate_block(eth1_block, period_start) - if success: - assert result +def run_get_signature_test(spec, state, obj, domain, get_signature_fn, privkey, pubkey): + signature = get_signature_fn(state, obj, privkey) + signing_root = spec.compute_signing_root(obj, domain) + assert bls.Verify(pubkey, signing_root, signature) + + +def run_get_committee_assignment(spec, state, epoch, validator_index, valid=True): + try: + assignment = spec.get_committee_assignment(state, epoch, validator_index) + committee, committee_index, slot = assignment + assert spec.compute_epoch_at_slot(slot) == epoch + assert committee == spec.get_beacon_committee(state, slot, committee_index) + assert committee_index < spec.get_committee_count_at_slot(state, slot) + assert validator_index in committee + assert valid + except AssertionError: + assert not valid else: - assert not result + assert valid + + +def run_is_candidate_block(spec, eth1_block, period_start, success=True): + assert success == spec.is_candidate_block(eth1_block, period_start) def get_min_new_period_epochs(spec): @@ -60,11 +77,25 @@ def test_check_if_validator_active(spec, state): def test_get_committee_assignment_current_epoch(spec, state): epoch = spec.get_current_epoch(state) validator_index = len(state.validators) - 1 - assignment = spec.get_committee_assignment(state, epoch, validator_index) - committee, committee_index, slot = assignment - assert spec.compute_epoch_at_slot(slot) == epoch - assert committee == spec.get_beacon_committee(state, slot, committee_index) - assert committee_index < spec.get_committee_count_at_slot(state, slot) + run_get_committee_assignment(spec, state, epoch, validator_index, valid=True) + + +@with_all_phases +@spec_state_test +@never_bls +def test_get_committee_assignment_next_epoch(spec, state): + epoch = spec.get_current_epoch(state) + 1 + validator_index = len(state.validators) - 1 + run_get_committee_assignment(spec, state, epoch, validator_index, valid=True) + + +@with_all_phases +@spec_state_test +@never_bls +def test_get_committee_assignment_out_bound_epoch(spec, state): + epoch = spec.get_current_epoch(state) + 2 + validator_index = len(state.validators) - 1 + run_get_committee_assignment(spec, state, epoch, validator_index, valid=False) @with_all_phases @@ -92,10 +123,16 @@ def test_get_epoch_signature(spec, state): block = spec.BeaconBlock() privkey = privkeys[0] pubkey = pubkeys[0] - signature = spec.get_epoch_signature(state, block, privkey) domain = spec.get_domain(state, spec.DOMAIN_RANDAO, spec.compute_epoch_at_slot(block.slot)) - signing_root = spec.compute_signing_root(spec.compute_epoch_at_slot(block.slot), domain) - assert bls.Verify(pubkey, signing_root, signature) + run_get_signature_test( + spec=spec, + state=state, + obj=block, + domain=domain, + get_signature_fn=spec.get_epoch_signature, + privkey=privkey, + pubkey=pubkey, + ) @with_all_phases @@ -130,7 +167,7 @@ def test_is_candidate_block(spec, state): @with_all_phases @spec_state_test -def test_get_eth1_data_default_vote(spec, state): +def test_get_eth1_vote_default_vote(spec, state): min_new_period_epochs = get_min_new_period_epochs(spec) for _ in range(min_new_period_epochs): next_epoch(spec, state) @@ -143,24 +180,61 @@ def test_get_eth1_data_default_vote(spec, state): @with_all_phases @spec_state_test -def test_get_eth1_data_consensus_vote(spec, state): +def test_get_eth1_vote_consensus_vote(spec, state): min_new_period_epochs = get_min_new_period_epochs(spec) - for _ in range(min_new_period_epochs): + for _ in range(min_new_period_epochs + 2): next_epoch(spec, state) period_start = spec.voting_period_start_time(state) votes_length = spec.get_current_epoch(state) % spec.EPOCHS_PER_ETH1_VOTING_PERIOD + assert votes_length >= 3 # We need to have the majority vote state.eth1_data_votes = () - eth1_chain = [] + + block_1 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE - 1) + block_2 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE) + eth1_chain = [block_1, block_2] eth1_data_votes = [] - block = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE) + + # Only the first vote is for block_1 + eth1_data_votes.append(spec.get_eth1_data(block_1)) + # Other votes are for block_2 + for _ in range(votes_length - 1): + eth1_data_votes.append(spec.get_eth1_data(block_2)) + + state.eth1_data_votes = eth1_data_votes + eth1_data = spec.get_eth1_vote(state, eth1_chain) + assert eth1_data.block_hash == block_2.hash_tree_root() + + +@with_all_phases +@spec_state_test +def test_get_eth1_vote_tie(spec, state): + min_new_period_epochs = get_min_new_period_epochs(spec) + for _ in range(min_new_period_epochs + 1): + next_epoch(spec, state) + + period_start = spec.voting_period_start_time(state) + votes_length = spec.get_current_epoch(state) % spec.EPOCHS_PER_ETH1_VOTING_PERIOD + assert votes_length > 0 and votes_length % 2 == 0 + + state.eth1_data_votes = () + block_1 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE - 1) + block_2 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE) + eth1_chain = [block_1, block_2] + eth1_data_votes = [] + # Half votes are for block_1, another half votes are for block_2 for i in range(votes_length): - eth1_chain.append(block) + if i % 2 == 0: + block = block_1 + else: + block = block_2 eth1_data_votes.append(spec.get_eth1_data(block)) state.eth1_data_votes = eth1_data_votes eth1_data = spec.get_eth1_vote(state, eth1_chain) - assert eth1_data.block_hash == block.hash_tree_root() + + # Tiebreak by smallest distance -> eth1_chain[0] + assert eth1_data.block_hash == eth1_chain[0].hash_tree_root() @with_all_phases @@ -185,10 +259,16 @@ def test_get_block_signature(spec, state): privkey = privkeys[0] pubkey = pubkeys[0] block = build_empty_block(spec, state) - signature = spec.get_block_signature(state, block, privkey) domain = spec.get_domain(state, spec.DOMAIN_BEACON_PROPOSER, spec.compute_epoch_at_slot(block.slot)) - signing_root = spec.compute_signing_root(block, domain) - assert bls.Verify(pubkey, signing_root, signature) + run_get_signature_test( + spec=spec, + state=state, + obj=block, + domain=domain, + get_signature_fn=spec.get_block_signature, + privkey=privkey, + pubkey=pubkey, + ) # Attesting @@ -200,10 +280,16 @@ def test_get_attestation_signature(spec, state): privkey = privkeys[0] pubkey = pubkeys[0] attestation_data = spec.AttestationData(slot=10) - signature = spec.get_attestation_signature(state, attestation_data, privkey) domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) - signing_root = spec.compute_signing_root(attestation_data, domain) - assert bls.Verify(pubkey, signing_root, signature) + run_get_signature_test( + spec=spec, + state=state, + obj=attestation_data, + domain=domain, + get_signature_fn=spec.get_attestation_signature, + privkey=privkey, + pubkey=pubkey, + ) # Attestation aggregation @@ -214,11 +300,17 @@ def test_get_attestation_signature(spec, state): def test_get_slot_signature(spec, state): privkey = privkeys[0] pubkey = pubkeys[0] - slot = 10 - signature = spec.get_slot_signature(state, spec.Slot(slot), privkey) + slot = spec.Slot(10) domain = spec.get_domain(state, spec.DOMAIN_SELECTION_PROOF, spec.compute_epoch_at_slot(slot)) - signing_root = spec.compute_signing_root(spec.Slot(slot), domain) - assert bls.Verify(pubkey, signing_root, signature) + run_get_signature_test( + spec=spec, + state=state, + obj=slot, + domain=domain, + get_signature_fn=spec.get_slot_signature, + privkey=privkey, + pubkey=pubkey, + ) @with_all_phases @@ -290,7 +382,13 @@ def test_get_aggregate_and_proof_signature(spec, state): pubkey = pubkeys[0] aggregate = get_mock_aggregate(spec) aggregate_and_proof = spec.get_aggregate_and_proof(state, spec.ValidatorIndex(1), aggregate, privkey) - signature = spec.get_aggregate_and_proof_signature(state, aggregate_and_proof, privkey) domain = spec.get_domain(state, spec.DOMAIN_AGGREGATE_AND_PROOF, spec.compute_epoch_at_slot(aggregate.data.slot)) - signing_root = spec.compute_signing_root(aggregate_and_proof, domain) - assert bls.Verify(pubkey, signing_root, signature) + run_get_signature_test( + spec=spec, + state=state, + obj=aggregate_and_proof, + domain=domain, + get_signature_fn=spec.get_aggregate_and_proof_signature, + privkey=privkey, + pubkey=pubkey, + ) From d311248d35684cd5aa6e47b286e075bb4feb46f4 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 27 Apr 2020 21:45:01 +0800 Subject: [PATCH 15/70] Increase `EPOCHS_PER_ETH1_VOTING_PERIOD` from `2` to `4` for testing eth1 votes consensus --- configs/minimal.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index b39a4fc01..c8b58146f 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -89,7 +89,7 @@ MIN_SEED_LOOKAHEAD: 1 # 2**2 (= 4) epochs MAX_SEED_LOOKAHEAD: 4 # [customized] higher frequency new deposits from eth1 for testing -EPOCHS_PER_ETH1_VOTING_PERIOD: 2 +EPOCHS_PER_ETH1_VOTING_PERIOD: 4 # [customized] smaller state SLOTS_PER_HISTORICAL_ROOT: 64 # 2**8 (= 256) epochs From 2dbc33327084d2814958f92eb0a838b9bc161903 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 27 Apr 2020 22:05:47 +0800 Subject: [PATCH 16/70] Make `compute_new_state_root` a pure function --- specs/phase0/validator.md | 7 ++++--- .../eth2spec/test/validator/test_validator_unittest.py | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 4576c4b90..cbe0c2d12 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -340,9 +340,10 @@ It is useful to be able to run a state transition function (working on a copy of ```python def compute_new_state_root(state: BeaconState, block: BeaconBlock) -> Root: - process_slots(state, block.slot) - process_block(state, block) - return hash_tree_root(state) + temp_state: BeaconState = state.copy() + signed_block = SignedBeaconBlock(message=block) + temp_state = state_transition(temp_state, signed_block, validate_result=False) + return hash_tree_root(temp_state) ``` ##### Signature diff --git a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py index a655cb486..5bb246ed5 100644 --- a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py +++ b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py @@ -246,6 +246,7 @@ def test_compute_new_state_root(spec, state): state_root = spec.compute_new_state_root(state, block) assert state_root != pre_state.hash_tree_root() + assert state == pre_state # dumb verification spec.process_slots(post_state, block.slot) From d128400da5cbcd16102fa760d4381ada07d4a577 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 27 Apr 2020 16:16:33 -0600 Subject: [PATCH 17/70] remove interop from header and gossip sections in network spec --- specs/phase0/p2p-interface.md | 55 +++++------------------------------ 1 file changed, 7 insertions(+), 48 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index f4f2f3d22..a3a9b48fe 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -4,7 +4,7 @@ This document contains the networking specification for Ethereum 2.0 clients. It consists of four main sections: -1. A specification of the network fundamentals detailing the two network configurations: interoperability test network and mainnet launch. +1. A specification of the network fundamentals. 2. A specification of the three network interaction *domains* of Eth2: (a) the gossip domain, (b) the discovery domain, and (c) the Req/Resp domain. 3. The rationale and further explanation for the design choices made in the previous two sections. 4. An analysis of the maturity/state of the libp2p features required by this spec across the languages in which Eth2 clients are being developed. @@ -120,42 +120,20 @@ It consists of four main sections: This section outlines the specification for the networking stack in Ethereum 2.0 clients. -Sections that have differing parameters for mainnet launch and interoperability testing are split into subsections. Sections that are not split have the same parameters for interoperability testing as mainnet launch. - ## Transport Even though libp2p is a multi-transport stack (designed to listen on multiple simultaneous transports and endpoints transparently), we hereby define a profile for basic interoperability. -#### Interop - All implementations MUST support the TCP libp2p transport, and it MUST be enabled for both dialing and listening (i.e. outbound and inbound connections). The libp2p TCP transport supports listening on IPv4 and IPv6 addresses (and on multiple simultaneously). -To facilitate connectivity and avert possible IPv6 routability/support issues, clients participating in the interoperability testnet MUST expose at least ONE IPv4 endpoint. +Clients must support listening on at least one of IPv4 or IPv6. Clients that do _not_ have support for listening on IPv4 SHOULD be cognizant of the potential disadvantages in terms of Internet-wide routability/support. Clients MAY choose to listen only on IPv6, but MUST be capable of dialing both IPv4 and IPv6 addresses. -All listening endpoints must be publicly dialable, and thus not rely on libp2p circuit relay, AutoNAT, or AutoRelay facilities. +All listening endpoints must be publicly dialable, and thus not rely on libp2p circuit relay, AutoNAT, or AutoRelay facilities. (Usage of circuit relay, AutoNAT, or AutoRelay will be specifically re-examined soon.) Nodes operating behind a NAT, or otherwise undialable by default (e.g. container runtime, firewall, etc.), MUST have their infrastructure configured to enable inbound traffic on the announced public listening endpoint. -#### Mainnet - -All requirements from the interoperability testnet apply, except for the IPv4 addressing scheme requirement. - -At this stage, clients are licensed to drop IPv4 support if they wish to do so, cognizant of the potential disadvantages in terms of Internet-wide routability/support. Clients MAY choose to listen only on IPv6, but MUST retain capability to dial both IPv4 and IPv6 addresses. - -Usage of circuit relay, AutoNAT, or AutoRelay will be specifically re-examined closer to the time. - ## Encryption and identification -#### Interop - -[SecIO](https://github.com/libp2p/specs/tree/master/secio) with `secp256k1` identities will be used for initial interoperability testing. - -The following SecIO parameters MUST be supported by all stacks: - -- Key agreement: ECDH-P256. -- Cipher: AES-128. -- Digest: SHA-256. - #### Mainnet The [Libp2p-noise](https://github.com/libp2p/specs/tree/master/noise) secure @@ -167,13 +145,7 @@ As specified in the libp2p specification, clients MUST support the `XX` handshak Clients MUST use exact equality when negotiating protocol versions to use and MAY use the version to give priority to higher version numbers. -#### Interop - -Connection-level and stream-level (see the [Rationale](#design-decision-rationale) section below for explanations) protocol negotiation MUST be conducted using [multistream-select v1.0](https://github.com/multiformats/multistream-select/). Its protocol ID is: `/multistream/1.0.0`. - -#### Mainnet - -Clients MUST support [multistream-select 1.0](https://github.com/multiformats/multistream-select/) and MAY support [multiselect 2.0](https://github.com/libp2p/specs/pull/95). Depending on the number of clients that have implementations for multiselect 2.0 by mainnet, [multistream-select 1.0](https://github.com/multiformats/multistream-select/) may be phased out. +Clients MUST support [multistream-select 1.0](https://github.com/multiformats/multistream-select/) and MAY support [multiselect 2.0](https://github.com/libp2p/specs/pull/95) when the spec solidifies. Once all clients have implementations for multiselect 2.0, multistream-select 1.0 MAY be phased out. ## Multiplexing @@ -181,7 +153,7 @@ During connection bootstrapping, libp2p dynamically negotiates a mutually suppor Two multiplexers are commonplace in libp2p implementations: [mplex](https://github.com/libp2p/specs/tree/master/mplex) and [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md). Their protocol IDs are, respectively: `/mplex/6.7.0` and `/yamux/1.0.0`. -Clients MUST support [mplex](https://github.com/libp2p/specs/tree/master/mplex) and MAY support [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md). If both are supported by the client, yamux must take precedence during negotiation. See the [Rationale](#design-decision-rationale) section below for tradeoffs. +Clients MUST support [mplex](https://github.com/libp2p/specs/tree/master/mplex) and MAY support [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md). If both are supported by the client, yamux MUST take precedence during negotiation. See the [Rationale](#design-decision-rationale) section below for tradeoffs. # Eth2 network interaction domains @@ -265,7 +237,6 @@ The payload is carried in the `data` field of a gossipsub message, and varies de |------------------------------------------------|-------------------------| | beacon_block | SignedBeaconBlock | | beacon_aggregate_and_proof | SignedAggregateAndProof | -| beacon_attestation\* | Attestation | | committee_index{subnet_id}\_beacon_attestation | Attestation | | voluntary_exit | SignedVoluntaryExit | | proposer_slashing | ProposerSlashing | @@ -275,8 +246,6 @@ Clients MUST reject (fail validation) messages containing an incorrect type, or When processing incoming gossip, clients MAY descore or disconnect peers who fail to observe these constraints. -\* The `beacon_attestation` topic is only for interop and will be removed prior to mainnet. - #### Global topics There are two primary global topics used to propagate beacon blocks and aggregate attestations to all nodes on the network. Their `Name`s are: @@ -323,11 +292,7 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio - The block being voted for (`attestation.data.beacon_block_root`) passes validation. - The signature of `attestation` is valid. -#### Interop - -Unaggregated and aggregated attestations from all shards are sent as `Attestation`s to the `beacon_attestation` topic. Clients are not required to publish aggregate attestations but must be able to process them. All validating clients SHOULD try to perform local attestation aggregation to prepare for block proposing. - -#### Mainnet +#### Attestations and Aggregation Attestation broadcasting is grouped into subnets defined by a topic. The number of subnets is defined via `ATTESTATION_SUBNET_COUNT`. For the `committee_index{subnet_id}_beacon_attestation` topics, `subnet_id` is set to `index % ATTESTATION_SUBNET_COUNT`, where `index` is the `CommitteeIndex` of the given committee. @@ -339,17 +304,11 @@ Aggregated attestations are sent to the `beacon_aggregate_and_proof` topic as `A Topics are post-fixed with an encoding. Encodings define how the payload of a gossipsub message is encoded. -#### Interop - -- `ssz` - All objects are [SSZ-encoded](#ssz-encoding). Example: The beacon block topic string is `/eth2/beacon_block/ssz`, and the data field of a gossipsub message is an ssz-encoded `SignedBeaconBlock`. - -#### Mainnet - - `ssz_snappy` - All objects are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy) block compression. Example: The beacon aggregate attestation topic string is `/eth2/446a7232/beacon_aggregate_and_proof/ssz_snappy`, the fork digest is `446a7232` and the data field of a gossipsub message is an `AggregateAndProof` that has been SSZ-encoded and then compressed with Snappy. Snappy has two formats: "block" and "frames" (streaming). Gossip messages remain relatively small (100s of bytes to 100s of kilobytes) so [basic snappy block compression](https://github.com/google/snappy/blob/master/format_description.txt) is used to avoid the additional overhead associated with snappy frames. -Implementations MUST use a single encoding. Changing an encoding will require coordination between participating implementations. +Implementations MUST use a single encoding for gossip. Changing an encoding will require coordination between participating implementations. ## The Req/Resp domain From 87586837c388a4005d09d7baf56213fc418211e9 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 27 Apr 2020 17:34:26 -0600 Subject: [PATCH 18/70] remove interop from phase 0 p2p specs --- specs/phase0/p2p-interface.md | 73 ++++++++++------------------------- 1 file changed, 21 insertions(+), 52 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index a3a9b48fe..f76502bb4 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -17,14 +17,8 @@ It consists of four main sections: - [Network fundamentals](#network-fundamentals) - [Transport](#transport) - - [Interop](#interop) - - [Mainnet](#mainnet) - [Encryption and identification](#encryption-and-identification) - - [Interop](#interop-1) - - [Mainnet](#mainnet-1) - [Protocol Negotiation](#protocol-negotiation) - - [Interop](#interop-2) - - [Mainnet](#mainnet-2) - [Multiplexing](#multiplexing) - [Eth2 network interaction domains](#eth2-network-interaction-domains) - [Configuration](#configuration) @@ -33,11 +27,8 @@ It consists of four main sections: - [Topics and messages](#topics-and-messages) - [Global topics](#global-topics) - [Attestation subnets](#attestation-subnets) - - [Interop](#interop-3) - - [Mainnet](#mainnet-3) + - [Attestations and Aggregation](#attestations-and-aggregation) - [Encodings](#encodings) - - [Interop](#interop-4) - - [Mainnet](#mainnet-4) - [The Req/Resp domain](#the-reqresp-domain) - [Protocol identification](#protocol-identification) - [Req/Resp interaction](#reqresp-interaction) @@ -56,29 +47,25 @@ It consists of four main sections: - [Integration into libp2p stacks](#integration-into-libp2p-stacks) - [ENR structure](#enr-structure) - [Attestation subnet bitfield](#attestation-subnet-bitfield) - - [Interop](#interop-5) - - [Mainnet](#mainnet-5) - - [`eth2` field](#eth2-field) - - [General capabilities](#general-capabilities) + - [`eth2` field](#eth2-field) + - [General capabilities](#general-capabilities) - [Topic advertisement](#topic-advertisement) - - [Mainnet](#mainnet-6) - [Design decision rationale](#design-decision-rationale) - [Transport](#transport-1) - [Why are we defining specific transports?](#why-are-we-defining-specific-transports) - [Can clients support other transports/handshakes than the ones mandated by the spec?](#can-clients-support-other-transportshandshakes-than-the-ones-mandated-by-the-spec) - [What are the advantages of using TCP/QUIC/Websockets?](#what-are-the-advantages-of-using-tcpquicwebsockets) - [Why do we not just support a single transport?](#why-do-we-not-just-support-a-single-transport) - - [Why are we not using QUIC for mainnet from the start?](#why-are-we-not-using-quic-for-mainnet-from-the-start) + - [Why are we not using QUIC from the start?](#why-are-we-not-using-quic-from-the-start) - [Multiplexing](#multiplexing-1) - [Why are we using mplex/yamux?](#why-are-we-using-mplexyamux) - [Protocol Negotiation](#protocol-negotiation-1) - - [When is multiselect 2.0 due and why are we using it for mainnet?](#when-is-multiselect-20-due-and-why-are-we-using-it-for-mainnet) + - [When is multiselect 2.0 due and why do we plan to migrate to it?](#when-is-multiselect-20-due-and-why-do-we-plan-to-migrate-to-it) - [What is the difference between connection-level and stream-level protocol negotiation?](#what-is-the-difference-between-connection-level-and-stream-level-protocol-negotiation) - [Encryption](#encryption) - - [Why are we using SecIO for interop? Why not for mainnet?](#why-are-we-using-secio-for-interop-why-not-for-mainnet) - - [Why are we using Noise/TLS 1.3 for mainnet?](#why-are-we-using-noisetls-13-for-mainnet) + - [Why are we not supporting SecIO?](#why-are-we-not-supporting-secio) + - [Why are we using Noise/TLS 1.3?](#why-are-we-using-noisetls-13) - [Why are we using encryption at all?](#why-are-we-using-encryption-at-all) - - [Will mainnnet networking be untested when it launches?](#will-mainnnet-networking-be-untested-when-it-launches) - [Gossipsub](#gossipsub) - [Why are we using a pub/sub algorithm for block and attestation propagation?](#why-are-we-using-a-pubsub-algorithm-for-block-and-attestation-propagation) - [Why are we using topics to segregate encodings, yet only support one encoding?](#why-are-we-using-topics-to-segregate-encodings-yet-only-support-one-encoding) @@ -134,10 +121,8 @@ Nodes operating behind a NAT, or otherwise undialable by default (e.g. container ## Encryption and identification -#### Mainnet - The [Libp2p-noise](https://github.com/libp2p/specs/tree/master/noise) secure -channel handshake with `secp256k1` identities will be used for mainnet. +channel handshake with `secp256k1` identities will be used for encryption. As specified in the libp2p specification, clients MUST support the `XX` handshake pattern. @@ -409,7 +394,7 @@ Here, `result` represents the 1-byte response code. The token of the negotiated protocol ID specifies the type of encoding to be used for the req/resp interaction. Two values are possible at this time: - `ssz`: the contents are [SSZ-encoded](../../ssz/simple-serialize.md). This encoding type MUST be supported by all clients. For objects containing a single field, only the field is SSZ-encoded not a container with a single field. For example, the `BeaconBlocksByRoot` request is an SSZ-encoded list of `Root`'s. -- `ssz_snappy`: The contents are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy) frames compression. MAY be supported in the interoperability testnet; MUST be supported in mainnet. +- `ssz_snappy`: The contents are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy) frames compression. This encoding type MUST be supported by all clients. #### SSZ-encoding strategy (with or without Snappy) @@ -642,7 +627,7 @@ The response MUST consist of a single `response_chunk`. ## The discovery domain: discv5 -Discovery Version 5 ([discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md)) is used for peer discovery, both in the interoperability testnet and mainnet. +Discovery Version 5 ([discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md)) is used for peer discovery. `discv5` is a standalone protocol, running on UDP on a dedicated port, meant for peer discovery only. `discv5` supports self-certified, flexible peer records (ENRs) and topic-based advertisement, both of which are (or will be) requirements in this context. @@ -682,15 +667,7 @@ If a node's `MetaData.attnets` has any non-zero bit, the ENR MUST include the `a If a node's `MetaData.attnets` is composed of all zeros, the ENR MAY optionally include the `attnets` entry or leave it out entirely. -#### Interop - -In the interoperability testnet, all peers will support all capabilities defined in this document (gossip, full Req/Resp suite, discovery protocol), therefore the ENR record does not need to carry Eth2 capability information, as it would be superfluous. - -Nonetheless, ENRs MUST carry a generic `eth2` key with nil value, denoting that the peer is indeed an Eth2 peer, in order to eschew connecting to Eth 1.0 peers. - -#### Mainnet - -##### `eth2` field +#### `eth2` field ENRs MUST carry a generic `eth2` key with an 16-byte value of the node's current fork digest, next fork version, and next fork epoch to ensure connections are made with peers on the intended eth2 network. @@ -722,14 +699,12 @@ Clients SHOULD connect to peers with `fork_digest`, `next_fork_version`, and `ne Clients MAY connect to peers with the same `fork_digest` but a different `next_fork_version`/`next_fork_epoch`. Unless `ENRForkID` is manually updated to matching prior to the earlier `next_fork_epoch` of the two clients, these connecting clients will be unable to successfully interact starting at the earlier `next_fork_epoch`. -##### General capabilities +#### General capabilities -On mainnet, ENRs MUST include a structure enumerating the capabilities offered by the peer in an efficient manner. The concrete solution is currently undefined. Proposals include using namespaced bloom filters mapping capabilities to specific protocol IDs supported under that capability. +ENRs MUST include a structure enumerating the capabilities offered by the peer in an efficient manner. The concrete solution is currently undefined. Proposals include using namespaced bloom filters mapping capabilities to specific protocol IDs supported under that capability. ### Topic advertisement -#### Mainnet - discv5's topic advertisement feature is not expected to be ready for mainnet launch of Phase 0. Once this feature is built out and stable, we expect to use topic advertisement as a rendezvous facility for peers on shards. Until then, the ENR [attestation subnet bitfield](#attestation-subnet-bitfield) will be used for discovery of peers on particular subnets. @@ -775,7 +750,7 @@ Modeling for upgradeability and dynamic transport selection from the get-go lays Clients can adopt new transports without breaking old ones, and the multi-transport ability enables constrained and sandboxed environments (e.g. browsers, embedded devices) to interact with the network as first-class citizens via suitable/native transports (e.g. WSS), without the need for proxying or trust delegation to servers. -### Why are we not using QUIC for mainnet from the start? +### Why are we not using QUIC from the start? The QUIC standard is still not finalized (at working draft 22 at the time of writing), and not all mainstream runtimes/languages have mature, standard, and/or fully-interoperable [QUIC support](https://github.com/quicwg/base-drafts/wiki/Implementations). One remarkable example is node.js, where the QUIC implementation is [in early development](https://github.com/nodejs/quic). @@ -791,13 +766,13 @@ Overlay multiplexers are not necessary with QUIC since the protocol provides nat ## Protocol Negotiation -### When is multiselect 2.0 due and why are we using it for mainnet? +### When is multiselect 2.0 due and why do we plan to migrate to it? multiselect 2.0 is currently being conceptualized. The debate started [on this issue](https://github.com/libp2p/specs/pull/95), but it got overloaded—as it tends to happen with large conceptual OSS discussions that touch the heart and core of a system. -In the following weeks (August 2019), there will be a renewed initiative to first define the requirements, constraints, assumptions, and features, in order to lock in basic consensus upfront and subsequently build on that consensus by submitting a specification for implementation. +At some point in 2020, we expect a renewed initiative to first define the requirements, constraints, assumptions, and features, in order to lock in basic consensus upfront and subsequently build on that consensus by submitting a specification for implementation. -We plan to use multiselect 2.0 for mainnet because it will: +We plan to eventually migrate to multiselect 2.0 because it will: 1. Reduce round trips during connection bootstrapping and stream protocol negotiation. 2. Enable efficient one-stream-per-request interaction patterns. @@ -819,17 +794,15 @@ At present, multistream-select 1.0 is used for both types of negotiation, but mu ## Encryption -### Why are we using SecIO for interop? Why not for mainnet? +### Why are we not supporting SecIO? SecIO has been the default encryption layer for libp2p for years. It is used in IPFS and Filecoin. And although it will be superseded shortly, it is proven to work at scale. -SecIO is the common denominator across the various language libraries at this stage. It is widely implemented. That’s why we have chosen to use it for initial interop to minimize overhead in getting to a basic interoperability testnet. - -We won’t be using it for mainnet because, amongst other things, it requires several round trips to be sound, and doesn’t support early data (0-RTT data), a mechanism that multiselect 2.0 will leverage to reduce round trips during connection bootstrapping. +Although SecIO has wide language support, we won’t be using it for mainnet because, amongst other things, it requires several round trips to be sound, and doesn’t support early data (0-RTT data), a mechanism that multiselect 2.0 will leverage to reduce round trips during connection bootstrapping. SecIO is not considered secure for the purposes of this spec. -### Why are we using Noise/TLS 1.3 for mainnet? +### Why are we using Noise/TLS 1.3? Copied from the Noise Protocol Framework [website](http://www.noiseprotocol.org): @@ -855,10 +828,6 @@ Transport level encryption secures message exchange and provides properties that Note that transport-level encryption is not exclusive of application-level encryption or cryptography. Transport-level encryption secures the communication itself, while application-level cryptography is necessary for the application’s use cases (e.g. signatures, randomness, etc.). -### Will mainnnet networking be untested when it launches? - -Before launching mainnet, the testnet will be switched over to mainnet networking parameters, including Noise handshakes, and other new protocols. This gives us an opportunity to drill coordinated network upgrades and verifying that there are no significant upgradeability gaps. - ## Gossipsub ### Why are we using a pub/sub algorithm for block and attestation propagation? @@ -967,7 +936,7 @@ Requests are segregated by protocol ID to: 2. Affording this level of granularity with a top-level protocol would imply creating as many variants (e.g. /protocol/43-{a,b,c,d,...}) as the cartesian product of RFCs inflight, O(n^2). 7. Allow us to simplify the payload of requests. Request-id’s and method-ids no longer need to be sent. The encoding/request type and version can all be handled by the framework. -**Caveat**: The protocol negotiation component in the current version of libp2p is called multistream-select 1.0. It is somewhat naïve and introduces overhead on every request when negotiating streams, although implementation-specific optimizations are possible to save this cost. Multiselect 2.0 will remove this overhead by memoizing previously selected protocols, and modeling shared protocol tables. Fortunately, this req/resp protocol is not the expected network bottleneck in the protocol so the additional overhead is not expected to hinder interop testing. More info is to be released from the libp2p community in the coming weeks. +**Caveat**: The protocol negotiation component in the current version of libp2p is called multistream-select 1.0. It is somewhat naïve and introduces overhead on every request when negotiating streams, although implementation-specific optimizations are possible to save this cost. Multiselect 2.0 will eventually remove this overhead by memoizing previously selected protocols, and modeling shared protocol tables. Fortunately, this req/resp protocol is not the expected network bottleneck in the protocol so the additional overhead is not expected to significantly hinder this domain. ### Why are messages length-prefixed with a protobuf varint in the SSZ-encoding? From 09cae4b3ccab47fcb7f9eba198af46d3bcdfa3d3 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 1 May 2020 15:17:41 +0200 Subject: [PATCH 19/70] Handle empty-aggregation-bits case, and add tests. See #1713 --- specs/phase0/beacon-chain.md | 4 ++-- specs/phase0/p2p-interface.md | 2 +- specs/phase1/beacon-chain.md | 3 ++- .../eth2spec/test/helpers/attestations.py | 3 +++ .../test_process_attestation.py | 23 +++++++++++++++++++ .../test_process_rewards_and_penalties.py | 3 +++ 6 files changed, 34 insertions(+), 4 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index cdf38dc1e..5ab499bfc 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -684,11 +684,11 @@ def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationDa ```python def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> bool: """ - Check if ``indexed_attestation`` has sorted and unique indices and a valid aggregate signature. + Check if ``indexed_attestation`` is not empty, has sorted and unique indices and has a valid aggregate signature. """ # Verify indices are sorted and unique indices = indexed_attestation.attesting_indices - if not indices == sorted(set(indices)): + if len(indices) == 0 or not indices == sorted(set(indices)): return False # Verify aggregate signature pubkeys = [state.validators[i].pubkey for i in indices] diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 7197581dc..a44de8c8e 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -293,7 +293,7 @@ There are two primary global topics used to propagate beacon blocks and aggregat - The `aggregate` is the first valid aggregate received for the aggregator with index `aggregate_and_proof.aggregator_index` for the epoch `aggregate.data.target.epoch`. - The block being voted for (`aggregate.data.beacon_block_root`) passes validation. - `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_aggregator(state, aggregate.data.slot, aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. - - The aggregator's validator index is within the aggregate's committee -- i.e. `aggregate_and_proof.aggregator_index in get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)`. + - The aggregator's validator index is within the aggregate's committee -- i.e. `aggregate_and_proof.aggregator_index in get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)`. This also means that it must never have an empty set of participants. - The `aggregate_and_proof.selection_proof` is a valid signature of the `aggregate.data.slot` by the validator with index `aggregate_and_proof.aggregator_index`. - The aggregator signature, `signed_aggregate_and_proof.signature`, is valid. - The signature of `aggregate` is valid. diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 596b3818f..1770b7a98 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -571,7 +571,8 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe attestation = indexed_attestation.attestation domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch) aggregation_bits = attestation.aggregation_bits - assert len(aggregation_bits) == len(indexed_attestation.committee) + if not any(aggregation_bits) or len(aggregation_bits) != len(indexed_attestation.committee): + return False if len(attestation.custody_bits_blocks) == 0: # fall back on phase0 behavior if there is no shard data. diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 8215f5c5b..377426006 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -114,6 +114,9 @@ def get_valid_late_attestation(spec, state, slot=None, index=None, signed=False) def get_valid_attestation(spec, state, slot=None, index=None, empty=False, signed=False, on_time=True): + # If empty is true, the attestation has 0 participants, and will not be signed. + # Thus strictly speaking invalid when no participant is added later. + if slot is None: slot = state.slot if index is None: diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 8663391aa..df3279faa 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -64,6 +64,29 @@ def test_invalid_attestation_signature(spec, state): yield from run_attestation_processing(spec, state, attestation, False) +@with_all_phases +@spec_state_test +@always_bls +def test_empty_participants_zeroes_sig(spec, state): + attestation = get_valid_attestation(spec, state, empty=True) + attestation.signature = spec.BLSSignature(b'\x00' * 96) + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) + + yield from run_attestation_processing(spec, state, attestation, False) + + +@with_all_phases +@spec_state_test +@always_bls +def test_empty_participants_seemingly_valid_sig(spec, state): + attestation = get_valid_attestation(spec, state, empty=True) + # Special BLS value, valid for zero pubkeys on some (but not all) BLS implementations. + attestation.signature = spec.BLSSignature(b'\xc0' + b'\x00' * 95) + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) + + yield from run_attestation_processing(spec, state, attestation, False) + + @with_all_phases @spec_state_test def test_before_inclusion_delay(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py index af695fe69..b862b5c48 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py @@ -22,6 +22,9 @@ def run_process_rewards_and_penalties(spec, state): def prepare_state_with_full_attestations(spec, state, empty=False): + # If empty is true, attestations have 0 participants, and are not signed. + # Thus strictly speaking invalid when no participant is added later. + # Go to start of next epoch to ensure can have full participation next_epoch(spec, state) From 47ed5b6500e3028e745458295cae395b9323ef9e Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 1 May 2020 16:10:28 +0200 Subject: [PATCH 20/70] Fix rewards testing for empty/weird participation cases, adding more as well --- .../eth2spec/test/helpers/attestations.py | 25 +++++--- .../test_process_attestation.py | 18 +----- .../test_process_rewards_and_penalties.py | 64 +++++++++++++++---- 3 files changed, 71 insertions(+), 36 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 377426006..85b543104 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -113,8 +113,8 @@ def get_valid_late_attestation(spec, state, slot=None, index=None, signed=False) return get_valid_attestation(spec, state, slot=slot, index=index, signed=signed, on_time=False) -def get_valid_attestation(spec, state, slot=None, index=None, empty=False, signed=False, on_time=True): - # If empty is true, the attestation has 0 participants, and will not be signed. +def get_valid_attestation(spec, state, slot=None, index=None, filter_participant_set=None, signed=False, on_time=True): + # If filter_participant_set is filters everything, the attestation has 0 participants, and cannot be signed. # Thus strictly speaking invalid when no participant is added later. if slot is None: @@ -136,10 +136,8 @@ def get_valid_attestation(spec, state, slot=None, index=None, empty=False, signe aggregation_bits=aggregation_bits, data=attestation_data, ) - if not empty: - fill_aggregate_attestation(spec, state, attestation) - if signed: - sign_attestation(spec, state, attestation) + # fill the attestation with (optionally filtered) participants, and optionally sign it + fill_aggregate_attestation(spec, state, attestation, signed=signed, filter_participant_set=filter_participant_set) if spec.fork == 'phase1' and on_time: attestation = convert_to_valid_on_time_attestation(spec, state, attestation, signed) @@ -232,16 +230,25 @@ def get_attestation_signature(spec, state, attestation_data, privkey): return bls.Sign(privkey, signing_root) -def fill_aggregate_attestation(spec, state, attestation, signed=False): +def fill_aggregate_attestation(spec, state, attestation, signed=False, filter_participant_set=None): + """ + `signed`: Signing is optional. + `filter_participant_set`: Optional, filters the full committee indices set (default) to a subset that participates + """ beacon_committee = spec.get_beacon_committee( state, attestation.data.slot, attestation.data.index, ) + # By default, have everyone participate + participants = set(beacon_committee) + # But optionally filter the participants to a smaller amount + if filter_participant_set is not None: + participants = filter_participant_set(participants) for i in range(len(beacon_committee)): - attestation.aggregation_bits[i] = True + attestation.aggregation_bits[i] = beacon_committee[i] in participants - if signed: + if signed and len(participants) > 0: sign_attestation(spec, state, attestation) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index df3279faa..bb25a384e 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -13,7 +13,6 @@ from eth2spec.test.helpers.attestations import ( sign_attestation, ) from eth2spec.test.helpers.state import ( - next_slot, next_slots, next_epoch, transition_to, @@ -68,7 +67,7 @@ def test_invalid_attestation_signature(spec, state): @spec_state_test @always_bls def test_empty_participants_zeroes_sig(spec, state): - attestation = get_valid_attestation(spec, state, empty=True) + attestation = get_valid_attestation(spec, state, filter_participant_set=lambda comm: []) # 0 participants attestation.signature = spec.BLSSignature(b'\x00' * 96) next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -79,7 +78,7 @@ def test_empty_participants_zeroes_sig(spec, state): @spec_state_test @always_bls def test_empty_participants_seemingly_valid_sig(spec, state): - attestation = get_valid_attestation(spec, state, empty=True) + attestation = get_valid_attestation(spec, state, filter_participant_set=lambda comm: []) # 0 participants # Special BLS value, valid for zero pubkeys on some (but not all) BLS implementations. attestation.signature = spec.BLSSignature(b'\xc0' + b'\x00' * 95) next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -283,19 +282,6 @@ def test_bad_source_root(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_all_phases -@spec_state_test -def test_empty_aggregation_bits(spec, state): - next_slot(spec, state) - attestation = get_valid_attestation(spec, state, empty=True) - next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) - - assert attestation.aggregation_bits == Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE]( - *([0b0] * len(attestation.aggregation_bits))) - - yield from run_attestation_processing(spec, state, attestation) - - @with_all_phases @spec_state_test def test_too_many_aggregation_bits(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py index b862b5c48..63aafe521 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py @@ -15,15 +15,15 @@ from eth2spec.test.helpers.attestations import ( ) from eth2spec.test.helpers.attester_slashings import get_indexed_attestation_participants from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_with +from random import Random def run_process_rewards_and_penalties(spec, state): yield from run_epoch_processing_with(spec, state, 'process_rewards_and_penalties') -def prepare_state_with_full_attestations(spec, state, empty=False): - # If empty is true, attestations have 0 participants, and are not signed. - # Thus strictly speaking invalid when no participant is added later. +def prepare_state_with_full_attestations(spec, state, participation_fn=None): + # participation_fn: (slot, committee_index, committee_indices_set) -> participants_indices_set # Go to start of next epoch to ensure can have full participation next_epoch(spec, state) @@ -36,8 +36,15 @@ def prepare_state_with_full_attestations(spec, state, empty=False): # create an attestation for each index in each slot in epoch if state.slot < next_epoch_start_slot: for committee_index in range(spec.get_committee_count_at_slot(state, state.slot)): - attestation = get_valid_attestation(spec, state, index=committee_index, empty=empty, signed=True) - attestations.append(attestation) + def temp_participants_filter(comm): + if participation_fn is None: + return comm + else: + return participation_fn(state.slot, committee_index, comm) + attestation = get_valid_attestation(spec, state, index=committee_index, + filter_participant_set=temp_participants_filter, signed=True) + if any(attestation.aggregation_bits): # Only if there is at least 1 participant. + attestations.append(attestation) # fill each created slot in state after inclusion delay if state.slot >= start_slot + spec.MIN_ATTESTATION_INCLUSION_DELAY: inclusion_slot = state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY @@ -192,20 +199,55 @@ def test_no_attestations_all_penalties(spec, state): assert state.balances[index] < pre_state.balances[index] -@with_all_phases -@spec_state_test -def test_empty_attestations(spec, state): - attestations = prepare_state_with_full_attestations(spec, state, empty=True) +def run_with_participation(spec, state, participation_fn): + participated = set() + + def participation_tracker(slot, comm_index, comm): + att_participants = participation_fn(slot, comm_index, comm) + participated.update(att_participants) + return att_participants + + attestations = prepare_state_with_full_attestations(spec, state, participation_fn=participation_tracker) pre_state = state.copy() yield from run_process_rewards_and_penalties(spec, state) attesting_indices = spec.get_unslashed_attesting_indices(state, attestations) - assert len(attesting_indices) == 0 + assert len(attesting_indices) == len(participated) for index in range(len(pre_state.validators)): - assert state.balances[index] < pre_state.balances[index] + if index in participated: + assert state.balances[index] > pre_state.balances[index] + else: + assert state.balances[index] < pre_state.balances[index] + + +@with_all_phases +@spec_state_test +def test_almost_empty_attestations(spec, state): + rng = Random(1234) + yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, 1)) + + +@with_all_phases +@spec_state_test +def test_random_fill_attestations(spec, state): + rng = Random(4567) + yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, len(comm) // 3)) + + +@with_all_phases +@spec_state_test +def test_almost_full_attestations(spec, state): + rng = Random(8901) + yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, len(comm) - 1)) + + +@with_all_phases +@spec_state_test +def test_full_attestation_participation(spec, state): + yield from run_with_participation(spec, state, lambda slot, comm_index, comm: comm) @with_all_phases From 5b3ed8a3e73b0a3825a1ac7ffa469e3759a03922 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 5 May 2020 09:45:24 -0600 Subject: [PATCH 21/70] bump VERSION.txt to 0.12.0 --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index a8839f70d..d33c3a212 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -0.11.2 \ No newline at end of file +0.12.0 \ No newline at end of file From 38f29ba0a851718bd31c47783f0e9b21360aa813 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 5 May 2020 11:44:53 -0600 Subject: [PATCH 22/70] remove PERSISTENT_COMMITTEE_PERIOD in favor of SHARD_COMMITTEE_PERIOD --- configs/mainnet.yaml | 6 ++-- configs/minimal.yaml | 4 +-- specs/phase0/beacon-chain.md | 5 ++-- specs/phase1/beacon-chain.md | 1 - .../test_process_voluntary_exit.py | 30 +++++++++---------- .../test/phase_0/sanity/test_blocks.py | 6 ++-- 6 files changed, 24 insertions(+), 28 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 7691b7481..60bd1c087 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -94,8 +94,8 @@ EPOCHS_PER_ETH1_VOTING_PERIOD: 32 SLOTS_PER_HISTORICAL_ROOT: 8192 # 2**8 (= 256) epochs ~27 hours MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 -# 2**11 (= 2,048) epochs 9 days -PERSISTENT_COMMITTEE_PERIOD: 2048 +# 2**8 (= 256) epochs ~27 hours +SHARD_COMMITTEE_PERIOD: 256 # 2**6 (= 64) epochs ~7 hours MAX_EPOCHS_PER_CROSSLINK: 64 # 2**2 (= 4) epochs 25.6 minutes @@ -175,8 +175,6 @@ ONLINE_PERIOD: 8 LIGHT_CLIENT_COMMITTEE_SIZE: 128 # 2**8 (= 256) | epochs | ~27 hours LIGHT_CLIENT_COMMITTEE_PERIOD: 256 -# 2**8 (= 256) | epochs | ~27 hours -SHARD_COMMITTEE_PERIOD: 256 # 2**18 (= 262,144) SHARD_BLOCK_CHUNK_SIZE: 262144 # 2**2 (= 4) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 38f12d297..5c1511e6d 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -95,7 +95,7 @@ SLOTS_PER_HISTORICAL_ROOT: 64 # 2**8 (= 256) epochs MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 # [customized] higher frequency of committee turnover and faster time to acceptable voluntary exit -PERSISTENT_COMMITTEE_PERIOD: 128 +SHARD_COMMITTEE_PERIOD: 64 # [customized] fast catchup crosslinks MAX_EPOCHS_PER_CROSSLINK: 4 # 2**2 (= 4) epochs @@ -178,8 +178,6 @@ ONLINE_PERIOD: 8 LIGHT_CLIENT_COMMITTEE_SIZE: 128 # 2**8 (= 256) | epochs LIGHT_CLIENT_COMMITTEE_PERIOD: 256 -# 2**8 (= 256) | epochs -SHARD_COMMITTEE_PERIOD: 256 # 2**18 (= 262,144) SHARD_BLOCK_CHUNK_SIZE: 262144 # 2**2 (= 4) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 62615b0e7..0b66cff3c 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -223,7 +223,8 @@ The following values are (non-configurable) constants used throughout the specif | `EPOCHS_PER_ETH1_VOTING_PERIOD` | `2**5` (= 32) | epochs | ~3.4 hours | | `SLOTS_PER_HISTORICAL_ROOT` | `2**13` (= 8,192) | slots | ~27 hours | | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours | -| `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | 9 days | +| `SHARD_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | + ### State list lengths @@ -1688,7 +1689,7 @@ def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVolu # Exits must specify an epoch when they become valid; they are not valid before then assert get_current_epoch(state) >= voluntary_exit.epoch # Verify the validator has been active long enough - assert get_current_epoch(state) >= validator.activation_epoch + PERSISTENT_COMMITTEE_PERIOD + assert get_current_epoch(state) >= validator.activation_epoch + SHARD_COMMITTEE_PERIOD # Verify signature domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch) signing_root = compute_signing_root(voluntary_exit, domain) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 35f6a6425..fc5fe9e14 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -100,7 +100,6 @@ Configuration is not namespaced. Instead it is strictly an extension; | `ONLINE_PERIOD` | `OnlineEpochs(2**3)` (= 8) | online epochs | ~51 min | | `LIGHT_CLIENT_COMMITTEE_SIZE` | `2**7` (= 128) | | `LIGHT_CLIENT_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | -| `SHARD_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | | `MAX_SHARD_BLOCK_SIZE` | `2**20` (= 1,048,576) | | | `TARGET_SHARD_BLOCK_SIZE` | `2**18` (= 262,144) | | | `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | | diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py index 19915750f..9464f80aa 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py @@ -34,8 +34,8 @@ def run_voluntary_exit_processing(spec, state, signed_voluntary_exit, valid=True @with_all_phases @spec_state_test def test_success(spec, state): - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -53,8 +53,8 @@ def test_success(spec, state): @spec_state_test @always_bls def test_invalid_signature(spec, state): - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -71,8 +71,8 @@ def test_invalid_signature(spec, state): @with_all_phases @spec_state_test def test_success_exit_queue(spec, state): - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH current_epoch = spec.get_current_epoch(state) @@ -115,8 +115,8 @@ def test_success_exit_queue(spec, state): @with_all_phases @spec_state_test def test_default_exit_epoch_subsequent_exit(spec, state): - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -137,8 +137,8 @@ def test_default_exit_epoch_subsequent_exit(spec, state): @with_all_phases @spec_state_test def test_validator_exit_in_future(spec, state): - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -156,8 +156,8 @@ def test_validator_exit_in_future(spec, state): @with_all_phases @spec_state_test def test_validator_invalid_validator_index(spec, state): - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -190,8 +190,8 @@ def test_validator_not_active(spec, state): @with_all_phases @spec_state_test def test_validator_already_exited(spec, state): - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow validator able to exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow validator able to exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -218,7 +218,7 @@ def test_validator_not_active_long_enough(spec, state): assert ( current_epoch - state.validators[validator_index].activation_epoch < - spec.PERSISTENT_COMMITTEE_PERIOD + spec.SHARD_COMMITTEE_PERIOD ) yield from run_voluntary_exit_processing(spec, state, signed_voluntary_exit, False) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py index 29a9dcca2..6ae71c16e 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py @@ -448,7 +448,7 @@ def test_attestation(spec, state): assert spec.hash_tree_root(state.previous_epoch_attestations) == pre_current_attestations_root -# In phase1 a committee is computed for PERSISTENT_COMMITTEE_PERIOD slots ago, +# In phase1 a committee is computed for SHARD_COMMITTEE_PERIOD slots ago, # exceeding the minimal-config randao mixes memory size. @with_phases(['phase0']) @spec_state_test @@ -458,8 +458,8 @@ def test_voluntary_exit(spec, state): spec.get_current_epoch(state) )[-1] - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH yield 'pre', state From b109e7da5a5c179e1081413c7bc3f8f8db95d95c Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 5 May 2020 14:46:02 -0600 Subject: [PATCH 23/70] add test for inconsistent head and target in attestation fork choice --- .../test/fork_choice/test_on_attestation.py | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 360c18ccd..b2d33d0aa 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -1,7 +1,7 @@ from eth2spec.test.context import PHASE0, with_all_phases, spec_state_test from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.attestations import get_valid_attestation, sign_attestation -from eth2spec.test.helpers.state import transition_to, state_transition_and_sign_block, next_epoch +from eth2spec.test.helpers.state import transition_to, state_transition_and_sign_block, next_epoch, next_slot def run_on_attestation(spec, state, store, attestation, valid=True): @@ -116,6 +116,44 @@ def test_on_attestation_mismatched_target_and_slot(spec, state): run_on_attestation(spec, state, store, attestation, False) +@with_all_phases +@spec_state_test +def test_on_attestation_inconsistent_target_and_head(spec, state): + store = spec.get_forkchoice_store(state) + spec.on_tick(store, store.time + 2 * spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH) + + # Create chain 1 as empty chain between genesis and start of 1st epoch + target_state_1 = state.copy() + next_epoch(spec, target_state_1) + + # Create chain 2 with different block in chain from chain 1 from chain 1 from chain 1 from chain 1 + target_state_2 = state.copy() + diff_block = build_empty_block_for_next_slot(spec, target_state_2) + signed_diff_block = state_transition_and_sign_block(spec, target_state_2, diff_block) + spec.on_block(store, signed_diff_block) + next_epoch(spec, target_state_2) + next_slot(spec, target_state_2) + + # Create and store block new head block on target state 1 + head_block = build_empty_block_for_next_slot(spec, target_state_1) + signed_head_block = state_transition_and_sign_block(spec, target_state_1, head_block) + spec.on_block(store, signed_head_block) + + # Attest to head of chain 1 + attestation = get_valid_attestation(spec, target_state_1, slot=head_block.slot, signed=False) + epoch = spec.compute_epoch_at_slot(attestation.data.slot) + + # Set attestation target to be from chain 2 + attestation.data.target = spec.Checkpoint(epoch=epoch, root=spec.get_block_root(target_state_2, epoch)) + sign_attestation(spec, state, attestation) + + assert attestation.data.target.epoch == spec.GENESIS_EPOCH + 1 + assert spec.compute_epoch_at_slot(attestation.data.slot) == spec.GENESIS_EPOCH + 1 + assert spec.get_block_root(target_state_1, epoch) != attestation.data.target.root + + run_on_attestation(spec, state, store, attestation, False) + + @with_all_phases @spec_state_test def test_on_attestation_target_not_in_store(spec, state): From 12aa84fc8a7a0c93f5ad52fbed539e662a2e8e59 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 7 May 2020 10:47:20 -0600 Subject: [PATCH 24/70] PR feedback --- .../eth2spec/test/helpers/attestations.py | 2 +- .../test_process_rewards_and_penalties.py | 20 +++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 8c2e10edf..2c15a5136 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -152,7 +152,7 @@ def get_valid_attestation(spec, shard_transition=None, signed=False, on_time=True): - # If filter_participant_set is filters everything, the attestation has 0 participants, and cannot be signed. + # If filter_participant_set filters everything, the attestation has 0 participants, and cannot be signed. # Thus strictly speaking invalid when no participant is added later. if slot is None: slot = state.slot diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py index 63aafe521..337f7f25c 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py @@ -22,9 +22,13 @@ def run_process_rewards_and_penalties(spec, state): yield from run_epoch_processing_with(spec, state, 'process_rewards_and_penalties') -def prepare_state_with_full_attestations(spec, state, participation_fn=None): - # participation_fn: (slot, committee_index, committee_indices_set) -> participants_indices_set +def prepare_state_with_attestations(spec, state, participation_fn=None): + """ + Prepare state with attestations according to the ``participation_fn``. + If no ``participation_fn``, default to "full" -- max committee participation at each slot. + participation_fn: (slot, committee_index, committee_indices_set) -> participants_indices_set + """ # Go to start of next epoch to ensure can have full participation next_epoch(spec, state) @@ -100,7 +104,7 @@ def test_genesis_epoch_full_attestations_no_rewards(spec, state): @with_all_phases @spec_state_test def test_full_attestations(spec, state): - attestations = prepare_state_with_full_attestations(spec, state) + attestations = prepare_state_with_attestations(spec, state) pre_state = state.copy() @@ -118,7 +122,7 @@ def test_full_attestations(spec, state): @with_all_phases @spec_state_test def test_full_attestations_random_incorrect_fields(spec, state): - attestations = prepare_state_with_full_attestations(spec, state) + attestations = prepare_state_with_attestations(spec, state) for i, attestation in enumerate(state.previous_epoch_attestations): if i % 3 == 0: # Mess up some head votes @@ -143,7 +147,7 @@ def test_full_attestations_random_incorrect_fields(spec, state): @with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.MAX_EFFECTIVE_BALANCE // 2) @single_phase def test_full_attestations_misc_balances(spec, state): - attestations = prepare_state_with_full_attestations(spec, state) + attestations = prepare_state_with_attestations(spec, state) pre_state = state.copy() @@ -175,7 +179,7 @@ def test_full_attestations_misc_balances(spec, state): @with_custom_state(balances_fn=low_single_balance, threshold_fn=zero_activation_threshold) @single_phase def test_full_attestations_one_validaor_one_gwei(spec, state): - attestations = prepare_state_with_full_attestations(spec, state) + attestations = prepare_state_with_attestations(spec, state) yield from run_process_rewards_and_penalties(spec, state) @@ -207,7 +211,7 @@ def run_with_participation(spec, state, participation_fn): participated.update(att_participants) return att_participants - attestations = prepare_state_with_full_attestations(spec, state, participation_fn=participation_tracker) + attestations = prepare_state_with_attestations(spec, state, participation_fn=participation_tracker) pre_state = state.copy() @@ -292,7 +296,7 @@ def test_duplicate_attestation(spec, state): @spec_state_test # Case when some eligible attestations are slashed. Modifies attesting_balance and consequently rewards/penalties. def test_attestations_some_slashed(spec, state): - attestations = prepare_state_with_full_attestations(spec, state) + attestations = prepare_state_with_attestations(spec, state) attesting_indices_before_slashings = list(spec.get_unslashed_attesting_indices(state, attestations)) # Slash maximum amount of validators allowed per epoch. From 1137e0332d66a7f523640232d2a1856b5b2d3e95 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 7 May 2020 14:38:32 -0600 Subject: [PATCH 25/70] move MAX_ATTESTER_SLASHINGS to 2, add multiple slashings per block tests --- configs/mainnet.yaml | 4 +- configs/minimal.yaml | 4 +- specs/phase0/beacon-chain.md | 2 +- .../test/helpers/attester_slashings.py | 22 ++- .../test/phase_0/sanity/test_blocks.py | 140 ++++++++++++++++-- 5 files changed, 154 insertions(+), 18 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 60bd1c087..42845c235 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -132,8 +132,8 @@ MIN_SLASHING_PENALTY_QUOTIENT: 32 # --------------------------------------------------------------- # 2**4 (= 16) MAX_PROPOSER_SLASHINGS: 16 -# 2**0 (= 1) -MAX_ATTESTER_SLASHINGS: 1 +# 2**1 (= 2) +MAX_ATTESTER_SLASHINGS: 2 # 2**7 (= 128) MAX_ATTESTATIONS: 128 # 2**4 (= 16) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 5c1511e6d..d8e346ffa 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -132,8 +132,8 @@ MIN_SLASHING_PENALTY_QUOTIENT: 32 # --------------------------------------------------------------- # 2**4 (= 16) MAX_PROPOSER_SLASHINGS: 16 -# 2**0 (= 1) -MAX_ATTESTER_SLASHINGS: 1 +# 2**1 (= 2) +MAX_ATTESTER_SLASHINGS: 2 # 2**7 (= 128) MAX_ATTESTATIONS: 128 # 2**4 (= 16) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 899778fd9..6d60d76e3 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -252,7 +252,7 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | | - | - | | `MAX_PROPOSER_SLASHINGS` | `2**4` (= 16) | -| `MAX_ATTESTER_SLASHINGS` | `2**0` (= 1) | +| `MAX_ATTESTER_SLASHINGS` | `2**1` (= 2) | | `MAX_ATTESTATIONS` | `2**7` (= 128) | | `MAX_DEPOSITS` | `2**4` (= 16) | | `MAX_VOLUNTARY_EXITS` | `2**4` (= 16) | diff --git a/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py b/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py index 975f34c20..e743ca8ff 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py @@ -1,5 +1,5 @@ from eth2spec.test.context import PHASE1 -from eth2spec.test.helpers.attestations import get_valid_attestation, sign_attestation +from eth2spec.test.helpers.attestations import get_valid_attestation, sign_attestation, sign_indexed_attestation def get_valid_attester_slashing(spec, state, signed_1=False, signed_2=False): @@ -17,6 +17,26 @@ def get_valid_attester_slashing(spec, state, signed_1=False, signed_2=False): ) +def get_valid_attester_slashing_by_indices(spec, state, indices_1, indices_2=None, signed_1=False, signed_2=False): + if indices_2 is None: + indices_2 = indices_1 + + assert indices_1 == sorted(indices_1) + assert indices_2 == sorted(indices_2) + + attester_slashing = get_valid_attester_slashing(spec, state) + + attester_slashing.attestation_1.attesting_indices = indices_1 + attester_slashing.attestation_2.attesting_indices = indices_2 + + if signed_1: + sign_indexed_attestation(spec, state, attester_slashing.attestation_1) + if signed_2: + sign_indexed_attestation(spec, state, attester_slashing.attestation_2) + + return attester_slashing + + def get_indexed_attestation_participants(spec, indexed_att): """ Wrapper around index-attestation to return the list of participant indices, regardless of spec phase. diff --git a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py index 1864006bd..1c0fa5eb9 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py @@ -4,7 +4,11 @@ from eth2spec.test.helpers.state import get_balance, state_transition_and_sign_b from eth2spec.test.helpers.block import build_empty_block_for_next_slot, build_empty_block, sign_block, \ transition_unsigned_block from eth2spec.test.helpers.keys import privkeys, pubkeys -from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing, get_indexed_attestation_participants +from eth2spec.test.helpers.attester_slashings import ( + get_valid_attester_slashing_by_indices, + get_valid_attester_slashing, + get_indexed_attestation_participants, +) from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing, check_proposer_slashing_effect from eth2spec.test.helpers.attestations import get_valid_attestation, fill_block_shard_transitions_by_attestations from eth2spec.test.helpers.deposits import prepare_state_and_deposit @@ -326,13 +330,14 @@ def test_multiple_different_proposer_slashings_same_block(spec, state): check_proposer_slashing_effect(spec, pre_state, state, slashed_index) -def check_attester_slashing_effect(spec, pre_state, state, validator_index): - slashed_validator = state.validators[validator_index] - assert slashed_validator.slashed - assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH - assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH - # lost whistleblower reward - assert get_balance(state, validator_index) < get_balance(pre_state, validator_index) +def check_attester_slashing_effect(spec, pre_state, state, slashed_indices): + for slashed_index in slashed_indices: + slashed_validator = state.validators[slashed_index] + assert slashed_validator.slashed + assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH + assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH + # lost whistleblower reward + assert get_balance(state, slashed_index) < get_balance(pre_state, slashed_index) proposer_index = spec.get_beacon_proposer_index(state) # gained whistleblower reward @@ -346,9 +351,9 @@ def test_attester_slashing(spec, state): pre_state = state.copy() attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) - validator_index = get_indexed_attestation_participants(spec, attester_slashing.attestation_1)[0] + slashed_indices = get_indexed_attestation_participants(spec, attester_slashing.attestation_1) - assert not state.validators[validator_index].slashed + assert not any(state.validators[i].slashed for i in slashed_indices) yield 'pre', state @@ -363,13 +368,124 @@ def test_attester_slashing(spec, state): yield 'blocks', [signed_block] yield 'post', state - check_attester_slashing_effect(spec, pre_state, state, validator_index) + check_attester_slashing_effect(spec, pre_state, state, slashed_indices) + + +@with_all_phases +@spec_state_test +def test_duplicate_attester_slashing(spec, state): + # Skip test if config cannot handle multiple AttesterSlashings per block + if spec.MAX_ATTESTER_SLASHINGS < 2: + return + + attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) + attester_slashings = [attester_slashing, attester_slashing.copy()] + slashed_indices = get_indexed_attestation_participants(spec, attester_slashing.attestation_1) + + assert not any(state.validators[i].slashed for i in slashed_indices) + + yield 'pre', state + + # + # Add to state via block transition + # + block = build_empty_block_for_next_slot(spec, state) + block.body.attester_slashings = attester_slashings + + signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + + yield 'blocks', [signed_block] + yield 'post', None + + +# All AttesterSlashing tests should be adopted for Phase 1 but helper support is not yet there + +@with_phases(['phase0']) +@spec_state_test +def test_multiple_attester_slashings_no_overlap(spec, state): + # Skip test if config cannot handle multiple AttesterSlashings per block + if spec.MAX_ATTESTER_SLASHINGS < 2: + return + + # copy for later balance lookups. + pre_state = state.copy() + + full_indices = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[:8] + half_length = len(full_indices) // 2 + + attester_slashing_1 = get_valid_attester_slashing_by_indices( + spec, state, + full_indices[:half_length], signed_1=True, signed_2=True, + ) + attester_slashing_2 = get_valid_attester_slashing_by_indices( + spec, state, + full_indices[half_length:], signed_1=True, signed_2=True, + ) + attester_slashings = [attester_slashing_1, attester_slashing_2] + + assert not any(state.validators[i].slashed for i in full_indices) + + yield 'pre', state + + # + # Add to state via block transition + # + block = build_empty_block_for_next_slot(spec, state) + block.body.attester_slashings = attester_slashings + + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state + + check_attester_slashing_effect(spec, pre_state, state, full_indices) + + +@with_phases(['phase0']) +@spec_state_test +def test_multiple_attester_slashings_partial_overlap(spec, state): + # Skip test if config cannot handle multiple AttesterSlashings per block + if spec.MAX_ATTESTER_SLASHINGS < 2: + return + + # copy for later balance lookups. + pre_state = state.copy() + + full_indices = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[:8] + one_third_length = len(full_indices) // 3 + + attester_slashing_1 = get_valid_attester_slashing_by_indices( + spec, state, + full_indices[:one_third_length * 2], signed_1=True, signed_2=True, + ) + attester_slashing_2 = get_valid_attester_slashing_by_indices( + spec, state, + full_indices[one_third_length:], signed_1=True, signed_2=True, + ) + attester_slashings = [attester_slashing_1, attester_slashing_2] + + assert not any(state.validators[i].slashed for i in full_indices) + + yield 'pre', state + + # + # Add to state via block transition + # + block = build_empty_block_for_next_slot(spec, state) + block.body.attester_slashings = attester_slashings + + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state + + check_attester_slashing_effect(spec, pre_state, state, full_indices) + # TODO: currently mainnet limits attester-slashings per block to 1. # When this is increased, it should be tested to cover various combinations # of duplicate slashings and overlaps of slashed attestations within the same block - @with_all_phases @spec_state_test def test_proposer_after_inactive_index(spec, state): From 01eaf6cc99091292a5778cc7d5828e544bab8561 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 8 May 2020 11:09:12 -0600 Subject: [PATCH 26/70] Clarify the response of BlocksByRange to address #1765 --- specs/phase0/p2p-interface.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index d697e5da3..dae82f464 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -537,7 +537,9 @@ The response MUST contain no more than `count` blocks. Clients MUST order blocks by increasing slot number. -Clients MUST respond with blocks from their view of the current fork choice. In particular, blocks from slots before the finalization MUST lead to the finalized block reported in the `Status` handshake. +Clients MUST respond with blocks from their view of the current fork choice -- that is, blocks from the single chain defined by the current head. Of note, blocks from slots before the finalization MUST lead to the finalized block reported in the `Status` handshake. + +Clients MUST respond with blocks that are consistent from a single chain within the context of the request. #### BeaconBlocksByRoot From fdcc6d65bc1a14c7ddda3cdce40062e361370da6 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 8 May 2020 12:00:05 -0600 Subject: [PATCH 27/70] gossipsub v1.1 and extended validators for gossip conditions --- specs/phase0/p2p-interface.md | 57 ++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index d697e5da3..1b1348e96 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -176,7 +176,7 @@ Where ## The gossip domain: gossipsub -Clients MUST support the [gossipsub](https://github.com/libp2p/specs/tree/master/pubsub/gossipsub) libp2p protocol. +Clients MUST support the [gossipsub v1](https://github.com/libp2p/specs/tree/master/pubsub/gossipsub) libp2p protocol including the [gossipsub v1.1](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.1.md) extension. **Protocol ID:** `/meshsub/1.0.0` @@ -231,38 +231,41 @@ Clients MUST reject (fail validation) messages containing an incorrect type, or When processing incoming gossip, clients MAY descore or disconnect peers who fail to observe these constraints. +Gossipsub v1.1 introduces [Extended Validators](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.1.md#extended-validators) for the application to aid in the gossipsub peer-scoring scheme. +We utilize `ACCEPT`, `REJECT`, and `IGNORE`. For each gossipsub topic, there are application specific validations. If all validations pass, return `ACCEPT`. If one or more validations fail while processing the items in order, return either `REJECT` or `IGNORE` as specified in the prefix of the particular condition. + #### Global topics There are two primary global topics used to propagate beacon blocks and aggregate attestations to all nodes on the network. Their `Name`s are: - `beacon_block` - This topic is used solely for propagating new signed beacon blocks to all nodes on the networks. Signed blocks are sent in their entirety. The following validations MUST pass before forwarding the `signed_beacon_block` on the network - - The block is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `signed_beacon_block.message.slot <= current_slot` (a client MAY queue future blocks for processing at the appropriate slot). - - The block is from a slot greater than the latest finalized slot -- i.e. validate that `signed_beacon_block.message.slot > compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)` (a client MAY choose to validate and store such blocks for additional purposes -- e.g. slashing detection, archive nodes, etc). - - The block is the first block with valid signature received for the proposer for the slot, `signed_beacon_block.message.slot`. - - The proposer signature, `signed_beacon_block.signature`, is valid with respect to the `proposer_index` pubkey. - - The block is proposed by the expected `proposer_index` for the block's slot in the context of the current shuffling (defined by `parent_root`/`slot`). If the `proposer_index` cannot immediately be verified against the expected shuffling, the block MAY be queued for later processing while proposers for the block's branch are calculated. + - _[IGNORE]_ The block is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `signed_beacon_block.message.slot <= current_slot` (a client MAY queue future blocks for processing at the appropriate slot). + - _[IGNORE]_ The block is from a slot greater than the latest finalized slot -- i.e. validate that `signed_beacon_block.message.slot > compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)` (a client MAY choose to validate and store such blocks for additional purposes -- e.g. slashing detection, archive nodes, etc). + - _[IGNORE]_ The block is the first block with valid signature received for the proposer for the slot, `signed_beacon_block.message.slot`. + - _[REJECT]_ The proposer signature, `signed_beacon_block.signature`, is valid with respect to the `proposer_index` pubkey. + - _[REJECT]_ The block is proposed by the expected `proposer_index` for the block's slot in the context of the current shuffling (defined by `parent_root`/`slot`). If the `proposer_index` cannot immediately be verified against the expected shuffling, the block MAY be queued for later processing while proposers for the block's branch are calculated -- in such a case _do not_ `REJECT`, instead `IGNORE` this message. - `beacon_aggregate_and_proof` - This topic is used to propagate aggregated attestations (as `SignedAggregateAndProof`s) to subscribing nodes (typically validators) to be included in future blocks. The following validations MUST pass before forwarding the `signed_aggregate_and_proof` on the network. (We define the following for convenience -- `aggregate_and_proof = signed_aggregate_and_proof.message` and `aggregate = aggregate_and_proof.aggregate`) - - `aggregate.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= aggregate.data.slot` (a client MAY queue future aggregates for processing at the appropriate slot). - - The valid aggregate attestation defined by `hash_tree_root(aggregate)` has _not_ already been seen (via aggregate gossip, within a verified block, or through the creation of an equivalent aggregate locally). - - The `aggregate` is the first valid aggregate received for the aggregator with index `aggregate_and_proof.aggregator_index` for the epoch `aggregate.data.target.epoch`. - - The block being voted for (`aggregate.data.beacon_block_root`) passes validation. - - `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_aggregator(state, aggregate.data.slot, aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. - - The aggregator's validator index is within the aggregate's committee -- i.e. `aggregate_and_proof.aggregator_index in get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)`. This also means that it must never have an empty set of participants. - - The `aggregate_and_proof.selection_proof` is a valid signature of the `aggregate.data.slot` by the validator with index `aggregate_and_proof.aggregator_index`. - - The aggregator signature, `signed_aggregate_and_proof.signature`, is valid. - - The signature of `aggregate` is valid. + - _[IGNORE]_ `aggregate.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= aggregate.data.slot` (a client MAY queue future aggregates for processing at the appropriate slot). + - _[IGNORE]_ The valid aggregate attestation defined by `hash_tree_root(aggregate)` has _not_ already been seen (via aggregate gossip, within a verified block, or through the creation of an equivalent aggregate locally). + - _[IGNORE]_ The `aggregate` is the first valid aggregate received for the aggregator with index `aggregate_and_proof.aggregator_index` for the epoch `aggregate.data.target.epoch`. + - _[REJECT]_ The block being voted for (`aggregate.data.beacon_block_root`) passes validation. + - _[REJECT]_ `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_aggregator(state, aggregate.data.slot, aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. + - _[REJECT]_ The aggregator's validator index is within the aggregate's committee -- i.e. `aggregate_and_proof.aggregator_index in get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)`. This also means that it must never have an empty set of participants. + - _[REJECT]_ The `aggregate_and_proof.selection_proof` is a valid signature of the `aggregate.data.slot` by the validator with index `aggregate_and_proof.aggregator_index`. + - _[REJECT]_ The aggregator signature, `signed_aggregate_and_proof.signature`, is valid. + - _[REJECT]_ The signature of `aggregate` is valid. Additional global topics are used to propagate lower frequency validator messages. Their `Name`s are: - `voluntary_exit` - This topic is used solely for propagating signed voluntary validator exits to proposers on the network. Signed voluntary exits are sent in their entirety. The following validations MUST pass before forwarding the `signed_voluntary_exit` on to the network - - The voluntary exit is the first valid voluntary exit received for the validator with index `signed_voluntary_exit.message.validator_index`. - - All of the conditions within `process_voluntary_exit` pass validation. + - _[IGNORE]_ The voluntary exit is the first valid voluntary exit received for the validator with index `signed_voluntary_exit.message.validator_index`. + - _[REJECT]_ All of the conditions within `process_voluntary_exit` pass validation. - `proposer_slashing` - This topic is used solely for propagating proposer slashings to proposers on the network. Proposer slashings are sent in their entirety. The following validations MUST pass before forwarding the `proposer_slashing` on to the network - - The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.index`. - - All of the conditions within `process_proposer_slashing` pass validation. + - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.index`. + - _[REJECT]_ All of the conditions within `process_proposer_slashing` pass validation. - `attester_slashing` - This topic is used solely for propagating attester slashings to proposers on the network. Attester slashings are sent in their entirety. Clients who receive an attester slashing on this topic MUST validate the conditions within `process_attester_slashing` before forwarding it across the network. - - At least one index in the intersection of the attesting indices of each attestation has not yet been seen in any prior `attester_slashing` (i.e. `attester_slashed_indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices)`, verify if `any(attester_slashed_indices.difference(prior_seen_attester_slashed_indices))`). - - All of the conditions within `process_attester_slashing` pass validation. + - _[IGNORE]_ At least one index in the intersection of the attesting indices of each attestation has not yet been seen in any prior `attester_slashing` (i.e. `attester_slashed_indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices)`, verify if `any(attester_slashed_indices.difference(prior_seen_attester_slashed_indices))`). + - _[REJECT]_ All of the conditions within `process_attester_slashing` pass validation. #### Attestation subnets @@ -270,12 +273,12 @@ Additional global topics are used to propagate lower frequency validator message Attestation subnets are used to propagate unaggregated attestations to subsections of the network. Their `Name`s are: - `committee_index{subnet_id}_beacon_attestation` - These topics are used to propagate unaggregated attestations to the subnet `subnet_id` (typically beacon and persistent committees) to be aggregated before being gossiped to `beacon_aggregate_and_proof`. The following validations MUST pass before forwarding the `attestation` on the subnet. - - The attestation's committee index (`attestation.data.index`) is for the correct subnet. - - `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` (a client MAY queue future attestations for processing at the appropriate slot). - - The attestation is unaggregated -- that is, it has exactly one participating validator (`len([bit for bit in attestation.aggregation_bits if bit == 0b1]) == 1`). - - There has been no other valid attestation seen on an attestation subnet that has an identical `attestation.data.target.epoch` and participating validator index. - - The block being voted for (`attestation.data.beacon_block_root`) passes validation. - - The signature of `attestation` is valid. + - _[REJECT]_ The attestation's committee index (`attestation.data.index`) is for the correct subnet. + - _[IGNORE]_ `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` (a client MAY queue future attestations for processing at the appropriate slot). + - _[REJECT]_ The attestation is unaggregated -- that is, it has exactly one participating validator (`len([bit for bit in attestation.aggregation_bits if bit == 0b1]) == 1`). + - _[IGNORE]_ There has been no other valid attestation seen on an attestation subnet that has an identical `attestation.data.target.epoch` and participating validator index. + - _[REJECT]_ The block being voted for (`attestation.data.beacon_block_root`) passes validation. + - _[REJECT]_ The signature of `attestation` is valid. #### Attestations and Aggregation From cdd0ed0f7b7d0dcfee36f71df8a29d81863a52a3 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 9 May 2020 11:48:48 +0800 Subject: [PATCH 28/70] Update to IETF BLS draft-irtf-cfrg-bls-signature-02 --- specs/phase0/beacon-chain.md | 4 ++-- specs/phase1/beacon-chain.md | 4 ++-- specs/phase1/custody-game.md | 2 +- specs/phase1/shard-transition.md | 5 ++++- tests/core/pyspec/eth2spec/test/helpers/keys.py | 2 +- tests/core/pyspec/eth2spec/utils/bls.py | 8 ++++---- tests/generators/bls/main.py | 10 +++++----- 7 files changed, 19 insertions(+), 16 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 899778fd9..83f9f7ec2 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -603,13 +603,13 @@ def bytes_to_int(data: bytes) -> uint64: #### BLS Signatures -Eth2 makes use of BLS signatures as specified in the [IETF draft BLS specification](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-00). Specifically, eth2 uses the `BLS_SIG_BLS12381G2-SHA256-SSWU-RO-_POP_` ciphersuite which implements the following interfaces: +Eth2 makes use of BLS signatures as specified in the [IETF draft BLS specification](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-02). Specifically, eth2 uses the `BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_` ciphersuite which implements the following interfaces: - `def Sign(SK: int, message: Bytes) -> BLSSignature` - `def Verify(PK: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` - `def Aggregate(signatures: Sequence[BLSSignature]) -> BLSSignature` - `def FastAggregateVerify(PKs: Sequence[BLSPubkey], message: Bytes, signature: BLSSignature) -> bool` -- `def AggregateVerify(pairs: Sequence[PK: BLSPubkey, message: Bytes], signature: BLSSignature) -> bool` +- `def AggregateVerify(PKs: Sequence[BLSPubkey], message: Sequence[Bytes], signature: BLSSignature) -> bool` Within these specifications, BLS signatures are treated as a module for notational clarity, thus to verify a signature `bls.Verify(...)` is used. diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 331243de6..83d63ee2a 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -596,7 +596,7 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe all_signing_roots.append(compute_signing_root(attestation_wrapper, domain)) else: assert not cbit - return bls.AggregateVerify(zip(all_pubkeys, all_signing_roots), signature=attestation.signature) + return bls.AggregateVerify(all_pubkeys, all_signing_roots, signature=attestation.signature) ``` #### `is_shard_attestation` @@ -764,7 +764,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr for header in headers ] # Verify combined proposer signature - assert bls.AggregateVerify(zip(pubkeys, signing_roots), signature=transition.proposer_signature_aggregate) + assert bls.AggregateVerify(pubkeys, signing_roots, signature=transition.proposer_signature_aggregate) # Save updated state state.shard_states[shard] = transition.shard_states[len(transition.shard_states) - 1] diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index eb243f8fb..5f5acd84f 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -300,7 +300,7 @@ def process_early_derived_secret_reveal(state: BeaconState, reveal: EarlyDerived domain = get_domain(state, DOMAIN_RANDAO, reveal.epoch) signing_roots = [compute_signing_root(root, domain) for root in [hash_tree_root(reveal.epoch), reveal.mask]] - assert bls.AggregateVerify(zip(pubkeys, signing_roots), reveal.reveal) + assert bls.AggregateVerify(pubkeys, signing_roots, reveal.reveal) if reveal.epoch >= get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING: # Full slashing when the secret was revealed so early it may be a valid custody diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index a8de508fb..5b6a72f28 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -280,7 +280,10 @@ def get_shard_transition(beacon_state: BeaconState, if proposal.signature != BLSSignature(): proposer_signatures.append(proposal.signature) - proposer_signature_aggregate = bls.Aggregate(proposer_signatures) + if len(proposer_signatures) > 0: + proposer_signature_aggregate = bls.Aggregate(proposer_signatures) + else: + proposer_signature_aggregate = BLSSignature() return ShardTransition( start_slot=start_slot, diff --git a/tests/core/pyspec/eth2spec/test/helpers/keys.py b/tests/core/pyspec/eth2spec/test/helpers/keys.py index 7f7820d3a..d813870e0 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/keys.py +++ b/tests/core/pyspec/eth2spec/test/helpers/keys.py @@ -2,5 +2,5 @@ from py_ecc.bls import G2ProofOfPossession as bls from eth2spec.phase0 import spec privkeys = [i + 1 for i in range(spec.SLOTS_PER_EPOCH * 256)] -pubkeys = [bls.PrivToPub(privkey) for privkey in privkeys] +pubkeys = [bls.SkToPk(privkey) for privkey in privkeys] pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} diff --git a/tests/core/pyspec/eth2spec/utils/bls.py b/tests/core/pyspec/eth2spec/utils/bls.py index 3b648fac9..7f265b555 100644 --- a/tests/core/pyspec/eth2spec/utils/bls.py +++ b/tests/core/pyspec/eth2spec/utils/bls.py @@ -29,13 +29,13 @@ def Verify(PK, message, signature): @only_with_bls(alt_return=True) -def AggregateVerify(pairs, signature): - return bls.AggregateVerify(pairs, signature) +def AggregateVerify(pubkeys, messages, signature): + return bls.AggregateVerify(pubkeys, messages, signature) @only_with_bls(alt_return=True) -def FastAggregateVerify(PKs, message, signature): - return bls.FastAggregateVerify(PKs, message, signature) +def FastAggregateVerify(pubkeys, message, signature): + return bls.FastAggregateVerify(pubkeys, message, signature) @only_with_bls(alt_return=STUB_SIGNATURE) diff --git a/tests/generators/bls/main.py b/tests/generators/bls/main.py index 455292ae3..7bb093593 100644 --- a/tests/generators/bls/main.py +++ b/tests/generators/bls/main.py @@ -69,7 +69,7 @@ def case02_verify(): for message in MESSAGES: # Valid signature signature = bls.G2ProofOfPossession.Sign(privkey, message) - pubkey = bls.G2ProofOfPossession.PrivToPub(privkey) + pubkey = bls.G2ProofOfPossession.SkToPk(privkey) identifier = f'{encode_hex(pubkey)}_{encode_hex(message)}' yield f'verify_valid_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { @@ -81,7 +81,7 @@ def case02_verify(): } # Invalid signatures -- wrong pubkey - wrong_pubkey = bls.G2ProofOfPossession.PrivToPub(PRIVKEYS[(i + 1) % len(PRIVKEYS)]) + wrong_pubkey = bls.G2ProofOfPossession.SkToPk(PRIVKEYS[(i + 1) % len(PRIVKEYS)]) identifier = f'{encode_hex(wrong_pubkey)}_{encode_hex(message)}' yield f'verify_wrong_pubkey_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { @@ -119,7 +119,7 @@ def case04_fast_aggregate_verify(): privkeys = PRIVKEYS[:i + 1] sigs = [bls.G2ProofOfPossession.Sign(privkey, message) for privkey in privkeys] aggregate_signature = bls.G2ProofOfPossession.Aggregate(sigs) - pubkeys = [bls.G2ProofOfPossession.PrivToPub(privkey) for privkey in privkeys] + pubkeys = [bls.G2ProofOfPossession.SkToPk(privkey) for privkey in privkeys] pubkeys_serial = [encode_hex(pubkey) for pubkey in pubkeys] # Valid signature @@ -134,7 +134,7 @@ def case04_fast_aggregate_verify(): } # Invalid signature -- extra pubkey - pubkeys_extra = pubkeys + [bls.G2ProofOfPossession.PrivToPub(PRIVKEYS[-1])] + pubkeys_extra = pubkeys + [bls.G2ProofOfPossession.SkToPk(PRIVKEYS[-1])] pubkeys_extra_serial = [encode_hex(pubkey) for pubkey in pubkeys_extra] identifier = f'{pubkeys_extra_serial}_{encode_hex(message)}' yield f'fast_aggregate_verify_extra_pubkey_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { @@ -164,7 +164,7 @@ def case05_aggregate_verify(): sigs = [] for privkey, message in zip(PRIVKEYS, MESSAGES): sig = bls.G2ProofOfPossession.Sign(privkey, message) - pubkey = bls.G2ProofOfPossession.PrivToPub(privkey) + pubkey = bls.G2ProofOfPossession.SkToPk(privkey) pairs.append({ 'pubkey': encode_hex(pubkey), 'message': encode_hex(message), From a8e3fe7551d063dd80e3578987c28e4306f593d0 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 9 May 2020 21:22:38 +0200 Subject: [PATCH 29/70] Update p2p reqresp with explicit handling of reqresp size bounds --- specs/phase0/p2p-interface.md | 48 ++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index d697e5da3..58f8653ca 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -98,6 +98,7 @@ It consists of four main sections: - [Why are we compressing, and at which layers?](#why-are-we-compressing-and-at-which-layers) - [Why are using Snappy for compression?](#why-are-using-snappy-for-compression) - [Can I get access to unencrypted bytes on the wire for debugging purposes?](#can-i-get-access-to-unencrypted-bytes-on-the-wire-for-debugging-purposes) + - [What are SSZ type size bounds?](#what-are-ssz-type-size-bounds) - [libp2p implementations matrix](#libp2p-implementations-matrix) @@ -329,9 +330,12 @@ result ::= “0” | “1” | “2” | [“128” ... ”255”] The encoding-dependent header may carry metadata or assertions such as the encoded payload length, for integrity and attack proofing purposes. Because req/resp streams are single-use and stream closures implicitly delimit the boundaries, it is not strictly necessary to length-prefix payloads; however, certain encodings like SSZ do, for added security. -A `response` is formed by zero or more `response_chunk`s. Responses that consist of a single SSZ-list (such as `BlocksByRange` and `BlocksByRoot`) send each list item as a `response_chunk`. All other response types (non-Lists) send a single `response_chunk`. The encoded-payload of a `response_chunk` has a maximum uncompressed byte size of `MAX_CHUNK_SIZE`. +A `response` is formed by zero or more `response_chunk`s. Responses that consist of a single SSZ-list (such as `BlocksByRange` and `BlocksByRoot`) send each list item as a `response_chunk`. All other response types (non-Lists) send a single `response_chunk`. -Clients MUST ensure the each encoded payload of a `response_chunk` is less than or equal to `MAX_CHUNK_SIZE`; if not, they SHOULD reset the stream immediately. Clients tracking peer reputation MAY decrement the score of the misbehaving peer under this circumstance. +For both `request`s and `response`s, he `encoding-dependent-header` MUST be valid, and the `encoded-payload` must be valid within the constraints of the `encoding-dependent-header`. +This includes type-specific bounds on payload size for some encoding strategies. Regardless of these type specific bounds, a global maximum uncompressed byte size of `MAX_CHUNK_SIZE` MUST be applied to all method response chunks. + +Clients MUST ensure that lengths are within these bounds; if not, they SHOULD reset the stream immediately. Clients tracking peer reputation MAY decrement the score of the misbehaving peer under this circumstance. #### Requesting side @@ -339,13 +343,22 @@ Once a new stream with the protocol ID for the request type has been negotiated, The requester MUST close the write side of the stream once it finishes writing the request message. At this point, the stream will be half-closed. -The requester MUST wait a maximum of `TTFB_TIMEOUT` for the first response byte to arrive (time to first byte—or TTFB—timeout). On that happening, the requester allows a further `RESP_TIMEOUT` for each subsequent `response_chunk` received. For responses consisting of potentially many `response_chunk`s (an SSZ-list) the requester SHOULD read from the stream until either; a) An error result is received in one of the chunks, b) The responder closes the stream, c) More than `MAX_CHUNK_SIZE` bytes have been read for a single `response_chunk` payload or d) More than the maximum number of requested chunks are read. For requests consisting of a single `response_chunk` and a length-prefix, the requester should read the exact number of bytes defined by the length-prefix before closing the stream. +The requester MUST wait a maximum of `TTFB_TIMEOUT` for the first response byte to arrive (time to first byte—or TTFB—timeout). On that happening, the requester allows a further `RESP_TIMEOUT` for each subsequent `response_chunk` received. If any of these timeouts fire, the requester SHOULD reset the stream and deem the req/resp operation to have failed. +A requester SHOULD read from the stream until either: +a) An error result is received in one of the chunks (the error payload MAY be read before stopping). +b) The responder closes the stream. +d) Any part of the `response_chunk` fails validation. +e) The maximum number of requested chunks are read. + +For requests consisting of a single valid `response_chunk`, the requester SHOULD read the chunk fully, as defined by the `encoding-dependent-header`, before closing the stream. + #### Responding side -Once a new stream with the protocol ID for the request type has been negotiated, the responder must process the incoming request message according to the encoding strategy, until EOF (denoting stream half-closure by the requester). +Once a new stream with the protocol ID for the request type has been negotiated, the responder SHOULD process the incoming request and MUST validate it before processing it. +Request processing and validation MUST be done according to the encoding strategy, until EOF (denoting stream half-closure by the requester). The responder MUST: @@ -415,18 +428,21 @@ If Snappy is applied, it can be passed through a buffered Snappy writer to compr *Reading*: After reading the expected SSZ byte length, the SSZ decoder can directly read the contents from the stream. If snappy is applied, it can be passed through a buffered Snappy reader to decompress frame by frame. -A reader SHOULD NOT read more than `max_encoded_len(n)` bytes after reading the SSZ length prefix `n` from the header. +Before reading the payload, the header MUST be validated: +- The unsigned protobuf varint used for the length-prefix MUST not be longer than 10 bytes, which is sufficient for any `uint64`. +- The length-prefix is within the expected [size bounds derived from the payload SSZ type](#what-are-ssz-type-size-bounds). + +After reading a valid header, the payload MAY be read, while maintaining the size constraints from the header. + +A reader SHOULD NOT read more than `max_encoded_len(n)` bytes after reading the SSZ length-prefix `n` from the header. - For `ssz` this is: `n` - For `ssz_snappy` this is: `32 + n + n // 6`. This is considered the [worst-case compression result](https://github.com/google/snappy/blob/537f4ad6240e586970fe554614542e9717df7902/snappy.cc#L98) by Snappy. A reader SHOULD consider the following cases as invalid input: -- A SSZ length prefix that, compared against the SSZ type information (vector lengths, list limits, integer sizes, etc.), is: - - Smaller than the expected minimum serialized length. - - Bigger than the expected maximum serialized length. -- Any remaining bytes, after having read the `n` SSZ bytes. An EOF is expected. -- An early EOF, before fully reading the declared length prefix worth of SSZ bytes. +- Any remaining bytes, after having read the `n` SSZ bytes. An EOF is expected if more bytes are read than required. +- An early EOF, before fully reading the declared length-prefix worth of SSZ bytes. -In case of an invalid input, a reader MUST: +In case of an invalid input (header or payload), a reader MUST: - From requests: send back an error message, response code `InvalidRequest`. The request itself is ignored. - From responses: ignore the response, the response MUST be considered bad server behavior. @@ -1055,7 +1071,7 @@ For all these reasons, generically negotiating compression algorithms may be tre At this stage, the wisest choice is to consider libp2p a messenger of bytes, and to make application layer participate in compressing those bytes. This looks different depending on the interaction layer: -- Gossip domain: since gossipsub has a framing protocol and exposes an API, we compress the payload (when dictated by the encoding token in the topic name) prior to publishing the message via the API. No length prefixing is necessary because protobuf takes care of bounding the field in the serialized form. +- Gossip domain: since gossipsub has a framing protocol and exposes an API, we compress the payload (when dictated by the encoding token in the topic name) prior to publishing the message via the API. No length-prefixing is necessary because protobuf takes care of bounding the field in the serialized form. - Req/Resp domain: since we define custom protocols that operate on byte streams, implementers are encouraged to encapsulate the encoding and compression logic behind MessageReader and MessageWriter components/strategies that can be layered on top of the raw byte streams. ### Why are using Snappy for compression? @@ -1070,6 +1086,14 @@ If your libp2p library relies on frameworks/runtimes such as Netty (jvm) or Node For specific ad-hoc testing scenarios, you can use the [plaintext/2.0.0 secure channel](https://github.com/libp2p/specs/blob/master/plaintext/README.md) (which is essentially no-op encryption or message authentication), in combination with tcpdump or Wireshark to inspect the wire. +### What are SSZ type size bounds? + +The SSZ encoding outputs of each type have size bounds: each dynamic type, such as a list, has a "limit", which can be used to compute the maximum valid output size. +Note that for some more complex dynamic-length objects, element offsets (4 bytes each) may need to be included. +Other types are static, they have a fixed size: no dynamic-length content is involved, and the minimum and maximum bounds are the same. + +For reference, the type bounds can be computed ahead of time, [as per this example](https://gist.github.com/protolambda/db75c7faa1e94f2464787a480e5d613e). It is advisable to derive these lengths from the SSZ type definitions in use, to ensure that version changes do not cause out-of-sync type bounds. + # libp2p implementations matrix This section will soon contain a matrix showing the maturity/state of the libp2p features required by this spec across the languages in which Eth2 clients are being developed. From 16363604266967f91bb60d3b00e23be7a4339452 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 11 May 2020 08:02:09 -0600 Subject: [PATCH 30/70] remove todo comment Co-authored-by: Hsiao-Wei Wang --- tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py index 1c0fa5eb9..4fee297ad 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py @@ -482,10 +482,6 @@ def test_multiple_attester_slashings_partial_overlap(spec, state): check_attester_slashing_effect(spec, pre_state, state, full_indices) -# TODO: currently mainnet limits attester-slashings per block to 1. -# When this is increased, it should be tested to cover various combinations -# of duplicate slashings and overlaps of slashed attestations within the same block - @with_all_phases @spec_state_test def test_proposer_after_inactive_index(spec, state): From b3dd99f4f4d624eed075a2510099a782d309920e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 19 Apr 2020 15:37:08 +1000 Subject: [PATCH 31/70] Loosen restrictions for aggregate propogation --- specs/phase0/p2p-interface.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 1b1348e96..4ffc45bf7 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -249,8 +249,9 @@ There are two primary global topics used to propagate beacon blocks and aggregat - _[IGNORE]_ The valid aggregate attestation defined by `hash_tree_root(aggregate)` has _not_ already been seen (via aggregate gossip, within a verified block, or through the creation of an equivalent aggregate locally). - _[IGNORE]_ The `aggregate` is the first valid aggregate received for the aggregator with index `aggregate_and_proof.aggregator_index` for the epoch `aggregate.data.target.epoch`. - _[REJECT]_ The block being voted for (`aggregate.data.beacon_block_root`) passes validation. + - _[REJECT]_ The attestation has participants -- that is, `len(get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)) >= 1`. - _[REJECT]_ `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_aggregator(state, aggregate.data.slot, aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. - - _[REJECT]_ The aggregator's validator index is within the aggregate's committee -- i.e. `aggregate_and_proof.aggregator_index in get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)`. This also means that it must never have an empty set of participants. + - _[REJECT]_ The aggregator's validator index is within the committee -- i.e. `aggregate_and_proof.aggregator_index in get_beacon_committee(state, aggregate.data.slot, aggregate.data.index)`. This also means that it must never have an empty set of participants. - _[REJECT]_ The `aggregate_and_proof.selection_proof` is a valid signature of the `aggregate.data.slot` by the validator with index `aggregate_and_proof.aggregator_index`. - _[REJECT]_ The aggregator signature, `signed_aggregate_and_proof.signature`, is valid. - _[REJECT]_ The signature of `aggregate` is valid. @@ -275,7 +276,7 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio - `committee_index{subnet_id}_beacon_attestation` - These topics are used to propagate unaggregated attestations to the subnet `subnet_id` (typically beacon and persistent committees) to be aggregated before being gossiped to `beacon_aggregate_and_proof`. The following validations MUST pass before forwarding the `attestation` on the subnet. - _[REJECT]_ The attestation's committee index (`attestation.data.index`) is for the correct subnet. - _[IGNORE]_ `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` (a client MAY queue future attestations for processing at the appropriate slot). - - _[REJECT]_ The attestation is unaggregated -- that is, it has exactly one participating validator (`len([bit for bit in attestation.aggregation_bits if bit == 0b1]) == 1`). + - _[REJECT]_ The attestation is unaggregated -- that is, it has exactly one participating validator (`len(get_attesting_indices(state, attestation.data, attestation.aggregation_bits)) == 1`). - _[IGNORE]_ There has been no other valid attestation seen on an attestation subnet that has an identical `attestation.data.target.epoch` and participating validator index. - _[REJECT]_ The block being voted for (`attestation.data.beacon_block_root`) passes validation. - _[REJECT]_ The signature of `attestation` is valid. From 7d4d3e43ef4a6b2002faf52263ae8d3e5bd63194 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 11 May 2020 08:38:22 -0600 Subject: [PATCH 32/70] remove aggregate clarifiyng text that is not longer valid Co-authored-by: Diederik Loerakker --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 4ffc45bf7..bfdc34ef9 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -251,7 +251,7 @@ There are two primary global topics used to propagate beacon blocks and aggregat - _[REJECT]_ The block being voted for (`aggregate.data.beacon_block_root`) passes validation. - _[REJECT]_ The attestation has participants -- that is, `len(get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)) >= 1`. - _[REJECT]_ `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_aggregator(state, aggregate.data.slot, aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. - - _[REJECT]_ The aggregator's validator index is within the committee -- i.e. `aggregate_and_proof.aggregator_index in get_beacon_committee(state, aggregate.data.slot, aggregate.data.index)`. This also means that it must never have an empty set of participants. + - _[REJECT]_ The aggregator's validator index is within the committee -- i.e. `aggregate_and_proof.aggregator_index in get_beacon_committee(state, aggregate.data.slot, aggregate.data.index)`. - _[REJECT]_ The `aggregate_and_proof.selection_proof` is a valid signature of the `aggregate.data.slot` by the validator with index `aggregate_and_proof.aggregator_index`. - _[REJECT]_ The aggregator signature, `signed_aggregate_and_proof.signature`, is valid. - _[REJECT]_ The signature of `aggregate` is valid. From b4bc2038e1a7727a18d05c8543504b7d7344cb8a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 11 May 2020 08:39:30 -0600 Subject: [PATCH 33/70] clarify that clients MAY stop block requests if fork choice changes --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index dae82f464..399e1b466 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -539,7 +539,7 @@ Clients MUST order blocks by increasing slot number. Clients MUST respond with blocks from their view of the current fork choice -- that is, blocks from the single chain defined by the current head. Of note, blocks from slots before the finalization MUST lead to the finalized block reported in the `Status` handshake. -Clients MUST respond with blocks that are consistent from a single chain within the context of the request. +Clients MUST respond with blocks that are consistent from a single chain within the context of the request. After the initial block clients MAY stop in the process of responding, if their fork choice changes the view of the chain in the context of the request. #### BeaconBlocksByRoot From fcf003859d576dec19c1e7c01093f484bfbf2314 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 11 May 2020 17:01:18 +0200 Subject: [PATCH 34/70] remove duplicate response diagram --- specs/phase0/p2p-interface.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 58f8653ca..1130dbe35 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -394,14 +394,6 @@ The `ErrorMessage` schema is: *Note*: The String type is encoded as UTF-8 bytes without NULL terminator when SSZ-encoded. As the `ErrorMessage` is not an SSZ-container, only the UTF-8 bytes will be sent when SSZ-encoded. -A response therefore has the form of one or more `response_chunk`s, each structured as follows: -``` - +--------+--------+--------+--------+--------+--------+ - | result | header (opt) | encoded_response | - +--------+--------+--------+--------+--------+--------+ -``` -Here, `result` represents the 1-byte response code. - ### Encoding strategies The token of the negotiated protocol ID specifies the type of encoding to be used for the req/resp interaction. Two values are possible at this time: From 65108aeee0936ba54661b3ddd3bdb48b10ed8315 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 11 May 2020 11:17:48 -0600 Subject: [PATCH 35/70] start on_block just slots test at time 0 --- tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py index 4438dff92..016326b30 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py @@ -184,7 +184,7 @@ def test_on_block_finalized_skip_slots_not_in_skip_chain(spec, state): def test_on_block_update_justified_checkpoint_within_safe_slots(spec, state): # Initialization store = spec.get_forkchoice_store(state) - time = 100 + time = 0 spec.on_tick(store, time) next_epoch(spec, state) @@ -215,7 +215,7 @@ def test_on_block_update_justified_checkpoint_within_safe_slots(spec, state): def test_on_block_outside_safe_slots_and_multiple_better_justified(spec, state): # Initialization store = spec.get_forkchoice_store(state) - time = 100 + time = 0 spec.on_tick(store, time) next_epoch(spec, state) From 30ecd9b602a62daa61d6e737f4bb0cc06eb38067 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 11 May 2020 11:22:34 -0600 Subject: [PATCH 36/70] p2p PR feedback --- specs/phase0/p2p-interface.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 1130dbe35..7c1a30e8a 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -332,7 +332,7 @@ The encoding-dependent header may carry metadata or assertions such as the encod A `response` is formed by zero or more `response_chunk`s. Responses that consist of a single SSZ-list (such as `BlocksByRange` and `BlocksByRoot`) send each list item as a `response_chunk`. All other response types (non-Lists) send a single `response_chunk`. -For both `request`s and `response`s, he `encoding-dependent-header` MUST be valid, and the `encoded-payload` must be valid within the constraints of the `encoding-dependent-header`. +For both `request`s and `response`s, the `encoding-dependent-header` MUST be valid, and the `encoded-payload` must be valid within the constraints of the `encoding-dependent-header`. This includes type-specific bounds on payload size for some encoding strategies. Regardless of these type specific bounds, a global maximum uncompressed byte size of `MAX_CHUNK_SIZE` MUST be applied to all method response chunks. Clients MUST ensure that lengths are within these bounds; if not, they SHOULD reset the stream immediately. Clients tracking peer reputation MAY decrement the score of the misbehaving peer under this circumstance. @@ -348,10 +348,10 @@ The requester MUST wait a maximum of `TTFB_TIMEOUT` for the first response byte If any of these timeouts fire, the requester SHOULD reset the stream and deem the req/resp operation to have failed. A requester SHOULD read from the stream until either: -a) An error result is received in one of the chunks (the error payload MAY be read before stopping). -b) The responder closes the stream. -d) Any part of the `response_chunk` fails validation. -e) The maximum number of requested chunks are read. +1. An error result is received in one of the chunks (the error payload MAY be read before stopping). +2. The responder closes the stream. +3. Any part of the `response_chunk` fails validation. +4. The maximum number of requested chunks are read. For requests consisting of a single valid `response_chunk`, the requester SHOULD read the chunk fully, as defined by the `encoding-dependent-header`, before closing the stream. From 511f8034969bc24e12aaa870fbbf6cadeaea7d79 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 11 May 2020 12:44:46 -0600 Subject: [PATCH 37/70] use all attnets across the epoch even when not max committees per slot --- specs/phase0/p2p-interface.md | 21 ++++++++++----------- specs/phase0/validator.md | 16 ++++++++++++++-- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index e06630ae5..98ee6edd2 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -151,7 +151,6 @@ This section outlines constants that are used in this spec. |---|---|---| | `GOSSIP_MAX_SIZE` | `2**20` (= 1048576, 1 MiB) | The maximum allowed size of uncompressed gossip messages. | | `MAX_CHUNK_SIZE` | `2**20` (1048576, 1 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | -| `ATTESTATION_SUBNET_COUNT` | `64` | The number of attestation subnets used in the gossipsub protocol. | | `TTFB_TIMEOUT` | `5s` | The maximum time to wait for first byte of request response (time-to-first-byte). | | `RESP_TIMEOUT` | `10s` | The maximum time for complete response transfer. | | `ATTESTATION_PROPAGATION_SLOT_RANGE` | `32` | The maximum number of slots during which an attestation can be propagated. | @@ -221,12 +220,12 @@ The payload is carried in the `data` field of a gossipsub message, and varies de | Name | Message Type | |------------------------------------------------|-------------------------| -| beacon_block | SignedBeaconBlock | -| beacon_aggregate_and_proof | SignedAggregateAndProof | -| committee_index{subnet_id}\_beacon_attestation | Attestation | -| voluntary_exit | SignedVoluntaryExit | -| proposer_slashing | ProposerSlashing | -| attester_slashing | AttesterSlashing | +| beacon\_block | SignedBeaconBlock | +| beacon\_aggregate\_and\_proof | SignedAggregateAndProof | +| beacon_attestation\_{subnet\_id} | Attestation | +| voluntary\_exit | SignedVoluntaryExit | +| proposer\_slashing | ProposerSlashing | +| attester\_slashing | AttesterSlashing | Clients MUST reject (fail validation) messages containing an incorrect type, or invalid payload. @@ -273,8 +272,8 @@ Additional global topics are used to propagate lower frequency validator message Attestation subnets are used to propagate unaggregated attestations to subsections of the network. Their `Name`s are: -- `committee_index{subnet_id}_beacon_attestation` - These topics are used to propagate unaggregated attestations to the subnet `subnet_id` (typically beacon and persistent committees) to be aggregated before being gossiped to `beacon_aggregate_and_proof`. The following validations MUST pass before forwarding the `attestation` on the subnet. - - _[REJECT]_ The attestation's committee index (`attestation.data.index`) is for the correct subnet. +- `beacon_attestation_{subnet_id}` - These topics are used to propagate unaggregated attestations to the subnet `subnet_id` (typically beacon and persistent committees) to be aggregated before being gossiped to `beacon_aggregate_and_proof`. The following validations MUST pass before forwarding the `attestation` on the subnet. + - _[REJECT]_ The attestation is for the correct subnet (i.e. `compute_subnet_for_attestation(state, attestation) == subnet_id`). - _[IGNORE]_ `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` (a client MAY queue future attestations for processing at the appropriate slot). - _[REJECT]_ The attestation is unaggregated -- that is, it has exactly one participating validator (`len([bit for bit in attestation.aggregation_bits if bit == 0b1]) == 1`). - _[IGNORE]_ There has been no other valid attestation seen on an attestation subnet that has an identical `attestation.data.target.epoch` and participating validator index. @@ -283,9 +282,9 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio #### Attestations and Aggregation -Attestation broadcasting is grouped into subnets defined by a topic. The number of subnets is defined via `ATTESTATION_SUBNET_COUNT`. For the `committee_index{subnet_id}_beacon_attestation` topics, `subnet_id` is set to `index % ATTESTATION_SUBNET_COUNT`, where `index` is the `CommitteeIndex` of the given committee. +Attestation broadcasting is grouped into subnets defined by a topic. The number of subnets is defined via `ATTESTATION_SUBNET_COUNT`. The correct subnet for an attestation can be calculated with `compute_subnet_for_attestation`. `beacon_attestation_{subnet_id}` topics, are rotated through throughout the epoch in a similar fashion to rotating through shards in committees in Phase 1. -Unaggregated attestations are sent to the subnet topic, `committee_index{attestation.data.index % ATTESTATION_SUBNET_COUNT}_beacon_attestation` as `Attestation`s. +Unaggregated attestations are sent to the subnet topic, `beacon_attestation_{compute_subnet_for_attestation(state, attestation)}` as `Attestation`s. Aggregated attestations are sent to the `beacon_aggregate_and_proof` topic as `AggregateAndProof`s. diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 62fdc0a93..398b882ee 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -90,6 +90,7 @@ All terminology, constants, functions, and protocol mechanics defined in the [Ph | `RANDOM_SUBNETS_PER_VALIDATOR` | `2**0` (= 1) | subnets | | | `EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | epochs | ~27 hours | | `SECONDS_PER_ETH1_BLOCK` | `14` | seconds | | +| `ATTESTATION_SUBNET_COUNT` | `64` | The number of attestation subnets used in the gossipsub protocol. | ## Becoming a validator @@ -418,7 +419,18 @@ def get_attestation_signature(state: BeaconState, attestation_data: AttestationD #### Broadcast attestation -Finally, the validator broadcasts `attestation` to the associated attestation subnet -- the `committee_index{attestation.data.index % ATTESTATION_SUBNET_COUNT}_beacon_attestation` pubsub topic. +Finally, the validator broadcasts `attestation` to the associated attestation subnet -- the `beacon_attestation_{compute_subnet_for_attestation(state, attestation)}` pubsub topic. + +```python +def compute_subnet_for_attestation(state: BeaconState, attestation: Attestation) -> uint64: + slots_since_epoch_start = attestation.data.slot % SLOTS_PER_EPOCH + committees_since_epoch_start = sum([ + get_committee_count_at_slot(state, Slot(slot)) + for slot in range(slots_since_epoch_start) + ]) + + return (committees_since_epoch_start + attestation.data.index) % ATTESTATION_SUBNET_COUNT +``` ### Attestation aggregation @@ -519,7 +531,7 @@ class SignedAggregateAndProof(Container): ## Phase 0 attestation subnet stability -Because Phase 0 does not have shards and thus does not have Shard Committees, there is no stable backbone to the attestation subnets (`committee_index{subnet_id}_beacon_attestation`). To provide this stability, each validator must: +Because Phase 0 does not have shards and thus does not have Shard Committees, there is no stable backbone to the attestation subnets (`beacon_attestation_{subnet_id}`). To provide this stability, each validator must: * Randomly select and remain subscribed to `RANDOM_SUBNETS_PER_VALIDATOR` attestation subnets * Maintain advertisement of the randomly selected subnets in their node's ENR `attnets` entry by setting the randomly selected `subnet_id` bits to `True` (e.g. `ENR["attnets"][subnet_id] = True`) for all persistent attestation subnets From 78d83b6c7d7b45b519cfd1d006925d6f964ea136 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 12 May 2020 04:24:56 +0800 Subject: [PATCH 38/70] Bump py_ecc to v3.0.0 (IETF BLS v2 + H2C v6) --- setup.py | 2 +- tests/generators/bls/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 37f3c16ef..a2c6b8f88 100644 --- a/setup.py +++ b/setup.py @@ -501,7 +501,7 @@ setup( "eth-utils>=1.3.0,<2", "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.9.4", - "py_ecc==2.0.0", + "py_ecc==3.0.0", "dataclasses==0.6", "remerkleable==0.1.13", "ruamel.yaml==0.16.5", diff --git a/tests/generators/bls/requirements.txt b/tests/generators/bls/requirements.txt index 24ea127c4..21920089f 100644 --- a/tests/generators/bls/requirements.txt +++ b/tests/generators/bls/requirements.txt @@ -1,4 +1,4 @@ -py_ecc==2.0.0 +py_ecc==3.0.0 eth-utils==1.6.0 ../../core/gen_helpers ../../../ From ed194de26b9d89cca2478cbc5c928e1c68d655e5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 13 May 2020 02:27:42 +0800 Subject: [PATCH 39/70] Bump py_ecc to v4.0.0 (IETF BLS v2 + H2C v7) --- setup.py | 2 +- tests/generators/bls/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index a2c6b8f88..5f0dce763 100644 --- a/setup.py +++ b/setup.py @@ -501,7 +501,7 @@ setup( "eth-utils>=1.3.0,<2", "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.9.4", - "py_ecc==3.0.0", + "py_ecc==4.0.0", "dataclasses==0.6", "remerkleable==0.1.13", "ruamel.yaml==0.16.5", diff --git a/tests/generators/bls/requirements.txt b/tests/generators/bls/requirements.txt index 21920089f..254705282 100644 --- a/tests/generators/bls/requirements.txt +++ b/tests/generators/bls/requirements.txt @@ -1,4 +1,4 @@ -py_ecc==3.0.0 +py_ecc==4.0.0 eth-utils==1.6.0 ../../core/gen_helpers ../../../ From 2718dcc4abe8d37af67426a0eaca86bc20e3161d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 13 May 2020 02:34:10 +0800 Subject: [PATCH 40/70] Update IETF standard description --- specs/phase0/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 83f9f7ec2..3d24ff043 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -603,7 +603,7 @@ def bytes_to_int(data: bytes) -> uint64: #### BLS Signatures -Eth2 makes use of BLS signatures as specified in the [IETF draft BLS specification](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-02). Specifically, eth2 uses the `BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_` ciphersuite which implements the following interfaces: + Eth2 makes use of BLS signatures as specified in the [IETF draft BLS specification draft-irtf-cfrg-bls-signature-02](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-02) but uses [Hashing to Elliptic Curves - draft-irtf-cfrg-hash-to-curve-07](https://tools.ietf.org/html/draft-irtf-cfrg-hash-to-curve-07) instead of draft-irtf-cfrg-hash-to-curve-06. Specifically, eth2 uses the `BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_` ciphersuite which implements the following interfaces: - `def Sign(SK: int, message: Bytes) -> BLSSignature` - `def Verify(PK: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` From 3c11a4dc020a21c2cdbf8b667bcec2be717e08ba Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 13 May 2020 12:56:50 +0800 Subject: [PATCH 41/70] Fix `AggregateVerify` param name: `message` -> `messages` Co-authored-by: Danny Ryan --- specs/phase0/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 3d24ff043..1c88c2a4c 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -609,7 +609,7 @@ def bytes_to_int(data: bytes) -> uint64: - `def Verify(PK: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` - `def Aggregate(signatures: Sequence[BLSSignature]) -> BLSSignature` - `def FastAggregateVerify(PKs: Sequence[BLSPubkey], message: Bytes, signature: BLSSignature) -> bool` -- `def AggregateVerify(PKs: Sequence[BLSPubkey], message: Sequence[Bytes], signature: BLSSignature) -> bool` +- `def AggregateVerify(PKs: Sequence[BLSPubkey], messages: Sequence[Bytes], signature: BLSSignature) -> bool` Within these specifications, BLS signatures are treated as a module for notational clarity, thus to verify a signature `bls.Verify(...)` is used. From 4a246ba5ac95f657fabea05506c86c8c48f453c0 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 13 May 2020 13:01:08 +0800 Subject: [PATCH 42/70] Apply feedback from Danny, add a note of the hash to curve configuration --- specs/phase0/beacon-chain.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 1c88c2a4c..f96842b20 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -613,6 +613,8 @@ def bytes_to_int(data: bytes) -> uint64: Within these specifications, BLS signatures are treated as a module for notational clarity, thus to verify a signature `bls.Verify(...)` is used. +*Note*: The non-standard configuration of the BLS and hash to curve specs is temporary and will be resolved once IETF releases BLS spec draft 3. + ### Predicates #### `is_active_validator` From b9e4bccb21a9c415c6454026fa90df3779a7d16f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 13 May 2020 16:26:20 +0800 Subject: [PATCH 43/70] Fix type error --- ssz/simple-serialize.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ssz/simple-serialize.md b/ssz/simple-serialize.md index 1c4f588eb..8c76ad916 100644 --- a/ssz/simple-serialize.md +++ b/ssz/simple-serialize.md @@ -211,8 +211,8 @@ We first define helper functions: * `List[B, N]` and `Vector[B, N]`, where `B` is a basic type: `(N * size_of(B) + 31) // 32` (dividing by chunk size, rounding up) * `List[C, N]` and `Vector[C, N]`, where `C` is a composite type: `N` * containers: `len(fields)` -* `bitfield_bytes(bits)`: return the bits of the bitlist or bitvector, packed in bytes, aligned to the start. Length-delimiting bit for bitlists is excluded. -* `pack`: Given ordered objects of the same basic type, serialize them, pack them into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. +* `pack(value)`: given ordered objects of the same basic type, serialize them, pack them into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. +* `pack_bits(bits)`: Given the `bits` of bitlist or bitvector, get `bitfield_bytes` by packing them in bytes and aligning to the start. Length-delimiting bit for bitlists is excluded. And them pack `bitfield_bytes` into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. * `next_pow_of_two(i)`: get the next power of 2 of `i`, if not already a power of 2, with 0 mapping to 1. Examples: `0->1, 1->1, 2->2, 3->4, 4->4, 6->8, 9->16` * `merkleize(chunks, limit=None)`: Given ordered `BYTES_PER_CHUNK`-byte chunks, merkleize the chunks, and return the root: * The merkleization depends on the effective input, which can be padded/limited: @@ -228,9 +228,9 @@ We first define helper functions: We now define Merkleization `hash_tree_root(value)` of an object `value` recursively: * `merkleize(pack(value))` if `value` is a basic object or a vector of basic objects. -* `merkleize(bitfield_bytes(value), limit=chunk_count(type))` if `value` is a bitvector. +* `merkleize(pack_bits(value), limit=chunk_count(type))` if `value` is a bitvector. * `mix_in_length(merkleize(pack(value), limit=chunk_count(type)), len(value))` if `value` is a list of basic objects. -* `mix_in_length(merkleize(bitfield_bytes(value), limit=chunk_count(type)), len(value))` if `value` is a bitlist. +* `mix_in_length(merkleize(pack_bits(value), limit=chunk_count(type)), len(value))` if `value` is a bitlist. * `merkleize([hash_tree_root(element) for element in value])` if `value` is a vector of composite objects or a container. * `mix_in_length(merkleize([hash_tree_root(element) for element in value], limit=chunk_count(type)), len(value))` if `value` is a list of composite objects. * `mix_in_type(merkleize(value.value), value.type_index)` if `value` is of union type. From 84cea96c42e23c3737bf04096bfb8ee5823ceeb7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 14 May 2020 15:01:03 +0800 Subject: [PATCH 44/70] Fix typo Co-authored-by: Diederik Loerakker --- ssz/simple-serialize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ssz/simple-serialize.md b/ssz/simple-serialize.md index 8c76ad916..b8a6bc9a2 100644 --- a/ssz/simple-serialize.md +++ b/ssz/simple-serialize.md @@ -212,7 +212,7 @@ We first define helper functions: * `List[C, N]` and `Vector[C, N]`, where `C` is a composite type: `N` * containers: `len(fields)` * `pack(value)`: given ordered objects of the same basic type, serialize them, pack them into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. -* `pack_bits(bits)`: Given the `bits` of bitlist or bitvector, get `bitfield_bytes` by packing them in bytes and aligning to the start. Length-delimiting bit for bitlists is excluded. And them pack `bitfield_bytes` into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. +* `pack_bits(bits)`: Given the `bits` of bitlist or bitvector, get `bitfield_bytes` by packing them in bytes and aligning to the start. The length-delimiting bit for bitlists is excluded. And then pack `bitfield_bytes` into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. * `next_pow_of_two(i)`: get the next power of 2 of `i`, if not already a power of 2, with 0 mapping to 1. Examples: `0->1, 1->1, 2->2, 3->4, 4->4, 6->8, 9->16` * `merkleize(chunks, limit=None)`: Given ordered `BYTES_PER_CHUNK`-byte chunks, merkleize the chunks, and return the root: * The merkleization depends on the effective input, which can be padded/limited: From aa436d91b2b3cc2434bccead18c991138139a81d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 14 May 2020 22:04:13 +0800 Subject: [PATCH 45/70] Use NO_SIGNATURE (0x00...) approach --- configs/mainnet.yaml | 2 ++ configs/minimal.yaml | 2 ++ specs/phase1/beacon-chain.md | 34 +++++++++++++++++++++++++------- specs/phase1/shard-transition.md | 4 ++-- 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 60bd1c087..0fb81a1c3 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -156,6 +156,8 @@ DOMAIN_SHARD_PROPOSAL: 0x80000000 DOMAIN_SHARD_COMMITTEE: 0x81000000 DOMAIN_LIGHT_CLIENT: 0x82000000 DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 +# Constant +NO_SIGNATURE: 0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 # Phase 1: Upgrade from Phase 0 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 5c1511e6d..6650eadc1 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -156,6 +156,8 @@ DOMAIN_SHARD_PROPOSAL: 0x80000000 DOMAIN_SHARD_COMMITTEE: 0x81000000 DOMAIN_LIGHT_CLIENT: 0x82000000 DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 +# Constant +NO_SIGNATURE: 0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 # Phase 1: Upgrade from Phase 0 diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 83d63ee2a..a52e2eb68 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -55,6 +55,8 @@ - [Updated `is_valid_indexed_attestation`](#updated-is_valid_indexed_attestation) - [`is_shard_attestation`](#is_shard_attestation) - [`is_winning_attestation`](#is_winning_attestation) + - [`optional_aggregate_verify`](#optional_aggregate_verify) + - [`optional_fast_aggregate_verify`](#optional_fast_aggregate_verify) - [Block processing](#block-processing) - [Operations](#operations) - [New Attestation processing](#new-attestation-processing) @@ -110,6 +112,7 @@ Configuration is not namespaced. Instead it is strictly an extension; | `DOMAIN_SHARD_PROPOSAL` | `DomainType('0x80000000')` | | | `DOMAIN_SHARD_COMMITTEE` | `DomainType('0x81000000')` | | | `DOMAIN_LIGHT_CLIENT` | `DomainType('0x82000000')` | | +| `NO_SIGNATURE` | `BLSSignature(b'\x00' * 96)` | | ## Updated containers @@ -633,6 +636,28 @@ def is_winning_attestation(state: BeaconState, ) ``` +#### `optional_aggregate_verify` + +```python +def optional_aggregate_verify(pubkeys: Sequence[BLSPubkey], + messages: Sequence[Bytes32], + signature: BLSSignature) -> bool: + if len(pubkeys) == 0: + return signature == NO_SIGNATURE + else: + return bls.AggregateVerify(pubkeys, messages, signature) +``` + +#### `optional_fast_aggregate_verify` + +```python +def optional_fast_aggregate_verify(pubkeys: Sequence[BLSPubkey], message: Bytes32, signature: BLSSignature) -> bool: + if len(pubkeys) == 0: + return signature == NO_SIGNATURE + else: + return bls.FastAggregateVerify(pubkeys, message, signature) +``` + ### Block processing ```python @@ -764,7 +789,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr for header in headers ] # Verify combined proposer signature - assert bls.AggregateVerify(pubkeys, signing_roots, signature=transition.proposer_signature_aggregate) + assert optional_aggregate_verify(pubkeys, signing_roots, transition.proposer_signature_aggregate) # Save updated state state.shard_states[shard] = transition.shard_states[len(transition.shard_states) - 1] @@ -942,12 +967,7 @@ def process_light_client_signatures(state: BeaconState, block_body: BeaconBlockB slot = compute_previous_slot(state.slot) signing_root = compute_signing_root(get_block_root_at_slot(state, slot), get_domain(state, DOMAIN_LIGHT_CLIENT, compute_epoch_at_slot(slot))) - if len(signer_pubkeys) == 0: - # TODO: handle the empty light_client_signature case? - assert block_body.light_client_signature == BLSSignature() - return - else: - assert bls.FastAggregateVerify(signer_pubkeys, signing_root, signature=block_body.light_client_signature) + assert optional_fast_aggregate_verify(signer_pubkeys, signing_root, block_body.light_client_signature) ``` ### Epoch transition diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 5b6a72f28..36b3b80ab 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -277,13 +277,13 @@ def get_shard_transition(beacon_state: BeaconState, proposer_signatures = [] for proposal in proposals: shard_block_lengths.append(len(proposal.message.body)) - if proposal.signature != BLSSignature(): + if proposal.signature != NO_SIGNATURE: proposer_signatures.append(proposal.signature) if len(proposer_signatures) > 0: proposer_signature_aggregate = bls.Aggregate(proposer_signatures) else: - proposer_signature_aggregate = BLSSignature() + proposer_signature_aggregate = NO_SIGNATURE return ShardTransition( start_slot=start_slot, From 6a3241be61b8be2437ef8a06bfdcc7b675915723 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 15 May 2020 00:48:53 +0800 Subject: [PATCH 46/70] Remove leading space Co-authored-by: Danny Ryan --- specs/phase0/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index f96842b20..2ab0686ef 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -603,7 +603,7 @@ def bytes_to_int(data: bytes) -> uint64: #### BLS Signatures - Eth2 makes use of BLS signatures as specified in the [IETF draft BLS specification draft-irtf-cfrg-bls-signature-02](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-02) but uses [Hashing to Elliptic Curves - draft-irtf-cfrg-hash-to-curve-07](https://tools.ietf.org/html/draft-irtf-cfrg-hash-to-curve-07) instead of draft-irtf-cfrg-hash-to-curve-06. Specifically, eth2 uses the `BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_` ciphersuite which implements the following interfaces: +Eth2 makes use of BLS signatures as specified in the [IETF draft BLS specification draft-irtf-cfrg-bls-signature-02](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-02) but uses [Hashing to Elliptic Curves - draft-irtf-cfrg-hash-to-curve-07](https://tools.ietf.org/html/draft-irtf-cfrg-hash-to-curve-07) instead of draft-irtf-cfrg-hash-to-curve-06. Specifically, eth2 uses the `BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_` ciphersuite which implements the following interfaces: - `def Sign(SK: int, message: Bytes) -> BLSSignature` - `def Verify(PK: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` From d27f2350a2079644595be83e5e03cb6a0f8ff696 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 15 May 2020 00:45:26 +0800 Subject: [PATCH 47/70] Update BLS test suite to BLS standard draft v2 format 1. Make sure that BLS -Verify APIs would only return `True` or `False` , no exceptions. 2. Use `eth2spec.utils.bls` instead of py_ecc for test generator 3. Add assertions in test generator 4. Add some special test cases for the -Verify APIs 5. Clean up the test format documents --- tests/core/pyspec/eth2spec/utils/bls.py | 26 +++- .../bls/{aggregate_sigs.md => aggregate.md} | 2 +- tests/formats/bls/aggregate_pubkeys.md | 19 --- tests/formats/bls/aggregate_verify.md | 17 +++ tests/formats/bls/fast_aggregate_verify.md | 17 +++ tests/formats/bls/msg_hash_g2_compressed.md | 21 ---- tests/formats/bls/msg_hash_g2_uncompressed.md | 21 ---- tests/formats/bls/priv_to_pub.md | 19 --- tests/formats/bls/{sign_msg.md => sign.md} | 6 - tests/formats/bls/verify.md | 17 +++ tests/generators/bls/main.py | 116 ++++++++++++++---- 11 files changed, 165 insertions(+), 116 deletions(-) rename tests/formats/bls/{aggregate_sigs.md => aggregate.md} (78%) delete mode 100644 tests/formats/bls/aggregate_pubkeys.md create mode 100644 tests/formats/bls/aggregate_verify.md create mode 100644 tests/formats/bls/fast_aggregate_verify.md delete mode 100644 tests/formats/bls/msg_hash_g2_compressed.md delete mode 100644 tests/formats/bls/msg_hash_g2_uncompressed.md delete mode 100644 tests/formats/bls/priv_to_pub.md rename tests/formats/bls/{sign_msg.md => sign.md} (67%) create mode 100644 tests/formats/bls/verify.md diff --git a/tests/core/pyspec/eth2spec/utils/bls.py b/tests/core/pyspec/eth2spec/utils/bls.py index 7f265b555..acf9f99c7 100644 --- a/tests/core/pyspec/eth2spec/utils/bls.py +++ b/tests/core/pyspec/eth2spec/utils/bls.py @@ -25,17 +25,32 @@ def only_with_bls(alt_return=None): @only_with_bls(alt_return=True) def Verify(PK, message, signature): - return bls.Verify(PK, message, signature) + try: + result = bls.Verify(PK, message, signature) + except Exception: + result = False + finally: + return result @only_with_bls(alt_return=True) def AggregateVerify(pubkeys, messages, signature): - return bls.AggregateVerify(pubkeys, messages, signature) + try: + result = bls.AggregateVerify(pubkeys, messages, signature) + except Exception: + result = False + finally: + return result @only_with_bls(alt_return=True) def FastAggregateVerify(pubkeys, message, signature): - return bls.FastAggregateVerify(pubkeys, message, signature) + try: + result = bls.FastAggregateVerify(pubkeys, message, signature) + except Exception: + result = False + finally: + return result @only_with_bls(alt_return=STUB_SIGNATURE) @@ -56,3 +71,8 @@ def signature_to_G2(signature): @only_with_bls(alt_return=STUB_PUBKEY) def AggregatePKs(pubkeys): return bls._AggregatePKs(pubkeys) + + +@only_with_bls(alt_return=STUB_SIGNATURE) +def SkToPk(SK): + return bls.SkToPk(SK) diff --git a/tests/formats/bls/aggregate_sigs.md b/tests/formats/bls/aggregate.md similarity index 78% rename from tests/formats/bls/aggregate_sigs.md rename to tests/formats/bls/aggregate.md index 2252dbaa8..0d7e7c631 100644 --- a/tests/formats/bls/aggregate_sigs.md +++ b/tests/formats/bls/aggregate.md @@ -16,4 +16,4 @@ output: BLS Signature -- expected output, single BLS signature ## Condition -The `aggregate_sigs` handler should aggregate the signatures in the `input`, and the result should match the expected `output`. +The `aggregate` handler should aggregate the signatures in the `input`, and the result should match the expected `output`. diff --git a/tests/formats/bls/aggregate_pubkeys.md b/tests/formats/bls/aggregate_pubkeys.md deleted file mode 100644 index 049ad6991..000000000 --- a/tests/formats/bls/aggregate_pubkeys.md +++ /dev/null @@ -1,19 +0,0 @@ -# Test format: BLS pubkey aggregation - -A BLS pubkey aggregation combines a series of pubkeys into a single pubkey. - -## Test case format - -The test data is declared in a `data.yaml` file: - -```yaml -input: List[BLS Pubkey] -- list of input BLS pubkeys -output: BLS Pubkey -- expected output, single BLS pubkey -``` - -`BLS Pubkey` here is encoded as a string: hexadecimal encoding of 48 bytes (96 nibbles), prefixed with `0x`. - - -## Condition - -The `aggregate_pubkeys` handler should aggregate the keys in the `input`, and the result should match the expected `output`. diff --git a/tests/formats/bls/aggregate_verify.md b/tests/formats/bls/aggregate_verify.md new file mode 100644 index 000000000..3985de9f4 --- /dev/null +++ b/tests/formats/bls/aggregate_verify.md @@ -0,0 +1,17 @@ +# Test format: BLS sign message + +Verify the signature against the given pubkeys and one messages. + +## Test case format + +The test data is declared in a `data.yaml` file: + +```yaml +input: + pubkeys: List[bytes48] -- the pubkeys + messages: List[bytes32] -- the messages + signature: bytes96 -- the signature to verify against pubkeys and messages +output: bool -- VALID or INVALID +``` + +All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. diff --git a/tests/formats/bls/fast_aggregate_verify.md b/tests/formats/bls/fast_aggregate_verify.md new file mode 100644 index 000000000..7e3899a15 --- /dev/null +++ b/tests/formats/bls/fast_aggregate_verify.md @@ -0,0 +1,17 @@ +# Test format: BLS sign message + +Verify the signature against the given pubkeys and one message. + +## Test case format + +The test data is declared in a `data.yaml` file: + +```yaml +input: + pubkeys: List[bytes48] -- the pubkey + message: bytes32 -- the message + signature: bytes96 -- the signature to verify against pubkeys and message +output: bool -- VALID or INVALID +``` + +All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. diff --git a/tests/formats/bls/msg_hash_g2_compressed.md b/tests/formats/bls/msg_hash_g2_compressed.md deleted file mode 100644 index 761e819f2..000000000 --- a/tests/formats/bls/msg_hash_g2_compressed.md +++ /dev/null @@ -1,21 +0,0 @@ -# Test format: BLS hash-compressed - -A BLS compressed-hash to G2. - -## Test case format - -The test data is declared in a `data.yaml` file: - -```yaml -input: - message: bytes32 - domain: bytes8 -- the BLS domain -output: List[bytes48] -- length of two -``` - -All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. - - -## Condition - -The `msg_hash_g2_compressed` handler should hash the `message`, with the given `domain`, to G2 with compression, and the result should match the expected `output`. diff --git a/tests/formats/bls/msg_hash_g2_uncompressed.md b/tests/formats/bls/msg_hash_g2_uncompressed.md deleted file mode 100644 index 5ee535a38..000000000 --- a/tests/formats/bls/msg_hash_g2_uncompressed.md +++ /dev/null @@ -1,21 +0,0 @@ -# Test format: BLS hash-uncompressed - -A BLS uncompressed-hash to G2. - -## Test case format - -The test data is declared in a `data.yaml` file: - -```yaml -input: - message: bytes32 - domain: bytes8 -- the BLS domain -output: List[List[bytes48]] -- 3 lists, each a length of two -``` - -All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. - - -## Condition - -The `msg_hash_g2_uncompressed` handler should hash the `message`, with the given `domain`, to G2, without compression, and the result should match the expected `output`. diff --git a/tests/formats/bls/priv_to_pub.md b/tests/formats/bls/priv_to_pub.md deleted file mode 100644 index 29c6b216a..000000000 --- a/tests/formats/bls/priv_to_pub.md +++ /dev/null @@ -1,19 +0,0 @@ -# Test format: BLS private key to pubkey - -A BLS private key to public key conversion. - -## Test case format - -The test data is declared in a `data.yaml` file: - -```yaml -input: bytes32 -- the private key -output: bytes48 -- the public key -``` - -All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. - - -## Condition - -The `priv_to_pub` handler should compute the public key for the given private key `input`, and the result should match the expected `output`. diff --git a/tests/formats/bls/sign_msg.md b/tests/formats/bls/sign.md similarity index 67% rename from tests/formats/bls/sign_msg.md rename to tests/formats/bls/sign.md index 6c4f88cd1..1c328755a 100644 --- a/tests/formats/bls/sign_msg.md +++ b/tests/formats/bls/sign.md @@ -10,13 +10,7 @@ The test data is declared in a `data.yaml` file: input: privkey: bytes32 -- the private key used for signing message: bytes32 -- input message to sign (a hash) - domain: bytes8 -- the BLS domain output: bytes96 -- expected signature ``` All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. - - -## Condition - -The `sign_msg` handler should sign the given `message`, with `domain`, using the given `privkey`, and the result should match the expected `output`. diff --git a/tests/formats/bls/verify.md b/tests/formats/bls/verify.md new file mode 100644 index 000000000..57ec8a33a --- /dev/null +++ b/tests/formats/bls/verify.md @@ -0,0 +1,17 @@ +# Test format: BLS sign message + +Verify the signature against the given one pubkey and one message. + +## Test case format + +The test data is declared in a `data.yaml` file: + +```yaml +input: + pubkey: bytes48 -- the pubkey + message: bytes32 -- the message + signature: bytes96 -- the signature to verify against pubkey and message +output: bool -- VALID or INVALID +``` + +All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. diff --git a/tests/generators/bls/main.py b/tests/generators/bls/main.py index 7bb093593..f97be3c90 100644 --- a/tests/generators/bls/main.py +++ b/tests/generators/bls/main.py @@ -10,7 +10,7 @@ from eth_utils import ( ) from gen_base import gen_runner, gen_typing -from py_ecc import bls +from eth2spec.utils import bls from hashlib import sha256 from eth2spec.test.context import PHASE0 @@ -19,11 +19,6 @@ def hash(x): return sha256(x).digest() -F2Q_COEFF_LEN = 48 -G2_COMPRESSED_Z_LEN = 48 -DST = bls.G2ProofOfPossession.DST - - def int_to_hex(n: int, byte_length: int = None) -> str: byte_value = int_to_big_endian(n) if byte_length: @@ -49,11 +44,15 @@ PRIVKEYS = [ hex_to_int('0x00000000000000000000000000000000328388aff0d4a5b7dc9205abd374e7e98f3cd9f3418edb4eafda5fb16473d216'), ] +NO_PUBKEY = b'\x00' * 48 +Z1_PUBKEY = b'\xc0' + b'\x00' * 47 +NO_SIGNATURE = b'\x00' * 96 +Z2_SIGNATURE = b'\xc0' + b'\x00' * 95 def case01_sign(): for privkey in PRIVKEYS: for message in MESSAGES: - sig = bls.G2ProofOfPossession.Sign(privkey, message) + sig = bls.Sign(privkey, message) identifier = f'{int_to_hex(privkey)}_{encode_hex(message)}' yield f'sign_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { @@ -68,9 +67,10 @@ def case02_verify(): for i, privkey in enumerate(PRIVKEYS): for message in MESSAGES: # Valid signature - signature = bls.G2ProofOfPossession.Sign(privkey, message) - pubkey = bls.G2ProofOfPossession.SkToPk(privkey) + signature = bls.Sign(privkey, message) + pubkey = bls.SkToPk(privkey) identifier = f'{encode_hex(pubkey)}_{encode_hex(message)}' + assert bls.Verify(pubkey, message, signature) yield f'verify_valid_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkey': encode_hex(pubkey), @@ -81,8 +81,9 @@ def case02_verify(): } # Invalid signatures -- wrong pubkey - wrong_pubkey = bls.G2ProofOfPossession.SkToPk(PRIVKEYS[(i + 1) % len(PRIVKEYS)]) + wrong_pubkey = bls.SkToPk(PRIVKEYS[(i + 1) % len(PRIVKEYS)]) identifier = f'{encode_hex(wrong_pubkey)}_{encode_hex(message)}' + assert not bls.Verify(wrong_pubkey, message, signature) yield f'verify_wrong_pubkey_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkey': encode_hex(wrong_pubkey), @@ -95,6 +96,7 @@ def case02_verify(): # Invalid signature -- tampered with signature tampered_signature = signature[:-4] + b'\xFF\xFF\xFF\xFF' identifier = f'{encode_hex(pubkey)}_{encode_hex(message)}' + assert not bls.Verify(pubkey, message, tampered_signature) yield f'verify_tampered_signature_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkey': encode_hex(pubkey), @@ -104,26 +106,37 @@ def case02_verify(): 'output': False, } + # Valid pubkey and signature with the point at infinity + assert bls.Verify(Z1_PUBKEY, message, Z2_SIGNATURE) + yield f'verify_infinity_pubkey_and_infinity_signature', { + 'input': { + 'pubkey': encode_hex(Z1_PUBKEY), + 'message': encode_hex(message), + 'signature': encode_hex(Z2_SIGNATURE), + }, + 'output': True, + } def case03_aggregate(): for message in MESSAGES: - sigs = [bls.G2ProofOfPossession.Sign(privkey, message) for privkey in PRIVKEYS] + sigs = [bls.Sign(privkey, message) for privkey in PRIVKEYS] yield f'aggregate_{encode_hex(message)}', { 'input': [encode_hex(sig) for sig in sigs], - 'output': encode_hex(bls.G2ProofOfPossession.Aggregate(sigs)), + 'output': encode_hex(bls.Aggregate(sigs)), } def case04_fast_aggregate_verify(): for i, message in enumerate(MESSAGES): privkeys = PRIVKEYS[:i + 1] - sigs = [bls.G2ProofOfPossession.Sign(privkey, message) for privkey in privkeys] - aggregate_signature = bls.G2ProofOfPossession.Aggregate(sigs) - pubkeys = [bls.G2ProofOfPossession.SkToPk(privkey) for privkey in privkeys] + sigs = [bls.Sign(privkey, message) for privkey in privkeys] + aggregate_signature = bls.Aggregate(sigs) + pubkeys = [bls.SkToPk(privkey) for privkey in privkeys] pubkeys_serial = [encode_hex(pubkey) for pubkey in pubkeys] # Valid signature identifier = f'{pubkeys_serial}_{encode_hex(message)}' + assert bls.FastAggregateVerify(pubkeys, message, aggregate_signature) yield f'fast_aggregate_verify_valid_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkeys': pubkeys_serial, @@ -134,9 +147,10 @@ def case04_fast_aggregate_verify(): } # Invalid signature -- extra pubkey - pubkeys_extra = pubkeys + [bls.G2ProofOfPossession.SkToPk(PRIVKEYS[-1])] + pubkeys_extra = pubkeys + [bls.SkToPk(PRIVKEYS[-1])] pubkeys_extra_serial = [encode_hex(pubkey) for pubkey in pubkeys_extra] identifier = f'{pubkeys_extra_serial}_{encode_hex(message)}' + assert not bls.FastAggregateVerify(pubkeys_extra, message, aggregate_signature) yield f'fast_aggregate_verify_extra_pubkey_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkeys': pubkeys_extra_serial, @@ -149,6 +163,7 @@ def case04_fast_aggregate_verify(): # Invalid signature -- tampered with signature tampered_signature = aggregate_signature[:-4] + b'\xff\xff\xff\xff' identifier = f'{pubkeys_serial}_{encode_hex(message)}' + assert not bls.FastAggregateVerify(pubkeys, message, tampered_signature) yield f'fast_aggregate_verify_tampered_signature_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkeys': pubkeys_serial, @@ -158,37 +173,86 @@ def case04_fast_aggregate_verify(): 'output': False, } + # Invalid pubkeys and signature -- len(pubkey) == 0 and signature == Z1_SIGNATURE + assert not bls.FastAggregateVerify([], message, Z2_SIGNATURE) + yield f'fast_aggregate_verify_na_pubkeys_and_infinity_signature', { + 'input': { + 'pubkeys': [], + 'message': encode_hex(message), + 'signature': encode_hex(Z2_SIGNATURE), + }, + 'output': False, + } + + # Invalid pubkeys and signature -- len(pubkey) == 0 and signature == 0x00... + assert not bls.FastAggregateVerify([], message, NO_SIGNATURE) + yield f'fast_aggregate_verify_na_pubkeys_and_na_signature', { + 'input': { + 'pubkeys': [], + 'message': encode_hex(message), + 'signature': encode_hex(NO_SIGNATURE), + }, + 'output': False, + } def case05_aggregate_verify(): - pairs = [] + pubekys = [] + pubkeys_serial = [] + messages = [] + messages_serial = [] sigs = [] for privkey, message in zip(PRIVKEYS, MESSAGES): - sig = bls.G2ProofOfPossession.Sign(privkey, message) - pubkey = bls.G2ProofOfPossession.SkToPk(privkey) - pairs.append({ - 'pubkey': encode_hex(pubkey), - 'message': encode_hex(message), - }) + sig = bls.Sign(privkey, message) + pubkey = bls.SkToPk(privkey) + pubekys.append(pubkey) + pubkeys_serial.append(encode_hex(pubkey)) + messages.append(message) + messages_serial.append(encode_hex(message)) sigs.append(sig) - aggregate_signature = bls.G2ProofOfPossession.Aggregate(sigs) + aggregate_signature = bls.Aggregate(sigs) + assert bls.AggregateVerify(pubekys, messages, aggregate_signature) yield f'aggregate_verify_valid', { 'input': { - 'pairs': pairs, + 'pubkeys': pubkeys_serial, + 'messages': messages_serial, 'signature': encode_hex(aggregate_signature), }, 'output': True, } tampered_signature = aggregate_signature[:4] + b'\xff\xff\xff\xff' + assert not bls.AggregateVerify(pubkey, messages, tampered_signature) yield f'aggregate_verify_tampered_signature', { 'input': { - 'pairs': pairs, + 'pubkeys': pubkeys_serial, + 'messages': messages_serial, 'signature': encode_hex(tampered_signature), }, 'output': False, } + # Invalid pubkeys and signature -- len(pubkey) == 0 and signature == Z1_SIGNATURE + assert not bls.AggregateVerify([], [], Z2_SIGNATURE) + yield f'aggregate_verify_na_pubkeys_and_infinity_signature', { + 'input': { + 'pubkeys': [], + 'message': [], + 'signature': encode_hex(Z2_SIGNATURE), + }, + 'output': False, + } + + # Invalid pubkeys and signature -- len(pubkey) == 0 and signature == 0x00... + assert not bls.AggregateVerify([], [], NO_SIGNATURE) + yield f'aggregate_verify_na_pubkeys_and_na_signature', { + 'input': { + 'pubkeys': [], + 'messages': [], + 'signature': encode_hex(NO_SIGNATURE), + }, + 'output': False, + } def create_provider(handler_name: str, test_case_fn: Callable[[], Iterable[Tuple[str, Dict[str, Any]]]]) -> gen_typing.TestProvider: From f0c4623871d0f66a9f9878c3444379c5ec80f6a3 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 15 May 2020 01:05:32 +0800 Subject: [PATCH 48/70] Apply PR feedback: add docstring --- specs/phase1/beacon-chain.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index a52e2eb68..9a07e0a77 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -642,6 +642,10 @@ def is_winning_attestation(state: BeaconState, def optional_aggregate_verify(pubkeys: Sequence[BLSPubkey], messages: Sequence[Bytes32], signature: BLSSignature) -> bool: + """ + If ``pubkeys`` is an empty list, the given ``signature`` should be a stub ``NO_SIGNATURE``. + Otherwise, verify it with standard BLS AggregateVerify API. + """ if len(pubkeys) == 0: return signature == NO_SIGNATURE else: @@ -652,6 +656,10 @@ def optional_aggregate_verify(pubkeys: Sequence[BLSPubkey], ```python def optional_fast_aggregate_verify(pubkeys: Sequence[BLSPubkey], message: Bytes32, signature: BLSSignature) -> bool: + """ + If ``pubkeys`` is an empty list, the given ``signature`` should be a stub ``NO_SIGNATURE``. + Otherwise, verify it with standard BLS FastAggregateVerify API. + """ if len(pubkeys) == 0: return signature == NO_SIGNATURE else: From 9a2559857cc862fe42940f1fcb0de3df28b82845 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 15 May 2020 01:17:11 +0800 Subject: [PATCH 49/70] Fix typo and remove unused variable --- tests/generators/bls/main.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/generators/bls/main.py b/tests/generators/bls/main.py index f97be3c90..092f1247b 100644 --- a/tests/generators/bls/main.py +++ b/tests/generators/bls/main.py @@ -15,6 +15,7 @@ from hashlib import sha256 from eth2spec.test.context import PHASE0 + def hash(x): return sha256(x).digest() @@ -44,11 +45,11 @@ PRIVKEYS = [ hex_to_int('0x00000000000000000000000000000000328388aff0d4a5b7dc9205abd374e7e98f3cd9f3418edb4eafda5fb16473d216'), ] -NO_PUBKEY = b'\x00' * 48 Z1_PUBKEY = b'\xc0' + b'\x00' * 47 NO_SIGNATURE = b'\x00' * 96 Z2_SIGNATURE = b'\xc0' + b'\x00' * 95 + def case01_sign(): for privkey in PRIVKEYS: for message in MESSAGES: @@ -117,6 +118,7 @@ def case02_verify(): 'output': True, } + def case03_aggregate(): for message in MESSAGES: sigs = [bls.Sign(privkey, message) for privkey in PRIVKEYS] @@ -195,6 +197,7 @@ def case04_fast_aggregate_verify(): 'output': False, } + def case05_aggregate_verify(): pubekys = [] pubkeys_serial = [] @@ -237,7 +240,7 @@ def case05_aggregate_verify(): yield f'aggregate_verify_na_pubkeys_and_infinity_signature', { 'input': { 'pubkeys': [], - 'message': [], + 'messages': [], 'signature': encode_hex(Z2_SIGNATURE), }, 'output': False, @@ -254,6 +257,7 @@ def case05_aggregate_verify(): 'output': False, } + def create_provider(handler_name: str, test_case_fn: Callable[[], Iterable[Tuple[str, Dict[str, Any]]]]) -> gen_typing.TestProvider: From ea99f0ab10314b4bc236aef90ec39618b7dc24ff Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 15 May 2020 03:03:47 +0800 Subject: [PATCH 50/70] Fix typo Co-authored-by: Danny Ryan --- tests/generators/bls/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/generators/bls/main.py b/tests/generators/bls/main.py index 092f1247b..ea7446e9b 100644 --- a/tests/generators/bls/main.py +++ b/tests/generators/bls/main.py @@ -175,7 +175,7 @@ def case04_fast_aggregate_verify(): 'output': False, } - # Invalid pubkeys and signature -- len(pubkey) == 0 and signature == Z1_SIGNATURE + # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == Z1_SIGNATURE assert not bls.FastAggregateVerify([], message, Z2_SIGNATURE) yield f'fast_aggregate_verify_na_pubkeys_and_infinity_signature', { 'input': { @@ -186,7 +186,7 @@ def case04_fast_aggregate_verify(): 'output': False, } - # Invalid pubkeys and signature -- len(pubkey) == 0 and signature == 0x00... + # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == 0x00... assert not bls.FastAggregateVerify([], message, NO_SIGNATURE) yield f'fast_aggregate_verify_na_pubkeys_and_na_signature', { 'input': { @@ -235,7 +235,7 @@ def case05_aggregate_verify(): 'output': False, } - # Invalid pubkeys and signature -- len(pubkey) == 0 and signature == Z1_SIGNATURE + # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == Z1_SIGNATURE assert not bls.AggregateVerify([], [], Z2_SIGNATURE) yield f'aggregate_verify_na_pubkeys_and_infinity_signature', { 'input': { @@ -246,7 +246,7 @@ def case05_aggregate_verify(): 'output': False, } - # Invalid pubkeys and signature -- len(pubkey) == 0 and signature == 0x00... + # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == 0x00... assert not bls.AggregateVerify([], [], NO_SIGNATURE) yield f'aggregate_verify_na_pubkeys_and_na_signature', { 'input': { From 82073a4a834de77baa44a64c74e608b8b39b50f6 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 15 May 2020 03:05:23 +0800 Subject: [PATCH 51/70] Fix typo --- tests/generators/bls/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/generators/bls/main.py b/tests/generators/bls/main.py index ea7446e9b..4ad42d0a6 100644 --- a/tests/generators/bls/main.py +++ b/tests/generators/bls/main.py @@ -199,7 +199,7 @@ def case04_fast_aggregate_verify(): def case05_aggregate_verify(): - pubekys = [] + pubkeys = [] pubkeys_serial = [] messages = [] messages_serial = [] @@ -207,14 +207,14 @@ def case05_aggregate_verify(): for privkey, message in zip(PRIVKEYS, MESSAGES): sig = bls.Sign(privkey, message) pubkey = bls.SkToPk(privkey) - pubekys.append(pubkey) + pubkeys.append(pubkey) pubkeys_serial.append(encode_hex(pubkey)) messages.append(message) messages_serial.append(encode_hex(message)) sigs.append(sig) aggregate_signature = bls.Aggregate(sigs) - assert bls.AggregateVerify(pubekys, messages, aggregate_signature) + assert bls.AggregateVerify(pubkeys, messages, aggregate_signature) yield f'aggregate_verify_valid', { 'input': { 'pubkeys': pubkeys_serial, From 3dd168335bf5a86a753a5f34de3afd2c1ad36bcf Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 14 May 2020 13:50:29 -0600 Subject: [PATCH 52/70] reformat compute_subnet_for_attestation to not use for loop --- specs/phase0/validator.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 398b882ee..75aae1f2e 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -423,11 +423,12 @@ Finally, the validator broadcasts `attestation` to the associated attestation su ```python def compute_subnet_for_attestation(state: BeaconState, attestation: Attestation) -> uint64: + """ + Compute the correct subnet for an attestation for Phase 0. + Note, this mimics expected Phase 1 behavior where attestations will be mapped to their shard subnet. + """ slots_since_epoch_start = attestation.data.slot % SLOTS_PER_EPOCH - committees_since_epoch_start = sum([ - get_committee_count_at_slot(state, Slot(slot)) - for slot in range(slots_since_epoch_start) - ]) + committees_since_epoch_start = get_committee_count_at_slot(state, attestation.data.slot) * slots_since_epoch_start return (committees_since_epoch_start + attestation.data.index) % ATTESTATION_SUBNET_COUNT ``` From ab3cbdae7528caf2d3db6d4d7cd3bd262133b3e1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 15 May 2020 04:05:51 +0800 Subject: [PATCH 53/70] Add a note of `len(attestations) > 0` --- specs/phase0/validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 62fdc0a93..d8c164669 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -446,7 +446,7 @@ def is_aggregator(state: BeaconState, slot: Slot, index: CommitteeIndex, slot_si If the validator is selected to aggregate (`is_aggregator()`), they construct an aggregate attestation via the following. -Collect `attestations` seen via gossip during the `slot` that have an equivalent `attestation_data` to that constructed by the validator, and create an `aggregate_attestation: Attestation` with the following fields. +Collect `attestations` seen via gossip during the `slot` that have an equivalent `attestation_data` to that constructed by the validator. If `len(attestations) > 0`, create an `aggregate_attestation: Attestation` with the following fields. ##### Data From 483f9a1d7b7faeafe635229881810247c754371b Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 14 May 2020 14:25:32 -0600 Subject: [PATCH 54/70] Update specs/phase0/p2p-interface.md Co-authored-by: Diederik Loerakker --- specs/phase0/p2p-interface.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 98ee6edd2..7b9d6b737 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -220,12 +220,14 @@ The payload is carried in the `data` field of a gossipsub message, and varies de | Name | Message Type | |------------------------------------------------|-------------------------| -| beacon\_block | SignedBeaconBlock | -| beacon\_aggregate\_and\_proof | SignedAggregateAndProof | -| beacon_attestation\_{subnet\_id} | Attestation | -| voluntary\_exit | SignedVoluntaryExit | -| proposer\_slashing | ProposerSlashing | -| attester\_slashing | AttesterSlashing | +| Name | Message Type | +|----------------------------------|---------------------------| +| `beacon_block` | `SignedBeaconBlock` | +| `beacon_aggregate_and_proof` | `SignedAggregateAndProof` | +| `beacon_attestation_{subnet_id}` | `Attestation` | +| `voluntary_exit` | `SignedVoluntaryExit` | +| `proposer_slashing` | `ProposerSlashing` | +| `attester_slashing` | `AttesterSlashing` | Clients MUST reject (fail validation) messages containing an incorrect type, or invalid payload. From a9c4516f382f2b13d94f514b0564b8537cda929d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 15 May 2020 18:50:08 +0800 Subject: [PATCH 55/70] PR feedback from proto: revert configs --- configs/mainnet.yaml | 2 -- configs/minimal.yaml | 2 -- 2 files changed, 4 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 0fb81a1c3..60bd1c087 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -156,8 +156,6 @@ DOMAIN_SHARD_PROPOSAL: 0x80000000 DOMAIN_SHARD_COMMITTEE: 0x81000000 DOMAIN_LIGHT_CLIENT: 0x82000000 DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 -# Constant -NO_SIGNATURE: 0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 # Phase 1: Upgrade from Phase 0 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 6650eadc1..5c1511e6d 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -156,8 +156,6 @@ DOMAIN_SHARD_PROPOSAL: 0x80000000 DOMAIN_SHARD_COMMITTEE: 0x81000000 DOMAIN_LIGHT_CLIENT: 0x82000000 DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 -# Constant -NO_SIGNATURE: 0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 # Phase 1: Upgrade from Phase 0 From d07e594f92ea41c1ed1899422ebaa15d9f20c85b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 15 May 2020 23:27:35 +0800 Subject: [PATCH 56/70] Add `Aggregate()` case --- tests/formats/bls/aggregate.md | 6 +++--- tests/generators/bls/main.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/formats/bls/aggregate.md b/tests/formats/bls/aggregate.md index 0d7e7c631..af8444540 100644 --- a/tests/formats/bls/aggregate.md +++ b/tests/formats/bls/aggregate.md @@ -8,11 +8,11 @@ The test data is declared in a `data.yaml` file: ```yaml input: List[BLS Signature] -- list of input BLS signatures -output: BLS Signature -- expected output, single BLS signature +output: BLS Signature -- expected output, single BLS signature or empty. ``` -`BLS Signature` here is encoded as a string: hexadecimal encoding of 96 bytes (192 nibbles), prefixed with `0x`. - +- `BLS Signature` here is encoded as a string: hexadecimal encoding of 96 bytes (192 nibbles), prefixed with `0x`. +- No output value if the input is invalid. ## Condition diff --git a/tests/generators/bls/main.py b/tests/generators/bls/main.py index 4ad42d0a6..8c6589b36 100644 --- a/tests/generators/bls/main.py +++ b/tests/generators/bls/main.py @@ -127,6 +127,19 @@ def case03_aggregate(): 'output': encode_hex(bls.Aggregate(sigs)), } + # Invalid pubkeys -- len(pubkeys) == 0 + try: + bls.Aggregate([]) + except Exception: + pass + else: + raise Exception("Should have been INVALID") + + yield f'aggregate_na_pubkeys', { + 'input': [], + 'output': None, + } + def case04_fast_aggregate_verify(): for i, message in enumerate(MESSAGES): From fd3cce0d2c791d59237d25a632110405e7d033f1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 15 May 2020 23:38:25 +0800 Subject: [PATCH 57/70] Update README --- tests/formats/bls/README.md | 11 +++++------ tests/generators/bls/README.md | 18 ++++-------------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/tests/formats/bls/README.md b/tests/formats/bls/README.md index 4d95bdfd7..65154ba1c 100644 --- a/tests/formats/bls/README.md +++ b/tests/formats/bls/README.md @@ -5,11 +5,10 @@ We do not recommend rolling your own crypto or using an untested BLS library. The BLS test suite runner has the following handlers: -- [`aggregate_pubkeys`](./aggregate_pubkeys.md) -- [`aggregate_sigs`](./aggregate_sigs.md) -- [`msg_hash_g2_compressed`](./msg_hash_g2_compressed.md) -- [`msg_hash_g2_uncompressed`](./msg_hash_g2_uncompressed.md) -- [`priv_to_pub`](./priv_to_pub.md) -- [`sign_msg`](./sign_msg.md) +- [`aggregate_verify`](./aggregate_verify.md) +- [`aggregate`](./aggregate.md) +- [`fast_aggregate_verify`](./fast_aggregate_verify.md) +- [`sign`](./sign.md) +- [`verify`](./verify.md) *Note*: Signature-verification and aggregate-verify test cases are not yet supported. diff --git a/tests/generators/bls/README.md b/tests/generators/bls/README.md index 878bb156b..24013f88e 100644 --- a/tests/generators/bls/README.md +++ b/tests/generators/bls/README.md @@ -1,21 +1,11 @@ # BLS Test Generator -Explanation of BLS12-381 type hierarchy -The base unit is bytes48 of which only 381 bits are used +The [BLS Signature APIs](../../../specs/phase0/beacon-chain.md#bls-signatures) -- FQ: uint381 modulo field modulus -- FQ2: (FQ, FQ) -- G2: (FQ2, FQ2, FQ2) +Information on the format of the tests can be found in the [BLS test formats documentation](../../formats/bls/README.md). ## Resources -- [Eth2 spec](../../../specs/phase0/beacon-chain.md#bls-signatures) +- [IETF BLS Signature Scheme](https://datatracker.ietf.org/doc/draft-irtf-cfrg-bls-signature/) - [Finite Field Arithmetic](http://www.springeronline.com/sgw/cda/pageitems/document/cda_downloaddocument/0,11996,0-0-45-110359-0,00.pdf) -- Chapter 2 of [Elliptic Curve Cryptography](http://cacr.uwaterloo.ca/ecc/). Darrel Hankerson, Alfred Menezes, and Scott Vanstone -- [Zcash BLS parameters](https://github.com/zkcrypto/pairing/tree/master/src/bls12_381) -- [Trinity implementation](https://github.com/ethereum/trinity/blob/master/eth2/_utils/bls.py) - -## Comments - -Compared to Zcash, Ethereum specs always requires the compressed form (c_flag / most significant bit always set). -Also note that pubkeys and privkeys are reversed. +- Chapter 2 of [Elliptic Curve Cryptography](http://cacr.uwaterloo.ca/ecc/). Darrel Hankerson, Alfred Menezes, and Scott Vanstone From b975a92e9016650bc90c4ea6619675933d70ddeb Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 May 2020 09:29:01 -0600 Subject: [PATCH 58/70] ensure at least one validator is slashed for slashing rewards tests --- tests/core/pyspec/eth2spec/test/helpers/rewards.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index c5eace226..0433e808a 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -83,8 +83,10 @@ def exit_random_validators(spec, state, rng): def slash_random_validators(spec, state, rng): # Slash ~1/2 of validators - for validator in state.validators: - validator.slashed = rng.choice([True, False]) + for index in range(len(state.validators)): + # slash at least one validator + if index == 0 or rng.choice([True, False]): + spec.slash_validator(state, index) def run_test_empty(spec, state, runner): From 97b6db497188da5c6fe9c1205b25642eb12232e2 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 May 2020 10:05:06 -0600 Subject: [PATCH 59/70] add rewards tests for validators not yet activated --- .../pyspec/eth2spec/test/helpers/deposits.py | 11 +++++++++ .../pyspec/eth2spec/test/helpers/rewards.py | 23 ++++++++++++++++++- .../test_process_registry_updates.py | 9 +------- .../phase_0/rewards/test_get_head_deltas.py | 6 +++++ .../test_get_inactivity_penalty_deltas.py | 21 +++++++++++++++++ .../test_get_inclusion_delay_deltas.py | 6 +++++ .../phase_0/rewards/test_get_source_deltas.py | 6 +++++ .../phase_0/rewards/test_get_target_deltas.py | 12 ++++++++++ 8 files changed, 85 insertions(+), 9 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/deposits.py b/tests/core/pyspec/eth2spec/test/helpers/deposits.py index a16f7a7bf..6a2e30497 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/deposits.py +++ b/tests/core/pyspec/eth2spec/test/helpers/deposits.py @@ -5,6 +5,17 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import List +def mock_deposit(spec, state, index): + """ + Mock validator at ``index`` as having just made a deposit + """ + assert spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)) + state.validators[index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + state.validators[index].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[index].effective_balance = spec.MAX_EFFECTIVE_BALANCE + assert not spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)) + + def build_deposit_data(spec, pubkey, privkey, amount, withdrawal_credentials, signed=False): deposit_data = spec.DepositData( pubkey=pubkey, diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 0433e808a..ac863725c 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -2,6 +2,7 @@ from random import Random from eth2spec.phase0 import spec as spec_phase0 from eth2spec.test.helpers.attestations import prepare_state_with_attestations +from eth2spec.test.helpers.deposits import mock_deposit from eth2spec.test.helpers.state import next_epoch from eth2spec.utils.ssz.ssz_typing import Container, uint64, List @@ -61,6 +62,17 @@ def run_attestation_component_deltas(spec, state, component_delta_fn, matching_a assert penalties[index] == 0 +def set_some_new_deposits(spec, state, rng): + num_validators = len(state.validators) + # last 10th of validators are new deposits + for i in range(len(state.validators))[0:num_validators // 10]: + index = num_validators - 1 - i + mock_deposit(spec, state, index) + # Set half to eligible for activation + if i % 2 == 0: + state.validators[index].activation_eligibility_epoch = spec.get_current_epoch(state) + + def exit_random_validators(spec, state, rng): if spec.get_current_epoch(state) < 5: # Move epochs forward to allow for some validators already exited/withdrawable @@ -69,10 +81,11 @@ def exit_random_validators(spec, state, rng): current_epoch = spec.get_current_epoch(state) # Exit ~1/2 of validators - for validator in state.validators: + for index in spec.get_active_validator_indices(state, current_epoch): if rng.choice([True, False]): continue + validator = state.validators[index] validator.exit_epoch = rng.choice([current_epoch - 1, current_epoch - 2, current_epoch - 3]) # ~1/2 are withdrawable if rng.choice([True, False]): @@ -133,6 +146,13 @@ def run_test_one_attestation_one_correct(spec, state, runner): yield from runner(spec, state) +def run_test_with_not_yet_activated_validators(spec, state, runner, rng=Random(5555)): + set_some_new_deposits(spec, state, rng) + prepare_state_with_attestations(spec, state) + + yield from runner(spec, state) + + def run_test_with_exited_validators(spec, state, runner, rng=Random(1337)): exit_random_validators(spec, state, rng) prepare_state_with_attestations(spec, state) @@ -190,6 +210,7 @@ def run_test_full_fraction_incorrect(spec, state, correct_target, correct_head, def run_test_full_random(spec, state, runner, rng=Random(8020)): + set_some_new_deposits(spec, state, rng) exit_random_validators(spec, state, rng) slash_random_validators(spec, state, rng) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py index a5f4d9227..b6597b1cf 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py @@ -1,3 +1,4 @@ +from eth2spec.test.helpers.deposits import mock_deposit from eth2spec.test.helpers.state import next_epoch, next_slots from eth2spec.test.context import spec_state_test, with_all_phases from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_with @@ -7,14 +8,6 @@ def run_process_registry_updates(spec, state): yield from run_epoch_processing_with(spec, state, 'process_registry_updates') -def mock_deposit(spec, state, index): - assert spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)) - state.validators[index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - state.validators[index].activation_epoch = spec.FAR_FUTURE_EPOCH - state.validators[index].effective_balance = spec.MAX_EFFECTIVE_BALANCE - assert not spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)) - - @with_all_phases @spec_state_test def test_add_to_activation_queue(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py index 2e4b9dbbc..e49a8567c 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py @@ -48,6 +48,12 @@ def test_one_attestation_one_correct(spec, state): yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state, run_get_head_deltas) +@with_all_phases +@spec_state_test +def test_with_not_yet_activated_validators(spec, state): + yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state, run_get_head_deltas) + + @with_all_phases @spec_state_test def test_with_exited_validators(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py index 4940cdc63..8588b3f52 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py @@ -115,6 +115,27 @@ def test_full_but_partial_participation_leak(spec, state): yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_inactivity_penalty_deltas) +@with_all_phases +@spec_state_test +def test_with_not_yet_activated_validators_no_leak(spec, state): + yield from rewards_helpers.run_test_with_not_yet_activated_validators( + spec, + state, + run_get_inactivity_penalty_deltas, + ) + + +@with_all_phases +@spec_state_test +def test_with_not_yet_activated_validators_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_with_not_yet_activated_validators( + spec, + state, + run_get_inactivity_penalty_deltas, + ) + + @with_all_phases @spec_state_test def test_with_exited_validators_no_leak(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py index 64e1a1796..94fef5777 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py @@ -83,6 +83,12 @@ def test_full_but_partial_participation(spec, state): yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_inclusion_delay_deltas) +@with_all_phases +@spec_state_test +def test_with_not_yet_activated_validators(spec, state): + yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state, run_get_inclusion_delay_deltas) + + @with_all_phases @spec_state_test def test_with_exited_validators(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py index 54f8f3b5d..1d7891eca 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py @@ -48,6 +48,12 @@ def test_one_attestation_one_correct(spec, state): yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state, run_get_source_deltas) +@with_all_phases +@spec_state_test +def test_with_not_yet_activated_validators(spec, state): + yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state, run_get_source_deltas) + + @with_all_phases @spec_state_test def test_with_exited_validators(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py index 0ae985086..5a02a246b 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py @@ -48,6 +48,18 @@ def test_one_attestation_one_correct(spec, state): yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state, run_get_target_deltas) +@with_all_phases +@spec_state_test +def test_with_not_yet_activated_validators(spec, state): + yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state, run_get_target_deltas) + + +@with_all_phases +@spec_state_test +def test_with_exited_validators(spec, state): + yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_target_deltas) + + @with_all_phases @spec_state_test def test_with_slashed_validators(spec, state): From 96b5733086f9d2900077132bd32885474f53e430 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 May 2020 12:06:24 -0600 Subject: [PATCH 60/70] cleanup set some new deposits helper for rewards tests --- tests/core/pyspec/eth2spec/test/helpers/rewards.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index ac863725c..9c445f968 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -64,13 +64,13 @@ def run_attestation_component_deltas(spec, state, component_delta_fn, matching_a def set_some_new_deposits(spec, state, rng): num_validators = len(state.validators) - # last 10th of validators are new deposits - for i in range(len(state.validators))[0:num_validators // 10]: - index = num_validators - 1 - i - mock_deposit(spec, state, index) - # Set half to eligible for activation - if i % 2 == 0: - state.validators[index].activation_eligibility_epoch = spec.get_current_epoch(state) + # Set ~1/10 to just recently deposited + for index in range(num_validators): + if rng.randrange(num_validators) < num_validators // 10: + mock_deposit(spec, state, index) + # Set ~half of selected to eligible for activation + if rng.choice([True, False]): + state.validators[index].activation_eligibility_epoch = spec.get_current_epoch(state) def exit_random_validators(spec, state, rng): From 8060505743911f6855254a386a07f5efd85cabeb Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 May 2020 16:00:59 -0600 Subject: [PATCH 61/70] refactor rewards/penalties tests to use a single structure --- .../pyspec/eth2spec/test/helpers/rewards.py | 250 +++++++++++++++--- ...est_get_source_deltas.py => test_basic.py} | 88 +++--- .../phase_0/rewards/test_get_head_deltas.py | 136 ---------- .../test_get_inactivity_penalty_deltas.py | 231 ---------------- .../test_get_inclusion_delay_deltas.py | 213 --------------- .../phase_0/rewards/test_get_target_deltas.py | 140 ---------- .../test/phase_0/rewards/test_leak.py | 165 ++++++++++++ .../test/phase_0/rewards/test_random.py | 22 ++ tests/formats/rewards/README.md | 57 ++-- tests/generators/rewards/main.py | 24 +- 10 files changed, 504 insertions(+), 822 deletions(-) rename tests/core/pyspec/eth2spec/test/phase_0/rewards/{test_get_source_deltas.py => test_basic.py} (67%) delete mode 100644 tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py delete mode 100644 tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py delete mode 100644 tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py delete mode 100644 tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py create mode 100644 tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py create mode 100644 tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 9c445f968..fc9f1f93a 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -25,17 +25,50 @@ def has_enough_for_reward(spec, state, index): ) -def run_attestation_component_deltas(spec, state, component_delta_fn, matching_att_fn): +def run_deltas(spec, state): """ - Run ``component_delta_fn``, yielding: + Run all deltas functions yielding: - pre-state ('pre') - - deltas ('deltas') + - source deltas ('source_deltas') + - target deltas ('target_deltas') + - head deltas ('head_deltas') + - inclusion delay deltas ('inclusion_delay_deltas') + - inactivity penalty deltas ('inactivity_penalty_deltas') """ yield 'pre', state + yield from run_attestation_component_deltas( + spec, + state, + spec.get_source_deltas, + spec.get_matching_source_attestations, + 'source_deltas', + ) + yield from run_attestation_component_deltas( + spec, + state, + spec.get_target_deltas, + spec.get_matching_target_attestations, + 'target_deltas', + ) + yield from run_attestation_component_deltas( + spec, + state, + spec.get_head_deltas, + spec.get_matching_head_attestations, + 'head_deltas', + ) + yield from run_get_inclusion_delay_deltas(spec, state) + yield from run_get_inactivity_penalty_deltas(spec, state) + +def run_attestation_component_deltas(spec, state, component_delta_fn, matching_att_fn, deltas_name): + """ + Run ``component_delta_fn``, yielding: + - deltas ('{``deltas_name``}') + """ rewards, penalties = component_delta_fn(state) - yield 'deltas', Deltas(rewards=rewards, penalties=penalties) + yield deltas_name, Deltas(rewards=rewards, penalties=penalties) matching_attestations = matching_att_fn(state, spec.get_previous_epoch(state)) matching_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) @@ -62,6 +95,81 @@ def run_attestation_component_deltas(spec, state, component_delta_fn, matching_a assert penalties[index] == 0 +def run_get_inclusion_delay_deltas(spec, state): + """ + Run ``get_inclusion_delay_deltas``, yielding: + - inclusion delay deltas ('inclusion_delay_deltas') + """ + rewards, penalties = spec.get_inclusion_delay_deltas(state) + + yield 'inclusion_delay_deltas', Deltas(rewards=rewards, penalties=penalties) + + eligible_attestations = spec.get_matching_source_attestations(state, spec.get_previous_epoch(state)) + attesting_indices = spec.get_unslashed_attesting_indices(state, eligible_attestations) + + rewarded_indices = set() + rewarded_proposer_indices = set() + # Ensure attesters with enough balance are rewarded for attestations + # Track those that are rewarded and track proposers that should be rewarded + for index in range(len(state.validators)): + if index in attesting_indices and has_enough_for_reward(spec, state, index): + assert rewards[index] > 0 + rewarded_indices.add(index) + + # Track proposer of earliest included attestation for the validator defined by index + earliest_attestation = min([ + a for a in eligible_attestations + if index in spec.get_attesting_indices(state, a.data, a.aggregation_bits) + ], key=lambda a: a.inclusion_delay) + rewarded_proposer_indices.add(earliest_attestation.proposer_index) + + # Ensure all expected proposers have been rewarded + # Track rewarde indices + proposing_indices = [a.proposer_index for a in eligible_attestations] + for index in proposing_indices: + if index in rewarded_proposer_indices: + assert rewards[index] > 0 + rewarded_indices.add(index) + + # Ensure all expected non-rewarded indices received no reward + for index in range(len(state.validators)): + assert penalties[index] == 0 + if index not in rewarded_indices: + assert rewards[index] == 0 + + +def run_get_inactivity_penalty_deltas(spec, state): + """ + Run ``get_inactivity_penalty_deltas``, yielding: + - inactivity penalty deltas ('inactivity_penalty_deltas') + """ + rewards, penalties = spec.get_inactivity_penalty_deltas(state) + + yield 'inactivity_penalty_deltas', Deltas(rewards=rewards, penalties=penalties) + + matching_attestations = spec.get_matching_target_attestations(state, spec.get_previous_epoch(state)) + matching_attesting_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) + + finality_delay = spec.get_previous_epoch(state) - state.finalized_checkpoint.epoch + eligible_indices = spec.get_eligible_validator_indices(state) + for index in range(len(state.validators)): + assert rewards[index] == 0 + if index not in eligible_indices: + assert penalties[index] == 0 + continue + + if finality_delay > spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY: + base_penalty = spec.BASE_REWARDS_PER_EPOCH * spec.get_base_reward(state, index) + if not has_enough_for_reward(spec, state, index): + assert penalties[index] == 0 + elif index in matching_attesting_indices: + assert penalties[index] == base_penalty + else: + assert penalties[index] > base_penalty + else: + assert penalties[index] == 0 + + def set_some_new_deposits(spec, state, rng): num_validators = len(state.validators) # Set ~1/10 to just recently deposited @@ -102,74 +210,74 @@ def slash_random_validators(spec, state, rng): spec.slash_validator(state, index) -def run_test_empty(spec, state, runner): +def run_test_empty(spec, state): # Do not add any attestations to state - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_full_all_correct(spec, state, runner): +def run_test_full_all_correct(spec, state): prepare_state_with_attestations(spec, state) - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_full_but_partial_participation(spec, state, runner, rng=Random(5522)): +def run_test_full_but_partial_participation(spec, state, rng=Random(5522)): prepare_state_with_attestations(spec, state) for a in state.previous_epoch_attestations: a.aggregation_bits = [rng.choice([True, False]) for _ in a.aggregation_bits] - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_partial(spec, state, fraction_filled, runner): +def run_test_partial(spec, state, fraction_filled): prepare_state_with_attestations(spec, state) # Remove portion of attestations num_attestations = int(len(state.previous_epoch_attestations) * fraction_filled) state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_half_full(spec, state, runner): - yield from run_test_partial(spec, state, 0.5, runner) +def run_test_half_full(spec, state): + yield from run_test_partial(spec, state, 0.5) -def run_test_one_attestation_one_correct(spec, state, runner): +def run_test_one_attestation_one_correct(spec, state): prepare_state_with_attestations(spec, state) # Remove all attestations except for the first one state.previous_epoch_attestations = state.previous_epoch_attestations[:1] - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_with_not_yet_activated_validators(spec, state, runner, rng=Random(5555)): +def run_test_with_not_yet_activated_validators(spec, state, rng=Random(5555)): set_some_new_deposits(spec, state, rng) prepare_state_with_attestations(spec, state) - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_with_exited_validators(spec, state, runner, rng=Random(1337)): +def run_test_with_exited_validators(spec, state, rng=Random(1337)): exit_random_validators(spec, state, rng) prepare_state_with_attestations(spec, state) - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_with_slashed_validators(spec, state, runner, rng=Random(3322)): +def run_test_with_slashed_validators(spec, state, rng=Random(3322)): exit_random_validators(spec, state, rng) slash_random_validators(spec, state, rng) prepare_state_with_attestations(spec, state) - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_some_very_low_effective_balances_that_attested(spec, state, runner): +def run_test_some_very_low_effective_balances_that_attested(spec, state): state.balances prepare_state_with_attestations(spec, state) @@ -178,10 +286,10 @@ def run_test_some_very_low_effective_balances_that_attested(spec, state, runner) for i, index in enumerate(range(5)): state.validators[index].effective_balance = i - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state, runner): +def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state): prepare_state_with_attestations(spec, state) # Remove attestation @@ -192,10 +300,10 @@ def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state, r for i, index in enumerate(indices): state.validators[index].effective_balance = i - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_full_fraction_incorrect(spec, state, correct_target, correct_head, fraction_incorrect, runner): +def run_test_full_fraction_incorrect(spec, state, correct_target, correct_head, fraction_incorrect): prepare_state_with_attestations(spec, state) # Make fraction_incorrect of pending attestations have bad target/head as specified @@ -206,10 +314,92 @@ def run_test_full_fraction_incorrect(spec, state, correct_target, correct_head, if not correct_head: pending_attestation.data.beacon_block_root = b'\x66' * 32 - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_full_random(spec, state, runner, rng=Random(8020)): +def run_test_full_delay_one_slot(spec, state): + prepare_state_with_attestations(spec, state) + for a in state.previous_epoch_attestations: + a.inclusion_delay += 1 + + yield from run_deltas(spec, state) + + +def run_test_full_delay_max_slots(spec, state): + prepare_state_with_attestations(spec, state) + for a in state.previous_epoch_attestations: + a.inclusion_delay += spec.SLOTS_PER_EPOCH + + yield from run_deltas(spec, state) + + +def run_test_full_mixed_delay(spec, state, rng=Random(1234)): + prepare_state_with_attestations(spec, state) + for a in state.previous_epoch_attestations: + a.inclusion_delay = rng.randint(1, spec.SLOTS_PER_EPOCH) + + yield from run_deltas(spec, state) + + +def run_test_proposer_not_in_attestations(spec, state): + prepare_state_with_attestations(spec, state) + + # Get an attestation where the proposer is not in the committee + non_proposer_attestations = [] + for a in state.previous_epoch_attestations: + if a.proposer_index not in spec.get_unslashed_attesting_indices(state, [a]): + non_proposer_attestations.append(a) + + assert any(non_proposer_attestations) + state.previous_epoch_attestations = non_proposer_attestations + + yield from run_deltas(spec, state) + + +def run_test_duplicate_attestations_at_later_slots(spec, state): + prepare_state_with_attestations(spec, state) + + # Remove 2/3 of attestations to make it more interesting + num_attestations = int(len(state.previous_epoch_attestations) * 0.33) + state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] + + # Get map of the proposer at each slot to make valid-looking duplicate attestations + per_slot_proposers = { + (a.data.slot + a.inclusion_delay): a.proposer_index + for a in state.previous_epoch_attestations + } + max_slot = max([a.data.slot + a.inclusion_delay for a in state.previous_epoch_attestations]) + later_attestations = [] + for a in state.previous_epoch_attestations: + # Only have proposers for previous epoch so do not create later + # duplicate if slot exceeds the max slot in previous_epoch_attestations + if a.data.slot + a.inclusion_delay >= max_slot: + continue + later_a = a.copy() + later_a.inclusion_delay += 1 + later_a.proposer_index = per_slot_proposers[later_a.data.slot + later_a.inclusion_delay] + later_attestations.append(later_a) + + assert any(later_attestations) + + state.previous_epoch_attestations = sorted( + state.previous_epoch_attestations + later_attestations, + key=lambda a: a.data.slot + a.inclusion_delay + ) + + yield from run_deltas(spec, state) + + +def run_test_all_balances_too_low_for_reward(spec, state): + prepare_state_with_attestations(spec, state) + + for index in range(len(state.validators)): + state.validators[index].effective_balance = 10 + + yield from run_deltas(spec, state) + + +def run_test_full_random(spec, state, rng=Random(8020)): set_some_new_deposits(spec, state, rng) exit_random_validators(spec, state, rng) slash_random_validators(spec, state, rng) @@ -228,4 +418,4 @@ def run_test_full_random(spec, state, runner, rng=Random(8020)): # Random inclusion delay pending_attestation.inclusion_delay = rng.randint(1, spec.SLOTS_PER_EPOCH) - yield from runner(spec, state) + yield from run_deltas(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_basic.py similarity index 67% rename from tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py rename to tests/core/pyspec/eth2spec/test/phase_0/rewards/test_basic.py index 1d7891eca..92277fdd7 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_basic.py @@ -1,89 +1,71 @@ from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.rewards import run_attestation_component_deltas import eth2spec.test.helpers.rewards as rewards_helpers -def run_get_source_deltas(spec, state): - """ - Run ``get_source_deltas``, yielding: - - pre-state ('pre') - - deltas ('deltas') - """ - - yield from run_attestation_component_deltas( - spec, - state, - spec.get_source_deltas, - spec.get_matching_source_attestations, - ) - - @with_all_phases @spec_state_test def test_empty(spec, state): - yield from rewards_helpers.run_test_empty(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_empty(spec, state) @with_all_phases @spec_state_test def test_full_all_correct(spec, state): - yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_full_all_correct(spec, state) @with_all_phases @spec_state_test def test_half_full(spec, state): - yield from rewards_helpers.run_test_half_full(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_half_full(spec, state) + + +@with_all_phases +@spec_state_test +def test_quarter_full(spec, state): + yield from rewards_helpers.run_test_partial(spec, state, 0.25) @with_all_phases @spec_state_test def test_full_but_partial_participation(spec, state): - yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_full_but_partial_participation(spec, state) @with_all_phases @spec_state_test def test_one_attestation_one_correct(spec, state): - yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state) @with_all_phases @spec_state_test def test_with_not_yet_activated_validators(spec, state): - yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state) @with_all_phases @spec_state_test def test_with_exited_validators(spec, state): - yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_with_exited_validators(spec, state) @with_all_phases @spec_state_test def test_with_slashed_validators(spec, state): - yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_with_slashed_validators(spec, state) @with_all_phases @spec_state_test def test_some_very_low_effective_balances_that_attested(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( - spec, - state, - run_get_source_deltas - ) + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested(spec, state) @with_all_phases @spec_state_test def test_some_very_low_effective_balances_that_did_not_attest(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( - spec, - state, - run_get_source_deltas, - ) + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest(spec, state) # @@ -101,7 +83,6 @@ def test_full_half_correct_target_incorrect_head(spec, state): correct_target=True, correct_head=False, fraction_incorrect=0.5, - runner=run_get_source_deltas ) @@ -113,7 +94,6 @@ def test_full_correct_target_incorrect_head(spec, state): correct_target=True, correct_head=False, fraction_incorrect=1.0, - runner=run_get_source_deltas ) @@ -125,7 +105,6 @@ def test_full_half_incorrect_target_incorrect_head(spec, state): correct_target=False, correct_head=False, fraction_incorrect=0.5, - runner=run_get_source_deltas ) @@ -137,11 +116,40 @@ def test_full_half_incorrect_target_correct_head(spec, state): correct_target=False, correct_head=True, fraction_incorrect=0.5, - runner=run_get_source_deltas ) @with_all_phases @spec_state_test -def test_full_random(spec, state): - yield from rewards_helpers.run_test_full_random(spec, state, run_get_source_deltas) +def test_full_delay_one_slot(spec, state): + yield from rewards_helpers.run_test_full_delay_one_slot(spec, state) + + +@with_all_phases +@spec_state_test +def test_full_delay_max_slots(spec, state): + yield from rewards_helpers.run_test_full_delay_max_slots(spec, state) + + +@with_all_phases +@spec_state_test +def test_full_mixed_delay(spec, state): + yield from rewards_helpers.run_test_full_mixed_delay(spec, state) + + +@with_all_phases +@spec_state_test +def test_proposer_not_in_attestations(spec, state): + yield from rewards_helpers.run_test_proposer_not_in_attestations(spec, state) + + +@with_all_phases +@spec_state_test +def test_duplicate_attestations_at_later_slots(spec, state): + yield from rewards_helpers.run_test_duplicate_attestations_at_later_slots(spec, state) + + +@with_all_phases +@spec_state_test +def test_all_balances_too_low_for_reward(spec, state): + yield from rewards_helpers.run_test_all_balances_too_low_for_reward(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py deleted file mode 100644 index e49a8567c..000000000 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py +++ /dev/null @@ -1,136 +0,0 @@ -from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.rewards import run_attestation_component_deltas -import eth2spec.test.helpers.rewards as rewards_helpers - - -def run_get_head_deltas(spec, state): - """ - Run ``get_head_deltas``, yielding: - - pre-state ('pre') - - deltas ('deltas') - """ - - yield from run_attestation_component_deltas( - spec, - state, - spec.get_head_deltas, - spec.get_matching_head_attestations, - ) - - -@with_all_phases -@spec_state_test -def test_empty(spec, state): - yield from rewards_helpers.run_test_empty(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_full_all_correct(spec, state): - yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_half_full(spec, state): - yield from rewards_helpers.run_test_half_full(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_full_but_partial_participation(spec, state): - yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_one_attestation_one_correct(spec, state): - yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_with_not_yet_activated_validators(spec, state): - yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_with_exited_validators(spec, state): - yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_with_slashed_validators(spec, state): - yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_attested(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_did_not_attest(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( - spec, - state, - run_get_head_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_full_half_correct_target_incorrect_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=True, - correct_head=False, - fraction_incorrect=0.5, - runner=run_get_head_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_correct_target_incorrect_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=True, - correct_head=False, - fraction_incorrect=1.0, - runner=run_get_head_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_half_incorrect_target_incorrect_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=False, - correct_head=False, - fraction_incorrect=0.5, - runner=run_get_head_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_half_incorrect_target_correct_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=False, - correct_head=True, - fraction_incorrect=0.5, - runner=run_get_head_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_random(spec, state): - yield from rewards_helpers.run_test_full_random(spec, state, run_get_head_deltas) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py deleted file mode 100644 index 8588b3f52..000000000 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py +++ /dev/null @@ -1,231 +0,0 @@ -from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.rewards import has_enough_for_reward -from eth2spec.test.helpers.state import next_epoch -from eth2spec.test.helpers.rewards import Deltas -import eth2spec.test.helpers.rewards as rewards_helpers - - -def run_get_inactivity_penalty_deltas(spec, state): - """ - Run ``get_inactivity_penalty_deltas``, yielding: - - pre-state ('pre') - - deltas ('deltas') - """ - - yield 'pre', state - - rewards, penalties = spec.get_inactivity_penalty_deltas(state) - - yield 'deltas', Deltas(rewards=rewards, penalties=penalties) - - matching_attestations = spec.get_matching_target_attestations(state, spec.get_previous_epoch(state)) - matching_attesting_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) - - finality_delay = spec.get_previous_epoch(state) - state.finalized_checkpoint.epoch - eligible_indices = spec.get_eligible_validator_indices(state) - for index in range(len(state.validators)): - assert rewards[index] == 0 - if index not in eligible_indices: - assert penalties[index] == 0 - continue - - if finality_delay > spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY: - base_penalty = spec.BASE_REWARDS_PER_EPOCH * spec.get_base_reward(state, index) - if not has_enough_for_reward(spec, state, index): - assert penalties[index] == 0 - elif index in matching_attesting_indices: - assert penalties[index] == base_penalty - else: - assert penalties[index] > base_penalty - else: - assert penalties[index] == 0 - - -def transition_state_to_leak(spec, state, epochs=None): - if epochs is None: - epochs = spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY - assert epochs >= spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY - - for _ in range(epochs): - next_epoch(spec, state) - - -@with_all_phases -@spec_state_test -def test_empty_no_leak(spec, state): - yield from rewards_helpers.run_test_empty(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_empty_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_empty(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_no_leak(spec, state): - yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_half_full_no_leak(spec, state): - yield from rewards_helpers.run_test_half_full(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_half_full_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_half_full(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_quarter_full_no_leak(spec, state): - yield from rewards_helpers.run_test_partial(spec, state, 0.25, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_quarter_full_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_partial(spec, state, 0.25, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_but_partial_participation_no_leak(spec, state): - yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_but_partial_participation_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_with_not_yet_activated_validators_no_leak(spec, state): - yield from rewards_helpers.run_test_with_not_yet_activated_validators( - spec, - state, - run_get_inactivity_penalty_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_with_not_yet_activated_validators_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_with_not_yet_activated_validators( - spec, - state, - run_get_inactivity_penalty_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_with_exited_validators_no_leak(spec, state): - yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_with_exited_validators_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_with_slashed_validators_no_leak(spec, state): - yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_with_slashed_validators_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_attested_no_leak(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( - spec, - state, - run_get_inactivity_penalty_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_attested_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( - spec, - state, - run_get_inactivity_penalty_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_did_not_attest_no_leak(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( - spec, - state, - run_get_inactivity_penalty_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_did_not_attest_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( - spec, - state, - run_get_inactivity_penalty_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_full_random_no_leak(spec, state): - yield from rewards_helpers.run_test_full_random(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_random_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_full_random(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_random_five_epoch_leak(spec, state): - transition_state_to_leak(spec, state, epochs=5) - yield from rewards_helpers.run_test_full_random(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_random_ten_epoch_leak(spec, state): - transition_state_to_leak(spec, state, epochs=10) - yield from rewards_helpers.run_test_full_random(spec, state, run_get_inactivity_penalty_deltas) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py deleted file mode 100644 index 94fef5777..000000000 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py +++ /dev/null @@ -1,213 +0,0 @@ -from random import Random - -from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.attestations import prepare_state_with_attestations -from eth2spec.test.helpers.rewards import Deltas, has_enough_for_reward -import eth2spec.test.helpers.rewards as rewards_helpers - - -def run_get_inclusion_delay_deltas(spec, state): - """ - Run ``get_inclusion_delay_deltas``, yielding: - - pre-state ('pre') - - deltas ('deltas') - """ - - yield 'pre', state - - rewards, penalties = spec.get_inclusion_delay_deltas(state) - - yield 'deltas', Deltas(rewards=rewards, penalties=penalties) - - eligible_attestations = spec.get_matching_source_attestations(state, spec.get_previous_epoch(state)) - attesting_indices = spec.get_unslashed_attesting_indices(state, eligible_attestations) - - rewarded_indices = set() - rewarded_proposer_indices = set() - # Ensure attesters with enough balance are rewarded for attestations - # Track those that are rewarded and track proposers that should be rewarded - for index in range(len(state.validators)): - if index in attesting_indices and has_enough_for_reward(spec, state, index): - assert rewards[index] > 0 - rewarded_indices.add(index) - - # Track proposer of earliest included attestation for the validator defined by index - earliest_attestation = min([ - a for a in eligible_attestations - if index in spec.get_attesting_indices(state, a.data, a.aggregation_bits) - ], key=lambda a: a.inclusion_delay) - rewarded_proposer_indices.add(earliest_attestation.proposer_index) - - # Ensure all expected proposers have been rewarded - # Track rewarde indices - proposing_indices = [a.proposer_index for a in eligible_attestations] - for index in proposing_indices: - if index in rewarded_proposer_indices: - assert rewards[index] > 0 - rewarded_indices.add(index) - - # Ensure all expected non-rewarded indices received no reward - for index in range(len(state.validators)): - assert penalties[index] == 0 - if index not in rewarded_indices: - assert rewards[index] == 0 - - -@with_all_phases -@spec_state_test -def test_empty(spec, state): - yield from rewards_helpers.run_test_empty(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_full(spec, state): - yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_half_full(spec, state): - yield from rewards_helpers.run_test_half_full(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_quarter_full(spec, state): - yield from rewards_helpers.run_test_partial(spec, state, 0.25, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_full_but_partial_participation(spec, state): - yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_with_not_yet_activated_validators(spec, state): - yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_with_exited_validators(spec, state): - yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_with_slashed_validators(spec, state): - yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_attested(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( - spec, - state, - run_get_inclusion_delay_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_random(spec, state): - yield from rewards_helpers.run_test_full_random(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_full_delay_one_slot(spec, state): - prepare_state_with_attestations(spec, state) - for a in state.previous_epoch_attestations: - a.inclusion_delay += 1 - - yield from run_get_inclusion_delay_deltas(spec, state) - - -@with_all_phases -@spec_state_test -def test_full_delay_max_slots(spec, state): - prepare_state_with_attestations(spec, state) - for a in state.previous_epoch_attestations: - a.inclusion_delay += spec.SLOTS_PER_EPOCH - - yield from run_get_inclusion_delay_deltas(spec, state) - - -@with_all_phases -@spec_state_test -def test_full_mixed_delay(spec, state): - rng = Random(1234) - - prepare_state_with_attestations(spec, state) - for a in state.previous_epoch_attestations: - a.inclusion_delay = rng.randint(1, spec.SLOTS_PER_EPOCH) - - yield from run_get_inclusion_delay_deltas(spec, state) - - -@with_all_phases -@spec_state_test -def test_proposer_not_in_attestations(spec, state): - prepare_state_with_attestations(spec, state) - - # Get an attestation where the proposer is not in the committee - non_proposer_attestations = [] - for a in state.previous_epoch_attestations: - if a.proposer_index not in spec.get_unslashed_attesting_indices(state, [a]): - non_proposer_attestations.append(a) - - assert any(non_proposer_attestations) - state.previous_epoch_attestations = non_proposer_attestations - - yield from run_get_inclusion_delay_deltas(spec, state) - - -@with_all_phases -@spec_state_test -def test_duplicate_attestations_at_later_slots(spec, state): - prepare_state_with_attestations(spec, state) - - # Remove 2/3 of attestations to make it more interesting - num_attestations = int(len(state.previous_epoch_attestations) * 0.33) - state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] - - # Get map of the proposer at each slot to make valid-looking duplicate attestations - per_slot_proposers = { - (a.data.slot + a.inclusion_delay): a.proposer_index - for a in state.previous_epoch_attestations - } - max_slot = max([a.data.slot + a.inclusion_delay for a in state.previous_epoch_attestations]) - later_attestations = [] - for a in state.previous_epoch_attestations: - # Only have proposers for previous epoch so do not create later - # duplicate if slot exceeds the max slot in previous_epoch_attestations - if a.data.slot + a.inclusion_delay >= max_slot: - continue - later_a = a.copy() - later_a.inclusion_delay += 1 - later_a.proposer_index = per_slot_proposers[later_a.data.slot + later_a.inclusion_delay] - later_attestations.append(later_a) - - assert any(later_attestations) - - state.previous_epoch_attestations = sorted( - state.previous_epoch_attestations + later_attestations, - key=lambda a: a.data.slot + a.inclusion_delay - ) - - yield from run_get_inclusion_delay_deltas(spec, state) - - -@with_all_phases -@spec_state_test -def test_all_balances_too_low_for_reward(spec, state): - prepare_state_with_attestations(spec, state) - - for index in range(len(state.validators)): - state.validators[index].effective_balance = 10 - - yield from run_get_inclusion_delay_deltas(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py deleted file mode 100644 index 5a02a246b..000000000 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py +++ /dev/null @@ -1,140 +0,0 @@ -from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.rewards import run_attestation_component_deltas -import eth2spec.test.helpers.rewards as rewards_helpers - - -def run_get_target_deltas(spec, state): - """ - Run ``get_target_deltas``, yielding: - - pre-state ('pre') - - deltas ('deltas') - """ - - yield from run_attestation_component_deltas( - spec, - state, - spec.get_target_deltas, - spec.get_matching_target_attestations, - ) - - -@with_all_phases -@spec_state_test -def test_empty(spec, state): - yield from rewards_helpers.run_test_empty(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_full_all_correct(spec, state): - yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_half_full(spec, state): - yield from rewards_helpers.run_test_half_full(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_full_but_partial_participation(spec, state): - yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_one_attestation_one_correct(spec, state): - yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_with_not_yet_activated_validators(spec, state): - yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_with_exited_validators(spec, state): - yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_with_slashed_validators(spec, state): - yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_attested(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( - spec, - state, - run_get_target_deltas - ) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_did_not_attest(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( - spec, - state, - run_get_target_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_full_half_correct_target_incorrect_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=True, - correct_head=False, - fraction_incorrect=0.5, - runner=run_get_target_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_correct_target_incorrect_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=True, - correct_head=False, - fraction_incorrect=1.0, - runner=run_get_target_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_half_incorrect_target_incorrect_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=False, - correct_head=False, - fraction_incorrect=0.5, - runner=run_get_target_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_half_incorrect_target_correct_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=False, - correct_head=True, - fraction_incorrect=0.5, - runner=run_get_target_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_random(spec, state): - yield from rewards_helpers.run_test_full_random(spec, state, run_get_target_deltas) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py new file mode 100644 index 000000000..562a99b4b --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py @@ -0,0 +1,165 @@ +from eth2spec.test.context import with_all_phases, spec_state_test +from eth2spec.test.helpers.state import next_epoch +import eth2spec.test.helpers.rewards as rewards_helpers + + +def transition_state_to_leak(spec, state, epochs=None): + if epochs is None: + epochs = spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + assert epochs >= spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + + for _ in range(epochs): + next_epoch(spec, state) + + +@with_all_phases +@spec_state_test +def test_empty_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_empty(spec, state) + + +@with_all_phases +@spec_state_test +def test_full_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_full_all_correct(spec, state) + + +@with_all_phases +@spec_state_test +def test_half_full_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_half_full(spec, state) + + +@with_all_phases +@spec_state_test +def test_quarter_full_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_partial(spec, state, 0.25) + + +@with_all_phases +@spec_state_test +def test_full_but_partial_participation_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_full_but_partial_participation(spec, state) + + +@with_all_phases +@spec_state_test +def test_one_attestation_one_correct_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state) + + +@with_all_phases +@spec_state_test +def test_with_not_yet_activated_validators_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state) + + +@with_all_phases +@spec_state_test +def test_with_exited_validators_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_with_exited_validators(spec, state) + + +@with_all_phases +@spec_state_test +def test_with_slashed_validators_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_with_slashed_validators(spec, state) + + +@with_all_phases +@spec_state_test +def test_some_very_low_effective_balances_that_attested_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested(spec, state) + + +@with_all_phases +@spec_state_test +def test_some_very_low_effective_balances_that_did_not_attest_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest(spec, state) + + +# +# NOTE: No source incorrect tests +# All PendingAttestations in state have source validated +# We choose to keep this invariant in these tests to not force clients to test with degenerate states +# + + +@with_all_phases +@spec_state_test +def test_full_half_correct_target_incorrect_head_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_full_fraction_incorrect( + spec, state, + correct_target=True, + correct_head=False, + fraction_incorrect=0.5, + ) + + +@with_all_phases +@spec_state_test +def test_full_correct_target_incorrect_head_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_full_fraction_incorrect( + spec, state, + correct_target=True, + correct_head=False, + fraction_incorrect=1.0, + ) + + +@with_all_phases +@spec_state_test +def test_full_half_incorrect_target_incorrect_head_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_full_fraction_incorrect( + spec, state, + correct_target=False, + correct_head=False, + fraction_incorrect=0.5, + ) + + +@with_all_phases +@spec_state_test +def test_full_half_incorrect_target_correct_head_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_full_fraction_incorrect( + spec, state, + correct_target=False, + correct_head=True, + fraction_incorrect=0.5, + ) + + +@with_all_phases +@spec_state_test +def test_full_random_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_full_random(spec, state) + + +@with_all_phases +@spec_state_test +def test_full_random_five_epoch_leak(spec, state): + transition_state_to_leak(spec, state, epochs=5) + yield from rewards_helpers.run_test_full_random(spec, state) + + +@with_all_phases +@spec_state_test +def test_full_random_ten_epoch_leak(spec, state): + transition_state_to_leak(spec, state, epochs=10) + yield from rewards_helpers.run_test_full_random(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py new file mode 100644 index 000000000..bda0ca687 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py @@ -0,0 +1,22 @@ +from random import Random + +from eth2spec.test.context import with_all_phases, spec_state_test +import eth2spec.test.helpers.rewards as rewards_helpers + + +@with_all_phases +@spec_state_test +def test_full_random_0(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state, rng=Random(1010)) + + +@with_all_phases +@spec_state_test +def test_full_random_1(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state, rng=Random(2020)) + + +@with_all_phases +@spec_state_test +def test_full_random_2(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state, rng=Random(3030)) diff --git a/tests/formats/rewards/README.md b/tests/formats/rewards/README.md index f70a20f9c..b00e7a32a 100644 --- a/tests/formats/rewards/README.md +++ b/tests/formats/rewards/README.md @@ -1,8 +1,15 @@ # Rewards tests -The different rewards deltas sub-functions are testing individually with the test handlers, each returning the related `rewards`/`penalties`. +All rewards deltas sub-functions are tested for each test case. There is no "change" factor, the rewards/penalties outputs are pure functions with just the pre-state as input. -Hence, the format is shared between each test-handler. (See test condition documentation on how to run the tests.) +(See test condition documentation on how to run the tests.) + +`Deltas` is defined as: +```python +class Deltas(Container): + rewards: List[uint64, VALIDATOR_REGISTRY_LIMIT] + penalties: List[uint64, VALIDATOR_REGISTRY_LIMIT] +``` ## Test case format @@ -22,31 +29,47 @@ A YAML-encoded `BeaconState`, the state before running the rewards sub-function. Also available as `pre.ssz`. -### `deltas.yaml` +### `source_deltas.yaml` -A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards sub-function +A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_source_deltas` function -Where `Deltas` is defined as: -```python -class Deltas(Container): - rewards: List[uint64, VALIDATOR_REGISTRY_LIMIT] - penalties: List[uint64, VALIDATOR_REGISTRY_LIMIT] -``` +Also available as `source_deltas.ssz`. -Also available as `deltas.ssz`. +### `target_deltas.yaml` + +A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_target_deltas` function + +Also available as `target_deltas.ssz`. + +### `head_deltas.yaml` + +A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_head_deltas` function + +Also available as `head_deltas.ssz`. + +### `inclusion_delay_deltas.yaml` + +A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_inclusion_delay_deltas` function + +Also available as `inclusion_delay_deltas.ssz`. + +### `inactivity_penalty_deltas.yaml` + +A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_inactivity_penalty_deltas` function + +Also available as `inactivity_penalty_deltas.ssz`. ## Condition A handler of the `rewards` test-runner should process these cases, - calling the corresponding rewards deltas function (same name in spec). -This excludes all other parts of `process_rewards_and_penalties` + calling the corresponding rewards deltas function for each set of deltas. -The provided pre-state is ready to be input into the designated handler. +The provided pre-state is ready to be input into each rewards deltas function. The provided `deltas` should match the return values of the - handler. Specifically the following must hold true: + deltas function. Specifically the following must hold true for each set of deltas: ```python - deltas.rewards == handler(state)[0] - deltas.penalties == handler(state)[1] + deltas.rewards == deltas_function(state)[0] + deltas.penalties == deltas_function(state)[1] ``` diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index fd95dcfaa..d8dae74fa 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -3,11 +3,9 @@ from typing import Iterable from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 from eth2spec.test.phase_0.rewards import ( - test_get_source_deltas, - test_get_target_deltas, - test_get_head_deltas, - test_get_inclusion_delay_deltas, - test_get_inactivity_penalty_deltas, + test_basic, + test_leak, + test_random, ) from gen_base import gen_runner, gen_typing from gen_from_tests.gen import generate_from_tests @@ -37,14 +35,10 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin if __name__ == "__main__": gen_runner.run_generator("epoch_processing", [ - create_provider('get_source_deltas', test_get_source_deltas, 'minimal'), - create_provider('get_source_deltas', test_get_source_deltas, 'mainnet'), - create_provider('get_target_deltas', test_get_target_deltas, 'minimal'), - create_provider('get_target_deltas', test_get_target_deltas, 'mainnet'), - create_provider('get_head_deltas', test_get_head_deltas, 'minimal'), - create_provider('get_head_deltas', test_get_head_deltas, 'mainnet'), - create_provider('get_inclusion_delay_deltas', test_get_inclusion_delay_deltas, 'minimal'), - create_provider('get_inclusion_delay_deltas', test_get_inclusion_delay_deltas, 'mainnet'), - create_provider('get_inactivity_penalty_deltas', test_get_inactivity_penalty_deltas, 'minimal'), - create_provider('get_inactivity_penalty_deltas', test_get_inactivity_penalty_deltas, 'mainnet'), + create_provider('get_deltas', test_basic, 'minimal'), + create_provider('get_deltas', test_basic, 'mainnet'), + create_provider('get_deltas', test_leak, 'minimal'), + create_provider('get_deltas', test_leak, 'mainnet'), + create_provider('get_deltas', test_random, 'minimal'), + create_provider('get_deltas', test_random, 'mainnet'), ]) From 0f20d8a9ba9da9ada7c2fee0a7d7cfa08ff6ba64 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 19 May 2020 01:55:17 +0200 Subject: [PATCH 62/70] leak state decorator, and test pre-state caching --- tests/core/pyspec/eth2spec/test/context.py | 53 ++++++++++------ .../test/phase_0/rewards/test_leak.py | 60 +++++++++++++------ 2 files changed, 78 insertions(+), 35 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 1a182fd31..303f680fb 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -9,6 +9,8 @@ from .utils import vector_test, with_meta_tags from random import Random from typing import Any, Callable, NewType, Sequence, TypedDict, Protocol +from lru import LRU + from importlib import reload @@ -48,28 +50,45 @@ class SpecForks(TypedDict, total=False): PHASE1: SpecPhase1 +def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Callable[[Any], int], + spec: Spec, phases: SpecForks): + + p0 = phases[PHASE0] + balances = balances_fn(p0) + activation_threshold = threshold_fn(p0) + + state = create_genesis_state(spec=p0, validator_balances=balances, + activation_threshold=activation_threshold) + if spec.fork == PHASE1: + # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper. + # Decide based on performance/consistency results later. + state = phases[PHASE1].upgrade_to_phase1(state) + # Shard state slot must lag behind BeaconState slot by at least 1 + # Will handle this more elegantly with fork mechanics + spec.process_slots(state, state.slot + 1) + + return state + + +_custom_state_cache_dict = LRU(size=10) + + def with_custom_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Callable[[Any], int]): def deco(fn): + def entry(*args, spec: Spec, phases: SpecForks, **kw): - try: - p0 = phases[PHASE0] - balances = balances_fn(p0) - activation_threshold = threshold_fn(p0) + # Use fork and file path to make a key for th + key = (spec.fork, spec.__file__, balances_fn, threshold_fn) + global _custom_state_cache_dict + if key not in _custom_state_cache_dict: + state = _prepare_state(balances_fn, threshold_fn, spec, phases) + _custom_state_cache_dict[key] = state.get_backing() - state = create_genesis_state(spec=p0, validator_balances=balances, - activation_threshold=activation_threshold) - if spec.fork == PHASE1: - # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper. - # Decide based on performance/consistency results later. - state = phases[PHASE1].upgrade_to_phase1(state) - # Shard state slot must lag behind BeaconState slot by at least 1 - # Will handle this more elegantly with fork mechanics - spec.process_slots(state, state.slot + 1) - - kw['state'] = state - except KeyError: - raise TypeError('Spec decorator must come within state decorator to inject spec into state.') + # Take a copy out of the LRU cache result. + # No copy is necessary, as we wrap the immutable backing with a new view. + state = spec.BeaconState(backing=_custom_state_cache_dict[key]) + kw['state'] = state return fn(*args, spec=spec, phases=phases, **kw) return entry return deco diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py index 562a99b4b..6080ec751 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py @@ -1,6 +1,7 @@ from eth2spec.test.context import with_all_phases, spec_state_test from eth2spec.test.helpers.state import next_epoch import eth2spec.test.helpers.rewards as rewards_helpers +from lru import LRU def transition_state_to_leak(spec, state, epochs=None): @@ -12,80 +13,103 @@ def transition_state_to_leak(spec, state, epochs=None): next_epoch(spec, state) +_cache_dict = LRU(size=10) + + +def leaking(epochs=None): + + def deco(fn): + def entry(*args, spec, state, **kw): + # If the pre-state is not already known in the LRU, then take it, make it leaking, and put it in the LRU. + # The input state is likely already cached, so the hash-tree-root is fine. + key = (state.hash_tree_root(), spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY, spec.SLOTS_PER_EPOCH, epochs) + global _cache_dict + if key not in _cache_dict: + transition_state_to_leak(spec, state, epochs=epochs) + _cache_dict[key] = state.get_backing() + + # Take a copy out of the LRU cache result. + # No copy is necessary, as we wrap the immutable backing with a new view. + state = spec.BeaconState(backing=_cache_dict[key]) + return fn(*args, spec=spec, state=state, **kw) + return entry + return deco + + @with_all_phases @spec_state_test +@leaking() def test_empty_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_empty(spec, state) @with_all_phases @spec_state_test +@leaking() def test_full_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_full_all_correct(spec, state) @with_all_phases @spec_state_test +@leaking() def test_half_full_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_half_full(spec, state) @with_all_phases @spec_state_test +@leaking() def test_quarter_full_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_partial(spec, state, 0.25) @with_all_phases @spec_state_test +@leaking() def test_full_but_partial_participation_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_full_but_partial_participation(spec, state) @with_all_phases @spec_state_test +@leaking() def test_one_attestation_one_correct_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state) @with_all_phases @spec_state_test +@leaking() def test_with_not_yet_activated_validators_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state) @with_all_phases @spec_state_test +@leaking() def test_with_exited_validators_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_with_exited_validators(spec, state) @with_all_phases @spec_state_test +@leaking() def test_with_slashed_validators_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_with_slashed_validators(spec, state) @with_all_phases @spec_state_test +@leaking() def test_some_very_low_effective_balances_that_attested_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested(spec, state) @with_all_phases @spec_state_test +@leaking() def test_some_very_low_effective_balances_that_did_not_attest_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest(spec, state) @@ -98,8 +122,8 @@ def test_some_very_low_effective_balances_that_did_not_attest_leak(spec, state): @with_all_phases @spec_state_test +@leaking() def test_full_half_correct_target_incorrect_head_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, correct_target=True, @@ -110,8 +134,8 @@ def test_full_half_correct_target_incorrect_head_leak(spec, state): @with_all_phases @spec_state_test +@leaking() def test_full_correct_target_incorrect_head_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, correct_target=True, @@ -122,8 +146,8 @@ def test_full_correct_target_incorrect_head_leak(spec, state): @with_all_phases @spec_state_test +@leaking() def test_full_half_incorrect_target_incorrect_head_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, correct_target=False, @@ -134,8 +158,8 @@ def test_full_half_incorrect_target_incorrect_head_leak(spec, state): @with_all_phases @spec_state_test +@leaking() def test_full_half_incorrect_target_correct_head_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, correct_target=False, @@ -146,20 +170,20 @@ def test_full_half_incorrect_target_correct_head_leak(spec, state): @with_all_phases @spec_state_test +@leaking() def test_full_random_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_full_random(spec, state) @with_all_phases @spec_state_test +@leaking(epochs=5) def test_full_random_five_epoch_leak(spec, state): - transition_state_to_leak(spec, state, epochs=5) yield from rewards_helpers.run_test_full_random(spec, state) @with_all_phases @spec_state_test +@leaking(epochs=10) def test_full_random_ten_epoch_leak(spec, state): - transition_state_to_leak(spec, state, epochs=10) yield from rewards_helpers.run_test_full_random(spec, state) From 75a0d60eb3339d457bfc70652cc6cacc8010461f Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 19 May 2020 02:25:32 +0200 Subject: [PATCH 63/70] cached epoch attestation preparation --- .../eth2spec/test/helpers/attestations.py | 21 +++++++++++ .../pyspec/eth2spec/test/helpers/rewards.py | 37 +++++++++---------- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index e4be6a521..8e0501e4a 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -6,6 +6,7 @@ from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.ssz.ssz_typing import Bitlist +from lru import LRU def run_attestation_processing(spec, state, attestation, valid=True): @@ -373,6 +374,26 @@ def prepare_state_with_attestations(spec, state, participation_fn=None): return attestations +_prep_state_cache_dict = LRU(size=10) + + +def cached_prepare_state_with_attestations(spec, state): + """ + Cached version of prepare_state_with_attestations, + but does not return anything, and does not support a participation fn argument + """ + # If the pre-state is not already known in the LRU, then take it, make it leaking, and put it in the LRU. + # The input state is likely already cached, so the hash-tree-root is fine. + key = (spec.fork, state.hash_tree_root()) + global _prep_state_cache_dict + if key not in _prep_state_cache_dict: + prepare_state_with_attestations(spec, state) + _prep_state_cache_dict[key] = state.get_backing() + + # Put the LRU cache result into the state view, as if we transitioned the original view + state.set_backing(_prep_state_cache_dict[key]) + + def fill_block_shard_transitions_by_attestations(spec, state, block): block.body.shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS for attestation in block.body.attestations: diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index fc9f1f93a..42e7a7614 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -1,7 +1,7 @@ from random import Random from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.test.helpers.attestations import prepare_state_with_attestations +from eth2spec.test.helpers.attestations import cached_prepare_state_with_attestations from eth2spec.test.helpers.deposits import mock_deposit from eth2spec.test.helpers.state import next_epoch from eth2spec.utils.ssz.ssz_typing import Container, uint64, List @@ -217,13 +217,13 @@ def run_test_empty(spec, state): def run_test_full_all_correct(spec, state): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) yield from run_deltas(spec, state) def run_test_full_but_partial_participation(spec, state, rng=Random(5522)): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) for a in state.previous_epoch_attestations: a.aggregation_bits = [rng.choice([True, False]) for _ in a.aggregation_bits] @@ -232,7 +232,7 @@ def run_test_full_but_partial_participation(spec, state, rng=Random(5522)): def run_test_partial(spec, state, fraction_filled): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) # Remove portion of attestations num_attestations = int(len(state.previous_epoch_attestations) * fraction_filled) @@ -246,7 +246,7 @@ def run_test_half_full(spec, state): def run_test_one_attestation_one_correct(spec, state): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) # Remove all attestations except for the first one state.previous_epoch_attestations = state.previous_epoch_attestations[:1] @@ -256,14 +256,14 @@ def run_test_one_attestation_one_correct(spec, state): def run_test_with_not_yet_activated_validators(spec, state, rng=Random(5555)): set_some_new_deposits(spec, state, rng) - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) yield from run_deltas(spec, state) def run_test_with_exited_validators(spec, state, rng=Random(1337)): exit_random_validators(spec, state, rng) - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) yield from run_deltas(spec, state) @@ -272,14 +272,13 @@ def run_test_with_slashed_validators(spec, state, rng=Random(3322)): exit_random_validators(spec, state, rng) slash_random_validators(spec, state, rng) - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) yield from run_deltas(spec, state) def run_test_some_very_low_effective_balances_that_attested(spec, state): - state.balances - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) # Set some balances to be very low (including 0) assert len(state.validators) >= 5 @@ -290,7 +289,7 @@ def run_test_some_very_low_effective_balances_that_attested(spec, state): def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) # Remove attestation attestation = state.previous_epoch_attestations[0] @@ -304,7 +303,7 @@ def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state): def run_test_full_fraction_incorrect(spec, state, correct_target, correct_head, fraction_incorrect): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) # Make fraction_incorrect of pending attestations have bad target/head as specified num_incorrect = int(fraction_incorrect * len(state.previous_epoch_attestations)) @@ -318,7 +317,7 @@ def run_test_full_fraction_incorrect(spec, state, correct_target, correct_head, def run_test_full_delay_one_slot(spec, state): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) for a in state.previous_epoch_attestations: a.inclusion_delay += 1 @@ -326,7 +325,7 @@ def run_test_full_delay_one_slot(spec, state): def run_test_full_delay_max_slots(spec, state): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) for a in state.previous_epoch_attestations: a.inclusion_delay += spec.SLOTS_PER_EPOCH @@ -334,7 +333,7 @@ def run_test_full_delay_max_slots(spec, state): def run_test_full_mixed_delay(spec, state, rng=Random(1234)): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) for a in state.previous_epoch_attestations: a.inclusion_delay = rng.randint(1, spec.SLOTS_PER_EPOCH) @@ -342,7 +341,7 @@ def run_test_full_mixed_delay(spec, state, rng=Random(1234)): def run_test_proposer_not_in_attestations(spec, state): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) # Get an attestation where the proposer is not in the committee non_proposer_attestations = [] @@ -357,7 +356,7 @@ def run_test_proposer_not_in_attestations(spec, state): def run_test_duplicate_attestations_at_later_slots(spec, state): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) # Remove 2/3 of attestations to make it more interesting num_attestations = int(len(state.previous_epoch_attestations) * 0.33) @@ -391,7 +390,7 @@ def run_test_duplicate_attestations_at_later_slots(spec, state): def run_test_all_balances_too_low_for_reward(spec, state): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) for index in range(len(state.validators)): state.validators[index].effective_balance = 10 @@ -404,7 +403,7 @@ def run_test_full_random(spec, state, rng=Random(8020)): exit_random_validators(spec, state, rng) slash_random_validators(spec, state, rng) - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) for pending_attestation in state.previous_epoch_attestations: # ~1/3 have bad target From 97e54b85d6c4c6663f97e4a9e3d51b10c1945300 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 19 May 2020 02:46:27 +0200 Subject: [PATCH 64/70] fix cache for config change during runtime --- tests/core/pyspec/eth2spec/test/context.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 303f680fb..93b474517 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -78,8 +78,9 @@ def with_custom_state(balances_fn: Callable[[Any], Sequence[int]], def deco(fn): def entry(*args, spec: Spec, phases: SpecForks, **kw): - # Use fork and file path to make a key for th - key = (spec.fork, spec.__file__, balances_fn, threshold_fn) + # make a key for the state + # genesis fork version separates configs during test-generation runtime. + key = (spec.fork, spec.GENESIS_FORK_VERSION, spec.__file__, balances_fn, threshold_fn) global _custom_state_cache_dict if key not in _custom_state_cache_dict: state = _prepare_state(balances_fn, threshold_fn, spec, phases) From 85f6712363802b6b1f4a1a9464f8847f02a57fb1 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 19 May 2020 03:42:58 +0200 Subject: [PATCH 65/70] fix comments based on PR feedback --- tests/core/pyspec/eth2spec/test/context.py | 2 +- tests/core/pyspec/eth2spec/test/helpers/attestations.py | 7 ++++--- .../pyspec/eth2spec/test/phase_0/rewards/test_leak.py | 9 +++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 93b474517..20214908e 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -86,7 +86,7 @@ def with_custom_state(balances_fn: Callable[[Any], Sequence[int]], state = _prepare_state(balances_fn, threshold_fn, spec, phases) _custom_state_cache_dict[key] = state.get_backing() - # Take a copy out of the LRU cache result. + # Take an entry out of the LRU. # No copy is necessary, as we wrap the immutable backing with a new view. state = spec.BeaconState(backing=_custom_state_cache_dict[key]) kw['state'] = state diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 8e0501e4a..79f752411 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -382,13 +382,14 @@ def cached_prepare_state_with_attestations(spec, state): Cached version of prepare_state_with_attestations, but does not return anything, and does not support a participation fn argument """ - # If the pre-state is not already known in the LRU, then take it, make it leaking, and put it in the LRU. - # The input state is likely already cached, so the hash-tree-root is fine. + # If the pre-state is not already known in the LRU, then take it, + # prepare it with attestations, and put it in the LRU. + # The input state is likely already cached, so the hash-tree-root does not affect speed. key = (spec.fork, state.hash_tree_root()) global _prep_state_cache_dict if key not in _prep_state_cache_dict: prepare_state_with_attestations(spec, state) - _prep_state_cache_dict[key] = state.get_backing() + _prep_state_cache_dict[key] = state.get_backing() # cache the tree structure, not the view wrapping it. # Put the LRU cache result into the state view, as if we transitioned the original view state.set_backing(_prep_state_cache_dict[key]) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py index 6080ec751..4e75079c0 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py @@ -20,15 +20,16 @@ def leaking(epochs=None): def deco(fn): def entry(*args, spec, state, **kw): - # If the pre-state is not already known in the LRU, then take it, make it leaking, and put it in the LRU. - # The input state is likely already cached, so the hash-tree-root is fine. + # If the pre-state is not already known in the LRU, then take it, + # transition it to leak, and put it in the LRU. + # The input state is likely already cached, so the hash-tree-root does not affect speed. key = (state.hash_tree_root(), spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY, spec.SLOTS_PER_EPOCH, epochs) global _cache_dict if key not in _cache_dict: transition_state_to_leak(spec, state, epochs=epochs) - _cache_dict[key] = state.get_backing() + _cache_dict[key] = state.get_backing() # cache the tree structure, not the view wrapping it. - # Take a copy out of the LRU cache result. + # Take an entry out of the LRU. # No copy is necessary, as we wrap the immutable backing with a new view. state = spec.BeaconState(backing=_cache_dict[key]) return fn(*args, spec=spec, state=state, **kw) From 6d45afeefbf5f76ea61eb0f4d71e90402c3744c8 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 May 2020 19:53:46 -0600 Subject: [PATCH 66/70] add some more random reawrds tests --- .../pyspec/eth2spec/test/helpers/rewards.py | 3 +++ .../test/phase_0/rewards/test_random.py | 25 ++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 42e7a7614..d62fee6ce 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -174,6 +174,9 @@ def set_some_new_deposits(spec, state, rng): num_validators = len(state.validators) # Set ~1/10 to just recently deposited for index in range(num_validators): + # If not already active, skip + if not spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)): + continue if rng.randrange(num_validators) < num_validators // 10: mock_deposit(spec, state, index) # Set ~half of selected to eligible for activation diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py index bda0ca687..83c7f7905 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py @@ -1,6 +1,13 @@ from random import Random -from eth2spec.test.context import with_all_phases, spec_state_test +from eth2spec.test.context import ( + with_all_phases, + spec_test, + spec_state_test, + with_custom_state, + single_phase, + low_balances, misc_balances, +) import eth2spec.test.helpers.rewards as rewards_helpers @@ -20,3 +27,19 @@ def test_full_random_1(spec, state): @spec_state_test def test_full_random_2(spec, state): yield from rewards_helpers.run_test_full_random(spec, state, rng=Random(3030)) + + +@with_all_phases +@with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@spec_test +@single_phase +def test_full_random_low_balances(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state) + + +@with_all_phases +@with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@spec_test +@single_phase +def test_full_random_misc_balances(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state) From 5c564f4d7c9463e553641fc1792f879a2aadecb5 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 May 2020 20:06:36 -0600 Subject: [PATCH 67/70] clean up reards generator --- tests/generators/rewards/main.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index d8dae74fa..c90943cab 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -14,7 +14,7 @@ from eth2spec.config import config_util from eth2spec.test.context import PHASE0 -def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: +def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) @@ -25,7 +25,7 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin def cases_fn() -> Iterable[gen_typing.TestCase]: return generate_from_tests( runner_name='rewards', - handler_name=handler_name, + handler_name='core', src=tests_src, fork_name=PHASE0, ) @@ -34,11 +34,11 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin if __name__ == "__main__": - gen_runner.run_generator("epoch_processing", [ - create_provider('get_deltas', test_basic, 'minimal'), - create_provider('get_deltas', test_basic, 'mainnet'), - create_provider('get_deltas', test_leak, 'minimal'), - create_provider('get_deltas', test_leak, 'mainnet'), - create_provider('get_deltas', test_random, 'minimal'), - create_provider('get_deltas', test_random, 'mainnet'), + gen_runner.run_generator("rewards", [ + create_provider(test_basic, 'minimal'), + create_provider(test_basic, 'mainnet'), + create_provider(test_leak, 'minimal'), + create_provider(test_leak, 'mainnet'), + create_provider(test_random, 'minimal'), + create_provider(test_random, 'mainnet'), ]) From 65b5aa3c636234d024c89c3f3a9df87700b5258a Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 19 May 2020 14:31:28 +0200 Subject: [PATCH 68/70] update test format docs --- tests/formats/epoch_processing/README.md | 2 +- tests/formats/rewards/README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/formats/epoch_processing/README.md b/tests/formats/epoch_processing/README.md index 7c5e2dc70..57c9441c8 100644 --- a/tests/formats/epoch_processing/README.md +++ b/tests/formats/epoch_processing/README.md @@ -38,7 +38,7 @@ The provided pre-state is already transitioned to just before the specific sub-t Sub-transitions: - `justification_and_finalization` -- *`rewards_and_penalties` - planned testing extension* +- `rewards_and_penalties` (limited to `minimal` config) - `registry_updates` - `slashings` - `final_updates` diff --git a/tests/formats/rewards/README.md b/tests/formats/rewards/README.md index b00e7a32a..b229d9f98 100644 --- a/tests/formats/rewards/README.md +++ b/tests/formats/rewards/README.md @@ -7,8 +7,8 @@ There is no "change" factor, the rewards/penalties outputs are pure functions wi `Deltas` is defined as: ```python class Deltas(Container): - rewards: List[uint64, VALIDATOR_REGISTRY_LIMIT] - penalties: List[uint64, VALIDATOR_REGISTRY_LIMIT] + rewards: List[Gwei, VALIDATOR_REGISTRY_LIMIT] + penalties: List[Gwei, VALIDATOR_REGISTRY_LIMIT] ``` ## Test case format From 724139a1f81526648dd0055d4ab4bdfa34796477 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 May 2020 10:23:12 -0600 Subject: [PATCH 69/70] change gossipsub protocol ID to /meshsub/1.1.0 --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 01d905a38..c8893ead9 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -178,7 +178,7 @@ Where Clients MUST support the [gossipsub v1](https://github.com/libp2p/specs/tree/master/pubsub/gossipsub) libp2p protocol including the [gossipsub v1.1](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.1.md) extension. -**Protocol ID:** `/meshsub/1.0.0` +**Protocol ID:** `/meshsub/1.1.0` **Gossipsub Parameters** From fdc7e846dfbffe172dd17805550a4980e16e7a36 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 May 2020 10:25:43 -0600 Subject: [PATCH 70/70] remove incorrect table format --- specs/phase0/p2p-interface.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index c8893ead9..0e8699555 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -218,8 +218,6 @@ where `base64` is the [URL-safe base64 alphabet](https://tools.ietf.org/html/rfc The payload is carried in the `data` field of a gossipsub message, and varies depending on the topic: -| Name | Message Type | -|------------------------------------------------|-------------------------| | Name | Message Type | |----------------------------------|---------------------------| | `beacon_block` | `SignedBeaconBlock` | @@ -552,7 +550,7 @@ Clients MUST order blocks by increasing slot number. Clients MUST respond with blocks from their view of the current fork choice -- that is, blocks from the single chain defined by the current head. Of note, blocks from slots before the finalization MUST lead to the finalized block reported in the `Status` handshake. -Clients MUST respond with blocks that are consistent from a single chain within the context of the request. After the initial block clients MAY stop in the process of responding, if their fork choice changes the view of the chain in the context of the request. +Clients MUST respond with blocks that are consistent from a single chain within the context of the request. After the initial block, clients MAY stop in the process of responding if their fork choice changes the view of the chain in the context of the request. #### BeaconBlocksByRoot