From 4c90c357b728b4e486f29cb4ed169d4652ec52a0 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 8 Oct 2020 20:34:59 +0200 Subject: [PATCH 001/222] remove yaml duplicates, compress SSZ test outputs --- tests/core/gen_helpers/gen_base/gen_runner.py | 6 ++++-- tests/core/gen_helpers/requirements.txt | 1 + tests/core/gen_helpers/setup.py | 1 + tests/core/pyspec/eth2spec/test/utils.py | 6 +----- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/core/gen_helpers/gen_base/gen_runner.py b/tests/core/gen_helpers/gen_base/gen_runner.py index a22073c00..1cf8ba8cf 100644 --- a/tests/core/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/gen_helpers/gen_base/gen_runner.py @@ -9,6 +9,7 @@ from ruamel.yaml import ( ) from gen_base.gen_typing import TestProvider +from snappy import compress from eth2spec.test import context from eth2spec.test.exceptions import SkippedTest @@ -180,7 +181,8 @@ def dump_yaml_fn(data: Any, name: str, file_mode: str, yaml_encoder: YAML): def dump_ssz_fn(data: AnyStr, name: str, file_mode: str): def dump(case_path: Path): - out_path = case_path / Path(name + '.ssz') + out_path = case_path / Path(name + '.ssz_snappy') + compressed = compress(data) with out_path.open(file_mode + 'b') as f: # write in raw binary mode - f.write(data) + f.write(compressed) return dump diff --git a/tests/core/gen_helpers/requirements.txt b/tests/core/gen_helpers/requirements.txt index e7cdd30ea..a2c194dea 100644 --- a/tests/core/gen_helpers/requirements.txt +++ b/tests/core/gen_helpers/requirements.txt @@ -1,3 +1,4 @@ ruamel.yaml==0.16.5 eth-utils==1.6.0 pytest>=4.4 +python-snappy==0.5.4 diff --git a/tests/core/gen_helpers/setup.py b/tests/core/gen_helpers/setup.py index e9fc1a787..d26530642 100644 --- a/tests/core/gen_helpers/setup.py +++ b/tests/core/gen_helpers/setup.py @@ -7,5 +7,6 @@ setup( "ruamel.yaml==0.16.5", "eth-utils==1.6.0", "pytest>=4.4", + "python-snappy==0.5.4", ] ) diff --git a/tests/core/pyspec/eth2spec/test/utils.py b/tests/core/pyspec/eth2spec/test/utils.py index 12ff68443..306f7d892 100644 --- a/tests/core/pyspec/eth2spec/test/utils.py +++ b/tests/core/pyspec/eth2spec/test/utils.py @@ -39,24 +39,20 @@ def vector_test(description: str = None): if value is None: continue if isinstance(value, View): - yield key, 'data', encode(value) yield key, 'ssz', serialize(value) elif isinstance(value, bytes): - yield key, 'data', encode(value) yield key, 'ssz', value elif isinstance(value, list) and all([isinstance(el, (View, bytes)) for el in value]): for i, el in enumerate(value): if isinstance(el, View): - yield f'{key}_{i}', 'data', encode(el) yield f'{key}_{i}', 'ssz', serialize(el) elif isinstance(el, bytes): - yield f'{key}_{i}', 'data', encode(el) yield f'{key}_{i}', 'ssz', el yield f'{key}_count', 'meta', len(value) else: # Not a ssz value. # The data will now just be yielded as any python data, - # something that should be encodeable by the generator runner. + # something that should be encodable by the generator runner. yield key, 'data', value # check generator mode, may be None/else. From 8bbc5f7fc3967fe2d65680c6450497795aab0b67 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 8 Oct 2020 21:02:18 +0200 Subject: [PATCH 002/222] update test format docs --- tests/formats/README.md | 28 ++++++++++-------- tests/formats/epoch_processing/README.md | 13 +++------ tests/formats/finality/README.md | 14 ++++----- tests/formats/genesis/initialization.md | 17 ++++------- tests/formats/genesis/validity.md | 5 ++-- tests/formats/operations/README.md | 18 ++++-------- tests/formats/rewards/README.md | 36 ++++++++---------------- tests/formats/sanity/blocks.md | 18 ++++-------- tests/formats/sanity/slots.md | 4 +-- tests/formats/ssz_generic/README.md | 8 +++--- tests/formats/ssz_static/core.md | 8 +++--- 11 files changed, 68 insertions(+), 101 deletions(-) diff --git a/tests/formats/README.md b/tests/formats/README.md index 36a5ec21b..d3933abdd 100644 --- a/tests/formats/README.md +++ b/tests/formats/README.md @@ -132,21 +132,25 @@ Cases are split up too. This enables diffing of parts of the test case, tracking ### `` -E.g. `pre.yaml`, `deposit.yaml`, `post.yaml`. - -Diffing a `pre.yaml` and `post.yaml` provides all the information for testing, good for readability of the change. -Then the difference between pre and post can be compared to anything that changes the pre state, e.g. `deposit.yaml` - These files allow for custom formats for some parts of the test. E.g. something encoded in SSZ. +Or to avoid large files, the SSZ can be compressed with Snappy. +E.g. `pre.ssz_snappy_snappy`, `deposit.ssz_snappy_snappy`, `post.ssz_snappy_snappy`. -Some yaml files have copies, but formatted as raw SSZ bytes: `pre.ssz`, `deposit.ssz`, `post.ssz`. -The yaml files are intended to be deprecated, and clients should shift to ssz inputs for efficiency. -Deprecation will start once a viewer of SSZ test-cases is in place, to maintain a standard of readable test cases. -This also means that some clients can drop legacy YAML -> JSON/other -> SSZ work-arounds. -(These were implemented to support the uint64 YAML, hex strings, etc. Things that were not idiomatic to their language.) +Diffing a `pre.ssz_snappy_snappy` and `post.ssz_snappy_snappy` provides all the information for testing, when decompressed and decoded. +Then the difference between pre and post can be compared to anything that changes the pre state, e.g. `deposit.ssz_snappy_snappy` + +YAML is generally used for test metadata, and for tests that do not use SSZ: e.g. shuffling and BLS tests. +In this case, there is no point in adding special SSZ types. And the size and efficiency of YAML is acceptable. + +#### Common output formats + +Between all types of tests, a few formats are common: + +- **`.yaml`**: A YAML file containing structured data to describe settings or test contents. +- **`.ssz_snappy`**: A file containing raw SSZ-encoded data. Previously widely used in tests, but replaced with compressed variant. +- **`.ssz_snappy_snappy`**: Like `.ssz_snappy`, but compressed with Snappy block compression. + Snappy block compression is already applied to SSZ in Eth2 gossip, available in client implementations, and thus chosen as compression method. -Yaml will not be deprecated for tests that do not use SSZ: e.g. shuffling and BLS tests. -In this case, there is no work around for loading necessary anyway, and the size and efficiency of yaml is acceptable. #### Special output parts diff --git a/tests/formats/epoch_processing/README.md b/tests/formats/epoch_processing/README.md index 57c9441c8..bc15e64b6 100644 --- a/tests/formats/epoch_processing/README.md +++ b/tests/formats/epoch_processing/README.md @@ -15,18 +15,13 @@ description: string -- Optional description of test case, purely for debuggin bls_setting: int -- see general test-format spec. ``` -### `pre.yaml` +### `pre.ssz_snappy` -A YAML-encoded `BeaconState`, the state before running the epoch sub-transition. +A SSZ-snappy encoded `BeaconState`, the state before running the epoch sub-transition. -Also available as `pre.ssz`. +### `post.ssz_snappy` - -### `post.yaml` - -A YAML-encoded `BeaconState`, the state after applying the epoch sub-transition. - -Also available as `post.ssz`. +A SSZ-snappy encoded `BeaconState`, the state after applying the epoch sub-transition. ## Condition diff --git a/tests/formats/finality/README.md b/tests/formats/finality/README.md index da9108a6a..2d279b441 100644 --- a/tests/formats/finality/README.md +++ b/tests/formats/finality/README.md @@ -14,11 +14,11 @@ bls_setting: int -- see general test-format spec. blocks_count: int -- the number of blocks processed in this test. ``` -### `pre.yaml` +### `pre.ssz_snappy` -A YAML-encoded `BeaconState`, the state before running the block transitions. +A SSZ-snappy encoded `BeaconState`, the state before running the block transitions. -Also available as `pre.ssz`. +Also available as `pre.ssz_snappy`. ### `blocks_.yaml` @@ -28,13 +28,11 @@ A series of files, with `` in range `[0, blocks_count)`. Blocks need to b Each file is a YAML-encoded `SignedBeaconBlock`. -Each block is also available as `blocks_.ssz` +Each block is also available as `blocks_.ssz_snappy` -### `post.yaml` +### `post.ssz_snappy` -A YAML-encoded `BeaconState`, the state after applying the block transitions. - -Also available as `post.ssz`. +A SSZ-snappy encoded `BeaconState`, the state after applying the block transitions. ## Condition diff --git a/tests/formats/genesis/initialization.md b/tests/formats/genesis/initialization.md index 428abb5bd..2e5452821 100644 --- a/tests/formats/genesis/initialization.md +++ b/tests/formats/genesis/initialization.md @@ -4,11 +4,9 @@ Tests the initialization of a genesis state based on Eth1 data. ## Test case format -### `eth1_block_hash.yaml` +### `eth1_block_hash.ssz_snappy` -A `Bytes32` hex encoded, with prefix 0x. The root of the Eth1 block. - -Also available as `eth1_block_hash.ssz`. +A SSZ-snappy encoded root of the Eth1 block. ### `eth1_timestamp.yaml` @@ -22,18 +20,15 @@ A yaml file to help read the deposit count: deposits_count: int -- Amount of deposits. ``` -### `deposits_.yaml` +### `deposits_.ssz_snappy` A series of files, with `` in range `[0, deposits_count)`. Deposits need to be processed in order. -Each file is a YAML-encoded `Deposit` object. +Each file is a SSZ-snappy encoded `Deposit` object. -Each deposit is also available as `deposits_.ssz`. +### `state.ssz_snappy` -### `state.yaml` +The expected genesis state. A SSZ-snappy encoded `BeaconState` object. -The expected genesis state. A YAML-encoded `BeaconState` object. - -Also available as `state.ssz`. ## Processing diff --git a/tests/formats/genesis/validity.md b/tests/formats/genesis/validity.md index 38f2b1b1f..1b3f79879 100644 --- a/tests/formats/genesis/validity.md +++ b/tests/formats/genesis/validity.md @@ -4,11 +4,10 @@ Tests if a genesis state is valid, i.e. if it counts as trigger to launch. ## Test case format -### `genesis.yaml` +### `genesis.ssz_snappy` -A `BeaconState`, the state to validate as genesis candidate. +A SSZ-snappy encoded `BeaconState`, the state to validate as genesis candidate. -Also available as `genesis.ssz`. ### `is_valid.yaml` diff --git a/tests/formats/operations/README.md b/tests/formats/operations/README.md index bb4636ec0..ca77ab966 100644 --- a/tests/formats/operations/README.md +++ b/tests/formats/operations/README.md @@ -12,23 +12,17 @@ description: string -- Optional description of test case, purely for debuggin bls_setting: int -- see general test-format spec. ``` -### `pre.yaml` +### `pre.ssz_snappy` -A YAML-encoded `BeaconState`, the state before applying the operation. +A SSZ-snappy encoded `BeaconState`, the state before applying the operation. -Also available as `pre.ssz`. +### `.ssz_snappy` -### `.yaml` +A SSZ-snappy encoded operation object, e.g. a `ProposerSlashing`, or `Deposit`. -A YAML-encoded operation object, e.g. a `ProposerSlashing`, or `Deposit`. +### `post.ssz_snappy` -Also available as `.ssz`. - -### `post.yaml` - -A YAML-encoded `BeaconState`, the state after applying the operation. No value if operation processing is aborted. - -Also available as `post.ssz`. +A SSZ-snappy encoded `BeaconState`, the state after applying the operation. No value if operation processing is aborted. ## Condition diff --git a/tests/formats/rewards/README.md b/tests/formats/rewards/README.md index b229d9f98..aee23c3e9 100644 --- a/tests/formats/rewards/README.md +++ b/tests/formats/rewards/README.md @@ -23,41 +23,29 @@ description: string -- Optional description of test case, purely for debuggin _Note_: No signature verification happens within rewards sub-functions. These tests can safely be run with or without BLS enabled. -### `pre.yaml` +### `pre.ssz_snappy` -A YAML-encoded `BeaconState`, the state before running the rewards sub-function. +A SSZ-snappy encoded `BeaconState`, the state before running the rewards sub-function. -Also available as `pre.ssz`. +### `source_deltas.ssz_snappy` -### `source_deltas.yaml` +A SSZ-snappy encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_source_deltas` function -A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_source_deltas` function +### `target_deltas.ssz_snappy` -Also available as `source_deltas.ssz`. +A SSZ-snappy encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_target_deltas` function -### `target_deltas.yaml` +### `head_deltas.ssz_snappy` -A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_target_deltas` function +A SSZ-snappy encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_head_deltas` function -Also available as `target_deltas.ssz`. +### `inclusion_delay_deltas.ssz_snappy` -### `head_deltas.yaml` +A SSZ-snappy encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_inclusion_delay_deltas` function -A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_head_deltas` function +### `inactivity_penalty_deltas.ssz_snappy` -Also available as `head_deltas.ssz`. - -### `inclusion_delay_deltas.yaml` - -A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_inclusion_delay_deltas` function - -Also available as `inclusion_delay_deltas.ssz`. - -### `inactivity_penalty_deltas.yaml` - -A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_inactivity_penalty_deltas` function - -Also available as `inactivity_penalty_deltas.ssz`. +A SSZ-snappy encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_inactivity_penalty_deltas` function ## Condition diff --git a/tests/formats/sanity/blocks.md b/tests/formats/sanity/blocks.md index 44b37ed5e..a8b38ccae 100644 --- a/tests/formats/sanity/blocks.md +++ b/tests/formats/sanity/blocks.md @@ -14,27 +14,21 @@ blocks_count: int -- the number of blocks processed in this test. ``` -### `pre.yaml` +### `pre.ssz_snappy` -A YAML-encoded `BeaconState`, the state before running the block transitions. - -Also available as `pre.ssz`. +A SSZ-snappy encoded `BeaconState`, the state before running the block transitions. -### `blocks_.yaml` +### `blocks_.ssz_snappy` A series of files, with `` in range `[0, blocks_count)`. Blocks need to be processed in order, following the main transition function (i.e. process slot and epoch transitions in between blocks as normal) -Each file is a YAML-encoded `SignedBeaconBlock`. +Each file is a SSZ-snappy encoded `SignedBeaconBlock`. -Each block is also available as `blocks_.ssz` +### `post.ssz_snappy` -### `post.yaml` - -A YAML-encoded `BeaconState`, the state after applying the block transitions. - -Also available as `post.ssz`. +A SSZ-snappy encoded `BeaconState`, the state after applying the block transitions. ## Condition diff --git a/tests/formats/sanity/slots.md b/tests/formats/sanity/slots.md index 353287ee2..72a24da51 100644 --- a/tests/formats/sanity/slots.md +++ b/tests/formats/sanity/slots.md @@ -16,7 +16,7 @@ bls_setting: int -- see general test-format spec. A YAML-encoded `BeaconState`, the state before running the transitions. -Also available as `pre.ssz`. +Also available as `pre.ssz_snappy`. ### `slots.yaml` @@ -27,7 +27,7 @@ An integer. The amount of slots to process (i.e. the difference in slots between A YAML-encoded `BeaconState`, the state after applying the transitions. -Also available as `post.ssz`. +Also available as `post.ssz_snappy`. ### Processing diff --git a/tests/formats/ssz_generic/README.md b/tests/formats/ssz_generic/README.md index 68bdbc15f..85a507985 100644 --- a/tests/formats/ssz_generic/README.md +++ b/tests/formats/ssz_generic/README.md @@ -33,7 +33,7 @@ Each of the handlers encodes the SSZ type declaration in the file-name. See [Typ ### `valid` -Valid has 3 parts: `meta.yaml`, `serialized.ssz`, `value.yaml` +Valid has 3 parts: `meta.yaml`, `serialized.ssz_snappy`, `value.yaml` ### `meta.yaml` @@ -46,9 +46,9 @@ root: Bytes32 -- Hash-tree-root of the object The `Bytes32` is encoded as a string, hexadecimal encoding, prefixed with `0x`. -### `serialized.ssz` +### `serialized.ssz_snappy` -The serialized form of the object, as raw SSZ bytes. +The serialized form of the object, as snappy-compressed SSZ bytes. ### `value.yaml` @@ -64,7 +64,7 @@ The conditions are the same for each type: ## `invalid` -Test cases in the `invalid` suite only include the `serialized.ssz` +Test cases in the `invalid` suite only include the `serialized.ssz_snappy` #### Condition diff --git a/tests/formats/ssz_static/core.md b/tests/formats/ssz_static/core.md index d6f99a32b..09ff04e20 100644 --- a/tests/formats/ssz_static/core.md +++ b/tests/formats/ssz_static/core.md @@ -18,7 +18,7 @@ One can iterate over the handlers, and select the type based on the handler name Suites are then the same format, but each specialized in one randomization mode. Some randomization modes may only produce a single test case (e.g. the all-zeroes case). -The output parts are: `roots.yaml`, `serialized.ssz`, `value.yaml` +The output parts are: `roots.yaml`, `serialized.ssz_snappy`, `value.yaml` ### `roots.yaml` @@ -26,13 +26,13 @@ The output parts are: `roots.yaml`, `serialized.ssz`, `value.yaml` root: bytes32 -- string, hash-tree-root of the value, hex encoded, with prefix 0x ``` -### `serialized.ssz` +### `serialized.ssz_snappy` -The raw encoded bytes. +The SSZ-snappy encoded bytes. ### `value.yaml` -The same value as `serialized.ssz`, represented as YAML. +The same value as `serialized.ssz_snappy`, represented as YAML. ## Condition From 3fb9b155518c68222fcef8af36b2bc7c81c230eb Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 8 Oct 2020 21:17:13 +0200 Subject: [PATCH 003/222] remove unused imports --- tests/core/pyspec/eth2spec/test/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/utils.py b/tests/core/pyspec/eth2spec/test/utils.py index 306f7d892..bad6c867b 100644 --- a/tests/core/pyspec/eth2spec/test/utils.py +++ b/tests/core/pyspec/eth2spec/test/utils.py @@ -1,5 +1,4 @@ from typing import Dict, Any -from eth2spec.debug.encode import encode from eth2spec.utils.ssz.ssz_typing import View from eth2spec.utils.ssz.ssz_impl import serialize From 7c6ede5eac9dee03817c78022874ff3b5c48a806 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Thu, 12 Nov 2020 17:28:05 +0800 Subject: [PATCH 004/222] Added standalone light client patch --- specs/lightclient/beacon-chain.md | 208 ++++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 specs/lightclient/beacon-chain.md diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md new file mode 100644 index 000000000..0eb964266 --- /dev/null +++ b/specs/lightclient/beacon-chain.md @@ -0,0 +1,208 @@ +# Ethereum 2.0 Light Client Support: Beacon Chain Changes + +## Table of contents + +- [Introduction](#introduction) +- [Configuration](#configuration) + - [Domain types](#domain-types) + - [Misc](#misc) +- [Updated containers](#updated-containers) + - [Extended `BeaconBlockBody`](#extended-beaconblockbody) + - [Extended `BeaconState`](#extended-beaconstate) +- [New containers](#new-containers) + - [`CompactCommittee`](#compactcommittee) +- [Helper functions](#helper-functions) + - [Misc](#misc-1) + - [`pack_compact_validator`](#pack_compact_validator) + - [`unpack_compact_validator`](#unpack_compact_validator) + - [`committee_to_compact_committee`](#committee_to_compact_committee) + - [Beacon state accessors](#beacon-state-accessors) + - [`get_light_client_committee`](#get_light_client_committee) + - [Block processing](#block-processing) + - [Light client processing](#light-client-processing) + - [Epoch processing](#epoch-transition) + - [Light client committee updates](#light-client-committee-updates) + +## Introduction + +This is a standalone patch to the ethereum beacon chain that adds light client support. + +## Configuration + +### Misc + +| Name | Value | +| - | - | +| `LIGHT_CLIENT_COMMITTEE_SIZE` | `uint64(2**7)` (= 128) | +| `LIGHT_CLIENT_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | +| `BASE_REWARDS_PER_EPOCH` | 5 | + +### Domain types + +| Name | Value | +| - | - | +| `DOMAIN_LIGHT_CLIENT` | `DomainType('0x82000000')` | + +## Updated containers + +### Extended `BeaconBlockBody` + +```python +class BeaconBlockBody(phase0.BeaconBlockBody): + # Bitfield of participants in this light client signature + light_client_bits: Bitvector[LIGHT_CLIENT_COMMITTEE_SIZE] + light_client_signature: BLSSignature +``` + +### Extended `BeaconState` + +```python +class BeaconState(phase0.BeaconState): + # Compact representations of the light client committee + current_light_committee: CompactCommittee + next_light_committee: CompactCommittee +``` + +## New containers + +### `CompactCommittee` + +```python +class CompactCommittee(Container): + pubkeys: List[BLSPubkey, MAX_VALIDATORS_PER_COMMITTEE] + compact_validators: List[uint64, MAX_VALIDATORS_PER_COMMITTEE] +``` + +## Helper functions + +### Misc + + +#### `pack_compact_validator` + +```python +def pack_compact_validator(index: ValidatorIndex, slashed: bool, balance_in_increments: uint64) -> uint64: + """ + Create a compact validator object representing index, slashed status, and compressed balance. + Takes as input balance-in-increments (// EFFECTIVE_BALANCE_INCREMENT) to preserve symmetry with + the unpacking function. + """ + return (index << 16) + (slashed << 15) + balance_in_increments +``` + +#### `unpack_compact_validator` + +```python +def unpack_compact_validator(compact_validator: uint64) -> Tuple[ValidatorIndex, bool, uint64]: + """ + Return validator index, slashed, balance // EFFECTIVE_BALANCE_INCREMENT + """ + return ( + ValidatorIndex(compact_validator >> 16), + bool((compact_validator >> 15) % 2), + compact_validator & (2**15 - 1), + ) +``` + +#### `committee_to_compact_committee` + +```python +def committee_to_compact_committee(state: BeaconState, committee: Sequence[ValidatorIndex]) -> CompactCommittee: + """ + Given a state and a list of validator indices, outputs the ``CompactCommittee`` representing them. + """ + validators = [state.validators[i] for i in committee] + compact_validators = [ + pack_compact_validator(i, v.slashed, v.effective_balance // EFFECTIVE_BALANCE_INCREMENT) + for i, v in zip(committee, validators) + ] + pubkeys = [v.pubkey for v in validators] + return CompactCommittee(pubkeys=pubkeys, compact_validators=compact_validators) +``` + +### Beacon state accessors + +#### `get_light_client_committee` + +```python +def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: + """ + Return the light client committee of no more than ``LIGHT_CLIENT_COMMITTEE_SIZE`` validators. + """ + source_epoch = (max(epoch // LIGHT_CLIENT_COMMITTEE_PERIOD, 1) - 1) * LIGHT_CLIENT_COMMITTEE_PERIOD + active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) + seed = get_seed(beacon_state, source_epoch, DOMAIN_LIGHT_CLIENT) + return [ + compute_shuffled_index(i, active_validator_indices, seed) + for i in range(min(active_validator_indices, LIGHT_CLIENT_COMMITTEE_SIZE)) + ] +``` + +### Block processing + +```python +def process_block(state: BeaconState, block: BeaconBlock) -> None: + phase0.process_block(state, block) + process_light_client_signature(state, block.body) +``` + +#### Light client processing + +```python +def process_light_client_signature(state: BeaconState, block_body: BeaconBlockBody) -> None: + committee = get_light_client_committee(state, get_current_epoch(state)) + previous_slot = max(state.slot, 1) - 1 + previous_block_root = get_block_root_at_slot(state, previous_slot) + + # Light clients sign over the previous block root + signing_root = compute_signing_root( + previous_block_root, + get_domain(state, DOMAIN_LIGHT_CLIENT, compute_epoch_at_slot(previous_slot)) + ) + + participants = [ + committee[i] for i in range(len(committee)) if block_body.light_client_bits[i] + ] + + signer_pubkeys = [ + state.validators[participant].pubkey for participant in participants + ] + + assert bls.FastAggregateVerify(signer_pubkeys, signing_root, block_body.light_client_signature) + + # Process rewards + total_reward = Gwei(0) + active_validator_count = len(get_active_validator_indices(beacon_state, get_current_epoch(state))) + for participant in participants: + reward = get_base_reward(state, participant) * active_validator_count // len(committee) + increase_balance(state, participant, reward) + total_reward += reward + + increase_balance(state, get_beacon_proposer_index(state), Gwei(total_reward // PROPOSER_REWARD_QUOTIENT)) +``` + +### Epoch processing + +This epoch transition overrides the phase0 epoch transition: + +```python +def process_epoch(state: BeaconState) -> None: + phase0.process_epoch(state) + process_light_client_committee_updates(state) +``` + +#### Light client committee updates + +```python +def process_light_client_committee_updates(state: BeaconState) -> None: + """ + Update light client committees. + """ + next_epoch = compute_epoch_at_slot(Slot(state.slot + 1)) + if next_epoch % LIGHT_CLIENT_COMMITTEE_PERIOD == 0: + state.current_light_committee = state.next_light_committee + new_committee = get_light_client_committee(state, next_epoch + LIGHT_CLIENT_COMMITTEE_PERIOD) + state.next_light_committee = committee_to_compact_committee(state, new_committee) +``` + + From 9e3690ad17adc2a087d39907c7a9830dcbfc020c Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 13 Nov 2020 10:19:37 +0800 Subject: [PATCH 005/222] Update specs/lightclient/beacon-chain.md Co-authored-by: Alex Stokes --- specs/lightclient/beacon-chain.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 0eb964266..77f0b8e0b 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -154,7 +154,7 @@ def process_light_client_signature(state: BeaconState, block_body: BeaconBlockBo previous_slot = max(state.slot, 1) - 1 previous_block_root = get_block_root_at_slot(state, previous_slot) - # Light clients sign over the previous block root + # Light client committees sign over the previous block root signing_root = compute_signing_root( previous_block_root, get_domain(state, DOMAIN_LIGHT_CLIENT, compute_epoch_at_slot(previous_slot)) @@ -205,4 +205,3 @@ def process_light_client_committee_updates(state: BeaconState) -> None: state.next_light_committee = committee_to_compact_committee(state, new_committee) ``` - From 620b812c2e5853bb3b737e736d8266eb522e2130 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 13 Nov 2020 10:21:30 +0800 Subject: [PATCH 006/222] Reduce reward by SLOTS_PER_EPOCH --- specs/lightclient/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 77f0b8e0b..fcf063c04 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -174,7 +174,7 @@ def process_light_client_signature(state: BeaconState, block_body: BeaconBlockBo total_reward = Gwei(0) active_validator_count = len(get_active_validator_indices(beacon_state, get_current_epoch(state))) for participant in participants: - reward = get_base_reward(state, participant) * active_validator_count // len(committee) + reward = get_base_reward(state, participant) * active_validator_count // len(committee) // SLOTS_PER_EPOCH increase_balance(state, participant, reward) total_reward += reward From 555e131e2c3826a8c90ed3fb4f38a0ff94bea9be Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sat, 14 Nov 2020 13:56:00 +0800 Subject: [PATCH 007/222] Small changes to make Justin happy --- specs/lightclient/beacon-chain.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index fcf063c04..5ba1b1f4b 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -33,7 +33,7 @@ This is a standalone patch to the ethereum beacon chain that adds light client s | Name | Value | | - | - | -| `LIGHT_CLIENT_COMMITTEE_SIZE` | `uint64(2**7)` (= 128) | +| `LIGHT_CLIENT_COMMITTEE_SIZE` | `uint64(2**8)` (= 256) | | `LIGHT_CLIENT_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | | `BASE_REWARDS_PER_EPOCH` | 5 | @@ -70,6 +70,7 @@ class BeaconState(phase0.BeaconState): ```python class CompactCommittee(Container): pubkeys: List[BLSPubkey, MAX_VALIDATORS_PER_COMMITTEE] + sum_of_pubkeys: BLSPubkey compact_validators: List[uint64, MAX_VALIDATORS_PER_COMMITTEE] ``` @@ -117,7 +118,11 @@ def committee_to_compact_committee(state: BeaconState, committee: Sequence[Valid for i, v in zip(committee, validators) ] pubkeys = [v.pubkey for v in validators] - return CompactCommittee(pubkeys=pubkeys, compact_validators=compact_validators) + return CompactCommittee( + pubkeys=pubkeys, + sum_of_pubkeys=bls.AggregatePubkeys(pubkeys), + compact_validators=compact_validators + ) ``` ### Beacon state accessors From e7d52d9056ae06b3ee6c34882ea244aa4634535a Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 15 Nov 2020 10:56:24 +0000 Subject: [PATCH 008/222] Significant polishing and a few substantive fixes See discussion for further details. --- specs/lightclient/beacon-chain.md | 246 ++++++++++++++---------------- 1 file changed, 118 insertions(+), 128 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 5ba1b1f4b..4f6e07dd5 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -1,31 +1,47 @@ -# Ethereum 2.0 Light Client Support: Beacon Chain Changes +# Ethereum 2.0 Light Client Support ## Table of contents - [Introduction](#introduction) +- [Custom types](#custom-types) +- [Constants](#constants) - [Configuration](#configuration) - - [Domain types](#domain-types) - [Misc](#misc) -- [Updated containers](#updated-containers) - - [Extended `BeaconBlockBody`](#extended-beaconblockbody) - - [Extended `BeaconState`](#extended-beaconstate) -- [New containers](#new-containers) - - [`CompactCommittee`](#compactcommittee) + - [Time parameters](#time-parameters) + - [Domain types](#domain-types) +- [Containers](#containers) + - [Extended containers](#extended-containers) + - [`BeaconBlockBody`](#beaconblockbody) + - [`BeaconState`](#beaconstate) + - [New containers](#new-containers) + - [`SyncCommittee`](#synccommittee) - [Helper functions](#helper-functions) - [Misc](#misc-1) - - [`pack_compact_validator`](#pack_compact_validator) - - [`unpack_compact_validator`](#unpack_compact_validator) - - [`committee_to_compact_committee`](#committee_to_compact_committee) + - [`compactify_validator`](#compactify_validator) + - [`decompactify_validator`](#decompactify_validator) - [Beacon state accessors](#beacon-state-accessors) - - [`get_light_client_committee`](#get_light_client_committee) + - [`get_sync_committee_indices`](#get_sync_committee_indices) + - [`get_sync_committee`](#get_sync_committee) - [Block processing](#block-processing) - - [Light client processing](#light-client-processing) + - [Sync committee processing](#sync-committee-processing) - [Epoch processing](#epoch-transition) - - [Light client committee updates](#light-client-committee-updates) - + - [Final updates](#updates-updates) + ## Introduction -This is a standalone patch to the ethereum beacon chain that adds light client support. +This is a standalone beacon chain patch adding light client support via sync committees. + +## Custom types + +| Name | SSZ equivalent | Description | +| - | - | - | +| `CompactValidator` | `uint64` | a compact validator | + +## Constants + +| Name | Value | +| - | - | +| `BASE_REWARDS_PER_EPOCH` | `uint64(5)` | ## Configuration @@ -33,114 +49,106 @@ This is a standalone patch to the ethereum beacon chain that adds light client s | Name | Value | | - | - | -| `LIGHT_CLIENT_COMMITTEE_SIZE` | `uint64(2**8)` (= 256) | -| `LIGHT_CLIENT_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | -| `BASE_REWARDS_PER_EPOCH` | 5 | +| `MAX_SYNC_COMMITTEE_SIZE` | `uint64(2**8)` (= 256) | + +### Time parameters + +| Name | Value | Unit | Duration | +| - | - | :-: | :-: | +| `EPOCHS_PER_SYNC_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | ### Domain types | Name | Value | | - | - | -| `DOMAIN_LIGHT_CLIENT` | `DomainType('0x82000000')` | +| `DOMAIN_SYNC_COMMITTEE` | `DomainType('0x07000000')` | -## Updated containers +## Containers -### Extended `BeaconBlockBody` +### Extended containers + +#### `BeaconBlockBody` ```python class BeaconBlockBody(phase0.BeaconBlockBody): - # Bitfield of participants in this light client signature - light_client_bits: Bitvector[LIGHT_CLIENT_COMMITTEE_SIZE] - light_client_signature: BLSSignature + sync_committee_bits: Bitlist[MAX_SYNC_COMMITTEE_SIZE] + sync_committee_signature: BLSSignature ``` -### Extended `BeaconState` +#### `BeaconState` ```python class BeaconState(phase0.BeaconState): - # Compact representations of the light client committee - current_light_committee: CompactCommittee - next_light_committee: CompactCommittee + current_sync_committee: SyncCommittee + next_sync_committee: SyncCommittee ``` -## New containers +### New containers -### `CompactCommittee` +#### `SyncCommittee` ```python -class CompactCommittee(Container): - pubkeys: List[BLSPubkey, MAX_VALIDATORS_PER_COMMITTEE] - sum_of_pubkeys: BLSPubkey - compact_validators: List[uint64, MAX_VALIDATORS_PER_COMMITTEE] +class SyncCommittee(Container): + pubkeys: List[BLSPubkey, MAX_SYNC_COMMITTEE_SIZE] + pubkeys_aggregate: BLSPubkey + compact_validators: List[CompactValidator, MAX_SYNC_COMMITTEE_SIZE] ``` ## Helper functions ### Misc - -#### `pack_compact_validator` +#### `compactify_validator` ```python -def pack_compact_validator(index: ValidatorIndex, slashed: bool, balance_in_increments: uint64) -> uint64: +def compactify_validator(index: ValidatorIndex, slashed: bool, effective_balance: Gwei) -> CompactValidator: """ - Create a compact validator object representing index, slashed status, and compressed balance. - Takes as input balance-in-increments (// EFFECTIVE_BALANCE_INCREMENT) to preserve symmetry with - the unpacking function. + Return the compact validator for a given validator index, slashed status and effective balance. """ - return (index << 16) + (slashed << 15) + balance_in_increments + return CompactValidator((index << 16) + (slashed << 15) + uint64(effective_balance // EFFECTIVE_BALANCE_INCREMENT)) ``` -#### `unpack_compact_validator` +#### `decompactify_validator` ```python -def unpack_compact_validator(compact_validator: uint64) -> Tuple[ValidatorIndex, bool, uint64]: +def decompactify_validator(compact_validator: CompactValidator) -> Tuple[ValidatorIndex, bool, Gwei]: """ - Return validator index, slashed, balance // EFFECTIVE_BALANCE_INCREMENT + Return the validator index, slashed status and effective balance for a given compact validator. """ - return ( - ValidatorIndex(compact_validator >> 16), - bool((compact_validator >> 15) % 2), - compact_validator & (2**15 - 1), - ) -``` - -#### `committee_to_compact_committee` - -```python -def committee_to_compact_committee(state: BeaconState, committee: Sequence[ValidatorIndex]) -> CompactCommittee: - """ - Given a state and a list of validator indices, outputs the ``CompactCommittee`` representing them. - """ - validators = [state.validators[i] for i in committee] - compact_validators = [ - pack_compact_validator(i, v.slashed, v.effective_balance // EFFECTIVE_BALANCE_INCREMENT) - for i, v in zip(committee, validators) - ] - pubkeys = [v.pubkey for v in validators] - return CompactCommittee( - pubkeys=pubkeys, - sum_of_pubkeys=bls.AggregatePubkeys(pubkeys), - compact_validators=compact_validators - ) + index = ValidatorIndex(compact_validator >> 16) # from bits 16-63 + slashed = bool((compact_validator >> 15) % 2) # from bit 15 + effective_balance = Gwei(compact_validator & (2**15 - 1)) * EFFECTIVE_BALANCE_INCREMENT # from bits 0-14 + return (index, slashed, effective_balance) ``` ### Beacon state accessors -#### `get_light_client_committee` +#### `get_sync_committee_indices` ```python -def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: +def get_sync_committee_indices(state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: """ - Return the light client committee of no more than ``LIGHT_CLIENT_COMMITTEE_SIZE`` validators. + Return the sync committee indices for a given state and epoch. """ - source_epoch = (max(epoch // LIGHT_CLIENT_COMMITTEE_PERIOD, 1) - 1) * LIGHT_CLIENT_COMMITTEE_PERIOD - active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) - seed = get_seed(beacon_state, source_epoch, DOMAIN_LIGHT_CLIENT) - return [ - compute_shuffled_index(i, active_validator_indices, seed) - for i in range(min(active_validator_indices, LIGHT_CLIENT_COMMITTEE_SIZE)) - ] + start_epoch = Epoch((max(epoch // EPOCHS_PER_SYNC_COMMITTEE_PERIOD, 1) - 1) * EPOCHS_PER_SYNC_COMMITTEE_PERIOD) + active_validator_count = uint64(len(get_active_validator_indices(state, start_epoch))) + sync_committee_size = min(active_validator_count, MAX_SYNC_COMMITTEE_SIZE) + seed = get_seed(state, base_epoch, DOMAIN_SYNC_COMMITTEE) + return [compute_shuffled_index(uint64(i), active_validator_count, seed) for i in range(sync_committee_size)] +``` + +### `get_sync_committee` + +```python +def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: + """ + Return the sync committee for a given state and epoch. + """ + indices = get_sync_committee_indices(state, epoch) + validators = [state.validators[index] for index in indices] + pubkeys = [validator.pubkey for validator in validators] + compact_validators = [compactify_validator(i, v.slashed, v.effective_balance) for i, v in zip(indices, validators)] + return SyncCommittee(pubkeys, bls.AggregatePubkeys(pubkeys), compact_validators) ``` ### Block processing @@ -148,65 +156,47 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque ```python def process_block(state: BeaconState, block: BeaconBlock) -> None: phase0.process_block(state, block) - process_light_client_signature(state, block.body) + process_sync_committee(state, block.body) ``` -#### Light client processing +#### Sync committee processing ```python -def process_light_client_signature(state: BeaconState, block_body: BeaconBlockBody) -> None: - committee = get_light_client_committee(state, get_current_epoch(state)) - previous_slot = max(state.slot, 1) - 1 - previous_block_root = get_block_root_at_slot(state, previous_slot) +def process_sync_committee(state: BeaconState, body: BeaconBlockBody) -> None: + # Verify sync committee bitfield length + committee_indices = get_sync_committee_indices(state, get_current_epoch(state)) + assert len(body.sync_committee_bits) == len(committee_indices) - # Light client committees sign over the previous block root - signing_root = compute_signing_root( - previous_block_root, - get_domain(state, DOMAIN_LIGHT_CLIENT, compute_epoch_at_slot(previous_slot)) - ) - - participants = [ - committee[i] for i in range(len(committee)) if block_body.light_client_bits[i] - ] - - signer_pubkeys = [ - state.validators[participant].pubkey for participant in participants - ] - - assert bls.FastAggregateVerify(signer_pubkeys, signing_root, block_body.light_client_signature) - - # Process rewards - total_reward = Gwei(0) - active_validator_count = len(get_active_validator_indices(beacon_state, get_current_epoch(state))) - for participant in participants: - reward = get_base_reward(state, participant) * active_validator_count // len(committee) // SLOTS_PER_EPOCH - increase_balance(state, participant, reward) - total_reward += reward + # Verify sync committee aggregate signature signing over the previous slot block root + previous_slot = max(state.slot, Slot(1)) - Slot(1) + participant_indices = [committee_indices[i] for i in range(len(committee_indices)) if body.sync_committee_bits[i]] + participant_pubkeys = [state.validators[participant_index].pubkey for participant_index in participant_indices] + domain = get_domain(state, DOMAIN_SYNC_COMMITTEE, compute_epoch_at_slot(previous_slot)) + signing_root = compute_signing_root(get_block_root_at_slot(state, previous_slot), domain) + assert bls.FastAggregateVerify(participant_pubkeys, signing_root, body.sync_committee_signature) - increase_balance(state, get_beacon_proposer_index(state), Gwei(total_reward // PROPOSER_REWARD_QUOTIENT)) + # Reward sync committee participants + participant_rewards = Gwei(0) + active_validator_count = uint64(len(get_active_validator_indices(state, get_current_epoch(state)))) + for participant_index in participant_indices: + base_reward = get_base_reward(state, participant_index) + reward = Gwei(base_reward * active_validator_count // len(committee_indices) // SLOTS_PER_EPOCH) + increase_balance(state, participant_index, reward) + participant_rewards += reward + + # Reward beacon proposer + increase_balance(state, get_beacon_proposer_index(state), Gwei(participant_rewards // PROPOSER_REWARD_QUOTIENT)) ``` ### Epoch processing -This epoch transition overrides the phase0 epoch transition: +#### Final updates ```python -def process_epoch(state: BeaconState) -> None: - phase0.process_epoch(state) - process_light_client_committee_updates(state) +def process_final_updates(state: BeaconState) -> None: + phase0.process_final_updates(state) + next_epoch = get_current_epoch(state) + Epoch(1) + if next_epoch % EPOCHS_PER_SYNC_COMMITTEE_PERIOD == 0: + state.current_sync_committee = state.next_sync_committee + state.next_sync_committee = get_sync_committee(state, next_epoch + EPOCHS_PER_SYNC_COMMITTEE_PERIOD) ``` - -#### Light client committee updates - -```python -def process_light_client_committee_updates(state: BeaconState) -> None: - """ - Update light client committees. - """ - next_epoch = compute_epoch_at_slot(Slot(state.slot + 1)) - if next_epoch % LIGHT_CLIENT_COMMITTEE_PERIOD == 0: - state.current_light_committee = state.next_light_committee - new_committee = get_light_client_committee(state, next_epoch + LIGHT_CLIENT_COMMITTEE_PERIOD) - state.next_light_committee = committee_to_compact_committee(state, new_committee) -``` - From 114e388d1230aed92987e8f12499bc14d18a0313 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 15 Nov 2020 17:23:44 +0000 Subject: [PATCH 009/222] =?UTF-8?q?Fix=20bugs=E2=80=94thanks=20@hwwhww?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- specs/lightclient/beacon-chain.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 4f6e07dd5..f64dc1271 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -130,14 +130,14 @@ def get_sync_committee_indices(state: BeaconState, epoch: Epoch) -> Sequence[Val """ Return the sync committee indices for a given state and epoch. """ - start_epoch = Epoch((max(epoch // EPOCHS_PER_SYNC_COMMITTEE_PERIOD, 1) - 1) * EPOCHS_PER_SYNC_COMMITTEE_PERIOD) - active_validator_count = uint64(len(get_active_validator_indices(state, start_epoch))) + base_epoch = Epoch((max(epoch // EPOCHS_PER_SYNC_COMMITTEE_PERIOD, 1) - 1) * EPOCHS_PER_SYNC_COMMITTEE_PERIOD) + active_validator_count = uint64(len(get_active_validator_indices(state, base_epoch))) sync_committee_size = min(active_validator_count, MAX_SYNC_COMMITTEE_SIZE) seed = get_seed(state, base_epoch, DOMAIN_SYNC_COMMITTEE) return [compute_shuffled_index(uint64(i), active_validator_count, seed) for i in range(sync_committee_size)] ``` -### `get_sync_committee` +#### `get_sync_committee` ```python def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: @@ -148,7 +148,7 @@ def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: validators = [state.validators[index] for index in indices] pubkeys = [validator.pubkey for validator in validators] compact_validators = [compactify_validator(i, v.slashed, v.effective_balance) for i, v in zip(indices, validators)] - return SyncCommittee(pubkeys, bls.AggregatePubkeys(pubkeys), compact_validators) + return SyncCommittee(pubkeys, bls.AggregatePKs(pubkeys), compact_validators) ``` ### Block processing From 1f210fd1f84c00b7005915914f0ae8e94b18cd12 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Mon, 16 Nov 2020 15:02:11 +0800 Subject: [PATCH 010/222] Added light client syncing protocol --- specs/lightclient/sync-protocol.md | 143 +++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 specs/lightclient/sync-protocol.md diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md new file mode 100644 index 000000000..ff9231365 --- /dev/null +++ b/specs/lightclient/sync-protocol.md @@ -0,0 +1,143 @@ +# Minimal Light Client Design + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + + + +- [Introduction](#introduction) +- [Custom types](#custom-types) +- [Constants](#constants) +- [Containers](#containers) + - [`LightClientUpdate`](#lightclientupdate) +- [Helpers](#helpers) + - [`LightClientMemory`](#lightclientmemory) +- [Light client state updates](#light-client-state-updates) + + + + +## Introduction + +Ethereum 2.0 is designed to be light client friendly. This allows low-resource clients such as mobile phones to access Ethereum 2.0 with reasonable safety and liveness. It also facilitates the development of "bridges" to external blockchains. This document suggests a minimal light client design for the beacon chain that uses the concept of "sync committees" introduced in [./beacon-chain.md](the the light-client-friendliness beacon chain extension). + +## Custom types + +We define the following Python custom types for type hinting and readability: + +| Name | SSZ equivalent | Description | +| - | - | - | + +## Constants + +| Name | Value | +| - | - | +| `SYNC_COMMITTEES_GENERALIZED_INDEX` | `GeneralizedIndexConcat(GeneralizedIndex(BeaconBlock, 'state_root'), GeneralizedIndex(BeaconState, 'current_sync_committee'))` | +| `FORK_GENERALIZED_INDEX` | `GeneralizedIndexConcat(GeneralizedIndex(BeaconBlock, 'state_root'), GeneralizedIndex(BeaconState, 'fork'))` | +| `BEACON_CHAIN_ROOT_IN_SHARD_BLOCK_HEADER_DEPTH` | `4` | +| `BEACON_CHAIN_ROOT_IN_SHARD_BLOCK_HEADER_INDEX` | **TBD** | +| `PERIOD_COMMITTEE_ROOT_IN_BEACON_STATE_DEPTH` | `5` | +| `PERIOD_COMMITTEE_ROOT_IN_BEACON_STATE_INDEX` | **TBD** | + +## Containers + +### `LightClientUpdate` + +```python +class LightClientUpdate(Container): + # Updated beacon header (and authenticating branch) + header: BeaconBlockHeader + # Sync committee signature to that header + aggregation_bits: Bitlist[MAX_SYNC_COMMITTEE_SIZE] + signature: BLSSignature + header_branch: Vector[Bytes32, BEACON_CHAIN_ROOT_IN_SHARD_BLOCK_HEADER_DEPTH] + # Updates fork version + new_fork: Fork + fork_branch: Vector[Bytes32, log_2(FORK_GENERALIZED_INDEX)] + # Updated period committee (and authenticating branch) + new_current_sync_committee: SyncCommittee + new_next_sync_committee: SyncCommittee + sync_committee_branch: Vector[Bytes32, log_2(SYNC_COMMITTEES_GENERALIZED_INDEX)] +``` + +## Helpers + +### `LightClientMemory` + +```python +class LightClientMemory(Container): + # Beacon header which is not expected to revert + header: BeaconBlockHeader + # Fork version data + fork_version: Version + # period committees corresponding to the beacon header + current_sync_committee: SyncCommittee + next_sync_committee: SyncCommittee +``` + +## Light client state updates + +The state of a light client is stored in a `memory` object of type `LightClientMemory`. To advance its state a light client requests an `update` object of type `LightClientUpdate` from the network by sending a request containing `(memory.shard, memory.header.slot, slot_range_end)`. It calls `validate_update(memory, update)` on each update that it receives in response. If `sum(update.aggregate_bits) * 3 > len(update.aggregate_bits) * 2` for any valid update, it accepts that update immediately; otherwise, it waits around for some time and then finally calls `update_memory(memory, update)` on the valid update with the highest `sum(update.aggregate_bits)`. + +#### `validate_update` + +```python +def validate_update(memory: LightClientMemory, update: LightClientUpdate) -> bool: + # Verify the update does not skip a period + current_period = compute_epoch_at_slot(memory.header.slot) // EPOCHS_PER_SYNC_COMMITTEE_PERIOD + new_epoch = compute_epoch_of_shard_slot(update.header.slot) + new_period = new_epoch // EPOCHS_PER_SYNC_COMMITTEE_PERIOD + assert new_period in (current_period, current_period + 1) + + # Verify that it actually updates to a newer slot + assert update.header.slot > memory.header.slot + + # Convenience as independent variable for convenience + committee = memory.current_sync_committee if new_period == current_period else memory.next_sync_committee + assert len(update.aggregation_bits) == len(committee) + + # Verify signature + active_pubkeys = [p for (bit, p) in zip(update.aggregation_bits, committee.pubkeys) if bit] + domain = compute_domain(DOMAIN_SYNC_COMMITTEE, memory.version) + signing_root = compute_signing_root(update.header, domain) + assert bls.FastAggregateVerify(pubkeys, signing_root, update.signature) + + # Verify Merkle branches of new info + assert is_valid_merkle_branch( + leaf=hash_tree_root(update.new_fork), + branch=update.fork_branch, + depth=log2(FORK_GENERALIZED_INDEX), + index=FORK_GENERALIZED_INDEX % 2**log2(FORK_GENERALIZED_INDEX), + root=hash_tree_root(update.header), + ) + assert is_valid_merkle_branch( + leaf=hash_tree_root(update.current_sync_committee), + branch=update.sync_committee_branch, + depth=log2(SYNC_COMMITTEES_GENERALIZED_INDEX), + index=SYNC_COMMITTEES_GENERALIZED_INDEX % 2**log2(SYNC_COMMITTEES_GENERALIZED_INDEX), + root=hash_tree_root(update.header), + ) + # Verify consistency of committees + if new_period == current_period: + assert update.new_current_sync_committee == memory.current_sync_committee + assert update.new_next_sync_committee == memory.next_sync_committee + else: + assert update.new_current_sync_committee == memory.next_sync_committee + + return True +``` + +#### `update_memory` + +``` +def update_memory(memory: LightClientMemory, update: LightClientUpdate) -> None: + memory.header = update.header + epoch = compute_epoch_at_slot(update.header.slot) + memory.fork_version = update.new_fork.previous_version if epoch < update.new_fork.epoch else update.new_fork.current_version + memory.current_sync_committee = update.new_current_sync_committee + memory.next_sync_committee == update.new_next_sync_committee +``` From 97dc916c9e0fbd48ac048d5212c5dca53b4b1e8c Mon Sep 17 00:00:00 2001 From: vbuterin Date: Mon, 16 Nov 2020 15:07:40 +0800 Subject: [PATCH 011/222] Python syntax highlighted updates --- specs/lightclient/sync-protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index ff9231365..aaff7c580 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -133,7 +133,7 @@ def validate_update(memory: LightClientMemory, update: LightClientUpdate) -> boo #### `update_memory` -``` +```python def update_memory(memory: LightClientMemory, update: LightClientUpdate) -> None: memory.header = update.header epoch = compute_epoch_at_slot(update.header.slot) From ca88dd6922ddba380754f20efd60ff6417f1631c Mon Sep 17 00:00:00 2001 From: vbuterin Date: Mon, 16 Nov 2020 15:08:43 +0800 Subject: [PATCH 012/222] Removed extraneous data --- specs/lightclient/sync-protocol.md | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index aaff7c580..bbe53c87f 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -25,23 +25,12 @@ Ethereum 2.0 is designed to be light client friendly. This allows low-resource clients such as mobile phones to access Ethereum 2.0 with reasonable safety and liveness. It also facilitates the development of "bridges" to external blockchains. This document suggests a minimal light client design for the beacon chain that uses the concept of "sync committees" introduced in [./beacon-chain.md](the the light-client-friendliness beacon chain extension). -## Custom types - -We define the following Python custom types for type hinting and readability: - -| Name | SSZ equivalent | Description | -| - | - | - | - ## Constants | Name | Value | | - | - | | `SYNC_COMMITTEES_GENERALIZED_INDEX` | `GeneralizedIndexConcat(GeneralizedIndex(BeaconBlock, 'state_root'), GeneralizedIndex(BeaconState, 'current_sync_committee'))` | | `FORK_GENERALIZED_INDEX` | `GeneralizedIndexConcat(GeneralizedIndex(BeaconBlock, 'state_root'), GeneralizedIndex(BeaconState, 'fork'))` | -| `BEACON_CHAIN_ROOT_IN_SHARD_BLOCK_HEADER_DEPTH` | `4` | -| `BEACON_CHAIN_ROOT_IN_SHARD_BLOCK_HEADER_INDEX` | **TBD** | -| `PERIOD_COMMITTEE_ROOT_IN_BEACON_STATE_DEPTH` | `5` | -| `PERIOD_COMMITTEE_ROOT_IN_BEACON_STATE_INDEX` | **TBD** | ## Containers From cbb3856ab9756f829d6af4107b884327df9e606e Mon Sep 17 00:00:00 2001 From: Justin Date: Mon, 16 Nov 2020 09:09:04 +0000 Subject: [PATCH 013/222] Fix ToC (cherry-picked from @hwwhww's PR) --- specs/lightclient/beacon-chain.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index f64dc1271..0c7ab2a3b 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -2,6 +2,10 @@ ## Table of contents + + + + - [Introduction](#introduction) - [Custom types](#custom-types) - [Constants](#constants) @@ -24,8 +28,11 @@ - [`get_sync_committee`](#get_sync_committee) - [Block processing](#block-processing) - [Sync committee processing](#sync-committee-processing) - - [Epoch processing](#epoch-transition) - - [Final updates](#updates-updates) + - [Epoch processing](#epoch-processing) + - [Final updates](#final-updates) + + + ## Introduction From 4df3547edfaf9703738dd1f517ae0583cc73534a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 17 Nov 2020 10:41:26 +0800 Subject: [PATCH 014/222] Make `lightclient` patch pass the linter (#2133) * Make `lightclient` an executable patch fork * fix conflicts * Fix ToC * Lightclient -> Light client * Try protolambda/remerkleable#8 * Fix sync-protocol.md ToC * Build lightclient/sync-protocol * Fix typo Co-authored-by: vbuterin Co-authored-by: vbuterin --- .gitignore | 1 + Makefile | 5 +-- setup.py | 49 ++++++++++++++++++++++++++++-- specs/lightclient/beacon-chain.md | 3 ++ specs/lightclient/sync-protocol.md | 3 +- 5 files changed, 56 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index bcd96f885..ed497112c 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ eth2.0-spec-tests/ # Dynamically built from Markdown spec tests/core/pyspec/eth2spec/phase0/ tests/core/pyspec/eth2spec/phase1/ +tests/core/pyspec/eth2spec/lightclient/ # coverage reports .htmlcov diff --git a/Makefile b/Makefile index 8fa104444..811abbad8 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENER # To check generator matching: #$(info $$GENERATOR_TARGETS is [${GENERATOR_TARGETS}]) -MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/phase0/*.md) $(wildcard $(SPEC_DIR)/phase1/*.md) $(wildcard $(SSZ_DIR)/*.md) $(wildcard $(SPEC_DIR)/networking/*.md) $(wildcard $(SPEC_DIR)/validator/*.md) +MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/phase0/*.md) $(wildcard $(SPEC_DIR)/phase1/*.md) $(wildcard $(SPEC_DIR)/lightclient/*.md) $(wildcard $(SSZ_DIR)/*.md) $(wildcard $(SPEC_DIR)/networking/*.md) $(wildcard $(SPEC_DIR)/validator/*.md) COV_HTML_OUT=.htmlcov COV_INDEX_FILE=$(PY_SPEC_DIR)/$(COV_HTML_OUT)/index.html @@ -49,6 +49,7 @@ partial_clean: rm -rf $(DEPOSIT_CONTRACT_TESTER_DIR)/.pytest_cache rm -rf $(PY_SPEC_DIR)/phase0 rm -rf $(PY_SPEC_DIR)/phase1 + rm -rf $(PY_SPEC_DIR)/lightclient rm -rf $(PY_SPEC_DIR)/$(COV_HTML_OUT) rm -rf $(PY_SPEC_DIR)/.coverage rm -rf $(PY_SPEC_DIR)/test-reports @@ -112,7 +113,7 @@ codespell: lint: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ flake8 --config $(LINTER_CONFIG_FILE) ./eth2spec \ - && mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.phase1 + && mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.phase1 -p eth2spec.lightclient lint_generators: pyspec . venv/bin/activate; cd $(TEST_GENERATORS_DIR); \ diff --git a/setup.py b/setup.py index 9c2de0751..6a2bf5707 100644 --- a/setup.py +++ b/setup.py @@ -52,8 +52,9 @@ def get_spec(file_name: str) -> SpecObject: else: # Handle function definitions & ssz_objects if pulling_from is not None: - if len(line) > 18 and line[:6] == 'class ' and line[-12:] == '(Container):': - name = line[6:-12] + if len(line) > 18 and line[:6] == 'class ' and (line[-12:] == '(Container):' or '(phase' in line): + end = -12 if line[-12:] == '(Container):' else line.find('(') + name = line[6:end] # Check consistency with markdown header assert name == current_name block_type = CodeBlockType.SSZ @@ -156,6 +157,40 @@ SSZObject = TypeVar('SSZObject', bound=View) CONFIG_NAME = 'mainnet' ''' +LIGHTCLIENT_IMPORT = '''from eth2spec.phase0 import spec as phase0 +from eth2spec.config.config_util import apply_constants_config +from typing import ( + Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable, Optional +) + +from dataclasses import ( + dataclass, + field, +) + +from lru import LRU + +from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes +from eth2spec.utils.ssz.ssz_typing import ( + View, boolean, Container, List, Vector, uint8, uint32, uint64, + Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, +) +from eth2spec.utils import bls + +from eth2spec.utils.hash_function import hash + +# Whenever lightclient is loaded, make sure we have the latest phase0 +from importlib import reload +reload(phase0) + + +SSZVariableName = str +GeneralizedIndex = NewType('GeneralizedIndex', int) +SSZObject = TypeVar('SSZObject', bound=View) + +CONFIG_NAME = 'mainnet' +''' + SUNDRY_CONSTANTS_FUNCTIONS = ''' def ceillog2(x: int) -> uint64: if x < 1: @@ -351,6 +386,7 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: fork_imports = { 'phase0': PHASE0_IMPORTS, 'phase1': PHASE1_IMPORTS, + 'lightclient': LIGHTCLIENT_IMPORT, } @@ -417,6 +453,15 @@ class PySpecCommand(Command): specs/phase1/shard-fork-choice.md specs/phase1/validator.md """ + elif self.spec_fork == "lightclient": + self.md_doc_paths = """ + specs/phase0/beacon-chain.md + specs/phase0/fork-choice.md + specs/phase0/validator.md + specs/phase0/weak-subjectivity.md + specs/lightclient/beacon-chain.md + specs/lightclient/sync-protocol.md + """ else: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 0c7ab2a3b..2d00d5170 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -6,6 +6,7 @@ + - [Introduction](#introduction) - [Custom types](#custom-types) - [Constants](#constants) @@ -78,6 +79,7 @@ This is a standalone beacon chain patch adding light client support via sync com ```python class BeaconBlockBody(phase0.BeaconBlockBody): + # Light client sync_committee_bits: Bitlist[MAX_SYNC_COMMITTEE_SIZE] sync_committee_signature: BLSSignature ``` @@ -86,6 +88,7 @@ class BeaconBlockBody(phase0.BeaconBlockBody): ```python class BeaconState(phase0.BeaconState): + # Light client current_sync_committee: SyncCommittee next_sync_committee: SyncCommittee ``` diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index bbe53c87f..8a07271b2 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -10,13 +10,14 @@ - [Introduction](#introduction) -- [Custom types](#custom-types) - [Constants](#constants) - [Containers](#containers) - [`LightClientUpdate`](#lightclientupdate) - [Helpers](#helpers) - [`LightClientMemory`](#lightclientmemory) - [Light client state updates](#light-client-state-updates) + - [`validate_update`](#validate_update) + - [`update_memory`](#update_memory) From f9e9d7cabf8b45be493e1cab0c175a36e4cb088a Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 17 Nov 2020 10:45:02 +0800 Subject: [PATCH 015/222] Update specs/lightclient/sync-protocol.md Co-authored-by: Alex Stokes --- specs/lightclient/sync-protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index 8a07271b2..d247187cd 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -92,7 +92,7 @@ def validate_update(memory: LightClientMemory, update: LightClientUpdate) -> boo # Verify signature active_pubkeys = [p for (bit, p) in zip(update.aggregation_bits, committee.pubkeys) if bit] - domain = compute_domain(DOMAIN_SYNC_COMMITTEE, memory.version) + domain = compute_domain(DOMAIN_SYNC_COMMITTEE, memory.fork_version) signing_root = compute_signing_root(update.header, domain) assert bls.FastAggregateVerify(pubkeys, signing_root, update.signature) From 117d31985f3927d2ae9bf48a6439e8b6aad215d1 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 17 Nov 2020 10:45:16 +0800 Subject: [PATCH 016/222] Update specs/lightclient/sync-protocol.md Co-authored-by: Alex Stokes --- specs/lightclient/sync-protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index d247187cd..e901f32c6 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -39,7 +39,7 @@ Ethereum 2.0 is designed to be light client friendly. This allows low-resource c ```python class LightClientUpdate(Container): - # Updated beacon header (and authenticating branch) + # Updated beacon header header: BeaconBlockHeader # Sync committee signature to that header aggregation_bits: Bitlist[MAX_SYNC_COMMITTEE_SIZE] From a5e6d77165f472d0fba756839fa79ef63a743df4 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 17 Nov 2020 13:25:59 +0800 Subject: [PATCH 017/222] Update specs/lightclient/sync-protocol.md Co-authored-by: Alex Stokes --- specs/lightclient/sync-protocol.md | 1 - 1 file changed, 1 deletion(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index e901f32c6..9b83d79e0 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -44,7 +44,6 @@ class LightClientUpdate(Container): # Sync committee signature to that header aggregation_bits: Bitlist[MAX_SYNC_COMMITTEE_SIZE] signature: BLSSignature - header_branch: Vector[Bytes32, BEACON_CHAIN_ROOT_IN_SHARD_BLOCK_HEADER_DEPTH] # Updates fork version new_fork: Fork fork_branch: Vector[Bytes32, log_2(FORK_GENERALIZED_INDEX)] From 1c146b2c031fc26902e2a6be4062f218ead262e5 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 17 Nov 2020 13:26:08 +0800 Subject: [PATCH 018/222] Update specs/lightclient/sync-protocol.md Co-authored-by: Alex Stokes --- specs/lightclient/sync-protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index 9b83d79e0..e03821c9d 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -47,7 +47,7 @@ class LightClientUpdate(Container): # Updates fork version new_fork: Fork fork_branch: Vector[Bytes32, log_2(FORK_GENERALIZED_INDEX)] - # Updated period committee (and authenticating branch) + # Updated sync committee (and authenticating branch) new_current_sync_committee: SyncCommittee new_next_sync_committee: SyncCommittee sync_committee_branch: Vector[Bytes32, log_2(SYNC_COMMITTEES_GENERALIZED_INDEX)] From 5e5d03d56f73fe5311e9a75a57e0426692479d35 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 17 Nov 2020 13:26:18 +0800 Subject: [PATCH 019/222] Update specs/lightclient/sync-protocol.md Co-authored-by: Alex Stokes --- specs/lightclient/sync-protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index e03821c9d..a24501426 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -63,7 +63,7 @@ class LightClientMemory(Container): header: BeaconBlockHeader # Fork version data fork_version: Version - # period committees corresponding to the beacon header + # sync committees corresponding to the beacon header current_sync_committee: SyncCommittee next_sync_committee: SyncCommittee ``` From 692a0aaaa5f599cbf0a0377e60346168cdc1fb26 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 17 Nov 2020 13:26:26 +0800 Subject: [PATCH 020/222] Update specs/lightclient/sync-protocol.md Co-authored-by: Alex Stokes --- specs/lightclient/sync-protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index a24501426..935c59481 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -85,7 +85,7 @@ def validate_update(memory: LightClientMemory, update: LightClientUpdate) -> boo # Verify that it actually updates to a newer slot assert update.header.slot > memory.header.slot - # Convenience as independent variable for convenience + # Independent variable for convenience committee = memory.current_sync_committee if new_period == current_period else memory.next_sync_committee assert len(update.aggregation_bits) == len(committee) From 99219c874f05c8331540be9f2dfeb0168ed69af0 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 17 Nov 2020 12:42:09 +0000 Subject: [PATCH 021/222] Revamp minimal light client (lots of polish and some bug fixes) --- specs/lightclient/sync-protocol.md | 197 +++++++++++++++++------------ 1 file changed, 113 insertions(+), 84 deletions(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index 935c59481..8fb94bfa5 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -1,4 +1,4 @@ -# Minimal Light Client Design +# Minimal Light Client **Notice**: This document is a work-in-progress for researchers and implementers. @@ -11,122 +11,151 @@ - [Introduction](#introduction) - [Constants](#constants) +- [Configuration](#configuration) + - [Misc](#misc) + - [Time parameters](#time-parameters) - [Containers](#containers) - - [`LightClientUpdate`](#lightclientupdate) -- [Helpers](#helpers) - - [`LightClientMemory`](#lightclientmemory) + - [`LightClientSnapshot`](#lightclientsnapshot) + - [`LightClientUpdate`](#lightclientupdate) + - [`LightClientStore`](#lightclientstore) - [Light client state updates](#light-client-state-updates) - - [`validate_update`](#validate_update) - - [`update_memory`](#update_memory) + - [`is_valid_light_client_update`](#is_valid_light_client_update) + - [`process_light_client_update`](#process_light_client_update) ## Introduction -Ethereum 2.0 is designed to be light client friendly. This allows low-resource clients such as mobile phones to access Ethereum 2.0 with reasonable safety and liveness. It also facilitates the development of "bridges" to external blockchains. This document suggests a minimal light client design for the beacon chain that uses the concept of "sync committees" introduced in [./beacon-chain.md](the the light-client-friendliness beacon chain extension). +Eth2 is designed to be light client friendly for constrained environments to access Eth2 with reasonable satefy and liveness. Such environments include resource-constrained devices (e.g. phones for trust-minimised wallets) and metered VMs (e.g. blockchain VMs for cross-chain bridges). + +This document suggests a minimal light client design for the beacon chain that uses sync committees introduced in [this beacon chain extension](./beacon-chain.md). ## Constants | Name | Value | | - | - | -| `SYNC_COMMITTEES_GENERALIZED_INDEX` | `GeneralizedIndexConcat(GeneralizedIndex(BeaconBlock, 'state_root'), GeneralizedIndex(BeaconState, 'current_sync_committee'))` | -| `FORK_GENERALIZED_INDEX` | `GeneralizedIndexConcat(GeneralizedIndex(BeaconBlock, 'state_root'), GeneralizedIndex(BeaconState, 'fork'))` | +| `NEXT_SYNC_COMMITTEE_INDEX` | `IndexConcat(Index(BeaconBlock, 'state_root'), Index(BeaconState, 'next_sync_committee'))` | +| `FORK_INDEX` | `IndexConcat(Index(BeaconBlock, 'state_root'), Index(BeaconState, 'fork'))` | + +## Configuration + +### Misc + +| Name | Value | +| - | - | +| `MAX_VALID_LIGHT_CLIENT_UPDATES` | `uint64(2**64 - 1)` | + +### Time parameters + +| Name | Value | Unit | Duration | +| - | - | :-: | :-: | +| `LIGHT_CLIENT_UPDATE_TIMEOUT` | `Slot(2**13)` | slots | ~27 hours | ## Containers -### `LightClientUpdate` +#### `LightClientSnapshot` ```python -class LightClientUpdate(Container): - # Updated beacon header +class LightClientSnapshot(Container): + # Beacon block header header: BeaconBlockHeader - # Sync committee signature to that header - aggregation_bits: Bitlist[MAX_SYNC_COMMITTEE_SIZE] - signature: BLSSignature - # Updates fork version - new_fork: Fork - fork_branch: Vector[Bytes32, log_2(FORK_GENERALIZED_INDEX)] - # Updated sync committee (and authenticating branch) - new_current_sync_committee: SyncCommittee - new_next_sync_committee: SyncCommittee - sync_committee_branch: Vector[Bytes32, log_2(SYNC_COMMITTEES_GENERALIZED_INDEX)] -``` - -## Helpers - -### `LightClientMemory` - -```python -class LightClientMemory(Container): - # Beacon header which is not expected to revert - header: BeaconBlockHeader - # Fork version data - fork_version: Version - # sync committees corresponding to the beacon header + # Fork data corresponding to the header + fork: Fork + # Sync committees corresponding to the header current_sync_committee: SyncCommittee next_sync_committee: SyncCommittee ``` -## Light client state updates - -The state of a light client is stored in a `memory` object of type `LightClientMemory`. To advance its state a light client requests an `update` object of type `LightClientUpdate` from the network by sending a request containing `(memory.shard, memory.header.slot, slot_range_end)`. It calls `validate_update(memory, update)` on each update that it receives in response. If `sum(update.aggregate_bits) * 3 > len(update.aggregate_bits) * 2` for any valid update, it accepts that update immediately; otherwise, it waits around for some time and then finally calls `update_memory(memory, update)` on the valid update with the highest `sum(update.aggregate_bits)`. - -#### `validate_update` +#### `LightClientUpdate` ```python -def validate_update(memory: LightClientMemory, update: LightClientUpdate) -> bool: - # Verify the update does not skip a period - current_period = compute_epoch_at_slot(memory.header.slot) // EPOCHS_PER_SYNC_COMMITTEE_PERIOD - new_epoch = compute_epoch_of_shard_slot(update.header.slot) - new_period = new_epoch // EPOCHS_PER_SYNC_COMMITTEE_PERIOD - assert new_period in (current_period, current_period + 1) - - # Verify that it actually updates to a newer slot - assert update.header.slot > memory.header.slot - - # Independent variable for convenience - committee = memory.current_sync_committee if new_period == current_period else memory.next_sync_committee - assert len(update.aggregation_bits) == len(committee) - - # Verify signature - active_pubkeys = [p for (bit, p) in zip(update.aggregation_bits, committee.pubkeys) if bit] - domain = compute_domain(DOMAIN_SYNC_COMMITTEE, memory.fork_version) - signing_root = compute_signing_root(update.header, domain) - assert bls.FastAggregateVerify(pubkeys, signing_root, update.signature) +class LightClientUpdate(Container): + # Updated snapshot + snapshot: LightClientSnapshot + # Merkle branches for the updated snapshot + fork_branch: Vector[Bytes32, log_2(FORK_INDEX)] + next_sync_committee_branch: Vector[Bytes32, log_2(NEXT_SYNC_COMMITTEE_INDEX)] + # Sync committee aggregate signature + sync_committee_bits: Bitlist[MAX_SYNC_COMMITTEE_SIZE] + sync_committee_signature: BLSSignature +``` - # Verify Merkle branches of new info +#### `LightClientStore` + +```python +class LightClientStore(Container): + snapshot: LightClientSnapshot + valid_updates: List[LightClientUpdate, MAX_VALID_LIGHT_CLIENT_UPDATES] +``` + +## Light client state updates + +A light client maintains its state in a `store` object of type `LightClientStore` and receives `update` objects of type `LightClientUpdate`. Every `update` triggers `process_light_client_update(store, update, current_slot)` where `current_slot` is the currect slot based on some local clock. + +#### `is_valid_light_client_update` + +```python +def is_valid_light_client_update(store: LightClientStore, update: LightClientUpdate) -> bool: + # Verify new slot is larger than old slot + old_snapshot = store.snapshot + new_snapshot = update.snapshot + assert new_snapshot.header.slot > old_snapshot.header.slot + + # Verify update does not skip a sync committee period + old_period = compute_epoch_at_slot(old_snapshot.header.slot) // EPOCHS_PER_SYNC_COMMITTEE_PERIOD + new_period = compute_epoch_at_slot(new_snapshot.header.slot) // EPOCHS_PER_SYNC_COMMITTEE_PERIOD + assert new_period in (old_period, old_period + 1) + + # Verify new snapshot sync committees + if new_period == old_period: + assert new_snapshot.current_sync_committee == old_snapshot.current_sync_committee + assert new_snapshot.next_sync_committee == old_snapshot.next_sync_committee + else new_period == old_period + 1: + assert new_snapshot.current_sync_committee == old_snapshot.next_sync_committee + assert is_valid_merkle_branch( + leaf=hash_tree_root(new_snapshot.next_sync_committee), + branch=update.next_sync_committee_branch, + depth=log2(NEXT_SYNC_COMMITTEE_INDEX), + index=NEXT_SYNC_COMMITTEE_INDEX % 2**log2(NEXT_SYNC_COMMITTEE_INDEX), + root=hash_tree_root(new_snapshot.header), + ) + + # Verify new snapshot fork assert is_valid_merkle_branch( - leaf=hash_tree_root(update.new_fork), + leaf=hash_tree_root(new_snapshot.fork), branch=update.fork_branch, - depth=log2(FORK_GENERALIZED_INDEX), - index=FORK_GENERALIZED_INDEX % 2**log2(FORK_GENERALIZED_INDEX), - root=hash_tree_root(update.header), + depth=log2(FORK_INDEX), + index=FORK_INDEX % 2**log2(FORK_INDEX), + root=hash_tree_root(new_snapshot.header), ) - assert is_valid_merkle_branch( - leaf=hash_tree_root(update.current_sync_committee), - branch=update.sync_committee_branch, - depth=log2(SYNC_COMMITTEES_GENERALIZED_INDEX), - index=SYNC_COMMITTEES_GENERALIZED_INDEX % 2**log2(SYNC_COMMITTEES_GENERALIZED_INDEX), - root=hash_tree_root(update.header), - ) - # Verify consistency of committees - if new_period == current_period: - assert update.new_current_sync_committee == memory.current_sync_committee - assert update.new_next_sync_committee == memory.next_sync_committee - else: - assert update.new_current_sync_committee == memory.next_sync_committee + + # Verify sync committee bitfield length + sync_committee = new_snapshot.current_sync_committee + assert len(update.sync_committee_bits) == len(sync_committee) + + # Verify sync committee aggregate signature + participant_pubkeys = [pubkey for (bit, pubkey) in zip(update.sync_committee_bits, sync_committee.pubkeys) if bit] + domain = compute_domain(DOMAIN_SYNC_COMMITTEE, fork_version.current_version) + signing_root = compute_signing_root(new_snapshot.header, domain) + assert bls.FastAggregateVerify(participant_pubkeys, signing_root, update.sync_committee_signature) return True ``` -#### `update_memory` +#### `process_update` ```python -def update_memory(memory: LightClientMemory, update: LightClientUpdate) -> None: - memory.header = update.header - epoch = compute_epoch_at_slot(update.header.slot) - memory.fork_version = update.new_fork.previous_version if epoch < update.new_fork.epoch else update.new_fork.current_version - memory.current_sync_committee = update.new_current_sync_committee - memory.next_sync_committee == update.new_next_sync_committee +def process_light_client_update(store: LightClientStore, update: LightClientUpdate, current_slot: Slot) -> None: + assert is_valid_light_client_update(store, update) + if sum(update.sync_committee_bits) * 3 > len(update.sync_committee_bits) * 2: + store.snapshot = update.snapshot + valid_updates = [] + else: + valid_updates.append(update) + + # Force an update after the update timeout has elapsed + if current_slot > old_snapshot.header.slot + LIGHT_CLIENT_UPDATE_TIMEOUT: + best_update = max(valid_updates, key=lambda update: sum(update.sync_committee_bits)) + store.snapshot = best_update.new_snapshot ``` From 7ffc9c5bc0147203b765ee370c76e13a96fb9ce3 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 17 Nov 2020 14:18:58 +0000 Subject: [PATCH 022/222] More polish and fixes to the sync protocol --- specs/lightclient/sync-protocol.md | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index 8fb94bfa5..6977d5862 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -36,7 +36,6 @@ This document suggests a minimal light client design for the beacon chain that u | Name | Value | | - | - | | `NEXT_SYNC_COMMITTEE_INDEX` | `IndexConcat(Index(BeaconBlock, 'state_root'), Index(BeaconState, 'next_sync_committee'))` | -| `FORK_INDEX` | `IndexConcat(Index(BeaconBlock, 'state_root'), Index(BeaconState, 'fork'))` | ## Configuration @@ -60,8 +59,6 @@ This document suggests a minimal light client design for the beacon chain that u class LightClientSnapshot(Container): # Beacon block header header: BeaconBlockHeader - # Fork data corresponding to the header - fork: Fork # Sync committees corresponding to the header current_sync_committee: SyncCommittee next_sync_committee: SyncCommittee @@ -73,12 +70,13 @@ class LightClientSnapshot(Container): class LightClientUpdate(Container): # Updated snapshot snapshot: LightClientSnapshot - # Merkle branches for the updated snapshot - fork_branch: Vector[Bytes32, log_2(FORK_INDEX)] - next_sync_committee_branch: Vector[Bytes32, log_2(NEXT_SYNC_COMMITTEE_INDEX)] + # Merkle branches for the next sync committee + next_sync_committee_branch: Vector[Bytes32, log2(NEXT_SYNC_COMMITTEE_INDEX)] # Sync committee aggregate signature sync_committee_bits: Bitlist[MAX_SYNC_COMMITTEE_SIZE] sync_committee_signature: BLSSignature + # Fork version corresponding to the aggregate signature + fork_version ``` #### `LightClientStore` @@ -121,22 +119,13 @@ def is_valid_light_client_update(store: LightClientStore, update: LightClientUpd root=hash_tree_root(new_snapshot.header), ) - # Verify new snapshot fork - assert is_valid_merkle_branch( - leaf=hash_tree_root(new_snapshot.fork), - branch=update.fork_branch, - depth=log2(FORK_INDEX), - index=FORK_INDEX % 2**log2(FORK_INDEX), - root=hash_tree_root(new_snapshot.header), - ) - # Verify sync committee bitfield length sync_committee = new_snapshot.current_sync_committee assert len(update.sync_committee_bits) == len(sync_committee) # Verify sync committee aggregate signature participant_pubkeys = [pubkey for (bit, pubkey) in zip(update.sync_committee_bits, sync_committee.pubkeys) if bit] - domain = compute_domain(DOMAIN_SYNC_COMMITTEE, fork_version.current_version) + domain = compute_domain(DOMAIN_SYNC_COMMITTEE, update.fork_version) signing_root = compute_signing_root(new_snapshot.header, domain) assert bls.FastAggregateVerify(participant_pubkeys, signing_root, update.sync_committee_signature) From 5e717a456d3046433a4add9f40f38bef9a303342 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 17 Nov 2020 15:30:09 +0000 Subject: [PATCH 023/222] More polish and fixes --- specs/lightclient/sync-protocol.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index 6977d5862..99f25d875 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -43,6 +43,7 @@ This document suggests a minimal light client design for the beacon chain that u | Name | Value | | - | - | +| `MIN_SYNC_COMMITTEE_PARTICIPANTS` | `1` | | `MAX_VALID_LIGHT_CLIENT_UPDATES` | `uint64(2**64 - 1)` | ### Time parameters @@ -75,7 +76,7 @@ class LightClientUpdate(Container): # Sync committee aggregate signature sync_committee_bits: Bitlist[MAX_SYNC_COMMITTEE_SIZE] sync_committee_signature: BLSSignature - # Fork version corresponding to the aggregate signature + # Fork version for the aggregate signature fork_version ``` @@ -122,6 +123,7 @@ def is_valid_light_client_update(store: LightClientStore, update: LightClientUpd # Verify sync committee bitfield length sync_committee = new_snapshot.current_sync_committee assert len(update.sync_committee_bits) == len(sync_committee) + assert sum(update.sync_committee_bits) > MIN_SYNC_COMMITTEE_PARTICIPANTS # Verify sync committee aggregate signature participant_pubkeys = [pubkey for (bit, pubkey) in zip(update.sync_committee_bits, sync_committee.pubkeys) if bit] @@ -136,15 +138,16 @@ def is_valid_light_client_update(store: LightClientStore, update: LightClientUpd ```python def process_light_client_update(store: LightClientStore, update: LightClientUpdate, current_slot: Slot) -> None: + # Validate update assert is_valid_light_client_update(store, update) + valid_updates.append(update) + if sum(update.sync_committee_bits) * 3 > len(update.sync_committee_bits) * 2: + # Immediate update when quorum is reached store.snapshot = update.snapshot valid_updates = [] - else: - valid_updates.append(update) - - # Force an update after the update timeout has elapsed - if current_slot > old_snapshot.header.slot + LIGHT_CLIENT_UPDATE_TIMEOUT: - best_update = max(valid_updates, key=lambda update: sum(update.sync_committee_bits)) - store.snapshot = best_update.new_snapshot + elif current_slot > old_snapshot.header.slot + LIGHT_CLIENT_UPDATE_TIMEOUT: + # Forced best update when the update timeout has elapsed + store.snapshot = max(valid_updates, key=lambda update: sum(update.sync_committee_bits)).new_snapshot + valid_updates = [] ``` From 830efa496a32835cce910fa4600547ddcc5ebd91 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 18 Nov 2020 16:15:30 +0800 Subject: [PATCH 024/222] Removed compact validators, make committee balance-based --- specs/lightclient/beacon-chain.md | 40 ++++++++----------------------- 1 file changed, 10 insertions(+), 30 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 2d00d5170..fbca70071 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -22,8 +22,6 @@ - [`SyncCommittee`](#synccommittee) - [Helper functions](#helper-functions) - [Misc](#misc-1) - - [`compactify_validator`](#compactify_validator) - - [`decompactify_validator`](#decompactify_validator) - [Beacon state accessors](#beacon-state-accessors) - [`get_sync_committee_indices`](#get_sync_committee_indices) - [`get_sync_committee`](#get_sync_committee) @@ -43,7 +41,6 @@ This is a standalone beacon chain patch adding light client support via sync com | Name | SSZ equivalent | Description | | - | - | - | -| `CompactValidator` | `uint64` | a compact validator | ## Constants @@ -101,36 +98,12 @@ class BeaconState(phase0.BeaconState): class SyncCommittee(Container): pubkeys: List[BLSPubkey, MAX_SYNC_COMMITTEE_SIZE] pubkeys_aggregate: BLSPubkey - compact_validators: List[CompactValidator, MAX_SYNC_COMMITTEE_SIZE] ``` ## Helper functions ### Misc -#### `compactify_validator` - -```python -def compactify_validator(index: ValidatorIndex, slashed: bool, effective_balance: Gwei) -> CompactValidator: - """ - Return the compact validator for a given validator index, slashed status and effective balance. - """ - return CompactValidator((index << 16) + (slashed << 15) + uint64(effective_balance // EFFECTIVE_BALANCE_INCREMENT)) -``` - -#### `decompactify_validator` - -```python -def decompactify_validator(compact_validator: CompactValidator) -> Tuple[ValidatorIndex, bool, Gwei]: - """ - Return the validator index, slashed status and effective balance for a given compact validator. - """ - index = ValidatorIndex(compact_validator >> 16) # from bits 16-63 - slashed = bool((compact_validator >> 15) % 2) # from bit 15 - effective_balance = Gwei(compact_validator & (2**15 - 1)) * EFFECTIVE_BALANCE_INCREMENT # from bits 0-14 - return (index, slashed, effective_balance) -``` - ### Beacon state accessors #### `get_sync_committee_indices` @@ -144,7 +117,15 @@ def get_sync_committee_indices(state: BeaconState, epoch: Epoch) -> Sequence[Val active_validator_count = uint64(len(get_active_validator_indices(state, base_epoch))) sync_committee_size = min(active_validator_count, MAX_SYNC_COMMITTEE_SIZE) seed = get_seed(state, base_epoch, DOMAIN_SYNC_COMMITTEE) - return [compute_shuffled_index(uint64(i), active_validator_count, seed) for i in range(sync_committee_size)] + i, output = 0, [] + while i < active_validator_count and len(output) < sync_committee_size: + candidate_index = indices[compute_shuffled_index(uint64(i), active_validator_count, seed)] + random_byte = hash(seed + uint_to_bytes(uint64(i // 32)))[i % 32] + effective_balance = state.validators[candidate_index].effective_balance + if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: + output.append(candidate_index) + i += 1 + return output ``` #### `get_sync_committee` @@ -157,8 +138,7 @@ def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: indices = get_sync_committee_indices(state, epoch) validators = [state.validators[index] for index in indices] pubkeys = [validator.pubkey for validator in validators] - compact_validators = [compactify_validator(i, v.slashed, v.effective_balance) for i, v in zip(indices, validators)] - return SyncCommittee(pubkeys, bls.AggregatePKs(pubkeys), compact_validators) + return SyncCommittee(pubkeys, bls.AggregatePKs(pubkeys)) ``` ### Block processing From 171b21301b73b9bad649199cd03e5ed58c757971 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 18 Nov 2020 16:27:19 +0800 Subject: [PATCH 025/222] Added support for updates that point to finalized ancestors --- specs/lightclient/sync-protocol.md | 31 +++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index 99f25d875..e460d86d9 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -35,7 +35,8 @@ This document suggests a minimal light client design for the beacon chain that u | Name | Value | | - | - | -| `NEXT_SYNC_COMMITTEE_INDEX` | `IndexConcat(Index(BeaconBlock, 'state_root'), Index(BeaconState, 'next_sync_committee'))` | +| `NEXT_SYNC_COMMITTEE_INDEX` | `Index(BeaconState, 'next_sync_committee')` | +| `FINALIZED_ROOT_INDEX` | `Index(BeaconState, 'finalized_checkpoint', 'root')` | ## Configuration @@ -71,13 +72,17 @@ class LightClientSnapshot(Container): class LightClientUpdate(Container): # Updated snapshot snapshot: LightClientSnapshot + # Header that the new snapshot is a finalized ancestor of + signed_header: BeaconBlockHeader + # Merkle branch proving ancestry of the header in the snapshot + ancestry_branch: Vector[Bytes32, log2(FINALIZED_ROOT_INDEX)] # Merkle branches for the next sync committee next_sync_committee_branch: Vector[Bytes32, log2(NEXT_SYNC_COMMITTEE_INDEX)] # Sync committee aggregate signature sync_committee_bits: Bitlist[MAX_SYNC_COMMITTEE_SIZE] sync_committee_signature: BLSSignature # Fork version for the aggregate signature - fork_version + fork_version: Version ``` #### `LightClientStore` @@ -105,6 +110,18 @@ def is_valid_light_client_update(store: LightClientStore, update: LightClientUpd old_period = compute_epoch_at_slot(old_snapshot.header.slot) // EPOCHS_PER_SYNC_COMMITTEE_PERIOD new_period = compute_epoch_at_slot(new_snapshot.header.slot) // EPOCHS_PER_SYNC_COMMITTEE_PERIOD assert new_period in (old_period, old_period + 1) + + # Verify relationship between signed header and ancestor header + if update.signed_header == new_snapshot.header: + assert update.ancestry_branch == [ZERO_HASH for _ in range(log2(FINALIZED_ROOT_INDEX))] + else: + assert is_valid_merkle_branch( + leaf=hash_tree_root(new_snapshot.header), + branch=update.ancestry_branch, + depth=log2(FINALIZED_ROOT_INDEX), + index=FINALIZED_ROOT_INDEX % 2**log2(FINALIZED_ROOT_INDEX), + root=update.signed_header.state_root, + ) # Verify new snapshot sync committees if new_period == old_period: @@ -117,7 +134,7 @@ def is_valid_light_client_update(store: LightClientStore, update: LightClientUpd branch=update.next_sync_committee_branch, depth=log2(NEXT_SYNC_COMMITTEE_INDEX), index=NEXT_SYNC_COMMITTEE_INDEX % 2**log2(NEXT_SYNC_COMMITTEE_INDEX), - root=hash_tree_root(new_snapshot.header), + root=new_snapshot.header.state_root, ) # Verify sync committee bitfield length @@ -128,7 +145,7 @@ def is_valid_light_client_update(store: LightClientStore, update: LightClientUpd # Verify sync committee aggregate signature participant_pubkeys = [pubkey for (bit, pubkey) in zip(update.sync_committee_bits, sync_committee.pubkeys) if bit] domain = compute_domain(DOMAIN_SYNC_COMMITTEE, update.fork_version) - signing_root = compute_signing_root(new_snapshot.header, domain) + signing_root = compute_signing_root(update.signed_header, domain) assert bls.FastAggregateVerify(participant_pubkeys, signing_root, update.sync_committee_signature) return True @@ -142,7 +159,11 @@ def process_light_client_update(store: LightClientStore, update: LightClientUpda assert is_valid_light_client_update(store, update) valid_updates.append(update) - if sum(update.sync_committee_bits) * 3 > len(update.sync_committee_bits) * 2: + # Immediate update "happy path" requires: + # (i) 2/3 participation + # (ii) an update that refers to the finalized ancestor of a signed block, and not the signed block directly + + if sum(update.sync_committee_bits) * 3 > len(update.sync_committee_bits) * 2 and update.snapshot.header != update.signed_header: # Immediate update when quorum is reached store.snapshot = update.snapshot valid_updates = [] From 664bc4b42e0e2d43432c750039644745d8f1fb74 Mon Sep 17 00:00:00 2001 From: Justin Date: Wed, 18 Nov 2020 09:19:32 +0000 Subject: [PATCH 026/222] Polish and fixes including fixed-size sync committees --- specs/lightclient/beacon-chain.md | 35 +++++++++++-------------------- 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index fbca70071..19ca9c85b 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -8,7 +8,6 @@ - [Introduction](#introduction) -- [Custom types](#custom-types) - [Constants](#constants) - [Configuration](#configuration) - [Misc](#misc) @@ -21,7 +20,6 @@ - [New containers](#new-containers) - [`SyncCommittee`](#synccommittee) - [Helper functions](#helper-functions) - - [Misc](#misc-1) - [Beacon state accessors](#beacon-state-accessors) - [`get_sync_committee_indices`](#get_sync_committee_indices) - [`get_sync_committee`](#get_sync_committee) @@ -37,11 +35,6 @@ This is a standalone beacon chain patch adding light client support via sync committees. -## Custom types - -| Name | SSZ equivalent | Description | -| - | - | - | - ## Constants | Name | Value | @@ -54,7 +47,7 @@ This is a standalone beacon chain patch adding light client support via sync com | Name | Value | | - | - | -| `MAX_SYNC_COMMITTEE_SIZE` | `uint64(2**8)` (= 256) | +| `SYNC_COMMITTEE_SIZE` | `uint64(2**8)` (= 256) | ### Time parameters @@ -77,7 +70,7 @@ This is a standalone beacon chain patch adding light client support via sync com ```python class BeaconBlockBody(phase0.BeaconBlockBody): # Light client - sync_committee_bits: Bitlist[MAX_SYNC_COMMITTEE_SIZE] + sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE] sync_committee_signature: BLSSignature ``` @@ -96,14 +89,12 @@ class BeaconState(phase0.BeaconState): ```python class SyncCommittee(Container): - pubkeys: List[BLSPubkey, MAX_SYNC_COMMITTEE_SIZE] + pubkeys: Vector[BLSPubkey, SYNC_COMMITTEE_SIZE] pubkeys_aggregate: BLSPubkey ``` ## Helper functions -### Misc - ### Beacon state accessors #### `get_sync_committee_indices` @@ -114,18 +105,19 @@ def get_sync_committee_indices(state: BeaconState, epoch: Epoch) -> Sequence[Val Return the sync committee indices for a given state and epoch. """ base_epoch = Epoch((max(epoch // EPOCHS_PER_SYNC_COMMITTEE_PERIOD, 1) - 1) * EPOCHS_PER_SYNC_COMMITTEE_PERIOD) - active_validator_count = uint64(len(get_active_validator_indices(state, base_epoch))) - sync_committee_size = min(active_validator_count, MAX_SYNC_COMMITTEE_SIZE) + active_validator_indices = get_active_validator_indices(state, base_epoch) + active_validator_count = uint64(len(active_validator_indices)) seed = get_seed(state, base_epoch, DOMAIN_SYNC_COMMITTEE) - i, output = 0, [] - while i < active_validator_count and len(output) < sync_committee_size: - candidate_index = indices[compute_shuffled_index(uint64(i), active_validator_count, seed)] + i, sync_committee_indices = 0, [] + while len(sync_committee_indices) < SYNC_COMMITTEE_SIZE: + shuffled_index = compute_shuffled_index(uint64(i % active_validator_count), active_validator_count, seed) + candidate_index = active_validator_indices[shuffled_index] random_byte = hash(seed + uint_to_bytes(uint64(i // 32)))[i % 32] effective_balance = state.validators[candidate_index].effective_balance if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: - output.append(candidate_index) + sync_committee_indices.append(candidate_index) i += 1 - return output + return sync_committee_indices ``` #### `get_sync_committee` @@ -153,12 +145,9 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: ```python def process_sync_committee(state: BeaconState, body: BeaconBlockBody) -> None: - # Verify sync committee bitfield length - committee_indices = get_sync_committee_indices(state, get_current_epoch(state)) - assert len(body.sync_committee_bits) == len(committee_indices) - # Verify sync committee aggregate signature signing over the previous slot block root previous_slot = max(state.slot, Slot(1)) - Slot(1) + committee_indices = get_sync_committee_indices(state, get_current_epoch(state)) participant_indices = [committee_indices[i] for i in range(len(committee_indices)) if body.sync_committee_bits[i]] participant_pubkeys = [state.validators[participant_index].pubkey for participant_index in participant_indices] domain = get_domain(state, DOMAIN_SYNC_COMMITTEE, compute_epoch_at_slot(previous_slot)) From d16900a7532156991515e6267e9d22b70f02a31a Mon Sep 17 00:00:00 2001 From: Justin Date: Wed, 18 Nov 2020 09:31:41 +0000 Subject: [PATCH 027/222] Fixes from @ralexstokes --- specs/lightclient/sync-protocol.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index e460d86d9..d35c9821d 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -76,10 +76,10 @@ class LightClientUpdate(Container): signed_header: BeaconBlockHeader # Merkle branch proving ancestry of the header in the snapshot ancestry_branch: Vector[Bytes32, log2(FINALIZED_ROOT_INDEX)] - # Merkle branches for the next sync committee + # Merkle branch for the next sync committee next_sync_committee_branch: Vector[Bytes32, log2(NEXT_SYNC_COMMITTEE_INDEX)] # Sync committee aggregate signature - sync_committee_bits: Bitlist[MAX_SYNC_COMMITTEE_SIZE] + sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE] sync_committee_signature: BLSSignature # Fork version for the aggregate signature fork_version: Version @@ -127,7 +127,7 @@ def is_valid_light_client_update(store: LightClientStore, update: LightClientUpd if new_period == old_period: assert new_snapshot.current_sync_committee == old_snapshot.current_sync_committee assert new_snapshot.next_sync_committee == old_snapshot.next_sync_committee - else new_period == old_period + 1: + else: assert new_snapshot.current_sync_committee == old_snapshot.next_sync_committee assert is_valid_merkle_branch( leaf=hash_tree_root(new_snapshot.next_sync_committee), @@ -140,7 +140,7 @@ def is_valid_light_client_update(store: LightClientStore, update: LightClientUpd # Verify sync committee bitfield length sync_committee = new_snapshot.current_sync_committee assert len(update.sync_committee_bits) == len(sync_committee) - assert sum(update.sync_committee_bits) > MIN_SYNC_COMMITTEE_PARTICIPANTS + assert sum(update.sync_committee_bits) >= MIN_SYNC_COMMITTEE_PARTICIPANTS # Verify sync committee aggregate signature participant_pubkeys = [pubkey for (bit, pubkey) in zip(update.sync_committee_bits, sync_committee.pubkeys) if bit] @@ -166,9 +166,9 @@ def process_light_client_update(store: LightClientStore, update: LightClientUpda if sum(update.sync_committee_bits) * 3 > len(update.sync_committee_bits) * 2 and update.snapshot.header != update.signed_header: # Immediate update when quorum is reached store.snapshot = update.snapshot - valid_updates = [] + store.valid_updates = [] elif current_slot > old_snapshot.header.slot + LIGHT_CLIENT_UPDATE_TIMEOUT: # Forced best update when the update timeout has elapsed - store.snapshot = max(valid_updates, key=lambda update: sum(update.sync_committee_bits)).new_snapshot - valid_updates = [] + store.snapshot = max(store.valid_updates, key=lambda update: sum(update.sync_committee_bits)).snapshot + store.valid_updates = [] ``` From 09ec58131d41792ac4479881ad89573331adcb38 Mon Sep 17 00:00:00 2001 From: Justin Date: Wed, 18 Nov 2020 10:33:42 +0000 Subject: [PATCH 028/222] Optimised updates as suggested by @vbuterin --- specs/lightclient/sync-protocol.md | 101 +++++++++++++++-------------- 1 file changed, 54 insertions(+), 47 deletions(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index d35c9821d..f78e6f090 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -20,6 +20,7 @@ - [`LightClientStore`](#lightclientstore) - [Light client state updates](#light-client-state-updates) - [`is_valid_light_client_update`](#is_valid_light_client_update) + - [`apply_light_client_update`](#apply_light_client_update) - [`process_light_client_update`](#process_light_client_update) @@ -35,8 +36,8 @@ This document suggests a minimal light client design for the beacon chain that u | Name | Value | | - | - | -| `NEXT_SYNC_COMMITTEE_INDEX` | `Index(BeaconState, 'next_sync_committee')` | | `FINALIZED_ROOT_INDEX` | `Index(BeaconState, 'finalized_checkpoint', 'root')` | +| `NEXT_SYNC_COMMITTEE_INDEX` | `Index(BeaconState, 'next_sync_committee')` | ## Configuration @@ -70,14 +71,14 @@ class LightClientSnapshot(Container): ```python class LightClientUpdate(Container): - # Updated snapshot - snapshot: LightClientSnapshot - # Header that the new snapshot is a finalized ancestor of - signed_header: BeaconBlockHeader - # Merkle branch proving ancestry of the header in the snapshot - ancestry_branch: Vector[Bytes32, log2(FINALIZED_ROOT_INDEX)] - # Merkle branch for the next sync committee + # Update beacon block header + header: BeaconBlockHeader + # Next sync committee corresponding to the header + next_sync_committee: SyncCommittee next_sync_committee_branch: Vector[Bytes32, log2(NEXT_SYNC_COMMITTEE_INDEX)] + # Finality proof for the update header + finality_header: BeaconBlockHeader + finality_branch: Vector[Bytes32, log2(FINALIZED_ROOT_INDEX)] # Sync committee aggregate signature sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE] sync_committee_signature: BLSSignature @@ -100,75 +101,81 @@ A light client maintains its state in a `store` object of type `LightClientStore #### `is_valid_light_client_update` ```python -def is_valid_light_client_update(store: LightClientStore, update: LightClientUpdate) -> bool: - # Verify new slot is larger than old slot - old_snapshot = store.snapshot - new_snapshot = update.snapshot - assert new_snapshot.header.slot > old_snapshot.header.slot +def is_valid_light_client_update(snapshot: LightClientSnapshot, update: LightClientUpdate) -> bool: + # Verify update slot is larger than snapshot slot + assert update.header.slot > snapshot.header.slot # Verify update does not skip a sync committee period - old_period = compute_epoch_at_slot(old_snapshot.header.slot) // EPOCHS_PER_SYNC_COMMITTEE_PERIOD - new_period = compute_epoch_at_slot(new_snapshot.header.slot) // EPOCHS_PER_SYNC_COMMITTEE_PERIOD - assert new_period in (old_period, old_period + 1) - - # Verify relationship between signed header and ancestor header - if update.signed_header == new_snapshot.header: - assert update.ancestry_branch == [ZERO_HASH for _ in range(log2(FINALIZED_ROOT_INDEX))] + snapshot_period = compute_epoch_at_slot(snapshot.header.slot) // EPOCHS_PER_SYNC_COMMITTEE_PERIOD + update_period = compute_epoch_at_slot(update.header.slot) // EPOCHS_PER_SYNC_COMMITTEE_PERIOD + assert update_period in (snapshot_period, snapshot_period + 1) + + # Verify update header root is the finalized root of the finality header, if specified + if update.finality_header == BeaconBlockHeader(): + signed_header = update.header + assert update.finality_branch == [ZERO_HASH for _ in range(log2(FINALIZED_ROOT_INDEX))] else: + signed_header = update.finality_header assert is_valid_merkle_branch( - leaf=hash_tree_root(new_snapshot.header), - branch=update.ancestry_branch, + leaf=hash_tree_root(update.header), + branch=update.finality_branch, depth=log2(FINALIZED_ROOT_INDEX), index=FINALIZED_ROOT_INDEX % 2**log2(FINALIZED_ROOT_INDEX), - root=update.signed_header.state_root, - ) + root=update.finality_header.state_root, + ) - # Verify new snapshot sync committees - if new_period == old_period: - assert new_snapshot.current_sync_committee == old_snapshot.current_sync_committee - assert new_snapshot.next_sync_committee == old_snapshot.next_sync_committee + # Verify update next sync committee if the update period incremented + if update_period == snapshot_period: + sync_committee = snapshot.current_sync_committee + assert update.next_sync_committee_branch == [ZERO_HASH for _ in range(log2(NEXT_SYNC_COMMITTEE_INDEX))] else: - assert new_snapshot.current_sync_committee == old_snapshot.next_sync_committee + sync_committee = snapshot.next_sync_committee assert is_valid_merkle_branch( - leaf=hash_tree_root(new_snapshot.next_sync_committee), + leaf=hash_tree_root(update.next_sync_committee), branch=update.next_sync_committee_branch, depth=log2(NEXT_SYNC_COMMITTEE_INDEX), index=NEXT_SYNC_COMMITTEE_INDEX % 2**log2(NEXT_SYNC_COMMITTEE_INDEX), - root=new_snapshot.header.state_root, + root=update.header.state_root, ) - # Verify sync committee bitfield length - sync_committee = new_snapshot.current_sync_committee - assert len(update.sync_committee_bits) == len(sync_committee) + # Verify sync committee has sufficient participants assert sum(update.sync_committee_bits) >= MIN_SYNC_COMMITTEE_PARTICIPANTS # Verify sync committee aggregate signature participant_pubkeys = [pubkey for (bit, pubkey) in zip(update.sync_committee_bits, sync_committee.pubkeys) if bit] domain = compute_domain(DOMAIN_SYNC_COMMITTEE, update.fork_version) - signing_root = compute_signing_root(update.signed_header, domain) + signing_root = compute_signing_root(signed_header, domain) assert bls.FastAggregateVerify(participant_pubkeys, signing_root, update.sync_committee_signature) return True ``` -#### `process_update` +#### `apply_light_client_update` + +```python +def apply_light_client_update(snapshot: LightClientSnapshot, update: LightClientUpdate) -> None: + snapshot_period = compute_epoch_at_slot(snapshot.header.slot) // EPOCHS_PER_SYNC_COMMITTEE_PERIOD + update_period = compute_epoch_at_slot(update.header.slot) // EPOCHS_PER_SYNC_COMMITTEE_PERIOD + if update_period == snapshot_period + 1: + snapshot.current_sync_committee = snapshot.next_sync_committee + snapshot.next_sync_committee = update.next_sync_committee + snapshot.header = update.header +``` + +#### `process_light_client_update` ```python def process_light_client_update(store: LightClientStore, update: LightClientUpdate, current_slot: Slot) -> None: # Validate update - assert is_valid_light_client_update(store, update) - valid_updates.append(update) + assert is_valid_light_client_update(store.snapshot, update) + store.valid_updates.append(update) - # Immediate update "happy path" requires: - # (i) 2/3 participation - # (ii) an update that refers to the finalized ancestor of a signed block, and not the signed block directly - - if sum(update.sync_committee_bits) * 3 > len(update.sync_committee_bits) * 2 and update.snapshot.header != update.signed_header: - # Immediate update when quorum is reached - store.snapshot = update.snapshot + if sum(update.sync_committee_bits) * 3 > len(update.sync_committee_bits) * 2 and update.header != update.finality_header: + # Apply update if 2/3 quorum is reached and we have a finality proof + apply_light_client_update(store, update) store.valid_updates = [] - elif current_slot > old_snapshot.header.slot + LIGHT_CLIENT_UPDATE_TIMEOUT: + elif current_slot > snapshot.header.slot + LIGHT_CLIENT_UPDATE_TIMEOUT: # Forced best update when the update timeout has elapsed - store.snapshot = max(store.valid_updates, key=lambda update: sum(update.sync_committee_bits)).snapshot + apply_light_client_update(store, max(store.valid_updates, key=lambda update: sum(update.sync_committee_bits))) store.valid_updates = [] ``` From bf48c1ce79aefb43ea5addf7fa98de05503bb0d0 Mon Sep 17 00:00:00 2001 From: Micah Zoltu Date: Sun, 22 Nov 2020 14:12:32 +0800 Subject: [PATCH 029/222] Redirects implementation list to one that is actually maintained. Unmaintained lists are bad, especially when there is a maintained one elsewhere. Lets keep a single list. @protolambda --- ssz/simple-serialize.md | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/ssz/simple-serialize.md b/ssz/simple-serialize.md index b0e7ec1e0..17b9aed41 100644 --- a/ssz/simple-serialize.md +++ b/ssz/simple-serialize.md @@ -247,16 +247,4 @@ We similarly define "summary types" and "expansion types". For example, [`Beacon ## Implementations -| Language | Project | Maintainer | Implementation | -|-|-|-|-| -| Python | Ethereum 2.0 | Ethereum Foundation | [https://github.com/ethereum/py-ssz](https://github.com/ethereum/py-ssz) | -| Rust | Lighthouse | Sigma Prime | [https://github.com/sigp/lighthouse/tree/master/consensus/ssz](https://github.com/sigp/lighthouse/tree/master/consensus/ssz) | -| Nim | Nimbus | Status | [https://github.com/status-im/nim-beacon-chain/blob/master/beacon_chain/ssz.nim](https://github.com/status-im/nim-beacon-chain/blob/master/beacon_chain/ssz.nim) | -| Rust | Shasper | ParityTech | [https://github.com/paritytech/shasper/tree/master/utils/ssz](https://github.com/paritytech/shasper/tree/master/utils/ssz) | -| TypeScript | Lodestar | ChainSafe Systems | [https://github.com/ChainSafe/ssz-js](https://github.com/ChainSafe/ssz) | -| Java | Cava | ConsenSys | [https://www.github.com/ConsenSys/cava/tree/master/ssz](https://www.github.com/ConsenSys/cava/tree/master/ssz) | -| Go | Prysm | Prysmatic Labs | [https://github.com/prysmaticlabs/go-ssz](https://github.com/prysmaticlabs/go-ssz) | -| Swift | Yeeth | Dean Eigenmann | [https://github.com/yeeth/SimpleSerialize.swift](https://github.com/yeeth/SimpleSerialize.swift) | -| C# | | Jordan Andrews | [https://github.com/codingupastorm/csharp-ssz](https://github.com/codingupastorm/csharp-ssz) | -| C# | Cortex | Sly Gryphon | [https://www.nuget.org/packages/Cortex.SimpleSerialize](https://www.nuget.org/packages/Cortex.SimpleSerialize) | -| C++ | | Jiyun Kim | [https://github.com/NAKsir-melody/cpp_ssz](https://github.com/NAKsir-melody/cpp_ssz) | +See https://github.com/ethereum/eth2.0-specs/issues/2138 for a list of current known implementations. From 5bfe61f86567511ddb8e07179b7cb390c1f99357 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 25 Nov 2020 19:38:45 +0800 Subject: [PATCH 030/222] Embiggened sync committee size and added sub-aggregates --- specs/lightclient/beacon-chain.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 19ca9c85b..c218a09bd 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -47,7 +47,8 @@ This is a standalone beacon chain patch adding light client support via sync com | Name | Value | | - | - | -| `SYNC_COMMITTEE_SIZE` | `uint64(2**8)` (= 256) | +| `SYNC_COMMITTEE_SIZE` | `uint64(2**10)` (= 1024) | +| `SYNC_PUBKEY_AGGREGATION_INTERVAL` | `uint64(2**6)` (= 64) | ### Time parameters @@ -90,7 +91,7 @@ class BeaconState(phase0.BeaconState): ```python class SyncCommittee(Container): pubkeys: Vector[BLSPubkey, SYNC_COMMITTEE_SIZE] - pubkeys_aggregate: BLSPubkey + pubkey_aggregates: Vector[BLSPubkey, SYNC_COMMITTEE_SIZE // SYNC_PUBKEY_AGGREGATION_INTERVAL] ``` ## Helper functions @@ -130,7 +131,11 @@ def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: indices = get_sync_committee_indices(state, epoch) validators = [state.validators[index] for index in indices] pubkeys = [validator.pubkey for validator in validators] - return SyncCommittee(pubkeys, bls.AggregatePKs(pubkeys)) + aggregates = [ + bls.AggregatePKs(pubkeys[i:i+SYNC_PUBKEY_AGGREGATION_INTERVAL]) + for i in range(0, len(pubkeys), SYNC_PUBKEY_AGGREGATION_INTERVAL) + ] + return SyncCommittee(pubkeys, aggregates) ``` ### Block processing From b2d25f745493260d73648e92e5e7c79bb30d98cb Mon Sep 17 00:00:00 2001 From: Justin Date: Wed, 25 Nov 2020 11:59:01 +0000 Subject: [PATCH 031/222] Nitpicks --- specs/lightclient/beacon-chain.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index c218a09bd..e8beb13a3 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -48,7 +48,7 @@ This is a standalone beacon chain patch adding light client support via sync com | Name | Value | | - | - | | `SYNC_COMMITTEE_SIZE` | `uint64(2**10)` (= 1024) | -| `SYNC_PUBKEY_AGGREGATION_INTERVAL` | `uint64(2**6)` (= 64) | +| `SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE` | `uint64(2**6)` (= 64) | ### Time parameters @@ -70,7 +70,7 @@ This is a standalone beacon chain patch adding light client support via sync com ```python class BeaconBlockBody(phase0.BeaconBlockBody): - # Light client + # Sync committee aggregate signature sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE] sync_committee_signature: BLSSignature ``` @@ -79,7 +79,7 @@ class BeaconBlockBody(phase0.BeaconBlockBody): ```python class BeaconState(phase0.BeaconState): - # Light client + # Sync committees current_sync_committee: SyncCommittee next_sync_committee: SyncCommittee ``` @@ -91,7 +91,7 @@ class BeaconState(phase0.BeaconState): ```python class SyncCommittee(Container): pubkeys: Vector[BLSPubkey, SYNC_COMMITTEE_SIZE] - pubkey_aggregates: Vector[BLSPubkey, SYNC_COMMITTEE_SIZE // SYNC_PUBKEY_AGGREGATION_INTERVAL] + pubkey_aggregates: Vector[BLSPubkey, SYNC_COMMITTEE_SIZE // SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE] ``` ## Helper functions @@ -132,8 +132,8 @@ def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: validators = [state.validators[index] for index in indices] pubkeys = [validator.pubkey for validator in validators] aggregates = [ - bls.AggregatePKs(pubkeys[i:i+SYNC_PUBKEY_AGGREGATION_INTERVAL]) - for i in range(0, len(pubkeys), SYNC_PUBKEY_AGGREGATION_INTERVAL) + bls.AggregatePKs(pubkeys[i:i + SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE]) + for i in range(0, len(pubkeys), SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE) ] return SyncCommittee(pubkeys, aggregates) ``` From 7bf5539b2f5a1bc753e3409ac746794b891b5477 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 7 Dec 2020 10:49:12 +0800 Subject: [PATCH 032/222] Set codespell<3.0.0,>=2.0.0 version and add `ether` to whitelist --- .circleci/config.yml | 2 +- .codespell-whitelist | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2767bec70..b57e15458 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -120,7 +120,7 @@ jobs: - checkout - run: name: Check codespell - command: pip install codespell --user && make codespell + command: pip install 'codespell<3.0.0,>=2.0.0' --user && make codespell lint: docker: - image: circleci/python:3.8 diff --git a/.codespell-whitelist b/.codespell-whitelist index ff694e380..6b8bab36f 100644 --- a/.codespell-whitelist +++ b/.codespell-whitelist @@ -1,2 +1,3 @@ uint -byteorder \ No newline at end of file +byteorder +ether From 8587e8c5f2a43d63cbd5a701dadf55c4af17a7a6 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 7 Dec 2020 10:59:05 +0800 Subject: [PATCH 033/222] Fix doctoc@2 version --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b57e15458..e00463cc5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -111,7 +111,7 @@ jobs: - checkout - run: name: Check table of contents - command: sudo npm install -g doctoc && make check_toc + command: sudo npm install -g doctoc@2 && make check_toc codespell: docker: - image: circleci/python:3.8 From 64412be0d7b5ba8b552b46afb152a2f0ac0d725f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 7 Dec 2020 11:00:29 +0800 Subject: [PATCH 034/222] Minor update ToC --- specs/phase0/beacon-chain.md | 1 - specs/phase0/deposit-contract.md | 1 - specs/phase0/fork-choice.md | 1 - specs/phase0/p2p-interface.md | 1 - specs/phase0/validator.md | 1 - specs/phase0/weak-subjectivity.md | 1 - specs/phase1/beacon-chain.md | 1 - specs/phase1/custody-game.md | 1 - specs/phase1/fork-choice.md | 1 - specs/phase1/light-client-sync.md | 1 - specs/phase1/phase1-fork.md | 1 - specs/phase1/shard-fork-choice.md | 1 - specs/phase1/shard-transition.md | 1 - specs/phase1/validator.md | 1 - ssz/merkle-proofs.md | 1 - ssz/simple-serialize.md | 1 - 16 files changed, 16 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index fcd681115..d336cb2f8 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -5,7 +5,6 @@ - - [Introduction](#introduction) - [Notation](#notation) - [Custom types](#custom-types) diff --git a/specs/phase0/deposit-contract.md b/specs/phase0/deposit-contract.md index 8b3035629..23d831917 100644 --- a/specs/phase0/deposit-contract.md +++ b/specs/phase0/deposit-contract.md @@ -5,7 +5,6 @@ - - [Introduction](#introduction) - [Constants](#constants) - [Configuration](#configuration) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index 8bc108caa..b5689ecd2 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -5,7 +5,6 @@ - - [Introduction](#introduction) - [Fork choice](#fork-choice) - [Configuration](#configuration) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 43a097f7e..5dc892991 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -14,7 +14,6 @@ It consists of four main sections: - - [Network fundamentals](#network-fundamentals) - [Transport](#transport) - [Encryption and identification](#encryption-and-identification) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index c6a91dcfc..cb45e65e8 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -8,7 +8,6 @@ This is an accompanying document to [Ethereum 2.0 Phase 0 -- The Beacon Chain](. - - [Introduction](#introduction) - [Prerequisites](#prerequisites) - [Constants](#constants) diff --git a/specs/phase0/weak-subjectivity.md b/specs/phase0/weak-subjectivity.md index 9d433de8e..b4d78cb12 100644 --- a/specs/phase0/weak-subjectivity.md +++ b/specs/phase0/weak-subjectivity.md @@ -6,7 +6,6 @@ - - [Introduction](#introduction) - [Prerequisites](#prerequisites) - [Constants](#constants) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 23ce88aa9..c8f93cc7f 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -7,7 +7,6 @@ - - [Introduction](#introduction) - [Custom types](#custom-types) - [Configuration](#configuration) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 82eb35d86..33c2a0397 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -7,7 +7,6 @@ - - [Introduction](#introduction) - [Constants](#constants) - [Misc](#misc) diff --git a/specs/phase1/fork-choice.md b/specs/phase1/fork-choice.md index 3845b2b74..d2e1bfefe 100644 --- a/specs/phase1/fork-choice.md +++ b/specs/phase1/fork-choice.md @@ -7,7 +7,6 @@ - - [Introduction](#introduction) - [Updated data structures](#updated-data-structures) - [Extended `Store`](#extended-store) diff --git a/specs/phase1/light-client-sync.md b/specs/phase1/light-client-sync.md index 4d14485dd..107baa0c6 100644 --- a/specs/phase1/light-client-sync.md +++ b/specs/phase1/light-client-sync.md @@ -8,7 +8,6 @@ - - [Introduction](#introduction) - [Custom types](#custom-types) - [Constants](#constants) diff --git a/specs/phase1/phase1-fork.md b/specs/phase1/phase1-fork.md index b4ace1066..d81ca64b3 100644 --- a/specs/phase1/phase1-fork.md +++ b/specs/phase1/phase1-fork.md @@ -7,7 +7,6 @@ - - [Introduction](#introduction) - [Configuration](#configuration) - [Fork to Phase 1](#fork-to-phase-1) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 380269d13..177c9c18c 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -7,7 +7,6 @@ - - [Introduction](#introduction) - [Fork choice](#fork-choice) - [Helpers](#helpers) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index f3a1f83ce..35d421cdd 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -7,7 +7,6 @@ - - [Introduction](#introduction) - [Helper functions](#helper-functions) - [Shard block verification functions](#shard-block-verification-functions) diff --git a/specs/phase1/validator.md b/specs/phase1/validator.md index 2bfea017e..c5893fbc6 100644 --- a/specs/phase1/validator.md +++ b/specs/phase1/validator.md @@ -8,7 +8,6 @@ - - [Introduction](#introduction) - [Prerequisites](#prerequisites) - [Constants](#constants) diff --git a/ssz/merkle-proofs.md b/ssz/merkle-proofs.md index 2f32e43eb..6772026fe 100644 --- a/ssz/merkle-proofs.md +++ b/ssz/merkle-proofs.md @@ -7,7 +7,6 @@ - - [Helper functions](#helper-functions) - [Generalized Merkle tree index](#generalized-merkle-tree-index) - [SSZ object to index](#ssz-object-to-index) diff --git a/ssz/simple-serialize.md b/ssz/simple-serialize.md index 17b9aed41..d36e444ec 100644 --- a/ssz/simple-serialize.md +++ b/ssz/simple-serialize.md @@ -5,7 +5,6 @@ - - [Constants](#constants) - [Typing](#typing) - [Basic types](#basic-types) From b817b7690ced79d06da521673befb4ede4f83be9 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 7 Dec 2020 12:13:47 +0800 Subject: [PATCH 035/222] Disable the deposit contract tests. Only keep the `build_deposit_contract` to verify the bytecode. --- .circleci/config.yml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e00463cc5..bd46e8542 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -216,15 +216,17 @@ workflows: - lint: requires: - test - - install_deposit_contract_web3_tester: - requires: - - checkout_specs - - test_deposit_contract_web3_tests: - requires: - - install_deposit_contract_web3_tester + # NOTE: Since phase 0 has been launched, we disabled the deposit contract tests. + # - install_deposit_contract_web3_tester: + # requires: + # - checkout_specs + # - test_deposit_contract_web3_tests: + # requires: + # - install_deposit_contract_web3_tester build_and_test_deposit_contract: jobs: - build_deposit_contract - - test_deposit_contract: - requires: - - build_deposit_contract + # NOTE: Since phase 0 has been launched, we disabled the deposit contract tests. + # - test_deposit_contract: + # requires: + # - build_deposit_contract From 56048cbf5637d5834ad4ab570888e7928ceba1a4 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 6 Dec 2020 20:56:06 +0000 Subject: [PATCH 036/222] Cosmetic cleanups to BLS signatures section Miscellaneous cosmetic cleanups to the " BLS signatures" section: 1) fix section capitalisation for spec consistency 2) avoid uncapitalised "eth2" for spec consistency 3) avoid contractions (such as `PK` for `pubkey`) for spec consistency 4) various copyedits --- specs/phase0/beacon-chain.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index d336cb2f8..857b6de94 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -57,7 +57,7 @@ - [Crypto](#crypto) - [`hash`](#hash) - [`hash_tree_root`](#hash_tree_root) - - [BLS Signatures](#bls-signatures) + - [BLS signatures](#bls-signatures) - [Predicates](#predicates) - [`is_active_validator`](#is_active_validator) - [`is_eligible_for_activation_queue`](#is_eligible_for_activation_queue) @@ -600,17 +600,17 @@ def bytes_to_uint64(data: bytes) -> uint64: `def hash_tree_root(object: SSZSerializable) -> Root` is a function for hashing objects into a single root by utilizing a hash tree structure, as defined in the [SSZ spec](../../ssz/simple-serialize.md#merkleization). -#### BLS Signatures +#### BLS signatures -Eth2 makes use of BLS signatures as specified in the [IETF draft BLS specification draft-irtf-cfrg-bls-signature-04](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-04). Specifically, eth2 uses the `BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_` ciphersuite which implements the following interfaces: +The [IETF BLS signature draft standard v4](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-04) with ciphersuite `BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_` define the following functions: -- `def Sign(SK: int, message: Bytes) -> BLSSignature` -- `def Verify(PK: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` +- `def Sign(secret_key: int, message: Bytes) -> BLSSignature` +- `def Verify(pubkey: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` - `def Aggregate(signatures: Sequence[BLSSignature]) -> BLSSignature` -- `def FastAggregateVerify(PKs: Sequence[BLSPubkey], message: Bytes, signature: BLSSignature) -> bool` -- `def AggregateVerify(PKs: Sequence[BLSPubkey], messages: Sequence[Bytes], signature: BLSSignature) -> bool` +- `def FastAggregateVerify(pubkeys: Sequence[BLSPubkey], message: Bytes, signature: BLSSignature) -> bool` +- `def AggregateVerify(pubkeys: Sequence[BLSPubkey], messages: Sequence[Bytes], signature: BLSSignature) -> bool` -Within these specifications, BLS signatures are treated as a module for notational clarity, thus to verify a signature `bls.Verify(...)` is used. +The above functions are accessed through the `bls` module, e.g. `bls.Verify`. ### Predicates From 3b7c02514b9c855acdddbe2918bd0e02eeb02c9b Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 7 Dec 2020 08:10:39 -0700 Subject: [PATCH 037/222] straightforward light client edits --- specs/lightclient/beacon-chain.md | 31 ++++++++++++++++++++++-------- specs/lightclient/sync-protocol.md | 8 ++++++-- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index e8beb13a3..25952599d 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -66,15 +66,28 @@ This is a standalone beacon chain patch adding light client support via sync com ### Extended containers -#### `BeaconBlockBody` +*Note*: Extended SSZ containers inherit all fields from the parent in the original +order and append any additional fields to the end. + +#### `BeaconBlock` ```python -class BeaconBlockBody(phase0.BeaconBlockBody): +class BeaconBlock(phase0.BeaconBlock): # Sync committee aggregate signature sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE] sync_committee_signature: BLSSignature ``` +#### `BeaconBlockHeader` + +```python +class BeaconBlockHeader(phase0.BeaconBlockHeader): + # Sync committee aggregate signature + sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE] + sync_committee_signature: BLSSignature +``` + + #### `BeaconState` ```python @@ -105,6 +118,7 @@ def get_sync_committee_indices(state: BeaconState, epoch: Epoch) -> Sequence[Val """ Return the sync committee indices for a given state and epoch. """ + MAX_RANDOM_BYTE = 2**8 - 1 base_epoch = Epoch((max(epoch // EPOCHS_PER_SYNC_COMMITTEE_PERIOD, 1) - 1) * EPOCHS_PER_SYNC_COMMITTEE_PERIOD) active_validator_indices = get_active_validator_indices(state, base_epoch) active_validator_count = uint64(len(active_validator_indices)) @@ -143,21 +157,22 @@ def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: ```python def process_block(state: BeaconState, block: BeaconBlock) -> None: phase0.process_block(state, block) - process_sync_committee(state, block.body) + process_sync_committee(state, block) ``` #### Sync committee processing ```python -def process_sync_committee(state: BeaconState, body: BeaconBlockBody) -> None: +def process_sync_committee(state: BeaconState, block: BeaconBlock) -> None: # Verify sync committee aggregate signature signing over the previous slot block root - previous_slot = max(state.slot, Slot(1)) - Slot(1) + previous_slot = Slot(max(state.slot, 1) - 1) committee_indices = get_sync_committee_indices(state, get_current_epoch(state)) - participant_indices = [committee_indices[i] for i in range(len(committee_indices)) if body.sync_committee_bits[i]] - participant_pubkeys = [state.validators[participant_index].pubkey for participant_index in participant_indices] + participant_indices = [index for index, bit in zip(committee_indices, body.sync_committee_bits) if bit] + committee_pubkeys = state.current_sync_committee.pubkeys + participant_pubkeys = [pubkey for pubkey, bit in zip(committee_pubkeys, body.sync_committee_bits) if bit] domain = get_domain(state, DOMAIN_SYNC_COMMITTEE, compute_epoch_at_slot(previous_slot)) signing_root = compute_signing_root(get_block_root_at_slot(state, previous_slot), domain) - assert bls.FastAggregateVerify(participant_pubkeys, signing_root, body.sync_committee_signature) + assert bls.FastAggregateVerify(participant_pubkeys, signing_root, block.sync_committee_signature) # Reward sync committee participants participant_rewards = Gwei(0) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index f78e6f090..84b6f6fd7 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -28,9 +28,13 @@ ## Introduction -Eth2 is designed to be light client friendly for constrained environments to access Eth2 with reasonable satefy and liveness. Such environments include resource-constrained devices (e.g. phones for trust-minimised wallets) and metered VMs (e.g. blockchain VMs for cross-chain bridges). +Eth2 is designed to be light client friendly for constrained environments to +access Eth2 with reasonable safety and liveness. +Such environments include resource-constrained devices (e.g. phones for trust-minimised wallets) +and metered VMs (e.g. blockchain VMs for cross-chain bridges). -This document suggests a minimal light client design for the beacon chain that uses sync committees introduced in [this beacon chain extension](./beacon-chain.md). +This document suggests a minimal light client design for the beacon chain that +uses sync committees introduced in [this beacon chain extension](./beacon-chain.md). ## Constants From 0b9e80d1eb0fe1609792fbd366b32b89837a94a6 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 6 Dec 2020 21:04:11 +0000 Subject: [PATCH 038/222] grammar --- specs/phase0/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 857b6de94..34ce3c9c5 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -602,7 +602,7 @@ def bytes_to_uint64(data: bytes) -> uint64: #### BLS signatures -The [IETF BLS signature draft standard v4](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-04) with ciphersuite `BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_` define the following functions: +The [IETF BLS signature draft standard v4](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-04) with ciphersuite `BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_` defines the following functions: - `def Sign(secret_key: int, message: Bytes) -> BLSSignature` - `def Verify(pubkey: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` From f2fe3906388bbc049616cd0f25755d73356fc36e Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 8 Dec 2020 09:20:50 +0000 Subject: [PATCH 039/222] secret_key => privkey --- specs/phase0/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 34ce3c9c5..b47558484 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -604,7 +604,7 @@ def bytes_to_uint64(data: bytes) -> uint64: The [IETF BLS signature draft standard v4](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-04) with ciphersuite `BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_` defines the following functions: -- `def Sign(secret_key: int, message: Bytes) -> BLSSignature` +- `def Sign(privkey: int, message: Bytes) -> BLSSignature` - `def Verify(pubkey: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` - `def Aggregate(signatures: Sequence[BLSSignature]) -> BLSSignature` - `def FastAggregateVerify(pubkeys: Sequence[BLSPubkey], message: Bytes, signature: BLSSignature) -> bool` From 6b728c18800cc0611d8630e35e15bba4003f6784 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 11 Dec 2020 14:46:16 -0700 Subject: [PATCH 040/222] add eth1 withdrawal credentials to spec --- specs/phase0/beacon-chain.md | 5 +++ specs/phase0/deposit-contract.md | 13 ++++-- specs/phase0/validator.md | 42 +++++++++++++++--- .../block_processing/test_process_deposit.py | 43 +++++++++++++++++++ 4 files changed, 93 insertions(+), 10 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index b47558484..7fdf027bc 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -13,6 +13,7 @@ - [Misc](#misc) - [Gwei values](#gwei-values) - [Initial values](#initial-values) + - [Validator withdrawal credential versions](#validator-withdrawal-credential-versions) - [Time parameters](#time-parameters) - [State list lengths](#state-list-lengths) - [Rewards and penalties](#rewards-and-penalties) @@ -209,7 +210,11 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | | - | - | | `GENESIS_FORK_VERSION` | `Version('0x00000000')` | + +### Validator withdrawal credential versions + | `BLS_WITHDRAWAL_PREFIX` | `Bytes1('0x00')` | +| `ETH1_ADDRESS_WITHDRAWAL_PREFIX` | `Bytes1('0x01')` | ### Time parameters diff --git a/specs/phase0/deposit-contract.md b/specs/phase0/deposit-contract.md index 23d831917..ef21d35e0 100644 --- a/specs/phase0/deposit-contract.md +++ b/specs/phase0/deposit-contract.md @@ -58,12 +58,17 @@ The amount of ETH (rounded down to the closest Gwei) sent to the deposit contrac #### Withdrawal credentials -One of the `DepositData` fields is `withdrawal_credentials`. It is a commitment to credentials for withdrawing validator balance (e.g. to another validator, or to shards). The first byte of `withdrawal_credentials` is a version number. As of now, the only expected format is as follows: +One of the `DepositData` fields is `withdrawal_credentials`. -* `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX` -* `withdrawal_credentials[1:] == hash(withdrawal_pubkey)[1:]` where `withdrawal_pubkey` is a BLS pubkey +This field is a commitment to credentials for withdrawing validator balance (e.g. to another validator, or to shards). +The first byte of `withdrawal_credentials` is a version number. The remaining +bytes are content specific to the version. -The private key corresponding to `withdrawal_pubkey` will be required to initiate a withdrawal. It can be stored separately until a withdrawal is required, e.g. in cold storage. +Currently, `BLS_WITHDRAWAL_PREFIX` and `ETH1_ADDRESS_WITHDRAWAL_PREFIX` +versioned withdrawal credentials are supported. Read more in the [validator guide](./validator.md). + +*Note*: The deposit contract does *not* validate withdrawal credentials. +Thus, new versions can be added without modifications of the deposit contract. #### `DepositEvent` log diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index cb45e65e8..4fafcbad4 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -15,7 +15,9 @@ This is an accompanying document to [Ethereum 2.0 Phase 0 -- The Beacon Chain](. - [Becoming a validator](#becoming-a-validator) - [Initialization](#initialization) - [BLS public key](#bls-public-key) - - [BLS withdrawal key](#bls-withdrawal-key) + - [Withdrawal credentials](#withdrawal-credentials) + - [BLS key credentials](#bls-key-credentials) + - [Eth1 address credentials](#eth1-address-credentials) - [Submit deposit](#submit-deposit) - [Process deposit](#process-deposit) - [Validator index](#validator-index) @@ -100,14 +102,42 @@ A validator must initialize many parameters locally before submitting a deposit Validator public keys are [G1 points](beacon-chain.md#bls-signatures) on the [BLS12-381 curve](https://z.cash/blog/new-snark-curve). A private key, `privkey`, must be securely generated along with the resultant `pubkey`. This `privkey` must be "hot", that is, constantly available to sign data throughout the lifetime of the validator. -#### BLS withdrawal key +#### Withdrawal credentials -A secondary withdrawal private key, `withdrawal_privkey`, must also be securely generated along with the resultant `withdrawal_pubkey`. This `withdrawal_privkey` does not have to be available for signing during the normal lifetime of a validator and can live in "cold storage". +The `withdrawal_credentials` ultimately control the deposited ETH once the +validator has exited and is withdrawable. The 0th byte of this 32-byte field, +called the "withdrawal prefix" defines the withdrawal credential version +while the remaining 31 bytes define the version-specific content. -The validator constructs their `withdrawal_credentials` via the following: +The following withdrawal credentials versions are currently supported. The +validator must choose and construct a version for the `withdrawal_credentials` field. -* Set `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX`. -* Set `withdrawal_credentials[1:] == hash(withdrawal_pubkey)[1:]`. +##### BLS key credentials + +BLS withdrawal credentials are controlled by a secondary withdrawal BLS private key, `bls_withdrawal_privkey`. +This key is securely generated along with the resultant `bls_withdrawal_pubkey`. +This `withdrawal_privkey` does not have to be available for signing during +the normal lifetime of a validator and can live in "cold storage". + +The validator constructs `withdrawal_credentials` as the following: +* Set `withdrawal_credentials[:1] = BLS_WITHDRAWAL_PREFIX`. +* Set `withdrawal_credentials[1:] = hash(bls_withdrawal_pubkey)[1:]`. + +##### Eth1 address credentials + +Eth1 address credentials are controlled by an eth1 address. This can be an either an externally owned account or a contract. + +The withdrawal to the address specified will be a normal ETH transfer (with no payload other than the validator's ETH) +triggered by an eth1 transaction that will handle gas price/limit and payment of fees. + +As long as such a withdrawal account/contract can receive ETH transfers, +the future withdrawal protocol is agnostic to all other implementation details. + +The validator selects a 20-byte eth1 address, `eth1_withdrawal_address`, and constructs `withdrawal_credentials` as the following: +* Set `withdrawal_credentials[:1] = ETH1_ADDRESS_WITHDRAWAL_PREFIX`. +* Set `withdrawal_credentials[12:] = eth1_withdrawal_address`. + +*Note*: Bytes `withdrawal_credentials[1:12]` remain the default `0x00` value. ### Submit deposit diff --git a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py index b7a0de6c8..36e76f46c 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py +++ b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py @@ -94,6 +94,49 @@ def test_new_deposit_over_max(spec, state): yield from run_deposit_processing(spec, state, deposit, validator_index) +@with_all_phases +@spec_state_test +def test_new_deposit_eth1_withdrawal_credentials(spec, state): + # fresh deposit = next validator index = validator appended to registry + validator_index = len(state.validators) + withdrawal_credentials = ( + spec.ETH1_ADDRESS_WITHDRAWAL_PREFIX + + b'\x00' * 11 # specified 0s + + b'\x59' * 20 # a 20-byte eth1 address + ) + amount = spec.MAX_EFFECTIVE_BALANCE + deposit = prepare_state_and_deposit( + spec, state, + validator_index, + amount, + withdrawal_credentials=withdrawal_credentials, + signed=True, + ) + + yield from run_deposit_processing(spec, state, deposit, validator_index) + + +@with_all_phases +@spec_state_test +def test_new_deposit_non_versioned_withdrawal_credentials(spec, state): + # fresh deposit = next validator index = validator appended to registry + validator_index = len(state.validators) + withdrawal_credentials = ( + b'\xFF' # Non specified withdrawal credentials version + + b'\x02' * 31 # Garabage bytes + ) + amount = spec.MAX_EFFECTIVE_BALANCE + deposit = prepare_state_and_deposit( + spec, state, + validator_index, + amount, + withdrawal_credentials=withdrawal_credentials, + signed=True, + ) + + yield from run_deposit_processing(spec, state, deposit, validator_index) + + @with_all_phases @spec_state_test @always_bls From 784e567d4b3e2dc9bdccb85d792c3d1696ac83a1 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 12 Dec 2020 20:38:40 +0000 Subject: [PATCH 041/222] Fix table for withdrawal credentials prefixes Minor cosmetic fixes (misformated table, section title). --- specs/phase0/beacon-chain.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 7fdf027bc..8df10c80a 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -13,7 +13,7 @@ - [Misc](#misc) - [Gwei values](#gwei-values) - [Initial values](#initial-values) - - [Validator withdrawal credential versions](#validator-withdrawal-credential-versions) + - [Withdrawal credentials prefixes](#withdrawal-credentials-prefixes) - [Time parameters](#time-parameters) - [State list lengths](#state-list-lengths) - [Rewards and penalties](#rewards-and-penalties) @@ -211,8 +211,10 @@ The following values are (non-configurable) constants used throughout the specif | - | - | | `GENESIS_FORK_VERSION` | `Version('0x00000000')` | -### Validator withdrawal credential versions +### Withdrawal credentials prefixes +| Name | Value | +| - | - | | `BLS_WITHDRAWAL_PREFIX` | `Bytes1('0x00')` | | `ETH1_ADDRESS_WITHDRAWAL_PREFIX` | `Bytes1('0x01')` | From ef3802b8e7dbe40880fa29ee0977193b7bbc6982 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 12 Dec 2020 20:40:59 +0000 Subject: [PATCH 042/222] Cleaner section title "Withdrawal prefixes" matches `[BLS]/[ETH1_ADDRESS]_WITHDRAWAL_PREFIX` --- specs/phase0/beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 8df10c80a..c0e026e97 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -13,7 +13,7 @@ - [Misc](#misc) - [Gwei values](#gwei-values) - [Initial values](#initial-values) - - [Withdrawal credentials prefixes](#withdrawal-credentials-prefixes) + - [Withdrawal prefixes](#withdrawal-prefixes) - [Time parameters](#time-parameters) - [State list lengths](#state-list-lengths) - [Rewards and penalties](#rewards-and-penalties) @@ -211,7 +211,7 @@ The following values are (non-configurable) constants used throughout the specif | - | - | | `GENESIS_FORK_VERSION` | `Version('0x00000000')` | -### Withdrawal credentials prefixes +### Withdrawal prefixes | Name | Value | | - | - | From 2cc9a12c982089439adc1a29b991cc805e5c29fc Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 12 Dec 2020 20:50:10 +0000 Subject: [PATCH 043/222] Copy-edit deposit-contract.md --- specs/phase0/deposit-contract.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/specs/phase0/deposit-contract.md b/specs/phase0/deposit-contract.md index ef21d35e0..2228aed90 100644 --- a/specs/phase0/deposit-contract.md +++ b/specs/phase0/deposit-contract.md @@ -60,15 +60,12 @@ The amount of ETH (rounded down to the closest Gwei) sent to the deposit contrac One of the `DepositData` fields is `withdrawal_credentials`. -This field is a commitment to credentials for withdrawing validator balance (e.g. to another validator, or to shards). -The first byte of `withdrawal_credentials` is a version number. The remaining -bytes are content specific to the version. +This field is a commitment to credentials for withdrawing validator balance, e.g. to another validator, to an Ethereum 1.0 address, or to a shard. +The first byte of `withdrawal_credentials` is a withdrawal prefix which specifies the withdrawal type. The remaining 31 bytes are specific to the withdrawal prefix. -Currently, `BLS_WITHDRAWAL_PREFIX` and `ETH1_ADDRESS_WITHDRAWAL_PREFIX` -versioned withdrawal credentials are supported. Read more in the [validator guide](./validator.md). +The withdrawal prefixes currently supported are `BLS_WITHDRAWAL_PREFIX` and `ETH1_ADDRESS_WITHDRAWAL_PREFIX`. Read more in the [validator guide](./validator.md). -*Note*: The deposit contract does *not* validate withdrawal credentials. -Thus, new versions can be added without modifications of the deposit contract. +*Note*: The deposit contract does not validate withdrawal credentials. Support for new withdrawal types can be added without modifying the deposit contract. #### `DepositEvent` log From ae0b10edb7143b0f285b33cf72548f083d0534f8 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 12 Dec 2020 21:46:39 +0000 Subject: [PATCH 044/222] Update validator.md --- specs/phase0/validator.md | 38 +++++++++++++------------------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 4fafcbad4..d68c4b4af 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -104,40 +104,28 @@ Validator public keys are [G1 points](beacon-chain.md#bls-signatures) on the [BL #### Withdrawal credentials -The `withdrawal_credentials` ultimately control the deposited ETH once the -validator has exited and is withdrawable. The 0th byte of this 32-byte field, -called the "withdrawal prefix" defines the withdrawal credential version -while the remaining 31 bytes define the version-specific content. +The `withdrawal_credentials` field specifies how a validator's withdrawable balance may be withdrawn. The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. The following withdrawal prefixes are currently supported. -The following withdrawal credentials versions are currently supported. The -validator must choose and construct a version for the `withdrawal_credentials` field. +##### `BLS_WITHDRAWAL_PREFIX` -##### BLS key credentials +Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair `(bls_withdrawal_privkey, bls_withdrawal_pubkey)` to trigger withdrawals. The `withdrawal_credentials` field must be constructed such that: -BLS withdrawal credentials are controlled by a secondary withdrawal BLS private key, `bls_withdrawal_privkey`. -This key is securely generated along with the resultant `bls_withdrawal_pubkey`. -This `withdrawal_privkey` does not have to be available for signing during -the normal lifetime of a validator and can live in "cold storage". +* `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX` +* `withdrawal_credentials[1:] == hash(bls_withdrawal_pubkey)[1:]` -The validator constructs `withdrawal_credentials` as the following: -* Set `withdrawal_credentials[:1] = BLS_WITHDRAWAL_PREFIX`. -* Set `withdrawal_credentials[1:] = hash(bls_withdrawal_pubkey)[1:]`. +*Note*: The `bls_withdrawal_pubkey` is not required for validating and can be kept in cold storage. -##### Eth1 address credentials +##### `ETH1_ADDRESS_WITHDRAWAL_PREFIX` -Eth1 address credentials are controlled by an eth1 address. This can be an either an externally owned account or a contract. +Withdrawal credentials with the Eth1 address withdrawal prefix specify a 20-byte Eth1 address `eth1_withdrawal_address` as the recipient for all withdrawals. The `eth1_withdrawal_address` can be the address of an externally owned account or of a contract. The `withdrawal_credentials` field must be constructed such that: -The withdrawal to the address specified will be a normal ETH transfer (with no payload other than the validator's ETH) -triggered by an eth1 transaction that will handle gas price/limit and payment of fees. +* `withdrawal_credentials[:1] = ETH1_ADDRESS_WITHDRAWAL_PREFIX` +* `withdrawal_credentials[1:12] == Bytes32()[1:12]` +* `withdrawal_credentials[12:] = eth1_withdrawal_address` -As long as such a withdrawal account/contract can receive ETH transfers, -the future withdrawal protocol is agnostic to all other implementation details. +Withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) triggered by an Eth1 transaction that will handle gas price/limit and payment of fees. As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers the future withdrawal protocol is agnostic to all other implementation details. -The validator selects a 20-byte eth1 address, `eth1_withdrawal_address`, and constructs `withdrawal_credentials` as the following: -* Set `withdrawal_credentials[:1] = ETH1_ADDRESS_WITHDRAWAL_PREFIX`. -* Set `withdrawal_credentials[12:] = eth1_withdrawal_address`. - -*Note*: Bytes `withdrawal_credentials[1:12]` remain the default `0x00` value. +**TODO**: Be explicit about the endianness of `eth1_withdrawal_address`. ### Submit deposit From b085436f2f4db99d8b7bf226e07bab11642aec60 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 12 Dec 2020 21:51:21 +0000 Subject: [PATCH 045/222] Update validator.md --- specs/phase0/validator.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index d68c4b4af..8fb13ead1 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -14,8 +14,8 @@ This is an accompanying document to [Ethereum 2.0 Phase 0 -- The Beacon Chain](. - [Misc](#misc) - [Becoming a validator](#becoming-a-validator) - [Initialization](#initialization) - - [BLS public key](#bls-public-key) - - [Withdrawal credentials](#withdrawal-credentials) + - [`BLS_WITHDRAWAL_PREFIX`](#bls_withdrawal_prefix) + - [`ETH1_ADDRESS_WITHDRAWAL_PREFIX`](#eth1_address_withdrawal_prefix) - [BLS key credentials](#bls-key-credentials) - [Eth1 address credentials](#eth1-address-credentials) - [Submit deposit](#submit-deposit) @@ -104,7 +104,7 @@ Validator public keys are [G1 points](beacon-chain.md#bls-signatures) on the [BL #### Withdrawal credentials -The `withdrawal_credentials` field specifies how a validator's withdrawable balance may be withdrawn. The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. The following withdrawal prefixes are currently supported. +The `withdrawal_credentials` field contrains how a validator's withdrawable balance may be withdrawn. The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. The following withdrawal prefixes are currently supported. ##### `BLS_WITHDRAWAL_PREFIX` @@ -113,7 +113,7 @@ Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair `(bls * `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX` * `withdrawal_credentials[1:] == hash(bls_withdrawal_pubkey)[1:]` -*Note*: The `bls_withdrawal_pubkey` is not required for validating and can be kept in cold storage. +*Note*: The `bls_withdrawal_privkey` is not required for validating and can be kept in cold storage. ##### `ETH1_ADDRESS_WITHDRAWAL_PREFIX` @@ -123,7 +123,7 @@ Withdrawal credentials with the Eth1 address withdrawal prefix specify a 20-byte * `withdrawal_credentials[1:12] == Bytes32()[1:12]` * `withdrawal_credentials[12:] = eth1_withdrawal_address` -Withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) triggered by an Eth1 transaction that will handle gas price/limit and payment of fees. As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers the future withdrawal protocol is agnostic to all other implementation details. +Withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) triggered by an Eth1 transaction that will handle the gas price and gas limit, as well the payment of fees. As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers the future withdrawal protocol is agnostic to all other implementation details. **TODO**: Be explicit about the endianness of `eth1_withdrawal_address`. From f9edecf23c5c18bc0f1871581275d0da4aaaf073 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 12 Dec 2020 21:56:30 +0000 Subject: [PATCH 046/222] Update validator.md --- specs/phase0/validator.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 8fb13ead1..fb40dd94f 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -104,11 +104,11 @@ Validator public keys are [G1 points](beacon-chain.md#bls-signatures) on the [BL #### Withdrawal credentials -The `withdrawal_credentials` field contrains how a validator's withdrawable balance may be withdrawn. The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. The following withdrawal prefixes are currently supported. +The `withdrawal_credentials` field constrains validator withdrawals. The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. The following withdrawal prefixes are currently supported. ##### `BLS_WITHDRAWAL_PREFIX` -Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair `(bls_withdrawal_privkey, bls_withdrawal_pubkey)` to trigger withdrawals. The `withdrawal_credentials` field must be constructed such that: +Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair `(bls_withdrawal_privkey, bls_withdrawal_pubkey)` to trigger withdrawals. The `withdrawal_credentials` field must be such that: * `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX` * `withdrawal_credentials[1:] == hash(bls_withdrawal_pubkey)[1:]` @@ -117,11 +117,11 @@ Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair `(bls ##### `ETH1_ADDRESS_WITHDRAWAL_PREFIX` -Withdrawal credentials with the Eth1 address withdrawal prefix specify a 20-byte Eth1 address `eth1_withdrawal_address` as the recipient for all withdrawals. The `eth1_withdrawal_address` can be the address of an externally owned account or of a contract. The `withdrawal_credentials` field must be constructed such that: +Withdrawal credentials with the Eth1 address withdrawal prefix specify a 20-byte Eth1 address `eth1_withdrawal_address` as the recipient for all withdrawals. The `eth1_withdrawal_address` can be the address of either an externally owned account or of a contract. The `withdrawal_credentials` field must be such that: -* `withdrawal_credentials[:1] = ETH1_ADDRESS_WITHDRAWAL_PREFIX` -* `withdrawal_credentials[1:12] == Bytes32()[1:12]` -* `withdrawal_credentials[12:] = eth1_withdrawal_address` +* `withdrawal_credentials[:1] == ETH1_ADDRESS_WITHDRAWAL_PREFIX` +* `withdrawal_credentials[1:12] == b'\x00' * 11` +* `withdrawal_credentials[12:] == eth1_withdrawal_address` Withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) triggered by an Eth1 transaction that will handle the gas price and gas limit, as well the payment of fees. As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers the future withdrawal protocol is agnostic to all other implementation details. From 85c1347ffc5f00d44829538d62626bedf0f0edc1 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 12 Dec 2020 21:57:58 +0000 Subject: [PATCH 047/222] Update validator.md --- specs/phase0/validator.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index fb40dd94f..3f60fc386 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -14,10 +14,10 @@ This is an accompanying document to [Ethereum 2.0 Phase 0 -- The Beacon Chain](. - [Misc](#misc) - [Becoming a validator](#becoming-a-validator) - [Initialization](#initialization) - - [`BLS_WITHDRAWAL_PREFIX`](#bls_withdrawal_prefix) - - [`ETH1_ADDRESS_WITHDRAWAL_PREFIX`](#eth1_address_withdrawal_prefix) - - [BLS key credentials](#bls-key-credentials) - - [Eth1 address credentials](#eth1-address-credentials) + - [BLS public key](#bls-public-key) + - [Withdrawal credentials](#withdrawal-credentials) + - [`BLS_WITHDRAWAL_PREFIX`](#bls_withdrawal_prefix) + - [`ETH1_ADDRESS_WITHDRAWAL_PREFIX`](#eth1_address_withdrawal_prefix) - [Submit deposit](#submit-deposit) - [Process deposit](#process-deposit) - [Validator index](#validator-index) From 645a3851af222a229936d6c93b9a4e7ce5839afb Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 12 Dec 2020 22:02:56 +0000 Subject: [PATCH 048/222] Update deposit-contract.md --- specs/phase0/deposit-contract.md | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/specs/phase0/deposit-contract.md b/specs/phase0/deposit-contract.md index 2228aed90..5c35da8d4 100644 --- a/specs/phase0/deposit-contract.md +++ b/specs/phase0/deposit-contract.md @@ -58,14 +58,9 @@ The amount of ETH (rounded down to the closest Gwei) sent to the deposit contrac #### Withdrawal credentials -One of the `DepositData` fields is `withdrawal_credentials`. +One of the `DepositData` fields is `withdrawal_credentials` which constrains validator withdrawals. The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. The withdrawal prefixes currently supported are `BLS_WITHDRAWAL_PREFIX` and `ETH1_ADDRESS_WITHDRAWAL_PREFIX`. Read more in the [validator guide](./validator.md#withdrawal-credentials). -This field is a commitment to credentials for withdrawing validator balance, e.g. to another validator, to an Ethereum 1.0 address, or to a shard. -The first byte of `withdrawal_credentials` is a withdrawal prefix which specifies the withdrawal type. The remaining 31 bytes are specific to the withdrawal prefix. - -The withdrawal prefixes currently supported are `BLS_WITHDRAWAL_PREFIX` and `ETH1_ADDRESS_WITHDRAWAL_PREFIX`. Read more in the [validator guide](./validator.md). - -*Note*: The deposit contract does not validate withdrawal credentials. Support for new withdrawal types can be added without modifying the deposit contract. +*Note*: The deposit contract does not validate the `withdrawal_credentials` field. Support for new withdrawal prefixes can be added without modifying the deposit contract. #### `DepositEvent` log From 2c90ffa2f499569fdaf7fea6af1a0e30ff14ae59 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 14 Dec 2020 13:09:25 -0700 Subject: [PATCH 049/222] copy edits --- specs/phase0/deposit-contract.md | 8 ++++++-- specs/phase0/validator.md | 21 +++++++++++++++------ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/specs/phase0/deposit-contract.md b/specs/phase0/deposit-contract.md index 5c35da8d4..02e762dae 100644 --- a/specs/phase0/deposit-contract.md +++ b/specs/phase0/deposit-contract.md @@ -58,9 +58,13 @@ The amount of ETH (rounded down to the closest Gwei) sent to the deposit contrac #### Withdrawal credentials -One of the `DepositData` fields is `withdrawal_credentials` which constrains validator withdrawals. The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. The withdrawal prefixes currently supported are `BLS_WITHDRAWAL_PREFIX` and `ETH1_ADDRESS_WITHDRAWAL_PREFIX`. Read more in the [validator guide](./validator.md#withdrawal-credentials). +One of the `DepositData` fields is `withdrawal_credentials` which constrains validator withdrawals. +The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. +The withdrawal prefixes currently supported are `BLS_WITHDRAWAL_PREFIX` and `ETH1_ADDRESS_WITHDRAWAL_PREFIX`. +Read more in the [validator guide](./validator.md#withdrawal-credentials). -*Note*: The deposit contract does not validate the `withdrawal_credentials` field. Support for new withdrawal prefixes can be added without modifying the deposit contract. +*Note*: The deposit contract does not validate the `withdrawal_credentials` field. +Support for new withdrawal prefixes can be added without modifying the deposit contract. #### `DepositEvent` log diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 3f60fc386..5106c2b73 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -104,11 +104,16 @@ Validator public keys are [G1 points](beacon-chain.md#bls-signatures) on the [BL #### Withdrawal credentials -The `withdrawal_credentials` field constrains validator withdrawals. The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. The following withdrawal prefixes are currently supported. +The `withdrawal_credentials` field constrains validator withdrawals. +The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. + +The following withdrawal prefixes are currently supported. ##### `BLS_WITHDRAWAL_PREFIX` -Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair `(bls_withdrawal_privkey, bls_withdrawal_pubkey)` to trigger withdrawals. The `withdrawal_credentials` field must be such that: +Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair +`(bls_withdrawal_privkey, bls_withdrawal_pubkey)` to trigger withdrawals. +The `withdrawal_credentials` field must be such that: * `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX` * `withdrawal_credentials[1:] == hash(bls_withdrawal_pubkey)[1:]` @@ -117,15 +122,19 @@ Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair `(bls ##### `ETH1_ADDRESS_WITHDRAWAL_PREFIX` -Withdrawal credentials with the Eth1 address withdrawal prefix specify a 20-byte Eth1 address `eth1_withdrawal_address` as the recipient for all withdrawals. The `eth1_withdrawal_address` can be the address of either an externally owned account or of a contract. The `withdrawal_credentials` field must be such that: +Withdrawal credentials with the Eth1 address withdrawal prefix specify +a 20-byte Eth1 address `eth1_withdrawal_address` as the recipient for all withdrawals. +The `eth1_withdrawal_address` can be the address of either an externally owned account or of a contract. +The `withdrawal_credentials` field must be such that: * `withdrawal_credentials[:1] == ETH1_ADDRESS_WITHDRAWAL_PREFIX` * `withdrawal_credentials[1:12] == b'\x00' * 11` * `withdrawal_credentials[12:] == eth1_withdrawal_address` -Withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) triggered by an Eth1 transaction that will handle the gas price and gas limit, as well the payment of fees. As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers the future withdrawal protocol is agnostic to all other implementation details. - -**TODO**: Be explicit about the endianness of `eth1_withdrawal_address`. +Withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) +triggered by an Eth1 transaction that will handle the gas price and gas limit, as well the payment of fees. +As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers +the future withdrawal protocol is agnostic to all other implementation details. ### Submit deposit From acfe49e3f311b02e127a31d0a453a07bb810bd49 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 15 Dec 2020 13:18:20 +0800 Subject: [PATCH 050/222] executable light client patch: beacon-chain.md (#2141) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Bump remerkleable to 0.1.18 * Disable `sync-protocol.md` for now. Make linter pass * Enable lightclient tests * Use *new* `optional_fast_aggregate_verify` * Fix ToC and codespell * Do not run phase1 tests with Lightclient patch * Fix the Eth1Data casting bug. Add a workaround. * Fix `run_on_attestation` testing helper * Revert * Rename `optional_fast_aggregate_verify` to `eth2_fast_aggregate_verify` * Apply Proto's suggestion * Apply Danny's suggestion * Fixing tests * Fix after rebasing * Rename `LIGHTCLIENT` -> `LIGHTCLIENT_PATCH` * New doctoc * Add lightclient patch configs * fix gitignore light client patch generator output * Upgrade state for light client patch * Add `lightclient-fork.md` to deal the fork boundary and fix `process_block_header` * Misc cleanups 1) Add a summary note for every function that is changed. 2) Avoid changing `process_block` (instead only change `process_block_header`). 3) Rename `G2_INFINITY_POINT_SIG` to `G2_POINT_AT_INFINITY` to avoid `SIG` contraction. 4) Misc cleanups * Update block.py * Update beacon-chain.md * Fix typo "minimal" -> "mainnet" Co-authored-by: Marin Petrunić * Use the new `BeaconBlockHeader` instead of phase 0 version * Update config files * Move `sync_committee_bits` and `sync_committee_signature` back to `BeaconBlockBody` Co-authored-by: protolambda Co-authored-by: Justin Co-authored-by: Marin Petrunić --- .gitignore | 2 +- Makefile | 6 +- configs/mainnet/lightclient_patch.yaml | 21 ++++ configs/minimal/lightclient_patch.yaml | 21 ++++ setup.py | 11 ++- specs/lightclient/beacon-chain.md | 95 ++++++++++++++----- specs/lightclient/lightclient-fork.md | 83 ++++++++++++++++ specs/lightclient/sync-protocol.md | 3 +- specs/phase0/validator.md | 4 +- tests/core/pyspec/eth2spec/test/context.py | 20 +++- .../pyspec/eth2spec/test/helpers/block.py | 5 + .../pyspec/eth2spec/test/helpers/rewards.py | 7 +- .../test_process_rewards_and_penalties.py | 12 ++- .../fork_choice/test_on_attestation.py | 6 +- .../test_process_attestation.py | 5 +- .../test_process_chunk_challenge.py | 23 ++--- .../test_process_custody_key_reveal.py | 11 ++- .../test_process_custody_slashing.py | 11 ++- ...est_process_early_derived_secret_reveal.py | 17 ++-- .../test_process_shard_transition.py | 9 +- .../test_process_challenge_deadlines.py | 3 +- .../test_process_custody_final_updates.py | 9 +- .../test_process_reveal_deadlines.py | 5 +- .../test/phase1/sanity/test_blocks.py | 16 ++-- .../test/phase1/sanity/test_shard_blocks.py | 23 ++--- .../fork_choice/test_on_shard_block.py | 15 ++- .../phase1/unittests/test_get_start_shard.py | 11 ++- 27 files changed, 340 insertions(+), 114 deletions(-) create mode 100644 configs/mainnet/lightclient_patch.yaml create mode 100644 configs/minimal/lightclient_patch.yaml create mode 100644 specs/lightclient/lightclient-fork.md diff --git a/.gitignore b/.gitignore index ed497112c..17d058225 100644 --- a/.gitignore +++ b/.gitignore @@ -17,7 +17,7 @@ eth2.0-spec-tests/ # Dynamically built from Markdown spec tests/core/pyspec/eth2spec/phase0/ tests/core/pyspec/eth2spec/phase1/ -tests/core/pyspec/eth2spec/lightclient/ +tests/core/pyspec/eth2spec/lightclient_patch/ # coverage reports .htmlcov diff --git a/Makefile b/Makefile index 811abbad8..987650948 100644 --- a/Makefile +++ b/Makefile @@ -86,11 +86,11 @@ install_test: test: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec + python -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov=eth2spec.lightclient_patch.spec -cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec find_test: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec + python -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov=eth2spec.lightclient_patch.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec citest: pyspec mkdir -p tests/core/pyspec/test-reports/eth2spec; . venv/bin/activate; cd $(PY_SPEC_DIR); \ @@ -113,7 +113,7 @@ codespell: lint: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ flake8 --config $(LINTER_CONFIG_FILE) ./eth2spec \ - && mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.phase1 -p eth2spec.lightclient + && mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.phase1 -p eth2spec.lightclient_patch lint_generators: pyspec . venv/bin/activate; cd $(TEST_GENERATORS_DIR); \ diff --git a/configs/mainnet/lightclient_patch.yaml b/configs/mainnet/lightclient_patch.yaml new file mode 100644 index 000000000..64c05a720 --- /dev/null +++ b/configs/mainnet/lightclient_patch.yaml @@ -0,0 +1,21 @@ +# Mainnet preset - lightclient patch + +CONFIG_NAME: "mainnet" + +# Misc +# --------------------------------------------------------------- +# 2**10 (=1,024) +SYNC_COMMITTEE_SIZE: 1024 +# 2**6 (=64) +SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE: 64 + + +# Time parameters +# --------------------------------------------------------------- +# 2**8 (= 256) +EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 256 + + +# Signature domains +# --------------------------------------------------------------- +DOMAIN_SYNC_COMMITTEE: 0x07000000 diff --git a/configs/minimal/lightclient_patch.yaml b/configs/minimal/lightclient_patch.yaml new file mode 100644 index 000000000..ba1179a2b --- /dev/null +++ b/configs/minimal/lightclient_patch.yaml @@ -0,0 +1,21 @@ +# Minimal preset - lightclient patch + +CONFIG_NAME: "minimal" + +# Misc +# --------------------------------------------------------------- +# [customized] +SYNC_COMMITTEE_SIZE: 64 +# [customized] +SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE: 16 + + +# Time parameters +# --------------------------------------------------------------- +# 2**8 (= 256) +EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 256 + + +# Signature domains +# --------------------------------------------------------------- +DOMAIN_SYNC_COMMITTEE: 0x07000000 diff --git a/setup.py b/setup.py index 6a2bf5707..bd043dccc 100644 --- a/setup.py +++ b/setup.py @@ -173,7 +173,7 @@ from lru import LRU from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes from eth2spec.utils.ssz.ssz_typing import ( View, boolean, Container, List, Vector, uint8, uint32, uint64, - Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, + Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils import bls @@ -386,7 +386,7 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: fork_imports = { 'phase0': PHASE0_IMPORTS, 'phase1': PHASE1_IMPORTS, - 'lightclient': LIGHTCLIENT_IMPORT, + 'lightclient_patch': LIGHTCLIENT_IMPORT, } @@ -453,15 +453,16 @@ class PySpecCommand(Command): specs/phase1/shard-fork-choice.md specs/phase1/validator.md """ - elif self.spec_fork == "lightclient": + elif self.spec_fork == "lightclient_patch": self.md_doc_paths = """ specs/phase0/beacon-chain.md specs/phase0/fork-choice.md specs/phase0/validator.md specs/phase0/weak-subjectivity.md specs/lightclient/beacon-chain.md - specs/lightclient/sync-protocol.md + specs/lightclient/lightclient-fork.md """ + # TODO: add specs/lightclient/sync-protocol.md back when the GeneralizedIndex helpers are included. else: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) @@ -584,7 +585,7 @@ setup( "py_ecc==5.0.0", "milagro_bls_binding==1.5.0", "dataclasses==0.6", - "remerkleable==0.1.17", + "remerkleable==0.1.18", "ruamel.yaml==0.16.5", "lru-dict==1.1.6" ] diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 25952599d..4eb1f24ab 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -6,10 +6,10 @@ - - [Introduction](#introduction) - [Constants](#constants) - [Configuration](#configuration) + - [Constants](#constants-1) - [Misc](#misc) - [Time parameters](#time-parameters) - [Domain types](#domain-types) @@ -20,12 +20,15 @@ - [New containers](#new-containers) - [`SyncCommittee`](#synccommittee) - [Helper functions](#helper-functions) + - [`Predicates`](#predicates) + - [`eth2_fast_aggregate_verify`](#eth2_fast_aggregate_verify) - [Beacon state accessors](#beacon-state-accessors) - [`get_sync_committee_indices`](#get_sync_committee_indices) - [`get_sync_committee`](#get_sync_committee) - [Block processing](#block-processing) - [Sync committee processing](#sync-committee-processing) - [Epoch processing](#epoch-processing) + - [Components of attestation deltas](#components-of-attestation-deltas) - [Final updates](#final-updates) @@ -43,6 +46,12 @@ This is a standalone beacon chain patch adding light client support via sync com ## Configuration +### Constants + +| Name | Value | +| - | - | +| `G2_POINT_AT_INFINITY` | `BLSSignature(b'\xc0' + b'\x00' * 95)` | + ### Misc | Name | Value | @@ -69,25 +78,15 @@ This is a standalone beacon chain patch adding light client support via sync com *Note*: Extended SSZ containers inherit all fields from the parent in the original order and append any additional fields to the end. -#### `BeaconBlock` +#### `BeaconBlockBody` ```python -class BeaconBlock(phase0.BeaconBlock): +class BeaconBlockBody(phase0.BeaconBlockBody): # Sync committee aggregate signature sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE] sync_committee_signature: BLSSignature ``` -#### `BeaconBlockHeader` - -```python -class BeaconBlockHeader(phase0.BeaconBlockHeader): - # Sync committee aggregate signature - sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE] - sync_committee_signature: BLSSignature -``` - - #### `BeaconState` ```python @@ -109,6 +108,20 @@ class SyncCommittee(Container): ## Helper functions +### `Predicates` + +#### `eth2_fast_aggregate_verify` + +```python +def eth2_fast_aggregate_verify(pubkeys: Sequence[BLSPubkey], message: Bytes32, signature: BLSSignature) -> bool: + """ + Wrapper to ``bls.FastAggregateVerify`` accepting the ``G2_POINT_AT_INFINITY`` signature when ``pubkeys`` is empty. + """ + if len(pubkeys) == 0 and signature == G2_POINT_AT_INFINITY: + return True + return bls.FastAggregateVerify(pubkeys, message, signature) +``` + ### Beacon state accessors #### `get_sync_committee_indices` @@ -117,13 +130,14 @@ class SyncCommittee(Container): def get_sync_committee_indices(state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: """ Return the sync committee indices for a given state and epoch. - """ + """ MAX_RANDOM_BYTE = 2**8 - 1 base_epoch = Epoch((max(epoch // EPOCHS_PER_SYNC_COMMITTEE_PERIOD, 1) - 1) * EPOCHS_PER_SYNC_COMMITTEE_PERIOD) active_validator_indices = get_active_validator_indices(state, base_epoch) active_validator_count = uint64(len(active_validator_indices)) seed = get_seed(state, base_epoch, DOMAIN_SYNC_COMMITTEE) - i, sync_committee_indices = 0, [] + i = 0 + sync_committee_indices: List[ValidatorIndex] = [] while len(sync_committee_indices) < SYNC_COMMITTEE_SIZE: shuffled_index = compute_shuffled_index(uint64(i % active_validator_count), active_validator_count, seed) candidate_index = active_validator_indices[shuffled_index] @@ -156,41 +170,74 @@ def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: ```python def process_block(state: BeaconState, block: BeaconBlock) -> None: - phase0.process_block(state, block) - process_sync_committee(state, block) + process_block_header(state, block) + process_randao(state, block.body) + process_eth1_data(state, block.body) + process_operations(state, block.body) + # Light client support + process_sync_committee(state, block.body) ``` #### Sync committee processing ```python -def process_sync_committee(state: BeaconState, block: BeaconBlock) -> None: +def process_sync_committee(state: BeaconState, body: BeaconBlockBody) -> None: # Verify sync committee aggregate signature signing over the previous slot block root - previous_slot = Slot(max(state.slot, 1) - 1) + previous_slot = Slot(max(int(state.slot), 1) - 1) committee_indices = get_sync_committee_indices(state, get_current_epoch(state)) participant_indices = [index for index, bit in zip(committee_indices, body.sync_committee_bits) if bit] committee_pubkeys = state.current_sync_committee.pubkeys participant_pubkeys = [pubkey for pubkey, bit in zip(committee_pubkeys, body.sync_committee_bits) if bit] domain = get_domain(state, DOMAIN_SYNC_COMMITTEE, compute_epoch_at_slot(previous_slot)) signing_root = compute_signing_root(get_block_root_at_slot(state, previous_slot), domain) - assert bls.FastAggregateVerify(participant_pubkeys, signing_root, block.sync_committee_signature) + assert eth2_fast_aggregate_verify(participant_pubkeys, signing_root, body.sync_committee_signature) # Reward sync committee participants - participant_rewards = Gwei(0) + proposer_reward = Gwei(0) active_validator_count = uint64(len(get_active_validator_indices(state, get_current_epoch(state)))) for participant_index in participant_indices: base_reward = get_base_reward(state, participant_index) - reward = Gwei(base_reward * active_validator_count // len(committee_indices) // SLOTS_PER_EPOCH) + max_participant_reward = base_reward - base_reward // PROPOSER_REWARD_QUOTIENT + reward = Gwei(max_participant_reward * active_validator_count // len(committee_indices) // SLOTS_PER_EPOCH) increase_balance(state, participant_index, reward) - participant_rewards += reward + proposer_reward += base_reward // PROPOSER_REWARD_QUOTIENT # Reward beacon proposer - increase_balance(state, get_beacon_proposer_index(state), Gwei(participant_rewards // PROPOSER_REWARD_QUOTIENT)) + increase_balance(state, get_beacon_proposer_index(state), proposer_reward) ``` ### Epoch processing +#### Components of attestation deltas + +*Note*: The function `get_inactivity_penalty_deltas` is modified with `BASE_REWARDS_PER_EPOCH` replaced by `BASE_REWARDS_PER_EPOCH - 1`. + +```python +def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Return inactivity reward/penalty deltas for each validator. + """ + penalties = [Gwei(0) for _ in range(len(state.validators))] + if is_in_inactivity_leak(state): + matching_target_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) + matching_target_attesting_indices = get_unslashed_attesting_indices(state, matching_target_attestations) + for index in get_eligible_validator_indices(state): + # Penalize validator so that optimal attestation performance is rewarded with one base reward per epoch + base_reward = get_base_reward(state, index) + penalties[index] += Gwei((BASE_REWARDS_PER_EPOCH - 1) * base_reward - get_proposer_reward(state, index)) + if index not in matching_target_attesting_indices: + effective_balance = state.validators[index].effective_balance + penalties[index] += Gwei(effective_balance * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) + + # No rewards associated with inactivity penalties + rewards = [Gwei(0) for _ in range(len(state.validators))] + return rewards, penalties +``` + #### Final updates +*Note*: The function `process_final_updates` is modified to handle sync committee updates. + ```python def process_final_updates(state: BeaconState) -> None: phase0.process_final_updates(state) diff --git a/specs/lightclient/lightclient-fork.md b/specs/lightclient/lightclient-fork.md new file mode 100644 index 000000000..bb67fa54a --- /dev/null +++ b/specs/lightclient/lightclient-fork.md @@ -0,0 +1,83 @@ +# Ethereum 2.0 Light Client Support -- From Phase 0 to Light Client Patch + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + +- [Introduction](#introduction) +- [Configuration](#configuration) +- [Fork to Light-client patch](#fork-to-light-client-patch) + - [Fork trigger](#fork-trigger) + - [Upgrading the state](#upgrading-the-state) + + + +## Introduction + +This document describes the process of moving from Phase 0 to Phase 1 of Ethereum 2.0. + +## Configuration + +Warning: this configuration is not definitive. + +| Name | Value | +| - | - | +| `LIGHTCLIENT_PATCH_FORK_VERSION` | `Version('0x01000000')` | +| `LIGHTCLIENT_PATCH_FORK_SLOT` | `Slot(0)` **TBD** | + +## Fork to Light-client patch + +### Fork trigger + +TBD. Social consensus, along with state conditions such as epoch boundary, finality, deposits, active validator count, etc. may be part of the decision process to trigger the fork. For now we assume the condition will be triggered at slot `LIGHTCLIENT_PATCH_FORK_SLOT`, where `LIGHTCLIENT_PATCH_FORK_SLOT % SLOTS_PER_EPOCH == 0`. + +### Upgrading the state + +After `process_slots` of Phase 0 finishes, if `state.slot == LIGHTCLIENT_PATCH_FORK_SLOT`, an irregular state change is made to upgrade to light-client patch. + +```python +def upgrade_to_lightclient_patch(pre: phase0.BeaconState) -> BeaconState: + epoch = get_current_epoch(pre) + post = BeaconState( + genesis_time=pre.genesis_time, + slot=pre.slot, + fork=Fork( + previous_version=pre.fork.current_version, + current_version=LIGHTCLIENT_PATCH_FORK_VERSION, + epoch=epoch, + ), + # History + latest_block_header=pre.latest_block_header, + block_roots=pre.block_roots, + state_roots=pre.state_roots, + historical_roots=pre.historical_roots, + # Eth1 + eth1_data=pre.eth1_data, + eth1_data_votes=pre.eth1_data_votes, + eth1_deposit_index=pre.eth1_deposit_index, + # Registry + validators=pre.validators, + balances=pre.balances, + # Randomness + randao_mixes=pre.randao_mixes, + # Slashings + slashings=pre.slashings, + # Attestations + # previous_epoch_attestations is cleared on upgrade. + previous_epoch_attestations=List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH](), + # empty in pre state, since the upgrade is performed just after an epoch boundary. + current_epoch_attestations=List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH](), + # Finality + justification_bits=pre.justification_bits, + previous_justified_checkpoint=pre.previous_justified_checkpoint, + current_justified_checkpoint=pre.current_justified_checkpoint, + finalized_checkpoint=pre.finalized_checkpoint, + # Light-client + current_sync_committee=SyncCommittee(), + next_sync_committee=SyncCommittee(), + ) + return post +``` diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index 84b6f6fd7..310aad2df 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -8,7 +8,6 @@ - - [Introduction](#introduction) - [Constants](#constants) - [Configuration](#configuration) @@ -100,7 +99,7 @@ class LightClientStore(Container): ## Light client state updates -A light client maintains its state in a `store` object of type `LightClientStore` and receives `update` objects of type `LightClientUpdate`. Every `update` triggers `process_light_client_update(store, update, current_slot)` where `current_slot` is the currect slot based on some local clock. +A light client maintains its state in a `store` object of type `LightClientStore` and receives `update` objects of type `LightClientUpdate`. Every `update` triggers `process_light_client_update(store, update, current_slot)` where `current_slot` is the current slot based on some local clock. #### `is_valid_light_client_update` diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index cb45e65e8..97b1acd92 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -327,7 +327,9 @@ def get_eth1_vote(state: BeaconState, eth1_chain: Sequence[Eth1Block]) -> Eth1Da valid_votes = [vote for vote in state.eth1_data_votes if vote in votes_to_consider] # Default vote on latest eth1 block data in the period range unless eth1 chain is not live - default_vote = votes_to_consider[len(votes_to_consider) - 1] if any(votes_to_consider) else state.eth1_data + # Non-substantive casting for linter + state_eth1_data: Eth1Data = state.eth1_data + default_vote = votes_to_consider[len(votes_to_consider) - 1] if any(votes_to_consider) else state_eth1_data return max( valid_votes, diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index cfd6724ed..d19547477 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -2,6 +2,7 @@ import pytest from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 +from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.utils import bls from .exceptions import SkippedTest @@ -19,6 +20,7 @@ from importlib import reload def reload_specs(): reload(spec_phase0) reload(spec_phase1) + reload(spec_lightclient_patch) # Some of the Spec module functionality is exposed here to deal with phase-specific changes. @@ -28,7 +30,9 @@ ConfigName = NewType("ConfigName", str) PHASE0 = SpecForkName('phase0') PHASE1 = SpecForkName('phase1') -ALL_PHASES = (PHASE0, PHASE1) +LIGHTCLIENT_PATCH = SpecForkName('lightclient_patch') + +ALL_PHASES = (PHASE0, PHASE1, LIGHTCLIENT_PATCH) MAINNET = ConfigName('mainnet') MINIMAL = ConfigName('minimal') @@ -47,14 +51,18 @@ class SpecPhase0(Spec): class SpecPhase1(Spec): - def upgrade_to_phase1(self, state: spec_phase0.BeaconState) -> spec_phase1.BeaconState: - ... + ... + + +class SpecLightclient(Spec): + ... # add transfer, bridge, etc. as the spec evolves class SpecForks(TypedDict, total=False): PHASE0: SpecPhase0 PHASE1: SpecPhase1 + LIGHTCLIENT_PATCH: SpecLightclient def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Callable[[Any], int], @@ -70,6 +78,8 @@ def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Ca # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper. # Decide based on performance/consistency results later. state = phases[PHASE1].upgrade_to_phase1(state) + elif spec.fork == LIGHTCLIENT_PATCH: # not generalizing this just yet, unclear final spec fork/patch order. + state = phases[LIGHTCLIENT_PATCH].upgrade_to_lightclient_patch(state) return state @@ -337,12 +347,16 @@ def with_phases(phases, other_phases=None): phase_dir[PHASE0] = spec_phase0 if PHASE1 in available_phases: phase_dir[PHASE1] = spec_phase1 + if LIGHTCLIENT_PATCH in available_phases: + phase_dir[LIGHTCLIENT_PATCH] = spec_lightclient_patch # return is ignored whenever multiple phases are ran. If if PHASE0 in run_phases: ret = fn(spec=spec_phase0, phases=phase_dir, *args, **kw) if PHASE1 in run_phases: ret = fn(spec=spec_phase1, phases=phase_dir, *args, **kw) + if LIGHTCLIENT_PATCH in run_phases: + ret = fn(spec=spec_lightclient_patch, phases=phase_dir, *args, **kw) return ret return wrapper return decorator diff --git a/tests/core/pyspec/eth2spec/test/helpers/block.py b/tests/core/pyspec/eth2spec/test/helpers/block.py index 69cb59021..c4d8f1931 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/block.py @@ -1,3 +1,4 @@ +from eth2spec.test.context import LIGHTCLIENT_PATCH from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.bls import only_with_bls @@ -89,6 +90,10 @@ def build_empty_block(spec, state, slot=None): empty_block.proposer_index = spec.get_beacon_proposer_index(state) empty_block.body.eth1_data.deposit_count = state.eth1_deposit_index empty_block.parent_root = parent_block_root + + if spec.fork == LIGHTCLIENT_PATCH: + empty_block.body.sync_committee_signature = spec.G2_POINT_AT_INFINITY + apply_randao_reveal(spec, state, empty_block) return empty_block diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index c11ba1ec1..dc355972f 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -2,6 +2,7 @@ from random import Random from lru import LRU from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.test.context import LIGHTCLIENT_PATCH from eth2spec.test.helpers.attestations import cached_prepare_state_with_attestations from eth2spec.test.helpers.deposits import mock_deposit from eth2spec.test.helpers.state import next_epoch @@ -159,8 +160,12 @@ def run_get_inactivity_penalty_deltas(spec, state): continue if spec.is_in_inactivity_leak(state): + if spec.fork == LIGHTCLIENT_PATCH: + cancel_base_rewards_per_epoch = spec.BASE_REWARDS_PER_EPOCH - 1 + else: + cancel_base_rewards_per_epoch = spec.BASE_REWARDS_PER_EPOCH base_reward = spec.get_base_reward(state, index) - base_penalty = spec.BASE_REWARDS_PER_EPOCH * base_reward - spec.get_proposer_reward(state, index) + base_penalty = cancel_base_rewards_per_epoch * base_reward - spec.get_proposer_reward(state, index) if not has_enough_for_reward(spec, state, index): assert penalties[index] == 0 elif index in matching_attesting_indices: diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py index 661a00014..dd3cbb791 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py @@ -1,4 +1,5 @@ from eth2spec.test.context import ( + LIGHTCLIENT_PATCH, spec_state_test, spec_test, with_all_phases, single_phase, with_phases, PHASE0, @@ -162,6 +163,9 @@ def run_with_participation(spec, state, participation_fn): pre_state = state.copy() + if spec.fork == LIGHTCLIENT_PATCH: + sync_committee_indices = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + yield from run_process_rewards_and_penalties(spec, state) attesting_indices = spec.get_unslashed_attesting_indices(state, attestations) @@ -172,9 +176,13 @@ def run_with_participation(spec, state, participation_fn): # Proposers can still make money during a leak if index in proposer_indices and index in participated: assert state.balances[index] > pre_state.balances[index] - # If not proposer but participated optimally, should have exactly neutral balance elif index in attesting_indices: - assert state.balances[index] == pre_state.balances[index] + if spec.fork == LIGHTCLIENT_PATCH and index in sync_committee_indices: + # The sync committee reward has not been canceled, so the sync committee participants still earn it + assert state.balances[index] >= pre_state.balances[index] + else: + # If not proposer but participated optimally, should have exactly neutral balance + assert state.balances[index] == pre_state.balances[index] else: assert state.balances[index] < pre_state.balances[index] else: diff --git a/tests/core/pyspec/eth2spec/test/phase0/unittests/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/phase0/unittests/fork_choice/test_on_attestation.py index 63b0572b1..05f0fb051 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/unittests/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase0/unittests/fork_choice/test_on_attestation.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import PHASE0, with_all_phases, spec_state_test +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, with_all_phases, spec_state_test from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.attestations import get_valid_attestation, sign_attestation from eth2spec.test.helpers.state import transition_to, state_transition_and_sign_block, next_epoch, next_slot @@ -18,12 +18,12 @@ def run_on_attestation(spec, state, store, attestation, valid=True): spec.on_attestation(store, attestation) sample_index = indexed_attestation.attesting_indices[0] - if spec.fork == PHASE0: + if spec.fork in (PHASE0, LIGHTCLIENT_PATCH): latest_message = spec.LatestMessage( epoch=attestation.data.target.epoch, root=attestation.data.beacon_block_root, ) - else: + elif spec.fork == PHASE1: latest_message = spec.LatestMessage( epoch=attestation.data.target.epoch, root=attestation.data.beacon_block_root, diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_attestation.py index a0cf7472f..5b2f952ae 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_attestation.py @@ -1,5 +1,6 @@ from eth2spec.test.context import ( PHASE0, + LIGHTCLIENT_PATCH, with_all_phases_except, spec_state_test, always_bls, @@ -12,7 +13,7 @@ from eth2spec.test.helpers.attestations import ( ) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @always_bls def test_on_time_success(spec, state): @@ -23,7 +24,7 @@ def test_on_time_success(spec, state): yield from run_attestation_processing(spec, state, attestation) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @always_bls def test_late_success(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py index e916010b2..27829e4a0 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py @@ -9,6 +9,7 @@ from eth2spec.test.helpers.attestations import ( from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot from eth2spec.test.context import ( PHASE0, + LIGHTCLIENT_PATCH, MINIMAL, expect_assertion_error, disable_process_reveal_deadlines, @@ -68,7 +69,7 @@ def run_custody_chunk_response_processing(spec, state, custody_response, valid=T yield 'post', state -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @with_configs([MINIMAL], reason="too slow") @disable_process_reveal_deadlines @@ -92,7 +93,7 @@ def test_challenge_appended(spec, state): yield from run_chunk_challenge_processing(spec, state, challenge) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -118,7 +119,7 @@ def test_challenge_empty_element_replaced(spec, state): yield from run_chunk_challenge_processing(spec, state, challenge) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -144,7 +145,7 @@ def test_duplicate_challenge(spec, state): yield from run_chunk_challenge_processing(spec, state, challenge, valid=False) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -172,7 +173,7 @@ def test_second_challenge(spec, state): yield from run_chunk_challenge_processing(spec, state, challenge1) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -197,7 +198,7 @@ def test_multiple_epochs_custody(spec, state): yield from run_chunk_challenge_processing(spec, state, challenge) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -222,7 +223,7 @@ def test_many_epochs_custody(spec, state): yield from run_chunk_challenge_processing(spec, state, challenge) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -243,7 +244,7 @@ def test_off_chain_attestation(spec, state): yield from run_chunk_challenge_processing(spec, state, challenge) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -275,7 +276,7 @@ def test_custody_response(spec, state): yield from run_custody_chunk_response_processing(spec, state, custody_response) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -306,7 +307,7 @@ def test_custody_response_chunk_index_2(spec, state): yield from run_custody_chunk_response_processing(spec, state, custody_response) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -338,7 +339,7 @@ def test_custody_response_multiple_epochs(spec, state): yield from run_custody_chunk_response_processing(spec, state, custody_response) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_key_reveal.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_key_reveal.py index cb96c97e1..00a6112bf 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_key_reveal.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_key_reveal.py @@ -1,6 +1,7 @@ from eth2spec.test.helpers.custody import get_valid_custody_key_reveal from eth2spec.test.context import ( PHASE0, + LIGHTCLIENT_PATCH, with_all_phases_except, spec_state_test, expect_assertion_error, @@ -39,7 +40,7 @@ def run_custody_key_reveal_processing(spec, state, custody_key_reveal, valid=Tru yield 'post', state -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @always_bls def test_success(spec, state): @@ -49,7 +50,7 @@ def test_success(spec, state): yield from run_custody_key_reveal_processing(spec, state, custody_key_reveal) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @always_bls def test_reveal_too_early(spec, state): @@ -58,7 +59,7 @@ def test_reveal_too_early(spec, state): yield from run_custody_key_reveal_processing(spec, state, custody_key_reveal, False) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @always_bls def test_wrong_period(spec, state): @@ -67,7 +68,7 @@ def test_wrong_period(spec, state): yield from run_custody_key_reveal_processing(spec, state, custody_key_reveal, False) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @always_bls def test_late_reveal(spec, state): @@ -77,7 +78,7 @@ def test_late_reveal(spec, state): yield from run_custody_key_reveal_processing(spec, state, custody_key_reveal) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @always_bls def test_double_reveal(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py index fc7efa5bc..1f46bcf05 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py @@ -11,6 +11,7 @@ from eth2spec.test.helpers.state import get_balance, transition_to from eth2spec.test.context import ( PHASE0, MINIMAL, + LIGHTCLIENT_PATCH, with_all_phases_except, spec_state_test, expect_assertion_error, @@ -112,7 +113,7 @@ def run_standard_custody_slashing_test(spec, yield from run_custody_slashing_processing(spec, state, slashing, valid=valid, correct=correct) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -120,7 +121,7 @@ def test_custody_slashing(spec, state): yield from run_standard_custody_slashing_test(spec, state) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -128,7 +129,7 @@ def test_incorrect_custody_slashing(spec, state): yield from run_standard_custody_slashing_test(spec, state, correct=False) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -136,7 +137,7 @@ def test_multiple_epochs_custody(spec, state): yield from run_standard_custody_slashing_test(spec, state, shard_lateness=spec.SLOTS_PER_EPOCH * 3) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -144,7 +145,7 @@ def test_many_epochs_custody(spec, state): yield from run_standard_custody_slashing_test(spec, state, shard_lateness=spec.SLOTS_PER_EPOCH * 5) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_early_derived_secret_reveal.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_early_derived_secret_reveal.py index 668561261..3094f795b 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_early_derived_secret_reveal.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_early_derived_secret_reveal.py @@ -2,6 +2,7 @@ from eth2spec.test.helpers.custody import get_valid_early_derived_secret_reveal from eth2spec.test.helpers.state import next_epoch_via_block, get_balance from eth2spec.test.context import ( PHASE0, + LIGHTCLIENT_PATCH, with_all_phases_except, spec_state_test, expect_assertion_error, @@ -41,7 +42,7 @@ def run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, v yield 'post', state -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @always_bls def test_success(spec, state): @@ -50,7 +51,7 @@ def test_success(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @never_bls def test_reveal_from_current_epoch(spec, state): @@ -59,7 +60,7 @@ def test_reveal_from_current_epoch(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @never_bls def test_reveal_from_past_epoch(spec, state): @@ -69,7 +70,7 @@ def test_reveal_from_past_epoch(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @always_bls def test_reveal_with_custody_padding(spec, state): @@ -81,7 +82,7 @@ def test_reveal_with_custody_padding(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, True) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @always_bls def test_reveal_with_custody_padding_minus_one(spec, state): @@ -93,7 +94,7 @@ def test_reveal_with_custody_padding_minus_one(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, True) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @never_bls def test_double_reveal(spec, state): @@ -114,7 +115,7 @@ def test_double_reveal(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal2, False) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @never_bls def test_revealer_is_slashed(spec, state): @@ -124,7 +125,7 @@ def test_revealer_is_slashed(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @never_bls def test_far_future_epoch(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_shard_transition.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_shard_transition.py index b0a51557a..d2b7962b6 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_shard_transition.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_shard_transition.py @@ -1,5 +1,6 @@ from eth2spec.test.context import ( PHASE0, + LIGHTCLIENT_PATCH, with_all_phases_except, only_full_crosslink, spec_state_test, @@ -90,21 +91,21 @@ def run_successful_crosslink_tests(spec, state, target_len_offset_slot): assert bool(pending_attestation.crosslink_success) is True -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @only_full_crosslink def test_basic_crosslinks(spec, state): yield from run_successful_crosslink_tests(spec, state, target_len_offset_slot=1) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @only_full_crosslink def test_multiple_offset_slots(spec, state): yield from run_successful_crosslink_tests(spec, state, target_len_offset_slot=2) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @only_full_crosslink def test_no_winning_root(spec, state): @@ -152,7 +153,7 @@ def test_no_winning_root(spec, state): assert state.shard_states == pre_shard_states -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @only_full_crosslink def test_wrong_shard_transition_root(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py index e270ff615..0350324a5 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py +++ b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py @@ -8,6 +8,7 @@ from eth2spec.test.helpers.attestations import ( from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot from eth2spec.test.context import ( PHASE0, + LIGHTCLIENT_PATCH, MINIMAL, spec_state_test, with_all_phases_except, @@ -25,7 +26,7 @@ def run_process_challenge_deadlines(spec, state): yield from run_epoch_processing_with(spec, state, 'process_challenge_deadlines') -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @with_configs([MINIMAL], reason="too slow") def test_validator_slashed_after_chunk_challenge(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_custody_final_updates.py b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_custody_final_updates.py index 0541411da..5994306d9 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_custody_final_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_custody_final_updates.py @@ -1,5 +1,6 @@ from eth2spec.test.context import ( PHASE0, + LIGHTCLIENT_PATCH, ) from eth2spec.test.helpers.custody import ( get_valid_chunk_challenge, @@ -29,7 +30,7 @@ def run_process_custody_final_updates(spec, state): yield from run_epoch_processing_with(spec, state, 'process_custody_final_updates') -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test def test_validator_withdrawal_delay(spec, state): transition_to_valid_shard_slot(spec, state) @@ -42,7 +43,7 @@ def test_validator_withdrawal_delay(spec, state): assert state.validators[0].withdrawable_epoch == spec.FAR_FUTURE_EPOCH -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test def test_validator_withdrawal_reenable_after_custody_reveal(spec, state): transition_to_valid_shard_slot(spec, state) @@ -67,7 +68,7 @@ def test_validator_withdrawal_reenable_after_custody_reveal(spec, state): assert state.validators[0].withdrawable_epoch < spec.FAR_FUTURE_EPOCH -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test def test_validator_withdrawal_suspend_after_chunk_challenge(spec, state): transition_to_valid_shard_slot(spec, state) @@ -116,7 +117,7 @@ def test_validator_withdrawal_suspend_after_chunk_challenge(spec, state): assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test def test_validator_withdrawal_resume_after_chunk_challenge_response(spec, state): transition_to_valid_shard_slot(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py index 5777e184a..3c2060ba5 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py +++ b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py @@ -4,6 +4,7 @@ from eth2spec.test.helpers.custody import ( from eth2spec.test.helpers.state import transition_to from eth2spec.test.context import ( PHASE0, + LIGHTCLIENT_PATCH, MINIMAL, with_all_phases_except, with_configs, @@ -17,7 +18,7 @@ def run_process_challenge_deadlines(spec, state): yield from run_epoch_processing_with(spec, state, 'process_challenge_deadlines') -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @with_configs([MINIMAL], reason="too slow") def test_validator_slashed_after_reveal_deadline(spec, state): @@ -37,7 +38,7 @@ def test_validator_slashed_after_reveal_deadline(spec, state): assert state.validators[0].slashed == 1 -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @with_configs([MINIMAL], reason="too slow") def test_validator_not_slashed_after_reveal(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py index 922b604ad..ba47adde9 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py @@ -1,7 +1,9 @@ from typing import Dict, Sequence from eth2spec.test.context import ( - PHASE0, MINIMAL, + PHASE0, + LIGHTCLIENT_PATCH, + MINIMAL, with_all_phases_except, spec_state_test, only_full_crosslink, @@ -98,7 +100,7 @@ def run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, comm assert post_shard_state.gasprice > pre_gasprice -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @only_full_crosslink def test_process_beacon_block_with_normal_shard_transition(spec, state): @@ -112,7 +114,7 @@ def test_process_beacon_block_with_normal_shard_transition(spec, state): yield from run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @only_full_crosslink def test_process_beacon_block_with_empty_proposal_transition(spec, state): @@ -131,7 +133,7 @@ def test_process_beacon_block_with_empty_proposal_transition(spec, state): # -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @only_full_crosslink def test_with_shard_transition_with_custody_challenge_and_response(spec, state): @@ -165,7 +167,7 @@ def test_with_shard_transition_with_custody_challenge_and_response(spec, state): yield from run_beacon_block(spec, state, block) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @with_configs([MINIMAL]) def test_custody_key_reveal(spec, state): @@ -179,7 +181,7 @@ def test_custody_key_reveal(spec, state): yield from run_beacon_block(spec, state, block) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test def test_early_derived_secret_reveal(spec, state): transition_to_valid_shard_slot(spec, state) @@ -190,7 +192,7 @@ def test_early_derived_secret_reveal(spec, state): yield from run_beacon_block(spec, state, block) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @only_full_crosslink def test_custody_slashing(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase1/sanity/test_shard_blocks.py b/tests/core/pyspec/eth2spec/test/phase1/sanity/test_shard_blocks.py index ab66314e5..1590d2a6e 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/sanity/test_shard_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase1/sanity/test_shard_blocks.py @@ -1,5 +1,6 @@ from eth2spec.test.context import ( PHASE0, + LIGHTCLIENT_PATCH, always_bls, expect_assertion_error, spec_state_test, @@ -43,7 +44,7 @@ def run_shard_blocks(spec, shard_state, signed_shard_block, beacon_parent_state, shard_state.latest_block_root == pre_shard_state.latest_block_root -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @always_bls @only_full_crosslink @@ -63,7 +64,7 @@ def test_valid_shard_block(spec, state): # -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @only_full_crosslink def test_invalid_shard_parent_root(spec, state): @@ -79,7 +80,7 @@ def test_invalid_shard_parent_root(spec, state): yield from run_shard_blocks(spec, shard_state, signed_shard_block, beacon_state, valid=False) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @only_full_crosslink def test_invalid_beacon_parent_root(spec, state): @@ -94,7 +95,7 @@ def test_invalid_beacon_parent_root(spec, state): yield from run_shard_blocks(spec, shard_state, signed_shard_block, beacon_state, valid=False) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @only_full_crosslink def test_invalid_slot(spec, state): @@ -110,7 +111,7 @@ def test_invalid_slot(spec, state): yield from run_shard_blocks(spec, shard_state, signed_shard_block, beacon_state, valid=False) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @only_full_crosslink def test_invalid_proposer_index(spec, state): @@ -130,7 +131,7 @@ def test_invalid_proposer_index(spec, state): yield from run_shard_blocks(spec, shard_state, signed_shard_block, beacon_state, valid=False) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @always_bls @only_full_crosslink @@ -151,7 +152,7 @@ def test_out_of_bound_offset(spec, state): yield from run_shard_blocks(spec, shard_state, signed_shard_block, beacon_state, valid=False) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @always_bls @only_full_crosslink @@ -170,7 +171,7 @@ def test_invalid_offset(spec, state): yield from run_shard_blocks(spec, shard_state, signed_shard_block, beacon_state, valid=False) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @always_bls @only_full_crosslink @@ -189,7 +190,7 @@ def test_empty_block_body(spec, state): # -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @always_bls @only_full_crosslink @@ -208,7 +209,7 @@ def test_invalid_signature(spec, state): # -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @always_bls @only_full_crosslink @@ -225,7 +226,7 @@ def test_max_offset(spec, state): yield from run_shard_blocks(spec, shard_state, signed_shard_block, beacon_state) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @always_bls @only_full_crosslink diff --git a/tests/core/pyspec/eth2spec/test/phase1/unittests/fork_choice/test_on_shard_block.py b/tests/core/pyspec/eth2spec/test/phase1/unittests/fork_choice/test_on_shard_block.py index 30eaa8d80..66d254ed1 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/unittests/fork_choice/test_on_shard_block.py +++ b/tests/core/pyspec/eth2spec/test/phase1/unittests/fork_choice/test_on_shard_block.py @@ -1,6 +1,13 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root -from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_except, never_bls, only_full_crosslink +from eth2spec.test.context import ( + PHASE0, + LIGHTCLIENT_PATCH, + spec_state_test, + with_all_phases_except, + never_bls, + only_full_crosslink, +) from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.shard_block import ( build_shard_block, @@ -145,7 +152,7 @@ def create_and_apply_beacon_and_shard_blocks(spec, state, store, shard, shard_bl return has_shard_committee -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @never_bls # Set to never_bls for testing `check_pending_shard_blocks` def test_basic(spec, state): @@ -206,7 +213,7 @@ def create_simple_fork(spec, state, store, shard): return head_block, forking_block -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @only_full_crosslink def test_shard_simple_fork(spec, state): @@ -231,7 +238,7 @@ def test_shard_simple_fork(spec, state): assert spec.get_shard_head(store, shard) == forking_block.message.hash_tree_root() -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test @only_full_crosslink def test_shard_latest_messages_for_different_shards(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase1/unittests/test_get_start_shard.py b/tests/core/pyspec/eth2spec/test/phase1/unittests/test_get_start_shard.py index a802d6c3c..030357655 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/unittests/test_get_start_shard.py +++ b/tests/core/pyspec/eth2spec/test/phase1/unittests/test_get_start_shard.py @@ -1,12 +1,13 @@ from eth2spec.test.context import ( PHASE0, + LIGHTCLIENT_PATCH, with_all_phases_except, spec_state_test, ) from eth2spec.test.helpers.state import next_epoch -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test def test_get_committee_count_delta(spec, state): assert spec.get_committee_count_delta(state, 0, 0) == 0 @@ -23,7 +24,7 @@ def test_get_committee_count_delta(spec, state): ) -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test def test_get_start_shard_current_epoch_start(spec, state): assert state.current_epoch_start_shard == 0 @@ -39,7 +40,7 @@ def test_get_start_shard_current_epoch_start(spec, state): assert start_shard == state.current_epoch_start_shard -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test def test_get_start_shard_next_slot(spec, state): next_epoch(spec, state) @@ -57,7 +58,7 @@ def test_get_start_shard_next_slot(spec, state): assert start_shard == expected_start_shard -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test def test_get_start_shard_previous_slot(spec, state): next_epoch(spec, state) @@ -76,7 +77,7 @@ def test_get_start_shard_previous_slot(spec, state): assert start_shard == expected_start_shard -@with_all_phases_except([PHASE0]) +@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) @spec_state_test def test_get_start_shard_far_past_epoch(spec, state): initial_epoch = spec.get_current_epoch(state) From bcde37c39ff551a0e5d71af9a379670c0853934c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 9 Dec 2020 17:35:22 +0800 Subject: [PATCH 051/222] Make `sync-protocol.md` pass the linter --- setup.py | 73 ++++++++++++++++--- specs/lightclient/sync-protocol.md | 27 ++++--- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 3 files changed, 80 insertions(+), 22 deletions(-) diff --git a/setup.py b/setup.py index bd043dccc..168e19cca 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,7 @@ class SpecObject(NamedTuple): functions: Dict[str, str] custom_types: Dict[str, str] constants: Dict[str, str] + ssz_dep_constants: Dict[str, str] # the constants that depend on ssz_objects ssz_objects: Dict[str, str] dataclasses: Dict[str, str] @@ -35,6 +36,7 @@ def get_spec(file_name: str) -> SpecObject: current_name = None # most recent section title functions: Dict[str, str] = {} constants: Dict[str, str] = {} + ssz_dep_constants: Dict[str, str] = {} ssz_objects: Dict[str, str] = {} dataclasses: Dict[str, str] = {} function_matcher = re.compile(FUNCTION_REGEX) @@ -88,10 +90,20 @@ def get_spec(file_name: str) -> SpecObject: if c not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789': is_constant_def = False if is_constant_def: - constants[row[0]] = row[1].replace('**TBD**', '2**32') + if row[1].startswith('get_generalized_index'): + ssz_dep_constants[row[0]] = row[1] + else: + constants[row[0]] = row[1].replace('**TBD**', '2**32') elif row[1].startswith('uint') or row[1].startswith('Bytes'): custom_types[row[0]] = row[1] - return SpecObject(functions, custom_types, constants, ssz_objects, dataclasses) + return SpecObject( + functions=functions, + custom_types=custom_types, + constants=constants, + ssz_dep_constants=ssz_dep_constants, + ssz_objects=ssz_objects, + dataclasses=dataclasses, + ) CONFIG_LOADER = ''' @@ -160,7 +172,7 @@ CONFIG_NAME = 'mainnet' LIGHTCLIENT_IMPORT = '''from eth2spec.phase0 import spec as phase0 from eth2spec.config.config_util import apply_constants_config from typing import ( - Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable, Optional + Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable, Optional, Union ) from dataclasses import ( @@ -174,6 +186,7 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes from eth2spec.utils.ssz.ssz_typing import ( View, boolean, Container, List, Vector, uint8, uint32, uint64, Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, + Path, ) from eth2spec.utils import bls @@ -277,6 +290,24 @@ get_start_shard = cache_this( _get_start_shard, lru_size=SLOTS_PER_EPOCH * 3)''' +LIGHTCLIENT_PATCH_SUNDRY_FUNCTIONS = ''' + +def get_generalized_index(ssz_class: Any, *path: Sequence[Union[int, SSZVariableName]]) -> GeneralizedIndex: + ssz_path = Path(ssz_class) + for item in path: + ssz_path = ssz_path / item + return GeneralizedIndex(ssz_path.gindex()) +''' + + +# The constants that depend on SSZ objects +# Will verify the value at the end of the spec +LIGHTCLIENT_PATCH_HARDCODED_SSZ_DEP_CONSTANTS = { + 'FINALIZED_ROOT_INDEX': 'GeneralizedIndex(105)', + 'NEXT_SYNC_COMMITTEE_INDEX': 'GeneralizedIndex(54)', +} + + def objects_to_spec(spec_object: SpecObject, imports: str, fork: str, ordered_class_objects: Dict[str, str]) -> str: """ Given all the objects that constitute a spec, combine them into a single pyfile. @@ -303,14 +334,30 @@ def objects_to_spec(spec_object: SpecObject, imports: str, fork: str, ordered_cl + '\n\n' + f"fork = \'{fork}\'\n" + '\n\n' + new_type_definitions + '\n' + SUNDRY_CONSTANTS_FUNCTIONS - + '\n\n' + constants_spec + ) + + if fork == 'lightclient_patch': + lightclient_patch_ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, LIGHTCLIENT_PATCH_HARDCODED_SSZ_DEP_CONSTANTS[x]), LIGHTCLIENT_PATCH_HARDCODED_SSZ_DEP_CONSTANTS)) + spec += ( + LIGHTCLIENT_PATCH_SUNDRY_FUNCTIONS + + '\n\n' + lightclient_patch_ssz_dep_constants + ) + + spec += ( + '\n\n' + constants_spec + '\n\n' + CONFIG_LOADER + '\n\n' + ordered_class_objects_spec + '\n\n' + functions_spec + '\n' + PHASE0_SUNDRY_FUNCTIONS ) + if fork == 'phase1': spec += '\n' + PHASE1_SUNDRY_FUNCTIONS + + if fork == 'lightclient_patch': + lightclient_patch_ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), LIGHTCLIENT_PATCH_HARDCODED_SSZ_DEP_CONSTANTS)) + spec += '\n\n' + lightclient_patch_ssz_dep_constants_verification + spec += '\n' return spec @@ -332,7 +379,7 @@ ignored_dependencies = [ 'Bytes1', 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector', 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', 'bytes', 'byte', 'ByteList', 'ByteVector', - 'Dict', 'dict', 'field', + 'Dict', 'dict', 'field', 'ceillog2', ] @@ -373,14 +420,22 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: """ Takes in two spec variants (as tuples of their objects) and combines them using the appropriate combiner function. """ - functions0, custom_types0, constants0, ssz_objects0, dataclasses0 = spec0 - functions1, custom_types1, constants1, ssz_objects1, dataclasses1 = spec1 + functions0, custom_types0, constants0, ssz_dep_constants0, ssz_objects0, dataclasses0 = spec0 + functions1, custom_types1, constants1, ssz_dep_constants1, ssz_objects1, dataclasses1 = spec1 functions = combine_functions(functions0, functions1) custom_types = combine_constants(custom_types0, custom_types1) constants = combine_constants(constants0, constants1) + ssz_dep_constants = combine_constants(ssz_dep_constants0, ssz_dep_constants1) ssz_objects = combine_ssz_objects(ssz_objects0, ssz_objects1, custom_types) dataclasses = combine_functions(dataclasses0, dataclasses1) - return SpecObject(functions, custom_types, constants, ssz_objects, dataclasses) + return SpecObject( + functions=functions, + custom_types=custom_types, + constants=constants, + ssz_dep_constants=ssz_dep_constants, + ssz_objects=ssz_objects, + dataclasses=dataclasses, + ) fork_imports = { @@ -461,8 +516,8 @@ class PySpecCommand(Command): specs/phase0/weak-subjectivity.md specs/lightclient/beacon-chain.md specs/lightclient/lightclient-fork.md + specs/lightclient/sync-protocol.md """ - # TODO: add specs/lightclient/sync-protocol.md back when the GeneralizedIndex helpers are included. else: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index 310aad2df..a8dc59639 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -39,8 +39,8 @@ uses sync committees introduced in [this beacon chain extension](./beacon-chain. | Name | Value | | - | - | -| `FINALIZED_ROOT_INDEX` | `Index(BeaconState, 'finalized_checkpoint', 'root')` | -| `NEXT_SYNC_COMMITTEE_INDEX` | `Index(BeaconState, 'next_sync_committee')` | +| `FINALIZED_ROOT_INDEX` | `get_generalized_index(BeaconState, 'finalized_checkpoint', 'root')` | +| `NEXT_SYNC_COMMITTEE_INDEX` | `get_generalized_index(BeaconState, 'next_sync_committee')` | ## Configuration @@ -78,10 +78,10 @@ class LightClientUpdate(Container): header: BeaconBlockHeader # Next sync committee corresponding to the header next_sync_committee: SyncCommittee - next_sync_committee_branch: Vector[Bytes32, log2(NEXT_SYNC_COMMITTEE_INDEX)] + next_sync_committee_branch: Vector[Bytes32, ceillog2(NEXT_SYNC_COMMITTEE_INDEX)] # Finality proof for the update header finality_header: BeaconBlockHeader - finality_branch: Vector[Bytes32, log2(FINALIZED_ROOT_INDEX)] + finality_branch: Vector[Bytes32, ceillog2(FINALIZED_ROOT_INDEX)] # Sync committee aggregate signature sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE] sync_committee_signature: BLSSignature @@ -116,28 +116,28 @@ def is_valid_light_client_update(snapshot: LightClientSnapshot, update: LightCli # Verify update header root is the finalized root of the finality header, if specified if update.finality_header == BeaconBlockHeader(): signed_header = update.header - assert update.finality_branch == [ZERO_HASH for _ in range(log2(FINALIZED_ROOT_INDEX))] + assert update.finality_branch == [Bytes32() for _ in range(ceillog2(FINALIZED_ROOT_INDEX))] else: signed_header = update.finality_header assert is_valid_merkle_branch( leaf=hash_tree_root(update.header), branch=update.finality_branch, - depth=log2(FINALIZED_ROOT_INDEX), - index=FINALIZED_ROOT_INDEX % 2**log2(FINALIZED_ROOT_INDEX), + depth=ceillog2(FINALIZED_ROOT_INDEX), + index=FINALIZED_ROOT_INDEX % 2**ceillog2(FINALIZED_ROOT_INDEX), root=update.finality_header.state_root, ) # Verify update next sync committee if the update period incremented if update_period == snapshot_period: sync_committee = snapshot.current_sync_committee - assert update.next_sync_committee_branch == [ZERO_HASH for _ in range(log2(NEXT_SYNC_COMMITTEE_INDEX))] + assert update.next_sync_committee_branch == [Bytes32() for _ in range(ceillog2(NEXT_SYNC_COMMITTEE_INDEX))] else: sync_committee = snapshot.next_sync_committee assert is_valid_merkle_branch( leaf=hash_tree_root(update.next_sync_committee), branch=update.next_sync_committee_branch, - depth=log2(NEXT_SYNC_COMMITTEE_INDEX), - index=NEXT_SYNC_COMMITTEE_INDEX % 2**log2(NEXT_SYNC_COMMITTEE_INDEX), + depth=ceillog2(NEXT_SYNC_COMMITTEE_INDEX), + index=NEXT_SYNC_COMMITTEE_INDEX % 2**ceillog2(NEXT_SYNC_COMMITTEE_INDEX), root=update.header.state_root, ) @@ -173,11 +173,14 @@ def process_light_client_update(store: LightClientStore, update: LightClientUpda assert is_valid_light_client_update(store.snapshot, update) store.valid_updates.append(update) - if sum(update.sync_committee_bits) * 3 > len(update.sync_committee_bits) * 2 and update.header != update.finality_header: + if ( + sum(update.sync_committee_bits) * 3 > len(update.sync_committee_bits) * 2 + and update.header != update.finality_header + ): # Apply update if 2/3 quorum is reached and we have a finality proof apply_light_client_update(store, update) store.valid_updates = [] - elif current_slot > snapshot.header.slot + LIGHT_CLIENT_UPDATE_TIMEOUT: + elif current_slot > store.snapshot.header.slot + LIGHT_CLIENT_UPDATE_TIMEOUT: # Forced best update when the update timeout has elapsed apply_light_client_update(store, max(store.valid_updates, key=lambda update: sum(update.sync_committee_bits))) store.valid_updates = [] diff --git a/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py b/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py index b3a0b9962..6626e26d6 100644 --- a/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -5,4 +5,4 @@ from remerkleable.complex import Container, Vector, List from remerkleable.basic import boolean, bit, uint, byte, uint8, uint16, uint32, uint64, uint128, uint256 from remerkleable.bitfields import Bitvector, Bitlist from remerkleable.byte_arrays import ByteVector, Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, ByteList -from remerkleable.core import BasicView, View +from remerkleable.core import BasicView, View, Path From cf6933ac457614f78052399e9d16dff1bc251640 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 10 Dec 2020 17:04:11 +0800 Subject: [PATCH 052/222] Fix depth calculation and add `get_subtree_index` helper --- specs/lightclient/sync-protocol.md | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index a8dc59639..073d570cb 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -17,6 +17,8 @@ - [`LightClientSnapshot`](#lightclientsnapshot) - [`LightClientUpdate`](#lightclientupdate) - [`LightClientStore`](#lightclientstore) +- [Helper functions](#helper-functions) + - [`get_subtree_index`](#get_subtree_index) - [Light client state updates](#light-client-state-updates) - [`is_valid_light_client_update`](#is_valid_light_client_update) - [`apply_light_client_update`](#apply_light_client_update) @@ -78,10 +80,10 @@ class LightClientUpdate(Container): header: BeaconBlockHeader # Next sync committee corresponding to the header next_sync_committee: SyncCommittee - next_sync_committee_branch: Vector[Bytes32, ceillog2(NEXT_SYNC_COMMITTEE_INDEX)] + next_sync_committee_branch: Vector[Bytes32, NEXT_SYNC_COMMITTEE_INDEX.bit_length()] # Finality proof for the update header finality_header: BeaconBlockHeader - finality_branch: Vector[Bytes32, ceillog2(FINALIZED_ROOT_INDEX)] + finality_branch: Vector[Bytes32, FINALIZED_ROOT_INDEX.bit_length()] # Sync committee aggregate signature sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE] sync_committee_signature: BLSSignature @@ -97,6 +99,15 @@ class LightClientStore(Container): valid_updates: List[LightClientUpdate, MAX_VALID_LIGHT_CLIENT_UPDATES] ``` +## Helper functions + +### `get_subtree_index` + +```python +def get_subtree_index(generalized_index: GeneralizedIndex) -> uint64: + return uint64(generalized_index % 2**((generalized_index).bit_length())) +``` + ## Light client state updates A light client maintains its state in a `store` object of type `LightClientStore` and receives `update` objects of type `LightClientUpdate`. Every `update` triggers `process_light_client_update(store, update, current_slot)` where `current_slot` is the current slot based on some local clock. @@ -116,28 +127,28 @@ def is_valid_light_client_update(snapshot: LightClientSnapshot, update: LightCli # Verify update header root is the finalized root of the finality header, if specified if update.finality_header == BeaconBlockHeader(): signed_header = update.header - assert update.finality_branch == [Bytes32() for _ in range(ceillog2(FINALIZED_ROOT_INDEX))] + assert update.finality_branch == [Bytes32() for _ in range(FINALIZED_ROOT_INDEX.bit_length())] else: signed_header = update.finality_header assert is_valid_merkle_branch( leaf=hash_tree_root(update.header), branch=update.finality_branch, - depth=ceillog2(FINALIZED_ROOT_INDEX), - index=FINALIZED_ROOT_INDEX % 2**ceillog2(FINALIZED_ROOT_INDEX), + depth=FINALIZED_ROOT_INDEX.bit_length(), + index=get_subtree_index(FINALIZED_ROOT_INDEX), root=update.finality_header.state_root, ) # Verify update next sync committee if the update period incremented if update_period == snapshot_period: sync_committee = snapshot.current_sync_committee - assert update.next_sync_committee_branch == [Bytes32() for _ in range(ceillog2(NEXT_SYNC_COMMITTEE_INDEX))] + assert update.next_sync_committee_branch == [Bytes32() for _ in range(NEXT_SYNC_COMMITTEE_INDEX.bit_length())] else: sync_committee = snapshot.next_sync_committee assert is_valid_merkle_branch( leaf=hash_tree_root(update.next_sync_committee), branch=update.next_sync_committee_branch, - depth=ceillog2(NEXT_SYNC_COMMITTEE_INDEX), - index=NEXT_SYNC_COMMITTEE_INDEX % 2**ceillog2(NEXT_SYNC_COMMITTEE_INDEX), + depth=NEXT_SYNC_COMMITTEE_INDEX.bit_length(), + index=get_subtree_index(NEXT_SYNC_COMMITTEE_INDEX), root=update.header.state_root, ) From d01a4ad82374dbfbabf047315536f90fe90bf928 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 16 Dec 2020 15:00:06 +0800 Subject: [PATCH 053/222] Fix depth calculation...again(!) and add unittests --- setup.py | 10 ++++-- specs/lightclient/sync-protocol.md | 14 ++++---- .../pyspec/eth2spec/test/helpers/custody.py | 22 +----------- .../pyspec/eth2spec/test/helpers/merkle.py | 21 +++++++++++ .../unittests/test_helpers.py | 35 +++++++++++++++++++ 5 files changed, 72 insertions(+), 30 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/helpers/merkle.py create mode 100644 tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_helpers.py diff --git a/setup.py b/setup.py index 168e19cca..8115fdfe5 100644 --- a/setup.py +++ b/setup.py @@ -209,6 +209,12 @@ def ceillog2(x: int) -> uint64: if x < 1: raise ValueError(f"ceillog2 accepts only positive values, x={x}") return uint64((x - 1).bit_length()) + + +def floorlog2(x: int) -> uint64: + if x < 1: + raise ValueError(f"floorlog2 accepts only positive values, x={x}") + return uint64(x.bit_length() - 1) ''' PHASE0_SUNDRY_FUNCTIONS = ''' def get_eth1_data(block: Eth1Block) -> Eth1Data: @@ -321,7 +327,7 @@ def objects_to_spec(spec_object: SpecObject, imports: str, fork: str, ordered_cl ) ) for k in list(spec_object.functions): - if "ceillog2" in k: + if "ceillog2" in k or "floorlog2" in k: del spec_object.functions[k] functions_spec = '\n\n'.join(spec_object.functions.values()) for k in list(spec_object.constants.keys()): @@ -379,7 +385,7 @@ ignored_dependencies = [ 'Bytes1', 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector', 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', 'bytes', 'byte', 'ByteList', 'ByteVector', - 'Dict', 'dict', 'field', 'ceillog2', + 'Dict', 'dict', 'field', 'ceillog2', 'floorlog2', ] diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index 073d570cb..156e6e78e 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -80,10 +80,10 @@ class LightClientUpdate(Container): header: BeaconBlockHeader # Next sync committee corresponding to the header next_sync_committee: SyncCommittee - next_sync_committee_branch: Vector[Bytes32, NEXT_SYNC_COMMITTEE_INDEX.bit_length()] + next_sync_committee_branch: Vector[Bytes32, floorlog2(NEXT_SYNC_COMMITTEE_INDEX)] # Finality proof for the update header finality_header: BeaconBlockHeader - finality_branch: Vector[Bytes32, FINALIZED_ROOT_INDEX.bit_length()] + finality_branch: Vector[Bytes32, floorlog2(FINALIZED_ROOT_INDEX)] # Sync committee aggregate signature sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE] sync_committee_signature: BLSSignature @@ -105,7 +105,7 @@ class LightClientStore(Container): ```python def get_subtree_index(generalized_index: GeneralizedIndex) -> uint64: - return uint64(generalized_index % 2**((generalized_index).bit_length())) + return uint64(generalized_index % 2**(floorlog2(generalized_index))) ``` ## Light client state updates @@ -127,13 +127,13 @@ def is_valid_light_client_update(snapshot: LightClientSnapshot, update: LightCli # Verify update header root is the finalized root of the finality header, if specified if update.finality_header == BeaconBlockHeader(): signed_header = update.header - assert update.finality_branch == [Bytes32() for _ in range(FINALIZED_ROOT_INDEX.bit_length())] + assert update.finality_branch == [Bytes32() for _ in range(floorlog2(FINALIZED_ROOT_INDEX))] else: signed_header = update.finality_header assert is_valid_merkle_branch( leaf=hash_tree_root(update.header), branch=update.finality_branch, - depth=FINALIZED_ROOT_INDEX.bit_length(), + depth=floorlog2(FINALIZED_ROOT_INDEX), index=get_subtree_index(FINALIZED_ROOT_INDEX), root=update.finality_header.state_root, ) @@ -141,13 +141,13 @@ def is_valid_light_client_update(snapshot: LightClientSnapshot, update: LightCli # Verify update next sync committee if the update period incremented if update_period == snapshot_period: sync_committee = snapshot.current_sync_committee - assert update.next_sync_committee_branch == [Bytes32() for _ in range(NEXT_SYNC_COMMITTEE_INDEX.bit_length())] + assert update.next_sync_committee_branch == [Bytes32() for _ in range(floorlog2(NEXT_SYNC_COMMITTEE_INDEX))] else: sync_committee = snapshot.next_sync_committee assert is_valid_merkle_branch( leaf=hash_tree_root(update.next_sync_committee), branch=update.next_sync_committee_branch, - depth=NEXT_SYNC_COMMITTEE_INDEX.bit_length(), + depth=floorlog2(NEXT_SYNC_COMMITTEE_INDEX), index=get_subtree_index(NEXT_SYNC_COMMITTEE_INDEX), root=update.header.state_root, ) diff --git a/tests/core/pyspec/eth2spec/test/helpers/custody.py b/tests/core/pyspec/eth2spec/test/helpers/custody.py index b3a8c0a95..8e9aafa66 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/custody.py +++ b/tests/core/pyspec/eth2spec/test/helpers/custody.py @@ -1,7 +1,7 @@ from eth2spec.test.helpers.keys import privkeys +from eth2spec.test.helpers.merkle import build_proof from eth2spec.utils import bls from eth2spec.utils.ssz.ssz_typing import Bitlist, ByteVector, ByteList -from remerkleable.tree import gindex_bit_iter BYTES_PER_CHUNK = 32 @@ -116,26 +116,6 @@ def custody_chunkify(spec, x): return [ByteVector[spec.BYTES_PER_CUSTODY_CHUNK](c) for c in chunks] -def build_proof(anchor, leaf_index): - if leaf_index <= 1: - return [] # Nothing to prove / invalid index - node = anchor - proof = [] - # Walk down, top to bottom to the leaf - bit_iter, _ = gindex_bit_iter(leaf_index) - for bit in bit_iter: - # Always take the opposite hand for the proof. - # 1 = right as leaf, thus get left - if bit: - proof.append(node.get_left().merkle_root()) - node = node.get_right() - else: - proof.append(node.get_right().merkle_root()) - node = node.get_left() - - return list(reversed(proof)) - - def get_valid_custody_chunk_response(spec, state, chunk_challenge, challenge_index, block_length_or_custody_data, invalid_chunk_data=False): diff --git a/tests/core/pyspec/eth2spec/test/helpers/merkle.py b/tests/core/pyspec/eth2spec/test/helpers/merkle.py new file mode 100644 index 000000000..d49827954 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/merkle.py @@ -0,0 +1,21 @@ +from remerkleable.tree import gindex_bit_iter + + +def build_proof(anchor, leaf_index): + if leaf_index <= 1: + return [] # Nothing to prove / invalid index + node = anchor + proof = [] + # Walk down, top to bottom to the leaf + bit_iter, _ = gindex_bit_iter(leaf_index) + for bit in bit_iter: + # Always take the opposite hand for the proof. + # 1 = right as leaf, thus get left + if bit: + proof.append(node.get_left().merkle_root()) + node = node.get_right() + else: + proof.append(node.get_right().merkle_root()) + node = node.get_left() + + return list(reversed(proof)) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_helpers.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_helpers.py new file mode 100644 index 000000000..33e749380 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_helpers.py @@ -0,0 +1,35 @@ +from eth2spec.test.context import ( + spec_state_test, + with_phases, + LIGHTCLIENT_PATCH, +) +from eth2spec.test.helpers.merkle import build_proof + + +@with_phases([LIGHTCLIENT_PATCH]) +@spec_state_test +def test_next_sync_committee_tree(spec, state): + state.next_sync_committee = spec.SyncCommittee( + pubkeys=[state.validators[i]for i in range(spec.SYNC_COMMITTEE_SIZE)] + ) + next_sync_committee_branch = build_proof(state.get_backing(), spec.NEXT_SYNC_COMMITTEE_INDEX) + assert spec.is_valid_merkle_branch( + leaf=state.next_sync_committee.hash_tree_root(), + branch=next_sync_committee_branch, + depth=spec.floorlog2(spec.NEXT_SYNC_COMMITTEE_INDEX), + index=spec.get_subtree_index(spec.NEXT_SYNC_COMMITTEE_INDEX), + root=state.hash_tree_root(), + ) + + +@with_phases([LIGHTCLIENT_PATCH]) +@spec_state_test +def test_finality_root_tree(spec, state): + finality_branch = build_proof(state.get_backing(), spec.FINALIZED_ROOT_INDEX) + assert spec.is_valid_merkle_branch( + leaf=state.finalized_checkpoint.root, + branch=finality_branch, + depth=spec.floorlog2(spec.FINALIZED_ROOT_INDEX), + index=spec.get_subtree_index(spec.FINALIZED_ROOT_INDEX), + root=state.hash_tree_root(), + ) From e63c96416a16109a7984603cb46dbef506c4f1a9 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 16 Dec 2020 15:10:54 +0800 Subject: [PATCH 054/222] Add a FIXME comment. --- specs/lightclient/beacon-chain.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 4eb1f24ab..eabcb3a86 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -240,6 +240,7 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S ```python def process_final_updates(state: BeaconState) -> None: + # FIXME: unfold the full `process_final_updates` to avoid side effects. phase0.process_final_updates(state) next_epoch = get_current_epoch(state) + Epoch(1) if next_epoch % EPOCHS_PER_SYNC_COMMITTEE_PERIOD == 0: From cc9a4cdc46131b6d320eb4ac4bbd663db16bf917 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 16 Dec 2020 17:12:51 -0700 Subject: [PATCH 055/222] add base sanity light client tests --- specs/lightclient/beacon-chain.md | 2 +- specs/lightclient/lightclient-fork.md | 6 +- .../lightclient_patch/sanity/test_blocks.py | 102 ++++++++++++++++++ 3 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index eabcb3a86..3b03ad853 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -163,7 +163,7 @@ def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: bls.AggregatePKs(pubkeys[i:i + SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE]) for i in range(0, len(pubkeys), SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE) ] - return SyncCommittee(pubkeys, aggregates) + return SyncCommittee(pubkeys=pubkeys, pubkey_aggregates=aggregates) ``` ### Block processing diff --git a/specs/lightclient/lightclient-fork.md b/specs/lightclient/lightclient-fork.md index bb67fa54a..568a9793b 100644 --- a/specs/lightclient/lightclient-fork.md +++ b/specs/lightclient/lightclient-fork.md @@ -75,9 +75,9 @@ def upgrade_to_lightclient_patch(pre: phase0.BeaconState) -> BeaconState: previous_justified_checkpoint=pre.previous_justified_checkpoint, current_justified_checkpoint=pre.current_justified_checkpoint, finalized_checkpoint=pre.finalized_checkpoint, - # Light-client - current_sync_committee=SyncCommittee(), - next_sync_committee=SyncCommittee(), ) + # Fill in sync committees + post.current_sync_committee = get_sync_committee(post, get_current_epoch(post)) + post.next_sync_committee = get_sync_committee(post, get_current_epoch(post) + EPOCHS_PER_SYNC_COMMITTEE_PERIOD) return post ``` diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py new file mode 100644 index 000000000..df8e2545c --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py @@ -0,0 +1,102 @@ +import random +from eth2spec.test.helpers.keys import privkeys, pubkeys +from eth2spec.utils import bls +from eth2spec.test.helpers.state import ( + state_transition_and_sign_block, + next_epoch, +) +from eth2spec.test.helpers.block import ( + build_empty_block_for_next_slot, +) +from eth2spec.test.context import ( + PHASE0, PHASE1, + with_all_phases_except, + spec_state_test, +) + + +def compute_light_client_signature(spec, state, slot, privkey): + domain = spec.get_domain(state, spec.DOMAIN_SYNC_COMMITTEE, spec.compute_epoch_at_slot(slot)) + if slot == state.slot: + block_root = build_empty_block_for_next_slot(spec, state).parent_root + else: + block_root = spec.get_block_root_at_slot(state, slot) + signing_root = spec.compute_signing_root(block_root, domain) + return bls.Sign(privkey, signing_root) + + +def compute_aggregate_light_client_signature(spec, state, slot, participants): + if len(participants) == 0: + return spec.G2_POINT_AT_INFINITY + + signatures = [] + for validator_index in participants: + privkey = privkeys[validator_index] + signatures.append( + compute_light_client_signature( + spec, + state, + slot, + privkey, + ) + ) + return bls.Aggregate(signatures) + + +def run_light_client_sanity_test(spec, state, fraction_full=1.0): + committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + participants = random.sample(committee, int(len(committee) * fraction_full)) + + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + block.body.sync_committee_bits = [index in participants for index in committee] + block.body.sync_committee_signature = compute_aggregate_light_client_signature( + spec, + state, + block.slot - 1, + participants, + ) + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_full_light_client_committee(spec, state): + next_epoch(spec, state) + yield from run_light_client_sanity_test(spec, state, fraction_full=1.0) + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_half_light_client_committee(spec, state): + next_epoch(spec, state) + yield from run_light_client_sanity_test(spec, state, fraction_full=0.5) + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_empty_light_client_committee(spec, state): + next_epoch(spec, state) + yield from run_light_client_sanity_test(spec, state, fraction_full=0.0) + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_full_light_client_committee_genesis(spec, state): + yield from run_light_client_sanity_test(spec, state, fraction_full=1.0) + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_half_light_client_committee_genesis(spec, state): + yield from run_light_client_sanity_test(spec, state, fraction_full=0.5) + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_empty_light_client_committee_genesis(spec, state): + yield from run_light_client_sanity_test(spec, state, fraction_full=0.0) From 89c5ca6bcd2fa9a1f57d342038bb08effa969fe7 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 17 Dec 2020 06:25:58 -0700 Subject: [PATCH 056/222] 'light_client' -> 'sync_committee' --- .../lightclient_patch/sanity/test_blocks.py | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py index df8e2545c..4fbdfc371 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py @@ -1,5 +1,5 @@ import random -from eth2spec.test.helpers.keys import privkeys, pubkeys +from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.test.helpers.state import ( state_transition_and_sign_block, @@ -15,7 +15,7 @@ from eth2spec.test.context import ( ) -def compute_light_client_signature(spec, state, slot, privkey): +def compute_sync_committee_signature(spec, state, slot, privkey): domain = spec.get_domain(state, spec.DOMAIN_SYNC_COMMITTEE, spec.compute_epoch_at_slot(slot)) if slot == state.slot: block_root = build_empty_block_for_next_slot(spec, state).parent_root @@ -25,7 +25,7 @@ def compute_light_client_signature(spec, state, slot, privkey): return bls.Sign(privkey, signing_root) -def compute_aggregate_light_client_signature(spec, state, slot, participants): +def compute_aggregate_sync_committee_signature(spec, state, slot, participants): if len(participants) == 0: return spec.G2_POINT_AT_INFINITY @@ -33,7 +33,7 @@ def compute_aggregate_light_client_signature(spec, state, slot, participants): for validator_index in participants: privkey = privkeys[validator_index] signatures.append( - compute_light_client_signature( + compute_sync_committee_signature( spec, state, slot, @@ -43,7 +43,7 @@ def compute_aggregate_light_client_signature(spec, state, slot, participants): return bls.Aggregate(signatures) -def run_light_client_sanity_test(spec, state, fraction_full=1.0): +def run_sync_committee_sanity_test(spec, state, fraction_full=1.0): committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) participants = random.sample(committee, int(len(committee) * fraction_full)) @@ -51,7 +51,7 @@ def run_light_client_sanity_test(spec, state, fraction_full=1.0): block = build_empty_block_for_next_slot(spec, state) block.body.sync_committee_bits = [index in participants for index in committee] - block.body.sync_committee_signature = compute_aggregate_light_client_signature( + block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, block.slot - 1, @@ -65,38 +65,38 @@ def run_light_client_sanity_test(spec, state, fraction_full=1.0): @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test -def test_full_light_client_committee(spec, state): +def test_full_sync_committee_committee(spec, state): next_epoch(spec, state) - yield from run_light_client_sanity_test(spec, state, fraction_full=1.0) + yield from run_sync_committee_sanity_test(spec, state, fraction_full=1.0) @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test -def test_half_light_client_committee(spec, state): +def test_half_sync_committee_committee(spec, state): next_epoch(spec, state) - yield from run_light_client_sanity_test(spec, state, fraction_full=0.5) + yield from run_sync_committee_sanity_test(spec, state, fraction_full=0.5) @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test -def test_empty_light_client_committee(spec, state): +def test_empty_sync_committee_committee(spec, state): next_epoch(spec, state) - yield from run_light_client_sanity_test(spec, state, fraction_full=0.0) + yield from run_sync_committee_sanity_test(spec, state, fraction_full=0.0) @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test -def test_full_light_client_committee_genesis(spec, state): - yield from run_light_client_sanity_test(spec, state, fraction_full=1.0) +def test_full_sync_committee_committee_genesis(spec, state): + yield from run_sync_committee_sanity_test(spec, state, fraction_full=1.0) @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test -def test_half_light_client_committee_genesis(spec, state): - yield from run_light_client_sanity_test(spec, state, fraction_full=0.5) +def test_half_sync_committee_committee_genesis(spec, state): + yield from run_sync_committee_sanity_test(spec, state, fraction_full=0.5) @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test -def test_empty_light_client_committee_genesis(spec, state): - yield from run_light_client_sanity_test(spec, state, fraction_full=0.0) +def test_empty_sync_committee_committee_genesis(spec, state): + yield from run_sync_committee_sanity_test(spec, state, fraction_full=0.0) From aa16da10994bb8ae645075c4f4a6cc19a3f824d4 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 18 Dec 2020 19:03:05 +0800 Subject: [PATCH 057/222] Updated readme (#2157) --- README.md | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index a3ca7c586..d5eecd620 100644 --- a/README.md +++ b/README.md @@ -11,29 +11,28 @@ This repository hosts the current Eth2 specifications. Discussions about design [![GitHub release](https://img.shields.io/github/v/release/ethereum/eth2.0-specs)](https://github.com/ethereum/eth2.0-specs/releases/) [![PyPI version](https://badge.fury.io/py/eth2spec.svg)](https://badge.fury.io/py/eth2spec) - Core specifications for Eth2 clients be found in [specs](specs/). These are divided into phases. Each subsequent phase depends upon the prior. The current phases specified are: ### Phase 0 + * [The Beacon Chain](specs/phase0/beacon-chain.md) * [Beacon Chain Fork Choice](specs/phase0/fork-choice.md) * [Deposit Contract](specs/phase0/deposit-contract.md) * [Honest Validator](specs/phase0/validator.md) * [P2P Networking](specs/phase0/p2p-interface.md) -### Phase 1 -* [From Phase 0 to Phase 1](specs/phase1/phase1-fork.md) -* [The Beacon Chain for Shards](specs/phase1/beacon-chain.md) -* [Custody Game](specs/phase1/custody-game.md) -* [Shard Transition and Fraud Proofs](specs/phase1/shard-transition.md) -* [Light client syncing protocol](specs/phase1/light-client-sync.md) -* [Beacon Chain Fork Choice for Shards](specs/phase1/fork-choice.md) +### Light clients -### Phase 2 +* [Beacon chain changes](specs/lightclient/beacon-chain.md) +* [Light client sync protocol](specs/lightclient/sync-protocol.md) -Phase 2 is still actively in R&D and does not yet have any formal specifications. +### Sharding -See the [Eth2 Phase 2 Wiki](https://hackmd.io/UzysWse1Th240HELswKqVA?view) for current progress, discussions, and definitions regarding this work. +The sharding spec is still actively in R&D; see the most recent available pull request [here](https://github.com/ethereum/eth2.0-specs/pull/2146), and some technical details [here](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/r1XzqYIOv). + +### Merge + +The merge is still actively in R&D; see an [ethresear.ch](https://ethresear.ch) post describing the proposed basic mechanism [here](https://ethresear.ch/t/the-eth1-eth2-transition/6265) and the section of [ethereum.org](https://ethereum.org) describing the merge at a high level [here](https://ethereum.org/en/eth2/docking/). ### Accompanying documents can be found in [specs](specs) and include: From 71c5eb5c2f3897651f7007de681fc6ab3ac96e14 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Fri, 18 Dec 2020 11:11:44 -0800 Subject: [PATCH 058/222] Update names in pyspec README Names for pip categories have been updated --- tests/core/pyspec/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/README.md b/tests/core/pyspec/README.md index f7092dbce..d84054f11 100644 --- a/tests/core/pyspec/README.md +++ b/tests/core/pyspec/README.md @@ -27,7 +27,7 @@ python setup.py pyspec --spec-fork=phase0 --md-doc-paths="specs/phase0/beacon-ch After installing, you can install the optional dependencies for testing and linting. With makefile: `make install_test`. -Or manually: run `pip install .[testing]` and `pip install .[linting]`. +Or manually: run `pip install .[test]` and `pip install .[lint]`. These tests are not intended for client-consumption. These tests are testing the spec itself, to verify consistency and provide feedback on modifications of the spec. From 9e2fa305673353a7a659166c0e3cdbd9efe8a243 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sat, 19 Dec 2020 07:22:11 +0800 Subject: [PATCH 059/222] Complete HF1 beacon-chain.md --- specs/lightclient/beacon-chain.md | 312 +++++++++++++++++++++++++++--- 1 file changed, 281 insertions(+), 31 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 3b03ad853..11187e25b 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -1,4 +1,4 @@ -# Ethereum 2.0 Light Client Support +# Ethereum 2.0 HF1 ## Table of contents @@ -36,13 +36,39 @@ ## Introduction -This is a standalone beacon chain patch adding light client support via sync committees. +This is a patch implementing the first hard fork to the beacon chain, tentatively named HF1 pending a permanent name. It has three main features: + +* Light client support via sync committees +* Incentive accounting reforms, reducing spec complexity and reducing the cost of processing chains that have very little or zero participation for a long span of epochs +* Fork choice rule changes to address weaknesses recently discovered in the existing fork choice ## Constants +### Participation flags + | Name | Value | -| - | - | -| `BASE_REWARDS_PER_EPOCH` | `uint64(5)` | +| - | - | +| `TIMELY_HEAD_FLAG` | `0` | +| `TIMELY_SOURCE_FLAG` | `1` | +| `TIMELY_TARGET_FLAG` | `2` | + +### Participation rewards + +| Name | Value | +| - | - | +| `TIMELY_HEAD_NUMERATOR` | `12` | +| `TIMELY_SOURCE_NUMERATOR` | `12` | +| `TIMELY_TARGET_NUMERATOR` | `32` | +| `REWARD_DENOMINATOR` | `64` | + +The reward fractions add up to 7/8, leaving the remaining 1/8 for proposer rewards and other future micro-rewards. + +### Misc + +| Name | Value | +| - | - | +| `PARTICIPATION_FLAGS_LENGTH` | `8` | +| `FLAGS_AND_NUMERATORS` | `((TIMELY_HEAD_FLAG, TIMELY_HEAD_NUMERATOR), (TIMELY_SOURCE_FLAG, TIMELY_SOURCE_NUMERATOR), (TIMELY_TARGET_FLAG, TIMELY_TARGET_NUMERATOR))` | ## Configuration @@ -90,10 +116,41 @@ class BeaconBlockBody(phase0.BeaconBlockBody): #### `BeaconState` ```python -class BeaconState(phase0.BeaconState): - # Sync committees +class BeaconState(Container): + # Versioning + genesis_time: uint64 + genesis_validators_root: Root + slot: Slot + fork: Fork + # History + latest_block_header: BeaconBlockHeader + block_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] + state_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] + historical_roots: List[Root, HISTORICAL_ROOTS_LIMIT] + # Eth1 + eth1_data: Eth1Data + eth1_data_votes: List[Eth1Data, EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH] + eth1_deposit_index: uint64 + # Registry + validators: List[Validator, VALIDATOR_REGISTRY_LIMIT] + balances: List[Gwei, VALIDATOR_REGISTRY_LIMIT] + # Randomness + randao_mixes: Vector[Bytes32, EPOCHS_PER_HISTORICAL_VECTOR] + # Slashings + slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR] # Per-epoch sums of slashed effective balances + # Participation + previous_epoch_participation: List[Bitvector[PARTICIPATION_FLAGS_LENGTH], VALIDATOR_REGISTRY_LIMIT] + current_epoch_participation: List[Bitvector[PARTICIPATION_FLAGS_LENGTH], VALIDATOR_REGISTRY_LIMIT] + # Finality + justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH] # Bit set for every recent justified epoch + previous_justified_checkpoint: Checkpoint + current_justified_checkpoint: Checkpoint + finalized_checkpoint: Checkpoint + # Light client sync committees current_sync_committee: SyncCommittee next_sync_committee: SyncCommittee + # Denominator to real-time-updated balances (NOT effective balances!) + balance_denominator: uint64 ``` ### New containers @@ -110,6 +167,24 @@ class SyncCommittee(Container): ### `Predicates` +#### `get_base_reward` + +*Note*: The function `get_base_reward` is modified with the removal of `BASE_REWARDS_PER_EPOCH`. Additionally, it is split into `get_base_reward_per_eth` to + +```python +def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: + return get_base_reward_per_eth(state) * state.validators[index].effective_balance // GWEI_PER_ETH +``` + +##### `get_base_reward_per_eth` + +```python +def get_base_reward_per_eth(state: BeaconState) -> Gwei: + total_balance = get_total_active_balance(state) + total_balance = get_total_active_balance(state) + effective_balance = state.validators[index].effective_balance + return Gwei(GWEI_PER_ETH * BASE_REWARD_FACTOR // integer_squareroot(total_balance)) + #### `eth2_fast_aggregate_verify` ```python @@ -166,6 +241,69 @@ def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: return SyncCommittee(pubkeys=pubkeys, pubkey_aggregates=aggregates) ``` +#### `get_unslashed_participating_indices` + +```python +def get_unslashed_participating_indices(state: BeaconState, flag: uint8, epoch: Epoch) -> Set[ValidatorIndex]: + assert epoch in (get_previous_epoch(state), get_current_epoch(state)) + if epoch == get_current_epoch(state): + epoch_participation = state.current_epoch_participation + else: + epoch_participation = state.previous_epoch_participation + participating_indices = [index in get_active_validator_indices(state, epoch) if epoch_participation[index][flag]] + return set(filter(lambda index: not state.validators[index].slashed, participating_indices)) +``` + +#### `get_flag_deltas` + +```python +def get_flag_deltas(state: BeaconState, flag: uint8, numerator: uint64) -> Tuple[Sequence[Gwei], Gwei]: + """ + Computes the rewards and penalties associated with a particular duty, by scanning through the participation + flags to determine who participated and who did not and assigning them the appropriate rewards and penalties. + """ + rewards = [Gwei(0)] * len(state.validators) + penalties = [Gwei(0)] * len(state.validators) + unslashed_participating_indices = get_unslashed_participating_indices(state, flag, get_previous_epoch(state)) + increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from balances to avoid uint64 overflow + unslashed_participating_increments = get_total_balance(state, unslashed_participating_indices) // increment + active_increments = get_total_active_balance(state) // increment + for index in get_eligible_validator_indices(state): + base_reward = get_base_reward(state, index) + if index in unslashed_participating_indices: + if is_in_inactivity_leak(state): + # Optimal participatition is fully rewarded to cancel the inactivity penalty + rewards[index] = base_reward * numerator // REWARD_DENOMINATOR + else: + rewards[index] = ( + (base_reward * unslashed_participating_increments // active_increments + base_reward) + * numerator // REWARD_DENOMINATOR + ) + return rewards, get_base_reward_per_eth(state) * numerator // REWARD_DENOMINATOR +``` + +##### `get_inactivity_penalty_deltas` + +*Note*: The function `get_inactivity_penalty_deltas` is modified in the selection of matching target indices and the removal of `BASE_REWARDS_PER_EPOCH`. + +```python +def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Compute the penalties associated with the inactivity leak, by scanning through the participation + flags to determine who participated and who did not, applying the leak penalty globally and applying + compensatory rewards to participants. + """ + rewards = [Gwei(0) for _ in range(len(state.validators))] + if is_in_inactivity_leak(state): + reward_numerator_sum = sum(numerator for (_, numerator) in FLAGS_AND_NUMERATORS) + matching_target_indices = get_unslashed_participating_indices(state, TIMELY_TARGET_FLAG, get_previous_epoch(state)) + for index in get_eligible_validator_indices(state): + if index in matching_target_attesting_indices: + effective_balance = state.validators[index].effective_balance + rewards[index] += Gwei(effective_balance * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) + return rewards, Gwei(GWEI_PER_ETH * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) +``` + ### Block processing ```python @@ -178,6 +316,52 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_sync_committee(state, block.body) ``` +#### New `process_attestation` + +*Note*: The function `process_attestation` is modified to do incentive accounting with epoch participation flags. + +```python +def process_attestation(state: BeaconState, attestation: Attestation) -> None: + data = attestation.data + assert data.target.epoch in (get_previous_epoch(state), get_current_epoch(state)) + assert data.target.epoch == compute_epoch_at_slot(data.slot) + assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= data.slot + SLOTS_PER_EPOCH + assert data.index < get_committee_count_per_slot(state, data.target.epoch) + committee = get_beacon_committee(state, data.slot, data.index) + assert len(attestation.aggregation_bits) == len(committee) + if data.target.epoch == get_current_epoch(state): + epoch_participation = state.current_epoch_participation + justified_checkpoint = state.current_justified_checkpoint + else: + epoch_participation = state.previous_epoch_participation + justified_checkpoint = state.previous_justified_checkpoint + # Matching roots + is_matching_head = data.beacon_block_root == get_block_root_at_slot(state, data.slot) + is_matching_source = data.source == justified_checkpoint + is_matching_target = data.target.root == get_block_root(state, data.target.epoch) + assert is_matching_source + # Participation flags + participation_flags = [] + if is_matching_head and state.slot <= data.slot + MIN_ATTESTATION_INCLUSION_DELAY: + participation_flags.append(TIMELY_HEAD_FLAG) + if is_matching_source and state.slot <= data.slot + integer_squareroot(SLOTS_PER_EPOCH): + participation_flags.append(TIMELY_SOURCE_FLAG) + if is_matching_target and state.slot <= data.slot + SLOTS_PER_EPOCH: + participation_flags.append(TIMELY_TARGET_FLAG) + # Update epoch participation flags + proposer_reward_numerator = 0 + for index in get_attesting_indices(state, data, attestation.aggregation_bits): + for flag, numerator in FLAGS_AND_NUMERATORS: + if flag in participation_flags and not epoch_participation[index][flag]: + epoch_participation[index][flag] = True + proposer_reward_numerator += get_base_reward(state, index) * numerator + # Reward proposer + proposer_reward = Gwei(proposer_reward_numerator // (REWARD_DENOMINATOR * PROPOSER_REWARD_QUOTIENT)) + increase_balance(state, get_beacon_proposer_index(state), proposer_reward) + # Verify signature + assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) +``` + #### Sync committee processing ```python @@ -208,42 +392,108 @@ def process_sync_committee(state: BeaconState, body: BeaconBlockBody) -> None: ### Epoch processing -#### Components of attestation deltas +#### New `process_justification_and_finalization` -*Note*: The function `get_inactivity_penalty_deltas` is modified with `BASE_REWARDS_PER_EPOCH` replaced by `BASE_REWARDS_PER_EPOCH - 1`. +*Note*: The function `process_justification_and_finalization` is modified with `matching_target_attestations` replaced by `matching_target_indices`. ```python -def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: - """ - Return inactivity reward/penalty deltas for each validator. - """ - penalties = [Gwei(0) for _ in range(len(state.validators))] - if is_in_inactivity_leak(state): - matching_target_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) - matching_target_attesting_indices = get_unslashed_attesting_indices(state, matching_target_attestations) - for index in get_eligible_validator_indices(state): - # Penalize validator so that optimal attestation performance is rewarded with one base reward per epoch - base_reward = get_base_reward(state, index) - penalties[index] += Gwei((BASE_REWARDS_PER_EPOCH - 1) * base_reward - get_proposer_reward(state, index)) - if index not in matching_target_attesting_indices: - effective_balance = state.validators[index].effective_balance - penalties[index] += Gwei(effective_balance * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) +def process_justification_and_finalization(state: BeaconState) -> None: + # Initial FFG checkpoint values have a `0x00` stub for `root`. + # Skip FFG updates in the first two epochs to avoid corner cases that might result in modifying this stub. + if get_current_epoch(state) <= GENESIS_EPOCH + 1: + return + previous_epoch = get_previous_epoch(state) + current_epoch = get_current_epoch(state) + old_previous_justified_checkpoint = state.previous_justified_checkpoint + old_current_justified_checkpoint = state.current_justified_checkpoint + # Process justifications + state.previous_justified_checkpoint = state.current_justified_checkpoint + state.justification_bits[1:] = state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits[0] = 0b0 + matching_target_indices = get_unslashed_participating_indices(state, TIMELY_TARGET_FLAG, previous_epoch) + if get_total_balance(state, matching_target_indices) * 3 >= get_total_active_balance(state) * 2: + state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, + root=get_block_root(state, previous_epoch)) + state.justification_bits[1] = 0b1 + matching_target_indices = get_unslashed_participating_indices(state, TIMELY_TARGET_FLAG, current_epoch) + if get_total_balance(state, matching_target_indices) * 3 >= get_total_active_balance(state) * 2: + state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, + root=get_block_root(state, current_epoch)) + state.justification_bits[0] = 0b1 + # Process finalizations + bits = state.justification_bits + # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source + if all(bits[1:4]) and old_previous_justified_checkpoint.epoch + 3 == current_epoch: + state.finalized_checkpoint = old_previous_justified_checkpoint + # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source + if all(bits[1:3]) and old_previous_justified_checkpoint.epoch + 2 == current_epoch: + state.finalized_checkpoint = old_previous_justified_checkpoint + # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source + if all(bits[0:3]) and old_current_justified_checkpoint.epoch + 2 == current_epoch: + state.finalized_checkpoint = old_current_justified_checkpoint + # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source + if all(bits[0:2]) and old_current_justified_checkpoint.epoch + 1 == current_epoch: + state.finalized_checkpoint = old_current_justified_checkpoint +``` - # No rewards associated with inactivity penalties - rewards = [Gwei(0) for _ in range(len(state.validators))] - return rewards, penalties +#### New `process_rewards_and_penalties` + +*Note*: The function `process_rewards_and_penalties` is modified to use participation flag deltas. + +```python +def process_rewards_and_penalties(state: BeaconState) -> None: + # No rewards are applied at the end of `GENESIS_EPOCH` because rewards are for work done in the previous epoch + if get_current_epoch(state) == GENESIS_EPOCH: + return + flag_deltas = [get_flag_deltas(state, flag, numerator) for (flag, numerator) in FLAGS_AND_NUMERATORS] + deltas = flag_deltas + [get_inactivity_penalty_deltas(state)] + for (rewards, penalty) in deltas: + for index in range(len(state.validators)): + increase_balance(state, ValidatorIndex(index), rewards[index]) + # Bounds-friendly expansion for `denom *= (1 + penalty // GWEI_PER_ETH)` + state.balance_denominator = ( + state.balance_denominator + + penalty + + (state.balance_denominator - GWEI_PER_ETH) * penalty // GWEI_PER_ETH + ) ``` #### Final updates -*Note*: The function `process_final_updates` is modified to handle sync committee updates. +*Note*: The function `process_final_updates` is modified to handle sync committee updates, replacement of `PendingAttestation`s with participation flags, and a sliding-denominator-aware hysteresis mechanism. ```python def process_final_updates(state: BeaconState) -> None: - # FIXME: unfold the full `process_final_updates` to avoid side effects. - phase0.process_final_updates(state) - next_epoch = get_current_epoch(state) + Epoch(1) + current_epoch = get_current_epoch(state) + next_epoch = Epoch(current_epoch + 1) + # Reset eth1 data votes + if next_epoch % EPOCHS_PER_ETH1_VOTING_PERIOD == 0: + state.eth1_data_votes = [] + # Update sync committees if next_epoch % EPOCHS_PER_SYNC_COMMITTEE_PERIOD == 0: state.current_sync_committee = state.next_sync_committee state.next_sync_committee = get_sync_committee(state, next_epoch + EPOCHS_PER_SYNC_COMMITTEE_PERIOD) -``` + # Update effective balances with hysteresis + for index, validator in enumerate(state.validators): + balance_increments = state.balances[index] * HYSTERESIS_QUOTIENT // state.balance_denominator + effective_increments = validator.effective_balance // GWEI_PER_ETH * HYSTERESIS_QUOTIENT + if ( + balance_increments + HYSTERESIS_DOWNWARD_MULTIPLIER < effective_increments + or effective_increments + HYSTERESIS_UPWARD_MULTIPLIER < balance_increments + or get_current_epoch(state) == validator.withdrawable_epoch + ): + validator.effective_balance = (state.balances[index] // state.balance_denominator) * GWEI_PER_ETH + if state.balance_denominator >= 2 * GWEI_PER_ETH: + state.balances = [x//2 for x in state.balances] + state.balance_denominator //= 2 + # Reset slashings + state.slashings[next_epoch % EPOCHS_PER_SLASHINGS_VECTOR] = Gwei(0) + # Set randao mix + state.randao_mixes[next_epoch % EPOCHS_PER_HISTORICAL_VECTOR] = get_randao_mix(state, current_epoch) + # Set historical root accumulator + if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0: + historical_batch = HistoricalBatch(block_roots=state.block_roots, state_roots=state.state_roots) + state.historical_roots.append(hash_tree_root(historical_batch)) + # Rotate current/previous epoch participation flags + state.previous_epoch_participation = state.current_epoch_participation + state.current_epoch_participation = [] From 682f6c02c7a1ede584127ca60e4fdd65549f8c0b Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sun, 20 Dec 2020 18:54:57 +0800 Subject: [PATCH 060/222] Update specs/lightclient/beacon-chain.md --- specs/lightclient/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 11187e25b..c4224fbaa 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -169,7 +169,7 @@ class SyncCommittee(Container): #### `get_base_reward` -*Note*: The function `get_base_reward` is modified with the removal of `BASE_REWARDS_PER_EPOCH`. Additionally, it is split into `get_base_reward_per_eth` to +*Note*: The function `get_base_reward` is modified with the removal of `BASE_REWARDS_PER_EPOCH`. Additionally, it is split into `get_base_reward_per_eth` to allow penalties to be computed to the denominator. ```python def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: From 452e3301bed9211199329ef45ceee9bf080a68a9 Mon Sep 17 00:00:00 2001 From: Ben Edgington Date: Mon, 21 Dec 2020 10:22:04 +0000 Subject: [PATCH 061/222] Fix typo --- specs/phase0/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index b47558484..08ddc853a 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -250,7 +250,7 @@ The following values are (non-configurable) constants used throughout the specif - The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**13` epochs (about 36 days) is the time it takes the inactivity penalty to reduce the balance of non-participating validators to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline validators after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)`; so after `INVERSE_SQRT_E_DROP_TIME` epochs, it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. Note this value will be upgraded to `2**24` after Phase 0 mainnet stabilizes to provide a faster recovery in the event of an inactivity leak. -- The `PROPORTIONAL_SLASHING_MULTIPLIER` is set to `1` at initial mainnet launch, resulting in one-third of the minimum accountable safety margin in the event of a finality attack. After Phase 0 mainnet stablizes, this value will be upgraded to `3` to provide the maximal minimum accoutable safety margin. +- The `PROPORTIONAL_SLASHING_MULTIPLIER` is set to `1` at initial mainnet launch, resulting in one-third of the minimum accountable safety margin in the event of a finality attack. After Phase 0 mainnet stablizes, this value will be upgraded to `3` to provide the maximal minimum accountable safety margin. ### Max operations per block From edfd04c21287f162fab5848a24ef0cc57c8e727c Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 22 Dec 2020 10:42:59 -0800 Subject: [PATCH 062/222] Refactor sync committee rewards to use helper This change is functionally equivalent but uses the helper we already have for proposer rewards. The argument for this change is better encapsulation of the reward which makes it easier in general to reason about properties of the spec ("are the attestation proposer rewards and the sync committee proposer rewards equivalent?") and a single point of maintenance in the event that rewards get refactored in the future (which makes refactoring safer overall). --- specs/lightclient/beacon-chain.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 3b03ad853..14a6cc850 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -197,10 +197,11 @@ def process_sync_committee(state: BeaconState, body: BeaconBlockBody) -> None: active_validator_count = uint64(len(get_active_validator_indices(state, get_current_epoch(state)))) for participant_index in participant_indices: base_reward = get_base_reward(state, participant_index) - max_participant_reward = base_reward - base_reward // PROPOSER_REWARD_QUOTIENT + proposer_reward = get_proposer_reward(state, participant_index) + max_participant_reward = base_reward - proposer_reward reward = Gwei(max_participant_reward * active_validator_count // len(committee_indices) // SLOTS_PER_EPOCH) increase_balance(state, participant_index, reward) - proposer_reward += base_reward // PROPOSER_REWARD_QUOTIENT + proposer_reward += proposer_reward # Reward beacon proposer increase_balance(state, get_beacon_proposer_index(state), proposer_reward) From cc80dd758cf5120b11af3747f5826de5cdcbd87c Mon Sep 17 00:00:00 2001 From: multisignature <72713080+multisignature@users.noreply.github.com> Date: Sat, 26 Dec 2020 06:53:42 +0000 Subject: [PATCH 063/222] Update README.md (#2164) * Update README.md I've removed an incorrectly placed comma. Since the comma is placed before 'and', and the conjunction isn't being used to either a) denote the last element of a list with more than two elements or b) separate two independent clauses, it's not necessary here. * Update validator.md Fixed a couple of minor errors. --- README.md | 2 +- specs/phase1/validator.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d5eecd620..f79c2b858 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Core specifications for Eth2 clients be found in [specs](specs/). These are divi ### Sharding -The sharding spec is still actively in R&D; see the most recent available pull request [here](https://github.com/ethereum/eth2.0-specs/pull/2146), and some technical details [here](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/r1XzqYIOv). +The sharding spec is still actively in R&D; see the most recent available pull request [here](https://github.com/ethereum/eth2.0-specs/pull/2146) and some technical details [here](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/r1XzqYIOv). ### Merge diff --git a/specs/phase1/validator.md b/specs/phase1/validator.md index c5893fbc6..ccc877be0 100644 --- a/specs/phase1/validator.md +++ b/specs/phase1/validator.md @@ -67,7 +67,7 @@ A validator is an entity that participates in the consensus of the Ethereum 2.0 This document is an extension of the [Phase 0 -- Validator](../phase0/validator.md). All behaviors and definitions defined in the Phase 0 doc carry over unless explicitly noted or overridden. -All terminology, constants, functions, and protocol mechanics defined in the [Phase 1 -- The Beacon Chain](./beacon-chain.md) and [Phase 1 -- Custody Game](./custody-game.md) docs are requisite for this document and used throughout. Please see the Phase 1 docs before continuing and use as a reference throughout. +All terminology, constants, functions, and protocol mechanics defined in the [Phase 1 -- The Beacon Chain](./beacon-chain.md) and [Phase 1 -- Custody Game](./custody-game.md) docs are requisite for this document and used throughout. Please see the Phase 1 docs before continuing and use them as a reference throughout. ## Constants @@ -352,7 +352,7 @@ Aggregation selection and the core of this duty are largely unchanged from Phase Note the timing of when to broadcast aggregates is altered in Phase 1+. -If the validator is selected to aggregate (`is_aggregator`), then they broadcast their best aggregate as a `SignedAggregateAndProof` to the global aggregate channel (`beacon_aggregate_and_proof`) three-fourths of the way through the `slot`-that is, `SECONDS_PER_SLOT * 3 / 4` seconds after the start of `slot`. +If the validator is selected to aggregate (`is_aggregator`), then they broadcast their best aggregate as a `SignedAggregateAndProof` to the global aggregate channel (`beacon_aggregate_and_proof`) three-fourths of the way through the `slot` -- that is, `SECONDS_PER_SLOT * 3 / 4` seconds after the start of `slot`. ##### `AggregateAndProof` From 6a8cb48f490664adcd41fead613aaf90c4f6dbb7 Mon Sep 17 00:00:00 2001 From: Saulius Grigaitis Date: Sun, 27 Dec 2020 00:18:22 +0200 Subject: [PATCH 064/222] Updated Sharding technical details link in readme. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f79c2b858..937841127 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Core specifications for Eth2 clients be found in [specs](specs/). These are divi ### Sharding -The sharding spec is still actively in R&D; see the most recent available pull request [here](https://github.com/ethereum/eth2.0-specs/pull/2146) and some technical details [here](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/r1XzqYIOv). +The sharding spec is still actively in R&D; see the most recent available pull request [here](https://github.com/ethereum/eth2.0-specs/pull/2146) and some technical details [here](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/B1YJPGkpD). ### Merge From 7d60e482b90c65ac6abd4c316f4dc4b7f27781e0 Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Tue, 5 Jan 2021 08:48:25 +0300 Subject: [PATCH 065/222] fix typo --- specs/phase0/weak-subjectivity.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/weak-subjectivity.md b/specs/phase0/weak-subjectivity.md index b4d78cb12..4bb950540 100644 --- a/specs/phase0/weak-subjectivity.md +++ b/specs/phase0/weak-subjectivity.md @@ -42,7 +42,7 @@ This document uses data structures, constants, functions, and terminology from ## Weak Subjectivity Checkpoint -Any `Checkpoint` can used be a Weak Subjectivity Checkpoint. +Any `Checkpoint` can be used as a Weak Subjectivity Checkpoint. These Weak Subjectivity Checkpoints are distributed by providers, downloaded by users and/or distributed as a part of clients, and used as input while syncing a client. From 844c879f1c7adfb2ae04135fae50ee86e2aa2423 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 5 Jan 2021 21:00:52 +0800 Subject: [PATCH 066/222] Bump `py_ecc` and `milagro_bls_binding` (#2169) * Bump py_ecc to v5.1.0 and milagro_bls_binding to v1.6.2 * python3.8 -> python3 for py39 compatibility * fix * Try python:3.9 * Revert: using Python3.8 in CI now --- .circleci/config.yml | 4 ++-- Makefile | 12 ++++++------ setup.py | 4 ++-- tests/generators/bls/requirements.txt | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bd46e8542..bb1df9909 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,13 +35,13 @@ commands: description: "Restore the cache with pyspec keys" steps: - restore_cached_venv: - venv_name: v22-pyspec + venv_name: v24-pyspec reqs_checksum: cache-{{ checksum "setup.py" }} save_pyspec_cached_venv: description: Save a venv into a cache with pyspec keys" steps: - save_cached_venv: - venv_name: v22-pyspec + venv_name: v24-pyspec reqs_checksum: cache-{{ checksum "setup.py" }} venv_path: ./venv restore_deposit_contract_tester_cached_venv: diff --git a/Makefile b/Makefile index 987650948..08e822bf6 100644 --- a/Makefile +++ b/Makefile @@ -82,19 +82,19 @@ pyspec: # installs the packages to run pyspec tests install_test: - python3.8 -m venv venv; . venv/bin/activate; pip3 install .[lint]; pip3 install -e .[test] + python3 -m venv venv; . venv/bin/activate; python3 -m pip install .[lint]; python3 -m pip install -e .[test] test: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov=eth2spec.lightclient_patch.spec -cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec + python3 -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov=eth2spec.lightclient_patch.spec -cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec find_test: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov=eth2spec.lightclient_patch.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec + python3 -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov=eth2spec.lightclient_patch.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec citest: pyspec mkdir -p tests/core/pyspec/test-reports/eth2spec; . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python -m pytest -n 4 --bls-type=milagro --junitxml=eth2spec/test_results.xml eth2spec + python3 -m pytest -n 4 --bls-type=milagro --junitxml=eth2spec/test_results.xml eth2spec open_cov: ((open "$(COV_INDEX_FILE)" || xdg-open "$(COV_INDEX_FILE)") &> /dev/null) & @@ -133,11 +133,11 @@ test_deposit_contract: dapp test -v --fuzz-runs 5 install_deposit_contract_web3_tester: - cd $(DEPOSIT_CONTRACT_TESTER_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt + cd $(DEPOSIT_CONTRACT_TESTER_DIR); python3 -m venv venv; . venv/bin/activate; python3 -m pip install -r requirements.txt test_deposit_contract_web3_tests: cd $(DEPOSIT_CONTRACT_TESTER_DIR); . venv/bin/activate; \ - python -m pytest . + python3 -m pytest . # Runs a generator, identified by param 1 define run_generator diff --git a/setup.py b/setup.py index bd043dccc..6b8520075 100644 --- a/setup.py +++ b/setup.py @@ -582,8 +582,8 @@ setup( "eth-utils>=1.3.0,<2", "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.9.4", - "py_ecc==5.0.0", - "milagro_bls_binding==1.5.0", + "py_ecc==5.1.0", + "milagro_bls_binding==1.6.2", "dataclasses==0.6", "remerkleable==0.1.18", "ruamel.yaml==0.16.5", diff --git a/tests/generators/bls/requirements.txt b/tests/generators/bls/requirements.txt index fd54b5b32..5f830773a 100644 --- a/tests/generators/bls/requirements.txt +++ b/tests/generators/bls/requirements.txt @@ -1,4 +1,4 @@ -py_ecc==5.0.0 +py_ecc==5.1.0 eth-utils==1.6.0 ../../core/gen_helpers ../../../ From 5cf05148167c38dd135f12de7d0bbc8feb478cb9 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 5 Jan 2021 21:56:32 +0800 Subject: [PATCH 067/222] Fix ToC and minor linter issues --- specs/lightclient/beacon-chain.md | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index c4224fbaa..29678ef45 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -8,9 +8,12 @@ - [Introduction](#introduction) - [Constants](#constants) + - [Participation flags](#participation-flags) + - [Participation rewards](#participation-rewards) + - [Misc](#misc) - [Configuration](#configuration) - [Constants](#constants-1) - - [Misc](#misc) + - [Misc](#misc-1) - [Time parameters](#time-parameters) - [Domain types](#domain-types) - [Containers](#containers) @@ -21,14 +24,21 @@ - [`SyncCommittee`](#synccommittee) - [Helper functions](#helper-functions) - [`Predicates`](#predicates) + - [`get_base_reward`](#get_base_reward) + - [`get_base_reward_per_eth`](#get_base_reward_per_eth) - [`eth2_fast_aggregate_verify`](#eth2_fast_aggregate_verify) - [Beacon state accessors](#beacon-state-accessors) - [`get_sync_committee_indices`](#get_sync_committee_indices) - [`get_sync_committee`](#get_sync_committee) + - [`get_unslashed_participating_indices`](#get_unslashed_participating_indices) + - [`get_flag_deltas`](#get_flag_deltas) + - [`get_inactivity_penalty_deltas`](#get_inactivity_penalty_deltas) - [Block processing](#block-processing) + - [New `process_attestation`](#new-process_attestation) - [Sync committee processing](#sync-committee-processing) - [Epoch processing](#epoch-processing) - - [Components of attestation deltas](#components-of-attestation-deltas) + - [New `process_justification_and_finalization`](#new-process_justification_and_finalization) + - [New `process_rewards_and_penalties`](#new-process_rewards_and_penalties) - [Final updates](#final-updates) @@ -77,6 +87,7 @@ The reward fractions add up to 7/8, leaving the remaining 1/8 for proposer rewar | Name | Value | | - | - | | `G2_POINT_AT_INFINITY` | `BLSSignature(b'\xc0' + b'\x00' * 95)` | +| `GWEI_PER_ETH` | `10**9` | ### Misc @@ -184,6 +195,7 @@ def get_base_reward_per_eth(state: BeaconState) -> Gwei: total_balance = get_total_active_balance(state) effective_balance = state.validators[index].effective_balance return Gwei(GWEI_PER_ETH * BASE_REWARD_FACTOR // integer_squareroot(total_balance)) +``` #### `eth2_fast_aggregate_verify` @@ -250,7 +262,8 @@ def get_unslashed_participating_indices(state: BeaconState, flag: uint8, epoch: epoch_participation = state.current_epoch_participation else: epoch_participation = state.previous_epoch_participation - participating_indices = [index in get_active_validator_indices(state, epoch) if epoch_participation[index][flag]] + participating_indices = [index for index in get_active_validator_indices(state, epoch) + if epoch_participation[index][flag]] return set(filter(lambda index: not state.validators[index].slashed, participating_indices)) ``` @@ -263,7 +276,6 @@ def get_flag_deltas(state: BeaconState, flag: uint8, numerator: uint64) -> Tuple flags to determine who participated and who did not and assigning them the appropriate rewards and penalties. """ rewards = [Gwei(0)] * len(state.validators) - penalties = [Gwei(0)] * len(state.validators) unslashed_participating_indices = get_unslashed_participating_indices(state, flag, get_previous_epoch(state)) increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from balances to avoid uint64 overflow unslashed_participating_increments = get_total_balance(state, unslashed_participating_indices) // increment @@ -279,7 +291,7 @@ def get_flag_deltas(state: BeaconState, flag: uint8, numerator: uint64) -> Tuple (base_reward * unslashed_participating_increments // active_increments + base_reward) * numerator // REWARD_DENOMINATOR ) - return rewards, get_base_reward_per_eth(state) * numerator // REWARD_DENOMINATOR + return rewards, get_base_reward_per_eth(state) * numerator // REWARD_DENOMINATOR ``` ##### `get_inactivity_penalty_deltas` @@ -296,12 +308,14 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S rewards = [Gwei(0) for _ in range(len(state.validators))] if is_in_inactivity_leak(state): reward_numerator_sum = sum(numerator for (_, numerator) in FLAGS_AND_NUMERATORS) - matching_target_indices = get_unslashed_participating_indices(state, TIMELY_TARGET_FLAG, get_previous_epoch(state)) + matching_target_indices = get_unslashed_participating_indices( + state, TIMELY_TARGET_FLAG, get_previous_epoch(state) + ) for index in get_eligible_validator_indices(state): if index in matching_target_attesting_indices: effective_balance = state.validators[index].effective_balance rewards[index] += Gwei(effective_balance * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) - return rewards, Gwei(GWEI_PER_ETH * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) + return rewards, Gwei(GWEI_PER_ETH * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) ``` ### Block processing @@ -484,7 +498,7 @@ def process_final_updates(state: BeaconState) -> None: ): validator.effective_balance = (state.balances[index] // state.balance_denominator) * GWEI_PER_ETH if state.balance_denominator >= 2 * GWEI_PER_ETH: - state.balances = [x//2 for x in state.balances] + state.balances = [x // 2 for x in state.balances] state.balance_denominator //= 2 # Reset slashings state.slashings[next_epoch % EPOCHS_PER_SLASHINGS_VECTOR] = Gwei(0) From 958173b5bfcb070945545bd317ecec613b9e307e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 5 Jan 2021 22:19:08 +0800 Subject: [PATCH 068/222] Fix typo --- specs/lightclient/beacon-chain.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 29678ef45..b4c776afd 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -25,7 +25,7 @@ - [Helper functions](#helper-functions) - [`Predicates`](#predicates) - [`get_base_reward`](#get_base_reward) - - [`get_base_reward_per_eth`](#get_base_reward_per_eth) + - [`get_base_reward_per_eth`](#get_base_reward_per_eth) - [`eth2_fast_aggregate_verify`](#eth2_fast_aggregate_verify) - [Beacon state accessors](#beacon-state-accessors) - [`get_sync_committee_indices`](#get_sync_committee_indices) @@ -187,13 +187,11 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: return get_base_reward_per_eth(state) * state.validators[index].effective_balance // GWEI_PER_ETH ``` -##### `get_base_reward_per_eth` +#### `get_base_reward_per_eth` ```python def get_base_reward_per_eth(state: BeaconState) -> Gwei: total_balance = get_total_active_balance(state) - total_balance = get_total_active_balance(state) - effective_balance = state.validators[index].effective_balance return Gwei(GWEI_PER_ETH * BASE_REWARD_FACTOR // integer_squareroot(total_balance)) ``` @@ -308,7 +306,7 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S rewards = [Gwei(0) for _ in range(len(state.validators))] if is_in_inactivity_leak(state): reward_numerator_sum = sum(numerator for (_, numerator) in FLAGS_AND_NUMERATORS) - matching_target_indices = get_unslashed_participating_indices( + matching_target_attesting_indices = get_unslashed_participating_indices( state, TIMELY_TARGET_FLAG, get_previous_epoch(state) ) for index in get_eligible_validator_indices(state): From 39d3a18d4845f4f994a2f306a7cf97102764afe4 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 6 Jan 2021 00:42:01 +0800 Subject: [PATCH 069/222] Fix `upgrade_to_lightclient_patch` --- specs/lightclient/lightclient-fork.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/lightclient/lightclient-fork.md b/specs/lightclient/lightclient-fork.md index 568a9793b..422d0a0a4 100644 --- a/specs/lightclient/lightclient-fork.md +++ b/specs/lightclient/lightclient-fork.md @@ -43,6 +43,7 @@ def upgrade_to_lightclient_patch(pre: phase0.BeaconState) -> BeaconState: epoch = get_current_epoch(pre) post = BeaconState( genesis_time=pre.genesis_time, + genesis_validators_root=pre.genesis_validators_root, slot=pre.slot, fork=Fork( previous_version=pre.fork.current_version, @@ -67,14 +68,15 @@ def upgrade_to_lightclient_patch(pre: phase0.BeaconState) -> BeaconState: slashings=pre.slashings, # Attestations # previous_epoch_attestations is cleared on upgrade. - previous_epoch_attestations=List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH](), + previous_epoch_participation=List[Bitvector[PARTICIPATION_FLAGS_LENGTH], VALIDATOR_REGISTRY_LIMIT](), # empty in pre state, since the upgrade is performed just after an epoch boundary. - current_epoch_attestations=List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH](), + current_epoch_participation=List[Bitvector[PARTICIPATION_FLAGS_LENGTH], VALIDATOR_REGISTRY_LIMIT](), # Finality justification_bits=pre.justification_bits, previous_justified_checkpoint=pre.previous_justified_checkpoint, current_justified_checkpoint=pre.current_justified_checkpoint, finalized_checkpoint=pre.finalized_checkpoint, + balance_denominator=GWEI_PER_ETH, ) # Fill in sync committees post.current_sync_committee = get_sync_committee(post, get_current_epoch(post)) From 53ad66a4e841c4bbb207dc1bc2cea304e8c17cf2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 6 Jan 2021 01:11:46 +0800 Subject: [PATCH 070/222] Arrange constants and configurations --- specs/lightclient/beacon-chain.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index b4c776afd..0c3ed2b73 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -12,7 +12,6 @@ - [Participation rewards](#participation-rewards) - [Misc](#misc) - [Configuration](#configuration) - - [Constants](#constants-1) - [Misc](#misc-1) - [Time parameters](#time-parameters) - [Domain types](#domain-types) @@ -79,16 +78,11 @@ The reward fractions add up to 7/8, leaving the remaining 1/8 for proposer rewar | - | - | | `PARTICIPATION_FLAGS_LENGTH` | `8` | | `FLAGS_AND_NUMERATORS` | `((TIMELY_HEAD_FLAG, TIMELY_HEAD_NUMERATOR), (TIMELY_SOURCE_FLAG, TIMELY_SOURCE_NUMERATOR), (TIMELY_TARGET_FLAG, TIMELY_TARGET_NUMERATOR))` | - -## Configuration - -### Constants - -| Name | Value | -| - | - | | `G2_POINT_AT_INFINITY` | `BLSSignature(b'\xc0' + b'\x00' * 95)` | | `GWEI_PER_ETH` | `10**9` | +## Configuration + ### Misc | Name | Value | From 9c75c3819da5f57b58a756096c0da00fcde50bcd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 6 Jan 2021 02:09:39 +0800 Subject: [PATCH 071/222] Fix `previous_epoch_participation` and `current_epoch_participation` initialization --- specs/lightclient/beacon-chain.md | 48 ++++++++++++++++++- specs/lightclient/lightclient-fork.md | 6 +-- .../eth2spec/test/helpers/attestations.py | 20 +++++--- 3 files changed, 63 insertions(+), 11 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 0c3ed2b73..4fff3a4f1 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -34,6 +34,7 @@ - [`get_inactivity_penalty_deltas`](#get_inactivity_penalty_deltas) - [Block processing](#block-processing) - [New `process_attestation`](#new-process_attestation) + - [New `process_deposit`](#new-process_deposit) - [Sync committee processing](#sync-committee-processing) - [Epoch processing](#epoch-processing) - [New `process_justification_and_finalization`](#new-process_justification_and_finalization) @@ -368,6 +369,51 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) ``` + +#### New `process_deposit` + +*Note*: The function `process_deposit` is modified to initialize `previous_epoch_participation` and `current_epoch_participation`. + +```python +def process_deposit(state: BeaconState, deposit: Deposit) -> None: + # Verify the Merkle branch + assert is_valid_merkle_branch( + leaf=hash_tree_root(deposit.data), + branch=deposit.proof, + depth=DEPOSIT_CONTRACT_TREE_DEPTH + 1, # Add 1 for the List length mix-in + index=state.eth1_deposit_index, + root=state.eth1_data.deposit_root, + ) + + # Deposits must be processed in order + state.eth1_deposit_index += 1 + + pubkey = deposit.data.pubkey + amount = deposit.data.amount + validator_pubkeys = [v.pubkey for v in state.validators] + if pubkey not in validator_pubkeys: + # Verify the deposit signature (proof of possession) which is not checked by the deposit contract + deposit_message = DepositMessage( + pubkey=deposit.data.pubkey, + withdrawal_credentials=deposit.data.withdrawal_credentials, + amount=deposit.data.amount, + ) + domain = compute_domain(DOMAIN_DEPOSIT) # Fork-agnostic domain since deposits are valid across forks + signing_root = compute_signing_root(deposit_message, domain) + if not bls.Verify(pubkey, signing_root, deposit.data.signature): + return + + # Add validator and balance entries + state.validators.append(get_validator_from_deposit(state, deposit)) + state.balances.append(amount) + state.previous_epoch_participation.append(Bitvector[PARTICIPATION_FLAGS_LENGTH]()) + state.current_epoch_participation.append(Bitvector[PARTICIPATION_FLAGS_LENGTH]()) + else: + # Increase balance by deposit amount + index = ValidatorIndex(validator_pubkeys.index(pubkey)) + increase_balance(state, index, amount) +``` + #### Sync committee processing ```python @@ -502,4 +548,4 @@ def process_final_updates(state: BeaconState) -> None: state.historical_roots.append(hash_tree_root(historical_batch)) # Rotate current/previous epoch participation flags state.previous_epoch_participation = state.current_epoch_participation - state.current_epoch_participation = [] + state.current_epoch_participation = [Bitvector[PARTICIPATION_FLAGS_LENGTH]() for _ in range(len(state.validators))] diff --git a/specs/lightclient/lightclient-fork.md b/specs/lightclient/lightclient-fork.md index 422d0a0a4..8e2791d4b 100644 --- a/specs/lightclient/lightclient-fork.md +++ b/specs/lightclient/lightclient-fork.md @@ -67,10 +67,8 @@ def upgrade_to_lightclient_patch(pre: phase0.BeaconState) -> BeaconState: # Slashings slashings=pre.slashings, # Attestations - # previous_epoch_attestations is cleared on upgrade. - previous_epoch_participation=List[Bitvector[PARTICIPATION_FLAGS_LENGTH], VALIDATOR_REGISTRY_LIMIT](), - # empty in pre state, since the upgrade is performed just after an epoch boundary. - current_epoch_participation=List[Bitvector[PARTICIPATION_FLAGS_LENGTH], VALIDATOR_REGISTRY_LIMIT](), + previous_epoch_participation=[Bitvector[PARTICIPATION_FLAGS_LENGTH]() for _ in range(len(pre.validators))], + current_epoch_participation=[Bitvector[PARTICIPATION_FLAGS_LENGTH]() for _ in range(len(pre.validators))], # Finality justification_bits=pre.justification_bits, previous_justified_checkpoint=pre.previous_justified_checkpoint, diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index b924da378..efc8d7a5d 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -2,7 +2,7 @@ from lru import LRU from typing import List -from eth2spec.test.context import expect_assertion_error, PHASE1 +from eth2spec.test.context import expect_assertion_error, PHASE1, LIGHTCLIENT_PATCH from eth2spec.test.helpers.state import state_transition_and_sign_block, next_epoch, next_slot from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.shard_transitions import get_shard_transition_of_committee @@ -30,17 +30,25 @@ def run_attestation_processing(spec, state, attestation, valid=True): yield 'post', None return - current_epoch_count = len(state.current_epoch_attestations) - previous_epoch_count = len(state.previous_epoch_attestations) + if spec.fork != LIGHTCLIENT_PATCH: + current_epoch_count = len(state.current_epoch_attestations) + previous_epoch_count = len(state.previous_epoch_attestations) # process attestation spec.process_attestation(state, attestation) # Make sure the attestation has been processed - if attestation.data.target.epoch == spec.get_current_epoch(state): - assert len(state.current_epoch_attestations) == current_epoch_count + 1 + if spec.fork != LIGHTCLIENT_PATCH: + if attestation.data.target.epoch == spec.get_current_epoch(state): + assert len(state.current_epoch_attestations) == current_epoch_count + 1 + else: + assert len(state.previous_epoch_attestations) == previous_epoch_count + 1 else: - assert len(state.previous_epoch_attestations) == previous_epoch_count + 1 + for index in spec.get_attesting_indices(state, attestation.data, attestation.aggregation_bits): + if attestation.data.target.epoch == spec.get_current_epoch(state): + assert state.current_epoch_participation[index][spec.TIMELY_TARGET_FLAG] + else: + assert state.previous_epoch_participation[index][spec.TIMELY_TARGET_FLAG] # yield post-state yield 'post', state From b8d3589b46cdfede1db990650fc196fbbfc6e5f7 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 5 Jan 2021 13:48:26 -0700 Subject: [PATCH 072/222] remove global quotient penalties from fork spec [temp] --- specs/lightclient/beacon-chain.md | 133 +++++++++++++++----------- specs/lightclient/lightclient-fork.md | 1 - 2 files changed, 77 insertions(+), 57 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 4fff3a4f1..afcb3a36d 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -49,7 +49,8 @@ This is a patch implementing the first hard fork to the beacon chain, tentatively named HF1 pending a permanent name. It has three main features: * Light client support via sync committees -* Incentive accounting reforms, reducing spec complexity and reducing the cost of processing chains that have very little or zero participation for a long span of epochs +* Incentive accounting reforms, reducing spec complexity + and [TODO] reducing the cost of processing chains that have very little or zero participation for a long span of epochs * Fork choice rule changes to address weaknesses recently discovered in the existing fork choice ## Constants @@ -78,9 +79,7 @@ The reward fractions add up to 7/8, leaving the remaining 1/8 for proposer rewar | Name | Value | | - | - | | `PARTICIPATION_FLAGS_LENGTH` | `8` | -| `FLAGS_AND_NUMERATORS` | `((TIMELY_HEAD_FLAG, TIMELY_HEAD_NUMERATOR), (TIMELY_SOURCE_FLAG, TIMELY_SOURCE_NUMERATOR), (TIMELY_TARGET_FLAG, TIMELY_TARGET_NUMERATOR))` | | `G2_POINT_AT_INFINITY` | `BLSSignature(b'\xc0' + b'\x00' * 95)` | -| `GWEI_PER_ETH` | `10**9` | ## Configuration @@ -155,8 +154,6 @@ class BeaconState(Container): # Light client sync committees current_sync_committee: SyncCommittee next_sync_committee: SyncCommittee - # Denominator to real-time-updated balances (NOT effective balances!) - balance_denominator: uint64 ``` ### New containers @@ -173,23 +170,6 @@ class SyncCommittee(Container): ### `Predicates` -#### `get_base_reward` - -*Note*: The function `get_base_reward` is modified with the removal of `BASE_REWARDS_PER_EPOCH`. Additionally, it is split into `get_base_reward_per_eth` to allow penalties to be computed to the denominator. - -```python -def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: - return get_base_reward_per_eth(state) * state.validators[index].effective_balance // GWEI_PER_ETH -``` - -#### `get_base_reward_per_eth` - -```python -def get_base_reward_per_eth(state: BeaconState) -> Gwei: - total_balance = get_total_active_balance(state) - return Gwei(GWEI_PER_ETH * BASE_REWARD_FACTOR // integer_squareroot(total_balance)) -``` - #### `eth2_fast_aggregate_verify` ```python @@ -202,6 +182,21 @@ def eth2_fast_aggregate_verify(pubkeys: Sequence[BLSPubkey], message: Bytes32, s return bls.FastAggregateVerify(pubkeys, message, signature) ``` +### Misc + +#### `flags_and_numerators` + +```python +def get_flags_and_numerators() -> Sequence[Tuple[int, int]]: + return ( + (TIMELY_HEAD_FLAG, TIMELY_HEAD_NUMERATOR), + (TIMELY_SOURCE_FLAG, TIMELY_SOURCE_NUMERATOR), + (TIMELY_TARGET_FLAG, TIMELY_TARGET_NUMERATOR) + ) +``` + + + ### Beacon state accessors #### `get_sync_committee_indices` @@ -246,6 +241,17 @@ def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: return SyncCommittee(pubkeys=pubkeys, pubkey_aggregates=aggregates) ``` +#### `get_base_reward` + +*Note*: The function `get_base_reward` is modified with the removal of `BASE_REWARDS_PER_EPOCH`. + +```python +def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: + total_balance = get_total_active_balance(state) + effective_balance = state.validators[index].effective_balance + return Gwei(effective_balance * BASE_REWARD_FACTOR // integer_squareroot(total_balance)) +``` + #### `get_unslashed_participating_indices` ```python @@ -255,20 +261,24 @@ def get_unslashed_participating_indices(state: BeaconState, flag: uint8, epoch: epoch_participation = state.current_epoch_participation else: epoch_participation = state.previous_epoch_participation - participating_indices = [index for index in get_active_validator_indices(state, epoch) - if epoch_participation[index][flag]] + participating_indices = [ + index for index in get_active_validator_indices(state, epoch) + if epoch_participation[index][flag] + ] return set(filter(lambda index: not state.validators[index].slashed, participating_indices)) ``` #### `get_flag_deltas` ```python -def get_flag_deltas(state: BeaconState, flag: uint8, numerator: uint64) -> Tuple[Sequence[Gwei], Gwei]: +def get_flag_deltas(state: BeaconState, flag: uint8, numerator: uint64) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: """ Computes the rewards and penalties associated with a particular duty, by scanning through the participation flags to determine who participated and who did not and assigning them the appropriate rewards and penalties. """ rewards = [Gwei(0)] * len(state.validators) + penalties = [Gwei(0)] * len(state.validators) + unslashed_participating_indices = get_unslashed_participating_indices(state, flag, get_previous_epoch(state)) increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from balances to avoid uint64 overflow unslashed_participating_increments = get_total_balance(state, unslashed_participating_indices) // increment @@ -281,10 +291,12 @@ def get_flag_deltas(state: BeaconState, flag: uint8, numerator: uint64) -> Tuple rewards[index] = base_reward * numerator // REWARD_DENOMINATOR else: rewards[index] = ( - (base_reward * unslashed_participating_increments // active_increments + base_reward) - * numerator // REWARD_DENOMINATOR + (base_reward * numerator * unslashed_participating_increments) + // (active_increments * REWARD_DENOMINATOR) ) - return rewards, get_base_reward_per_eth(state) * numerator // REWARD_DENOMINATOR + else: + penalties[index] = base_reward * numerator // REWARD_DENOMINATOR + return rewards, penalties ``` ##### `get_inactivity_penalty_deltas` @@ -298,17 +310,21 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S flags to determine who participated and who did not, applying the leak penalty globally and applying compensatory rewards to participants. """ - rewards = [Gwei(0) for _ in range(len(state.validators))] + penalties = [Gwei(0) for _ in range(len(state.validators))] if is_in_inactivity_leak(state): - reward_numerator_sum = sum(numerator for (_, numerator) in FLAGS_AND_NUMERATORS) + reward_numerator_sum = sum(numerator for (_, numerator) in get_flags_and_numerators()) matching_target_attesting_indices = get_unslashed_participating_indices( state, TIMELY_TARGET_FLAG, get_previous_epoch(state) ) for index in get_eligible_validator_indices(state): - if index in matching_target_attesting_indices: + # If validator is performing optimally this cancels all attestation rewards for a neutral balance + penalties[index] += Gwei(get_base_reward(state, index) * reward_numerator_sum // REWARD_DENOMINATOR) + if index not in matching_target_attesting_indices: effective_balance = state.validators[index].effective_balance - rewards[index] += Gwei(effective_balance * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) - return rewards, Gwei(GWEI_PER_ETH * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) + penalties[index] += Gwei(effective_balance * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) + + rewards = [Gwei(0) for _ in range(len(state.validators))] + return rewards, penalties ``` ### Block processing @@ -334,19 +350,26 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: assert data.target.epoch == compute_epoch_at_slot(data.slot) assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= data.slot + SLOTS_PER_EPOCH assert data.index < get_committee_count_per_slot(state, data.target.epoch) + committee = get_beacon_committee(state, data.slot, data.index) assert len(attestation.aggregation_bits) == len(committee) + if data.target.epoch == get_current_epoch(state): epoch_participation = state.current_epoch_participation justified_checkpoint = state.current_justified_checkpoint else: epoch_participation = state.previous_epoch_participation justified_checkpoint = state.previous_justified_checkpoint + # Matching roots is_matching_head = data.beacon_block_root == get_block_root_at_slot(state, data.slot) is_matching_source = data.source == justified_checkpoint is_matching_target = data.target.root == get_block_root(state, data.target.epoch) assert is_matching_source + + # Verify signature + assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) + # Participation flags participation_flags = [] if is_matching_head and state.slot <= data.slot + MIN_ATTESTATION_INCLUSION_DELAY: @@ -355,18 +378,18 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: participation_flags.append(TIMELY_SOURCE_FLAG) if is_matching_target and state.slot <= data.slot + SLOTS_PER_EPOCH: participation_flags.append(TIMELY_TARGET_FLAG) + # Update epoch participation flags proposer_reward_numerator = 0 for index in get_attesting_indices(state, data, attestation.aggregation_bits): - for flag, numerator in FLAGS_AND_NUMERATORS: + for flag, numerator in get_flags_and_numerators(): if flag in participation_flags and not epoch_participation[index][flag]: epoch_participation[index][flag] = True proposer_reward_numerator += get_base_reward(state, index) * numerator + # Reward proposer proposer_reward = Gwei(proposer_reward_numerator // (REWARD_DENOMINATOR * PROPOSER_REWARD_QUOTIENT)) increase_balance(state, get_beacon_proposer_index(state), proposer_reward) - # Verify signature - assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) ``` @@ -406,6 +429,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: # Add validator and balance entries state.validators.append(get_validator_from_deposit(state, deposit)) state.balances.append(amount) + # [Added in hf-1] Initialize empty participation flags for new validator state.previous_epoch_participation.append(Bitvector[PARTICIPATION_FLAGS_LENGTH]()) state.current_epoch_participation.append(Bitvector[PARTICIPATION_FLAGS_LENGTH]()) else: @@ -458,6 +482,7 @@ def process_justification_and_finalization(state: BeaconState) -> None: current_epoch = get_current_epoch(state) old_previous_justified_checkpoint = state.previous_justified_checkpoint old_current_justified_checkpoint = state.current_justified_checkpoint + # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint state.justification_bits[1:] = state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] @@ -472,6 +497,7 @@ def process_justification_and_finalization(state: BeaconState) -> None: state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) state.justification_bits[0] = 0b1 + # Process finalizations bits = state.justification_bits # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source @@ -497,22 +523,17 @@ def process_rewards_and_penalties(state: BeaconState) -> None: # No rewards are applied at the end of `GENESIS_EPOCH` because rewards are for work done in the previous epoch if get_current_epoch(state) == GENESIS_EPOCH: return - flag_deltas = [get_flag_deltas(state, flag, numerator) for (flag, numerator) in FLAGS_AND_NUMERATORS] + flag_deltas = [get_flag_deltas(state, flag, numerator) for (flag, numerator) in get_flags_and_numerators()] deltas = flag_deltas + [get_inactivity_penalty_deltas(state)] - for (rewards, penalty) in deltas: + for (rewards, penalties) in deltas: for index in range(len(state.validators)): increase_balance(state, ValidatorIndex(index), rewards[index]) - # Bounds-friendly expansion for `denom *= (1 + penalty // GWEI_PER_ETH)` - state.balance_denominator = ( - state.balance_denominator + - penalty + - (state.balance_denominator - GWEI_PER_ETH) * penalty // GWEI_PER_ETH - ) + decrease_balance(state, ValidatorIndex(index), penalties[index]) ``` #### Final updates -*Note*: The function `process_final_updates` is modified to handle sync committee updates, replacement of `PendingAttestation`s with participation flags, and a sliding-denominator-aware hysteresis mechanism. +*Note*: The function `process_final_updates` is modified to handle sync committee updates and with the replacement of `PendingAttestation`s with participation flags. ```python def process_final_updates(state: BeaconState) -> None: @@ -521,23 +542,21 @@ def process_final_updates(state: BeaconState) -> None: # Reset eth1 data votes if next_epoch % EPOCHS_PER_ETH1_VOTING_PERIOD == 0: state.eth1_data_votes = [] - # Update sync committees + # [Added in hf-1] Update sync committees if next_epoch % EPOCHS_PER_SYNC_COMMITTEE_PERIOD == 0: state.current_sync_committee = state.next_sync_committee state.next_sync_committee = get_sync_committee(state, next_epoch + EPOCHS_PER_SYNC_COMMITTEE_PERIOD) # Update effective balances with hysteresis for index, validator in enumerate(state.validators): - balance_increments = state.balances[index] * HYSTERESIS_QUOTIENT // state.balance_denominator - effective_increments = validator.effective_balance // GWEI_PER_ETH * HYSTERESIS_QUOTIENT + balance = state.balances[index] + HYSTERESIS_INCREMENT = uint64(EFFECTIVE_BALANCE_INCREMENT // HYSTERESIS_QUOTIENT) + DOWNWARD_THRESHOLD = HYSTERESIS_INCREMENT * HYSTERESIS_DOWNWARD_MULTIPLIER + UPWARD_THRESHOLD = HYSTERESIS_INCREMENT * HYSTERESIS_UPWARD_MULTIPLIER if ( - balance_increments + HYSTERESIS_DOWNWARD_MULTIPLIER < effective_increments - or effective_increments + HYSTERESIS_UPWARD_MULTIPLIER < balance_increments - or get_current_epoch(state) == validator.withdrawable_epoch + balance + DOWNWARD_THRESHOLD < validator.effective_balance + or validator.effective_balance + UPWARD_THRESHOLD < balance ): - validator.effective_balance = (state.balances[index] // state.balance_denominator) * GWEI_PER_ETH - if state.balance_denominator >= 2 * GWEI_PER_ETH: - state.balances = [x // 2 for x in state.balances] - state.balance_denominator //= 2 + validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) # Reset slashings state.slashings[next_epoch % EPOCHS_PER_SLASHINGS_VECTOR] = Gwei(0) # Set randao mix @@ -546,6 +565,8 @@ def process_final_updates(state: BeaconState) -> None: if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0: historical_batch = HistoricalBatch(block_roots=state.block_roots, state_roots=state.state_roots) state.historical_roots.append(hash_tree_root(historical_batch)) - # Rotate current/previous epoch participation flags + # [Added in hf-1] Rotate current/previous epoch participation flags state.previous_epoch_participation = state.current_epoch_participation state.current_epoch_participation = [Bitvector[PARTICIPATION_FLAGS_LENGTH]() for _ in range(len(state.validators))] + # [Removed in hf-1] Rotate current/previous epoch attestations +``` diff --git a/specs/lightclient/lightclient-fork.md b/specs/lightclient/lightclient-fork.md index 8e2791d4b..a10e8c5f6 100644 --- a/specs/lightclient/lightclient-fork.md +++ b/specs/lightclient/lightclient-fork.md @@ -74,7 +74,6 @@ def upgrade_to_lightclient_patch(pre: phase0.BeaconState) -> BeaconState: previous_justified_checkpoint=pre.previous_justified_checkpoint, current_justified_checkpoint=pre.current_justified_checkpoint, finalized_checkpoint=pre.finalized_checkpoint, - balance_denominator=GWEI_PER_ETH, ) # Fill in sync committees post.current_sync_committee = get_sync_committee(post, get_current_epoch(post)) From 271b9dff83521aac4e24e24e168d046949133f1b Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 5 Jan 2021 14:17:34 -0700 Subject: [PATCH 073/222] toc --- specs/lightclient/beacon-chain.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index afcb3a36d..1112ea126 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -23,12 +23,13 @@ - [`SyncCommittee`](#synccommittee) - [Helper functions](#helper-functions) - [`Predicates`](#predicates) - - [`get_base_reward`](#get_base_reward) - - [`get_base_reward_per_eth`](#get_base_reward_per_eth) - [`eth2_fast_aggregate_verify`](#eth2_fast_aggregate_verify) + - [Misc](#misc-2) + - [`flags_and_numerators`](#flags_and_numerators) - [Beacon state accessors](#beacon-state-accessors) - [`get_sync_committee_indices`](#get_sync_committee_indices) - [`get_sync_committee`](#get_sync_committee) + - [`get_base_reward`](#get_base_reward) - [`get_unslashed_participating_indices`](#get_unslashed_participating_indices) - [`get_flag_deltas`](#get_flag_deltas) - [`get_inactivity_penalty_deltas`](#get_inactivity_penalty_deltas) From 70e25e63900db8ff0fdf088750cbe2de821b9204 Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Tue, 5 Jan 2021 21:13:35 -0800 Subject: [PATCH 074/222] Update specs/phase0/weak-subjectivity.md Co-authored-by: Hsiao-Wei Wang --- specs/phase0/weak-subjectivity.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/weak-subjectivity.md b/specs/phase0/weak-subjectivity.md index 4bb950540..a99887926 100644 --- a/specs/phase0/weak-subjectivity.md +++ b/specs/phase0/weak-subjectivity.md @@ -42,7 +42,7 @@ This document uses data structures, constants, functions, and terminology from ## Weak Subjectivity Checkpoint -Any `Checkpoint` can be used as a Weak Subjectivity Checkpoint. +Any `Checkpoint` object can be used as a Weak Subjectivity Checkpoint. These Weak Subjectivity Checkpoints are distributed by providers, downloaded by users and/or distributed as a part of clients, and used as input while syncing a client. From a3bf632b4f01c21151c5011c70f3efac290dafd2 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Wed, 6 Jan 2021 08:39:21 -0800 Subject: [PATCH 075/222] Bugfix in sync committee proposer rewards The variable used to accumulate proposer rewards across the sync committee processing was shadowed by the per-participant proposer reward. This means the total proposer reward would simply be twice the output of `get_proposer_reward` for the last participant in the sync committee. I believe we want to sum all contributions to the proposer reward across sync committee participants which is what this PR does. --- specs/lightclient/beacon-chain.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 14a6cc850..2fbe190e4 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -193,7 +193,7 @@ def process_sync_committee(state: BeaconState, body: BeaconBlockBody) -> None: assert eth2_fast_aggregate_verify(participant_pubkeys, signing_root, body.sync_committee_signature) # Reward sync committee participants - proposer_reward = Gwei(0) + total_proposer_reward = Gwei(0) active_validator_count = uint64(len(get_active_validator_indices(state, get_current_epoch(state)))) for participant_index in participant_indices: base_reward = get_base_reward(state, participant_index) @@ -201,10 +201,10 @@ def process_sync_committee(state: BeaconState, body: BeaconBlockBody) -> None: max_participant_reward = base_reward - proposer_reward reward = Gwei(max_participant_reward * active_validator_count // len(committee_indices) // SLOTS_PER_EPOCH) increase_balance(state, participant_index, reward) - proposer_reward += proposer_reward + total_proposer_reward += proposer_reward # Reward beacon proposer - increase_balance(state, get_beacon_proposer_index(state), proposer_reward) + increase_balance(state, get_beacon_proposer_index(state), total_proposer_reward) ``` ### Epoch processing From b94af435dab7c97e137f83bbfb99d824159b5f81 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 5 Jan 2021 12:38:50 -0800 Subject: [PATCH 076/222] Refactor helpers into separate module --- .../test/lightclient_patch/__init__.py | 0 .../test/lightclient_patch/helpers.py | 33 +++++++++++++++++++ .../lightclient_patch/sanity/test_blocks.py | 33 ++----------------- 3 files changed, 36 insertions(+), 30 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/lightclient_patch/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/lightclient_patch/helpers.py diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/__init__.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/helpers.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/helpers.py new file mode 100644 index 000000000..b7b2381e3 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/helpers.py @@ -0,0 +1,33 @@ +from eth2spec.test.helpers.keys import privkeys +from eth2spec.test.helpers.block import ( + build_empty_block_for_next_slot, +) +from eth2spec.utils import bls + + +def compute_sync_committee_signature(spec, state, slot, privkey): + domain = spec.get_domain(state, spec.DOMAIN_SYNC_COMMITTEE, spec.compute_epoch_at_slot(slot)) + if slot == state.slot: + block_root = build_empty_block_for_next_slot(spec, state).parent_root + else: + block_root = spec.get_block_root_at_slot(state, slot) + signing_root = spec.compute_signing_root(block_root, domain) + return bls.Sign(privkey, signing_root) + + +def compute_aggregate_sync_committee_signature(spec, state, slot, participants): + if len(participants) == 0: + return spec.G2_POINT_AT_INFINITY + + signatures = [] + for validator_index in participants: + privkey = privkeys[validator_index] + signatures.append( + compute_sync_committee_signature( + spec, + state, + slot, + privkey, + ) + ) + return bls.Aggregate(signatures) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py index 4fbdfc371..1698ecce1 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py @@ -1,6 +1,4 @@ import random -from eth2spec.test.helpers.keys import privkeys -from eth2spec.utils import bls from eth2spec.test.helpers.state import ( state_transition_and_sign_block, next_epoch, @@ -8,6 +6,9 @@ from eth2spec.test.helpers.state import ( from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, ) +from eth2spec.test.lightclient_patch.helpers import ( + compute_aggregate_sync_committee_signature, +) from eth2spec.test.context import ( PHASE0, PHASE1, with_all_phases_except, @@ -15,34 +16,6 @@ from eth2spec.test.context import ( ) -def compute_sync_committee_signature(spec, state, slot, privkey): - domain = spec.get_domain(state, spec.DOMAIN_SYNC_COMMITTEE, spec.compute_epoch_at_slot(slot)) - if slot == state.slot: - block_root = build_empty_block_for_next_slot(spec, state).parent_root - else: - block_root = spec.get_block_root_at_slot(state, slot) - signing_root = spec.compute_signing_root(block_root, domain) - return bls.Sign(privkey, signing_root) - - -def compute_aggregate_sync_committee_signature(spec, state, slot, participants): - if len(participants) == 0: - return spec.G2_POINT_AT_INFINITY - - signatures = [] - for validator_index in participants: - privkey = privkeys[validator_index] - signatures.append( - compute_sync_committee_signature( - spec, - state, - slot, - privkey, - ) - ) - return bls.Aggregate(signatures) - - def run_sync_committee_sanity_test(spec, state, fraction_full=1.0): committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) participants = random.sample(committee, int(len(committee) * fraction_full)) From 955a01c49bc816642523ce772210bae5dcbe8b08 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 5 Jan 2021 12:39:13 -0800 Subject: [PATCH 077/222] Add basic test for invalid sync committee bits --- .../block_processing/__init__.py | 0 .../test_process_sync_committee.py | 36 +++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/__init__.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py new file mode 100644 index 000000000..578c1a1c6 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -0,0 +1,36 @@ +import random +from eth2spec.test.helpers.block import ( + build_empty_block_for_next_slot, +) +from eth2spec.test.lightclient_patch.helpers import ( + compute_aggregate_sync_committee_signature, +) +from eth2spec.test.context import ( + PHASE0, PHASE1, + expect_assertion_error, + with_all_phases_except, + spec_state_test, +) + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_invalid_sync_committee_bits(spec, state): + committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + random_participant = random.choice(committee) + + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + # Exclude one participant whose signature was included. + block.body.sync_committee_bits = [index != random_participant for index in committee] + block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + block.slot - 1, + committee, + ) + + yield 'blocks', [block] + expect_assertion_error(lambda: spec.process_sync_committee(state, block.body)) + yield 'post', None From 781f3444098eceda820436d440228a60af6751eb Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 5 Jan 2021 15:18:56 -0800 Subject: [PATCH 078/222] Add test for invalid sync committee signature --- .../test_process_sync_committee.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index 578c1a1c6..90d4af3cc 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -34,3 +34,26 @@ def test_invalid_sync_committee_bits(spec, state): yield 'blocks', [block] expect_assertion_error(lambda: spec.process_sync_committee(state, block.body)) yield 'post', None + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_invalid_sync_committee_signature(spec, state): + committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + random_participant = random.choice(committee) + + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + # Exclude one signature even though the block claims the entire committee participated. + block.body.sync_committee_bits = [True] * len(committee) + block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + block.slot - 1, + [index for index in committee if index != random_participant], + ) + + yield 'blocks', [block] + expect_assertion_error(lambda: spec.process_sync_committee(state, block.body)) + yield 'post', None From 547cb0f38f78e490760e544ac549feb0c48bd622 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 5 Jan 2021 17:04:40 -0800 Subject: [PATCH 079/222] Add epoch processing test for sync committee updates --- .../epoch_processing/__init__.py | 0 .../test_process_final_updates.py | 37 +++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/__init__.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py new file mode 100644 index 000000000..227b2461d --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py @@ -0,0 +1,37 @@ +from eth2spec.test.context import ( + PHASE0, PHASE1, + with_all_phases_except, + spec_state_test, +) +from eth2spec.test.helpers.state import transition_to +from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import ( + run_epoch_processing_with, +) + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_sync_committees_progress(spec, state): + current_epoch = spec.get_current_epoch(state) + # NOTE: if not in the genesis epoch, period math below needs to be + # adjusted relative to the current epoch + assert current_epoch == 0 + + first_sync_committee = state.current_sync_committee + second_sync_committee = state.next_sync_committee + + slot_at_end_of_current_period = spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH - 1 + transition_to(spec, state, slot_at_end_of_current_period) + + # Ensure assignments have not changed: + assert state.current_sync_committee == first_sync_committee + assert state.next_sync_committee == second_sync_committee + + yield from run_epoch_processing_with(spec, state, 'process_final_updates') + + # Can compute the third committee having computed final balances in the last epoch + # of this `EPOCHS_PER_SYNC_COMMITTEE_PERIOD` + third_sync_committee = spec.get_sync_committee(state, 2 * spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD) + + assert state.current_sync_committee == second_sync_committee + assert state.next_sync_committee == third_sync_committee From cc7ae4abd0551f72c22b0493755f41a2349a4563 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Wed, 6 Jan 2021 09:44:17 -0800 Subject: [PATCH 080/222] Add test for sync committee block rewards --- .../test_process_sync_committee.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index 90d4af3cc..1a3b41fa4 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -2,6 +2,9 @@ import random from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, ) +from eth2spec.test.helpers.state import ( + state_transition_and_sign_block, +) from eth2spec.test.lightclient_patch.helpers import ( compute_aggregate_sync_committee_signature, ) @@ -57,3 +60,53 @@ def test_invalid_sync_committee_signature(spec, state): yield 'blocks', [block] expect_assertion_error(lambda: spec.process_sync_committee(state, block.body)) yield 'post', None + + +def compute_sync_committee_participant_reward(spec, state, participant_index, active_validator_count, committee_size): + base_reward = spec.get_base_reward(state, participant_index) + proposer_reward = spec.get_proposer_reward(state, participant_index) + max_participant_reward = base_reward - proposer_reward + return max_participant_reward * active_validator_count // committee_size // spec.SLOTS_PER_EPOCH + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_sync_committee_rewards(spec, state): + committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + committee_size = len(committee) + active_validator_count = len(spec.get_active_validator_indices(state, spec.get_current_epoch(state))) + + yield 'pre', state + + pre_balances = state.balances.copy() + + block = build_empty_block_for_next_slot(spec, state) + block.body.sync_committee_bits = [True] * committee_size + block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + block.slot - 1, + committee, + ) + + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state + + for index in range(len(state.validators)): + expected_reward = 0 + + if index == block.proposer_index: + expected_reward += sum([spec.get_proposer_reward(state, index) for index in committee]) + + if index in committee: + expected_reward += compute_sync_committee_participant_reward( + spec, + state, + index, + active_validator_count, + committee_size + ) + + assert state.balances[index] == pre_balances[index] + expected_reward From ce879143266f9f60a1bbb65e6c96adc6f6d7e7c5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 7 Jan 2021 11:14:32 +0800 Subject: [PATCH 081/222] Fix the 2/3 threshold calculation Co-authored-by: Danny Ryan --- specs/lightclient/sync-protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index 156e6e78e..562b7b5bb 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -185,7 +185,7 @@ def process_light_client_update(store: LightClientStore, update: LightClientUpda store.valid_updates.append(update) if ( - sum(update.sync_committee_bits) * 3 > len(update.sync_committee_bits) * 2 + sum(update.sync_committee_bits) * 3 >= len(update.sync_committee_bits) * 2 and update.header != update.finality_header ): # Apply update if 2/3 quorum is reached and we have a finality proof From 7e82b54131ad2252c0b8749128297da1b1633753 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 7 Jan 2021 09:45:20 -0800 Subject: [PATCH 082/222] Update tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py Add clarifying comment Co-authored-by: Danny Ryan --- .../block_processing/test_process_sync_committee.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index 1a3b41fa4..a4e31335a 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -31,7 +31,7 @@ def test_invalid_sync_committee_bits(spec, state): spec, state, block.slot - 1, - committee, + committee, # full committee signs ) yield 'blocks', [block] From 049075b44a3bd7d2ebc363d29ecddf471b03ef3a Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 7 Jan 2021 10:42:21 -0800 Subject: [PATCH 083/222] Refactor sync committee helpers --- .../{lightclient_patch/helpers.py => helpers/sync_committee.py} | 0 .../block_processing/test_process_sync_committee.py | 2 +- .../eth2spec/test/lightclient_patch/sanity/test_blocks.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename tests/core/pyspec/eth2spec/test/{lightclient_patch/helpers.py => helpers/sync_committee.py} (100%) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/helpers.py b/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/lightclient_patch/helpers.py rename to tests/core/pyspec/eth2spec/test/helpers/sync_committee.py diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index a4e31335a..f29d1ef2b 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -5,7 +5,7 @@ from eth2spec.test.helpers.block import ( from eth2spec.test.helpers.state import ( state_transition_and_sign_block, ) -from eth2spec.test.lightclient_patch.helpers import ( +from eth2spec.test.helpers.sync_committee import ( compute_aggregate_sync_committee_signature, ) from eth2spec.test.context import ( diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py index 1698ecce1..9033a0f15 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py @@ -6,7 +6,7 @@ from eth2spec.test.helpers.state import ( from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, ) -from eth2spec.test.lightclient_patch.helpers import ( +from eth2spec.test.helpers.sync_committee import ( compute_aggregate_sync_committee_signature, ) from eth2spec.test.context import ( From 1a3fefcc931d6961a099e30afa5f8d282fc2c4eb Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 7 Jan 2021 10:55:51 -0800 Subject: [PATCH 084/222] Refactor epoch processing test helpers --- .../run_epoch_process_base.py => helpers/epoch_processing.py} | 0 .../epoch_processing/test_process_final_updates.py | 2 +- .../test/phase0/epoch_processing/test_process_final_updates.py | 2 +- .../test_process_justification_and_finalization.py | 2 +- .../phase0/epoch_processing/test_process_registry_updates.py | 2 +- .../epoch_processing/test_process_rewards_and_penalties.py | 2 +- .../test/phase0/epoch_processing/test_process_slashings.py | 2 +- .../phase1/epoch_processing/test_process_challenge_deadlines.py | 2 +- .../epoch_processing/test_process_custody_final_updates.py | 2 +- .../phase1/epoch_processing/test_process_reveal_deadlines.py | 2 +- 10 files changed, 9 insertions(+), 9 deletions(-) rename tests/core/pyspec/eth2spec/test/{phase0/epoch_processing/run_epoch_process_base.py => helpers/epoch_processing.py} (100%) diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/run_epoch_process_base.py b/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/phase0/epoch_processing/run_epoch_process_base.py rename to tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py index 227b2461d..012438cd8 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py @@ -4,7 +4,7 @@ from eth2spec.test.context import ( spec_state_test, ) from eth2spec.test.helpers.state import transition_to -from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import ( +from eth2spec.test.helpers.epoch_processing import ( run_epoch_processing_with, ) diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_final_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_final_updates.py index 27676a59a..f9b5c872b 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_final_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_final_updates.py @@ -1,5 +1,5 @@ from eth2spec.test.context import spec_state_test, with_all_phases -from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import ( +from eth2spec.test.helpers.epoch_processing import ( run_epoch_processing_with, run_epoch_processing_to ) from eth2spec.test.helpers.state import transition_to diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py index f8504fc4f..921289f5b 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py @@ -1,5 +1,5 @@ from eth2spec.test.context import spec_state_test, with_all_phases -from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import ( +from eth2spec.test.helpers.epoch_processing import ( run_epoch_processing_with ) from eth2spec.test.helpers.state import transition_to diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_registry_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_registry_updates.py index c734010f3..ee40984a6 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_registry_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_registry_updates.py @@ -1,7 +1,7 @@ from eth2spec.test.helpers.deposits import mock_deposit from eth2spec.test.helpers.state import next_epoch, next_slots from eth2spec.test.context import spec_state_test, with_all_phases -from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import run_epoch_processing_with +from eth2spec.test.helpers.epoch_processing import run_epoch_processing_with def run_process_registry_updates(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py index dd3cbb791..8560b66a8 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py @@ -19,7 +19,7 @@ from eth2spec.test.helpers.attestations import ( ) from eth2spec.test.helpers.rewards import leaking from eth2spec.test.helpers.attester_slashings import get_indexed_attestation_participants -from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import run_epoch_processing_with +from eth2spec.test.helpers.epoch_processing import run_epoch_processing_with from random import Random diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py index 3e96f8b58..2e09f5c8a 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py @@ -1,5 +1,5 @@ from eth2spec.test.context import spec_state_test, with_all_phases -from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import ( +from eth2spec.test.helpers.epoch_processing import ( run_epoch_processing_with, run_epoch_processing_to ) from eth2spec.test.helpers.state import next_epoch diff --git a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py index 0350324a5..1d8adecbc 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py +++ b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py @@ -15,7 +15,7 @@ from eth2spec.test.context import ( with_configs, ) from eth2spec.test.phase0.block_processing.test_process_attestation import run_attestation_processing -from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import run_epoch_processing_with +from eth2spec.test.helpers.epoch_processing import run_epoch_processing_with from eth2spec.test.phase1.block_processing.test_process_chunk_challenge import ( run_chunk_challenge_processing, diff --git a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_custody_final_updates.py b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_custody_final_updates.py index 5994306d9..82ecde7ab 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_custody_final_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_custody_final_updates.py @@ -17,7 +17,7 @@ from eth2spec.test.context import ( spec_state_test, ) from eth2spec.test.phase0.block_processing.test_process_attestation import run_attestation_processing -from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import run_epoch_processing_with +from eth2spec.test.helpers.epoch_processing import run_epoch_processing_with from eth2spec.test.phase1.block_processing.test_process_chunk_challenge import ( run_chunk_challenge_processing, diff --git a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py index 3c2060ba5..b95082491 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py +++ b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py @@ -10,7 +10,7 @@ from eth2spec.test.context import ( with_configs, spec_state_test, ) -from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import run_epoch_processing_with +from eth2spec.test.helpers.epoch_processing import run_epoch_processing_with from eth2spec.test.phase1.block_processing.test_process_custody_key_reveal import run_custody_key_reveal_processing From 61d141b4db8978d6e3b825b6f7ccdc37ea559ab6 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 7 Jan 2021 10:57:54 -0800 Subject: [PATCH 085/222] Use more clear names for tests --- .../block_processing/test_process_sync_committee.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index f29d1ef2b..e36500918 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -18,7 +18,7 @@ from eth2spec.test.context import ( @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test -def test_invalid_sync_committee_bits(spec, state): +def test_invalid_signature_missing_participant(spec, state): committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) random_participant = random.choice(committee) @@ -31,7 +31,7 @@ def test_invalid_sync_committee_bits(spec, state): spec, state, block.slot - 1, - committee, # full committee signs + committee, # full committee signs ) yield 'blocks', [block] @@ -41,7 +41,7 @@ def test_invalid_sync_committee_bits(spec, state): @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test -def test_invalid_sync_committee_signature(spec, state): +def test_invalid_signature_extra_participant(spec, state): committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) random_participant = random.choice(committee) From ac6dbd1c3531c24cd2d0f07d3a5cda180ca26dbb Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 7 Jan 2021 11:28:21 -0800 Subject: [PATCH 086/222] Add sync committee test for signature over incorrect block --- .../test_process_sync_committee.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index e36500918..5f49e2131 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -1,6 +1,7 @@ import random from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, + transition_unsigned_block, ) from eth2spec.test.helpers.state import ( state_transition_and_sign_block, @@ -110,3 +111,42 @@ def test_sync_committee_rewards(spec, state): ) assert state.balances[index] == pre_balances[index] + expected_reward + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_invalid_signature_past_block(spec, state): + committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + + yield 'pre', state + + blocks = [] + for _ in range(2): + # NOTE: need to transition twice to move beyond the degenerate case at genesis + block = build_empty_block_for_next_slot(spec, state) + # Valid sync committee signature here... + block.body.sync_committee_bits = [True] * len(committee) + block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + block.slot - 1, + committee, + ) + + signed_block = state_transition_and_sign_block(spec, state, block) + blocks.append(signed_block) + + invalid_block = build_empty_block_for_next_slot(spec, state) + # Invalid signature from a slot other than the previous + invalid_block.body.sync_committee_bits = [True] * len(committee) + invalid_block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + invalid_block.slot - 2, + committee, + ) + blocks.append(invalid_block) + + expect_assertion_error(lambda: transition_unsigned_block(spec, state, invalid_block)) + + yield 'blocks', blocks + yield 'post', None From 500158828502bafb980a48ade5285d384bd9a183 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 7 Jan 2021 12:27:24 -0800 Subject: [PATCH 087/222] Add additional sync committee tests --- .../test_process_sync_committee.py | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index 5f49e2131..522e248fb 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -5,6 +5,7 @@ from eth2spec.test.helpers.block import ( ) from eth2spec.test.helpers.state import ( state_transition_and_sign_block, + transition_to, ) from eth2spec.test.helpers.sync_committee import ( compute_aggregate_sync_committee_signature, @@ -112,6 +113,7 @@ def test_sync_committee_rewards(spec, state): assert state.balances[index] == pre_balances[index] + expected_reward + @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test def test_invalid_signature_past_block(spec, state): @@ -150,3 +152,68 @@ def test_invalid_signature_past_block(spec, state): yield 'blocks', blocks yield 'post', None + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_invalid_signature_previous_committee(spec, state): + # NOTE: the `state` provided is at genesis and the process to select + # sync committees currently returns the same committee for the first and second + # periods at genesis. + # To get a distinct committee so we can generate an "old" signature, we need to advance + # 2 EPOCHS_PER_SYNC_COMMITTEE_PERIOD periods. + current_epoch = spec.get_current_epoch(state) + previous_committee = state.next_sync_committee + + epoch_in_future_sync_commitee_period = current_epoch + 2 * spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD + slot_in_future_sync_committee_period = epoch_in_future_sync_commitee_period * spec.SLOTS_PER_EPOCH + transition_to(spec, state, slot_in_future_sync_committee_period) + + pubkeys = [validator.pubkey for validator in state.validators] + committee = [pubkeys.index(pubkey) for pubkey in previous_committee.pubkeys] + + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + block.body.sync_committee_bits = [True] * len(committee) + block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + block.slot - 1, + committee, + ) + + yield 'blocks', [block] + expect_assertion_error(lambda: spec.process_sync_committee(state, block.body)) + yield 'post', None + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_valid_signature_next_committee(spec, state): + current_epoch = spec.get_current_epoch(state) + next_committee = state.next_sync_committee + epoch_in_next_sync_committee_period = current_epoch + spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD + slot_in_next_sync_committee_period = epoch_in_next_sync_committee_period * spec.SLOTS_PER_EPOCH + transition_to(spec, state, slot_in_next_sync_committee_period) + + assert state.current_sync_committee == next_committee + + pubkeys = [validator.pubkey for validator in state.validators] + committee = [pubkeys.index(pubkey) for pubkey in next_committee.pubkeys] + + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + block.body.sync_committee_bits = [True] * len(committee) + block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + block.slot - 1, + committee, + ) + + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state From f113413e5b3fabb8d41efcbd3dc5c52601399cb2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 11 Jan 2021 18:06:37 +0800 Subject: [PATCH 088/222] Fix ToC --- specs/lightclient/beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 1112ea126..892f5bbb3 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -32,7 +32,7 @@ - [`get_base_reward`](#get_base_reward) - [`get_unslashed_participating_indices`](#get_unslashed_participating_indices) - [`get_flag_deltas`](#get_flag_deltas) - - [`get_inactivity_penalty_deltas`](#get_inactivity_penalty_deltas) + - [New `get_inactivity_penalty_deltas`](#new-get_inactivity_penalty_deltas) - [Block processing](#block-processing) - [New `process_attestation`](#new-process_attestation) - [New `process_deposit`](#new-process_deposit) @@ -300,7 +300,7 @@ def get_flag_deltas(state: BeaconState, flag: uint8, numerator: uint64) -> Tuple return rewards, penalties ``` -##### `get_inactivity_penalty_deltas` +#### New `get_inactivity_penalty_deltas` *Note*: The function `get_inactivity_penalty_deltas` is modified in the selection of matching target indices and the removal of `BASE_REWARDS_PER_EPOCH`. From e518c4d04d1fb741d4b22a71d48a6f4b38582b6d Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 12 Jan 2021 09:31:07 -0800 Subject: [PATCH 089/222] update test to use fresh sync committees the way the test infra is built we end up with two identical sync committees at epoch 0. --- .../test_process_sync_committee.py | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index 522e248fb..37cd0992b 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -190,27 +190,40 @@ def test_invalid_signature_previous_committee(spec, state): @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test -def test_valid_signature_next_committee(spec, state): +def test_valid_signature_future_committee(spec, state): + # NOTE: the `state` provided is at genesis and the process to select + # sync committees currently returns the same committee for the first and second + # periods at genesis. + # To get a distinct committee so we can generate an "old" signature, we need to advance + # 2 EPOCHS_PER_SYNC_COMMITTEE_PERIOD periods. current_epoch = spec.get_current_epoch(state) - next_committee = state.next_sync_committee - epoch_in_next_sync_committee_period = current_epoch + spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD - slot_in_next_sync_committee_period = epoch_in_next_sync_committee_period * spec.SLOTS_PER_EPOCH - transition_to(spec, state, slot_in_next_sync_committee_period) + old_current_sync_committee = state.current_sync_committee + old_next_sync_committee = state.next_sync_committee - assert state.current_sync_committee == next_committee + epoch_in_future_sync_committee_period = current_epoch + 2 * spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD + slot_in_future_sync_committee_period = epoch_in_future_sync_committee_period * spec.SLOTS_PER_EPOCH + transition_to(spec, state, slot_in_future_sync_committee_period) + + sync_committee = state.current_sync_committee + + expected_sync_committee = spec.get_sync_committee(state, epoch_in_future_sync_committee_period) + + assert sync_committee == expected_sync_committee + assert sync_committee != old_current_sync_committee + assert sync_committee != old_next_sync_committee pubkeys = [validator.pubkey for validator in state.validators] - committee = [pubkeys.index(pubkey) for pubkey in next_committee.pubkeys] + committee_indices = [pubkeys.index(pubkey) for pubkey in sync_committee.pubkeys] yield 'pre', state block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = [True] * len(committee) + block.body.sync_committee_bits = [True] * len(committee_indices) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, block.slot - 1, - committee, + committee_indices, ) signed_block = state_transition_and_sign_block(spec, state, block) From 252c331255adc4920ea88758b007825e895be6fb Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 13 Jan 2021 14:29:18 +0800 Subject: [PATCH 090/222] Fix existing tests on #2176 (accounting reform) (#2180) * Fixing tests * Add `is_post_lightclient_patch` helper to determine the fork version condition --- tests/core/pyspec/eth2spec/test/context.py | 8 + .../eth2spec/test/helpers/attestations.py | 9 +- .../pyspec/eth2spec/test/helpers/block.py | 4 +- .../pyspec/eth2spec/test/helpers/rewards.py | 137 +++++++++++++----- ..._process_justification_and_finalization.py | 55 ++++--- .../test_process_rewards_and_penalties.py | 21 +-- .../test/phase0/rewards/test_basic.py | 22 +-- .../eth2spec/test/phase0/rewards/test_leak.py | 12 +- .../test/phase0/sanity/test_blocks.py | 24 ++- 9 files changed, 197 insertions(+), 95 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index d19547477..4f07a790a 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -390,3 +390,11 @@ def only_full_crosslink(fn): return None return fn(*args, spec=spec, state=state, **kw) return wrapper + + +def is_post_lightclient_patch(spec): + if spec.fork in [PHASE0, PHASE1]: + # TODO: PHASE1 fork is temporarily parallel to LIGHTCLIENT_PATCH. + # Will make PHASE1 fork inherit LIGHTCLIENT_PATCH later. + return False + return True diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index efc8d7a5d..cf62482da 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -2,7 +2,7 @@ from lru import LRU from typing import List -from eth2spec.test.context import expect_assertion_error, PHASE1, LIGHTCLIENT_PATCH +from eth2spec.test.context import expect_assertion_error, PHASE1, is_post_lightclient_patch from eth2spec.test.helpers.state import state_transition_and_sign_block, next_epoch, next_slot from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.shard_transitions import get_shard_transition_of_committee @@ -30,7 +30,7 @@ def run_attestation_processing(spec, state, attestation, valid=True): yield 'post', None return - if spec.fork != LIGHTCLIENT_PATCH: + if not is_post_lightclient_patch(spec): current_epoch_count = len(state.current_epoch_attestations) previous_epoch_count = len(state.previous_epoch_attestations) @@ -38,7 +38,7 @@ def run_attestation_processing(spec, state, attestation, valid=True): spec.process_attestation(state, attestation) # Make sure the attestation has been processed - if spec.fork != LIGHTCLIENT_PATCH: + if not is_post_lightclient_patch(spec): if attestation.data.target.epoch == spec.get_current_epoch(state): assert len(state.current_epoch_attestations) == current_epoch_count + 1 else: @@ -323,7 +323,8 @@ def prepare_state_with_attestations(spec, state, participation_fn=None): next_slot(spec, state) assert state.slot == next_epoch_start_slot + spec.MIN_ATTESTATION_INCLUSION_DELAY - assert len(state.previous_epoch_attestations) == len(attestations) + if not is_post_lightclient_patch(spec): + assert len(state.previous_epoch_attestations) == len(attestations) return attestations diff --git a/tests/core/pyspec/eth2spec/test/helpers/block.py b/tests/core/pyspec/eth2spec/test/helpers/block.py index c4d8f1931..7501c8268 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/block.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import LIGHTCLIENT_PATCH +from eth2spec.test.context import is_post_lightclient_patch from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.bls import only_with_bls @@ -91,7 +91,7 @@ def build_empty_block(spec, state, slot=None): empty_block.body.eth1_data.deposit_count = state.eth1_deposit_index empty_block.parent_root = parent_block_root - if spec.fork == LIGHTCLIENT_PATCH: + if is_post_lightclient_patch(spec): empty_block.body.sync_committee_signature = spec.G2_POINT_AT_INFINITY apply_randao_reveal(spec, state, empty_block) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index dc355972f..770519384 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -2,11 +2,11 @@ from random import Random from lru import LRU from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.test.context import LIGHTCLIENT_PATCH +from eth2spec.test.context import is_post_lightclient_patch from eth2spec.test.helpers.attestations import cached_prepare_state_with_attestations from eth2spec.test.helpers.deposits import mock_deposit from eth2spec.test.helpers.state import next_epoch -from eth2spec.utils.ssz.ssz_typing import Container, uint64, List +from eth2spec.utils.ssz.ssz_typing import Container, uint64, List, Bitvector class Deltas(Container): @@ -38,24 +38,35 @@ def run_deltas(spec, state): - inactivity penalty deltas ('inactivity_penalty_deltas') """ yield 'pre', state + + if is_post_lightclient_patch(spec): + def get_source_deltas(state): + return spec.get_flag_deltas(state, spec.TIMELY_SOURCE_FLAG, spec.TIMELY_SOURCE_NUMERATOR) + + def get_head_deltas(state): + return spec.get_flag_deltas(state, spec.TIMELY_HEAD_FLAG, spec.TIMELY_HEAD_NUMERATOR) + + def get_target_deltas(state): + return spec.get_flag_deltas(state, spec.TIMELY_TARGET_FLAG, spec.TIMELY_TARGET_NUMERATOR) + yield from run_attestation_component_deltas( spec, state, - spec.get_source_deltas, + spec.get_source_deltas if not is_post_lightclient_patch(spec) else get_source_deltas, spec.get_matching_source_attestations, 'source_deltas', ) yield from run_attestation_component_deltas( spec, state, - spec.get_target_deltas, + spec.get_target_deltas if not is_post_lightclient_patch(spec) else get_target_deltas, spec.get_matching_target_attestations, 'target_deltas', ) yield from run_attestation_component_deltas( spec, state, - spec.get_head_deltas, + spec.get_head_deltas if not is_post_lightclient_patch(spec) else get_head_deltas, spec.get_matching_head_attestations, 'head_deltas', ) @@ -63,6 +74,16 @@ def run_deltas(spec, state): yield from run_get_inactivity_penalty_deltas(spec, state) +def deltas_name_to_flag(spec, deltas_name): + if 'source' in deltas_name: + return spec.TIMELY_SOURCE_FLAG + elif 'head' in deltas_name: + return spec.TIMELY_HEAD_FLAG + elif 'target' in deltas_name: + return spec.TIMELY_TARGET_FLAG + raise ValueError("Wrong deltas_name %s" % deltas_name) + + def run_attestation_component_deltas(spec, state, component_delta_fn, matching_att_fn, deltas_name): """ Run ``component_delta_fn``, yielding: @@ -72,8 +93,14 @@ def run_attestation_component_deltas(spec, state, component_delta_fn, matching_a yield deltas_name, Deltas(rewards=rewards, penalties=penalties) - matching_attestations = matching_att_fn(state, spec.get_previous_epoch(state)) - matching_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) + if not is_post_lightclient_patch(spec): + matching_attestations = matching_att_fn(state, spec.get_previous_epoch(state)) + matching_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) + else: + matching_indices = spec.get_unslashed_participating_indices( + state, deltas_name_to_flag(spec, deltas_name), spec.get_previous_epoch(state) + ) + eligible_indices = spec.get_eligible_validator_indices(state) for index in range(len(state.validators)): if index not in eligible_indices: @@ -102,6 +129,12 @@ def run_get_inclusion_delay_deltas(spec, state): Run ``get_inclusion_delay_deltas``, yielding: - inclusion delay deltas ('inclusion_delay_deltas') """ + if is_post_lightclient_patch(spec): + # No inclusion_delay_deltas + yield 'inclusion_delay_deltas', Deltas(rewards=[0] * len(state.validators), + penalties=[0] * len(state.validators)) + return + rewards, penalties = spec.get_inclusion_delay_deltas(state) yield 'inclusion_delay_deltas', Deltas(rewards=rewards, penalties=penalties) @@ -149,8 +182,14 @@ def run_get_inactivity_penalty_deltas(spec, state): yield 'inactivity_penalty_deltas', Deltas(rewards=rewards, penalties=penalties) - matching_attestations = spec.get_matching_target_attestations(state, spec.get_previous_epoch(state)) - matching_attesting_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) + if not is_post_lightclient_patch(spec): + matching_attestations = spec.get_matching_target_attestations(state, spec.get_previous_epoch(state)) + matching_attesting_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) + else: + matching_attesting_indices = spec.get_unslashed_participating_indices( + state, spec.TIMELY_TARGET_FLAG, spec.get_previous_epoch(state) + ) + reward_numerator_sum = sum(numerator for (_, numerator) in spec.get_flags_and_numerators()) eligible_indices = spec.get_eligible_validator_indices(state) for index in range(len(state.validators)): @@ -160,12 +199,14 @@ def run_get_inactivity_penalty_deltas(spec, state): continue if spec.is_in_inactivity_leak(state): - if spec.fork == LIGHTCLIENT_PATCH: - cancel_base_rewards_per_epoch = spec.BASE_REWARDS_PER_EPOCH - 1 - else: + # Compute base_penalty + if not is_post_lightclient_patch(spec): cancel_base_rewards_per_epoch = spec.BASE_REWARDS_PER_EPOCH - base_reward = spec.get_base_reward(state, index) - base_penalty = cancel_base_rewards_per_epoch * base_reward - spec.get_proposer_reward(state, index) + base_reward = spec.get_base_reward(state, index) + base_penalty = cancel_base_rewards_per_epoch * base_reward - spec.get_proposer_reward(state, index) + else: + base_penalty = spec.get_base_reward(state, index) * reward_numerator_sum // spec.REWARD_DENOMINATOR + if not has_enough_for_reward(spec, state, index): assert penalties[index] == 0 elif index in matching_attesting_indices: @@ -267,8 +308,13 @@ def run_test_full_all_correct(spec, state): def run_test_full_but_partial_participation(spec, state, rng=Random(5522)): cached_prepare_state_with_attestations(spec, state) - for a in state.previous_epoch_attestations: - a.aggregation_bits = [rng.choice([True, False]) for _ in a.aggregation_bits] + if not is_post_lightclient_patch(spec): + for a in state.previous_epoch_attestations: + a.aggregation_bits = [rng.choice([True, False]) for _ in a.aggregation_bits] + else: + for index in range(len(state.validators)): + if rng.choice([True, False]): + state.previous_epoch_participation[index] = Bitvector[spec.PARTICIPATION_FLAGS_LENGTH]() yield from run_deltas(spec, state) @@ -277,8 +323,12 @@ def run_test_partial(spec, state, fraction_filled): cached_prepare_state_with_attestations(spec, state) # Remove portion of attestations - num_attestations = int(len(state.previous_epoch_attestations) * fraction_filled) - state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] + if not is_post_lightclient_patch(spec): + num_attestations = int(len(state.previous_epoch_attestations) * fraction_filled) + state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] + else: + for index in range(int(len(state.validators) * fraction_filled)): + state.previous_epoch_participation[index] = Bitvector[spec.PARTICIPATION_FLAGS_LENGTH]() yield from run_deltas(spec, state) @@ -333,13 +383,18 @@ def run_test_some_very_low_effective_balances_that_attested(spec, state): def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state): cached_prepare_state_with_attestations(spec, state) - # Remove attestation - attestation = state.previous_epoch_attestations[0] - state.previous_epoch_attestations = state.previous_epoch_attestations[1:] - # Set removed indices effective balance to very low amount - indices = spec.get_unslashed_attesting_indices(state, [attestation]) - for i, index in enumerate(indices): - state.validators[index].effective_balance = i + if not is_post_lightclient_patch(spec): + # Remove attestation + attestation = state.previous_epoch_attestations[0] + state.previous_epoch_attestations = state.previous_epoch_attestations[1:] + # Set removed indices effective balance to very low amount + indices = spec.get_unslashed_attesting_indices(state, [attestation]) + for i, index in enumerate(indices): + state.validators[index].effective_balance = i + else: + index = 0 + state.validators[index].effective_balance = 1 + state.previous_epoch_participation[index] = Bitvector[spec.PARTICIPATION_FLAGS_LENGTH]() yield from run_deltas(spec, state) @@ -447,16 +502,26 @@ def run_test_full_random(spec, state, rng=Random(8020)): cached_prepare_state_with_attestations(spec, state) - for pending_attestation in state.previous_epoch_attestations: - # ~1/3 have bad target - if rng.randint(0, 2) == 0: - pending_attestation.data.target.root = b'\x55' * 32 - # ~1/3 have bad head - if rng.randint(0, 2) == 0: - pending_attestation.data.beacon_block_root = b'\x66' * 32 - # ~50% participation - pending_attestation.aggregation_bits = [rng.choice([True, False]) for _ in pending_attestation.aggregation_bits] - # Random inclusion delay - pending_attestation.inclusion_delay = rng.randint(1, spec.SLOTS_PER_EPOCH) + if not is_post_lightclient_patch(spec): + for pending_attestation in state.previous_epoch_attestations: + # ~1/3 have bad target + if rng.randint(0, 2) == 0: + pending_attestation.data.target.root = b'\x55' * 32 + # ~1/3 have bad head + if rng.randint(0, 2) == 0: + pending_attestation.data.beacon_block_root = b'\x66' * 32 + # ~50% participation + pending_attestation.aggregation_bits = [rng.choice([True, False]) + for _ in pending_attestation.aggregation_bits] + # Random inclusion delay + pending_attestation.inclusion_delay = rng.randint(1, spec.SLOTS_PER_EPOCH) + else: + for index in range(len(state.validators)): + # ~1/3 have bad target + state.previous_epoch_participation[index][spec.TIMELY_TARGET_FLAG] = rng.randint(0, 2) != 0 + # ~1/3 have bad head + state.previous_epoch_participation[index][spec.TIMELY_HEAD_FLAG] = rng.randint(0, 2) != 0 + # ~50% participation + state.previous_epoch_participation[index][spec.TIMELY_SOURCE_FLAG] = rng.choice([True, False]) yield from run_deltas(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py index f8504fc4f..14623ea9a 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.context import is_post_lightclient_patch, spec_state_test, with_all_phases from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import ( run_epoch_processing_with ) @@ -16,12 +16,20 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support previous_epoch = spec.get_previous_epoch(state) current_epoch = spec.get_current_epoch(state) - if current_epoch == epoch: - attestations = state.current_epoch_attestations - elif previous_epoch == epoch: - attestations = state.previous_epoch_attestations + if not is_post_lightclient_patch(spec): + if current_epoch == epoch: + attestations = state.current_epoch_attestations + elif previous_epoch == epoch: + attestations = state.previous_epoch_attestations + else: + raise Exception(f"cannot include attestations in epoch ${epoch} from epoch ${current_epoch}") else: - raise Exception(f"cannot include attestations in epoch ${epoch} from epoch ${current_epoch}") + if current_epoch == epoch: + epoch_participation = state.current_epoch_participation + elif previous_epoch == epoch: + epoch_participation = state.previous_epoch_participation + else: + raise Exception(f"cannot include attestations in epoch ${epoch} from epoch ${current_epoch}") total_balance = spec.get_total_active_balance(state) remaining_balance = int(total_balance * 2 // 3) # can become negative @@ -52,19 +60,28 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support for i in range(max(len(committee) // 5, 1)): aggregation_bits[i] = 0 - attestations.append(spec.PendingAttestation( - aggregation_bits=aggregation_bits, - data=spec.AttestationData( - slot=slot, - beacon_block_root=b'\xff' * 32, # irrelevant to testing - source=source, - target=target, - index=index, - ), - inclusion_delay=1, - )) - if messed_up_target: - attestations[len(attestations) - 1].data.target.root = b'\x99' * 32 + # Update state + if not is_post_lightclient_patch(spec): + attestations.append(spec.PendingAttestation( + aggregation_bits=aggregation_bits, + data=spec.AttestationData( + slot=slot, + beacon_block_root=b'\xff' * 32, # irrelevant to testing + source=source, + target=target, + index=index, + ), + inclusion_delay=1, + )) + if messed_up_target: + attestations[len(attestations) - 1].data.target.root = b'\x99' * 32 + else: + for i, index in enumerate(committee): + if aggregation_bits[i]: + epoch_participation[index][spec.TIMELY_HEAD_FLAG] = True + epoch_participation[index][spec.TIMELY_SOURCE_FLAG] = True + if not messed_up_target: + epoch_participation[index][spec.TIMELY_TARGET_FLAG] = True def get_checkpoints(spec, epoch): diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py index dd3cbb791..cdc0be8cd 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py @@ -1,11 +1,11 @@ from eth2spec.test.context import ( - LIGHTCLIENT_PATCH, spec_state_test, spec_test, with_all_phases, single_phase, - with_phases, PHASE0, + with_phases, PHASE0, PHASE1, with_custom_state, zero_activation_threshold, misc_balances, low_single_balance, + is_post_lightclient_patch, ) from eth2spec.test.helpers.state import ( next_epoch, @@ -66,7 +66,7 @@ def test_genesis_epoch_full_attestations_no_rewards(spec, state): assert state.balances[index] == pre_state.balances[index] -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test def test_full_attestations_random_incorrect_fields(spec, state): attestations = prepare_state_with_attestations(spec, state) @@ -159,11 +159,11 @@ def run_with_participation(spec, state, participation_fn): return att_participants attestations = prepare_state_with_attestations(spec, state, participation_fn=participation_tracker) - proposer_indices = [a.proposer_index for a in state.previous_epoch_attestations] - pre_state = state.copy() - if spec.fork == LIGHTCLIENT_PATCH: + if not is_post_lightclient_patch(spec): + proposer_indices = [a.proposer_index for a in state.previous_epoch_attestations] + else: sync_committee_indices = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) yield from run_process_rewards_and_penalties(spec, state) @@ -173,11 +173,11 @@ def run_with_participation(spec, state, participation_fn): for index in range(len(pre_state.validators)): if spec.is_in_inactivity_leak(state): - # Proposers can still make money during a leak - if index in proposer_indices and index in participated: + # Proposers can still make money during a leak before LIGHTCLIENT_PATCH + if not is_post_lightclient_patch(spec) and index in proposer_indices and index in participated: assert state.balances[index] > pre_state.balances[index] elif index in attesting_indices: - if spec.fork == LIGHTCLIENT_PATCH and index in sync_committee_indices: + if is_post_lightclient_patch(spec) and index in sync_committee_indices: # The sync committee reward has not been canceled, so the sync committee participants still earn it assert state.balances[index] >= pre_state.balances[index] else: @@ -428,7 +428,8 @@ def test_attestations_some_slashed(spec, state): for i in range(spec.MIN_PER_EPOCH_CHURN_LIMIT): spec.slash_validator(state, attesting_indices_before_slashings[i]) - assert len(state.previous_epoch_attestations) == len(attestations) + if not is_post_lightclient_patch(spec): + assert len(state.previous_epoch_attestations) == len(attestations) pre_state = state.copy() diff --git a/tests/core/pyspec/eth2spec/test/phase0/rewards/test_basic.py b/tests/core/pyspec/eth2spec/test/phase0/rewards/test_basic.py index 92277fdd7..7871d3fcf 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/rewards/test_basic.py +++ b/tests/core/pyspec/eth2spec/test/phase0/rewards/test_basic.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import with_all_phases, spec_state_test +from eth2spec.test.context import PHASE0, PHASE1, with_all_phases, with_phases, spec_state_test import eth2spec.test.helpers.rewards as rewards_helpers @@ -32,7 +32,7 @@ def test_full_but_partial_participation(spec, state): yield from rewards_helpers.run_test_full_but_partial_participation(spec, state) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test def test_one_attestation_one_correct(spec, state): yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state) @@ -75,7 +75,7 @@ def test_some_very_low_effective_balances_that_did_not_attest(spec, state): # -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test def test_full_half_correct_target_incorrect_head(spec, state): yield from rewards_helpers.run_test_full_fraction_incorrect( @@ -86,7 +86,7 @@ def test_full_half_correct_target_incorrect_head(spec, state): ) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test def test_full_correct_target_incorrect_head(spec, state): yield from rewards_helpers.run_test_full_fraction_incorrect( @@ -97,7 +97,7 @@ def test_full_correct_target_incorrect_head(spec, state): ) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test def test_full_half_incorrect_target_incorrect_head(spec, state): yield from rewards_helpers.run_test_full_fraction_incorrect( @@ -108,7 +108,7 @@ def test_full_half_incorrect_target_incorrect_head(spec, state): ) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test def test_full_half_incorrect_target_correct_head(spec, state): yield from rewards_helpers.run_test_full_fraction_incorrect( @@ -119,31 +119,31 @@ def test_full_half_incorrect_target_correct_head(spec, state): ) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test def test_full_delay_one_slot(spec, state): yield from rewards_helpers.run_test_full_delay_one_slot(spec, state) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test def test_full_delay_max_slots(spec, state): yield from rewards_helpers.run_test_full_delay_max_slots(spec, state) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test def test_full_mixed_delay(spec, state): yield from rewards_helpers.run_test_full_mixed_delay(spec, state) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test def test_proposer_not_in_attestations(spec, state): yield from rewards_helpers.run_test_proposer_not_in_attestations(spec, state) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test def test_duplicate_attestations_at_later_slots(spec, state): yield from rewards_helpers.run_test_duplicate_attestations_at_later_slots(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase0/rewards/test_leak.py b/tests/core/pyspec/eth2spec/test/phase0/rewards/test_leak.py index b0f9767b2..b2ed6f5d8 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/rewards/test_leak.py +++ b/tests/core/pyspec/eth2spec/test/phase0/rewards/test_leak.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import with_all_phases, spec_state_test +from eth2spec.test.context import PHASE0, PHASE1, with_all_phases, with_phases, spec_state_test from eth2spec.test.helpers.rewards import leaking import eth2spec.test.helpers.rewards as rewards_helpers @@ -38,7 +38,7 @@ def test_full_but_partial_participation_leak(spec, state): yield from rewards_helpers.run_test_full_but_partial_participation(spec, state) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test @leaking() def test_one_attestation_one_correct_leak(spec, state): @@ -87,7 +87,7 @@ def test_some_very_low_effective_balances_that_did_not_attest_leak(spec, state): # -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test @leaking() def test_full_half_correct_target_incorrect_head_leak(spec, state): @@ -99,7 +99,7 @@ def test_full_half_correct_target_incorrect_head_leak(spec, state): ) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test @leaking() def test_full_correct_target_incorrect_head_leak(spec, state): @@ -111,7 +111,7 @@ def test_full_correct_target_incorrect_head_leak(spec, state): ) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test @leaking() def test_full_half_incorrect_target_incorrect_head_leak(spec, state): @@ -123,7 +123,7 @@ def test_full_half_incorrect_target_incorrect_head_leak(spec, state): ) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test @leaking() def test_full_half_incorrect_target_correct_head_leak(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py index 358fd5211..9cc8ab721 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py @@ -35,6 +35,7 @@ from eth2spec.test.context import ( with_configs, with_custom_state, large_validator_set, + is_post_lightclient_patch, ) @@ -780,15 +781,19 @@ def test_attestation(spec, state): spec, state, shard_transition=shard_transition, index=index, signed=True, on_time=True ) + if not is_post_lightclient_patch(spec): + pre_current_attestations_len = len(state.current_epoch_attestations) + # Add to state via block transition - pre_current_attestations_len = len(state.current_epoch_attestations) attestation_block.body.attestations.append(attestation) signed_attestation_block = state_transition_and_sign_block(spec, state, attestation_block) - assert len(state.current_epoch_attestations) == pre_current_attestations_len + 1 - - # Epoch transition should move to previous_epoch_attestations - pre_current_attestations_root = spec.hash_tree_root(state.current_epoch_attestations) + if not is_post_lightclient_patch(spec): + assert len(state.current_epoch_attestations) == pre_current_attestations_len + 1 + # Epoch transition should move to previous_epoch_attestations + pre_current_attestations_root = spec.hash_tree_root(state.current_epoch_attestations) + else: + pre_current_epoch_participation_root = spec.hash_tree_root(state.current_epoch_participation) epoch_block = build_empty_block(spec, state, state.slot + spec.SLOTS_PER_EPOCH) signed_epoch_block = state_transition_and_sign_block(spec, state, epoch_block) @@ -796,8 +801,13 @@ def test_attestation(spec, state): yield 'blocks', [signed_attestation_block, signed_epoch_block] yield 'post', state - assert len(state.current_epoch_attestations) == 0 - assert spec.hash_tree_root(state.previous_epoch_attestations) == pre_current_attestations_root + if not is_post_lightclient_patch(spec): + assert len(state.current_epoch_attestations) == 0 + assert spec.hash_tree_root(state.previous_epoch_attestations) == pre_current_attestations_root + else: + for index in range(len(state.validators)): + assert state.current_epoch_participation[index] == spec.Bitvector[spec.PARTICIPATION_FLAGS_LENGTH]() + assert spec.hash_tree_root(state.previous_epoch_participation) == pre_current_epoch_participation_root # In phase1 a committee is computed for SHARD_COMMITTEE_PERIOD slots ago, From 50765edc300d9d06aad35d5428157dc961d9090e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 13 Jan 2021 15:11:24 +0800 Subject: [PATCH 091/222] Set minimal config's `EPOCHS_PER_SYNC_COMMITTEE_PERIOD` to 8 --- configs/minimal/lightclient_patch.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/minimal/lightclient_patch.yaml b/configs/minimal/lightclient_patch.yaml index ba1179a2b..0ee39a30b 100644 --- a/configs/minimal/lightclient_patch.yaml +++ b/configs/minimal/lightclient_patch.yaml @@ -12,8 +12,8 @@ SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE: 16 # Time parameters # --------------------------------------------------------------- -# 2**8 (= 256) -EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 256 +# [customized] +EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 8 # Signature domains From 002dfaa89192f29ce5e742962cf2c41f4927e78c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 13 Jan 2021 17:29:33 +0800 Subject: [PATCH 092/222] Set minimal config's `SYNC_COMMITTEE_SIZE` to 32 --- configs/minimal/lightclient_patch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/minimal/lightclient_patch.yaml b/configs/minimal/lightclient_patch.yaml index 0ee39a30b..afe7d897e 100644 --- a/configs/minimal/lightclient_patch.yaml +++ b/configs/minimal/lightclient_patch.yaml @@ -5,7 +5,7 @@ CONFIG_NAME: "minimal" # Misc # --------------------------------------------------------------- # [customized] -SYNC_COMMITTEE_SIZE: 64 +SYNC_COMMITTEE_SIZE: 32 # [customized] SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE: 16 From b2658f1091482a3993b311696d5c50b744bde51f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 13 Jan 2021 17:28:23 +0800 Subject: [PATCH 093/222] Fix SyncCommittee 1. Make `get_sync_committee_indices` do not return duplicate indices 2. Pad default values to Vectors --- specs/lightclient/beacon-chain.md | 17 +++++++++++++++-- .../eth2spec/test/helpers/sync_committee.py | 6 ++++++ .../test_process_sync_committee.py | 17 ++++++++++------- .../lightclient_patch/sanity/test_blocks.py | 5 ++++- 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 2fbe190e4..1b434faf4 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -50,6 +50,7 @@ This is a standalone beacon chain patch adding light client support via sync com | Name | Value | | - | - | +| `G1_POINT_AT_INFINITY` | `BLSPubkey(b'\xc0' + b'\x00' * 47)` | | `G2_POINT_AT_INFINITY` | `BLSSignature(b'\xc0' + b'\x00' * 95)` | ### Misc @@ -138,12 +139,19 @@ def get_sync_committee_indices(state: BeaconState, epoch: Epoch) -> Sequence[Val seed = get_seed(state, base_epoch, DOMAIN_SYNC_COMMITTEE) i = 0 sync_committee_indices: List[ValidatorIndex] = [] - while len(sync_committee_indices) < SYNC_COMMITTEE_SIZE: + if len(active_validator_indices) < SYNC_COMMITTEE_SIZE: + committee_size = len(active_validator_indices) + else: + committee_size = SYNC_COMMITTEE_SIZE + while len(sync_committee_indices) < committee_size: shuffled_index = compute_shuffled_index(uint64(i % active_validator_count), active_validator_count, seed) candidate_index = active_validator_indices[shuffled_index] random_byte = hash(seed + uint_to_bytes(uint64(i // 32)))[i % 32] effective_balance = state.validators[candidate_index].effective_balance - if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: + if ( + effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte + and candidate_index not in sync_committee_indices + ): sync_committee_indices.append(candidate_index) i += 1 return sync_committee_indices @@ -163,6 +171,11 @@ def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: bls.AggregatePKs(pubkeys[i:i + SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE]) for i in range(0, len(pubkeys), SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE) ] + # Pad G1_POINT_AT_INFINITY to the BLSPubkey Vectors + if len(pubkeys) < SYNC_COMMITTEE_SIZE: + pubkeys += [G1_POINT_AT_INFINITY] * (SYNC_COMMITTEE_SIZE - len(pubkeys)) + aggregates_length = SYNC_COMMITTEE_SIZE // SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE + aggregates += [G1_POINT_AT_INFINITY] * (aggregates_length - len(aggregates)) return SyncCommittee(pubkeys=pubkeys, pubkey_aggregates=aggregates) ``` diff --git a/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py b/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py index b7b2381e3..21557dedd 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py @@ -31,3 +31,9 @@ def compute_aggregate_sync_committee_signature(spec, state, slot, participants): ) ) return bls.Aggregate(signatures) + + +def get_padded_sync_committee_bits(spec, sync_committee_bits): + if len(sync_committee_bits) < spec.SYNC_COMMITTEE_SIZE: + return sync_committee_bits + [False] * (spec.SYNC_COMMITTEE_SIZE - len(sync_committee_bits)) + return sync_committee_bits diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index 37cd0992b..14f3a9f0a 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -9,6 +9,7 @@ from eth2spec.test.helpers.state import ( ) from eth2spec.test.helpers.sync_committee import ( compute_aggregate_sync_committee_signature, + get_padded_sync_committee_bits, ) from eth2spec.test.context import ( PHASE0, PHASE1, @@ -28,7 +29,9 @@ def test_invalid_signature_missing_participant(spec, state): block = build_empty_block_for_next_slot(spec, state) # Exclude one participant whose signature was included. - block.body.sync_committee_bits = [index != random_participant for index in committee] + block.body.sync_committee_bits = get_padded_sync_committee_bits( + spec, [index != random_participant for index in committee] + ) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, @@ -51,7 +54,7 @@ def test_invalid_signature_extra_participant(spec, state): block = build_empty_block_for_next_slot(spec, state) # Exclude one signature even though the block claims the entire committee participated. - block.body.sync_committee_bits = [True] * len(committee) + block.body.sync_committee_bits = get_padded_sync_committee_bits(spec, [True] * len(committee)) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, @@ -83,7 +86,7 @@ def test_sync_committee_rewards(spec, state): pre_balances = state.balances.copy() block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = [True] * committee_size + block.body.sync_committee_bits = get_padded_sync_committee_bits(spec, [True] * committee_size) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, @@ -126,7 +129,7 @@ def test_invalid_signature_past_block(spec, state): # NOTE: need to transition twice to move beyond the degenerate case at genesis block = build_empty_block_for_next_slot(spec, state) # Valid sync committee signature here... - block.body.sync_committee_bits = [True] * len(committee) + block.body.sync_committee_bits = get_padded_sync_committee_bits(spec, [True] * len(committee)) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, @@ -139,7 +142,7 @@ def test_invalid_signature_past_block(spec, state): invalid_block = build_empty_block_for_next_slot(spec, state) # Invalid signature from a slot other than the previous - invalid_block.body.sync_committee_bits = [True] * len(committee) + invalid_block.body.sync_committee_bits = get_padded_sync_committee_bits(spec, [True] * len(committee)) invalid_block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, @@ -175,7 +178,7 @@ def test_invalid_signature_previous_committee(spec, state): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = [True] * len(committee) + block.body.sync_committee_bits = get_padded_sync_committee_bits(spec, [True] * len(committee)) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, @@ -218,7 +221,7 @@ def test_valid_signature_future_committee(spec, state): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = [True] * len(committee_indices) + block.body.sync_committee_bits = get_padded_sync_committee_bits(spec, [True] * len(committee_indices)) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py index 9033a0f15..93f8cd3ee 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py @@ -8,6 +8,7 @@ from eth2spec.test.helpers.block import ( ) from eth2spec.test.helpers.sync_committee import ( compute_aggregate_sync_committee_signature, + get_padded_sync_committee_bits, ) from eth2spec.test.context import ( PHASE0, PHASE1, @@ -23,7 +24,9 @@ def run_sync_committee_sanity_test(spec, state, fraction_full=1.0): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = [index in participants for index in committee] + block.body.sync_committee_bits = get_padded_sync_committee_bits( + spec, [index in participants for index in committee] + ) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, From 2a6699290f14bccddc9992b4a1e49eb71a6eb52b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 14 Jan 2021 01:47:40 +0800 Subject: [PATCH 094/222] Revert "Fix SyncCommittee" This reverts commit b2658f1091482a3993b311696d5c50b744bde51f. --- specs/lightclient/beacon-chain.md | 17 ++--------------- .../eth2spec/test/helpers/sync_committee.py | 6 ------ .../test_process_sync_committee.py | 17 +++++++---------- .../lightclient_patch/sanity/test_blocks.py | 5 +---- 4 files changed, 10 insertions(+), 35 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 1b434faf4..2fbe190e4 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -50,7 +50,6 @@ This is a standalone beacon chain patch adding light client support via sync com | Name | Value | | - | - | -| `G1_POINT_AT_INFINITY` | `BLSPubkey(b'\xc0' + b'\x00' * 47)` | | `G2_POINT_AT_INFINITY` | `BLSSignature(b'\xc0' + b'\x00' * 95)` | ### Misc @@ -139,19 +138,12 @@ def get_sync_committee_indices(state: BeaconState, epoch: Epoch) -> Sequence[Val seed = get_seed(state, base_epoch, DOMAIN_SYNC_COMMITTEE) i = 0 sync_committee_indices: List[ValidatorIndex] = [] - if len(active_validator_indices) < SYNC_COMMITTEE_SIZE: - committee_size = len(active_validator_indices) - else: - committee_size = SYNC_COMMITTEE_SIZE - while len(sync_committee_indices) < committee_size: + while len(sync_committee_indices) < SYNC_COMMITTEE_SIZE: shuffled_index = compute_shuffled_index(uint64(i % active_validator_count), active_validator_count, seed) candidate_index = active_validator_indices[shuffled_index] random_byte = hash(seed + uint_to_bytes(uint64(i // 32)))[i % 32] effective_balance = state.validators[candidate_index].effective_balance - if ( - effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte - and candidate_index not in sync_committee_indices - ): + if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: sync_committee_indices.append(candidate_index) i += 1 return sync_committee_indices @@ -171,11 +163,6 @@ def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: bls.AggregatePKs(pubkeys[i:i + SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE]) for i in range(0, len(pubkeys), SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE) ] - # Pad G1_POINT_AT_INFINITY to the BLSPubkey Vectors - if len(pubkeys) < SYNC_COMMITTEE_SIZE: - pubkeys += [G1_POINT_AT_INFINITY] * (SYNC_COMMITTEE_SIZE - len(pubkeys)) - aggregates_length = SYNC_COMMITTEE_SIZE // SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE - aggregates += [G1_POINT_AT_INFINITY] * (aggregates_length - len(aggregates)) return SyncCommittee(pubkeys=pubkeys, pubkey_aggregates=aggregates) ``` diff --git a/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py b/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py index 21557dedd..b7b2381e3 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py @@ -31,9 +31,3 @@ def compute_aggregate_sync_committee_signature(spec, state, slot, participants): ) ) return bls.Aggregate(signatures) - - -def get_padded_sync_committee_bits(spec, sync_committee_bits): - if len(sync_committee_bits) < spec.SYNC_COMMITTEE_SIZE: - return sync_committee_bits + [False] * (spec.SYNC_COMMITTEE_SIZE - len(sync_committee_bits)) - return sync_committee_bits diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index 14f3a9f0a..37cd0992b 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -9,7 +9,6 @@ from eth2spec.test.helpers.state import ( ) from eth2spec.test.helpers.sync_committee import ( compute_aggregate_sync_committee_signature, - get_padded_sync_committee_bits, ) from eth2spec.test.context import ( PHASE0, PHASE1, @@ -29,9 +28,7 @@ def test_invalid_signature_missing_participant(spec, state): block = build_empty_block_for_next_slot(spec, state) # Exclude one participant whose signature was included. - block.body.sync_committee_bits = get_padded_sync_committee_bits( - spec, [index != random_participant for index in committee] - ) + block.body.sync_committee_bits = [index != random_participant for index in committee] block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, @@ -54,7 +51,7 @@ def test_invalid_signature_extra_participant(spec, state): block = build_empty_block_for_next_slot(spec, state) # Exclude one signature even though the block claims the entire committee participated. - block.body.sync_committee_bits = get_padded_sync_committee_bits(spec, [True] * len(committee)) + block.body.sync_committee_bits = [True] * len(committee) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, @@ -86,7 +83,7 @@ def test_sync_committee_rewards(spec, state): pre_balances = state.balances.copy() block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = get_padded_sync_committee_bits(spec, [True] * committee_size) + block.body.sync_committee_bits = [True] * committee_size block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, @@ -129,7 +126,7 @@ def test_invalid_signature_past_block(spec, state): # NOTE: need to transition twice to move beyond the degenerate case at genesis block = build_empty_block_for_next_slot(spec, state) # Valid sync committee signature here... - block.body.sync_committee_bits = get_padded_sync_committee_bits(spec, [True] * len(committee)) + block.body.sync_committee_bits = [True] * len(committee) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, @@ -142,7 +139,7 @@ def test_invalid_signature_past_block(spec, state): invalid_block = build_empty_block_for_next_slot(spec, state) # Invalid signature from a slot other than the previous - invalid_block.body.sync_committee_bits = get_padded_sync_committee_bits(spec, [True] * len(committee)) + invalid_block.body.sync_committee_bits = [True] * len(committee) invalid_block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, @@ -178,7 +175,7 @@ def test_invalid_signature_previous_committee(spec, state): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = get_padded_sync_committee_bits(spec, [True] * len(committee)) + block.body.sync_committee_bits = [True] * len(committee) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, @@ -221,7 +218,7 @@ def test_valid_signature_future_committee(spec, state): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = get_padded_sync_committee_bits(spec, [True] * len(committee_indices)) + block.body.sync_committee_bits = [True] * len(committee_indices) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py index 93f8cd3ee..9033a0f15 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py @@ -8,7 +8,6 @@ from eth2spec.test.helpers.block import ( ) from eth2spec.test.helpers.sync_committee import ( compute_aggregate_sync_committee_signature, - get_padded_sync_committee_bits, ) from eth2spec.test.context import ( PHASE0, PHASE1, @@ -24,9 +23,7 @@ def run_sync_committee_sanity_test(spec, state, fraction_full=1.0): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = get_padded_sync_committee_bits( - spec, [index in participants for index in committee] - ) + block.body.sync_committee_bits = [index in participants for index in committee] block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, From 65696ca68be3badd36a784ddd3c5a0e2d62ae469 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 16 Jan 2021 22:48:22 +0100 Subject: [PATCH 095/222] fix typo: same condition, but non-aggregate attestation here --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 5dc892991..5a0953f76 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -419,7 +419,7 @@ The following validations MUST pass before forwarding the `attestation` on the s - _[REJECT]_ The signature of `attestation` is valid. - _[IGNORE]_ The block being voted for (`attestation.data.beacon_block_root`) has been seen (via both gossip and non-gossip sources) - (a client MAY queue aggregates for processing once block is retrieved). + (a client MAY queue attestations for processing once block is retrieved). - _[REJECT]_ The block being voted for (`attestation.data.beacon_block_root`) passes validation. - _[REJECT]_ The attestation's target block is an ancestor of the block named in the LMD vote -- i.e. `get_ancestor(store, attestation.data.beacon_block_root, compute_start_slot_at_epoch(attestation.data.target.epoch)) == attestation.data.target.root` From 37874f9b790ba457f160f9a77e625af5bd212974 Mon Sep 17 00:00:00 2001 From: Aditya Asgaonkar Date: Tue, 19 Jan 2021 01:53:12 -0800 Subject: [PATCH 096/222] Update WS calc --- specs/phase0/weak-subjectivity.md | 101 ++++++++++++++++++++---------- 1 file changed, 69 insertions(+), 32 deletions(-) diff --git a/specs/phase0/weak-subjectivity.md b/specs/phase0/weak-subjectivity.md index a99887926..6e2515185 100644 --- a/specs/phase0/weak-subjectivity.md +++ b/specs/phase0/weak-subjectivity.md @@ -59,38 +59,70 @@ a safety margin of at least `1/3 - SAFETY_DECAY/100`. ### Calculating the Weak Subjectivity Period -*Note*: `compute_weak_subjectivity_period()` is planned to be updated when a more accurate calculation is made. +A detailed analysis of the calculation of the weak subjectivity period is made in [this report](https://github.com/runtimeverification/beacon-chain-verification/blob/master/weak-subjectivity/weak-subjectivity-analysis.pdf). The expressions in the report use fractions, whereas we only use uint64 arithmetic in eth2.0-specs. The expressions have been simplified to avoid computing fractions, and more details can be found [here](https://www.overleaf.com/read/wgjzjdjpvpsd). ```python -def compute_weak_subjectivity_period(state: BeaconState) -> uint64: - weak_subjectivity_period = MIN_VALIDATOR_WITHDRAWABILITY_DELAY - validator_count = len(get_active_validator_indices(state, get_current_epoch(state))) - if validator_count >= MIN_PER_EPOCH_CHURN_LIMIT * CHURN_LIMIT_QUOTIENT: - weak_subjectivity_period += SAFETY_DECAY * CHURN_LIMIT_QUOTIENT // (2 * 100) - else: - weak_subjectivity_period += SAFETY_DECAY * validator_count // (2 * 100 * MIN_PER_EPOCH_CHURN_LIMIT) - return weak_subjectivity_period -``` +def get_active_validator_count(state: BeaconState) -> uint64: + active_validator_count = len(get_active_validator_indices(state, get_current_epoch(state))) + return active_validator_count -*Details about the calculation*: -- `100` appears in the denominator to get the actual percentage ratio from `SAFETY_DECAY` -- For more information about other terms in this equation, refer to - [Weak Subjectivity in Eth2.0](https://notes.ethereum.org/@adiasg/weak-subjectvity-eth2) +def compute_avg_active_validator_balance(state: BeaconState) -> Gwei: + total_active_balance = get_total_active_balance(state) + active_validator_count = get_active_validator_count(state) + avg_active_validator_balance = total_active_balance // active_validator_count + return avg_active_validator_balance//10**9 + +def compute_weak_subjectivity_period(state: BeaconState) -> uint64: + ws_period = MIN_VALIDATOR_WITHDRAWABILITY_DELAY + N = get_active_validator_count(state) + t = compute_avg_active_validator_balance(state) + T = MAX_EFFECTIVE_BALANCE//10**9 + delta = get_validator_churn_limit(state) + Delta = MAX_DEPOSITS * SLOTS_PER_EPOCH + D = SAFETY_DECAY + + case = ( + T*(200+3*D) < t*(200+12*D) + ) + + if case == 1: + arg1 = ( + N*(t*(200+12*D) - T*(200+3*D)) // (600*delta*(2*t+T)) + ) + arg2 = ( + N*(200+3*D) // (600*Delta) + ) + ws_period += max(arg1, arg2) + else: + ws_period += ( + 3*N*D*t // (200*Delta*(T-t)) + ) + + return ws_period +``` A brief reference for what these values look like in practice: -| `validator_count` | `weak_subjectivity_period` | -| ---- | ---- | -| 1024 | 268 | -| 2048 | 281 | -| 4096 | 307 | -| 8192 | 358 | -| 16384 | 460 | -| 32768 | 665 | -| 65536 | 1075 | -| 131072 | 1894 | -| 262144 | 3532 | -| 524288 | 3532 | +| SAFETY_DECAY | validator_count | average_active_validator_balance | weak_subjectivity_period | +| ---- | ---- | ---- | ---- | +| 10 | 8192 | 28 | 318 | +| 10 | 8192 | 32 | 358 | +| 10 | 16384 | 28 | 380 | +| 10 | 16384 | 32 | 460 | +| 10 | 32768 | 28 | 504 | +| 10 | 32768 | 32 | 665 | +| 20 | 8192 | 28 | 411 | +| 20 | 8192 | 32 | 460 | +| 20 | 16384 | 28 | 566 | +| 20 | 16384 | 32 | 665 | +| 20 | 32768 | 28 | 876 | +| 20 | 32768 | 32 | 1075 | +| 33 | 8192 | 28 | 532 | +| 33 | 8192 | 32 | 593 | +| 33 | 16384 | 28 | 808 | +| 33 | 16384 | 32 | 931 | +| 33 | 32768 | 28 | 1360 | +| 33 | 32768 | 32 | 1607 | ## Weak Subjectivity Sync @@ -101,17 +133,21 @@ Clients should allow users to input a Weak Subjectivity Checkpoint at startup, a 1. Input a Weak Subjectivity Checkpoint as a CLI parameter in `block_root:epoch_number` format, where `block_root` (an "0x" prefixed 32-byte hex string) and `epoch_number` (an integer) represent a valid `Checkpoint`. Example of the format: -``` + +```python 0x8584188b86a9296932785cc2827b925f9deebacce6d72ad8d53171fa046b43d9:9544 ``` -2. - *IF* `epoch_number > store.finalized_checkpoint.epoch`, - then *ASSERT* during block sync that block with root `block_root` is in the sync path at epoch `epoch_number`. - Emit descriptive critical error if this assert fails, then exit client process. + +2. Check the weak subjectivity requirements: + - *IF* `epoch_number > store.finalized_checkpoint.epoch`, + then *ASSERT* during block sync that block with root `block_root` is in the sync path at epoch `epoch_number`. + Emit descriptive critical error if this assert fails, then exit client process. - *IF* `epoch_number <= store.finalized_checkpoint.epoch`, - then *ASSERT* that the block in the canonical chain at epoch `epoch_number` has root `block_root`. - Emit descriptive critical error if this assert fails, then exit client process. + then *ASSERT* that the block in the canonical chain at epoch `epoch_number` has root `block_root`. + Emit descriptive critical error if this assert fails, then exit client process. ### Checking for Stale Weak Subjectivity Checkpoint + Clients may choose to validate that the input Weak Subjectivity Checkpoint is not stale at the time of startup. To support this mechanism, the client needs to take the state at the Weak Subjectivity Checkpoint as a CLI parameter input (or fetch the state associated with the input Weak Subjectivity Checkpoint from some source). @@ -130,4 +166,5 @@ def is_within_weak_subjectivity_period(store: Store, ws_state: BeaconState, ws_c ``` ## Distributing Weak Subjectivity Checkpoints + This section will be updated soon. From 93c4c62900de17ebffa5786fe5f118392de9fc36 Mon Sep 17 00:00:00 2001 From: Aditya Asgaonkar Date: Tue, 19 Jan 2021 02:04:55 -0800 Subject: [PATCH 097/222] Remove python tag from fenced code block --- specs/phase0/weak-subjectivity.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/weak-subjectivity.md b/specs/phase0/weak-subjectivity.md index 6e2515185..f5dcdba98 100644 --- a/specs/phase0/weak-subjectivity.md +++ b/specs/phase0/weak-subjectivity.md @@ -134,7 +134,7 @@ Clients should allow users to input a Weak Subjectivity Checkpoint at startup, a where `block_root` (an "0x" prefixed 32-byte hex string) and `epoch_number` (an integer) represent a valid `Checkpoint`. Example of the format: -```python +``` 0x8584188b86a9296932785cc2827b925f9deebacce6d72ad8d53171fa046b43d9:9544 ``` From c5d9aa2502d57fe264702c2ad5a396ffaf1a3b5d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 19 Jan 2021 20:00:43 +0800 Subject: [PATCH 098/222] Fix test cases for minimal and mainnet configs --- .../test_process_sync_committee.py | 81 +++++++++++++++++-- 1 file changed, 76 insertions(+), 5 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index 37cd0992b..fb44e855e 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -12,8 +12,10 @@ from eth2spec.test.helpers.sync_committee import ( ) from eth2spec.test.context import ( PHASE0, PHASE1, + MAINNET, MINIMAL, expect_assertion_error, with_all_phases_except, + with_configs, spec_state_test, ) @@ -72,12 +74,18 @@ def compute_sync_committee_participant_reward(spec, state, participant_index, ac @with_all_phases_except([PHASE0, PHASE1]) +@with_configs([MINIMAL], reason="to create nonduplicate committee") @spec_state_test -def test_sync_committee_rewards(spec, state): +def test_sync_committee_rewards_nonduplicate_committee(spec, state): committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) committee_size = len(committee) active_validator_count = len(spec.get_active_validator_indices(state, spec.get_current_epoch(state))) + # Preconditions of this test case + # Note that the committee members MAY still be duplicate even with enough active validator count probabilistically. + assert active_validator_count >= spec.SYNC_COMMITTEE_SIZE + assert committee_size == len(set(committee)) + yield 'pre', state pre_balances = state.balances.copy() @@ -114,6 +122,67 @@ def test_sync_committee_rewards(spec, state): assert state.balances[index] == pre_balances[index] + expected_reward +@with_all_phases_except([PHASE0, PHASE1]) +@with_configs([MAINNET], reason="to create duplicate committee") +@spec_state_test +def test_sync_committee_rewards_duplicate_committee(spec, state): + committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + committee_size = len(committee) + active_validator_count = len(spec.get_active_validator_indices(state, spec.get_current_epoch(state))) + + # Preconditions of this test case + # With mainnet config, where active validators are less than SYNC_COMMITTEE_SIZE, + # the committee members SHOULD be duplicate. + assert active_validator_count < spec.SYNC_COMMITTEE_SIZE + assert committee_size > len(set(committee)) + + yield 'pre', state + + pre_balances = state.balances.copy() + + block = build_empty_block_for_next_slot(spec, state) + block.body.sync_committee_bits = [True] * committee_size + block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + block.slot - 1, + committee, + ) + + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state + + duplicate_count = {} + for i, x in enumerate(committee): + if i != committee.index(x): + if x not in duplicate_count: + duplicate_count[x] = 1 + duplicate_count[x] += 1 + + for index in range(len(state.validators)): + expected_reward = 0 + + if index == block.proposer_index: + expected_reward += sum([spec.get_proposer_reward(state, index) for index in committee]) + + if index in committee: + reward = compute_sync_committee_participant_reward( + spec, + state, + index, + active_validator_count, + committee_size, + ) + if index not in duplicate_count: + expected_reward += reward + else: + expected_reward += reward * duplicate_count[index] + + assert state.balances[index] == pre_balances[index] + expected_reward + + @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test def test_invalid_signature_past_block(spec, state): @@ -163,24 +232,26 @@ def test_invalid_signature_previous_committee(spec, state): # To get a distinct committee so we can generate an "old" signature, we need to advance # 2 EPOCHS_PER_SYNC_COMMITTEE_PERIOD periods. current_epoch = spec.get_current_epoch(state) - previous_committee = state.next_sync_committee epoch_in_future_sync_commitee_period = current_epoch + 2 * spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD slot_in_future_sync_committee_period = epoch_in_future_sync_commitee_period * spec.SLOTS_PER_EPOCH transition_to(spec, state, slot_in_future_sync_committee_period) + # Create incorrect_committee for generating invalid signature. pubkeys = [validator.pubkey for validator in state.validators] - committee = [pubkeys.index(pubkey) for pubkey in previous_committee.pubkeys] + correct_committee = [pubkeys.index(pubkey) for pubkey in state.current_sync_committee.pubkeys] + incorrect_committee = [(correct_committee[0] + 1) % len(pubkeys)] + correct_committee[1:] + assert correct_committee != incorrect_committee yield 'pre', state block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = [True] * len(committee) + block.body.sync_committee_bits = [True] * len(incorrect_committee) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, block.slot - 1, - committee, + incorrect_committee, ) yield 'blocks', [block] From c877d142bd30fb8325e2f5a99ee9fdaa2aef4111 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 19 Jan 2021 20:24:25 +0800 Subject: [PATCH 099/222] Add duplicate elements warning to the docstring --- specs/lightclient/beacon-chain.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 2fbe190e4..8f0c514f0 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -130,7 +130,8 @@ def eth2_fast_aggregate_verify(pubkeys: Sequence[BLSPubkey], message: Bytes32, s def get_sync_committee_indices(state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: """ Return the sync committee indices for a given state and epoch. - """ + Note that there may be duplicate indices in the resulting list. + """ MAX_RANDOM_BYTE = 2**8 - 1 base_epoch = Epoch((max(epoch // EPOCHS_PER_SYNC_COMMITTEE_PERIOD, 1) - 1) * EPOCHS_PER_SYNC_COMMITTEE_PERIOD) active_validator_indices = get_active_validator_indices(state, base_epoch) From fb0c6d54f87eba3d9454484cd4ae14e5397d0bb2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 19 Jan 2021 20:12:36 +0800 Subject: [PATCH 100/222] Add @ralexstokes's fix --- specs/lightclient/sync-protocol.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index 562b7b5bb..b8d443fb2 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -185,10 +185,12 @@ def process_light_client_update(store: LightClientStore, update: LightClientUpda store.valid_updates.append(update) if ( - sum(update.sync_committee_bits) * 3 >= len(update.sync_committee_bits) * 2 - and update.header != update.finality_header + sum(update.sync_committee_bits) * 3 > len(update.sync_committee_bits) * 2 + and update.finality_header != BeaconBlockHeader() ): - # Apply update if 2/3 quorum is reached and we have a finality proof + # Apply update if (1) 2/3 quorum is reached and (2) we have a finality proof. + # Note that (2) means that the current light client design needs finality. + # It may be changed to re-organizable light client design. See the on-going issue eth2.0-specs#2182. apply_light_client_update(store, update) store.valid_updates = [] elif current_slot > store.snapshot.header.slot + LIGHT_CLIENT_UPDATE_TIMEOUT: From f3d7dee71f15622f0dd258c25046e31bbdbb9510 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 19 Jan 2021 20:44:21 +0800 Subject: [PATCH 101/222] Apply @djrtwo's suggestion --- specs/lightclient/sync-protocol.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index b8d443fb2..7d442897e 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -20,7 +20,7 @@ - [Helper functions](#helper-functions) - [`get_subtree_index`](#get_subtree_index) - [Light client state updates](#light-client-state-updates) - - [`is_valid_light_client_update`](#is_valid_light_client_update) + - [`validate_light_client_update`](#validate_light_client_update) - [`apply_light_client_update`](#apply_light_client_update) - [`process_light_client_update`](#process_light_client_update) @@ -112,10 +112,10 @@ def get_subtree_index(generalized_index: GeneralizedIndex) -> uint64: A light client maintains its state in a `store` object of type `LightClientStore` and receives `update` objects of type `LightClientUpdate`. Every `update` triggers `process_light_client_update(store, update, current_slot)` where `current_slot` is the current slot based on some local clock. -#### `is_valid_light_client_update` +#### `validate_light_client_update` ```python -def is_valid_light_client_update(snapshot: LightClientSnapshot, update: LightClientUpdate) -> bool: +def validate_light_client_update(snapshot: LightClientSnapshot, update: LightClientUpdate) -> None: # Verify update slot is larger than snapshot slot assert update.header.slot > snapshot.header.slot @@ -160,8 +160,6 @@ def is_valid_light_client_update(snapshot: LightClientSnapshot, update: LightCli domain = compute_domain(DOMAIN_SYNC_COMMITTEE, update.fork_version) signing_root = compute_signing_root(signed_header, domain) assert bls.FastAggregateVerify(participant_pubkeys, signing_root, update.sync_committee_signature) - - return True ``` #### `apply_light_client_update` @@ -180,8 +178,7 @@ def apply_light_client_update(snapshot: LightClientSnapshot, update: LightClient ```python def process_light_client_update(store: LightClientStore, update: LightClientUpdate, current_slot: Slot) -> None: - # Validate update - assert is_valid_light_client_update(store.snapshot, update) + validate_light_client_update(store.snapshot, update) store.valid_updates.append(update) if ( From 12593e8782ef4d66f557377d537aa843d68f86d0 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 19 Jan 2021 12:52:40 +0000 Subject: [PATCH 102/222] Update comments --- specs/lightclient/beacon-chain.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 8f0c514f0..54006325e 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -129,8 +129,7 @@ def eth2_fast_aggregate_verify(pubkeys: Sequence[BLSPubkey], message: Bytes32, s ```python def get_sync_committee_indices(state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: """ - Return the sync committee indices for a given state and epoch. - Note that there may be duplicate indices in the resulting list. + Return the sequence of sync committee indices (which may include duplicate indices) for a given state and epoch. """ MAX_RANDOM_BYTE = 2**8 - 1 base_epoch = Epoch((max(epoch // EPOCHS_PER_SYNC_COMMITTEE_PERIOD, 1) - 1) * EPOCHS_PER_SYNC_COMMITTEE_PERIOD) @@ -144,7 +143,7 @@ def get_sync_committee_indices(state: BeaconState, epoch: Epoch) -> Sequence[Val candidate_index = active_validator_indices[shuffled_index] random_byte = hash(seed + uint_to_bytes(uint64(i // 32)))[i % 32] effective_balance = state.validators[candidate_index].effective_balance - if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: + if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: # Sample with replacement sync_committee_indices.append(candidate_index) i += 1 return sync_committee_indices From daa47987045f041a09922a3ae1b244c882aaf5aa Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 19 Jan 2021 21:21:45 +0800 Subject: [PATCH 103/222] Break down process_final_updates --- specs/phase0/beacon-chain.md | 52 ++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 08ddc853a..b05ebe6de 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -113,7 +113,12 @@ - [`process_rewards_and_penalties`](#process_rewards_and_penalties) - [Registry updates](#registry-updates) - [Slashings](#slashings) - - [Final updates](#final-updates) + - [Eth1 data votes updates](#eth1-data-votes-updates) + - [Effective balances updates](#effective-balances-updates) + - [Slashings balances updates](#slashings-balances-updates) + - [Randao mixes updates](#randao-mixes-updates) + - [Historical roots updates](#historical-roots-updates) + - [Participation records rotation](#participation-records-rotation) - [Block processing](#block-processing) - [Block header](#block-header) - [RANDAO](#randao) @@ -1250,7 +1255,12 @@ def process_epoch(state: BeaconState) -> None: process_rewards_and_penalties(state) process_registry_updates(state) process_slashings(state) - process_final_updates(state) + process_eth1_data_votes_updates(state) + process_effective_balances_updates(state) + process_slashings_updates(state) + process_randao_mixes_updates(state) + process_historical_roots_updates(state) + process_participation_record_updates(state) ``` #### Helper functions @@ -1557,15 +1567,20 @@ def process_slashings(state: BeaconState) -> None: decrease_balance(state, ValidatorIndex(index), penalty) ``` -#### Final updates - +#### Eth1 data votes updates ```python -def process_final_updates(state: BeaconState) -> None: +def process_eth1_data_votes_updates(state: BeaconState) -> None: current_epoch = get_current_epoch(state) next_epoch = Epoch(current_epoch + 1) # Reset eth1 data votes if next_epoch % EPOCHS_PER_ETH1_VOTING_PERIOD == 0: state.eth1_data_votes = [] +``` + +#### Effective balances updates + +```python +def process_effective_balances_updates(state: BeaconState) -> None: # Update effective balances with hysteresis for index, validator in enumerate(state.validators): balance = state.balances[index] @@ -1577,14 +1592,41 @@ def process_final_updates(state: BeaconState) -> None: or validator.effective_balance + UPWARD_THRESHOLD < balance ): validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) +``` + +#### Slashings balances updates + +```python +def process_slashings_updates(state: BeaconState) -> None: + next_epoch = Epoch(get_current_epoch(state) + 1) # Reset slashings state.slashings[next_epoch % EPOCHS_PER_SLASHINGS_VECTOR] = Gwei(0) +``` + +#### Randao mixes updates + +```python +def process_randao_mixes_updates(state: BeaconState) -> None: + current_epoch = get_current_epoch(state) + next_epoch = Epoch(current_epoch + 1) # Set randao mix state.randao_mixes[next_epoch % EPOCHS_PER_HISTORICAL_VECTOR] = get_randao_mix(state, current_epoch) +``` + +#### Historical roots updates +```python +def process_historical_roots_updates(state: BeaconState) -> None: # Set historical root accumulator + next_epoch = Epoch(get_current_epoch(state) + 1) if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0: historical_batch = HistoricalBatch(block_roots=state.block_roots, state_roots=state.state_roots) state.historical_roots.append(hash_tree_root(historical_batch)) +``` + +#### Participation records rotation + +```python +def process_participation_record_updates(state: BeaconState) -> None: # Rotate current/previous epoch attestations state.previous_epoch_attestations = state.current_epoch_attestations state.current_epoch_attestations = [] From fa6094837bfeab56e35d74d81f3ca15adc9221d1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 19 Jan 2021 21:41:34 +0800 Subject: [PATCH 104/222] Update lightclient patch and phase1 specs --- specs/lightclient/beacon-chain.md | 26 +++++++++++++++++++------- specs/phase1/beacon-chain.md | 10 +++++++--- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 2fbe190e4..60526d8c0 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -29,7 +29,7 @@ - [Sync committee processing](#sync-committee-processing) - [Epoch processing](#epoch-processing) - [Components of attestation deltas](#components-of-attestation-deltas) - - [Final updates](#final-updates) + - [Sync committee updates](#sync-committee-updates) @@ -209,6 +209,22 @@ def process_sync_committee(state: BeaconState, body: BeaconBlockBody) -> None: ### Epoch processing +```python +def process_epoch(state: BeaconState) -> None: + process_justification_and_finalization(state) + process_rewards_and_penalties(state) + process_registry_updates(state) + process_slashings(state) + process_eth1_data_votes_updates(state) + process_effective_balances_updates(state) + process_slashings_updates(state) + process_randao_mixes_updates(state) + process_historical_roots_updates(state) + process_participation_record_updates(state) + # Light client patch + process_sync_committee_updates(state) +``` + #### Components of attestation deltas *Note*: The function `get_inactivity_penalty_deltas` is modified with `BASE_REWARDS_PER_EPOCH` replaced by `BASE_REWARDS_PER_EPOCH - 1`. @@ -235,14 +251,10 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S return rewards, penalties ``` -#### Final updates - -*Note*: The function `process_final_updates` is modified to handle sync committee updates. +#### Sync committee updates ```python -def process_final_updates(state: BeaconState) -> None: - # FIXME: unfold the full `process_final_updates` to avoid side effects. - phase0.process_final_updates(state) +def process_sync_committee_updates(state: BeaconState) -> None: next_epoch = get_current_epoch(state) + Epoch(1) if next_epoch % EPOCHS_PER_SYNC_COMMITTEE_PERIOD == 0: state.current_sync_committee = state.next_sync_committee diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index c8f93cc7f..2b8e460ed 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -1054,10 +1054,14 @@ def process_epoch(state: BeaconState) -> None: process_justification_and_finalization(state) process_rewards_and_penalties(state) process_registry_updates(state) - process_reveal_deadlines(state) - process_challenge_deadlines(state) process_slashings(state) - process_final_updates(state) # phase 0 final updates + process_eth1_data_votes_updates(state) + process_effective_balances_updates(state) + process_slashings_updates(state) + process_randao_mixes_updates(state) + process_historical_roots_updates(state) + process_participation_record_updates(state) + # Phase 1 process_phase_1_final_updates(state) ``` From 93d19bdf4025c8eb7b2b3ab493de22113d9bae36 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 19 Jan 2021 21:34:48 +0800 Subject: [PATCH 105/222] Update and add tests --- .../eth2spec/test/helpers/epoch_processing.py | 13 ++++- .../test_process_final_updates.py | 2 +- ...est_process_effective_balances_updates.py} | 57 ++----------------- .../test_process_eth1_data_votes_updates.py | 43 ++++++++++++++ .../test_process_historical_roots_updates.py | 20 +++++++ ...st_process_participation_record_updates.py | 21 +++++++ .../test_process_randao_mixes_updates.py | 21 +++++++ .../test_process_slashings_updates.py | 20 +++++++ tests/formats/epoch_processing/README.md | 11 +++- 9 files changed, 150 insertions(+), 58 deletions(-) rename tests/core/pyspec/eth2spec/test/phase0/epoch_processing/{test_process_final_updates.py => test_process_effective_balances_updates.py} (52%) create mode 100644 tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_eth1_data_votes_updates.py create mode 100644 tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_historical_roots_updates.py create mode 100644 tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_participation_record_updates.py create mode 100644 tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_randao_mixes_updates.py create mode 100644 tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings_updates.py diff --git a/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py b/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py index b8692227f..ca1b9f802 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py +++ b/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py @@ -1,13 +1,22 @@ process_calls = [ + # PHASE0 'process_justification_and_finalization', 'process_rewards_and_penalties', 'process_registry_updates', 'process_reveal_deadlines', 'process_challenge_deadlines', 'process_slashings', - 'process_final_updates', - 'after_process_final_updates', + 'process_eth1_data_votes_updates', + 'process_effective_balances_updates', + 'process_slashings_updates', + 'process_randao_mixes_updates', + 'process_historical_roots_updates', + 'process_participation_record_updates', + # LIGHTCLIENT_PATCH + 'process_sync_committee_updates', + # PHASE1 + 'process_phase_1_final_updates', ] diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py index 012438cd8..06473c645 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py @@ -27,7 +27,7 @@ def test_sync_committees_progress(spec, state): assert state.current_sync_committee == first_sync_committee assert state.next_sync_committee == second_sync_committee - yield from run_epoch_processing_with(spec, state, 'process_final_updates') + yield from run_epoch_processing_with(spec, state, 'process_sync_committee_updates') # Can compute the third committee having computed final balances in the last epoch # of this `EPOCHS_PER_SYNC_COMMITTEE_PERIOD` diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_final_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_effective_balances_updates.py similarity index 52% rename from tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_final_updates.py rename to tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_effective_balances_updates.py index f9b5c872b..ce8f0e348 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_final_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_effective_balances_updates.py @@ -2,45 +2,10 @@ from eth2spec.test.context import spec_state_test, with_all_phases from eth2spec.test.helpers.epoch_processing import ( run_epoch_processing_with, run_epoch_processing_to ) -from eth2spec.test.helpers.state import transition_to -def run_process_final_updates(spec, state): - yield from run_epoch_processing_with(spec, state, 'process_final_updates') - - -@with_all_phases -@spec_state_test -def test_eth1_vote_no_reset(spec, state): - assert spec.EPOCHS_PER_ETH1_VOTING_PERIOD > 1 - # skip ahead to the end of the epoch - transition_to(spec, state, spec.SLOTS_PER_EPOCH - 1) - - for i in range(state.slot + 1): # add a vote for each skipped slot. - state.eth1_data_votes.append( - spec.Eth1Data(deposit_root=b'\xaa' * 32, - deposit_count=state.eth1_deposit_index, - block_hash=b'\xbb' * 32)) - - yield from run_process_final_updates(spec, state) - - assert len(state.eth1_data_votes) == spec.SLOTS_PER_EPOCH - - -@with_all_phases -@spec_state_test -def test_eth1_vote_reset(spec, state): - # skip ahead to the end of the voting period - state.slot = (spec.EPOCHS_PER_ETH1_VOTING_PERIOD * spec.SLOTS_PER_EPOCH) - 1 - for i in range(state.slot + 1): # add a vote for each skipped slot. - state.eth1_data_votes.append( - spec.Eth1Data(deposit_root=b'\xaa' * 32, - deposit_count=state.eth1_deposit_index, - block_hash=b'\xbb' * 32)) - - yield from run_process_final_updates(spec, state) - - assert len(state.eth1_data_votes) == 0 +def run_process_effective_balances_updates(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_effective_balances_updates') @with_all_phases @@ -48,7 +13,7 @@ def test_eth1_vote_reset(spec, state): def test_effective_balance_hysteresis(spec, state): # Prepare state up to the final-updates. # Then overwrite the balances, we only want to focus to be on the hysteresis based changes. - run_epoch_processing_to(spec, state, 'process_final_updates') + run_epoch_processing_to(spec, state, 'process_effective_balances_updates') # Set some edge cases for balances max = spec.MAX_EFFECTIVE_BALANCE min = spec.EJECTION_BALANCE @@ -79,21 +44,7 @@ def test_effective_balance_hysteresis(spec, state): state.validators[i].effective_balance = pre_eff state.balances[i] = bal - yield 'pre', state - spec.process_final_updates(state) - yield 'post', state + yield from run_process_effective_balances_updates(spec, state) for i, (_, _, post_eff, name) in enumerate(cases): assert state.validators[i].effective_balance == post_eff, name - - -@with_all_phases -@spec_state_test -def test_historical_root_accumulator(spec, state): - # skip ahead to near the end of the historical roots period (excl block before epoch processing) - state.slot = spec.SLOTS_PER_HISTORICAL_ROOT - 1 - history_len = len(state.historical_roots) - - yield from run_process_final_updates(spec, state) - - assert len(state.historical_roots) == history_len + 1 diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_eth1_data_votes_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_eth1_data_votes_updates.py new file mode 100644 index 000000000..c9d7222d7 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_eth1_data_votes_updates.py @@ -0,0 +1,43 @@ +from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.helpers.epoch_processing import ( + run_epoch_processing_with, +) +from eth2spec.test.helpers.state import transition_to + + +def run_process_eth1_data_votes_updates(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_eth1_data_votes_updates') + + +@with_all_phases +@spec_state_test +def test_eth1_vote_no_reset(spec, state): + assert spec.EPOCHS_PER_ETH1_VOTING_PERIOD > 1 + # skip ahead to the end of the epoch + transition_to(spec, state, spec.SLOTS_PER_EPOCH - 1) + + for i in range(state.slot + 1): # add a vote for each skipped slot. + state.eth1_data_votes.append( + spec.Eth1Data(deposit_root=b'\xaa' * 32, + deposit_count=state.eth1_deposit_index, + block_hash=b'\xbb' * 32)) + + yield from run_process_eth1_data_votes_updates(spec, state) + + assert len(state.eth1_data_votes) == spec.SLOTS_PER_EPOCH + + +@with_all_phases +@spec_state_test +def test_eth1_vote_reset(spec, state): + # skip ahead to the end of the voting period + state.slot = (spec.EPOCHS_PER_ETH1_VOTING_PERIOD * spec.SLOTS_PER_EPOCH) - 1 + for i in range(state.slot + 1): # add a vote for each skipped slot. + state.eth1_data_votes.append( + spec.Eth1Data(deposit_root=b'\xaa' * 32, + deposit_count=state.eth1_deposit_index, + block_hash=b'\xbb' * 32)) + + yield from run_process_eth1_data_votes_updates(spec, state) + + assert len(state.eth1_data_votes) == 0 diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_historical_roots_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_historical_roots_updates.py new file mode 100644 index 000000000..d1ab9524f --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_historical_roots_updates.py @@ -0,0 +1,20 @@ +from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.helpers.epoch_processing import ( + run_epoch_processing_with +) + + +def run_process_historical_roots_updates(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_historical_roots_updates') + + +@with_all_phases +@spec_state_test +def test_historical_root_accumulator(spec, state): + # skip ahead to near the end of the historical roots period (excl block before epoch processing) + state.slot = spec.SLOTS_PER_HISTORICAL_ROOT - 1 + history_len = len(state.historical_roots) + + yield from run_process_historical_roots_updates(spec, state) + + assert len(state.historical_roots) == history_len + 1 diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_participation_record_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_participation_record_updates.py new file mode 100644 index 000000000..f5e1513e3 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_participation_record_updates.py @@ -0,0 +1,21 @@ +from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.helpers.epoch_processing import ( + run_epoch_processing_with +) + + +def run_process_participation_record_updates(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_participation_record_updates') + + +@with_all_phases +@spec_state_test +def test_updated_participation_record(spec, state): + state.previous_epoch_attestations = [spec.PendingAttestation(proposer_index=100)] + current_epoch_attestations = [spec.PendingAttestation(proposer_index=200)] + state.current_epoch_attestations = current_epoch_attestations + + yield from run_process_participation_record_updates(spec, state) + + assert state.previous_epoch_attestations == current_epoch_attestations + assert state.current_epoch_attestations == [] diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_randao_mixes_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_randao_mixes_updates.py new file mode 100644 index 000000000..b7e4a0b8f --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_randao_mixes_updates.py @@ -0,0 +1,21 @@ +from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.helpers.epoch_processing import ( + run_epoch_processing_with +) + + +def run_process_randao_mixes_updates(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_randao_mixes_updates') + + +@with_all_phases +@spec_state_test +def test_updated_randao_mixes(spec, state): + next_epoch = spec.get_current_epoch(state) + 1 + state.randao_mixes[next_epoch % spec.EPOCHS_PER_HISTORICAL_VECTOR] = b'\x56' * 32 + + yield from run_process_randao_mixes_updates(spec, state) + + assert state.randao_mixes[next_epoch % spec.EPOCHS_PER_HISTORICAL_VECTOR] == spec.get_randao_mix( + state, spec.get_current_epoch(state) + ) diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings_updates.py new file mode 100644 index 000000000..559b52c09 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings_updates.py @@ -0,0 +1,20 @@ +from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.helpers.epoch_processing import ( + run_epoch_processing_with +) + + +def run_process_slashings_updates(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_slashings_updates') + + +@with_all_phases +@spec_state_test +def test_flush_slashings(spec, state): + next_epoch = spec.get_current_epoch(state) + 1 + state.slashings[next_epoch % spec.EPOCHS_PER_SLASHINGS_VECTOR] = 100 + assert state.slashings[next_epoch % spec.EPOCHS_PER_SLASHINGS_VECTOR] != 0 + + yield from run_process_slashings_updates(spec, state) + + assert state.slashings[next_epoch % spec.EPOCHS_PER_SLASHINGS_VECTOR] == 0 diff --git a/tests/formats/epoch_processing/README.md b/tests/formats/epoch_processing/README.md index 57c9441c8..39e8050d8 100644 --- a/tests/formats/epoch_processing/README.md +++ b/tests/formats/epoch_processing/README.md @@ -37,10 +37,17 @@ The provided pre-state is already transitioned to just before the specific sub-t Sub-transitions: +Sub-transitions: + - `justification_and_finalization` -- `rewards_and_penalties` (limited to `minimal` config) +- `rewards_and_penalties` - `registry_updates` - `slashings` -- `final_updates` +- `eth1_data_votes_updates` +- `effective_balances_updates` +- `slashings_updates` +- `randao_mixes_updates` +- `historical_roots_updates` +- `participation_record_updates` The resulting state should match the expected `post` state. From e53213a585e00851f61d540ec24a3c19a0b30b57 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 20 Jan 2021 13:06:24 +0800 Subject: [PATCH 106/222] Minor refactor --- specs/phase0/beacon-chain.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index b05ebe6de..bfa6bdd35 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1570,8 +1570,7 @@ def process_slashings(state: BeaconState) -> None: #### Eth1 data votes updates ```python def process_eth1_data_votes_updates(state: BeaconState) -> None: - current_epoch = get_current_epoch(state) - next_epoch = Epoch(current_epoch + 1) + next_epoch = Epoch(get_current_epoch(state) + 1) # Reset eth1 data votes if next_epoch % EPOCHS_PER_ETH1_VOTING_PERIOD == 0: state.eth1_data_votes = [] From 900eb4a83cfb808ab176d35fc3ddd60436dbc2c2 Mon Sep 17 00:00:00 2001 From: Aditya Asgaonkar Date: Wed, 20 Jan 2021 19:33:25 -0800 Subject: [PATCH 107/222] Fixed CI errors --- specs/phase0/weak-subjectivity.md | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/specs/phase0/weak-subjectivity.md b/specs/phase0/weak-subjectivity.md index f5dcdba98..2dd28df3d 100644 --- a/specs/phase0/weak-subjectivity.md +++ b/specs/phase0/weak-subjectivity.md @@ -8,7 +8,9 @@ - [Introduction](#introduction) - [Prerequisites](#prerequisites) +- [Custom Types](#custom-types) - [Constants](#constants) +- [Configuration](#configuration) - [Weak Subjectivity Checkpoint](#weak-subjectivity-checkpoint) - [Weak Subjectivity Period](#weak-subjectivity-period) - [Calculating the Weak Subjectivity Period](#calculating-the-weak-subjectivity-period) @@ -34,10 +36,22 @@ For more information about weak subjectivity and why it is required, please refe This document uses data structures, constants, functions, and terminology from [Phase 0 -- The Beacon Chain](./beacon-chain.md) and [Phase 0 -- Beacon Chain Fork Choice](./fork-choice.md). +## Custom Types + +| Name | SSZ Equivalent | Description | +|---|---|---| +| `Ether` | `uint64` | an amount in Ether | + ## Constants -| Name | Value | -|----------------|--------------| +| Name | Value | +|---|---| +| `ETH_TO_GWEI` | `uint64(10**9)` | + +## Configuration + +| Name | Value | +|---|---| | `SAFETY_DECAY` | `uint64(10)` | ## Weak Subjectivity Checkpoint @@ -66,11 +80,12 @@ def get_active_validator_count(state: BeaconState) -> uint64: active_validator_count = len(get_active_validator_indices(state, get_current_epoch(state))) return active_validator_count -def compute_avg_active_validator_balance(state: BeaconState) -> Gwei: +def compute_avg_active_validator_balance(state: BeaconState) -> Ether: total_active_balance = get_total_active_balance(state) active_validator_count = get_active_validator_count(state) - avg_active_validator_balance = total_active_balance // active_validator_count - return avg_active_validator_balance//10**9 + avg_active_validator_balance_gwei = total_active_balance // active_validator_count + avg_active_validator_balance_eth = avg_active_validator_balance_gwei // ETH_TO_GWEI + return avg_active_validator_balance_eth def compute_weak_subjectivity_period(state: BeaconState) -> uint64: ws_period = MIN_VALIDATOR_WITHDRAWABILITY_DELAY @@ -103,7 +118,7 @@ def compute_weak_subjectivity_period(state: BeaconState) -> uint64: A brief reference for what these values look like in practice: -| SAFETY_DECAY | validator_count | average_active_validator_balance | weak_subjectivity_period | +| Safety Decay | Validator Count | Average Active Validator Balance | Weak Subjectivity Period | | ---- | ---- | ---- | ---- | | 10 | 8192 | 28 | 318 | | 10 | 8192 | 32 | 358 | From 17a04c2728c67410941b064cbc9897545c3e2acc Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 21 Jan 2021 23:02:22 +0800 Subject: [PATCH 108/222] PR feedback from @ralexstokes --- .../test_process_sync_committee.py | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index fb44e855e..cec3a1a65 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -1,3 +1,4 @@ +from collections import Counter import random from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, @@ -154,12 +155,7 @@ def test_sync_committee_rewards_duplicate_committee(spec, state): yield 'blocks', [signed_block] yield 'post', state - duplicate_count = {} - for i, x in enumerate(committee): - if i != committee.index(x): - if x not in duplicate_count: - duplicate_count[x] = 1 - duplicate_count[x] += 1 + multiplicities = Counter(committee) for index in range(len(state.validators)): expected_reward = 0 @@ -175,10 +171,7 @@ def test_sync_committee_rewards_duplicate_committee(spec, state): active_validator_count, committee_size, ) - if index not in duplicate_count: - expected_reward += reward - else: - expected_reward += reward * duplicate_count[index] + expected_reward += reward * multiplicities[index] assert state.balances[index] == pre_balances[index] + expected_reward @@ -224,6 +217,7 @@ def test_invalid_signature_past_block(spec, state): @with_all_phases_except([PHASE0, PHASE1]) +@with_configs([MINIMAL], reason="to produce different committee sets") @spec_state_test def test_invalid_signature_previous_committee(spec, state): # NOTE: the `state` provided is at genesis and the process to select @@ -232,26 +226,27 @@ def test_invalid_signature_previous_committee(spec, state): # To get a distinct committee so we can generate an "old" signature, we need to advance # 2 EPOCHS_PER_SYNC_COMMITTEE_PERIOD periods. current_epoch = spec.get_current_epoch(state) + old_sync_committee = state.next_sync_committee epoch_in_future_sync_commitee_period = current_epoch + 2 * spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD slot_in_future_sync_committee_period = epoch_in_future_sync_commitee_period * spec.SLOTS_PER_EPOCH transition_to(spec, state, slot_in_future_sync_committee_period) - # Create incorrect_committee for generating invalid signature. - pubkeys = [validator.pubkey for validator in state.validators] - correct_committee = [pubkeys.index(pubkey) for pubkey in state.current_sync_committee.pubkeys] - incorrect_committee = [(correct_committee[0] + 1) % len(pubkeys)] + correct_committee[1:] - assert correct_committee != incorrect_committee - yield 'pre', state + # Use the previous sync committee to produce the signature. + pubkeys = [validator.pubkey for validator in state.validators] + # Ensure that the pubkey sets are different. + assert set(old_sync_committee.pubkeys) != set(state.current_sync_committee.pubkeys) + committee = [pubkeys.index(pubkey) for pubkey in old_sync_committee.pubkeys] + block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = [True] * len(incorrect_committee) + block.body.sync_committee_bits = [True] * len(committee) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, block.slot - 1, - incorrect_committee, + committee, ) yield 'blocks', [block] From 3847e425b1cae8292a3fb833730068db7c7269e2 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Wed, 20 Jan 2021 16:59:27 -0800 Subject: [PATCH 109/222] refactor to use helper for duplicates in light client committees, rather than config changes --- .../test_process_sync_committee.py | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index fb44e855e..5ef04ef40 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -18,6 +18,27 @@ from eth2spec.test.context import ( with_configs, spec_state_test, ) +from eth2spec.utils.hash_function import hash + + +def get_committee_indices(spec, state, duplicates=False): + ''' + This utility function allows the caller to ensure there are or are not + duplicate validator indices in the returned committee based on + the boolean ``duplicates``. + ''' + state = state.copy() + current_epoch = spec.get_current_epoch(state) + randao_index = current_epoch % spec.EPOCHS_PER_HISTORICAL_VECTOR + while True: + committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + if duplicates: + if len(committee) != len(set(committee)): + return committee + else: + if len(committee) == len(set(committee)): + return committee + state.randao_mixes[randao_index] = hash(state.randao_mixes[randao_index]) @with_all_phases_except([PHASE0, PHASE1]) @@ -77,7 +98,7 @@ def compute_sync_committee_participant_reward(spec, state, participant_index, ac @with_configs([MINIMAL], reason="to create nonduplicate committee") @spec_state_test def test_sync_committee_rewards_nonduplicate_committee(spec, state): - committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + committee = get_committee_indices(spec, state, duplicates=False) committee_size = len(committee) active_validator_count = len(spec.get_active_validator_indices(state, spec.get_current_epoch(state))) @@ -126,7 +147,7 @@ def test_sync_committee_rewards_nonduplicate_committee(spec, state): @with_configs([MAINNET], reason="to create duplicate committee") @spec_state_test def test_sync_committee_rewards_duplicate_committee(spec, state): - committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + committee = get_committee_indices(spec, state, duplicates=True) committee_size = len(committee) active_validator_count = len(spec.get_active_validator_indices(state, spec.get_current_epoch(state))) From 0e415fe7c7d04ce1c1bacdee1288d40b6158c0b8 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 21 Jan 2021 15:33:43 -0800 Subject: [PATCH 110/222] comments no longer apply --- .../block_processing/test_process_sync_committee.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index 5ef04ef40..98db7b7ee 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -103,7 +103,6 @@ def test_sync_committee_rewards_nonduplicate_committee(spec, state): active_validator_count = len(spec.get_active_validator_indices(state, spec.get_current_epoch(state))) # Preconditions of this test case - # Note that the committee members MAY still be duplicate even with enough active validator count probabilistically. assert active_validator_count >= spec.SYNC_COMMITTEE_SIZE assert committee_size == len(set(committee)) @@ -152,8 +151,6 @@ def test_sync_committee_rewards_duplicate_committee(spec, state): active_validator_count = len(spec.get_active_validator_indices(state, spec.get_current_epoch(state))) # Preconditions of this test case - # With mainnet config, where active validators are less than SYNC_COMMITTEE_SIZE, - # the committee members SHOULD be duplicate. assert active_validator_count < spec.SYNC_COMMITTEE_SIZE assert committee_size > len(set(committee)) From c932fc279897bcc4a7bdf109a97d70ceac7cd666 Mon Sep 17 00:00:00 2001 From: Aditya Asgaonkar Date: Thu, 21 Jan 2021 17:07:45 -0800 Subject: [PATCH 111/222] Fix linter errors --- specs/phase0/weak-subjectivity.md | 40 +++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/specs/phase0/weak-subjectivity.md b/specs/phase0/weak-subjectivity.md index 2dd28df3d..75cdbec40 100644 --- a/specs/phase0/weak-subjectivity.md +++ b/specs/phase0/weak-subjectivity.md @@ -14,9 +14,13 @@ - [Weak Subjectivity Checkpoint](#weak-subjectivity-checkpoint) - [Weak Subjectivity Period](#weak-subjectivity-period) - [Calculating the Weak Subjectivity Period](#calculating-the-weak-subjectivity-period) + - [`get_active_validator_count`](#get_active_validator_count) + - [`compute_avg_active_validator_balance`](#compute_avg_active_validator_balance) + - [`compute_weak_subjectivity_period`](#compute_weak_subjectivity_period) - [Weak Subjectivity Sync](#weak-subjectivity-sync) - [Weak Subjectivity Sync Procedure](#weak-subjectivity-sync-procedure) - [Checking for Stale Weak Subjectivity Checkpoint](#checking-for-stale-weak-subjectivity-checkpoint) + - [`is_within_weak_subjectivity_period`](#is_within_weak_subjectivity_period) - [Distributing Weak Subjectivity Checkpoints](#distributing-weak-subjectivity-checkpoints) @@ -75,43 +79,53 @@ a safety margin of at least `1/3 - SAFETY_DECAY/100`. A detailed analysis of the calculation of the weak subjectivity period is made in [this report](https://github.com/runtimeverification/beacon-chain-verification/blob/master/weak-subjectivity/weak-subjectivity-analysis.pdf). The expressions in the report use fractions, whereas we only use uint64 arithmetic in eth2.0-specs. The expressions have been simplified to avoid computing fractions, and more details can be found [here](https://www.overleaf.com/read/wgjzjdjpvpsd). +#### `get_active_validator_count` + ```python def get_active_validator_count(state: BeaconState) -> uint64: active_validator_count = len(get_active_validator_indices(state, get_current_epoch(state))) return active_validator_count +``` +#### `compute_avg_active_validator_balance` + +```python def compute_avg_active_validator_balance(state: BeaconState) -> Ether: total_active_balance = get_total_active_balance(state) active_validator_count = get_active_validator_count(state) avg_active_validator_balance_gwei = total_active_balance // active_validator_count avg_active_validator_balance_eth = avg_active_validator_balance_gwei // ETH_TO_GWEI return avg_active_validator_balance_eth +``` +#### `compute_weak_subjectivity_period` + +```python def compute_weak_subjectivity_period(state: BeaconState) -> uint64: ws_period = MIN_VALIDATOR_WITHDRAWABILITY_DELAY N = get_active_validator_count(state) t = compute_avg_active_validator_balance(state) - T = MAX_EFFECTIVE_BALANCE//10**9 + T = MAX_EFFECTIVE_BALANCE // 10**9 delta = get_validator_churn_limit(state) Delta = MAX_DEPOSITS * SLOTS_PER_EPOCH D = SAFETY_DECAY case = ( - T*(200+3*D) < t*(200+12*D) + T * (200 + 3 * D) < t * (200 + 12 * D) ) if case == 1: - arg1 = ( - N*(t*(200+12*D) - T*(200+3*D)) // (600*delta*(2*t+T)) - ) - arg2 = ( - N*(200+3*D) // (600*Delta) - ) - ws_period += max(arg1, arg2) + arg1 = ( + N * (t * (200 + 12 * D) - T * (200 + 3 * D)) // (600 * delta * (2 * t + T)) + ) + arg2 = ( + N * (200 + 3 * D) // (600 * Delta) + ) + ws_period += max(arg1, arg2) else: - ws_period += ( - 3*N*D*t // (200*Delta*(T-t)) - ) + ws_period += ( + 3 * N * D * t // (200 * Delta * (T - t)) + ) return ws_period ``` @@ -168,6 +182,8 @@ To support this mechanism, the client needs to take the state at the Weak Subjec a CLI parameter input (or fetch the state associated with the input Weak Subjectivity Checkpoint from some source). The check can be implemented in the following way: +#### `is_within_weak_subjectivity_period` + ```python def is_within_weak_subjectivity_period(store: Store, ws_state: BeaconState, ws_checkpoint: Checkpoint) -> bool: # Clients may choose to validate the input state against the input Weak Subjectivity Checkpoint From 15e48f712bab8f16e0131ebcddb98b043c0c6fba Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 27 Jan 2021 02:55:37 +0800 Subject: [PATCH 112/222] Fix Phase 1 epoch_processing and fix epoch_processing testgen --- specs/phase1/beacon-chain.md | 5 +++-- ...ates.py => test_process_sync_committee_updates.py} | 0 tests/generators/README.md | 2 +- tests/generators/epoch_processing/main.py | 11 ++++++++--- 4 files changed, 12 insertions(+), 6 deletions(-) rename tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/{test_process_final_updates.py => test_process_sync_committee_updates.py} (100%) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 2b8e460ed..274d3f5b6 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -1054,6 +1054,8 @@ def process_epoch(state: BeaconState) -> None: process_justification_and_finalization(state) process_rewards_and_penalties(state) process_registry_updates(state) + process_reveal_deadlines(state) # Phase 1 + process_challenge_deadlines(state) # Phase 1 process_slashings(state) process_eth1_data_votes_updates(state) process_effective_balances_updates(state) @@ -1061,8 +1063,7 @@ def process_epoch(state: BeaconState) -> None: process_randao_mixes_updates(state) process_historical_roots_updates(state) process_participation_record_updates(state) - # Phase 1 - process_phase_1_final_updates(state) + process_phase_1_final_updates(state) # Phase 1 ``` #### Phase 1 final updates diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_sync_committee_updates.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py rename to tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_sync_committee_updates.py diff --git a/tests/generators/README.md b/tests/generators/README.md index 9446551fb..077a8443c 100644 --- a/tests/generators/README.md +++ b/tests/generators/README.md @@ -184,7 +184,7 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin if __name__ == "__main__": gen_runner.run_generator("epoch_processing", [ - create_provider('final_updates', test_process_final_updates, 'minimal'), + create_provider('justification_and_finalization', test_process_justification_and_finalization, 'minimal'), ... ]) diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index fe8e0ee92..ede966465 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -33,16 +33,21 @@ def create_provider(fork_name: str, handler_name: str, if __name__ == "__main__": phase_0_mods = {key: 'eth2spec.test.phase0.epoch_processing.test_process_' + key for key in [ - 'final_updates', 'justification_and_finalization', - 'registry_updates', 'rewards_and_penalties', + 'registry_updates', 'slashings', + 'eth1_data_votes_updates', + 'effective_balances_updates', + 'slashings_updates', + 'randao_mixes_updates', + 'historical_roots_updates', + 'participation_record_updates', ]} phase_1_mods = {**{key: 'eth2spec.test.phase1.epoch_processing.test_process_' + key for key in [ + 'reveal_deadlines', 'challenge_deadlines', 'custody_final_updates', - 'reveal_deadlines', ]}, **phase_0_mods} # also run the previous phase 0 tests (but against phase 1 spec) gen_runner.run_generator(f"epoch_processing", [ From 1b00c10ed337eaeb49618bcdddbc5694604259c5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 27 Jan 2021 14:42:50 +0800 Subject: [PATCH 113/222] Apply @michaelsproul's feedback --- specs/lightclient/beacon-chain.md | 10 +++++----- specs/phase0/beacon-chain.md | 20 +++++++++---------- specs/phase1/beacon-chain.md | 10 +++++----- .../eth2spec/test/helpers/epoch_processing.py | 10 +++++----- ...test_process_effective_balance_updates.py} | 8 ++++---- ...tes.py => test_process_eth1_data_reset.py} | 8 ++++---- ...> test_process_historical_roots_update.py} | 6 +++--- ....py => test_process_randao_mixes_reset.py} | 6 +++--- ...tes.py => test_process_slashings_reset.py} | 6 +++--- tests/formats/epoch_processing/README.md | 10 +++++----- tests/generators/epoch_processing/main.py | 10 +++++----- 11 files changed, 52 insertions(+), 52 deletions(-) rename tests/core/pyspec/eth2spec/test/phase0/epoch_processing/{test_process_effective_balances_updates.py => test_process_effective_balance_updates.py} (93%) rename tests/core/pyspec/eth2spec/test/phase0/epoch_processing/{test_process_eth1_data_votes_updates.py => test_process_eth1_data_reset.py} (86%) rename tests/core/pyspec/eth2spec/test/phase0/epoch_processing/{test_process_historical_roots_updates.py => test_process_historical_roots_update.py} (79%) rename tests/core/pyspec/eth2spec/test/phase0/epoch_processing/{test_process_randao_mixes_updates.py => test_process_randao_mixes_reset.py} (81%) rename tests/core/pyspec/eth2spec/test/phase0/epoch_processing/{test_process_slashings_updates.py => test_process_slashings_reset.py} (82%) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index e431cf8c8..d27def8d9 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -215,11 +215,11 @@ def process_epoch(state: BeaconState) -> None: process_rewards_and_penalties(state) process_registry_updates(state) process_slashings(state) - process_eth1_data_votes_updates(state) - process_effective_balances_updates(state) - process_slashings_updates(state) - process_randao_mixes_updates(state) - process_historical_roots_updates(state) + process_eth1_data_reset(state) + process_effective_balance_updates(state) + process_slashings_reset(state) + process_randao_mixes_reset(state) + process_historical_roots_update(state) process_participation_record_updates(state) # Light client patch process_sync_committee_updates(state) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index bfa6bdd35..fcee714d9 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1255,11 +1255,11 @@ def process_epoch(state: BeaconState) -> None: process_rewards_and_penalties(state) process_registry_updates(state) process_slashings(state) - process_eth1_data_votes_updates(state) - process_effective_balances_updates(state) - process_slashings_updates(state) - process_randao_mixes_updates(state) - process_historical_roots_updates(state) + process_eth1_data_reset(state) + process_effective_balance_updates(state) + process_slashings_reset(state) + process_randao_mixes_reset(state) + process_historical_roots_update(state) process_participation_record_updates(state) ``` @@ -1569,7 +1569,7 @@ def process_slashings(state: BeaconState) -> None: #### Eth1 data votes updates ```python -def process_eth1_data_votes_updates(state: BeaconState) -> None: +def process_eth1_data_reset(state: BeaconState) -> None: next_epoch = Epoch(get_current_epoch(state) + 1) # Reset eth1 data votes if next_epoch % EPOCHS_PER_ETH1_VOTING_PERIOD == 0: @@ -1579,7 +1579,7 @@ def process_eth1_data_votes_updates(state: BeaconState) -> None: #### Effective balances updates ```python -def process_effective_balances_updates(state: BeaconState) -> None: +def process_effective_balance_updates(state: BeaconState) -> None: # Update effective balances with hysteresis for index, validator in enumerate(state.validators): balance = state.balances[index] @@ -1596,7 +1596,7 @@ def process_effective_balances_updates(state: BeaconState) -> None: #### Slashings balances updates ```python -def process_slashings_updates(state: BeaconState) -> None: +def process_slashings_reset(state: BeaconState) -> None: next_epoch = Epoch(get_current_epoch(state) + 1) # Reset slashings state.slashings[next_epoch % EPOCHS_PER_SLASHINGS_VECTOR] = Gwei(0) @@ -1605,7 +1605,7 @@ def process_slashings_updates(state: BeaconState) -> None: #### Randao mixes updates ```python -def process_randao_mixes_updates(state: BeaconState) -> None: +def process_randao_mixes_reset(state: BeaconState) -> None: current_epoch = get_current_epoch(state) next_epoch = Epoch(current_epoch + 1) # Set randao mix @@ -1614,7 +1614,7 @@ def process_randao_mixes_updates(state: BeaconState) -> None: #### Historical roots updates ```python -def process_historical_roots_updates(state: BeaconState) -> None: +def process_historical_roots_update(state: BeaconState) -> None: # Set historical root accumulator next_epoch = Epoch(get_current_epoch(state) + 1) if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0: diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 274d3f5b6..21e9751fe 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -1057,11 +1057,11 @@ def process_epoch(state: BeaconState) -> None: process_reveal_deadlines(state) # Phase 1 process_challenge_deadlines(state) # Phase 1 process_slashings(state) - process_eth1_data_votes_updates(state) - process_effective_balances_updates(state) - process_slashings_updates(state) - process_randao_mixes_updates(state) - process_historical_roots_updates(state) + process_eth1_data_reset(state) + process_effective_balance_updates(state) + process_slashings_reset(state) + process_randao_mixes_reset(state) + process_historical_roots_update(state) process_participation_record_updates(state) process_phase_1_final_updates(state) # Phase 1 ``` diff --git a/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py b/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py index ca1b9f802..b1ebb5d49 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py +++ b/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py @@ -7,11 +7,11 @@ process_calls = [ 'process_reveal_deadlines', 'process_challenge_deadlines', 'process_slashings', - 'process_eth1_data_votes_updates', - 'process_effective_balances_updates', - 'process_slashings_updates', - 'process_randao_mixes_updates', - 'process_historical_roots_updates', + 'process_eth1_data_reset', + 'process_effective_balance_updates', + 'process_slashings_reset', + 'process_randao_mixes_reset', + 'process_historical_roots_update', 'process_participation_record_updates', # LIGHTCLIENT_PATCH 'process_sync_committee_updates', diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_effective_balances_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_effective_balance_updates.py similarity index 93% rename from tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_effective_balances_updates.py rename to tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_effective_balance_updates.py index ce8f0e348..93411f657 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_effective_balances_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_effective_balance_updates.py @@ -4,8 +4,8 @@ from eth2spec.test.helpers.epoch_processing import ( ) -def run_process_effective_balances_updates(spec, state): - yield from run_epoch_processing_with(spec, state, 'process_effective_balances_updates') +def run_process_effective_balance_updates(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_effective_balance_updates') @with_all_phases @@ -13,7 +13,7 @@ def run_process_effective_balances_updates(spec, state): def test_effective_balance_hysteresis(spec, state): # Prepare state up to the final-updates. # Then overwrite the balances, we only want to focus to be on the hysteresis based changes. - run_epoch_processing_to(spec, state, 'process_effective_balances_updates') + run_epoch_processing_to(spec, state, 'process_effective_balance_updates') # Set some edge cases for balances max = spec.MAX_EFFECTIVE_BALANCE min = spec.EJECTION_BALANCE @@ -44,7 +44,7 @@ def test_effective_balance_hysteresis(spec, state): state.validators[i].effective_balance = pre_eff state.balances[i] = bal - yield from run_process_effective_balances_updates(spec, state) + yield from run_process_effective_balance_updates(spec, state) for i, (_, _, post_eff, name) in enumerate(cases): assert state.validators[i].effective_balance == post_eff, name diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_eth1_data_votes_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_eth1_data_reset.py similarity index 86% rename from tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_eth1_data_votes_updates.py rename to tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_eth1_data_reset.py index c9d7222d7..71af69f79 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_eth1_data_votes_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_eth1_data_reset.py @@ -5,8 +5,8 @@ from eth2spec.test.helpers.epoch_processing import ( from eth2spec.test.helpers.state import transition_to -def run_process_eth1_data_votes_updates(spec, state): - yield from run_epoch_processing_with(spec, state, 'process_eth1_data_votes_updates') +def run_process_eth1_data_reset(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_eth1_data_reset') @with_all_phases @@ -22,7 +22,7 @@ def test_eth1_vote_no_reset(spec, state): deposit_count=state.eth1_deposit_index, block_hash=b'\xbb' * 32)) - yield from run_process_eth1_data_votes_updates(spec, state) + yield from run_process_eth1_data_reset(spec, state) assert len(state.eth1_data_votes) == spec.SLOTS_PER_EPOCH @@ -38,6 +38,6 @@ def test_eth1_vote_reset(spec, state): deposit_count=state.eth1_deposit_index, block_hash=b'\xbb' * 32)) - yield from run_process_eth1_data_votes_updates(spec, state) + yield from run_process_eth1_data_reset(spec, state) assert len(state.eth1_data_votes) == 0 diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_historical_roots_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_historical_roots_update.py similarity index 79% rename from tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_historical_roots_updates.py rename to tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_historical_roots_update.py index d1ab9524f..02ce7ccba 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_historical_roots_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_historical_roots_update.py @@ -4,8 +4,8 @@ from eth2spec.test.helpers.epoch_processing import ( ) -def run_process_historical_roots_updates(spec, state): - yield from run_epoch_processing_with(spec, state, 'process_historical_roots_updates') +def run_process_historical_roots_update(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_historical_roots_update') @with_all_phases @@ -15,6 +15,6 @@ def test_historical_root_accumulator(spec, state): state.slot = spec.SLOTS_PER_HISTORICAL_ROOT - 1 history_len = len(state.historical_roots) - yield from run_process_historical_roots_updates(spec, state) + yield from run_process_historical_roots_update(spec, state) assert len(state.historical_roots) == history_len + 1 diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_randao_mixes_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_randao_mixes_reset.py similarity index 81% rename from tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_randao_mixes_updates.py rename to tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_randao_mixes_reset.py index b7e4a0b8f..1d35965b5 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_randao_mixes_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_randao_mixes_reset.py @@ -4,8 +4,8 @@ from eth2spec.test.helpers.epoch_processing import ( ) -def run_process_randao_mixes_updates(spec, state): - yield from run_epoch_processing_with(spec, state, 'process_randao_mixes_updates') +def run_process_randao_mixes_reset(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_randao_mixes_reset') @with_all_phases @@ -14,7 +14,7 @@ def test_updated_randao_mixes(spec, state): next_epoch = spec.get_current_epoch(state) + 1 state.randao_mixes[next_epoch % spec.EPOCHS_PER_HISTORICAL_VECTOR] = b'\x56' * 32 - yield from run_process_randao_mixes_updates(spec, state) + yield from run_process_randao_mixes_reset(spec, state) assert state.randao_mixes[next_epoch % spec.EPOCHS_PER_HISTORICAL_VECTOR] == spec.get_randao_mix( state, spec.get_current_epoch(state) diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings_reset.py similarity index 82% rename from tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings_updates.py rename to tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings_reset.py index 559b52c09..24c350b25 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings_reset.py @@ -4,8 +4,8 @@ from eth2spec.test.helpers.epoch_processing import ( ) -def run_process_slashings_updates(spec, state): - yield from run_epoch_processing_with(spec, state, 'process_slashings_updates') +def run_process_slashings_reset(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_slashings_reset') @with_all_phases @@ -15,6 +15,6 @@ def test_flush_slashings(spec, state): state.slashings[next_epoch % spec.EPOCHS_PER_SLASHINGS_VECTOR] = 100 assert state.slashings[next_epoch % spec.EPOCHS_PER_SLASHINGS_VECTOR] != 0 - yield from run_process_slashings_updates(spec, state) + yield from run_process_slashings_reset(spec, state) assert state.slashings[next_epoch % spec.EPOCHS_PER_SLASHINGS_VECTOR] == 0 diff --git a/tests/formats/epoch_processing/README.md b/tests/formats/epoch_processing/README.md index 39e8050d8..1f88b4832 100644 --- a/tests/formats/epoch_processing/README.md +++ b/tests/formats/epoch_processing/README.md @@ -43,11 +43,11 @@ Sub-transitions: - `rewards_and_penalties` - `registry_updates` - `slashings` -- `eth1_data_votes_updates` -- `effective_balances_updates` -- `slashings_updates` -- `randao_mixes_updates` -- `historical_roots_updates` +- `eth1_data_reset` +- `effective_balance_updates` +- `slashings_reset` +- `randao_mixes_reset` +- `historical_roots_update` - `participation_record_updates` The resulting state should match the expected `post` state. diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index ede966465..ea7639605 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -37,11 +37,11 @@ if __name__ == "__main__": 'rewards_and_penalties', 'registry_updates', 'slashings', - 'eth1_data_votes_updates', - 'effective_balances_updates', - 'slashings_updates', - 'randao_mixes_updates', - 'historical_roots_updates', + 'eth1_data_reset', + 'effective_balance_updates', + 'slashings_reset', + 'randao_mixes_reset', + 'historical_roots_update', 'participation_record_updates', ]} phase_1_mods = {**{key: 'eth2spec.test.phase1.epoch_processing.test_process_' + key for key in [ From 742d21e914b247e7621467cfcdb2b3339bbcd14d Mon Sep 17 00:00:00 2001 From: Aditya Asgaonkar Date: Wed, 27 Jan 2021 15:16:15 -0800 Subject: [PATCH 114/222] Updates based on review --- specs/phase0/weak-subjectivity.md | 77 ++++++++++++------------------- 1 file changed, 29 insertions(+), 48 deletions(-) diff --git a/specs/phase0/weak-subjectivity.md b/specs/phase0/weak-subjectivity.md index 75cdbec40..42034ed1a 100644 --- a/specs/phase0/weak-subjectivity.md +++ b/specs/phase0/weak-subjectivity.md @@ -14,8 +14,6 @@ - [Weak Subjectivity Checkpoint](#weak-subjectivity-checkpoint) - [Weak Subjectivity Period](#weak-subjectivity-period) - [Calculating the Weak Subjectivity Period](#calculating-the-weak-subjectivity-period) - - [`get_active_validator_count`](#get_active_validator_count) - - [`compute_avg_active_validator_balance`](#compute_avg_active_validator_balance) - [`compute_weak_subjectivity_period`](#compute_weak_subjectivity_period) - [Weak Subjectivity Sync](#weak-subjectivity-sync) - [Weak Subjectivity Sync Procedure](#weak-subjectivity-sync-procedure) @@ -77,44 +75,33 @@ a safety margin of at least `1/3 - SAFETY_DECAY/100`. ### Calculating the Weak Subjectivity Period -A detailed analysis of the calculation of the weak subjectivity period is made in [this report](https://github.com/runtimeverification/beacon-chain-verification/blob/master/weak-subjectivity/weak-subjectivity-analysis.pdf). The expressions in the report use fractions, whereas we only use uint64 arithmetic in eth2.0-specs. The expressions have been simplified to avoid computing fractions, and more details can be found [here](https://www.overleaf.com/read/wgjzjdjpvpsd). +A detailed analysis of the calculation of the weak subjectivity period is made in [this report](https://github.com/runtimeverification/beacon-chain-verification/blob/master/weak-subjectivity/weak-subjectivity-analysis.pdf). -#### `get_active_validator_count` +*Note*: The expressions in the report use fractions, whereas eth2.0-specs uses only `uint64` arithmetic. The expressions have been simplified to avoid computing fractions, and more details can be found [here](https://www.overleaf.com/read/wgjzjdjpvpsd). -```python -def get_active_validator_count(state: BeaconState) -> uint64: - active_validator_count = len(get_active_validator_indices(state, get_current_epoch(state))) - return active_validator_count -``` - -#### `compute_avg_active_validator_balance` - -```python -def compute_avg_active_validator_balance(state: BeaconState) -> Ether: - total_active_balance = get_total_active_balance(state) - active_validator_count = get_active_validator_count(state) - avg_active_validator_balance_gwei = total_active_balance // active_validator_count - avg_active_validator_balance_eth = avg_active_validator_balance_gwei // ETH_TO_GWEI - return avg_active_validator_balance_eth -``` +*Note*: The calculations here use `Ether` instead of `Gwei`, because the large magitude of balances in `Gwei` can cause an overflow while computing using `uint64` arithmetic operations. Using `Ether` reduces the magintude of the multiplicative factors by an order of `ETH_TO_GWEI` (`= 10**9`) and avoid the scope for overflows in `uint64`. #### `compute_weak_subjectivity_period` ```python def compute_weak_subjectivity_period(state: BeaconState) -> uint64: + """ + Returns the weak subjectivity period for the current ``state``. + This computation takes into account the effect of: + - validator set churn (bounded by ``get_validator_churn_limit()`` per epoch), and + - validator balance top-ups (bounded by ``MAX_DEPOSITS * SLOTS_PER_EPOCH`` per epoch). + A detailed calculation can be found at: + https://github.com/runtimeverification/beacon-chain-verification/blob/master/weak-subjectivity/weak-subjectivity-analysis.pdf + """ ws_period = MIN_VALIDATOR_WITHDRAWABILITY_DELAY - N = get_active_validator_count(state) - t = compute_avg_active_validator_balance(state) - T = MAX_EFFECTIVE_BALANCE // 10**9 + N = len(get_active_validator_indices(state, get_current_epoch(state))) + t = get_total_active_balance(state) // N // ETH_TO_GWEI + T = MAX_EFFECTIVE_BALANCE // ETH_TO_GWEI delta = get_validator_churn_limit(state) Delta = MAX_DEPOSITS * SLOTS_PER_EPOCH D = SAFETY_DECAY - case = ( - T * (200 + 3 * D) < t * (200 + 12 * D) - ) - - if case == 1: + if T * (200 + 3 * D) < t * (200 + 12 * D): arg1 = ( N * (t * (200 + 12 * D) - T * (200 + 3 * D)) // (600 * delta * (2 * t + T)) ) @@ -130,28 +117,22 @@ def compute_weak_subjectivity_period(state: BeaconState) -> uint64: return ws_period ``` -A brief reference for what these values look like in practice: +A brief reference for what these values look like in practice ([reference script](https://gist.github.com/adiasg/3aceab409b36aa9a9d9156c1baa3c248)): -| Safety Decay | Validator Count | Average Active Validator Balance | Weak Subjectivity Period | +| Safety Decay | Avg. Val. Balance (ETH) | Val. Count | Weak Sub. Period (Epochs) | | ---- | ---- | ---- | ---- | -| 10 | 8192 | 28 | 318 | -| 10 | 8192 | 32 | 358 | -| 10 | 16384 | 28 | 380 | -| 10 | 16384 | 32 | 460 | -| 10 | 32768 | 28 | 504 | -| 10 | 32768 | 32 | 665 | -| 20 | 8192 | 28 | 411 | -| 20 | 8192 | 32 | 460 | -| 20 | 16384 | 28 | 566 | -| 20 | 16384 | 32 | 665 | -| 20 | 32768 | 28 | 876 | -| 20 | 32768 | 32 | 1075 | -| 33 | 8192 | 28 | 532 | -| 33 | 8192 | 32 | 593 | -| 33 | 16384 | 28 | 808 | -| 33 | 16384 | 32 | 931 | -| 33 | 32768 | 28 | 1360 | -| 33 | 32768 | 32 | 1607 | +| 10 | 28 | 32768 | 504 | +| 10 | 28 | 65536 | 752 | +| 10 | 28 | 131072 | 1248 | +| 10 | 28 | 262144 | 2241 | +| 10 | 28 | 524288 | 2241 | +| 10 | 28 | 1048576 | 2241 | +| 10 | 32 | 32768 | 665 | +| 10 | 32 | 65536 | 1075 | +| 10 | 32 | 131072 | 1894 | +| 10 | 32 | 262144 | 3532 | +| 10 | 32 | 524288 | 3532 | +| 10 | 32 | 1048576 | 3532 | ## Weak Subjectivity Sync From a28f52729dde6f79a87c87eddcfd1f82fbc5e014 Mon Sep 17 00:00:00 2001 From: Aditya Asgaonkar Date: Wed, 27 Jan 2021 15:58:19 -0800 Subject: [PATCH 115/222] Rename variables for clarity --- specs/phase0/weak-subjectivity.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/phase0/weak-subjectivity.md b/specs/phase0/weak-subjectivity.md index 42034ed1a..f91a04d7e 100644 --- a/specs/phase0/weak-subjectivity.md +++ b/specs/phase0/weak-subjectivity.md @@ -102,13 +102,13 @@ def compute_weak_subjectivity_period(state: BeaconState) -> uint64: D = SAFETY_DECAY if T * (200 + 3 * D) < t * (200 + 12 * D): - arg1 = ( + epochs_for_validator_set_churn = ( N * (t * (200 + 12 * D) - T * (200 + 3 * D)) // (600 * delta * (2 * t + T)) ) - arg2 = ( + epochs_for_balance_top_ups = ( N * (200 + 3 * D) // (600 * Delta) ) - ws_period += max(arg1, arg2) + ws_period += max(epochs_for_validator_set_churn, epochs_for_balance_top_ups) else: ws_period += ( 3 * N * D * t // (200 * Delta * (T - t)) From 007a6f0eccfc3a6a90555d222f08d3b77a0d243d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 29 Jan 2021 10:37:19 -0700 Subject: [PATCH 116/222] a couple of hf1 notes --- specs/lightclient/beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index bbce43ad5..224be971d 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -473,8 +473,8 @@ def process_sync_committee(state: BeaconState, body: BeaconBlockBody) -> None: ```python def process_epoch(state: BeaconState) -> None: - process_justification_and_finalization(state) - process_rewards_and_penalties(state) + process_justification_and_finalization(state) # [Updated in HF1] + process_rewards_and_penalties(state) # [Updated in HF1] process_registry_updates(state) process_slashings(state) process_eth1_data_reset(state) From 6ce4b1b0e795aa5233c57cac97ffbd20a112fe2c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 29 Jan 2021 17:18:49 +0800 Subject: [PATCH 117/222] Fix tests --- .../eth2spec/test/helpers/epoch_processing.py | 47 +++++++++++-------- ...st_process_participation_record_updates.py | 4 +- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py b/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py index b1ebb5d49..40c7cb8b0 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py +++ b/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py @@ -1,23 +1,30 @@ -process_calls = [ - # PHASE0 - 'process_justification_and_finalization', - 'process_rewards_and_penalties', - 'process_registry_updates', - 'process_reveal_deadlines', - 'process_challenge_deadlines', - 'process_slashings', - 'process_eth1_data_reset', - 'process_effective_balance_updates', - 'process_slashings_reset', - 'process_randao_mixes_reset', - 'process_historical_roots_update', - 'process_participation_record_updates', - # LIGHTCLIENT_PATCH - 'process_sync_committee_updates', - # PHASE1 - 'process_phase_1_final_updates', -] +from eth2spec.test.context import is_post_lightclient_patch + + +def get_process_calls(spec): + return [ + # PHASE0 + 'process_justification_and_finalization', + 'process_rewards_and_penalties', + 'process_registry_updates', + 'process_reveal_deadlines', + 'process_challenge_deadlines', + 'process_slashings', + 'process_eth1_data_reset', + 'process_effective_balance_updates', + 'process_slashings_reset', + 'process_randao_mixes_reset', + 'process_historical_roots_update', + # LIGHTCLIENT_PATCH replaced `process_participation_record_updates` with + # `process_epoch_participation_updates` + 'process_epoch_participation_updates' if is_post_lightclient_patch(spec) else ( + 'process_participation_record_updates' + ), + 'process_sync_committee_updates', + # PHASE1 + 'process_phase_1_final_updates', + ] def run_epoch_processing_to(spec, state, process_name: str): @@ -34,7 +41,7 @@ def run_epoch_processing_to(spec, state, process_name: str): spec.process_slot(state) # process components of epoch transition before final-updates - for name in process_calls: + for name in get_process_calls(spec): if name == process_name: break # only run when present. Later phases introduce more to the epoch-processing. diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_participation_record_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_participation_record_updates.py index f5e1513e3..978ef5739 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_participation_record_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_participation_record_updates.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.context import PHASE0, spec_state_test, with_phases from eth2spec.test.helpers.epoch_processing import ( run_epoch_processing_with ) @@ -8,7 +8,7 @@ def run_process_participation_record_updates(spec, state): yield from run_epoch_processing_with(spec, state, 'process_participation_record_updates') -@with_all_phases +@with_phases([PHASE0]) @spec_state_test def test_updated_participation_record(spec, state): state.previous_epoch_attestations = [spec.PendingAttestation(proposer_index=100)] From ad01c85ff6497d303b176b1cc3a5a041f58ed6ad Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 1 Feb 2021 07:06:29 -0700 Subject: [PATCH 118/222] minor reorder to process_epoch calls --- specs/lightclient/beacon-chain.md | 2 +- tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 224be971d..bb00a5c59 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -483,8 +483,8 @@ def process_epoch(state: BeaconState) -> None: process_randao_mixes_reset(state) process_historical_roots_update(state) # [Added in HF1] - process_sync_committee_updates(state) process_participation_flag_updates(state) + process_sync_committee_updates(state) # [Removed in HF1] -- process_participation_record_updates(state) ``` diff --git a/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py b/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py index 40c7cb8b0..52479cfeb 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py +++ b/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py @@ -16,9 +16,8 @@ def get_process_calls(spec): 'process_slashings_reset', 'process_randao_mixes_reset', 'process_historical_roots_update', - # LIGHTCLIENT_PATCH replaced `process_participation_record_updates` with - # `process_epoch_participation_updates` - 'process_epoch_participation_updates' if is_post_lightclient_patch(spec) else ( + # HF1 replaced `process_participation_record_updates` with `process_participation_flag_updates` + 'process_participation_flag_updates' if is_post_lightclient_patch(spec) else ( 'process_participation_record_updates' ), 'process_sync_committee_updates', From b029c75d88f911902228b92668b7d0d8f17a894e Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 1 Feb 2021 07:52:06 -0700 Subject: [PATCH 119/222] must be correct target to get correct head --- specs/lightclient/beacon-chain.md | 2 +- .../pyspec/eth2spec/test/helpers/rewards.py | 19 +++++++++++++------ .../test/phase0/rewards/test_random.py | 6 ++++++ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index bb00a5c59..bcb90f756 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -374,7 +374,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: # Participation flags participation_flags = [] - if is_matching_head and state.slot <= data.slot + MIN_ATTESTATION_INCLUSION_DELAY: + if is_matching_head and is_matching_target and state.slot <= data.slot + MIN_ATTESTATION_INCLUSION_DELAY: participation_flags.append(TIMELY_HEAD_FLAG) if is_matching_source and state.slot <= data.slot + integer_squareroot(SLOTS_PER_EPOCH): participation_flags.append(TIMELY_SOURCE_FLAG) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 770519384..81eb1f955 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -517,11 +517,18 @@ def run_test_full_random(spec, state, rng=Random(8020)): pending_attestation.inclusion_delay = rng.randint(1, spec.SLOTS_PER_EPOCH) else: for index in range(len(state.validators)): - # ~1/3 have bad target - state.previous_epoch_participation[index][spec.TIMELY_TARGET_FLAG] = rng.randint(0, 2) != 0 - # ~1/3 have bad head - state.previous_epoch_participation[index][spec.TIMELY_HEAD_FLAG] = rng.randint(0, 2) != 0 - # ~50% participation - state.previous_epoch_participation[index][spec.TIMELY_SOURCE_FLAG] = rng.choice([True, False]) + # ~1/3 have bad head or bad target or not timely enough + is_timely_correct_head = rng.randint(0, 2) != 0 + state.previous_epoch_participation[index][spec.TIMELY_HEAD_FLAG] = is_timely_correct_head + if is_timely_correct_head: + # If timely head, then must be timely target + state.previous_epoch_participation[index][spec.TIMELY_TARGET_FLAG] = True + # If timely head, then must be timely source + state.previous_epoch_participation[index][spec.TIMELY_SOURCE_FLAG] = True + else: + # ~50% of remaining have bad target or not timely enough + state.previous_epoch_participation[index][spec.TIMELY_TARGET_FLAG] = rng.choice([True, False]) + # ~50% of remaining have bad source or not timely enough + state.previous_epoch_participation[index][spec.TIMELY_SOURCE_FLAG] = rng.choice([True, False]) yield from run_deltas(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase0/rewards/test_random.py b/tests/core/pyspec/eth2spec/test/phase0/rewards/test_random.py index 83c7f7905..ae44c6640 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/rewards/test_random.py +++ b/tests/core/pyspec/eth2spec/test/phase0/rewards/test_random.py @@ -29,6 +29,12 @@ def test_full_random_2(spec, state): yield from rewards_helpers.run_test_full_random(spec, state, rng=Random(3030)) +@with_all_phases +@spec_state_test +def test_full_random_3(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state, rng=Random(4040)) + + @with_all_phases @with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test From 1ba491711947b4f908bd68b0639584b11c20442c Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 1 Feb 2021 08:35:58 -0700 Subject: [PATCH 120/222] add process_attestation tests to cover various timing and correctness scenarios --- .../eth2spec/test/helpers/attestations.py | 7 +- .../test_process_attestation.py | 159 +++++++++++++++++- 2 files changed, 160 insertions(+), 6 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index cf62482da..571e19fef 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -44,11 +44,8 @@ def run_attestation_processing(spec, state, attestation, valid=True): else: assert len(state.previous_epoch_attestations) == previous_epoch_count + 1 else: - for index in spec.get_attesting_indices(state, attestation.data, attestation.aggregation_bits): - if attestation.data.target.epoch == spec.get_current_epoch(state): - assert state.current_epoch_participation[index][spec.TIMELY_TARGET_FLAG] - else: - assert state.previous_epoch_participation[index][spec.TIMELY_TARGET_FLAG] + # After accounting reform, there are cases when processing an attestation does not result in any flag updates + pass # yield post-state yield 'post', state diff --git a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attestation.py index b31cf167c..71000763b 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attestation.py @@ -2,10 +2,13 @@ from eth2spec.test.context import ( spec_state_test, always_bls, never_bls, with_all_phases, + with_all_phases_except, spec_test, low_balances, with_custom_state, - single_phase) + single_phase, + PHASE1, +) from eth2spec.test.helpers.attestations import ( run_attestation_processing, get_valid_attestation, @@ -329,3 +332,157 @@ def test_too_few_aggregation_bits(spec, state): attestation.aggregation_bits = attestation.aggregation_bits[:-1] yield from run_attestation_processing(spec, state, attestation, False) + + +# +# Full correct atttestation contents at different slot inclusions +# + +@with_all_phases +@spec_state_test +def test_correct_min_inclusion_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=True) + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) + + yield from run_attestation_processing(spec, state, attestation) + + +@with_all_phases +@spec_state_test +def test_correct_sqrt_epoch_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=True, on_time=False) + next_slots(spec, state, spec.integer_squareroot(spec.SLOTS_PER_EPOCH)) + + yield from run_attestation_processing(spec, state, attestation) + + +@with_all_phases +@spec_state_test +def test_correct_epoch_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=True, on_time=False) + next_slots(spec, state, spec.SLOTS_PER_EPOCH) + + yield from run_attestation_processing(spec, state, attestation) + + +# +# Incorrect head but correct source/target at different slot inclusions +# + +@with_all_phases_except([PHASE1]) +@spec_state_test +def test_incorrect_head_min_inclusion_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=False) + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) + + attestation.data.beacon_block_root = b'\x42' * 32 + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation) + + +@with_all_phases +@spec_state_test +def test_incorrect_head_sqrt_epoch_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=False, on_time=False) + next_slots(spec, state, spec.integer_squareroot(spec.SLOTS_PER_EPOCH)) + + attestation.data.beacon_block_root = b'\x42' * 32 + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation) + + +@with_all_phases +@spec_state_test +def test_incorrect_head_epoch_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=False, on_time=False) + next_slots(spec, state, spec.SLOTS_PER_EPOCH) + + attestation.data.beacon_block_root = b'\x42' * 32 + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation) + + +# +# Incorrect head and target but correct source at different slot inclusions +# + +@with_all_phases_except([PHASE1]) +@spec_state_test +def test_incorrect_head_and_target_min_inclusion_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=False) + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) + + attestation.data.beacon_block_root = b'\x42' * 32 + attestation.data.target.root = b'\x42' * 32 + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation) + + +@with_all_phases +@spec_state_test +def test_incorrect_head_and_target_sqrt_epoch_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=False, on_time=False) + next_slots(spec, state, spec.integer_squareroot(spec.SLOTS_PER_EPOCH)) + + attestation.data.beacon_block_root = b'\x42' * 32 + attestation.data.target.root = b'\x42' * 32 + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation) + + +@with_all_phases +@spec_state_test +def test_incorrect_head_and_target_epoch_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=False, on_time=False) + next_slots(spec, state, spec.SLOTS_PER_EPOCH) + + attestation.data.beacon_block_root = b'\x42' * 32 + attestation.data.target.root = b'\x42' * 32 + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation) + + +# +# Correct head and source but incorrect target at different slot inclusions +# + +@with_all_phases_except([PHASE1]) +@spec_state_test +def test_incorrect_target_min_inclusion_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=False) + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) + + attestation.data.target.root = b'\x42' * 32 + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation) + + +@with_all_phases +@spec_state_test +def test_incorrect_target_sqrt_epoch_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=False, on_time=False) + next_slots(spec, state, spec.integer_squareroot(spec.SLOTS_PER_EPOCH)) + + attestation.data.target.root = b'\x42' * 32 + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation) + + +@with_all_phases +@spec_state_test +def test_incorrect_target_epoch_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=False, on_time=False) + next_slots(spec, state, spec.SLOTS_PER_EPOCH) + + attestation.data.target.root = b'\x42' * 32 + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation) From 3677073812c02c2ed613e14d874e444b97a89210 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 1 Feb 2021 21:46:27 +0100 Subject: [PATCH 121/222] bitvector[8] -> uint8, for efficient packing in flags merkle tree --- specs/lightclient/beacon-chain.md | 53 +++++++++++++------ specs/lightclient/lightclient-fork.md | 4 +- .../pyspec/eth2spec/test/helpers/rewards.py | 29 ++++++---- ..._process_justification_and_finalization.py | 6 +-- .../test/phase0/sanity/test_blocks.py | 2 +- 5 files changed, 63 insertions(+), 31 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index bcb90f756..1ac6d2a2c 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -55,15 +55,26 @@ This is a patch implementing the first hard fork to the beacon chain, tentativel and [TODO] reducing the cost of processing chains that have very little or zero participation for a long span of epochs * Fork choice rule changes to address weaknesses recently discovered in the existing fork choice +## Custom types + +| Name | SSZ equivalent | Description | +| - | - | - | +| `ValidatorFlags` | `uint8` | Bitflags to track validator actions with | + ## Constants -### Participation flags +### Validator action flags + +This is formatted as an enum, with values `2**i` that can be combined as bit-flags. +The `0` value is reserved as default. Remaining bits in `ValidatorFlags` may be used in future hardforks. + +**Note**: unlike Phase0, a `TIMELY_TARGET_FLAG` does not imply a `TIMELY_SOURCE_FLAG`. | Name | Value | | - | - | -| `TIMELY_HEAD_FLAG` | `0` | -| `TIMELY_SOURCE_FLAG` | `1` | -| `TIMELY_TARGET_FLAG` | `2` | +| `TIMELY_HEAD_FLAG` | `ValidatorFlags(2**0)` (= 1) | +| `TIMELY_SOURCE_FLAG` | `ValidatorFlags(2**1)` (= 2) | +| `TIMELY_TARGET_FLAG` | `ValidatorFlags(2**2)` (= 4) | ### Participation rewards @@ -80,7 +91,6 @@ The reward fractions add up to 7/8, leaving the remaining 1/8 for proposer rewar | Name | Value | | - | - | -| `PARTICIPATION_FLAGS_LENGTH` | `8` | | `G2_POINT_AT_INFINITY` | `BLSSignature(b'\xc0' + b'\x00' * 95)` | ## Configuration @@ -146,8 +156,8 @@ class BeaconState(Container): # Slashings slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR] # Per-epoch sums of slashed effective balances # Participation - previous_epoch_participation: List[Bitvector[PARTICIPATION_FLAGS_LENGTH], VALIDATOR_REGISTRY_LIMIT] - current_epoch_participation: List[Bitvector[PARTICIPATION_FLAGS_LENGTH], VALIDATOR_REGISTRY_LIMIT] + previous_epoch_participation: List[ValidatorFlags, VALIDATOR_REGISTRY_LIMIT] + current_epoch_participation: List[ValidatorFlags, VALIDATOR_REGISTRY_LIMIT] # Finality justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH] # Bit set for every recent justified epoch previous_justified_checkpoint: Checkpoint @@ -197,7 +207,15 @@ def get_flags_and_numerators() -> Sequence[Tuple[int, int]]: ) ``` +```python +def add_flags(flags: ValidatorFlags, add: ValidatorFlags) -> ValidatorFlags: + return flags | add +``` +```python +def has_flags(flags: ValidatorFlags, has: ValidatorFlags) -> bool: + return flags & has == has +``` ### Beacon state accessors @@ -257,7 +275,10 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: #### `get_unslashed_participating_indices` ```python -def get_unslashed_participating_indices(state: BeaconState, flag: uint8, epoch: Epoch) -> Set[ValidatorIndex]: +def get_unslashed_participating_indices(state: BeaconState, flags: ValidatorFlags, epoch: Epoch) -> Set[ValidatorIndex]: + """ + Retrieves the active validator indices of the given epoch, who are not slashed, and have all of the given flags. + """ assert epoch in (get_previous_epoch(state), get_current_epoch(state)) if epoch == get_current_epoch(state): epoch_participation = state.current_epoch_participation @@ -265,7 +286,7 @@ def get_unslashed_participating_indices(state: BeaconState, flag: uint8, epoch: epoch_participation = state.previous_epoch_participation participating_indices = [ index for index in get_active_validator_indices(state, epoch) - if epoch_participation[index][flag] + if has_flags(epoch_participation[index], flags) ] return set(filter(lambda index: not state.validators[index].slashed, participating_indices)) ``` @@ -273,7 +294,9 @@ def get_unslashed_participating_indices(state: BeaconState, flag: uint8, epoch: #### `get_flag_deltas` ```python -def get_flag_deltas(state: BeaconState, flag: uint8, numerator: uint64) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: +def get_flag_deltas(state: BeaconState, + flag: ValidatorFlags, + numerator: uint64) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: """ Computes the rewards and penalties associated with a particular duty, by scanning through the participation flags to determine who participated and who did not and assigning them the appropriate rewards and penalties. @@ -385,8 +408,8 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: proposer_reward_numerator = 0 for index in get_attesting_indices(state, data, attestation.aggregation_bits): for flag, numerator in get_flags_and_numerators(): - if flag in participation_flags and not epoch_participation[index][flag]: - epoch_participation[index][flag] = True + if flag in participation_flags and not has_flags(epoch_participation[index], flag): + epoch_participation[index] = add_flags(epoch_participation[index], flag) proposer_reward_numerator += get_base_reward(state, index) * numerator # Reward proposer @@ -432,8 +455,8 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: state.validators.append(get_validator_from_deposit(state, deposit)) state.balances.append(amount) # [Added in hf-1] Initialize empty participation flags for new validator - state.previous_epoch_participation.append(Bitvector[PARTICIPATION_FLAGS_LENGTH]()) - state.current_epoch_participation.append(Bitvector[PARTICIPATION_FLAGS_LENGTH]()) + state.previous_epoch_participation.append(ValidatorFlags(0)) + state.current_epoch_participation.append(ValidatorFlags(0)) else: # Increase balance by deposit amount index = ValidatorIndex(validator_pubkeys.index(pubkey)) @@ -572,5 +595,5 @@ def process_participation_flag_updates(state: BeaconState) -> None: Call to ``process_participation_flag_updates`` added to ``process_epoch`` in HF1 """ state.previous_epoch_participation = state.current_epoch_participation - state.current_epoch_participation = [Bitvector[PARTICIPATION_FLAGS_LENGTH]() for _ in range(len(state.validators))] + state.current_epoch_participation = [ValidatorFlags(0) for _ in range(len(state.validators))] ``` diff --git a/specs/lightclient/lightclient-fork.md b/specs/lightclient/lightclient-fork.md index a10e8c5f6..f5f3d1b7b 100644 --- a/specs/lightclient/lightclient-fork.md +++ b/specs/lightclient/lightclient-fork.md @@ -67,8 +67,8 @@ def upgrade_to_lightclient_patch(pre: phase0.BeaconState) -> BeaconState: # Slashings slashings=pre.slashings, # Attestations - previous_epoch_participation=[Bitvector[PARTICIPATION_FLAGS_LENGTH]() for _ in range(len(pre.validators))], - current_epoch_participation=[Bitvector[PARTICIPATION_FLAGS_LENGTH]() for _ in range(len(pre.validators))], + previous_epoch_participation=[ValidatorFlags(0) for _ in range(len(pre.validators))], + current_epoch_participation=[ValidatorFlags(0) for _ in range(len(pre.validators))], # Finality justification_bits=pre.justification_bits, previous_justified_checkpoint=pre.previous_justified_checkpoint, diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 81eb1f955..feaac018b 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -6,7 +6,7 @@ from eth2spec.test.context import is_post_lightclient_patch from eth2spec.test.helpers.attestations import cached_prepare_state_with_attestations from eth2spec.test.helpers.deposits import mock_deposit from eth2spec.test.helpers.state import next_epoch -from eth2spec.utils.ssz.ssz_typing import Container, uint64, List, Bitvector +from eth2spec.utils.ssz.ssz_typing import Container, uint64, List class Deltas(Container): @@ -314,7 +314,7 @@ def run_test_full_but_partial_participation(spec, state, rng=Random(5522)): else: for index in range(len(state.validators)): if rng.choice([True, False]): - state.previous_epoch_participation[index] = Bitvector[spec.PARTICIPATION_FLAGS_LENGTH]() + state.previous_epoch_participation[index] = spec.ValidatorFlags(0) yield from run_deltas(spec, state) @@ -328,7 +328,7 @@ def run_test_partial(spec, state, fraction_filled): state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] else: for index in range(int(len(state.validators) * fraction_filled)): - state.previous_epoch_participation[index] = Bitvector[spec.PARTICIPATION_FLAGS_LENGTH]() + state.previous_epoch_participation[index] = spec.ValidatorFlags(0) yield from run_deltas(spec, state) @@ -394,7 +394,7 @@ def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state): else: index = 0 state.validators[index].effective_balance = 1 - state.previous_epoch_participation[index] = Bitvector[spec.PARTICIPATION_FLAGS_LENGTH]() + state.previous_epoch_participation[index] = spec.ValidatorFlags(0) yield from run_deltas(spec, state) @@ -519,16 +519,25 @@ def run_test_full_random(spec, state, rng=Random(8020)): for index in range(len(state.validators)): # ~1/3 have bad head or bad target or not timely enough is_timely_correct_head = rng.randint(0, 2) != 0 - state.previous_epoch_participation[index][spec.TIMELY_HEAD_FLAG] = is_timely_correct_head + flags = state.previous_epoch_participation[index] + + def set_flag(f, v): + nonlocal flags + if v: + flags |= f + else: + flags &= 0xff ^ f + + set_flag(spec.TIMELY_HEAD_FLAG, is_timely_correct_head) if is_timely_correct_head: # If timely head, then must be timely target - state.previous_epoch_participation[index][spec.TIMELY_TARGET_FLAG] = True + set_flag(spec.TIMELY_TARGET_FLAG, True) # If timely head, then must be timely source - state.previous_epoch_participation[index][spec.TIMELY_SOURCE_FLAG] = True + set_flag(spec.TIMELY_SOURCE_FLAG, True) else: # ~50% of remaining have bad target or not timely enough - state.previous_epoch_participation[index][spec.TIMELY_TARGET_FLAG] = rng.choice([True, False]) + set_flag(spec.TIMELY_TARGET_FLAG, rng.choice([True, False])) # ~50% of remaining have bad source or not timely enough - state.previous_epoch_participation[index][spec.TIMELY_SOURCE_FLAG] = rng.choice([True, False]) - + set_flag(spec.TIMELY_SOURCE_FLAG, rng.choice([True, False])) + state.previous_epoch_participation[index] = flags yield from run_deltas(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py index 0d1741efb..89783f987 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py @@ -78,10 +78,10 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support else: for i, index in enumerate(committee): if aggregation_bits[i]: - epoch_participation[index][spec.TIMELY_HEAD_FLAG] = True - epoch_participation[index][spec.TIMELY_SOURCE_FLAG] = True + epoch_participation[index] |= spec.TIMELY_HEAD_FLAG + epoch_participation[index] |= spec.TIMELY_SOURCE_FLAG if not messed_up_target: - epoch_participation[index][spec.TIMELY_TARGET_FLAG] = True + epoch_participation[index] |= spec.TIMELY_TARGET_FLAG def get_checkpoints(spec, epoch): diff --git a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py index 9cc8ab721..1834b290f 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py @@ -806,7 +806,7 @@ def test_attestation(spec, state): assert spec.hash_tree_root(state.previous_epoch_attestations) == pre_current_attestations_root else: for index in range(len(state.validators)): - assert state.current_epoch_participation[index] == spec.Bitvector[spec.PARTICIPATION_FLAGS_LENGTH]() + assert state.current_epoch_participation[index] == 0 assert spec.hash_tree_root(state.previous_epoch_participation) == pre_current_epoch_participation_root From e865670111c2fb89b7031f3ea6d67bf2426b03a9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 1 Feb 2021 21:47:00 +0100 Subject: [PATCH 122/222] add missing decorators for testruns in no-bls mode --- .../block_processing/test_process_sync_committee.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index a8be1af63..430c59b10 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -18,6 +18,7 @@ from eth2spec.test.context import ( with_all_phases_except, with_configs, spec_state_test, + always_bls, ) from eth2spec.utils.hash_function import hash @@ -196,6 +197,7 @@ def test_sync_committee_rewards_duplicate_committee(spec, state): @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test +@always_bls def test_invalid_signature_past_block(spec, state): committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) @@ -237,6 +239,7 @@ def test_invalid_signature_past_block(spec, state): @with_all_phases_except([PHASE0, PHASE1]) @with_configs([MINIMAL], reason="to produce different committee sets") @spec_state_test +@always_bls def test_invalid_signature_previous_committee(spec, state): # NOTE: the `state` provided is at genesis and the process to select # sync committees currently returns the same committee for the first and second From 71c28e67a13469ffc0452687f4ce6fad99a73107 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 1 Feb 2021 21:48:55 +0100 Subject: [PATCH 123/222] toc update --- specs/lightclient/beacon-chain.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 1ac6d2a2c..179aec6d1 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -7,8 +7,9 @@ - [Introduction](#introduction) +- [Custom types](#custom-types) - [Constants](#constants) - - [Participation flags](#participation-flags) + - [Validator action flags](#validator-action-flags) - [Participation rewards](#participation-rewards) - [Misc](#misc) - [Configuration](#configuration) From b4ba6c57de0bb2138379a2e1761d5ab2bc3ed90e Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 1 Feb 2021 22:02:12 +0100 Subject: [PATCH 124/222] linter: first tuple element type is ValidatorFlags, not just int --- specs/lightclient/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 179aec6d1..9f0572740 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -200,7 +200,7 @@ def eth2_fast_aggregate_verify(pubkeys: Sequence[BLSPubkey], message: Bytes32, s #### `flags_and_numerators` ```python -def get_flags_and_numerators() -> Sequence[Tuple[int, int]]: +def get_flags_and_numerators() -> Sequence[Tuple[ValidatorFlags, int]]: return ( (TIMELY_HEAD_FLAG, TIMELY_HEAD_NUMERATOR), (TIMELY_SOURCE_FLAG, TIMELY_SOURCE_NUMERATOR), From 1c1ba5cba291e725972601c62e97649565d1f299 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 2 Feb 2021 12:35:00 -0700 Subject: [PATCH 125/222] minor PR feedback --- specs/lightclient/beacon-chain.md | 7 ++- .../test_process_attestation.py | 55 +++++++++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 9f0572740..3508fe991 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -69,7 +69,8 @@ This is a patch implementing the first hard fork to the beacon chain, tentativel This is formatted as an enum, with values `2**i` that can be combined as bit-flags. The `0` value is reserved as default. Remaining bits in `ValidatorFlags` may be used in future hardforks. -**Note**: unlike Phase0, a `TIMELY_TARGET_FLAG` does not imply a `TIMELY_SOURCE_FLAG`. +**Note**: Unlike Phase0, a `TIMELY_TARGET_FLAG` does not necessarily imply a `TIMELY_SOURCE_FLAG` +due to the varying slot delay requirements of each. | Name | Value | | - | - | @@ -278,7 +279,7 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: ```python def get_unslashed_participating_indices(state: BeaconState, flags: ValidatorFlags, epoch: Epoch) -> Set[ValidatorIndex]: """ - Retrieves the active validator indices of the given epoch, who are not slashed, and have all of the given flags. + Retrieve the active validator indices of the given epoch, which are not slashed, and have all of the given flags. """ assert epoch in (get_previous_epoch(state), get_current_epoch(state)) if epoch == get_current_epoch(state): @@ -299,7 +300,7 @@ def get_flag_deltas(state: BeaconState, flag: ValidatorFlags, numerator: uint64) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: """ - Computes the rewards and penalties associated with a particular duty, by scanning through the participation + Compute the rewards and penalties associated with a particular duty, by scanning through the participation flags to determine who participated and who did not and assigning them the appropriate rewards and penalties. """ rewards = [Gwei(0)] * len(state.validators) diff --git a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attestation.py index 71000763b..99a82879d 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attestation.py @@ -365,6 +365,17 @@ def test_correct_epoch_delay(spec, state): yield from run_attestation_processing(spec, state, attestation) +@with_all_phases +@spec_state_test +def test_correct_after_epoch_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=True, on_time=False) + + # increment past latest inclusion slot + next_slots(spec, state, spec.SLOTS_PER_EPOCH + 1) + + yield from run_attestation_processing(spec, state, attestation, False) + + # # Incorrect head but correct source/target at different slot inclusions # @@ -405,10 +416,27 @@ def test_incorrect_head_epoch_delay(spec, state): yield from run_attestation_processing(spec, state, attestation) +@with_all_phases +@spec_state_test +def test_incorrect_head_after_epoch_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=False, on_time=False) + + # increment past latest inclusion slot + next_slots(spec, state, spec.SLOTS_PER_EPOCH + 1) + + attestation.data.beacon_block_root = b'\x42' * 32 + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation, False) + + # # Incorrect head and target but correct source at different slot inclusions # +# Note: current phase 1 spec checks +# `assert data.beacon_block_root == get_block_root_at_slot(state, compute_previous_slot(state.slot))` +# so this test can't pass that until phase 1 refactor is merged @with_all_phases_except([PHASE1]) @spec_state_test def test_incorrect_head_and_target_min_inclusion_delay(spec, state): @@ -448,6 +476,20 @@ def test_incorrect_head_and_target_epoch_delay(spec, state): yield from run_attestation_processing(spec, state, attestation) +@with_all_phases +@spec_state_test +def test_incorrect_head_and_target_after_epoch_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=False, on_time=False) + # increment past latest inclusion slot + next_slots(spec, state, spec.SLOTS_PER_EPOCH + 1) + + attestation.data.beacon_block_root = b'\x42' * 32 + attestation.data.target.root = b'\x42' * 32 + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation, False) + + # # Correct head and source but incorrect target at different slot inclusions # @@ -486,3 +528,16 @@ def test_incorrect_target_epoch_delay(spec, state): sign_attestation(spec, state, attestation) yield from run_attestation_processing(spec, state, attestation) + + +@with_all_phases +@spec_state_test +def test_incorrect_target_after_epoch_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=False, on_time=False) + # increment past latest inclusion slot + next_slots(spec, state, spec.SLOTS_PER_EPOCH + 1) + + attestation.data.target.root = b'\x42' * 32 + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation, False) From 8b217d9277fd87755cca256e0836999793be297d Mon Sep 17 00:00:00 2001 From: Aditya Asgaonkar Date: Tue, 2 Feb 2021 12:17:56 -0800 Subject: [PATCH 126/222] Fix typo Co-authored-by: Danny Ryan --- specs/phase0/weak-subjectivity.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/weak-subjectivity.md b/specs/phase0/weak-subjectivity.md index f91a04d7e..4137efade 100644 --- a/specs/phase0/weak-subjectivity.md +++ b/specs/phase0/weak-subjectivity.md @@ -79,7 +79,7 @@ A detailed analysis of the calculation of the weak subjectivity period is made i *Note*: The expressions in the report use fractions, whereas eth2.0-specs uses only `uint64` arithmetic. The expressions have been simplified to avoid computing fractions, and more details can be found [here](https://www.overleaf.com/read/wgjzjdjpvpsd). -*Note*: The calculations here use `Ether` instead of `Gwei`, because the large magitude of balances in `Gwei` can cause an overflow while computing using `uint64` arithmetic operations. Using `Ether` reduces the magintude of the multiplicative factors by an order of `ETH_TO_GWEI` (`= 10**9`) and avoid the scope for overflows in `uint64`. +*Note*: The calculations here use `Ether` instead of `Gwei`, because the large magnitude of balances in `Gwei` can cause an overflow while computing using `uint64` arithmetic operations. Using `Ether` reduces the magnitude of the multiplicative factors by an order of `ETH_TO_GWEI` (`= 10**9`) and avoid the scope for overflows in `uint64`. #### `compute_weak_subjectivity_period` From 34cea67b91cd0d36504a6376c93e13c589832a7d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 4 Feb 2021 08:45:25 -0700 Subject: [PATCH 127/222] ValidatorFlags -> ValidatorFlag --- specs/lightclient/beacon-chain.md | 30 +++++++++---------- specs/lightclient/lightclient-fork.md | 4 +-- .../pyspec/eth2spec/test/helpers/rewards.py | 6 ++-- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 3508fe991..ff69c1312 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -60,23 +60,23 @@ This is a patch implementing the first hard fork to the beacon chain, tentativel | Name | SSZ equivalent | Description | | - | - | - | -| `ValidatorFlags` | `uint8` | Bitflags to track validator actions with | +| `ValidatorFlag` | `uint8` | Bitflags to track validator actions with | ## Constants ### Validator action flags This is formatted as an enum, with values `2**i` that can be combined as bit-flags. -The `0` value is reserved as default. Remaining bits in `ValidatorFlags` may be used in future hardforks. +The `0` value is reserved as default. Remaining bits in `ValidatorFlag` may be used in future hardforks. **Note**: Unlike Phase0, a `TIMELY_TARGET_FLAG` does not necessarily imply a `TIMELY_SOURCE_FLAG` due to the varying slot delay requirements of each. | Name | Value | | - | - | -| `TIMELY_HEAD_FLAG` | `ValidatorFlags(2**0)` (= 1) | -| `TIMELY_SOURCE_FLAG` | `ValidatorFlags(2**1)` (= 2) | -| `TIMELY_TARGET_FLAG` | `ValidatorFlags(2**2)` (= 4) | +| `TIMELY_HEAD_FLAG` | `ValidatorFlag(2**0)` (= 1) | +| `TIMELY_SOURCE_FLAG` | `ValidatorFlag(2**1)` (= 2) | +| `TIMELY_TARGET_FLAG` | `ValidatorFlag(2**2)` (= 4) | ### Participation rewards @@ -158,8 +158,8 @@ class BeaconState(Container): # Slashings slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR] # Per-epoch sums of slashed effective balances # Participation - previous_epoch_participation: List[ValidatorFlags, VALIDATOR_REGISTRY_LIMIT] - current_epoch_participation: List[ValidatorFlags, VALIDATOR_REGISTRY_LIMIT] + previous_epoch_participation: List[ValidatorFlag, VALIDATOR_REGISTRY_LIMIT] + current_epoch_participation: List[ValidatorFlag, VALIDATOR_REGISTRY_LIMIT] # Finality justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH] # Bit set for every recent justified epoch previous_justified_checkpoint: Checkpoint @@ -201,7 +201,7 @@ def eth2_fast_aggregate_verify(pubkeys: Sequence[BLSPubkey], message: Bytes32, s #### `flags_and_numerators` ```python -def get_flags_and_numerators() -> Sequence[Tuple[ValidatorFlags, int]]: +def get_flags_and_numerators() -> Sequence[Tuple[ValidatorFlag, int]]: return ( (TIMELY_HEAD_FLAG, TIMELY_HEAD_NUMERATOR), (TIMELY_SOURCE_FLAG, TIMELY_SOURCE_NUMERATOR), @@ -210,12 +210,12 @@ def get_flags_and_numerators() -> Sequence[Tuple[ValidatorFlags, int]]: ``` ```python -def add_flags(flags: ValidatorFlags, add: ValidatorFlags) -> ValidatorFlags: +def add_flags(flags: ValidatorFlag, add: ValidatorFlag) -> ValidatorFlag: return flags | add ``` ```python -def has_flags(flags: ValidatorFlags, has: ValidatorFlags) -> bool: +def has_flags(flags: ValidatorFlag, has: ValidatorFlag) -> bool: return flags & has == has ``` @@ -277,7 +277,7 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: #### `get_unslashed_participating_indices` ```python -def get_unslashed_participating_indices(state: BeaconState, flags: ValidatorFlags, epoch: Epoch) -> Set[ValidatorIndex]: +def get_unslashed_participating_indices(state: BeaconState, flags: ValidatorFlag, epoch: Epoch) -> Set[ValidatorIndex]: """ Retrieve the active validator indices of the given epoch, which are not slashed, and have all of the given flags. """ @@ -297,7 +297,7 @@ def get_unslashed_participating_indices(state: BeaconState, flags: ValidatorFlag ```python def get_flag_deltas(state: BeaconState, - flag: ValidatorFlags, + flag: ValidatorFlag, numerator: uint64) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: """ Compute the rewards and penalties associated with a particular duty, by scanning through the participation @@ -457,8 +457,8 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: state.validators.append(get_validator_from_deposit(state, deposit)) state.balances.append(amount) # [Added in hf-1] Initialize empty participation flags for new validator - state.previous_epoch_participation.append(ValidatorFlags(0)) - state.current_epoch_participation.append(ValidatorFlags(0)) + state.previous_epoch_participation.append(ValidatorFlag(0)) + state.current_epoch_participation.append(ValidatorFlag(0)) else: # Increase balance by deposit amount index = ValidatorIndex(validator_pubkeys.index(pubkey)) @@ -597,5 +597,5 @@ def process_participation_flag_updates(state: BeaconState) -> None: Call to ``process_participation_flag_updates`` added to ``process_epoch`` in HF1 """ state.previous_epoch_participation = state.current_epoch_participation - state.current_epoch_participation = [ValidatorFlags(0) for _ in range(len(state.validators))] + state.current_epoch_participation = [ValidatorFlag(0) for _ in range(len(state.validators))] ``` diff --git a/specs/lightclient/lightclient-fork.md b/specs/lightclient/lightclient-fork.md index f5f3d1b7b..aa0171b86 100644 --- a/specs/lightclient/lightclient-fork.md +++ b/specs/lightclient/lightclient-fork.md @@ -67,8 +67,8 @@ def upgrade_to_lightclient_patch(pre: phase0.BeaconState) -> BeaconState: # Slashings slashings=pre.slashings, # Attestations - previous_epoch_participation=[ValidatorFlags(0) for _ in range(len(pre.validators))], - current_epoch_participation=[ValidatorFlags(0) for _ in range(len(pre.validators))], + previous_epoch_participation=[ValidatorFlag(0) for _ in range(len(pre.validators))], + current_epoch_participation=[ValidatorFlag(0) for _ in range(len(pre.validators))], # Finality justification_bits=pre.justification_bits, previous_justified_checkpoint=pre.previous_justified_checkpoint, diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index feaac018b..2499bcffe 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -314,7 +314,7 @@ def run_test_full_but_partial_participation(spec, state, rng=Random(5522)): else: for index in range(len(state.validators)): if rng.choice([True, False]): - state.previous_epoch_participation[index] = spec.ValidatorFlags(0) + state.previous_epoch_participation[index] = spec.ValidatorFlag(0) yield from run_deltas(spec, state) @@ -328,7 +328,7 @@ def run_test_partial(spec, state, fraction_filled): state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] else: for index in range(int(len(state.validators) * fraction_filled)): - state.previous_epoch_participation[index] = spec.ValidatorFlags(0) + state.previous_epoch_participation[index] = spec.ValidatorFlag(0) yield from run_deltas(spec, state) @@ -394,7 +394,7 @@ def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state): else: index = 0 state.validators[index].effective_balance = 1 - state.previous_epoch_participation[index] = spec.ValidatorFlags(0) + state.previous_epoch_participation[index] = spec.ValidatorFlag(0) yield from run_deltas(spec, state) From 9313815976803c2e93c8fbb4b2518340c4ce83c7 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 4 Feb 2021 08:47:46 -0700 Subject: [PATCH 128/222] put 'validator' in flags methods --- specs/lightclient/beacon-chain.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index ff69c1312..e11f50076 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -210,12 +210,12 @@ def get_flags_and_numerators() -> Sequence[Tuple[ValidatorFlag, int]]: ``` ```python -def add_flags(flags: ValidatorFlag, add: ValidatorFlag) -> ValidatorFlag: +def add_validator_flags(flags: ValidatorFlag, add: ValidatorFlag) -> ValidatorFlag: return flags | add ``` ```python -def has_flags(flags: ValidatorFlag, has: ValidatorFlag) -> bool: +def has_validator_flags(flags: ValidatorFlag, has: ValidatorFlag) -> bool: return flags & has == has ``` @@ -288,7 +288,7 @@ def get_unslashed_participating_indices(state: BeaconState, flags: ValidatorFlag epoch_participation = state.previous_epoch_participation participating_indices = [ index for index in get_active_validator_indices(state, epoch) - if has_flags(epoch_participation[index], flags) + if has_validator_flags(epoch_participation[index], flags) ] return set(filter(lambda index: not state.validators[index].slashed, participating_indices)) ``` @@ -410,8 +410,8 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: proposer_reward_numerator = 0 for index in get_attesting_indices(state, data, attestation.aggregation_bits): for flag, numerator in get_flags_and_numerators(): - if flag in participation_flags and not has_flags(epoch_participation[index], flag): - epoch_participation[index] = add_flags(epoch_participation[index], flag) + if flag in participation_flags and not has_validator_flags(epoch_participation[index], flag): + epoch_participation[index] = add_validator_flags(epoch_participation[index], flag) proposer_reward_numerator += get_base_reward(state, index) * numerator # Reward proposer From b08600156e03c17ab2c6250bb85b7734a10afe94 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 4 Feb 2021 11:05:00 -0600 Subject: [PATCH 129/222] hww feedback Co-authored-by: Hsiao-Wei Wang --- specs/lightclient/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index e11f50076..2773e1760 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -507,10 +507,10 @@ def process_epoch(state: BeaconState) -> None: process_slashings_reset(state) process_randao_mixes_reset(state) process_historical_roots_update(state) + # [Removed in HF1] -- process_participation_record_updates(state) # [Added in HF1] process_participation_flag_updates(state) process_sync_committee_updates(state) - # [Removed in HF1] -- process_participation_record_updates(state) ``` #### New `process_justification_and_finalization` From 0fd0db1ffc219e8686497874d975e18087f1bd40 Mon Sep 17 00:00:00 2001 From: Phong Phan Date: Sat, 6 Feb 2021 20:20:35 +0700 Subject: [PATCH 130/222] Fix some typos (#2195) * Fix typo for P2P Networking document * Fix link typo of P2P networking document * fix typo for light clients beacon chain document --- specs/lightclient/beacon-chain.md | 2 +- specs/phase0/p2p-interface.md | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 2773e1760..fff8726bf 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -314,7 +314,7 @@ def get_flag_deltas(state: BeaconState, base_reward = get_base_reward(state, index) if index in unslashed_participating_indices: if is_in_inactivity_leak(state): - # Optimal participatition is fully rewarded to cancel the inactivity penalty + # Optimal participation is fully rewarded to cancel the inactivity penalty rewards[index] = base_reward * numerator // REWARD_DENOMINATOR else: rewards[index] = ( diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 5a0953f76..a08a2d136 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -101,7 +101,7 @@ It consists of four main sections: - [Compression/Encoding](#compressionencoding) - [Why are we using SSZ for encoding?](#why-are-we-using-ssz-for-encoding) - [Why are we compressing, and at which layers?](#why-are-we-compressing-and-at-which-layers) - - [Why are using Snappy for compression?](#why-are-using-snappy-for-compression) + - [Why are we using Snappy for compression?](#why-are-we-using-snappy-for-compression) - [Can I get access to unencrypted bytes on the wire for debugging purposes?](#can-i-get-access-to-unencrypted-bytes-on-the-wire-for-debugging-purposes) - [What are SSZ type size bounds?](#what-are-ssz-type-size-bounds) - [libp2p implementations matrix](#libp2p-implementations-matrix) @@ -937,7 +937,7 @@ where the fields of `ENRForkID` are defined as * `next_fork_epoch` is the epoch at which the next fork is planned and the `current_fork_version` will be updated. If no future fork is planned, set `next_fork_epoch = FAR_FUTURE_EPOCH` to signal this fact -*Note*: `fork_digest` is composed of values that are not not known until the genesis block/state are available. +*Note*: `fork_digest` is composed of values that are not known until the genesis block/state are available. Due to this, clients SHOULD NOT form ENRs and begin peer discovery until genesis values are known. One notable exception to this rule is the distribution of bootnode ENRs prior to genesis. In this case, bootnode ENRs SHOULD be initially distributed with `eth2` field set as @@ -1223,7 +1223,7 @@ the node's fork choice prevents integration of these messages into the actual co Depending on the number of validators, it may be more efficient to group shard subnets and might provide better stability for the gossipsub channel. The exact grouping will be dependent on more involved network tests. This constant allows for more flexibility in setting up the network topology for attestation aggregation (as aggregation should happen on each subnet). -The value is currently set to to be equal `MAX_COMMITTEES_PER_SLOT` if/until network tests indicate otherwise. +The value is currently set to be equal to `MAX_COMMITTEES_PER_SLOT` if/until network tests indicate otherwise. ### Why are attestations limited to be broadcast on gossip channels within `SLOTS_PER_EPOCH` slots? @@ -1369,7 +1369,7 @@ Thus, it may happen that we need to transmit an empty list - there are several w Semantically, it is not an error that a block is missing during a slot making option 2 unnatural. -Option 1 allows allows the responder to signal "no block", but this information may be wrong - for example in the case of a malicious node. +Option 1 allows the responder to signal "no block", but this information may be wrong - for example in the case of a malicious node. Under option 0, there is no way for a client to distinguish between a slot without a block and an incomplete response, but given that it already must contain logic to handle the uncertainty of a malicious peer, option 0 was chosen. @@ -1495,7 +1495,7 @@ This looks different depending on the interaction layer: implementers are encouraged to encapsulate the encoding and compression logic behind MessageReader and MessageWriter components/strategies that can be layered on top of the raw byte streams. -### Why are using Snappy for compression? +### Why are we using Snappy for compression? Snappy is used in Ethereum 1.0. It is well maintained by Google, has good benchmarks, and can calculate the size of the uncompressed object without inflating it in memory. From 7d715220bb7128ea98597deedab3ae82ca5cc854 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 11 Feb 2021 06:39:42 +1100 Subject: [PATCH 131/222] Don't propagate blocks with a faulty slot (#2196) --- specs/phase0/p2p-interface.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index a08a2d136..249d08799 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -313,6 +313,7 @@ The following validations MUST pass before forwarding the `signed_beacon_block` (via both gossip and non-gossip sources) (a client MAY queue blocks for processing once the parent block is retrieved). - _[REJECT]_ The block's parent (defined by `block.parent_root`) passes validation. +- _[REJECT]_ The block is from a higher slot than its parent. - _[REJECT]_ The current `finalized_checkpoint` is an ancestor of `block` -- i.e. `get_ancestor(store, block.parent_root, compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)) == store.finalized_checkpoint.root` From 2b8b0d9e2bcd301c012d09a6ce0d4680cd17ab6f Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 11 Feb 2021 14:53:29 -0700 Subject: [PATCH 132/222] update penalty config values for hf1 --- configs/mainnet/lightclient_patch.yaml | 10 +++ configs/minimal/lightclient_patch.yaml | 10 +++ specs/lightclient/beacon-chain.md | 77 ++++++++++++++++++- .../test_process_slashings.py | 48 ++++++++++-- 4 files changed, 134 insertions(+), 11 deletions(-) diff --git a/configs/mainnet/lightclient_patch.yaml b/configs/mainnet/lightclient_patch.yaml index 64c05a720..6c5b16edf 100644 --- a/configs/mainnet/lightclient_patch.yaml +++ b/configs/mainnet/lightclient_patch.yaml @@ -2,6 +2,16 @@ CONFIG_NAME: "mainnet" +# Updated penalty values +# --------------------------------------------------------------- +# 3 * 2**24) (= 50,331,648) +HF1_INACTIVITY_PENALTY_QUOTIENT: 50331648 +# 2**6 (= 64) +HF1_MIN_SLASHING_PENALTY_QUOTIENT: 64 +# 2 +HF1_PROPORTIONAL_SLASHING_MULTIPLIER: 2 + + # Misc # --------------------------------------------------------------- # 2**10 (=1,024) diff --git a/configs/minimal/lightclient_patch.yaml b/configs/minimal/lightclient_patch.yaml index afe7d897e..7ab5f34ba 100644 --- a/configs/minimal/lightclient_patch.yaml +++ b/configs/minimal/lightclient_patch.yaml @@ -2,6 +2,16 @@ CONFIG_NAME: "minimal" +# Updated penalty values +# --------------------------------------------------------------- +# 3 * 2**24) (= 50,331,648) +HF1_INACTIVITY_PENALTY_QUOTIENT: 50331648 +# 2**6 (= 64) +HF1_MIN_SLASHING_PENALTY_QUOTIENT: 64 +# 2 +HF1_PROPORTIONAL_SLASHING_MULTIPLIER: 2 + + # Misc # --------------------------------------------------------------- # [customized] diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index fff8726bf..ccf9b2ebc 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -13,6 +13,7 @@ - [Participation rewards](#participation-rewards) - [Misc](#misc) - [Configuration](#configuration) + - [Updated penalty values](#updated-penalty-values) - [Misc](#misc-1) - [Time parameters](#time-parameters) - [Domain types](#domain-types) @@ -34,6 +35,8 @@ - [`get_unslashed_participating_indices`](#get_unslashed_participating_indices) - [`get_flag_deltas`](#get_flag_deltas) - [New `get_inactivity_penalty_deltas`](#new-get_inactivity_penalty_deltas) + - [Beacon state mutators](#beacon-state-mutators) + - [New `slash_validator`](#new-slash_validator) - [Block processing](#block-processing) - [New `process_attestation`](#new-process_attestation) - [New `process_deposit`](#new-process_deposit) @@ -41,6 +44,7 @@ - [Epoch processing](#epoch-processing) - [New `process_justification_and_finalization`](#new-process_justification_and_finalization) - [New `process_rewards_and_penalties`](#new-process_rewards_and_penalties) + - [New `process_slashings`](#new-process_slashings) - [Sync committee updates](#sync-committee-updates) - [Participation flags updates](#participation-flags-updates) @@ -49,7 +53,8 @@ ## Introduction -This is a patch implementing the first hard fork to the beacon chain, tentatively named HF1 pending a permanent name. It has three main features: +This is a patch implementing the first hard fork to the beacon chain, tentatively named HF1 pending a permanent name. +It has three main features: * Light client support via sync committees * Incentive accounting reforms, reducing spec complexity @@ -97,6 +102,18 @@ The reward fractions add up to 7/8, leaving the remaining 1/8 for proposer rewar ## Configuration +### Updated penalty values + +This patch updates a few configuration values to move penalty constants toward their final, maxmium security values. + +*Note*: The spec does *not* override previous configuration values but instead creates new values and replaces usage throughout. + +| Name | Value | +| - | - | +| `HF1_INACTIVITY_PENALTY_QUOTIENT` | `uint64(3 * 2**24)` (= 50,331,648) | +| `HF1_MIN_SLASHING_PENALTY_QUOTIENT` | `uint64(2**6)` (=64) | +| `HF1_PROPORTIONAL_SLASHING_MULTIPLIER` | `uint64(2)` | + ### Misc | Name | Value | @@ -328,7 +345,8 @@ def get_flag_deltas(state: BeaconState, #### New `get_inactivity_penalty_deltas` -*Note*: The function `get_inactivity_penalty_deltas` is modified in the selection of matching target indices and the removal of `BASE_REWARDS_PER_EPOCH`. +*Note*: The function `get_inactivity_penalty_deltas` is modified in the selection of matching target indices +and the removal of `BASE_REWARDS_PER_EPOCH`. ```python def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: @@ -348,12 +366,47 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S penalties[index] += Gwei(get_base_reward(state, index) * reward_numerator_sum // REWARD_DENOMINATOR) if index not in matching_target_attesting_indices: effective_balance = state.validators[index].effective_balance - penalties[index] += Gwei(effective_balance * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) + penalties[index] += Gwei( + effective_balance * get_finality_delay(state) + // HF1_INACTIVITY_PENALTY_QUOTIENT + ) rewards = [Gwei(0) for _ in range(len(state.validators))] return rewards, penalties ``` +### Beacon state mutators + +#### New `slash_validator` + +*Note*: The function `slash_validator` is modified +with the substitution of `MIN_SLASHING_PENALTY_QUOTIENT` with `HF1_MIN_SLASHING_PENALTY_QUOTIENT`. + +```python +def slash_validator(state: BeaconState, + slashed_index: ValidatorIndex, + whistleblower_index: ValidatorIndex=None) -> None: + """ + Slash the validator with index ``slashed_index``. + """ + epoch = get_current_epoch(state) + initiate_validator_exit(state, slashed_index) + validator = state.validators[slashed_index] + validator.slashed = True + validator.withdrawable_epoch = max(validator.withdrawable_epoch, Epoch(epoch + EPOCHS_PER_SLASHINGS_VECTOR)) + state.slashings[epoch % EPOCHS_PER_SLASHINGS_VECTOR] += validator.effective_balance + decrease_balance(state, slashed_index, validator.effective_balance // HF1_MIN_SLASHING_PENALTY_QUOTIENT) + + # Apply proposer and whistleblower rewards + proposer_index = get_beacon_proposer_index(state) + if whistleblower_index is None: + whistleblower_index = proposer_index + whistleblower_reward = Gwei(validator.effective_balance // WHISTLEBLOWER_REWARD_QUOTIENT) + proposer_reward = Gwei(whistleblower_reward // PROPOSER_REWARD_QUOTIENT) + increase_balance(state, proposer_index, proposer_reward) + increase_balance(state, whistleblower_index, Gwei(whistleblower_reward - proposer_reward)) +``` + ### Block processing ```python @@ -576,6 +629,24 @@ def process_rewards_and_penalties(state: BeaconState) -> None: decrease_balance(state, ValidatorIndex(index), penalties[index]) ``` +#### New `process_slashings` + +*Note*: The function `process_slashings` is modified +with the substitution of `PROPORTIONAL_SLASHING_MULTIPLIER` with `HF1_PROPORTIONAL_SLASHING_MULTIPLIER`. + +```python +def process_slashings(state: BeaconState) -> None: + epoch = get_current_epoch(state) + total_balance = get_total_active_balance(state) + adjusted_total_slashing_balance = min(sum(state.slashings) * HF1_PROPORTIONAL_SLASHING_MULTIPLIER, total_balance) + for index, validator in enumerate(state.validators): + if validator.slashed and epoch + EPOCHS_PER_SLASHINGS_VECTOR // 2 == validator.withdrawable_epoch: + increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from penalty numerator to avoid uint64 overflow + penalty_numerator = validator.effective_balance // increment * adjusted_total_slashing_balance + penalty = penalty_numerator // total_balance * increment + decrease_balance(state, ValidatorIndex(index), penalty) +``` + #### Sync committee updates ```python diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py index 2e09f5c8a..8bb4ac218 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.context import spec_state_test, with_all_phases, is_post_lightclient_patch from eth2spec.test.helpers.epoch_processing import ( run_epoch_processing_with, run_epoch_processing_to ) @@ -23,12 +23,19 @@ def slash_validators(spec, state, indices, out_epochs): ] = total_slashed_balance +def get_slashing_multipler(spec): + if is_post_lightclient_patch(spec): + return spec.HF1_PROPORTIONAL_SLASHING_MULTIPLIER + else: + return spec.PROPORTIONAL_SLASHING_MULTIPLIER + + @with_all_phases @spec_state_test def test_max_penalties(spec, state): # Slashed count to ensure that enough validators are slashed to induce maximum penalties slashed_count = min( - (len(state.validators) // spec.PROPORTIONAL_SLASHING_MULTIPLIER) + 1, + (len(state.validators) // get_slashing_multipler(spec)) + 1, # Can't slash more than validator count! len(state.validators) ) @@ -40,7 +47,7 @@ def test_max_penalties(spec, state): total_balance = spec.get_total_active_balance(state) total_penalties = sum(state.slashings) - assert total_balance // spec.PROPORTIONAL_SLASHING_MULTIPLIER <= total_penalties + assert total_balance // get_slashing_multipler(spec) <= total_penalties yield from run_process_slashings(spec, state) @@ -50,7 +57,30 @@ def test_max_penalties(spec, state): @with_all_phases @spec_state_test -def test_small_penalty(spec, state): +def test_low_penalty(spec, state): + # Slashed count is one tenth of validator set + slashed_count = (len(state.validators) // 10) + 1 + out_epoch = spec.get_current_epoch(state) + (spec.EPOCHS_PER_SLASHINGS_VECTOR // 2) + + slashed_indices = list(range(slashed_count)) + slash_validators(spec, state, slashed_indices, [out_epoch] * slashed_count) + + pre_state = state.copy() + + yield from run_process_slashings(spec, state) + + for i in slashed_indices: + assert 0 < state.balances[i] < pre_state.balances[i] + + +@with_all_phases +@spec_state_test +def test_minimal_penalty(spec, state): + # + # When very few slashings, the resulting slashing penalty gets rounded down + # to zero so the result of `process_slashings` is null + # + # Just the bare minimum for this one validator state.balances[0] = state.validators[0].effective_balance = spec.EJECTION_BALANCE # All the other validators get the maximum. @@ -74,11 +104,13 @@ def test_small_penalty(spec, state): expected_penalty = ( state.validators[0].effective_balance // spec.EFFECTIVE_BALANCE_INCREMENT - * (3 * total_penalties) + * (get_slashing_multipler(spec) * total_penalties) // total_balance * spec.EFFECTIVE_BALANCE_INCREMENT ) - assert state.balances[0] == pre_slash_balances[0] - expected_penalty + + assert expected_penalty == 0 + assert state.balances[0] == pre_slash_balances[0] @with_all_phases @@ -96,7 +128,7 @@ def test_scaled_penalties(spec, state): state.slashings[5] = base + (incr * 6) state.slashings[spec.EPOCHS_PER_SLASHINGS_VECTOR - 1] = base + (incr * 7) - slashed_count = len(state.validators) // (spec.PROPORTIONAL_SLASHING_MULTIPLIER + 1) + slashed_count = len(state.validators) // (get_slashing_multipler(spec) + 1) assert slashed_count > 10 @@ -134,7 +166,7 @@ def test_scaled_penalties(spec, state): v = state.validators[i] expected_penalty = ( v.effective_balance // spec.EFFECTIVE_BALANCE_INCREMENT - * (spec.PROPORTIONAL_SLASHING_MULTIPLIER * total_penalties) + * (get_slashing_multipler(spec) * total_penalties) // (total_balance) * spec.EFFECTIVE_BALANCE_INCREMENT ) From 24a244eb9b340fe707888180e02bcb3918cd7998 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 13 Feb 2021 22:48:34 +0800 Subject: [PATCH 133/222] Fix typo: `get_slashing_multipler` -> `get_slashing_multiplier` --- .../epoch_processing/test_process_slashings.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py index 8bb4ac218..34f1e89c6 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py @@ -23,7 +23,7 @@ def slash_validators(spec, state, indices, out_epochs): ] = total_slashed_balance -def get_slashing_multipler(spec): +def get_slashing_multiplier(spec): if is_post_lightclient_patch(spec): return spec.HF1_PROPORTIONAL_SLASHING_MULTIPLIER else: @@ -35,7 +35,7 @@ def get_slashing_multipler(spec): def test_max_penalties(spec, state): # Slashed count to ensure that enough validators are slashed to induce maximum penalties slashed_count = min( - (len(state.validators) // get_slashing_multipler(spec)) + 1, + (len(state.validators) // get_slashing_multiplier(spec)) + 1, # Can't slash more than validator count! len(state.validators) ) @@ -47,7 +47,7 @@ def test_max_penalties(spec, state): total_balance = spec.get_total_active_balance(state) total_penalties = sum(state.slashings) - assert total_balance // get_slashing_multipler(spec) <= total_penalties + assert total_balance // get_slashing_multiplier(spec) <= total_penalties yield from run_process_slashings(spec, state) @@ -104,7 +104,7 @@ def test_minimal_penalty(spec, state): expected_penalty = ( state.validators[0].effective_balance // spec.EFFECTIVE_BALANCE_INCREMENT - * (get_slashing_multipler(spec) * total_penalties) + * (get_slashing_multiplier(spec) * total_penalties) // total_balance * spec.EFFECTIVE_BALANCE_INCREMENT ) @@ -128,7 +128,7 @@ def test_scaled_penalties(spec, state): state.slashings[5] = base + (incr * 6) state.slashings[spec.EPOCHS_PER_SLASHINGS_VECTOR - 1] = base + (incr * 7) - slashed_count = len(state.validators) // (get_slashing_multipler(spec) + 1) + slashed_count = len(state.validators) // (get_slashing_multiplier(spec) + 1) assert slashed_count > 10 @@ -166,7 +166,7 @@ def test_scaled_penalties(spec, state): v = state.validators[i] expected_penalty = ( v.effective_balance // spec.EFFECTIVE_BALANCE_INCREMENT - * (get_slashing_multipler(spec) * total_penalties) + * (get_slashing_multiplier(spec) * total_penalties) // (total_balance) * spec.EFFECTIVE_BALANCE_INCREMENT ) From dda7010c0c02f2bd740509e315b5985db7590f84 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 13 Feb 2021 23:02:06 +0800 Subject: [PATCH 134/222] Fix the tests that use `MIN_SLASHING_PENALTY_QUOTIENT`. (The mainnet tests failed before this fix) --- .../pyspec/eth2spec/test/helpers/proposer_slashings.py | 10 +++++++++- .../block_processing/test_process_attester_slashing.py | 3 ++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py b/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py index 87b4f5ca0..89acb3417 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py +++ b/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -1,8 +1,16 @@ +from eth2spec.test.context import is_post_lightclient_patch from eth2spec.test.helpers.block_header import sign_block_header from eth2spec.test.helpers.keys import pubkey_to_privkey from eth2spec.test.helpers.state import get_balance +def get_min_slashing_penalty_quotient(spec): + if is_post_lightclient_patch(spec): + return spec.HF1_MIN_SLASHING_PENALTY_QUOTIENT + else: + return spec.MIN_SLASHING_PENALTY_QUOTIENT + + def check_proposer_slashing_effect(spec, pre_state, state, slashed_index): slashed_validator = state.validators[slashed_index] assert slashed_validator.slashed @@ -10,7 +18,7 @@ def check_proposer_slashing_effect(spec, pre_state, state, slashed_index): assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH proposer_index = spec.get_beacon_proposer_index(state) - slash_penalty = state.validators[slashed_index].effective_balance // spec.MIN_SLASHING_PENALTY_QUOTIENT + slash_penalty = state.validators[slashed_index].effective_balance // get_min_slashing_penalty_quotient(spec) whistleblower_reward = state.validators[slashed_index].effective_balance // spec.WHISTLEBLOWER_REWARD_QUOTIENT if proposer_index != slashed_index: # slashed validator lost initial slash penalty diff --git a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attester_slashing.py b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attester_slashing.py index 82d490311..21d9363f7 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attester_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attester_slashing.py @@ -4,6 +4,7 @@ from eth2spec.test.context import ( from eth2spec.test.helpers.attestations import sign_indexed_attestation from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing, \ get_indexed_attestation_participants, get_attestation_2_data, get_attestation_1_data +from eth2spec.test.helpers.proposer_slashings import get_min_slashing_penalty_quotient from eth2spec.test.helpers.state import ( get_balance, next_epoch_via_block, @@ -70,7 +71,7 @@ def run_attester_slashing_processing(spec, state, attester_slashing, valid=True) expected_balance = ( pre_proposer_balance + total_proposer_rewards - - pre_slashings[proposer_index] // spec.MIN_SLASHING_PENALTY_QUOTIENT + - pre_slashings[proposer_index] // get_min_slashing_penalty_quotient(spec) ) assert get_balance(state, proposer_index) == expected_balance From 600a4daddf1456778b71a86f9dad7e25d25aaf52 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 Feb 2021 18:38:20 +0800 Subject: [PATCH 135/222] Turn off phase1 testgen and turn on lightclient_patch testgen --- tests/core/gen_helpers/gen_from_tests/gen.py | 12 +++++- tests/core/pyspec/eth2spec/test/context.py | 4 ++ .../test/lightclient_patch/sanity/__init__.py | 0 tests/generators/epoch_processing/main.py | 37 +++++++++++------- tests/generators/finality/main.py | 39 ++++++++++++------- tests/generators/operations/main.py | 37 +++++++++++------- tests/generators/rewards/main.py | 34 +++++++++------- tests/generators/sanity/main.py | 34 +++++++++------- 8 files changed, 126 insertions(+), 71 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/__init__.py diff --git a/tests/core/gen_helpers/gen_from_tests/gen.py b/tests/core/gen_helpers/gen_from_tests/gen.py index 902b0954a..c09b477bb 100644 --- a/tests/core/gen_helpers/gen_from_tests/gen.py +++ b/tests/core/gen_helpers/gen_from_tests/gen.py @@ -1,5 +1,5 @@ from inspect import getmembers, isfunction -from typing import Any, Iterable +from typing import Any, Iterable, Dict from gen_base.gen_typing import TestCase @@ -38,3 +38,13 @@ def generate_from_tests(runner_name: str, handler_name: str, src: Any, # TODO: with_all_phases and other per-phase tooling, should be replaced with per-fork equivalent. case_fn=lambda: tfn(generator_mode=True, phase=fork_name, bls_active=bls_active) ) + + +def get_provider(create_provider_fn, config_name, fork_name, all_mods): + for key, mod_name in all_mods[fork_name].items(): + yield create_provider_fn( + fork_name=fork_name, + handler_name=key, + tests_src_mod_name=mod_name, + config_name=config_name, + ) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index d19547477..75832794f 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -37,6 +37,10 @@ ALL_PHASES = (PHASE0, PHASE1, LIGHTCLIENT_PATCH) MAINNET = ConfigName('mainnet') MINIMAL = ConfigName('minimal') +ALL_CONFIGS = (MINIMAL, MAINNET) + +# The forks that output to the test vectors. +TESTGEN_FORKS = (PHASE0, LIGHTCLIENT_PATCH) # TODO: currently phases are defined as python modules. # It would be better if they would be more well-defined interfaces for stronger typing. diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/__init__.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index ea7639605..835fe0087 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -1,12 +1,13 @@ from typing import Iterable from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests +from gen_from_tests.gen import generate_from_tests, get_provider from importlib import reload, import_module from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1 +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, ALL_CONFIGS from eth2spec.utils import bls @@ -15,6 +16,7 @@ def create_provider(fork_name: str, handler_name: str, def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) + reload(spec_lightclient_patch) reload(spec_phase1) bls.use_milagro() return config_name @@ -44,21 +46,28 @@ if __name__ == "__main__": 'historical_roots_update', 'participation_record_updates', ]} + lightclient_patch_mods = { + **{key: 'eth2spec.test.lightclient_patch.epoch_processing.test_process_' + key for key in [ + 'sync_committee_updates', + ]}, + **phase_0_mods, + } # also run the previous phase 0 tests phase_1_mods = {**{key: 'eth2spec.test.phase1.epoch_processing.test_process_' + key for key in [ 'reveal_deadlines', 'challenge_deadlines', 'custody_final_updates', ]}, **phase_0_mods} # also run the previous phase 0 tests (but against phase 1 spec) - gen_runner.run_generator(f"epoch_processing", [ - create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items() - ]) - gen_runner.run_generator(f"epoch_processing", [ - create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items() - ]) - gen_runner.run_generator(f"epoch_processing", [ - create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items() - ]) - gen_runner.run_generator(f"epoch_processing", [ - create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items() - ]) + all_mods = { + PHASE0: phase_0_mods, + LIGHTCLIENT_PATCH: lightclient_patch_mods, + PHASE1: phase_1_mods, + } + + for config_name in ALL_CONFIGS: + for fork_name in TESTGEN_FORKS: + if fork_name in all_mods: + gen_runner.run_generator(f"epoch_processing", get_provider( + create_provider_fn=create_provider, config_name=config_name, + fork_name=fork_name, all_mods=all_mods, + )) diff --git a/tests/generators/finality/main.py b/tests/generators/finality/main.py index ef2d8293f..6ea3d82c8 100644 --- a/tests/generators/finality/main.py +++ b/tests/generators/finality/main.py @@ -1,27 +1,30 @@ from typing import Iterable -from importlib import reload +from importlib import reload, import_module from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests +from gen_from_tests.gen import generate_from_tests, get_provider -from eth2spec.test.context import PHASE0, PHASE1 -from eth2spec.test.phase0.finality import test_finality +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, ALL_CONFIGS from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 from eth2spec.utils import bls -def create_provider(fork_name: str, handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: +def create_provider(fork_name: str, handler_name: str, + tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) + reload(spec_lightclient_patch) reload(spec_phase1) bls.use_milagro() return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: + tests_src = import_module(tests_src_mod_name) return generate_from_tests( runner_name='finality', handler_name=handler_name, @@ -33,11 +36,21 @@ def create_provider(fork_name: str, handler_name: str, tests_src, config_name: s if __name__ == "__main__": - # No additional phase 1 specific rewards tests, yet. - key = 'finality' - gen_runner.run_generator("finality", [ - create_provider(PHASE0, 'finality', test_finality, 'minimal'), - create_provider(PHASE0, 'finality', test_finality, 'mainnet'), - create_provider(PHASE1, 'finality', test_finality, 'minimal'), - create_provider(PHASE1, 'finality', test_finality, 'mainnet'), - ]) + phase_0_mods = {'finality': 'eth2spec.test.phase0.finality.test_finality'} + # No additional lightclient_patch or phase 1 specific finality tests, yet. + lightclient_patch_mods = phase_0_mods + phase_1_mods = phase_0_mods + + all_mods = { + PHASE0: phase_0_mods, + LIGHTCLIENT_PATCH: lightclient_patch_mods, + PHASE1: phase_1_mods, + } + + for config_name in ALL_CONFIGS: + for fork_name in TESTGEN_FORKS: + if fork_name in all_mods: + gen_runner.run_generator(f"finality", get_provider( + create_provider_fn=create_provider, config_name=config_name, + fork_name=fork_name, all_mods=all_mods, + )) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index 1acf45e47..20643f8d7 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -1,12 +1,13 @@ from typing import Iterable from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests +from gen_from_tests.gen import generate_from_tests, get_provider from importlib import reload, import_module from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1 +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, ALL_CONFIGS from eth2spec.utils import bls @@ -15,6 +16,7 @@ def create_provider(fork_name: str, handler_name: str, def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) + reload(spec_lightclient_patch) reload(spec_phase1) bls.use_milagro() return config_name @@ -40,6 +42,12 @@ if __name__ == "__main__": 'proposer_slashing', 'voluntary_exit', ]} + lightclient_patch_mods = { + **{key: 'eth2spec.test.lightclient_patch.block_processing.test_process_' + key for key in [ + 'sync_committee', + ]}, + **phase_0_mods, + } # also run the previous phase 0 tests phase_1_mods = {**{key: 'eth2spec.test.phase1.block_processing.test_process_' + key for key in [ 'attestation', 'chunk_challenge', @@ -49,15 +57,16 @@ if __name__ == "__main__": 'shard_transition', ]}, **phase_0_mods} # also run the previous phase 0 tests (but against phase 1 spec) - gen_runner.run_generator(f"operations", [ - create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items() - ]) - gen_runner.run_generator(f"operations", [ - create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items() - ]) - gen_runner.run_generator(f"operations", [ - create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items() - ]) - gen_runner.run_generator(f"operations", [ - create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items() - ]) + all_mods = { + PHASE0: phase_0_mods, + LIGHTCLIENT_PATCH: lightclient_patch_mods, + PHASE1: phase_1_mods, + } + + for config_name in ALL_CONFIGS: + for fork_name in TESTGEN_FORKS: + if fork_name in all_mods: + gen_runner.run_generator(f"operations", get_provider( + create_provider_fn=create_provider, config_name=config_name, + fork_name=fork_name, all_mods=all_mods, + )) diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index 23d0633b0..cd3206a3f 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -1,12 +1,13 @@ from typing import Iterable from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests +from gen_from_tests.gen import generate_from_tests, get_provider from importlib import reload, import_module from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1 +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, ALL_CONFIGS from eth2spec.utils import bls @@ -15,6 +16,7 @@ def create_provider(fork_name: str, handler_name: str, def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) + reload(spec_lightclient_patch) reload(spec_phase1) bls.use_milagro() return config_name @@ -37,18 +39,20 @@ if __name__ == "__main__": 'leak', 'random', ]} - # No additional phase 1 specific rewards tests, yet. + # No additional lightclient_patch or phase 1 specific rewards tests, yet. + lightclient_patch_mods = phase_0_mods phase_1_mods = phase_0_mods - gen_runner.run_generator(f"rewards", [ - create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items() - ]) - gen_runner.run_generator(f"rewards", [ - create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items() - ]) - gen_runner.run_generator(f"rewards", [ - create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items() - ]) - gen_runner.run_generator(f"rewards", [ - create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items() - ]) + all_mods = { + PHASE0: phase_0_mods, + LIGHTCLIENT_PATCH: lightclient_patch_mods, + PHASE1: phase_1_mods, + } + + for config_name in ALL_CONFIGS: + for fork_name in TESTGEN_FORKS: + if fork_name in all_mods: + gen_runner.run_generator(f"rewards", get_provider( + create_provider_fn=create_provider, config_name=config_name, + fork_name=fork_name, all_mods=all_mods, + )) diff --git a/tests/generators/sanity/main.py b/tests/generators/sanity/main.py index 83166f0cf..0aabf5d4c 100644 --- a/tests/generators/sanity/main.py +++ b/tests/generators/sanity/main.py @@ -1,12 +1,13 @@ from typing import Iterable from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests +from gen_from_tests.gen import generate_from_tests, get_provider from importlib import reload, import_module from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1 +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, ALL_CONFIGS from eth2spec.utils import bls @@ -15,6 +16,7 @@ def create_provider(fork_name: str, handler_name: str, def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) + reload(spec_lightclient_patch) reload(spec_phase1) bls.use_milagro() return config_name @@ -36,20 +38,24 @@ if __name__ == "__main__": 'blocks', 'slots', ]} + lightclient_patch_mods = {**{key: 'eth2spec.test.lightclient_patch.sanity.test_' + key for key in [ + 'blocks', + ]}, **phase_0_mods} # also run the previous phase 0 tests phase_1_mods = {**{key: 'eth2spec.test.phase1.sanity.test_' + key for key in [ 'blocks', # more phase 1 specific block tests 'shard_blocks', ]}, **phase_0_mods} # also run the previous phase 0 tests (but against phase 1 spec) - gen_runner.run_generator(f"sanity", [ - create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items() - ]) - gen_runner.run_generator(f"sanity", [ - create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items() - ]) - gen_runner.run_generator(f"sanity", [ - create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items() - ]) - gen_runner.run_generator(f"sanity", [ - create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items() - ]) + all_mods = { + PHASE0: phase_0_mods, + LIGHTCLIENT_PATCH: lightclient_patch_mods, + PHASE1: phase_1_mods, + } + + for config_name in ALL_CONFIGS: + for fork_name in TESTGEN_FORKS: + if fork_name in all_mods: + gen_runner.run_generator(f"sanity", get_provider( + create_provider_fn=create_provider, config_name=config_name, + fork_name=fork_name, all_mods=all_mods, + )) From c7d975981c04626b72cdc3566914af0aa552b7bb Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 Feb 2021 22:25:58 +0800 Subject: [PATCH 136/222] Refactor state tests generators --- tests/core/gen_helpers/gen_from_tests/gen.py | 53 ++++++++++++++++++-- tests/generators/epoch_processing/main.py | 39 ++------------ tests/generators/finality/main.py | 41 ++------------- tests/generators/operations/main.py | 39 ++------------ tests/generators/rewards/main.py | 39 ++------------ tests/generators/sanity/main.py | 39 ++------------ 6 files changed, 69 insertions(+), 181 deletions(-) diff --git a/tests/core/gen_helpers/gen_from_tests/gen.py b/tests/core/gen_helpers/gen_from_tests/gen.py index c09b477bb..3ec8904fc 100644 --- a/tests/core/gen_helpers/gen_from_tests/gen.py +++ b/tests/core/gen_helpers/gen_from_tests/gen.py @@ -1,11 +1,17 @@ +from importlib import reload, import_module from inspect import getmembers, isfunction -from typing import Any, Iterable, Dict +from typing import Any, Callable, Dict, Iterable -from gen_base.gen_typing import TestCase +from eth2spec.config import config_util +from eth2spec.utils import bls + +from eth2spec.test.context import ALL_CONFIGS, TESTGEN_FORKS, SpecForkName, ConfigName +from gen_base import gen_runner +from gen_base.gen_typing import TestCase, TestProvider def generate_from_tests(runner_name: str, handler_name: str, src: Any, - fork_name: str, bls_active: bool = True) -> Iterable[TestCase]: + fork_name: SpecForkName, bls_active: bool = True) -> Iterable[TestCase]: """ Generate a list of test cases by running tests from the given src in generator-mode. :param runner_name: to categorize the test in general as. @@ -40,7 +46,10 @@ def generate_from_tests(runner_name: str, handler_name: str, src: Any, ) -def get_provider(create_provider_fn, config_name, fork_name, all_mods): +def get_provider(create_provider_fn: Callable[[SpecForkName, str, str, ConfigName], TestProvider], + config_name: ConfigName, + fork_name: SpecForkName, + all_mods: Dict[str, Dict[str, str]]) -> Iterable[TestProvider]: for key, mod_name in all_mods[fork_name].items(): yield create_provider_fn( fork_name=fork_name, @@ -48,3 +57,39 @@ def get_provider(create_provider_fn, config_name, fork_name, all_mods): tests_src_mod_name=mod_name, config_name=config_name, ) + + +def get_create_provider_fn(runner_name: str, config_name: ConfigName, specs: Iterable[Any] + ) -> Callable[[SpecForkName, str, str, ConfigName], TestProvider]: + def prepare_fn(configs_path: str) -> str: + config_util.prepare_config(configs_path, config_name) + for spec in specs: + reload(spec) + bls.use_milagro() + return config_name + + def create_provider(fork_name: SpecForkName, handler_name: str, + tests_src_mod_name: str, config_name: ConfigName) -> TestProvider: + def cases_fn() -> Iterable[TestCase]: + tests_src = import_module(tests_src_mod_name) + return generate_from_tests( + runner_name=runner_name, + handler_name=handler_name, + src=tests_src, + fork_name=fork_name, + ) + + return TestProvider(prepare=prepare_fn, make_cases=cases_fn) + return create_provider + + +def run_state_test_generators(runner_name: str, specs: Iterable[Any], all_mods: Dict[str, Dict[str, str]]) -> None: + for config_name in ALL_CONFIGS: + for fork_name in TESTGEN_FORKS: + if fork_name in all_mods: + gen_runner.run_generator(runner_name, get_provider( + create_provider_fn=get_create_provider_fn(runner_name, config_name, specs), + config_name=config_name, + fork_name=fork_name, + all_mods=all_mods, + )) diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index 835fe0087..1306da523 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -1,36 +1,11 @@ -from typing import Iterable - -from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests, get_provider -from importlib import reload, import_module -from eth2spec.config import config_util +from gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, ALL_CONFIGS -from eth2spec.utils import bls +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH -def create_provider(fork_name: str, handler_name: str, - tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: - def prepare_fn(configs_path: str) -> str: - config_util.prepare_config(configs_path, config_name) - reload(spec_phase0) - reload(spec_lightclient_patch) - reload(spec_phase1) - bls.use_milagro() - return config_name - - def cases_fn() -> Iterable[gen_typing.TestCase]: - tests_src = import_module(tests_src_mod_name) - return generate_from_tests( - runner_name='epoch_processing', - handler_name=handler_name, - src=tests_src, - fork_name=fork_name, - ) - - return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) +specs = (spec_phase0, spec_lightclient_patch, spec_phase1) if __name__ == "__main__": @@ -64,10 +39,4 @@ if __name__ == "__main__": PHASE1: phase_1_mods, } - for config_name in ALL_CONFIGS: - for fork_name in TESTGEN_FORKS: - if fork_name in all_mods: - gen_runner.run_generator(f"epoch_processing", get_provider( - create_provider_fn=create_provider, config_name=config_name, - fork_name=fork_name, all_mods=all_mods, - )) + run_state_test_generators(runner_name="epoch_processing", specs=specs, all_mods=all_mods) diff --git a/tests/generators/finality/main.py b/tests/generators/finality/main.py index 6ea3d82c8..0a62b1aa5 100644 --- a/tests/generators/finality/main.py +++ b/tests/generators/finality/main.py @@ -1,38 +1,11 @@ -from typing import Iterable -from importlib import reload, import_module - -from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests, get_provider - -from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, ALL_CONFIGS -from eth2spec.config import config_util +from gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.utils import bls +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH -def create_provider(fork_name: str, handler_name: str, - tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: - - def prepare_fn(configs_path: str) -> str: - config_util.prepare_config(configs_path, config_name) - reload(spec_phase0) - reload(spec_lightclient_patch) - reload(spec_phase1) - bls.use_milagro() - return config_name - - def cases_fn() -> Iterable[gen_typing.TestCase]: - tests_src = import_module(tests_src_mod_name) - return generate_from_tests( - runner_name='finality', - handler_name=handler_name, - src=tests_src, - fork_name=fork_name, - ) - - return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) +specs = (spec_phase0, spec_lightclient_patch, spec_phase1) if __name__ == "__main__": @@ -47,10 +20,4 @@ if __name__ == "__main__": PHASE1: phase_1_mods, } - for config_name in ALL_CONFIGS: - for fork_name in TESTGEN_FORKS: - if fork_name in all_mods: - gen_runner.run_generator(f"finality", get_provider( - create_provider_fn=create_provider, config_name=config_name, - fork_name=fork_name, all_mods=all_mods, - )) + run_state_test_generators(runner_name="finality", specs=specs, all_mods=all_mods) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index 20643f8d7..d40dbe9cd 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -1,36 +1,11 @@ -from typing import Iterable - -from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests, get_provider -from importlib import reload, import_module -from eth2spec.config import config_util +from gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, ALL_CONFIGS -from eth2spec.utils import bls +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH -def create_provider(fork_name: str, handler_name: str, - tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: - def prepare_fn(configs_path: str) -> str: - config_util.prepare_config(configs_path, config_name) - reload(spec_phase0) - reload(spec_lightclient_patch) - reload(spec_phase1) - bls.use_milagro() - return config_name - - def cases_fn() -> Iterable[gen_typing.TestCase]: - tests_src = import_module(tests_src_mod_name) - return generate_from_tests( - runner_name='operations', - handler_name=handler_name, - src=tests_src, - fork_name=fork_name, - ) - - return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) +specs = (spec_phase0, spec_lightclient_patch, spec_phase1) if __name__ == "__main__": @@ -63,10 +38,4 @@ if __name__ == "__main__": PHASE1: phase_1_mods, } - for config_name in ALL_CONFIGS: - for fork_name in TESTGEN_FORKS: - if fork_name in all_mods: - gen_runner.run_generator(f"operations", get_provider( - create_provider_fn=create_provider, config_name=config_name, - fork_name=fork_name, all_mods=all_mods, - )) + run_state_test_generators(runner_name="operations", specs=specs, all_mods=all_mods) diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index cd3206a3f..addb6aef0 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -1,36 +1,11 @@ -from typing import Iterable - -from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests, get_provider -from importlib import reload, import_module -from eth2spec.config import config_util +from gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, ALL_CONFIGS -from eth2spec.utils import bls +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH -def create_provider(fork_name: str, handler_name: str, - tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: - def prepare_fn(configs_path: str) -> str: - config_util.prepare_config(configs_path, config_name) - reload(spec_phase0) - reload(spec_lightclient_patch) - reload(spec_phase1) - bls.use_milagro() - return config_name - - def cases_fn() -> Iterable[gen_typing.TestCase]: - tests_src = import_module(tests_src_mod_name) - return generate_from_tests( - runner_name='rewards', - handler_name=handler_name, - src=tests_src, - fork_name=fork_name, - ) - - return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) +specs = (spec_phase0, spec_lightclient_patch, spec_phase1) if __name__ == "__main__": @@ -49,10 +24,4 @@ if __name__ == "__main__": PHASE1: phase_1_mods, } - for config_name in ALL_CONFIGS: - for fork_name in TESTGEN_FORKS: - if fork_name in all_mods: - gen_runner.run_generator(f"rewards", get_provider( - create_provider_fn=create_provider, config_name=config_name, - fork_name=fork_name, all_mods=all_mods, - )) + run_state_test_generators(runner_name="rewards", specs=specs, all_mods=all_mods) diff --git a/tests/generators/sanity/main.py b/tests/generators/sanity/main.py index 0aabf5d4c..fc5227a53 100644 --- a/tests/generators/sanity/main.py +++ b/tests/generators/sanity/main.py @@ -1,36 +1,11 @@ -from typing import Iterable - -from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests, get_provider -from importlib import reload, import_module -from eth2spec.config import config_util +from gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, ALL_CONFIGS -from eth2spec.utils import bls +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH -def create_provider(fork_name: str, handler_name: str, - tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: - def prepare_fn(configs_path: str) -> str: - config_util.prepare_config(configs_path, config_name) - reload(spec_phase0) - reload(spec_lightclient_patch) - reload(spec_phase1) - bls.use_milagro() - return config_name - - def cases_fn() -> Iterable[gen_typing.TestCase]: - tests_src = import_module(tests_src_mod_name) - return generate_from_tests( - runner_name='sanity', - handler_name=handler_name, - src=tests_src, - fork_name=fork_name, - ) - - return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) +specs = (spec_phase0, spec_lightclient_patch, spec_phase1) if __name__ == "__main__": @@ -52,10 +27,4 @@ if __name__ == "__main__": PHASE1: phase_1_mods, } - for config_name in ALL_CONFIGS: - for fork_name in TESTGEN_FORKS: - if fork_name in all_mods: - gen_runner.run_generator(f"sanity", get_provider( - create_provider_fn=create_provider, config_name=config_name, - fork_name=fork_name, all_mods=all_mods, - )) + run_state_test_generators(runner_name="sanity", specs=specs, all_mods=all_mods) From c9ba641800dfba8a4838d5030260b1b6e5c7efd4 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 15 Feb 2021 11:22:11 -0700 Subject: [PATCH 137/222] note penalties in hf1 list --- specs/lightclient/beacon-chain.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index ccf9b2ebc..bf9b6a092 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -54,11 +54,12 @@ ## Introduction This is a patch implementing the first hard fork to the beacon chain, tentatively named HF1 pending a permanent name. -It has three main features: +It has four main features: * Light client support via sync committees * Incentive accounting reforms, reducing spec complexity and [TODO] reducing the cost of processing chains that have very little or zero participation for a long span of epochs +* Update penalty configuration values, moving them toward their planned maximally punitive configuration * Fork choice rule changes to address weaknesses recently discovered in the existing fork choice ## Custom types From 7050cb0add082f7a406a8d086f332a369b806fa7 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 16 Feb 2021 11:55:01 -0700 Subject: [PATCH 138/222] minor 0x01 PR feedback --- specs/phase0/validator.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 5106c2b73..9e2ee7b1f 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -125,14 +125,16 @@ The `withdrawal_credentials` field must be such that: Withdrawal credentials with the Eth1 address withdrawal prefix specify a 20-byte Eth1 address `eth1_withdrawal_address` as the recipient for all withdrawals. The `eth1_withdrawal_address` can be the address of either an externally owned account or of a contract. + The `withdrawal_credentials` field must be such that: * `withdrawal_credentials[:1] == ETH1_ADDRESS_WITHDRAWAL_PREFIX` * `withdrawal_credentials[1:12] == b'\x00' * 11` * `withdrawal_credentials[12:] == eth1_withdrawal_address` -Withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) -triggered by an Eth1 transaction that will handle the gas price and gas limit, as well the payment of fees. +After the merge of eth1 into eth2, +withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) +triggered by a user transaction that will set the gas price and gas limit as well pay fees. As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers the future withdrawal protocol is agnostic to all other implementation details. From 9cc8567d682ee831adf4a019de31fd0ebbdd30bb Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 18 Feb 2021 15:17:47 +0800 Subject: [PATCH 139/222] Move `gen_helpers` into a module of `eth2spec` package --- .../{ => pyspec/eth2spec}/gen_helpers/README.md | 0 .../eth2spec/gen_helpers}/__init__.py | 0 .../eth2spec/gen_helpers/gen_base}/__init__.py | 0 .../eth2spec}/gen_helpers/gen_base/gen_runner.py | 13 +++++++------ .../eth2spec}/gen_helpers/gen_base/gen_typing.py | 0 .../eth2spec/gen_helpers/gen_from_tests/__init__.py | 0 .../eth2spec}/gen_helpers/gen_from_tests/gen.py | 13 +++++++------ .../eth2spec}/gen_helpers/requirements.txt | 0 .../core/{ => pyspec/eth2spec}/gen_helpers/setup.py | 0 tests/generators/README.md | 2 +- tests/generators/bls/main.py | 2 +- tests/generators/bls/requirements.txt | 4 +--- tests/generators/epoch_processing/main.py | 2 +- tests/generators/epoch_processing/requirements.txt | 4 ++-- tests/generators/finality/main.py | 2 +- tests/generators/finality/requirements.txt | 4 ++-- tests/generators/genesis/main.py | 4 ++-- tests/generators/genesis/requirements.txt | 4 ++-- tests/generators/operations/main.py | 2 +- tests/generators/operations/requirements.txt | 5 ++--- tests/generators/rewards/main.py | 2 +- tests/generators/rewards/requirements.txt | 4 ++-- tests/generators/sanity/main.py | 3 ++- tests/generators/sanity/requirements.txt | 4 ++-- tests/generators/shuffling/main.py | 2 +- tests/generators/ssz_generic/main.py | 2 +- tests/generators/ssz_generic/requirements.txt | 3 +-- tests/generators/ssz_static/main.py | 2 +- tests/generators/ssz_static/requirements.txt | 4 ++-- 29 files changed, 43 insertions(+), 44 deletions(-) rename tests/core/{ => pyspec/eth2spec}/gen_helpers/README.md (100%) rename tests/core/{gen_helpers/gen_base => pyspec/eth2spec/gen_helpers}/__init__.py (100%) rename tests/core/{gen_helpers/gen_from_tests => pyspec/eth2spec/gen_helpers/gen_base}/__init__.py (100%) rename tests/core/{ => pyspec/eth2spec}/gen_helpers/gen_base/gen_runner.py (95%) rename tests/core/{ => pyspec/eth2spec}/gen_helpers/gen_base/gen_typing.py (100%) create mode 100644 tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/__init__.py rename tests/core/{ => pyspec/eth2spec}/gen_helpers/gen_from_tests/gen.py (92%) rename tests/core/{ => pyspec/eth2spec}/gen_helpers/requirements.txt (100%) rename tests/core/{ => pyspec/eth2spec}/gen_helpers/setup.py (100%) diff --git a/tests/core/gen_helpers/README.md b/tests/core/pyspec/eth2spec/gen_helpers/README.md similarity index 100% rename from tests/core/gen_helpers/README.md rename to tests/core/pyspec/eth2spec/gen_helpers/README.md diff --git a/tests/core/gen_helpers/gen_base/__init__.py b/tests/core/pyspec/eth2spec/gen_helpers/__init__.py similarity index 100% rename from tests/core/gen_helpers/gen_base/__init__.py rename to tests/core/pyspec/eth2spec/gen_helpers/__init__.py diff --git a/tests/core/gen_helpers/gen_from_tests/__init__.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/__init__.py similarity index 100% rename from tests/core/gen_helpers/gen_from_tests/__init__.py rename to tests/core/pyspec/eth2spec/gen_helpers/gen_base/__init__.py diff --git a/tests/core/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py similarity index 95% rename from tests/core/gen_helpers/gen_base/gen_runner.py rename to tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index a22073c00..26427c6f8 100644 --- a/tests/core/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -8,11 +8,11 @@ from ruamel.yaml import ( YAML, ) -from gen_base.gen_typing import TestProvider - from eth2spec.test import context from eth2spec.test.exceptions import SkippedTest +from .gen_typing import TestProvider + # Flag that the runner does NOT run test via pytest context.is_pytest = False @@ -119,10 +119,11 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): print(f"generating tests with config '{config_name}' ...") for test_case in tprov.make_cases(): - case_dir = Path(output_dir) / Path(config_name) / Path(test_case.fork_name) \ - / Path(test_case.runner_name) / Path(test_case.handler_name) \ - / Path(test_case.suite_name) / Path(test_case.case_name) - + case_dir = ( + Path(output_dir) / Path(config_name) / Path(test_case.fork_name) + / Path(test_case.runner_name) / Path(test_case.handler_name) + / Path(test_case.suite_name) / Path(test_case.case_name) + ) if case_dir.exists(): if not args.force: print(f'Skipping already existing test: {case_dir}') diff --git a/tests/core/gen_helpers/gen_base/gen_typing.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_typing.py similarity index 100% rename from tests/core/gen_helpers/gen_base/gen_typing.py rename to tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_typing.py diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/__init__.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/gen_helpers/gen_from_tests/gen.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py similarity index 92% rename from tests/core/gen_helpers/gen_from_tests/gen.py rename to tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py index 3ec8904fc..f5a0f378b 100644 --- a/tests/core/gen_helpers/gen_from_tests/gen.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py @@ -4,10 +4,10 @@ from typing import Any, Callable, Dict, Iterable from eth2spec.config import config_util from eth2spec.utils import bls - from eth2spec.test.context import ALL_CONFIGS, TESTGEN_FORKS, SpecForkName, ConfigName -from gen_base import gen_runner -from gen_base.gen_typing import TestCase, TestProvider + +from eth2spec.gen_helpers.gen_base import gen_runner +from eth2spec.gen_helpers.gen_base.gen_typing import TestCase, TestProvider def generate_from_tests(runner_name: str, handler_name: str, src: Any, @@ -56,11 +56,12 @@ def get_provider(create_provider_fn: Callable[[SpecForkName, str, str, ConfigNam handler_name=key, tests_src_mod_name=mod_name, config_name=config_name, - ) + ) -def get_create_provider_fn(runner_name: str, config_name: ConfigName, specs: Iterable[Any] - ) -> Callable[[SpecForkName, str, str, ConfigName], TestProvider]: +def get_create_provider_fn( + runner_name: str, config_name: ConfigName, specs: Iterable[Any] +) -> Callable[[SpecForkName, str, str, ConfigName], TestProvider]: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) for spec in specs: diff --git a/tests/core/gen_helpers/requirements.txt b/tests/core/pyspec/eth2spec/gen_helpers/requirements.txt similarity index 100% rename from tests/core/gen_helpers/requirements.txt rename to tests/core/pyspec/eth2spec/gen_helpers/requirements.txt diff --git a/tests/core/gen_helpers/setup.py b/tests/core/pyspec/eth2spec/gen_helpers/setup.py similarity index 100% rename from tests/core/gen_helpers/setup.py rename to tests/core/pyspec/eth2spec/gen_helpers/setup.py diff --git a/tests/generators/README.md b/tests/generators/README.md index 077a8443c..b819d93b8 100644 --- a/tests/generators/README.md +++ b/tests/generators/README.md @@ -103,7 +103,7 @@ Write a `main.py` file. The shuffling test generator is a good minimal starting ```python from eth2spec.phase0 import spec as spec from eth_utils import to_tuple -from gen_base import gen_runner, gen_typing +from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from preset_loader import loader from typing import Iterable diff --git a/tests/generators/bls/main.py b/tests/generators/bls/main.py index 6552b8654..fecc2df7d 100644 --- a/tests/generators/bls/main.py +++ b/tests/generators/bls/main.py @@ -13,7 +13,7 @@ import milagro_bls_binding as milagro_bls from eth2spec.utils import bls from eth2spec.test.context import PHASE0 -from gen_base import gen_runner, gen_typing +from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing def to_bytes(i): diff --git a/tests/generators/bls/requirements.txt b/tests/generators/bls/requirements.txt index 5f830773a..df386450b 100644 --- a/tests/generators/bls/requirements.txt +++ b/tests/generators/bls/requirements.txt @@ -1,4 +1,2 @@ -py_ecc==5.1.0 -eth-utils==1.6.0 -../../core/gen_helpers +pytest>=4.4 ../../../ diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index 1306da523..50a1e2b57 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -1,4 +1,4 @@ -from gen_from_tests.gen import run_state_test_generators +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 diff --git a/tests/generators/epoch_processing/requirements.txt b/tests/generators/epoch_processing/requirements.txt index b82314298..df386450b 100644 --- a/tests/generators/epoch_processing/requirements.txt +++ b/tests/generators/epoch_processing/requirements.txt @@ -1,2 +1,2 @@ -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../ diff --git a/tests/generators/finality/main.py b/tests/generators/finality/main.py index 0a62b1aa5..8b961f9f4 100644 --- a/tests/generators/finality/main.py +++ b/tests/generators/finality/main.py @@ -1,4 +1,4 @@ -from gen_from_tests.gen import run_state_test_generators +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 diff --git a/tests/generators/finality/requirements.txt b/tests/generators/finality/requirements.txt index b82314298..df386450b 100644 --- a/tests/generators/finality/requirements.txt +++ b/tests/generators/finality/requirements.txt @@ -1,2 +1,2 @@ -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../ diff --git a/tests/generators/genesis/main.py b/tests/generators/genesis/main.py index ce055b44a..854af6572 100644 --- a/tests/generators/genesis/main.py +++ b/tests/generators/genesis/main.py @@ -3,8 +3,8 @@ from typing import Iterable from eth2spec.test.context import PHASE0 from eth2spec.test.phase0.genesis import test_initialization, test_validity -from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests +from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing +from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests from eth2spec.phase0 import spec as spec from importlib import reload from eth2spec.config import config_util diff --git a/tests/generators/genesis/requirements.txt b/tests/generators/genesis/requirements.txt index b82314298..df386450b 100644 --- a/tests/generators/genesis/requirements.txt +++ b/tests/generators/genesis/requirements.txt @@ -1,2 +1,2 @@ -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../ diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index d40dbe9cd..00a58288f 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -1,4 +1,4 @@ -from gen_from_tests.gen import run_state_test_generators +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 diff --git a/tests/generators/operations/requirements.txt b/tests/generators/operations/requirements.txt index a6ea61aea..df386450b 100644 --- a/tests/generators/operations/requirements.txt +++ b/tests/generators/operations/requirements.txt @@ -1,3 +1,2 @@ -eth-utils==1.6.0 -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../ diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index addb6aef0..4124587cc 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -1,4 +1,4 @@ -from gen_from_tests.gen import run_state_test_generators +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 diff --git a/tests/generators/rewards/requirements.txt b/tests/generators/rewards/requirements.txt index b82314298..df386450b 100644 --- a/tests/generators/rewards/requirements.txt +++ b/tests/generators/rewards/requirements.txt @@ -1,2 +1,2 @@ -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../ diff --git a/tests/generators/sanity/main.py b/tests/generators/sanity/main.py index fc5227a53..5155798ff 100644 --- a/tests/generators/sanity/main.py +++ b/tests/generators/sanity/main.py @@ -1,9 +1,10 @@ -from gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators + specs = (spec_phase0, spec_lightclient_patch, spec_phase1) diff --git a/tests/generators/sanity/requirements.txt b/tests/generators/sanity/requirements.txt index b82314298..df386450b 100644 --- a/tests/generators/sanity/requirements.txt +++ b/tests/generators/sanity/requirements.txt @@ -1,2 +1,2 @@ -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../ diff --git a/tests/generators/shuffling/main.py b/tests/generators/shuffling/main.py index 6069de77a..091207bca 100644 --- a/tests/generators/shuffling/main.py +++ b/tests/generators/shuffling/main.py @@ -2,7 +2,7 @@ from eth_utils import to_tuple from typing import Iterable from importlib import reload -from gen_base import gen_runner, gen_typing +from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.config import config_util from eth2spec.phase0 import spec as spec diff --git a/tests/generators/ssz_generic/main.py b/tests/generators/ssz_generic/main.py index 8cfb2e3eb..737f8cda6 100644 --- a/tests/generators/ssz_generic/main.py +++ b/tests/generators/ssz_generic/main.py @@ -1,5 +1,5 @@ from typing import Iterable -from gen_base import gen_runner, gen_typing +from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing import ssz_basic_vector import ssz_bitlist import ssz_bitvector diff --git a/tests/generators/ssz_generic/requirements.txt b/tests/generators/ssz_generic/requirements.txt index af061a3b1..df386450b 100644 --- a/tests/generators/ssz_generic/requirements.txt +++ b/tests/generators/ssz_generic/requirements.txt @@ -1,3 +1,2 @@ -eth-utils==1.6.0 -../../core/gen_helpers +pytest>=4.4 ../../../ diff --git a/tests/generators/ssz_static/main.py b/tests/generators/ssz_static/main.py index 38ff18615..8b18705e7 100644 --- a/tests/generators/ssz_static/main.py +++ b/tests/generators/ssz_static/main.py @@ -3,7 +3,7 @@ from typing import Iterable from importlib import reload from inspect import getmembers, isclass -from gen_base import gen_runner, gen_typing +from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.debug import random_value, encode from eth2spec.config import config_util diff --git a/tests/generators/ssz_static/requirements.txt b/tests/generators/ssz_static/requirements.txt index b82314298..df386450b 100644 --- a/tests/generators/ssz_static/requirements.txt +++ b/tests/generators/ssz_static/requirements.txt @@ -1,2 +1,2 @@ -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../ From e58dcb40acc4348047e681fc069b1c7abb33fa16 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 18 Feb 2021 17:51:01 +0800 Subject: [PATCH 140/222] Clean up and kick the cache --- setup.py | 3 +-- .../core/pyspec/eth2spec/gen_helpers/requirements.txt | 3 --- tests/core/pyspec/eth2spec/gen_helpers/setup.py | 11 ----------- 3 files changed, 1 insertion(+), 16 deletions(-) delete mode 100644 tests/core/pyspec/eth2spec/gen_helpers/requirements.txt delete mode 100644 tests/core/pyspec/eth2spec/gen_helpers/setup.py diff --git a/setup.py b/setup.py index 6b8520075..9c2696006 100644 --- a/setup.py +++ b/setup.py @@ -562,13 +562,12 @@ setup( url="https://github.com/ethereum/eth2.0-specs", include_package_data=False, package_data={'configs': ['*.yaml'], - 'specs': ['**/*.md'], 'eth2spec': ['VERSION.txt']}, package_dir={ "eth2spec": "tests/core/pyspec/eth2spec", "configs": "configs", - "specs": "specs" + "specs": "specs", }, packages=find_packages(where='tests/core/pyspec') + ['configs', 'specs'], py_modules=["eth2spec"], diff --git a/tests/core/pyspec/eth2spec/gen_helpers/requirements.txt b/tests/core/pyspec/eth2spec/gen_helpers/requirements.txt deleted file mode 100644 index e7cdd30ea..000000000 --- a/tests/core/pyspec/eth2spec/gen_helpers/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -ruamel.yaml==0.16.5 -eth-utils==1.6.0 -pytest>=4.4 diff --git a/tests/core/pyspec/eth2spec/gen_helpers/setup.py b/tests/core/pyspec/eth2spec/gen_helpers/setup.py deleted file mode 100644 index e9fc1a787..000000000 --- a/tests/core/pyspec/eth2spec/gen_helpers/setup.py +++ /dev/null @@ -1,11 +0,0 @@ -from distutils.core import setup - -setup( - name='gen_helpers', - packages=['gen_base', 'gen_from_tests'], - install_requires=[ - "ruamel.yaml==0.16.5", - "eth-utils==1.6.0", - "pytest>=4.4", - ] -) From 27507fb3e220f41e640eba19fd2e152a23af2986 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 19 Feb 2021 12:34:12 +0800 Subject: [PATCH 141/222] Add `get_head` test vectors --- .../eth2spec/test/helpers/fork_choice.py | 31 +- .../test/phase0/fork_choice/__init__.py | 0 .../test/phase0/fork_choice/test_get_head.py | 288 ++++++++++++++++++ tests/generators/fork_choice/README.md | 3 + tests/generators/fork_choice/main.py | 40 +++ tests/generators/fork_choice/requirements.txt | 2 + 6 files changed, 361 insertions(+), 3 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/phase0/fork_choice/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_head.py create mode 100644 tests/generators/fork_choice/README.md create mode 100644 tests/generators/fork_choice/main.py create mode 100644 tests/generators/fork_choice/requirements.txt diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py index 85437e98a..040bee975 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py @@ -1,3 +1,5 @@ +from eth_utils import encode_hex + from eth2spec.phase0 import spec as phase0_spec @@ -8,17 +10,25 @@ def get_anchor_root(spec, state): return spec.hash_tree_root(anchor_block_header) -def add_block_to_store(spec, store, signed_block): +def add_block_to_store(spec, store, signed_block, test_steps=None): + if test_steps is None: + test_steps = [] + pre_state = store.block_states[signed_block.message.parent_root] block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT if store.time < block_time: spec.on_tick(store, block_time) + test_steps.append({'tick': int(block_time)}) spec.on_block(store, signed_block) + test_steps.append({'block': get_block_file_name(signed_block)}) -def add_attestation_to_store(spec, store, attestation): +def add_attestation_to_store(spec, store, attestation, test_steps=None): + if test_steps is None: + test_steps = [] + parent_block = store.blocks[attestation.data.beacon_block_root] pre_state = store.block_states[spec.hash_tree_root(parent_block)] block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT @@ -26,12 +36,27 @@ def add_attestation_to_store(spec, store, attestation): if store.time < next_epoch_time: spec.on_tick(store, next_epoch_time) + test_steps.append({'tick': int(next_epoch_time)}) spec.on_attestation(store, attestation) + test_steps.append({'attestation': get_attestation_file_name(attestation)}) def get_genesis_forkchoice_store(spec, genesis_state): + store, _ = get_genesis_forkchoice_store_and_block(spec, genesis_state) + return store + + +def get_genesis_forkchoice_store_and_block(spec, genesis_state): assert genesis_state.slot == spec.GENESIS_SLOT # The genesis block must be a Phase 0 `BeaconBlock` genesis_block = phase0_spec.BeaconBlock(state_root=genesis_state.hash_tree_root()) - return spec.get_forkchoice_store(genesis_state, genesis_block) + return spec.get_forkchoice_store(genesis_state, genesis_block), genesis_block + + +def get_block_file_name(block): + return f"block_{encode_hex(block.hash_tree_root())}" + + +def get_attestation_file_name(attestation): + return f"attestation_{encode_hex(attestation.hash_tree_root())}" diff --git a/tests/core/pyspec/eth2spec/test/phase0/fork_choice/__init__.py b/tests/core/pyspec/eth2spec/test/phase0/fork_choice/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_head.py b/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_head.py new file mode 100644 index 000000000..6648000fa --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_head.py @@ -0,0 +1,288 @@ +from eth_utils import encode_hex + +from eth2spec.test.context import MINIMAL, with_all_phases, with_configs, spec_state_test +from eth2spec.test.helpers.attestations import get_valid_attestation, next_epoch_with_attestations +from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.helpers.fork_choice import ( + add_attestation_to_store, + add_block_to_store, get_anchor_root, + get_genesis_forkchoice_store_and_block, + get_attestation_file_name, + get_block_file_name, +) +from eth2spec.test.helpers.state import ( + next_epoch, + state_transition_and_sign_block, +) + + +@with_all_phases +@with_configs([MINIMAL]) +@spec_state_test +def test_genesis(spec, state): + test_steps = [] + # Initialization + store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state) + yield 'anchor_state', state + yield 'anchor_block', anchor_block + + anchor_root = get_anchor_root(spec, state) + head = spec.get_head(store) + assert head == anchor_root + test_steps.append({ + 'checks': { + 'head': encode_hex(head) + } + }) + + yield 'steps', test_steps + + +@with_all_phases +@with_configs([MINIMAL]) +@spec_state_test +def test_chain_no_attestations(spec, state): + test_steps = [] + # Initialization + store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state) + yield 'anchor_state', state + yield 'anchor_block', anchor_block + + anchor_root = get_anchor_root(spec, state) + head = spec.get_head(store) + assert head == anchor_root + test_steps.append({ + 'checks': { + 'head': encode_hex(head) + } + }) + + # On receiving a block of `GENESIS_SLOT + 1` slot + block_1 = build_empty_block_for_next_slot(spec, state) + signed_block_1 = state_transition_and_sign_block(spec, state, block_1) + add_block_to_store(spec, store, signed_block_1, test_steps) + yield get_block_file_name(signed_block_1), signed_block_1 + + # On receiving a block of next epoch + block_2 = build_empty_block_for_next_slot(spec, state) + signed_block_2 = state_transition_and_sign_block(spec, state, block_2) + add_block_to_store(spec, store, signed_block_2, test_steps) + yield get_block_file_name(signed_block_2), signed_block_2 + + head = spec.get_head(store) + assert head == spec.hash_tree_root(block_2) + test_steps.append({ + 'checks': { + 'head': encode_hex(head) + } + }) + + yield 'steps', test_steps + + +@with_all_phases +@with_configs([MINIMAL]) +@spec_state_test +def test_split_tie_breaker_no_attestations(spec, state): + test_steps = [] + genesis_state = state.copy() + + # Initialization + store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state) + yield 'anchor_state', state + yield 'anchor_block', anchor_block + anchor_root = get_anchor_root(spec, state) + head = spec.get_head(store) + assert head == anchor_root + test_steps.append({ + 'checks': { + 'head': encode_hex(head) + } + }) + + # block at slot 1 + block_1_state = genesis_state.copy() + block_1 = build_empty_block_for_next_slot(spec, block_1_state) + signed_block_1 = state_transition_and_sign_block(spec, block_1_state, block_1) + add_block_to_store(spec, store, signed_block_1, test_steps) + yield get_block_file_name(signed_block_1), signed_block_1 + + # additional block at slot 1 + block_2_state = genesis_state.copy() + block_2 = build_empty_block_for_next_slot(spec, block_2_state) + block_2.body.graffiti = b'\x42' * 32 + signed_block_2 = state_transition_and_sign_block(spec, block_2_state, block_2) + add_block_to_store(spec, store, signed_block_2, test_steps) + yield get_block_file_name(signed_block_2), signed_block_2 + + highest_root = max(spec.hash_tree_root(block_1), spec.hash_tree_root(block_2)) + head = spec.get_head(store) + assert head == highest_root + test_steps.append({ + 'checks': { + 'head': encode_hex(head) + } + }) + + yield 'steps', test_steps + + +@with_all_phases +@spec_state_test +def test_shorter_chain_but_heavier_weight(spec, state): + test_steps = [] + genesis_state = state.copy() + + # Initialization + store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state) + yield 'anchor_state', state + yield 'anchor_block', anchor_block + anchor_root = get_anchor_root(spec, state) + head = spec.get_head(store) + assert head == anchor_root + test_steps.append({ + 'checks': { + 'head': encode_hex(head) + } + }) + + # build longer tree + long_state = genesis_state.copy() + for _ in range(3): + long_block = build_empty_block_for_next_slot(spec, long_state) + signed_long_block = state_transition_and_sign_block(spec, long_state, long_block) + add_block_to_store(spec, store, signed_long_block, test_steps) + yield get_block_file_name(signed_long_block), signed_long_block + + # build short tree + short_state = genesis_state.copy() + short_block = build_empty_block_for_next_slot(spec, short_state) + short_block.body.graffiti = b'\x42' * 32 + signed_short_block = state_transition_and_sign_block(spec, short_state, short_block) + add_block_to_store(spec, store, signed_short_block, test_steps) + yield get_block_file_name(signed_short_block), signed_short_block + + short_attestation = get_valid_attestation(spec, short_state, short_block.slot, signed=True) + add_attestation_to_store(spec, store, short_attestation, test_steps) + + head = spec.get_head(store) + assert head == spec.hash_tree_root(short_block) + test_steps.append({ + 'checks': { + 'head': encode_hex(head) + } + }) + + yield 'steps', test_steps + + +@with_all_phases +@spec_state_test +def test_filtered_block_tree(spec, state): + test_steps = [] + # Initialization + store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state) + yield 'anchor_state', state + yield 'anchor_block', anchor_block + + anchor_root = get_anchor_root(spec, state) + + # transition state past initial couple of epochs + next_epoch(spec, state) + next_epoch(spec, state) + + head = spec.get_head(store) + assert head == anchor_root + test_steps.append({ + 'checks': { + 'head': encode_hex(head) + } + }) + + # fill in attestations for entire epoch, justifying the recent epoch + prev_state, signed_blocks, state = next_epoch_with_attestations(spec, state, True, False) + attestations = [ + attestation for signed_block in signed_blocks + for attestation in signed_block.message.body.attestations + ] + assert state.current_justified_checkpoint.epoch > prev_state.current_justified_checkpoint.epoch + + # tick time forward and add blocks and attestations to store + current_time = state.slot * spec.SECONDS_PER_SLOT + store.genesis_time + spec.on_tick(store, current_time) + for signed_block in signed_blocks: + spec.on_block(store, signed_block) + test_steps.append({'block': get_block_file_name(signed_block)}) + yield get_block_file_name(signed_block), signed_block + + for attestation in attestations: + spec.on_attestation(store, attestation) + test_steps.append({'attestation': get_attestation_file_name(attestation)}) + yield get_attestation_file_name(attestation), attestation + + assert store.justified_checkpoint == state.current_justified_checkpoint + + # the last block in the branch should be the head + expected_head_root = spec.hash_tree_root(signed_blocks[-1].message) + head = spec.get_head(store) + assert head == expected_head_root + + test_steps.append({ + 'checks': { + 'justified_checkpoint_root': encode_hex(store.justified_checkpoint.hash_tree_root()), + 'head': encode_hex(head), + } + }) + + # + # create branch containing the justified block but not containing enough on + # chain votes to justify that block + # + + # build a chain without attestations off of previous justified block + non_viable_state = store.block_states[store.justified_checkpoint.root].copy() + + # ensure that next wave of votes are for future epoch + next_epoch(spec, non_viable_state) + next_epoch(spec, non_viable_state) + next_epoch(spec, non_viable_state) + assert spec.get_current_epoch(non_viable_state) > store.justified_checkpoint.epoch + + # create rogue block that will be attested to in this non-viable branch + rogue_block = build_empty_block_for_next_slot(spec, non_viable_state) + signed_rogue_block = state_transition_and_sign_block(spec, non_viable_state, rogue_block) + + # create an epoch's worth of attestations for the rogue block + next_epoch(spec, non_viable_state) + attestations = [] + for i in range(spec.SLOTS_PER_EPOCH): + slot = rogue_block.slot + i + for index in range(spec.get_committee_count_per_slot(non_viable_state, spec.compute_epoch_at_slot(slot))): + attestation = get_valid_attestation(spec, non_viable_state, slot, index, signed=True) + attestations.append(attestation) + + # tick time forward to be able to include up to the latest attestation + current_time = (attestations[-1].data.slot + 1) * spec.SECONDS_PER_SLOT + store.genesis_time + spec.on_tick(store, current_time) + test_steps.append({'tick': int(current_time)}) + + # include rogue block and associated attestations in the store + spec.on_block(store, signed_rogue_block) + test_steps.append({'block': get_block_file_name(signed_rogue_block)}) + yield get_block_file_name(signed_rogue_block), signed_rogue_block + + for attestation in attestations: + spec.on_attestation(store, attestation) + test_steps.append({'attestation': get_attestation_file_name(attestation)}) + yield get_attestation_file_name(attestation), attestation + + # ensure that get_head still returns the head from the previous branch + head = spec.get_head(store) + assert head == expected_head_root + test_steps.append({ + 'checks': { + 'head': encode_hex(head) + } + }) + + yield 'steps', test_steps diff --git a/tests/generators/fork_choice/README.md b/tests/generators/fork_choice/README.md new file mode 100644 index 000000000..e450ca92f --- /dev/null +++ b/tests/generators/fork_choice/README.md @@ -0,0 +1,3 @@ +# Fork choice tests + +TODO diff --git a/tests/generators/fork_choice/main.py b/tests/generators/fork_choice/main.py new file mode 100644 index 000000000..f684399eb --- /dev/null +++ b/tests/generators/fork_choice/main.py @@ -0,0 +1,40 @@ +from typing import Iterable + +from gen_base import gen_runner, gen_typing +from gen_from_tests.gen import generate_from_tests +from importlib import reload, import_module +from eth2spec.config import config_util +from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.test.context import PHASE0 +from eth2spec.utils import bls + + +def create_provider(fork_name: str, handler_name: str, + tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: + def prepare_fn(configs_path: str) -> str: + config_util.prepare_config(configs_path, config_name) + reload(spec_phase0) + bls.use_milagro() + return config_name + + def cases_fn() -> Iterable[gen_typing.TestCase]: + tests_src = import_module(tests_src_mod_name) + return generate_from_tests( + runner_name='fork_choice', + handler_name=handler_name, + src=tests_src, + fork_name=fork_name, + ) + + return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) + + +if __name__ == "__main__": + phase_0_mods = {key: 'eth2spec.test.phase0.fork_choice.test_' + key for key in [ + 'get_head', + ]} + + # TODO: add other configs and forks + gen_runner.run_generator(f"fork_choice", [ + create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items() + ]) diff --git a/tests/generators/fork_choice/requirements.txt b/tests/generators/fork_choice/requirements.txt new file mode 100644 index 000000000..b82314298 --- /dev/null +++ b/tests/generators/fork_choice/requirements.txt @@ -0,0 +1,2 @@ +../../core/gen_helpers +../../../ \ No newline at end of file From a28b2d73493f8a978c6afec967156a752fe87fbd Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 20 Feb 2021 15:29:34 +0000 Subject: [PATCH 142/222] Bump Milagro dependency for M1 support --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6b8520075..df95d99d3 100644 --- a/setup.py +++ b/setup.py @@ -583,7 +583,7 @@ setup( "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.9.4", "py_ecc==5.1.0", - "milagro_bls_binding==1.6.2", + "milagro_bls_binding==1.6.3", "dataclasses==0.6", "remerkleable==0.1.18", "ruamel.yaml==0.16.5", From de4cad5d3596abbcb0f3bb74d9cd478933b89f2e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 22 Feb 2021 18:29:31 +0800 Subject: [PATCH 143/222] Update docs --- .../pyspec/eth2spec/gen_helpers/README.md | 2 +- .../gen_helpers/gen_from_tests/gen.py | 3 + tests/generators/README.md | 57 ++++++++++--------- 3 files changed, 34 insertions(+), 28 deletions(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/README.md b/tests/core/pyspec/eth2spec/gen_helpers/README.md index 20b48db83..d39ee66ae 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/README.md +++ b/tests/core/pyspec/eth2spec/gen_helpers/README.md @@ -4,7 +4,7 @@ A util to quickly write new test suite generators with. -See [Generators documentation](../../generators/README.md) for integration details. +See [Generators documentation](../../../../generators/README.md) for integration details. Options: diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py index f5a0f378b..67d29b194 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py @@ -85,6 +85,9 @@ def get_create_provider_fn( def run_state_test_generators(runner_name: str, specs: Iterable[Any], all_mods: Dict[str, Dict[str, str]]) -> None: + """ + Generate all available state tests of `TESTGEN_FORKS` forks of `ALL_CONFIGS` configs of the given runner. + """ for config_name in ALL_CONFIGS: for fork_name in TESTGEN_FORKS: if fork_name in all_mods: diff --git a/tests/generators/README.md b/tests/generators/README.md index b819d93b8..26094bbae 100644 --- a/tests/generators/README.md +++ b/tests/generators/README.md @@ -78,9 +78,8 @@ It's recommended to extend the base-generator. Create a `requirements.txt` in the root of your generator directory: ``` -../../core/gen_helpers -../../core/config_helpers -../../core/pyspec +pytest>=4.4 +../../../ ``` The config helper and pyspec is optional, but preferred. We encourage generators to derive tests from the spec itself in order to prevent code duplication and outdated tests. @@ -163,35 +162,40 @@ To extend this, one could decide to parametrize the `shuffling_test_cases` funct Another example, to generate tests from pytests: ```python -def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: +from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.lightclient_patch import spec as spec_lightclient_patch +from eth2spec.phase1 import spec as spec_phase1 +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH - def prepare_fn(configs_path: str) -> str: - presets = loader.load_presets(configs_path, config_name) - spec_phase0.apply_constants_preset(presets) - spec_phase1.apply_constants_preset(presets) - return config_name +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators - def cases_fn() -> Iterable[gen_typing.TestCase]: - return generate_from_tests( - runner_name='epoch_processing', - handler_name=handler_name, - src=tests_src, - fork_name='phase0' - ) - return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) +specs = (spec_phase0, spec_lightclient_patch, spec_phase1) if __name__ == "__main__": - gen_runner.run_generator("epoch_processing", [ - create_provider('justification_and_finalization', test_process_justification_and_finalization, 'minimal'), - ... - ]) + phase_0_mods = {key: 'eth2spec.test.phase0.sanity.test_' + key for key in [ + 'blocks', + 'slots', + ]} + lightclient_patch_mods = {**{key: 'eth2spec.test.lightclient_patch.sanity.test_' + key for key in [ + 'blocks', + ]}, **phase_0_mods} # also run the previous phase 0 tests + phase_1_mods = {**{key: 'eth2spec.test.phase1.sanity.test_' + key for key in [ + 'blocks', # more phase 1 specific block tests + 'shard_blocks', + ]}, **phase_0_mods} # also run the previous phase 0 tests (but against phase 1 spec) + all_mods = { + PHASE0: phase_0_mods, + LIGHTCLIENT_PATCH: lightclient_patch_mods, + PHASE1: phase_1_mods, + } + + run_state_test_generators(runner_name="sanity", specs=specs, all_mods=all_mods) ``` -Here multiple phases load the configuration, and the stream of test cases is derived from a pytest file using the `generate_from_tests` utility. - +Here multiple phases load the configuration, and the stream of test cases is derived from a pytest file using the `eth2spec.gen_helpers.gen_from_tests.gen.run_state_test_generators` utility. Note that this helper generates all available tests of `TESTGEN_FORKS` forks of `ALL_CONFIGS` configs of the given runner. Recommendations: - You can have more than just one test provider. @@ -200,8 +204,7 @@ Recommendations: - Use config `minimal` for performance and simplicity, but also implement a suite with the `mainnet` config where necessary. - You may be able to write your test case provider in a way where it does not make assumptions on constants. If so, you can generate test cases with different configurations for the same scenario (see example). -- See [`tests/core/gen_helpers/README.md`](../core/gen_helpers/README.md) for command line options for generators. - +- See [`tests/core/gen_helpers/README.md`](../core/pyspec/eth2spec/gen_helpers/README.md) for command line options for generators. ## How to add a new test generator @@ -216,8 +219,8 @@ To add a new test generator that builds `New Tests`: 4. Your generator is called with `-o some/file/path/for_testing/can/be_anything -c some/other/path/to_configs/`. The base generator helps you handle this; you only have to define test case providers. 5. Finally, add any linting or testing commands to the - [circleci config file](../.circleci/config.yml) if desired to increase code quality. - Or add it to the [`Makefile`](../Makefile), if it can be run locally. + [circleci config file](../../.circleci/config.yml) if desired to increase code quality. + Or add it to the [`Makefile`](../../Makefile), if it can be run locally. *Note*: You do not have to change the makefile. However, if necessary (e.g. not using Python, or mixing in other languages), submit an issue, and it can be a special case. From f6b8171350f702501c997abf9bd837489b89dfb6 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 22 Feb 2021 19:45:10 +0800 Subject: [PATCH 144/222] Update shuffling generator requirements.txt --- tests/generators/shuffling/requirements.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/generators/shuffling/requirements.txt b/tests/generators/shuffling/requirements.txt index a6ea61aea..df386450b 100644 --- a/tests/generators/shuffling/requirements.txt +++ b/tests/generators/shuffling/requirements.txt @@ -1,3 +1,2 @@ -eth-utils==1.6.0 -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../ From 9f634dc6f6d269339d8e5fd7d069286fa75c2687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marin=20Petruni=C4=87?= Date: Tue, 23 Feb 2021 09:56:38 +0100 Subject: [PATCH 145/222] Update test generation docs --- tests/generators/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/generators/README.md b/tests/generators/README.md index 26094bbae..7d0b5240c 100644 --- a/tests/generators/README.md +++ b/tests/generators/README.md @@ -36,7 +36,7 @@ Prerequisites: ### Cleaning -This removes the existing virtual environments (`/test_generators//venv`) and generated tests (`/yaml_tests/`). +This removes the existing virtual environments (`/tests/generators//venv`) and generated tests (`../eth2.0-spec-tests/tests`). ```bash make clean @@ -47,7 +47,7 @@ make clean This runs all of the generators. ```bash -make -j 4 gen_yaml_tests +make -j 4 generate_tests ``` The `-j N` flag makes the generators run in parallel, with `N` being the amount of cores. @@ -55,7 +55,7 @@ The `-j N` flag makes the generators run in parallel, with `N` being the amount ### Running a single generator -The makefile auto-detects generators in the `test_generators` directory and provides a tests-gen target for each generator. See example: +The makefile auto-detects generators in the `tests/generators` directory and provides a tests-gen target for each generator. See example: ```bash make ./eth2.0-spec-tests/tests/shuffling/ @@ -210,7 +210,7 @@ Recommendations: To add a new test generator that builds `New Tests`: -1. Create a new directory `new_tests` within the `test_generators` directory. +1. Create a new directory `new_tests` within the `tests/generators` directory. Note that `new_tests` is also the name of the directory in which the tests will appear in the tests repository later. 2. Your generator is assumed to have a `requirements.txt` file, with any dependencies it may need. Leave it empty if your generator has none. From e63754e6293b73cf056c3e5edc54569dba35439e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marin=20Petruni=C4=87?= Date: Tue, 23 Feb 2021 13:23:32 +0100 Subject: [PATCH 146/222] change how generator targets are invoked --- tests/generators/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/generators/README.md b/tests/generators/README.md index 7d0b5240c..0f1ed478f 100644 --- a/tests/generators/README.md +++ b/tests/generators/README.md @@ -55,10 +55,10 @@ The `-j N` flag makes the generators run in parallel, with `N` being the amount ### Running a single generator -The makefile auto-detects generators in the `tests/generators` directory and provides a tests-gen target for each generator. See example: +The makefile auto-detects generators in the `tests/generators` directory and provides a tests-gen target (gen_) for each generator. See example: ```bash -make ./eth2.0-spec-tests/tests/shuffling/ +make gen_ssz_static ``` ## Developing a generator From 1d4f467516436d0eb9d97692aef2fcd891f79a29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marin=20Petruni=C4=87?= Date: Tue, 23 Feb 2021 13:30:26 +0100 Subject: [PATCH 147/222] disable phase1/enable lightclient ssz static types test --- tests/generators/ssz_static/main.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/generators/ssz_static/main.py b/tests/generators/ssz_static/main.py index 8b18705e7..854961ae4 100644 --- a/tests/generators/ssz_static/main.py +++ b/tests/generators/ssz_static/main.py @@ -9,7 +9,8 @@ from eth2spec.debug import random_value, encode from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1 +from eth2spec.lightclient_patch import spec as spec_lightclient_patch +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH from eth2spec.utils.ssz.ssz_typing import Container from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, @@ -64,6 +65,7 @@ def create_provider(fork_name, config_name: str, seed: int, mode: random_value.R config_util.prepare_config(configs_path, config_name) reload(spec_phase0) reload(spec_phase1) + reload(spec_lightclient_patch) return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: @@ -71,6 +73,8 @@ def create_provider(fork_name, config_name: str, seed: int, mode: random_value.R spec = spec_phase0 if fork_name == PHASE1: spec = spec_phase1 + if fork_name == LIGHTCLIENT_PATCH: + spec = spec_lightclient_patch for (i, (name, ssz_type)) in enumerate(get_spec_ssz_types(spec)): yield from ssz_static_cases(fork_name, seed * 1000 + i, name, ssz_type, mode, chaos, count) @@ -90,7 +94,7 @@ if __name__ == "__main__": settings.append((seed, "mainnet", random_value.RandomizationMode.mode_random, False, 5)) seed += 1 - for fork in [PHASE0, PHASE1]: + for fork in [PHASE0, LIGHTCLIENT_PATCH]: gen_runner.run_generator("ssz_static", [ create_provider(fork, config_name, seed, mode, chaos, cases_if_random) for (seed, config_name, mode, chaos, cases_if_random) in settings From 993bcdf082dbdbf2103bae608d409f54f75b3088 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 23 Feb 2021 21:17:12 +0800 Subject: [PATCH 148/222] Use constants --- tests/generators/ssz_static/main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/generators/ssz_static/main.py b/tests/generators/ssz_static/main.py index 854961ae4..fb8635fa0 100644 --- a/tests/generators/ssz_static/main.py +++ b/tests/generators/ssz_static/main.py @@ -10,7 +10,7 @@ from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 from eth2spec.lightclient_patch import spec as spec_lightclient_patch -from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, MINIMAL, MAINNET from eth2spec.utils.ssz.ssz_typing import Container from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, @@ -87,14 +87,14 @@ if __name__ == "__main__": settings = [] seed = 1 for mode in random_value.RandomizationMode: - settings.append((seed, "minimal", mode, False, 30)) + settings.append((seed, MINIMAL, mode, False, 30)) seed += 1 - settings.append((seed, "minimal", random_value.RandomizationMode.mode_random, True, 30)) + settings.append((seed, MINIMAL, random_value.RandomizationMode.mode_random, True, 30)) seed += 1 - settings.append((seed, "mainnet", random_value.RandomizationMode.mode_random, False, 5)) + settings.append((seed, MAINNET, random_value.RandomizationMode.mode_random, False, 5)) seed += 1 - for fork in [PHASE0, LIGHTCLIENT_PATCH]: + for fork in TESTGEN_FORKS: gen_runner.run_generator("ssz_static", [ create_provider(fork, config_name, seed, mode, chaos, cases_if_random) for (seed, config_name, mode, chaos, cases_if_random) in settings From d28ec2fc75dc744d4d41201c2554b728c8e1d29a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marin=20Petruni=C4=87?= Date: Tue, 23 Feb 2021 14:46:53 +0100 Subject: [PATCH 149/222] fix lint --- tests/generators/ssz_static/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/ssz_static/main.py b/tests/generators/ssz_static/main.py index fb8635fa0..e21fc4141 100644 --- a/tests/generators/ssz_static/main.py +++ b/tests/generators/ssz_static/main.py @@ -10,7 +10,7 @@ from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 from eth2spec.lightclient_patch import spec as spec_lightclient_patch -from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, MINIMAL, MAINNET +from eth2spec.test.context import PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, MINIMAL, MAINNET from eth2spec.utils.ssz.ssz_typing import Container from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, From deace8768e87ccd3cdc64d759c1752f764fdc3b8 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 1 Mar 2021 18:19:12 -0700 Subject: [PATCH 150/222] port @justindrake's general cleanups from #2212 --- configs/mainnet/lightclient_patch.yaml | 2 +- configs/minimal/lightclient_patch.yaml | 2 +- specs/lightclient/beacon-chain.md | 227 +++++++++--------- specs/lightclient/lightclient-fork.md | 6 +- .../pyspec/eth2spec/test/helpers/rewards.py | 47 ++-- ..._process_justification_and_finalization.py | 6 +- .../test/phase0/sanity/test_blocks.py | 2 +- 7 files changed, 150 insertions(+), 142 deletions(-) diff --git a/configs/mainnet/lightclient_patch.yaml b/configs/mainnet/lightclient_patch.yaml index 6c5b16edf..a9ddc16f6 100644 --- a/configs/mainnet/lightclient_patch.yaml +++ b/configs/mainnet/lightclient_patch.yaml @@ -17,7 +17,7 @@ HF1_PROPORTIONAL_SLASHING_MULTIPLIER: 2 # 2**10 (=1,024) SYNC_COMMITTEE_SIZE: 1024 # 2**6 (=64) -SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE: 64 +SYNC_SUBCOMMITTEE_SIZE: 64 # Time parameters diff --git a/configs/minimal/lightclient_patch.yaml b/configs/minimal/lightclient_patch.yaml index 7ab5f34ba..56ce34591 100644 --- a/configs/minimal/lightclient_patch.yaml +++ b/configs/minimal/lightclient_patch.yaml @@ -17,7 +17,7 @@ HF1_PROPORTIONAL_SLASHING_MULTIPLIER: 2 # [customized] SYNC_COMMITTEE_SIZE: 32 # [customized] -SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE: 16 +SYNC_SUBCOMMITTEE_SIZE: 16 # Time parameters diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index bf9b6a092..de0fdf8c5 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -9,8 +9,8 @@ - [Introduction](#introduction) - [Custom types](#custom-types) - [Constants](#constants) - - [Validator action flags](#validator-action-flags) - - [Participation rewards](#participation-rewards) + - [Participation flag indices](#participation-flag-indices) + - [Participation flag fractions](#participation-flag-fractions) - [Misc](#misc) - [Configuration](#configuration) - [Updated penalty values](#updated-penalty-values) @@ -27,7 +27,9 @@ - [`Predicates`](#predicates) - [`eth2_fast_aggregate_verify`](#eth2_fast_aggregate_verify) - [Misc](#misc-2) - - [`flags_and_numerators`](#flags_and_numerators) + - [`get_flag_indices_and_numerators`](#get_flag_indices_and_numerators) + - [`add_flag`](#add_flag) + - [`has_flag`](#has_flag) - [Beacon state accessors](#beacon-state-accessors) - [`get_sync_committee_indices`](#get_sync_committee_indices) - [`get_sync_committee`](#get_sync_committee) @@ -38,15 +40,15 @@ - [Beacon state mutators](#beacon-state-mutators) - [New `slash_validator`](#new-slash_validator) - [Block processing](#block-processing) - - [New `process_attestation`](#new-process_attestation) + - [Modified `process_attestation`](#modified-process_attestation) - [New `process_deposit`](#new-process_deposit) - [Sync committee processing](#sync-committee-processing) - [Epoch processing](#epoch-processing) - - [New `process_justification_and_finalization`](#new-process_justification_and_finalization) - - [New `process_rewards_and_penalties`](#new-process_rewards_and_penalties) - - [New `process_slashings`](#new-process_slashings) - - [Sync committee updates](#sync-committee-updates) + - [Justification and finalization](#justification-and-finalization) + - [Rewards and penalties](#rewards-and-penalties) + - [Slashings](#slashings) - [Participation flags updates](#participation-flags-updates) + - [Sync committee updates](#sync-committee-updates) @@ -66,34 +68,29 @@ It has four main features: | Name | SSZ equivalent | Description | | - | - | - | -| `ValidatorFlag` | `uint8` | Bitflags to track validator actions with | +| `ParticipationFlags` | `uint8` | A succinct representation of 8 boolean participation flags | ## Constants -### Validator action flags - -This is formatted as an enum, with values `2**i` that can be combined as bit-flags. -The `0` value is reserved as default. Remaining bits in `ValidatorFlag` may be used in future hardforks. - -**Note**: Unlike Phase0, a `TIMELY_TARGET_FLAG` does not necessarily imply a `TIMELY_SOURCE_FLAG` -due to the varying slot delay requirements of each. +### Participation flag indices | Name | Value | | - | - | -| `TIMELY_HEAD_FLAG` | `ValidatorFlag(2**0)` (= 1) | -| `TIMELY_SOURCE_FLAG` | `ValidatorFlag(2**1)` (= 2) | -| `TIMELY_TARGET_FLAG` | `ValidatorFlag(2**2)` (= 4) | +| `TIMELY_HEAD_FLAG_INDEX` | `0` | +| `TIMELY_SOURCE_FLAG_INDEX` | `1` | +| `TIMELY_TARGET_FLAG_INDEX` | `2` | -### Participation rewards +### Participation flag fractions | Name | Value | | - | - | -| `TIMELY_HEAD_NUMERATOR` | `12` | -| `TIMELY_SOURCE_NUMERATOR` | `12` | -| `TIMELY_TARGET_NUMERATOR` | `32` | -| `REWARD_DENOMINATOR` | `64` | +| `TIMELY_HEAD_FLAG_NUMERATOR` | `12` | +| `TIMELY_SOURCE_FLAG_NUMERATOR` | `12` | +| `TIMELY_TARGET_FLAG_NUMERATOR` | `32` | +| `FLAG_DENOMINATOR` | `64` | -The reward fractions add up to 7/8, leaving the remaining 1/8 for proposer rewards and other future micro-rewards. +**Note**: The participatition flag fractions add up to 7/8. +The remaining 1/8 is for proposer incentives and other future micro-incentives. ### Misc @@ -119,8 +116,8 @@ This patch updates a few configuration values to move penalty constants toward t | Name | Value | | - | - | -| `SYNC_COMMITTEE_SIZE` | `uint64(2**10)` (= 1024) | -| `SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE` | `uint64(2**6)` (= 64) | +| `SYNC_COMMITTEE_SIZE` | `uint64(2**10)` (= 1,024) | +| `SYNC_SUBCOMMITTEE_SIZE` | `uint64(2**6)` (= 64) | ### Time parameters @@ -144,10 +141,19 @@ order and append any additional fields to the end. #### `BeaconBlockBody` ```python -class BeaconBlockBody(phase0.BeaconBlockBody): +class BeaconBlockBody(Container): + randao_reveal: BLSSignature + eth1_data: Eth1Data # Eth1 data vote + graffiti: Bytes32 # Arbitrary data + # Operations + proposer_slashings: List[ProposerSlashing, MAX_PROPOSER_SLASHINGS] + attester_slashings: List[AttesterSlashing, MAX_ATTESTER_SLASHINGS] + attestations: List[Attestation, MAX_ATTESTATIONS] + deposits: List[Deposit, MAX_DEPOSITS] + voluntary_exits: List[SignedVoluntaryExit, MAX_VOLUNTARY_EXITS] # Sync committee aggregate signature - sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE] - sync_committee_signature: BLSSignature + sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE] # [New in HF1] + sync_committee_signature: BLSSignature # [New in HF1] ``` #### `BeaconState` @@ -176,8 +182,8 @@ class BeaconState(Container): # Slashings slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR] # Per-epoch sums of slashed effective balances # Participation - previous_epoch_participation: List[ValidatorFlag, VALIDATOR_REGISTRY_LIMIT] - current_epoch_participation: List[ValidatorFlag, VALIDATOR_REGISTRY_LIMIT] + previous_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT] + current_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT] # Finality justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH] # Bit set for every recent justified epoch previous_justified_checkpoint: Checkpoint @@ -195,7 +201,7 @@ class BeaconState(Container): ```python class SyncCommittee(Container): pubkeys: Vector[BLSPubkey, SYNC_COMMITTEE_SIZE] - pubkey_aggregates: Vector[BLSPubkey, SYNC_COMMITTEE_SIZE // SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE] + pubkey_aggregates: Vector[BLSPubkey, SYNC_COMMITTEE_SIZE // SYNC_SUBCOMMITTEE_SIZE] ``` ## Helper functions @@ -216,25 +222,31 @@ def eth2_fast_aggregate_verify(pubkeys: Sequence[BLSPubkey], message: Bytes32, s ### Misc -#### `flags_and_numerators` +#### `get_flag_indices_and_numerators` ```python -def get_flags_and_numerators() -> Sequence[Tuple[ValidatorFlag, int]]: +def get_flag_indices_and_numerators() -> Sequence[Tuple[int, int]]: return ( - (TIMELY_HEAD_FLAG, TIMELY_HEAD_NUMERATOR), - (TIMELY_SOURCE_FLAG, TIMELY_SOURCE_NUMERATOR), - (TIMELY_TARGET_FLAG, TIMELY_TARGET_NUMERATOR) + (TIMELY_HEAD_FLAG_INDEX, TIMELY_HEAD_FLAG_NUMERATOR), + (TIMELY_SOURCE_FLAG_INDEX, TIMELY_SOURCE_FLAG_NUMERATOR), + (TIMELY_TARGET_FLAG_INDEX, TIMELY_TARGET_FLAG_NUMERATOR), ) ``` -```python -def add_validator_flags(flags: ValidatorFlag, add: ValidatorFlag) -> ValidatorFlag: - return flags | add -``` +#### `add_flag` ```python -def has_validator_flags(flags: ValidatorFlag, has: ValidatorFlag) -> bool: - return flags & has == has +def add_flag(flags: ParticipationFlags, flag_index: int) -> ParticipationFlags: + flag = ParticipationFlags(2**flag_index) + return flags | flag +``` + +#### `has_flag` + +```python +def has_flag(flags: ParticipationFlags, flag_index: int) -> bool: + flag = ParticipationFlags(2**flag_index) + return flags & flag == flag ``` ### Beacon state accessors @@ -275,8 +287,8 @@ def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: validators = [state.validators[index] for index in indices] pubkeys = [validator.pubkey for validator in validators] aggregates = [ - bls.AggregatePKs(pubkeys[i:i + SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE]) - for i in range(0, len(pubkeys), SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE) + bls.AggregatePKs(pubkeys[i:i + SYNC_SUBCOMMITTEE_SIZE]) + for i in range(0, len(pubkeys), SYNC_SUBCOMMITTEE_SIZE) ] return SyncCommittee(pubkeys=pubkeys, pubkey_aggregates=aggregates) ``` @@ -295,19 +307,17 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: #### `get_unslashed_participating_indices` ```python -def get_unslashed_participating_indices(state: BeaconState, flags: ValidatorFlag, epoch: Epoch) -> Set[ValidatorIndex]: +def get_unslashed_participating_indices(state: BeaconState, flag_index: int, epoch: Epoch) -> Set[ValidatorIndex]: """ - Retrieve the active validator indices of the given epoch, which are not slashed, and have all of the given flags. + Retrieve the active and unslashed validator indices for the given epoch and flag index. """ assert epoch in (get_previous_epoch(state), get_current_epoch(state)) if epoch == get_current_epoch(state): epoch_participation = state.current_epoch_participation else: epoch_participation = state.previous_epoch_participation - participating_indices = [ - index for index in get_active_validator_indices(state, epoch) - if has_validator_flags(epoch_participation[index], flags) - ] + active_validator_indices = get_active_validator_indices(state, epoch) + participating_indices = [i for i in active_validator_indices if has_flag(epoch_participation[i], flag_index)] return set(filter(lambda index: not state.validators[index].slashed, participating_indices)) ``` @@ -315,7 +325,7 @@ def get_unslashed_participating_indices(state: BeaconState, flags: ValidatorFlag ```python def get_flag_deltas(state: BeaconState, - flag: ValidatorFlag, + flag_index: int, numerator: uint64) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: """ Compute the rewards and penalties associated with a particular duty, by scanning through the participation @@ -324,7 +334,7 @@ def get_flag_deltas(state: BeaconState, rewards = [Gwei(0)] * len(state.validators) penalties = [Gwei(0)] * len(state.validators) - unslashed_participating_indices = get_unslashed_participating_indices(state, flag, get_previous_epoch(state)) + unslashed_participating_indices = get_unslashed_participating_indices(state, flag_index, get_previous_epoch(state)) increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from balances to avoid uint64 overflow unslashed_participating_increments = get_total_balance(state, unslashed_participating_indices) // increment active_increments = get_total_active_balance(state) // increment @@ -333,14 +343,14 @@ def get_flag_deltas(state: BeaconState, if index in unslashed_participating_indices: if is_in_inactivity_leak(state): # Optimal participation is fully rewarded to cancel the inactivity penalty - rewards[index] = base_reward * numerator // REWARD_DENOMINATOR + rewards[index] = base_reward * numerator // FLAG_DENOMINATOR else: rewards[index] = ( (base_reward * numerator * unslashed_participating_increments) - // (active_increments * REWARD_DENOMINATOR) + // (active_increments * FLAG_DENOMINATOR) ) else: - penalties[index] = base_reward * numerator // REWARD_DENOMINATOR + penalties[index] = base_reward * numerator // FLAG_DENOMINATOR return rewards, penalties ``` @@ -358,13 +368,13 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S """ penalties = [Gwei(0) for _ in range(len(state.validators))] if is_in_inactivity_leak(state): - reward_numerator_sum = sum(numerator for (_, numerator) in get_flags_and_numerators()) + reward_numerator_sum = sum(numerator for (_, numerator) in get_flag_indices_and_numerators()) matching_target_attesting_indices = get_unslashed_participating_indices( - state, TIMELY_TARGET_FLAG, get_previous_epoch(state) + state, TIMELY_TARGET_FLAG_INDEX, get_previous_epoch(state) ) for index in get_eligible_validator_indices(state): # If validator is performing optimally this cancels all attestation rewards for a neutral balance - penalties[index] += Gwei(get_base_reward(state, index) * reward_numerator_sum // REWARD_DENOMINATOR) + penalties[index] += Gwei(get_base_reward(state, index) * reward_numerator_sum // FLAG_DENOMINATOR) if index not in matching_target_attesting_indices: effective_balance = state.validators[index].effective_balance penalties[index] += Gwei( @@ -415,12 +425,11 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) process_randao(state, block.body) process_eth1_data(state, block.body) - process_operations(state, block.body) - # Light client support - process_sync_committee(state, block.body) + process_operations(state, block.body) # [Modified in HF1] + process_sync_committee(state, block.body) # [New in HF1] ``` -#### New `process_attestation` +#### Modified `process_attestation` *Note*: The function `process_attestation` is modified to do incentive accounting with epoch participation flags. @@ -451,25 +460,25 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: # Verify signature assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) - # Participation flags - participation_flags = [] + # Participation flag indices + participation_flag_indices = [] if is_matching_head and is_matching_target and state.slot <= data.slot + MIN_ATTESTATION_INCLUSION_DELAY: - participation_flags.append(TIMELY_HEAD_FLAG) + participation_flag_indices.append(TIMELY_HEAD_FLAG_INDEX) if is_matching_source and state.slot <= data.slot + integer_squareroot(SLOTS_PER_EPOCH): - participation_flags.append(TIMELY_SOURCE_FLAG) + participation_flag_indices.append(TIMELY_SOURCE_FLAG_INDEX) if is_matching_target and state.slot <= data.slot + SLOTS_PER_EPOCH: - participation_flags.append(TIMELY_TARGET_FLAG) + participation_flag_indices.append(TIMELY_TARGET_FLAG_INDEX) # Update epoch participation flags proposer_reward_numerator = 0 for index in get_attesting_indices(state, data, attestation.aggregation_bits): - for flag, numerator in get_flags_and_numerators(): - if flag in participation_flags and not has_validator_flags(epoch_participation[index], flag): - epoch_participation[index] = add_validator_flags(epoch_participation[index], flag) - proposer_reward_numerator += get_base_reward(state, index) * numerator + for flag_index, flag_numerator in get_flag_indices_and_numerators(): + if flag_index in participation_flag_indices and not has_flag(epoch_participation[index], flag_index): + epoch_participation[index] = add_flag(epoch_participation[index], flag_index) + proposer_reward_numerator += get_base_reward(state, index) * flag_numerator # Reward proposer - proposer_reward = Gwei(proposer_reward_numerator // (REWARD_DENOMINATOR * PROPOSER_REWARD_QUOTIENT)) + proposer_reward = Gwei(proposer_reward_numerator // (FLAG_DENOMINATOR * PROPOSER_REWARD_QUOTIENT)) increase_balance(state, get_beacon_proposer_index(state), proposer_reward) ``` @@ -511,8 +520,8 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: state.validators.append(get_validator_from_deposit(state, deposit)) state.balances.append(amount) # [Added in hf-1] Initialize empty participation flags for new validator - state.previous_epoch_participation.append(ValidatorFlag(0)) - state.current_epoch_participation.append(ValidatorFlag(0)) + state.previous_epoch_participation.append(ParticipationFlags(0b0000_0000)) + state.current_epoch_participation.append(ParticipationFlags(0b0000_0000)) else: # Increase balance by deposit amount index = ValidatorIndex(validator_pubkeys.index(pubkey)) @@ -534,26 +543,26 @@ def process_sync_committee(state: BeaconState, body: BeaconBlockBody) -> None: assert eth2_fast_aggregate_verify(participant_pubkeys, signing_root, body.sync_committee_signature) # Reward sync committee participants - total_proposer_reward = Gwei(0) + proposer_rewards = Gwei(0) active_validator_count = uint64(len(get_active_validator_indices(state, get_current_epoch(state)))) for participant_index in participant_indices: - base_reward = get_base_reward(state, participant_index) proposer_reward = get_proposer_reward(state, participant_index) + proposer_rewards += proposer_reward + base_reward = get_base_reward(state, participant_index) max_participant_reward = base_reward - proposer_reward - reward = Gwei(max_participant_reward * active_validator_count // len(committee_indices) // SLOTS_PER_EPOCH) + reward = Gwei(max_participant_reward * active_validator_count // (len(committee_indices) * SLOTS_PER_EPOCH)) increase_balance(state, participant_index, reward) - total_proposer_reward += proposer_reward # Reward beacon proposer - increase_balance(state, get_beacon_proposer_index(state), total_proposer_reward) + increase_balance(state, get_beacon_proposer_index(state), proposer_rewards) ``` ### Epoch processing ```python def process_epoch(state: BeaconState) -> None: - process_justification_and_finalization(state) # [Updated in HF1] - process_rewards_and_penalties(state) # [Updated in HF1] + process_justification_and_finalization(state) # [Modified in HF1] + process_rewards_and_penalties(state) # [Modified in HF1] process_registry_updates(state) process_slashings(state) process_eth1_data_reset(state) @@ -561,13 +570,11 @@ def process_epoch(state: BeaconState) -> None: process_slashings_reset(state) process_randao_mixes_reset(state) process_historical_roots_update(state) - # [Removed in HF1] -- process_participation_record_updates(state) - # [Added in HF1] - process_participation_flag_updates(state) - process_sync_committee_updates(state) + process_participation_flag_updates(state) # [New in HF1] + process_sync_committee_updates(state) # [New in HF1] ``` -#### New `process_justification_and_finalization` +#### Justification and finalization *Note*: The function `process_justification_and_finalization` is modified with `matching_target_attestations` replaced by `matching_target_indices`. @@ -586,12 +593,12 @@ def process_justification_and_finalization(state: BeaconState) -> None: state.previous_justified_checkpoint = state.current_justified_checkpoint state.justification_bits[1:] = state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] state.justification_bits[0] = 0b0 - matching_target_indices = get_unslashed_participating_indices(state, TIMELY_TARGET_FLAG, previous_epoch) + matching_target_indices = get_unslashed_participating_indices(state, TIMELY_TARGET_FLAG_INDEX, previous_epoch) if get_total_balance(state, matching_target_indices) * 3 >= get_total_active_balance(state) * 2: state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, root=get_block_root(state, previous_epoch)) state.justification_bits[1] = 0b1 - matching_target_indices = get_unslashed_participating_indices(state, TIMELY_TARGET_FLAG, current_epoch) + matching_target_indices = get_unslashed_participating_indices(state, TIMELY_TARGET_FLAG_INDEX, current_epoch) if get_total_balance(state, matching_target_indices) * 3 >= get_total_active_balance(state) * 2: state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) @@ -613,16 +620,19 @@ def process_justification_and_finalization(state: BeaconState) -> None: state.finalized_checkpoint = old_current_justified_checkpoint ``` -#### New `process_rewards_and_penalties` +#### Rewards and penalties -*Note*: The function `process_rewards_and_penalties` is modified to use participation flag deltas. +*Note*: The function `process_rewards_and_penalties` is modified to support the incentive reforms. ```python def process_rewards_and_penalties(state: BeaconState) -> None: # No rewards are applied at the end of `GENESIS_EPOCH` because rewards are for work done in the previous epoch if get_current_epoch(state) == GENESIS_EPOCH: return - flag_deltas = [get_flag_deltas(state, flag, numerator) for (flag, numerator) in get_flags_and_numerators()] + flag_deltas = [ + get_flag_deltas(state, flag_index, flag_numerator) + for (flag_index, flag_numerator) in get_flag_indices_and_numerators() + ] deltas = flag_deltas + [get_inactivity_penalty_deltas(state)] for (rewards, penalties) in deltas: for index in range(len(state.validators)): @@ -630,10 +640,9 @@ def process_rewards_and_penalties(state: BeaconState) -> None: decrease_balance(state, ValidatorIndex(index), penalties[index]) ``` -#### New `process_slashings` +#### Slashings -*Note*: The function `process_slashings` is modified -with the substitution of `PROPORTIONAL_SLASHING_MULTIPLIER` with `HF1_PROPORTIONAL_SLASHING_MULTIPLIER`. +*Note*: The function `process_slashings` is modified to use `HF1_PROPORTIONAL_SLASHING_MULTIPLIER`. ```python def process_slashings(state: BeaconState) -> None: @@ -648,26 +657,24 @@ def process_slashings(state: BeaconState) -> None: decrease_balance(state, ValidatorIndex(index), penalty) ``` +#### Participation flags updates + +*Note*: The function `process_participation_flag_updates` is new. + +```python +def process_participation_flag_updates(state: BeaconState) -> None: + state.previous_epoch_participation = state.current_epoch_participation + state.current_epoch_participation = [ParticipationFlags(0b0000_0000) for _ in range(len(state.validators))] +``` + #### Sync committee updates +*Note*: The function `process_sync_committee_updates` is new. + ```python def process_sync_committee_updates(state: BeaconState) -> None: - """ - Call to ``proces_sync_committee_updates`` added to ``process_epoch`` in HF1 - """ next_epoch = get_current_epoch(state) + Epoch(1) if next_epoch % EPOCHS_PER_SYNC_COMMITTEE_PERIOD == 0: state.current_sync_committee = state.next_sync_committee state.next_sync_committee = get_sync_committee(state, next_epoch + EPOCHS_PER_SYNC_COMMITTEE_PERIOD) ``` - -#### Participation flags updates - -```python -def process_participation_flag_updates(state: BeaconState) -> None: - """ - Call to ``process_participation_flag_updates`` added to ``process_epoch`` in HF1 - """ - state.previous_epoch_participation = state.current_epoch_participation - state.current_epoch_participation = [ValidatorFlag(0) for _ in range(len(state.validators))] -``` diff --git a/specs/lightclient/lightclient-fork.md b/specs/lightclient/lightclient-fork.md index aa0171b86..157e67dc9 100644 --- a/specs/lightclient/lightclient-fork.md +++ b/specs/lightclient/lightclient-fork.md @@ -66,9 +66,9 @@ def upgrade_to_lightclient_patch(pre: phase0.BeaconState) -> BeaconState: randao_mixes=pre.randao_mixes, # Slashings slashings=pre.slashings, - # Attestations - previous_epoch_participation=[ValidatorFlag(0) for _ in range(len(pre.validators))], - current_epoch_participation=[ValidatorFlag(0) for _ in range(len(pre.validators))], + # Participation + previous_epoch_participation=[ParticipationFlags(0) for _ in range(len(pre.validators))], + current_epoch_participation=[ParticipationFlags(0) for _ in range(len(pre.validators))], # Finality justification_bits=pre.justification_bits, previous_justified_checkpoint=pre.previous_justified_checkpoint, diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 2499bcffe..19ed3f691 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -41,13 +41,13 @@ def run_deltas(spec, state): if is_post_lightclient_patch(spec): def get_source_deltas(state): - return spec.get_flag_deltas(state, spec.TIMELY_SOURCE_FLAG, spec.TIMELY_SOURCE_NUMERATOR) + return spec.get_flag_deltas(state, spec.TIMELY_SOURCE_FLAG_INDEX, spec.TIMELY_SOURCE_FLAG_NUMERATOR) def get_head_deltas(state): - return spec.get_flag_deltas(state, spec.TIMELY_HEAD_FLAG, spec.TIMELY_HEAD_NUMERATOR) + return spec.get_flag_deltas(state, spec.TIMELY_HEAD_FLAG_INDEX, spec.TIMELY_HEAD_FLAG_NUMERATOR) def get_target_deltas(state): - return spec.get_flag_deltas(state, spec.TIMELY_TARGET_FLAG, spec.TIMELY_TARGET_NUMERATOR) + return spec.get_flag_deltas(state, spec.TIMELY_TARGET_FLAG_INDEX, spec.TIMELY_TARGET_FLAG_NUMERATOR) yield from run_attestation_component_deltas( spec, @@ -74,13 +74,13 @@ def run_deltas(spec, state): yield from run_get_inactivity_penalty_deltas(spec, state) -def deltas_name_to_flag(spec, deltas_name): +def deltas_name_to_flag_index(spec, deltas_name): if 'source' in deltas_name: - return spec.TIMELY_SOURCE_FLAG + return spec.TIMELY_SOURCE_FLAG_INDEX elif 'head' in deltas_name: - return spec.TIMELY_HEAD_FLAG + return spec.TIMELY_HEAD_FLAG_INDEX elif 'target' in deltas_name: - return spec.TIMELY_TARGET_FLAG + return spec.TIMELY_TARGET_FLAG_INDEX raise ValueError("Wrong deltas_name %s" % deltas_name) @@ -98,7 +98,7 @@ def run_attestation_component_deltas(spec, state, component_delta_fn, matching_a matching_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) else: matching_indices = spec.get_unslashed_participating_indices( - state, deltas_name_to_flag(spec, deltas_name), spec.get_previous_epoch(state) + state, deltas_name_to_flag_index(spec, deltas_name), spec.get_previous_epoch(state) ) eligible_indices = spec.get_eligible_validator_indices(state) @@ -187,9 +187,9 @@ def run_get_inactivity_penalty_deltas(spec, state): matching_attesting_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) else: matching_attesting_indices = spec.get_unslashed_participating_indices( - state, spec.TIMELY_TARGET_FLAG, spec.get_previous_epoch(state) + state, spec.TIMELY_TARGET_FLAG_INDEX, spec.get_previous_epoch(state) ) - reward_numerator_sum = sum(numerator for (_, numerator) in spec.get_flags_and_numerators()) + reward_numerator_sum = sum(numerator for (_, numerator) in spec.get_flag_indices_and_numerators()) eligible_indices = spec.get_eligible_validator_indices(state) for index in range(len(state.validators)): @@ -205,7 +205,7 @@ def run_get_inactivity_penalty_deltas(spec, state): base_reward = spec.get_base_reward(state, index) base_penalty = cancel_base_rewards_per_epoch * base_reward - spec.get_proposer_reward(state, index) else: - base_penalty = spec.get_base_reward(state, index) * reward_numerator_sum // spec.REWARD_DENOMINATOR + base_penalty = spec.get_base_reward(state, index) * reward_numerator_sum // spec.FLAG_DENOMINATOR if not has_enough_for_reward(spec, state, index): assert penalties[index] == 0 @@ -314,7 +314,7 @@ def run_test_full_but_partial_participation(spec, state, rng=Random(5522)): else: for index in range(len(state.validators)): if rng.choice([True, False]): - state.previous_epoch_participation[index] = spec.ValidatorFlag(0) + state.previous_epoch_participation[index] = spec.ParticipationFlags(0b0000_0000) yield from run_deltas(spec, state) @@ -328,7 +328,7 @@ def run_test_partial(spec, state, fraction_filled): state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] else: for index in range(int(len(state.validators) * fraction_filled)): - state.previous_epoch_participation[index] = spec.ValidatorFlag(0) + state.previous_epoch_participation[index] = spec.ParticipationFlags(0b0000_0000) yield from run_deltas(spec, state) @@ -394,7 +394,7 @@ def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state): else: index = 0 state.validators[index].effective_balance = 1 - state.previous_epoch_participation[index] = spec.ValidatorFlag(0) + state.previous_epoch_participation[index] = spec.ParticipationFlags(0b0000_0000) yield from run_deltas(spec, state) @@ -521,23 +521,24 @@ def run_test_full_random(spec, state, rng=Random(8020)): is_timely_correct_head = rng.randint(0, 2) != 0 flags = state.previous_epoch_participation[index] - def set_flag(f, v): + def set_flag(index, value): nonlocal flags - if v: - flags |= f + flag = spec.ParticipationFlags(2**index) + if value: + flags |= flag else: - flags &= 0xff ^ f + flags &= 0xff ^ flag - set_flag(spec.TIMELY_HEAD_FLAG, is_timely_correct_head) + set_flag(spec.TIMELY_HEAD_FLAG_INDEX, is_timely_correct_head) if is_timely_correct_head: # If timely head, then must be timely target - set_flag(spec.TIMELY_TARGET_FLAG, True) + set_flag(spec.TIMELY_TARGET_FLAG_INDEX, True) # If timely head, then must be timely source - set_flag(spec.TIMELY_SOURCE_FLAG, True) + set_flag(spec.TIMELY_SOURCE_FLAG_INDEX, True) else: # ~50% of remaining have bad target or not timely enough - set_flag(spec.TIMELY_TARGET_FLAG, rng.choice([True, False])) + set_flag(spec.TIMELY_TARGET_FLAG_INDEX, rng.choice([True, False])) # ~50% of remaining have bad source or not timely enough - set_flag(spec.TIMELY_SOURCE_FLAG, rng.choice([True, False])) + set_flag(spec.TIMELY_SOURCE_FLAG_INDEX, rng.choice([True, False])) state.previous_epoch_participation[index] = flags yield from run_deltas(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py index 89783f987..274d67134 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py @@ -78,10 +78,10 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support else: for i, index in enumerate(committee): if aggregation_bits[i]: - epoch_participation[index] |= spec.TIMELY_HEAD_FLAG - epoch_participation[index] |= spec.TIMELY_SOURCE_FLAG + epoch_participation[index] |= spec.ParticipationFlags(2**spec.TIMELY_HEAD_FLAG_INDEX) + epoch_participation[index] |= spec.ParticipationFlags(2**spec.TIMELY_SOURCE_FLAG_INDEX) if not messed_up_target: - epoch_participation[index] |= spec.TIMELY_TARGET_FLAG + epoch_participation[index] |= spec.ParticipationFlags(2**spec.TIMELY_TARGET_FLAG_INDEX) def get_checkpoints(spec, epoch): diff --git a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py index 1834b290f..98ffbd590 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py @@ -806,7 +806,7 @@ def test_attestation(spec, state): assert spec.hash_tree_root(state.previous_epoch_attestations) == pre_current_attestations_root else: for index in range(len(state.validators)): - assert state.current_epoch_participation[index] == 0 + assert state.current_epoch_participation[index] == spec.ParticipationFlags(0b0000_0000) assert spec.hash_tree_root(state.previous_epoch_participation) == pre_current_epoch_participation_root From 8ea5e37608bd6734e2764665d35a0433579b5ad0 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 2 Mar 2021 07:56:30 -0600 Subject: [PATCH 151/222] add another 'modified' tag Co-authored-by: Hsiao-Wei Wang --- specs/lightclient/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index de0fdf8c5..bc070908a 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -564,7 +564,7 @@ def process_epoch(state: BeaconState) -> None: process_justification_and_finalization(state) # [Modified in HF1] process_rewards_and_penalties(state) # [Modified in HF1] process_registry_updates(state) - process_slashings(state) + process_slashings(state) # [Modified in HF1] process_eth1_data_reset(state) process_effective_balance_updates(state) process_slashings_reset(state) From e1023f55d1ab36c0c148da991df62f043b3c6188 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Fri, 5 Mar 2021 18:05:15 -0800 Subject: [PATCH 152/222] Fix a small typo --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 249d08799..02a578fb8 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -292,7 +292,7 @@ If one or more validations fail while processing the items in order, return eith There are two primary global topics used to propagate beacon blocks (`beacon_block`) and aggregate attestations (`beacon_aggregate_and_proof`) to all nodes on the network. -There are three additional global topics are used to propagate lower frequency validator messages +There are three additional global topics that are used to propagate lower frequency validator messages (`voluntary_exit`, `proposer_slashing`, and `attester_slashing`). ##### `beacon_block` From 6bd1efc73bf45ae5658e115a8b2956144d4f4559 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 8 Mar 2021 17:16:13 -0700 Subject: [PATCH 153/222] rename fork files --- setup.py | 4 ++-- specs/lightclient/{lightclient-fork.md => fork.md} | 0 specs/phase1/{phase1-fork.md => fork.md} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename specs/lightclient/{lightclient-fork.md => fork.md} (100%) rename specs/phase1/{phase1-fork.md => fork.md} (100%) diff --git a/setup.py b/setup.py index 6cfa0910a..ca85a83ac 100644 --- a/setup.py +++ b/setup.py @@ -449,7 +449,7 @@ class PySpecCommand(Command): specs/phase1/beacon-chain.md specs/phase1/shard-transition.md specs/phase1/fork-choice.md - specs/phase1/phase1-fork.md + specs/phase1/fork.md specs/phase1/shard-fork-choice.md specs/phase1/validator.md """ @@ -460,7 +460,7 @@ class PySpecCommand(Command): specs/phase0/validator.md specs/phase0/weak-subjectivity.md specs/lightclient/beacon-chain.md - specs/lightclient/lightclient-fork.md + specs/lightclient/fork.md """ # TODO: add specs/lightclient/sync-protocol.md back when the GeneralizedIndex helpers are included. else: diff --git a/specs/lightclient/lightclient-fork.md b/specs/lightclient/fork.md similarity index 100% rename from specs/lightclient/lightclient-fork.md rename to specs/lightclient/fork.md diff --git a/specs/phase1/phase1-fork.md b/specs/phase1/fork.md similarity index 100% rename from specs/phase1/phase1-fork.md rename to specs/phase1/fork.md From d6961f636de17b31ecb3350b0a5710a1d102d669 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 8 Mar 2021 17:16:29 -0700 Subject: [PATCH 154/222] add base hf1 fork function tests --- tests/core/pyspec/eth2spec/test/context.py | 10 +- .../test/lightclient_patch/fork/test_fork.py | 104 ++++++++++++++++++ 2 files changed, 108 insertions(+), 6 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 5c2a3bf4d..3f247b400 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -346,13 +346,11 @@ def with_phases(phases, other_phases=None): # A new state-creation helper for phase 1 may be in place, and then phase1+ tests can run without phase0 available_phases.add(PHASE0) + # Populate all phases for multi-phase tests phase_dir = {} - if PHASE0 in available_phases: - phase_dir[PHASE0] = spec_phase0 - if PHASE1 in available_phases: - phase_dir[PHASE1] = spec_phase1 - if LIGHTCLIENT_PATCH in available_phases: - phase_dir[LIGHTCLIENT_PATCH] = spec_lightclient_patch + phase_dir[PHASE0] = spec_phase0 + phase_dir[PHASE1] = spec_phase1 + phase_dir[LIGHTCLIENT_PATCH] = spec_lightclient_patch # return is ignored whenever multiple phases are ran. If if PHASE0 in run_phases: diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py new file mode 100644 index 000000000..08c6e6d29 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py @@ -0,0 +1,104 @@ +from operator import attrgetter + +from eth2spec.test.context import ( + PHASE0, LIGHTCLIENT_PATCH, + spec_state_test, with_phases, + with_custom_state, + spec_test, with_state, + low_balances, misc_balances, large_validator_set, +) +from eth2spec.test.helpers.state import ( + next_slots, + next_epoch, + next_epoch_via_block, + transition_to_slot_via_block, +) + + +def run_fork_test(spec, pre_state): + yield 'pre', pre_state + + post_state = spec.upgrade_to_lightclient_patch(pre_state) + + # Stable fields + stable_fields = [ + 'genesis_time', 'genesis_validators_root', 'slot', + # History + 'latest_block_header', 'block_roots', 'state_roots', 'historical_roots', + # Eth1 + 'eth1_data', 'eth1_data_votes', 'eth1_deposit_index', + # Registry + 'validators', 'balances', + # Randomness + 'randao_mixes', + # Slashings + 'slashings', + # Finality + 'justification_bits', 'previous_justified_checkpoint', 'current_justified_checkpoint', 'finalized_checkpoint', + ] + for field in stable_fields: + assert getattr(pre_state, field) == getattr(post_state, field) + + # Modified fields + modified_fields = ['fork'] + for field in modified_fields: + assert getattr(pre_state, field) != getattr(post_state, field) + + assert pre_state.fork.current_version == post_state.fork.previous_version + assert post_state.fork.current_version == spec.LIGHTCLIENT_PATCH_FORK_VERSION + assert post_state.fork.epoch == spec.get_current_epoch(post_state) + + yield 'post', post_state + + +@with_phases(([PHASE0])) +@with_state +@spec_test +def test_fork_base_state(spec, phases, state): + yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) + + +@with_phases(([PHASE0])) +@with_state +@spec_test +def test_fork_next_epoch(spec, phases, state): + next_epoch(spec, state) + yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) + + +@with_phases(([PHASE0])) +@with_state +@spec_test +def test_fork_next_epoch_with_block(spec, phases, state): + next_epoch_via_block(spec, state) + yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) + + +@with_phases(([PHASE0])) +@with_state +@spec_test +def test_fork_many_next_epoch(spec, phases, state): + for _ in range(3): + next_epoch(spec, state) + yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) + + +@with_phases(([PHASE0])) +@with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@spec_test +def test_fork_random_low_balances(spec, phases, state): + yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) + + +@with_phases(([PHASE0])) +@with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@spec_test +def test_fork_random_misc_balances(spec, phases, state): + yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) + + +@with_phases(([PHASE0])) +@with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@spec_test +def test_fork_random_large_validator_set(spec, phases, state): + yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) From 6c406753f150563cac1253cfef8d99d9111bd906 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 8 Mar 2021 19:11:31 -0700 Subject: [PATCH 155/222] working through test gens --- tests/core/pyspec/eth2spec/test/context.py | 18 +++++-- .../test/lightclient_patch/fork/__init__.py | 0 .../test/lightclient_patch/fork/test_fork.py | 44 +++++++++++------ tests/formats/forks/README.md | 47 +++++++++++++++++++ tests/generators/forks/main.py | 40 ++++++++++++++++ tests/generators/forks/requirements.txt | 2 + 6 files changed, 133 insertions(+), 18 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/lightclient_patch/fork/__init__.py create mode 100644 tests/formats/forks/README.md create mode 100644 tests/generators/forks/main.py create mode 100644 tests/generators/forks/requirements.txt diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 3f247b400..9c45049a2 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -70,7 +70,7 @@ class SpecForks(TypedDict, total=False): def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Callable[[Any], int], - spec: Spec, phases: SpecForks): + spec: Spec, phases: SpecForks, is_fork_test: bool): p0 = phases[PHASE0] balances = balances_fn(p0) @@ -82,7 +82,7 @@ def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Ca # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper. # Decide based on performance/consistency results later. state = phases[PHASE1].upgrade_to_phase1(state) - elif spec.fork == LIGHTCLIENT_PATCH: # not generalizing this just yet, unclear final spec fork/patch order. + elif spec.fork == LIGHTCLIENT_PATCH and not fork_test: # do not upgrade if spec ttttest state = phases[LIGHTCLIENT_PATCH].upgrade_to_lightclient_patch(state) return state @@ -98,10 +98,11 @@ def with_custom_state(balances_fn: Callable[[Any], Sequence[int]], def entry(*args, spec: Spec, phases: SpecForks, **kw): # make a key for the state # genesis fork version separates configs during test-generation runtime. - key = (spec.fork, spec.GENESIS_FORK_VERSION, spec.__file__, balances_fn, threshold_fn) + is_fork_test = kw.pop('fork_test') if 'fork_test' in kw else False + key = (spec.fork, spec.GENESIS_FORK_VERSION, spec.__file__, balances_fn, threshold_fn, is_fork_test) global _custom_state_cache_dict if key not in _custom_state_cache_dict: - state = _prepare_state(balances_fn, threshold_fn, spec, phases) + state = _prepare_state(balances_fn, threshold_fn, spec, phases, is_fork_test) _custom_state_cache_dict[key] = state.get_backing() # Take an entry out of the LRU. @@ -287,6 +288,15 @@ def bls_switch(fn): return entry +def fork_test(fn): + """ + """ + def entry(*args, **kw): + # override fork test setting + kw['fork_test'] = True + return entry + + def disable_process_reveal_deadlines(fn): """ Decorator to make a function execute with `process_reveal_deadlines` OFF. diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/__init__.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py index 08c6e6d29..527ad746c 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py @@ -1,20 +1,22 @@ -from operator import attrgetter - from eth2spec.test.context import ( - PHASE0, LIGHTCLIENT_PATCH, - spec_state_test, with_phases, - with_custom_state, + LIGHTCLIENT_PATCH, + with_phases, + with_custom_state, fork_test, spec_test, with_state, low_balances, misc_balances, large_validator_set, ) +from eth2spec.test.utils import with_meta_tags from eth2spec.test.helpers.state import ( - next_slots, next_epoch, next_epoch_via_block, - transition_to_slot_via_block, ) +HF1_FORK_TEST_META_TAGS = { + 'fork': 'altair', +} + + def run_fork_test(spec, pre_state): yield 'pre', pre_state @@ -51,54 +53,68 @@ def run_fork_test(spec, pre_state): yield 'post', post_state -@with_phases(([PHASE0])) +@with_phases(([LIGHTCLIENT_PATCH])) @with_state @spec_test +@fork_test +@with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_base_state(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([PHASE0])) +@with_phases(([LIGHTCLIENT_PATCH])) @with_state @spec_test +@fork_test +@with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_next_epoch(spec, phases, state): next_epoch(spec, state) yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([PHASE0])) +@with_phases(([LIGHTCLIENT_PATCH])) @with_state @spec_test +@fork_test +@with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_next_epoch_with_block(spec, phases, state): next_epoch_via_block(spec, state) yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([PHASE0])) +@with_phases(([LIGHTCLIENT_PATCH])) @with_state @spec_test +@fork_test +@with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_many_next_epoch(spec, phases, state): for _ in range(3): next_epoch(spec, state) yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([PHASE0])) +@with_phases(([LIGHTCLIENT_PATCH])) @with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test +@fork_test +@with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_random_low_balances(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([PHASE0])) +@with_phases(([LIGHTCLIENT_PATCH])) @with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test +@fork_test +@with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_random_misc_balances(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([PHASE0])) +@with_phases(([LIGHTCLIENT_PATCH])) @with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test +@fork_test +@with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_random_large_validator_set(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) diff --git a/tests/formats/forks/README.md b/tests/formats/forks/README.md new file mode 100644 index 000000000..1a7ee64ad --- /dev/null +++ b/tests/formats/forks/README.md @@ -0,0 +1,47 @@ +# Forks + +The aim of the fork tests is to ensure that a pre-fork state can be transformed + into a valid post-fork state, utilizing the `upgrade` function found in the relevant `fork.md` spec. + +There is only one handler: `core`. Each fork (after genesis) is handled with the same format, + and the particular fork boundary being tested is noted in `meta.yaml`. + +## Test case format + +### `meta.yaml` + +A yaml file to signify which fork boundary is being tested. + +```yaml +fork: str -- Fork being transitioned to +``` + +#### Fork strings + +Key of valid `fork` strings that might be found in `meta.yaml` + +| String ID | Pre-fork | Post-fork | Function | +| - | - | - | - | +| `altair` | Phase 0 | Altair | `upgrade_to_lightclient_patch` | + +### `pre.yaml` + +A YAML-encoded `BeaconState`, the state before running the fork transition. + +Also available as `pre.ssz`. + +### `post.yaml` + +A YAML-encoded `BeaconState`, the state after applying the fork transition. + +Also available as `post.ssz`. + +*Note*: This type is the `BeaconState` after the fork and is *not* the same type as `pre`. + +## Processing + +To process this test, pass `pre` into the upgrade function defined by the `fork` in `meta.yaml`. + +## Condition + +The resulting state should match the expected `post`. diff --git a/tests/generators/forks/main.py b/tests/generators/forks/main.py new file mode 100644 index 000000000..a32ece712 --- /dev/null +++ b/tests/generators/forks/main.py @@ -0,0 +1,40 @@ +from importlib import reload +from typing import Iterable + +from eth2spec.test.context import LIGHTCLIENT_PATCH +from eth2spec.config import config_util +from eth2spec.test.lightclient_patch.fork import test_fork as test_altair_forks +from eth2spec.phase0 import spec as spec_phase0 + +from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing +from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests + + +pre_specs = { + LIGHTCLIENT_PATCH: spec_phase0, +} + + +def create_provider(fork_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: + + def prepare_fn(configs_path: str) -> str: + config_util.prepare_config(configs_path, config_name) + reload(pre_specs[fork_name]) + return config_name + + def cases_fn() -> Iterable[gen_typing.TestCase]: + return generate_from_tests( + runner_name='forks', + handler_name='core', + src=tests_src, + fork_name=fork_name, + ) + + return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) + + +if __name__ == "__main__": + gen_runner.run_generator("forks", [ + create_provider(LIGHTCLIENT_PATCH, test_altair_forks, 'minimal'), + create_provider(LIGHTCLIENT_PATCH, test_altair_forks, 'minimal'), + ]) diff --git a/tests/generators/forks/requirements.txt b/tests/generators/forks/requirements.txt new file mode 100644 index 000000000..816df6e63 --- /dev/null +++ b/tests/generators/forks/requirements.txt @@ -0,0 +1,2 @@ +pytest>=4.4 +../../../ \ No newline at end of file From 50fb3da0729429179e6a24abaf1150b6c023a94b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Mar 2021 20:31:06 +0800 Subject: [PATCH 156/222] Make test_fork.py truly pass --- tests/core/pyspec/eth2spec/test/context.py | 3 +- .../test/lightclient_patch/fork/test_fork.py | 45 ++++++++++--------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 9c45049a2..f3d8e2a8f 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -82,7 +82,7 @@ def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Ca # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper. # Decide based on performance/consistency results later. state = phases[PHASE1].upgrade_to_phase1(state) - elif spec.fork == LIGHTCLIENT_PATCH and not fork_test: # do not upgrade if spec ttttest + elif spec.fork == LIGHTCLIENT_PATCH and not is_fork_test: # do not upgrade if spec ttttest state = phases[LIGHTCLIENT_PATCH].upgrade_to_lightclient_patch(state) return state @@ -294,6 +294,7 @@ def fork_test(fn): def entry(*args, **kw): # override fork test setting kw['fork_test'] = True + return fn(*args, **kw) return entry diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py index 527ad746c..88ff68a06 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py @@ -1,5 +1,5 @@ from eth2spec.test.context import ( - LIGHTCLIENT_PATCH, + PHASE0, LIGHTCLIENT_PATCH, with_phases, with_custom_state, fork_test, spec_test, with_state, @@ -53,39 +53,39 @@ def run_fork_test(spec, pre_state): yield 'post', post_state -@with_phases(([LIGHTCLIENT_PATCH])) -@with_state -@spec_test @fork_test +@with_phases(([PHASE0])) +@spec_test +@with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_base_state(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([LIGHTCLIENT_PATCH])) -@with_state -@spec_test @fork_test +@with_phases(([PHASE0])) +@spec_test +@with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_next_epoch(spec, phases, state): next_epoch(spec, state) yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([LIGHTCLIENT_PATCH])) -@with_state -@spec_test @fork_test +@with_phases(([PHASE0])) +@spec_test +@with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_next_epoch_with_block(spec, phases, state): next_epoch_via_block(spec, state) yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([LIGHTCLIENT_PATCH])) -@with_state -@spec_test @fork_test +@with_phases(([PHASE0])) +@spec_test +@with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_many_next_epoch(spec, phases, state): for _ in range(3): @@ -93,28 +93,29 @@ def test_fork_many_next_epoch(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([LIGHTCLIENT_PATCH])) -@with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) -@spec_test @fork_test +@with_phases(([PHASE0])) +@spec_test +@with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_random_low_balances(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([LIGHTCLIENT_PATCH])) -@with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) -@spec_test @fork_test +@with_phases(([PHASE0])) +@with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_random_misc_balances(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([LIGHTCLIENT_PATCH])) -@with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE) -@spec_test @fork_test +@with_phases(([PHASE0])) +@with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_random_large_validator_set(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) From f97ea9e172e88a7527e7c7fa97e104a90df99a69 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Mar 2021 20:55:39 +0800 Subject: [PATCH 157/222] Generate basic tests. Still having problem with generating `with_custom_state` tests --- .../gen_helpers/gen_from_tests/gen.py | 11 +++++++--- tests/generators/forks/main.py | 21 +++++++------------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py index 67d29b194..f04bf46a7 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py @@ -1,6 +1,6 @@ from importlib import reload, import_module from inspect import getmembers, isfunction -from typing import Any, Callable, Dict, Iterable +from typing import Any, Callable, Dict, Iterable, Optional from eth2spec.config import config_util from eth2spec.utils import bls @@ -11,7 +11,7 @@ from eth2spec.gen_helpers.gen_base.gen_typing import TestCase, TestProvider def generate_from_tests(runner_name: str, handler_name: str, src: Any, - fork_name: SpecForkName, bls_active: bool = True) -> Iterable[TestCase]: + fork_name: SpecForkName, bls_active: bool = True, phase: Optional[str]=None) -> Iterable[TestCase]: """ Generate a list of test cases by running tests from the given src in generator-mode. :param runner_name: to categorize the test in general as. @@ -20,12 +20,17 @@ def generate_from_tests(runner_name: str, handler_name: str, src: Any, :param fork_name: to run tests against particular phase and/or fork. (if multiple forks are applicable, indicate the last fork) :param bls_active: optional, to override BLS switch preference. Defaults to True. + :param phase: optional, specific phase name :return: an iterable of test cases. """ fn_names = [ name for (name, _) in getmembers(src, isfunction) if name.startswith('test_') ] + + if phase is None: + phase = fork_name + print("generating test vectors from tests source: %s" % src.__name__) for name in fn_names: tfn = getattr(src, name) @@ -42,7 +47,7 @@ def generate_from_tests(runner_name: str, handler_name: str, src: Any, suite_name='pyspec_tests', case_name=case_name, # TODO: with_all_phases and other per-phase tooling, should be replaced with per-fork equivalent. - case_fn=lambda: tfn(generator_mode=True, phase=fork_name, bls_active=bls_active) + case_fn=lambda: tfn(generator_mode=True, phase=phase, bls_active=bls_active) ) diff --git a/tests/generators/forks/main.py b/tests/generators/forks/main.py index a32ece712..d39af802a 100644 --- a/tests/generators/forks/main.py +++ b/tests/generators/forks/main.py @@ -1,7 +1,7 @@ from importlib import reload from typing import Iterable -from eth2spec.test.context import LIGHTCLIENT_PATCH +from eth2spec.test.context import PHASE0, LIGHTCLIENT_PATCH, MINIMAL, MAINNET from eth2spec.config import config_util from eth2spec.test.lightclient_patch.fork import test_fork as test_altair_forks from eth2spec.phase0 import spec as spec_phase0 @@ -10,24 +10,20 @@ from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests -pre_specs = { - LIGHTCLIENT_PATCH: spec_phase0, -} - - -def create_provider(fork_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: +def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) - reload(pre_specs[fork_name]) + reload(spec_phase0) return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: return generate_from_tests( - runner_name='forks', - handler_name='core', + runner_name='fork', + handler_name='fork', src=tests_src, - fork_name=fork_name, + fork_name=LIGHTCLIENT_PATCH, + phase=PHASE0, ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) @@ -35,6 +31,5 @@ def create_provider(fork_name: str, tests_src, config_name: str) -> gen_typing.T if __name__ == "__main__": gen_runner.run_generator("forks", [ - create_provider(LIGHTCLIENT_PATCH, test_altair_forks, 'minimal'), - create_provider(LIGHTCLIENT_PATCH, test_altair_forks, 'minimal'), + create_provider(test_altair_forks, MINIMAL), ]) From 7a10c7108ad5b21efac726afe2a5799383457a3f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Mar 2021 21:17:02 +0800 Subject: [PATCH 158/222] Fix decorator calls --- .../eth2spec/test/lightclient_patch/fork/test_fork.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py index 88ff68a06..c5b9b8981 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py @@ -95,9 +95,8 @@ def test_fork_many_next_epoch(spec, phases, state): @fork_test @with_phases(([PHASE0])) -@spec_test @with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) -@with_state +@spec_test @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_random_low_balances(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) @@ -106,7 +105,7 @@ def test_fork_random_low_balances(spec, phases, state): @fork_test @with_phases(([PHASE0])) @with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) -@with_state +@spec_test @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_random_misc_balances(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) @@ -115,7 +114,7 @@ def test_fork_random_misc_balances(spec, phases, state): @fork_test @with_phases(([PHASE0])) @with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE) -@with_state +@spec_test @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_random_large_validator_set(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) From f025ec40c584e45517aeed1f93ce4fa4db1f0f3d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Mar 2021 21:21:32 +0800 Subject: [PATCH 159/222] Fix linter error --- tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py index f04bf46a7..057aa1a2c 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py @@ -11,7 +11,8 @@ from eth2spec.gen_helpers.gen_base.gen_typing import TestCase, TestProvider def generate_from_tests(runner_name: str, handler_name: str, src: Any, - fork_name: SpecForkName, bls_active: bool = True, phase: Optional[str]=None) -> Iterable[TestCase]: + fork_name: SpecForkName, bls_active: bool = True, + phase: Optional[str]=None) -> Iterable[TestCase]: """ Generate a list of test cases by running tests from the given src in generator-mode. :param runner_name: to categorize the test in general as. From 5e864af67a161878c294a41dce579a20f98b62bd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Mar 2021 21:32:37 +0800 Subject: [PATCH 160/222] Reload Altair spec --- tests/generators/forks/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/generators/forks/main.py b/tests/generators/forks/main.py index d39af802a..dd4c0051c 100644 --- a/tests/generators/forks/main.py +++ b/tests/generators/forks/main.py @@ -5,6 +5,7 @@ from eth2spec.test.context import PHASE0, LIGHTCLIENT_PATCH, MINIMAL, MAINNET from eth2spec.config import config_util from eth2spec.test.lightclient_patch.fork import test_fork as test_altair_forks from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests @@ -15,6 +16,7 @@ def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) + reload(spec_lightclient_patch) return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: From f71a3c6b22fe23f28eb49216618a20ef194d1025 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Mar 2021 21:34:45 +0800 Subject: [PATCH 161/222] Generate with mainnet config --- tests/generators/forks/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/generators/forks/main.py b/tests/generators/forks/main.py index dd4c0051c..b22707786 100644 --- a/tests/generators/forks/main.py +++ b/tests/generators/forks/main.py @@ -34,4 +34,5 @@ def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider: if __name__ == "__main__": gen_runner.run_generator("forks", [ create_provider(test_altair_forks, MINIMAL), + create_provider(test_altair_forks, MAINNET), ]) From d59f1945e61a832cba2b8aab617b1e39ae5819fb Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 9 Mar 2021 12:52:04 -0700 Subject: [PATCH 162/222] port leak-score feature without 64-epoch --- specs/lightclient/beacon-chain.md | 60 +++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index bc070908a..5736c22a2 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -118,6 +118,7 @@ This patch updates a few configuration values to move penalty constants toward t | - | - | | `SYNC_COMMITTEE_SIZE` | `uint64(2**10)` (= 1,024) | | `SYNC_SUBCOMMITTEE_SIZE` | `uint64(2**6)` (= 64) | +| `LEAK_SCORE_BIAS` | 4 | ### Time parameters @@ -192,6 +193,8 @@ class BeaconState(Container): # Light client sync committees current_sync_committee: SyncCommittee next_sync_committee: SyncCommittee + # is online in an inactivity leak, inactivity leak penalties are proportional to this value + leak_score: List[uint64, VALIDATOR_REGISTRY_LIMIT] ``` ### New containers @@ -366,23 +369,26 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S flags to determine who participated and who did not, applying the leak penalty globally and applying compensatory rewards to participants. """ - penalties = [Gwei(0) for _ in range(len(state.validators))] - if is_in_inactivity_leak(state): - reward_numerator_sum = sum(numerator for (_, numerator) in get_flag_indices_and_numerators()) - matching_target_attesting_indices = get_unslashed_participating_indices( - state, TIMELY_TARGET_FLAG_INDEX, get_previous_epoch(state) - ) - for index in get_eligible_validator_indices(state): - # If validator is performing optimally this cancels all attestation rewards for a neutral balance - penalties[index] += Gwei(get_base_reward(state, index) * reward_numerator_sum // FLAG_DENOMINATOR) - if index not in matching_target_attesting_indices: - effective_balance = state.validators[index].effective_balance - penalties[index] += Gwei( - effective_balance * get_finality_delay(state) - // HF1_INACTIVITY_PENALTY_QUOTIENT - ) - rewards = [Gwei(0) for _ in range(len(state.validators))] + penalties = [Gwei(0) for _ in range(len(state.validators))] + + if not is_in_inactivity_leak(state): + return rewards, penalties + + reward_numerator_sum = sum(numerator for (_, numerator) in get_flag_indices_and_numerators()) + matching_target_attesting_indices = get_unslashed_participating_indices( + state, TIMELY_TARGET_FLAG_INDEX, get_previous_epoch(state) + ) + for index in get_eligible_validator_indices(state): + # If validator is performing optimally this cancels all attestation rewards for a neutral balance + penalties[index] += Gwei(get_base_reward(state, index) * reward_numerator_sum // FLAG_DENOMINATOR) + if index not in matching_target_attesting_indices and state.leak_score[index] >= LEAK_SCORE_BIAS: + effective_balance = state.validators[index].effective_balance + leak_penalty = Gwei( + effective_balance * state.leak_score[index] // LEAK_SCORE_BIAS // HF1_INACTIVITY_PENALTY_QUOTIENT + ) + penalties[index] += leak_penalty + return rewards, penalties ``` @@ -519,9 +525,9 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: # Add validator and balance entries state.validators.append(get_validator_from_deposit(state, deposit)) state.balances.append(amount) - # [Added in hf-1] Initialize empty participation flags for new validator - state.previous_epoch_participation.append(ParticipationFlags(0b0000_0000)) - state.current_epoch_participation.append(ParticipationFlags(0b0000_0000)) + state.previous_epoch_participation.append(ParticipationFlags(0b0000_0000)) # New in HF1 + state.current_epoch_participation.append(ParticipationFlags(0b0000_0000)) # New in HF1 + state.leak_score.append(0) # New in HF1 else: # Increase balance by deposit amount index = ValidatorIndex(validator_pubkeys.index(pubkey)) @@ -562,6 +568,7 @@ def process_sync_committee(state: BeaconState, body: BeaconBlockBody) -> None: ```python def process_epoch(state: BeaconState) -> None: process_justification_and_finalization(state) # [Modified in HF1] + process_leak_score_updates(state) # [New in HF1] process_rewards_and_penalties(state) # [Modified in HF1] process_registry_updates(state) process_slashings(state) # [Modified in HF1] @@ -620,6 +627,21 @@ def process_justification_and_finalization(state: BeaconState) -> None: state.finalized_checkpoint = old_current_justified_checkpoint ``` +#### Leak scores + +```python +def process_leak_score_updates(state: BeaconState) -> None: + matching_target_attesting_indices = get_unslashed_participating_indices( + state, TIMELY_TARGET_FLAG_INDEX, get_previous_epoch(state) + ) + for index in get_eligible_validator_indices(state): + if index in matching_target_attesting_indices: + if state.leak_score[index] > 0: + state.leak_score[index] -= 1 + elif is_in_inactivity_leak(state): + state.leak_score[index] += LEAK_SCORE_BIAS +``` + #### Rewards and penalties *Note*: The function `process_rewards_and_penalties` is modified to support the incentive reforms. From 70a28340582dee52ad91d9d12d6f08364c76760c Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 9 Mar 2021 13:21:16 -0700 Subject: [PATCH 163/222] incorporate justin's pr --- specs/lightclient/beacon-chain.md | 44 +++++++++++++++---------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 5736c22a2..c49e34c6c 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -41,10 +41,11 @@ - [New `slash_validator`](#new-slash_validator) - [Block processing](#block-processing) - [Modified `process_attestation`](#modified-process_attestation) - - [New `process_deposit`](#new-process_deposit) + - [Modified `process_deposit`](#modified-process_deposit) - [Sync committee processing](#sync-committee-processing) - [Epoch processing](#epoch-processing) - [Justification and finalization](#justification-and-finalization) + - [Leak scores](#leak-scores) - [Rewards and penalties](#rewards-and-penalties) - [Slashings](#slashings) - [Participation flags updates](#participation-flags-updates) @@ -183,8 +184,8 @@ class BeaconState(Container): # Slashings slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR] # Per-epoch sums of slashed effective balances # Participation - previous_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT] - current_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT] + previous_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT] # [New in HF1] + current_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT] # [New in HF1] # Finality justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH] # Bit set for every recent justified epoch previous_justified_checkpoint: Checkpoint @@ -193,8 +194,8 @@ class BeaconState(Container): # Light client sync committees current_sync_committee: SyncCommittee next_sync_committee: SyncCommittee - # is online in an inactivity leak, inactivity leak penalties are proportional to this value - leak_score: List[uint64, VALIDATOR_REGISTRY_LIMIT] + # Leak + leak_scores: List[uint64, VALIDATOR_REGISTRY_LIMIT] ``` ### New containers @@ -287,13 +288,10 @@ def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: Return the sync committee for a given state and epoch. """ indices = get_sync_committee_indices(state, epoch) - validators = [state.validators[index] for index in indices] - pubkeys = [validator.pubkey for validator in validators] - aggregates = [ - bls.AggregatePKs(pubkeys[i:i + SYNC_SUBCOMMITTEE_SIZE]) - for i in range(0, len(pubkeys), SYNC_SUBCOMMITTEE_SIZE) - ] - return SyncCommittee(pubkeys=pubkeys, pubkey_aggregates=aggregates) + pubkeys = [state.validators[index].pubkey for index in indices] + subcommitees = [pubkeys[i:i + SYNC_SUBCOMMITTEE_SIZE] for i in range(0, len(pubkeys), SYNC_SUBCOMMITTEE_SIZE)] + pubkey_aggregates = [bls.AggregatePKs(subcommitee) for subcommitee in subcommitees] + return SyncCommittee(pubkeys=pubkeys, pubkey_aggregates=pubkey_aggregates) ``` #### `get_base_reward` @@ -382,10 +380,10 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S for index in get_eligible_validator_indices(state): # If validator is performing optimally this cancels all attestation rewards for a neutral balance penalties[index] += Gwei(get_base_reward(state, index) * reward_numerator_sum // FLAG_DENOMINATOR) - if index not in matching_target_attesting_indices and state.leak_score[index] >= LEAK_SCORE_BIAS: + if index not in matching_target_attesting_indices and state.leak_scores[index] >= LEAK_SCORE_BIAS: effective_balance = state.validators[index].effective_balance leak_penalty = Gwei( - effective_balance * state.leak_score[index] // LEAK_SCORE_BIAS // HF1_INACTIVITY_PENALTY_QUOTIENT + effective_balance * state.leak_scores[index] // LEAK_SCORE_BIAS // HF1_INACTIVITY_PENALTY_QUOTIENT ) penalties[index] += leak_penalty @@ -489,9 +487,9 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: ``` -#### New `process_deposit` +#### Modified `process_deposit` -*Note*: The function `process_deposit` is modified to initialize `previous_epoch_participation` and `current_epoch_participation`. +*Note*: The function `process_deposit` is modified to initialize `leak_scores`, `previous_epoch_participation`, `current_epoch_participation`. ```python def process_deposit(state: BeaconState, deposit: Deposit) -> None: @@ -527,7 +525,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: state.balances.append(amount) state.previous_epoch_participation.append(ParticipationFlags(0b0000_0000)) # New in HF1 state.current_epoch_participation.append(ParticipationFlags(0b0000_0000)) # New in HF1 - state.leak_score.append(0) # New in HF1 + state.leak_scores.append(0) # New in HF1 else: # Increase balance by deposit amount index = ValidatorIndex(validator_pubkeys.index(pubkey)) @@ -568,7 +566,7 @@ def process_sync_committee(state: BeaconState, body: BeaconBlockBody) -> None: ```python def process_epoch(state: BeaconState) -> None: process_justification_and_finalization(state) # [Modified in HF1] - process_leak_score_updates(state) # [New in HF1] + process_leak_updates(state) # [New in HF1] process_rewards_and_penalties(state) # [Modified in HF1] process_registry_updates(state) process_slashings(state) # [Modified in HF1] @@ -629,17 +627,19 @@ def process_justification_and_finalization(state: BeaconState) -> None: #### Leak scores +*Note*: The function `process_leak_updates` is new. + ```python -def process_leak_score_updates(state: BeaconState) -> None: +def process_leak_updates(state: BeaconState) -> None: matching_target_attesting_indices = get_unslashed_participating_indices( state, TIMELY_TARGET_FLAG_INDEX, get_previous_epoch(state) ) for index in get_eligible_validator_indices(state): if index in matching_target_attesting_indices: - if state.leak_score[index] > 0: - state.leak_score[index] -= 1 + if state.leak_scores[index] > 0: + state.leak_scores[index] -= 1 elif is_in_inactivity_leak(state): - state.leak_score[index] += LEAK_SCORE_BIAS + state.leak_scores[index] += LEAK_SCORE_BIAS ``` #### Rewards and penalties From e2abdb74ae6340ecf2be5f871bd070254dc05e37 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 9 Mar 2021 15:16:26 -0700 Subject: [PATCH 164/222] port testing --- configs/mainnet/lightclient_patch.yaml | 2 + configs/minimal/lightclient_patch.yaml | 2 + specs/lightclient/beacon-chain.md | 9 +-- specs/lightclient/lightclient-fork.md | 7 +- .../pyspec/eth2spec/test/helpers/rewards.py | 9 ++- .../lightclient_patch/sanity/test_blocks.py | 24 ++++++ .../test_process_attester_slashing.py | 55 ++++++++++++-- .../test_process_effective_balance_updates.py | 12 +-- .../test_process_rewards_and_penalties.py | 74 ++++++++----------- 9 files changed, 125 insertions(+), 69 deletions(-) diff --git a/configs/mainnet/lightclient_patch.yaml b/configs/mainnet/lightclient_patch.yaml index a9ddc16f6..b151f5cdb 100644 --- a/configs/mainnet/lightclient_patch.yaml +++ b/configs/mainnet/lightclient_patch.yaml @@ -18,6 +18,8 @@ HF1_PROPORTIONAL_SLASHING_MULTIPLIER: 2 SYNC_COMMITTEE_SIZE: 1024 # 2**6 (=64) SYNC_SUBCOMMITTEE_SIZE: 64 +# 2**2 (=4) +LEAK_SCORE_BIAS: 4 # Time parameters diff --git a/configs/minimal/lightclient_patch.yaml b/configs/minimal/lightclient_patch.yaml index 56ce34591..cbdbaf8a7 100644 --- a/configs/minimal/lightclient_patch.yaml +++ b/configs/minimal/lightclient_patch.yaml @@ -18,6 +18,8 @@ HF1_PROPORTIONAL_SLASHING_MULTIPLIER: 2 SYNC_COMMITTEE_SIZE: 32 # [customized] SYNC_SUBCOMMITTEE_SIZE: 16 +# 2**2 (=4) +LEAK_SCORE_BIAS: 4 # Time parameters diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index c49e34c6c..5bebc8836 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -18,7 +18,7 @@ - [Time parameters](#time-parameters) - [Domain types](#domain-types) - [Containers](#containers) - - [Extended containers](#extended-containers) + - [Modified containers](#modified-containers) - [`BeaconBlockBody`](#beaconblockbody) - [`BeaconState`](#beaconstate) - [New containers](#new-containers) @@ -135,10 +135,7 @@ This patch updates a few configuration values to move penalty constants toward t ## Containers -### Extended containers - -*Note*: Extended SSZ containers inherit all fields from the parent in the original -order and append any additional fields to the end. +### Modified containers #### `BeaconBlockBody` @@ -195,7 +192,7 @@ class BeaconState(Container): current_sync_committee: SyncCommittee next_sync_committee: SyncCommittee # Leak - leak_scores: List[uint64, VALIDATOR_REGISTRY_LIMIT] + leak_scores: List[uint64, VALIDATOR_REGISTRY_LIMIT] # [New in HF1] ``` ### New containers diff --git a/specs/lightclient/lightclient-fork.md b/specs/lightclient/lightclient-fork.md index 157e67dc9..17a65793a 100644 --- a/specs/lightclient/lightclient-fork.md +++ b/specs/lightclient/lightclient-fork.md @@ -42,6 +42,7 @@ After `process_slots` of Phase 0 finishes, if `state.slot == LIGHTCLIENT_PATCH_F def upgrade_to_lightclient_patch(pre: phase0.BeaconState) -> BeaconState: epoch = get_current_epoch(pre) post = BeaconState( + # Versioning genesis_time=pre.genesis_time, genesis_validators_root=pre.genesis_validators_root, slot=pre.slot, @@ -67,13 +68,15 @@ def upgrade_to_lightclient_patch(pre: phase0.BeaconState) -> BeaconState: # Slashings slashings=pre.slashings, # Participation - previous_epoch_participation=[ParticipationFlags(0) for _ in range(len(pre.validators))], - current_epoch_participation=[ParticipationFlags(0) for _ in range(len(pre.validators))], + previous_epoch_participation=[ParticipationFlags(0b0000_0000) for _ in range(len(pre.validators))], + current_epoch_participation=[ParticipationFlags(0b0000_0000) for _ in range(len(pre.validators))], # Finality justification_bits=pre.justification_bits, previous_justified_checkpoint=pre.previous_justified_checkpoint, current_justified_checkpoint=pre.current_justified_checkpoint, finalized_checkpoint=pre.finalized_checkpoint, + # Leak + leak_scores=[0 for _ in range(len(pre.validators))], ) # Fill in sync committees post.current_sync_committee = get_sync_committee(post, get_current_epoch(post)) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 19ed3f691..a92a59458 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -34,7 +34,8 @@ def run_deltas(spec, state): - source deltas ('source_deltas') - target deltas ('target_deltas') - head deltas ('head_deltas') - - inclusion delay deltas ('inclusion_delay_deltas') + - not if is_post_lightclient_patch(spec) + - inclusion delay deltas ('inclusion_delay_deltas') - inactivity penalty deltas ('inactivity_penalty_deltas') """ yield 'pre', state @@ -70,7 +71,8 @@ def run_deltas(spec, state): spec.get_matching_head_attestations, 'head_deltas', ) - yield from run_get_inclusion_delay_deltas(spec, state) + if not is_post_lightclient_patch(spec): + yield from run_get_inclusion_delay_deltas(spec, state) yield from run_get_inactivity_penalty_deltas(spec, state) @@ -219,7 +221,8 @@ def run_get_inactivity_penalty_deltas(spec, state): def transition_state_to_leak(spec, state, epochs=None): if epochs is None: - epochs = spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + # +1 to trigger leak_score transitions + epochs = spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + 1 assert epochs >= spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY for _ in range(epochs): diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py index 9033a0f15..3689c0783 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py @@ -2,9 +2,11 @@ import random from eth2spec.test.helpers.state import ( state_transition_and_sign_block, next_epoch, + next_epoch_via_block, ) from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, + build_empty_block, ) from eth2spec.test.helpers.sync_committee import ( compute_aggregate_sync_committee_signature, @@ -73,3 +75,25 @@ def test_half_sync_committee_committee_genesis(spec, state): @spec_state_test def test_empty_sync_committee_committee_genesis(spec, state): yield from run_sync_committee_sanity_test(spec, state, fraction_full=0.0) + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_leak_scores(spec, state): + for _ in range(spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + 2): + next_epoch_via_block(spec, state) + + assert spec.is_in_inactivity_leak(state) + previous_leak_scores = state.leak_scores.copy() + + yield 'pre', state + + # Block transition to next epoch + block = build_empty_block(spec, state, slot=state.slot + spec.SLOTS_PER_EPOCH) + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state + + for pre, post in zip(previous_leak_scores, state.leak_scores): + assert post == pre + spec.LEAK_SCORE_BIAS diff --git a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attester_slashing.py b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attester_slashing.py index 21d9363f7..7345e62ba 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attester_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attester_slashing.py @@ -1,5 +1,9 @@ +import random + from eth2spec.test.context import ( - spec_state_test, expect_assertion_error, always_bls, with_all_phases + spec_state_test, expect_assertion_error, always_bls, with_all_phases, + with_custom_state, spec_test, single_phase, + low_balances, misc_balances, ) from eth2spec.test.helpers.attestations import sign_indexed_attestation from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing, \ @@ -32,15 +36,19 @@ def run_attester_slashing_processing(spec, state, attester_slashing, valid=True) proposer_index = spec.get_beacon_proposer_index(state) pre_proposer_balance = get_balance(state, proposer_index) - pre_slashings = {slashed_index: get_balance(state, slashed_index) for slashed_index in slashed_indices} + pre_slashing_balances = {slashed_index: get_balance(state, slashed_index) for slashed_index in slashed_indices} + pre_slashing_effectives = { + slashed_index: state.validators[slashed_index].effective_balance + for slashed_index in slashed_indices + } pre_withdrawalable_epochs = { slashed_index: state.validators[slashed_index].withdrawable_epoch for slashed_index in slashed_indices } total_proposer_rewards = sum( - balance // spec.WHISTLEBLOWER_REWARD_QUOTIENT - for balance in pre_slashings.values() + effective_balance // spec.WHISTLEBLOWER_REWARD_QUOTIENT + for effective_balance in pre_slashing_effectives.values() ) # Process slashing @@ -61,7 +69,7 @@ def run_attester_slashing_processing(spec, state, attester_slashing, valid=True) assert slashed_validator.withdrawable_epoch == expected_withdrawable_epoch else: assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH - assert get_balance(state, slashed_index) < pre_slashings[slashed_index] + assert get_balance(state, slashed_index) < pre_slashing_balances[slashed_index] if proposer_index not in slashed_indices: # gained whistleblower reward @@ -71,7 +79,7 @@ def run_attester_slashing_processing(spec, state, attester_slashing, valid=True) expected_balance = ( pre_proposer_balance + total_proposer_rewards - - pre_slashings[proposer_index] // get_min_slashing_penalty_quotient(spec) + - pre_slashing_effectives[proposer_index] // get_min_slashing_penalty_quotient(spec) ) assert get_balance(state, proposer_index) == expected_balance @@ -118,6 +126,41 @@ def test_success_already_exited_recent(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing) +@with_all_phases +@with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@spec_test +@single_phase +def test_success_low_balances(spec, state): + attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) + + yield from run_attester_slashing_processing(spec, state, attester_slashing) + + +@with_all_phases +@with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@spec_test +@single_phase +def test_success_misc_balances(spec, state): + attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) + + yield from run_attester_slashing_processing(spec, state, attester_slashing) + + +@with_all_phases +@with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@spec_test +@single_phase +def test_success_with_effective_balance_disparity(spec, state): + # Jitter balances to be different from effective balances + for i in range(len(state.balances)): + pre = int(state.balances[i]) + state.balances[i] += random.randrange(max(pre - 5000, 0), pre + 5000) + + attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) + + yield from run_attester_slashing_processing(spec, state, attester_slashing) + + @with_all_phases @spec_state_test @always_bls diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_effective_balance_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_effective_balance_updates.py index 93411f657..dc4c047a2 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_effective_balance_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_effective_balance_updates.py @@ -1,11 +1,5 @@ from eth2spec.test.context import spec_state_test, with_all_phases -from eth2spec.test.helpers.epoch_processing import ( - run_epoch_processing_with, run_epoch_processing_to -) - - -def run_process_effective_balance_updates(spec, state): - yield from run_epoch_processing_with(spec, state, 'process_effective_balance_updates') +from eth2spec.test.helpers.epoch_processing import run_epoch_processing_to @with_all_phases @@ -44,7 +38,9 @@ def test_effective_balance_hysteresis(spec, state): state.validators[i].effective_balance = pre_eff state.balances[i] = bal - yield from run_process_effective_balance_updates(spec, state) + yield 'pre', state + spec.process_effective_balance_updates(state) + yield 'post', state for i, (_, _, post_eff, name) in enumerate(cases): assert state.validators[i].effective_balance == post_eff, name diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py index 7bb86b45e..2de054505 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py @@ -27,6 +27,31 @@ def run_process_rewards_and_penalties(spec, state): yield from run_epoch_processing_with(spec, state, 'process_rewards_and_penalties') +def validate_resulting_balances(spec, pre_state, post_state, attestations): + attesting_indices = spec.get_unslashed_attesting_indices(post_state, attestations) + current_epoch = spec.get_current_epoch(post_state) + + for index in range(len(pre_state.validators)): + if not spec.is_active_validator(pre_state.validators[index], current_epoch): + assert post_state.balances[index] == pre_state.balances[index] + elif not is_post_lightclient_patch(spec): + proposer_indices = [a.proposer_index for a in post_state.previous_epoch_attestations] + if spec.is_in_inactivity_leak(post_state): + # Proposers can still make money during a leak before LIGHTCLIENT_PATCH + if index in proposer_indices and index in attesting_indices: + assert post_state.balances[index] > pre_state.balances[index] + elif index in attesting_indices: + # If not proposer but participated optimally, should have exactly neutral balance + assert post_state.balances[index] == pre_state.balances[index] + else: + assert post_state.balances[index] < pre_state.balances[index] + else: + if index in attesting_indices: + assert post_state.balances[index] > pre_state.balances[index] + else: + assert post_state.balances[index] < pre_state.balances[index] + + @with_all_phases @spec_state_test def test_genesis_epoch_no_attestations_no_penalties(spec, state): @@ -100,19 +125,10 @@ def test_full_attestations_misc_balances(spec, state): yield from run_process_rewards_and_penalties(spec, state) - attesting_indices = spec.get_unslashed_attesting_indices(state, attestations) - assert len(attesting_indices) > 0 - assert len(attesting_indices) != len(pre_state.validators) - assert any(v.effective_balance != spec.MAX_EFFECTIVE_BALANCE for v in state.validators) - for index in range(len(pre_state.validators)): - if index in attesting_indices: - assert state.balances[index] > pre_state.balances[index] - elif spec.is_active_validator(pre_state.validators[index], spec.compute_epoch_at_slot(state.slot)): - assert state.balances[index] < pre_state.balances[index] - else: - assert state.balances[index] == pre_state.balances[index] + validate_resulting_balances(spec, pre_state, state, attestations) # Check if base rewards are consistent with effective balance. brs = {} + attesting_indices = spec.get_unslashed_attesting_indices(state, attestations) for index in attesting_indices: br = spec.get_base_reward(state, index) if br in brs: @@ -146,8 +162,7 @@ def test_no_attestations_all_penalties(spec, state): yield from run_process_rewards_and_penalties(spec, state) - for index in range(len(pre_state.validators)): - assert state.balances[index] < pre_state.balances[index] + validate_resulting_balances(spec, pre_state, state, []) def run_with_participation(spec, state, participation_fn): @@ -161,35 +176,12 @@ def run_with_participation(spec, state, participation_fn): attestations = prepare_state_with_attestations(spec, state, participation_fn=participation_tracker) pre_state = state.copy() - if not is_post_lightclient_patch(spec): - proposer_indices = [a.proposer_index for a in state.previous_epoch_attestations] - else: - sync_committee_indices = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) - yield from run_process_rewards_and_penalties(spec, state) attesting_indices = spec.get_unslashed_attesting_indices(state, attestations) assert len(attesting_indices) == len(participated) - for index in range(len(pre_state.validators)): - if spec.is_in_inactivity_leak(state): - # Proposers can still make money during a leak before LIGHTCLIENT_PATCH - if not is_post_lightclient_patch(spec) and index in proposer_indices and index in participated: - assert state.balances[index] > pre_state.balances[index] - elif index in attesting_indices: - if is_post_lightclient_patch(spec) and index in sync_committee_indices: - # The sync committee reward has not been canceled, so the sync committee participants still earn it - assert state.balances[index] >= pre_state.balances[index] - else: - # If not proposer but participated optimally, should have exactly neutral balance - assert state.balances[index] == pre_state.balances[index] - else: - assert state.balances[index] < pre_state.balances[index] - else: - if index in participated: - assert state.balances[index] > pre_state.balances[index] - else: - assert state.balances[index] < pre_state.balances[index] + validate_resulting_balances(spec, pre_state, state, attestations) @with_all_phases @@ -438,10 +430,4 @@ def test_attestations_some_slashed(spec, state): attesting_indices = spec.get_unslashed_attesting_indices(state, attestations) assert len(attesting_indices) > 0 assert len(attesting_indices_before_slashings) - len(attesting_indices) == spec.MIN_PER_EPOCH_CHURN_LIMIT - for index in range(len(pre_state.validators)): - if index in attesting_indices: - # non-slashed attester should gain reward - assert state.balances[index] > pre_state.balances[index] - else: - # Slashed non-proposer attester should have penalty - assert state.balances[index] < pre_state.balances[index] + validate_resulting_balances(spec, pre_state, state, attestations) From 956a7a2ef1a20da3472307768d41e28b69c80840 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 9 Mar 2021 16:09:08 -0700 Subject: [PATCH 165/222] Update tests/core/pyspec/eth2spec/test/context.py --- tests/core/pyspec/eth2spec/test/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index f3d8e2a8f..916c74e6a 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -82,7 +82,7 @@ def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Ca # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper. # Decide based on performance/consistency results later. state = phases[PHASE1].upgrade_to_phase1(state) - elif spec.fork == LIGHTCLIENT_PATCH and not is_fork_test: # do not upgrade if spec ttttest + elif spec.fork == LIGHTCLIENT_PATCH and not is_fork_test: # do not upgrade if spec test state = phases[LIGHTCLIENT_PATCH].upgrade_to_lightclient_patch(state) return state From f9b54ea03ba57ee3335721596513c45f1f2eaa7c Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 9 Mar 2021 16:18:30 -0700 Subject: [PATCH 166/222] remove fork_test --- tests/core/pyspec/eth2spec/test/context.py | 19 ++++--------------- .../test/lightclient_patch/fork/test_fork.py | 9 +-------- 2 files changed, 5 insertions(+), 23 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 916c74e6a..1272485de 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -70,7 +70,7 @@ class SpecForks(TypedDict, total=False): def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Callable[[Any], int], - spec: Spec, phases: SpecForks, is_fork_test: bool): + spec: Spec, phases: SpecForks): p0 = phases[PHASE0] balances = balances_fn(p0) @@ -82,7 +82,7 @@ def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Ca # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper. # Decide based on performance/consistency results later. state = phases[PHASE1].upgrade_to_phase1(state) - elif spec.fork == LIGHTCLIENT_PATCH and not is_fork_test: # do not upgrade if spec test + elif spec.fork == LIGHTCLIENT_PATCH: state = phases[LIGHTCLIENT_PATCH].upgrade_to_lightclient_patch(state) return state @@ -98,11 +98,10 @@ def with_custom_state(balances_fn: Callable[[Any], Sequence[int]], def entry(*args, spec: Spec, phases: SpecForks, **kw): # make a key for the state # genesis fork version separates configs during test-generation runtime. - is_fork_test = kw.pop('fork_test') if 'fork_test' in kw else False - key = (spec.fork, spec.GENESIS_FORK_VERSION, spec.__file__, balances_fn, threshold_fn, is_fork_test) + key = (spec.fork, spec.GENESIS_FORK_VERSION, spec.__file__, balances_fn, threshold_fn) global _custom_state_cache_dict if key not in _custom_state_cache_dict: - state = _prepare_state(balances_fn, threshold_fn, spec, phases, is_fork_test) + state = _prepare_state(balances_fn, threshold_fn, spec, phases) _custom_state_cache_dict[key] = state.get_backing() # Take an entry out of the LRU. @@ -288,16 +287,6 @@ def bls_switch(fn): return entry -def fork_test(fn): - """ - """ - def entry(*args, **kw): - # override fork test setting - kw['fork_test'] = True - return fn(*args, **kw) - return entry - - def disable_process_reveal_deadlines(fn): """ Decorator to make a function execute with `process_reveal_deadlines` OFF. diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py index c5b9b8981..5904805cf 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py @@ -1,7 +1,7 @@ from eth2spec.test.context import ( PHASE0, LIGHTCLIENT_PATCH, with_phases, - with_custom_state, fork_test, + with_custom_state, spec_test, with_state, low_balances, misc_balances, large_validator_set, ) @@ -53,7 +53,6 @@ def run_fork_test(spec, pre_state): yield 'post', post_state -@fork_test @with_phases(([PHASE0])) @spec_test @with_state @@ -62,7 +61,6 @@ def test_fork_base_state(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@fork_test @with_phases(([PHASE0])) @spec_test @with_state @@ -72,7 +70,6 @@ def test_fork_next_epoch(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@fork_test @with_phases(([PHASE0])) @spec_test @with_state @@ -82,7 +79,6 @@ def test_fork_next_epoch_with_block(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@fork_test @with_phases(([PHASE0])) @spec_test @with_state @@ -93,7 +89,6 @@ def test_fork_many_next_epoch(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@fork_test @with_phases(([PHASE0])) @with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test @@ -102,7 +97,6 @@ def test_fork_random_low_balances(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@fork_test @with_phases(([PHASE0])) @with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test @@ -111,7 +105,6 @@ def test_fork_random_misc_balances(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@fork_test @with_phases(([PHASE0])) @with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test From 338be1f636b7eae6a6bb3182d4b6682984e788c7 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 9 Mar 2021 16:27:06 -0700 Subject: [PATCH 167/222] clean up fork gens --- tests/formats/forks/README.md | 2 +- tests/generators/forks/main.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/formats/forks/README.md b/tests/formats/forks/README.md index 1a7ee64ad..57cc09227 100644 --- a/tests/formats/forks/README.md +++ b/tests/formats/forks/README.md @@ -3,7 +3,7 @@ The aim of the fork tests is to ensure that a pre-fork state can be transformed into a valid post-fork state, utilizing the `upgrade` function found in the relevant `fork.md` spec. -There is only one handler: `core`. Each fork (after genesis) is handled with the same format, +There is only one handler: `fork`. Each fork (after genesis) is handled with the same format, and the particular fork boundary being tested is noted in `meta.yaml`. ## Test case format diff --git a/tests/generators/forks/main.py b/tests/generators/forks/main.py index b22707786..1a603a0dc 100644 --- a/tests/generators/forks/main.py +++ b/tests/generators/forks/main.py @@ -11,7 +11,7 @@ from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests -def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider: +def create_provider(tests_src, config_name: str, phase: str, fork_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) @@ -24,8 +24,9 @@ def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider: runner_name='fork', handler_name='fork', src=tests_src, - fork_name=LIGHTCLIENT_PATCH, - phase=PHASE0, + fork_name=fork_name, + phase=phase, + ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) @@ -33,6 +34,6 @@ def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider: if __name__ == "__main__": gen_runner.run_generator("forks", [ - create_provider(test_altair_forks, MINIMAL), - create_provider(test_altair_forks, MAINNET), + create_provider(test_altair_forks, MINIMAL, PHASE0, LIGHTCLIENT_PATCH), + create_provider(test_altair_forks, MAINNET, PHASE0, LIGHTCLIENT_PATCH), ]) From 91f6956b3a0a6f210ec4d865f47836cde11e24a5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 10 Mar 2021 21:55:50 +0800 Subject: [PATCH 168/222] Bump py_ecc to 5.2.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6cfa0910a..efe2f379f 100644 --- a/setup.py +++ b/setup.py @@ -581,7 +581,7 @@ setup( "eth-utils>=1.3.0,<2", "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.9.4", - "py_ecc==5.1.0", + "py_ecc==5.2.0", "milagro_bls_binding==1.6.3", "dataclasses==0.6", "remerkleable==0.1.18", From e792c27c915f96e9b576b0a21f3c2cab73a93894 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 10 Mar 2021 12:27:50 -0700 Subject: [PATCH 169/222] @hwwhww review Co-authored-by: Hsiao-Wei Wang --- .../pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py | 4 ++-- .../eth2spec/test/lightclient_patch/fork/test_fork.py | 8 ++++---- tests/generators/forks/main.py | 1 - 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py index 057aa1a2c..c090869d5 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py @@ -18,10 +18,10 @@ def generate_from_tests(runner_name: str, handler_name: str, src: Any, :param runner_name: to categorize the test in general as. :param handler_name: to categorize the test specialization as. :param src: to retrieve tests from (discovered using inspect.getmembers). - :param fork_name: to run tests against particular phase and/or fork. + :param fork_name: the folder name for these tests. (if multiple forks are applicable, indicate the last fork) :param bls_active: optional, to override BLS switch preference. Defaults to True. - :param phase: optional, specific phase name + :param phase: optional, to run tests against a particular spec version. Default to `fork_name` value. :return: an iterable of test cases. """ fn_names = [ diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py index 5904805cf..dada24af9 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py @@ -17,10 +17,10 @@ HF1_FORK_TEST_META_TAGS = { } -def run_fork_test(spec, pre_state): +def run_fork_test(post_spec, pre_state): yield 'pre', pre_state - post_state = spec.upgrade_to_lightclient_patch(pre_state) + post_state = post_spec.upgrade_to_lightclient_patch(pre_state) # Stable fields stable_fields = [ @@ -47,8 +47,8 @@ def run_fork_test(spec, pre_state): assert getattr(pre_state, field) != getattr(post_state, field) assert pre_state.fork.current_version == post_state.fork.previous_version - assert post_state.fork.current_version == spec.LIGHTCLIENT_PATCH_FORK_VERSION - assert post_state.fork.epoch == spec.get_current_epoch(post_state) + assert post_state.fork.current_version == post_spec.LIGHTCLIENT_PATCH_FORK_VERSION + assert post_state.fork.epoch == post_spec.get_current_epoch(post_state) yield 'post', post_state diff --git a/tests/generators/forks/main.py b/tests/generators/forks/main.py index 1a603a0dc..190d4620c 100644 --- a/tests/generators/forks/main.py +++ b/tests/generators/forks/main.py @@ -26,7 +26,6 @@ def create_provider(tests_src, config_name: str, phase: str, fork_name: str) -> src=tests_src, fork_name=fork_name, phase=phase, - ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) From 1f3e73703c10c5e9c8c3dc182632a535222a8d7f Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 10 Mar 2021 12:38:30 -0700 Subject: [PATCH 170/222] use 'other_phases' for fork tests --- tests/core/pyspec/eth2spec/test/context.py | 11 +++++++---- .../test/lightclient_patch/fork/test_fork.py | 14 +++++++------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 1272485de..e197124c0 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -340,7 +340,7 @@ def with_phases(phases, other_phases=None): available_phases = set(run_phases) if other_phases is not None: - available_phases += set(other_phases) + available_phases |= set(other_phases) # TODO: test state is dependent on phase0 but is immediately transitioned to phase1. # A new state-creation helper for phase 1 may be in place, and then phase1+ tests can run without phase0 @@ -348,9 +348,12 @@ def with_phases(phases, other_phases=None): # Populate all phases for multi-phase tests phase_dir = {} - phase_dir[PHASE0] = spec_phase0 - phase_dir[PHASE1] = spec_phase1 - phase_dir[LIGHTCLIENT_PATCH] = spec_lightclient_patch + if PHASE0 in available_phases: + phase_dir[PHASE0] = spec_phase0 + if PHASE1 in available_phases: + phase_dir[PHASE1] = spec_phase1 + if LIGHTCLIENT_PATCH in available_phases: + phase_dir[LIGHTCLIENT_PATCH] = spec_lightclient_patch # return is ignored whenever multiple phases are ran. If if PHASE0 in run_phases: diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py index dada24af9..27d181510 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py @@ -53,7 +53,7 @@ def run_fork_test(post_spec, pre_state): yield 'post', post_state -@with_phases(([PHASE0])) +@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH]) @spec_test @with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) @@ -61,7 +61,7 @@ def test_fork_base_state(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([PHASE0])) +@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH]) @spec_test @with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) @@ -70,7 +70,7 @@ def test_fork_next_epoch(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([PHASE0])) +@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH]) @spec_test @with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) @@ -79,7 +79,7 @@ def test_fork_next_epoch_with_block(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([PHASE0])) +@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH]) @spec_test @with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) @@ -89,7 +89,7 @@ def test_fork_many_next_epoch(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([PHASE0])) +@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH]) @with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test @with_meta_tags(HF1_FORK_TEST_META_TAGS) @@ -97,7 +97,7 @@ def test_fork_random_low_balances(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([PHASE0])) +@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH]) @with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test @with_meta_tags(HF1_FORK_TEST_META_TAGS) @@ -105,7 +105,7 @@ def test_fork_random_misc_balances(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([PHASE0])) +@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH]) @with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test @with_meta_tags(HF1_FORK_TEST_META_TAGS) From 37c49ffcdcc7eba5df5a62b5b885f9618468b616 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 10 Mar 2021 13:11:03 -0700 Subject: [PATCH 171/222] ensure rewards are tested properly post altair fork --- .../test_process_rewards_and_penalties.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py index 2de054505..6d05a498e 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py @@ -50,6 +50,18 @@ def validate_resulting_balances(spec, pre_state, post_state, attestations): assert post_state.balances[index] > pre_state.balances[index] else: assert post_state.balances[index] < pre_state.balances[index] + else: + if spec.is_in_inactivity_leak(post_state): + if index in attesting_indices: + # If not proposer but participated optimally, should have exactly neutral balance + assert post_state.balances[index] == pre_state.balances[index] + else: + assert post_state.balances[index] < pre_state.balances[index] + else: + if index in attesting_indices: + assert post_state.balances[index] > pre_state.balances[index] + else: + assert post_state.balances[index] < pre_state.balances[index] @with_all_phases From b2a172ab21ec8cde863f7a06c1078063af994ec4 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 10 Mar 2021 13:12:48 -0700 Subject: [PATCH 172/222] @hwwhww review Co-authored-by: Hsiao-Wei Wang --- specs/lightclient/beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 5bebc8836..040d00eae 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -189,8 +189,8 @@ class BeaconState(Container): current_justified_checkpoint: Checkpoint finalized_checkpoint: Checkpoint # Light client sync committees - current_sync_committee: SyncCommittee - next_sync_committee: SyncCommittee + current_sync_committee: SyncCommittee # [New in HF1] + next_sync_committee: SyncCommittee # [New in HF1] # Leak leak_scores: List[uint64, VALIDATOR_REGISTRY_LIMIT] # [New in HF1] ``` From 7205f70192b5ec9dc409ee2f99eebef37ea123e1 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 10 Mar 2021 16:48:53 -0700 Subject: [PATCH 173/222] patch remaining generator docs to reflect snappy_ssz encoding --- tests/formats/README.md | 10 +++++----- tests/formats/forks/README.md | 12 ++++-------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/tests/formats/README.md b/tests/formats/README.md index d3933abdd..7808538ad 100644 --- a/tests/formats/README.md +++ b/tests/formats/README.md @@ -134,10 +134,10 @@ Cases are split up too. This enables diffing of parts of the test case, tracking These files allow for custom formats for some parts of the test. E.g. something encoded in SSZ. Or to avoid large files, the SSZ can be compressed with Snappy. -E.g. `pre.ssz_snappy_snappy`, `deposit.ssz_snappy_snappy`, `post.ssz_snappy_snappy`. +E.g. `pre.ssz_snappy`, `deposit.ssz_snappy`, `post.ssz_snappy`. -Diffing a `pre.ssz_snappy_snappy` and `post.ssz_snappy_snappy` provides all the information for testing, when decompressed and decoded. -Then the difference between pre and post can be compared to anything that changes the pre state, e.g. `deposit.ssz_snappy_snappy` +Diffing a `pre.ssz_snappy` and `post.ssz_snappy` provides all the information for testing, when decompressed and decoded. +Then the difference between pre and post can be compared to anything that changes the pre state, e.g. `deposit.ssz_snappy` YAML is generally used for test metadata, and for tests that do not use SSZ: e.g. shuffling and BLS tests. In this case, there is no point in adding special SSZ types. And the size and efficiency of YAML is acceptable. @@ -147,8 +147,8 @@ In this case, there is no point in adding special SSZ types. And the size and ef Between all types of tests, a few formats are common: - **`.yaml`**: A YAML file containing structured data to describe settings or test contents. -- **`.ssz_snappy`**: A file containing raw SSZ-encoded data. Previously widely used in tests, but replaced with compressed variant. -- **`.ssz_snappy_snappy`**: Like `.ssz_snappy`, but compressed with Snappy block compression. +- **`.ssz`**: A file containing raw SSZ-encoded data. Previously widely used in tests, but replaced with compressed variant. +- **`.ssz_snappy`**: Like `.ssz`, but compressed with Snappy block compression. Snappy block compression is already applied to SSZ in Eth2 gossip, available in client implementations, and thus chosen as compression method. diff --git a/tests/formats/forks/README.md b/tests/formats/forks/README.md index 57cc09227..cbafd4e2a 100644 --- a/tests/formats/forks/README.md +++ b/tests/formats/forks/README.md @@ -24,17 +24,13 @@ Key of valid `fork` strings that might be found in `meta.yaml` | - | - | - | - | | `altair` | Phase 0 | Altair | `upgrade_to_lightclient_patch` | -### `pre.yaml` +### `pre.ssz_snappy` -A YAML-encoded `BeaconState`, the state before running the fork transition. +A SSZ-snappy encoded `BeaconState`, the state before running the fork transition. -Also available as `pre.ssz`. +### `post.ssz_snappy` -### `post.yaml` - -A YAML-encoded `BeaconState`, the state after applying the fork transition. - -Also available as `post.ssz`. +A SSZ-snappy encoded `BeaconState`, the state after applying the fork transition. *Note*: This type is the `BeaconState` after the fork and is *not* the same type as `pre`. From c36106e63060c81760827fd1c740ea77ec3ba37a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 10 Mar 2021 17:18:11 -0700 Subject: [PATCH 174/222] put snappy in 'generator' extra dep build and use for generator builds --- setup.py | 2 +- tests/generators/README.md | 2 +- tests/generators/bls/requirements.txt | 2 +- tests/generators/epoch_processing/requirements.txt | 2 +- tests/generators/finality/requirements.txt | 2 +- tests/generators/forks/requirements.txt | 2 +- tests/generators/genesis/requirements.txt | 2 +- tests/generators/operations/requirements.txt | 2 +- tests/generators/rewards/requirements.txt | 2 +- tests/generators/shuffling/requirements.txt | 2 +- tests/generators/ssz_generic/requirements.txt | 2 +- tests/generators/ssz_static/requirements.txt | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index 7923c0a44..aa74fcb05 100644 --- a/setup.py +++ b/setup.py @@ -576,6 +576,7 @@ setup( extras_require={ "test": ["pytest>=4.4", "pytest-cov", "pytest-xdist"], "lint": ["flake8==3.7.7", "mypy==0.750"], + "generator": ["python-snappy==0.5.4"], }, install_requires=[ "eth-utils>=1.3.0,<2", @@ -587,6 +588,5 @@ setup( "remerkleable==0.1.18", "ruamel.yaml==0.16.5", "lru-dict==1.1.6", - "python-snappy==0.5.4", ] ) diff --git a/tests/generators/README.md b/tests/generators/README.md index 0f1ed478f..6ccf9f118 100644 --- a/tests/generators/README.md +++ b/tests/generators/README.md @@ -79,7 +79,7 @@ It's recommended to extend the base-generator. Create a `requirements.txt` in the root of your generator directory: ``` pytest>=4.4 -../../../ +../../../[generator] ``` The config helper and pyspec is optional, but preferred. We encourage generators to derive tests from the spec itself in order to prevent code duplication and outdated tests. diff --git a/tests/generators/bls/requirements.txt b/tests/generators/bls/requirements.txt index df386450b..182248686 100644 --- a/tests/generators/bls/requirements.txt +++ b/tests/generators/bls/requirements.txt @@ -1,2 +1,2 @@ pytest>=4.4 -../../../ +../../../[generator] diff --git a/tests/generators/epoch_processing/requirements.txt b/tests/generators/epoch_processing/requirements.txt index df386450b..182248686 100644 --- a/tests/generators/epoch_processing/requirements.txt +++ b/tests/generators/epoch_processing/requirements.txt @@ -1,2 +1,2 @@ pytest>=4.4 -../../../ +../../../[generator] diff --git a/tests/generators/finality/requirements.txt b/tests/generators/finality/requirements.txt index df386450b..182248686 100644 --- a/tests/generators/finality/requirements.txt +++ b/tests/generators/finality/requirements.txt @@ -1,2 +1,2 @@ pytest>=4.4 -../../../ +../../../[generator] diff --git a/tests/generators/forks/requirements.txt b/tests/generators/forks/requirements.txt index 816df6e63..735f863fa 100644 --- a/tests/generators/forks/requirements.txt +++ b/tests/generators/forks/requirements.txt @@ -1,2 +1,2 @@ pytest>=4.4 -../../../ \ No newline at end of file +../../../[generator] \ No newline at end of file diff --git a/tests/generators/genesis/requirements.txt b/tests/generators/genesis/requirements.txt index df386450b..182248686 100644 --- a/tests/generators/genesis/requirements.txt +++ b/tests/generators/genesis/requirements.txt @@ -1,2 +1,2 @@ pytest>=4.4 -../../../ +../../../[generator] diff --git a/tests/generators/operations/requirements.txt b/tests/generators/operations/requirements.txt index df386450b..182248686 100644 --- a/tests/generators/operations/requirements.txt +++ b/tests/generators/operations/requirements.txt @@ -1,2 +1,2 @@ pytest>=4.4 -../../../ +../../../[generator] diff --git a/tests/generators/rewards/requirements.txt b/tests/generators/rewards/requirements.txt index df386450b..182248686 100644 --- a/tests/generators/rewards/requirements.txt +++ b/tests/generators/rewards/requirements.txt @@ -1,2 +1,2 @@ pytest>=4.4 -../../../ +../../../[generator] diff --git a/tests/generators/shuffling/requirements.txt b/tests/generators/shuffling/requirements.txt index df386450b..182248686 100644 --- a/tests/generators/shuffling/requirements.txt +++ b/tests/generators/shuffling/requirements.txt @@ -1,2 +1,2 @@ pytest>=4.4 -../../../ +../../../[generator] diff --git a/tests/generators/ssz_generic/requirements.txt b/tests/generators/ssz_generic/requirements.txt index df386450b..182248686 100644 --- a/tests/generators/ssz_generic/requirements.txt +++ b/tests/generators/ssz_generic/requirements.txt @@ -1,2 +1,2 @@ pytest>=4.4 -../../../ +../../../[generator] diff --git a/tests/generators/ssz_static/requirements.txt b/tests/generators/ssz_static/requirements.txt index df386450b..182248686 100644 --- a/tests/generators/ssz_static/requirements.txt +++ b/tests/generators/ssz_static/requirements.txt @@ -1,2 +1,2 @@ pytest>=4.4 -../../../ +../../../[generator] From 4569ddea5d675ee1e743bc5fc5adf2d889923e96 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 10 Mar 2021 18:49:50 -0700 Subject: [PATCH 175/222] add missing sanity requirements.txt for generators --- tests/generators/sanity/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/sanity/requirements.txt b/tests/generators/sanity/requirements.txt index df386450b..182248686 100644 --- a/tests/generators/sanity/requirements.txt +++ b/tests/generators/sanity/requirements.txt @@ -1,2 +1,2 @@ pytest>=4.4 -../../../ +../../../[generator] From dd341adc2019bc63e3b2d42b115a8fde74814db4 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 10 Mar 2021 18:55:21 -0700 Subject: [PATCH 176/222] remove faulty de-duplication condition for seen aggregates --- specs/phase0/p2p-interface.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 02a578fb8..28b166651 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -335,8 +335,6 @@ The following validations MUST pass before forwarding the `signed_aggregate_and_ (a client MAY queue future aggregates for processing at the appropriate slot). - _[REJECT]_ The aggregate attestation's epoch matches its target -- i.e. `aggregate.data.target.epoch == compute_epoch_at_slot(aggregate.data.slot)` -- _[IGNORE]_ The valid aggregate attestation defined by `hash_tree_root(aggregate)` has _not_ already been seen - (via aggregate gossip, within a verified block, or through the creation of an equivalent aggregate locally). - _[IGNORE]_ The `aggregate` is the first valid aggregate received for the aggregator with index `aggregate_and_proof.aggregator_index` for the epoch `aggregate.data.target.epoch`. - _[REJECT]_ The attestation has participants -- From 86fe6bc0943a7fdb29645941637df086a35b5b58 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 11 Mar 2021 14:27:23 +0800 Subject: [PATCH 177/222] Fix apply_light_client_update calls --- specs/lightclient/sync-protocol.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index 7d442897e..306fca3b2 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -14,9 +14,9 @@ - [Misc](#misc) - [Time parameters](#time-parameters) - [Containers](#containers) - - [`LightClientSnapshot`](#lightclientsnapshot) - - [`LightClientUpdate`](#lightclientupdate) - - [`LightClientStore`](#lightclientstore) + - [`LightClientSnapshot`](#lightclientsnapshot) + - [`LightClientUpdate`](#lightclientupdate) + - [`LightClientStore`](#lightclientstore) - [Helper functions](#helper-functions) - [`get_subtree_index`](#get_subtree_index) - [Light client state updates](#light-client-state-updates) @@ -61,7 +61,7 @@ uses sync committees introduced in [this beacon chain extension](./beacon-chain. ## Containers -#### `LightClientSnapshot` +### `LightClientSnapshot` ```python class LightClientSnapshot(Container): @@ -72,7 +72,7 @@ class LightClientSnapshot(Container): next_sync_committee: SyncCommittee ``` -#### `LightClientUpdate` +### `LightClientUpdate` ```python class LightClientUpdate(Container): @@ -91,7 +91,7 @@ class LightClientUpdate(Container): fork_version: Version ``` -#### `LightClientStore` +### `LightClientStore` ```python class LightClientStore(Container): @@ -188,10 +188,11 @@ def process_light_client_update(store: LightClientStore, update: LightClientUpda # Apply update if (1) 2/3 quorum is reached and (2) we have a finality proof. # Note that (2) means that the current light client design needs finality. # It may be changed to re-organizable light client design. See the on-going issue eth2.0-specs#2182. - apply_light_client_update(store, update) + apply_light_client_update(store.snapshot, update) store.valid_updates = [] elif current_slot > store.snapshot.header.slot + LIGHT_CLIENT_UPDATE_TIMEOUT: # Forced best update when the update timeout has elapsed - apply_light_client_update(store, max(store.valid_updates, key=lambda update: sum(update.sync_committee_bits))) + apply_light_client_update(store.snapshot, + max(store.valid_updates, key=lambda update: sum(update.sync_committee_bits))) store.valid_updates = [] ``` From 9d3556668b2bfb096c386de3e0432c4b8296605c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 11 Mar 2021 21:02:05 +0800 Subject: [PATCH 178/222] Fix domain generation --- specs/lightclient/sync-protocol.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index 306fca3b2..28705803b 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -115,7 +115,8 @@ A light client maintains its state in a `store` object of type `LightClientStore #### `validate_light_client_update` ```python -def validate_light_client_update(snapshot: LightClientSnapshot, update: LightClientUpdate) -> None: +def validate_light_client_update(snapshot: LightClientSnapshot, update: LightClientUpdate, + genesis_validators_root: Root) -> None: # Verify update slot is larger than snapshot slot assert update.header.slot > snapshot.header.slot @@ -157,7 +158,7 @@ def validate_light_client_update(snapshot: LightClientSnapshot, update: LightCli # Verify sync committee aggregate signature participant_pubkeys = [pubkey for (bit, pubkey) in zip(update.sync_committee_bits, sync_committee.pubkeys) if bit] - domain = compute_domain(DOMAIN_SYNC_COMMITTEE, update.fork_version) + domain = compute_domain(DOMAIN_SYNC_COMMITTEE, update.fork_version, genesis_validators_root) signing_root = compute_signing_root(signed_header, domain) assert bls.FastAggregateVerify(participant_pubkeys, signing_root, update.sync_committee_signature) ``` @@ -177,8 +178,9 @@ def apply_light_client_update(snapshot: LightClientSnapshot, update: LightClient #### `process_light_client_update` ```python -def process_light_client_update(store: LightClientStore, update: LightClientUpdate, current_slot: Slot) -> None: - validate_light_client_update(store.snapshot, update) +def process_light_client_update(store: LightClientStore, update: LightClientUpdate, current_slot: Slot, + genesis_validators_root: Root) -> None: + validate_light_client_update(store.snapshot, update, genesis_validators_root) store.valid_updates.append(update) if ( From 2de64cbda15cb944f4378db532f210954a62ae69 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 11 Mar 2021 21:04:16 +0800 Subject: [PATCH 179/222] Add `process_light_client_update` tests --- configs/mainnet/lightclient_patch.yaml | 6 + configs/minimal/lightclient_patch.yaml | 6 + .../eth2spec/test/helpers/sync_committee.py | 14 +- .../unittests/test_helpers.py | 2 +- .../unittests/test_sync_protocol.py | 212 ++++++++++++++++++ 5 files changed, 233 insertions(+), 7 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_sync_protocol.py diff --git a/configs/mainnet/lightclient_patch.yaml b/configs/mainnet/lightclient_patch.yaml index a9ddc16f6..ce4b509c4 100644 --- a/configs/mainnet/lightclient_patch.yaml +++ b/configs/mainnet/lightclient_patch.yaml @@ -29,3 +29,9 @@ EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 256 # Signature domains # --------------------------------------------------------------- DOMAIN_SYNC_COMMITTEE: 0x07000000 + + +# Sync protocol +# --------------------------------------------------------------- +# 2**13 (=8192) +LIGHT_CLIENT_UPDATE_TIMEOUT: 8192 diff --git a/configs/minimal/lightclient_patch.yaml b/configs/minimal/lightclient_patch.yaml index 56ce34591..4e8cf3fd3 100644 --- a/configs/minimal/lightclient_patch.yaml +++ b/configs/minimal/lightclient_patch.yaml @@ -29,3 +29,9 @@ EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 8 # Signature domains # --------------------------------------------------------------- DOMAIN_SYNC_COMMITTEE: 0x07000000 + + +# Sync protocol +# --------------------------------------------------------------- +# [customized] +LIGHT_CLIENT_UPDATE_TIMEOUT: 32 diff --git a/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py b/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py index b7b2381e3..da85fad60 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py @@ -5,17 +5,18 @@ from eth2spec.test.helpers.block import ( from eth2spec.utils import bls -def compute_sync_committee_signature(spec, state, slot, privkey): +def compute_sync_committee_signature(spec, state, slot, privkey, block_root=None): domain = spec.get_domain(state, spec.DOMAIN_SYNC_COMMITTEE, spec.compute_epoch_at_slot(slot)) - if slot == state.slot: - block_root = build_empty_block_for_next_slot(spec, state).parent_root - else: - block_root = spec.get_block_root_at_slot(state, slot) + if block_root is None: + if slot == state.slot: + block_root = build_empty_block_for_next_slot(spec, state).parent_root + else: + block_root = spec.get_block_root_at_slot(state, slot) signing_root = spec.compute_signing_root(block_root, domain) return bls.Sign(privkey, signing_root) -def compute_aggregate_sync_committee_signature(spec, state, slot, participants): +def compute_aggregate_sync_committee_signature(spec, state, slot, participants, block_root=None): if len(participants) == 0: return spec.G2_POINT_AT_INFINITY @@ -28,6 +29,7 @@ def compute_aggregate_sync_committee_signature(spec, state, slot, participants): state, slot, privkey, + block_root=block_root, ) ) return bls.Aggregate(signatures) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_helpers.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_helpers.py index 33e749380..48054d088 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_helpers.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_helpers.py @@ -9,7 +9,7 @@ from eth2spec.test.helpers.merkle import build_proof @with_phases([LIGHTCLIENT_PATCH]) @spec_state_test def test_next_sync_committee_tree(spec, state): - state.next_sync_committee = spec.SyncCommittee( + state.next_sync_committee: object = spec.SyncCommittee( pubkeys=[state.validators[i]for i in range(spec.SYNC_COMMITTEE_SIZE)] ) next_sync_committee_branch = build_proof(state.get_backing(), spec.NEXT_SYNC_COMMITTEE_INDEX) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_sync_protocol.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_sync_protocol.py new file mode 100644 index 000000000..b9fdb66ba --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_sync_protocol.py @@ -0,0 +1,212 @@ +from eth2spec.test.context import ( + LIGHTCLIENT_PATCH, + spec_state_test, + with_phases, +) +from eth2spec.test.helpers.attestations import next_epoch_with_attestations +from eth2spec.test.helpers.block import ( + build_empty_block, + build_empty_block_for_next_slot, +) +from eth2spec.test.helpers.state import ( + next_slots, + state_transition_and_sign_block, +) +from eth2spec.test.helpers.sync_committee import ( + compute_aggregate_sync_committee_signature, +) +from eth2spec.test.helpers.merkle import build_proof + + +@with_phases([LIGHTCLIENT_PATCH]) +@spec_state_test +def test_process_light_client_update_not_updated(spec, state): + pre_snapshot = spec.LightClientSnapshot( + header=spec.BeaconBlockHeader(), + current_sync_committee=state.current_sync_committee, + next_sync_committee=state.next_sync_committee, + ) + store = spec.LightClientStore( + snapshot=pre_snapshot, + valid_updates=[] + ) + + # Block at slot 1 doesn't increase sync committee period, so it won't update snapshot + block = build_empty_block_for_next_slot(spec, state) + signed_block = state_transition_and_sign_block(spec, state, block) + block_header = spec.BeaconBlockHeader( + slot=signed_block.message.slot, + proposer_index=signed_block.message.proposer_index, + parent_root=signed_block.message.parent_root, + state_root=signed_block.message.state_root, + body_root=signed_block.message.body.hash_tree_root(), + ) + # Sync committee signing the header + committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + sync_committee_bits = [True] * len(committee) + sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + block.slot, + committee, + ) + next_sync_committee_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.NEXT_SYNC_COMMITTEE_INDEX))] + + # Ensure that finality checkpoint is genesis + assert state.finalized_checkpoint.epoch == 0 + # Finality is unchanged + finality_header = spec.BeaconBlockHeader() + finality_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.FINALIZED_ROOT_INDEX))] + + update = spec.LightClientUpdate( + header=block_header, + next_sync_committee=state.next_sync_committee, + next_sync_committee_branch=next_sync_committee_branch, + finality_header=finality_header, + finality_branch=finality_branch, + sync_committee_bits=sync_committee_bits, + sync_committee_signature=sync_committee_signature, + fork_version=state.fork.current_version, + ) + + spec.process_light_client_update(store, update, state.slot, state.genesis_validators_root) + + assert len(store.valid_updates) == 1 + assert store.valid_updates[0] == update + assert store.snapshot == pre_snapshot + + +@with_phases([LIGHTCLIENT_PATCH]) +@spec_state_test +def test_process_light_client_update_timeout(spec, state): + pre_snapshot = spec.LightClientSnapshot( + header=spec.BeaconBlockHeader(), + current_sync_committee=state.current_sync_committee, + next_sync_committee=state.next_sync_committee, + ) + store = spec.LightClientStore( + snapshot=pre_snapshot, + valid_updates=[] + ) + + # Forward to next sync committee period + next_slots(spec, state, spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD)) + snapshot_period = spec.compute_epoch_at_slot(pre_snapshot.header.slot) // spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD + update_period = spec.compute_epoch_at_slot(state.slot) // spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD + assert snapshot_period + 1 == update_period + + block = build_empty_block_for_next_slot(spec, state) + signed_block = state_transition_and_sign_block(spec, state, block) + block_header = spec.BeaconBlockHeader( + slot=signed_block.message.slot, + proposer_index=signed_block.message.proposer_index, + parent_root=signed_block.message.parent_root, + state_root=signed_block.message.state_root, + body_root=signed_block.message.body.hash_tree_root(), + ) + + # Sync committee is updated + next_sync_committee_branch = build_proof(state.get_backing(), spec.NEXT_SYNC_COMMITTEE_INDEX) + # Finality is unchanged + finality_header = spec.BeaconBlockHeader() + finality_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.FINALIZED_ROOT_INDEX))] + + # Sync committee signing the finalized_block_header + committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + sync_committee_bits = [True] * len(committee) + sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + block_header.slot, + committee, + block_root=spec.Root(block_header.hash_tree_root()), + ) + + update = spec.LightClientUpdate( + header=block_header, + next_sync_committee=state.next_sync_committee, + next_sync_committee_branch=next_sync_committee_branch, + finality_header=finality_header, + finality_branch=finality_branch, + sync_committee_bits=sync_committee_bits, + sync_committee_signature=sync_committee_signature, + fork_version=state.fork.current_version, + ) + + spec.process_light_client_update(store, update, state.slot, state.genesis_validators_root) + + # snapshot has been updated + assert len(store.valid_updates) == 0 + assert store.snapshot.header == update.header + + +@with_phases([LIGHTCLIENT_PATCH]) +@spec_state_test +def test_process_light_client_update_finality_updated(spec, state): + pre_snapshot = spec.LightClientSnapshot( + header=spec.BeaconBlockHeader(), + current_sync_committee=state.current_sync_committee, + next_sync_committee=state.next_sync_committee, + ) + store = spec.LightClientStore( + snapshot=pre_snapshot, + valid_updates=[] + ) + + # Change finality + blocks = [] + next_slots(spec, state, spec.SLOTS_PER_EPOCH * 2) + for epoch in range(3): + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, True) + blocks += new_blocks + # Ensure that finality checkpoint has changed + assert state.finalized_checkpoint.epoch == 3 + # Ensure that it's same period + snapshot_period = spec.compute_epoch_at_slot(pre_snapshot.header.slot) // spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD + update_period = spec.compute_epoch_at_slot(state.slot) // spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD + assert snapshot_period == update_period + + # Updated sync_committee and finality + next_sync_committee_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.NEXT_SYNC_COMMITTEE_INDEX))] + finalized_block_header = blocks[spec.SLOTS_PER_EPOCH - 1].message + assert finalized_block_header.slot == spec.compute_start_slot_at_epoch(state.finalized_checkpoint.epoch) + assert finalized_block_header.hash_tree_root() == state.finalized_checkpoint.root + finality_branch = build_proof(state.get_backing(), spec.FINALIZED_ROOT_INDEX) + + # Sync committee signing the finalized_block_header + committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + sync_committee_bits = [True] * len(committee) + sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + finalized_block_header.slot, + committee, + block_root=spec.Root(finalized_block_header.hash_tree_root()), + ) + + # Build block header + block = build_empty_block(spec, state) + block_header = spec.BeaconBlockHeader( + slot=block.slot, + proposer_index=block.proposer_index, + parent_root=block.parent_root, + state_root=state.hash_tree_root(), + body_root=block.body.hash_tree_root(), + ) + + update = spec.LightClientUpdate( + header=finalized_block_header, + next_sync_committee=state.next_sync_committee, + next_sync_committee_branch=next_sync_committee_branch, + finality_header=block_header, + finality_branch=finality_branch, + sync_committee_bits=sync_committee_bits, + sync_committee_signature=sync_committee_signature, + fork_version=state.fork.current_version, + ) + + spec.process_light_client_update(store, update, state.slot, state.genesis_validators_root) + + # snapshot has been updated + assert len(store.valid_updates) == 0 + assert store.snapshot.header == update.header From 69489d1541e973707cccf18f8d4961ab3b945f63 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 11 Mar 2021 21:12:46 +0800 Subject: [PATCH 180/222] Rename folders and file names to `altair` --- configs/mainnet/{lightclient_patch.yaml => altair.yaml} | 0 configs/minimal/{lightclient_patch.yaml => altair.yaml} | 0 specs/{lightclient => altair}/beacon-chain.md | 0 specs/{lightclient => altair}/fork.md | 0 specs/{lightclient => altair}/sync-protocol.md | 0 .../eth2spec/test/{lightclient_patch => altair}/__init__.py | 0 .../{lightclient_patch => altair}/block_processing/__init__.py | 0 .../block_processing/test_process_sync_committee.py | 0 .../{lightclient_patch => altair}/epoch_processing/__init__.py | 0 .../epoch_processing/test_process_sync_committee_updates.py | 0 .../eth2spec/test/{lightclient_patch => altair}/fork/__init__.py | 0 .../eth2spec/test/{lightclient_patch => altair}/fork/test_fork.py | 0 .../test/{lightclient_patch => altair}/sanity/__init__.py | 0 .../test/{lightclient_patch => altair}/sanity/test_blocks.py | 0 14 files changed, 0 insertions(+), 0 deletions(-) rename configs/mainnet/{lightclient_patch.yaml => altair.yaml} (100%) rename configs/minimal/{lightclient_patch.yaml => altair.yaml} (100%) rename specs/{lightclient => altair}/beacon-chain.md (100%) rename specs/{lightclient => altair}/fork.md (100%) rename specs/{lightclient => altair}/sync-protocol.md (100%) rename tests/core/pyspec/eth2spec/test/{lightclient_patch => altair}/__init__.py (100%) rename tests/core/pyspec/eth2spec/test/{lightclient_patch => altair}/block_processing/__init__.py (100%) rename tests/core/pyspec/eth2spec/test/{lightclient_patch => altair}/block_processing/test_process_sync_committee.py (100%) rename tests/core/pyspec/eth2spec/test/{lightclient_patch => altair}/epoch_processing/__init__.py (100%) rename tests/core/pyspec/eth2spec/test/{lightclient_patch => altair}/epoch_processing/test_process_sync_committee_updates.py (100%) rename tests/core/pyspec/eth2spec/test/{lightclient_patch => altair}/fork/__init__.py (100%) rename tests/core/pyspec/eth2spec/test/{lightclient_patch => altair}/fork/test_fork.py (100%) rename tests/core/pyspec/eth2spec/test/{lightclient_patch => altair}/sanity/__init__.py (100%) rename tests/core/pyspec/eth2spec/test/{lightclient_patch => altair}/sanity/test_blocks.py (100%) diff --git a/configs/mainnet/lightclient_patch.yaml b/configs/mainnet/altair.yaml similarity index 100% rename from configs/mainnet/lightclient_patch.yaml rename to configs/mainnet/altair.yaml diff --git a/configs/minimal/lightclient_patch.yaml b/configs/minimal/altair.yaml similarity index 100% rename from configs/minimal/lightclient_patch.yaml rename to configs/minimal/altair.yaml diff --git a/specs/lightclient/beacon-chain.md b/specs/altair/beacon-chain.md similarity index 100% rename from specs/lightclient/beacon-chain.md rename to specs/altair/beacon-chain.md diff --git a/specs/lightclient/fork.md b/specs/altair/fork.md similarity index 100% rename from specs/lightclient/fork.md rename to specs/altair/fork.md diff --git a/specs/lightclient/sync-protocol.md b/specs/altair/sync-protocol.md similarity index 100% rename from specs/lightclient/sync-protocol.md rename to specs/altair/sync-protocol.md diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/__init__.py b/tests/core/pyspec/eth2spec/test/altair/__init__.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/lightclient_patch/__init__.py rename to tests/core/pyspec/eth2spec/test/altair/__init__.py diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/__init__.py b/tests/core/pyspec/eth2spec/test/altair/block_processing/__init__.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/__init__.py rename to tests/core/pyspec/eth2spec/test/altair/block_processing/__init__.py diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py rename to tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/__init__.py b/tests/core/pyspec/eth2spec/test/altair/epoch_processing/__init__.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/__init__.py rename to tests/core/pyspec/eth2spec/test/altair/epoch_processing/__init__.py diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_sync_committee_updates.py b/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_sync_committee_updates.py rename to tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/__init__.py b/tests/core/pyspec/eth2spec/test/altair/fork/__init__.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/lightclient_patch/fork/__init__.py rename to tests/core/pyspec/eth2spec/test/altair/fork/__init__.py diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py b/tests/core/pyspec/eth2spec/test/altair/fork/test_fork.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py rename to tests/core/pyspec/eth2spec/test/altair/fork/test_fork.py diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/__init__.py b/tests/core/pyspec/eth2spec/test/altair/sanity/__init__.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/__init__.py rename to tests/core/pyspec/eth2spec/test/altair/sanity/__init__.py diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/altair/sanity/test_blocks.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py rename to tests/core/pyspec/eth2spec/test/altair/sanity/test_blocks.py From b44e576e95a4f80b30ae04c721d6b97333af1159 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 11 Mar 2021 21:22:38 +0800 Subject: [PATCH 181/222] LIGHTCLIENT_PATCH/HF1 -> ALTAIR --- .gitignore | 2 +- Makefile | 10 ++-- README.md | 7 +-- configs/mainnet/altair.yaml | 8 +-- configs/minimal/altair.yaml | 8 +-- setup.py | 12 ++--- specs/altair/beacon-chain.md | 38 +++++++------- specs/altair/fork.md | 12 ++--- .../eth2spec/test/altair/fork/test_fork.py | 50 +++++++++---------- tests/core/pyspec/eth2spec/test/context.py | 30 +++++------ .../eth2spec/test/helpers/attestations.py | 8 +-- .../pyspec/eth2spec/test/helpers/block.py | 4 +- .../eth2spec/test/helpers/epoch_processing.py | 6 +-- .../test/helpers/proposer_slashings.py | 6 +-- .../pyspec/eth2spec/test/helpers/rewards.py | 26 +++++----- ..._process_justification_and_finalization.py | 6 +-- .../test_process_rewards_and_penalties.py | 12 ++--- .../test_process_slashings.py | 6 +-- .../test/phase0/sanity/test_blocks.py | 8 +-- .../fork_choice/test_on_attestation.py | 4 +- .../test_process_attestation.py | 6 +-- .../test_process_chunk_challenge.py | 24 ++++----- .../test_process_custody_key_reveal.py | 12 ++--- .../test_process_custody_slashing.py | 12 ++--- ...est_process_early_derived_secret_reveal.py | 18 +++---- .../test_process_shard_transition.py | 10 ++-- .../test_process_challenge_deadlines.py | 4 +- .../test_process_custody_final_updates.py | 10 ++-- .../test_process_reveal_deadlines.py | 6 +-- .../test/phase1/sanity/test_blocks.py | 14 +++--- .../test/phase1/sanity/test_shard_blocks.py | 24 ++++----- .../fork_choice/test_on_shard_block.py | 8 +-- .../phase1/unittests/test_get_start_shard.py | 12 ++--- tests/formats/forks/README.md | 2 +- tests/generators/README.md | 10 ++-- tests/generators/epoch_processing/main.py | 12 ++--- tests/generators/finality/main.py | 12 ++--- tests/generators/forks/main.py | 12 ++--- tests/generators/operations/main.py | 12 ++--- tests/generators/rewards/main.py | 12 ++--- tests/generators/sanity/main.py | 10 ++-- tests/generators/ssz_static/main.py | 10 ++-- 42 files changed, 258 insertions(+), 257 deletions(-) diff --git a/.gitignore b/.gitignore index 17d058225..56a3d605c 100644 --- a/.gitignore +++ b/.gitignore @@ -17,7 +17,7 @@ eth2.0-spec-tests/ # Dynamically built from Markdown spec tests/core/pyspec/eth2spec/phase0/ tests/core/pyspec/eth2spec/phase1/ -tests/core/pyspec/eth2spec/lightclient_patch/ +tests/core/pyspec/eth2spec/altair/ # coverage reports .htmlcov diff --git a/Makefile b/Makefile index 08e822bf6..97da705ef 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENER # To check generator matching: #$(info $$GENERATOR_TARGETS is [${GENERATOR_TARGETS}]) -MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/phase0/*.md) $(wildcard $(SPEC_DIR)/phase1/*.md) $(wildcard $(SPEC_DIR)/lightclient/*.md) $(wildcard $(SSZ_DIR)/*.md) $(wildcard $(SPEC_DIR)/networking/*.md) $(wildcard $(SPEC_DIR)/validator/*.md) +MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/phase0/*.md) $(wildcard $(SPEC_DIR)/phase1/*.md) $(wildcard $(SPEC_DIR)/altair/*.md) $(wildcard $(SSZ_DIR)/*.md) $(wildcard $(SPEC_DIR)/networking/*.md) $(wildcard $(SPEC_DIR)/validator/*.md) COV_HTML_OUT=.htmlcov COV_INDEX_FILE=$(PY_SPEC_DIR)/$(COV_HTML_OUT)/index.html @@ -49,7 +49,7 @@ partial_clean: rm -rf $(DEPOSIT_CONTRACT_TESTER_DIR)/.pytest_cache rm -rf $(PY_SPEC_DIR)/phase0 rm -rf $(PY_SPEC_DIR)/phase1 - rm -rf $(PY_SPEC_DIR)/lightclient + rm -rf $(PY_SPEC_DIR)/altair rm -rf $(PY_SPEC_DIR)/$(COV_HTML_OUT) rm -rf $(PY_SPEC_DIR)/.coverage rm -rf $(PY_SPEC_DIR)/test-reports @@ -86,11 +86,11 @@ install_test: test: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python3 -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov=eth2spec.lightclient_patch.spec -cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec + python3 -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov=eth2spec.altair.spec -cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec find_test: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python3 -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov=eth2spec.lightclient_patch.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec + python3 -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov=eth2spec.altair.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec citest: pyspec mkdir -p tests/core/pyspec/test-reports/eth2spec; . venv/bin/activate; cd $(PY_SPEC_DIR); \ @@ -113,7 +113,7 @@ codespell: lint: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ flake8 --config $(LINTER_CONFIG_FILE) ./eth2spec \ - && mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.phase1 -p eth2spec.lightclient_patch + && mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.phase1 -p eth2spec.altair lint_generators: pyspec . venv/bin/activate; cd $(TEST_GENERATORS_DIR); \ diff --git a/README.md b/README.md index 937841127..42ad7e71f 100644 --- a/README.md +++ b/README.md @@ -21,10 +21,11 @@ Core specifications for Eth2 clients be found in [specs](specs/). These are divi * [Honest Validator](specs/phase0/validator.md) * [P2P Networking](specs/phase0/p2p-interface.md) -### Light clients +### Altair -* [Beacon chain changes](specs/lightclient/beacon-chain.md) -* [Light client sync protocol](specs/lightclient/sync-protocol.md) +* [Beacon chain changes](specs/altair/beacon-chain.md) +* [Altair fork](specs/altair/fork.md) +* [Light client sync protocol](specs/altair/sync-protocol.md) ### Sharding diff --git a/configs/mainnet/altair.yaml b/configs/mainnet/altair.yaml index a9ddc16f6..8ba51d283 100644 --- a/configs/mainnet/altair.yaml +++ b/configs/mainnet/altair.yaml @@ -1,15 +1,15 @@ -# Mainnet preset - lightclient patch +# Mainnet preset - Altair CONFIG_NAME: "mainnet" # Updated penalty values # --------------------------------------------------------------- # 3 * 2**24) (= 50,331,648) -HF1_INACTIVITY_PENALTY_QUOTIENT: 50331648 +ALTAIR_INACTIVITY_PENALTY_QUOTIENT: 50331648 # 2**6 (= 64) -HF1_MIN_SLASHING_PENALTY_QUOTIENT: 64 +ALTAIR_MIN_SLASHING_PENALTY_QUOTIENT: 64 # 2 -HF1_PROPORTIONAL_SLASHING_MULTIPLIER: 2 +ALTAIR_PROPORTIONAL_SLASHING_MULTIPLIER: 2 # Misc diff --git a/configs/minimal/altair.yaml b/configs/minimal/altair.yaml index 56ce34591..38cf4db76 100644 --- a/configs/minimal/altair.yaml +++ b/configs/minimal/altair.yaml @@ -1,15 +1,15 @@ -# Minimal preset - lightclient patch +# Minimal preset - Altair CONFIG_NAME: "minimal" # Updated penalty values # --------------------------------------------------------------- # 3 * 2**24) (= 50,331,648) -HF1_INACTIVITY_PENALTY_QUOTIENT: 50331648 +ALTAIR_INACTIVITY_PENALTY_QUOTIENT: 50331648 # 2**6 (= 64) -HF1_MIN_SLASHING_PENALTY_QUOTIENT: 64 +ALTAIR_MIN_SLASHING_PENALTY_QUOTIENT: 64 # 2 -HF1_PROPORTIONAL_SLASHING_MULTIPLIER: 2 +ALTAIR_PROPORTIONAL_SLASHING_MULTIPLIER: 2 # Misc diff --git a/setup.py b/setup.py index aa74fcb05..d0a2371e8 100644 --- a/setup.py +++ b/setup.py @@ -179,7 +179,7 @@ from eth2spec.utils import bls from eth2spec.utils.hash_function import hash -# Whenever lightclient is loaded, make sure we have the latest phase0 +# Whenever altair is loaded, make sure we have the latest phase0 from importlib import reload reload(phase0) @@ -386,7 +386,7 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: fork_imports = { 'phase0': PHASE0_IMPORTS, 'phase1': PHASE1_IMPORTS, - 'lightclient_patch': LIGHTCLIENT_IMPORT, + 'altair': LIGHTCLIENT_IMPORT, } @@ -453,16 +453,16 @@ class PySpecCommand(Command): specs/phase1/shard-fork-choice.md specs/phase1/validator.md """ - elif self.spec_fork == "lightclient_patch": + elif self.spec_fork == "altair": self.md_doc_paths = """ specs/phase0/beacon-chain.md specs/phase0/fork-choice.md specs/phase0/validator.md specs/phase0/weak-subjectivity.md - specs/lightclient/beacon-chain.md - specs/lightclient/fork.md + specs/altair/beacon-chain.md + specs/altair/fork.md """ - # TODO: add specs/lightclient/sync-protocol.md back when the GeneralizedIndex helpers are included. + # TODO: add specs/altair/sync-protocol.md back when the GeneralizedIndex helpers are included. else: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index bc070908a..d2e060cdf 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -1,4 +1,4 @@ -# Ethereum 2.0 HF1 +# Ethereum 2.0 Altair ## Table of contents @@ -55,7 +55,7 @@ ## Introduction -This is a patch implementing the first hard fork to the beacon chain, tentatively named HF1 pending a permanent name. +Altair is a patch implementing the first hard fork to the beacon chain. It has four main features: * Light client support via sync committees @@ -108,9 +108,9 @@ This patch updates a few configuration values to move penalty constants toward t | Name | Value | | - | - | -| `HF1_INACTIVITY_PENALTY_QUOTIENT` | `uint64(3 * 2**24)` (= 50,331,648) | -| `HF1_MIN_SLASHING_PENALTY_QUOTIENT` | `uint64(2**6)` (=64) | -| `HF1_PROPORTIONAL_SLASHING_MULTIPLIER` | `uint64(2)` | +| `ALTAIR_INACTIVITY_PENALTY_QUOTIENT` | `uint64(3 * 2**24)` (= 50,331,648) | +| `ALTAIR_MIN_SLASHING_PENALTY_QUOTIENT` | `uint64(2**6)` (=64) | +| `ALTAIR_PROPORTIONAL_SLASHING_MULTIPLIER` | `uint64(2)` | ### Misc @@ -152,8 +152,8 @@ class BeaconBlockBody(Container): deposits: List[Deposit, MAX_DEPOSITS] voluntary_exits: List[SignedVoluntaryExit, MAX_VOLUNTARY_EXITS] # Sync committee aggregate signature - sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE] # [New in HF1] - sync_committee_signature: BLSSignature # [New in HF1] + sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE] # [New in Altair] + sync_committee_signature: BLSSignature # [New in Altair] ``` #### `BeaconState` @@ -379,7 +379,7 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S effective_balance = state.validators[index].effective_balance penalties[index] += Gwei( effective_balance * get_finality_delay(state) - // HF1_INACTIVITY_PENALTY_QUOTIENT + // ALTAIR_INACTIVITY_PENALTY_QUOTIENT ) rewards = [Gwei(0) for _ in range(len(state.validators))] @@ -391,7 +391,7 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S #### New `slash_validator` *Note*: The function `slash_validator` is modified -with the substitution of `MIN_SLASHING_PENALTY_QUOTIENT` with `HF1_MIN_SLASHING_PENALTY_QUOTIENT`. +with the substitution of `MIN_SLASHING_PENALTY_QUOTIENT` with `ALTAIR_MIN_SLASHING_PENALTY_QUOTIENT`. ```python def slash_validator(state: BeaconState, @@ -406,7 +406,7 @@ def slash_validator(state: BeaconState, validator.slashed = True validator.withdrawable_epoch = max(validator.withdrawable_epoch, Epoch(epoch + EPOCHS_PER_SLASHINGS_VECTOR)) state.slashings[epoch % EPOCHS_PER_SLASHINGS_VECTOR] += validator.effective_balance - decrease_balance(state, slashed_index, validator.effective_balance // HF1_MIN_SLASHING_PENALTY_QUOTIENT) + decrease_balance(state, slashed_index, validator.effective_balance // ALTAIR_MIN_SLASHING_PENALTY_QUOTIENT) # Apply proposer and whistleblower rewards proposer_index = get_beacon_proposer_index(state) @@ -425,8 +425,8 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) process_randao(state, block.body) process_eth1_data(state, block.body) - process_operations(state, block.body) # [Modified in HF1] - process_sync_committee(state, block.body) # [New in HF1] + process_operations(state, block.body) # [Modified in Altair] + process_sync_committee(state, block.body) # [New in Altair] ``` #### Modified `process_attestation` @@ -561,17 +561,17 @@ def process_sync_committee(state: BeaconState, body: BeaconBlockBody) -> None: ```python def process_epoch(state: BeaconState) -> None: - process_justification_and_finalization(state) # [Modified in HF1] - process_rewards_and_penalties(state) # [Modified in HF1] + process_justification_and_finalization(state) # [Modified in Altair] + process_rewards_and_penalties(state) # [Modified in Altair] process_registry_updates(state) - process_slashings(state) # [Modified in HF1] + process_slashings(state) # [Modified in Altair] process_eth1_data_reset(state) process_effective_balance_updates(state) process_slashings_reset(state) process_randao_mixes_reset(state) process_historical_roots_update(state) - process_participation_flag_updates(state) # [New in HF1] - process_sync_committee_updates(state) # [New in HF1] + process_participation_flag_updates(state) # [New in Altair] + process_sync_committee_updates(state) # [New in Altair] ``` #### Justification and finalization @@ -642,13 +642,13 @@ def process_rewards_and_penalties(state: BeaconState) -> None: #### Slashings -*Note*: The function `process_slashings` is modified to use `HF1_PROPORTIONAL_SLASHING_MULTIPLIER`. +*Note*: The function `process_slashings` is modified to use `ALTAIR_PROPORTIONAL_SLASHING_MULTIPLIER`. ```python def process_slashings(state: BeaconState) -> None: epoch = get_current_epoch(state) total_balance = get_total_active_balance(state) - adjusted_total_slashing_balance = min(sum(state.slashings) * HF1_PROPORTIONAL_SLASHING_MULTIPLIER, total_balance) + adjusted_total_slashing_balance = min(sum(state.slashings) * ALTAIR_PROPORTIONAL_SLASHING_MULTIPLIER, total_balance) for index, validator in enumerate(state.validators): if validator.slashed and epoch + EPOCHS_PER_SLASHINGS_VECTOR // 2 == validator.withdrawable_epoch: increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from penalty numerator to avoid uint64 overflow diff --git a/specs/altair/fork.md b/specs/altair/fork.md index 157e67dc9..4972983af 100644 --- a/specs/altair/fork.md +++ b/specs/altair/fork.md @@ -25,21 +25,21 @@ Warning: this configuration is not definitive. | Name | Value | | - | - | -| `LIGHTCLIENT_PATCH_FORK_VERSION` | `Version('0x01000000')` | -| `LIGHTCLIENT_PATCH_FORK_SLOT` | `Slot(0)` **TBD** | +| `ALTAIR_FORK_VERSION` | `Version('0x01000000')` | +| `ALTAIR_FORK_SLOT` | `Slot(0)` **TBD** | ## Fork to Light-client patch ### Fork trigger -TBD. Social consensus, along with state conditions such as epoch boundary, finality, deposits, active validator count, etc. may be part of the decision process to trigger the fork. For now we assume the condition will be triggered at slot `LIGHTCLIENT_PATCH_FORK_SLOT`, where `LIGHTCLIENT_PATCH_FORK_SLOT % SLOTS_PER_EPOCH == 0`. +TBD. Social consensus, along with state conditions such as epoch boundary, finality, deposits, active validator count, etc. may be part of the decision process to trigger the fork. For now we assume the condition will be triggered at slot `ALTAIR_FORK_SLOT`, where `ALTAIR_FORK_SLOT % SLOTS_PER_EPOCH == 0`. ### Upgrading the state -After `process_slots` of Phase 0 finishes, if `state.slot == LIGHTCLIENT_PATCH_FORK_SLOT`, an irregular state change is made to upgrade to light-client patch. +After `process_slots` of Phase 0 finishes, if `state.slot == ALTAIR_FORK_SLOT`, an irregular state change is made to upgrade to light-client patch. ```python -def upgrade_to_lightclient_patch(pre: phase0.BeaconState) -> BeaconState: +def upgrade_to_altair(pre: phase0.BeaconState) -> BeaconState: epoch = get_current_epoch(pre) post = BeaconState( genesis_time=pre.genesis_time, @@ -47,7 +47,7 @@ def upgrade_to_lightclient_patch(pre: phase0.BeaconState) -> BeaconState: slot=pre.slot, fork=Fork( previous_version=pre.fork.current_version, - current_version=LIGHTCLIENT_PATCH_FORK_VERSION, + current_version=ALTAIR_FORK_VERSION, epoch=epoch, ), # History diff --git a/tests/core/pyspec/eth2spec/test/altair/fork/test_fork.py b/tests/core/pyspec/eth2spec/test/altair/fork/test_fork.py index 27d181510..9667c2e37 100644 --- a/tests/core/pyspec/eth2spec/test/altair/fork/test_fork.py +++ b/tests/core/pyspec/eth2spec/test/altair/fork/test_fork.py @@ -1,5 +1,5 @@ from eth2spec.test.context import ( - PHASE0, LIGHTCLIENT_PATCH, + PHASE0, ALTAIR, with_phases, with_custom_state, spec_test, with_state, @@ -12,7 +12,7 @@ from eth2spec.test.helpers.state import ( ) -HF1_FORK_TEST_META_TAGS = { +ALTAIR_FORK_TEST_META_TAGS = { 'fork': 'altair', } @@ -20,7 +20,7 @@ HF1_FORK_TEST_META_TAGS = { def run_fork_test(post_spec, pre_state): yield 'pre', pre_state - post_state = post_spec.upgrade_to_lightclient_patch(pre_state) + post_state = post_spec.upgrade_to_altair(pre_state) # Stable fields stable_fields = [ @@ -47,67 +47,67 @@ def run_fork_test(post_spec, pre_state): assert getattr(pre_state, field) != getattr(post_state, field) assert pre_state.fork.current_version == post_state.fork.previous_version - assert post_state.fork.current_version == post_spec.LIGHTCLIENT_PATCH_FORK_VERSION + assert post_state.fork.current_version == post_spec.ALTAIR_FORK_VERSION assert post_state.fork.epoch == post_spec.get_current_epoch(post_state) yield 'post', post_state -@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH]) +@with_phases(phases=[PHASE0], other_phases=[ALTAIR]) @spec_test @with_state -@with_meta_tags(HF1_FORK_TEST_META_TAGS) +@with_meta_tags(ALTAIR_FORK_TEST_META_TAGS) def test_fork_base_state(spec, phases, state): - yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) + yield from run_fork_test(phases[ALTAIR], state) -@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH]) +@with_phases(phases=[PHASE0], other_phases=[ALTAIR]) @spec_test @with_state -@with_meta_tags(HF1_FORK_TEST_META_TAGS) +@with_meta_tags(ALTAIR_FORK_TEST_META_TAGS) def test_fork_next_epoch(spec, phases, state): next_epoch(spec, state) - yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) + yield from run_fork_test(phases[ALTAIR], state) -@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH]) +@with_phases(phases=[PHASE0], other_phases=[ALTAIR]) @spec_test @with_state -@with_meta_tags(HF1_FORK_TEST_META_TAGS) +@with_meta_tags(ALTAIR_FORK_TEST_META_TAGS) def test_fork_next_epoch_with_block(spec, phases, state): next_epoch_via_block(spec, state) - yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) + yield from run_fork_test(phases[ALTAIR], state) -@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH]) +@with_phases(phases=[PHASE0], other_phases=[ALTAIR]) @spec_test @with_state -@with_meta_tags(HF1_FORK_TEST_META_TAGS) +@with_meta_tags(ALTAIR_FORK_TEST_META_TAGS) def test_fork_many_next_epoch(spec, phases, state): for _ in range(3): next_epoch(spec, state) - yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) + yield from run_fork_test(phases[ALTAIR], state) -@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH]) +@with_phases(phases=[PHASE0], other_phases=[ALTAIR]) @with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test -@with_meta_tags(HF1_FORK_TEST_META_TAGS) +@with_meta_tags(ALTAIR_FORK_TEST_META_TAGS) def test_fork_random_low_balances(spec, phases, state): - yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) + yield from run_fork_test(phases[ALTAIR], state) -@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH]) +@with_phases(phases=[PHASE0], other_phases=[ALTAIR]) @with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test -@with_meta_tags(HF1_FORK_TEST_META_TAGS) +@with_meta_tags(ALTAIR_FORK_TEST_META_TAGS) def test_fork_random_misc_balances(spec, phases, state): - yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) + yield from run_fork_test(phases[ALTAIR], state) -@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH]) +@with_phases(phases=[PHASE0], other_phases=[ALTAIR]) @with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test -@with_meta_tags(HF1_FORK_TEST_META_TAGS) +@with_meta_tags(ALTAIR_FORK_TEST_META_TAGS) def test_fork_random_large_validator_set(spec, phases, state): - yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) + yield from run_fork_test(phases[ALTAIR], state) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index e197124c0..d631bb4d2 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -2,7 +2,7 @@ import pytest from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.lightclient_patch import spec as spec_lightclient_patch +from eth2spec.altair import spec as spec_altair from eth2spec.utils import bls from .exceptions import SkippedTest @@ -20,7 +20,7 @@ from importlib import reload def reload_specs(): reload(spec_phase0) reload(spec_phase1) - reload(spec_lightclient_patch) + reload(spec_altair) # Some of the Spec module functionality is exposed here to deal with phase-specific changes. @@ -30,9 +30,9 @@ ConfigName = NewType("ConfigName", str) PHASE0 = SpecForkName('phase0') PHASE1 = SpecForkName('phase1') -LIGHTCLIENT_PATCH = SpecForkName('lightclient_patch') +ALTAIR = SpecForkName('altair') -ALL_PHASES = (PHASE0, PHASE1, LIGHTCLIENT_PATCH) +ALL_PHASES = (PHASE0, PHASE1, ALTAIR) MAINNET = ConfigName('mainnet') MINIMAL = ConfigName('minimal') @@ -40,7 +40,7 @@ MINIMAL = ConfigName('minimal') ALL_CONFIGS = (MINIMAL, MAINNET) # The forks that output to the test vectors. -TESTGEN_FORKS = (PHASE0, LIGHTCLIENT_PATCH) +TESTGEN_FORKS = (PHASE0, ALTAIR) # TODO: currently phases are defined as python modules. # It would be better if they would be more well-defined interfaces for stronger typing. @@ -66,7 +66,7 @@ class SpecLightclient(Spec): class SpecForks(TypedDict, total=False): PHASE0: SpecPhase0 PHASE1: SpecPhase1 - LIGHTCLIENT_PATCH: SpecLightclient + ALTAIR: SpecLightclient def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Callable[[Any], int], @@ -82,8 +82,8 @@ def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Ca # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper. # Decide based on performance/consistency results later. state = phases[PHASE1].upgrade_to_phase1(state) - elif spec.fork == LIGHTCLIENT_PATCH: - state = phases[LIGHTCLIENT_PATCH].upgrade_to_lightclient_patch(state) + elif spec.fork == ALTAIR: + state = phases[ALTAIR].upgrade_to_altair(state) return state @@ -352,16 +352,16 @@ def with_phases(phases, other_phases=None): phase_dir[PHASE0] = spec_phase0 if PHASE1 in available_phases: phase_dir[PHASE1] = spec_phase1 - if LIGHTCLIENT_PATCH in available_phases: - phase_dir[LIGHTCLIENT_PATCH] = spec_lightclient_patch + if ALTAIR in available_phases: + phase_dir[ALTAIR] = spec_altair # return is ignored whenever multiple phases are ran. If if PHASE0 in run_phases: ret = fn(spec=spec_phase0, phases=phase_dir, *args, **kw) if PHASE1 in run_phases: ret = fn(spec=spec_phase1, phases=phase_dir, *args, **kw) - if LIGHTCLIENT_PATCH in run_phases: - ret = fn(spec=spec_lightclient_patch, phases=phase_dir, *args, **kw) + if ALTAIR in run_phases: + ret = fn(spec=spec_altair, phases=phase_dir, *args, **kw) return ret return wrapper return decorator @@ -397,9 +397,9 @@ def only_full_crosslink(fn): return wrapper -def is_post_lightclient_patch(spec): +def is_post_altair(spec): if spec.fork in [PHASE0, PHASE1]: - # TODO: PHASE1 fork is temporarily parallel to LIGHTCLIENT_PATCH. - # Will make PHASE1 fork inherit LIGHTCLIENT_PATCH later. + # TODO: PHASE1 fork is temporarily parallel to ALTAIR. + # Will make PHASE1 fork inherit ALTAIR later. return False return True diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 571e19fef..0b3e0484a 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -2,7 +2,7 @@ from lru import LRU from typing import List -from eth2spec.test.context import expect_assertion_error, PHASE1, is_post_lightclient_patch +from eth2spec.test.context import expect_assertion_error, PHASE1, is_post_altair from eth2spec.test.helpers.state import state_transition_and_sign_block, next_epoch, next_slot from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.shard_transitions import get_shard_transition_of_committee @@ -30,7 +30,7 @@ def run_attestation_processing(spec, state, attestation, valid=True): yield 'post', None return - if not is_post_lightclient_patch(spec): + if not is_post_altair(spec): current_epoch_count = len(state.current_epoch_attestations) previous_epoch_count = len(state.previous_epoch_attestations) @@ -38,7 +38,7 @@ def run_attestation_processing(spec, state, attestation, valid=True): spec.process_attestation(state, attestation) # Make sure the attestation has been processed - if not is_post_lightclient_patch(spec): + if not is_post_altair(spec): if attestation.data.target.epoch == spec.get_current_epoch(state): assert len(state.current_epoch_attestations) == current_epoch_count + 1 else: @@ -320,7 +320,7 @@ def prepare_state_with_attestations(spec, state, participation_fn=None): next_slot(spec, state) assert state.slot == next_epoch_start_slot + spec.MIN_ATTESTATION_INCLUSION_DELAY - if not is_post_lightclient_patch(spec): + if not is_post_altair(spec): assert len(state.previous_epoch_attestations) == len(attestations) return attestations diff --git a/tests/core/pyspec/eth2spec/test/helpers/block.py b/tests/core/pyspec/eth2spec/test/helpers/block.py index 7501c8268..084826d65 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/block.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import is_post_lightclient_patch +from eth2spec.test.context import is_post_altair from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.bls import only_with_bls @@ -91,7 +91,7 @@ def build_empty_block(spec, state, slot=None): empty_block.body.eth1_data.deposit_count = state.eth1_deposit_index empty_block.parent_root = parent_block_root - if is_post_lightclient_patch(spec): + if is_post_altair(spec): empty_block.body.sync_committee_signature = spec.G2_POINT_AT_INFINITY apply_randao_reveal(spec, state, empty_block) diff --git a/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py b/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py index 52479cfeb..e8e3dd492 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py +++ b/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py @@ -1,5 +1,5 @@ -from eth2spec.test.context import is_post_lightclient_patch +from eth2spec.test.context import is_post_altair def get_process_calls(spec): @@ -16,8 +16,8 @@ def get_process_calls(spec): 'process_slashings_reset', 'process_randao_mixes_reset', 'process_historical_roots_update', - # HF1 replaced `process_participation_record_updates` with `process_participation_flag_updates` - 'process_participation_flag_updates' if is_post_lightclient_patch(spec) else ( + # Altair replaced `process_participation_record_updates` with `process_participation_flag_updates` + 'process_participation_flag_updates' if is_post_altair(spec) else ( 'process_participation_record_updates' ), 'process_sync_committee_updates', diff --git a/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py b/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py index 89acb3417..97d09b141 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py +++ b/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -1,12 +1,12 @@ -from eth2spec.test.context import is_post_lightclient_patch +from eth2spec.test.context import is_post_altair from eth2spec.test.helpers.block_header import sign_block_header from eth2spec.test.helpers.keys import pubkey_to_privkey from eth2spec.test.helpers.state import get_balance def get_min_slashing_penalty_quotient(spec): - if is_post_lightclient_patch(spec): - return spec.HF1_MIN_SLASHING_PENALTY_QUOTIENT + if is_post_altair(spec): + return spec.ALTAIR_MIN_SLASHING_PENALTY_QUOTIENT else: return spec.MIN_SLASHING_PENALTY_QUOTIENT diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 19ed3f691..4f8b5ea5e 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -2,7 +2,7 @@ from random import Random from lru import LRU from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.test.context import is_post_lightclient_patch +from eth2spec.test.context import is_post_altair from eth2spec.test.helpers.attestations import cached_prepare_state_with_attestations from eth2spec.test.helpers.deposits import mock_deposit from eth2spec.test.helpers.state import next_epoch @@ -39,7 +39,7 @@ def run_deltas(spec, state): """ yield 'pre', state - if is_post_lightclient_patch(spec): + if is_post_altair(spec): def get_source_deltas(state): return spec.get_flag_deltas(state, spec.TIMELY_SOURCE_FLAG_INDEX, spec.TIMELY_SOURCE_FLAG_NUMERATOR) @@ -52,21 +52,21 @@ def run_deltas(spec, state): yield from run_attestation_component_deltas( spec, state, - spec.get_source_deltas if not is_post_lightclient_patch(spec) else get_source_deltas, + spec.get_source_deltas if not is_post_altair(spec) else get_source_deltas, spec.get_matching_source_attestations, 'source_deltas', ) yield from run_attestation_component_deltas( spec, state, - spec.get_target_deltas if not is_post_lightclient_patch(spec) else get_target_deltas, + spec.get_target_deltas if not is_post_altair(spec) else get_target_deltas, spec.get_matching_target_attestations, 'target_deltas', ) yield from run_attestation_component_deltas( spec, state, - spec.get_head_deltas if not is_post_lightclient_patch(spec) else get_head_deltas, + spec.get_head_deltas if not is_post_altair(spec) else get_head_deltas, spec.get_matching_head_attestations, 'head_deltas', ) @@ -93,7 +93,7 @@ def run_attestation_component_deltas(spec, state, component_delta_fn, matching_a yield deltas_name, Deltas(rewards=rewards, penalties=penalties) - if not is_post_lightclient_patch(spec): + if not is_post_altair(spec): matching_attestations = matching_att_fn(state, spec.get_previous_epoch(state)) matching_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) else: @@ -129,7 +129,7 @@ def run_get_inclusion_delay_deltas(spec, state): Run ``get_inclusion_delay_deltas``, yielding: - inclusion delay deltas ('inclusion_delay_deltas') """ - if is_post_lightclient_patch(spec): + if is_post_altair(spec): # No inclusion_delay_deltas yield 'inclusion_delay_deltas', Deltas(rewards=[0] * len(state.validators), penalties=[0] * len(state.validators)) @@ -182,7 +182,7 @@ def run_get_inactivity_penalty_deltas(spec, state): yield 'inactivity_penalty_deltas', Deltas(rewards=rewards, penalties=penalties) - if not is_post_lightclient_patch(spec): + if not is_post_altair(spec): matching_attestations = spec.get_matching_target_attestations(state, spec.get_previous_epoch(state)) matching_attesting_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) else: @@ -200,7 +200,7 @@ def run_get_inactivity_penalty_deltas(spec, state): if spec.is_in_inactivity_leak(state): # Compute base_penalty - if not is_post_lightclient_patch(spec): + if not is_post_altair(spec): cancel_base_rewards_per_epoch = spec.BASE_REWARDS_PER_EPOCH base_reward = spec.get_base_reward(state, index) base_penalty = cancel_base_rewards_per_epoch * base_reward - spec.get_proposer_reward(state, index) @@ -308,7 +308,7 @@ def run_test_full_all_correct(spec, state): def run_test_full_but_partial_participation(spec, state, rng=Random(5522)): cached_prepare_state_with_attestations(spec, state) - if not is_post_lightclient_patch(spec): + if not is_post_altair(spec): for a in state.previous_epoch_attestations: a.aggregation_bits = [rng.choice([True, False]) for _ in a.aggregation_bits] else: @@ -323,7 +323,7 @@ def run_test_partial(spec, state, fraction_filled): cached_prepare_state_with_attestations(spec, state) # Remove portion of attestations - if not is_post_lightclient_patch(spec): + if not is_post_altair(spec): num_attestations = int(len(state.previous_epoch_attestations) * fraction_filled) state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] else: @@ -383,7 +383,7 @@ def run_test_some_very_low_effective_balances_that_attested(spec, state): def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state): cached_prepare_state_with_attestations(spec, state) - if not is_post_lightclient_patch(spec): + if not is_post_altair(spec): # Remove attestation attestation = state.previous_epoch_attestations[0] state.previous_epoch_attestations = state.previous_epoch_attestations[1:] @@ -502,7 +502,7 @@ def run_test_full_random(spec, state, rng=Random(8020)): cached_prepare_state_with_attestations(spec, state) - if not is_post_lightclient_patch(spec): + if not is_post_altair(spec): for pending_attestation in state.previous_epoch_attestations: # ~1/3 have bad target if rng.randint(0, 2) == 0: diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py index 274d67134..9db6076f8 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import is_post_lightclient_patch, spec_state_test, with_all_phases +from eth2spec.test.context import is_post_altair, spec_state_test, with_all_phases from eth2spec.test.helpers.epoch_processing import ( run_epoch_processing_with, ) @@ -16,7 +16,7 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support previous_epoch = spec.get_previous_epoch(state) current_epoch = spec.get_current_epoch(state) - if not is_post_lightclient_patch(spec): + if not is_post_altair(spec): if current_epoch == epoch: attestations = state.current_epoch_attestations elif previous_epoch == epoch: @@ -61,7 +61,7 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support aggregation_bits[i] = 0 # Update state - if not is_post_lightclient_patch(spec): + if not is_post_altair(spec): attestations.append(spec.PendingAttestation( aggregation_bits=aggregation_bits, data=spec.AttestationData( diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py index 7bb86b45e..229dafe53 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py @@ -5,7 +5,7 @@ from eth2spec.test.context import ( with_custom_state, zero_activation_threshold, misc_balances, low_single_balance, - is_post_lightclient_patch, + is_post_altair, ) from eth2spec.test.helpers.state import ( next_epoch, @@ -161,7 +161,7 @@ def run_with_participation(spec, state, participation_fn): attestations = prepare_state_with_attestations(spec, state, participation_fn=participation_tracker) pre_state = state.copy() - if not is_post_lightclient_patch(spec): + if not is_post_altair(spec): proposer_indices = [a.proposer_index for a in state.previous_epoch_attestations] else: sync_committee_indices = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) @@ -173,11 +173,11 @@ def run_with_participation(spec, state, participation_fn): for index in range(len(pre_state.validators)): if spec.is_in_inactivity_leak(state): - # Proposers can still make money during a leak before LIGHTCLIENT_PATCH - if not is_post_lightclient_patch(spec) and index in proposer_indices and index in participated: + # Proposers can still make money during a leak before ALTAIR + if not is_post_altair(spec) and index in proposer_indices and index in participated: assert state.balances[index] > pre_state.balances[index] elif index in attesting_indices: - if is_post_lightclient_patch(spec) and index in sync_committee_indices: + if is_post_altair(spec) and index in sync_committee_indices: # The sync committee reward has not been canceled, so the sync committee participants still earn it assert state.balances[index] >= pre_state.balances[index] else: @@ -428,7 +428,7 @@ def test_attestations_some_slashed(spec, state): for i in range(spec.MIN_PER_EPOCH_CHURN_LIMIT): spec.slash_validator(state, attesting_indices_before_slashings[i]) - if not is_post_lightclient_patch(spec): + if not is_post_altair(spec): assert len(state.previous_epoch_attestations) == len(attestations) pre_state = state.copy() diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py index 34f1e89c6..b31f6fb81 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import spec_state_test, with_all_phases, is_post_lightclient_patch +from eth2spec.test.context import spec_state_test, with_all_phases, is_post_altair from eth2spec.test.helpers.epoch_processing import ( run_epoch_processing_with, run_epoch_processing_to ) @@ -24,8 +24,8 @@ def slash_validators(spec, state, indices, out_epochs): def get_slashing_multiplier(spec): - if is_post_lightclient_patch(spec): - return spec.HF1_PROPORTIONAL_SLASHING_MULTIPLIER + if is_post_altair(spec): + return spec.ALTAIR_PROPORTIONAL_SLASHING_MULTIPLIER else: return spec.PROPORTIONAL_SLASHING_MULTIPLIER diff --git a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py index 98ffbd590..8af0411c4 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py @@ -35,7 +35,7 @@ from eth2spec.test.context import ( with_configs, with_custom_state, large_validator_set, - is_post_lightclient_patch, + is_post_altair, ) @@ -781,14 +781,14 @@ def test_attestation(spec, state): spec, state, shard_transition=shard_transition, index=index, signed=True, on_time=True ) - if not is_post_lightclient_patch(spec): + if not is_post_altair(spec): pre_current_attestations_len = len(state.current_epoch_attestations) # Add to state via block transition attestation_block.body.attestations.append(attestation) signed_attestation_block = state_transition_and_sign_block(spec, state, attestation_block) - if not is_post_lightclient_patch(spec): + if not is_post_altair(spec): assert len(state.current_epoch_attestations) == pre_current_attestations_len + 1 # Epoch transition should move to previous_epoch_attestations pre_current_attestations_root = spec.hash_tree_root(state.current_epoch_attestations) @@ -801,7 +801,7 @@ def test_attestation(spec, state): yield 'blocks', [signed_attestation_block, signed_epoch_block] yield 'post', state - if not is_post_lightclient_patch(spec): + if not is_post_altair(spec): assert len(state.current_epoch_attestations) == 0 assert spec.hash_tree_root(state.previous_epoch_attestations) == pre_current_attestations_root else: diff --git a/tests/core/pyspec/eth2spec/test/phase0/unittests/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/phase0/unittests/fork_choice/test_on_attestation.py index 05f0fb051..926ebec80 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/unittests/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase0/unittests/fork_choice/test_on_attestation.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, with_all_phases, spec_state_test +from eth2spec.test.context import PHASE0, PHASE1, ALTAIR, with_all_phases, spec_state_test from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.attestations import get_valid_attestation, sign_attestation from eth2spec.test.helpers.state import transition_to, state_transition_and_sign_block, next_epoch, next_slot @@ -18,7 +18,7 @@ def run_on_attestation(spec, state, store, attestation, valid=True): spec.on_attestation(store, attestation) sample_index = indexed_attestation.attesting_indices[0] - if spec.fork in (PHASE0, LIGHTCLIENT_PATCH): + if spec.fork in (PHASE0, ALTAIR): latest_message = spec.LatestMessage( epoch=attestation.data.target.epoch, root=attestation.data.beacon_block_root, diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_attestation.py index 5b2f952ae..2f641eacb 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_attestation.py @@ -1,6 +1,6 @@ from eth2spec.test.context import ( PHASE0, - LIGHTCLIENT_PATCH, + ALTAIR, with_all_phases_except, spec_state_test, always_bls, @@ -13,7 +13,7 @@ from eth2spec.test.helpers.attestations import ( ) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @always_bls def test_on_time_success(spec, state): @@ -24,7 +24,7 @@ def test_on_time_success(spec, state): yield from run_attestation_processing(spec, state, attestation) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @always_bls def test_late_success(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py index 27829e4a0..249074999 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py @@ -9,7 +9,7 @@ from eth2spec.test.helpers.attestations import ( from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot from eth2spec.test.context import ( PHASE0, - LIGHTCLIENT_PATCH, + ALTAIR, MINIMAL, expect_assertion_error, disable_process_reveal_deadlines, @@ -69,7 +69,7 @@ def run_custody_chunk_response_processing(spec, state, custody_response, valid=T yield 'post', state -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @with_configs([MINIMAL], reason="too slow") @disable_process_reveal_deadlines @@ -93,7 +93,7 @@ def test_challenge_appended(spec, state): yield from run_chunk_challenge_processing(spec, state, challenge) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -119,7 +119,7 @@ def test_challenge_empty_element_replaced(spec, state): yield from run_chunk_challenge_processing(spec, state, challenge) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -145,7 +145,7 @@ def test_duplicate_challenge(spec, state): yield from run_chunk_challenge_processing(spec, state, challenge, valid=False) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -173,7 +173,7 @@ def test_second_challenge(spec, state): yield from run_chunk_challenge_processing(spec, state, challenge1) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -198,7 +198,7 @@ def test_multiple_epochs_custody(spec, state): yield from run_chunk_challenge_processing(spec, state, challenge) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -223,7 +223,7 @@ def test_many_epochs_custody(spec, state): yield from run_chunk_challenge_processing(spec, state, challenge) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -244,7 +244,7 @@ def test_off_chain_attestation(spec, state): yield from run_chunk_challenge_processing(spec, state, challenge) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -276,7 +276,7 @@ def test_custody_response(spec, state): yield from run_custody_chunk_response_processing(spec, state, custody_response) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -307,7 +307,7 @@ def test_custody_response_chunk_index_2(spec, state): yield from run_custody_chunk_response_processing(spec, state, custody_response) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -339,7 +339,7 @@ def test_custody_response_multiple_epochs(spec, state): yield from run_custody_chunk_response_processing(spec, state, custody_response) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_key_reveal.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_key_reveal.py index 00a6112bf..2ea70703a 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_key_reveal.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_key_reveal.py @@ -1,7 +1,7 @@ from eth2spec.test.helpers.custody import get_valid_custody_key_reveal from eth2spec.test.context import ( PHASE0, - LIGHTCLIENT_PATCH, + ALTAIR, with_all_phases_except, spec_state_test, expect_assertion_error, @@ -40,7 +40,7 @@ def run_custody_key_reveal_processing(spec, state, custody_key_reveal, valid=Tru yield 'post', state -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @always_bls def test_success(spec, state): @@ -50,7 +50,7 @@ def test_success(spec, state): yield from run_custody_key_reveal_processing(spec, state, custody_key_reveal) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @always_bls def test_reveal_too_early(spec, state): @@ -59,7 +59,7 @@ def test_reveal_too_early(spec, state): yield from run_custody_key_reveal_processing(spec, state, custody_key_reveal, False) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @always_bls def test_wrong_period(spec, state): @@ -68,7 +68,7 @@ def test_wrong_period(spec, state): yield from run_custody_key_reveal_processing(spec, state, custody_key_reveal, False) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @always_bls def test_late_reveal(spec, state): @@ -78,7 +78,7 @@ def test_late_reveal(spec, state): yield from run_custody_key_reveal_processing(spec, state, custody_key_reveal) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @always_bls def test_double_reveal(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py index 1f46bcf05..732d7da05 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py @@ -11,7 +11,7 @@ from eth2spec.test.helpers.state import get_balance, transition_to from eth2spec.test.context import ( PHASE0, MINIMAL, - LIGHTCLIENT_PATCH, + ALTAIR, with_all_phases_except, spec_state_test, expect_assertion_error, @@ -113,7 +113,7 @@ def run_standard_custody_slashing_test(spec, yield from run_custody_slashing_processing(spec, state, slashing, valid=valid, correct=correct) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -121,7 +121,7 @@ def test_custody_slashing(spec, state): yield from run_standard_custody_slashing_test(spec, state) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -129,7 +129,7 @@ def test_incorrect_custody_slashing(spec, state): yield from run_standard_custody_slashing_test(spec, state, correct=False) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -137,7 +137,7 @@ def test_multiple_epochs_custody(spec, state): yield from run_standard_custody_slashing_test(spec, state, shard_lateness=spec.SLOTS_PER_EPOCH * 3) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") @@ -145,7 +145,7 @@ def test_many_epochs_custody(spec, state): yield from run_standard_custody_slashing_test(spec, state, shard_lateness=spec.SLOTS_PER_EPOCH * 5) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @disable_process_reveal_deadlines @with_configs([MINIMAL], reason="too slow") diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_early_derived_secret_reveal.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_early_derived_secret_reveal.py index 3094f795b..12cdfdff0 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_early_derived_secret_reveal.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_early_derived_secret_reveal.py @@ -2,7 +2,7 @@ from eth2spec.test.helpers.custody import get_valid_early_derived_secret_reveal from eth2spec.test.helpers.state import next_epoch_via_block, get_balance from eth2spec.test.context import ( PHASE0, - LIGHTCLIENT_PATCH, + ALTAIR, with_all_phases_except, spec_state_test, expect_assertion_error, @@ -42,7 +42,7 @@ def run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, v yield 'post', state -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @always_bls def test_success(spec, state): @@ -51,7 +51,7 @@ def test_success(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @never_bls def test_reveal_from_current_epoch(spec, state): @@ -60,7 +60,7 @@ def test_reveal_from_current_epoch(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @never_bls def test_reveal_from_past_epoch(spec, state): @@ -70,7 +70,7 @@ def test_reveal_from_past_epoch(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @always_bls def test_reveal_with_custody_padding(spec, state): @@ -82,7 +82,7 @@ def test_reveal_with_custody_padding(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, True) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @always_bls def test_reveal_with_custody_padding_minus_one(spec, state): @@ -94,7 +94,7 @@ def test_reveal_with_custody_padding_minus_one(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, True) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @never_bls def test_double_reveal(spec, state): @@ -115,7 +115,7 @@ def test_double_reveal(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal2, False) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @never_bls def test_revealer_is_slashed(spec, state): @@ -125,7 +125,7 @@ def test_revealer_is_slashed(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @never_bls def test_far_future_epoch(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_shard_transition.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_shard_transition.py index d2b7962b6..2ee2d34c2 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_shard_transition.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_shard_transition.py @@ -1,6 +1,6 @@ from eth2spec.test.context import ( PHASE0, - LIGHTCLIENT_PATCH, + ALTAIR, with_all_phases_except, only_full_crosslink, spec_state_test, @@ -91,21 +91,21 @@ def run_successful_crosslink_tests(spec, state, target_len_offset_slot): assert bool(pending_attestation.crosslink_success) is True -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @only_full_crosslink def test_basic_crosslinks(spec, state): yield from run_successful_crosslink_tests(spec, state, target_len_offset_slot=1) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @only_full_crosslink def test_multiple_offset_slots(spec, state): yield from run_successful_crosslink_tests(spec, state, target_len_offset_slot=2) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @only_full_crosslink def test_no_winning_root(spec, state): @@ -153,7 +153,7 @@ def test_no_winning_root(spec, state): assert state.shard_states == pre_shard_states -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @only_full_crosslink def test_wrong_shard_transition_root(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py index 1d8adecbc..be058bb4b 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py +++ b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py @@ -8,7 +8,7 @@ from eth2spec.test.helpers.attestations import ( from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot from eth2spec.test.context import ( PHASE0, - LIGHTCLIENT_PATCH, + ALTAIR, MINIMAL, spec_state_test, with_all_phases_except, @@ -26,7 +26,7 @@ def run_process_challenge_deadlines(spec, state): yield from run_epoch_processing_with(spec, state, 'process_challenge_deadlines') -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @with_configs([MINIMAL], reason="too slow") def test_validator_slashed_after_chunk_challenge(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_custody_final_updates.py b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_custody_final_updates.py index 82ecde7ab..f2d9acc9a 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_custody_final_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_custody_final_updates.py @@ -1,6 +1,6 @@ from eth2spec.test.context import ( PHASE0, - LIGHTCLIENT_PATCH, + ALTAIR, ) from eth2spec.test.helpers.custody import ( get_valid_chunk_challenge, @@ -30,7 +30,7 @@ def run_process_custody_final_updates(spec, state): yield from run_epoch_processing_with(spec, state, 'process_custody_final_updates') -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test def test_validator_withdrawal_delay(spec, state): transition_to_valid_shard_slot(spec, state) @@ -43,7 +43,7 @@ def test_validator_withdrawal_delay(spec, state): assert state.validators[0].withdrawable_epoch == spec.FAR_FUTURE_EPOCH -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test def test_validator_withdrawal_reenable_after_custody_reveal(spec, state): transition_to_valid_shard_slot(spec, state) @@ -68,7 +68,7 @@ def test_validator_withdrawal_reenable_after_custody_reveal(spec, state): assert state.validators[0].withdrawable_epoch < spec.FAR_FUTURE_EPOCH -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test def test_validator_withdrawal_suspend_after_chunk_challenge(spec, state): transition_to_valid_shard_slot(spec, state) @@ -117,7 +117,7 @@ def test_validator_withdrawal_suspend_after_chunk_challenge(spec, state): assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test def test_validator_withdrawal_resume_after_chunk_challenge_response(spec, state): transition_to_valid_shard_slot(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py index b95082491..7b2094aea 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py +++ b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py @@ -4,7 +4,7 @@ from eth2spec.test.helpers.custody import ( from eth2spec.test.helpers.state import transition_to from eth2spec.test.context import ( PHASE0, - LIGHTCLIENT_PATCH, + ALTAIR, MINIMAL, with_all_phases_except, with_configs, @@ -18,7 +18,7 @@ def run_process_challenge_deadlines(spec, state): yield from run_epoch_processing_with(spec, state, 'process_challenge_deadlines') -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @with_configs([MINIMAL], reason="too slow") def test_validator_slashed_after_reveal_deadline(spec, state): @@ -38,7 +38,7 @@ def test_validator_slashed_after_reveal_deadline(spec, state): assert state.validators[0].slashed == 1 -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @with_configs([MINIMAL], reason="too slow") def test_validator_not_slashed_after_reveal(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py index ba47adde9..1f17fa911 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py @@ -2,7 +2,7 @@ from typing import Dict, Sequence from eth2spec.test.context import ( PHASE0, - LIGHTCLIENT_PATCH, + ALTAIR, MINIMAL, with_all_phases_except, spec_state_test, @@ -100,7 +100,7 @@ def run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, comm assert post_shard_state.gasprice > pre_gasprice -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @only_full_crosslink def test_process_beacon_block_with_normal_shard_transition(spec, state): @@ -114,7 +114,7 @@ def test_process_beacon_block_with_normal_shard_transition(spec, state): yield from run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @only_full_crosslink def test_process_beacon_block_with_empty_proposal_transition(spec, state): @@ -133,7 +133,7 @@ def test_process_beacon_block_with_empty_proposal_transition(spec, state): # -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @only_full_crosslink def test_with_shard_transition_with_custody_challenge_and_response(spec, state): @@ -167,7 +167,7 @@ def test_with_shard_transition_with_custody_challenge_and_response(spec, state): yield from run_beacon_block(spec, state, block) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @with_configs([MINIMAL]) def test_custody_key_reveal(spec, state): @@ -181,7 +181,7 @@ def test_custody_key_reveal(spec, state): yield from run_beacon_block(spec, state, block) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test def test_early_derived_secret_reveal(spec, state): transition_to_valid_shard_slot(spec, state) @@ -192,7 +192,7 @@ def test_early_derived_secret_reveal(spec, state): yield from run_beacon_block(spec, state, block) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @only_full_crosslink def test_custody_slashing(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase1/sanity/test_shard_blocks.py b/tests/core/pyspec/eth2spec/test/phase1/sanity/test_shard_blocks.py index 1590d2a6e..d27dacc7d 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/sanity/test_shard_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase1/sanity/test_shard_blocks.py @@ -1,6 +1,6 @@ from eth2spec.test.context import ( PHASE0, - LIGHTCLIENT_PATCH, + ALTAIR, always_bls, expect_assertion_error, spec_state_test, @@ -44,7 +44,7 @@ def run_shard_blocks(spec, shard_state, signed_shard_block, beacon_parent_state, shard_state.latest_block_root == pre_shard_state.latest_block_root -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @always_bls @only_full_crosslink @@ -64,7 +64,7 @@ def test_valid_shard_block(spec, state): # -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @only_full_crosslink def test_invalid_shard_parent_root(spec, state): @@ -80,7 +80,7 @@ def test_invalid_shard_parent_root(spec, state): yield from run_shard_blocks(spec, shard_state, signed_shard_block, beacon_state, valid=False) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @only_full_crosslink def test_invalid_beacon_parent_root(spec, state): @@ -95,7 +95,7 @@ def test_invalid_beacon_parent_root(spec, state): yield from run_shard_blocks(spec, shard_state, signed_shard_block, beacon_state, valid=False) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @only_full_crosslink def test_invalid_slot(spec, state): @@ -111,7 +111,7 @@ def test_invalid_slot(spec, state): yield from run_shard_blocks(spec, shard_state, signed_shard_block, beacon_state, valid=False) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @only_full_crosslink def test_invalid_proposer_index(spec, state): @@ -131,7 +131,7 @@ def test_invalid_proposer_index(spec, state): yield from run_shard_blocks(spec, shard_state, signed_shard_block, beacon_state, valid=False) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @always_bls @only_full_crosslink @@ -152,7 +152,7 @@ def test_out_of_bound_offset(spec, state): yield from run_shard_blocks(spec, shard_state, signed_shard_block, beacon_state, valid=False) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @always_bls @only_full_crosslink @@ -171,7 +171,7 @@ def test_invalid_offset(spec, state): yield from run_shard_blocks(spec, shard_state, signed_shard_block, beacon_state, valid=False) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @always_bls @only_full_crosslink @@ -190,7 +190,7 @@ def test_empty_block_body(spec, state): # -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @always_bls @only_full_crosslink @@ -209,7 +209,7 @@ def test_invalid_signature(spec, state): # -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @always_bls @only_full_crosslink @@ -226,7 +226,7 @@ def test_max_offset(spec, state): yield from run_shard_blocks(spec, shard_state, signed_shard_block, beacon_state) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @always_bls @only_full_crosslink diff --git a/tests/core/pyspec/eth2spec/test/phase1/unittests/fork_choice/test_on_shard_block.py b/tests/core/pyspec/eth2spec/test/phase1/unittests/fork_choice/test_on_shard_block.py index 66d254ed1..225800303 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/unittests/fork_choice/test_on_shard_block.py +++ b/tests/core/pyspec/eth2spec/test/phase1/unittests/fork_choice/test_on_shard_block.py @@ -2,7 +2,7 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.test.context import ( PHASE0, - LIGHTCLIENT_PATCH, + ALTAIR, spec_state_test, with_all_phases_except, never_bls, @@ -152,7 +152,7 @@ def create_and_apply_beacon_and_shard_blocks(spec, state, store, shard, shard_bl return has_shard_committee -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @never_bls # Set to never_bls for testing `check_pending_shard_blocks` def test_basic(spec, state): @@ -213,7 +213,7 @@ def create_simple_fork(spec, state, store, shard): return head_block, forking_block -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @only_full_crosslink def test_shard_simple_fork(spec, state): @@ -238,7 +238,7 @@ def test_shard_simple_fork(spec, state): assert spec.get_shard_head(store, shard) == forking_block.message.hash_tree_root() -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test @only_full_crosslink def test_shard_latest_messages_for_different_shards(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase1/unittests/test_get_start_shard.py b/tests/core/pyspec/eth2spec/test/phase1/unittests/test_get_start_shard.py index 030357655..646dc6eb7 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/unittests/test_get_start_shard.py +++ b/tests/core/pyspec/eth2spec/test/phase1/unittests/test_get_start_shard.py @@ -1,13 +1,13 @@ from eth2spec.test.context import ( PHASE0, - LIGHTCLIENT_PATCH, + ALTAIR, with_all_phases_except, spec_state_test, ) from eth2spec.test.helpers.state import next_epoch -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test def test_get_committee_count_delta(spec, state): assert spec.get_committee_count_delta(state, 0, 0) == 0 @@ -24,7 +24,7 @@ def test_get_committee_count_delta(spec, state): ) -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test def test_get_start_shard_current_epoch_start(spec, state): assert state.current_epoch_start_shard == 0 @@ -40,7 +40,7 @@ def test_get_start_shard_current_epoch_start(spec, state): assert start_shard == state.current_epoch_start_shard -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test def test_get_start_shard_next_slot(spec, state): next_epoch(spec, state) @@ -58,7 +58,7 @@ def test_get_start_shard_next_slot(spec, state): assert start_shard == expected_start_shard -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test def test_get_start_shard_previous_slot(spec, state): next_epoch(spec, state) @@ -77,7 +77,7 @@ def test_get_start_shard_previous_slot(spec, state): assert start_shard == expected_start_shard -@with_all_phases_except([PHASE0, LIGHTCLIENT_PATCH]) +@with_all_phases_except([PHASE0, ALTAIR]) @spec_state_test def test_get_start_shard_far_past_epoch(spec, state): initial_epoch = spec.get_current_epoch(state) diff --git a/tests/formats/forks/README.md b/tests/formats/forks/README.md index cbafd4e2a..2226a3f20 100644 --- a/tests/formats/forks/README.md +++ b/tests/formats/forks/README.md @@ -22,7 +22,7 @@ Key of valid `fork` strings that might be found in `meta.yaml` | String ID | Pre-fork | Post-fork | Function | | - | - | - | - | -| `altair` | Phase 0 | Altair | `upgrade_to_lightclient_patch` | +| `altair` | Phase 0 | Altair | `upgrade_to_altair` | ### `pre.ssz_snappy` diff --git a/tests/generators/README.md b/tests/generators/README.md index 6ccf9f118..f629c8b74 100644 --- a/tests/generators/README.md +++ b/tests/generators/README.md @@ -163,14 +163,14 @@ Another example, to generate tests from pytests: ```python from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.lightclient_patch import spec as spec_lightclient_patch +from eth2spec.altair import spec as spec_altair from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH +from eth2spec.test.context import PHASE0, PHASE1, ALTAIR from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators -specs = (spec_phase0, spec_lightclient_patch, spec_phase1) +specs = (spec_phase0, spec_altair, spec_phase1) if __name__ == "__main__": @@ -178,7 +178,7 @@ if __name__ == "__main__": 'blocks', 'slots', ]} - lightclient_patch_mods = {**{key: 'eth2spec.test.lightclient_patch.sanity.test_' + key for key in [ + altair_mods = {**{key: 'eth2spec.test.altair.sanity.test_' + key for key in [ 'blocks', ]}, **phase_0_mods} # also run the previous phase 0 tests phase_1_mods = {**{key: 'eth2spec.test.phase1.sanity.test_' + key for key in [ @@ -188,7 +188,7 @@ if __name__ == "__main__": all_mods = { PHASE0: phase_0_mods, - LIGHTCLIENT_PATCH: lightclient_patch_mods, + ALTAIR: altair_mods, PHASE1: phase_1_mods, } diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index 50a1e2b57..5207330a7 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -1,11 +1,11 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.lightclient_patch import spec as spec_lightclient_patch +from eth2spec.altair import spec as spec_altair from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH +from eth2spec.test.context import PHASE0, PHASE1, ALTAIR -specs = (spec_phase0, spec_lightclient_patch, spec_phase1) +specs = (spec_phase0, spec_altair, spec_phase1) if __name__ == "__main__": @@ -21,8 +21,8 @@ if __name__ == "__main__": 'historical_roots_update', 'participation_record_updates', ]} - lightclient_patch_mods = { - **{key: 'eth2spec.test.lightclient_patch.epoch_processing.test_process_' + key for key in [ + altair_mods = { + **{key: 'eth2spec.test.altair.epoch_processing.test_process_' + key for key in [ 'sync_committee_updates', ]}, **phase_0_mods, @@ -35,7 +35,7 @@ if __name__ == "__main__": all_mods = { PHASE0: phase_0_mods, - LIGHTCLIENT_PATCH: lightclient_patch_mods, + ALTAIR: altair_mods, PHASE1: phase_1_mods, } diff --git a/tests/generators/finality/main.py b/tests/generators/finality/main.py index 8b961f9f4..5598028a2 100644 --- a/tests/generators/finality/main.py +++ b/tests/generators/finality/main.py @@ -1,22 +1,22 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.lightclient_patch import spec as spec_lightclient_patch +from eth2spec.altair import spec as spec_altair from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH +from eth2spec.test.context import PHASE0, PHASE1, ALTAIR -specs = (spec_phase0, spec_lightclient_patch, spec_phase1) +specs = (spec_phase0, spec_altair, spec_phase1) if __name__ == "__main__": phase_0_mods = {'finality': 'eth2spec.test.phase0.finality.test_finality'} - # No additional lightclient_patch or phase 1 specific finality tests, yet. - lightclient_patch_mods = phase_0_mods + # No additional altair or phase 1 specific finality tests, yet. + altair_mods = phase_0_mods phase_1_mods = phase_0_mods all_mods = { PHASE0: phase_0_mods, - LIGHTCLIENT_PATCH: lightclient_patch_mods, + ALTAIR: altair_mods, PHASE1: phase_1_mods, } diff --git a/tests/generators/forks/main.py b/tests/generators/forks/main.py index 190d4620c..3bec19735 100644 --- a/tests/generators/forks/main.py +++ b/tests/generators/forks/main.py @@ -1,11 +1,11 @@ from importlib import reload from typing import Iterable -from eth2spec.test.context import PHASE0, LIGHTCLIENT_PATCH, MINIMAL, MAINNET +from eth2spec.test.context import PHASE0, ALTAIR, MINIMAL, MAINNET from eth2spec.config import config_util -from eth2spec.test.lightclient_patch.fork import test_fork as test_altair_forks +from eth2spec.test.altair.fork import test_fork as test_altair_forks from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.lightclient_patch import spec as spec_lightclient_patch +from eth2spec.altair import spec as spec_altair from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests @@ -16,7 +16,7 @@ def create_provider(tests_src, config_name: str, phase: str, fork_name: str) -> def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) - reload(spec_lightclient_patch) + reload(spec_altair) return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: @@ -33,6 +33,6 @@ def create_provider(tests_src, config_name: str, phase: str, fork_name: str) -> if __name__ == "__main__": gen_runner.run_generator("forks", [ - create_provider(test_altair_forks, MINIMAL, PHASE0, LIGHTCLIENT_PATCH), - create_provider(test_altair_forks, MAINNET, PHASE0, LIGHTCLIENT_PATCH), + create_provider(test_altair_forks, MINIMAL, PHASE0, ALTAIR), + create_provider(test_altair_forks, MAINNET, PHASE0, ALTAIR), ]) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index 00a58288f..f5c181710 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -1,11 +1,11 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.lightclient_patch import spec as spec_lightclient_patch +from eth2spec.altair import spec as spec_altair from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH +from eth2spec.test.context import PHASE0, PHASE1, ALTAIR -specs = (spec_phase0, spec_lightclient_patch, spec_phase1) +specs = (spec_phase0, spec_altair, spec_phase1) if __name__ == "__main__": @@ -17,8 +17,8 @@ if __name__ == "__main__": 'proposer_slashing', 'voluntary_exit', ]} - lightclient_patch_mods = { - **{key: 'eth2spec.test.lightclient_patch.block_processing.test_process_' + key for key in [ + altair_mods = { + **{key: 'eth2spec.test.altair.block_processing.test_process_' + key for key in [ 'sync_committee', ]}, **phase_0_mods, @@ -34,7 +34,7 @@ if __name__ == "__main__": all_mods = { PHASE0: phase_0_mods, - LIGHTCLIENT_PATCH: lightclient_patch_mods, + ALTAIR: altair_mods, PHASE1: phase_1_mods, } diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index 4124587cc..3a7e8f63d 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -1,11 +1,11 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.lightclient_patch import spec as spec_lightclient_patch +from eth2spec.altair import spec as spec_altair from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH +from eth2spec.test.context import PHASE0, PHASE1, ALTAIR -specs = (spec_phase0, spec_lightclient_patch, spec_phase1) +specs = (spec_phase0, spec_altair, spec_phase1) if __name__ == "__main__": @@ -14,13 +14,13 @@ if __name__ == "__main__": 'leak', 'random', ]} - # No additional lightclient_patch or phase 1 specific rewards tests, yet. - lightclient_patch_mods = phase_0_mods + # No additional altair or phase 1 specific rewards tests, yet. + altair_mods = phase_0_mods phase_1_mods = phase_0_mods all_mods = { PHASE0: phase_0_mods, - LIGHTCLIENT_PATCH: lightclient_patch_mods, + ALTAIR: altair_mods, PHASE1: phase_1_mods, } diff --git a/tests/generators/sanity/main.py b/tests/generators/sanity/main.py index 5155798ff..a0ec2aaae 100644 --- a/tests/generators/sanity/main.py +++ b/tests/generators/sanity/main.py @@ -1,12 +1,12 @@ from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.lightclient_patch import spec as spec_lightclient_patch +from eth2spec.altair import spec as spec_altair from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH +from eth2spec.test.context import PHASE0, PHASE1, ALTAIR from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators -specs = (spec_phase0, spec_lightclient_patch, spec_phase1) +specs = (spec_phase0, spec_altair, spec_phase1) if __name__ == "__main__": @@ -14,7 +14,7 @@ if __name__ == "__main__": 'blocks', 'slots', ]} - lightclient_patch_mods = {**{key: 'eth2spec.test.lightclient_patch.sanity.test_' + key for key in [ + altair_mods = {**{key: 'eth2spec.test.altair.sanity.test_' + key for key in [ 'blocks', ]}, **phase_0_mods} # also run the previous phase 0 tests phase_1_mods = {**{key: 'eth2spec.test.phase1.sanity.test_' + key for key in [ @@ -24,7 +24,7 @@ if __name__ == "__main__": all_mods = { PHASE0: phase_0_mods, - LIGHTCLIENT_PATCH: lightclient_patch_mods, + ALTAIR: altair_mods, PHASE1: phase_1_mods, } diff --git a/tests/generators/ssz_static/main.py b/tests/generators/ssz_static/main.py index e21fc4141..e06e3edc8 100644 --- a/tests/generators/ssz_static/main.py +++ b/tests/generators/ssz_static/main.py @@ -9,8 +9,8 @@ from eth2spec.debug import random_value, encode from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.lightclient_patch import spec as spec_lightclient_patch -from eth2spec.test.context import PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, MINIMAL, MAINNET +from eth2spec.altair import spec as spec_altair +from eth2spec.test.context import PHASE1, ALTAIR, TESTGEN_FORKS, MINIMAL, MAINNET from eth2spec.utils.ssz.ssz_typing import Container from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, @@ -65,7 +65,7 @@ def create_provider(fork_name, config_name: str, seed: int, mode: random_value.R config_util.prepare_config(configs_path, config_name) reload(spec_phase0) reload(spec_phase1) - reload(spec_lightclient_patch) + reload(spec_altair) return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: @@ -73,8 +73,8 @@ def create_provider(fork_name, config_name: str, seed: int, mode: random_value.R spec = spec_phase0 if fork_name == PHASE1: spec = spec_phase1 - if fork_name == LIGHTCLIENT_PATCH: - spec = spec_lightclient_patch + if fork_name == ALTAIR: + spec = spec_altair for (i, (name, ssz_type)) in enumerate(get_spec_ssz_types(spec)): yield from ssz_static_cases(fork_name, seed * 1000 + i, name, ssz_type, mode, chaos, count) From 72832c8b9c99e8950e74d3e8c1d0650ca4f04f36 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 11 Mar 2021 22:12:20 +0800 Subject: [PATCH 182/222] Fix test --- .../unittests/test_sync_protocol.py | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_sync_protocol.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_sync_protocol.py index b9fdb66ba..d65e0ad01 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_sync_protocol.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_sync_protocol.py @@ -1,6 +1,8 @@ from eth2spec.test.context import ( LIGHTCLIENT_PATCH, + MINIMAL, spec_state_test, + with_configs, with_phases, ) from eth2spec.test.helpers.attestations import next_epoch_with_attestations @@ -78,6 +80,7 @@ def test_process_light_client_update_not_updated(spec, state): @with_phases([LIGHTCLIENT_PATCH]) @spec_state_test +@with_configs([MINIMAL], reason="too slow") def test_process_light_client_update_timeout(spec, state): pre_snapshot = spec.LightClientSnapshot( header=spec.BeaconBlockHeader(), @@ -105,12 +108,6 @@ def test_process_light_client_update_timeout(spec, state): body_root=signed_block.message.body.hash_tree_root(), ) - # Sync committee is updated - next_sync_committee_branch = build_proof(state.get_backing(), spec.NEXT_SYNC_COMMITTEE_INDEX) - # Finality is unchanged - finality_header = spec.BeaconBlockHeader() - finality_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.FINALIZED_ROOT_INDEX))] - # Sync committee signing the finalized_block_header committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) sync_committee_bits = [True] * len(committee) @@ -122,6 +119,12 @@ def test_process_light_client_update_timeout(spec, state): block_root=spec.Root(block_header.hash_tree_root()), ) + # Sync committee is updated + next_sync_committee_branch = build_proof(state.get_backing(), spec.NEXT_SYNC_COMMITTEE_INDEX) + # Finality is unchanged + finality_header = spec.BeaconBlockHeader() + finality_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.FINALIZED_ROOT_INDEX))] + update = spec.LightClientUpdate( header=block_header, next_sync_committee=state.next_sync_committee, @@ -142,6 +145,7 @@ def test_process_light_client_update_timeout(spec, state): @with_phases([LIGHTCLIENT_PATCH]) @spec_state_test +@with_configs([MINIMAL], reason="too slow") def test_process_light_client_update_finality_updated(spec, state): pre_snapshot = spec.LightClientSnapshot( header=spec.BeaconBlockHeader(), @@ -173,17 +177,6 @@ def test_process_light_client_update_finality_updated(spec, state): assert finalized_block_header.hash_tree_root() == state.finalized_checkpoint.root finality_branch = build_proof(state.get_backing(), spec.FINALIZED_ROOT_INDEX) - # Sync committee signing the finalized_block_header - committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) - sync_committee_bits = [True] * len(committee) - sync_committee_signature = compute_aggregate_sync_committee_signature( - spec, - state, - finalized_block_header.slot, - committee, - block_root=spec.Root(finalized_block_header.hash_tree_root()), - ) - # Build block header block = build_empty_block(spec, state) block_header = spec.BeaconBlockHeader( @@ -194,11 +187,22 @@ def test_process_light_client_update_finality_updated(spec, state): body_root=block.body.hash_tree_root(), ) + # Sync committee signing the finalized_block_header + committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + sync_committee_bits = [True] * len(committee) + sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + block_header.slot, + committee, + block_root=spec.Root(block_header.hash_tree_root()), + ) + update = spec.LightClientUpdate( header=finalized_block_header, next_sync_committee=state.next_sync_committee, next_sync_committee_branch=next_sync_committee_branch, - finality_header=block_header, + finality_header=block_header, # block_header is the signed header finality_branch=finality_branch, sync_committee_bits=sync_committee_bits, sync_committee_signature=sync_committee_signature, From b9c95b722d9400c29bb0258619e94c665d43d89a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 11 Mar 2021 08:55:55 -0700 Subject: [PATCH 183/222] one more hf-1 to Altair conversion --- specs/altair/beacon-chain.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index d2e060cdf..51817fd21 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -519,9 +519,8 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: # Add validator and balance entries state.validators.append(get_validator_from_deposit(state, deposit)) state.balances.append(amount) - # [Added in hf-1] Initialize empty participation flags for new validator - state.previous_epoch_participation.append(ParticipationFlags(0b0000_0000)) - state.current_epoch_participation.append(ParticipationFlags(0b0000_0000)) + state.previous_epoch_participation.append(ParticipationFlags(0b0000_0000)) # [New in Altair] + state.current_epoch_participation.append(ParticipationFlags(0b0000_0000)) # [New in Altair] else: # Increase balance by deposit amount index = ValidatorIndex(validator_pubkeys.index(pubkey)) From 8e303b60b961420bb1df6b8ea93be5521a32e02a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 12 Mar 2021 00:06:52 +0800 Subject: [PATCH 184/222] Fix generator to ssz_snappy mode --- tests/generators/fork_choice/main.py | 45 +++++++------------ tests/generators/fork_choice/requirements.txt | 4 +- 2 files changed, 17 insertions(+), 32 deletions(-) diff --git a/tests/generators/fork_choice/main.py b/tests/generators/fork_choice/main.py index f684399eb..79920d8ad 100644 --- a/tests/generators/fork_choice/main.py +++ b/tests/generators/fork_choice/main.py @@ -1,40 +1,25 @@ -from typing import Iterable - -from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests -from importlib import reload, import_module -from eth2spec.config import config_util +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.test.context import PHASE0 -from eth2spec.utils import bls +from eth2spec.lightclient_patch import spec as spec_lightclient_patch +from eth2spec.phase1 import spec as spec_phase1 +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH -def create_provider(fork_name: str, handler_name: str, - tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: - def prepare_fn(configs_path: str) -> str: - config_util.prepare_config(configs_path, config_name) - reload(spec_phase0) - bls.use_milagro() - return config_name - - def cases_fn() -> Iterable[gen_typing.TestCase]: - tests_src = import_module(tests_src_mod_name) - return generate_from_tests( - runner_name='fork_choice', - handler_name=handler_name, - src=tests_src, - fork_name=fork_name, - ) - - return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) +specs = (spec_phase0, spec_lightclient_patch, spec_phase1) if __name__ == "__main__": phase_0_mods = {key: 'eth2spec.test.phase0.fork_choice.test_' + key for key in [ 'get_head', ]} + # No additional lightclient_patch or phase 1 specific finality tests, yet. + lightclient_patch_mods = phase_0_mods + phase_1_mods = phase_0_mods - # TODO: add other configs and forks - gen_runner.run_generator(f"fork_choice", [ - create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items() - ]) + all_mods = { + PHASE0: phase_0_mods, + LIGHTCLIENT_PATCH: lightclient_patch_mods, + PHASE1: phase_1_mods, + } + + run_state_test_generators(runner_name="fork_choice", specs=specs, all_mods=all_mods) diff --git a/tests/generators/fork_choice/requirements.txt b/tests/generators/fork_choice/requirements.txt index b82314298..182248686 100644 --- a/tests/generators/fork_choice/requirements.txt +++ b/tests/generators/fork_choice/requirements.txt @@ -1,2 +1,2 @@ -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../[generator] From 78a48974184b760ede20b8e92adbcb7efb39b87f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 12 Mar 2021 00:33:04 +0800 Subject: [PATCH 185/222] Add fork choice rule format doc --- tests/formats/fork_choice/README.md | 101 +++++++++++++++++++++++++ tests/generators/fork_choice/README.md | 4 +- 2 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 tests/formats/fork_choice/README.md diff --git a/tests/formats/fork_choice/README.md b/tests/formats/fork_choice/README.md new file mode 100644 index 000000000..20f6d7d64 --- /dev/null +++ b/tests/formats/fork_choice/README.md @@ -0,0 +1,101 @@ +# Fork choice tests + +The aim of the tests for the fork choice rules. + +## Test case format + +### `meta.yaml` + +```yaml +description: string -- Optional. Description of test case, purely for debugging purposes. +bls_setting: int -- see general test-format spec. +``` + +### `anchor_state.ssz_snappy` + +A YAML-encoded `BeaconState`, the state to initialize store with `get_forkchoice_store(anchor_state: BeaconState, anchor_block: BeaconBlock)` helper. + +### `anchor_block.ssz_snappy` + +A YAML-encoded `BeaconBlock`, the block to initialize store with `get_forkchoice_store(anchor_state: BeaconState, anchor_block: BeaconBlock)` helper. + +### `steps.yaml` + +The steps to execute in sequence. There may be multiple items of the following types: + +#### `on_tick` execution step + +The parameter that is required for executing `on_tick(store, time)`. + +```yaml +{ tick: int } -- to execute `on_tick(store, time)` +``` + +After this step, the `store` object may have been updated. + +#### `on_attestation` execution step + +The parameter that is required for executing `on_attestation(store, attestation)`. + +```yaml +{ attestation: string }: -- the name of the `attestation_<32-byte-root>.ssz_snappy` file. To execute `on_attestation(store, attestation)` with the given attestation. +``` +The file is located in the same folder (see below). + +After this step, the `store` object may have been updated. + +#### `on_block` execution step + +The parameter that is required for executing `on_block(store, block)`. + +```yaml +{ block: string }: -- the name of the `block_<32-byte-root>.ssz_snappy` file. To execute `on_block(store, block)` with the given attestation. +``` +The file is located in the same folder (see below). + +After this step, the `store` object may have been updated. + +#### Checks step + +The checks to verify the current status of `store` . + +```yaml +checks: {: value} -- the assertions. +``` + +`` is the field member of [`Store`](../../../specs/phase0/fork-choice.md#store) object that maintained by client implementation. Currently, the possible fields included: + +```yaml +time: int -- store.time +genesis_time: int -- store.genesis_time +justified_checkpoint_root: string -- store.justified_checkpoint.root +finalized_checkpoint_root: string -- store.finalized_checkpoint_root.root +best_justified_checkpoint_root: string -- store.best_justified_checkpoint_root.root +``` + +For example: +```yaml +- checks: { + justified_checkpoint_root: '0x347468b606d03f8429afd491f94e32cd3a2295c2536e808c863a9d132a521dc4', + head: '0x17aa608f5fce87592c6f02ca6ca3c49ca70b5cef5456697709b2e5894e3879c2' +} +``` + +### `attestation_<32-byte-root>.ssz_snappy` + +`<32-byte-root>` is the hash tree root of the given attestation. + +Each file is a YAML-encoded `Attestation`. + +### `block_<32-byte-root>.ssz_snappy` + +`<32-byte-root>` is the hash tree root of the given block. + +Each file is a YAML-encoded `SignedBeaconBlock`. + +## Condition + +1. Deserialize `anchor_state.ssz_snappy` and `anchor_block.ssz_snappy` to initialize the local store object by with `get_forkchoice_store(anchor_state, anchor_block)` helper. +2. Go through `steps.yaml` + - For each execution, look up the corresponding ssz_snappy file. Execute the corresponding helper function on your store. + - For each `checks` step, the assertions must be satisfied. diff --git a/tests/generators/fork_choice/README.md b/tests/generators/fork_choice/README.md index e450ca92f..e67b115ba 100644 --- a/tests/generators/fork_choice/README.md +++ b/tests/generators/fork_choice/README.md @@ -1,3 +1,5 @@ # Fork choice tests -TODO +Fork choice tests cover the different forking cases with fork choice helper functions. + +Information on the format of the tests can be found in the [fork choice test formats documentation](../../formats/fork_choice/README.md). From 9254f1bf8c61560ccf9312ce2f65e0b8688ee206 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 12 Mar 2021 00:39:38 +0800 Subject: [PATCH 186/222] Fix conflicts --- tests/generators/fork_choice/main.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/generators/fork_choice/main.py b/tests/generators/fork_choice/main.py index 79920d8ad..492e596f4 100644 --- a/tests/generators/fork_choice/main.py +++ b/tests/generators/fork_choice/main.py @@ -1,24 +1,24 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.lightclient_patch import spec as spec_lightclient_patch +from eth2spec.altair import spec as spec_altair from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH +from eth2spec.test.context import PHASE0, PHASE1, ALTAIR -specs = (spec_phase0, spec_lightclient_patch, spec_phase1) +specs = (spec_phase0, spec_altair, spec_phase1) if __name__ == "__main__": phase_0_mods = {key: 'eth2spec.test.phase0.fork_choice.test_' + key for key in [ 'get_head', ]} - # No additional lightclient_patch or phase 1 specific finality tests, yet. - lightclient_patch_mods = phase_0_mods + # No additional Altair or phase 1 specific finality tests, yet. + altair_mods = phase_0_mods phase_1_mods = phase_0_mods all_mods = { PHASE0: phase_0_mods, - LIGHTCLIENT_PATCH: lightclient_patch_mods, + ALTAIR: altair_mods, PHASE1: phase_1_mods, } From a1e74b2c18a8885dab633c85470b99e7e162afcc Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 12 Mar 2021 00:51:31 +0800 Subject: [PATCH 187/222] Fix conflicts --- .../pyspec/eth2spec/test/altair/unittests/test_helpers.py | 6 +++--- .../eth2spec/test/altair/unittests/test_sync_protocol.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/unittests/test_helpers.py b/tests/core/pyspec/eth2spec/test/altair/unittests/test_helpers.py index 48054d088..d89ff6d84 100644 --- a/tests/core/pyspec/eth2spec/test/altair/unittests/test_helpers.py +++ b/tests/core/pyspec/eth2spec/test/altair/unittests/test_helpers.py @@ -1,12 +1,12 @@ from eth2spec.test.context import ( spec_state_test, with_phases, - LIGHTCLIENT_PATCH, + ALTAIR, ) from eth2spec.test.helpers.merkle import build_proof -@with_phases([LIGHTCLIENT_PATCH]) +@with_phases([ALTAIR]) @spec_state_test def test_next_sync_committee_tree(spec, state): state.next_sync_committee: object = spec.SyncCommittee( @@ -22,7 +22,7 @@ def test_next_sync_committee_tree(spec, state): ) -@with_phases([LIGHTCLIENT_PATCH]) +@with_phases([ALTAIR]) @spec_state_test def test_finality_root_tree(spec, state): finality_branch = build_proof(state.get_backing(), spec.FINALIZED_ROOT_INDEX) diff --git a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py index d65e0ad01..4c9b98e0a 100644 --- a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py +++ b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py @@ -1,5 +1,5 @@ from eth2spec.test.context import ( - LIGHTCLIENT_PATCH, + ALTAIR, MINIMAL, spec_state_test, with_configs, @@ -20,7 +20,7 @@ from eth2spec.test.helpers.sync_committee import ( from eth2spec.test.helpers.merkle import build_proof -@with_phases([LIGHTCLIENT_PATCH]) +@with_phases([ALTAIR]) @spec_state_test def test_process_light_client_update_not_updated(spec, state): pre_snapshot = spec.LightClientSnapshot( @@ -78,7 +78,7 @@ def test_process_light_client_update_not_updated(spec, state): assert store.snapshot == pre_snapshot -@with_phases([LIGHTCLIENT_PATCH]) +@with_phases([ALTAIR]) @spec_state_test @with_configs([MINIMAL], reason="too slow") def test_process_light_client_update_timeout(spec, state): @@ -143,7 +143,7 @@ def test_process_light_client_update_timeout(spec, state): assert store.snapshot.header == update.header -@with_phases([LIGHTCLIENT_PATCH]) +@with_phases([ALTAIR]) @spec_state_test @with_configs([MINIMAL], reason="too slow") def test_process_light_client_update_finality_updated(spec, state): From 22fe06829b0accebc920d744508a93980eba54ee Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 12 Mar 2021 20:26:26 +0800 Subject: [PATCH 188/222] Refactor `objects_to_spec` --- setup.py | 59 +++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/setup.py b/setup.py index 8557f38b1..6022f9d1e 100644 --- a/setup.py +++ b/setup.py @@ -10,6 +10,12 @@ from typing import Dict, NamedTuple, List FUNCTION_REGEX = r'^def [\w_]*' +# Definitions in context.py +PHASE0 = 'phase0' +ALTAIR = 'altair' +PHASE1 = 'phase1' + + class SpecObject(NamedTuple): functions: Dict[str, str] custom_types: Dict[str, str] @@ -302,8 +308,7 @@ def get_generalized_index(ssz_class: Any, *path: Sequence[Union[int, SSZVariable ssz_path = Path(ssz_class) for item in path: ssz_path = ssz_path / item - return GeneralizedIndex(ssz_path.gindex()) -''' + return GeneralizedIndex(ssz_path.gindex())''' # The constants that depend on SSZ objects @@ -314,6 +319,18 @@ ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS = { } +def is_phase0(fork): + return fork == PHASE0 + + +def is_altair(fork): + return fork == ALTAIR + + +def is_phase1(fork): + return fork == PHASE1 + + def objects_to_spec(spec_object: SpecObject, imports: str, fork: str, ordered_class_objects: Dict[str, str]) -> str: """ Given all the objects that constitute a spec, combine them into a single pyfile. @@ -335,34 +352,32 @@ def objects_to_spec(spec_object: SpecObject, imports: str, fork: str, ordered_cl spec_object.constants[k] += " # noqa: E501" constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, spec_object.constants[x]), spec_object.constants)) ordered_class_objects_spec = '\n\n'.join(ordered_class_objects.values()) + + if is_altair(fork): + altair_ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS[x]), ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS)) + spec = ( imports + '\n\n' + f"fork = \'{fork}\'\n" + '\n\n' + new_type_definitions + '\n' + SUNDRY_CONSTANTS_FUNCTIONS - ) - - if fork == 'altair': - altair_ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS[x]), ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS)) - spec += ( - ALTAIR_SUNDRY_FUNCTIONS - + '\n\n' + altair_ssz_dep_constants - ) - - spec += ( - '\n\n' + constants_spec + # The constants that some SSZ containers require. Need to be defined before `constants_spec` + + ('\n\n' + altair_ssz_dep_constants if is_altair(fork) else '') + + '\n\n' + constants_spec + '\n\n' + CONFIG_LOADER + '\n\n' + ordered_class_objects_spec + '\n\n' + functions_spec + # Functions to make pyspec work + '\n' + PHASE0_SUNDRY_FUNCTIONS + + ('\n' + ALTAIR_SUNDRY_FUNCTIONS if is_altair(fork) else '') + + ('\n' + PHASE1_SUNDRY_FUNCTIONS if is_phase1(fork) else '') ) - if fork == 'phase1': - spec += '\n' + PHASE1_SUNDRY_FUNCTIONS - - if fork == 'altair': + # Since some contants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are + # as same as the spec definition. + if is_altair(fork): altair_ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS)) - spec += '\n\n' + altair_ssz_dep_constants_verification + spec += '\n\n\n' + altair_ssz_dep_constants_verification spec += '\n' return spec @@ -484,7 +499,7 @@ class PySpecCommand(Command): def initialize_options(self): """Set default values for options.""" # Each user option must be listed here with their default value. - self.spec_fork = 'phase0' + self.spec_fork = PHASE0 self.md_doc_paths = '' self.out_dir = 'pyspec_output' @@ -493,14 +508,14 @@ class PySpecCommand(Command): if len(self.md_doc_paths) == 0: print("no paths were specified, using default markdown file paths for pyspec" " build (spec fork: %s)" % self.spec_fork) - if self.spec_fork == "phase0": + if is_phase0(self.spec_fork): self.md_doc_paths = """ specs/phase0/beacon-chain.md specs/phase0/fork-choice.md specs/phase0/validator.md specs/phase0/weak-subjectivity.md """ - elif self.spec_fork == "phase1": + elif is_phase1(self.spec_fork): self.md_doc_paths = """ specs/phase0/beacon-chain.md specs/phase0/fork-choice.md @@ -514,7 +529,7 @@ class PySpecCommand(Command): specs/phase1/shard-fork-choice.md specs/phase1/validator.md """ - elif self.spec_fork == "altair": + elif is_altair(self.spec_fork): self.md_doc_paths = """ specs/phase0/beacon-chain.md specs/phase0/fork-choice.md From 310301236f5b292dd0cd86434fc23569fdaf51da Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 12 Mar 2021 20:29:28 +0800 Subject: [PATCH 189/222] Fix typo --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6022f9d1e..3bd64869f 100644 --- a/setup.py +++ b/setup.py @@ -373,7 +373,7 @@ def objects_to_spec(spec_object: SpecObject, imports: str, fork: str, ordered_cl + ('\n' + PHASE1_SUNDRY_FUNCTIONS if is_phase1(fork) else '') ) - # Since some contants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are + # Since some constants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are # as same as the spec definition. if is_altair(fork): altair_ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS)) From 23b28b62d07abad40aae147f477867fd4bae7856 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 12 Mar 2021 20:40:29 +0800 Subject: [PATCH 190/222] Apply suggestions from @djrtwo code review Co-authored-by: Danny Ryan --- tests/formats/fork_choice/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/formats/fork_choice/README.md b/tests/formats/fork_choice/README.md index 20f6d7d64..6710346b9 100644 --- a/tests/formats/fork_choice/README.md +++ b/tests/formats/fork_choice/README.md @@ -13,11 +13,11 @@ bls_setting: int -- see general test-format spec. ### `anchor_state.ssz_snappy` -A YAML-encoded `BeaconState`, the state to initialize store with `get_forkchoice_store(anchor_state: BeaconState, anchor_block: BeaconBlock)` helper. +An SSZ-snappy encoded `BeaconState`, the state to initialize store with `get_forkchoice_store(anchor_state: BeaconState, anchor_block: BeaconBlock)` helper. ### `anchor_block.ssz_snappy` -A YAML-encoded `BeaconBlock`, the block to initialize store with `get_forkchoice_store(anchor_state: BeaconState, anchor_block: BeaconBlock)` helper. +An SSZ-snappy encoded `BeaconBlock`, the block to initialize store with `get_forkchoice_store(anchor_state: BeaconState, anchor_block: BeaconBlock)` helper. ### `steps.yaml` @@ -85,17 +85,17 @@ For example: `<32-byte-root>` is the hash tree root of the given attestation. -Each file is a YAML-encoded `Attestation`. +Each file is an SSZ-snappy encoded `Attestation`. ### `block_<32-byte-root>.ssz_snappy` `<32-byte-root>` is the hash tree root of the given block. -Each file is a YAML-encoded `SignedBeaconBlock`. +Each file is an SSZ-snappy encoded `SignedBeaconBlock`. ## Condition 1. Deserialize `anchor_state.ssz_snappy` and `anchor_block.ssz_snappy` to initialize the local store object by with `get_forkchoice_store(anchor_state, anchor_block)` helper. -2. Go through `steps.yaml` - - For each execution, look up the corresponding ssz_snappy file. Execute the corresponding helper function on your store. - - For each `checks` step, the assertions must be satisfied. +2. Iterate sequentially through `steps.yaml` + - For each execution, look up the corresponding ssz_snappy file. Execute the corresponding helper function on the current store. + - For each `checks` step, the assertions on the current store must be satisfied. From d29926efe6913a683ba2be0c833c74f90461c9cd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 12 Mar 2021 21:05:23 +0800 Subject: [PATCH 191/222] Remove the duplicate get_head unit test --- .../unittests/fork_choice/test_get_head.py | 175 ------------------ 1 file changed, 175 deletions(-) delete mode 100644 tests/core/pyspec/eth2spec/test/phase0/unittests/fork_choice/test_get_head.py diff --git a/tests/core/pyspec/eth2spec/test/phase0/unittests/fork_choice/test_get_head.py b/tests/core/pyspec/eth2spec/test/phase0/unittests/fork_choice/test_get_head.py deleted file mode 100644 index b470ab079..000000000 --- a/tests/core/pyspec/eth2spec/test/phase0/unittests/fork_choice/test_get_head.py +++ /dev/null @@ -1,175 +0,0 @@ -from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.attestations import get_valid_attestation, next_epoch_with_attestations -from eth2spec.test.helpers.block import build_empty_block_for_next_slot -from eth2spec.test.helpers.fork_choice import ( - add_attestation_to_store, - add_block_to_store, get_anchor_root, - get_genesis_forkchoice_store, -) -from eth2spec.test.helpers.state import ( - next_epoch, - state_transition_and_sign_block, -) - - -@with_all_phases -@spec_state_test -def test_genesis(spec, state): - # Initialization - store = get_genesis_forkchoice_store(spec, state) - anchor_root = get_anchor_root(spec, state) - assert spec.get_head(store) == anchor_root - - -@with_all_phases -@spec_state_test -def test_chain_no_attestations(spec, state): - # Initialization - store = get_genesis_forkchoice_store(spec, state) - anchor_root = get_anchor_root(spec, state) - assert spec.get_head(store) == anchor_root - - # On receiving a block of `GENESIS_SLOT + 1` slot - block_1 = build_empty_block_for_next_slot(spec, state) - signed_block_1 = state_transition_and_sign_block(spec, state, block_1) - add_block_to_store(spec, store, signed_block_1) - - # On receiving a block of next epoch - block_2 = build_empty_block_for_next_slot(spec, state) - signed_block_2 = state_transition_and_sign_block(spec, state, block_2) - add_block_to_store(spec, store, signed_block_2) - - assert spec.get_head(store) == spec.hash_tree_root(block_2) - - -@with_all_phases -@spec_state_test -def test_split_tie_breaker_no_attestations(spec, state): - genesis_state = state.copy() - - # Initialization - store = get_genesis_forkchoice_store(spec, state) - anchor_root = get_anchor_root(spec, state) - assert spec.get_head(store) == anchor_root - - # block at slot 1 - block_1_state = genesis_state.copy() - block_1 = build_empty_block_for_next_slot(spec, block_1_state) - signed_block_1 = state_transition_and_sign_block(spec, block_1_state, block_1) - add_block_to_store(spec, store, signed_block_1) - - # additional block at slot 1 - block_2_state = genesis_state.copy() - block_2 = build_empty_block_for_next_slot(spec, block_2_state) - block_2.body.graffiti = b'\x42' * 32 - signed_block_2 = state_transition_and_sign_block(spec, block_2_state, block_2) - add_block_to_store(spec, store, signed_block_2) - - highest_root = max(spec.hash_tree_root(block_1), spec.hash_tree_root(block_2)) - - assert spec.get_head(store) == highest_root - - -@with_all_phases -@spec_state_test -def test_shorter_chain_but_heavier_weight(spec, state): - genesis_state = state.copy() - - # Initialization - store = get_genesis_forkchoice_store(spec, state) - anchor_root = get_anchor_root(spec, state) - assert spec.get_head(store) == anchor_root - - # build longer tree - long_state = genesis_state.copy() - for _ in range(3): - long_block = build_empty_block_for_next_slot(spec, long_state) - signed_long_block = state_transition_and_sign_block(spec, long_state, long_block) - add_block_to_store(spec, store, signed_long_block) - - # build short tree - short_state = genesis_state.copy() - short_block = build_empty_block_for_next_slot(spec, short_state) - short_block.body.graffiti = b'\x42' * 32 - signed_short_block = state_transition_and_sign_block(spec, short_state, short_block) - add_block_to_store(spec, store, signed_short_block) - - short_attestation = get_valid_attestation(spec, short_state, short_block.slot, signed=True) - add_attestation_to_store(spec, store, short_attestation) - - assert spec.get_head(store) == spec.hash_tree_root(short_block) - - -@with_all_phases -@spec_state_test -def test_filtered_block_tree(spec, state): - # Initialization - store = get_genesis_forkchoice_store(spec, state) - anchor_root = get_anchor_root(spec, state) - - # transition state past initial couple of epochs - next_epoch(spec, state) - next_epoch(spec, state) - - assert spec.get_head(store) == anchor_root - - # fill in attestations for entire epoch, justifying the recent epoch - prev_state, signed_blocks, state = next_epoch_with_attestations(spec, state, True, False) - attestations = [ - attestation for signed_block in signed_blocks - for attestation in signed_block.message.body.attestations - ] - assert state.current_justified_checkpoint.epoch > prev_state.current_justified_checkpoint.epoch - - # tick time forward and add blocks and attestations to store - current_time = state.slot * spec.SECONDS_PER_SLOT + store.genesis_time - spec.on_tick(store, current_time) - for signed_block in signed_blocks: - spec.on_block(store, signed_block) - for attestation in attestations: - spec.on_attestation(store, attestation) - - assert store.justified_checkpoint == state.current_justified_checkpoint - - # the last block in the branch should be the head - expected_head_root = spec.hash_tree_root(signed_blocks[-1].message) - assert spec.get_head(store) == expected_head_root - - # - # create branch containing the justified block but not containing enough on - # chain votes to justify that block - # - - # build a chain without attestations off of previous justified block - non_viable_state = store.block_states[store.justified_checkpoint.root].copy() - - # ensure that next wave of votes are for future epoch - next_epoch(spec, non_viable_state) - next_epoch(spec, non_viable_state) - next_epoch(spec, non_viable_state) - assert spec.get_current_epoch(non_viable_state) > store.justified_checkpoint.epoch - - # create rogue block that will be attested to in this non-viable branch - rogue_block = build_empty_block_for_next_slot(spec, non_viable_state) - signed_rogue_block = state_transition_and_sign_block(spec, non_viable_state, rogue_block) - - # create an epoch's worth of attestations for the rogue block - next_epoch(spec, non_viable_state) - attestations = [] - for i in range(spec.SLOTS_PER_EPOCH): - slot = rogue_block.slot + i - for index in range(spec.get_committee_count_per_slot(non_viable_state, spec.compute_epoch_at_slot(slot))): - attestation = get_valid_attestation(spec, non_viable_state, slot, index, signed=True) - attestations.append(attestation) - - # tick time forward to be able to include up to the latest attestation - current_time = (attestations[-1].data.slot + 1) * spec.SECONDS_PER_SLOT + store.genesis_time - spec.on_tick(store, current_time) - - # include rogue block and associated attestations in the store - spec.on_block(store, signed_rogue_block) - for attestation in attestations: - spec.on_attestation(store, attestation) - - # ensure that get_head still returns the head from the previous branch - assert spec.get_head(store) == expected_head_root From e77ba9182182d45490f54dbed60ac890ba8ced57 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 12 Mar 2021 22:13:07 +0800 Subject: [PATCH 192/222] Apply proto's feedback, fix+refactor test_get_head, fix test format doc Note that to execute on_attestation after on_block Output more checking field Disable mainnet config test_filtered_block_tree Fix after rectoring + use more run_on_block Fix and refactor `tick_and_run_on_attestation` --- .../eth2spec/test/helpers/fork_choice.py | 65 ++++++++- .../test/phase0/fork_choice/test_get_head.py | 133 +++++++----------- tests/formats/fork_choice/README.md | 38 +++-- 3 files changed, 137 insertions(+), 99 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py index 040bee975..2ca37768b 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py @@ -10,7 +10,17 @@ def get_anchor_root(spec, state): return spec.hash_tree_root(anchor_block_header) -def add_block_to_store(spec, store, signed_block, test_steps=None): +def add_block_to_store(spec, store, signed_block): + pre_state = store.block_states[signed_block.message.parent_root] + block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT + + if store.time < block_time: + spec.on_tick(store, block_time) + + spec.on_block(store, signed_block) + + +def tick_and_run_on_block(spec, store, signed_block, test_steps=None): if test_steps is None: test_steps = [] @@ -18,14 +28,12 @@ def add_block_to_store(spec, store, signed_block, test_steps=None): block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT if store.time < block_time: - spec.on_tick(store, block_time) - test_steps.append({'tick': int(block_time)}) + on_tick_and_append_step(spec, store, block_time, test_steps) - spec.on_block(store, signed_block) - test_steps.append({'block': get_block_file_name(signed_block)}) + yield from run_on_block(spec, store, signed_block, test_steps) -def add_attestation_to_store(spec, store, attestation, test_steps=None): +def tick_and_run_on_attestation(spec, store, attestation, test_steps=None): if test_steps is None: test_steps = [] @@ -39,6 +47,7 @@ def add_attestation_to_store(spec, store, attestation, test_steps=None): test_steps.append({'tick': int(next_epoch_time)}) spec.on_attestation(store, attestation) + yield get_attestation_file_name(attestation), attestation test_steps.append({'attestation': get_attestation_file_name(attestation)}) @@ -60,3 +69,47 @@ def get_block_file_name(block): def get_attestation_file_name(attestation): return f"attestation_{encode_hex(attestation.hash_tree_root())}" + + +def on_tick_and_append_step(spec, store, time, test_steps): + spec.on_tick(store, time) + test_steps.append({'tick': int(time)}) + + +def run_on_block(spec, store, signed_block, test_steps, valid=True): + if not valid: + try: + spec.on_block(store, signed_block) + + except AssertionError: + return + else: + assert False + + spec.on_block(store, signed_block) + yield get_block_file_name(signed_block), signed_block + test_steps.append({'block': get_block_file_name(signed_block)}) + + # An on_block step implies receiving block's attestations + for attestation in signed_block.message.body.attestations: + spec.on_attestation(store, attestation) + + assert store.blocks[signed_block.message.hash_tree_root()] == signed_block.message + test_steps.append({ + 'checks': { + 'time': int(store.time), + 'head': get_formatted_head_output(spec, store), + 'justified_checkpoint_root': encode_hex(store.justified_checkpoint.root), + 'finalized_checkpoint_root': encode_hex(store.finalized_checkpoint.root), + 'best_justified_checkpoint': encode_hex(store.best_justified_checkpoint.root), + } + }) + + +def get_formatted_head_output(spec, store): + head = spec.get_head(store) + slot = store.blocks[head].slot + return { + 'slot': int(slot), + 'root': encode_hex(head), + } diff --git a/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_head.py b/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_head.py index 6648000fa..318db496a 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_head.py +++ b/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_head.py @@ -1,14 +1,22 @@ from eth_utils import encode_hex -from eth2spec.test.context import MINIMAL, with_all_phases, with_configs, spec_state_test +from eth2spec.test.context import ( + MINIMAL, + is_post_altair, + spec_state_test, + with_all_phases, + with_configs, +) from eth2spec.test.helpers.attestations import get_valid_attestation, next_epoch_with_attestations from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.fork_choice import ( - add_attestation_to_store, - add_block_to_store, get_anchor_root, + tick_and_run_on_attestation, + tick_and_run_on_block, + get_anchor_root, get_genesis_forkchoice_store_and_block, - get_attestation_file_name, - get_block_file_name, + get_formatted_head_output, + on_tick_and_append_step, + run_on_block, ) from eth2spec.test.helpers.state import ( next_epoch, @@ -17,7 +25,6 @@ from eth2spec.test.helpers.state import ( @with_all_phases -@with_configs([MINIMAL]) @spec_state_test def test_genesis(spec, state): test_steps = [] @@ -27,19 +34,21 @@ def test_genesis(spec, state): yield 'anchor_block', anchor_block anchor_root = get_anchor_root(spec, state) - head = spec.get_head(store) - assert head == anchor_root + assert spec.get_head(store) == anchor_root test_steps.append({ 'checks': { - 'head': encode_hex(head) + 'genesis_time': int(store.genesis_time), + 'head': get_formatted_head_output(spec, store), } }) yield 'steps', test_steps + if is_post_altair(spec): + yield 'description', 'meta', f"Although it's not phase 0, we may use {spec.fork} spec to start testnets." + @with_all_phases -@with_configs([MINIMAL]) @spec_state_test def test_chain_no_attestations(spec, state): test_steps = [] @@ -49,31 +58,27 @@ def test_chain_no_attestations(spec, state): yield 'anchor_block', anchor_block anchor_root = get_anchor_root(spec, state) - head = spec.get_head(store) - assert head == anchor_root + assert spec.get_head(store) == anchor_root test_steps.append({ 'checks': { - 'head': encode_hex(head) + 'head': get_formatted_head_output(spec, store), } }) # On receiving a block of `GENESIS_SLOT + 1` slot block_1 = build_empty_block_for_next_slot(spec, state) signed_block_1 = state_transition_and_sign_block(spec, state, block_1) - add_block_to_store(spec, store, signed_block_1, test_steps) - yield get_block_file_name(signed_block_1), signed_block_1 + yield from tick_and_run_on_block(spec, store, signed_block_1, test_steps) # On receiving a block of next epoch block_2 = build_empty_block_for_next_slot(spec, state) signed_block_2 = state_transition_and_sign_block(spec, state, block_2) - add_block_to_store(spec, store, signed_block_2, test_steps) - yield get_block_file_name(signed_block_2), signed_block_2 + yield from tick_and_run_on_block(spec, store, signed_block_2, test_steps) - head = spec.get_head(store) - assert head == spec.hash_tree_root(block_2) + assert spec.get_head(store) == spec.hash_tree_root(block_2) test_steps.append({ 'checks': { - 'head': encode_hex(head) + 'head': get_formatted_head_output(spec, store), } }) @@ -81,7 +86,6 @@ def test_chain_no_attestations(spec, state): @with_all_phases -@with_configs([MINIMAL]) @spec_state_test def test_split_tie_breaker_no_attestations(spec, state): test_steps = [] @@ -92,11 +96,10 @@ def test_split_tie_breaker_no_attestations(spec, state): yield 'anchor_state', state yield 'anchor_block', anchor_block anchor_root = get_anchor_root(spec, state) - head = spec.get_head(store) - assert head == anchor_root + assert spec.get_head(store) == anchor_root test_steps.append({ 'checks': { - 'head': encode_hex(head) + 'head': get_formatted_head_output(spec, store), } }) @@ -104,23 +107,20 @@ def test_split_tie_breaker_no_attestations(spec, state): block_1_state = genesis_state.copy() block_1 = build_empty_block_for_next_slot(spec, block_1_state) signed_block_1 = state_transition_and_sign_block(spec, block_1_state, block_1) - add_block_to_store(spec, store, signed_block_1, test_steps) - yield get_block_file_name(signed_block_1), signed_block_1 + yield from tick_and_run_on_block(spec, store, signed_block_1, test_steps) # additional block at slot 1 block_2_state = genesis_state.copy() block_2 = build_empty_block_for_next_slot(spec, block_2_state) block_2.body.graffiti = b'\x42' * 32 signed_block_2 = state_transition_and_sign_block(spec, block_2_state, block_2) - add_block_to_store(spec, store, signed_block_2, test_steps) - yield get_block_file_name(signed_block_2), signed_block_2 + yield from tick_and_run_on_block(spec, store, signed_block_2, test_steps) highest_root = max(spec.hash_tree_root(block_1), spec.hash_tree_root(block_2)) - head = spec.get_head(store) - assert head == highest_root + assert spec.get_head(store) == highest_root test_steps.append({ 'checks': { - 'head': encode_hex(head) + 'head': get_formatted_head_output(spec, store), } }) @@ -138,11 +138,10 @@ def test_shorter_chain_but_heavier_weight(spec, state): yield 'anchor_state', state yield 'anchor_block', anchor_block anchor_root = get_anchor_root(spec, state) - head = spec.get_head(store) - assert head == anchor_root + assert spec.get_head(store) == anchor_root test_steps.append({ 'checks': { - 'head': encode_hex(head) + 'head': get_formatted_head_output(spec, store), } }) @@ -151,25 +150,22 @@ def test_shorter_chain_but_heavier_weight(spec, state): for _ in range(3): long_block = build_empty_block_for_next_slot(spec, long_state) signed_long_block = state_transition_and_sign_block(spec, long_state, long_block) - add_block_to_store(spec, store, signed_long_block, test_steps) - yield get_block_file_name(signed_long_block), signed_long_block + yield from tick_and_run_on_block(spec, store, signed_long_block, test_steps) # build short tree short_state = genesis_state.copy() short_block = build_empty_block_for_next_slot(spec, short_state) short_block.body.graffiti = b'\x42' * 32 signed_short_block = state_transition_and_sign_block(spec, short_state, short_block) - add_block_to_store(spec, store, signed_short_block, test_steps) - yield get_block_file_name(signed_short_block), signed_short_block + yield from tick_and_run_on_block(spec, store, signed_short_block, test_steps) short_attestation = get_valid_attestation(spec, short_state, short_block.slot, signed=True) - add_attestation_to_store(spec, store, short_attestation, test_steps) + yield from tick_and_run_on_attestation(spec, store, short_attestation, test_steps) - head = spec.get_head(store) - assert head == spec.hash_tree_root(short_block) + assert spec.get_head(store) == spec.hash_tree_root(short_block) test_steps.append({ 'checks': { - 'head': encode_hex(head) + 'head': get_formatted_head_output(spec, store), } }) @@ -178,59 +174,44 @@ def test_shorter_chain_but_heavier_weight(spec, state): @with_all_phases @spec_state_test +@with_configs([MINIMAL], reason="too slow") def test_filtered_block_tree(spec, state): test_steps = [] # Initialization store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state) yield 'anchor_state', state yield 'anchor_block', anchor_block - anchor_root = get_anchor_root(spec, state) + assert spec.get_head(store) == anchor_root + test_steps.append({ + 'checks': { + 'head': get_formatted_head_output(spec, store), + } + }) # transition state past initial couple of epochs next_epoch(spec, state) next_epoch(spec, state) - - head = spec.get_head(store) - assert head == anchor_root - test_steps.append({ - 'checks': { - 'head': encode_hex(head) - } - }) - # fill in attestations for entire epoch, justifying the recent epoch prev_state, signed_blocks, state = next_epoch_with_attestations(spec, state, True, False) - attestations = [ - attestation for signed_block in signed_blocks - for attestation in signed_block.message.body.attestations - ] assert state.current_justified_checkpoint.epoch > prev_state.current_justified_checkpoint.epoch # tick time forward and add blocks and attestations to store current_time = state.slot * spec.SECONDS_PER_SLOT + store.genesis_time - spec.on_tick(store, current_time) + on_tick_and_append_step(spec, store, current_time, test_steps) for signed_block in signed_blocks: - spec.on_block(store, signed_block) - test_steps.append({'block': get_block_file_name(signed_block)}) - yield get_block_file_name(signed_block), signed_block - - for attestation in attestations: - spec.on_attestation(store, attestation) - test_steps.append({'attestation': get_attestation_file_name(attestation)}) - yield get_attestation_file_name(attestation), attestation + yield from run_on_block(spec, store, signed_block, test_steps) assert store.justified_checkpoint == state.current_justified_checkpoint # the last block in the branch should be the head expected_head_root = spec.hash_tree_root(signed_blocks[-1].message) - head = spec.get_head(store) - assert head == expected_head_root + assert spec.get_head(store) == expected_head_root test_steps.append({ 'checks': { + 'head': get_formatted_head_output(spec, store), 'justified_checkpoint_root': encode_hex(store.justified_checkpoint.hash_tree_root()), - 'head': encode_hex(head), } }) @@ -263,25 +244,19 @@ def test_filtered_block_tree(spec, state): # tick time forward to be able to include up to the latest attestation current_time = (attestations[-1].data.slot + 1) * spec.SECONDS_PER_SLOT + store.genesis_time - spec.on_tick(store, current_time) - test_steps.append({'tick': int(current_time)}) + on_tick_and_append_step(spec, store, current_time, test_steps) # include rogue block and associated attestations in the store - spec.on_block(store, signed_rogue_block) - test_steps.append({'block': get_block_file_name(signed_rogue_block)}) - yield get_block_file_name(signed_rogue_block), signed_rogue_block + yield from run_on_block(spec, store, signed_rogue_block, test_steps) for attestation in attestations: - spec.on_attestation(store, attestation) - test_steps.append({'attestation': get_attestation_file_name(attestation)}) - yield get_attestation_file_name(attestation), attestation + yield from tick_and_run_on_attestation(spec, store, attestation, test_steps) # ensure that get_head still returns the head from the previous branch - head = spec.get_head(store) - assert head == expected_head_root + assert spec.get_head(store) == expected_head_root test_steps.append({ 'checks': { - 'head': encode_hex(head) + 'head': get_formatted_head_output(spec, store) } }) diff --git a/tests/formats/fork_choice/README.md b/tests/formats/fork_choice/README.md index 6710346b9..832ce9dd1 100644 --- a/tests/formats/fork_choice/README.md +++ b/tests/formats/fork_choice/README.md @@ -1,6 +1,6 @@ # Fork choice tests -The aim of the tests for the fork choice rules. +The aim of the fork choice tests is to provide test coverage of the various components of the fork choice. ## Test case format @@ -28,7 +28,7 @@ The steps to execute in sequence. There may be multiple items of the following t The parameter that is required for executing `on_tick(store, time)`. ```yaml -{ tick: int } -- to execute `on_tick(store, time)` +{ tick: int } -- to execute `on_tick(store, time)` ``` After this step, the `store` object may have been updated. @@ -38,7 +38,7 @@ After this step, the `store` object may have been updated. The parameter that is required for executing `on_attestation(store, attestation)`. ```yaml -{ attestation: string }: -- the name of the `attestation_<32-byte-root>.ssz_snappy` file. To execute `on_attestation(store, attestation)` with the given attestation. +{ attestation: string } -- the name of the `attestation_<32-byte-root>.ssz_snappy` file. To execute `on_attestation(store, attestation)` with the given attestation. ``` The file is located in the same folder (see below). @@ -49,7 +49,7 @@ After this step, the `store` object may have been updated. The parameter that is required for executing `on_block(store, block)`. ```yaml -{ block: string }: -- the name of the `block_<32-byte-root>.ssz_snappy` file. To execute `on_block(store, block)` with the given attestation. +{ block: string } -- the name of the `block_<32-byte-root>.ssz_snappy` file. To execute `on_block(store, block)` with the given attestation. ``` The file is located in the same folder (see below). @@ -57,30 +57,39 @@ After this step, the `store` object may have been updated. #### Checks step -The checks to verify the current status of `store` . +The checks to verify the current status of `store`. ```yaml -checks: {: value} -- the assertions. +checks: {: value} -- the assertions. ``` -`` is the field member of [`Store`](../../../specs/phase0/fork-choice.md#store) object that maintained by client implementation. Currently, the possible fields included: +`` is the field member or property of [`Store`](../../../specs/phase0/fork-choice.md#store) object that maintained by client implementation. Currently, the possible fields included: ```yaml +head: { -- Encoded 32-byte value from get_head(store) + slot: slot, + root: string, +} time: int -- store.time genesis_time: int -- store.genesis_time -justified_checkpoint_root: string -- store.justified_checkpoint.root -finalized_checkpoint_root: string -- store.finalized_checkpoint_root.root -best_justified_checkpoint_root: string -- store.best_justified_checkpoint_root.root +justified_checkpoint_root: string -- Encoded 32-byte value from store.justified_checkpoint.root +finalized_checkpoint_root: string -- Encoded 32-byte value from store.finalized_checkpoint.root +best_justified_checkpoint_root: string -- Encoded 32-byte value from store.best_justified_checkpoint.root ``` For example: ```yaml -- checks: { - justified_checkpoint_root: '0x347468b606d03f8429afd491f94e32cd3a2295c2536e808c863a9d132a521dc4', - head: '0x17aa608f5fce87592c6f02ca6ca3c49ca70b5cef5456697709b2e5894e3879c2' -} +- checks: + time: 144 + genesis_time: 0 + head: {slot: 17, root: '0xd2724c86002f7e1f8656ab44a341a409ad80e6e70a5225fd94835566deebb66f'} + justified_checkpoint_root: '0xcea6ecd3d3188e32ebf611f960eebd45b6c6f477a7cff242fa567a42653bfc7c' + finalized_checkpoint_root: '0xcea6ecd3d3188e32ebf611f960eebd45b6c6f477a7cff242fa567a42653bfc7c' + best_justified_checkpoint: '0xcea6ecd3d3188e32ebf611f960eebd45b6c6f477a7cff242fa567a42653bfc7c' ``` +*Note*: Each `checks` step may include one or multiple items. Each item has to be checked against the current store. + ### `attestation_<32-byte-root>.ssz_snappy` `<32-byte-root>` is the hash tree root of the given attestation. @@ -98,4 +107,5 @@ Each file is an SSZ-snappy encoded `SignedBeaconBlock`. 1. Deserialize `anchor_state.ssz_snappy` and `anchor_block.ssz_snappy` to initialize the local store object by with `get_forkchoice_store(anchor_state, anchor_block)` helper. 2. Iterate sequentially through `steps.yaml` - For each execution, look up the corresponding ssz_snappy file. Execute the corresponding helper function on the current store. + - For the `on_block` execution step: if `len(block.message.body.attestations) > 0`, execute each attestation with `on_attestation(store, attestation)` after executing `on_block(store, block)`. - For each `checks` step, the assertions on the current store must be satisfied. From 1824bdf1811c9256af5db8de9765e7491cf4cf69 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 13 Mar 2021 01:21:04 +0800 Subject: [PATCH 193/222] Update Altair configs --- configs/mainnet/altair.yaml | 11 +++++++++++ configs/minimal/altair.yaml | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/configs/mainnet/altair.yaml b/configs/mainnet/altair.yaml index 2e4839c16..d7daf6028 100644 --- a/configs/mainnet/altair.yaml +++ b/configs/mainnet/altair.yaml @@ -33,7 +33,18 @@ EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 256 DOMAIN_SYNC_COMMITTEE: 0x07000000 +# Fork +# --------------------------------------------------------------- +ALTAIR_FORK_VERSION: 0x01000000 +# TBD +ALTAIR_FORK_SLOT: 0 + + # Sync protocol # --------------------------------------------------------------- +# 1 +MIN_SYNC_COMMITTEE_PARTICIPANTS: 1 +# 2**13 +MAX_VALID_LIGHT_CLIENT_UPDATES: 8192 # 2**13 (=8192) LIGHT_CLIENT_UPDATE_TIMEOUT: 8192 diff --git a/configs/minimal/altair.yaml b/configs/minimal/altair.yaml index 80233404d..dc0491d60 100644 --- a/configs/minimal/altair.yaml +++ b/configs/minimal/altair.yaml @@ -33,7 +33,18 @@ EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 8 DOMAIN_SYNC_COMMITTEE: 0x07000000 +# Fork +# --------------------------------------------------------------- +ALTAIR_FORK_VERSION: 0x01000000 +# [customized] +ALTAIR_FORK_SLOT: 0 + + # Sync protocol # --------------------------------------------------------------- +# 1 +MIN_SYNC_COMMITTEE_PARTICIPANTS: 1 +# [customized] +MAX_VALID_LIGHT_CLIENT_UPDATES: 32 # [customized] LIGHT_CLIENT_UPDATE_TIMEOUT: 32 From 1c0238075e12ef537a3cb4c0695b430b536d9d79 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 13 Mar 2021 12:42:51 +0800 Subject: [PATCH 194/222] Fix some leftover from #2097 --- tests/core/pyspec/eth2spec/gen_helpers/README.md | 4 ++-- tests/formats/epoch_processing/README.md | 4 ++-- tests/formats/finality/README.md | 4 ++-- tests/formats/forks/README.md | 4 ++-- tests/formats/genesis/initialization.md | 4 ++-- tests/formats/genesis/validity.md | 2 +- tests/formats/operations/README.md | 6 +++--- tests/formats/rewards/README.md | 12 ++++++------ tests/formats/sanity/blocks.md | 4 ++-- tests/formats/sanity/slots.md | 8 ++++---- 10 files changed, 26 insertions(+), 26 deletions(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/README.md b/tests/core/pyspec/eth2spec/gen_helpers/README.md index d39ee66ae..5bd76b99f 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/README.md +++ b/tests/core/pyspec/eth2spec/gen_helpers/README.md @@ -43,8 +43,8 @@ The yielding pattern is: 3 value style: `yield `. Test part output kinds: -- `ssz`: value is expected to be a `bytes`, and the raw data is written to a `.ssz` file. -- `data`: value is expected to be any python object that can be dumped as YAML. Output is written to `.yaml` +- `ssz`: value is expected to be a `bytes`, and the raw data is written to a `.ssz_snappy` file. +- `data`: value is expected to be any Python object that can be dumped as YAML. Output is written to `.yaml` - `meta`: these key-value pairs are collected into a dict, and then collectively written to a metadata file named `meta.yaml`, if anything is yielded with `meta` empty. diff --git a/tests/formats/epoch_processing/README.md b/tests/formats/epoch_processing/README.md index 6469682dc..3ac2a28c4 100644 --- a/tests/formats/epoch_processing/README.md +++ b/tests/formats/epoch_processing/README.md @@ -17,11 +17,11 @@ bls_setting: int -- see general test-format spec. ### `pre.ssz_snappy` -A SSZ-snappy encoded `BeaconState`, the state before running the epoch sub-transition. +An SSZ-snappy encoded `BeaconState`, the state before running the epoch sub-transition. ### `post.ssz_snappy` -A SSZ-snappy encoded `BeaconState`, the state after applying the epoch sub-transition. +An SSZ-snappy encoded `BeaconState`, the state after applying the epoch sub-transition. ## Condition diff --git a/tests/formats/finality/README.md b/tests/formats/finality/README.md index 2d279b441..af39f5c8c 100644 --- a/tests/formats/finality/README.md +++ b/tests/formats/finality/README.md @@ -16,7 +16,7 @@ blocks_count: int -- the number of blocks processed in this test. ### `pre.ssz_snappy` -A SSZ-snappy encoded `BeaconState`, the state before running the block transitions. +An SSZ-snappy encoded `BeaconState`, the state before running the block transitions. Also available as `pre.ssz_snappy`. @@ -32,7 +32,7 @@ Each block is also available as `blocks_.ssz_snappy` ### `post.ssz_snappy` -A SSZ-snappy encoded `BeaconState`, the state after applying the block transitions. +An SSZ-snappy encoded `BeaconState`, the state after applying the block transitions. ## Condition diff --git a/tests/formats/forks/README.md b/tests/formats/forks/README.md index 2226a3f20..36ce942d7 100644 --- a/tests/formats/forks/README.md +++ b/tests/formats/forks/README.md @@ -26,11 +26,11 @@ Key of valid `fork` strings that might be found in `meta.yaml` ### `pre.ssz_snappy` -A SSZ-snappy encoded `BeaconState`, the state before running the fork transition. +An SSZ-snappy encoded `BeaconState`, the state before running the fork transition. ### `post.ssz_snappy` -A SSZ-snappy encoded `BeaconState`, the state after applying the fork transition. +An SSZ-snappy encoded `BeaconState`, the state after applying the fork transition. *Note*: This type is the `BeaconState` after the fork and is *not* the same type as `pre`. diff --git a/tests/formats/genesis/initialization.md b/tests/formats/genesis/initialization.md index 2e5452821..4b365c2ef 100644 --- a/tests/formats/genesis/initialization.md +++ b/tests/formats/genesis/initialization.md @@ -6,7 +6,7 @@ Tests the initialization of a genesis state based on Eth1 data. ### `eth1_block_hash.ssz_snappy` -A SSZ-snappy encoded root of the Eth1 block. +An SSZ-snappy encoded root of the Eth1 block. ### `eth1_timestamp.yaml` @@ -27,7 +27,7 @@ Each file is a SSZ-snappy encoded `Deposit` object. ### `state.ssz_snappy` -The expected genesis state. A SSZ-snappy encoded `BeaconState` object. +The expected genesis state. An SSZ-snappy encoded `BeaconState` object. ## Processing diff --git a/tests/formats/genesis/validity.md b/tests/formats/genesis/validity.md index 1b3f79879..f2e91222a 100644 --- a/tests/formats/genesis/validity.md +++ b/tests/formats/genesis/validity.md @@ -6,7 +6,7 @@ Tests if a genesis state is valid, i.e. if it counts as trigger to launch. ### `genesis.ssz_snappy` -A SSZ-snappy encoded `BeaconState`, the state to validate as genesis candidate. +An SSZ-snappy encoded `BeaconState`, the state to validate as genesis candidate. ### `is_valid.yaml` diff --git a/tests/formats/operations/README.md b/tests/formats/operations/README.md index ca77ab966..51d92a017 100644 --- a/tests/formats/operations/README.md +++ b/tests/formats/operations/README.md @@ -14,15 +14,15 @@ bls_setting: int -- see general test-format spec. ### `pre.ssz_snappy` -A SSZ-snappy encoded `BeaconState`, the state before applying the operation. +An SSZ-snappy encoded `BeaconState`, the state before applying the operation. ### `.ssz_snappy` -A SSZ-snappy encoded operation object, e.g. a `ProposerSlashing`, or `Deposit`. +An SSZ-snappy encoded operation object, e.g. a `ProposerSlashing`, or `Deposit`. ### `post.ssz_snappy` -A SSZ-snappy encoded `BeaconState`, the state after applying the operation. No value if operation processing is aborted. +An SSZ-snappy encoded `BeaconState`, the state after applying the operation. No value if operation processing is aborted. ## Condition diff --git a/tests/formats/rewards/README.md b/tests/formats/rewards/README.md index aee23c3e9..a6682042f 100644 --- a/tests/formats/rewards/README.md +++ b/tests/formats/rewards/README.md @@ -25,27 +25,27 @@ _Note_: No signature verification happens within rewards sub-functions. These ### `pre.ssz_snappy` -A SSZ-snappy encoded `BeaconState`, the state before running the rewards sub-function. +An SSZ-snappy encoded `BeaconState`, the state before running the rewards sub-function. ### `source_deltas.ssz_snappy` -A SSZ-snappy encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_source_deltas` function +An SSZ-snappy encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_source_deltas` function ### `target_deltas.ssz_snappy` -A SSZ-snappy encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_target_deltas` function +An SSZ-snappy encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_target_deltas` function ### `head_deltas.ssz_snappy` -A SSZ-snappy encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_head_deltas` function +An SSZ-snappy encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_head_deltas` function ### `inclusion_delay_deltas.ssz_snappy` -A SSZ-snappy encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_inclusion_delay_deltas` function +An SSZ-snappy encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_inclusion_delay_deltas` function ### `inactivity_penalty_deltas.ssz_snappy` -A SSZ-snappy encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_inactivity_penalty_deltas` function +An SSZ-snappy encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_inactivity_penalty_deltas` function ## Condition diff --git a/tests/formats/sanity/blocks.md b/tests/formats/sanity/blocks.md index a8b38ccae..7ea646b9e 100644 --- a/tests/formats/sanity/blocks.md +++ b/tests/formats/sanity/blocks.md @@ -16,7 +16,7 @@ blocks_count: int -- the number of blocks processed in this test. ### `pre.ssz_snappy` -A SSZ-snappy encoded `BeaconState`, the state before running the block transitions. +An SSZ-snappy encoded `BeaconState`, the state before running the block transitions. ### `blocks_.ssz_snappy` @@ -28,7 +28,7 @@ Each file is a SSZ-snappy encoded `SignedBeaconBlock`. ### `post.ssz_snappy` -A SSZ-snappy encoded `BeaconState`, the state after applying the block transitions. +An SSZ-snappy encoded `BeaconState`, the state after applying the block transitions. ## Condition diff --git a/tests/formats/sanity/slots.md b/tests/formats/sanity/slots.md index 72a24da51..f1b8a1321 100644 --- a/tests/formats/sanity/slots.md +++ b/tests/formats/sanity/slots.md @@ -12,9 +12,9 @@ bls_setting: int -- see general test-format spec. ``` -### `pre.yaml` +### `pre.ssz_snappy` -A YAML-encoded `BeaconState`, the state before running the transitions. +An SSZ-snappy `BeaconState`, the state before running the transitions. Also available as `pre.ssz_snappy`. @@ -23,9 +23,9 @@ Also available as `pre.ssz_snappy`. An integer. The amount of slots to process (i.e. the difference in slots between pre and post), always a positive number. -### `post.yaml` +### `post.ssz_snappy` -A YAML-encoded `BeaconState`, the state after applying the transitions. +An SSZ-snappy `BeaconState`, the state after applying the transitions. Also available as `post.ssz_snappy`. From c979e690030d22661b1d7e5c433ecf4a809683db Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 14 Mar 2021 22:54:30 +0100 Subject: [PATCH 195/222] Altair in description and titles --- specs/altair/beacon-chain.md | 2 +- specs/altair/fork.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 733411361..43c0a9710 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -1,4 +1,4 @@ -# Ethereum 2.0 Altair +# Ethereum 2.0 Altair Beacon chain changes ## Table of contents diff --git a/specs/altair/fork.md b/specs/altair/fork.md index ad157f272..e80cee351 100644 --- a/specs/altair/fork.md +++ b/specs/altair/fork.md @@ -1,4 +1,4 @@ -# Ethereum 2.0 Light Client Support -- From Phase 0 to Light Client Patch +# Ethereum 2.0 Altair fork **Notice**: This document is a work-in-progress for researchers and implementers. @@ -17,7 +17,7 @@ ## Introduction -This document describes the process of moving from Phase 0 to Phase 1 of Ethereum 2.0. +This document describes the process of the first upgrade of Ethereum 2.0: the Altair hardfork, introducing Light Client support. ## Configuration @@ -36,7 +36,7 @@ TBD. Social consensus, along with state conditions such as epoch boundary, final ### Upgrading the state -After `process_slots` of Phase 0 finishes, if `state.slot == ALTAIR_FORK_SLOT`, an irregular state change is made to upgrade to light-client patch. +After `process_slots` of Phase 0 finishes, if `state.slot == ALTAIR_FORK_SLOT`, an irregular state change is made to upgrade to Altair. ```python def upgrade_to_altair(pre: phase0.BeaconState) -> BeaconState: From e7ebd08d69c16dcb57e344c90ef48020fa2a6041 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Mon, 15 Mar 2021 11:08:41 +0000 Subject: [PATCH 196/222] Altair cosmetic cleanups plus a couple substantive changes --- configs/mainnet/altair.yaml | 10 +- configs/minimal/altair.yaml | 6 +- specs/altair/beacon-chain.md | 157 ++++++++---------- specs/altair/fork.md | 4 +- specs/phase0/beacon-chain.md | 2 +- specs/phase0/validator.md | 2 +- .../test/altair/sanity/test_blocks.py | 8 +- .../pyspec/eth2spec/test/helpers/rewards.py | 13 +- .../test_process_rewards_and_penalties.py | 10 +- .../eth2spec/test/phase0/rewards/test_leak.py | 38 ++--- 10 files changed, 112 insertions(+), 138 deletions(-) diff --git a/configs/mainnet/altair.yaml b/configs/mainnet/altair.yaml index 13b62bd7a..29a2ceeda 100644 --- a/configs/mainnet/altair.yaml +++ b/configs/mainnet/altair.yaml @@ -4,7 +4,7 @@ CONFIG_NAME: "mainnet" # Updated penalty values # --------------------------------------------------------------- -# 3 * 2**24) (= 50,331,648) +# 3 * 2**24 (= 50,331,648) ALTAIR_INACTIVITY_PENALTY_QUOTIENT: 50331648 # 2**6 (= 64) ALTAIR_MIN_SLASHING_PENALTY_QUOTIENT: 64 @@ -14,12 +14,12 @@ ALTAIR_PROPORTIONAL_SLASHING_MULTIPLIER: 2 # Misc # --------------------------------------------------------------- -# 2**10 (=1,024) +# 2**10 (= 1,024) SYNC_COMMITTEE_SIZE: 1024 -# 2**6 (=64) +# 2**6 (= 64) SYNC_SUBCOMMITTEE_SIZE: 64 -# 2**2 (=4) -LEAK_SCORE_BIAS: 4 +# 2**2 (= 4) +INACTIVITY_SCORE_BIAS: 4 # Time parameters diff --git a/configs/minimal/altair.yaml b/configs/minimal/altair.yaml index e98fee699..2e9d1f793 100644 --- a/configs/minimal/altair.yaml +++ b/configs/minimal/altair.yaml @@ -4,7 +4,7 @@ CONFIG_NAME: "minimal" # Updated penalty values # --------------------------------------------------------------- -# 3 * 2**24) (= 50,331,648) +# 3 * 2**24 (= 50,331,648) ALTAIR_INACTIVITY_PENALTY_QUOTIENT: 50331648 # 2**6 (= 64) ALTAIR_MIN_SLASHING_PENALTY_QUOTIENT: 64 @@ -18,8 +18,8 @@ ALTAIR_PROPORTIONAL_SLASHING_MULTIPLIER: 2 SYNC_COMMITTEE_SIZE: 32 # [customized] SYNC_SUBCOMMITTEE_SIZE: 16 -# 2**2 (=4) -LEAK_SCORE_BIAS: 4 +# 2**2 (= 4) +INACTIVITY_SCORE_BIAS: 4 # Time parameters diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 733411361..608637510 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -35,17 +35,17 @@ - [`get_sync_committee`](#get_sync_committee) - [`get_base_reward`](#get_base_reward) - [`get_unslashed_participating_indices`](#get_unslashed_participating_indices) - - [`get_flag_deltas`](#get_flag_deltas) - - [New `get_inactivity_penalty_deltas`](#new-get_inactivity_penalty_deltas) + - [`get_flag_index_deltas`](#get_flag_index_deltas) + - [Modified `get_inactivity_penalty_deltas`](#modified-get_inactivity_penalty_deltas) - [Beacon state mutators](#beacon-state-mutators) - - [New `slash_validator`](#new-slash_validator) + - [Modified `slash_validator`](#modified-slash_validator) - [Block processing](#block-processing) - [Modified `process_attestation`](#modified-process_attestation) - [Modified `process_deposit`](#modified-process_deposit) - [Sync committee processing](#sync-committee-processing) - [Epoch processing](#epoch-processing) - [Justification and finalization](#justification-and-finalization) - - [Leak scores](#leak-scores) + - [Inactivity scores](#inactivity-scores) - [Rewards and penalties](#rewards-and-penalties) - [Slashings](#slashings) - [Participation flags updates](#participation-flags-updates) @@ -56,20 +56,17 @@ ## Introduction -Altair is a patch implementing the first hard fork to the beacon chain. -It has four main features: +Altair is the first beacon chain hard fork. Its main features are: -* Light client support via sync committees -* Incentive accounting reforms, reducing spec complexity - and [TODO] reducing the cost of processing chains that have very little or zero participation for a long span of epochs -* Update penalty configuration values, moving them toward their planned maximally punitive configuration -* Fork choice rule changes to address weaknesses recently discovered in the existing fork choice +* sync committees to support light clients +* incentive accounting reforms to reduce spec complexity +* penalty parameter updates to move them to their planned maximally punitive configuration ## Custom types | Name | SSZ equivalent | Description | | - | - | - | -| `ParticipationFlags` | `uint8` | A succinct representation of 8 boolean participation flags | +| `ParticipationFlags` | `uint8` | a succinct representation of 8 boolean participation flags | ## Constants @@ -90,8 +87,7 @@ It has four main features: | `TIMELY_TARGET_FLAG_NUMERATOR` | `32` | | `FLAG_DENOMINATOR` | `64` | -**Note**: The participatition flag fractions add up to 7/8. -The remaining 1/8 is for proposer incentives and other future micro-incentives. +**Note**: The sum of the participatition flag fractions (7/8) plus the proposer reward fraction (1/8) equals 1. ### Misc @@ -103,23 +99,23 @@ The remaining 1/8 is for proposer incentives and other future micro-incentives. ### Updated penalty values -This patch updates a few configuration values to move penalty constants toward their final, maxmium security values. +This patch updates a few configuration values to move penalty parameters toward their final, maxmium security values. *Note*: The spec does *not* override previous configuration values but instead creates new values and replaces usage throughout. | Name | Value | | - | - | | `ALTAIR_INACTIVITY_PENALTY_QUOTIENT` | `uint64(3 * 2**24)` (= 50,331,648) | -| `ALTAIR_MIN_SLASHING_PENALTY_QUOTIENT` | `uint64(2**6)` (=64) | +| `ALTAIR_MIN_SLASHING_PENALTY_QUOTIENT` | `uint64(2**6)` (= 64) | | `ALTAIR_PROPORTIONAL_SLASHING_MULTIPLIER` | `uint64(2)` | ### Misc | Name | Value | -| - | - | +| - | - | | `SYNC_COMMITTEE_SIZE` | `uint64(2**10)` (= 1,024) | | `SYNC_SUBCOMMITTEE_SIZE` | `uint64(2**6)` (= 64) | -| `LEAK_SCORE_BIAS` | 4 | +| `INACTIVITY_SCORE_BIAS` | `uint64(4)` | ### Time parameters @@ -181,18 +177,18 @@ class BeaconState(Container): # Slashings slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR] # Per-epoch sums of slashed effective balances # Participation - previous_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT] # [New in Altair] - current_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT] # [New in Altair] + previous_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT] # [Modified in Altair] + current_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT] # [Modofied in Altair] # Finality justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH] # Bit set for every recent justified epoch previous_justified_checkpoint: Checkpoint current_justified_checkpoint: Checkpoint finalized_checkpoint: Checkpoint - # Light client sync committees + # Inactivity + inactivity_scores: List[uint64, VALIDATOR_REGISTRY_LIMIT] # [New in Altair] + # Sync current_sync_committee: SyncCommittee # [New in Altair] next_sync_committee: SyncCommittee # [New in Altair] - # Leak - leak_scores: List[uint64, VALIDATOR_REGISTRY_LIMIT] # [New in Altair] ``` ### New containers @@ -307,7 +303,7 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: ```python def get_unslashed_participating_indices(state: BeaconState, flag_index: int, epoch: Epoch) -> Set[ValidatorIndex]: """ - Retrieve the active and unslashed validator indices for the given epoch and flag index. + Return the active and unslashed validator indices for the given epoch and flag index. """ assert epoch in (get_previous_epoch(state), get_current_epoch(state)) if epoch == get_current_epoch(state): @@ -319,19 +315,17 @@ def get_unslashed_participating_indices(state: BeaconState, flag_index: int, epo return set(filter(lambda index: not state.validators[index].slashed, participating_indices)) ``` -#### `get_flag_deltas` +#### `get_flag_index_deltas` ```python -def get_flag_deltas(state: BeaconState, - flag_index: int, - numerator: uint64) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: +def get_flag_index_deltas(state: BeaconState, + flag_index: int, + numerator: uint64) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: """ - Compute the rewards and penalties associated with a particular duty, by scanning through the participation - flags to determine who participated and who did not and assigning them the appropriate rewards and penalties. + Return the deltas for a given flag index by scanning through the participation flags. """ rewards = [Gwei(0)] * len(state.validators) penalties = [Gwei(0)] * len(state.validators) - unslashed_participating_indices = get_unslashed_participating_indices(state, flag_index, get_previous_epoch(state)) increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from balances to avoid uint64 overflow unslashed_participating_increments = get_total_balance(state, unslashed_participating_indices) // increment @@ -340,19 +334,17 @@ def get_flag_deltas(state: BeaconState, base_reward = get_base_reward(state, index) if index in unslashed_participating_indices: if is_in_inactivity_leak(state): - # Optimal participation is fully rewarded to cancel the inactivity penalty - rewards[index] = base_reward * numerator // FLAG_DENOMINATOR + # This flag reward cancels the inactivity penalty corresponding to the flag index + rewards[index] += Gwei(base_reward * numerator // FLAG_DENOMINATOR) else: - rewards[index] = ( - (base_reward * numerator * unslashed_participating_increments) - // (active_increments * FLAG_DENOMINATOR) - ) + reward_numerator = base_reward * numerator * unslashed_participating_increments + rewards[index] += Gwei(reward_numerator // (active_increments * FLAG_DENOMINATOR)) else: - penalties[index] = base_reward * numerator // FLAG_DENOMINATOR + penalties[index] += Gwei(base_reward * numerator // FLAG_DENOMINATOR) return rewards, penalties ``` -#### New `get_inactivity_penalty_deltas` +#### Modified `get_inactivity_penalty_deltas` *Note*: The function `get_inactivity_penalty_deltas` is modified in the selection of matching target indices and the removal of `BASE_REWARDS_PER_EPOCH`. @@ -360,39 +352,29 @@ and the removal of `BASE_REWARDS_PER_EPOCH`. ```python def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: """ - Compute the penalties associated with the inactivity leak, by scanning through the participation - flags to determine who participated and who did not, applying the leak penalty globally and applying - compensatory rewards to participants. + Return the inactivity penalty deltas by considering timely target participation flags and inactivity scores. """ rewards = [Gwei(0) for _ in range(len(state.validators))] penalties = [Gwei(0) for _ in range(len(state.validators))] - - if not is_in_inactivity_leak(state): - return rewards, penalties - - reward_numerator_sum = sum(numerator for (_, numerator) in get_flag_indices_and_numerators()) - matching_target_attesting_indices = get_unslashed_participating_indices( - state, TIMELY_TARGET_FLAG_INDEX, get_previous_epoch(state) - ) - for index in get_eligible_validator_indices(state): - # If validator is performing optimally this cancels all attestation rewards for a neutral balance - penalties[index] += Gwei(get_base_reward(state, index) * reward_numerator_sum // FLAG_DENOMINATOR) - if index not in matching_target_attesting_indices and state.leak_scores[index] >= LEAK_SCORE_BIAS: - effective_balance = state.validators[index].effective_balance - leak_penalty = Gwei( - effective_balance * state.leak_scores[index] // LEAK_SCORE_BIAS // ALTAIR_INACTIVITY_PENALTY_QUOTIENT - ) - penalties[index] += leak_penalty - + if is_in_inactivity_leak(state): + previous_epoch = get_previous_epoch(state) + matching_target_indices = get_unslashed_participating_indices(state, TIMELY_TARGET_FLAG_INDEX, previous_epoch) + for index in get_eligible_validator_indices(state): + for (_, numerator) in get_flag_indices_and_numerators(): + # This inactivity penalty cancels the flag reward corresponding to the flag index + penalties[index] += Gwei(get_base_reward(state, index) * numerator // FLAG_DENOMINATOR) + if index not in matching_target_indices: + penalty_numerator = state.validators[index].effective_balance * state.inactivity_scores[index] + penalty_denominator = INACTIVITY_SCORE_BIAS * ALTAIR_INACTIVITY_PENALTY_QUOTIENT + penalties[index] += Gwei(penalty_numerator // penalty_denominator) return rewards, penalties ``` ### Beacon state mutators -#### New `slash_validator` +#### Modified `slash_validator` -*Note*: The function `slash_validator` is modified -with the substitution of `MIN_SLASHING_PENALTY_QUOTIENT` with `ALTAIR_MIN_SLASHING_PENALTY_QUOTIENT`. +*Note*: The function `slash_validator` is modified to use `ALTAIR_MIN_SLASHING_PENALTY_QUOTIENT`. ```python def slash_validator(state: BeaconState, @@ -483,10 +465,9 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: increase_balance(state, get_beacon_proposer_index(state), proposer_reward) ``` - #### Modified `process_deposit` -*Note*: The function `process_deposit` is modified to initialize `leak_scores`, `previous_epoch_participation`, `current_epoch_participation`. +*Note*: The function `process_deposit` is modified to initialize `inactivity_scores`, `previous_epoch_participation`, `current_epoch_participation`. ```python def process_deposit(state: BeaconState, deposit: Deposit) -> None: @@ -504,7 +485,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: pubkey = deposit.data.pubkey amount = deposit.data.amount - validator_pubkeys = [v.pubkey for v in state.validators] + validator_pubkeys = [validator.pubkey for validator in state.validators] if pubkey not in validator_pubkeys: # Verify the deposit signature (proof of possession) which is not checked by the deposit contract deposit_message = DepositMessage( @@ -514,15 +495,13 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: ) domain = compute_domain(DOMAIN_DEPOSIT) # Fork-agnostic domain since deposits are valid across forks signing_root = compute_signing_root(deposit_message, domain) - if not bls.Verify(pubkey, signing_root, deposit.data.signature): - return - - # Add validator and balance entries - state.validators.append(get_validator_from_deposit(state, deposit)) - state.balances.append(amount) - state.previous_epoch_participation.append(ParticipationFlags(0b0000_0000)) # [New in Altair] - state.current_epoch_participation.append(ParticipationFlags(0b0000_0000)) # [New in Altair] - state.leak_scores.append(0) # [New in Altair] + # Initialize validator if the deposit signature is valid + if bls.Verify(pubkey, signing_root, deposit.data.signature): + state.validators.append(get_validator_from_deposit(state, deposit)) + state.balances.append(amount) + state.previous_epoch_participation.append(ParticipationFlags(0b0000_0000)) + state.current_epoch_participation.append(ParticipationFlags(0b0000_0000)) + state.inactivity_scores.append(0) else: # Increase balance by deposit amount index = ValidatorIndex(validator_pubkeys.index(pubkey)) @@ -563,7 +542,7 @@ def process_sync_committee(state: BeaconState, body: BeaconBlockBody) -> None: ```python def process_epoch(state: BeaconState) -> None: process_justification_and_finalization(state) # [Modified in Altair] - process_leak_updates(state) # [New in Altair] + process_inactivity_updates(state) # [New in Altair] process_rewards_and_penalties(state) # [Modified in Altair] process_registry_updates(state) process_slashings(state) # [Modified in Altair] @@ -622,36 +601,32 @@ def process_justification_and_finalization(state: BeaconState) -> None: state.finalized_checkpoint = old_current_justified_checkpoint ``` -#### Leak scores +#### Inactivity scores -*Note*: The function `process_leak_updates` is new. +*Note*: The function `process_inactivity_updates` is new. ```python -def process_leak_updates(state: BeaconState) -> None: - matching_target_attesting_indices = get_unslashed_participating_indices( - state, TIMELY_TARGET_FLAG_INDEX, get_previous_epoch(state) - ) +def process_inactivity_updates(state: BeaconState) -> None: for index in get_eligible_validator_indices(state): - if index in matching_target_attesting_indices: - if state.leak_scores[index] > 0: - state.leak_scores[index] -= 1 + if index in get_unslashed_participating_indices(state, TIMELY_TARGET_FLAG_INDEX, get_previous_epoch(state)): + if state.inactivity_scores[index] > 0: + state.inactivity_scores[index] -= 1 elif is_in_inactivity_leak(state): - state.leak_scores[index] += LEAK_SCORE_BIAS + state.inactivity_scores[index] += INACTIVITY_SCORE_BIAS ``` #### Rewards and penalties -*Note*: The function `process_rewards_and_penalties` is modified to support the incentive reforms. +*Note*: The function `process_rewards_and_penalties` is modified to support the incentive accounting reforms. ```python def process_rewards_and_penalties(state: BeaconState) -> None: # No rewards are applied at the end of `GENESIS_EPOCH` because rewards are for work done in the previous epoch if get_current_epoch(state) == GENESIS_EPOCH: return - flag_deltas = [ - get_flag_deltas(state, flag_index, flag_numerator) - for (flag_index, flag_numerator) in get_flag_indices_and_numerators() - ] + + flag_indices_and_numerators = get_flag_indices_and_numerators() + flag_deltas = [get_flag_index_deltas(state, index, numerator) for (index, numerator) in flag_indices_and_numerators] deltas = flag_deltas + [get_inactivity_penalty_deltas(state)] for (rewards, penalties) in deltas: for index in range(len(state.validators)): diff --git a/specs/altair/fork.md b/specs/altair/fork.md index ad157f272..73d86a2c8 100644 --- a/specs/altair/fork.md +++ b/specs/altair/fork.md @@ -75,8 +75,8 @@ def upgrade_to_altair(pre: phase0.BeaconState) -> BeaconState: previous_justified_checkpoint=pre.previous_justified_checkpoint, current_justified_checkpoint=pre.current_justified_checkpoint, finalized_checkpoint=pre.finalized_checkpoint, - # Leak - leak_scores=[0 for _ in range(len(pre.validators))], + # Inactivity + inactivity_scores=[0 for _ in range(len(pre.validators))], ) # Fill in sync committees post.current_sync_committee = get_sync_committee(post, get_current_epoch(post)) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index a71e86a85..9fcd11852 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -257,7 +257,7 @@ The following values are (non-configurable) constants used throughout the specif | `WHISTLEBLOWER_REWARD_QUOTIENT` | `uint64(2**9)` (= 512) | | `PROPOSER_REWARD_QUOTIENT` | `uint64(2**3)` (= 8) | | `INACTIVITY_PENALTY_QUOTIENT` | `uint64(2**26)` (= 67,108,864) | -| `MIN_SLASHING_PENALTY_QUOTIENT` | `uint64(2**7)` (=128) | +| `MIN_SLASHING_PENALTY_QUOTIENT` | `uint64(2**7)` (= 128) | | `PROPORTIONAL_SLASHING_MULTIPLIER` | `uint64(1)` | - The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**13` epochs (about 36 days) is the time it takes the inactivity penalty to reduce the balance of non-participating validators to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline validators after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)`; so after `INVERSE_SQRT_E_DROP_TIME` epochs, it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. Note this value will be upgraded to `2**24` after Phase 0 mainnet stabilizes to provide a faster recovery in the event of an inactivity leak. diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 14a5ecbba..281078d20 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -647,5 +647,5 @@ A validator client should be considered standalone and should consider the beaco 1) Private keys -- private keys should be protected from being exported accidentally or by an attacker. 2) Slashing -- before a validator client signs a message it should validate the data, check it against a local slashing database (do not sign a slashable attestation or block) and update its internal slashing database with the newly signed object. 3) Recovered validator -- Recovering a validator from a private key will result in an empty local slashing db. Best practice is to import (from a trusted source) that validator's attestation history. See [EIP 3076](https://github.com/ethereum/EIPs/pull/3076/files) for a standard slashing interchange format. -4) Far future signing requests -- A validator client can be requested to sign a far into the future attestation, resulting in a valid non-slashable request. If the validator client signs this message, it will result in it blocking itself from attesting any other attestation until the beacon-chain reaches that far into the future epoch. This will result in an inactivity leak and potential ejection due to low balance. +4) Far future signing requests -- A validator client can be requested to sign a far into the future attestation, resulting in a valid non-slashable request. If the validator client signs this message, it will result in it blocking itself from attesting any other attestation until the beacon-chain reaches that far into the future epoch. This will result in an inactivity penalty and potential ejection due to low balance. A validator client should prevent itself from signing such requests by: a) keeping a local time clock if possible and following best practices to stop time server attacks and b) refusing to sign, by default, any message that has a large (>6h) gap from the current slashing protection database indicated a time "jump" or a long offline event. The administrator can manually override this protection to restart the validator after a genuine long offline event. diff --git a/tests/core/pyspec/eth2spec/test/altair/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/altair/sanity/test_blocks.py index 3689c0783..7bd2a91ba 100644 --- a/tests/core/pyspec/eth2spec/test/altair/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/altair/sanity/test_blocks.py @@ -79,12 +79,12 @@ def test_empty_sync_committee_committee_genesis(spec, state): @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test -def test_leak_scores(spec, state): +def test_inactivity_scores(spec, state): for _ in range(spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + 2): next_epoch_via_block(spec, state) assert spec.is_in_inactivity_leak(state) - previous_leak_scores = state.leak_scores.copy() + previous_inactivity_scores = state.inactivity_scores.copy() yield 'pre', state @@ -95,5 +95,5 @@ def test_leak_scores(spec, state): yield 'blocks', [signed_block] yield 'post', state - for pre, post in zip(previous_leak_scores, state.leak_scores): - assert post == pre + spec.LEAK_SCORE_BIAS + for pre, post in zip(previous_inactivity_scores, state.inactivity_scores): + assert post == pre + spec.INACTIVITY_SCORE_BIAS diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index ab4b9a282..b44611ebf 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -42,13 +42,13 @@ def run_deltas(spec, state): if is_post_altair(spec): def get_source_deltas(state): - return spec.get_flag_deltas(state, spec.TIMELY_SOURCE_FLAG_INDEX, spec.TIMELY_SOURCE_FLAG_NUMERATOR) + return spec.get_flag_index_deltas(state, spec.TIMELY_SOURCE_FLAG_INDEX, spec.TIMELY_SOURCE_FLAG_NUMERATOR) def get_head_deltas(state): - return spec.get_flag_deltas(state, spec.TIMELY_HEAD_FLAG_INDEX, spec.TIMELY_HEAD_FLAG_NUMERATOR) + return spec.get_flag_index_deltas(state, spec.TIMELY_HEAD_FLAG_INDEX, spec.TIMELY_HEAD_FLAG_NUMERATOR) def get_target_deltas(state): - return spec.get_flag_deltas(state, spec.TIMELY_TARGET_FLAG_INDEX, spec.TIMELY_TARGET_FLAG_NUMERATOR) + return spec.get_flag_index_deltas(state, spec.TIMELY_TARGET_FLAG_INDEX, spec.TIMELY_TARGET_FLAG_NUMERATOR) yield from run_attestation_component_deltas( spec, @@ -191,7 +191,6 @@ def run_get_inactivity_penalty_deltas(spec, state): matching_attesting_indices = spec.get_unslashed_participating_indices( state, spec.TIMELY_TARGET_FLAG_INDEX, spec.get_previous_epoch(state) ) - reward_numerator_sum = sum(numerator for (_, numerator) in spec.get_flag_indices_and_numerators()) eligible_indices = spec.get_eligible_validator_indices(state) for index in range(len(state.validators)): @@ -207,7 +206,7 @@ def run_get_inactivity_penalty_deltas(spec, state): base_reward = spec.get_base_reward(state, index) base_penalty = cancel_base_rewards_per_epoch * base_reward - spec.get_proposer_reward(state, index) else: - base_penalty = spec.get_base_reward(state, index) * reward_numerator_sum // spec.FLAG_DENOMINATOR + base_penalty = sum(spec.get_base_reward(state, index) * numerator // spec.FLAG_DENOMINATOR for (_, numerator) in spec.get_flag_indices_and_numerators()) if not has_enough_for_reward(spec, state, index): assert penalties[index] == 0 @@ -221,7 +220,7 @@ def run_get_inactivity_penalty_deltas(spec, state): def transition_state_to_leak(spec, state, epochs=None): if epochs is None: - # +1 to trigger leak_score transitions + # +1 to trigger inactivity_score transitions epochs = spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + 1 assert epochs >= spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY @@ -232,7 +231,7 @@ def transition_state_to_leak(spec, state, epochs=None): _cache_dict = LRU(size=10) -def leaking(epochs=None): +def inactivity_penalty_active(epochs=None): def deco(fn): def entry(*args, spec, state, **kw): diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py index f7c9b394e..2ca20f047 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py @@ -17,7 +17,7 @@ from eth2spec.test.helpers.attestations import ( sign_attestation, prepare_state_with_attestations, ) -from eth2spec.test.helpers.rewards import leaking +from eth2spec.test.helpers.rewards import inactivity_penalty_active from eth2spec.test.helpers.attester_slashings import get_indexed_attestation_participants from eth2spec.test.helpers.epoch_processing import run_epoch_processing_with from random import Random @@ -205,7 +205,7 @@ def test_almost_empty_attestations(spec, state): @with_all_phases @spec_state_test -@leaking() +@inactivity_penalty_active() def test_almost_empty_attestations_with_leak(spec, state): rng = Random(1234) yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, 1)) @@ -220,7 +220,7 @@ def test_random_fill_attestations(spec, state): @with_all_phases @spec_state_test -@leaking() +@inactivity_penalty_active() def test_random_fill_attestations_with_leak(spec, state): rng = Random(4567) yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, len(comm) // 3)) @@ -235,7 +235,7 @@ def test_almost_full_attestations(spec, state): @with_all_phases @spec_state_test -@leaking() +@inactivity_penalty_active() def test_almost_full_attestations_with_leak(spec, state): rng = Random(8901) yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, len(comm) - 1)) @@ -249,7 +249,7 @@ def test_full_attestation_participation(spec, state): @with_all_phases @spec_state_test -@leaking() +@inactivity_penalty_active() def test_full_attestation_participation_with_leak(spec, state): yield from run_with_participation(spec, state, lambda slot, comm_index, comm: comm) diff --git a/tests/core/pyspec/eth2spec/test/phase0/rewards/test_leak.py b/tests/core/pyspec/eth2spec/test/phase0/rewards/test_leak.py index b2ed6f5d8..4ded4050f 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/rewards/test_leak.py +++ b/tests/core/pyspec/eth2spec/test/phase0/rewards/test_leak.py @@ -1,81 +1,81 @@ from eth2spec.test.context import PHASE0, PHASE1, with_all_phases, with_phases, spec_state_test -from eth2spec.test.helpers.rewards import leaking +from eth2spec.test.helpers.rewards import inactivity_penalty_active import eth2spec.test.helpers.rewards as rewards_helpers @with_all_phases @spec_state_test -@leaking() +@inactivity_penalty_active() def test_empty_leak(spec, state): yield from rewards_helpers.run_test_empty(spec, state) @with_all_phases @spec_state_test -@leaking() +@inactivity_penalty_active() def test_full_leak(spec, state): yield from rewards_helpers.run_test_full_all_correct(spec, state) @with_all_phases @spec_state_test -@leaking() +@inactivity_penalty_active() def test_half_full_leak(spec, state): yield from rewards_helpers.run_test_half_full(spec, state) @with_all_phases @spec_state_test -@leaking() +@inactivity_penalty_active() def test_quarter_full_leak(spec, state): yield from rewards_helpers.run_test_partial(spec, state, 0.25) @with_all_phases @spec_state_test -@leaking() +@inactivity_penalty_active() def test_full_but_partial_participation_leak(spec, state): yield from rewards_helpers.run_test_full_but_partial_participation(spec, state) @with_phases([PHASE0, PHASE1]) @spec_state_test -@leaking() +@inactivity_penalty_active() def test_one_attestation_one_correct_leak(spec, state): yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state) @with_all_phases @spec_state_test -@leaking() +@inactivity_penalty_active() def test_with_not_yet_activated_validators_leak(spec, state): yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state) @with_all_phases @spec_state_test -@leaking() +@inactivity_penalty_active() def test_with_exited_validators_leak(spec, state): yield from rewards_helpers.run_test_with_exited_validators(spec, state) @with_all_phases @spec_state_test -@leaking() +@inactivity_penalty_active() def test_with_slashed_validators_leak(spec, state): yield from rewards_helpers.run_test_with_slashed_validators(spec, state) @with_all_phases @spec_state_test -@leaking() +@inactivity_penalty_active() def test_some_very_low_effective_balances_that_attested_leak(spec, state): yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested(spec, state) @with_all_phases @spec_state_test -@leaking() +@inactivity_penalty_active() def test_some_very_low_effective_balances_that_did_not_attest_leak(spec, state): yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest(spec, state) @@ -89,7 +89,7 @@ def test_some_very_low_effective_balances_that_did_not_attest_leak(spec, state): @with_phases([PHASE0, PHASE1]) @spec_state_test -@leaking() +@inactivity_penalty_active() def test_full_half_correct_target_incorrect_head_leak(spec, state): yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, @@ -101,7 +101,7 @@ def test_full_half_correct_target_incorrect_head_leak(spec, state): @with_phases([PHASE0, PHASE1]) @spec_state_test -@leaking() +@inactivity_penalty_active() def test_full_correct_target_incorrect_head_leak(spec, state): yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, @@ -113,7 +113,7 @@ def test_full_correct_target_incorrect_head_leak(spec, state): @with_phases([PHASE0, PHASE1]) @spec_state_test -@leaking() +@inactivity_penalty_active() def test_full_half_incorrect_target_incorrect_head_leak(spec, state): yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, @@ -125,7 +125,7 @@ def test_full_half_incorrect_target_incorrect_head_leak(spec, state): @with_phases([PHASE0, PHASE1]) @spec_state_test -@leaking() +@inactivity_penalty_active() def test_full_half_incorrect_target_correct_head_leak(spec, state): yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, @@ -137,20 +137,20 @@ def test_full_half_incorrect_target_correct_head_leak(spec, state): @with_all_phases @spec_state_test -@leaking() +@inactivity_penalty_active() def test_full_random_leak(spec, state): yield from rewards_helpers.run_test_full_random(spec, state) @with_all_phases @spec_state_test -@leaking(epochs=5) +@inactivity_penalty_active(epochs=5) def test_full_random_five_epoch_leak(spec, state): yield from rewards_helpers.run_test_full_random(spec, state) @with_all_phases @spec_state_test -@leaking(epochs=10) +@inactivity_penalty_active(epochs=10) def test_full_random_ten_epoch_leak(spec, state): yield from rewards_helpers.run_test_full_random(spec, state) From a2268a432c62e7ecb02cc9729e843514885b2f3a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 Mar 2021 19:20:36 +0800 Subject: [PATCH 197/222] Fix missing --cov-report --- Makefile | 2 +- setup.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 97da705ef..951237808 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ install_test: test: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python3 -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov=eth2spec.altair.spec -cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec + python3 -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov=eth2spec.altair.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec find_test: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ diff --git a/setup.py b/setup.py index 3bd64869f..3b48143dc 100644 --- a/setup.py +++ b/setup.py @@ -539,7 +539,6 @@ class PySpecCommand(Command): specs/altair/fork.md specs/altair/sync-protocol.md """ - # TODO: add specs/altair/sync-protocol.md back when the GeneralizedIndex helpers are included. else: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) From 5a16f99302f32f3b9fe101b6741a6c7efd0df491 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Mon, 15 Mar 2021 11:21:05 +0000 Subject: [PATCH 198/222] Fix line too long --- tests/core/pyspec/eth2spec/test/helpers/rewards.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index b44611ebf..3b5bd0549 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -201,12 +201,15 @@ def run_get_inactivity_penalty_deltas(spec, state): if spec.is_in_inactivity_leak(state): # Compute base_penalty + base_reward = spec.get_base_reward(state, index) if not is_post_altair(spec): cancel_base_rewards_per_epoch = spec.BASE_REWARDS_PER_EPOCH - base_reward = spec.get_base_reward(state, index) base_penalty = cancel_base_rewards_per_epoch * base_reward - spec.get_proposer_reward(state, index) else: - base_penalty = sum(spec.get_base_reward(state, index) * numerator // spec.FLAG_DENOMINATOR for (_, numerator) in spec.get_flag_indices_and_numerators()) + base_penalty = sum( + base_reward * numerator // spec.FLAG_DENOMINATOR + for (_, numerator) in spec.get_flag_indices_and_numerators() + ) if not has_enough_for_reward(spec, state, index): assert penalties[index] == 0 From 8e815dd3f76024bc248c52352dc0a9a0abc91337 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Mon, 15 Mar 2021 11:25:59 +0000 Subject: [PATCH 199/222] revert rename of to --- .../pyspec/eth2spec/test/helpers/rewards.py | 2 +- .../test_process_rewards_and_penalties.py | 10 ++--- .../eth2spec/test/phase0/rewards/test_leak.py | 38 +++++++++---------- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 3b5bd0549..8d34500b6 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -234,7 +234,7 @@ def transition_state_to_leak(spec, state, epochs=None): _cache_dict = LRU(size=10) -def inactivity_penalty_active(epochs=None): +def leaking(epochs=None): def deco(fn): def entry(*args, spec, state, **kw): diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py index 2ca20f047..f7c9b394e 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py @@ -17,7 +17,7 @@ from eth2spec.test.helpers.attestations import ( sign_attestation, prepare_state_with_attestations, ) -from eth2spec.test.helpers.rewards import inactivity_penalty_active +from eth2spec.test.helpers.rewards import leaking from eth2spec.test.helpers.attester_slashings import get_indexed_attestation_participants from eth2spec.test.helpers.epoch_processing import run_epoch_processing_with from random import Random @@ -205,7 +205,7 @@ def test_almost_empty_attestations(spec, state): @with_all_phases @spec_state_test -@inactivity_penalty_active() +@leaking() def test_almost_empty_attestations_with_leak(spec, state): rng = Random(1234) yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, 1)) @@ -220,7 +220,7 @@ def test_random_fill_attestations(spec, state): @with_all_phases @spec_state_test -@inactivity_penalty_active() +@leaking() def test_random_fill_attestations_with_leak(spec, state): rng = Random(4567) yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, len(comm) // 3)) @@ -235,7 +235,7 @@ def test_almost_full_attestations(spec, state): @with_all_phases @spec_state_test -@inactivity_penalty_active() +@leaking() def test_almost_full_attestations_with_leak(spec, state): rng = Random(8901) yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, len(comm) - 1)) @@ -249,7 +249,7 @@ def test_full_attestation_participation(spec, state): @with_all_phases @spec_state_test -@inactivity_penalty_active() +@leaking() def test_full_attestation_participation_with_leak(spec, state): yield from run_with_participation(spec, state, lambda slot, comm_index, comm: comm) diff --git a/tests/core/pyspec/eth2spec/test/phase0/rewards/test_leak.py b/tests/core/pyspec/eth2spec/test/phase0/rewards/test_leak.py index 4ded4050f..b2ed6f5d8 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/rewards/test_leak.py +++ b/tests/core/pyspec/eth2spec/test/phase0/rewards/test_leak.py @@ -1,81 +1,81 @@ from eth2spec.test.context import PHASE0, PHASE1, with_all_phases, with_phases, spec_state_test -from eth2spec.test.helpers.rewards import inactivity_penalty_active +from eth2spec.test.helpers.rewards import leaking import eth2spec.test.helpers.rewards as rewards_helpers @with_all_phases @spec_state_test -@inactivity_penalty_active() +@leaking() def test_empty_leak(spec, state): yield from rewards_helpers.run_test_empty(spec, state) @with_all_phases @spec_state_test -@inactivity_penalty_active() +@leaking() def test_full_leak(spec, state): yield from rewards_helpers.run_test_full_all_correct(spec, state) @with_all_phases @spec_state_test -@inactivity_penalty_active() +@leaking() def test_half_full_leak(spec, state): yield from rewards_helpers.run_test_half_full(spec, state) @with_all_phases @spec_state_test -@inactivity_penalty_active() +@leaking() def test_quarter_full_leak(spec, state): yield from rewards_helpers.run_test_partial(spec, state, 0.25) @with_all_phases @spec_state_test -@inactivity_penalty_active() +@leaking() def test_full_but_partial_participation_leak(spec, state): yield from rewards_helpers.run_test_full_but_partial_participation(spec, state) @with_phases([PHASE0, PHASE1]) @spec_state_test -@inactivity_penalty_active() +@leaking() def test_one_attestation_one_correct_leak(spec, state): yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state) @with_all_phases @spec_state_test -@inactivity_penalty_active() +@leaking() def test_with_not_yet_activated_validators_leak(spec, state): yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state) @with_all_phases @spec_state_test -@inactivity_penalty_active() +@leaking() def test_with_exited_validators_leak(spec, state): yield from rewards_helpers.run_test_with_exited_validators(spec, state) @with_all_phases @spec_state_test -@inactivity_penalty_active() +@leaking() def test_with_slashed_validators_leak(spec, state): yield from rewards_helpers.run_test_with_slashed_validators(spec, state) @with_all_phases @spec_state_test -@inactivity_penalty_active() +@leaking() def test_some_very_low_effective_balances_that_attested_leak(spec, state): yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested(spec, state) @with_all_phases @spec_state_test -@inactivity_penalty_active() +@leaking() def test_some_very_low_effective_balances_that_did_not_attest_leak(spec, state): yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest(spec, state) @@ -89,7 +89,7 @@ def test_some_very_low_effective_balances_that_did_not_attest_leak(spec, state): @with_phases([PHASE0, PHASE1]) @spec_state_test -@inactivity_penalty_active() +@leaking() def test_full_half_correct_target_incorrect_head_leak(spec, state): yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, @@ -101,7 +101,7 @@ def test_full_half_correct_target_incorrect_head_leak(spec, state): @with_phases([PHASE0, PHASE1]) @spec_state_test -@inactivity_penalty_active() +@leaking() def test_full_correct_target_incorrect_head_leak(spec, state): yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, @@ -113,7 +113,7 @@ def test_full_correct_target_incorrect_head_leak(spec, state): @with_phases([PHASE0, PHASE1]) @spec_state_test -@inactivity_penalty_active() +@leaking() def test_full_half_incorrect_target_incorrect_head_leak(spec, state): yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, @@ -125,7 +125,7 @@ def test_full_half_incorrect_target_incorrect_head_leak(spec, state): @with_phases([PHASE0, PHASE1]) @spec_state_test -@inactivity_penalty_active() +@leaking() def test_full_half_incorrect_target_correct_head_leak(spec, state): yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, @@ -137,20 +137,20 @@ def test_full_half_incorrect_target_correct_head_leak(spec, state): @with_all_phases @spec_state_test -@inactivity_penalty_active() +@leaking() def test_full_random_leak(spec, state): yield from rewards_helpers.run_test_full_random(spec, state) @with_all_phases @spec_state_test -@inactivity_penalty_active(epochs=5) +@leaking(epochs=5) def test_full_random_five_epoch_leak(spec, state): yield from rewards_helpers.run_test_full_random(spec, state) @with_all_phases @spec_state_test -@inactivity_penalty_active(epochs=10) +@leaking(epochs=10) def test_full_random_ten_epoch_leak(spec, state): yield from rewards_helpers.run_test_full_random(spec, state) From 178a99480495ba7c5aca2ec693d6ef42321f9eac Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 15 Mar 2021 07:33:14 -0600 Subject: [PATCH 200/222] Apply suggestions from code review Co-authored-by: Hsiao-Wei Wang --- specs/altair/fork.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/altair/fork.md b/specs/altair/fork.md index e80cee351..4d6c0c88a 100644 --- a/specs/altair/fork.md +++ b/specs/altair/fork.md @@ -17,7 +17,7 @@ ## Introduction -This document describes the process of the first upgrade of Ethereum 2.0: the Altair hardfork, introducing Light Client support. +This document describes the process of the first upgrade of Ethereum 2.0: the Altair hard fork, introducing light client support and other improvements. ## Configuration From 2a0e0eb1194b260318d5ef9b9e68b75c386f78ce Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 15 Mar 2021 08:42:50 -0600 Subject: [PATCH 201/222] fix typo --- specs/altair/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 608637510..249c730dc 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -178,7 +178,7 @@ class BeaconState(Container): slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR] # Per-epoch sums of slashed effective balances # Participation previous_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT] # [Modified in Altair] - current_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT] # [Modofied in Altair] + current_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT] # [Modified in Altair] # Finality justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH] # Bit set for every recent justified epoch previous_justified_checkpoint: Checkpoint From b4c1520bc2a0799c680a21a592b0c46a9332e352 Mon Sep 17 00:00:00 2001 From: Justin Date: Mon, 15 Mar 2021 15:02:57 +0000 Subject: [PATCH 202/222] Minor formatting fix --- specs/altair/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index a90ca9155..5c266a81c 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -87,7 +87,7 @@ Altair is the first beacon chain hard fork. Its main features are: | `TIMELY_TARGET_FLAG_NUMERATOR` | `32` | | `FLAG_DENOMINATOR` | `64` | -**Note**: The sum of the participatition flag fractions (7/8) plus the proposer reward fraction (1/8) equals 1. +*Note*: The sum of the participatition flag fractions (7/8) plus the proposer reward fraction (1/8) equals 1. ### Misc From 32d0c8087f6a0558749fa14f9338f060c4edeb8b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 Mar 2021 23:17:59 +0800 Subject: [PATCH 203/222] Fix `NEXT_SYNC_COMMITTEE_INDEX` --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3b48143dc..ef6c65289 100644 --- a/setup.py +++ b/setup.py @@ -315,7 +315,7 @@ def get_generalized_index(ssz_class: Any, *path: Sequence[Union[int, SSZVariable # Will verify the value at the end of the spec ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS = { 'FINALIZED_ROOT_INDEX': 'GeneralizedIndex(105)', - 'NEXT_SYNC_COMMITTEE_INDEX': 'GeneralizedIndex(54)', + 'NEXT_SYNC_COMMITTEE_INDEX': 'GeneralizedIndex(55)', } From 78211a3649a0d5229be1fdb5789d4de8ddbffa89 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 Mar 2021 23:52:33 +0800 Subject: [PATCH 204/222] Enable Altair genesis tests --- .../phase0/genesis/test_initialization.py | 31 +++++++++++++++---- .../test/phase0/genesis/test_validity.py | 31 +++++++++++++++---- 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/phase0/genesis/test_initialization.py b/tests/core/pyspec/eth2spec/test/phase0/genesis/test_initialization.py index b39f5ecfd..0382f4af1 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/genesis/test_initialization.py +++ b/tests/core/pyspec/eth2spec/test/phase0/genesis/test_initialization.py @@ -1,14 +1,21 @@ -from eth2spec.test.context import PHASE0, spec_test, with_phases, single_phase +from eth2spec.test.context import PHASE0, ALTAIR, spec_test, with_phases, single_phase, is_post_altair from eth2spec.test.helpers.deposits import ( prepare_full_genesis_deposits, prepare_random_genesis_deposits, ) -@with_phases(([PHASE0])) +def get_post_altair_description(spec): + return f"Although it's not phase 0, we may use {spec.fork} spec to start testnets." + + +@with_phases(([PHASE0, ALTAIR])) @spec_test @single_phase def test_initialize_beacon_state_from_eth1(spec): + if is_post_altair(spec): + yield 'description', 'meta', get_post_altair_description(spec) + deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT deposits, deposit_root, _ = prepare_full_genesis_deposits( spec, @@ -38,10 +45,13 @@ def test_initialize_beacon_state_from_eth1(spec): yield 'state', state -@with_phases([PHASE0]) +@with_phases([PHASE0, ALTAIR]) @spec_test @single_phase def test_initialize_beacon_state_some_small_balances(spec): + if is_post_altair(spec): + yield 'description', 'meta', get_post_altair_description(spec) + main_deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT main_deposits, _, deposit_data_list = prepare_full_genesis_deposits( spec, spec.MAX_EFFECTIVE_BALANCE, @@ -79,10 +89,13 @@ def test_initialize_beacon_state_some_small_balances(spec): yield 'state', state -@with_phases([PHASE0]) +@with_phases([PHASE0, ALTAIR]) @spec_test @single_phase def test_initialize_beacon_state_one_topup_activation(spec): + if is_post_altair(spec): + yield 'description', 'meta', get_post_altair_description(spec) + # Submit all but one deposit as MAX_EFFECTIVE_BALANCE main_deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT - 1 main_deposits, _, deposit_data_list = prepare_full_genesis_deposits( @@ -125,10 +138,13 @@ def test_initialize_beacon_state_one_topup_activation(spec): yield 'state', state -@with_phases([PHASE0]) +@with_phases([PHASE0, ALTAIR]) @spec_test @single_phase def test_initialize_beacon_state_random_invalid_genesis(spec): + if is_post_altair(spec): + yield 'description', 'meta', get_post_altair_description(spec) + # Make a bunch of random deposits deposits, _, deposit_data_list = prepare_random_genesis_deposits( spec, @@ -149,10 +165,13 @@ def test_initialize_beacon_state_random_invalid_genesis(spec): yield 'state', state -@with_phases([PHASE0]) +@with_phases([PHASE0, ALTAIR]) @spec_test @single_phase def test_initialize_beacon_state_random_valid_genesis(spec): + if is_post_altair(spec): + yield 'description', 'meta', get_post_altair_description(spec) + # Make a bunch of random deposits random_deposits, _, deposit_data_list = prepare_random_genesis_deposits( spec, diff --git a/tests/core/pyspec/eth2spec/test/phase0/genesis/test_validity.py b/tests/core/pyspec/eth2spec/test/phase0/genesis/test_validity.py index 28cd3544b..da420b9bd 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/genesis/test_validity.py +++ b/tests/core/pyspec/eth2spec/test/phase0/genesis/test_validity.py @@ -1,9 +1,13 @@ -from eth2spec.test.context import PHASE0, spec_test, with_phases, single_phase +from eth2spec.test.context import PHASE0, ALTAIR, spec_test, with_phases, single_phase, is_post_altair from eth2spec.test.helpers.deposits import ( prepare_full_genesis_deposits, ) +def get_post_altair_description(spec): + return f"Although it's not phase 0, we may use {spec.fork} spec to start testnets." + + def create_valid_beacon_state(spec): deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT deposits, _, _ = prepare_full_genesis_deposits( @@ -30,39 +34,51 @@ def run_is_valid_genesis_state(spec, state, valid=True): assert is_valid == valid -@with_phases([PHASE0]) +@with_phases([PHASE0, ALTAIR]) @spec_test @single_phase def test_is_valid_genesis_state_true(spec): + if is_post_altair(spec): + yield 'description', 'meta', get_post_altair_description(spec) + state = create_valid_beacon_state(spec) yield from run_is_valid_genesis_state(spec, state, valid=True) -@with_phases([PHASE0]) +@with_phases([PHASE0, ALTAIR]) @spec_test @single_phase def test_is_valid_genesis_state_false_invalid_timestamp(spec): + if is_post_altair(spec): + yield 'description', 'meta', get_post_altair_description(spec) + state = create_valid_beacon_state(spec) state.genesis_time = spec.MIN_GENESIS_TIME - 1 yield from run_is_valid_genesis_state(spec, state, valid=False) -@with_phases([PHASE0]) +@with_phases([PHASE0, ALTAIR]) @spec_test @single_phase def test_is_valid_genesis_state_true_more_balance(spec): + if is_post_altair(spec): + yield 'description', 'meta', get_post_altair_description(spec) + state = create_valid_beacon_state(spec) state.validators[0].effective_balance = spec.MAX_EFFECTIVE_BALANCE + 1 yield from run_is_valid_genesis_state(spec, state, valid=True) -@with_phases([PHASE0]) +@with_phases([PHASE0, ALTAIR]) @spec_test @single_phase def test_is_valid_genesis_state_true_one_more_validator(spec): + if is_post_altair(spec): + yield 'description', 'meta', get_post_altair_description(spec) + deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT + 1 deposits, _, _ = prepare_full_genesis_deposits( spec, @@ -78,10 +94,13 @@ def test_is_valid_genesis_state_true_one_more_validator(spec): yield from run_is_valid_genesis_state(spec, state, valid=True) -@with_phases([PHASE0]) +@with_phases([PHASE0, ALTAIR]) @spec_test @single_phase def test_is_valid_genesis_state_false_not_enough_validator(spec): + if is_post_altair(spec): + yield 'description', 'meta', get_post_altair_description(spec) + deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT - 1 deposits, _, _ = prepare_full_genesis_deposits( spec, From d590eebd36a5e4457ab9e94512bc703ffa7ecfeb Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 Mar 2021 23:53:13 +0800 Subject: [PATCH 205/222] `SpecLightclient` -> `SpecAltair` --- tests/core/pyspec/eth2spec/test/context.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index d631bb4d2..ddb01adcb 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -58,7 +58,7 @@ class SpecPhase1(Spec): ... -class SpecLightclient(Spec): +class SpecAltair(Spec): ... @@ -66,7 +66,7 @@ class SpecLightclient(Spec): class SpecForks(TypedDict, total=False): PHASE0: SpecPhase0 PHASE1: SpecPhase1 - ALTAIR: SpecLightclient + ALTAIR: SpecAltair def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Callable[[Any], int], From 40fa73a311314c0abd8e47c5dc069ee09bd39842 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 16 Mar 2021 00:05:13 +0800 Subject: [PATCH 206/222] Fork to Altair --- specs/altair/fork.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/altair/fork.md b/specs/altair/fork.md index b40464653..1fb87d554 100644 --- a/specs/altair/fork.md +++ b/specs/altair/fork.md @@ -9,7 +9,7 @@ - [Introduction](#introduction) - [Configuration](#configuration) -- [Fork to Light-client patch](#fork-to-light-client-patch) +- [Fork to Altair](#fork-to-altair) - [Fork trigger](#fork-trigger) - [Upgrading the state](#upgrading-the-state) @@ -28,7 +28,7 @@ Warning: this configuration is not definitive. | `ALTAIR_FORK_VERSION` | `Version('0x01000000')` | | `ALTAIR_FORK_SLOT` | `Slot(0)` **TBD** | -## Fork to Light-client patch +## Fork to Altair ### Fork trigger From 1a4bbdfd791ba6afdc0ca1584d3c5ecf15de6031 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 16 Mar 2021 00:16:27 +0800 Subject: [PATCH 207/222] Disable `test_sync_committees_progress` + mainnet config --- .../epoch_processing/test_process_sync_committee_updates.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py b/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py index 06473c645..0cf1e7a49 100644 --- a/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py +++ b/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py @@ -1,7 +1,9 @@ from eth2spec.test.context import ( PHASE0, PHASE1, - with_all_phases_except, + MINIMAL, spec_state_test, + with_all_phases_except, + with_configs, ) from eth2spec.test.helpers.state import transition_to from eth2spec.test.helpers.epoch_processing import ( @@ -11,6 +13,7 @@ from eth2spec.test.helpers.epoch_processing import ( @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test +@with_configs([MINIMAL], reason="too slow") def test_sync_committees_progress(spec, state): current_epoch = spec.get_current_epoch(state) # NOTE: if not in the genesis epoch, period math below needs to be From 5d9f4b072c4f97ccdf3bc0f75b2528b6e880829c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 16 Mar 2021 00:38:30 +0800 Subject: [PATCH 208/222] Update genesis testgen and format --- tests/formats/genesis/initialization.md | 1 + tests/formats/genesis/validity.md | 8 ++++ tests/generators/genesis/main.py | 51 +++++++++---------------- 3 files changed, 28 insertions(+), 32 deletions(-) diff --git a/tests/formats/genesis/initialization.md b/tests/formats/genesis/initialization.md index 4b365c2ef..7a791dcfa 100644 --- a/tests/formats/genesis/initialization.md +++ b/tests/formats/genesis/initialization.md @@ -17,6 +17,7 @@ An integer. The timestamp of the block, in seconds. A yaml file to help read the deposit count: ```yaml +description: string -- Optional. Description of test case, purely for debugging purposes. deposits_count: int -- Amount of deposits. ``` diff --git a/tests/formats/genesis/validity.md b/tests/formats/genesis/validity.md index f2e91222a..15236c3ba 100644 --- a/tests/formats/genesis/validity.md +++ b/tests/formats/genesis/validity.md @@ -4,6 +4,14 @@ Tests if a genesis state is valid, i.e. if it counts as trigger to launch. ## Test case format +### `meta.yaml` + +A yaml file to help read the deposit count: + +```yaml +description: string -- Optional. Description of test case, purely for debugging purposes. +``` + ### `genesis.ssz_snappy` An SSZ-snappy encoded `BeaconState`, the state to validate as genesis candidate. diff --git a/tests/generators/genesis/main.py b/tests/generators/genesis/main.py index 854af6572..4fede298f 100644 --- a/tests/generators/genesis/main.py +++ b/tests/generators/genesis/main.py @@ -1,37 +1,24 @@ -from typing import Iterable - -from eth2spec.test.context import PHASE0 -from eth2spec.test.phase0.genesis import test_initialization, test_validity - -from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing -from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests -from eth2spec.phase0 import spec as spec -from importlib import reload -from eth2spec.config import config_util -from eth2spec.utils import bls +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators +from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.altair import spec as spec_altair +from eth2spec.phase1 import spec as spec_phase1 +from eth2spec.test.context import PHASE0, PHASE1, ALTAIR -def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: - - def prepare_fn(configs_path: str) -> str: - config_util.prepare_config(configs_path, config_name) - reload(spec) - bls.use_milagro() - return config_name - - def cases_fn() -> Iterable[gen_typing.TestCase]: - return generate_from_tests( - runner_name='genesis', - handler_name=handler_name, - src=tests_src, - fork_name=PHASE0, - ) - - return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) +specs = (spec_phase0, spec_altair, spec_phase1) if __name__ == "__main__": - gen_runner.run_generator("genesis", [ - create_provider('initialization', test_initialization, 'minimal'), - create_provider('validity', test_validity, 'minimal'), - ]) + phase_0_mods = {key: 'eth2spec.test.phase0.genesis.test_' + key for key in [ + 'initialization', + 'validity', + ]} + altair_mods = phase_0_mods + phase_1_mods = phase_0_mods + all_mods = { + PHASE0: phase_0_mods, + ALTAIR: altair_mods, + PHASE1: phase_1_mods, + } + + run_state_test_generators(runner_name="genesis", specs=specs, all_mods=all_mods) From 734863a6d61390e719fc581a3c1150856dbccb59 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 16 Mar 2021 00:46:57 +0800 Subject: [PATCH 209/222] Skip mainnet genesis tests --- .../test/phase0/genesis/test_initialization.py | 14 +++++++++++++- .../eth2spec/test/phase0/genesis/test_validity.py | 14 +++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/phase0/genesis/test_initialization.py b/tests/core/pyspec/eth2spec/test/phase0/genesis/test_initialization.py index 0382f4af1..939152309 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/genesis/test_initialization.py +++ b/tests/core/pyspec/eth2spec/test/phase0/genesis/test_initialization.py @@ -1,4 +1,11 @@ -from eth2spec.test.context import PHASE0, ALTAIR, spec_test, with_phases, single_phase, is_post_altair +from eth2spec.test.context import ( + PHASE0, ALTAIR, MINIMAL, + is_post_altair, + single_phase, + spec_test, + with_configs, + with_phases, +) from eth2spec.test.helpers.deposits import ( prepare_full_genesis_deposits, prepare_random_genesis_deposits, @@ -12,6 +19,7 @@ def get_post_altair_description(spec): @with_phases(([PHASE0, ALTAIR])) @spec_test @single_phase +@with_configs([MINIMAL], reason="too slow") def test_initialize_beacon_state_from_eth1(spec): if is_post_altair(spec): yield 'description', 'meta', get_post_altair_description(spec) @@ -48,6 +56,7 @@ def test_initialize_beacon_state_from_eth1(spec): @with_phases([PHASE0, ALTAIR]) @spec_test @single_phase +@with_configs([MINIMAL], reason="too slow") def test_initialize_beacon_state_some_small_balances(spec): if is_post_altair(spec): yield 'description', 'meta', get_post_altair_description(spec) @@ -92,6 +101,7 @@ def test_initialize_beacon_state_some_small_balances(spec): @with_phases([PHASE0, ALTAIR]) @spec_test @single_phase +@with_configs([MINIMAL], reason="too slow") def test_initialize_beacon_state_one_topup_activation(spec): if is_post_altair(spec): yield 'description', 'meta', get_post_altair_description(spec) @@ -141,6 +151,7 @@ def test_initialize_beacon_state_one_topup_activation(spec): @with_phases([PHASE0, ALTAIR]) @spec_test @single_phase +@with_configs([MINIMAL], reason="too slow") def test_initialize_beacon_state_random_invalid_genesis(spec): if is_post_altair(spec): yield 'description', 'meta', get_post_altair_description(spec) @@ -168,6 +179,7 @@ def test_initialize_beacon_state_random_invalid_genesis(spec): @with_phases([PHASE0, ALTAIR]) @spec_test @single_phase +@with_configs([MINIMAL], reason="too slow") def test_initialize_beacon_state_random_valid_genesis(spec): if is_post_altair(spec): yield 'description', 'meta', get_post_altair_description(spec) diff --git a/tests/core/pyspec/eth2spec/test/phase0/genesis/test_validity.py b/tests/core/pyspec/eth2spec/test/phase0/genesis/test_validity.py index da420b9bd..dcddf2d54 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/genesis/test_validity.py +++ b/tests/core/pyspec/eth2spec/test/phase0/genesis/test_validity.py @@ -1,4 +1,11 @@ -from eth2spec.test.context import PHASE0, ALTAIR, spec_test, with_phases, single_phase, is_post_altair +from eth2spec.test.context import ( + PHASE0, ALTAIR, MINIMAL, + is_post_altair, + spec_test, + single_phase, + with_configs, + with_phases, +) from eth2spec.test.helpers.deposits import ( prepare_full_genesis_deposits, ) @@ -37,6 +44,7 @@ def run_is_valid_genesis_state(spec, state, valid=True): @with_phases([PHASE0, ALTAIR]) @spec_test @single_phase +@with_configs([MINIMAL], reason="too slow") def test_is_valid_genesis_state_true(spec): if is_post_altair(spec): yield 'description', 'meta', get_post_altair_description(spec) @@ -49,6 +57,7 @@ def test_is_valid_genesis_state_true(spec): @with_phases([PHASE0, ALTAIR]) @spec_test @single_phase +@with_configs([MINIMAL], reason="too slow") def test_is_valid_genesis_state_false_invalid_timestamp(spec): if is_post_altair(spec): yield 'description', 'meta', get_post_altair_description(spec) @@ -62,6 +71,7 @@ def test_is_valid_genesis_state_false_invalid_timestamp(spec): @with_phases([PHASE0, ALTAIR]) @spec_test @single_phase +@with_configs([MINIMAL], reason="too slow") def test_is_valid_genesis_state_true_more_balance(spec): if is_post_altair(spec): yield 'description', 'meta', get_post_altair_description(spec) @@ -75,6 +85,7 @@ def test_is_valid_genesis_state_true_more_balance(spec): @with_phases([PHASE0, ALTAIR]) @spec_test @single_phase +@with_configs([MINIMAL], reason="too slow") def test_is_valid_genesis_state_true_one_more_validator(spec): if is_post_altair(spec): yield 'description', 'meta', get_post_altair_description(spec) @@ -97,6 +108,7 @@ def test_is_valid_genesis_state_true_one_more_validator(spec): @with_phases([PHASE0, ALTAIR]) @spec_test @single_phase +@with_configs([MINIMAL], reason="too slow") def test_is_valid_genesis_state_false_not_enough_validator(spec): if is_post_altair(spec): yield 'description', 'meta', get_post_altair_description(spec) From 64dbcdce35979dbadb08ce952cd86db339bb2f1a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 16 Mar 2021 01:10:18 +0800 Subject: [PATCH 210/222] Use @with_all_phases --- .../test/phase0/genesis/test_initialization.py | 14 +++++++------- .../eth2spec/test/phase0/genesis/test_validity.py | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/phase0/genesis/test_initialization.py b/tests/core/pyspec/eth2spec/test/phase0/genesis/test_initialization.py index 939152309..1ec40c56f 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/genesis/test_initialization.py +++ b/tests/core/pyspec/eth2spec/test/phase0/genesis/test_initialization.py @@ -1,10 +1,10 @@ from eth2spec.test.context import ( - PHASE0, ALTAIR, MINIMAL, + MINIMAL, is_post_altair, single_phase, spec_test, with_configs, - with_phases, + with_all_phases, ) from eth2spec.test.helpers.deposits import ( prepare_full_genesis_deposits, @@ -16,7 +16,7 @@ def get_post_altair_description(spec): return f"Although it's not phase 0, we may use {spec.fork} spec to start testnets." -@with_phases(([PHASE0, ALTAIR])) +@with_all_phases @spec_test @single_phase @with_configs([MINIMAL], reason="too slow") @@ -53,7 +53,7 @@ def test_initialize_beacon_state_from_eth1(spec): yield 'state', state -@with_phases([PHASE0, ALTAIR]) +@with_all_phases @spec_test @single_phase @with_configs([MINIMAL], reason="too slow") @@ -98,7 +98,7 @@ def test_initialize_beacon_state_some_small_balances(spec): yield 'state', state -@with_phases([PHASE0, ALTAIR]) +@with_all_phases @spec_test @single_phase @with_configs([MINIMAL], reason="too slow") @@ -148,7 +148,7 @@ def test_initialize_beacon_state_one_topup_activation(spec): yield 'state', state -@with_phases([PHASE0, ALTAIR]) +@with_all_phases @spec_test @single_phase @with_configs([MINIMAL], reason="too slow") @@ -176,7 +176,7 @@ def test_initialize_beacon_state_random_invalid_genesis(spec): yield 'state', state -@with_phases([PHASE0, ALTAIR]) +@with_all_phases @spec_test @single_phase @with_configs([MINIMAL], reason="too slow") diff --git a/tests/core/pyspec/eth2spec/test/phase0/genesis/test_validity.py b/tests/core/pyspec/eth2spec/test/phase0/genesis/test_validity.py index dcddf2d54..a7bcaa6d0 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/genesis/test_validity.py +++ b/tests/core/pyspec/eth2spec/test/phase0/genesis/test_validity.py @@ -1,10 +1,10 @@ from eth2spec.test.context import ( - PHASE0, ALTAIR, MINIMAL, + MINIMAL, is_post_altair, spec_test, single_phase, with_configs, - with_phases, + with_all_phases, ) from eth2spec.test.helpers.deposits import ( prepare_full_genesis_deposits, @@ -41,7 +41,7 @@ def run_is_valid_genesis_state(spec, state, valid=True): assert is_valid == valid -@with_phases([PHASE0, ALTAIR]) +@with_all_phases @spec_test @single_phase @with_configs([MINIMAL], reason="too slow") @@ -54,7 +54,7 @@ def test_is_valid_genesis_state_true(spec): yield from run_is_valid_genesis_state(spec, state, valid=True) -@with_phases([PHASE0, ALTAIR]) +@with_all_phases @spec_test @single_phase @with_configs([MINIMAL], reason="too slow") @@ -68,7 +68,7 @@ def test_is_valid_genesis_state_false_invalid_timestamp(spec): yield from run_is_valid_genesis_state(spec, state, valid=False) -@with_phases([PHASE0, ALTAIR]) +@with_all_phases @spec_test @single_phase @with_configs([MINIMAL], reason="too slow") @@ -82,7 +82,7 @@ def test_is_valid_genesis_state_true_more_balance(spec): yield from run_is_valid_genesis_state(spec, state, valid=True) -@with_phases([PHASE0, ALTAIR]) +@with_all_phases @spec_test @single_phase @with_configs([MINIMAL], reason="too slow") @@ -105,7 +105,7 @@ def test_is_valid_genesis_state_true_one_more_validator(spec): yield from run_is_valid_genesis_state(spec, state, valid=True) -@with_phases([PHASE0, ALTAIR]) +@with_all_phases @spec_test @single_phase @with_configs([MINIMAL], reason="too slow") From 4afb6533a2e895f49140506770d057b5ae0a753d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 15 Mar 2021 11:46:53 -0600 Subject: [PATCH 211/222] suffix ALTAIR on update constants --- configs/mainnet/altair.yaml | 6 +++--- configs/minimal/altair.yaml | 6 +++--- specs/altair/beacon-chain.md | 16 ++++++++-------- .../eth2spec/test/helpers/proposer_slashings.py | 2 +- .../epoch_processing/test_process_slashings.py | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/configs/mainnet/altair.yaml b/configs/mainnet/altair.yaml index ee3785f14..f30e9fcdc 100644 --- a/configs/mainnet/altair.yaml +++ b/configs/mainnet/altair.yaml @@ -5,11 +5,11 @@ CONFIG_NAME: "mainnet" # Updated penalty values # --------------------------------------------------------------- # 3 * 2**24 (= 50,331,648) -ALTAIR_INACTIVITY_PENALTY_QUOTIENT: 50331648 +INACTIVITY_PENALTY_QUOTIENT_ALTAIR: 50331648 # 2**6 (= 64) -ALTAIR_MIN_SLASHING_PENALTY_QUOTIENT: 64 +MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR: 64 # 2 -ALTAIR_PROPORTIONAL_SLASHING_MULTIPLIER: 2 +PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR: 2 # Misc diff --git a/configs/minimal/altair.yaml b/configs/minimal/altair.yaml index a70c0404b..afdaf9eb5 100644 --- a/configs/minimal/altair.yaml +++ b/configs/minimal/altair.yaml @@ -5,11 +5,11 @@ CONFIG_NAME: "minimal" # Updated penalty values # --------------------------------------------------------------- # 3 * 2**24 (= 50,331,648) -ALTAIR_INACTIVITY_PENALTY_QUOTIENT: 50331648 +INACTIVITY_PENALTY_QUOTIENT_ALTAIR: 50331648 # 2**6 (= 64) -ALTAIR_MIN_SLASHING_PENALTY_QUOTIENT: 64 +MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR: 64 # 2 -ALTAIR_PROPORTIONAL_SLASHING_MULTIPLIER: 2 +PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR: 2 # Misc diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 5c266a81c..9b6428148 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -105,9 +105,9 @@ This patch updates a few configuration values to move penalty parameters toward | Name | Value | | - | - | -| `ALTAIR_INACTIVITY_PENALTY_QUOTIENT` | `uint64(3 * 2**24)` (= 50,331,648) | -| `ALTAIR_MIN_SLASHING_PENALTY_QUOTIENT` | `uint64(2**6)` (= 64) | -| `ALTAIR_PROPORTIONAL_SLASHING_MULTIPLIER` | `uint64(2)` | +| `INACTIVITY_PENALTY_QUOTIENT_ALTAIR` | `uint64(3 * 2**24)` (= 50,331,648) | +| `MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR` | `uint64(2**6)` (= 64) | +| `PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR` | `uint64(2)` | ### Misc @@ -365,7 +365,7 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S penalties[index] += Gwei(get_base_reward(state, index) * numerator // FLAG_DENOMINATOR) if index not in matching_target_indices: penalty_numerator = state.validators[index].effective_balance * state.inactivity_scores[index] - penalty_denominator = INACTIVITY_SCORE_BIAS * ALTAIR_INACTIVITY_PENALTY_QUOTIENT + penalty_denominator = INACTIVITY_SCORE_BIAS * INACTIVITY_PENALTY_QUOTIENT_ALTAIR penalties[index] += Gwei(penalty_numerator // penalty_denominator) return rewards, penalties ``` @@ -374,7 +374,7 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S #### Modified `slash_validator` -*Note*: The function `slash_validator` is modified to use `ALTAIR_MIN_SLASHING_PENALTY_QUOTIENT`. +*Note*: The function `slash_validator` is modified to use `MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR`. ```python def slash_validator(state: BeaconState, @@ -389,7 +389,7 @@ def slash_validator(state: BeaconState, validator.slashed = True validator.withdrawable_epoch = max(validator.withdrawable_epoch, Epoch(epoch + EPOCHS_PER_SLASHINGS_VECTOR)) state.slashings[epoch % EPOCHS_PER_SLASHINGS_VECTOR] += validator.effective_balance - decrease_balance(state, slashed_index, validator.effective_balance // ALTAIR_MIN_SLASHING_PENALTY_QUOTIENT) + decrease_balance(state, slashed_index, validator.effective_balance // MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR) # Apply proposer and whistleblower rewards proposer_index = get_beacon_proposer_index(state) @@ -636,13 +636,13 @@ def process_rewards_and_penalties(state: BeaconState) -> None: #### Slashings -*Note*: The function `process_slashings` is modified to use `ALTAIR_PROPORTIONAL_SLASHING_MULTIPLIER`. +*Note*: The function `process_slashings` is modified to use `PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR`. ```python def process_slashings(state: BeaconState) -> None: epoch = get_current_epoch(state) total_balance = get_total_active_balance(state) - adjusted_total_slashing_balance = min(sum(state.slashings) * ALTAIR_PROPORTIONAL_SLASHING_MULTIPLIER, total_balance) + adjusted_total_slashing_balance = min(sum(state.slashings) * PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR, total_balance) for index, validator in enumerate(state.validators): if validator.slashed and epoch + EPOCHS_PER_SLASHINGS_VECTOR // 2 == validator.withdrawable_epoch: increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from penalty numerator to avoid uint64 overflow diff --git a/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py b/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py index 97d09b141..d3520e580 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py +++ b/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -6,7 +6,7 @@ from eth2spec.test.helpers.state import get_balance def get_min_slashing_penalty_quotient(spec): if is_post_altair(spec): - return spec.ALTAIR_MIN_SLASHING_PENALTY_QUOTIENT + return spec.MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR else: return spec.MIN_SLASHING_PENALTY_QUOTIENT diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py index b31f6fb81..b7ae3cf4e 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py @@ -25,7 +25,7 @@ def slash_validators(spec, state, indices, out_epochs): def get_slashing_multiplier(spec): if is_post_altair(spec): - return spec.ALTAIR_PROPORTIONAL_SLASHING_MULTIPLIER + return spec.PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR else: return spec.PROPORTIONAL_SLASHING_MULTIPLIER From 06eed71856ad4a838ff3d7831f4df29fb06a086e Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 15 Mar 2021 13:10:19 -0600 Subject: [PATCH 212/222] fix altair test generators --- .../eth2spec/test/altair/fork/test_fork.py | 4 ++++ .../pyspec/eth2spec/test/helpers/rewards.py | 22 ++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/fork/test_fork.py b/tests/core/pyspec/eth2spec/test/altair/fork/test_fork.py index 9667c2e37..fb056b72a 100644 --- a/tests/core/pyspec/eth2spec/test/altair/fork/test_fork.py +++ b/tests/core/pyspec/eth2spec/test/altair/fork/test_fork.py @@ -1,7 +1,9 @@ from eth2spec.test.context import ( PHASE0, ALTAIR, + MINIMAL, with_phases, with_custom_state, + with_configs, spec_test, with_state, low_balances, misc_balances, large_validator_set, ) @@ -106,6 +108,8 @@ def test_fork_random_misc_balances(spec, phases, state): @with_phases(phases=[PHASE0], other_phases=[ALTAIR]) +@with_configs([MINIMAL], + reason="mainnet config leads to larger validator set than limit of public/private keys pre-generated") @with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test @with_meta_tags(ALTAIR_FORK_TEST_META_TAGS) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 8d34500b6..9360392d4 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -27,6 +27,26 @@ def has_enough_for_reward(spec, state, index): ) +def has_enough_for_leak_penalty(spec, state, index): + """ + Check if effective_balance and state of leak is high enough for a leak penalty. + + At very low balances / leak values, it is possible for a validator have a positive effective_balance + and be in a leak, but have zero leak penalty. + """ + + if is_post_altair(spec): + return ( + state.validators[index].effective_balance * state.inactivity_scores[index] + > spec.INACTIVITY_SCORE_BIAS * spec.INACTIVITY_PENALTY_QUOTIENT_ALTAIR + ) + else: + return ( + state.validators[index].effective_balance * spec.get_finality_delay(state) + > spec.INACTIVITY_PENALTY_QUOTIENT + ) + + def run_deltas(spec, state): """ Run all deltas functions yielding: @@ -213,7 +233,7 @@ def run_get_inactivity_penalty_deltas(spec, state): if not has_enough_for_reward(spec, state, index): assert penalties[index] == 0 - elif index in matching_attesting_indices: + elif index in matching_attesting_indices or not has_enough_for_leak_penalty(spec, state, index): assert penalties[index] == base_penalty else: assert penalties[index] > base_penalty From 1eaa3c174238a2c45bcc5e97dd62ac7a8900f96c Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 15 Mar 2021 21:43:49 +0100 Subject: [PATCH 213/222] Define SyncAggregate to bundle sync committee bits and signature, update tests to better isolate the state-change, introduce helper function for future tests, and update test doc --- specs/altair/beacon-chain.md | 24 ++- .../test_process_sync_committee.py | 184 +++++++++--------- .../test/altair/sanity/test_blocks.py | 14 +- .../pyspec/eth2spec/test/helpers/block.py | 2 +- .../eth2spec/test/helpers/block_processing.py | 60 ++++++ tests/formats/operations/README.md | 17 +- 6 files changed, 188 insertions(+), 113 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/helpers/block_processing.py diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 5c266a81c..38090cf11 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -22,6 +22,7 @@ - [`BeaconBlockBody`](#beaconblockbody) - [`BeaconState`](#beaconstate) - [New containers](#new-containers) + - [`SyncAggregate`](#syncaggregate) - [`SyncCommittee`](#synccommittee) - [Helper functions](#helper-functions) - [`Predicates`](#predicates) @@ -146,9 +147,8 @@ class BeaconBlockBody(Container): attestations: List[Attestation, MAX_ATTESTATIONS] deposits: List[Deposit, MAX_DEPOSITS] voluntary_exits: List[SignedVoluntaryExit, MAX_VOLUNTARY_EXITS] - # Sync committee aggregate signature - sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE] # [New in Altair] - sync_committee_signature: BLSSignature # [New in Altair] + # [New in Altair] + sync_aggregate: SyncAggregate ``` #### `BeaconState` @@ -193,6 +193,14 @@ class BeaconState(Container): ### New containers +#### `SyncAggregate` + +```python +class SyncAggregate(Container): + sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE] + sync_committee_signature: BLSSignature +``` + #### `SyncCommittee` ```python @@ -409,7 +417,7 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_randao(state, block.body) process_eth1_data(state, block.body) process_operations(state, block.body) # [Modified in Altair] - process_sync_committee(state, block.body) # [New in Altair] + process_sync_committee(state, block.body.sync_aggregate) # [New in Altair] ``` #### Modified `process_attestation` @@ -511,16 +519,16 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: #### Sync committee processing ```python -def process_sync_committee(state: BeaconState, body: BeaconBlockBody) -> None: +def process_sync_committee(state: BeaconState, aggregate: SyncAggregate) -> None: # Verify sync committee aggregate signature signing over the previous slot block root previous_slot = Slot(max(int(state.slot), 1) - 1) committee_indices = get_sync_committee_indices(state, get_current_epoch(state)) - participant_indices = [index for index, bit in zip(committee_indices, body.sync_committee_bits) if bit] + participant_indices = [index for index, bit in zip(committee_indices, aggregate.sync_committee_bits) if bit] committee_pubkeys = state.current_sync_committee.pubkeys - participant_pubkeys = [pubkey for pubkey, bit in zip(committee_pubkeys, body.sync_committee_bits) if bit] + participant_pubkeys = [pubkey for pubkey, bit in zip(committee_pubkeys, aggregate.sync_committee_bits) if bit] domain = get_domain(state, DOMAIN_SYNC_COMMITTEE, compute_epoch_at_slot(previous_slot)) signing_root = compute_signing_root(get_block_root_at_slot(state, previous_slot), domain) - assert eth2_fast_aggregate_verify(participant_pubkeys, signing_root, body.sync_committee_signature) + assert eth2_fast_aggregate_verify(participant_pubkeys, signing_root, aggregate.sync_committee_signature) # Reward sync committee participants proposer_rewards = Gwei(0) diff --git a/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py index 430c59b10..58cf48314 100644 --- a/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py @@ -2,8 +2,8 @@ from collections import Counter import random from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, - transition_unsigned_block, ) +from eth2spec.test.helpers.block_processing import run_block_processing_to from eth2spec.test.helpers.state import ( state_transition_and_sign_block, transition_to, @@ -23,12 +23,29 @@ from eth2spec.test.context import ( from eth2spec.utils.hash_function import hash +def run_sync_committee_processing(spec, state, block, expect_exception=False): + """ + Processes everything up to the sync committee work, then runs the sync committee work in isolation, and + produces a pre-state and post-state (None if exception) specifically for sync-committee processing changes. + """ + # process up to the sync committee work + call = run_block_processing_to(spec, state, block, 'process_sync_committee') + yield 'pre', state + yield 'sync_aggregate', block.body.sync_aggregate + if expect_exception: + expect_assertion_error(lambda: call(state, block)) + yield 'post', None + else: + call(state, block) + yield 'post', state + + def get_committee_indices(spec, state, duplicates=False): - ''' + """ This utility function allows the caller to ensure there are or are not duplicate validator indices in the returned committee based on the boolean ``duplicates``. - ''' + """ state = state.copy() current_epoch = spec.get_current_epoch(state) randao_index = current_epoch % spec.EPOCHS_PER_HISTORICAL_VECTOR @@ -45,6 +62,7 @@ def get_committee_indices(spec, state, duplicates=False): @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test +@always_bls def test_invalid_signature_missing_participant(spec, state): committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) random_participant = random.choice(committee) @@ -53,40 +71,38 @@ def test_invalid_signature_missing_participant(spec, state): block = build_empty_block_for_next_slot(spec, state) # Exclude one participant whose signature was included. - block.body.sync_committee_bits = [index != random_participant for index in committee] - block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( - spec, - state, - block.slot - 1, - committee, # full committee signs + block.body.sync_aggregate = spec.SyncAggregate( + sync_committee_bits=[index != random_participant for index in committee], + sync_committee_signature=compute_aggregate_sync_committee_signature( + spec, + state, + block.slot - 1, + committee, # full committee signs + ) ) - - yield 'blocks', [block] - expect_assertion_error(lambda: spec.process_sync_committee(state, block.body)) - yield 'post', None + yield from run_sync_committee_processing(spec, state, block, expect_exception=True) @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test +@always_bls def test_invalid_signature_extra_participant(spec, state): committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) random_participant = random.choice(committee) - yield 'pre', state - block = build_empty_block_for_next_slot(spec, state) # Exclude one signature even though the block claims the entire committee participated. - block.body.sync_committee_bits = [True] * len(committee) - block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( - spec, - state, - block.slot - 1, - [index for index in committee if index != random_participant], + block.body.sync_aggregate = spec.SyncAggregate( + sync_committee_bits=[True] * len(committee), + sync_committee_signature=compute_aggregate_sync_committee_signature( + spec, + state, + block.slot - 1, + [index for index in committee if index != random_participant], + ) ) - yield 'blocks', [block] - expect_assertion_error(lambda: spec.process_sync_committee(state, block.body)) - yield 'post', None + yield from run_sync_committee_processing(spec, state, block, expect_exception=True) def compute_sync_committee_participant_reward(spec, state, participant_index, active_validator_count, committee_size): @@ -113,18 +129,17 @@ def test_sync_committee_rewards_nonduplicate_committee(spec, state): pre_balances = state.balances.copy() block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = [True] * committee_size - block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( - spec, - state, - block.slot - 1, - committee, + block.body.sync_aggregate = spec.SyncAggregate( + sync_committee_bits=[True] * committee_size, + sync_committee_signature=compute_aggregate_sync_committee_signature( + spec, + state, + block.slot - 1, + committee, + ) ) - signed_block = state_transition_and_sign_block(spec, state, block) - - yield 'blocks', [signed_block] - yield 'post', state + yield from run_sync_committee_processing(spec, state, block) for index in range(len(state.validators)): expected_reward = 0 @@ -156,23 +171,18 @@ def test_sync_committee_rewards_duplicate_committee(spec, state): assert active_validator_count < spec.SYNC_COMMITTEE_SIZE assert committee_size > len(set(committee)) - yield 'pre', state - pre_balances = state.balances.copy() - block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = [True] * committee_size - block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( - spec, - state, - block.slot - 1, - committee, + block.body.sync_aggregate = spec.SyncAggregate( + sync_committee_bits=[True] * committee_size, + sync_committee_signature=compute_aggregate_sync_committee_signature( + spec, + state, + block.slot - 1, + committee, + ) ) - - signed_block = state_transition_and_sign_block(spec, state, block) - - yield 'blocks', [signed_block] - yield 'post', state + yield from run_sync_committee_processing(spec, state, block) multiplicities = Counter(committee) @@ -201,19 +211,19 @@ def test_sync_committee_rewards_duplicate_committee(spec, state): def test_invalid_signature_past_block(spec, state): committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) - yield 'pre', state - blocks = [] for _ in range(2): # NOTE: need to transition twice to move beyond the degenerate case at genesis block = build_empty_block_for_next_slot(spec, state) # Valid sync committee signature here... - block.body.sync_committee_bits = [True] * len(committee) - block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( - spec, - state, - block.slot - 1, - committee, + block.body.sync_aggregate = spec.SyncAggregate( + sync_committee_bits=[True] * len(committee), + sync_committee_signature=compute_aggregate_sync_committee_signature( + spec, + state, + block.slot - 1, + committee, + ) ) signed_block = state_transition_and_sign_block(spec, state, block) @@ -221,19 +231,17 @@ def test_invalid_signature_past_block(spec, state): invalid_block = build_empty_block_for_next_slot(spec, state) # Invalid signature from a slot other than the previous - invalid_block.body.sync_committee_bits = [True] * len(committee) - invalid_block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( - spec, - state, - invalid_block.slot - 2, - committee, + invalid_block.body.sync_aggregate = spec.SyncAggregate( + sync_committee_bits=[True] * len(committee), + sync_committee_signature=compute_aggregate_sync_committee_signature( + spec, + state, + invalid_block.slot - 2, + committee, + ) ) - blocks.append(invalid_block) - expect_assertion_error(lambda: transition_unsigned_block(spec, state, invalid_block)) - - yield 'blocks', blocks - yield 'post', None + yield from run_sync_committee_processing(spec, state, invalid_block, expect_exception=True) @with_all_phases_except([PHASE0, PHASE1]) @@ -253,8 +261,6 @@ def test_invalid_signature_previous_committee(spec, state): slot_in_future_sync_committee_period = epoch_in_future_sync_commitee_period * spec.SLOTS_PER_EPOCH transition_to(spec, state, slot_in_future_sync_committee_period) - yield 'pre', state - # Use the previous sync committee to produce the signature. pubkeys = [validator.pubkey for validator in state.validators] # Ensure that the pubkey sets are different. @@ -262,21 +268,22 @@ def test_invalid_signature_previous_committee(spec, state): committee = [pubkeys.index(pubkey) for pubkey in old_sync_committee.pubkeys] block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = [True] * len(committee) - block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( - spec, - state, - block.slot - 1, - committee, + block.body.sync_aggregate = spec.SyncAggregate( + sync_committee_bits=[True] * len(committee), + sync_committee_signature=compute_aggregate_sync_committee_signature( + spec, + state, + block.slot - 1, + committee, + ) ) - yield 'blocks', [block] - expect_assertion_error(lambda: spec.process_sync_committee(state, block.body)) - yield 'post', None + yield from run_sync_committee_processing(spec, state, block, expect_exception=True) @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test +@always_bls def test_valid_signature_future_committee(spec, state): # NOTE: the `state` provided is at genesis and the process to select # sync committees currently returns the same committee for the first and second @@ -302,18 +309,15 @@ def test_valid_signature_future_committee(spec, state): pubkeys = [validator.pubkey for validator in state.validators] committee_indices = [pubkeys.index(pubkey) for pubkey in sync_committee.pubkeys] - yield 'pre', state - block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = [True] * len(committee_indices) - block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( - spec, - state, - block.slot - 1, - committee_indices, + block.body.sync_aggregate = spec.SyncAggregate( + sync_committee_bits=[True] * len(committee_indices), + sync_committee_signature=compute_aggregate_sync_committee_signature( + spec, + state, + block.slot - 1, + committee_indices, + ) ) - signed_block = state_transition_and_sign_block(spec, state, block) - - yield 'blocks', [signed_block] - yield 'post', state + yield from run_sync_committee_processing(spec, state, block) diff --git a/tests/core/pyspec/eth2spec/test/altair/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/altair/sanity/test_blocks.py index 7bd2a91ba..81e0df713 100644 --- a/tests/core/pyspec/eth2spec/test/altair/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/altair/sanity/test_blocks.py @@ -25,12 +25,14 @@ def run_sync_committee_sanity_test(spec, state, fraction_full=1.0): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = [index in participants for index in committee] - block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( - spec, - state, - block.slot - 1, - participants, + block.body.sync_aggregate = spec.SyncAggregate( + sync_committee_bits=[index in participants for index in committee], + sync_committee_signature=compute_aggregate_sync_committee_signature( + spec, + state, + block.slot - 1, + participants, + ) ) signed_block = state_transition_and_sign_block(spec, state, block) diff --git a/tests/core/pyspec/eth2spec/test/helpers/block.py b/tests/core/pyspec/eth2spec/test/helpers/block.py index 084826d65..6949b54bc 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/block.py @@ -92,7 +92,7 @@ def build_empty_block(spec, state, slot=None): empty_block.parent_root = parent_block_root if is_post_altair(spec): - empty_block.body.sync_committee_signature = spec.G2_POINT_AT_INFINITY + empty_block.body.sync_aggregate.sync_committee_signature = spec.G2_POINT_AT_INFINITY apply_randao_reveal(spec, state, empty_block) return empty_block diff --git a/tests/core/pyspec/eth2spec/test/helpers/block_processing.py b/tests/core/pyspec/eth2spec/test/helpers/block_processing.py new file mode 100644 index 000000000..5c8ecb36c --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/block_processing.py @@ -0,0 +1,60 @@ +def for_ops(state, operations, fn) -> None: + for operation in operations: + fn(state, operation) + + +def get_process_calls(spec): + return { + # PHASE0 + 'process_block_header': + lambda state, block: spec.process_block_header(state, block), + 'process_randao': + lambda state, block: spec.process_randao(state, block.body), + 'process_eth1_data': + lambda state, block: spec.process_eth1_data(state, block.body), + 'process_proposer_slashing': + lambda state, block: for_ops(state, block.body.proposer_slashings, spec.process_proposer_slashing), + 'process_attester_slashing': + lambda state, block: for_ops(state, block.body.attester_slashings, spec.process_attester_slashing), + 'process_attestation': + lambda state, block: for_ops(state, block.body.attestations, spec.process_attestation), + 'process_deposit': + lambda state, block: for_ops(state, block.body.deposits, spec.process_deposit), + 'process_voluntary_exit': + lambda state, block: for_ops(state, block.body.voluntary_exits, spec.process_voluntary_exit), + # Altair + 'process_sync_committee': + lambda state, block: spec.process_sync_committee(state, block.body.sync_aggregate), + # PHASE1 + 'process_custody_game_operations': + lambda state, block: spec.process_custody_game_operations(state, block.body), + 'process_shard_transitions': + lambda state, block: spec.process_shard_transitions( + state, block.body.shard_transitions, block.body.attestations), + } + + +def run_block_processing_to(spec, state, block, process_name: str): + """ + Processes to the block transition, up to, but not including, the sub-transition named ``process_name``. + Returns a Callable[[state, block], None] for the remaining ``process_name`` transition. + + Tests should create full blocks to ensure a valid state transition, even if the operation itself is isolated. + (e.g. latest_header in the beacon state is up-to-date in a sync-committee test). + + A test prepares a pre-state by calling this function, output the pre-state, + and it can then proceed to run the returned callable, and output a post-state. + """ + print(f"state.slot {state.slot} block.slot {block.slot}") + # transition state to slot before block state transition + if state.slot < block.slot: + spec.process_slots(state, block.slot) + print(f"state.slot {state.slot} block.slot {block.slot} A") + + # process components of block transition + for name, call in get_process_calls(spec).items(): + if name == process_name: + return call + # only run when present. Later phases introduce more to the block-processing. + if hasattr(spec, name): + call(state, block) diff --git a/tests/formats/operations/README.md b/tests/formats/operations/README.md index 51d92a017..033c5fdef 100644 --- a/tests/formats/operations/README.md +++ b/tests/formats/operations/README.md @@ -33,14 +33,15 @@ This excludes the other parts of the block-transition. Operations: -| *`operation-name`* | *`operation-object`* | *`input name`* | *`processing call`* | -|-------------------------|-----------------------|----------------------|--------------------------------------------------------| -| `attestation` | `Attestation` | `attestation` | `process_attestation(state, attestation)` | -| `attester_slashing` | `AttesterSlashing` | `attester_slashing` | `process_attester_slashing(state, attester_slashing)` | -| `block_header` | `BeaconBlock` | **`block`** | `process_block_header(state, block)` | -| `deposit` | `Deposit` | `deposit` | `process_deposit(state, deposit)` | -| `proposer_slashing` | `ProposerSlashing` | `proposer_slashing` | `process_proposer_slashing(state, proposer_slashing)` | -| `voluntary_exit` | `SignedVoluntaryExit` | `voluntary_exit` | `process_voluntary_exit(state, voluntary_exit)` | +| *`operation-name`* | *`operation-object`* | *`input name`* | *`processing call`* | +|-------------------------|-----------------------|----------------------|-----------------------------------------------------------------| +| `attestation` | `Attestation` | `attestation` | `process_attestation(state, attestation)` | +| `attester_slashing` | `AttesterSlashing` | `attester_slashing` | `process_attester_slashing(state, attester_slashing)` | +| `block_header` | `BeaconBlock` | **`block`** | `process_block_header(state, block)` | +| `deposit` | `Deposit` | `deposit` | `process_deposit(state, deposit)` | +| `proposer_slashing` | `ProposerSlashing` | `proposer_slashing` | `process_proposer_slashing(state, proposer_slashing)` | +| `voluntary_exit` | `SignedVoluntaryExit` | `voluntary_exit` | `process_voluntary_exit(state, voluntary_exit)` | +| `sync_aggregate` | `SyncAggregate` | `sync_aggregate` | `process_sync_committee(state, sync_aggregate)` (new in Altair) | Note that `block_header` is not strictly an operation (and is a full `Block`), but processed in the same manner, and hence included here. From 27e88a24843cf220625710ffb6c510416430ccfd Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Tue, 16 Mar 2021 15:30:25 +0000 Subject: [PATCH 214/222] (WIP) refactor sync committee rewards --- specs/altair/beacon-chain.md | 98 ++++++++++--------- .../test_process_sync_committee.py | 53 +++++----- .../pyspec/eth2spec/test/helpers/rewards.py | 10 +- 3 files changed, 85 insertions(+), 76 deletions(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 54f716c09..c25f748b0 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -10,7 +10,7 @@ - [Custom types](#custom-types) - [Constants](#constants) - [Participation flag indices](#participation-flag-indices) - - [Participation flag fractions](#participation-flag-fractions) + - [Incentivization weights](#incentivization-weights) - [Misc](#misc) - [Configuration](#configuration) - [Updated penalty values](#updated-penalty-values) @@ -28,12 +28,13 @@ - [`Predicates`](#predicates) - [`eth2_fast_aggregate_verify`](#eth2_fast_aggregate_verify) - [Misc](#misc-2) - - [`get_flag_indices_and_numerators`](#get_flag_indices_and_numerators) + - [`get_flag_indices_and_weights`](#get_flag_indices_and_weights) - [`add_flag`](#add_flag) - [`has_flag`](#has_flag) - [Beacon state accessors](#beacon-state-accessors) - [`get_sync_committee_indices`](#get_sync_committee_indices) - [`get_sync_committee`](#get_sync_committee) + - [`get_base_reward_per_increment`](#get_base_reward_per_increment) - [`get_base_reward`](#get_base_reward) - [`get_unslashed_participating_indices`](#get_unslashed_participating_indices) - [`get_flag_index_deltas`](#get_flag_index_deltas) @@ -61,7 +62,7 @@ Altair is the first beacon chain hard fork. Its main features are: * sync committees to support light clients * incentive accounting reforms to reduce spec complexity -* penalty parameter updates to move them to their planned maximally punitive configuration +* penalty parameter updates towards their planned maximally punitive values ## Custom types @@ -79,16 +80,17 @@ Altair is the first beacon chain hard fork. Its main features are: | `TIMELY_SOURCE_FLAG_INDEX` | `1` | | `TIMELY_TARGET_FLAG_INDEX` | `2` | -### Participation flag fractions +### Incentivization weights | Name | Value | | - | - | -| `TIMELY_HEAD_FLAG_NUMERATOR` | `12` | -| `TIMELY_SOURCE_FLAG_NUMERATOR` | `12` | -| `TIMELY_TARGET_FLAG_NUMERATOR` | `32` | -| `FLAG_DENOMINATOR` | `64` | +| `TIMELY_HEAD_WEIGHT` | `12` | +| `TIMELY_SOURCE_WEIGHT` | `12` | +| `TIMELY_TARGET_WEIGHT` | `24` | +| `SYNC_REWARD_WEIGHT` | `8` | +| `WEIGHT_DENOMINATOR` | `64` | -*Note*: The sum of the participatition flag fractions (7/8) plus the proposer reward fraction (1/8) equals 1. +*Note*: The sum of the weight fractions (7/8) plus the proposer inclusion fraction (1/8) equals 1. ### Misc @@ -227,14 +229,14 @@ def eth2_fast_aggregate_verify(pubkeys: Sequence[BLSPubkey], message: Bytes32, s ### Misc -#### `get_flag_indices_and_numerators` +#### `get_flag_indices_and_weights` ```python -def get_flag_indices_and_numerators() -> Sequence[Tuple[int, int]]: +def get_flag_indices_and_weights() -> Sequence[Tuple[int, int]]: return ( - (TIMELY_HEAD_FLAG_INDEX, TIMELY_HEAD_FLAG_NUMERATOR), - (TIMELY_SOURCE_FLAG_INDEX, TIMELY_SOURCE_FLAG_NUMERATOR), - (TIMELY_TARGET_FLAG_INDEX, TIMELY_TARGET_FLAG_NUMERATOR), + (TIMELY_HEAD_FLAG_INDEX, TIMELY_HEAD_WEIGHT), + (TIMELY_SOURCE_FLAG_INDEX, TIMELY_SOURCE_WEIGHT), + (TIMELY_TARGET_FLAG_INDEX, TIMELY_TARGET_WEIGHT), ) ``` @@ -295,15 +297,21 @@ def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: return SyncCommittee(pubkeys=pubkeys, pubkey_aggregates=pubkey_aggregates) ``` +#### `get_base_reward_per_increment` + +```python +def get_base_reward_per_increment(state: BeaconState) -> Gwei: + return Gwei(EFFECTIVE_BALANCE_INCREMENT * BASE_REWARD_FACTOR // integer_squareroot(get_total_active_balance(state))) +``` + #### `get_base_reward` *Note*: The function `get_base_reward` is modified with the removal of `BASE_REWARDS_PER_EPOCH`. ```python def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: - total_balance = get_total_active_balance(state) - effective_balance = state.validators[index].effective_balance - return Gwei(effective_balance * BASE_REWARD_FACTOR // integer_squareroot(total_balance)) + increments = state.validators[index].effective_balance // EFFECTIVE_BALANCE_INCREMENT + return Gwei(increments * get_base_reward_per_increment(state)) ``` #### `get_unslashed_participating_indices` @@ -326,9 +334,7 @@ def get_unslashed_participating_indices(state: BeaconState, flag_index: int, epo #### `get_flag_index_deltas` ```python -def get_flag_index_deltas(state: BeaconState, - flag_index: int, - numerator: uint64) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: +def get_flag_index_deltas(state: BeaconState, flag_index: int, weight: uint64) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: """ Return the deltas for a given flag index by scanning through the participation flags. """ @@ -343,12 +349,12 @@ def get_flag_index_deltas(state: BeaconState, if index in unslashed_participating_indices: if is_in_inactivity_leak(state): # This flag reward cancels the inactivity penalty corresponding to the flag index - rewards[index] += Gwei(base_reward * numerator // FLAG_DENOMINATOR) + rewards[index] += Gwei(base_reward * weight // WEIGHT_DENOMINATOR) else: - reward_numerator = base_reward * numerator * unslashed_participating_increments - rewards[index] += Gwei(reward_numerator // (active_increments * FLAG_DENOMINATOR)) + reward_numerator = base_reward * weight * unslashed_participating_increments + rewards[index] += Gwei(reward_numerator // (active_increments * WEIGHT_DENOMINATOR)) else: - penalties[index] += Gwei(base_reward * numerator // FLAG_DENOMINATOR) + penalties[index] += Gwei(base_reward * weight // WEIGHT_DENOMINATOR) return rewards, penalties ``` @@ -368,9 +374,9 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S previous_epoch = get_previous_epoch(state) matching_target_indices = get_unslashed_participating_indices(state, TIMELY_TARGET_FLAG_INDEX, previous_epoch) for index in get_eligible_validator_indices(state): - for (_, numerator) in get_flag_indices_and_numerators(): + for (_, weight) in get_flag_indices_and_weights(): # This inactivity penalty cancels the flag reward corresponding to the flag index - penalties[index] += Gwei(get_base_reward(state, index) * numerator // FLAG_DENOMINATOR) + penalties[index] += Gwei(get_base_reward(state, index) * weight // WEIGHT_DENOMINATOR) if index not in matching_target_indices: penalty_numerator = state.validators[index].effective_balance * state.inactivity_scores[index] penalty_denominator = INACTIVITY_SCORE_BIAS * INACTIVITY_PENALTY_QUOTIENT_ALTAIR @@ -463,13 +469,13 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: # Update epoch participation flags proposer_reward_numerator = 0 for index in get_attesting_indices(state, data, attestation.aggregation_bits): - for flag_index, flag_numerator in get_flag_indices_and_numerators(): + for flag_index, weight in get_flag_indices_and_weights(): if flag_index in participation_flag_indices and not has_flag(epoch_participation[index], flag_index): epoch_participation[index] = add_flag(epoch_participation[index], flag_index) - proposer_reward_numerator += get_base_reward(state, index) * flag_numerator + proposer_reward_numerator += get_base_reward(state, index) * weight # Reward proposer - proposer_reward = Gwei(proposer_reward_numerator // (FLAG_DENOMINATOR * PROPOSER_REWARD_QUOTIENT)) + proposer_reward = Gwei(proposer_reward_numerator // (WEIGHT_DENOMINATOR * PROPOSER_REWARD_QUOTIENT)) increase_balance(state, get_beacon_proposer_index(state), proposer_reward) ``` @@ -523,26 +529,28 @@ def process_sync_committee(state: BeaconState, aggregate: SyncAggregate) -> None # Verify sync committee aggregate signature signing over the previous slot block root previous_slot = Slot(max(int(state.slot), 1) - 1) committee_indices = get_sync_committee_indices(state, get_current_epoch(state)) - participant_indices = [index for index, bit in zip(committee_indices, aggregate.sync_committee_bits) if bit] + included_indices = [index for index, bit in zip(committee_indices, aggregate.sync_committee_bits) if bit] committee_pubkeys = state.current_sync_committee.pubkeys - participant_pubkeys = [pubkey for pubkey, bit in zip(committee_pubkeys, aggregate.sync_committee_bits) if bit] + included_pubkeys = [pubkey for pubkey, bit in zip(committee_pubkeys, aggregate.sync_committee_bits) if bit] domain = get_domain(state, DOMAIN_SYNC_COMMITTEE, compute_epoch_at_slot(previous_slot)) signing_root = compute_signing_root(get_block_root_at_slot(state, previous_slot), domain) - assert eth2_fast_aggregate_verify(participant_pubkeys, signing_root, aggregate.sync_committee_signature) + assert eth2_fast_aggregate_verify(included_pubkeys, signing_root, aggregate.sync_committee_signature) - # Reward sync committee participants - proposer_rewards = Gwei(0) - active_validator_count = uint64(len(get_active_validator_indices(state, get_current_epoch(state)))) - for participant_index in participant_indices: - proposer_reward = get_proposer_reward(state, participant_index) - proposer_rewards += proposer_reward - base_reward = get_base_reward(state, participant_index) - max_participant_reward = base_reward - proposer_reward - reward = Gwei(max_participant_reward * active_validator_count // (len(committee_indices) * SLOTS_PER_EPOCH)) - increase_balance(state, participant_index, reward) + # Compute the maximum sync rewards for the slot + total_active_increments = get_total_active_balance(state) // EFFECTIVE_BALANCE_INCREMENT + total_base_rewards = Gwei(get_base_reward_per_increment(state) * total_active_increments) + max_epoch_rewards = Gwei(total_base_rewards * SYNC_REWARD_WEIGHT // WEIGHT_DENOMINATOR) + max_slot_rewards = Gwei(max_epoch_rewards * len(included_indices) // len(committee_indices) // SLOTS_PER_EPOCH) - # Reward beacon proposer - increase_balance(state, get_beacon_proposer_index(state), proposer_rewards) + # Compute the participant and proposer sync rewards + committee_effective_balance = sum([state.validators[index].effective_balance for index in included_indices]) + committee_effective_balance = max(EFFECTIVE_BALANCE_INCREMENT, committee_effective_balance) + for included_index in included_indices: + effective_balance = state.validators[included_index].effective_balance + inclusion_reward = Gwei(max_slot_rewards * effective_balance // committee_effective_balance) + proposer_reward = Gwei(inclusion_reward // PROPOSER_REWARD_QUOTIENT) + increase_balance(state, get_beacon_proposer_index(state), proposer_reward) + increase_balance(state, included_index, inclusion_reward - proposer_reward) ``` ### Epoch processing @@ -633,7 +641,7 @@ def process_rewards_and_penalties(state: BeaconState) -> None: if get_current_epoch(state) == GENESIS_EPOCH: return - flag_indices_and_numerators = get_flag_indices_and_numerators() + flag_indices_and_numerators = get_flag_indices_and_weights() flag_deltas = [get_flag_index_deltas(state, index, numerator) for (index, numerator) in flag_indices_and_numerators] deltas = flag_deltas + [get_inactivity_penalty_deltas(state)] for (rewards, penalties) in deltas: diff --git a/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py index 58cf48314..135e5bf8d 100644 --- a/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py @@ -105,12 +105,20 @@ def test_invalid_signature_extra_participant(spec, state): yield from run_sync_committee_processing(spec, state, block, expect_exception=True) -def compute_sync_committee_participant_reward(spec, state, participant_index, active_validator_count, committee_size): - base_reward = spec.get_base_reward(state, participant_index) - proposer_reward = spec.get_proposer_reward(state, participant_index) - max_participant_reward = base_reward - proposer_reward - return max_participant_reward * active_validator_count // committee_size // spec.SLOTS_PER_EPOCH +def compute_sync_committee_participant_reward(spec, state, participant_index, committee, committee_bits): + total_active_increments = spec.get_total_active_balance(state) // spec.EFFECTIVE_BALANCE_INCREMENT + total_base_rewards = spec.Gwei(spec.get_base_reward_per_increment(state) * total_active_increments) + max_epoch_rewards = spec.Gwei(total_base_rewards * spec.SYNC_REWARD_WEIGHT // spec.WEIGHT_DENOMINATOR) + included_indices = [index for index, bit in zip(committee, committee_bits) if bit] + max_slot_rewards = spec.Gwei(max_epoch_rewards * len(included_indices) // len(committee) // spec.SLOTS_PER_EPOCH) + # Compute the participant and proposer sync rewards + committee_effective_balance = sum([state.validators[index].effective_balance for index in included_indices]) + committee_effective_balance = max(spec.EFFECTIVE_BALANCE_INCREMENT, committee_effective_balance) + effective_balance = state.validators[participant_index].effective_balance + inclusion_reward = spec.Gwei(max_slot_rewards * effective_balance // committee_effective_balance) + proposer_reward = spec.Gwei(inclusion_reward // spec.PROPOSER_REWARD_QUOTIENT) + return spec.Gwei(inclusion_reward - proposer_reward) @with_all_phases_except([PHASE0, PHASE1]) @with_configs([MINIMAL], reason="to create nonduplicate committee") @@ -129,8 +137,9 @@ def test_sync_committee_rewards_nonduplicate_committee(spec, state): pre_balances = state.balances.copy() block = build_empty_block_for_next_slot(spec, state) + committee_bits = [True] * committee_size block.body.sync_aggregate = spec.SyncAggregate( - sync_committee_bits=[True] * committee_size, + sync_committee_bits=committee_bits, sync_committee_signature=compute_aggregate_sync_committee_signature( spec, state, @@ -142,21 +151,16 @@ def test_sync_committee_rewards_nonduplicate_committee(spec, state): yield from run_sync_committee_processing(spec, state, block) for index in range(len(state.validators)): - expected_reward = 0 - - if index == block.proposer_index: - expected_reward += sum([spec.get_proposer_reward(state, index) for index in committee]) - if index in committee: - expected_reward += compute_sync_committee_participant_reward( + participant_reward = compute_sync_committee_participant_reward( spec, state, index, - active_validator_count, - committee_size + committee, + committee_bits, ) - assert state.balances[index] == pre_balances[index] + expected_reward + assert state.balances[index] == pre_balances[index] + participant_reward @with_all_phases_except([PHASE0, PHASE1]) @@ -173,8 +177,9 @@ def test_sync_committee_rewards_duplicate_committee(spec, state): pre_balances = state.balances.copy() block = build_empty_block_for_next_slot(spec, state) + committee_bits = [True] * committee_size block.body.sync_aggregate = spec.SyncAggregate( - sync_committee_bits=[True] * committee_size, + sync_committee_bits=committee_bits, sync_committee_signature=compute_aggregate_sync_committee_signature( spec, state, @@ -187,22 +192,18 @@ def test_sync_committee_rewards_duplicate_committee(spec, state): multiplicities = Counter(committee) for index in range(len(state.validators)): - expected_reward = 0 - - if index == block.proposer_index: - expected_reward += sum([spec.get_proposer_reward(state, index) for index in committee]) - + inclusion_rewards = 0 if index in committee: - reward = compute_sync_committee_participant_reward( + participant_reward = compute_sync_committee_participant_reward( spec, state, index, - active_validator_count, - committee_size, + committee, + committee_bits, ) - expected_reward += reward * multiplicities[index] + inclusion_rewards += reward * multiplicities[index] - assert state.balances[index] == pre_balances[index] + expected_reward + # assert state.balances[index] == pre_balances[index] + inclusion_rewards @with_all_phases_except([PHASE0, PHASE1]) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 8d34500b6..e51da228d 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -42,13 +42,13 @@ def run_deltas(spec, state): if is_post_altair(spec): def get_source_deltas(state): - return spec.get_flag_index_deltas(state, spec.TIMELY_SOURCE_FLAG_INDEX, spec.TIMELY_SOURCE_FLAG_NUMERATOR) + return spec.get_flag_index_deltas(state, spec.TIMELY_SOURCE_FLAG_INDEX, spec.TIMELY_SOURCE_WEIGHT) def get_head_deltas(state): - return spec.get_flag_index_deltas(state, spec.TIMELY_HEAD_FLAG_INDEX, spec.TIMELY_HEAD_FLAG_NUMERATOR) + return spec.get_flag_index_deltas(state, spec.TIMELY_HEAD_FLAG_INDEX, spec.TIMELY_HEAD_WEIGHT) def get_target_deltas(state): - return spec.get_flag_index_deltas(state, spec.TIMELY_TARGET_FLAG_INDEX, spec.TIMELY_TARGET_FLAG_NUMERATOR) + return spec.get_flag_index_deltas(state, spec.TIMELY_TARGET_FLAG_INDEX, spec.TIMELY_TARGET_WEIGHT) yield from run_attestation_component_deltas( spec, @@ -207,8 +207,8 @@ def run_get_inactivity_penalty_deltas(spec, state): base_penalty = cancel_base_rewards_per_epoch * base_reward - spec.get_proposer_reward(state, index) else: base_penalty = sum( - base_reward * numerator // spec.FLAG_DENOMINATOR - for (_, numerator) in spec.get_flag_indices_and_numerators() + base_reward * numerator // spec.WEIGHT_DENOMINATOR + for (_, numerator) in spec.get_flag_indices_and_weights() ) if not has_enough_for_reward(spec, state, index): From c9de95541feba6ffd1f2307f31be13e39f184d7a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 16 Mar 2021 23:56:34 +0800 Subject: [PATCH 215/222] Use a dummy `INCOMPLETE` file to indicate that the test generation is incomplete --- .../gen_helpers/gen_base/gen_runner.py | 35 +++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index f14f7e323..ee2410894 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -1,3 +1,5 @@ +import os +import shutil import argparse from pathlib import Path import sys @@ -102,8 +104,11 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): yaml = YAML(pure=True) yaml.default_flow_style = None + log_file = Path(output_dir) / 'testgen_error_log.txt' + print(f"Generating tests into {output_dir}") print(f"Reading configs from {args.configs_path}") + print(f'Error log file: {log_file}') configs = args.config_list if configs is None: @@ -126,14 +131,25 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): / Path(test_case.runner_name) / Path(test_case.handler_name) / Path(test_case.suite_name) / Path(test_case.case_name) ) + incomplete_tag_file = case_dir / "INCOMPLETE" + if case_dir.exists(): - if not args.force: + if not args.force and not incomplete_tag_file.exists(): print(f'Skipping already existing test: {case_dir}') continue - print(f'Warning, output directory {case_dir} already exist,' - f' old files are not deleted but will be overwritten when a new version is produced') + else: + print(f'Warning, output directory {case_dir} already exist,' + f' old files will be deleted and it will generate test vector files with the latest version') + # Clear the existing case_dir folder + shutil.rmtree(case_dir) print(f'Generating test: {case_dir}') + + # Add `INCOMPLETE` tag file to indicate that the test generation has not completed. + case_dir.mkdir(parents=True, exist_ok=True) + with incomplete_tag_file.open("w") as f: + f.write("\n") + try: def output_part(out_kind: str, name: str, fn: Callable[[Path, ], None]): # make sure the test case directory is created before any test part is written. @@ -170,6 +186,19 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): except Exception as e: print(f"ERROR: failed to generate vector(s) for test {case_dir}: {e}") traceback.print_exc() + # Write to log file + with log_file.open("a+") as f: + f.write(f"ERROR: failed to generate vector(s) for test {case_dir}: {e}") + traceback.print_exc(file=f) + f.write('\n') + else: + # If no written_part, the only file was incomplete_tag_file. Clear the existing case_dir folder. + if not written_part: + shutil.rmtree(case_dir) + else: + # Only remove `INCOMPLETE` tag file + os.remove(incomplete_tag_file) + print(f"completed {generator_name}") From d3c9eacd1438655b31fa18a79889e35d1165720f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 16 Mar 2021 23:59:20 +0800 Subject: [PATCH 216/222] Disable MAINNET test_valid_signature_future_committee --- .../test/altair/block_processing/test_process_sync_committee.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py index 58cf48314..d24294272 100644 --- a/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py @@ -284,6 +284,7 @@ def test_invalid_signature_previous_committee(spec, state): @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test @always_bls +@with_configs([MINIMAL], reason="too slow") def test_valid_signature_future_committee(spec, state): # NOTE: the `state` provided is at genesis and the process to select # sync committees currently returns the same committee for the first and second From b0a9fc827736a4249e408e64135ba62d7ed65c0b Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 16 Mar 2021 11:18:26 -0600 Subject: [PATCH 217/222] clean up and extend sync committee rewards tests --- .../test_process_sync_committee.py | 152 +++++++++++------- 1 file changed, 92 insertions(+), 60 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py index 135e5bf8d..61faa9af0 100644 --- a/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py @@ -65,7 +65,8 @@ def get_committee_indices(spec, state, duplicates=False): @always_bls def test_invalid_signature_missing_participant(spec, state): committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) - random_participant = random.choice(committee) + rng = random.Random(2020) + random_participant = rng.choice(committee) yield 'pre', state @@ -88,7 +89,8 @@ def test_invalid_signature_missing_participant(spec, state): @always_bls def test_invalid_signature_extra_participant(spec, state): committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) - random_participant = random.choice(committee) + rng = random.Random(3030) + random_participant = rng.choice(committee) block = build_empty_block_for_next_slot(spec, state) # Exclude one signature even though the block claims the entire committee participated. @@ -105,7 +107,7 @@ def test_invalid_signature_extra_participant(spec, state): yield from run_sync_committee_processing(spec, state, block, expect_exception=True) -def compute_sync_committee_participant_reward(spec, state, participant_index, committee, committee_bits): +def compute_sync_committee_inclusion_reward(spec, state, participant_index, committee, committee_bits): total_active_increments = spec.get_total_active_balance(state) // spec.EFFECTIVE_BALANCE_INCREMENT total_base_rewards = spec.Gwei(spec.get_base_reward_per_increment(state) * total_active_increments) max_epoch_rewards = spec.Gwei(total_base_rewards * spec.SYNC_REWARD_WEIGHT // spec.WEIGHT_DENOMINATOR) @@ -116,9 +118,82 @@ def compute_sync_committee_participant_reward(spec, state, participant_index, co committee_effective_balance = sum([state.validators[index].effective_balance for index in included_indices]) committee_effective_balance = max(spec.EFFECTIVE_BALANCE_INCREMENT, committee_effective_balance) effective_balance = state.validators[participant_index].effective_balance - inclusion_reward = spec.Gwei(max_slot_rewards * effective_balance // committee_effective_balance) + return spec.Gwei(max_slot_rewards * effective_balance // committee_effective_balance) + + +def compute_sync_committee_participant_reward(spec, state, participant_index, committee, committee_bits): + included_indices = [index for index, bit in zip(committee, committee_bits) if bit] + multiplicities = Counter(included_indices) + + inclusion_reward = compute_sync_committee_inclusion_reward( + spec, state, participant_index, committee, committee_bits, + ) proposer_reward = spec.Gwei(inclusion_reward // spec.PROPOSER_REWARD_QUOTIENT) - return spec.Gwei(inclusion_reward - proposer_reward) + return spec.Gwei((inclusion_reward - proposer_reward) * multiplicities[participant_index]) + + +def compute_sync_committee_proposer_reward(spec, state, committee, committee_bits): + proposer_reward = 0 + for index, bit in zip(committee, committee_bits): + if not bit: + continue + inclusion_reward = compute_sync_committee_inclusion_reward( + spec, state, index, committee, committee_bits, + ) + proposer_reward += spec.Gwei(inclusion_reward // spec.PROPOSER_REWARD_QUOTIENT) + return proposer_reward + + +def validate_sync_committee_rewards(spec, pre_state, post_state, committee, committee_bits, proposer_index): + for index in range(len(post_state.validators)): + reward = 0 + if index in committee: + reward += compute_sync_committee_participant_reward( + spec, + pre_state, + index, + committee, + committee_bits, + ) + + if proposer_index == index: + reward += compute_sync_committee_proposer_reward( + spec, + pre_state, + committee, + committee_bits, + ) + + assert post_state.balances[index] == pre_state.balances[index] + reward + + +def run_successful_sync_committee_test(spec, state, committee, committee_bits): + yield 'pre', state + + pre_state = state.copy() + + block = build_empty_block_for_next_slot(spec, state) + block.body.sync_aggregate = spec.SyncAggregate( + sync_committee_bits=committee_bits, + sync_committee_signature=compute_aggregate_sync_committee_signature( + spec, + state, + block.slot - 1, + [index for index, bit in zip(committee, committee_bits) if bit], + ) + ) + + yield from run_sync_committee_processing(spec, state, block) + + validate_sync_committee_rewards( + spec, + pre_state, + state, + committee, + committee_bits, + block.proposer_index, + ) + @with_all_phases_except([PHASE0, PHASE1]) @with_configs([MINIMAL], reason="to create nonduplicate committee") @@ -126,41 +201,25 @@ def compute_sync_committee_participant_reward(spec, state, participant_index, co def test_sync_committee_rewards_nonduplicate_committee(spec, state): committee = get_committee_indices(spec, state, duplicates=False) committee_size = len(committee) + committee_bits = [True] * committee_size active_validator_count = len(spec.get_active_validator_indices(state, spec.get_current_epoch(state))) # Preconditions of this test case assert active_validator_count >= spec.SYNC_COMMITTEE_SIZE assert committee_size == len(set(committee)) - yield 'pre', state + yield from run_successful_sync_committee_test(spec, state, committee, committee_bits) - pre_balances = state.balances.copy() - block = build_empty_block_for_next_slot(spec, state) - committee_bits = [True] * committee_size - block.body.sync_aggregate = spec.SyncAggregate( - sync_committee_bits=committee_bits, - sync_committee_signature=compute_aggregate_sync_committee_signature( - spec, - state, - block.slot - 1, - committee, - ) - ) +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +@always_bls +def test_sync_committee_rewards_not_full_participants(spec, state): + committee = get_committee_indices(spec, state, duplicates=False) + rng = random.Random(1010) + committee_bits = [rng.choice([True, False]) for _ in committee] - yield from run_sync_committee_processing(spec, state, block) - - for index in range(len(state.validators)): - if index in committee: - participant_reward = compute_sync_committee_participant_reward( - spec, - state, - index, - committee, - committee_bits, - ) - - assert state.balances[index] == pre_balances[index] + participant_reward + yield from run_successful_sync_committee_test(spec, state, committee, committee_bits) @with_all_phases_except([PHASE0, PHASE1]) @@ -169,41 +228,14 @@ def test_sync_committee_rewards_nonduplicate_committee(spec, state): def test_sync_committee_rewards_duplicate_committee(spec, state): committee = get_committee_indices(spec, state, duplicates=True) committee_size = len(committee) + committee_bits = [True] * committee_size active_validator_count = len(spec.get_active_validator_indices(state, spec.get_current_epoch(state))) # Preconditions of this test case assert active_validator_count < spec.SYNC_COMMITTEE_SIZE assert committee_size > len(set(committee)) - pre_balances = state.balances.copy() - block = build_empty_block_for_next_slot(spec, state) - committee_bits = [True] * committee_size - block.body.sync_aggregate = spec.SyncAggregate( - sync_committee_bits=committee_bits, - sync_committee_signature=compute_aggregate_sync_committee_signature( - spec, - state, - block.slot - 1, - committee, - ) - ) - yield from run_sync_committee_processing(spec, state, block) - - multiplicities = Counter(committee) - - for index in range(len(state.validators)): - inclusion_rewards = 0 - if index in committee: - participant_reward = compute_sync_committee_participant_reward( - spec, - state, - index, - committee, - committee_bits, - ) - inclusion_rewards += reward * multiplicities[index] - - # assert state.balances[index] == pre_balances[index] + inclusion_rewards + yield from run_successful_sync_committee_test(spec, state, committee, committee_bits) @with_all_phases_except([PHASE0, PHASE1]) From 2ee559eca1c5f94e9a9a2b3d05619a121e6ebeae Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 17 Mar 2021 02:40:59 +0800 Subject: [PATCH 218/222] Fix `SkippedTest` cases. Should remove the folder --- .../core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index ee2410894..e4f1318ae 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -145,6 +145,8 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): print(f'Generating test: {case_dir}') + written_part = False + # Add `INCOMPLETE` tag file to indicate that the test generation has not completed. case_dir.mkdir(parents=True, exist_ok=True) with incomplete_tag_file.open("w") as f: @@ -159,7 +161,6 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): except IOError as e: sys.exit(f'Error when dumping test "{case_dir}", part "{name}", kind "{out_kind}": {e}') - written_part = False meta = dict() try: @@ -173,6 +174,7 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): output_part("ssz", name, dump_ssz_fn(data, name, file_mode)) except SkippedTest as e: print(e) + shutil.rmtree(case_dir) continue # Once all meta data is collected (if any), write it to a meta data file. @@ -182,7 +184,6 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): if not written_part: print(f"test case {case_dir} did not produce any test case parts") - except Exception as e: print(f"ERROR: failed to generate vector(s) for test {case_dir}: {e}") traceback.print_exc() From f360a9823f91d4cbc0714eb90b62646f515dd490 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 17 Mar 2021 02:41:44 +0800 Subject: [PATCH 219/222] Add Makefile commands `detect_generator_error_log` and `detect_generator_incomplete` --- Makefile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 951237808..d1ebe0fec 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ COV_INDEX_FILE=$(PY_SPEC_DIR)/$(COV_HTML_OUT)/index.html CURRENT_DIR = ${CURDIR} LINTER_CONFIG_FILE = $(CURRENT_DIR)/linter.ini +GENERATOR_ERROR_LOG_FILE = $(CURRENT_DIR)/$(TEST_VECTOR_DIR)/testgen_error_log.txt export DAPP_SKIP_BUILD:=1 export DAPP_SRC:=$(SOLIDITY_DEPOSIT_CONTRACT_DIR) @@ -35,7 +36,8 @@ export DAPP_JSON:=build/combined.json .PHONY: clean partial_clean all test citest lint generate_tests pyspec install_test open_cov \ install_deposit_contract_tester test_deposit_contract install_deposit_contract_compiler \ - compile_deposit_contract test_compile_deposit_contract check_toc + compile_deposit_contract test_compile_deposit_contract check_toc \ + detect_generator_incomplete detect_generator_error_log all: $(PY_SPEC_ALL_TARGETS) @@ -171,3 +173,9 @@ $(TEST_VECTOR_DIR)/: # (creation of output dir is a dependency) gen_%: $(TEST_VECTOR_DIR) $(call run_generator,$*) + +detect_generator_incomplete: $(TEST_VECTOR_DIR) + find $(TEST_VECTOR_DIR) -name "INCOMPLETE" + +detect_generator_error_log: $(TEST_VECTOR_DIR) + [ -f $(GENERATOR_ERROR_LOG_FILE) ] && echo "[ERROR] $(GENERATOR_ERROR_LOG_FILE) file exists" || echo "[PASSED] error log file does not exist" From 893134d2ab76e419c5d6c46303ac50570e70f07a Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Wed, 17 Feb 2021 20:47:52 -0800 Subject: [PATCH 220/222] Add validator guide and initial P2P spec for Altair --- specs/altair/beacon-chain.md | 2 + specs/altair/p2p-interface.md | 129 +++++++++++ specs/altair/validator.md | 415 ++++++++++++++++++++++++++++++++++ 3 files changed, 546 insertions(+) create mode 100644 specs/altair/p2p-interface.md create mode 100644 specs/altair/validator.md diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 54f716c09..ab507a4f7 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -129,6 +129,8 @@ This patch updates a few configuration values to move penalty parameters toward | Name | Value | | - | - | | `DOMAIN_SYNC_COMMITTEE` | `DomainType('0x07000000')` | +| `DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF` | `DomainType('0x08000000')` | +| `DOMAIN_CONTRIBUTION_AND_PROOF` | `DomainType('0x09000000')` | ## Containers diff --git a/specs/altair/p2p-interface.md b/specs/altair/p2p-interface.md new file mode 100644 index 000000000..ba45eb578 --- /dev/null +++ b/specs/altair/p2p-interface.md @@ -0,0 +1,129 @@ +# Ethereum Altair networking specification + +This document contains the networking specification for Ethereum 2.0 clients added during the Altair deployment. +This document should be viewed as additive to the [document from Phase 0](../phase0/p2p-interface.md) and will be referred to as the "phase 0 document" hereafter. +Readers should understand the phase 0 document and use it as a basis to understand the changes outlined in this document. + +In particular, Altair introduces changes to the gossip domain and a small change to ENR advertisement in the discovery domain. +The phase 0 document should be consulted for specs on the network fundamentals, the existing gossip domain, the req/resp domain and the discovery domain. + +## Table of contents + + + + + + - [Warning](#warning) +- [Modifications in Altair](#modifications-in-altair) + - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) + - [Topics and messages](#topics-and-messages) + - [Global topics](#global-topics) + - [`beacon_block`](#beacon_block) + - [`sync_committee_contribution_and_proof`](#sync_committee_contribution_and_proof) + - [Sync committee subnets](#sync-committee-subnets) + - [`sync_committee_{subnet_id}`](#sync_committee_subnet_id) + - [Sync committees and aggregation](#sync-committees-and-aggregation) + - [The Req/Resp domain](#the-reqresp-domain) + - [The discovery domain: discv5](#the-discovery-domain-discv5) + + + + +## Warning + +This document is currently illustrative for early Altair testnets and some parts are subject to change. +Refer to the note in the [validator guide](./validator.md) for further details. + +# Modifications in Altair + +## The gossip domain: gossipsub + +Gossip meshes are added in Altair to support the consensus activities of the sync committees. +Validators use an aggregation scheme to balance the processing and networking load across all of the relevant actors. + +### Topics and messages + +Topics follow the same specification as in the phase 0 document. +New topics are added in Altair to support the sync committees and the beacon block topic is updated with the modified type. + +The specification around the creation, validation, and dissemination of messages has not changed from the Phase 0 document. + +The new topics along with the type of the `data` field of a gossipsub message are given in this table: + +| Name | Message Type | +| - | - | +| `beacon_block` | `SignedBeaconBlock` (modified) | +| `sync_committee_contribution_and_proof` | `SignedContributionAndProof` | +| `sync_committee_{subnet_id}` | `SyncCommitteeSignature` | + +Definitions of these new types can be found in the [Altair validator guide](./validator.md#containers). + +#### Global topics + +Altair changes the type of the global beacon block topic and adds one global topic to propagate partially aggregated sync committee signatures to all potential proposers of beacon blocks. + +##### `beacon_block` + +The existing specification for this topic does not change from the Phase 0 document, but the type of the payload does change to the (modified) `SignedBeaconBlock`. +This type changes due to the inclusion of the inner `BeaconBlockBody` that is modified in Altair. +See the [state transition document](./beacon-chain.md#beaconblockbody) for Altair for further details. + +##### `sync_committee_contribution_and_proof` + +This topic is used to propagate partially aggregated sync committee signatures to be included in future blocks. + +The following validations MUST pass before forwarding the `signed_contribution_and_proof` on the network; define `contribution_and_proof = signed_contribution_and_proof.message` and `contribution = contribution_and_proof.contribution` for convenience: + +- _[IGNORE]_ The contribution's slot is for the current slot, i.e. `contribution.slot == current_slot`. +- _[IGNORE]_ The block being signed over (`contribution.beacon_block_root`) has been seen (via both gossip and non-gossip sources). +- _[REJECT]_ The subcommittee index is in the allowed range, i.e. `contribution.subcommittee_index < SYNC_COMMITTEE_SUBNET_COUNT`. +- _[IGNORE]_ The sync committee contribution is the first valid contribution received for the aggregator with index `contribution_and_proof.aggregator_index` for the slot `contribution.slot`. +- _[REJECT]_ The aggregator's validator index is within the current sync committee -- + i.e. `state.validators[aggregate_and_proof.aggregator_index].pubkey in state.current_sync_committee.pubkeys`. +- _[REJECT]_ The `contribution_and_proof.selection_proof` is a valid signature of the `contribution.slot` by the validator with index `contribution_and_proof.aggregator_index`. +- _[REJECT]_ `contribution_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_sync_committee_aggregator(state, contribution.slot, contribution_and_proof.selection_proof)` returns `True`. +- _[REJECT]_ The aggregator signature, `signed_contribution_and_proof.signature`, is valid. +- _[REJECT]_ The aggregate signature is valid for the message `beacon_block_root` and aggregate pubkey derived from the participation info in `aggregation_bits` for the subcommittee specified by the `subcommittee_index`. + +#### Sync committee subnets + +Sync committee subnets are used to propagate unaggregated sync committee signatures to subsections of the network. + +##### `sync_committee_{subnet_id}` + +The `sync_committee_{subnet_id}` topics are used to propagate unaggregated sync committee signatures to the subnet `subnet_id` to be aggregated before being gossiped to the global `sync_committee_contribution_and_proof` topic. + +The following validations MUST pass before forwarding the `sync_committee_signature` on the network: + +- _[IGNORE]_ The signature's slot is for the current slot, i.e. `sync_committee_signature.slot == current_slot`. +- _[IGNORE]_ The block being signed over (`sync_committee_signature.beacon_block_root`) has been seen (via both gossip and non-gossip sources). +- _[IGNORE]_ There has been no other valid sync committee signature for the declared `slot` for the validator referenced by `sync_committee_signature.validator_index`. +- _[REJECT]_ The validator producing this `sync_committee_signature` is in the current sync committee, i.e. `state.validators[sync_committee_signature.validator_index].pubkey in state.current_sync_committee.pubkeys`. +- _[REJECT]_ The `subnet_id` is correct, i.e. `subnet_id in compute_subnets_for_sync_committee(state, sync_committee_signature.validator_index)`. +- _[REJECT]_ The `signature` is valid for the message `beacon_block_root` for the validator referenced by `validator_index`. + +#### Sync committees and aggregation + +The aggregation scheme closely follows the design of the attestation aggregation scheme. +Sync committee signatures are broadcast into "subnets" defined by a topic. +The number of subnets is defined by `SYNC_COMMITTEE_SUBNET_COUNT` in the [Altair validator guide](./validator.md#constants). +Sync committee members are divided into "subcommittees" which are then assigned to a subnet for the duration of tenure in the sync committee. +Individual validators can be duplicated in the broader sync committee such that they are included multiple times in a given subcommittee or across multiple subcommittees. + +Unaggregated signatures (along with metadata) are sent as `SyncCommitteeSignature`s on the `sync_committee_{subnet_id}` topics. + +Aggregated sync committee signatures are packaged into (signed) `SyncCommitteeContribution` along with proofs and gossiped to the `sync_committee_contribution_and_proof` topic. + +## The Req/Resp domain + +* (TODO) specify how to handle different types of blocks on RPC responses. + +## The discovery domain: discv5 + +The `attnets` key of the ENR is used as defined in the phase 0 document. + +An additional bitfield is added to the ENR under the key `syncnets` to facilitate sync committee subnet discovery. +The length of this bitfield is `SYNC_COMMITTEE_SUBNET_COUNT` where each bit corresponds to a distinct `subnet_id` for a specific sync committee subnet. +The `i`th bit is set in this bitfield if the validator is currently subscribed to the `sync_committee_{i}` topic. + +See the [validator document](./validator.md#sync-committee-subnet-stability) for further details on how the new bits are used. diff --git a/specs/altair/validator.md b/specs/altair/validator.md new file mode 100644 index 000000000..59c1ab81f --- /dev/null +++ b/specs/altair/validator.md @@ -0,0 +1,415 @@ +# Ethereum 2.0 Altair -- Honest Validator + +This is an accompanying document to [Ethereum 2.0 Altair -- The Beacon Chain](./beacon-chain.md), which describes the expected actions of a "validator" participating in the Ethereum 2.0 protocol. + +## Table of contents + + + + + +- [Introduction](#introduction) +- [Prerequisites](#prerequisites) +- [Warning](#warning) +- [Constants](#constants) + - [Misc](#misc) +- [Containers](#containers) + - [`SyncCommitteeSignature`](#synccommitteesignature) + - [`SyncCommitteeContribution`](#synccommitteecontribution) + - [`ContributionAndProof`](#contributionandproof) + - [`SignedContributionAndProof`](#signedcontributionandproof) +- [Validator assignments](#validator-assignments) + - [Sync Committee](#sync-committee) + - [Lookahead](#lookahead) +- [Beacon chain responsibilities](#beacon-chain-responsibilities) + - [Block proposal](#block-proposal) + - [Preparing a `BeaconBlock`](#preparing-a-beaconblock) + - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody) + - [Sync committee](#sync-committee) + - [Packaging into a `SignedBeaconBlock`](#packaging-into-a-signedbeaconblock) + - [Attesting and attestation aggregation](#attesting-and-attestation-aggregation) + - [Sync committees](#sync-committees) + - [Sync committee signatures](#sync-committee-signatures) + - [Prepare sync committee signature](#prepare-sync-committee-signature) + - [Broadcast sync committee signature](#broadcast-sync-committee-signature) + - [Sync committee contributions](#sync-committee-contributions) + - [Aggregation selection](#aggregation-selection) + - [Construct sync committee contribution](#construct-sync-committee-contribution) + - [Slot](#slot) + - [Beacon block root](#beacon-block-root) + - [Subcommittee index](#subcommittee-index) + - [Aggregate bits](#aggregate-bits) + - [Signature](#signature) + - [Broadcast sync committee contribution](#broadcast-sync-committee-contribution) +- [Sync committee subnet stability](#sync-committee-subnet-stability) + + + + +## Introduction + +This document represents the expected behavior of an "honest validator" with respect to Altair of the Ethereum 2.0 protocol. +It builds on the [previous document for the behavior of an "honest validator" from Phase 0](../phase0/validator.md) of the Ethereum 2.0 protocol. +This previous document is referred to below as the "Phase 0 document". + +Altair introduces a new type of committee: the sync committee. Sync committees are responsible for signing each block of the canonical chain and there exists an efficient algorithm for light clients to sync the chain using the output of the sync committees. +See the [sync protocol](./sync-protocol.md) for further details on the light client sync. +Under this network upgrade, validators track their participation in this new committee type and produce the relevant signatures as required. +Block proposers incorporate the (aggregated) sync committee signatures into each block they produce. + +## Prerequisites + +All terminology, constants, functions, and protocol mechanics defined in the [Altair -- The Beacon Chain](./beacon-chain.md) doc are requisite for this document and used throughout. +Please see this document before continuing and use as a reference throughout. + +## Warning + +This document is currently illustrative for early Altair testnets and some parts are subject to change, especially pending implementation and profiling of Altair testnets. + +## Constants + +### Misc + +| Name | Value | Unit | +| - | - | :-: | +| `TARGET_AGGREGATORS_PER_SYNC_SUBCOMMITTEE` | `2**2` (= 4) | validators | +| `SYNC_COMMITTEE_SUBNET_COUNT` | `8` | The number of sync committee subnets used in the gossipsub aggregation protocol. | + +## Containers + +### `SyncCommitteeSignature` + +```python +class SyncCommitteeSignature(Container): + # Slot to which this contribution pertains + slot: Slot + # Block root for this signature + beacon_block_root: Root + # Index of the validator that produced this signature + validator_index: ValidatorIndex + # Signature by the validator over the block root of `slot` + signature: BLSSignature +``` + +### `SyncCommitteeContribution` + +```python +class SyncCommitteeContribution(Container): + # Slot to which this contribution pertains + slot: Slot + # Block root for this contribution + beacon_block_root: Root + # The subcommittee this contribution pertains to out of the broader sync committee + subcommittee_index: uint64 + # A bit is set if a signature from the validator at the corresponding + # index in the subcommittee is present in the aggregate `signature`. + aggregation_bits: Bitvector[SYNC_COMMITTEE_SIZE // SYNC_COMMITTEE_SUBNET_COUNT] + # Signature by the validator(s) over the block root of `slot` + signature: BLSSignature +``` + +### `ContributionAndProof` + +```python +class ContributionAndProof(Container): + aggregator_index: ValidatorIndex + contribution: SyncCommitteeContribution + selection_proof: BLSSignature +``` + +### `SignedContributionAndProof` + +```python +class SignedContributionAndProof(Container): + message: ContributionAndProof + signature: BLSSignature +``` + +## Validator assignments + +A validator determines beacon committee assignments and beacon block proposal duties as defined in the Phase 0 document. + +### Sync Committee + +To determine sync committee assignments, a validator can run the following function: `is_assigned_to_sync_committee(state, epoch, validator_index)` where `epoch` is an epoch number within the current or next sync committee period. +This function is a predicate indicating the presence or absence of the validator in the corresponding sync committee for the queried sync committee period. + +```python +def compute_sync_committee_period(epoch: Epoch) -> uint64: + return epoch // EPOCHS_PER_SYNC_COMMITTEE_PERIOD + +def is_assigned_to_sync_committee(state: BeaconState, + epoch: Epoch, + validator_index: ValidatorIndex) -> bool: + sync_committee_period = compute_sync_committee_period(epoch) + current_epoch = get_current_epoch(state) + current_sync_committee_period = compute_sync_committee_period(current_epoch) + next_sync_committee_period = current_sync_committee_period + 1 + assert sync_committee_period in (current_sync_committee_period, next_sync_committee_period) + + pubkey = state.validators[validator_index].pubkey + if sync_committee_period == current_sync_committee_period: + return pubkey in state.current_sync_committee.pubkeys + else: # sync_committee_period == next_sync_committee_period + return pubkey in state.next_sync_committee.pubkeys +``` + +### Lookahead + +The sync committee shufflings give validators 1 sync committee period of lookahead which amounts to `EPOCHS_PER_SYNC_COMMITTEE_PERIOD` epochs. +At any given `epoch`, the `BeaconState` contains the current `SyncCommittee` and the next `SyncCommittee`. +Once every `EPOCHS_PER_SYNC_COMMITTEE_PERIOD` epochs, the next `SyncCommittee` becomes the current `SyncCommittee` and the next committee is computed and stored. + +*Note*: The data required to compute a given committee is not cached in the `BeaconState` after committees are calculated at the period boundaries. +This means that calling `get_sync_commitee()` in a given `epoch` can return a different result than what was computed during the relevant epoch transition. +For this reason, *always* get committee assignments via the fields of the `BeaconState` (`current_sync_committee` and `next_sync_committee`) or use the above reference code. + +A validator should plan for future sync committee assignments by noting which sync committee periods they are selected for participation. +Specifically, a validator should: +* Upon (re)syncing the chain and upon sync committee period boundaries, check for assignments in the current and next sync committee periods. +* If the validator is in the current sync committee period, they can perform the responsibilities below for sync committee rewards. +* If the validator is in the next sync committee period, they should wait until the next `EPOCHS_PER_SYNC_COMMITTEE_PERIOD` boundary and then perform the responsibilities throughout that period. + +## Beacon chain responsibilities + +A validator maintains the responsibilities given in the Phase 0 document. + +Block proposals are modified to incorporate the sync committee signatures as detailed below. + +When assigned to a sync committee, validators have a new responsibility to sign beacon block roots during each slot of the sync committee period. +These signatures are aggregated and routed to the proposer over gossip for inclusion into a beacon block. +Assignments to a particular sync committee are infrequent at normal validator counts; however, an action every slot is required when in the current active sync committee. + +### Block proposal + +Refer to the phase 0 document for the majority of the [block proposal responsibility](../phase0/validator.md#block-proposal). +The validator should follow those instructions to prepare a `SignedBeaconBlock` for inclusion into the chain. All changes are additive to phase 0 and noted below. + +#### Preparing a `BeaconBlock` + +No change to [Preparing for a `BeaconBlock`](../phase0/validator.md#preparing-for-a-beaconblock). + +#### Constructing the `BeaconBlockBody` + +Each section of [Constructing the `BeaconBlockBody`](../phase0/validator.md#constructing-the-beaconblockbody) should be followed. +After constructing the `BeaconBlockBody` as per that section, the proposer has an additional task to include the sync committee signatures: + +##### Sync committee + +The proposer receives a number of `SyncCommitteeContribution`s (wrapped in `SignedContributionAndProof`s on the wire) from validators in the sync committee who are selected to partially aggregate signatures from independent subcommittees formed by breaking the full sync committee into `SYNC_COMMITTEE_SUBNET_COUNT` pieces (see below for details). + +The proposer collects these contributions for further aggregation when preparing a block. +Proposers should select the best contribution seen across all aggregators for each subnet/subcommittee when preparing a block. +A contribution with more valid signatures is better than a contribution with fewer signatures. + +Recall `block.body.sync_aggregate.sync_committee_bits` is a `Bitvector` where the `i`th bit is `True` if the corresponding validator in the sync committee has produced a valid signature, +and that `block.body.sync_aggregate.sync_committee_signature` is the aggregate BLS signature combining all of the valid signatures. + +Given a collection of the best seen `contributions` (with no repeating `subcommittee_index` values) and the `BeaconBlock` under construction, +the proposer processes them as follows: + +```python +def process_sync_committee_contributions(state: BeaconState, + block: BeaconBlock, + contributions: Set[SyncCommitteeContribution]) -> None: + sync_aggregate = SyncAggregate() + signatures = [] + + for contribution in contributions: + subcommittee_index = contribution.subcommittee_index + for index, participated in enumerate(contribution.aggregation_bits): + if participated: + participant_index = SYNC_COMMITTEE_SIZE // SYNC_COMMITTEE_SUBNET_COUNT * subcommittee_index + index + sync_aggregate.sync_committee_bits[participant_index] = True + signatures.append(contribution.signature) + + sync_aggregate.sync_committee_signature = bls.Aggregate(signatures) + + block.body.sync_aggregate = sync_aggregate +``` + +*Note*: The resulting block must pass the validations for the `SyncAggregate` defined in `process_sync_committee` defined in the [state transition document](./beacon-chain.md#sync-committee-processing). +In particular, this means `SyncCommitteeContribution`s received from gossip must have a `beacon_block_root` that matches the proposer's local view of the chain. + +#### Packaging into a `SignedBeaconBlock` + +No change to [Packaging into a `SignedBeaconBlock`](../phase0/validator.md#packaging-into-a-signedbeaconblock). + +### Attesting and attestation aggregation + +Refer to the phase 0 document for the [attesting](../phase0/validator.md#attesting) and [attestation aggregation](../phase0/validator.md#attestation-aggregation) responsibilities. +There is no change compared to the phase 0 document. + +### Sync committees + +Sync committee members employ an aggregation scheme to reduce load on the global proposer channel that is monitored by all potential proposers to be able to include the full output of the sync committee every slot. +Sync committee members produce individual signatures on subnets (similar to the attestation subnets) via `SyncCommitteeSignature`s which are then collected by aggregators sampled from the sync subcommittees to produce a `SyncCommitteeContribution` which is gossiped to proposers. +This process occurs each slot. + +#### Sync committee signatures + +##### Prepare sync committee signature + +If a validator is in the current sync committee (i.e. `is_assigned_to_sync_committee` above returns `True`), then for every slot in the current sync committee period the validator should prepare a `SyncCommitteeSignature` according to the logic in `get_sync_committee_signature` as soon as they have determined the head block of the current slot. + +This logic is triggered upon the same conditions as when producing an attestation. +Meaning, a sync committee member should produce and broadcast a `SyncCommitteeSignature` either when (a) the validator has received a valid block from the expected block proposer for the current `slot` or (b) one-third of the slot has transpired (`SECONDS_PER_SLOT / 3` seconds after the start of the slot) -- whichever comes first. + +`get_sync_committee_signature` assumes `state` is the head state corresponding to processing the block at the current slot as determined by the fork choice (including any empty slots processed with `process_slots`), `validator_index` is the index of the validator in the registry `state.validators` controlled by `privkey`, and `privkey` is the BLS private key for the validator. + +```python +def get_sync_committee_signature(state: BeaconState, + validator_index: ValidatorIndex, + privkey: int) -> SyncCommitteeSignature: + epoch = get_current_epoch(state) + domain = get_domain(state, DOMAIN_SYNC_COMMITTEE, epoch) + signing_root = compute_signing_root(get_block_root_at_slot(state, state.slot), domain) + signature = bls.Sign(privkey, signing_root) + + return SyncCommitteeSignature(slot=state.slot, validator_index=validator_index, signature=signature) +``` + +##### Broadcast sync committee signature + +The validator broadcasts the assembled signature to the assigned subnet, the `sync_committee_{subnet_id}` pubsub topic. + +The `subnet_id` is derived from the position in the sync committee such that the sync committee is divided into "subcommittees". +It can be computed via `compute_subnets_for_sync_committee` where `state` is a `BeaconState` during the matching sync committee period. +This function returns multiple subnets if a given validator index is included multiple times in a given sync committee across multiple subcommittees. + +```python +def compute_subnets_for_sync_committee(state: BeaconState, validator_index: ValidatorIndex) -> Sequence[uint64]: + target_pubkey = state.validators[validator_index].pubkey + sync_committee_indices = [index for index, pubkey in enumerate(state.current_sync_committee.pubkeys) if pubkey == target_pubkey] + return [ + uint64(index // (SYNC_COMMITEE_SIZE // SYNC_COMMITTEE_SUBNET_COUNT)) + for index in sync_committee_indices + ] +``` + +*Note*: Subnet assignment does not change during the duration of a validator's assignment to a given sync committee. + +*Note*: If a validator has multiple `subnet_id` results from `compute_subnets_for_sync_committee`, the validator should broadcast a copy of the `sync_committee_signature` on each of the distinct subnets. + +#### Sync committee contributions + +Each slot some sync committee members in each subcommittee are selected to aggregate the `SyncCommitteeSignature`s into a `SyncCommitteeContribution` which is broadcast on a global channel for inclusion into the next block. + +##### Aggregation selection + +A validator is selected to aggregate based on the computation in `is_sync_committee_aggregator` where `state` is a `BeaconState` as supplied to `get_sync_committee_slot_signature` and `signature` is the BLS signature returned by `get_sync_committee_slot_signature`. +The signature function takes a `BeaconState` with the relevant sync committees for the queried `slot` (i.e. `state.slot` is within the span covered by the current or next sync committee period), the `subcommittee_index` equal to the `subnet_id`, and the `privkey` is the BLS private key associated with the validator. + +```python +class SyncCommitteeSigningData(Container): + slot: Slot + subcommittee_index: uint64 +``` + +```python +def get_sync_committee_slot_signature(state: BeaconState, slot: Slot, subcommittee_index: uint64, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF, compute_epoch_at_slot(slot)) + signing_data = SyncCommitteeSigningData( + slot=slot, + subcommittee_index=subcommittee_index, + ) + signing_root = compute_signing_root(signing_data, domain) + return bls.Sign(privkey, signing_root) +``` + +```python +def is_sync_committee_aggregator(state: BeaconState, signature: BLSSignature) -> bool: + modulo = max(1, SYNC_COMMITTEE_SIZE // SYNC_COMMITTEE_SUBNET_COUNT // TARGET_AGGREGATORS_PER_SYNC_SUBCOMMITTEE) + return bytes_to_uint64(hash(signature)[0:8]) % modulo == 0 +``` + +*NOTE*: the set of aggregators generally changes every slot; however, the assignments can be computed ahead of time as soon as the committee is known + +##### Construct sync committee contribution + +If a validator is selected to aggregate the `SyncCommitteeSignature`s produced on a subnet during a given `slot`, they construct an aggregated `SyncCommitteeContribution`. + +Given all of the (valid) collected `sync_committee_signatures: Set[SyncCommitteeSignature]` from the `sync_committee_{subnet_id}` gossip during the selected `slot` with an equivalent `beacon_block_root` to that of the aggregator, the aggregator creates a `contribution: SyncCommitteeContribution` with the following fields: + +###### Slot + +Set `contribution.slot = state.slot` where `state` is the `BeaconState` for the slot in question. + +###### Beacon block root + +Set `contribution.beacon_block_root = beacon_block_root` from the `beacon_block_root` found in the `sync_committee_signatures`. + +###### Subcommittee index + +Set `contribution.subcommittee_index` to the index for the subcommittee index corresponding to the subcommittee assigned to this subnet. This index matches the `subnet_id` used to derive the topic name. + +###### Aggregate bits + +Let `contribution.aggregation_bits` be a `Bitvector[SYNC_COMMITTEE_SIZE // SYNC_COMMITTEE_SUBNET_COUNT]`, where the `index`th bit is set in the `Bitvector` for each corresponding validator included in this aggregate from the corresponding subcommittee. +An aggregator needs to find the index in the sync committee (as returned by `get_sync_committee_indices`) for a given validator referenced by `sync_committee_signature.validator_index` and map the sync committee index to an index in the subcommittee (along with the prior `subcommittee_index`). This index within the subcommittee is the one set in the `Bitvector`. +For example, a validator with index `2044` could be at index `15` in the current sync committee. This sync committee index maps to `subcommittee_index` `1` with position `7` in the `Bitvector` for the contribution. +Also note that a validator **could be included multiple times** in a given subcommittee such that multiple bits are set for a single `SyncCommitteeSignature`. + +###### Signature + +Set `contribution.signature = aggregate_signature` where `aggregate_signature` is obtained by assembling the appropriate collection of `BLSSignature`s from the set of `sync_committee_signatures` and using the `bls.Aggregate` function to produce an aggregate `BLSSignature`. +The collection of input signatures should include one signature per validator who had a bit set in the `aggregation_bits` bitfield, with repeated signatures if one validator maps to multiple indices within the subcommittee. + +##### Broadcast sync committee contribution + +If the validator is selected to aggregate (`is_sync_committee_aggregator`), then they broadcast their best aggregate as a `SignedContributionAndProof` to the global aggregate channel (`sync_committee_contribution_and_proof` topic) two-thirds of the way through the `slot`-that is, `SECONDS_PER_SLOT * 2 / 3` seconds after the start of `slot`. + +Selection proofs are provided in `ContributionAndProof` to prove to the gossip channel that the validator has been selected as an aggregator. + +`ContributionAndProof` messages are signed by the aggregator and broadcast inside of `SignedContributionAndProof` objects to prevent a class of DoS attacks and message forgeries. + +First, `contribution_and_proof = get_contribution_and_proof(state, validator_index, contribution, privkey)` is constructed. + +```python +def get_contribution_and_proof(state: BeaconState, + aggregator_index: ValidatorIndex, + contribution: SyncCommitteeContribution, + privkey: int) -> ContributionAndProof: + selection_proof = get_sync_committee_slot_signature( + state, + contribution.slot, + contribution.subcommittee_index, + privkey, + ) + return ContributionAndProof( + aggregator_index=aggregator_index, + contribution=contribution, + selection_proof=selection_proof, + ) +``` + +Then `signed_contribution_and_proof = SignedContributionAndProof(message=contribution_and_proof, signature=signature)` is constructed and broadast. Where `signature` is obtained from: + +```python +def get_contribution_and_proof_signature(state: BeaconState, + contribution_and_proof: ContributionAndProof, + privkey: int) -> BLSSignature: + contribution = contribution_and_proof.contribution + domain = get_domain(state, DOMAIN_CONTRIBUTION_AND_PROOF, compute_epoch_at_slot(contribution.slot)) + signing_root = compute_signing_root(contribution_and_proof, domain) + return bls.Sign(privkey, signing_root) +``` + +## Sync committee subnet stability + +The sync committee subnets need special care to ensure stability given the relatively low number of validators involved in the sync committee at any particular time. +To provide this stability, a validator must do the following: + +* Maintain advertisement of the subnet the validator in the sync committee is assigned to in their node's ENR as soon as they have joined the subnet. +Subnet assignments are known `EPOCHS_PER_SYNC_COMMITTEE_PERIOD` epochs in advance and can be computed with `compute_subnets_for_sync_committee` defined above. +ENR advertisement is indicated by setting the appropriate bit(s) of the bitfield found under the `syncnets` key in the ENR corresponding to the derived `subnet_id`(s). +Any bits modified for the sync committee responsibilities are unset in the ENR after any validators have left the sync committee. + +*Note*: The first sync committee from phase 0 to the Altair fork will not be known until the fork happens which implies subnet assignments are not known until then. +Early sync committee members should listen for topic subscriptions from peers and employ discovery via the ENR advertisements near the fork boundary to form initial subnets +Some early sync committee rewards may be missed while the initial subnets form. + +* To join a sync committee subnet, select a random number of epochs before the end of the current sync committee period between 1 and `SYNC_COMMITTEE_SUBNET_COUNT`, inclusive. +Validators should join their member subnet at the beginning of the epoch they have randomly selected. +For example, if the next sync committee period starts at epoch `853,248` and the validator randomly selects an offset of `3`, they should join the subnet at the beginning of epoch `853,245`. +Validators should leverage the lookahead period on sync committee assignments so that they can join the appropriate subnets ahead of their assigned sync committee period. \ No newline at end of file From 0fe5e1ef6e5a31613a2c82d5fb06dc987b7167f7 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 16 Mar 2021 00:32:54 +0100 Subject: [PATCH 221/222] update of req-resp, v2 sync methods, gossip topics --- specs/altair/p2p-interface.md | 112 +++++++++++++++++++++++++++++++--- 1 file changed, 102 insertions(+), 10 deletions(-) diff --git a/specs/altair/p2p-interface.md b/specs/altair/p2p-interface.md index ba45eb578..c087fd49a 100644 --- a/specs/altair/p2p-interface.md +++ b/specs/altair/p2p-interface.md @@ -1,11 +1,11 @@ # Ethereum Altair networking specification -This document contains the networking specification for Ethereum 2.0 clients added during the Altair deployment. -This document should be viewed as additive to the [document from Phase 0](../phase0/p2p-interface.md) and will be referred to as the "phase 0 document" hereafter. -Readers should understand the phase 0 document and use it as a basis to understand the changes outlined in this document. +This document contains the networking specification for Ethereum 2.0 clients added during the Altair deployment. +This document should be viewed as additive to the [document from Phase 0](../phase0/p2p-interface.md) and will be referred to as the "Phase 0 document" hereafter. +Readers should understand the Phase 0 document and use it as a basis to understand the changes outlined in this document. + +Altair adds new messages, topics and data to the Req-Resp, Gossip and Discovery domain. Some Phase 0 features will be deprecated, but not removed immediately. -In particular, Altair introduces changes to the gossip domain and a small change to ENR advertisement in the discovery domain. -The phase 0 document should be consulted for specs on the network fundamentals, the existing gossip domain, the req/resp domain and the discovery domain. ## Table of contents @@ -23,7 +23,14 @@ The phase 0 document should be consulted for specs on the network fundamentals, - [Sync committee subnets](#sync-committee-subnets) - [`sync_committee_{subnet_id}`](#sync_committee_subnet_id) - [Sync committees and aggregation](#sync-committees-and-aggregation) + - [Transitioning the gossip](#transitioning-the-gossip) - [The Req/Resp domain](#the-reqresp-domain) + - [Req-Resp interaction](#req-resp-interaction) + - [`ForkDigest`-context](#forkdigest-context) + - [Messages](#messages) + - [BeaconBlocksByRange v2](#beaconblocksbyrange-v2) + - [BeaconBlocksByRoot v2](#beaconblocksbyroot-v2) + - [Transitioning from v1 to v2](#transitioning-from-v1-to-v2) - [The discovery domain: discv5](#the-discovery-domain-discv5) @@ -43,7 +50,7 @@ Validators use an aggregation scheme to balance the processing and networking lo ### Topics and messages -Topics follow the same specification as in the phase 0 document. +Topics follow the same specification as in the Phase 0 document. New topics are added in Altair to support the sync committees and the beacon block topic is updated with the modified type. The specification around the creation, validation, and dissemination of messages has not changed from the Phase 0 document. @@ -58,14 +65,18 @@ The new topics along with the type of the `data` field of a gossipsub message ar Definitions of these new types can be found in the [Altair validator guide](./validator.md#containers). +Note that the `ForkDigestValue` path segment of the topic separates the old and the new `beacon_block` topics. + #### Global topics Altair changes the type of the global beacon block topic and adds one global topic to propagate partially aggregated sync committee signatures to all potential proposers of beacon blocks. ##### `beacon_block` -The existing specification for this topic does not change from the Phase 0 document, but the type of the payload does change to the (modified) `SignedBeaconBlock`. -This type changes due to the inclusion of the inner `BeaconBlockBody` that is modified in Altair. +The existing specification for this topic does not change from the Phase 0 document, +but the type of the payload does change to the (modified) `SignedBeaconBlock`. +This type changes due to the inclusion of the inner `BeaconBlockBody` that is modified in Altair. + See the [state transition document](./beacon-chain.md#beaconblockbody) for Altair for further details. ##### `sync_committee_contribution_and_proof` @@ -114,13 +125,94 @@ Unaggregated signatures (along with metadata) are sent as `SyncCommitteeSignatur Aggregated sync committee signatures are packaged into (signed) `SyncCommitteeContribution` along with proofs and gossiped to the `sync_committee_contribution_and_proof` topic. +### Transitioning the gossip + +With any fork, the fork version, and thus the `ForkDigestValue`, change. +Message types are unique per topic, and so for a smooth transition a node must temporarily subscribe to both the old and new topics. + +The topics that are not removed in a fork are updated with a new `ForkDigestValue`. In advance of the fork, a node SHOULD subscribe to the post-fork variants of the topics. + +Subscriptions are expected to be well-received, all updated nodes should subscribe as well. +Topic-meshes can be grafted quickly as the nodes are already connected and exchanging gossip control messages. + +Messages SHOULD NOT be re-broadcast from one fork to the other. +A node's behavior before the fork and after the fork are as follows: +Pre-fork: +- Peers who propagate messages on the post-fork topics MAY be scored negatively proportionally to time till fork, + to account for clock discrepancy. +- Messages can be IGNORED on the post-fork topics, with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` margin. + +Post-fork: +- Peers who propagate messages on the pre-fork topics MUST NOT be scored negatively. Lagging IWANT may force them to. +- Messages on pre and post-fork variants of topics share application-level caches. + E.g. an attestation on the both the old and new topic is ignored like any duplicate. +- Two epochs after the fork, pre-fork topics SHOULD be unsubscribed from. This is well after the configured `seen_ttl`. + ## The Req/Resp domain -* (TODO) specify how to handle different types of blocks on RPC responses. +### Req-Resp interaction + +An additional `` field is introduced to the `response_chunk` as defined in the Phase 0 document: + +``` +response_chunk ::= | | | +``` + +All Phase 0 methods are compatible: `` is empty by default. +On a non-zero `` with `ErrorMessage` payload, the `` is also empty. + +In Altair and later forks, `` functions as a short meta-data, +defined per req-resp method, and can parametrize the payload decoder. + +#### `ForkDigest`-context + +Starting with Altair, and in future forks, SSZ type definitions may change. +For this common case, we define the `ForkDigest`-context: + +A fixed-width 4 byte ``, set to the `ForkDigest` matching the chunk: + `compute_fork_digest(fork_version, genesis_validators_root)`. + +### Messages + +#### BeaconBlocksByRange v2 + +**Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks_by_range/2/` + +Request and Response remain unchanged. A `ForkDigest`-context is used to select the fork namespace of the Response type. + +Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: + +| `fork_version` | Chunk SSZ type | +| `GENESIS_FORK_VERSION` | `phase0.SignedBeaconBlock` | +| `ALTAIR_FORK_VERSION` | `altair.SignedBeaconBlock` | + +#### BeaconBlocksByRoot v2 + +**Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks_by_root/2/` + +Request and Response remain unchanged. A `ForkDigest`-context is used to select the fork namespace of the Response type. + +Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: + +| `fork_version` | Chunk SSZ type | +| `GENESIS_FORK_VERSION` | `phase0.SignedBeaconBlock` | +| `ALTAIR_FORK_VERSION` | `altair.SignedBeaconBlock` | + +### Transitioning from v1 to v2 + +In advance of the fork, implementations can opt in to both run the v1 and v2 for a smooth transition. +This is non-breaking, and is recommended as soon as the fork specification is stable. + +The v1 variants will be deprecated, and implementations should use v2 when available +(as negotiatied with peers via LibP2P multistream-select). + +The v1 method MAY be unregistered at the fork boundary. +In the event of a request on v1 for an Altair specific payload, +the responder MUST return the **InvalidRequest** response code. ## The discovery domain: discv5 -The `attnets` key of the ENR is used as defined in the phase 0 document. +The `attnets` key of the ENR is used as defined in the Phase 0 document. An additional bitfield is added to the ENR under the key `syncnets` to facilitate sync committee subnet discovery. The length of this bitfield is `SYNC_COMMITTEE_SUBNET_COUNT` where each bit corresponds to a distinct `subnet_id` for a specific sync committee subnet. From 7e1f7c392ea4244b6f3be932ebbadfc4ae3bb5c2 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 16 Mar 2021 20:03:19 -0600 Subject: [PATCH 222/222] bump version.txt to 1.1.0-alpha.1 --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index 7f207341d..5655aea02 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -1.0.1 \ No newline at end of file +1.1.0-alpha.1 \ No newline at end of file