Merge branch 'dev' into JustinDrake-patch-14

This commit is contained in:
Justin
2019-05-07 10:16:38 +01:00
committed by GitHub
51 changed files with 1142 additions and 816 deletions

View File

@@ -0,0 +1 @@
ruamel.yaml==0.15.87

View File

@@ -1,20 +1,9 @@
from setuptools import setup, find_packages
deps = {
'preset_loader': [
"ruamel.yaml==0.15.87",
],
}
deps['dev'] = (
deps['preset_loader']
)
install_requires = deps['preset_loader']
from distutils.core import setup
setup(
name='config_helpers',
packages=find_packages(exclude=["tests", "tests.*"]),
install_requires=install_requires,
packages=['preset_loader'],
install_requires=[
"ruamel.yaml==0.15.87"
]
)

View File

@@ -1,4 +1,10 @@
from typing import Callable, Dict, Tuple, Any
from typing import (
Any,
Callable,
Dict,
Tuple,
)
TestCase = Dict[str, Any]
TestSuite = Dict[str, Any]

View File

@@ -0,0 +1,2 @@
ruamel.yaml==0.15.87
eth-utils==1.4.1

View File

@@ -1,21 +1,10 @@
from setuptools import setup, find_packages
deps = {
'gen_base': [
"ruamel.yaml==0.15.87",
"eth-utils==1.4.1",
],
}
deps['dev'] = (
deps['gen_base']
)
install_requires = deps['gen_base']
from distutils.core import setup
setup(
name='gen_helpers',
packages=find_packages(exclude=["tests", "tests.*"]),
install_requires=install_requires,
packages=['gen_base'],
install_requires=[
"ruamel.yaml==0.15.87",
"eth-utils==1.4.1"
]
)

View File

@@ -1,11 +1,11 @@
# ETH 2.0 PySpec
# Eth 2.0 Executable Python Spec (PySpec)
The Python executable spec is built from the ETH 2.0 specification,
The executable Python spec is built from the Eth 2.0 specification,
complemented with the necessary helper functions for hashing, BLS, and more.
With this executable spec,
test-generators can easily create test-vectors for client implementations,
and the spec itself can be verified to be consistent and coherent, through sanity tests implemented with pytest.
and the spec itself can be verified to be consistent and coherent through sanity tests implemented with pytest.
## Building
@@ -14,12 +14,12 @@ All the dynamic parts of the spec can be build at once with `make pyspec`.
Alternatively, you can build a sub-set of the pyspec: `make phase0`.
Or, to build a single file, specify the path, e.g. `make test_libs/pyspec/eth2spec/phase0/spec.py`
Or, to build a single file, specify the path, e.g. `make test_libs/pyspec/eth2spec/phase0/spec.py`.
## Py-tests
After building, you can install the dependencies for running the `pyspec` tests with `make install_test`
After building, you can install the dependencies for running the `pyspec` tests with `make install_test`.
These tests are not intended for client-consumption.
These tests are sanity tests, to verify if the spec itself is consistent.
@@ -28,7 +28,7 @@ These tests are sanity tests, to verify if the spec itself is consistent.
#### Automated
Run `make test` from the root of the spec repository.
Run `make test` from the root of the specs repository.
#### Manual
@@ -38,9 +38,9 @@ Install dependencies:
```bash
python3 -m venv venv
. venv/bin/activate
pip3 install -e .[dev]
pip3 install -r requirements-testing.txt
```
Note: make sure to run `make -B pyspec` from the root of the specs repository,
*Note*: Make sure to run `make -B pyspec` from the root of the specs repository,
to build the parts of the pyspec module derived from the markdown specs.
The `-B` flag may be helpful to force-overwrite the `pyspec` output after you made a change to the markdown source files.
@@ -58,4 +58,4 @@ The pyspec is not a replacement.
## License
Same as the spec itself, see [LICENSE](../../LICENSE) file in spec repository root.
Same as the spec itself; see [LICENSE](../../LICENSE) file in the specs repository root.

View File

@@ -3,7 +3,7 @@ from typing import Any
from .hash_function import hash
BYTES_PER_CHUNK = 32
BYTES_PER_LENGTH_PREFIX = 4
BYTES_PER_LENGTH_OFFSET = 4
ZERO_CHUNK = b'\x00' * BYTES_PER_CHUNK
@@ -111,19 +111,34 @@ def coerce_to_bytes(x):
raise Exception("Expecting bytes")
def encode_bytes(value):
serialized_bytes = coerce_to_bytes(value)
assert len(serialized_bytes) < 2 ** (8 * BYTES_PER_LENGTH_PREFIX)
serialized_length = len(serialized_bytes).to_bytes(BYTES_PER_LENGTH_PREFIX, 'little')
return serialized_length + serialized_bytes
def encode_series(values, types):
# Recursively serialize
parts = [(is_constant_sized(types[i]), serialize_value(values[i], types[i])) for i in range(len(values))]
# Compute and check lengths
fixed_lengths = [len(serialized) if constant_size else BYTES_PER_LENGTH_OFFSET
for (constant_size, serialized) in parts]
variable_lengths = [len(serialized) if not constant_size else 0
for (constant_size, serialized) in parts]
def encode_variable_size_container(values, types):
return encode_bytes(encode_fixed_size_container(values, types))
# Check if integer is not out of bounds (Python)
assert sum(fixed_lengths + variable_lengths) < 2 ** (BYTES_PER_LENGTH_OFFSET * 8)
# Interleave offsets of variable-size parts with fixed-size parts.
# Avoid quadratic complexity in calculation of offsets.
offset = sum(fixed_lengths)
variable_parts = []
fixed_parts = []
for (constant_size, serialized) in parts:
if constant_size:
fixed_parts.append(serialized)
else:
fixed_parts.append(offset.to_bytes(BYTES_PER_LENGTH_OFFSET, 'little'))
variable_parts.append(serialized)
offset += len(serialized)
def encode_fixed_size_container(values, types):
return b''.join([serialize_value(v, typ) for (v, typ) in zip(values, types)])
# Return the concatenation of the fixed-size parts (offsets interleaved) with the variable-size parts
return b"".join(fixed_parts + variable_parts)
def serialize_value(value, typ=None):
@@ -142,18 +157,13 @@ def serialize_value(value, typ=None):
elif isinstance(typ, list) and len(typ) == 2:
# (regardless of element type, sanity-check if the length reported in the vector type matches the value length)
assert len(value) == typ[1]
# If value is fixed-size (i.e. element type is fixed-size):
if is_constant_sized(typ):
return encode_fixed_size_container(value, [typ[0]] * len(value))
# If value is variable-size (i.e. element type is variable-size)
else:
return encode_variable_size_container(value, [typ[0]] * len(value))
# "bytes" (variable size)
elif isinstance(typ, str) and typ == 'bytes':
return encode_bytes(value)
return encode_series(value, [typ[0]] * len(value))
# List
elif isinstance(typ, list) and len(typ) == 1:
return encode_variable_size_container(value, [typ[0]] * len(value))
return encode_series(value, [typ[0]] * len(value))
# "bytes" (variable size)
elif isinstance(typ, str) and typ == 'bytes':
return coerce_to_bytes(value)
# "bytesN" (fixed size)
elif isinstance(typ, str) and len(typ) > 5 and typ[:5] == 'bytes':
assert len(value) == int(typ[5:]), (value, int(typ[5:]))
@@ -162,10 +172,7 @@ def serialize_value(value, typ=None):
elif hasattr(typ, 'fields'):
values = [getattr(value, field) for field in typ.fields.keys()]
types = list(typ.fields.values())
if is_constant_sized(typ):
return encode_fixed_size_container(values, types)
else:
return encode_variable_size_container(values, types)
return encode_series(values, types)
else:
print(value, typ)
raise Exception("Type not recognized")

View File

@@ -0,0 +1,3 @@
-r requirements.txt
pytest>=3.6,<3.7
../config_helpers

View File

@@ -0,0 +1,4 @@
eth-utils>=1.3.0,<2
eth-typing>=2.1.0,<3.0.0
pycryptodome==3.7.3
py_ecc>=1.6.0

View File

@@ -1,28 +1,13 @@
from setuptools import setup, find_packages
deps = {
'pyspec': [
setup(
name='pyspec',
packages=find_packages(),
tests_require=["pytest"],
install_requires=[
"eth-utils>=1.3.0,<2",
"eth-typing>=2.1.0,<3.0.0",
"pycryptodome==3.7.3",
"py_ecc>=1.6.0",
],
'test': [
"pytest>=3.6,<3.7",
],
}
deps['dev'] = (
deps['pyspec'] +
deps['test']
)
install_requires = deps['pyspec']
setup(
name='pyspec',
packages=find_packages(exclude=["tests", "tests.*"]),
install_requires=install_requires,
extras_require=deps,
]
)

View File

@@ -36,8 +36,7 @@ def run_attestation_processing(state, attestation, valid=True):
process_attestation(post_state, attestation)
current_epoch = get_current_epoch(state)
target_epoch = slot_to_epoch(attestation.data.slot)
if target_epoch == current_epoch:
if attestation.data.target_epoch == current_epoch:
assert len(post_state.current_epoch_attestations) == len(state.current_epoch_attestations) + 1
else:
assert len(post_state.previous_epoch_attestations) == len(state.previous_epoch_attestations) + 1

View File

@@ -65,7 +65,7 @@ def test_success_surround(state):
# set attestion1 to surround attestation 2
attester_slashing.attestation_1.data.source_epoch = attester_slashing.attestation_2.data.source_epoch - 1
attester_slashing.attestation_1.data.slot = attester_slashing.attestation_2.data.slot + spec.SLOTS_PER_EPOCH
attester_slashing.attestation_1.data.target_epoch = attester_slashing.attestation_2.data.target_epoch + 1
pre_state, post_state = run_attester_slashing_processing(state, attester_slashing)
@@ -85,7 +85,7 @@ def test_same_data(state):
def test_no_double_or_surround(state):
attester_slashing = get_valid_attester_slashing(state)
attester_slashing.attestation_1.data.slot += spec.SLOTS_PER_EPOCH
attester_slashing.attestation_1.data.target_epoch += 1
pre_state, post_state = run_attester_slashing_processing(state, attester_slashing, False)

View File

@@ -0,0 +1,109 @@
from copy import deepcopy
import pytest
import eth2spec.phase1.spec as spec
from eth2spec.phase1.spec import (
get_current_epoch,
process_randao_key_reveal,
RANDAO_PENALTY_EPOCHS,
CUSTODY_PERIOD_TO_RANDAO_PADDING,
RANDAO_PENALTY_MAX_FUTURE_EPOCHS,
)
from tests.helpers_phase1 import (
get_valid_randao_key_reveal,
)
mark entire file as 'randao_key_reveals'
pytestmark = pytest.mark.randao_key_reveals
def run_randao_key_reveal_processing(state, randao_key_reveal, valid=True):
"""
Run ``process_randao_key_reveal`` returning the pre and post state.
If ``valid == False``, run expecting ``AssertionError``
"""
post_state = deepcopy(state)
if not valid:
with pytest.raises(AssertionError):
process_randao_key_reveal(post_state, randao_key_reveal)
return state, None
process_randao_key_reveal(post_state, randao_key_reveal)
slashed_validator = post_state.validator_registry[randao_key_reveal.revealed_index]
if randao_key_reveal.epoch >= get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING:
assert slashed_validator.slashed
assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH
assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH
# lost whistleblower reward
# FIXME: Currently broken because get_base_reward in genesis epoch is 0
assert (
post_state.balances[randao_key_reveal.revealed_index] <
state.balances[randao_key_reveal.revealed_index]
)
return state, post_state
def test_success(state):
randao_key_reveal = get_valid_randao_key_reveal(state)
pre_state, post_state = run_randao_key_reveal_processing(state, randao_key_reveal)
return pre_state, randao_key_reveal, post_state
def test_reveal_from_current_epoch(state):
randao_key_reveal = get_valid_randao_key_reveal(state, get_current_epoch(state))
pre_state, post_state = run_randao_key_reveal_processing(state, randao_key_reveal, False)
return pre_state, randao_key_reveal, post_state
# Not currently possible as we are testing at epoch 0
#
#def test_reveal_from_past_epoch(state):
# randao_key_reveal = get_valid_randao_key_reveal(state, get_current_epoch(state) - 1)
#
# pre_state, post_state = run_randao_key_reveal_processing(state, randao_key_reveal, False)
#
# return pre_state, randao_key_reveal, post_state
def test_reveal_with_custody_padding(state):
randao_key_reveal = get_valid_randao_key_reveal(state, get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING)
pre_state, post_state = run_randao_key_reveal_processing(state, randao_key_reveal, True)
return pre_state, randao_key_reveal, post_state
def test_reveal_with_custody_padding_minus_one(state):
randao_key_reveal = get_valid_randao_key_reveal(state, get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING - 1)
pre_state, post_state = run_randao_key_reveal_processing(state, randao_key_reveal, True)
return pre_state, randao_key_reveal, post_state
def test_double_reveal(state):
randao_key_reveal1 = get_valid_randao_key_reveal(state, get_current_epoch(state) + RANDAO_PENALTY_EPOCHS + 1)
pre_state, intermediate_state = run_randao_key_reveal_processing(state, randao_key_reveal1)
randao_key_reveal2 = get_valid_randao_key_reveal(intermediate_state, get_current_epoch(pre_state) + RANDAO_PENALTY_EPOCHS + 1)
intermediate_state_, post_state = run_randao_key_reveal_processing(intermediate_state, randao_key_reveal2, False)
return pre_state, [randao_key_reveal1, randao_key_reveal2], post_state
def test_revealer_is_slashed(state):
randao_key_reveal = get_valid_randao_key_reveal(state, get_current_epoch(state))
state.validator_registry[randao_key_reveal.revealed_index].slashed = True
pre_state, post_state = run_randao_key_reveal_processing(state, randao_key_reveal, False)
return pre_state, randao_key_reveal, post_state
def test_far_future_epoch(state):
randao_key_reveal = get_valid_randao_key_reveal(state, get_current_epoch(state) + RANDAO_PENALTY_MAX_FUTURE_EPOCHS)
pre_state, post_state = run_randao_key_reveal_processing(state, randao_key_reveal, False)
return pre_state, randao_key_reveal, post_state

View File

@@ -13,7 +13,7 @@ from tests.helpers import (
add_attestation_to_state,
build_empty_block_for_next_slot,
fill_aggregate_attestation,
get_crosslink_committee_for_attestation,
get_crosslink_committee,
get_valid_attestation,
next_epoch,
next_slot,
@@ -86,7 +86,7 @@ def test_single_crosslink_update_from_previous_epoch(state):
assert post_state.previous_crosslinks[shard] != post_state.current_crosslinks[shard]
assert pre_state.current_crosslinks[shard] != post_state.current_crosslinks[shard]
# ensure rewarded
for index in get_crosslink_committee_for_attestation(state, attestation.data):
for index in get_crosslink_committee(state, attestation.data.target_epoch, attestation.data.shard):
assert crosslink_deltas[0][index] > 0
assert crosslink_deltas[1][index] == 0
@@ -127,7 +127,7 @@ def test_double_late_crosslink(state):
# ensure that the current crosslinks were not updated by the second attestation
assert post_state.previous_crosslinks[shard] == post_state.current_crosslinks[shard]
# ensure no reward, only penalties for the failed crosslink
for index in get_crosslink_committee_for_attestation(state, attestation_2.data):
for index in get_crosslink_committee(state, attestation_2.data.target_epoch, attestation_2.data.shard):
assert crosslink_deltas[0][index] == 0
assert crosslink_deltas[1][index] > 0

View File

@@ -0,0 +1,67 @@
from copy import deepcopy
import pytest
import eth2spec.phase0.spec as spec
from eth2spec.phase0.spec import (
get_current_epoch,
is_active_validator,
)
from tests.helpers import (
next_epoch,
)
# mark entire file as 'state'
pytestmark = pytest.mark.state
def test_activation(state):
index = 0
assert is_active_validator(state.validator_registry[index], get_current_epoch(state))
# Mock a new deposit
state.validator_registry[index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH
state.validator_registry[index].activation_epoch = spec.FAR_FUTURE_EPOCH
state.validator_registry[index].effective_balance = spec.MAX_EFFECTIVE_BALANCE
assert not is_active_validator(state.validator_registry[index], get_current_epoch(state))
pre_state = deepcopy(state)
blocks = []
for _ in range(spec.ACTIVATION_EXIT_DELAY + 1):
block = next_epoch(state)
blocks.append(block)
assert state.validator_registry[index].activation_eligibility_epoch != spec.FAR_FUTURE_EPOCH
assert state.validator_registry[index].activation_epoch != spec.FAR_FUTURE_EPOCH
assert is_active_validator(
state.validator_registry[index],
get_current_epoch(state),
)
return pre_state, blocks, state
def test_ejection(state):
index = 0
assert is_active_validator(state.validator_registry[index], get_current_epoch(state))
assert state.validator_registry[index].exit_epoch == spec.FAR_FUTURE_EPOCH
# Mock an ejection
state.validator_registry[index].effective_balance = spec.EJECTION_BALANCE
pre_state = deepcopy(state)
blocks = []
for _ in range(spec.ACTIVATION_EXIT_DELAY + 1):
block = next_epoch(state)
blocks.append(block)
assert state.validator_registry[index].exit_epoch != spec.FAR_FUTURE_EPOCH
assert not is_active_validator(
state.validator_registry[index],
get_current_epoch(state),
)
return pre_state, blocks, state

View File

@@ -26,7 +26,7 @@ from eth2spec.phase0.spec import (
get_attesting_indices,
get_block_root,
get_block_root_at_slot,
get_crosslink_committees_at_slot,
get_crosslink_committee,
get_current_epoch,
get_domain,
get_epoch_start_slot,
@@ -176,11 +176,11 @@ def build_attestation_data(state, slot, shard):
crosslinks = state.current_crosslinks if slot_to_epoch(slot) == get_current_epoch(state) else state.previous_crosslinks
return AttestationData(
slot=slot,
shard=shard,
beacon_block_root=block_root,
source_epoch=justified_epoch,
source_root=justified_block_root,
target_epoch=slot_to_epoch(slot),
target_root=epoch_boundary_root,
crosslink_data_root=spec.ZERO_HASH,
previous_crosslink_root=hash_tree_root(crosslinks[shard]),
@@ -278,14 +278,6 @@ def get_valid_attester_slashing(state):
)
def get_crosslink_committee_for_attestation(state, attestation_data):
"""
Return the crosslink committee corresponding to ``attestation_data``.
"""
crosslink_committees = get_crosslink_committees_at_slot(state, attestation_data.slot)
return [committee for committee, shard in crosslink_committees if shard == attestation_data.shard][0]
def get_valid_attestation(state, slot=None):
if slot is None:
slot = state.slot
@@ -298,7 +290,7 @@ def get_valid_attestation(state, slot=None):
attestation_data = build_attestation_data(state, slot, shard)
crosslink_committee = get_crosslink_committee_for_attestation(state, attestation_data)
crosslink_committee = get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.shard)
committee_size = len(crosslink_committee)
bitfield_length = (committee_size + 7) // 8
@@ -385,13 +377,13 @@ def get_attestation_signature(state, attestation_data, privkey, custody_bit=0b0)
domain=get_domain(
state=state,
domain_type=spec.DOMAIN_ATTESTATION,
message_epoch=slot_to_epoch(attestation_data.slot),
message_epoch=attestation_data.target_epoch,
)
)
def fill_aggregate_attestation(state, attestation):
crosslink_committee = get_crosslink_committee_for_attestation(state, attestation.data)
crosslink_committee = get_crosslink_committee(state, attestation.data.target_epoch, attestation.data.shard)
for i in range(len(crosslink_committee)):
attestation.aggregation_bitfield = set_bitfield_bit(attestation.aggregation_bitfield, i)
@@ -404,11 +396,29 @@ def add_attestation_to_state(state, attestation, slot):
def next_slot(state):
"""
Transition to the next slot via an empty block.
Return the empty block that triggered the transition.
"""
block = build_empty_block_for_next_slot(state)
state_transition(state, block)
return block
def next_epoch(state):
"""
Transition to the start slot of the next epoch via an empty block.
Return the empty block that triggered the transition.
"""
block = build_empty_block_for_next_slot(state)
block.slot += spec.SLOTS_PER_EPOCH - (state.slot % spec.SLOTS_PER_EPOCH)
state_transition(state, block)
return block
def get_state_root(state, slot) -> bytes:
"""
Return the state root at a recent ``slot``.
"""
assert slot < state.slot <= slot + spec.SLOTS_PER_HISTORICAL_ROOT
return state.latest_state_roots[slot % spec.SLOTS_PER_HISTORICAL_ROOT]

View File

@@ -0,0 +1,50 @@
from py_ecc import bls
import eth2spec.phase1.spec as spec
from eth2spec.phase0.spec import (
# constants
ZERO_HASH,
CUSTODY_PERIOD_TO_RANDAO_PADDING,
# SSZ
RandaoKeyReveal,
# functions
get_active_validator_indices,
get_current_epoch,
get_domain,
hash_tree_root,
)
def get_valid_randao_key_reveal(state, epoch=None):
current_epoch = get_current_epoch(state)
revealed_index = get_active_validator_indices(state, current_epoch)[-1]
masker_index = get_active_validator_indices(state, current_epoch)[0]
if epoch is None:
epoch = current_epoch + CUSTODY_PERIOD_TO_RANDAO_PADDING
reveal = bls.sign(
message_hash=hash_tree_root(epoch),
privkey=pubkey_to_privkey[state.validator_registry[revealed_index].pubkey],
domain=get_domain(
state=state,
domain_type=spec.DOMAIN_RANDAO,
message_epoch=epoch,
),
)
mask = bls.sign(
message_hash=hash_tree_root(epoch),
privkey=pubkey_to_privkey[state.validator_registry[masker_index].pubkey],
domain=get_domain(
state=state,
domain_type=spec.DOMAIN_RANDAO,
message_epoch=epoch,
),
)
return RandaoKeyReveal(
revealed_index=revealed_index,
epoch=epoch,
reveal=reveal,
masker_index=masker_index,
mask=mask,
)

View File

@@ -9,6 +9,7 @@ from eth2spec.utils.minimal_ssz import signing_root
from eth2spec.phase0.spec import (
# constants
ZERO_HASH,
SLOTS_PER_HISTORICAL_ROOT,
# SSZ
Deposit,
Transfer,
@@ -17,7 +18,6 @@ from eth2spec.phase0.spec import (
get_active_validator_indices,
get_beacon_proposer_index,
get_block_root_at_slot,
get_state_root,
get_current_epoch,
get_domain,
process_slot,
@@ -36,6 +36,7 @@ from .helpers import (
build_deposit_data,
build_empty_block_for_next_slot,
fill_aggregate_attestation,
get_state_root,
get_valid_attestation,
get_valid_attester_slashing,
get_valid_proposer_slashing,

View File

@@ -1,29 +0,0 @@
from setuptools import setup, find_packages
deps = {
'pyspec': [
"eth-utils>=1.3.0,<2",
"eth-typing>=2.1.0,<3.0.0",
"pycryptodome==3.7.3",
"py_ecc>=1.6.0",
],
'test': [
"pytest>=3.6,<3.7",
],
}
deps['dev'] = (
deps['pyspec'] +
deps['test']
)
install_requires = deps['pyspec']
setup(
name='pyspec',
packages=find_packages(exclude=["tests", "tests.*"]),
install_requires=install_requires,
extras_require=deps,
)