diff --git a/Makefile b/Makefile index 1c7f84ec6..65c8bc33d 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,11 @@ YAML_TEST_TARGETS = $(patsubst $(GENERATOR_DIR)/%, $(YAML_TEST_DIR)/%, $(GENERAT GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENERATORS)) PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase0/spec.py +PY_SPEC_PHASE_0_DEPS = $(SPEC_DIR)/core/0_*.md + PY_SPEC_PHASE_1_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase1/custody_game.py $(PY_SPEC_DIR)/eth2spec/phase1/shard_data_chains.py +PY_SPEC_PHASE_1_DEPS = $(SPEC_DIR)/core/1_*.md + PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) @@ -50,13 +54,13 @@ phase0: $(PY_SPEC_PHASE_0_TARGETS) # "make phase1" to create pyspec for phase1 phase1: $(PY_SPEC_PHASE_1_TARGETS) -$(PY_SPEC_DIR)/eth2spec/phase0/spec.py: +$(PY_SPEC_DIR)/eth2spec/phase0/spec.py: $(PY_SPEC_PHASE_0_DEPS) python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $@ -$(PY_SPEC_DIR)/eth2spec/phase1/custody_game.py: +$(PY_SPEC_DIR)/eth2spec/phase1/custody_game.py: $(PY_SPEC_PHASE_1_DEPS) python3 $(SCRIPT_DIR)/phase0/build_spec.py -p1 $(SPEC_DIR)/core/1_custody-game.md $@ -$(PY_SPEC_DIR)/eth2spec/phase1/shard_data_chains.py: +$(PY_SPEC_DIR)/eth2spec/phase1/shard_data_chains.py: $(PY_SPEC_PHASE_0_DEPS) python3 $(SCRIPT_DIR)/phase0/build_spec.py -p1 $(SPEC_DIR)/core/1_shard-data-chains.md $@ diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index 80b378fbb..e3009004f 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -89,11 +89,29 @@ def build_phase1_spec(sourcefile, outfile): code_lines = [] code_lines.append(""" from eth2spec.phase0.spec import * +from eth2spec.phase0.spec import apply_constants_preset as apply_constants_preset_phase0 """) code_lines += function_puller.get_spec(sourcefile) + code_lines.append(""" +# Access to overwrite spec constants based on configuration +def apply_constants_preset(preset: Dict[str, Any]): + + apply_constants_preset_phase0(preset) + + global_vars = globals() + for k, v in preset.items(): + global_vars[k] = v + + # Deal with derived constants + global_vars['GENESIS_EPOCH'] = slot_to_epoch(GENESIS_SLOT) + + # Initialize SSZ types again, to account for changed lengths + init_SSZ_types() +""") + with open(outfile, 'w') as out: out.write("\n".join(code_lines)) diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index 830e52a32..26ac2f17f 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -39,10 +39,11 @@ def get_spec(file_name: str, phase:int = 0) -> List[str]: else: if pulling_from == linenum and line == '{': if is_update_section: - code_lines.append('%s = SSZTypeExtension(%s, {' % (current_name, current_name)) + code_lines.append('%s = SSZTypeExtension("%s", {' % (current_name, current_name)) + current_typedef = ['global_vars["%s"] = SSZTypeExtension("%s", {' % (current_name, current_name)] else: - code_lines.append('%s = SSZType({' % current_name) - current_typedef = ['global_vars["%s"] = SSZType({' % current_name] + code_lines.append('%s = SSZType("%s", {' % (current_name, current_name)) + current_typedef = ['global_vars["%s"] = SSZType("%s", {' % (current_name, current_name)] elif pulling_from is not None: # Add some whitespace between functions if line[:3] == 'def': diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 5de52a4ea..501bfce46 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -661,12 +661,12 @@ def process_bit_challenge_response(state: BeaconState, ### Handling of custody-related deadlines -Run `process_reveal_deadlines(state)` immediately after `process_ejections(state)`: +Run `process_reveal_deadlines(state)` immediately after `process_registry_updates(state)`: ```python def process_reveal_deadlines(state: BeaconState) -> None: for index, validator in enumerate(state.validator_registry): - if (validator.latest_custody_reveal_period + + if (validator.next_custody_reveal_period + (CUSTODY_RESPONSE_DEADLINE // EPOCHS_PER_CUSTODY_PERIOD) < get_validators_custody_reveal_period(state, index)): slash_validator(state, index) @@ -693,6 +693,7 @@ Append this to `process_final_updates(state)`: ```python def after_process_final_updates(state: BeaconState) -> None: + current_epoch = get_current_epoch(state) # Clean up exposed RANDAO key reveals state.exposed_derived_secrets[current_epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] = [] ``` diff --git a/test_libs/pyspec/eth2spec/phase1/__init__.py b/test_libs/pyspec/eth2spec/phase1/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_libs/pyspec/eth2spec/phase1/state_transition.py b/test_libs/pyspec/eth2spec/phase1/state_transition.py new file mode 100644 index 000000000..7e6411287 --- /dev/null +++ b/test_libs/pyspec/eth2spec/phase1/state_transition.py @@ -0,0 +1,77 @@ +from . import spec + +from typing import ( + Any, + Callable, + List +) + +from .spec import ( + BeaconState, + BeaconBlock, + Slot, +) + +from eth2spec.phase0.state_transition import ( + expected_deposit_count, + process_operation_type, + process_operations as process_operations_phase0, + process_block as process_block_phase0, + process_epoch_transition as process_epoch_transition_phase0, + state_transition_to as state_transition_to_phase0, + state_transition as state_transition_phase0 +) + +def process_operations(state: BeaconState, block: BeaconBlock) -> None: + process_operations_phase0(state, block) + + process_operation_type( + state, + block.body.custody_key_reveals, + spec.MAX_CUSTODY_KEY_REVEALS, + spec.process_custody_key_reveal, + ) + + process_operation_type( + state, + block.body.early_derived_secret_reveals, + spec.MAX_EARLY_DERIVED_SECRET_REVEALS, + spec.process_early_derived_secret_reveal, + ) + +def process_block(state: BeaconState, + block: BeaconBlock, + verify_state_root: bool=False) -> None: + spec.process_block_header(state, block) + spec.process_randao(state, block) + spec.process_eth1_data(state, block) + + process_operations(state, block) + if verify_state_root: + spec.verify_block_state_root(state, block) + +def process_epoch_transition(state: BeaconState) -> None: + spec.process_justification_and_finalization(state) + spec.process_crosslinks(state) + # TODO: Eligible + spec.process_rewards_and_penalties(state) + spec.process_registry_updates(state) + spec.process_reveal_deadlines(state) + spec.process_challenge_deadlines(state) + spec.process_slashings(state) + spec.process_final_updates(state) + spec.after_process_final_updates(state) + +def state_transition_to(state: BeaconState, up_to: Slot) -> BeaconState: + while state.slot < up_to: + spec.cache_state(state) + if (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0: + process_epoch_transition(state) + spec.advance_slot(state) + + +def state_transition(state: BeaconState, + block: BeaconBlock, + verify_state_root: bool=False) -> BeaconState: + state_transition_to(state, block.slot) + process_block(state, block, verify_state_root) \ No newline at end of file diff --git a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py index cfd2e56ac..8a210f41b 100644 --- a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py +++ b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py @@ -6,9 +6,14 @@ BYTES_PER_CHUNK = 32 BYTES_PER_LENGTH_OFFSET = 4 ZERO_CHUNK = b'\x00' * BYTES_PER_CHUNK +cached_typedefs = {} -def SSZType(fields): +def SSZType(name, fields): class SSZObject(): + if name != None: + __name__ = name + __qualname__ = name + def __init__(self, **kwargs): for f, t in fields.items(): if f not in kwargs: @@ -25,8 +30,11 @@ def SSZType(fields): def __str__(self): output = [] for field in self.fields: - output.append(f'{field}: {getattr(self, field)}') + output.append(f'{field}: {repr(getattr(self, field))},') return "\n".join(output) + + def __repr__(self): + return name + "(**{\n " + str(self).replace("\n", "\n ") + "\n})" def serialize(self): return serialize_value(self, self.__class__) @@ -35,12 +43,16 @@ def SSZType(fields): return hash_tree_root(self, self.__class__) SSZObject.fields = fields + + if name != None: + cached_typedefs[name] = SSZObject + return SSZObject def SSZTypeExtension(original_type, new_fields): - fields = original_type.fields.copy() - fields.update(original_type.fields) - return SSZType(fields) + typedef = cached_typedefs[original_type] + typedef.fields.update(new_fields) + return typedef class Vector(): def __init__(self, items): @@ -319,7 +331,7 @@ def truncate(container): key: container.fields[key] for key in field_keys[:-1] } - truncated_class = SSZType(truncated_fields) + truncated_class = SSZType(None, truncated_fields) kwargs = { field: getattr(container, field) for field in field_keys[:-1] diff --git a/test_libs/pyspec/tests/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/block_processing/test_process_attestation.py index bcf71376c..18706c9e5 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attestation.py @@ -1,12 +1,12 @@ from copy import deepcopy import pytest -import eth2spec.phase0.spec as spec +import eth2spec.phase1.spec as spec -from eth2spec.phase0.state_transition import ( +from eth2spec.phase1.state_transition import ( state_transition, ) -from eth2spec.phase0.spec import ( +from eth2spec.phase1.spec import ( get_current_epoch, process_attestation, slot_to_epoch, diff --git a/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py index 2ea16f13d..880ca829a 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py @@ -1,8 +1,8 @@ from copy import deepcopy import pytest -import eth2spec.phase0.spec as spec -from eth2spec.phase0.spec import ( +import eth2spec.phase1.spec as spec +from eth2spec.phase1.spec import ( get_beacon_proposer_index, process_attester_slashing, ) diff --git a/test_libs/pyspec/tests/block_processing/test_process_block_header.py b/test_libs/pyspec/tests/block_processing/test_process_block_header.py index b35b0a9c1..bcaf3c6d8 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/tests/block_processing/test_process_block_header.py @@ -2,7 +2,7 @@ from copy import deepcopy import pytest -from eth2spec.phase0.spec import ( +from eth2spec.phase1.spec import ( get_beacon_proposer_index, cache_state, advance_slot, diff --git a/test_libs/pyspec/tests/block_processing/test_process_deposit.py b/test_libs/pyspec/tests/block_processing/test_process_deposit.py index bbfb390ef..768f61678 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/tests/block_processing/test_process_deposit.py @@ -1,9 +1,9 @@ from copy import deepcopy import pytest -import eth2spec.phase0.spec as spec +import eth2spec.phase1.spec as spec -from eth2spec.phase0.spec import ( +from eth2spec.phase1.spec import ( ZERO_HASH, process_deposit, ) diff --git a/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py index 475221036..513e607b1 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py @@ -1,8 +1,8 @@ from copy import deepcopy import pytest -import eth2spec.phase0.spec as spec -from eth2spec.phase0.spec import ( +import eth2spec.phase1.spec as spec +from eth2spec.phase1.spec import ( get_current_epoch, process_proposer_slashing, ) diff --git a/test_libs/pyspec/tests/block_processing/test_process_transfer.py b/test_libs/pyspec/tests/block_processing/test_process_transfer.py index 0eeaa7792..02f448fd0 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/tests/block_processing/test_process_transfer.py @@ -1,9 +1,9 @@ from copy import deepcopy import pytest -import eth2spec.phase0.spec as spec +import eth2spec.phase1.spec as spec -from eth2spec.phase0.spec import ( +from eth2spec.phase1.spec import ( get_active_validator_indices, get_beacon_proposer_index, get_current_epoch, diff --git a/test_libs/pyspec/tests/block_processing/test_voluntary_exit.py b/test_libs/pyspec/tests/block_processing/test_voluntary_exit.py index c58c5238a..c789b55d8 100644 --- a/test_libs/pyspec/tests/block_processing/test_voluntary_exit.py +++ b/test_libs/pyspec/tests/block_processing/test_voluntary_exit.py @@ -1,9 +1,9 @@ from copy import deepcopy import pytest -import eth2spec.phase0.spec as spec +import eth2spec.phase1.spec as spec -from eth2spec.phase0.spec import ( +from eth2spec.phase1.spec import ( get_active_validator_indices, get_churn_limit, get_current_epoch, diff --git a/test_libs/pyspec/tests/conftest.py b/test_libs/pyspec/tests/conftest.py index 9840dc7b2..98b64c73b 100644 --- a/test_libs/pyspec/tests/conftest.py +++ b/test_libs/pyspec/tests/conftest.py @@ -1,6 +1,6 @@ import pytest -from eth2spec.phase0 import spec +from eth2spec.phase1 import spec from preset_loader import loader from .helpers import ( @@ -20,7 +20,6 @@ def config(request): presets = loader.load_presets('../../configs/', config_name) spec.apply_constants_preset(presets) - @pytest.fixture def num_validators(config): return spec.SLOTS_PER_EPOCH * 8 diff --git a/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py index d6765e3a7..9a6254715 100644 --- a/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py @@ -1,12 +1,12 @@ from copy import deepcopy import pytest -import eth2spec.phase0.spec as spec +import eth2spec.phase1.spec as spec -from eth2spec.phase0.state_transition import ( +from eth2spec.phase1.state_transition import ( state_transition, ) -from eth2spec.phase0.spec import ( +from eth2spec.phase1.spec import ( cache_state, get_crosslink_deltas, process_crosslinks, diff --git a/test_libs/pyspec/tests/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/tests/epoch_processing/test_process_registry_updates.py index 11f5de2ad..15f7ae6ec 100644 --- a/test_libs/pyspec/tests/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/tests/epoch_processing/test_process_registry_updates.py @@ -2,9 +2,9 @@ from copy import deepcopy import pytest -import eth2spec.phase0.spec as spec +import eth2spec.phase1.spec as spec -from eth2spec.phase0.spec import ( +from eth2spec.phase1.spec import ( get_current_epoch, is_active_validator, ) diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index 3b9b6904d..3156a7f28 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -2,12 +2,12 @@ from copy import deepcopy from py_ecc import bls -from eth2spec.phase0.state_transition import ( +from eth2spec.phase1.state_transition import ( state_transition, ) -import eth2spec.phase0.spec as spec +import eth2spec.phase1.spec as spec from eth2spec.utils.minimal_ssz import signing_root -from eth2spec.phase0.spec import ( +from eth2spec.phase1.spec import ( # constants ZERO_HASH, # SSZ @@ -420,3 +420,6 @@ def get_state_root(state, slot) -> bytes: """ assert slot < state.slot <= slot + spec.SLOTS_PER_HISTORICAL_ROOT return state.latest_state_roots[slot % spec.SLOTS_PER_HISTORICAL_ROOT] + +# Stub to be overwritten by config +import_spec = None \ No newline at end of file diff --git a/test_libs/pyspec/tests/test_finality.py b/test_libs/pyspec/tests/test_finality.py index ca048c2b2..46f08cfbb 100644 --- a/test_libs/pyspec/tests/test_finality.py +++ b/test_libs/pyspec/tests/test_finality.py @@ -2,9 +2,9 @@ from copy import deepcopy import pytest -import eth2spec.phase0.spec as spec +import eth2spec.phase1.spec as spec -from eth2spec.phase0.state_transition import ( +from eth2spec.phase1.state_transition import ( state_transition, ) from .helpers import ( diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index 1b4d20f4c..ea488fd1f 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -3,10 +3,10 @@ from copy import deepcopy import pytest from py_ecc import bls -import eth2spec.phase0.spec as spec +import eth2spec.phase1.spec as spec from eth2spec.utils.minimal_ssz import signing_root -from eth2spec.phase0.spec import ( +from eth2spec.phase1.spec import ( # constants ZERO_HASH, SLOTS_PER_HISTORICAL_ROOT, @@ -25,7 +25,7 @@ from eth2spec.phase0.spec import ( verify_merkle_branch, hash, ) -from eth2spec.phase0.state_transition import ( +from eth2spec.phase1.state_transition import ( state_transition, ) from eth2spec.utils.merkle_minimal import ( @@ -326,7 +326,7 @@ def test_voluntary_exit(state): def test_transfer(state): # overwrite default 0 to test - spec.MAX_TRANSFERS = 1 + spec.apply_constants_preset({"MAX_TRANSFERS": 1}) pre_state = deepcopy(state) current_epoch = get_current_epoch(pre_state)