From a604d03dff66aed47a0790525651267008511fae Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Tue, 7 May 2019 12:13:22 +0100 Subject: [PATCH] Basic phase 1 pulling + correcting syntax errors in phase 1 --- Makefile | 10 +++++ scripts/phase0/build_spec.py | 37 +++++++++++++--- scripts/phase0/function_puller.py | 19 ++++++-- specs/core/1_custody-game.md | 19 +++++--- specs/core/1_shard-data-chains.md | 44 +++++++++---------- .../pyspec/eth2spec/utils/minimal_ssz.py | 4 ++ 6 files changed, 97 insertions(+), 36 deletions(-) diff --git a/Makefile b/Makefile index 73d8adea8..420fc3276 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ YAML_TEST_TARGETS = $(patsubst $(GENERATOR_DIR)/%, $(YAML_TEST_DIR)/%, $(GENERAT GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENERATORS)) PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase0/spec.py +PY_SPEC_PHASE_1_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase1/custody-game.py $(PY_SPEC_DIR)/eth2spec/phase1/shard-data-chains.py PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) @@ -25,6 +26,7 @@ clean: rm -rf $(GENERATOR_VENVS) rm -rf $(PY_SPEC_DIR)/venv $(PY_SPEC_DIR)/.pytest_cache rm -rf $(PY_SPEC_ALL_TARGETS) + rm -rf $(PY_SPEC_PHASE_1_TARGETS) # "make gen_yaml_tests" to run generators gen_yaml_tests: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_TARGETS) @@ -45,10 +47,18 @@ pyspec: $(PY_SPEC_ALL_TARGETS) # "make phase0" to create pyspec for phase0 phase0: $(PY_SPEC_PHASE_0_TARGETS) +# "make phase1" to create pyspec for phase1 +phase1: $(PY_SPEC_PHASE_1_TARGETS) $(PY_SPEC_DIR)/eth2spec/phase0/spec.py: python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $@ +$(PY_SPEC_DIR)/eth2spec/phase1/custody-game.py: + python3 $(SCRIPT_DIR)/phase0/build_spec.py -p1 $(SPEC_DIR)/core/1_custody-game.md $@ + +$(PY_SPEC_DIR)/eth2spec/phase1/shard-data-chains.py: + python3 $(SCRIPT_DIR)/phase0/build_spec.py -p1 $(SPEC_DIR)/core/1_shard-data-chains.md $@ + CURRENT_DIR = ${CURDIR} diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index da5845951..80b378fbb 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -1,6 +1,6 @@ import sys import function_puller - +from optparse import OptionParser def build_phase0_spec(sourcefile, outfile): code_lines = [] @@ -85,8 +85,35 @@ def apply_constants_preset(preset: Dict[str, Any]): out.write("\n".join(code_lines)) -if __name__ == '__main__': - if len(sys.argv) < 3: - print("Usage: ") - build_phase0_spec(sys.argv[1], sys.argv[2]) +def build_phase1_spec(sourcefile, outfile): + code_lines = [] + code_lines.append(""" +from eth2spec.phase0.spec import * + +""") + + code_lines += function_puller.get_spec(sourcefile) + + with open(outfile, 'w') as out: + out.write("\n".join(code_lines)) + + +if __name__ == '__main__': + parser = OptionParser() + parser.add_option("-p", "--phase", dest="phase", type="int", default=0, + help="Build for phase #") + + (options, args) = parser.parse_args() + + if len(args) < 2: + parser.print_help() + + if options.phase == 0: + build_phase0_spec(args[0], args[1]) + print(args[0]) + print(args[1]) + elif options.phase == 1: + build_phase1_spec(args[0], args[1]) + else: + print("Invalid phase: {0}".format(options["phase"])) diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index 1fad41fa9..393f821d5 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -2,16 +2,26 @@ import sys from typing import List -def get_spec(file_name: str) -> List[str]: +def get_spec(file_name: str, phase:int = 0) -> List[str]: code_lines = [] pulling_from = None current_name = None current_typedef = None + is_update_section = False + update_section_depth = None type_defs = [] - for linenum, line in enumerate(open(sys.argv[1]).readlines()): + for linenum, line in enumerate(open(file_name).readlines()): line = line.rstrip() if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`': current_name = line[line[:-1].rfind('`') + 1: -1] + if pulling_from is None and len(line) > 0 and line[0] == '#' and line.endswith('updates'): + is_update_section = True + update_section_depth = max(i for i in range(10) if line.startswith('#' * i)) + elif pulling_from is None and len(line) > 0 and line[0] == '#' and is_update_section: + section_depth = max(i for i in range(10) if line.startswith('#' * i)) + if section_depth <= update_section_depth: + is_update_section = False + update_section_depth = None if line[:9] == '```python': assert pulling_from is None pulling_from = linenum + 1 @@ -28,7 +38,10 @@ def get_spec(file_name: str) -> List[str]: current_typedef = None else: if pulling_from == linenum and line == '{': - code_lines.append('%s = SSZType({' % current_name) + if is_update_section: + code_lines.append('%s = SSZTypeExtension({' % current_name) + else: + code_lines.append('%s = SSZType({' % current_name) current_typedef = ['global_vars["%s"] = SSZType({' % current_name] elif pulling_from is not None: # Add some whitespace between functions diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index d56526611..09e876ec3 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -220,16 +220,19 @@ Add the following fields to the end of the specified container objects. Fields w #### `Validator` ```python +{ # next_custody_reveal_period is initialised to the custody period # (of the particular validator) in which the validator is activated # = get_validators_custody_reveal_period(...) 'next_custody_reveal_period': 'uint64', 'max_reveal_lateness': 'uint64', +} ``` #### `BeaconState` ```python +{ 'custody_chunk_challenge_records': [CustodyChunkChallengeRecord], 'custody_bit_challenge_records': [CustodyBitChallengeRecord], 'custody_challenge_index': 'uint64', @@ -237,16 +240,19 @@ Add the following fields to the end of the specified container objects. Fields w # Future derived secrets already exposed; contains the indices of the exposed validator # at RANDAO reveal period % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS 'exposed_derived_secrets': [['uint64'], EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS], +} ``` #### `BeaconBlockBody` ```python +{ 'custody_chunk_challenges': [CustodyChunkChallenge], 'custody_bit_challenges': [CustodyBitChallenge], 'custody_responses': [CustodyResponse], 'custody_key_reveals': [CustodyKeyReveal], 'early_derived_secret_reveals': [EarlyDerivedSecretReveal], +} ``` ## Helpers @@ -299,7 +305,7 @@ def get_randao_epoch_for_custody_period(period: int, validator_index: ValidatorI ### `get_validators_custody_reveal_period` - ```python +```python def get_validators_custody_reveal_period(state: BeaconState, validator_index: ValidatorIndex, epoch: Epoch=None) -> int: @@ -484,7 +490,7 @@ def process_chunk_challenge(state: BeaconState, new_record = CustodyChunkChallengeRecord( challenge_index=state.custody_challenge_index, challenger_index=get_beacon_proposer_index(state), - responder_index=challenge.responder_index + responder_index=challenge.responder_index, deadline=get_current_epoch(state) + CUSTODY_RESPONSE_DEADLINE, crosslink_data_root=challenge.attestation.data.crosslink_data_root, depth=depth, @@ -538,7 +544,7 @@ def process_bit_challenge(state: BeaconState, get_validators_custody_reveal_period( state=state, index=challenge.responder_index, - epoch=slot_to_epoch(attestation.data.slot), + epoch=slot_to_epoch(attestation.data.slot)), challenge.responder_index ) assert bls_verify( @@ -585,11 +591,11 @@ For each `response` in `block.body.custody_responses`, run the following functio ```python def process_custody_response(state: BeaconState, response: CustodyResponse) -> None: - chunk_challenge = next(record for record in state.custody_chunk_challenge_records if record.challenge_index == response.challenge_index, None) + chunk_challenge = next((record for record in state.custody_chunk_challenge_records if record.challenge_index == response.challenge_index), None) if chunk_challenge is not None: return process_chunk_challenge_response(state, response, chunk_challenge) - bit_challenge = next(record for record in state.custody_bit_challenge_records if record.challenge_index == response.challenge_index, None) + bit_challenge = next((record for record in state.custody_bit_challenge_records if record.challenge_index == response.challenge_index), None) if bit_challenge is not None: return process_bit_challenge_response(state, response, bit_challenge) @@ -657,7 +663,7 @@ def process_bit_challenge_response(state: BeaconState, Run `process_reveal_deadlines(state)` immediately after `process_ejections(state)`: - ```python +```python def process_reveal_deadlines(state: BeaconState) -> None: for index, validator in enumerate(state.validator_registry): if (validator.latest_custody_reveal_period + @@ -686,6 +692,7 @@ def process_challenge_deadlines(state: BeaconState) -> None: Append this to `process_final_updates(state)`: ```python +def after_process_final_updates(state: BeaconState) -> None: # Clean up exposed RANDAO key reveals state.exposed_derived_secrets[current_epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] = [] ``` diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 33ef8632b..ffb7926f8 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -53,8 +53,8 @@ This document describes the shard data layer and the shard fork choice rule in P | Name | Value | Unit | Duration | | - | - | :-: | :-: | -| `CROSSLINK_LOOKBACK` | 2**0 (= 1) | epochs | 6.2 minutes | -| `PERSISTENT_COMMITTEE_PERIOD` | 2**11 (= 2,048) | epochs | ~9 days | +| `CROSSLINK_LOOKBACK` | `2**0` (= 1) | epochs | 6.2 minutes | +| `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | ~9 days | ### Signature domains @@ -198,14 +198,14 @@ def get_shard_proposer_index(state: BeaconState, ```python def get_shard_header(block: ShardBlock) -> ShardBlockHeader: return ShardBlockHeader( - slot: block.slot, - shard: block.shard, - beacon_chain_root: block.beacon_chain_root, - previous_block_root: block.previous_block_root, - body_root: hash_tree_root(block.body), - state_root: block.state_root, - attestations: block.attestations, - signature: block.signature, + slot=block.slot, + shard=block.shard, + beacon_chain_root=block.beacon_chain_root, + previous_block_root=block.previous_block_root, + body_root=hash_tree_root(block.body), + state_root=block.state_root, + attestations=block.attestations, + signature=block.signature, ) ``` @@ -219,7 +219,7 @@ def verify_shard_attestation_signature(state: BeaconState, assert verify_bitfield(attestation.aggregation_bitfield, len(persistent_committee)) pubkeys = [] for i, index in enumerate(persistent_committee): - if get_bitfield_bit(attestation.aggregation_bitfield, i) == 0b1 + if get_bitfield_bit(attestation.aggregation_bitfield, i) == 0b1: validator = state.validator_registry[index] assert is_active_validator(validator, get_current_epoch(state)) pubkeys.append(validator.pubkey) @@ -273,7 +273,7 @@ def is_valid_shard_block(beacon_blocks: List[BeaconBlock], beacon_state: BeaconState, valid_shard_blocks: List[ShardBlock], unix_time: uint64, - candidate: ShardBlock) -> bool + candidate: ShardBlock) -> bool: # Check if block is already determined valid for _, block in enumerate(valid_shard_blocks): if candidate == block: @@ -289,7 +289,7 @@ def is_valid_shard_block(beacon_blocks: List[BeaconBlock], # Check beacon block beacon_block = beacon_blocks[block.slot] assert block.beacon_block_root == signing_root(beacon_block) - assert beacon_block.slot <= block.slot: + assert beacon_block.slot <= block.slot # Check state root assert block.state_root == ZERO_HASH # [to be removed in phase 2] @@ -299,9 +299,9 @@ def is_valid_shard_block(beacon_blocks: List[BeaconBlock], assert candidate.previous_block_root == ZERO_HASH else: parent_block = next( - block for block in valid_shard_blocks if - signing_root(block) == candidate.previous_block_root - , None) + (block for block in valid_shard_blocks if + signing_root(block) == candidate.previous_block_root), + None) assert parent_block != None assert parent_block.shard == block.shard assert parent_block.slot < block.slot @@ -342,9 +342,9 @@ def is_valid_shard_attestation(valid_shard_blocks: List[ShardBlock], candidate: Attestation) -> bool: # Check shard block shard_block = next( - block for block in valid_shard_blocks if - signing_root(block) == candidate.attestation.data.shard_block_root - , None) + (block for block in valid_shard_blocks if + signing_root(block) == candidate.attestation.data.shard_block_root), + None) assert shard_block != None assert shard_block.slot == attestation.data.slot assert shard_block.shard == attestation.data.shard @@ -381,9 +381,9 @@ def is_valid_beacon_attestation(shard: Shard, assert candidate.data.previous_crosslink.crosslink_data_root == ZERO_HASH else: previous_attestation = next( - attestation for attestation in valid_attestations if - attestation.data.crosslink_data_root == candidate.data.previous_crosslink.crosslink_data_root - , None) + (attestation for attestation in valid_attestations if + attestation.data.crosslink_data_root == candidate.data.previous_crosslink.crosslink_data_root), + None) assert previous_attestation != None assert candidate.data.previous_attestation.epoch < slot_to_epoch(candidate.data.slot) diff --git a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py index 9cc2baebb..cfd2e56ac 100644 --- a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py +++ b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py @@ -37,6 +37,10 @@ def SSZType(fields): SSZObject.fields = fields return SSZObject +def SSZTypeExtension(original_type, new_fields): + fields = original_type.fields.copy() + fields.update(original_type.fields) + return SSZType(fields) class Vector(): def __init__(self, items):