From 08faa86d706c31cd9690cb483d0df32f66696396 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 12 May 2019 23:56:53 +0200 Subject: [PATCH] POC ssz types spec build + update spec defs, typing still needs work --- scripts/phase0/build_spec.py | 4 +- scripts/phase0/function_puller.py | 41 ++-- specs/core/0_beacon-chain.md | 230 +++++++++--------- .../pyspec/eth2spec/utils/ssz/__init__.py | 0 4 files changed, 141 insertions(+), 134 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/utils/ssz/__init__.py diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index c8cd7348b..b226194d6 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -12,11 +12,11 @@ from typing import ( NewType, Tuple, ) -from eth2spec.utils.minimal_ssz import ( - SSZType, +from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, signing_root, ) +from eth2spec.utils.ssz.ssz_typing import * from eth2spec.utils.bls_stub import ( bls_aggregate_pubkeys, bls_verify, diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index 750f19590..f94687344 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -2,6 +2,22 @@ import sys from typing import List +def translate_ssz_type_line(line: str) -> str: + if ':' not in line: + return line + start = line[:line.index(':')] + field_type = line[line.index(':')+2:] + if field_type.startswith('['): + if ',' in line: + # TODO: translate [Foobar, SOME_THING] to Vector[Foobar, SSZLen(SOME_THING)] cleanly. + # just brute it here + field_type = 'Vector[%s, SSLen(%s)]' % (field_type[1:field_type.index(',')], field_type[field_type.index(',')+2:len(field_type)-1]) + else: + field_type = 'List[%s]' % field_type[1:len(field_type)-1] + line = start + ': ' + field_type + return line + + def get_spec(file_name: str) -> List[str]: code_lines = [] pulling_from = None @@ -21,24 +37,23 @@ def get_spec(file_name: str) -> List[str]: else: if current_typedef is not None: assert code_lines[-1] == '}' - code_lines[-1] = '})' - current_typedef[-1] = '})' - type_defs.append((current_name, current_typedef)) + code_lines[-1] = '' + code_lines.append('') pulling_from = None current_typedef = None else: if pulling_from == linenum and line == '{': - code_lines.append('%s = SSZType({' % current_name) - current_typedef = ['global_vars["%s"] = SSZType({' % current_name] + code_lines.append('class %s(SSZContainer):' % current_name) + current_typedef = current_name + type_defs.append(current_name) elif pulling_from is not None: # Add some whitespace between functions if line[:3] == 'def': code_lines.append('') code_lines.append('') - code_lines.append(line) - # Remember type def lines if current_typedef is not None: - current_typedef.append(line) + line = translate_ssz_type_line(line) + code_lines.append(line) elif pulling_from is None and len(line) > 0 and line[0] == '|': row = line[1:].split('|') if len(row) >= 2: @@ -56,16 +71,8 @@ def get_spec(file_name: str) -> List[str]: code_lines.append(row[0] + ' = ' + (row[1].replace('**TBD**', '0x1234567890123456789012345678901234567890'))) # Build type-def re-initialization code_lines.append('\n') - code_lines.append('def init_SSZ_types():') - code_lines.append(' global_vars = globals()') - for ssz_type_name, ssz_type in type_defs: - code_lines.append('') - for type_line in ssz_type: - if len(type_line) > 0: - code_lines.append(' ' + type_line) - code_lines.append('\n') code_lines.append('ssz_types = [\n') - for (ssz_type_name, _) in type_defs: + for ssz_type_name in type_defs: code_lines.append(f' {ssz_type_name},\n') code_lines.append(']') code_lines.append('\n') diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index bbca333cd..ff5a082ef 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -270,11 +270,11 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Previous fork version - 'previous_version': 'bytes4', + previous_version: bytes4 # Current fork version - 'current_version': 'bytes4', + current_version: bytes4 # Fork epoch number - 'epoch': 'uint64', + epoch: uint64 } ``` @@ -283,13 +283,13 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Shard number - 'shard': 'uint64', + shard: uint64 # Epoch number - 'epoch': 'uint64', + epoch: uint64 # Root of the previous crosslink - 'parent_root': 'bytes32', + parent_root: bytes32 # Root of the crosslinked shard data since the previous crosslink - 'data_root': 'bytes32', + data_root: bytes32 } ``` @@ -298,11 +298,11 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Root of the deposit tree - 'deposit_root': 'bytes32', + deposit_root: bytes32 # Total number of deposits - 'deposit_count': 'uint64', + deposit_count: uint64 # Block hash - 'block_hash': 'bytes32', + block_hash: bytes32 } ``` @@ -311,16 +311,16 @@ The types are defined topologically to aid in facilitating an executable version ```python { # LMD GHOST vote - 'beacon_block_root': 'bytes32', - + beacon_block_root: bytes32 + # FFG vote - 'source_epoch': 'uint64', - 'source_root': 'bytes32', - 'target_epoch': 'uint64', - 'target_root': 'bytes32', - + source_epoch: uint64 + source_root: bytes32 + target_epoch: uint64 + target_root: bytes32 + # Crosslink vote - 'crosslink': Crosslink, + crosslink: Crosslink } ``` @@ -329,9 +329,9 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Attestation data - 'data': AttestationData, + data: AttestationData # Custody bit - 'custody_bit': 'bool', + custody_bit: bool } ``` @@ -340,12 +340,12 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Validator indices - 'custody_bit_0_indices': ['uint64'], - 'custody_bit_1_indices': ['uint64'], + custody_bit_0_indices: [uint64] + custody_bit_1_indices: [uint64] # Attestation data - 'data': AttestationData, + data: AttestationData # Aggregate signature - 'signature': 'bytes96', + signature: bytes96 } ``` @@ -354,13 +354,13 @@ The types are defined topologically to aid in facilitating an executable version ```python { # BLS pubkey - 'pubkey': 'bytes48', + pubkey: bytes48 # Withdrawal credentials - 'withdrawal_credentials': 'bytes32', + withdrawal_credentials: bytes32 # Amount in Gwei - 'amount': 'uint64', + amount: uint64 # Container self-signature - 'signature': 'bytes96', + signature: bytes96 } ``` @@ -368,11 +368,11 @@ The types are defined topologically to aid in facilitating an executable version ```python { - 'slot': 'uint64', - 'parent_root': 'bytes32', - 'state_root': 'bytes32', - 'body_root': 'bytes32', - 'signature': 'bytes96', + slot: uint64 + parent_root: bytes32 + state_root: bytes32 + body_root: bytes32 + signature: bytes96 } ``` #### `Validator` @@ -380,21 +380,21 @@ The types are defined topologically to aid in facilitating an executable version ```python { # BLS public key - 'pubkey': 'bytes48', + pubkey: bytes48 # Withdrawal credentials - 'withdrawal_credentials': 'bytes32', + withdrawal_credentials: bytes32 # Epoch when became eligible for activation - 'activation_eligibility_epoch': 'uint64', + activation_eligibility_epoch: uint64 # Epoch when validator activated - 'activation_epoch': 'uint64', + activation_epoch: uint64 # Epoch when validator exited - 'exit_epoch': 'uint64', + exit_epoch: uint64 # Epoch when validator is eligible to withdraw - 'withdrawable_epoch': 'uint64', + withdrawable_epoch: uint64 # Was the validator slashed - 'slashed': 'bool', + slashed: bool # Effective balance - 'effective_balance': 'uint64', + effective_balance: uint64 } ``` @@ -403,13 +403,13 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Attester aggregation bitfield - 'aggregation_bitfield': 'bytes', + aggregation_bitfield: bytes # Attestation data - 'data': AttestationData, + data: AttestationData # Inclusion delay - 'inclusion_delay': 'uint64', + inclusion_delay: uint64 # Proposer index - 'proposer_index': 'uint64', + proposer_index: uint64 } ``` @@ -418,9 +418,9 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Block roots - 'block_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], + block_roots: [bytes32, SLOTS_PER_HISTORICAL_ROOT] # State roots - 'state_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], + state_roots: [bytes32, SLOTS_PER_HISTORICAL_ROOT] } ``` @@ -431,11 +431,11 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Proposer index - 'proposer_index': 'uint64', + proposer_index: uint64 # First block header - 'header_1': BeaconBlockHeader, + header_1: BeaconBlockHeader # Second block header - 'header_2': BeaconBlockHeader, + header_2: BeaconBlockHeader } ``` @@ -444,9 +444,9 @@ The types are defined topologically to aid in facilitating an executable version ```python { # First attestation - 'attestation_1': IndexedAttestation, + attestation_1: IndexedAttestation # Second attestation - 'attestation_2': IndexedAttestation, + attestation_2: IndexedAttestation } ``` @@ -455,13 +455,13 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Attester aggregation bitfield - 'aggregation_bitfield': 'bytes', + aggregation_bitfield: bytes # Attestation data - 'data': AttestationData, + data: AttestationData # Custody bitfield - 'custody_bitfield': 'bytes', + custody_bitfield: bytes # BLS aggregate signature - 'signature': 'bytes96', + signature: bytes96 } ``` @@ -470,11 +470,11 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Branch in the deposit tree - 'proof': ['bytes32', DEPOSIT_CONTRACT_TREE_DEPTH], + proof: [bytes32, DEPOSIT_CONTRACT_TREE_DEPTH] # Index in the deposit tree - 'index': 'uint64', + index: uint64 # Data - 'data': DepositData, + data: DepositData } ``` @@ -483,11 +483,11 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Minimum epoch for processing exit - 'epoch': 'uint64', + epoch: uint64 # Index of the exiting validator - 'validator_index': 'uint64', + validator_index: uint64 # Validator signature - 'signature': 'bytes96', + signature: bytes96 } ``` @@ -496,19 +496,19 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Sender index - 'sender': 'uint64', + sender: uint64 # Recipient index - 'recipient': 'uint64', + recipient: uint64 # Amount in Gwei - 'amount': 'uint64', + amount: uint64 # Fee in Gwei for block proposer - 'fee': 'uint64', + fee: uint64 # Inclusion slot - 'slot': 'uint64', + slot: uint64 # Sender withdrawal pubkey - 'pubkey': 'bytes48', + pubkey: bytes48 # Sender signature - 'signature': 'bytes96', + signature: bytes96 } ``` @@ -518,15 +518,15 @@ The types are defined topologically to aid in facilitating an executable version ```python { - 'randao_reveal': 'bytes96', - 'eth1_data': Eth1Data, - 'graffiti': 'bytes32', - 'proposer_slashings': [ProposerSlashing], - 'attester_slashings': [AttesterSlashing], - 'attestations': [Attestation], - 'deposits': [Deposit], - 'voluntary_exits': [VoluntaryExit], - 'transfers': [Transfer], + randao_reveal: bytes96 + eth1_data: Eth1Data + graffiti: bytes32 + proposer_slashings: [ProposerSlashing] + attester_slashings: [AttesterSlashing] + attestations: [Attestation] + deposits: [Deposit] + voluntary_exits: [VoluntaryExit] + transfers: [Transfer] } ``` @@ -535,11 +535,11 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Header - 'slot': 'uint64', - 'parent_root': 'bytes32', - 'state_root': 'bytes32', - 'body': BeaconBlockBody, - 'signature': 'bytes96', + slot: uint64 + parent_root: bytes32 + state_root: bytes32 + body: BeaconBlockBody + signature: bytes96 } ``` @@ -550,45 +550,45 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Misc - 'slot': 'uint64', - 'genesis_time': 'uint64', - 'fork': Fork, # For versioning hard forks - + slot: uint64 + genesis_time: uint64 + fork: Fork # For versioning hard forks + # Validator registry - 'validator_registry': [Validator], - 'balances': ['uint64'], - + validator_registry: [Validator] + balances: [uint64] + # Randomness and committees - 'latest_randao_mixes': ['bytes32', LATEST_RANDAO_MIXES_LENGTH], - 'latest_start_shard': 'uint64', - + latest_randao_mixes: [bytes32, LATEST_RANDAO_MIXES_LENGTH] + latest_start_shard: uint64 + # Finality - 'previous_epoch_attestations': [PendingAttestation], - 'current_epoch_attestations': [PendingAttestation], - 'previous_justified_epoch': 'uint64', - 'current_justified_epoch': 'uint64', - 'previous_justified_root': 'bytes32', - 'current_justified_root': 'bytes32', - 'justification_bitfield': 'uint64', - 'finalized_epoch': 'uint64', - 'finalized_root': 'bytes32', - + previous_epoch_attestations: [PendingAttestation] + current_epoch_attestations: [PendingAttestation] + previous_justified_epoch: uint64 + current_justified_epoch: uint64 + previous_justified_root: bytes32 + current_justified_root: bytes32 + justification_bitfield: uint64 + finalized_epoch: uint64 + finalized_root: bytes32 + # Recent state - 'current_crosslinks': [Crosslink, SHARD_COUNT], - 'previous_crosslinks': [Crosslink, SHARD_COUNT], - 'latest_block_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], - 'latest_state_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], - 'latest_active_index_roots': ['bytes32', LATEST_ACTIVE_INDEX_ROOTS_LENGTH], + current_crosslinks: [Crosslink, SHARD_COUNT] + previous_crosslinks: [Crosslink, SHARD_COUNT] + latest_block_roots: [bytes32, SLOTS_PER_HISTORICAL_ROOT] + latest_state_roots: [bytes32, SLOTS_PER_HISTORICAL_ROOT] + latest_active_index_roots: [bytes32, LATEST_ACTIVE_INDEX_ROOTS_LENGTH] # Balances slashed at every withdrawal period - 'latest_slashed_balances': ['uint64', LATEST_SLASHED_EXIT_LENGTH], + latest_slashed_balances: [uint64, LATEST_SLASHED_EXIT_LENGTH] # `latest_block_header.state_root == ZERO_HASH` temporarily - 'latest_block_header': BeaconBlockHeader, - 'historical_roots': ['bytes32'], - + latest_block_header: BeaconBlockHeader + historical_roots: [bytes32] + # Ethereum 1.0 chain data - 'latest_eth1_data': Eth1Data, - 'eth1_data_votes': [Eth1Data], - 'deposit_index': 'uint64', + latest_eth1_data: Eth1Data + eth1_data_votes: [Eth1Data] + deposit_index: uint64 } ``` diff --git a/test_libs/pyspec/eth2spec/utils/ssz/__init__.py b/test_libs/pyspec/eth2spec/utils/ssz/__init__.py new file mode 100644 index 000000000..e69de29bb