From bcde37c39ff551a0e5d71af9a379670c0853934c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 9 Dec 2020 17:35:22 +0800 Subject: [PATCH] Make `sync-protocol.md` pass the linter --- setup.py | 73 ++++++++++++++++--- specs/lightclient/sync-protocol.md | 27 ++++--- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 3 files changed, 80 insertions(+), 22 deletions(-) diff --git a/setup.py b/setup.py index bd043dccc..168e19cca 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,7 @@ class SpecObject(NamedTuple): functions: Dict[str, str] custom_types: Dict[str, str] constants: Dict[str, str] + ssz_dep_constants: Dict[str, str] # the constants that depend on ssz_objects ssz_objects: Dict[str, str] dataclasses: Dict[str, str] @@ -35,6 +36,7 @@ def get_spec(file_name: str) -> SpecObject: current_name = None # most recent section title functions: Dict[str, str] = {} constants: Dict[str, str] = {} + ssz_dep_constants: Dict[str, str] = {} ssz_objects: Dict[str, str] = {} dataclasses: Dict[str, str] = {} function_matcher = re.compile(FUNCTION_REGEX) @@ -88,10 +90,20 @@ def get_spec(file_name: str) -> SpecObject: if c not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789': is_constant_def = False if is_constant_def: - constants[row[0]] = row[1].replace('**TBD**', '2**32') + if row[1].startswith('get_generalized_index'): + ssz_dep_constants[row[0]] = row[1] + else: + constants[row[0]] = row[1].replace('**TBD**', '2**32') elif row[1].startswith('uint') or row[1].startswith('Bytes'): custom_types[row[0]] = row[1] - return SpecObject(functions, custom_types, constants, ssz_objects, dataclasses) + return SpecObject( + functions=functions, + custom_types=custom_types, + constants=constants, + ssz_dep_constants=ssz_dep_constants, + ssz_objects=ssz_objects, + dataclasses=dataclasses, + ) CONFIG_LOADER = ''' @@ -160,7 +172,7 @@ CONFIG_NAME = 'mainnet' LIGHTCLIENT_IMPORT = '''from eth2spec.phase0 import spec as phase0 from eth2spec.config.config_util import apply_constants_config from typing import ( - Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable, Optional + Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable, Optional, Union ) from dataclasses import ( @@ -174,6 +186,7 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes from eth2spec.utils.ssz.ssz_typing import ( View, boolean, Container, List, Vector, uint8, uint32, uint64, Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, + Path, ) from eth2spec.utils import bls @@ -277,6 +290,24 @@ get_start_shard = cache_this( _get_start_shard, lru_size=SLOTS_PER_EPOCH * 3)''' +LIGHTCLIENT_PATCH_SUNDRY_FUNCTIONS = ''' + +def get_generalized_index(ssz_class: Any, *path: Sequence[Union[int, SSZVariableName]]) -> GeneralizedIndex: + ssz_path = Path(ssz_class) + for item in path: + ssz_path = ssz_path / item + return GeneralizedIndex(ssz_path.gindex()) +''' + + +# The constants that depend on SSZ objects +# Will verify the value at the end of the spec +LIGHTCLIENT_PATCH_HARDCODED_SSZ_DEP_CONSTANTS = { + 'FINALIZED_ROOT_INDEX': 'GeneralizedIndex(105)', + 'NEXT_SYNC_COMMITTEE_INDEX': 'GeneralizedIndex(54)', +} + + def objects_to_spec(spec_object: SpecObject, imports: str, fork: str, ordered_class_objects: Dict[str, str]) -> str: """ Given all the objects that constitute a spec, combine them into a single pyfile. @@ -303,14 +334,30 @@ def objects_to_spec(spec_object: SpecObject, imports: str, fork: str, ordered_cl + '\n\n' + f"fork = \'{fork}\'\n" + '\n\n' + new_type_definitions + '\n' + SUNDRY_CONSTANTS_FUNCTIONS - + '\n\n' + constants_spec + ) + + if fork == 'lightclient_patch': + lightclient_patch_ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, LIGHTCLIENT_PATCH_HARDCODED_SSZ_DEP_CONSTANTS[x]), LIGHTCLIENT_PATCH_HARDCODED_SSZ_DEP_CONSTANTS)) + spec += ( + LIGHTCLIENT_PATCH_SUNDRY_FUNCTIONS + + '\n\n' + lightclient_patch_ssz_dep_constants + ) + + spec += ( + '\n\n' + constants_spec + '\n\n' + CONFIG_LOADER + '\n\n' + ordered_class_objects_spec + '\n\n' + functions_spec + '\n' + PHASE0_SUNDRY_FUNCTIONS ) + if fork == 'phase1': spec += '\n' + PHASE1_SUNDRY_FUNCTIONS + + if fork == 'lightclient_patch': + lightclient_patch_ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), LIGHTCLIENT_PATCH_HARDCODED_SSZ_DEP_CONSTANTS)) + spec += '\n\n' + lightclient_patch_ssz_dep_constants_verification + spec += '\n' return spec @@ -332,7 +379,7 @@ ignored_dependencies = [ 'Bytes1', 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector', 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', 'bytes', 'byte', 'ByteList', 'ByteVector', - 'Dict', 'dict', 'field', + 'Dict', 'dict', 'field', 'ceillog2', ] @@ -373,14 +420,22 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: """ Takes in two spec variants (as tuples of their objects) and combines them using the appropriate combiner function. """ - functions0, custom_types0, constants0, ssz_objects0, dataclasses0 = spec0 - functions1, custom_types1, constants1, ssz_objects1, dataclasses1 = spec1 + functions0, custom_types0, constants0, ssz_dep_constants0, ssz_objects0, dataclasses0 = spec0 + functions1, custom_types1, constants1, ssz_dep_constants1, ssz_objects1, dataclasses1 = spec1 functions = combine_functions(functions0, functions1) custom_types = combine_constants(custom_types0, custom_types1) constants = combine_constants(constants0, constants1) + ssz_dep_constants = combine_constants(ssz_dep_constants0, ssz_dep_constants1) ssz_objects = combine_ssz_objects(ssz_objects0, ssz_objects1, custom_types) dataclasses = combine_functions(dataclasses0, dataclasses1) - return SpecObject(functions, custom_types, constants, ssz_objects, dataclasses) + return SpecObject( + functions=functions, + custom_types=custom_types, + constants=constants, + ssz_dep_constants=ssz_dep_constants, + ssz_objects=ssz_objects, + dataclasses=dataclasses, + ) fork_imports = { @@ -461,8 +516,8 @@ class PySpecCommand(Command): specs/phase0/weak-subjectivity.md specs/lightclient/beacon-chain.md specs/lightclient/lightclient-fork.md + specs/lightclient/sync-protocol.md """ - # TODO: add specs/lightclient/sync-protocol.md back when the GeneralizedIndex helpers are included. else: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index 310aad2df..a8dc59639 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -39,8 +39,8 @@ uses sync committees introduced in [this beacon chain extension](./beacon-chain. | Name | Value | | - | - | -| `FINALIZED_ROOT_INDEX` | `Index(BeaconState, 'finalized_checkpoint', 'root')` | -| `NEXT_SYNC_COMMITTEE_INDEX` | `Index(BeaconState, 'next_sync_committee')` | +| `FINALIZED_ROOT_INDEX` | `get_generalized_index(BeaconState, 'finalized_checkpoint', 'root')` | +| `NEXT_SYNC_COMMITTEE_INDEX` | `get_generalized_index(BeaconState, 'next_sync_committee')` | ## Configuration @@ -78,10 +78,10 @@ class LightClientUpdate(Container): header: BeaconBlockHeader # Next sync committee corresponding to the header next_sync_committee: SyncCommittee - next_sync_committee_branch: Vector[Bytes32, log2(NEXT_SYNC_COMMITTEE_INDEX)] + next_sync_committee_branch: Vector[Bytes32, ceillog2(NEXT_SYNC_COMMITTEE_INDEX)] # Finality proof for the update header finality_header: BeaconBlockHeader - finality_branch: Vector[Bytes32, log2(FINALIZED_ROOT_INDEX)] + finality_branch: Vector[Bytes32, ceillog2(FINALIZED_ROOT_INDEX)] # Sync committee aggregate signature sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE] sync_committee_signature: BLSSignature @@ -116,28 +116,28 @@ def is_valid_light_client_update(snapshot: LightClientSnapshot, update: LightCli # Verify update header root is the finalized root of the finality header, if specified if update.finality_header == BeaconBlockHeader(): signed_header = update.header - assert update.finality_branch == [ZERO_HASH for _ in range(log2(FINALIZED_ROOT_INDEX))] + assert update.finality_branch == [Bytes32() for _ in range(ceillog2(FINALIZED_ROOT_INDEX))] else: signed_header = update.finality_header assert is_valid_merkle_branch( leaf=hash_tree_root(update.header), branch=update.finality_branch, - depth=log2(FINALIZED_ROOT_INDEX), - index=FINALIZED_ROOT_INDEX % 2**log2(FINALIZED_ROOT_INDEX), + depth=ceillog2(FINALIZED_ROOT_INDEX), + index=FINALIZED_ROOT_INDEX % 2**ceillog2(FINALIZED_ROOT_INDEX), root=update.finality_header.state_root, ) # Verify update next sync committee if the update period incremented if update_period == snapshot_period: sync_committee = snapshot.current_sync_committee - assert update.next_sync_committee_branch == [ZERO_HASH for _ in range(log2(NEXT_SYNC_COMMITTEE_INDEX))] + assert update.next_sync_committee_branch == [Bytes32() for _ in range(ceillog2(NEXT_SYNC_COMMITTEE_INDEX))] else: sync_committee = snapshot.next_sync_committee assert is_valid_merkle_branch( leaf=hash_tree_root(update.next_sync_committee), branch=update.next_sync_committee_branch, - depth=log2(NEXT_SYNC_COMMITTEE_INDEX), - index=NEXT_SYNC_COMMITTEE_INDEX % 2**log2(NEXT_SYNC_COMMITTEE_INDEX), + depth=ceillog2(NEXT_SYNC_COMMITTEE_INDEX), + index=NEXT_SYNC_COMMITTEE_INDEX % 2**ceillog2(NEXT_SYNC_COMMITTEE_INDEX), root=update.header.state_root, ) @@ -173,11 +173,14 @@ def process_light_client_update(store: LightClientStore, update: LightClientUpda assert is_valid_light_client_update(store.snapshot, update) store.valid_updates.append(update) - if sum(update.sync_committee_bits) * 3 > len(update.sync_committee_bits) * 2 and update.header != update.finality_header: + if ( + sum(update.sync_committee_bits) * 3 > len(update.sync_committee_bits) * 2 + and update.header != update.finality_header + ): # Apply update if 2/3 quorum is reached and we have a finality proof apply_light_client_update(store, update) store.valid_updates = [] - elif current_slot > snapshot.header.slot + LIGHT_CLIENT_UPDATE_TIMEOUT: + elif current_slot > store.snapshot.header.slot + LIGHT_CLIENT_UPDATE_TIMEOUT: # Forced best update when the update timeout has elapsed apply_light_client_update(store, max(store.valid_updates, key=lambda update: sum(update.sync_committee_bits))) store.valid_updates = [] diff --git a/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py b/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py index b3a0b9962..6626e26d6 100644 --- a/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -5,4 +5,4 @@ from remerkleable.complex import Container, Vector, List from remerkleable.basic import boolean, bit, uint, byte, uint8, uint16, uint32, uint64, uint128, uint256 from remerkleable.bitfields import Bitvector, Bitlist from remerkleable.byte_arrays import ByteVector, Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, ByteList -from remerkleable.core import BasicView, View +from remerkleable.core import BasicView, View, Path