From 658ede2191e7da57a37a9c12f5c21cc4bc4c7020 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 9 Apr 2021 20:34:51 +0800 Subject: [PATCH 1/5] Refactor pyspec builder with `SpecAdjustment` classes --- setup.py | 339 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 206 insertions(+), 133 deletions(-) diff --git a/setup.py b/setup.py index 514f75c50..65b04fcf2 100644 --- a/setup.py +++ b/setup.py @@ -6,15 +6,35 @@ from distutils.util import convert_path import os import re from typing import Dict, NamedTuple, List +from abc import ABC, abstractmethod + FUNCTION_REGEX = r'^def [\w_]*' - # Definitions in context.py PHASE0 = 'phase0' ALTAIR = 'altair' MERGE = 'merge' +CONFIG_LOADER = ''' +apply_constants_config(globals()) +''' + +# The helper functions that are used when defining constants +CONSTANT_DEP_SUNDRY_CONSTANTS_FUNCTIONS = ''' +def ceillog2(x: int) -> uint64: + if x < 1: + raise ValueError(f"ceillog2 accepts only positive values, x={x}") + return uint64((x - 1).bit_length()) + + +def floorlog2(x: int) -> uint64: + if x < 1: + raise ValueError(f"floorlog2 accepts only positive values, x={x}") + return uint64(x.bit_length() - 1) +''' + + class SpecObject(NamedTuple): functions: Dict[str, str] custom_types: Dict[str, str] @@ -111,11 +131,55 @@ def get_spec(file_name: str) -> SpecObject: ) -CONFIG_LOADER = ''' -apply_constants_config(globals()) -''' +class SpecAdjustment(ABC): + @classmethod + @abstractmethod + def imports_and_predefinitions(cls) -> str: + """ + Importing functions and defining special types/constants for building pyspec. + """ + raise NotImplementedError() -PHASE0_IMPORTS = '''from eth2spec.config.config_util import apply_constants_config + @classmethod + @abstractmethod + def sundry_functions(cls) -> str: + """ + The functions that are (1) defined abstractly in specs or (2) adjusted for getting better performance. + """ + raise NotImplementedError() + + @classmethod + @abstractmethod + def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: + """ + The constants that are required for SSZ objects. + """ + raise NotImplementedError() + + @classmethod + @abstractmethod + def hardcoded_custom_type_dep_constants(cls) -> Dict[str, str]: + """ + The constants that are required for custom types. + """ + raise NotImplementedError() + + @classmethod + @abstractmethod + def invariant_checks(cls) -> str: + """ + The invariant checks + """ + raise NotImplementedError() + + +# +# Phase0SpecAdjustment +# +class Phase0SpecAdjustment(SpecAdjustment): + @classmethod + def imports_and_predefinitions(cls) -> str: + return '''from eth2spec.config.config_util import apply_constants_config from typing import ( Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar ) @@ -141,82 +205,9 @@ SSZObject = TypeVar('SSZObject', bound=View) CONFIG_NAME = 'mainnet' ''' -ALTAIR_IMPORTS = '''from eth2spec.phase0 import spec as phase0 -from eth2spec.config.config_util import apply_constants_config -from typing import ( - Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable, Optional, Union -) - -from dataclasses import ( - dataclass, - field, -) - -from lru import LRU - -from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes -from eth2spec.utils.ssz.ssz_typing import ( - View, boolean, Container, List, Vector, uint8, uint32, uint64, - Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, - Path, -) -from eth2spec.utils import bls - -from eth2spec.utils.hash_function import hash - -# Whenever altair is loaded, make sure we have the latest phase0 -from importlib import reload -reload(phase0) - - -SSZVariableName = str -GeneralizedIndex = NewType('GeneralizedIndex', int) -SSZObject = TypeVar('SSZObject', bound=View) - -CONFIG_NAME = 'mainnet' -''' - -MERGE_IMPORTS = '''from eth2spec.phase0 import spec as phase0 -from eth2spec.config.config_util import apply_constants_config -from typing import ( - Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar -) - -from dataclasses import ( - dataclass, - field, -) - -from lru import LRU - -from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes -from eth2spec.utils.ssz.ssz_typing import ( - View, boolean, Container, List, Vector, uint8, uint32, uint64, uint256, - Bytes1, Bytes4, Bytes20, Bytes32, Bytes48, Bytes96, Bitlist, - ByteList, ByteVector -) -from eth2spec.utils import bls - -from eth2spec.utils.hash_function import hash - -SSZObject = TypeVar('SSZObject', bound=View) - -CONFIG_NAME = 'mainnet' -''' - -SUNDRY_CONSTANTS_FUNCTIONS = ''' -def ceillog2(x: int) -> uint64: - if x < 1: - raise ValueError(f"ceillog2 accepts only positive values, x={x}") - return uint64((x - 1).bit_length()) - - -def floorlog2(x: int) -> uint64: - if x < 1: - raise ValueError(f"floorlog2 accepts only positive values, x={x}") - return uint64(x.bit_length() - 1) -''' -PHASE0_SUNDRY_FUNCTIONS = ''' + @classmethod + def sundry_functions(cls) -> str: + return ''' def get_eth1_data(block: Eth1Block) -> Eth1Data: """ A stub function return mocking Eth1Data. @@ -287,9 +278,62 @@ get_attesting_indices = cache_this( ), _get_attesting_indices, lru_size=SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT * 3)''' + @classmethod + def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: + return {} -ALTAIR_SUNDRY_FUNCTIONS = ''' + @classmethod + def hardcoded_custom_type_dep_constants(cls) -> Dict[str, str]: + return {} + @classmethod + def invariant_checks(cls) -> str: + return '' + + +# +# AltairSpecAdjustment +# +class AltairSpecAdjustment(Phase0SpecAdjustment): + @classmethod + def imports_and_predefinitions(cls) -> str: + return '''from eth2spec.phase0 import spec as phase0 +from eth2spec.config.config_util import apply_constants_config +from typing import ( + Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable, Optional, Union +) + +from dataclasses import ( + dataclass, + field, +) + +from lru import LRU + +from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes +from eth2spec.utils.ssz.ssz_typing import ( + View, boolean, Container, List, Vector, uint8, uint32, uint64, + Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, + Path, +) +from eth2spec.utils import bls + +from eth2spec.utils.hash_function import hash + +# Whenever altair is loaded, make sure we have the latest phase0 +from importlib import reload +reload(phase0) + + +SSZVariableName = str +GeneralizedIndex = NewType('GeneralizedIndex', int) +SSZObject = TypeVar('SSZObject', bound=View) + +CONFIG_NAME = 'mainnet' +''' + @classmethod + def sundry_functions(cls) -> str: + return super().sundry_functions() + '\n\n' + ''' def get_generalized_index(ssz_class: Any, *path: Sequence[Union[int, SSZVariableName]]) -> GeneralizedIndex: ssz_path = Path(ssz_class) for item in path: @@ -297,7 +341,59 @@ def get_generalized_index(ssz_class: Any, *path: Sequence[Union[int, SSZVariable return GeneralizedIndex(ssz_path.gindex())''' -MERGE_SUNDRY_FUNCTIONS = """ + @classmethod + def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: + constants = { + 'FINALIZED_ROOT_INDEX': 'GeneralizedIndex(105)', + 'NEXT_SYNC_COMMITTEE_INDEX': 'GeneralizedIndex(55)', + } + return {**super().hardcoded_ssz_dep_constants(), **constants} + + @classmethod + def invariant_checks(cls) -> str: + return ''' +assert ( + TIMELY_HEAD_WEIGHT + TIMELY_SOURCE_WEIGHT + TIMELY_TARGET_WEIGHT + SYNC_REWARD_WEIGHT + PROPOSER_WEIGHT +) == WEIGHT_DENOMINATOR''' + + +# +# MergeSpecAdjustment +# +class MergeSpecAdjustment(Phase0SpecAdjustment): + @classmethod + def imports_and_predefinitions(cls): + return '''from eth2spec.phase0 import spec as phase0 +from eth2spec.config.config_util import apply_constants_config +from typing import ( + Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar +) + +from dataclasses import ( + dataclass, + field, +) + +from lru import LRU + +from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes +from eth2spec.utils.ssz.ssz_typing import ( + View, boolean, Container, List, Vector, uint8, uint32, uint64, uint256, + Bytes1, Bytes4, Bytes20, Bytes32, Bytes48, Bytes96, Bitlist, + ByteList, ByteVector +) +from eth2spec.utils import bls + +from eth2spec.utils.hash_function import hash + +SSZObject = TypeVar('SSZObject', bound=View) + +CONFIG_NAME = 'mainnet' +''' + + @classmethod + def sundry_functions(cls) -> str: + return super().sundry_functions() + '\n\n' + """ ExecutionState = Any @@ -321,22 +417,18 @@ def produce_execution_payload(parent_hash: Bytes32) -> ExecutionPayload: pass""" -# The constants that depend on SSZ objects -# Will verify the value at the end of the spec -ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS = { - 'FINALIZED_ROOT_INDEX': 'GeneralizedIndex(105)', - 'NEXT_SYNC_COMMITTEE_INDEX': 'GeneralizedIndex(55)', -} + @classmethod + def hardcoded_custom_type_dep_constants(cls) -> str: + constants = { + 'MAX_BYTES_PER_OPAQUE_TRANSACTION': 'uint64(2**20)', + } + return {**super().hardcoded_custom_type_dep_constants(), **constants} -ALTAIR_INVAIANT_CHECKS = ''' -assert ( - TIMELY_HEAD_WEIGHT + TIMELY_SOURCE_WEIGHT + TIMELY_TARGET_WEIGHT + SYNC_REWARD_WEIGHT + PROPOSER_WEIGHT -) == WEIGHT_DENOMINATOR''' - - -MERGE_HARDCODED_CUSTOM_TYPE_DEP_CONSTANTS = { - 'MAX_BYTES_PER_OPAQUE_TRANSACTION': 'uint64(2**20)', +spec_adjustments = { + PHASE0: Phase0SpecAdjustment, + ALTAIR: AltairSpecAdjustment, + MERGE: MergeSpecAdjustment, } @@ -352,7 +444,7 @@ def is_merge(fork): return fork == MERGE -def objects_to_spec(spec_object: SpecObject, imports: str, fork: str, ordered_class_objects: Dict[str, str]) -> str: +def objects_to_spec(spec_object: SpecObject, adjustment: SpecAdjustment, fork: str, ordered_class_objects: Dict[str, str]) -> str: """ Given all the objects that constitute a spec, combine them into a single pyfile. """ @@ -382,41 +474,29 @@ def objects_to_spec(spec_object: SpecObject, imports: str, fork: str, ordered_cl spec_object.constants[k] += " # noqa: E501" constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, spec_object.constants[x]), spec_object.constants)) ordered_class_objects_spec = '\n\n'.join(ordered_class_objects.values()) - - if is_altair(fork): - altair_ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS[x]), ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS)) - - if is_merge(fork): - merge_custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, MERGE_HARDCODED_CUSTOM_TYPE_DEP_CONSTANTS[x]), MERGE_HARDCODED_CUSTOM_TYPE_DEP_CONSTANTS)) - - + ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, adjustment.hardcoded_ssz_dep_constants()[x]), adjustment.hardcoded_ssz_dep_constants())) + ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), adjustment.hardcoded_ssz_dep_constants())) + custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, adjustment.hardcoded_custom_type_dep_constants()[x]), adjustment.hardcoded_custom_type_dep_constants())) spec = ( - imports + adjustment.imports_and_predefinitions() + '\n\n' + f"fork = \'{fork}\'\n" # The constants that some SSZ containers require. Need to be defined before `new_type_definitions` - + ('\n\n' + merge_custom_type_dep_constants + '\n' if is_merge(fork) else '') + + ('\n\n' + custom_type_dep_constants + '\n' if custom_type_dep_constants != '' else '') + '\n\n' + new_type_definitions - + '\n' + SUNDRY_CONSTANTS_FUNCTIONS + + '\n' + CONSTANT_DEP_SUNDRY_CONSTANTS_FUNCTIONS # The constants that some SSZ containers require. Need to be defined before `constants_spec` - + ('\n\n' + altair_ssz_dep_constants if is_altair(fork) else '') + + ('\n\n' + ssz_dep_constants if ssz_dep_constants != '' else '') + '\n\n' + constants_spec + '\n\n' + CONFIG_LOADER + '\n\n' + ordered_class_objects_spec + '\n\n' + functions_spec - # Functions to make pyspec work - + '\n' + PHASE0_SUNDRY_FUNCTIONS - + ('\n' + ALTAIR_SUNDRY_FUNCTIONS if is_altair(fork) else '') - + ('\n' + MERGE_SUNDRY_FUNCTIONS if is_merge(fork) else '') + + '\n' + adjustment.sundry_functions() + # Since some constants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are + # as same as the spec definition. + + ('\n\n\n' + ssz_dep_constants_verification if ssz_dep_constants_verification != '' else '') + + ('\n' + adjustment.invariant_checks() if adjustment.invariant_checks() != '' else '') + + '\n' ) - - # Since some constants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are - # as same as the spec definition. - if is_altair(fork): - altair_ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS)) - spec += '\n\n\n' + altair_ssz_dep_constants_verification - spec += '\n' + ALTAIR_INVAIANT_CHECKS - - spec += '\n' return spec @@ -496,13 +576,6 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: ) -fork_imports = { - 'phase0': PHASE0_IMPORTS, - 'altair': ALTAIR_IMPORTS, - 'merge': MERGE_IMPORTS, -} - - def build_spec(fork: str, source_files: List[str]) -> str: all_specs = [get_spec(spec) for spec in source_files] @@ -513,7 +586,7 @@ def build_spec(fork: str, source_files: List[str]) -> str: class_objects = {**spec_object.ssz_objects, **spec_object.dataclasses} dependency_order_class_objects(class_objects, spec_object.custom_types) - return objects_to_spec(spec_object, fork_imports[fork], fork, class_objects) + return objects_to_spec(spec_object, spec_adjustments[fork], fork, class_objects) class PySpecCommand(Command): @@ -611,7 +684,7 @@ class BuildPyCommand(build_py): self.run_command('pyspec') def run(self): - for spec_fork in fork_imports: + for spec_fork in spec_adjustments: self.run_pyspec_cmd(spec_fork=spec_fork) super(BuildPyCommand, self).run() @@ -639,7 +712,7 @@ class PyspecDevCommand(Command): def run(self): print("running build_py command") - for spec_fork in fork_imports: + for spec_fork in spec_adjustments: self.run_pyspec_cmd(spec_fork=spec_fork) commands = { From 1ffa436836c9117a0157290ff40b86478ddefa66 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 9 Apr 2021 21:28:58 +0800 Subject: [PATCH 2/5] Update `imports_and_predefinitions` --- setup.py | 48 +++++++++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/setup.py b/setup.py index 65b04fcf2..5498f2803 100644 --- a/setup.py +++ b/setup.py @@ -179,25 +179,22 @@ class SpecAdjustment(ABC): class Phase0SpecAdjustment(SpecAdjustment): @classmethod def imports_and_predefinitions(cls) -> str: - return '''from eth2spec.config.config_util import apply_constants_config -from typing import ( - Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar -) - + return '''from lru import LRU from dataclasses import ( dataclass, field, ) +from typing import ( + Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar +) -from lru import LRU - +from eth2spec.config.config_util import apply_constants_config from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes from eth2spec.utils.ssz.ssz_typing import ( View, boolean, Container, List, Vector, uint8, uint32, uint64, Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils import bls - from eth2spec.utils.hash_function import hash SSZObject = TypeVar('SSZObject', bound=View) @@ -297,19 +294,18 @@ get_attesting_indices = cache_this( class AltairSpecAdjustment(Phase0SpecAdjustment): @classmethod def imports_and_predefinitions(cls) -> str: - return '''from eth2spec.phase0 import spec as phase0 -from eth2spec.config.config_util import apply_constants_config -from typing import ( - Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable, Optional, Union -) - + return '''from lru import LRU from dataclasses import ( dataclass, field, ) +from typing import ( + Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable, Optional, Union +) -from lru import LRU +from eth2spec.config.config_util import apply_constants_config +from eth2spec.phase0 import spec as phase0 from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes from eth2spec.utils.ssz.ssz_typing import ( View, boolean, Container, List, Vector, uint8, uint32, uint64, @@ -320,7 +316,7 @@ from eth2spec.utils import bls from eth2spec.utils.hash_function import hash -# Whenever altair is loaded, make sure we have the latest phase0 +# Whenever this spec version is loaded, make sure we have the latest phase0 from importlib import reload reload(phase0) @@ -363,19 +359,17 @@ assert ( class MergeSpecAdjustment(Phase0SpecAdjustment): @classmethod def imports_and_predefinitions(cls): - return '''from eth2spec.phase0 import spec as phase0 -from eth2spec.config.config_util import apply_constants_config -from typing import ( - Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar -) - + return '''from lru import LRU from dataclasses import ( dataclass, field, ) +from typing import ( + Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar +) -from lru import LRU - +from eth2spec.phase0 import spec as phase0 +from eth2spec.config.config_util import apply_constants_config from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes from eth2spec.utils.ssz.ssz_typing import ( View, boolean, Container, List, Vector, uint8, uint32, uint64, uint256, @@ -383,9 +377,13 @@ from eth2spec.utils.ssz.ssz_typing import ( ByteList, ByteVector ) from eth2spec.utils import bls - from eth2spec.utils.hash_function import hash + +# Whenever this spec version is loaded, make sure we have the latest phase0 +from importlib import reload +reload(phase0) + SSZObject = TypeVar('SSZObject', bound=View) CONFIG_NAME = 'mainnet' From ceb352be12fbca77e710574c0961db237b67652c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 9 Apr 2021 22:17:01 +0800 Subject: [PATCH 3/5] Refactor `imports_and_predefinitions` into `imports` and `preparations` --- setup.py | 101 ++++++++++++++++++++++--------------------------------- 1 file changed, 40 insertions(+), 61 deletions(-) diff --git a/setup.py b/setup.py index 5498f2803..8201df574 100644 --- a/setup.py +++ b/setup.py @@ -134,9 +134,17 @@ def get_spec(file_name: str) -> SpecObject: class SpecAdjustment(ABC): @classmethod @abstractmethod - def imports_and_predefinitions(cls) -> str: + def imports(cls) -> str: """ - Importing functions and defining special types/constants for building pyspec. + Import objects from other libaries. + """ + raise NotImplementedError() + + @classmethod + @abstractmethod + def preparations(cls) -> str: + """ + Define special types/constants for building pyspec or call functions. """ raise NotImplementedError() @@ -178,7 +186,7 @@ class SpecAdjustment(ABC): # class Phase0SpecAdjustment(SpecAdjustment): @classmethod - def imports_and_predefinitions(cls) -> str: + def imports(cls) -> str: return '''from lru import LRU from dataclasses import ( dataclass, @@ -192,12 +200,15 @@ from eth2spec.config.config_util import apply_constants_config from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes from eth2spec.utils.ssz.ssz_typing import ( View, boolean, Container, List, Vector, uint8, uint32, uint64, - Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, -) + Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist) +from eth2spec.utils.ssz.ssz_typing import Bitvector # noqa: F401 from eth2spec.utils import bls from eth2spec.utils.hash_function import hash +''' -SSZObject = TypeVar('SSZObject', bound=View) + @classmethod + def preparations(cls) -> str: + return '''SSZObject = TypeVar('SSZObject', bound=View) CONFIG_NAME = 'mainnet' ''' @@ -293,40 +304,25 @@ get_attesting_indices = cache_this( # class AltairSpecAdjustment(Phase0SpecAdjustment): @classmethod - def imports_and_predefinitions(cls) -> str: - return '''from lru import LRU -from dataclasses import ( - dataclass, - field, -) -from typing import ( - Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable, Optional, Union -) - - -from eth2spec.config.config_util import apply_constants_config -from eth2spec.phase0 import spec as phase0 -from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes -from eth2spec.utils.ssz.ssz_typing import ( - View, boolean, Container, List, Vector, uint8, uint32, uint64, - Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, - Path, -) -from eth2spec.utils import bls - -from eth2spec.utils.hash_function import hash - -# Whenever this spec version is loaded, make sure we have the latest phase0 + def imports(cls) -> str: + return super().imports() + '\n' + ''' +from typing import NewType, Union from importlib import reload -reload(phase0) +from eth2spec.phase0 import spec as phase0 +from eth2spec.utils.ssz.ssz_typing import Path +''' + + @classmethod + def preparations(cls): + return super().preparations() + '\n' + ''' +# Whenever this spec version is loaded, make sure we have the latest phase0 +reload(phase0) SSZVariableName = str GeneralizedIndex = NewType('GeneralizedIndex', int) -SSZObject = TypeVar('SSZObject', bound=View) - -CONFIG_NAME = 'mainnet' ''' + @classmethod def sundry_functions(cls) -> str: return super().sundry_functions() + '\n\n' + ''' @@ -358,35 +354,17 @@ assert ( # class MergeSpecAdjustment(Phase0SpecAdjustment): @classmethod - def imports_and_predefinitions(cls): - return '''from lru import LRU -from dataclasses import ( - dataclass, - field, -) -from typing import ( - Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar -) - + def imports(cls): + return super().imports() + '\n' + ''' from eth2spec.phase0 import spec as phase0 -from eth2spec.config.config_util import apply_constants_config -from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes -from eth2spec.utils.ssz.ssz_typing import ( - View, boolean, Container, List, Vector, uint8, uint32, uint64, uint256, - Bytes1, Bytes4, Bytes20, Bytes32, Bytes48, Bytes96, Bitlist, - ByteList, ByteVector -) -from eth2spec.utils import bls -from eth2spec.utils.hash_function import hash - - -# Whenever this spec version is loaded, make sure we have the latest phase0 +from eth2spec.utils.ssz.ssz_typing import Bytes20, ByteList, ByteVector, uint256 from importlib import reload +''' + + @classmethod + def preparations(cls): + return super().preparations() + '\n' + ''' reload(phase0) - -SSZObject = TypeVar('SSZObject', bound=View) - -CONFIG_NAME = 'mainnet' ''' @classmethod @@ -476,7 +454,8 @@ def objects_to_spec(spec_object: SpecObject, adjustment: SpecAdjustment, fork: s ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), adjustment.hardcoded_ssz_dep_constants())) custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, adjustment.hardcoded_custom_type_dep_constants()[x]), adjustment.hardcoded_custom_type_dep_constants())) spec = ( - adjustment.imports_and_predefinitions() + adjustment.imports() + + adjustment.preparations() + '\n\n' + f"fork = \'{fork}\'\n" # The constants that some SSZ containers require. Need to be defined before `new_type_definitions` + ('\n\n' + custom_type_dep_constants + '\n' if custom_type_dep_constants != '' else '') From 3320ebb8654b419d6b38129162ad2accf783572c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 9 Apr 2021 22:26:32 +0800 Subject: [PATCH 4/5] Fix typo and add `SpecAdjustment.fork` property --- setup.py | 46 ++++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/setup.py b/setup.py index 8201df574..2101eebc4 100644 --- a/setup.py +++ b/setup.py @@ -132,11 +132,16 @@ def get_spec(file_name: str) -> SpecObject: class SpecAdjustment(ABC): + @property + @abstractmethod + def fork(self) -> str: + raise NotImplementedError() + @classmethod @abstractmethod def imports(cls) -> str: """ - Import objects from other libaries. + Import objects from other libraries. """ raise NotImplementedError() @@ -185,6 +190,8 @@ class SpecAdjustment(ABC): # Phase0SpecAdjustment # class Phase0SpecAdjustment(SpecAdjustment): + fork: str = PHASE0 + @classmethod def imports(cls) -> str: return '''from lru import LRU @@ -208,8 +215,8 @@ from eth2spec.utils.hash_function import hash @classmethod def preparations(cls) -> str: - return '''SSZObject = TypeVar('SSZObject', bound=View) - + return ''' +SSZObject = TypeVar('SSZObject', bound=View) CONFIG_NAME = 'mainnet' ''' @@ -303,6 +310,8 @@ get_attesting_indices = cache_this( # AltairSpecAdjustment # class AltairSpecAdjustment(Phase0SpecAdjustment): + fork: str = ALTAIR + @classmethod def imports(cls) -> str: return super().imports() + '\n' + ''' @@ -353,6 +362,8 @@ assert ( # MergeSpecAdjustment # class MergeSpecAdjustment(Phase0SpecAdjustment): + fork: str = MERGE + @classmethod def imports(cls): return super().imports() + '\n' + ''' @@ -402,25 +413,12 @@ def produce_execution_payload(parent_hash: Bytes32) -> ExecutionPayload: spec_adjustments = { - PHASE0: Phase0SpecAdjustment, - ALTAIR: AltairSpecAdjustment, - MERGE: MergeSpecAdjustment, + adjustment.fork: adjustment + for adjustment in (Phase0SpecAdjustment, AltairSpecAdjustment, MergeSpecAdjustment) } -def is_phase0(fork): - return fork == PHASE0 - - -def is_altair(fork): - return fork == ALTAIR - - -def is_merge(fork): - return fork == MERGE - - -def objects_to_spec(spec_object: SpecObject, adjustment: SpecAdjustment, fork: str, ordered_class_objects: Dict[str, str]) -> str: +def objects_to_spec(spec_object: SpecObject, adjustment: SpecAdjustment, ordered_class_objects: Dict[str, str]) -> str: """ Given all the objects that constitute a spec, combine them into a single pyfile. """ @@ -456,7 +454,7 @@ def objects_to_spec(spec_object: SpecObject, adjustment: SpecAdjustment, fork: s spec = ( adjustment.imports() + adjustment.preparations() - + '\n\n' + f"fork = \'{fork}\'\n" + + '\n\n' + f"fork = \'{adjustment.fork}\'\n" # The constants that some SSZ containers require. Need to be defined before `new_type_definitions` + ('\n\n' + custom_type_dep_constants + '\n' if custom_type_dep_constants != '' else '') + '\n\n' + new_type_definitions @@ -563,7 +561,7 @@ def build_spec(fork: str, source_files: List[str]) -> str: class_objects = {**spec_object.ssz_objects, **spec_object.dataclasses} dependency_order_class_objects(class_objects, spec_object.custom_types) - return objects_to_spec(spec_object, spec_adjustments[fork], fork, class_objects) + return objects_to_spec(spec_object, spec_adjustments[fork], class_objects) class PySpecCommand(Command): @@ -595,14 +593,14 @@ class PySpecCommand(Command): if len(self.md_doc_paths) == 0: print("no paths were specified, using default markdown file paths for pyspec" " build (spec fork: %s)" % self.spec_fork) - if is_phase0(self.spec_fork): + if self.spec_fork == PHASE0: self.md_doc_paths = """ specs/phase0/beacon-chain.md specs/phase0/fork-choice.md specs/phase0/validator.md specs/phase0/weak-subjectivity.md """ - elif is_altair(self.spec_fork): + elif self.spec_fork == ALTAIR: self.md_doc_paths = """ specs/phase0/beacon-chain.md specs/phase0/fork-choice.md @@ -613,7 +611,7 @@ class PySpecCommand(Command): specs/altair/validator.md specs/altair/sync-protocol.md """ - elif is_merge(self.spec_fork): + elif self.spec_fork == MERGE: self.md_doc_paths = """ specs/phase0/beacon-chain.md specs/phase0/fork-choice.md From 5349645a8f012c38dc7abe2e060a593c4ab409d0 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 16 Apr 2021 11:29:10 +0800 Subject: [PATCH 5/5] Rename `SpecAdjustment` to `SpecBuilder` and add `build_spec` interface --- setup.py | 59 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/setup.py b/setup.py index c6e9f55ae..66a1b7cfe 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from distutils import dir_util from distutils.util import convert_path import os import re -from typing import Dict, NamedTuple, List +from typing import Dict, NamedTuple, List, Sequence from abc import ABC, abstractmethod @@ -131,7 +131,7 @@ def get_spec(file_name: str) -> SpecObject: ) -class SpecAdjustment(ABC): +class SpecBuilder(ABC): @property @abstractmethod def fork(self) -> str: @@ -185,11 +185,16 @@ class SpecAdjustment(ABC): """ raise NotImplementedError() + @classmethod + @abstractmethod + def build_spec(cls, source_files: List[str]) -> str: + raise NotImplementedError() + # -# Phase0SpecAdjustment +# Phase0SpecBuilder # -class Phase0SpecAdjustment(SpecAdjustment): +class Phase0SpecBuilder(SpecBuilder): fork: str = PHASE0 @classmethod @@ -305,11 +310,15 @@ get_attesting_indices = cache_this( def invariant_checks(cls) -> str: return '' + @classmethod + def build_spec(cls, source_files: Sequence[str]) -> str: + return _build_spec(cls.fork, source_files) + # -# AltairSpecAdjustment +# AltairSpecBuilder # -class AltairSpecAdjustment(Phase0SpecAdjustment): +class AltairSpecBuilder(Phase0SpecBuilder): fork: str = ALTAIR @classmethod @@ -359,9 +368,9 @@ assert ( # -# MergeSpecAdjustment +# MergeSpecBuilder # -class MergeSpecAdjustment(Phase0SpecAdjustment): +class MergeSpecBuilder(Phase0SpecBuilder): fork: str = MERGE @classmethod @@ -412,13 +421,13 @@ def produce_execution_payload(parent_hash: Hash32, timestamp: uint64) -> Executi return {**super().hardcoded_custom_type_dep_constants(), **constants} -spec_adjustments = { - adjustment.fork: adjustment - for adjustment in (Phase0SpecAdjustment, AltairSpecAdjustment, MergeSpecAdjustment) +spec_builders = { + builder.fork: builder + for builder in (Phase0SpecBuilder, AltairSpecBuilder, MergeSpecBuilder) } -def objects_to_spec(spec_object: SpecObject, adjustment: SpecAdjustment, ordered_class_objects: Dict[str, str]) -> str: +def objects_to_spec(spec_object: SpecObject, builder: SpecBuilder, ordered_class_objects: Dict[str, str]) -> str: """ Given all the objects that constitute a spec, combine them into a single pyfile. """ @@ -448,13 +457,13 @@ def objects_to_spec(spec_object: SpecObject, adjustment: SpecAdjustment, ordered spec_object.constants[k] += " # noqa: E501" constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, spec_object.constants[x]), spec_object.constants)) ordered_class_objects_spec = '\n\n'.join(ordered_class_objects.values()) - ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, adjustment.hardcoded_ssz_dep_constants()[x]), adjustment.hardcoded_ssz_dep_constants())) - ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), adjustment.hardcoded_ssz_dep_constants())) - custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, adjustment.hardcoded_custom_type_dep_constants()[x]), adjustment.hardcoded_custom_type_dep_constants())) + ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, builder.hardcoded_ssz_dep_constants()[x]), builder.hardcoded_ssz_dep_constants())) + ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), builder.hardcoded_ssz_dep_constants())) + custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, builder.hardcoded_custom_type_dep_constants()[x]), builder.hardcoded_custom_type_dep_constants())) spec = ( - adjustment.imports() - + adjustment.preparations() - + '\n\n' + f"fork = \'{adjustment.fork}\'\n" + builder.imports() + + builder.preparations() + + '\n\n' + f"fork = \'{builder.fork}\'\n" # The constants that some SSZ containers require. Need to be defined before `new_type_definitions` + ('\n\n' + custom_type_dep_constants + '\n' if custom_type_dep_constants != '' else '') + '\n\n' + new_type_definitions @@ -465,11 +474,11 @@ def objects_to_spec(spec_object: SpecObject, adjustment: SpecAdjustment, ordered + '\n\n' + CONFIG_LOADER + '\n\n' + ordered_class_objects_spec + '\n\n' + functions_spec - + '\n' + adjustment.sundry_functions() + + '\n' + builder.sundry_functions() # Since some constants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are # as same as the spec definition. + ('\n\n\n' + ssz_dep_constants_verification if ssz_dep_constants_verification != '' else '') - + ('\n' + adjustment.invariant_checks() if adjustment.invariant_checks() != '' else '') + + ('\n' + builder.invariant_checks() if builder.invariant_checks() != '' else '') + '\n' ) return spec @@ -551,7 +560,7 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: ) -def build_spec(fork: str, source_files: List[str]) -> str: +def _build_spec(fork: str, source_files: Sequence[str]) -> str: all_specs = [get_spec(spec) for spec in source_files] spec_object = all_specs[0] @@ -561,7 +570,7 @@ def build_spec(fork: str, source_files: List[str]) -> str: class_objects = {**spec_object.ssz_objects, **spec_object.dataclasses} dependency_order_class_objects(class_objects, spec_object.custom_types) - return objects_to_spec(spec_object, spec_adjustments[fork], class_objects) + return objects_to_spec(spec_object, spec_builders[fork], class_objects) class PySpecCommand(Command): @@ -631,7 +640,7 @@ class PySpecCommand(Command): raise Exception('Pyspec markdown input file "%s" does not exist.' % filename) def run(self): - spec_str = build_spec(self.spec_fork, self.parsed_md_doc_paths) + spec_str = spec_builders[self.spec_fork].build_spec(self.parsed_md_doc_paths) if self.dry_run: self.announce('dry run successfully prepared contents for spec.' f' out dir: "{self.out_dir}", spec fork: "{self.spec_fork}"') @@ -659,7 +668,7 @@ class BuildPyCommand(build_py): self.run_command('pyspec') def run(self): - for spec_fork in spec_adjustments: + for spec_fork in spec_builders: self.run_pyspec_cmd(spec_fork=spec_fork) super(BuildPyCommand, self).run() @@ -687,7 +696,7 @@ class PyspecDevCommand(Command): def run(self): print("running build_py command") - for spec_fork in spec_adjustments: + for spec_fork in spec_builders: self.run_pyspec_cmd(spec_fork=spec_fork) commands = {