diff --git a/setup.py b/setup.py index 36e9dc656..5dbcff276 100644 --- a/setup.py +++ b/setup.py @@ -783,7 +783,7 @@ def parse_config_vars(conf: Dict[str, str]) -> Dict[str, str]: for k, v in conf.items(): if isinstance(v, str) and (v.startswith("0x") or k == 'PRESET_BASE' or k == 'CONFIG_NAME'): # Represent byte data with string, to avoid misinterpretation as big-endian int. - # Everything is either byte data or an integer, with PRESET_BASE as one exception. + # Everything except PRESET_BASE and CONFIG_NAME is either byte data or an integer. out[k] = f"'{v}'" else: out[k] = str(int(v)) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index 6850af188..07daa9839 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -1,3 +1,4 @@ +from eth_utils import encode_hex import os import time import shutil @@ -95,6 +96,20 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): yaml = YAML(pure=True) yaml.default_flow_style = None + # Spec config is using a YAML subset + cfg_yaml = YAML(pure=True) + cfg_yaml.default_flow_style = False # Emit separate line for each key + + def cfg_represent_bytes(self, data): + return self.represent_int(encode_hex(data)) + + cfg_yaml.representer.add_representer(bytes, cfg_represent_bytes) + + def cfg_represent_quoted_str(self, data): + return self.represent_scalar(u'tag:yaml.org,2002:str', data, style="'") + + cfg_yaml.representer.add_representer(context.quoted_str, cfg_represent_quoted_str) + log_file = Path(output_dir) / 'testgen_error_log.txt' print(f"Generating tests into {output_dir}") @@ -167,10 +182,14 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): written_part = True if out_kind == "meta": meta[name] = data - if out_kind == "data": - output_part("data", name, dump_yaml_fn(data, name, file_mode, yaml)) - if out_kind == "ssz": - output_part("ssz", name, dump_ssz_fn(data, name, file_mode)) + elif out_kind == "cfg": + output_part(out_kind, name, dump_yaml_fn(data, name, file_mode, cfg_yaml)) + elif out_kind == "data": + output_part(out_kind, name, dump_yaml_fn(data, name, file_mode, yaml)) + elif out_kind == "ssz": + output_part(out_kind, name, dump_ssz_fn(data, name, file_mode)) + else: + assert False # Unknown kind except SkippedTest as e: print(e) skipped_test_count += 1 diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_typing.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_typing.py index 669238d1c..2bb66d06c 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_typing.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_typing.py @@ -10,9 +10,10 @@ from dataclasses import dataclass # Elements: name, out_kind, data # # out_kind is the type of data: +# - "meta" for generic data to collect into a meta data dict +# - "cfg" for a spec config dictionary # - "data" for generic # - "ssz" for SSZ encoded bytes -# - "meta" for generic data to collect into a meta data dict. TestCasePart = NewType("TestCasePart", Tuple[str, str, Any]) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 06313c195..3a0ee5ebb 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -1,7 +1,6 @@ import pytest from dataclasses import dataclass import importlib -from eth_utils import encode_hex from eth2spec.phase0 import mainnet as spec_phase0_mainnet, minimal as spec_phase0_minimal from eth2spec.altair import mainnet as spec_altair_mainnet, minimal as spec_altair_minimal @@ -506,18 +505,22 @@ def with_presets(preset_bases, reason=None): return decorator +class quoted_str(str): + pass + + def _get_basic_dict(ssz_dict: Dict[str, Any]) -> Dict[str, Any]: """ - Get dict of Python built-in types from a dict of SSZ objects. + Get dict of basic types from a dict of SSZ objects. """ result = {} for k, v in ssz_dict.items(): if isinstance(v, int): value = int(v) elif isinstance(v, bytes): - value = encode_hex(v) + value = bytes(bytearray(v)) else: - value = str(v) + value = quoted_str(v) result[k] = value return result @@ -553,7 +556,7 @@ def with_config_overrides(config_overrides): # To output the changed config to could be serialized with yaml test vectors, # the dict SSZ objects have to be converted into Python built-in types. output_config = _get_basic_dict(modified_config) - yield 'config', 'data', output_config + yield 'config', 'cfg', output_config spec.config = spec.Configuration(**modified_config) diff --git a/tests/core/pyspec/eth2spec/test/utils/utils.py b/tests/core/pyspec/eth2spec/test/utils/utils.py index bad6c867b..14a7ceeb9 100644 --- a/tests/core/pyspec/eth2spec/test/utils/utils.py +++ b/tests/core/pyspec/eth2spec/test/utils/utils.py @@ -17,6 +17,7 @@ def vector_test(description: str = None): # this wraps the function, to yield type-annotated entries of data. # Valid types are: # - "meta": all key-values with this type can be collected by the generator, to put somewhere together. + # - "cfg": spec config dictionary # - "ssz": raw SSZ bytes # - "data": a python structure to be encoded by the user. def entry(*args, **kw):