Merge pull request #2097 from ethereum/tests-compression

Tests compression with Snappy, remove YAML duplicates

setup.py

@@ -576,6 +576,7 @@ setup(
     extras_require={
         "test": ["pytest>=4.4", "pytest-cov", "pytest-xdist"],
         "lint": ["flake8==3.7.7", "mypy==0.750"],
+        "generator": ["python-snappy==0.5.4"],
     },
     install_requires=[
         "eth-utils>=1.3.0,<2",
@@ -586,6 +587,6 @@ setup(
         "dataclasses==0.6",
         "remerkleable==0.1.18",
         "ruamel.yaml==0.16.5",
-        "lru-dict==1.1.6"
+        "lru-dict==1.1.6",
     ]
 )

tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py

@@ -8,6 +8,8 @@ from ruamel.yaml import (
     YAML,
 )
 
+from snappy import compress
+
 from eth2spec.test import context
 from eth2spec.test.exceptions import SkippedTest
 
@@ -181,7 +183,8 @@ def dump_yaml_fn(data: Any, name: str, file_mode: str, yaml_encoder: YAML):
 
 def dump_ssz_fn(data: AnyStr, name: str, file_mode: str):
     def dump(case_path: Path):
-        out_path = case_path / Path(name + '.ssz')
+        out_path = case_path / Path(name + '.ssz_snappy')
+        compressed = compress(data)
         with out_path.open(file_mode + 'b') as f:  # write in raw binary mode
-            f.write(data)
+            f.write(compressed)
     return dump

tests/core/pyspec/eth2spec/test/utils.py

@@ -1,5 +1,4 @@
 from typing import Dict, Any
-from eth2spec.debug.encode import encode
 from eth2spec.utils.ssz.ssz_typing import View
 from eth2spec.utils.ssz.ssz_impl import serialize
 
@@ -39,24 +38,20 @@ def vector_test(description: str = None):
                     if value is None:
                         continue
                     if isinstance(value, View):
-                        yield key, 'data', encode(value)
                         yield key, 'ssz', serialize(value)
                     elif isinstance(value, bytes):
-                        yield key, 'data', encode(value)
                         yield key, 'ssz', value
                     elif isinstance(value, list) and all([isinstance(el, (View, bytes)) for el in value]):
                         for i, el in enumerate(value):
                             if isinstance(el, View):
-                                yield f'{key}_{i}', 'data', encode(el)
                                 yield f'{key}_{i}', 'ssz', serialize(el)
                             elif isinstance(el, bytes):
-                                yield f'{key}_{i}', 'data', encode(el)
                                 yield f'{key}_{i}', 'ssz', el
                         yield f'{key}_count', 'meta', len(value)
                     else:
                         # Not a ssz value.
                         # The data will now just be yielded as any python data,
-                        #  something that should be encodeable by the generator runner.
+                        #  something that should be encodable by the generator runner.
                         yield key, 'data', value
 
             # check generator mode, may be None/else.

tests/formats/README.md

@@ -132,21 +132,25 @@ Cases are split up too. This enables diffing of parts of the test case, tracking
 
 ### `<output part>`
 
-E.g. `pre.yaml`, `deposit.yaml`, `post.yaml`.
-
-Diffing a `pre.yaml` and `post.yaml` provides all the information for testing, good for readability of the change. 
-Then the difference between pre and post can be compared to anything that changes the pre state, e.g. `deposit.yaml`
-
 These files allow for custom formats for some parts of the test. E.g. something encoded in SSZ.
+Or to avoid large files, the SSZ can be compressed with Snappy.
+E.g. `pre.ssz_snappy`, `deposit.ssz_snappy`, `post.ssz_snappy`.
 
-Some yaml files have copies, but formatted as raw SSZ bytes: `pre.ssz`, `deposit.ssz`, `post.ssz`.
-The yaml files are intended to be deprecated, and clients should shift to ssz inputs for efficiency.
-Deprecation will start once a viewer of SSZ test-cases is in place, to maintain a standard of readable test cases.
-This also means that some clients can drop legacy YAML -> JSON/other -> SSZ work-arounds.
-(These were implemented to support the uint64 YAML, hex strings, etc. Things that were not idiomatic to their language.)
+Diffing a `pre.ssz_snappy` and `post.ssz_snappy` provides all the information for testing, when decompressed and decoded.
+Then the difference between pre and post can be compared to anything that changes the pre state, e.g. `deposit.ssz_snappy`
+
+YAML is generally used for test metadata, and for tests that do not use SSZ: e.g. shuffling and BLS tests.
+In this case, there is no point in adding special SSZ types. And the size and efficiency of YAML is acceptable.
+
+#### Common output formats
+
+Between all types of tests, a few formats are common:
+
+- **`.yaml`**: A YAML file containing structured data to describe settings or test contents.
+- **`.ssz`**: A file containing raw SSZ-encoded data. Previously widely used in tests, but replaced with compressed variant.
+- **`.ssz_snappy`**: Like `.ssz`, but compressed with Snappy block compression.
+  Snappy block compression is already applied to SSZ in Eth2 gossip, available in client implementations, and thus chosen as compression method.
 
-Yaml will not be deprecated for tests that do not use SSZ: e.g. shuffling and BLS tests.
-In this case, there is no work around for loading necessary anyway, and the size and efficiency of yaml is acceptable.
 
 #### Special output parts
 

tests/formats/epoch_processing/README.md

@@ -15,18 +15,13 @@ description: string    -- Optional description of test case, purely for debuggin
 bls_setting: int       -- see general test-format spec.
 ```
 
-### `pre.yaml`
+### `pre.ssz_snappy`
 
-A YAML-encoded `BeaconState`, the state before running the epoch sub-transition.
+A SSZ-snappy encoded `BeaconState`, the state before running the epoch sub-transition.
 
-Also available as `pre.ssz`.
+### `post.ssz_snappy`
 
-
-### `post.yaml`
-
-A YAML-encoded `BeaconState`, the state after applying the epoch sub-transition.
-
-Also available as `post.ssz`.
+A SSZ-snappy encoded `BeaconState`, the state after applying the epoch sub-transition.
 
 ## Condition
 

tests/formats/finality/README.md

@@ -14,11 +14,11 @@ bls_setting: int       -- see general test-format spec.
 blocks_count: int      -- the number of blocks processed in this test.
 ```
 
-### `pre.yaml`
+### `pre.ssz_snappy`
 
-A YAML-encoded `BeaconState`, the state before running the block transitions.
+A SSZ-snappy encoded `BeaconState`, the state before running the block transitions.
 
-Also available as `pre.ssz`.
+Also available as `pre.ssz_snappy`.
 
 
 ### `blocks_<index>.yaml`
@@ -28,13 +28,11 @@ A series of files, with `<index>` in range `[0, blocks_count)`. Blocks need to b
 
 Each file is a YAML-encoded `SignedBeaconBlock`.
 
-Each block is also available as `blocks_<index>.ssz`
+Each block is also available as `blocks_<index>.ssz_snappy`
 
-### `post.yaml`
+### `post.ssz_snappy`
 
-A YAML-encoded `BeaconState`, the state after applying the block transitions.
-
-Also available as `post.ssz`.
+A SSZ-snappy encoded `BeaconState`, the state after applying the block transitions.
 
 
 ## Condition

tests/formats/forks/README.md

@@ -24,17 +24,13 @@ Key of valid `fork` strings that might be found in `meta.yaml`
 | - | - | - | - |
 | `altair` | Phase 0 | Altair | `upgrade_to_lightclient_patch` |
 
-### `pre.yaml`
+### `pre.ssz_snappy`
 
-A YAML-encoded `BeaconState`, the state before running the fork transition.
+A SSZ-snappy encoded `BeaconState`, the state before running the fork transition.
 
-Also available as `pre.ssz`.
+### `post.ssz_snappy`
 
-### `post.yaml`
-
-A YAML-encoded `BeaconState`, the state after applying the fork transition.
-
-Also available as `post.ssz`.
+A SSZ-snappy encoded `BeaconState`, the state after applying the fork transition.
 
 *Note*: This type is the `BeaconState` after the fork and is *not* the same type as `pre`.
 

tests/formats/genesis/initialization.md

@@ -4,11 +4,9 @@ Tests the initialization of a genesis state based on Eth1 data.
 
 ## Test case format
 
-### `eth1_block_hash.yaml`
+### `eth1_block_hash.ssz_snappy`
 
-A `Bytes32` hex encoded, with prefix 0x. The root of the Eth1 block.
-
-Also available as `eth1_block_hash.ssz`.
+A SSZ-snappy encoded root of the Eth1 block.
 
 ### `eth1_timestamp.yaml`
 
@@ -22,18 +20,15 @@ A yaml file to help read the deposit count:
 deposits_count: int    -- Amount of deposits.
 ```
 
-### `deposits_<index>.yaml`
+### `deposits_<index>.ssz_snappy`
 
 A series of files, with `<index>` in range `[0, deposits_count)`. Deposits need to be processed in order.
-Each file is a YAML-encoded `Deposit` object.
+Each file is a SSZ-snappy encoded `Deposit` object.
 
-Each deposit is also available as `deposits_<index>.ssz`.
+###  `state.ssz_snappy`
 
-###  `state.yaml`
+The expected genesis state. A SSZ-snappy encoded `BeaconState` object.
 
-The expected genesis state. A YAML-encoded `BeaconState` object.
-
-Also available as `state.ssz`.
 
 ## Processing
 

tests/formats/genesis/validity.md

@@ -4,11 +4,10 @@ Tests if a genesis state is valid, i.e. if it counts as trigger to launch.
 
 ## Test case format
 
-### `genesis.yaml`
+### `genesis.ssz_snappy`
 
-A `BeaconState`, the state to validate as genesis candidate.
+A SSZ-snappy encoded `BeaconState`, the state to validate as genesis candidate.
 
-Also available as `genesis.ssz`. 
 
 ### `is_valid.yaml`
 

tests/formats/operations/README.md

@@ -12,23 +12,17 @@ description: string    -- Optional description of test case, purely for debuggin
 bls_setting: int       -- see general test-format spec.
 ```
 
-### `pre.yaml`
+### `pre.ssz_snappy`
 
-A YAML-encoded `BeaconState`, the state before applying the operation.
+A SSZ-snappy encoded `BeaconState`, the state before applying the operation.
 
-Also available as `pre.ssz`.
+### `<input-name>.ssz_snappy`
 
-### `<input-name>.yaml`
+A SSZ-snappy encoded operation object, e.g. a `ProposerSlashing`, or `Deposit`.
 
-A YAML-encoded operation object, e.g. a `ProposerSlashing`, or `Deposit`.
+### `post.ssz_snappy`
 
-Also available as `<input-name>.ssz`.
-
-### `post.yaml`
-
-A YAML-encoded `BeaconState`, the state after applying the operation. No value if operation processing is aborted.
-
-Also available as `post.ssz`.
+A SSZ-snappy encoded `BeaconState`, the state after applying the operation. No value if operation processing is aborted.
 
 
 ## Condition

tests/formats/rewards/README.md

@@ -23,41 +23,29 @@ description: string    -- Optional description of test case, purely for debuggin
 _Note_: No signature verification happens within rewards sub-functions. These
  tests can safely be run with or without BLS enabled.
 
-### `pre.yaml`
+### `pre.ssz_snappy`
 
-A YAML-encoded `BeaconState`, the state before running the rewards sub-function.
+A SSZ-snappy encoded `BeaconState`, the state before running the rewards sub-function.
 
-Also available as `pre.ssz`.
+### `source_deltas.ssz_snappy`
 
-### `source_deltas.yaml`
+A SSZ-snappy encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_source_deltas` function
 
-A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_source_deltas` function
+### `target_deltas.ssz_snappy`
 
-Also available as `source_deltas.ssz`.
+A SSZ-snappy encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_target_deltas` function
 
-### `target_deltas.yaml`
+### `head_deltas.ssz_snappy`
 
-A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_target_deltas` function
+A SSZ-snappy encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_head_deltas` function
 
-Also available as `target_deltas.ssz`.
+### `inclusion_delay_deltas.ssz_snappy`
 
-### `head_deltas.yaml`
+A SSZ-snappy encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_inclusion_delay_deltas` function
 
-A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_head_deltas` function
+### `inactivity_penalty_deltas.ssz_snappy`
 
-Also available as `head_deltas.ssz`.
-
-### `inclusion_delay_deltas.yaml`
-
-A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_inclusion_delay_deltas` function
-
-Also available as `inclusion_delay_deltas.ssz`.
-
-### `inactivity_penalty_deltas.yaml`
-
-A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_inactivity_penalty_deltas` function
-
-Also available as `inactivity_penalty_deltas.ssz`.
+A SSZ-snappy encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_inactivity_penalty_deltas` function
 
 ## Condition
 

tests/formats/sanity/blocks.md

@@ -14,27 +14,21 @@ blocks_count: int              -- the number of blocks processed in this test.
 ```
 
 
-### `pre.yaml`
+### `pre.ssz_snappy`
 
-A YAML-encoded `BeaconState`, the state before running the block transitions.
-
-Also available as `pre.ssz`.
+A SSZ-snappy encoded `BeaconState`, the state before running the block transitions.
 
 
-### `blocks_<index>.yaml`
+### `blocks_<index>.ssz_snappy`
 
 A series of files, with `<index>` in range `[0, blocks_count)`. Blocks need to be processed in order,
  following the main transition function (i.e. process slot and epoch transitions in between blocks as normal)
 
-Each file is a YAML-encoded `SignedBeaconBlock`.
+Each file is a SSZ-snappy encoded `SignedBeaconBlock`.
 
-Each block is also available as `blocks_<index>.ssz`
+### `post.ssz_snappy`
 
-### `post.yaml`
-
-A YAML-encoded `BeaconState`, the state after applying the block transitions.
-
-Also available as `post.ssz`.
+A SSZ-snappy encoded `BeaconState`, the state after applying the block transitions.
 
 
 ## Condition

tests/formats/sanity/slots.md

@@ -16,7 +16,7 @@ bls_setting: int       -- see general test-format spec.
 
 A YAML-encoded `BeaconState`, the state before running the transitions.
 
-Also available as `pre.ssz`.
+Also available as `pre.ssz_snappy`.
 
 
 ### `slots.yaml`
@@ -27,7 +27,7 @@ An integer. The amount of slots to process (i.e. the difference in slots between
 
 A YAML-encoded `BeaconState`, the state after applying the transitions.
 
-Also available as `post.ssz`.
+Also available as `post.ssz_snappy`.
 
 
 ### Processing

tests/formats/ssz_generic/README.md

@@ -33,7 +33,7 @@ Each of the handlers encodes the SSZ type declaration in the file-name. See [Typ
 
 ### `valid`
 
-Valid has 3 parts: `meta.yaml`, `serialized.ssz`, `value.yaml`
+Valid has 3 parts: `meta.yaml`, `serialized.ssz_snappy`, `value.yaml`
 
 ### `meta.yaml`
 
@@ -46,9 +46,9 @@ root: Bytes32             -- Hash-tree-root of the object
 
 The `Bytes32` is encoded as a string, hexadecimal encoding, prefixed with `0x`.
 
-### `serialized.ssz`
+### `serialized.ssz_snappy`
 
-The serialized form of the object, as raw SSZ bytes.
+The serialized form of the object, as snappy-compressed SSZ bytes.
 
 ### `value.yaml`
 
@@ -64,7 +64,7 @@ The conditions are the same for each type:
 
 ## `invalid`
 
-Test cases in the `invalid` suite only include the `serialized.ssz`
+Test cases in the `invalid` suite only include the `serialized.ssz_snappy`
 
 #### Condition
 

tests/formats/ssz_static/core.md

@@ -18,7 +18,7 @@ One can iterate over the handlers, and select the type based on the handler name
 Suites are then the same format, but each specialized in one randomization mode.
 Some randomization modes may only produce a single test case (e.g. the all-zeroes case).
 
-The output parts are: `roots.yaml`, `serialized.ssz`, `value.yaml`
+The output parts are: `roots.yaml`, `serialized.ssz_snappy`, `value.yaml`
 
 ### `roots.yaml`
 
@@ -26,13 +26,13 @@ The output parts are: `roots.yaml`, `serialized.ssz`, `value.yaml`
 root: bytes32         -- string, hash-tree-root of the value, hex encoded, with prefix 0x
 ```
 
-### `serialized.ssz`
+### `serialized.ssz_snappy`
 
-The raw encoded bytes.
+The SSZ-snappy encoded bytes.
 
 ### `value.yaml`
 
-The same value as `serialized.ssz`, represented as YAML.
+The same value as `serialized.ssz_snappy`, represented as YAML.
 
 
 ## Condition

tests/generators/README.md

@@ -79,7 +79,7 @@ It's recommended to extend the base-generator.
 Create a `requirements.txt` in the root of your generator directory:
 ```
 pytest>=4.4
-../../../
+../../../[generator]
 ```
 
 The config helper and pyspec is optional, but preferred. We encourage generators to derive tests from the spec itself in order to prevent code duplication and outdated tests.

tests/generators/bls/requirements.txt

@@ -1,2 +1,2 @@
 pytest>=4.4
-../../../
+../../../[generator]

tests/generators/epoch_processing/requirements.txt

@@ -1,2 +1,2 @@
 pytest>=4.4
-../../../
+../../../[generator]

tests/generators/finality/requirements.txt

@@ -1,2 +1,2 @@
 pytest>=4.4
-../../../
+../../../[generator]

tests/generators/forks/requirements.txt

@@ -1,2 +1,2 @@
 pytest>=4.4
-../../../
+../../../[generator]

tests/generators/genesis/requirements.txt

@@ -1,2 +1,2 @@
 pytest>=4.4
-../../../
+../../../[generator]

tests/generators/operations/requirements.txt

@@ -1,2 +1,2 @@
 pytest>=4.4
-../../../
+../../../[generator]

tests/generators/rewards/requirements.txt

@@ -1,2 +1,2 @@
 pytest>=4.4
-../../../
+../../../[generator]

tests/generators/shuffling/requirements.txt

@@ -1,2 +1,2 @@
 pytest>=4.4
-../../../
+../../../[generator]

tests/generators/ssz_generic/requirements.txt

@@ -1,2 +1,2 @@
 pytest>=4.4
-../../../
+../../../[generator]

tests/generators/ssz_static/requirements.txt

@@ -1,2 +1,2 @@
 pytest>=4.4
-../../../
+../../../[generator]