Add to epoch processing tests generation states before and after full epoch processing (#4155)

tests/core/pyspec/eth2spec/test/electra/epoch_processing/pending_deposits/test_apply_pending_deposit.py

@@ -330,25 +330,6 @@ def test_apply_pending_deposit_top_up__less_effective_balance(spec, state):
     assert state.validators[validator_index].effective_balance == initial_effective_balance
 
 
-@with_electra_and_later
-@spec_state_test
-def test_apply_pending_deposit_top_up__zero_balance(spec, state):
-    validator_index = 0
-    amount = spec.MIN_ACTIVATION_BALANCE // 4
-    pending_deposit = prepare_pending_deposit(spec, validator_index, amount, signed=True)
-
-    initial_balance = 0
-    initial_effective_balance = 0
-    state.balances[validator_index] = initial_balance
-    state.validators[validator_index].effective_balance = initial_effective_balance
-
-    yield from run_pending_deposit_applying(spec, state, pending_deposit, validator_index)
-
-    assert state.balances[validator_index] == initial_balance + amount
-    # unchanged effective balance
-    assert state.validators[validator_index].effective_balance == initial_effective_balance
-
-
 @with_electra_and_later
 @spec_state_test
 @always_bls

tests/core/pyspec/eth2spec/test/helpers/deposits.py

@@ -1,7 +1,11 @@
 from random import Random
 
 from eth2spec.test.context import expect_assertion_error
-from eth2spec.test.helpers.epoch_processing import run_epoch_processing_to
+from eth2spec.test.helpers.epoch_processing import (
+    run_epoch_processing_from,
+    run_epoch_processing_to,
+    run_process_slots_up_to_epoch_boundary,
+)
 from eth2spec.test.helpers.forks import is_post_altair, is_post_electra
 from eth2spec.test.helpers.keys import privkeys, pubkeys
 from eth2spec.test.helpers.state import get_balance
@@ -424,6 +428,8 @@ def run_pending_deposit_applying(spec, state, pending_deposit, validator_index,
     Enqueue ``pending_deposit`` and run epoch processing with ``process_pending_deposits``, yielding:
       - pre-state ('pre')
       - post-state ('post').
+      - pre-epoch-state ('pre_epoch'), state before epoch transition
+      - post-epoch-state ('post_epoch'), state after epoch transition
     """
     assert is_post_electra(spec)
 
@@ -439,10 +445,6 @@ def run_pending_deposit_applying(spec, state, pending_deposit, validator_index,
     # append pending deposit
     state.pending_deposits.append(pending_deposit)
 
-    # run to the very beginning of the epoch processing to avoid
-    # any updates to the validator registry (e.g. ejections)
-    run_epoch_processing_to(spec, state, "process_justification_and_finalization")
-
     pre_validator_count = len(state.validators)
     pre_balance = 0
     pre_effective_balance = 0
@@ -453,12 +455,18 @@ def run_pending_deposit_applying(spec, state, pending_deposit, validator_index,
         pre_balance = get_balance(state, validator_index)
         pre_effective_balance = state.validators[validator_index].effective_balance
 
+    run_process_slots_up_to_epoch_boundary(spec, state)
+    yield "pre_epoch", state
+    run_epoch_processing_to(spec, state, "process_pending_deposits", enable_slots_processing=False)
+
     yield "pre", state
-
     spec.process_pending_deposits(state)
-
     yield "post", state
 
+    continue_state = state.copy()
+    run_epoch_processing_from(spec, continue_state, "process_pending_deposits")
+    yield "post_epoch", continue_state
+
     if effective:
         if is_top_up:
             # Top-ups don't add validators

tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py

@@ -37,18 +37,12 @@ def get_process_calls(spec):
     ]
 
 
-def run_epoch_processing_to(spec, state, process_name: str):
+def run_epoch_processing_to(spec, state, process_name: str, enable_slots_processing=True):
     """
     Processes to the next epoch transition, up to, but not including, the sub-transition named ``process_name``
     """
-    slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH)
-
-    # transition state to slot before epoch state transition
-    if state.slot < slot - 1:
-        spec.process_slots(state, slot - 1)
-
-    # start transitioning, do one slot update before the epoch itself.
-    spec.process_slot(state)
+    if enable_slots_processing:
+        run_process_slots_up_to_epoch_boundary(spec, state)
 
     # process components of epoch transition before final-updates
     for name in get_process_calls(spec):
@@ -59,13 +53,51 @@ def run_epoch_processing_to(spec, state, process_name: str):
             getattr(spec, name)(state)
 
 
+def run_process_slots_up_to_epoch_boundary(spec, state):
+    """
+    Processes slots until the next epoch transition
+    """
+    slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH)
+
+    # transition state to slot before epoch state transition
+    if state.slot < slot - 1:
+        spec.process_slots(state, slot - 1)
+
+    # start transitioning, do one slot update before the epoch itself.
+    spec.process_slot(state)
+
+
+def run_epoch_processing_from(spec, state, process_name: str):
+    """
+    Processes to the next epoch transition, from, but not including, the sub-transition named ``process_name``
+    """
+    assert (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0
+
+    processing = False
+    for name in get_process_calls(spec):
+        if name == process_name:
+            processing = True
+            continue
+        # only run when present. Later phases introduce more to the epoch-processing.
+        if processing and hasattr(spec, name):
+            getattr(spec, name)(state)
+
+
 def run_epoch_processing_with(spec, state, process_name: str):
     """
     Processes to the next epoch transition, up to and including the sub-transition named ``process_name``
       - pre-state ('pre'), state before calling ``process_name``
       - post-state ('post'), state after calling ``process_name``
+      - pre-epoch-state ('pre_epoch'), state before epoch transition
+      - post-epoch-state ('post_epoch'), state after epoch transition
+    The state passed by reference will be modified to be the ``process_name``post state.
     """
-    run_epoch_processing_to(spec, state, process_name)
+    run_process_slots_up_to_epoch_boundary(spec, state)
+    yield "pre_epoch", state
+    run_epoch_processing_to(spec, state, process_name, enable_slots_processing=False)
     yield "pre", state
     getattr(spec, process_name)(state)
     yield "post", state
+    continue_state = state.copy()
+    run_epoch_processing_from(spec, continue_state, process_name)
+    yield "post_epoch", continue_state

tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_effective_balance_updates.py

@@ -1,5 +1,9 @@
 from eth2spec.test.context import spec_state_test, with_all_phases
-from eth2spec.test.helpers.epoch_processing import run_epoch_processing_to
+from eth2spec.test.helpers.epoch_processing import (
+    run_epoch_processing_from,
+    run_epoch_processing_to,
+    run_process_slots_up_to_epoch_boundary,
+)
 from eth2spec.test.helpers.forks import is_post_electra
 from eth2spec.test.helpers.withdrawals import (
     set_compounding_withdrawal_credential,
@@ -16,7 +20,11 @@ def run_test_effective_balance_hysteresis(spec, state, with_compounding_credenti
     assert is_post_electra(spec) or not with_compounding_credentials
     # Prepare state up to the final-updates.
     # Then overwrite the balances, we only want to focus to be on the hysteresis based changes.
-    run_epoch_processing_to(spec, state, "process_effective_balance_updates")
+    run_process_slots_up_to_epoch_boundary(spec, state)
+    yield "pre_epoch", state
+    run_epoch_processing_to(
+        spec, state, "process_effective_balance_updates", enable_slots_processing=False
+    )
     # Set some edge cases for balances
     max = (
         spec.MAX_EFFECTIVE_BALANCE_ELECTRA
@@ -92,3 +100,6 @@ def run_test_effective_balance_hysteresis(spec, state, with_compounding_credenti
 
     for i, (_, _, post_eff, name) in enumerate(cases):
         assert state.validators[i].effective_balance == post_eff, name
+
+    run_epoch_processing_from(spec, state, "process_effective_balance_updates")
+    yield "post_epoch", state

tests/formats/epoch_processing/README.md

@@ -23,6 +23,14 @@ An SSZ-snappy encoded `BeaconState`, the state before running the epoch sub-tran
 
 An SSZ-snappy encoded `BeaconState`, the state after applying the epoch sub-transition. No value if the sub-transition processing is aborted.
 
+### `pre_epoch.ssz_snappy`
+
+An SSZ-snappy encoded `BeaconState`, the state before running the epoch transition.
+
+### `post_epoch.ssz_snappy`
+
+An SSZ-snappy encoded `BeaconState`, the state after applying the epoch transition. No value if the transition processing is aborted.
+
 ## Condition
 
 A handler of the `epoch_processing` test-runner should process these cases,
@@ -50,3 +58,15 @@ Sub-transitions:
 - `pending_deposits` (>=Electra)
 
 The resulting state should match the expected `post` state.
+
+## Condition (alternative)
+
+Instead of having a different handler for each sub-transition, a single handler for all cases should load `pre_full` state, call `process_epoch` and then assert that the result state should match `post_full` state.
+
+This has the advantages:
+
+- Less code to maintain for the epoch processing handler.
+- Works with single pass epoch processing.
+- Can detect bugs related to data dependencies between different sub-transitions.
+
+As a disadvantage this condition takes more resources to compute, but just a constant amount per test vector.