Merge branch 'dev' into pr3181

.github/workflows/run-tests.yml

@@ -0,0 +1,133 @@
+name: Run spec tests and linter
+
+defaults:
+  run:
+    shell: zsh {0}
+
+env:
+  TEST_PRESET_TYPE: "minimal"
+  DEFAULT_BRANCH: "dev"
+
+# Run tests on workflow_Dispatch
+on: 
+  push:
+    branches:
+      - dev 
+      - master
+  pull_request:    
+  workflow_dispatch:
+    inputs:
+      test_preset_type:
+        default: minimal
+        description: Type of test to run, either mainnet or minimal
+        type: string
+        required: true
+      commitRef:
+        description: The branch, tag or SHA to checkout and build from
+        default: dev
+        required: true  
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  precleanup:
+      runs-on: self-hosted
+      if: always()
+      steps:
+        - name: 'Cleanup build folder'
+          run: |
+            ls -la ./
+            rm -rf ./* || true
+            rm -rf ./.??* || true
+            ls -la ./
+  setup-env:
+    runs-on: self-hosted
+    needs: precleanup
+    steps:
+      - name: Checkout this repo
+        uses: actions/checkout@v3.2.0
+        with:
+          ref: ${{ github.event.inputs.commitRef || env.DEFAULT_BRANCH }}
+      - uses: actions/cache@v3.2.2
+        id: cache-git
+        with:
+          path: ./*
+          key: ${{ github.sha }}
+
+  table_of_contents:
+   runs-on: self-hosted
+   needs: setup-env
+   steps:
+      - uses: actions/cache@v3.2.2
+        id: restore-build
+        with:
+          path: ./*
+          key: ${{ github.sha }}
+      - name: Check table of contents
+        run: sudo npm install -g doctoc@2 && make check_toc
+
+  codespell:
+   runs-on: self-hosted
+   needs: setup-env
+   steps:
+      - name: Check codespell
+        run: pip install 'codespell<3.0.0,>=2.0.0' --user &&  make codespell
+
+  lint:
+    runs-on: self-hosted
+    needs: setup-env
+    steps:
+      - name: Run linter for pyspec
+        run: make lint
+      - name: Run linter for test generators
+        run: make lint_generators
+
+  pyspec-tests:
+    runs-on: self-hosted
+    needs: [setup-env,lint,codespell,table_of_contents]
+    strategy:
+      matrix:
+        version: ["phase0", "altair", "bellatrix", "capella", "eip4844"]
+    steps:
+      - uses: actions/cache@v3.2.2
+        id: restore-build
+        with:
+          path: ./*
+          key: ${{ github.sha }}
+      - name: set TEST_PRESET_TYPE
+        if: github.event.inputs.test_preset_type != ''        
+        run: |
+          echo "spec_test_preset_type=${{ github.event.inputs.test_preset_type || env.TEST_PRESET_TYPE }}" >> $GITHUB_ENV
+      - name: set TEST_PRESET_TYPE
+        if: ${{ (github.event_name == 'push' && github.ref_name != 'master') || github.event_name == 'pull_request' }}
+        run: |
+          echo "spec_test_preset_type=${{ env.TEST_PRESET_TYPE}}" >> $GITHUB_ENV    
+      - name: set TEST_PRESET_TYPE
+        if: ${{ github.event_name == 'push' && github.ref_name == 'master' }}
+        run: |
+          echo "spec_test_preset_type=mainnet" >> $GITHUB_ENV    
+      - name: set TEST_PRESET_TYPE
+        if: github.event.schedule=='0 0 * * *'
+        run: |
+          echo "spec_test_preset_type=mainnet" >> $GITHUB_ENV
+      - name: Install pyspec requirements
+        run: make install_test
+      - name: test-${{ matrix.version }}
+        run: make citest fork=${{ matrix.version }} TEST_PRESET_TYPE=${{env.spec_test_preset_type}}
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: test-${{ matrix.version }}
+          path: tests/core/pyspec/test-reports
+
+  cleanup:
+      runs-on: self-hosted
+      needs: [setup-env,pyspec-tests,codespell,lint,table_of_contents]
+      if: always()
+      steps:
+        - name: 'Cleanup build folder'
+          run: |
+            ls -la ./
+            rm -rf ./* || true
+            rm -rf ./.??* || true
+            ls -la ./

Makefile

@@ -13,7 +13,7 @@ SOLIDITY_DEPOSIT_CONTRACT_SOURCE = ${SOLIDITY_DEPOSIT_CONTRACT_DIR}/deposit_cont
 SOLIDITY_FILE_NAME = deposit_contract.json
 DEPOSIT_CONTRACT_TESTER_DIR = ${SOLIDITY_DEPOSIT_CONTRACT_DIR}/web3_tester
 CONFIGS_DIR = ./configs
-
+TEST_PRESET_TYPE ?= minimal
 # Collect a list of generator names
 GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/.)))
 # Map this list of generator paths to "gen_{generator name}" entries
@@ -96,30 +96,30 @@ generate_tests: $(GENERATOR_TARGETS)
 
 # "make pyspec" to create the pyspec for all phases.
 pyspec:
-	. venv/bin/activate; python3 setup.py pyspecdev
+	python3 -m venv venv; . venv/bin/activate; python3 setup.py pyspecdev
 
 # installs the packages to run pyspec tests
 install_test:
 	python3 -m venv venv; . venv/bin/activate; python3 -m pip install -e .[lint]; python3 -m pip install -e .[test]
 
-# Testing against `minimal` config by default
+# Testing against `minimal` or `mainnet` config by default
 test: pyspec
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.minimal --cov=eth2spec.altair.minimal --cov=eth2spec.bellatrix.minimal --cov=eth2spec.capella.minimal --cov=eth2spec.eip4844.minimal --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
+	python3 -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.$(TEST_PRESET_TYPE) --cov=eth2spec.altair.$(TEST_PRESET_TYPE) --cov=eth2spec.bellatrix.$(TEST_PRESET_TYPE) --cov=eth2spec.capella.$(TEST_PRESET_TYPE) --cov=eth2spec.eip4844.$(TEST_PRESET_TYPE) --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
 
-# Testing against `minimal` config by default
+# Testing against `minimal` or `mainnet` config by default
 find_test: pyspec
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.minimal --cov=eth2spec.altair.minimal --cov=eth2spec.bellatrix.minimal --cov=eth2spec.capella.minimal --cov=eth2spec.eip4844.minimal --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
+	python3 -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.$(TEST_PRESET_TYPE) --cov=eth2spec.altair.$(TEST_PRESET_TYPE) --cov=eth2spec.bellatrix.$(TEST_PRESET_TYPE) --cov=eth2spec.capella.$(TEST_PRESET_TYPE) --cov=eth2spec.eip4844.$(TEST_PRESET_TYPE) --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
 
 citest: pyspec
 	mkdir -p $(TEST_REPORT_DIR);
 ifdef fork
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -n 4 --bls-type=milagro --fork=$(fork) --junitxml=test-reports/test_results.xml eth2spec
+	python3 -m pytest -n 16 --bls-type=milagro --preset=$(TEST_PRESET_TYPE) --fork=$(fork) --junitxml=test-reports/test_results.xml eth2spec
 else
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -n 4 --bls-type=milagro --junitxml=test-reports/test_results.xml eth2spec
+	python3 -m pytest -n 16 --bls-type=milagro  --preset=$(TEST_PRESET_TYPE) --junitxml=test-reports/test_results.xml eth2spec
 endif
 
 
@@ -135,7 +135,7 @@ check_toc: $(MARKDOWN_FILES:=.toc)
 	rm $*.tmp
 
 codespell:
-	codespell . --skip ./.git -I .codespell-whitelist
+	codespell . --skip "./.git,./venv,$(PY_SPEC_DIR)/.mypy_cache" -I .codespell-whitelist
 
 # TODO: add future protocol upgrade patch packages to linting.
 # NOTE: we use `pylint` just for catching unused arguments in spec code

setup.py

@@ -660,6 +660,7 @@ def get_empty_list_result(fn):  # type: ignore
 process_withdrawals = no_op(process_withdrawals)
 process_bls_to_execution_change = no_op(process_bls_to_execution_change)
 get_expected_withdrawals = get_empty_list_result(get_expected_withdrawals)
+process_historical_summaries_update = no_op(process_historical_summaries_update)
 
 
 #

specs/capella/beacon-chain.md

@@ -18,6 +18,7 @@
     - [`Withdrawal`](#withdrawal)
     - [`BLSToExecutionChange`](#blstoexecutionchange)
     - [`SignedBLSToExecutionChange`](#signedblstoexecutionchange)
+    - [`HistoricalSummary`](#historicalsummary)
   - [Extended Containers](#extended-containers)
     - [`ExecutionPayload`](#executionpayload)
     - [`ExecutionPayloadHeader`](#executionpayloadheader)
@@ -29,6 +30,8 @@
     - [`is_fully_withdrawable_validator`](#is_fully_withdrawable_validator)
     - [`is_partially_withdrawable_validator`](#is_partially_withdrawable_validator)
 - [Beacon chain state transition function](#beacon-chain-state-transition-function)
+  - [Epoch processing](#epoch-processing)
+    - [Historical summaries updates](#historical-summaries-updates)
   - [Block processing](#block-processing)
     - [New `get_expected_withdrawals`](#new-get_expected_withdrawals)
     - [New `process_withdrawals`](#new-process_withdrawals)
@@ -115,6 +118,18 @@ class SignedBLSToExecutionChange(Container):
     signature: BLSSignature
 ```
 
+#### `HistoricalSummary`
+
+```python
+class HistoricalSummary(Container):
+    """
+    `HistoricalSummary` matches the components of the phase0 `HistoricalBatch`
+    making the two hash_tree_root-compatible.
+    """
+    block_summary_root: Root
+    state_summary_root: Root
+```
+
 ### Extended Containers
 
 #### `ExecutionPayload`
@@ -196,7 +211,7 @@ class BeaconState(Container):
     latest_block_header: BeaconBlockHeader
     block_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT]
     state_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT]
-    historical_roots: List[Root, HISTORICAL_ROOTS_LIMIT]
+    historical_roots: List[Root, HISTORICAL_ROOTS_LIMIT]  # Frozen in Capella, replaced by historical_summaries
     # Eth1
     eth1_data: Eth1Data
     eth1_data_votes: List[Eth1Data, EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH]
@@ -226,6 +241,8 @@ class BeaconState(Container):
     # Withdrawals
     next_withdrawal_index: WithdrawalIndex  # [New in Capella]
     next_withdrawal_validator_index: ValidatorIndex  # [New in Capella]
+    # Deep history valid from Capella onwards
+    historical_summaries: List[HistoricalSummary, HISTORICAL_ROOTS_LIMIT]  # [New in Capella]
 ```
 
 ## Helpers
@@ -270,6 +287,40 @@ def is_partially_withdrawable_validator(validator: Validator, balance: Gwei) ->
 
 ## Beacon chain state transition function
 
+### Epoch processing
+
+*Note*: The function `process_historical_summaries_update` replaces `process_historical_roots_update` in Bellatrix.
+
+```python
+def process_epoch(state: BeaconState) -> None:
+    process_justification_and_finalization(state)
+    process_inactivity_updates(state)
+    process_rewards_and_penalties(state)
+    process_registry_updates(state)
+    process_slashings(state)
+    process_eth1_data_reset(state)
+    process_effective_balance_updates(state)
+    process_slashings_reset(state)
+    process_randao_mixes_reset(state)
+    process_historical_summaries_update(state)  # [Modified in Capella]
+    process_participation_flag_updates(state)
+    process_sync_committee_updates(state)
+```
+
+#### Historical summaries updates
+
+```python
+def process_historical_summaries_update(state: BeaconState) -> None:
+    # Set historical block root accumulator.
+    next_epoch = Epoch(get_current_epoch(state) + 1)
+    if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0:
+        historical_summary = HistoricalSummary(
+            block_summary_root=hash_tree_root(state.block_roots),
+            state_summary_root=hash_tree_root(state.state_roots),
+        )
+        state.historical_summaries.append(historical_summary)
+```
+
 ### Block processing
 
 ```python

specs/eip4844/polynomial-commitments.md

@@ -370,8 +370,9 @@ def verify_kzg_proof_impl(polynomial_kzg: KZGCommitment,
 ```python
 def compute_kzg_proof(polynomial: Polynomial, z: BLSFieldElement) -> KZGProof:
     """
-    Compute KZG proof at point `z` with `polynomial` being in evaluation form
-    Do this by computing the quotient polynomial in evaluation form: q(x) = (p(x) - p(z)) / (x - z)
+    Compute KZG proof at point `z` with `polynomial` being in evaluation form.
+    Do this by computing the quotient polynomial in evaluation form: q(x) = (p(x) - p(z)) / (x - z).
+    Public method.
     """
     y = evaluate_polynomial_in_evaluation_form(polynomial, z)
     polynomial_shifted = [BLSFieldElement((int(p) - int(y)) % BLS_MODULUS) for p in polynomial]

tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_historical_batches_update.py

@@ -0,0 +1,27 @@
+from eth2spec.test.context import (
+    CAPELLA,
+    spec_state_test,
+    with_phases,
+)
+from eth2spec.test.helpers.epoch_processing import (
+    run_epoch_processing_with
+)
+
+
+def run_process_historical_summaries_update(spec, state):
+    yield from run_epoch_processing_with(spec, state, 'process_historical_summaries_update')
+
+
+@with_phases([CAPELLA])
+@spec_state_test
+def test_historical_summaries_accumulator(spec, state):
+    # skip ahead to near the end of the historical batch period (excl block before epoch processing)
+    state.slot = spec.SLOTS_PER_HISTORICAL_ROOT - 1
+    pre_historical_summaries = state.historical_summaries.copy()
+
+    yield from run_process_historical_summaries_update(spec, state)
+
+    assert len(state.historical_summaries) == len(pre_historical_summaries) + 1
+    summary = state.historical_summaries[len(state.historical_summaries) - 1]
+    assert summary.block_summary_root == state.block_roots.hash_tree_root()
+    assert summary.state_summary_root == state.state_roots.hash_tree_root()

tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/test_process_historical_batches_update.py

@@ -0,0 +1,23 @@
+from eth2spec.test.context import (
+    spec_state_test,
+    with_eip4844_and_later,
+)
+from eth2spec.test.helpers.epoch_processing import (
+    run_epoch_processing_with
+)
+
+
+def run_process_historical_summaries_update(spec, state):
+    yield from run_epoch_processing_with(spec, state, 'process_historical_summaries_update')
+
+
+@with_eip4844_and_later
+@spec_state_test
+def test_no_op(spec, state):
+    # skip ahead to near the end of the historical batch period (excl block before epoch processing)
+    state.slot = spec.SLOTS_PER_HISTORICAL_ROOT - 1
+    historical_summaries_len = len(state.historical_summaries)
+
+    yield from run_process_historical_summaries_update(spec, state)
+
+    assert len(state.historical_summaries) == historical_summaries_len

tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py

@@ -1,5 +1,8 @@
 
-from eth2spec.test.helpers.forks import is_post_altair
+from eth2spec.test.helpers.forks import (
+    is_post_altair,
+    is_post_capella,
+)
 
 
 def get_process_calls(spec):
@@ -22,7 +25,10 @@ def get_process_calls(spec):
         'process_effective_balance_updates',
         'process_slashings_reset',
         'process_randao_mixes_reset',
-        'process_historical_roots_update',
+        # Capella replaced `process_historical_roots_update` with `process_historical_summaries_update`
+        'process_historical_summaries_update' if is_post_capella(spec) else (
+            'process_historical_roots_update'
+        ),
         # Altair replaced `process_participation_record_updates` with `process_participation_flag_updates`
         'process_participation_flag_updates' if is_post_altair(spec) else (
             'process_participation_record_updates'

tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_historical_roots_update.py

@@ -1,4 +1,8 @@
-from eth2spec.test.context import spec_state_test, with_all_phases
+from eth2spec.test.context import (
+    PHASE0, ALTAIR, BELLATRIX,
+    spec_state_test,
+    with_phases,
+)
 from eth2spec.test.helpers.epoch_processing import (
     run_epoch_processing_with
 )
@@ -8,7 +12,7 @@ def run_process_historical_roots_update(spec, state):
     yield from run_epoch_processing_with(spec, state, 'process_historical_roots_update')
 
 
-@with_all_phases
+@with_phases([PHASE0, ALTAIR, BELLATRIX])
 @spec_state_test
 def test_historical_root_accumulator(spec, state):
     # skip ahead to near the end of the historical roots period (excl block before epoch processing)

tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py

@@ -29,8 +29,8 @@ from eth2spec.test.helpers.sync_committee import (
     compute_committee_indices,
     compute_sync_committee_participant_reward_and_penalty,
 )
-from eth2spec.test.helpers.constants import PHASE0, MINIMAL
-from eth2spec.test.helpers.forks import is_post_altair, is_post_bellatrix
+from eth2spec.test.helpers.constants import PHASE0, EIP4844, MINIMAL
+from eth2spec.test.helpers.forks import is_post_altair, is_post_bellatrix, is_post_capella
 from eth2spec.test.context import (
     spec_test, spec_state_test, dump_skipping_message,
     with_phases, with_all_phases, single_phase,
@@ -1026,7 +1026,10 @@ def test_balance_driven_status_transitions(spec, state):
 @always_bls
 def test_historical_batch(spec, state):
     state.slot += spec.SLOTS_PER_HISTORICAL_ROOT - (state.slot % spec.SLOTS_PER_HISTORICAL_ROOT) - 1
-    pre_historical_roots_len = len(state.historical_roots)
+    pre_historical_roots = state.historical_roots.copy()
+
+    if is_post_capella(spec):
+        pre_historical_summaries = state.historical_summaries.copy()
 
     yield 'pre', state
 
@@ -1038,7 +1041,18 @@ def test_historical_batch(spec, state):
 
     assert state.slot == block.slot
     assert spec.get_current_epoch(state) % (spec.SLOTS_PER_HISTORICAL_ROOT // spec.SLOTS_PER_EPOCH) == 0
-    assert len(state.historical_roots) == pre_historical_roots_len + 1
+
+    # check history update
+    if is_post_capella(spec):
+        # Frozen `historical_roots`
+        assert state.historical_roots == pre_historical_roots
+        if spec.fork == EIP4844:
+            # TODO: no-op for now in EIP4844 testnet
+            assert state.historical_summaries == pre_historical_summaries
+        else:
+            assert len(state.historical_summaries) == len(pre_historical_summaries) + 1
+    else:
+        assert len(state.historical_roots) == len(pre_historical_roots) + 1
 
 
 @with_all_phases

tests/core/pyspec/eth2spec/test/phase0/sanity/test_slots.py

@@ -1,3 +1,9 @@
+from eth2spec.test.helpers.constants import (
+    EIP4844,
+)
+from eth2spec.test.helpers.forks import (
+    is_post_capella,
+)
 from eth2spec.test.helpers.state import get_state_root
 from eth2spec.test.context import (
     spec_state_test,
@@ -61,3 +67,30 @@ def test_over_epoch_boundary(spec, state):
     yield 'slots', int(slots)
     spec.process_slots(state, state.slot + slots)
     yield 'post', state
+
+
+@with_all_phases
+@spec_state_test
+def test_historical_accumulator(spec, state):
+    pre_historical_roots = state.historical_roots.copy()
+
+    if is_post_capella(spec):
+        pre_historical_summaries = state.historical_summaries.copy()
+
+    yield 'pre', state
+    slots = spec.SLOTS_PER_HISTORICAL_ROOT
+    yield 'slots', int(slots)
+    spec.process_slots(state, state.slot + slots)
+    yield 'post', state
+
+    # check history update
+    if is_post_capella(spec):
+        # Frozen `historical_roots`
+        assert state.historical_roots == pre_historical_roots
+        if spec.fork == EIP4844:
+            # TODO: no-op for now in EIP4844 testnet
+            assert state.historical_summaries == pre_historical_summaries
+        else:
+            assert len(state.historical_summaries) == len(pre_historical_summaries) + 1
+    else:
+        assert len(state.historical_roots) == len(pre_historical_roots) + 1

tests/generators/epoch_processing/main.py

@@ -27,13 +27,15 @@ if __name__ == "__main__":
     # so no additional tests required.
     bellatrix_mods = altair_mods
 
-    # No epoch-processing changes in Capella and previous testing repeats with new types,
-    # so no additional tests required.
-    capella_mods = bellatrix_mods
+    _new_capella_mods = {key: 'eth2spec.test.capella.epoch_processing.test_process_' + key for key in [
+        'historical_summaries_update',
+    ]}
+    capella_mods = combine_mods(_new_capella_mods, bellatrix_mods)
 
-    # No epoch-processing changes in EIP4844 and previous testing repeats with new types,
-    # so no additional tests required.
-    eip4844_mods = capella_mods
+    _new_eip4844_mods = {key: 'eth2spec.test.eip4844.epoch_processing.test_process_' + key for key in [
+        'historical_summaries_update',
+    ]}
+    eip4844_mods = combine_mods(_new_eip4844_mods, capella_mods)
 
     # TODO Custody Game testgen is disabled for now
     # custody_game_mods = {**{key: 'eth2spec.test.custody_game.epoch_processing.test_process_' + key for key in [