From b75e3df3dea01bc99d7f48397cb1050ab801ceaa Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Wed, 21 Aug 2024 19:00:11 +0200 Subject: [PATCH] test: add gh workflow to run hive tests with experimental flag (#10355) --- .github/assets/hive/build_simulators.sh | 2 +- .../hive/expected_failures_experimental.yaml | 72 ++++++++++++ .github/workflows/hive.yml | 104 +++++++++++------- 3 files changed, 136 insertions(+), 42 deletions(-) create mode 100644 .github/assets/hive/expected_failures_experimental.yaml diff --git a/.github/assets/hive/build_simulators.sh b/.github/assets/hive/build_simulators.sh index 45583d549a..b33f4db4ee 100755 --- a/.github/assets/hive/build_simulators.sh +++ b/.github/assets/hive/build_simulators.sh @@ -35,4 +35,4 @@ wait # Make sure we don't rebuild images on the CI jobs git apply ../.github/assets/hive/no_sim_build.diff go build . -mv ./hive ../hive_assets/ \ No newline at end of file +mv ./hive ../hive_assets/ diff --git a/.github/assets/hive/expected_failures_experimental.yaml b/.github/assets/hive/expected_failures_experimental.yaml new file mode 100644 index 0000000000..26aa540681 --- /dev/null +++ b/.github/assets/hive/expected_failures_experimental.yaml @@ -0,0 +1,72 @@ +# https://github.com/paradigmxyz/reth/issues/7015 +# https://github.com/paradigmxyz/reth/issues/6332 +rpc-compat: + - debug_getRawBlock/get-invalid-number (reth) + - debug_getRawHeader/get-invalid-number (reth) + - debug_getRawReceipts/get-invalid-number (reth) + - debug_getRawTransaction/get-invalid-hash (reth) + + - eth_call/call-callenv (reth) + - eth_createAccessList/create-al-contract-eip1559 (reth) + - eth_createAccessList/create-al-contract (reth) + - eth_feeHistory/fee-history (reth) + - eth_getStorageAt/get-storage-invalid-key-too-large (reth) + - eth_getStorageAt/get-storage-invalid-key (reth) + - eth_getTransactionReceipt/get-access-list (reth) + - eth_getTransactionReceipt/get-blob-tx (reth) + - eth_getTransactionReceipt/get-dynamic-fee (reth) + +# https://github.com/paradigmxyz/reth/issues/8732 +engine-withdrawals: + - Withdrawals Fork On Genesis (Paris) (reth) + - Withdrawals Fork on Block 1 (Paris) (reth) + - Withdrawals Fork on Block 2 (Paris) (reth) + - Withdrawals Fork on Block 3 (Paris) (reth) + - Withdraw to a single account (Paris) (reth) + - Withdraw to two accounts (Paris) (reth) + - Withdraw many accounts (Paris) (reth) + - Withdraw zero amount (Paris) (reth) + - Empty Withdrawals (Paris) (reth) + - Corrupted Block Hash Payload (INVALID) (Paris) (reth) + - Withdrawals Fork on Block 1 - 8 Block Re-Org NewPayload (Paris) (reth) + - Withdrawals Fork on Block 1 - 8 Block Re-Org, Sync (Paris) (reth) + - Withdrawals Fork on Block 8 - 10 Block Re-Org NewPayload (Paris) (reth) + - Withdrawals Fork on Block 8 - 10 Block Re-Org Sync (Paris) (reth) + - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org (Paris) (reth) + - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org Sync (Paris) (reth) + - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org (Paris) (reth) + - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org Sync (Paris) (reth) + +# https://github.com/paradigmxyz/reth/issues/8305 +# https://github.com/paradigmxyz/reth/issues/6217 +engine-api: + - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Nonce, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Paris) (reth) + - Re-org to Previously Validated Sidechain Payload (Paris) (reth) + - Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs=False, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs=True, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs=False, Invalid P10 (Paris) (reth) + - Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs=True, Invalid P10 (Paris) (reth) + +# https://github.com/paradigmxyz/reth/issues/8305 +# https://github.com/paradigmxyz/reth/issues/6217 +# https://github.com/paradigmxyz/reth/issues/8306 +# https://github.com/paradigmxyz/reth/issues/7144 +engine-cancun: + - Blob Transaction Ordering, Multiple Clients (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth) + - Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth) + - Invalid NewPayload, BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + - Invalid NewPayload, Blob Count on BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + - Invalid NewPayload, ExcessBlobGas, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + - Re-org to Previously Validated Sidechain Payload (Cancun) (reth) + - Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs=False, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs=True, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs=False, Invalid P10 (Cancun) (reth) + - Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs=True, Invalid P10 (Cancun) (reth) + +# https://github.com/paradigmxyz/reth/issues/8579 +sync: + - sync reth -> reth diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml index c524e9eb7f..d7f8919155 100644 --- a/.github/workflows/hive.yml +++ b/.github/workflows/hive.yml @@ -23,7 +23,7 @@ jobs: group: Reth steps: - uses: actions/checkout@v4 - - run: mkdir artifacts + - run: mkdir artifacts - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 with: @@ -82,10 +82,13 @@ jobs: strategy: fail-fast: false matrix: + engine: [regular, experimental] # ethereum/rpc to be deprecated: # https://github.com/ethereum/hive/pull/1117 - sim: [smoke/genesis, smoke/network, ethereum/sync] - include: + scenario: + - sim: smoke/genesis + - sim: smoke/network + - sim: ethereum/sync - sim: devp2p limit: discv4 - sim: devp2p @@ -139,46 +142,46 @@ jobs: - eth_getTransactionReceipt - eth_sendRawTransaction - eth_syncing - # debug_ rpc methods - - sim: ethereum/rpc-compat - include: [debug_] + # debug_ rpc methods + - debug_ # Pyspec cancun jobs - - sim: pyspec - include: [cancun/eip4844] - - sim: pyspec - include: [cancun/eip4788] - - sim: pyspec - include: [cancun/eip6780] - - sim: pyspec - include: [cancun/eip5656] - - sim: pyspec - include: [cancun/eip1153] - - sim: pyspec - include: [cancun/eip7516] + # TODO: uncomment when https://github.com/ethereum/hive/issues/1147 is fixed + #- sim: pyspec + # include: [cancun/eip4844] + #- sim: pyspec + # include: [cancun/eip4788] + #- sim: pyspec + # include: [cancun/eip6780] + #- sim: pyspec + # include: [cancun/eip5656] + #- sim: pyspec + # include: [cancun/eip1153] + #- sim: pyspec + # include: [cancun/eip7516] # Pyspec shanghai jobs - - sim: pyspec - include: [shanghai/eip3651] - - sim: pyspec - include: [shanghai/eip3855] - - sim: pyspec - include: [shanghai/eip3860] - - sim: pyspec - include: [shanghai/eip4895] + #- sim: pyspec + # include: [shanghai/eip3651] + #- sim: pyspec + # include: [shanghai/eip3855] + #- sim: pyspec + # include: [shanghai/eip3860] + #- sim: pyspec + # include: [shanghai/eip4895] # Pyspec merge and earlier jobs - - sim: pyspec - include: [merge/] - - sim: pyspec - include: [berlin/] - - sim: pyspec - include: [istanbul/] - - sim: pyspec - include: [homestead/] - - sim: pyspec - include: [frontier/] - needs: + #- sim: pyspec + # include: [merge/] + #- sim: pyspec + # include: [berlin/] + #- sim: pyspec + # include: [istanbul/] + #- sim: pyspec + # include: [homestead/] + #- sim: pyspec + # include: [frontier/] + needs: - prepare-reth - prepare-hive - name: run + name: run ${{ matrix.engine }} - ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }} runs-on: group: Reth permissions: @@ -215,12 +218,31 @@ jobs: ref: master path: hivetests - - name: Run ${{ matrix.sim }} simulator - run: .github/assets/hive/run_simulator.sh "${{ matrix.sim }}$" "${{matrix.limit}}/${{join(matrix.include, '|')}}" + - name: Modify client for experimental engine + if: matrix.engine == 'experimental' + run: | + sed -ie 's/RUST_LOG=info $reth node $FLAGS/RUST_LOG=info $reth node --engine.experimental $FLAGS/' hivetests/clients/reth/reth.sh + + - name: Run simulator + run: | + LIMIT="${{ matrix.scenario.limit }}" + TESTS="${{ join(matrix.scenario.include, '|') }}" + if [ -n "$LIMIT" ] && [ -n "$TESTS" ]; then + FILTER="$LIMIT/$TESTS" + elif [ -n "$LIMIT" ]; then + FILTER="$LIMIT" + elif [ -n "$TESTS" ]; then + FILTER="/$TESTS" + else + FILTER="/" + fi + echo "filter: $FILTER" + .github/assets/hive/run_simulator.sh "${{ matrix.scenario.sim }}" "$FILTER" - name: Parse hive output run: | - find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion .github/assets/hive/expected_failures.yaml + FAILURE_FILE="${{ matrix.engine == 'experimental' && '.github/assets/hive/expected_failures_experimental.yaml' || '.github/assets/hive/expected_failures.yaml' }}" + find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion $FAILURE_FILE - name: Print simulator output if: ${{ failure() }}