github/prysm
1
1
Fork 0
mirror of https://github.com/OffchainLabs/prysm.git synced 2026-01-09 15:37:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

Spec test coverage report hack (#13718)

Browse Source 
* Spec test report hack

* no export

* fix shell complaint

* shell fix?

* shell again?

* chmod +x ./hack/spectest-report.sh

* Review + improvements

* Remove unwanted change

* Add exclusion list

* Fix path + add eip6110 to exclusion

* Fix bazel path nonsense

* Add extra detail about specific test

* Cleanup exclusion list

* Add fail conditions

* Add mkdir

* Shorten filename + mkdir only if new

* Fix names

* Add to exclusion list

* Add report to .gitignore

* Back to stupid names

* Add Bazel flags option

---------

Co-authored-by: Preston Van Loon <pvanloon@offchainlabs.com>
This commit is contained in:
Sammy Rosso
2024-03-25 17:10:32 +01:00
committed by GitHub
parent 6782df917a
commit 14d7416c16
6 changed files with 396 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

296
testing/spectest/exclusions.txt Normal file
View File

@@ -0,0 +1,296 @@
# LightClient
tests/mainnet/altair/light_client/single_merkle_proof
tests/mainnet/bellatrix/light_client/single_merkle_proof
tests/mainnet/capella/light_client/single_merkle_proof
tests/mainnet/deneb/light_client/single_merkle_proof
tests/minimal/altair/light_client/single_merkle_proof
tests/minimal/altair/light_client/sync
tests/minimal/altair/light_client/update_ranking
tests/minimal/bellatrix/light_client/single_merkle_proof
tests/minimal/bellatrix/light_client/sync
tests/minimal/bellatrix/light_client/update_ranking
tests/minimal/capella/light_client/single_merkle_proof
tests/minimal/capella/light_client/sync
tests/minimal/capella/light_client/update_ranking
tests/minimal/deneb/light_client/single_merkle_proof
tests/minimal/deneb/light_client/sync
tests/minimal/deneb/light_client/update_ranking
# SSZ Generic
tests/general/phase0/ssz_generic/basic_vector
tests/general/phase0/ssz_generic/bitlist
tests/general/phase0/ssz_generic/bitvector
tests/general/phase0/ssz_generic/boolean
tests/general/phase0/ssz_generic/containers
tests/general/phase0/ssz_generic/uints
# EIP6110
tests/mainnet/eip6110/epoch_processing/effective_balance_updates
tests/mainnet/eip6110/epoch_processing/eth1_data_reset
tests/mainnet/eip6110/epoch_processing/historical_summaries_update
tests/mainnet/eip6110/epoch_processing/inactivity_updates
tests/mainnet/eip6110/epoch_processing/justification_and_finalization
tests/mainnet/eip6110/epoch_processing/participation_flag_updates
tests/mainnet/eip6110/epoch_processing/randao_mixes_reset
tests/mainnet/eip6110/epoch_processing/registry_updates
tests/mainnet/eip6110/epoch_processing/rewards_and_penalties
tests/mainnet/eip6110/epoch_processing/slashings
tests/mainnet/eip6110/epoch_processing/slashings_reset
tests/mainnet/eip6110/finality/finality
tests/mainnet/eip6110/fork_choice/ex_ante
tests/mainnet/eip6110/fork_choice/get_head
tests/mainnet/eip6110/fork_choice/get_proposer_head
tests/mainnet/eip6110/fork_choice/on_block
tests/mainnet/eip6110/fork_choice/should_override_forkchoice_update
tests/mainnet/eip6110/operations/attestation
tests/mainnet/eip6110/operations/attester_slashing
tests/mainnet/eip6110/operations/block_header
tests/mainnet/eip6110/operations/bls_to_execution_change
tests/mainnet/eip6110/operations/deposit
tests/mainnet/eip6110/operations/deposit_receipt
tests/mainnet/eip6110/operations/execution_payload
tests/mainnet/eip6110/operations/proposer_slashing
tests/mainnet/eip6110/operations/sync_aggregate
tests/mainnet/eip6110/operations/voluntary_exit
tests/mainnet/eip6110/operations/withdrawals
tests/mainnet/eip6110/rewards/basic
tests/mainnet/eip6110/rewards/leak
tests/mainnet/eip6110/rewards/random
tests/mainnet/eip6110/sanity/blocks
tests/mainnet/eip6110/sanity/slots
tests/mainnet/eip6110/ssz_static/AggregateAndProof
tests/mainnet/eip6110/ssz_static/Attestation
tests/mainnet/eip6110/ssz_static/AttestationData
tests/mainnet/eip6110/ssz_static/AttesterSlashing
tests/mainnet/eip6110/ssz_static/BLSToExecutionChange
tests/mainnet/eip6110/ssz_static/BeaconBlock
tests/mainnet/eip6110/ssz_static/BeaconBlockBody
tests/mainnet/eip6110/ssz_static/BeaconBlockHeader
tests/mainnet/eip6110/ssz_static/BeaconState
tests/mainnet/eip6110/ssz_static/BlobIdentifier
tests/mainnet/eip6110/ssz_static/BlobSidecar
tests/mainnet/eip6110/ssz_static/Checkpoint
tests/mainnet/eip6110/ssz_static/ContributionAndProof
tests/mainnet/eip6110/ssz_static/Deposit
tests/mainnet/eip6110/ssz_static/DepositData
tests/mainnet/eip6110/ssz_static/DepositMessage
tests/mainnet/eip6110/ssz_static/DepositReceipt
tests/mainnet/eip6110/ssz_static/Eth1Block
tests/mainnet/eip6110/ssz_static/Eth1Data
tests/mainnet/eip6110/ssz_static/ExecutionPayload
tests/mainnet/eip6110/ssz_static/ExecutionPayloadHeader
tests/mainnet/eip6110/ssz_static/Fork
tests/mainnet/eip6110/ssz_static/ForkData
tests/mainnet/eip6110/ssz_static/HistoricalBatch
tests/mainnet/eip6110/ssz_static/HistoricalSummary
tests/mainnet/eip6110/ssz_static/IndexedAttestation
tests/mainnet/eip6110/ssz_static/LightClientBootstrap
tests/mainnet/eip6110/ssz_static/LightClientFinalityUpdate
tests/mainnet/eip6110/ssz_static/LightClientHeader
tests/mainnet/eip6110/ssz_static/LightClientOptimisticUpdate
tests/mainnet/eip6110/ssz_static/LightClientUpdate
tests/mainnet/eip6110/ssz_static/PendingAttestation
tests/mainnet/eip6110/ssz_static/PowBlock
tests/mainnet/eip6110/ssz_static/ProposerSlashing
tests/mainnet/eip6110/ssz_static/SignedAggregateAndProof
tests/mainnet/eip6110/ssz_static/SignedBLSToExecutionChange
tests/mainnet/eip6110/ssz_static/SignedBeaconBlock
tests/mainnet/eip6110/ssz_static/SignedBeaconBlockHeader
tests/mainnet/eip6110/ssz_static/SignedContributionAndProof
tests/mainnet/eip6110/ssz_static/SignedVoluntaryExit
tests/mainnet/eip6110/ssz_static/SigningData
tests/mainnet/eip6110/ssz_static/SyncAggregate
tests/mainnet/eip6110/ssz_static/SyncAggregatorSelectionData
tests/mainnet/eip6110/ssz_static/SyncCommittee
tests/mainnet/eip6110/ssz_static/SyncCommitteeContribution
tests/mainnet/eip6110/ssz_static/SyncCommitteeMessage
tests/mainnet/eip6110/ssz_static/Validator
tests/mainnet/eip6110/ssz_static/VoluntaryExit
tests/mainnet/eip6110/ssz_static/Withdrawal
tests/mainnet/eip6110/sync/optimistic
tests/mainnet/eip6110/transition/core
tests/minimal/eip6110/epoch_processing/effective_balance_updates
tests/minimal/eip6110/epoch_processing/eth1_data_reset
tests/minimal/eip6110/epoch_processing/historical_summaries_update
tests/minimal/eip6110/epoch_processing/inactivity_updates
tests/minimal/eip6110/epoch_processing/justification_and_finalization
tests/minimal/eip6110/epoch_processing/participation_flag_updates
tests/minimal/eip6110/epoch_processing/randao_mixes_reset
tests/minimal/eip6110/epoch_processing/registry_updates
tests/minimal/eip6110/epoch_processing/rewards_and_penalties
tests/minimal/eip6110/epoch_processing/slashings
tests/minimal/eip6110/epoch_processing/slashings_reset
tests/minimal/eip6110/epoch_processing/sync_committee_updates
tests/minimal/eip6110/finality/finality
tests/minimal/eip6110/fork_choice/ex_ante
tests/minimal/eip6110/fork_choice/get_head
tests/minimal/eip6110/fork_choice/get_proposer_head
tests/minimal/eip6110/fork_choice/on_block
tests/minimal/eip6110/fork_choice/reorg
tests/minimal/eip6110/fork_choice/should_override_forkchoice_update
tests/minimal/eip6110/fork_choice/withholding
tests/minimal/eip6110/genesis/initialization
tests/minimal/eip6110/genesis/validity
tests/minimal/eip6110/operations/attestation
tests/minimal/eip6110/operations/attester_slashing
tests/minimal/eip6110/operations/block_header
tests/minimal/eip6110/operations/bls_to_execution_change
tests/minimal/eip6110/operations/deposit
tests/minimal/eip6110/operations/deposit_receipt
tests/minimal/eip6110/operations/execution_payload
tests/minimal/eip6110/operations/proposer_slashing
tests/minimal/eip6110/operations/sync_aggregate
tests/minimal/eip6110/operations/voluntary_exit
tests/minimal/eip6110/operations/withdrawals
tests/minimal/eip6110/rewards/basic
tests/minimal/eip6110/rewards/leak
tests/minimal/eip6110/rewards/random
tests/minimal/eip6110/sanity/blocks
tests/minimal/eip6110/sanity/slots
tests/minimal/eip6110/ssz_static/AggregateAndProof
tests/minimal/eip6110/ssz_static/Attestation
tests/minimal/eip6110/ssz_static/AttestationData
tests/minimal/eip6110/ssz_static/AttesterSlashing
tests/minimal/eip6110/ssz_static/BLSToExecutionChange
tests/minimal/eip6110/ssz_static/BeaconBlock
tests/minimal/eip6110/ssz_static/BeaconBlockBody
tests/minimal/eip6110/ssz_static/BeaconBlockHeader
tests/minimal/eip6110/ssz_static/BeaconState
tests/minimal/eip6110/ssz_static/BlobIdentifier
tests/minimal/eip6110/ssz_static/BlobSidecar
tests/minimal/eip6110/ssz_static/Checkpoint
tests/minimal/eip6110/ssz_static/ContributionAndProof
tests/minimal/eip6110/ssz_static/Deposit
tests/minimal/eip6110/ssz_static/DepositData
tests/minimal/eip6110/ssz_static/DepositMessage
tests/minimal/eip6110/ssz_static/DepositReceipt
tests/minimal/eip6110/ssz_static/Eth1Block
tests/minimal/eip6110/ssz_static/Eth1Data
tests/minimal/eip6110/ssz_static/ExecutionPayload
tests/minimal/eip6110/ssz_static/ExecutionPayloadHeader
tests/minimal/eip6110/ssz_static/Fork
tests/minimal/eip6110/ssz_static/ForkData
tests/minimal/eip6110/ssz_static/HistoricalBatch
tests/minimal/eip6110/ssz_static/HistoricalSummary
tests/minimal/eip6110/ssz_static/IndexedAttestation
tests/minimal/eip6110/ssz_static/LightClientBootstrap
tests/minimal/eip6110/ssz_static/LightClientFinalityUpdate
tests/minimal/eip6110/ssz_static/LightClientHeader
tests/minimal/eip6110/ssz_static/LightClientOptimisticUpdate
tests/minimal/eip6110/ssz_static/LightClientUpdate
tests/minimal/eip6110/ssz_static/PendingAttestation
tests/minimal/eip6110/ssz_static/PowBlock
tests/minimal/eip6110/ssz_static/ProposerSlashing
tests/minimal/eip6110/ssz_static/SignedAggregateAndProof
tests/minimal/eip6110/ssz_static/SignedBLSToExecutionChange
tests/minimal/eip6110/ssz_static/SignedBeaconBlock
tests/minimal/eip6110/ssz_static/SignedBeaconBlockHeader
tests/minimal/eip6110/ssz_static/SignedContributionAndProof
tests/minimal/eip6110/ssz_static/SignedVoluntaryExit
tests/minimal/eip6110/ssz_static/SigningData
tests/minimal/eip6110/ssz_static/SyncAggregate
tests/minimal/eip6110/ssz_static/SyncAggregatorSelectionData
tests/minimal/eip6110/ssz_static/SyncCommittee
tests/minimal/eip6110/ssz_static/SyncCommitteeContribution
tests/minimal/eip6110/ssz_static/SyncCommitteeMessage
tests/minimal/eip6110/ssz_static/Validator
tests/minimal/eip6110/ssz_static/VoluntaryExit
tests/minimal/eip6110/ssz_static/Withdrawal
tests/minimal/eip6110/sync/optimistic
tests/minimal/eip6110/transition/core
# Whisk
tests/mainnet/whisk/ssz_static/AggregateAndProof
tests/mainnet/whisk/ssz_static/Attestation
tests/mainnet/whisk/ssz_static/AttestationData
tests/mainnet/whisk/ssz_static/AttesterSlashing
tests/mainnet/whisk/ssz_static/BLSToExecutionChange
tests/mainnet/whisk/ssz_static/BeaconBlock
tests/mainnet/whisk/ssz_static/BeaconBlockBody
tests/mainnet/whisk/ssz_static/BeaconBlockHeader
tests/mainnet/whisk/ssz_static/BeaconState
tests/mainnet/whisk/ssz_static/Checkpoint
tests/mainnet/whisk/ssz_static/ContributionAndProof
tests/mainnet/whisk/ssz_static/Deposit
tests/mainnet/whisk/ssz_static/DepositData
tests/mainnet/whisk/ssz_static/DepositMessage
tests/mainnet/whisk/ssz_static/Eth1Block
tests/mainnet/whisk/ssz_static/Eth1Data
tests/mainnet/whisk/ssz_static/ExecutionPayload
tests/mainnet/whisk/ssz_static/ExecutionPayloadHeader
tests/mainnet/whisk/ssz_static/Fork
tests/mainnet/whisk/ssz_static/ForkData
tests/mainnet/whisk/ssz_static/HistoricalBatch
tests/mainnet/whisk/ssz_static/HistoricalSummary
tests/mainnet/whisk/ssz_static/IndexedAttestation
tests/mainnet/whisk/ssz_static/LightClientBootstrap
tests/mainnet/whisk/ssz_static/LightClientFinalityUpdate
tests/mainnet/whisk/ssz_static/LightClientHeader
tests/mainnet/whisk/ssz_static/LightClientOptimisticUpdate
tests/mainnet/whisk/ssz_static/LightClientUpdate
tests/mainnet/whisk/ssz_static/PendingAttestation
tests/mainnet/whisk/ssz_static/PowBlock
tests/mainnet/whisk/ssz_static/ProposerSlashing
tests/mainnet/whisk/ssz_static/SignedAggregateAndProof
tests/mainnet/whisk/ssz_static/SignedBLSToExecutionChange
tests/mainnet/whisk/ssz_static/SignedBeaconBlock
tests/mainnet/whisk/ssz_static/SignedBeaconBlockHeader
tests/mainnet/whisk/ssz_static/SignedContributionAndProof
tests/mainnet/whisk/ssz_static/SignedVoluntaryExit
tests/mainnet/whisk/ssz_static/SigningData
tests/mainnet/whisk/ssz_static/SyncAggregate
tests/mainnet/whisk/ssz_static/SyncAggregatorSelectionData
tests/mainnet/whisk/ssz_static/SyncCommittee
tests/mainnet/whisk/ssz_static/SyncCommitteeContribution
tests/mainnet/whisk/ssz_static/SyncCommitteeMessage
tests/mainnet/whisk/ssz_static/Validator
tests/mainnet/whisk/ssz_static/VoluntaryExit
tests/mainnet/whisk/ssz_static/WhiskTracker
tests/mainnet/whisk/ssz_static/Withdrawal
tests/minimal/whisk/ssz_static/AggregateAndProof
tests/minimal/whisk/ssz_static/Attestation
tests/minimal/whisk/ssz_static/AttestationData
tests/minimal/whisk/ssz_static/AttesterSlashing
tests/minimal/whisk/ssz_static/BLSToExecutionChange
tests/minimal/whisk/ssz_static/BeaconBlock
tests/minimal/whisk/ssz_static/BeaconBlockBody
tests/minimal/whisk/ssz_static/BeaconBlockHeader
tests/minimal/whisk/ssz_static/BeaconState
tests/minimal/whisk/ssz_static/Checkpoint
tests/minimal/whisk/ssz_static/ContributionAndProof
tests/minimal/whisk/ssz_static/Deposit
tests/minimal/whisk/ssz_static/DepositData
tests/minimal/whisk/ssz_static/DepositMessage
tests/minimal/whisk/ssz_static/Eth1Block
tests/minimal/whisk/ssz_static/Eth1Data
tests/minimal/whisk/ssz_static/ExecutionPayload
tests/minimal/whisk/ssz_static/ExecutionPayloadHeader
tests/minimal/whisk/ssz_static/Fork
tests/minimal/whisk/ssz_static/ForkData
tests/minimal/whisk/ssz_static/HistoricalBatch
tests/minimal/whisk/ssz_static/HistoricalSummary
tests/minimal/whisk/ssz_static/IndexedAttestation
tests/minimal/whisk/ssz_static/LightClientBootstrap
tests/minimal/whisk/ssz_static/LightClientFinalityUpdate
tests/minimal/whisk/ssz_static/LightClientHeader
tests/minimal/whisk/ssz_static/LightClientOptimisticUpdate
tests/minimal/whisk/ssz_static/LightClientUpdate
tests/minimal/whisk/ssz_static/PendingAttestation
tests/minimal/whisk/ssz_static/PowBlock
tests/minimal/whisk/ssz_static/ProposerSlashing
tests/minimal/whisk/ssz_static/SignedAggregateAndProof
tests/minimal/whisk/ssz_static/SignedBLSToExecutionChange
tests/minimal/whisk/ssz_static/SignedBeaconBlock
tests/minimal/whisk/ssz_static/SignedBeaconBlockHeader
tests/minimal/whisk/ssz_static/SignedContributionAndProof
tests/minimal/whisk/ssz_static/SignedVoluntaryExit
tests/minimal/whisk/ssz_static/SigningData
tests/minimal/whisk/ssz_static/SyncAggregate
tests/minimal/whisk/ssz_static/SyncAggregatorSelectionData
tests/minimal/whisk/ssz_static/SyncCommittee
tests/minimal/whisk/ssz_static/SyncCommitteeContribution
tests/minimal/whisk/ssz_static/SyncCommitteeMessage
tests/minimal/whisk/ssz_static/Validator
tests/minimal/whisk/ssz_static/VoluntaryExit
tests/minimal/whisk/ssz_static/WhiskTracker
tests/minimal/whisk/ssz_static/Withdrawal

1
testing/spectest/utils/BUILD.bazel
View File

@@ -11,6 +11,7 @@ go_library(
visibility = ["//testing/spectest:__subpackages__"],
deps = [
"//config/params:go_default_library",
"//io/file:go_default_library",
"//testing/require:go_default_library",
"@com_github_ghodss_yaml//:go_default_library",
"@com_github_json_iterator_go//:go_default_library",

18
testing/spectest/utils/utils.go
View File

@@ -1,6 +1,7 @@
package utils
import (
"fmt"
"os"
"path"
"testing"
@@ -8,6 +9,7 @@ import (
"github.com/bazelbuild/rules_go/go/tools/bazel"
"github.com/ghodss/yaml"
jsoniter "github.com/json-iterator/go"
"github.com/prysmaticlabs/prysm/v5/io/file"
"github.com/prysmaticlabs/prysm/v5/testing/require"
)
@@ -42,6 +44,20 @@ func TestFolders(t testing.TB, config, forkOrPhase, folderPath string) ([]os.Dir
if len(testFolders) == 0 {
t.Fatalf("No test folders found at %s", testsFolderPath)
}
err = saveSpecTest(testsFolderPath)
require.NoError(t, err)
return testFolders, testsFolderPath
}
func saveSpecTest(testFolder string) error {
baseDir := os.Getenv("SPEC_TEST_REPORT_OUTPUT_DIR")
if baseDir == "" {
return nil // Do nothing if spec test report not requested.
}
fullPath := path.Join(baseDir, fmt.Sprintf("%x_tests.txt", testFolder))
err := file.WriteFile(fullPath, []byte(testFolder))
if err != nil {
return err
}
return nil
}