diff --git a/WORKSPACE b/WORKSPACE index 376a6e1ae6..53bb1251a6 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -253,16 +253,16 @@ filegroup( url = "https://github.com/ethereum/EIPs/archive/5480440fe51742ed23342b68cf106cefd427e39d.tar.gz", ) -consensus_spec_version = "v1.6.0-alpha.1" +consensus_spec_version = "v1.6.0-alpha.4" load("@prysm//tools:download_spectests.bzl", "consensus_spec_tests") consensus_spec_tests( name = "consensus_spec_tests", flavors = { - "general": "sha256-o4t9p3R+fQHF4KOykGmwlG3zDw5wUdVWprkzId8aIsk=", - "minimal": "sha256-sU7ToI8t3MR8x0vVjC8ERmAHZDWpEmnAC9FWIpHi5x4=", - "mainnet": "sha256-YKS4wngg0LgI9Upp4MYJ77aG+8+e/G4YeqEIlp06LZw=", + "general": "sha256-MaN4zu3o0vWZypUHS5r4D8WzJF4wANoadM8qm6iyDs4=", + "minimal": "sha256-aZGNPp/bBvJgq3Wf6vyR0H6G3DOkbSuggEmOL4jEmtg=", + "mainnet": "sha256-C7jjosvpzUgw3GPajlsWBV02ZbkZ5Uv4ikmOqfDGajI=", }, version = consensus_spec_version, ) @@ -278,7 +278,7 @@ filegroup( visibility = ["//visibility:public"], ) """, - integrity = "sha256-Nv4TEuEJPQIM4E6T9J0FOITsmappmXZjGtlhe1HEXnU=", + integrity = "sha256-qreawRS77l8CebiNww8z727qUItw7KlHY1Xqj7IrPdk=", strip_prefix = "consensus-specs-" + consensus_spec_version[1:], url = "https://github.com/ethereum/consensus-specs/archive/refs/tags/%s.tar.gz" % consensus_spec_version, ) diff --git a/beacon-chain/verification/data_column.go b/beacon-chain/verification/data_column.go index d4d8e6638b..71d2232a52 100644 --- a/beacon-chain/verification/data_column.go +++ b/beacon-chain/verification/data_column.go @@ -47,6 +47,10 @@ var ( RequireSidecarKzgProofVerified, } + // SpectestDataColumnSidecarRequirements is used by the forkchoice spectests when verifying data columns used in the on_block tests. + SpectestDataColumnSidecarRequirements = requirementList(GossipDataColumnSidecarRequirements).excluding( + RequireSidecarParentSeen, RequireSidecarParentValid) + errColumnsInvalid = errors.New("data columns failed verification") errBadTopicLength = errors.New("topic length is invalid") errBadTopic = errors.New("topic is not of the one expected") diff --git a/changelog/ttsao_update-consensus-spec-v160-alpha4.md b/changelog/ttsao_update-consensus-spec-v160-alpha4.md new file mode 100644 index 0000000000..6f2e3bf968 --- /dev/null +++ b/changelog/ttsao_update-consensus-spec-v160-alpha4.md @@ -0,0 +1,3 @@ +### Changed + +- Update consensus spec to v1.6.0-alpha.4 and implement data column support for forkchoice spectests diff --git a/config/params/loader_test.go b/config/params/loader_test.go index 11adb22b61..19e03f19a6 100644 --- a/config/params/loader_test.go +++ b/config/params/loader_test.go @@ -37,11 +37,12 @@ var placeholderFields = []string{ "EIP7805_FORK_EPOCH", "EIP7805_FORK_VERSION", "EPOCHS_PER_SHUFFLING_PHASE", + "INCLUSION_LIST_SUBMISSION_DEADLINE", "MAX_BYTES_PER_INCLUSION_LIST", "MAX_REQUEST_BLOB_SIDECARS_FULU", "MAX_REQUEST_INCLUSION_LIST", "MAX_REQUEST_PAYLOADS", // Compile time constant on BeaconBlockBody.ExecutionRequests - "PROPOSER_INCLUSION_LIST_CUT_OFF", + "PROPOSER_INCLUSION_LIST_CUTOFF", "PROPOSER_SCORE_BOOST_EIP7732", "PROPOSER_SELECTION_GAP", "TARGET_NUMBER_OF_PEERS", diff --git a/testing/spectest/shared/common/forkchoice/BUILD.bazel b/testing/spectest/shared/common/forkchoice/BUILD.bazel index 65b26a907d..ddc4162632 100644 --- a/testing/spectest/shared/common/forkchoice/BUILD.bazel +++ b/testing/spectest/shared/common/forkchoice/BUILD.bazel @@ -17,6 +17,7 @@ go_library( "//beacon-chain/blockchain/testing:go_default_library", "//beacon-chain/cache:go_default_library", "//beacon-chain/cache/depositsnapshot:go_default_library", + "//beacon-chain/core/helpers:go_default_library", "//beacon-chain/core/time:go_default_library", "//beacon-chain/core/transition:go_default_library", "//beacon-chain/db/filesystem:go_default_library", diff --git a/testing/spectest/shared/common/forkchoice/runner.go b/testing/spectest/shared/common/forkchoice/runner.go index 530057a05b..ef0f1c7110 100644 --- a/testing/spectest/shared/common/forkchoice/runner.go +++ b/testing/spectest/shared/common/forkchoice/runner.go @@ -10,6 +10,7 @@ import ( "strings" "testing" + "github.com/OffchainLabs/prysm/v6/beacon-chain/core/helpers" "github.com/OffchainLabs/prysm/v6/beacon-chain/core/transition" "github.com/OffchainLabs/prysm/v6/beacon-chain/state" state_native "github.com/OffchainLabs/prysm/v6/beacon-chain/state/state-native" @@ -62,6 +63,7 @@ func runTest(t *testing.T, config string, fork int, basePath string) { // nolint for _, folder := range testFolders { t.Run(folder.Name(), func(t *testing.T) { + helpers.ClearCache() preStepsFile, err := util.BazelFileBytes(testsFolderPath, folder.Name(), "steps.yaml") require.NoError(t, err) var steps []Step @@ -148,6 +150,9 @@ func runTest(t *testing.T, config string, fork int, basePath string) { // nolint } } runBlobStep(t, step, beaconBlock, fork, folder, testsFolderPath, builder) + if len(step.DataColumns) > 0 { + runDataColumnStep(t, step, beaconBlock, fork, folder, testsFolderPath, builder) + } if beaconBlock != nil { if step.Valid != nil && !*step.Valid { builder.InvalidBlock(t, beaconBlock) @@ -293,10 +298,154 @@ func runBlobStep(t *testing.T, } } +func runDataColumnStep(t *testing.T, + step Step, + beaconBlock interfaces.ReadOnlySignedBeaconBlock, + fork int, + folder os.DirEntry, + testsFolderPath string, + builder *Builder, +) { + columnFiles := step.DataColumns + + require.NotNil(t, beaconBlock) + require.Equal(t, true, fork >= version.Fulu) + + block := beaconBlock.Block() + root, err := block.HashTreeRoot() + require.NoError(t, err) + kzgs, err := block.Body().BlobKzgCommitments() + require.NoError(t, err) + sh, err := beaconBlock.Header() + require.NoError(t, err) + // Use the same error that the verification system returns for data columns + errDataColumnsInvalid := errors.New("data columns failed verification") + requireVerifyExpected := errAssertionForStep(step, errDataColumnsInvalid) + + var allColumns []blocks.RODataColumn + + for columnIndex, columnFile := range columnFiles { + if columnFile == nil || *columnFile == "null" { + continue + } + + dataColumnFile, err := util.BazelFileBytes(testsFolderPath, folder.Name(), fmt.Sprint(*columnFile, ".ssz_snappy")) + require.NoError(t, err) + dataColumnSSZ, err := snappy.Decode(nil /* dst */, dataColumnFile) + require.NoError(t, err) + + var pb *ethpb.DataColumnSidecar + + if step.Valid != nil && !*step.Valid { + pb = ðpb.DataColumnSidecar{} + if err := pb.UnmarshalSSZ(dataColumnSSZ); err != nil { + pb = ðpb.DataColumnSidecar{ + Index: uint64(columnIndex), + Column: [][]byte{}, + KzgCommitments: kzgs, + KzgProofs: make([][]byte, 0), + SignedBlockHeader: sh, + } + } + } else { + numCells := len(kzgs) + column := make([][]byte, numCells) + for cellIndex := 0; cellIndex < numCells; cellIndex++ { + cell := make([]byte, 2048) + cellStart := cellIndex * 2048 + cellEnd := cellStart + 2048 + if cellEnd <= len(dataColumnSSZ) { + copy(cell, dataColumnSSZ[cellStart:cellEnd]) + } + column[cellIndex] = cell + } + + inclusionProof, err := blocks.MerkleProofKZGCommitments(block.Body()) + require.NoError(t, err) + + pb = ðpb.DataColumnSidecar{ + Index: uint64(columnIndex), + Column: column, + KzgCommitments: kzgs, + SignedBlockHeader: sh, + KzgCommitmentsInclusionProof: inclusionProof, + } + } + + ro, err := blocks.NewRODataColumnWithRoot(pb, root) + require.NoError(t, err) + allColumns = append(allColumns, ro) + } + + if len(allColumns) > 0 { + ini, err := builder.vwait.WaitForInitializer(context.Background()) + require.NoError(t, err) + // Use different verification requirements based on whether this is a valid or invalid test case + var forkchoiceReqs []verification.Requirement + if step.Valid != nil && !*step.Valid { + forkchoiceReqs = verification.SpectestDataColumnSidecarRequirements + } else { + forkchoiceReqs = []verification.Requirement{ + verification.RequireNotFromFutureSlot, + verification.RequireSlotAboveFinalized, + verification.RequireValidProposerSignature, + verification.RequireSidecarParentSlotLower, + verification.RequireSidecarDescendsFromFinalized, + verification.RequireSidecarInclusionProven, + verification.RequireSidecarProposerExpected, + } + } + dv := ini.NewDataColumnsVerifier(allColumns, forkchoiceReqs) + ctx := t.Context() + + if step.Valid != nil && !*step.Valid { + if err := dv.ValidFields(); err != nil { + t.Logf("ValidFields error: %s", err.Error()) + } + } + + if err := dv.NotFromFutureSlot(); err != nil { + t.Logf("NotFromFutureSlot error: %s", err.Error()) + } + if err := dv.SlotAboveFinalized(); err != nil { + t.Logf("SlotAboveFinalized error: %s", err.Error()) + } + if err := dv.SidecarInclusionProven(); err != nil { + t.Logf("SidecarInclusionProven error: %s", err.Error()) + } + if err := dv.ValidProposerSignature(ctx); err != nil { + t.Logf("ValidProposerSignature error: %s", err.Error()) + } + if err := dv.SidecarParentSlotLower(); err != nil { + t.Logf("SidecarParentSlotLower error: %s", err.Error()) + } + if err := dv.SidecarDescendsFromFinalized(); err != nil { + t.Logf("SidecarDescendsFromFinalized error: %s", err.Error()) + } + if err := dv.SidecarProposerExpected(ctx); err != nil { + t.Logf("SidecarProposerExpected error: %s", err.Error()) + } + + vdc, err := dv.VerifiedRODataColumns() + requireVerifyExpected(t, err) + + if err == nil { + for _, column := range vdc { + require.NoError(t, builder.service.ReceiveDataColumn(column)) + } + } + } +} + func errAssertionForStep(step Step, expect error) func(t *testing.T, err error) { if !*step.Valid { return func(t *testing.T, err error) { - require.ErrorIs(t, err, expect) + if expect.Error() == "data columns failed verification" { + require.NotNil(t, err) + require.Equal(t, true, strings.Contains(err.Error(), expect.Error())) + } else { + require.ErrorIs(t, err, expect) + } } } return func(t *testing.T, err error) { diff --git a/testing/spectest/shared/common/forkchoice/type.go b/testing/spectest/shared/common/forkchoice/type.go index 7e0ead5fd9..6ba52689ca 100644 --- a/testing/spectest/shared/common/forkchoice/type.go +++ b/testing/spectest/shared/common/forkchoice/type.go @@ -11,6 +11,7 @@ type Step struct { PayloadStatus *MockEngineResp `json:"payload_status"` PowBlock *string `json:"pow_block"` Check *Check `json:"checks"` + DataColumns []*string `json:"columns"` } type Check struct {