Blob verification spectest (#13707)

* use real blob verifier in forkchoice spectest

* wip

* Use real blob sidecar for test

* Set file db correctly

* correctly handle blob cases where valid=false

* work-around spectest's weird Fork in genesis state

* gaz

* revert T-money's log level change

* rm whitespace

* unskip minimal test

* Preston's feedback

---------

Co-authored-by: Kasey Kirkham <kasey@users.noreply.github.com>
Co-authored-by: terence tsao <terence@prysmaticlabs.com>

testing/spectest/mainnet/deneb/forkchoice/forkchoice_test.go

@@ -8,6 +8,5 @@ import (
 )
 
 func TestMainnet_Deneb_Forkchoice(t *testing.T) {
-	t.Skip("This will fail until we re-integrate proof verification")
 	forkchoice.Run(t, "mainnet", version.Deneb)
 }

testing/spectest/minimal/deneb/forkchoice/forkchoice_test.go

@@ -8,6 +8,5 @@ import (
 )
 
 func TestMinimal_Deneb_Forkchoice(t *testing.T) {
-	t.Skip("blocked by go-kzg-4844 minimal trusted setup")
 	forkchoice.Run(t, "minimal", version.Deneb)
 }

testing/spectest/shared/common/forkchoice/BUILD.bazel

@@ -22,6 +22,7 @@ go_library(
         "//beacon-chain/db/filesystem:go_default_library",
         "//beacon-chain/db/testing:go_default_library",
         "//beacon-chain/execution:go_default_library",
+        "//beacon-chain/forkchoice:go_default_library",
         "//beacon-chain/forkchoice/doubly-linked-tree:go_default_library",
         "//beacon-chain/operations/attestations:go_default_library",
         "//beacon-chain/startup:go_default_library",

testing/spectest/shared/common/forkchoice/builder.go

@@ -10,7 +10,9 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/prysmaticlabs/prysm/v5/beacon-chain/blockchain"
 	"github.com/prysmaticlabs/prysm/v5/beacon-chain/execution"
+	"github.com/prysmaticlabs/prysm/v5/beacon-chain/startup"
 	"github.com/prysmaticlabs/prysm/v5/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/v5/beacon-chain/verification"
 	"github.com/prysmaticlabs/prysm/v5/config/params"
 	"github.com/prysmaticlabs/prysm/v5/consensus-types/interfaces"
 	"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
@@ -23,16 +25,27 @@ type Builder struct {
 	service  *blockchain.Service
 	lastTick int64
 	execMock *engineMock
+	vwait    *verification.InitializerWaiter
 }
 
 func NewBuilder(t testing.TB, initialState state.BeaconState, initialBlock interfaces.ReadOnlySignedBeaconBlock) *Builder {
 	execMock := &engineMock{
 		powBlocks: make(map[[32]byte]*ethpb.PowBlock),
 	}
-	service := startChainService(t, initialState, initialBlock, execMock)
+	cw := startup.NewClockSynchronizer()
+	service, sg, fc := startChainService(t, initialState, initialBlock, execMock, cw)
+	// blob spectests use a weird Fork in the genesis beacon state that has different previous and current versions.
+	// This trips up the lite fork lookup code in the blob verifier that figures out the fork
+	// based on the slot of the block. So just for spectests we override that behavior and get the fork from the state
+	// which matches the behavior of block verification.
+	getFork := func(targetEpoch primitives.Epoch) (*ethpb.Fork, error) {
+		return initialState.Fork(), nil
+	}
+	bvw := verification.NewInitializerWaiter(cw, fc, sg, verification.WithForkLookup(getFork))
 	return &Builder{
 		service:  service,
 		execMock: execMock,
+		vwait:    bvw,
 	}
 }
 
@@ -88,7 +101,7 @@ func (bb *Builder) block(t testing.TB, b interfaces.ReadOnlySignedBeaconBlock) [
 // InvalidBlock receives the invalid block and notifies forkchoice.
 func (bb *Builder) InvalidBlock(t testing.TB, b interfaces.ReadOnlySignedBeaconBlock) {
 	r := bb.block(t, b)
-	ctx, cancel := context.WithTimeout(context.TODO(), time.Duration(params.BeaconConfig().SecondsPerSlot)*time.Second)
+	ctx, cancel := context.WithTimeout(context.TODO(), time.Second)
 	defer cancel()
 	require.Equal(t, true, bb.service.ReceiveBlock(ctx, b, r, nil) != nil)
 }
@@ -96,7 +109,7 @@ func (bb *Builder) InvalidBlock(t testing.TB, b interfaces.ReadOnlySignedBeaconB
 // ValidBlock receives the valid block and notifies forkchoice.
 func (bb *Builder) ValidBlock(t testing.TB, b interfaces.ReadOnlySignedBeaconBlock) {
 	r := bb.block(t, b)
-	ctx, cancel := context.WithTimeout(context.TODO(), time.Duration(params.BeaconConfig().SecondsPerSlot)*time.Second)
+	ctx, cancel := context.WithTimeout(context.TODO(), time.Second)
 	defer cancel()
 	require.NoError(t, bb.service.ReceiveBlock(ctx, b, r, nil))
 }

testing/spectest/shared/common/forkchoice/runner.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"strings"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -115,7 +116,7 @@ func runTest(t *testing.T, config string, fork int, basePath string) {
 							t.Fatalf("unknown fork version: %v", fork)
 						}
 					}
-					runBlobStep(t, step.Blobs, beaconBlock, fork, folder, testsFolderPath, step.Proofs, builder)
+					runBlobStep(t, step, beaconBlock, fork, folder, testsFolderPath, builder)
 					if beaconBlock != nil {
 						if step.Valid != nil && !*step.Valid {
 							builder.InvalidBlock(t, beaconBlock)
@@ -281,14 +282,15 @@ func unmarshalSignedDenebBlock(t *testing.T, raw []byte) interfaces.SignedBeacon
 }
 
 func runBlobStep(t *testing.T,
-	blobs *string,
+	step Step,
 	beaconBlock interfaces.ReadOnlySignedBeaconBlock,
 	fork int,
 	folder os.DirEntry,
 	testsFolderPath string,
-	proofs []*string,
 	builder *Builder,
 ) {
+	blobs := step.Blobs
+	proofs := step.Proofs
 	if blobs != nil && *blobs != "null" {
 		require.NotNil(t, beaconBlock)
 		require.Equal(t, true, fork >= version.Deneb)
@@ -305,44 +307,94 @@ func runBlobStep(t *testing.T,
 		require.NoError(t, err)
 		sh, err := beaconBlock.Header()
 		require.NoError(t, err)
-		for index := uint64(0); index*fieldparams.BlobLength < uint64(len(blobsSSZ)); index++ {
+		requireVerifyExpected := errAssertionForStep(step, verification.ErrBlobInvalid)
+		for index := 0; index*fieldparams.BlobLength < len(blobsSSZ); index++ {
 			var proof []byte
-			if index < uint64(len(proofs)) {
+			if index < len(proofs) {
 				proofPTR := proofs[index]
 				require.NotNil(t, proofPTR)
 				proof, err = hexutil.Decode(*proofPTR)
 				require.NoError(t, err)
 			}
 
-			var kzg []byte
-			if uint64(len(kzgs)) < index {
-				kzg = kzgs[index]
-			}
-			if len(kzg) == 0 {
-				kzg = make([]byte, 48)
-			}
 			blob := [fieldparams.BlobLength]byte{}
 			copy(blob[:], blobsSSZ[index*fieldparams.BlobLength:])
-			fakeProof := make([][]byte, fieldparams.KzgCommitmentInclusionProofDepth)
-			for i := range fakeProof {
-				fakeProof[i] = make([]byte, fieldparams.RootLength)
-			}
 			if len(proof) == 0 {
 				proof = make([]byte, 48)
 			}
+
+			inclusionProof, err := blocks.MerkleProofKZGCommitment(block.Body(), index)
+			require.NoError(t, err)
 			pb := &ethpb.BlobSidecar{
-				Index:                    index,
+				Index:                    uint64(index),
 				Blob:                     blob[:],
-				KzgCommitment:            kzg,
+				KzgCommitment:            kzgs[index],
 				KzgProof:                 proof,
 				SignedBlockHeader:        sh,
-				CommitmentInclusionProof: fakeProof,
+				CommitmentInclusionProof: inclusionProof,
 			}
 			ro, err := blocks.NewROBlobWithRoot(pb, root)
 			require.NoError(t, err)
-			vsc, err := verification.BlobSidecarNoop(ro)
+			ini, err := builder.vwait.WaitForInitializer(context.Background())
 			require.NoError(t, err)
-			require.NoError(t, builder.service.ReceiveBlob(context.Background(), vsc))
+			bv := ini.NewBlobVerifier(ro, verification.SpectestSidecarRequirements)
+			ctx := context.Background()
+			if err := bv.BlobIndexInBounds(); err != nil {
+				t.Logf("BlobIndexInBounds error: %s", err.Error())
+			}
+			if err := bv.NotFromFutureSlot(); err != nil {
+				t.Logf("NotFromFutureSlot error: %s", err.Error())
+			}
+			if err := bv.SlotAboveFinalized(); err != nil {
+				t.Logf("SlotAboveFinalized error: %s", err.Error())
+			}
+			if err := bv.SidecarInclusionProven(); err != nil {
+				t.Logf("SidecarInclusionProven error: %s", err.Error())
+			}
+			if err := bv.SidecarKzgProofVerified(); err != nil {
+				t.Logf("SidecarKzgProofVerified error: %s", err.Error())
+			}
+			if err := bv.ValidProposerSignature(ctx); err != nil {
+				t.Logf("ValidProposerSignature error: %s", err.Error())
+			}
+			if err := bv.SidecarParentSlotLower(); err != nil {
+				t.Logf("SidecarParentSlotLower error: %s", err.Error())
+			}
+			if err := bv.SidecarDescendsFromFinalized(); err != nil {
+				t.Logf("SidecarDescendsFromFinalized error: %s", err.Error())
+			}
+			if err := bv.SidecarProposerExpected(ctx); err != nil {
+				t.Logf("SidecarProposerExpected error: %s", err.Error())
+			}
+
+			vsc, err := bv.VerifiedROBlob()
+			requireVerifyExpected(t, err)
+
+			if err == nil {
+				require.NoError(t, builder.service.ReceiveBlob(context.Background(), vsc))
+			}
+		}
+	}
+}
+
+func errAssertionForStep(step Step, expect error) func(t *testing.T, err error) {
+	if !*step.Valid {
+		return func(t *testing.T, err error) {
+			require.ErrorIs(t, err, expect)
+		}
+	}
+	return func(t *testing.T, err error) {
+		if err != nil {
+			require.ErrorIs(t, err, verification.ErrBlobInvalid)
+			me, ok := err.(verification.VerificationMultiError)
+			require.Equal(t, true, ok)
+			fails := me.Failures()
+			// we haven't performed any verification, so all the results should be this type
+			fmsg := make([]string, 0, len(fails))
+			for k, v := range fails {
+				fmsg = append(fmsg, fmt.Sprintf("%s - %s", v.Error(), k.String()))
+			}
+			t.Fatal(strings.Join(fmsg, ";"))
 		}
 	}
 }

testing/spectest/shared/common/forkchoice/service.go

@@ -16,6 +16,7 @@ import (
 	coreTime "github.com/prysmaticlabs/prysm/v5/beacon-chain/core/time"
 	"github.com/prysmaticlabs/prysm/v5/beacon-chain/db/filesystem"
 	testDB "github.com/prysmaticlabs/prysm/v5/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/v5/beacon-chain/forkchoice"
 	doublylinkedtree "github.com/prysmaticlabs/prysm/v5/beacon-chain/forkchoice/doubly-linked-tree"
 	"github.com/prysmaticlabs/prysm/v5/beacon-chain/operations/attestations"
 	"github.com/prysmaticlabs/prysm/v5/beacon-chain/startup"
@@ -34,7 +35,8 @@ func startChainService(t testing.TB,
 	st state.BeaconState,
 	block interfaces.ReadOnlySignedBeaconBlock,
 	engineMock *engineMock,
-) *blockchain.Service {
+	clockSync *startup.ClockSynchronizer,
+) (*blockchain.Service, *stategen.State, forkchoice.ForkChoicer) {
 	ctx := context.Background()
 	db := testDB.SetupDB(t)
 	require.NoError(t, db.SaveBlock(ctx, block))
@@ -58,28 +60,30 @@ func startChainService(t testing.TB,
 	require.NoError(t, err)
 
 	fc := doublylinkedtree.New()
+	sg := stategen.New(db, fc)
 	opts := append([]blockchain.Option{},
 		blockchain.WithExecutionEngineCaller(engineMock),
 		blockchain.WithFinalizedStateAtStartUp(st),
 		blockchain.WithDatabase(db),
 		blockchain.WithAttestationService(attPool),
 		blockchain.WithForkChoiceStore(fc),
-		blockchain.WithStateGen(stategen.New(db, fc)),
+		blockchain.WithStateGen(sg),
 		blockchain.WithStateNotifier(&mock.MockStateNotifier{}),
 		blockchain.WithAttestationPool(attestations.NewPool()),
 		blockchain.WithDepositCache(depositCache),
 		blockchain.WithTrackedValidatorsCache(cache.NewTrackedValidatorsCache()),
 		blockchain.WithPayloadIDCache(cache.NewPayloadIDCache()),
-		blockchain.WithClockSynchronizer(startup.NewClockSynchronizer()),
+		blockchain.WithClockSynchronizer(clockSync),
 		blockchain.WithBlobStorage(filesystem.NewEphemeralBlobStorage(t)),
 		blockchain.WithSyncChecker(mock.MockChecker{}),
+		blockchain.WithBlobStorage(filesystem.NewEphemeralBlobStorage(t)),
 	)
 	service, err := blockchain.NewService(context.Background(), opts...)
 	require.NoError(t, err)
 	// force start kzg context here until Deneb fork epoch is decided
 	require.NoError(t, kzg.Start())
 	require.NoError(t, service.StartFromSavedState(st))
-	return service
+	return service, sg, fc
 }
 
 type engineMock struct {