Validate blobs feature (#12574)

beacon-chain/blockchain/BUILD.bazel

@@ -35,6 +35,7 @@ go_library(
     deps = [
         "//async:go_default_library",
         "//async/event:go_default_library",
+        "//beacon-chain/blockchain/kzg:go_default_library",
         "//beacon-chain/cache:go_default_library",
         "//beacon-chain/cache/depositcache:go_default_library",
         "//beacon-chain/core/altair:go_default_library",

beacon-chain/blockchain/execution_engine.go

@@ -27,6 +27,8 @@ import (
 	"go.opencensus.io/trace"
 )
 
+const blobCommitmentVersionKZG uint8 = 0x01
+
 var defaultLatestValidHash = bytesutil.PadTo([]byte{0xff}, 32)
 
 // notifyForkchoiceUpdateArg is the argument for the forkchoice update notification `notifyForkchoiceUpdate`.
@@ -214,14 +216,10 @@ func (s *Service) notifyNewPayload(ctx context.Context, preStateVersion int,
 
 	var lastValidHash []byte
 	if blk.Version() >= version.Deneb {
-		var kzgs [][]byte
-		kzgs, err = blk.Block().Body().BlobKzgCommitments()
+		var versionedHashes [][32]byte
+		versionedHashes, err = kzgCommitmentsToVersionedHashes(blk.Block().Body())
 		if err != nil {
-			return false, errors.Wrap(invalidBlock{error: err}, "could not get blob kzg commitments")
-		}
-		versionedHashes := make([][32]byte, len(kzgs))
-		for i := range versionedHashes {
-			versionedHashes[i] = kzgToVersionedHash(kzgs[i])
+			return false, errors.Wrap(err, "could not get versioned hashes to feed the engine")
 		}
 		lastValidHash, err = s.cfg.ExecutionEngineCaller.NewPayload(ctx, payload, versionedHashes)
 	} else {
@@ -377,13 +375,16 @@ func (s *Service) removeInvalidBlockAndState(ctx context.Context, blkRoots [][32
 	return nil
 }
 
-const (
-	blobCommitmentVersionKZG uint8 = 0x01
-)
+func kzgCommitmentsToVersionedHashes(body interfaces.ReadOnlyBeaconBlockBody) ([][32]byte, error) {
+	commitments, err := body.BlobKzgCommitments()
+	if err != nil {
+		return nil, errors.Wrap(invalidBlock{error: err}, "could not get blob kzg commitments")
+	}
 
-// kzgToVersionedHash implements kzg_to_versioned_hash from EIP-4844
-func kzgToVersionedHash(kzg []byte) (h [32]byte) {
-	h = sha256.Sum256(kzg)
-	h[0] = blobCommitmentVersionKZG
-	return
+	versionedHashes := make([][32]byte, len(commitments))
+	for i, commitment := range commitments {
+		versionedHashes[i] = sha256.Sum256(commitment)
+		versionedHashes[i][0] = blobCommitmentVersionKZG
+	}
+	return versionedHashes, nil
 }

beacon-chain/blockchain/execution_engine_test.go

@@ -1081,3 +1081,28 @@ func TestService_getPayloadHash(t *testing.T) {
 	require.NoError(t, err)
 	require.DeepEqual(t, [32]byte{'a'}, h)
 }
+
+func TestKZGCommitmentToVersionedHashes(t *testing.T) {
+	kzg1 := make([]byte, 96)
+	kzg1[10] = 'a'
+	kzg2 := make([]byte, 96)
+	kzg2[1] = 'b'
+	commitments := [][]byte{kzg1, kzg2}
+
+	blk := &ethpb.SignedBeaconBlockDeneb{
+		Block: &ethpb.BeaconBlockDeneb{
+			Body: &ethpb.BeaconBlockBodyDeneb{
+				BlobKzgCommitments: commitments,
+			},
+		},
+	}
+	b, err := consensusblocks.NewSignedBeaconBlock(blk)
+	require.NoError(t, err)
+	vhs, err := kzgCommitmentsToVersionedHashes(b.Block().Body())
+	require.NoError(t, err)
+	vh0 := [32]byte{1, 207, 35, 21, 201, 118, 88, 167, 237, 84, 173, 161, 129, 118, 94, 35, 179, 250, 219, 81, 80, 250, 179, 149, 9, 246, 49, 192, 185, 175, 69, 102}
+	vh1 := [32]byte{1, 226, 124, 226, 142, 82, 126, 176, 113, 150, 179, 26, 240, 245, 250, 24, 130, 172, 231, 1, 166, 130, 2, 42, 183, 121, 248, 22, 172, 57, 212, 126}
+	require.Equal(t, 2, len(vhs))
+	require.Equal(t, vhs[0], vh0)
+	require.Equal(t, vhs[1], vh1)
+}

beacon-chain/blockchain/kzg/BUILD.bazel

@@ -0,0 +1,31 @@
+load("@prysm//tools/go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "trusted_setup.go",
+        "validation.go",
+    ],
+    embedsrcs = ["trusted_setup.json"],
+    importpath = "github.com/prysmaticlabs/prysm/v4/beacon-chain/blockchain/kzg",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//proto/prysm/v1alpha1:go_default_library",
+        "@com_github_crate_crypto_go_kzg_4844//:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "trusted_setup_test.go",
+        "validation_test.go",
+    ],
+    embed = [":go_default_library"],
+    deps = [
+        "//proto/prysm/v1alpha1:go_default_library",
+        "//testing/require:go_default_library",
+        "@com_github_crate_crypto_go_kzg_4844//:go_default_library",
+    ],
+)

beacon-chain/blockchain/kzg/trusted_setup.go

@@ -0,0 +1,28 @@
+package kzg
+
+import (
+	_ "embed"
+	"encoding/json"
+
+	GoKZG "github.com/crate-crypto/go-kzg-4844"
+	"github.com/pkg/errors"
+)
+
+var (
+	//go:embed trusted_setup.json
+	embeddedTrustedSetup []byte // 1.2Mb
+	kzgContext           *GoKZG.Context
+)
+
+func Start() error {
+	parsedSetup := GoKZG.JSONTrustedSetup{}
+	err := json.Unmarshal(embeddedTrustedSetup, &parsedSetup)
+	if err != nil {
+		return errors.Wrap(err, "could not parse trusted setup JSON")
+	}
+	kzgContext, err = GoKZG.NewContext4096(&parsedSetup)
+	if err != nil {
+		return errors.Wrap(err, "could not initialize go-kzg context")
+	}
+	return nil
+}

beacon-chain/blockchain/kzg/trusted_setup_test.go

@@ -0,0 +1,12 @@
+package kzg
+
+import (
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/v4/testing/require"
+)
+
+func TestStart(t *testing.T) {
+	require.NoError(t, Start())
+	require.NotNil(t, kzgContext)
+}

beacon-chain/blockchain/kzg/validation.go

@@ -0,0 +1,47 @@
+package kzg
+
+import (
+	"fmt"
+
+	GoKZG "github.com/crate-crypto/go-kzg-4844"
+	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
+)
+
+// IsDataAvailable checks that
+// - all blobs in the block are available
+// - Expected KZG commitments match the number of blobs in the block
+// - That the number of proofs match the number of blobs
+// - That the proofs are verified against the KZG commitments
+func IsDataAvailable(commitments [][]byte, sidecars []*ethpb.BlobSidecar) error {
+	if len(commitments) != len(sidecars) {
+		return fmt.Errorf("could not check data availability, expected %d commitments, obtained %d",
+			len(commitments), len(sidecars))
+	}
+	if len(commitments) == 0 {
+		return nil
+	}
+	blobs := make([]GoKZG.Blob, len(commitments))
+	proofs := make([]GoKZG.KZGProof, len(commitments))
+	cmts := make([]GoKZG.KZGCommitment, len(commitments))
+	for i, sidecar := range sidecars {
+		blobs[i] = bytesToBlob(sidecar.Blob)
+		proofs[i] = bytesToKZGProof(sidecar.KzgProof)
+		cmts[i] = bytesToCommitment(commitments[i])
+	}
+	return kzgContext.VerifyBlobKZGProofBatch(blobs, cmts, proofs)
+}
+
+func bytesToBlob(blob []byte) (ret GoKZG.Blob) {
+	copy(ret[:], blob)
+	return
+}
+
+func bytesToCommitment(commitment []byte) (ret GoKZG.KZGCommitment) {
+	copy(ret[:], commitment)
+	return
+}
+
+func bytesToKZGProof(proof []byte) (ret GoKZG.KZGProof) {
+	copy(ret[:], proof)
+	return
+}

beacon-chain/blockchain/kzg/validation_test.go

@@ -0,0 +1,25 @@
+package kzg
+
+import (
+	"testing"
+
+	GoKZG "github.com/crate-crypto/go-kzg-4844"
+	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/v4/testing/require"
+)
+
+func TestIsDataAvailable(t *testing.T) {
+	sidecars := make([]*ethpb.BlobSidecar, 0)
+	commitments := make([][]byte, 0)
+	require.NoError(t, IsDataAvailable(commitments, sidecars))
+}
+
+func TestBytesToAny(t *testing.T) {
+	bytes := []byte{0x01, 0x02}
+	blob := GoKZG.Blob{0x01, 0x02}
+	commitment := GoKZG.KZGCommitment{0x01, 0x02}
+	proof := GoKZG.KZGProof{0x01, 0x02}
+	require.DeepEqual(t, blob, bytesToBlob(bytes))
+	require.DeepEqual(t, commitment, bytesToCommitment(bytes))
+	require.DeepEqual(t, proof, bytesToKZGProof(bytes))
+}

beacon-chain/blockchain/process_block.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/blockchain/kzg"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed"
 	statefeed "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed/state"
@@ -497,6 +498,33 @@ func (s *Service) runLateBlockTasks() {
 	}
 }
 
+func (s *Service) isDataAvailable(ctx context.Context, root [32]byte, signed interfaces.ReadOnlySignedBeaconBlock) error {
+	if signed.Version() < version.Deneb {
+		return nil
+	}
+	block := signed.Block()
+	if block == nil {
+		return errors.New("invalid nil beacon block")
+	}
+	// We are only required to check within MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS
+	if slots.ToEpoch(block.Slot())+params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest > primitives.Epoch(s.CurrentSlot()) {
+		return nil
+	}
+	body := block.Body()
+	if body == nil {
+		return errors.New("invalid nil beacon block body")
+	}
+	kzgCommitments, err := body.BlobKzgCommitments()
+	if err != nil {
+		return errors.Wrap(err, "could not get KZG commitments")
+	}
+	sidecars, err := s.cfg.BeaconDB.BlobSidecarsByRoot(ctx, root)
+	if err != nil {
+		return errors.Wrap(err, "could not get blob sidecars")
+	}
+	return kzg.IsDataAvailable(kzgCommitments, sidecars)
+}
+
 // lateBlockTasks  is called 4 seconds into the slot and performs tasks
 // related to late blocks. It emits a MissedSlot state feed event.
 // It calls FCU and sets the right attributes if we are proposing next slot

beacon-chain/blockchain/receive_block.go

@@ -90,13 +90,15 @@ func (s *Service) ReceiveBlock(ctx context.Context, block interfaces.ReadOnlySig
 	if err := eg.Wait(); err != nil {
 		return err
 	}
+	if err := s.isDataAvailable(ctx, blockRoot, blockCopy); err != nil {
+		return errors.Wrap(err, "could not validate blob data availability")
+	}
 	// The rest of block processing takes a lock on forkchoice.
 	s.cfg.ForkChoiceStore.Lock()
 	defer s.cfg.ForkChoiceStore.Unlock()
 	if err := s.savePostStateInfo(ctx, blockRoot, blockCopy, postState); err != nil {
 		return errors.Wrap(err, "could not save post state info")
 	}
-
 	if err := s.postBlockProcess(ctx, blockCopy, blockRoot, postState, isValidPayload); err != nil {
 		err := errors.Wrap(err, "could not process block")
 		tracing.AnnotateError(span, err)

beacon-chain/blockchain/service.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/v4/async/event"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/blockchain/kzg"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/cache/depositcache"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed"
@@ -91,6 +92,10 @@ var ErrMissingClockSetter = errors.New("blockchain Service initialized without a
 // NewService instantiates a new block service instance that will
 // be registered into a running beacon node.
 func NewService(ctx context.Context, opts ...Option) (*Service, error) {
+	err := kzg.Start()
+	if err != nil {
+		return nil, errors.Wrap(err, "could not initialize go-kzg context")
+	}
 	ctx, cancel := context.WithCancel(ctx)
 	srv := &Service{
 		ctx:                  ctx,
@@ -108,7 +113,6 @@ func NewService(ctx context.Context, opts ...Option) (*Service, error) {
 	if srv.clockSetter == nil {
 		return nil, ErrMissingClockSetter
 	}
-	var err error
 	srv.wsVerifier, err = NewWeakSubjectivityVerifier(srv.cfg.WeakSubjectivityCheckpt, srv.cfg.BeaconDB)
 	if err != nil {
 		return nil, err

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/aristanetworks/goarista v0.0.0-20200805130819-fd197cf57d96
 	github.com/bazelbuild/rules_go v0.23.2
 	github.com/btcsuite/btcd/btcec/v2 v2.3.2
+	github.com/crate-crypto/go-kzg-4844 v0.3.0
 	github.com/d4l3k/messagediff v1.2.1
 	github.com/dgraph-io/ristretto v0.0.4-0.20210318174700-74754f61e018
 	github.com/dustin/go-humanize v1.0.0
@@ -115,7 +116,6 @@ require (
 	github.com/containerd/cgroups v1.1.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
-	github.com/crate-crypto/go-kzg-4844 v0.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
 	github.com/deckarep/golang-set/v2 v2.1.0 // indirect

testing/spectest/general/deneb/kzg/BUILD.bazel

@@ -0,0 +1,19 @@
+load("@prysm//tools/go:def.bzl", "go_test")
+
+go_test(
+    name = "go_default_test",
+    size = "small",
+    srcs = ["verify_blob_kzg_proof_batch_test.go"],
+    data = [
+        "@consensus_spec_tests_general//:test_data",
+    ],
+    tags = ["spectest"],
+    deps = [
+        "//beacon-chain/blockchain/kzg:go_default_library",
+        "//proto/prysm/v1alpha1:go_default_library",
+        "//testing/require:go_default_library",
+        "//testing/spectest/utils:go_default_library",
+        "//testing/util:go_default_library",
+        "@com_github_ghodss_yaml//:go_default_library",
+    ],
+)

testing/spectest/general/deneb/kzg/verify_blob_kzg_proof_batch_test.go

@@ -0,0 +1,73 @@
+package kzg
+
+import (
+	"encoding/hex"
+	"path"
+	"testing"
+
+	"github.com/ghodss/yaml"
+	kzgPrysm "github.com/prysmaticlabs/prysm/v4/beacon-chain/blockchain/kzg"
+	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/v4/testing/require"
+	"github.com/prysmaticlabs/prysm/v4/testing/spectest/utils"
+	"github.com/prysmaticlabs/prysm/v4/testing/util"
+)
+
+type KZGTestDataInput struct {
+	Blobs       []string `json:"blobs"`
+	Commitments []string `json:"commitments"`
+	Proofs      []string `json:"proofs"`
+}
+
+type KZGTestData struct {
+	Input  KZGTestDataInput `json:"input"`
+	Output bool             `json:"output"`
+}
+
+func TestVerifyBlobKZGProofBatch(t *testing.T) {
+	require.NoError(t, kzgPrysm.Start())
+	testFolders, testFolderPath := utils.TestFolders(t, "general", "deneb", "kzg/verify_blob_kzg_proof_batch/small")
+	if len(testFolders) == 0 {
+		t.Fatalf("No test folders found for %s/%s/%s", "general", "deneb", "kzg/verify_blob_kzg_proof_batch/small")
+	}
+	for _, folder := range testFolders {
+		t.Run(folder.Name(), func(t *testing.T) {
+			file, err := util.BazelFileBytes(path.Join(testFolderPath, folder.Name(), "data.yaml"))
+			require.NoError(t, err)
+			test := &KZGTestData{}
+			require.NoError(t, yaml.Unmarshal(file, test))
+			var sidecars []*ethpb.BlobSidecar
+			blobs := test.Input.Blobs
+			commitments := test.Input.Commitments
+			proofs := test.Input.Proofs
+			if len(proofs) != len(blobs) {
+				require.Equal(t, false, test.Output)
+				return
+			}
+			var kzgs [][]byte
+			// Need separate loops to test length checks in
+			// `IsDataAvailable`
+			for i, blob := range blobs {
+				blobBytes, err := hex.DecodeString(blob[2:])
+				require.NoError(t, err)
+				proofBytes, err := hex.DecodeString(proofs[i][2:])
+				require.NoError(t, err)
+				sidecar := &ethpb.BlobSidecar{
+					Blob:     blobBytes,
+					KzgProof: proofBytes,
+				}
+				sidecars = append(sidecars, sidecar)
+			}
+			for _, commitment := range commitments {
+				commitmentBytes, err := hex.DecodeString(commitment[2:])
+				require.NoError(t, err)
+				kzgs = append(kzgs, commitmentBytes)
+			}
+			if test.Output {
+				require.NoError(t, kzgPrysm.IsDataAvailable(kzgs, sidecars))
+			} else {
+				require.NotNil(t, kzgPrysm.IsDataAvailable(kzgs, sidecars))
+			}
+		})
+	}
+}