PeerDAS: Implement core. (#15192)

* Fulu: Implement params.

* KZG tests: Re-implement `getRandBlob` to avoid tests cyclical dependencies.

Not ideal, but any better idea welcome.

* Fulu testing util: Implement `GenerateCellsAndProofs`.

* Create `RODataColumn`.

* Implement `MerkleProofKZGCommitments`.

* Export `leavesFromCommitments`.

* Implement peerDAS core.

* Add changelog.

* Update beacon-chain/core/peerdas/das_core.go

Co-authored-by: terence <terence@prysmaticlabs.com>

* Fix Terence's comment: Use `IsNil`.

* Fix Terence's comment: Avoid useless `filteredIndices`.

* Fix Terence's comment: Simplify odd/even cases.

* Fix Terence's comment: Use `IsNil`.

* Spectests: Add Fulu networking

* Fix Terence's comment: `CustodyGroups`: Stick to the spec by returning a (sorted) slice.

* Fix Terence's comment: `CustodyGroups`: Handle correctly the `maxUint256` case.

* Update beacon-chain/core/peerdas/das_core.go

Co-authored-by: terence <terence@prysmaticlabs.com>

* Fix Terence's comment: `ComputeColumnsForCustodyGroup`: Add test if `custodyGroup == numberOfCustodyGroup`

* `CustodyGroups`: Test if `custodyGroupCount > numberOfCustodyGroup`.

* `CustodyGroups`: Add a shortcut if all custody groups are needed.

* `ComputeCystodyGroupForColumn`: Move from `p2p_interface.go` to `das_core.go`.

* Fix Terence's comment: Fix `ComputeCustodyGroupForColumn`.

* Fix Terence's comment: Remove `constructCellsAndProofs` function.

* Fix Terence's comment: `ValidatorsCustodyRequirement`: Use effective balance instead of balance.

* `MerkleProofKZGCommitments`: Add tests

* Remove peer sampling.

* `DataColumnSidecars`: Add missing tests.

* Fix Jame's comment.

* Fix James' comment.

* Fix James' comment.

* Fix James' coment.

* Fix James' comment.

---------

Co-authored-by: terence <terence@prysmaticlabs.com>

consensus-types/blocks/BUILD.bazel

@@ -12,6 +12,7 @@ go_library(
         "proto.go",
         "roblob.go",
         "roblock.go",
+        "rodatacolumn.go",
         "setters.go",
         "types.go",
     ],
@@ -51,6 +52,7 @@ go_test(
         "proto_test.go",
         "roblob_test.go",
         "roblock_test.go",
+        "rodatacolumn_test.go",
     ],
     embed = [":go_default_library"],
     deps = [

consensus-types/blocks/kzg.go

@@ -80,8 +80,38 @@ func MerkleProofKZGCommitment(body interfaces.ReadOnlyBeaconBlockBody, index int
 	return proof, nil
 }
 
-// leavesFromCommitments hashes each commitment to construct a slice of roots
-func leavesFromCommitments(commitments [][]byte) [][]byte {
+// MerkleProofKZGCommitments constructs a Merkle proof of inclusion of the KZG
+// commitments into the Beacon Block with the given `body`
+func MerkleProofKZGCommitments(body interfaces.ReadOnlyBeaconBlockBody) ([][]byte, error) {
+	bodyVersion := body.Version()
+	if bodyVersion < version.Deneb {
+		return nil, errUnsupportedBeaconBlockBody
+	}
+
+	membersRoots, err := topLevelRoots(body)
+	if err != nil {
+		return nil, errors.Wrap(err, "top level roots")
+	}
+
+	sparse, err := trie.GenerateTrieFromItems(membersRoots, logBodyLength)
+	if err != nil {
+		return nil, errors.Wrap(err, "generate trie from items")
+	}
+
+	proof, err := sparse.MerkleProof(kzgPosition)
+	if err != nil {
+		return nil, errors.Wrap(err, "merkle proof")
+	}
+
+	// Remove the last element as it is a mix in with the number of
+	// elements in the trie.
+	proof = proof[:len(proof)-1]
+
+	return proof, nil
+}
+
+// LeavesFromCommitments hashes each commitment to construct a slice of roots
+func LeavesFromCommitments(commitments [][]byte) [][]byte {
 	leaves := make([][]byte, len(commitments))
 	for i, kzg := range commitments {
 		chunk := makeChunk(kzg)
@@ -105,7 +135,7 @@ func bodyProof(commitments [][]byte, index int) ([][]byte, error) {
 	if index < 0 || index >= len(commitments) {
 		return nil, errInvalidIndex
 	}
-	leaves := leavesFromCommitments(commitments)
+	leaves := LeavesFromCommitments(commitments)
 	sparse, err := trie.GenerateTrieFromItems(leaves, field_params.LogMaxBlobCommitments)
 	if err != nil {
 		return nil, err

consensus-types/blocks/kzg_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	fieldparams "github.com/OffchainLabs/prysm/v6/config/fieldparams"
+	"github.com/OffchainLabs/prysm/v6/consensus-types/interfaces"
 	"github.com/OffchainLabs/prysm/v6/container/trie"
 	enginev1 "github.com/OffchainLabs/prysm/v6/proto/engine/v1"
 	ethpb "github.com/OffchainLabs/prysm/v6/proto/prysm/v1alpha1"
@@ -32,7 +33,7 @@ func Test_MerkleProofKZGCommitment_Altair(t *testing.T) {
 	require.ErrorIs(t, errUnsupportedBeaconBlockBody, err)
 }
 
-func Test_MerkleProofKZGCommitment(t *testing.T) {
+func buildTestKzgsAndBody(t *testing.T) ([][]byte, interfaces.ReadOnlyBeaconBlockBody) {
 	kzgs := make([][]byte, 3)
 	kzgs[0] = make([]byte, 48)
 	_, err := rand.Read(kzgs[0])
@@ -69,8 +70,15 @@ func Test_MerkleProofKZGCommitment(t *testing.T) {
 
 	body, err := NewBeaconBlockBody(pbBody)
 	require.NoError(t, err)
-	index := 1
-	_, err = MerkleProofKZGCommitment(body, 10)
+
+	return kzgs, body
+}
+
+func Test_MerkleProofKZGCommitment(t *testing.T) {
+	const index = 1
+
+	kzgs, body := buildTestKzgsAndBody(t)
+	_, err := MerkleProofKZGCommitment(body, 10)
 	require.ErrorIs(t, errInvalidIndex, err)
 	proof, err := MerkleProofKZGCommitment(body, index)
 	require.NoError(t, err)
@@ -104,6 +112,40 @@ func Test_MerkleProofKZGCommitment(t *testing.T) {
 	require.Equal(t, true, trie.VerifyMerkleProof(root[:], chunk[0][:], uint64(index+KZGOffset), proof))
 }
 
+func TestMerkleProofKZGCommitments(t *testing.T) {
+	t.Run("invalid version", func(t *testing.T) {
+		pbBody := &ethpb.BeaconBlockBodyAltair{}
+
+		body, err := NewBeaconBlockBody(pbBody)
+		require.NoError(t, err)
+		_, err = MerkleProofKZGCommitments(body)
+		require.ErrorIs(t, errUnsupportedBeaconBlockBody, err)
+	})
+
+	t.Run("nominal", func(t *testing.T) {
+		kzgs, body := buildTestKzgsAndBody(t)
+
+		proof, err := MerkleProofKZGCommitments(body)
+		require.NoError(t, err)
+
+		commitmentsRoot, err := getBlobKzgCommitmentsRoot(kzgs)
+		require.NoError(t, err)
+
+		bodyMembersRoots, err := topLevelRoots(body)
+		require.NoError(t, err, "Failed to get top level roots")
+
+		bodySparse, err := trie.GenerateTrieFromItems(bodyMembersRoots, logBodyLength)
+		require.NoError(t, err, "Failed to generate trie from member roots")
+
+		require.Equal(t, bodyLength, bodySparse.NumOfItems())
+
+		root, err := body.HashTreeRoot()
+		require.NoError(t, err)
+
+		require.Equal(t, true, trie.VerifyMerkleProof(root[:], commitmentsRoot[:], kzgPosition, proof))
+	})
+}
+
 // This test explains the calculation of the KZG commitment root's Merkle index
 // in the Body's Merkle tree based on the index of the KZG commitment list in the Body.
 func Test_KZGRootIndex(t *testing.T) {
@@ -139,7 +181,7 @@ func ceilLog2(x uint32) (uint32, error) {
 }
 
 func getBlobKzgCommitmentsRoot(commitments [][]byte) ([32]byte, error) {
-	commitmentsLeaves := leavesFromCommitments(commitments)
+	commitmentsLeaves := LeavesFromCommitments(commitments)
 	commitmentsSparse, err := trie.GenerateTrieFromItems(
 		commitmentsLeaves,
 		fieldparams.LogMaxBlobCommitments,

consensus-types/blocks/rodatacolumn.go

@@ -0,0 +1,68 @@
+package blocks
+
+import (
+	fieldparams "github.com/OffchainLabs/prysm/v6/config/fieldparams"
+	ethpb "github.com/OffchainLabs/prysm/v6/proto/prysm/v1alpha1"
+)
+
+// RODataColumn represents a read-only data column sidecar with its block root.
+type RODataColumn struct {
+	*ethpb.DataColumnSidecar
+	root [fieldparams.RootLength]byte
+}
+
+func roDataColumnNilCheck(dc *ethpb.DataColumnSidecar) error {
+	// Check if the data column is nil.
+	if dc == nil {
+		return errNilDataColumn
+	}
+
+	// Check if the data column header is nil.
+	if dc.SignedBlockHeader == nil || dc.SignedBlockHeader.Header == nil {
+		return errNilBlockHeader
+	}
+
+	// Check if the data column signature is nil.
+	if len(dc.SignedBlockHeader.Signature) == 0 {
+		return errMissingBlockSignature
+	}
+
+	return nil
+}
+
+// NewRODataColumn creates a new RODataColumn by computing the HashTreeRoot of the header.
+func NewRODataColumn(dc *ethpb.DataColumnSidecar) (RODataColumn, error) {
+	if err := roDataColumnNilCheck(dc); err != nil {
+		return RODataColumn{}, err
+	}
+	root, err := dc.SignedBlockHeader.Header.HashTreeRoot()
+	if err != nil {
+		return RODataColumn{}, err
+	}
+	return RODataColumn{DataColumnSidecar: dc, root: root}, nil
+}
+
+// NewRODataColumnWithRoot creates a new RODataColumn with a given root.
+func NewRODataColumnWithRoot(dc *ethpb.DataColumnSidecar, root [fieldparams.RootLength]byte) (RODataColumn, error) {
+	// Check if the data column is nil.
+	if err := roDataColumnNilCheck(dc); err != nil {
+		return RODataColumn{}, err
+	}
+
+	return RODataColumn{DataColumnSidecar: dc, root: root}, nil
+}
+
+// BlockRoot returns the root of the block.
+func (dc *RODataColumn) BlockRoot() [fieldparams.RootLength]byte {
+	return dc.root
+}
+
+// VerifiedRODataColumn represents an RODataColumn that has undergone full verification (eg block sig, inclusion proof, commitment check).
+type VerifiedRODataColumn struct {
+	RODataColumn
+}
+
+// NewVerifiedRODataColumn "upgrades" an RODataColumn to a VerifiedRODataColumn. This method should only be used by the verification package.
+func NewVerifiedRODataColumn(roDataColumn RODataColumn) VerifiedRODataColumn {
+	return VerifiedRODataColumn{RODataColumn: roDataColumn}
+}

consensus-types/blocks/rodatacolumn_test.go

@@ -0,0 +1,125 @@
+package blocks
+
+import (
+	"testing"
+
+	fieldparams "github.com/OffchainLabs/prysm/v6/config/fieldparams"
+	"github.com/OffchainLabs/prysm/v6/encoding/bytesutil"
+	ethpb "github.com/OffchainLabs/prysm/v6/proto/prysm/v1alpha1"
+	"github.com/OffchainLabs/prysm/v6/testing/assert"
+	"github.com/OffchainLabs/prysm/v6/testing/require"
+)
+
+func TestNewRODataColumnWithAndWithoutRoot(t *testing.T) {
+	cases := []struct {
+		name   string
+		dcFunc func(t *testing.T) *ethpb.DataColumnSidecar
+		err    error
+		root   []byte
+	}{
+		{
+			name: "nil signed data column",
+			dcFunc: func(t *testing.T) *ethpb.DataColumnSidecar {
+				return nil
+			},
+			err:  errNilDataColumn,
+			root: bytesutil.PadTo([]byte("sup"), fieldparams.RootLength),
+		},
+		{
+			name: "nil signed block header",
+			dcFunc: func(t *testing.T) *ethpb.DataColumnSidecar {
+				return &ethpb.DataColumnSidecar{
+					SignedBlockHeader: nil,
+				}
+			},
+			err:  errNilBlockHeader,
+			root: bytesutil.PadTo([]byte("sup"), fieldparams.RootLength),
+		},
+		{
+			name: "nil inner header",
+			dcFunc: func(t *testing.T) *ethpb.DataColumnSidecar {
+				return &ethpb.DataColumnSidecar{
+					SignedBlockHeader: &ethpb.SignedBeaconBlockHeader{
+						Header: nil,
+					},
+				}
+			},
+			err:  errNilBlockHeader,
+			root: bytesutil.PadTo([]byte("sup"), fieldparams.RootLength),
+		},
+		{
+			name: "nil signature",
+			dcFunc: func(t *testing.T) *ethpb.DataColumnSidecar {
+				return &ethpb.DataColumnSidecar{
+					SignedBlockHeader: &ethpb.SignedBeaconBlockHeader{
+						Header: &ethpb.BeaconBlockHeader{
+							ParentRoot: make([]byte, fieldparams.RootLength),
+							StateRoot:  make([]byte, fieldparams.RootLength),
+							BodyRoot:   make([]byte, fieldparams.RootLength),
+						},
+						Signature: nil,
+					},
+				}
+			},
+			err:  errMissingBlockSignature,
+			root: bytesutil.PadTo([]byte("sup"), fieldparams.RootLength),
+		},
+		{
+			name: "nominal",
+			dcFunc: func(t *testing.T) *ethpb.DataColumnSidecar {
+				return &ethpb.DataColumnSidecar{
+					SignedBlockHeader: &ethpb.SignedBeaconBlockHeader{
+						Header: &ethpb.BeaconBlockHeader{
+							ParentRoot: make([]byte, fieldparams.RootLength),
+							StateRoot:  make([]byte, fieldparams.RootLength),
+							BodyRoot:   make([]byte, fieldparams.RootLength),
+						},
+						Signature: make([]byte, fieldparams.BLSSignatureLength),
+					},
+				}
+			},
+			root: bytesutil.PadTo([]byte("sup"), fieldparams.RootLength),
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name+" NewRODataColumn", func(t *testing.T) {
+			dataColumnSidecar := c.dcFunc(t)
+			roDataColumnSidecar, err := NewRODataColumn(dataColumnSidecar)
+
+			if c.err != nil {
+				require.ErrorIs(t, err, c.err)
+				return
+			}
+
+			require.NoError(t, err)
+			hr, err := dataColumnSidecar.SignedBlockHeader.Header.HashTreeRoot()
+			require.NoError(t, err)
+			require.Equal(t, hr, roDataColumnSidecar.BlockRoot())
+		})
+
+		if len(c.root) == 0 {
+			continue
+		}
+
+		t.Run(c.name+" NewRODataColumnWithRoot", func(t *testing.T) {
+			b := c.dcFunc(t)
+
+			// We want the same validation when specifying a root.
+			bl, err := NewRODataColumnWithRoot(b, bytesutil.ToBytes32(c.root))
+			if c.err != nil {
+				require.ErrorIs(t, err, c.err)
+				return
+			}
+
+			assert.Equal(t, bytesutil.ToBytes32(c.root), bl.BlockRoot())
+		})
+	}
+}
+
+func TestDataColumn_BlockRoot(t *testing.T) {
+	root := [fieldparams.RootLength]byte{1}
+	dataColumn := &RODataColumn{
+		root: root,
+	}
+	assert.Equal(t, root, dataColumn.BlockRoot())
+}

consensus-types/blocks/types.go

@@ -29,6 +29,7 @@ var (
 	// ErrUnsupportedVersion for beacon block methods.
 	ErrUnsupportedVersion    = errors.New("unsupported beacon block version")
 	errNilBlob               = errors.New("received nil blob sidecar")
+	errNilDataColumn         = errors.New("received nil data column sidecar")
 	errNilBlock              = errors.New("received nil beacon block")
 	errNilBlockBody          = errors.New("received nil beacon block body")
 	errIncorrectBlockVersion = errors.New(incorrectBlockVersion)