remove unused var

Co-authored-by: Bastin <43618253+Inspector-Butters@users.noreply.github.com>

beacon-chain/p2p/types/BUILD.bazel

@@ -25,6 +25,7 @@ go_library(
         "//consensus-types/primitives:go_default_library",
         "//consensus-types/wrapper:go_default_library",
         "//encoding/bytesutil:go_default_library",
+        "//encoding/ssz:go_default_library",
         "//proto/engine/v1:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//proto/prysm/v1alpha1/metadata:go_default_library",
@@ -45,10 +46,10 @@ go_test(
         "//config/params:go_default_library",
         "//consensus-types/primitives:go_default_library",
         "//encoding/bytesutil:go_default_library",
+        "//encoding/ssz:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//runtime/version:go_default_library",
         "//testing/assert:go_default_library",
         "//testing/require:go_default_library",
-        "@com_github_prysmaticlabs_fastssz//:go_default_library",
     ],
 )

beacon-chain/p2p/types/types.go

@@ -5,19 +5,18 @@ package types
 
 import (
 	"bytes"
-	"encoding/binary"
 	"sort"
 
 	fieldparams "github.com/OffchainLabs/prysm/v6/config/fieldparams"
 	"github.com/OffchainLabs/prysm/v6/config/params"
+	"github.com/OffchainLabs/prysm/v6/encoding/ssz"
 	eth "github.com/OffchainLabs/prysm/v6/proto/prysm/v1alpha1"
 	"github.com/pkg/errors"
-	ssz "github.com/prysmaticlabs/fastssz"
+	fastssz "github.com/prysmaticlabs/fastssz"
 )
 
 const (
-	maxErrorLength       = 256
-	bytesPerLengthOffset = 4
+	maxErrorLength = 256
 )
 
 // SSZBytes is a bytes slice that satisfies the fast-ssz interface.
@@ -25,11 +24,11 @@ type SSZBytes []byte
 
 // HashTreeRoot hashes the uint64 object following the SSZ standard.
 func (b *SSZBytes) HashTreeRoot() ([32]byte, error) {
-	return ssz.HashWithDefaultHasher(b)
+	return fastssz.HashWithDefaultHasher(b)
 }
 
 // HashTreeRootWith hashes the uint64 object with the given hasher.
-func (b *SSZBytes) HashTreeRootWith(hh *ssz.Hasher) error {
+func (b *SSZBytes) HashTreeRootWith(hh *fastssz.Hasher) error {
 	indx := hh.Index()
 	hh.PutBytes(*b)
 	hh.Merkleize(indx)
@@ -74,7 +73,7 @@ func (r *BeaconBlockByRootsReq) UnmarshalSSZ(buf []byte) error {
 		return errors.Errorf("expected buffer with length of up to %d but received length %d", maxLength, bufLen)
 	}
 	if bufLen%fieldparams.RootLength != 0 {
-		return ssz.ErrIncorrectByteSize
+		return fastssz.ErrIncorrectByteSize
 	}
 	numOfRoots := bufLen / fieldparams.RootLength
 	roots := make([][fieldparams.RootLength]byte, 0, numOfRoots)
@@ -131,14 +130,6 @@ func (m *ErrorMessage) UnmarshalSSZ(buf []byte) error {
 // BlobSidecarsByRootReq is used to specify a list of blob targets (root+index) in a BlobSidecarsByRoot RPC request.
 type BlobSidecarsByRootReq []*eth.BlobIdentifier
 
-// BlobIdentifier is a fixed size value, so we can compute its fixed size at start time (see init below)
-var blobIdSize int
-
-// SizeSSZ returns the size of the serialized representation.
-func (b *BlobSidecarsByRootReq) SizeSSZ() int {
-	return len(*b) * blobIdSize
-}
-
 // MarshalSSZTo appends the serialized BlobSidecarsByRootReq value to the provided byte slice.
 func (b *BlobSidecarsByRootReq) MarshalSSZTo(dst []byte) ([]byte, error) {
 	// A List without an enclosing container is marshaled exactly like a vector, no length offset required.
@@ -151,38 +142,22 @@ func (b *BlobSidecarsByRootReq) MarshalSSZTo(dst []byte) ([]byte, error) {
 
 // MarshalSSZ serializes the BlobSidecarsByRootReq value to a byte slice.
 func (b *BlobSidecarsByRootReq) MarshalSSZ() ([]byte, error) {
-	buf := make([]byte, len(*b)*blobIdSize)
-	for i, id := range *b {
-		by, err := id.MarshalSSZ()
-		if err != nil {
-			return nil, err
-		}
-		copy(buf[i*blobIdSize:(i+1)*blobIdSize], by)
-	}
-	return buf, nil
+	return ssz.MarshalListFixedElement[*eth.BlobIdentifier](*b)
 }
 
+func newBSBR() *eth.BlobIdentifier { return &eth.BlobIdentifier{} }
+
 // UnmarshalSSZ unmarshals the provided bytes buffer into the
 // BlobSidecarsByRootReq value.
 func (b *BlobSidecarsByRootReq) UnmarshalSSZ(buf []byte) error {
-	bufLen := len(buf)
-	maxLength := int(params.BeaconConfig().MaxRequestBlobSidecarsElectra) * blobIdSize
-	if bufLen > maxLength {
-		return errors.Wrapf(ssz.ErrIncorrectListSize, "expected buffer with length of up to %d but received length %d", maxLength, bufLen)
+	v, err := ssz.UnmarshalListFixedElement[*eth.BlobIdentifier](buf, newBSBR)
+	if err != nil {
+		return errors.Wrapf(err, "failed to unmarshal BlobSidecarsByRootReq")
 	}
-	if bufLen%blobIdSize != 0 {
-		return errors.Wrapf(ssz.ErrIncorrectByteSize, "size=%d", bufLen)
-	}
-	count := bufLen / blobIdSize
-	*b = make([]*eth.BlobIdentifier, count)
-	for i := 0; i < count; i++ {
-		id := &eth.BlobIdentifier{}
-		err := id.UnmarshalSSZ(buf[i*blobIdSize : (i+1)*blobIdSize])
-		if err != nil {
-			return err
-		}
-		(*b)[i] = id
+	if len(v) > int(params.BeaconConfig().MaxRequestBlobSidecarsElectra) {
+		return ErrMaxBlobReqExceeded
 	}
+	*b = v
 	return nil
 }
 
@@ -213,102 +188,25 @@ func (s BlobSidecarsByRootReq) Len() int {
 // ====================================
 // DataColumnsByRootIdentifiers section
 // ====================================
-var _ ssz.Marshaler = DataColumnsByRootIdentifiers{}
-var _ ssz.Unmarshaler = (*DataColumnsByRootIdentifiers)(nil)
+var _ fastssz.Marshaler = &DataColumnsByRootIdentifiers{}
+var _ fastssz.Unmarshaler = &DataColumnsByRootIdentifiers{}
 
 // DataColumnsByRootIdentifiers is used to specify a list of data column targets (root+index) in a DataColumnSidecarsByRoot RPC request.
 type DataColumnsByRootIdentifiers []*eth.DataColumnsByRootIdentifier
 
-// DataColumnIdentifier is a fixed size value, so we can compute its fixed size at start time (see init below)
-var dataColumnIdSize int
+func newDCRI() *eth.DataColumnsByRootIdentifier { return &eth.DataColumnsByRootIdentifier{} }
 
-// UnmarshalSSZ implements ssz.Unmarshaler. It unmarshals the provided bytes buffer into the DataColumnSidecarsByRootReq value.
 func (d *DataColumnsByRootIdentifiers) UnmarshalSSZ(buf []byte) error {
-	// Exit early if the buffer is too small.
-	if len(buf) < bytesPerLengthOffset {
-		return nil
+	v, err := ssz.UnmarshalListVariableElement[*eth.DataColumnsByRootIdentifier](buf, newDCRI)
+	if err != nil {
+		return errors.Wrapf(err, "failed to unmarshal DataColumnsByRootIdentifiers")
 	}
-
-	// Get the size of the offsets.
-	offsetEnd := binary.LittleEndian.Uint32(buf[:bytesPerLengthOffset])
-	if offsetEnd%bytesPerLengthOffset != 0 {
-		return errors.Errorf("expected offsets size to be a multiple of %d but got %d", bytesPerLengthOffset, offsetEnd)
-	}
-
-	count := offsetEnd / bytesPerLengthOffset
-	if count < 1 {
-		return nil
-	}
-
-	maxSize := params.BeaconConfig().MaxRequestBlocksDeneb
-	if uint64(count) > maxSize {
-		return errors.Errorf("data column identifiers list exceeds max size: %d > %d", count, maxSize)
-	}
-
-	if offsetEnd > uint32(len(buf)) {
-		return errors.Errorf("offsets value %d larger than buffer %d", offsetEnd, len(buf))
-	}
-	valueStart := offsetEnd
-
-	// Decode the identifers.
-	*d = make([]*eth.DataColumnsByRootIdentifier, count)
-	var start uint32
-	end := uint32(len(buf))
-	for i := count; i > 0; i-- {
-		offsetEnd -= bytesPerLengthOffset
-		start = binary.LittleEndian.Uint32(buf[offsetEnd : offsetEnd+bytesPerLengthOffset])
-		if start > end {
-			return errors.Errorf("expected offset[%d] %d to be less than %d", i-1, start, end)
-		}
-		if start < valueStart {
-			return errors.Errorf("offset[%d] %d indexes before value section %d", i-1, start, valueStart)
-		}
-		// Decode the identifier.
-		ident := &eth.DataColumnsByRootIdentifier{}
-		if err := ident.UnmarshalSSZ(buf[start:end]); err != nil {
-			return err
-		}
-		(*d)[i-1] = ident
-		end = start
-	}
-
+	*d = v
 	return nil
 }
 
-func (d DataColumnsByRootIdentifiers) MarshalSSZ() ([]byte, error) {
-	var err error
-	count := len(d)
-	maxSize := params.BeaconConfig().MaxRequestBlocksDeneb
-	if uint64(count) > maxSize {
-		return nil, errors.Errorf("data column identifiers list exceeds max size: %d > %d", count, maxSize)
-	}
-
-	if len(d) == 0 {
-		return []byte{}, nil
-	}
-	sizes := make([]uint32, count)
-	valTotal := uint32(0)
-	for i, elem := range d {
-		if elem == nil {
-			return nil, errors.New("nil item in DataColumnsByRootIdentifiers list")
-		}
-		sizes[i] = uint32(elem.SizeSSZ())
-		valTotal += sizes[i]
-	}
-	offSize := uint32(4 * len(d))
-	out := make([]byte, offSize, offSize+valTotal)
-	for i := range sizes {
-		binary.LittleEndian.PutUint32(out[i*4:i*4+4], offSize)
-		offSize += sizes[i]
-	}
-	for _, elem := range d {
-		out, err = elem.MarshalSSZTo(out)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return out, nil
+func (d *DataColumnsByRootIdentifiers) MarshalSSZ() ([]byte, error) {
+	return ssz.MarshalListVariableElement[*eth.DataColumnsByRootIdentifier](*d)
 }
 
 // MarshalSSZTo implements ssz.Marshaler. It appends the serialized DataColumnSidecarsByRootReq value to the provided byte slice.
@@ -329,11 +227,3 @@ func (d DataColumnsByRootIdentifiers) SizeSSZ() int {
 	}
 	return size
 }
-
-func init() {
-	blobSizer := &eth.BlobIdentifier{}
-	blobIdSize = blobSizer.SizeSSZ()
-
-	dataColumnSizer := &eth.DataColumnSidecarsByRangeRequest{}
-	dataColumnIdSize = dataColumnSizer.SizeSSZ()
-}

beacon-chain/p2p/types/types_test.go

@@ -8,10 +8,10 @@ import (
 	"github.com/OffchainLabs/prysm/v6/config/params"
 	"github.com/OffchainLabs/prysm/v6/consensus-types/primitives"
 	"github.com/OffchainLabs/prysm/v6/encoding/bytesutil"
+	"github.com/OffchainLabs/prysm/v6/encoding/ssz"
 	eth "github.com/OffchainLabs/prysm/v6/proto/prysm/v1alpha1"
 	"github.com/OffchainLabs/prysm/v6/testing/assert"
 	"github.com/OffchainLabs/prysm/v6/testing/require"
-	ssz "github.com/prysmaticlabs/fastssz"
 )
 
 func generateBlobIdentifiers(n int) []*eth.BlobIdentifier {
@@ -51,7 +51,7 @@ func TestBlobSidecarsByRootReq_MarshalSSZ(t *testing.T) {
 		{
 			name:         "beyond max list",
 			ids:          generateBlobIdentifiers(int(params.BeaconConfig().MaxRequestBlobSidecarsElectra) + 1),
-			unmarshalErr: ssz.ErrIncorrectListSize,
+			unmarshalErr: ErrMaxBlobReqExceeded,
 		},
 		{
 			name: "wonky unmarshal size",
@@ -60,7 +60,7 @@ func TestBlobSidecarsByRootReq_MarshalSSZ(t *testing.T) {
 				in = append(in, byte(0))
 				return in
 			},
-			unmarshalErr: ssz.ErrIncorrectByteSize,
+			unmarshalErr: ssz.ErrInvalidFixedEncodingLen,
 		},
 	}
 
@@ -305,7 +305,8 @@ func TestDataColumnSidecarsByRootReq_MarshalUnmarshal(t *testing.T) {
 			name: "size too big",
 			ids:  generateDataColumnIdentifiers(1),
 			unmarshalMod: func(in []byte) []byte {
-				maxLen := params.BeaconConfig().MaxRequestDataColumnSidecars * uint64(dataColumnIdSize)
+				sizer := &eth.DataColumnSidecarsByRangeRequest{}
+				maxLen := params.BeaconConfig().MaxRequestDataColumnSidecars * uint64(sizer.SizeSSZ())
 				add := make([]byte, maxLen)
 				in = append(in, add...)
 				return in

changelog/kasey_generic-list-serdes.md

@@ -0,0 +1,2 @@
+## Added
+- Methods to generically encode/decode independent lists of ssz values.

encoding/ssz/BUILD.bazel

@@ -3,9 +3,11 @@ load("@prysm//tools/go:def.bzl", "go_library", "go_test")
 go_library(
     name = "go_default_library",
     srcs = [
+        "errors.go",
         "hashers.go",
         "helpers.go",
         "htrutils.go",
+        "list.go",
         "merkleize.go",
         "slice_root.go",
     ],

encoding/ssz/errors.go

@@ -0,0 +1,17 @@
+package ssz
+
+import "github.com/pkg/errors"
+
+var (
+	ErrInvalidEncodingLength     = errors.New("invalid encoded length")
+	ErrInvalidFixedEncodingLen   = errors.Wrap(ErrInvalidEncodingLength, "not multiple of fixed size")
+	ErrEncodingSmallerThanOffset = errors.Wrap(ErrInvalidEncodingLength, "smaller than a single offset")
+	ErrInvalidOffset             = errors.New("invalid offset")
+	ErrOffsetIntoFixed           = errors.Wrap(ErrInvalidOffset, "does not point past fixed section of encoding")
+	ErrOffsetExceedsBuffer       = errors.Wrap(ErrInvalidOffset, "exceeds buffer length")
+	ErrNegativeRelativeOffset    = errors.Wrap(ErrInvalidOffset, "less than previous offset")
+	ErrOffsetInsufficient        = errors.Wrap(ErrInvalidOffset, "insufficient difference relative to previous")
+	ErrOffsetSectionMisaligned   = errors.Wrap(ErrInvalidOffset, "offset bytes are not a multiple of offset size")
+
+	ErrOffsetDecodedMismatch = errors.New("unmarshaled size does not relative offsets")
+)

encoding/ssz/list.go

@@ -0,0 +1,210 @@
+package ssz
+
+import (
+	"encoding/binary"
+	"math"
+
+	"github.com/pkg/errors"
+)
+
+const offsetLen = 4 // each list element offset is a 4-byte uint32
+
+type Marshalable interface {
+	MarshalSSZTo(buf []byte) ([]byte, error)
+	SizeSSZ() int
+}
+
+type Unmarshalable interface {
+	UnmarshalSSZ(buf []byte) error
+	SizeSSZ() int
+}
+
+// MarshalListFixedElement encodes a slice of fixed sized elements as an ssz list.
+// A list of fixed-size elements is marshaled by concatenating the marshaled bytes
+// of each element in the list.
+//
+// MarshalListVariableElement should be used for variable-sized elements.
+// SSZ Lists have different encoding rules depending whether their elements are fixed- or variable-sized,
+// and we can't differentiate them by the ssz interface, so it is the caller's responsibility to
+// pick the correct method.
+func MarshalListFixedElement[T Marshalable](elems []T) ([]byte, error) {
+	if len(elems) == 0 {
+		return nil, nil
+	}
+	size := elems[0].SizeSSZ()
+	buf := make([]byte, 0, len(elems)*size)
+	for _, elem := range elems {
+		if elem.SizeSSZ() != size {
+			return nil, ErrInvalidFixedEncodingLen
+		}
+		var err error
+		buf, err = elem.MarshalSSZTo(buf)
+		if err != nil {
+			return nil, errors.Wrap(err, "marshal ssz")
+		}
+	}
+	return buf, nil
+}
+
+// MarshalListVariableElement marshals a list of variable-sized elements.
+// A list of variable-sized elements is marshaled by first writing the offsets of each element to the
+// beginning of the byte sequence (the fixed size section of the variable sized list container), followed
+// by the encoded values of each element at the indicated offset relative to the beginning of the byte sequence.
+//
+// MarshalListFixedElement should be used for fixed-size elements.
+// SSZ Lists have different encoding rules depending whether their elements are fixed- or variable-sized,
+// and we can't differentiate them by the ssz interface, so it is the caller's responsibility to
+// pick the correct method.
+func MarshalListVariableElement[T Marshalable](elems []T) ([]byte, error) {
+	var err error
+	var total uint32
+	nElems := len(elems)
+	if nElems == 0 {
+		return nil, nil
+	}
+	sizes := make([]uint32, nElems)
+	for i, e := range elems {
+		sizes[i], err = safeUint32(e.SizeSSZ())
+		if err != nil {
+			return nil, err
+		}
+		total += sizes[i]
+	}
+	nextOffset, err := safeUint32(nElems * offsetLen)
+	if err != nil {
+		return nil, err
+	}
+	buf := make([]byte, 0, total+nextOffset)
+	for _, size := range sizes {
+		buf = binary.LittleEndian.AppendUint32(buf, nextOffset)
+		nextOffset += size
+	}
+	for _, elem := range elems {
+		buf, err = elem.MarshalSSZTo(buf)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return buf, nil
+}
+
+// UnmarshalListVariableElement unmarshals a ssz-encoded list of variable-sized elements.
+// Because this generic method is parameterized by a [T Unmarshalable] interface type,
+// it is unable to initialize elements of the list internally. That is why the caller must
+// provide the `newt` function that returns a new instance of the type [T] to be unmarshaled.
+// This func will be called for each element in the list to create a new instance of [T].
+//
+// UnmarshalListFixedElement should be used for fixed-size elements.
+// SSZ Lists have different encoding rules depending whether their elements are fixed- or variable-sized,
+// and we can't differentiate them by the ssz interface, so it is the caller's responsibility to
+// pick the correct method.
+func UnmarshalListVariableElement[T Unmarshalable](buf []byte, newt func() T) ([]T, error) {
+	bufLen := len(buf)
+	if bufLen == 0 {
+		return nil, nil
+	}
+	if bufLen < offsetLen {
+		return nil, ErrEncodingSmallerThanOffset
+	}
+	fixedSize := uint32(newt().SizeSSZ())
+	bufLen32 := uint32(bufLen)
+
+	first := binary.LittleEndian.Uint32(buf)
+	// Rather than just return a zero element list in this case,
+	// we want to explicitly reject this input as invalid
+	if first < offsetLen {
+		return nil, ErrOffsetIntoFixed
+	}
+	if first%offsetLen != 0 {
+		return nil, ErrOffsetSectionMisaligned
+	}
+	if first > bufLen32 {
+		return nil, ErrOffsetExceedsBuffer
+	}
+
+	nElems := int(first) / offsetLen // lint:ignore uintcast -- int has higher precision than uint32 on 64 bit systems, so this is 100% safe
+	if nElems == 0 {
+		return nil, nil
+	}
+	buf = buf[offsetLen:]
+	sizes := make([]uint32, nElems)
+
+	// We've already looked at the offset of the first element (to perform validation on it)
+	// so we just need to iterate over the remaining offsets, aka nElems-1 times.
+	// The size of each element is computed relative to the next offset, so this loop is effectively
+	// looking ahead +1 (starting with a `buf` that has already had the first offset sliced off),
+	// with the final element handled as a special case outside the loop (using the size of the entire buffer
+	// as the ending bound).
+	previous := first
+	for i := 0; i < nElems-1; i++ {
+		next := binary.LittleEndian.Uint32(buf)
+		if next > bufLen32 {
+			return nil, ErrOffsetExceedsBuffer
+		}
+		if next < previous {
+			return nil, ErrNegativeRelativeOffset
+		}
+		sizes[i] = next - previous
+		if sizes[i] < fixedSize {
+			return nil, ErrOffsetInsufficient
+		}
+		buf = buf[offsetLen:]
+		previous = next
+	}
+	sizes[len(sizes)-1] = bufLen32 - previous
+	elements := make([]T, nElems)
+	for i, size := range sizes {
+		elem := newt()
+		if err := elem.UnmarshalSSZ(buf[:size]); err != nil {
+			return nil, errors.Wrap(err, "unmarshal ssz")
+		}
+		szi := int(size) // lint:ignore uintcast -- int has higher precision than uint32 on 64 bit systems, so this is 100% safe
+		if elem.SizeSSZ() != szi {
+			return nil, ErrOffsetDecodedMismatch
+		}
+		elements[i] = elem
+		buf = buf[size:]
+	}
+	return elements, nil
+}
+
+// UnmarshalListFixedElement unmarshals a ssz-encoded list of variable-sized elements.
+// A List of fixed-size elements is encoded as a concatenation of the marshaled bytes of each
+// element, so after performing some safety checks on the alignment and size of the buffer,
+// we simply iterate over the buffer in chunks of the fixed size and unmarshal each element.
+// Because this generic method is parameterized by a [T Unmarshalable] interface type,
+// it is unable to initialize elements of the list internally. That is why the caller must
+// provide the `newt` function that returns a new instance of the type [T] to be unmarshaled.
+// This func will be called for each element in the list to create a new instance of [T].
+//
+// UnmarshalListFixedElement should be used for fixed-size elements.
+// SSZ Lists have different encoding rules depending whether their elements are fixed- or variable-sized,
+// and we can't differentiate them by the ssz interface, so it is the caller's responsibility to
+// pick the correct method.
+func UnmarshalListFixedElement[T Unmarshalable](buf []byte, newt func() T) ([]T, error) {
+	bufLen := len(buf)
+	if bufLen == 0 {
+		return nil, nil
+	}
+	fixedSize := newt().SizeSSZ()
+	if bufLen%fixedSize != 0 {
+		return nil, ErrInvalidFixedEncodingLen
+	}
+	nElems := bufLen / fixedSize
+	elements := make([]T, nElems)
+	for i := 0; i < nElems; i++ {
+		elem := newt()
+		if err := elem.UnmarshalSSZ(buf[i*fixedSize : (i+1)*fixedSize]); err != nil {
+			return nil, errors.Wrap(err, "unmarshal ssz")
+		}
+		elements[i] = elem
+	}
+	return elements, nil
+}
+
+func safeUint32(val int) (uint32, error) {
+	if val < 0 || val > math.MaxUint32 {
+		return 0, errors.New("value exceeds uint32 range")
+	}
+	return uint32(val), nil // lint:ignore uintcast -- integer value explicitly checked to prevent truncation
+}