Compare commits

...

1 Commits

Author SHA1 Message Date
Kasey
fa6a09b1e8 generic ssz list encoding
Co-authored-by: Bastin <43618253+Inspector-Butters@users.noreply.github.com>
2025-07-02 21:50:36 -05:00
7 changed files with 331 additions and 138 deletions

View File

@@ -25,6 +25,7 @@ go_library(
"//consensus-types/primitives:go_default_library",
"//consensus-types/wrapper:go_default_library",
"//encoding/bytesutil:go_default_library",
"//encoding/ssz:go_default_library",
"//proto/engine/v1:go_default_library",
"//proto/prysm/v1alpha1:go_default_library",
"//proto/prysm/v1alpha1/metadata:go_default_library",
@@ -45,10 +46,10 @@ go_test(
"//config/params:go_default_library",
"//consensus-types/primitives:go_default_library",
"//encoding/bytesutil:go_default_library",
"//encoding/ssz:go_default_library",
"//proto/prysm/v1alpha1:go_default_library",
"//runtime/version:go_default_library",
"//testing/assert:go_default_library",
"//testing/require:go_default_library",
"@com_github_prysmaticlabs_fastssz//:go_default_library",
],
)

View File

@@ -5,19 +5,18 @@ package types
import (
"bytes"
"encoding/binary"
"sort"
fieldparams "github.com/OffchainLabs/prysm/v6/config/fieldparams"
"github.com/OffchainLabs/prysm/v6/config/params"
"github.com/OffchainLabs/prysm/v6/encoding/ssz"
eth "github.com/OffchainLabs/prysm/v6/proto/prysm/v1alpha1"
"github.com/pkg/errors"
ssz "github.com/prysmaticlabs/fastssz"
fastssz "github.com/prysmaticlabs/fastssz"
)
const (
maxErrorLength = 256
bytesPerLengthOffset = 4
maxErrorLength = 256
)
// SSZBytes is a bytes slice that satisfies the fast-ssz interface.
@@ -25,11 +24,11 @@ type SSZBytes []byte
// HashTreeRoot hashes the uint64 object following the SSZ standard.
func (b *SSZBytes) HashTreeRoot() ([32]byte, error) {
return ssz.HashWithDefaultHasher(b)
return fastssz.HashWithDefaultHasher(b)
}
// HashTreeRootWith hashes the uint64 object with the given hasher.
func (b *SSZBytes) HashTreeRootWith(hh *ssz.Hasher) error {
func (b *SSZBytes) HashTreeRootWith(hh *fastssz.Hasher) error {
indx := hh.Index()
hh.PutBytes(*b)
hh.Merkleize(indx)
@@ -74,7 +73,7 @@ func (r *BeaconBlockByRootsReq) UnmarshalSSZ(buf []byte) error {
return errors.Errorf("expected buffer with length of up to %d but received length %d", maxLength, bufLen)
}
if bufLen%fieldparams.RootLength != 0 {
return ssz.ErrIncorrectByteSize
return fastssz.ErrIncorrectByteSize
}
numOfRoots := bufLen / fieldparams.RootLength
roots := make([][fieldparams.RootLength]byte, 0, numOfRoots)
@@ -128,17 +127,13 @@ func (m *ErrorMessage) UnmarshalSSZ(buf []byte) error {
return nil
}
var BlobSidecarsByRootReqSerdes = ssz.NewListFixedElementSerdes[*eth.BlobIdentifier](func() *eth.BlobIdentifier {
return &eth.BlobIdentifier{}
})
// BlobSidecarsByRootReq is used to specify a list of blob targets (root+index) in a BlobSidecarsByRoot RPC request.
type BlobSidecarsByRootReq []*eth.BlobIdentifier
// BlobIdentifier is a fixed size value, so we can compute its fixed size at start time (see init below)
var blobIdSize int
// SizeSSZ returns the size of the serialized representation.
func (b *BlobSidecarsByRootReq) SizeSSZ() int {
return len(*b) * blobIdSize
}
// MarshalSSZTo appends the serialized BlobSidecarsByRootReq value to the provided byte slice.
func (b *BlobSidecarsByRootReq) MarshalSSZTo(dst []byte) ([]byte, error) {
// A List without an enclosing container is marshaled exactly like a vector, no length offset required.
@@ -151,38 +146,20 @@ func (b *BlobSidecarsByRootReq) MarshalSSZTo(dst []byte) ([]byte, error) {
// MarshalSSZ serializes the BlobSidecarsByRootReq value to a byte slice.
func (b *BlobSidecarsByRootReq) MarshalSSZ() ([]byte, error) {
buf := make([]byte, len(*b)*blobIdSize)
for i, id := range *b {
by, err := id.MarshalSSZ()
if err != nil {
return nil, err
}
copy(buf[i*blobIdSize:(i+1)*blobIdSize], by)
}
return buf, nil
return BlobSidecarsByRootReqSerdes.Marshal(*b)
}
// UnmarshalSSZ unmarshals the provided bytes buffer into the
// BlobSidecarsByRootReq value.
func (b *BlobSidecarsByRootReq) UnmarshalSSZ(buf []byte) error {
bufLen := len(buf)
maxLength := int(params.BeaconConfig().MaxRequestBlobSidecarsElectra) * blobIdSize
if bufLen > maxLength {
return errors.Wrapf(ssz.ErrIncorrectListSize, "expected buffer with length of up to %d but received length %d", maxLength, bufLen)
v, err := BlobSidecarsByRootReqSerdes.Unmarshal(buf)
if err != nil {
return errors.Wrapf(err, "failed to unmarshal BlobSidecarsByRootReq")
}
if bufLen%blobIdSize != 0 {
return errors.Wrapf(ssz.ErrIncorrectByteSize, "size=%d", bufLen)
}
count := bufLen / blobIdSize
*b = make([]*eth.BlobIdentifier, count)
for i := 0; i < count; i++ {
id := &eth.BlobIdentifier{}
err := id.UnmarshalSSZ(buf[i*blobIdSize : (i+1)*blobIdSize])
if err != nil {
return err
}
(*b)[i] = id
if len(v) > int(params.BeaconConfig().MaxRequestBlobSidecarsElectra) {
return ErrMaxBlobReqExceeded
}
*b = v
return nil
}
@@ -213,102 +190,28 @@ func (s BlobSidecarsByRootReq) Len() int {
// ====================================
// DataColumnsByRootIdentifiers section
// ====================================
var _ ssz.Marshaler = DataColumnsByRootIdentifiers{}
var _ ssz.Unmarshaler = (*DataColumnsByRootIdentifiers)(nil)
var _ fastssz.Marshaler = DataColumnsByRootIdentifiers{}
var _ fastssz.Unmarshaler = &DataColumnsByRootIdentifiers{}
// DataColumnsByRootIdentifiers is used to specify a list of data column targets (root+index) in a DataColumnSidecarsByRoot RPC request.
type DataColumnsByRootIdentifiers []*eth.DataColumnsByRootIdentifier
// DataColumnIdentifier is a fixed size value, so we can compute its fixed size at start time (see init below)
var dataColumnIdSize int
var DataColumnsByRootIdentifiersSerdes = ssz.NewListVariableElementSerdes[*eth.DataColumnsByRootIdentifier](func() *eth.DataColumnsByRootIdentifier {
return &eth.DataColumnsByRootIdentifier{}
})
// UnmarshalSSZ implements ssz.Unmarshaler. It unmarshals the provided bytes buffer into the DataColumnSidecarsByRootReq value.
func (d *DataColumnsByRootIdentifiers) UnmarshalSSZ(buf []byte) error {
// Exit early if the buffer is too small.
if len(buf) < bytesPerLengthOffset {
return nil
//v, err := DataColumnsByRootIdentifiersSerdes.Unmarshal(buf)
v, err := DataColumnsByRootIdentifiersSerdes.Unmarshal(buf)
if err != nil {
return errors.Wrapf(err, "failed to unmarshal DataColumnsByRootIdentifiers")
}
// Get the size of the offsets.
offsetEnd := binary.LittleEndian.Uint32(buf[:bytesPerLengthOffset])
if offsetEnd%bytesPerLengthOffset != 0 {
return errors.Errorf("expected offsets size to be a multiple of %d but got %d", bytesPerLengthOffset, offsetEnd)
}
count := offsetEnd / bytesPerLengthOffset
if count < 1 {
return nil
}
maxSize := params.BeaconConfig().MaxRequestBlocksDeneb
if uint64(count) > maxSize {
return errors.Errorf("data column identifiers list exceeds max size: %d > %d", count, maxSize)
}
if offsetEnd > uint32(len(buf)) {
return errors.Errorf("offsets value %d larger than buffer %d", offsetEnd, len(buf))
}
valueStart := offsetEnd
// Decode the identifers.
*d = make([]*eth.DataColumnsByRootIdentifier, count)
var start uint32
end := uint32(len(buf))
for i := count; i > 0; i-- {
offsetEnd -= bytesPerLengthOffset
start = binary.LittleEndian.Uint32(buf[offsetEnd : offsetEnd+bytesPerLengthOffset])
if start > end {
return errors.Errorf("expected offset[%d] %d to be less than %d", i-1, start, end)
}
if start < valueStart {
return errors.Errorf("offset[%d] %d indexes before value section %d", i-1, start, valueStart)
}
// Decode the identifier.
ident := &eth.DataColumnsByRootIdentifier{}
if err := ident.UnmarshalSSZ(buf[start:end]); err != nil {
return err
}
(*d)[i-1] = ident
end = start
}
*d = v
return nil
}
func (d DataColumnsByRootIdentifiers) MarshalSSZ() ([]byte, error) {
var err error
count := len(d)
maxSize := params.BeaconConfig().MaxRequestBlocksDeneb
if uint64(count) > maxSize {
return nil, errors.Errorf("data column identifiers list exceeds max size: %d > %d", count, maxSize)
}
if len(d) == 0 {
return []byte{}, nil
}
sizes := make([]uint32, count)
valTotal := uint32(0)
for i, elem := range d {
if elem == nil {
return nil, errors.New("nil item in DataColumnsByRootIdentifiers list")
}
sizes[i] = uint32(elem.SizeSSZ())
valTotal += sizes[i]
}
offSize := uint32(4 * len(d))
out := make([]byte, offSize, offSize+valTotal)
for i := range sizes {
binary.LittleEndian.PutUint32(out[i*4:i*4+4], offSize)
offSize += sizes[i]
}
for _, elem := range d {
out, err = elem.MarshalSSZTo(out)
if err != nil {
return nil, err
}
}
return out, nil
return DataColumnsByRootIdentifiersSerdes.Marshal(d)
}
// MarshalSSZTo implements ssz.Marshaler. It appends the serialized DataColumnSidecarsByRootReq value to the provided byte slice.
@@ -329,11 +232,3 @@ func (d DataColumnsByRootIdentifiers) SizeSSZ() int {
}
return size
}
func init() {
blobSizer := &eth.BlobIdentifier{}
blobIdSize = blobSizer.SizeSSZ()
dataColumnSizer := &eth.DataColumnSidecarsByRangeRequest{}
dataColumnIdSize = dataColumnSizer.SizeSSZ()
}

View File

@@ -8,10 +8,10 @@ import (
"github.com/OffchainLabs/prysm/v6/config/params"
"github.com/OffchainLabs/prysm/v6/consensus-types/primitives"
"github.com/OffchainLabs/prysm/v6/encoding/bytesutil"
"github.com/OffchainLabs/prysm/v6/encoding/ssz"
eth "github.com/OffchainLabs/prysm/v6/proto/prysm/v1alpha1"
"github.com/OffchainLabs/prysm/v6/testing/assert"
"github.com/OffchainLabs/prysm/v6/testing/require"
ssz "github.com/prysmaticlabs/fastssz"
)
func generateBlobIdentifiers(n int) []*eth.BlobIdentifier {
@@ -51,7 +51,7 @@ func TestBlobSidecarsByRootReq_MarshalSSZ(t *testing.T) {
{
name: "beyond max list",
ids: generateBlobIdentifiers(int(params.BeaconConfig().MaxRequestBlobSidecarsElectra) + 1),
unmarshalErr: ssz.ErrIncorrectListSize,
unmarshalErr: ErrMaxBlobReqExceeded,
},
{
name: "wonky unmarshal size",
@@ -60,7 +60,7 @@ func TestBlobSidecarsByRootReq_MarshalSSZ(t *testing.T) {
in = append(in, byte(0))
return in
},
unmarshalErr: ssz.ErrIncorrectByteSize,
unmarshalErr: ssz.ErrInvalidFixedEncodingLen,
},
}
@@ -305,7 +305,8 @@ func TestDataColumnSidecarsByRootReq_MarshalUnmarshal(t *testing.T) {
name: "size too big",
ids: generateDataColumnIdentifiers(1),
unmarshalMod: func(in []byte) []byte {
maxLen := params.BeaconConfig().MaxRequestDataColumnSidecars * uint64(dataColumnIdSize)
sizer := &eth.DataColumnSidecarsByRangeRequest{}
maxLen := params.BeaconConfig().MaxRequestDataColumnSidecars * uint64(sizer.SizeSSZ())
add := make([]byte, maxLen)
in = append(in, add...)
return in

View File

@@ -0,0 +1,2 @@
## Added
- Methods to generically encode/decode independent lists of ssz values.

View File

@@ -3,9 +3,11 @@ load("@prysm//tools/go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = [
"errors.go",
"hashers.go",
"helpers.go",
"htrutils.go",
"list.go",
"merkleize.go",
"slice_root.go",
],

17
encoding/ssz/errors.go Normal file
View File

@@ -0,0 +1,17 @@
package ssz
import "github.com/pkg/errors"
var (
ErrInvalidEncodingLength = errors.New("invalid encoded length")
ErrInvalidFixedEncodingLen = errors.Wrap(ErrInvalidEncodingLength, "not multiple of fixed size")
ErrEncodingSmallerThanOffset = errors.Wrap(ErrInvalidEncodingLength, "smaller than a single offset")
ErrInvalidOffset = errors.New("invalid offset")
ErrOffsetIntoFixed = errors.Wrap(ErrInvalidOffset, "does not point past fixed section of encoding")
ErrOffsetExceedsBuffer = errors.Wrap(ErrInvalidOffset, "exceeds buffer length")
ErrNegativeRelativeOffset = errors.Wrap(ErrInvalidOffset, "less than previous offset")
ErrOffsetInsufficient = errors.Wrap(ErrInvalidOffset, "insufficient difference relative to previous")
ErrOffsetSectionMisaligned = errors.Wrap(ErrInvalidOffset, "offset bytes are not a multiple of offset size")
ErrOffsetDecodedMismatch = errors.New("unmarshaled size does not relative offsets")
)

275
encoding/ssz/list.go Normal file
View File

@@ -0,0 +1,275 @@
package ssz
import (
"encoding/binary"
"math"
"github.com/pkg/errors"
)
const offsetLen = 4 // Each variable-sized list element offset is a 4-byte uint32.
// Marshalable describes the methodset required for a type to be generically marshaled.
type Marshalable interface {
MarshalSSZTo(buf []byte) ([]byte, error)
SizeSSZ() int
}
// Unmarshalable describes the methodset required for a type to be generically unmarshaled.
type Unmarshalable interface {
UnmarshalSSZ(buf []byte) error
SizeSSZ() int
}
// SerDesable is a union interface that combines both Marshalable and Unmarshalable interfaces.
// The name means "serializable/deserializable", indicating that types implementing this interface
// can be marshaled to and unmarshaled from a byte sequence.
type SerDesable interface {
Marshalable
Unmarshalable
}
// ListSerdes is a type that manages the serialization and deserialization of a list of elements
// for a given type that supports the SerDesable interface.
type ListSerdes[T SerDesable] struct {
new func() T
marshal func([]T) ([]byte, error)
unmarshal func([]byte, func() T) ([]T, error)
}
// NewListFixedElementSerdes creates a new ListSerdes parameterized by the given type [T SerDesable]
// where the type [T] is expected to be a fixed-size ssz type.
// and holds onto to the constructor func so users of the ListSerdes don't need to specify it.
func NewListFixedElementSerdes[T SerDesable](newt func() T) ListSerdes[T] {
return ListSerdes[T]{
new: newt,
marshal: MarshalListFixedElement[T],
unmarshal: UnmarshalListFixedElement[T],
}
}
// NewListVariableElementSerdes creates a new ListSerdes parameterized by the given type [T SerDesable]
// where the type [T] is expected to be a fixed-size ssz type.
// and holds onto to the constructor func so users of the ListSerdes don't need to specify it.
func NewListVariableElementSerdes[T SerDesable](newt func() T) ListSerdes[T] {
return ListSerdes[T]{
new: newt,
marshal: MarshalListVariableElement[T],
unmarshal: UnmarshalListVariableElement[T],
}
}
// Marshal encodes a slice of elements of type [T] as an ssz-encoded List.
func (ls ListSerdes[T]) Marshal(elems []T) ([]byte, error) {
if len(elems) == 0 {
return nil, nil
}
return ls.marshal(elems)
}
// Unmarshal decodes an ssz-encoded List of elements of type [T].
func (ls ListSerdes[T]) Unmarshal(buf []byte) ([]T, error) {
if len(buf) == 0 {
return nil, nil
}
return ls.unmarshal(buf, ls.new)
}
// MarshalListFixedElement encodes a slice of fixed-sized elements as an ssz list.
// A list of fixed-size elements is marshaled by concatenating the marshaled bytes
// of each element in the list.
//
// For variable-sized elements, use MarshalListVariableElement instead.
// SSZ Lists have different encoding rules depending whether their elements are fixed- or variable-sized,
// and we can't differentiate them by the ssz interface, so it is the caller's responsibility to
// pick the correct method.
//
// This method should only be used for container types that have code-generated methods.
// For lists of primitive types (eg a List[Vector[byte, 32], N]), please use generated code, or hand-tuned methods.
func MarshalListFixedElement[T Marshalable](elems []T) ([]byte, error) {
if len(elems) == 0 {
return nil, nil
}
size := elems[0].SizeSSZ()
buf := make([]byte, 0, len(elems)*size)
for _, elem := range elems {
if elem.SizeSSZ() != size {
return nil, ErrInvalidFixedEncodingLen
}
var err error
buf, err = elem.MarshalSSZTo(buf)
if err != nil {
return nil, errors.Wrap(err, "marshal ssz")
}
}
return buf, nil
}
// UnmarshalListFixedElement unmarshals an ssz-encoded list of fixed-sized elements.
// A List of fixed-size elements is encoded as a concatenation of the marshaled bytes of each
// element, so after performing some safety checks on the alignment and size of the buffer,
// we simply iterate over the buffer in chunks of the fixed size and unmarshal each element.
// Because this generic method is parameterized by a [T Unmarshalable] interface type,
// it is unable to initialize elements of the list internally. That is why the caller must
// provide the `newt` function that returns a new instance of the type [T] to be unmarshaled.
// This func will be called for each element in the list to create a new instance of [T].
//
// For variable-sized elements, use UnmarshalListVariableElement instead.
// SSZ Lists have different encoding rules depending whether their elements are fixed- or variable-sized,
// and we can't differentiate them by the ssz interface, so it is the caller's responsibility to
// pick the correct method.
//
// This method should only be used for container types that have code-generated methods.
// For lists of primitive types (eg a List[Vector[byte, 32], N]), please use generated code, or hand-tuned methods.
func UnmarshalListFixedElement[T Unmarshalable](buf []byte, newt func() T) ([]T, error) {
bufLen := len(buf)
if bufLen == 0 {
return nil, nil
}
fixedSize := newt().SizeSSZ()
if bufLen%fixedSize != 0 {
return nil, ErrInvalidFixedEncodingLen
}
nElems := bufLen / fixedSize
elements := make([]T, nElems)
for i := range elements {
elem := newt()
if err := elem.UnmarshalSSZ(buf[i*fixedSize : (i+1)*fixedSize]); err != nil {
return nil, errors.Wrap(err, "unmarshal ssz")
}
elements[i] = elem
}
return elements, nil
}
// MarshalListVariableElement marshals a slice of variable-sized elements as an ssz list.
// A list of variable-sized elements is marshaled by first writing the offsets of each element to the
// beginning of the byte sequence (the fixed size section of the variable sized list container), followed
// by the encoded values of each element at the indicated offset relative to the beginning of the byte sequence.
//
// For fixed-sized elements, use MarshalListFixedElement instead.
// SSZ Lists have different encoding rules depending whether their elements are fixed- or variable-sized,
// and we can't differentiate them by the ssz interface, so it is the caller's responsibility to
// pick the correct method.
//
// This method should only be used for container types that have code-generated methods.
// For lists of primitive types (eg a List[List[byte, N], N]), please use generated code, or hand-tuned methods.
func MarshalListVariableElement[T Marshalable](elems []T) ([]byte, error) {
var err error
var total uint32
nElems := len(elems)
if nElems == 0 {
return nil, nil
}
sizes := make([]uint32, nElems)
for i, e := range elems {
sizes[i], err = safeUint32(e.SizeSSZ())
if err != nil {
return nil, err
}
total += sizes[i]
}
nextOffset, err := safeUint32(nElems * offsetLen)
if err != nil {
return nil, err
}
buf := make([]byte, 0, total+nextOffset)
for _, size := range sizes {
buf = binary.LittleEndian.AppendUint32(buf, nextOffset)
nextOffset += size
}
for _, elem := range elems {
buf, err = elem.MarshalSSZTo(buf)
if err != nil {
return nil, err
}
}
return buf, nil
}
// UnmarshalListVariableElement unmarshals an ssz-encoded list of variable-sized elements.
// Because this generic method is parameterized by a [T Unmarshalable] interface type,
// it is unable to initialize elements of the list internally. That is why the caller must
// provide the `newt` function that returns a new instance of the type [T] to be unmarshaled.
// This func will be called for each element in the list to create a new instance of [T].
//
// For fixed-sized elements, use UnmarshalListFixedElement instead.
// SSZ Lists have different encoding rules depending whether their elements are fixed- or variable-sized,
// and we can't differentiate them by the ssz interface, so it is the caller's responsibility to
// pick the correct method.
//
// This method should only be used for container types that have code-generated methods.
// For lists of primitive types (eg a List[List[byte, N], N]), please use generated code, or hand-tuned methods.
func UnmarshalListVariableElement[T Unmarshalable](buf []byte, newt func() T) ([]T, error) {
bufLen := len(buf)
if bufLen == 0 {
return nil, nil
}
if bufLen < offsetLen {
return nil, ErrEncodingSmallerThanOffset
}
fixedSize := uint32(newt().SizeSSZ())
bufLen32 := uint32(bufLen)
first := binary.LittleEndian.Uint32(buf)
// Rather than just return a zero element list in this case,
// we want to explicitly reject this input as invalid
if first < offsetLen {
return nil, ErrOffsetIntoFixed
}
if first%offsetLen != 0 {
return nil, ErrOffsetSectionMisaligned
}
if first > bufLen32 {
return nil, ErrOffsetExceedsBuffer
}
nElems := int(first) / offsetLen // lint:ignore uintcast -- int has higher precision than uint32 on 64 bit systems, so this is 100% safe
sizes := make([]uint32, nElems)
// We've already looked at the offset of the first element (to perform validation on it)
// so we just need to iterate over the remaining offsets, aka nElems-1 times.
// The size of each element is computed relative to the next offset, so this loop is effectively
// looking ahead +1 (starting with a `buf` that has already had the first offset sliced off),
// with the final element handled as a special case outside the loop (using the size of the entire buffer
// as the ending bound).
previous := first
buf = buf[offsetLen:]
for i := 0; i < nElems-1; i++ {
next := binary.LittleEndian.Uint32(buf)
if next > bufLen32 {
return nil, ErrOffsetExceedsBuffer
}
if next < previous {
return nil, ErrNegativeRelativeOffset
}
sizes[i] = next - previous
if sizes[i] < fixedSize {
return nil, ErrOffsetInsufficient
}
buf = buf[offsetLen:]
previous = next
}
sizes[len(sizes)-1] = bufLen32 - previous
elements := make([]T, len(sizes))
for i, size := range sizes {
elem := newt()
if err := elem.UnmarshalSSZ(buf[:size]); err != nil {
return nil, errors.Wrap(err, "unmarshal ssz")
}
szi := int(size) // lint:ignore uintcast -- int has higher precision than uint32 on 64 bit systems, so this is 100% safe
if elem.SizeSSZ() != szi {
return nil, ErrOffsetDecodedMismatch
}
elements[i] = elem
buf = buf[size:]
}
return elements, nil
}
func safeUint32(val int) (uint32, error) {
if val < 0 || val > math.MaxUint32 {
return 0, errors.New("value exceeds uint32 range")
}
return uint32(val), nil // lint:ignore uintcast -- integer value explicitly checked to prevent truncation
}