Native beacon state: fieldtrie package (#10146)

* Native beacon state: `fieldtrie` package

* reuse ProofFromMerkleLayers

* more reuse + fixes

* more reuse + fixes

* move validateIndices to helpers

* remove TestAppendBeyondIndicesLimit

* do not duplicate package

* correct errors

* handle32ByteArrays function

* type switch

Co-authored-by: Raul Jordan <raul@prysmaticlabs.com>
Co-authored-by: Nishant Das <nishdas93@gmail.com>

beacon-chain/state/fieldtrie/BUILD.bazel

@@ -9,6 +9,7 @@ go_library(
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/state/fieldtrie",
     visibility = ["//beacon-chain:__subpackages__"],
     deps = [
+        "//beacon-chain/state/state-native/custom-types:go_default_library",
         "//beacon-chain/state/stateutil:go_default_library",
         "//beacon-chain/state/types:go_default_library",
         "//crypto/hash:go_default_library",

beacon-chain/state/fieldtrie/field_trie.go

@@ -55,7 +55,7 @@ func NewFieldTrie(field types.FieldIndex, dataType types.DataType, elements inte
 			reference:   stateutil.NewRef(1),
 			RWMutex:     new(sync.RWMutex),
 			length:      length,
-			numOfElems:  reflect.ValueOf(elements).Len(),
+			numOfElems:  reflect.Indirect(reflect.ValueOf(elements)).Len(),
 		}, nil
 	case types.CompositeArray, types.CompressedArray:
 		return &FieldTrie{
@@ -65,7 +65,7 @@ func NewFieldTrie(field types.FieldIndex, dataType types.DataType, elements inte
 			reference:   stateutil.NewRef(1),
 			RWMutex:     new(sync.RWMutex),
 			length:      length,
-			numOfElems:  reflect.ValueOf(elements).Len(),
+			numOfElems:  reflect.Indirect(reflect.ValueOf(elements)).Len(),
 		}, nil
 	default:
 		return nil, errors.Errorf("unrecognized data type in field map: %v", reflect.TypeOf(dataType).Name())

beacon-chain/state/fieldtrie/field_trie_helpers.go

@@ -6,6 +6,7 @@ import (
 	"reflect"
 
 	"github.com/pkg/errors"
+	customtypes "github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/custom-types"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/types"
 	"github.com/prysmaticlabs/prysm/crypto/hash"
@@ -55,7 +56,7 @@ func validateElements(field types.FieldIndex, dataType types.DataType, elements
 		}
 		length *= comLength
 	}
-	val := reflect.ValueOf(elements)
+	val := reflect.Indirect(reflect.ValueOf(elements))
 	if val.Len() > int(length) {
 		return errors.Errorf("elements length is larger than expected for field %s: %d > %d", field.String(version.Phase0), val.Len(), length)
 	}
@@ -65,13 +66,33 @@ func validateElements(field types.FieldIndex, dataType types.DataType, elements
 // fieldConverters converts the corresponding field and the provided elements to the appropriate roots.
 func fieldConverters(field types.FieldIndex, indices []uint64, elements interface{}, convertAll bool) ([][32]byte, error) {
 	switch field {
-	case types.BlockRoots, types.StateRoots, types.RandaoMixes:
-		val, ok := elements.([][]byte)
-		if !ok {
-			return nil, errors.Errorf("Wanted type of %v but got %v",
-				reflect.TypeOf([][]byte{}).Name(), reflect.TypeOf(elements).Name())
+	case types.BlockRoots:
+		switch val := elements.(type) {
+		case [][]byte:
+			return handleByteArrays(val, indices, convertAll)
+		case *customtypes.BlockRoots:
+			return handle32ByteArrays(val[:], indices, convertAll)
+		default:
+			return nil, errors.Errorf("Incorrect type used for block roots")
+		}
+	case types.StateRoots:
+		switch val := elements.(type) {
+		case [][]byte:
+			return handleByteArrays(val, indices, convertAll)
+		case *customtypes.StateRoots:
+			return handle32ByteArrays(val[:], indices, convertAll)
+		default:
+			return nil, errors.Errorf("Incorrect type used for state roots")
+		}
+	case types.RandaoMixes:
+		switch val := elements.(type) {
+		case [][]byte:
+			return handleByteArrays(val, indices, convertAll)
+		case *customtypes.RandaoMixes:
+			return handle32ByteArrays(val[:], indices, convertAll)
+		default:
+			return nil, errors.Errorf("Incorrect type used for randao mixes")
 		}
-		return handleByteArrays(val, indices, convertAll)
 	case types.Eth1DataVotes:
 		val, ok := elements.([]*ethpb.Eth1Data)
 		if !ok {
@@ -92,7 +113,7 @@ func fieldConverters(field types.FieldIndex, indices []uint64, elements interfac
 			return nil, errors.Errorf("Wanted type of %v but got %v",
 				reflect.TypeOf([]*ethpb.PendingAttestation{}).Name(), reflect.TypeOf(elements).Name())
 		}
-		return handlePendingAttestation(val, indices, convertAll)
+		return handlePendingAttestationSlice(val, indices, convertAll)
 	case types.Balances:
 		val, ok := elements.([]uint64)
 		if !ok {
@@ -133,6 +154,33 @@ func handleByteArrays(val [][]byte, indices []uint64, convertAll bool) ([][32]by
 	return roots, nil
 }
 
+// handle32ByteArrays computes and returns 32 byte arrays in a slice of root format.
+func handle32ByteArrays(val [][32]byte, indices []uint64, convertAll bool) ([][32]byte, error) {
+	length := len(indices)
+	if convertAll {
+		length = len(val)
+	}
+	roots := make([][32]byte, 0, length)
+	rootCreator := func(input [32]byte) {
+		roots = append(roots, input)
+	}
+	if convertAll {
+		for i := range val {
+			rootCreator(val[i])
+		}
+		return roots, nil
+	}
+	if len(val) > 0 {
+		for _, idx := range indices {
+			if idx > uint64(len(val))-1 {
+				return nil, fmt.Errorf("index %d greater than number of byte arrays %d", idx, len(val))
+			}
+			rootCreator(val[idx])
+		}
+	}
+	return roots, nil
+}
+
 // handleValidatorSlice returns the validator indices in a slice of root format.
 func handleValidatorSlice(val []*ethpb.Validator, indices []uint64, convertAll bool) ([][32]byte, error) {
 	length := len(indices)
@@ -211,7 +259,8 @@ func handleEth1DataSlice(val []*ethpb.Eth1Data, indices []uint64, convertAll boo
 	return roots, nil
 }
 
-func handlePendingAttestation(val []*ethpb.PendingAttestation, indices []uint64, convertAll bool) ([][32]byte, error) {
+// handlePendingAttestationSlice returns the root of a slice of pending attestations.
+func handlePendingAttestationSlice(val []*ethpb.PendingAttestation, indices []uint64, convertAll bool) ([][32]byte, error) {
 	length := len(indices)
 	if convertAll {
 		length = len(val)
@@ -249,6 +298,7 @@ func handlePendingAttestation(val []*ethpb.PendingAttestation, indices []uint64,
 	return roots, nil
 }
 
+// handleBalanceSlice returns the root of a slice of validator balances.
 func handleBalanceSlice(val, indices []uint64, convertAll bool) ([][32]byte, error) {
 	if convertAll {
 		balancesMarshaling := make([][]byte, 0)

beacon-chain/state/fieldtrie/helpers_test.go

@@ -15,7 +15,7 @@ import (
 func Test_handlePendingAttestation_OutOfRange(t *testing.T) {
 	items := make([]*ethpb.PendingAttestation, 1)
 	indices := []uint64{3}
-	_, err := handlePendingAttestation(items, indices, false)
+	_, err := handlePendingAttestationSlice(items, indices, false)
 	assert.ErrorContains(t, "index 3 greater than number of pending attestations 1", err)
 }
 

beacon-chain/state/state-native/fieldtrie/BUILD.bazel

@@ -1,40 +0,0 @@
-load("@prysm//tools/go:def.bzl", "go_library", "go_test")
-
-go_library(
-    name = "go_default_library",
-    srcs = [
-        "field_trie.go",
-        "field_trie_helpers.go",
-    ],
-    importpath = "github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/fieldtrie",
-    visibility = ["//beacon-chain:__subpackages__"],
-    deps = [
-        "//beacon-chain/state/stateutil:go_default_library",
-        "//beacon-chain/state/types:go_default_library",
-        "//crypto/hash:go_default_library",
-        "//encoding/bytesutil:go_default_library",
-        "//encoding/ssz:go_default_library",
-        "//proto/prysm/v1alpha1:go_default_library",
-        "//runtime/version:go_default_library",
-        "@com_github_pkg_errors//:go_default_library",
-    ],
-)
-
-go_test(
-    name = "go_default_test",
-    srcs = [
-        "field_trie_test.go",
-        "helpers_test.go",
-    ],
-    embed = [":go_default_library"],
-    deps = [
-        "//beacon-chain/state/stateutil:go_default_library",
-        "//beacon-chain/state/types:go_default_library",
-        "//config/params:go_default_library",
-        "//proto/prysm/v1alpha1:go_default_library",
-        "//testing/assert:go_default_library",
-        "//testing/require:go_default_library",
-        "//testing/util:go_default_library",
-        "@com_github_prysmaticlabs_eth2_types//:go_default_library",
-    ],
-)

beacon-chain/state/state-native/fieldtrie/field_trie.go

@@ -1,200 +0,0 @@
-package fieldtrie
-
-import (
-	"reflect"
-	"sync"
-
-	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/types"
-)
-
-// FieldTrie is the representation of the representative
-// trie of the particular field.
-type FieldTrie struct {
-	*sync.RWMutex
-	reference   *stateutil.Reference
-	fieldLayers [][]*[32]byte
-	field       types.FieldIndex
-	dataType    types.DataType
-	length      uint64
-	numOfElems  int
-}
-
-// NewFieldTrie is the constructor for the field trie data structure. It creates the corresponding
-// trie according to the given parameters. Depending on whether the field is a basic/composite array
-// which is either fixed/variable length, it will appropriately determine the trie.
-func NewFieldTrie(field types.FieldIndex, dataType types.DataType, elements interface{}, length uint64) (*FieldTrie, error) {
-	if elements == nil {
-		return &FieldTrie{
-			field:      field,
-			dataType:   dataType,
-			reference:  stateutil.NewRef(1),
-			RWMutex:    new(sync.RWMutex),
-			length:     length,
-			numOfElems: 0,
-		}, nil
-	}
-	fieldRoots, err := fieldConverters(field, []uint64{}, elements, true)
-	if err != nil {
-		return nil, err
-	}
-	if err := validateElements(field, dataType, elements, length); err != nil {
-		return nil, err
-	}
-	switch dataType {
-	case types.BasicArray:
-		fl, err := stateutil.ReturnTrieLayer(fieldRoots, length)
-		if err != nil {
-			return nil, err
-		}
-		return &FieldTrie{
-			fieldLayers: fl,
-			field:       field,
-			dataType:    dataType,
-			reference:   stateutil.NewRef(1),
-			RWMutex:     new(sync.RWMutex),
-			length:      length,
-			numOfElems:  reflect.ValueOf(elements).Len(),
-		}, nil
-	case types.CompositeArray, types.CompressedArray:
-		return &FieldTrie{
-			fieldLayers: stateutil.ReturnTrieLayerVariable(fieldRoots, length),
-			field:       field,
-			dataType:    dataType,
-			reference:   stateutil.NewRef(1),
-			RWMutex:     new(sync.RWMutex),
-			length:      length,
-			numOfElems:  reflect.ValueOf(elements).Len(),
-		}, nil
-	default:
-		return nil, errors.Errorf("unrecognized data type in field map: %v", reflect.TypeOf(dataType).Name())
-	}
-
-}
-
-// RecomputeTrie rebuilds the affected branches in the trie according to the provided
-// changed indices and elements. This recomputes the trie according to the particular
-// field the trie is based on.
-func (f *FieldTrie) RecomputeTrie(indices []uint64, elements interface{}) ([32]byte, error) {
-	f.Lock()
-	defer f.Unlock()
-	var fieldRoot [32]byte
-	if len(indices) == 0 {
-		return f.TrieRoot()
-	}
-	fieldRoots, err := fieldConverters(f.field, indices, elements, false)
-	if err != nil {
-		return [32]byte{}, err
-	}
-	if err := f.validateIndices(indices); err != nil {
-		return [32]byte{}, err
-	}
-	switch f.dataType {
-	case types.BasicArray:
-		fieldRoot, f.fieldLayers, err = stateutil.RecomputeFromLayer(fieldRoots, indices, f.fieldLayers)
-		if err != nil {
-			return [32]byte{}, err
-		}
-		f.numOfElems = reflect.ValueOf(elements).Len()
-		return fieldRoot, nil
-	case types.CompositeArray:
-		fieldRoot, f.fieldLayers, err = stateutil.RecomputeFromLayerVariable(fieldRoots, indices, f.fieldLayers)
-		if err != nil {
-			return [32]byte{}, err
-		}
-		f.numOfElems = reflect.ValueOf(elements).Len()
-		return stateutil.AddInMixin(fieldRoot, uint64(len(f.fieldLayers[0])))
-	case types.CompressedArray:
-		numOfElems, err := f.field.ElemsInChunk()
-		if err != nil {
-			return [32]byte{}, err
-		}
-		// We remove the duplicates here in order to prevent
-		// duplicated insertions into the trie.
-		newIndices := []uint64{}
-		indexExists := make(map[uint64]bool)
-		newRoots := make([][32]byte, 0, len(fieldRoots)/int(numOfElems))
-		for i, idx := range indices {
-			startIdx := idx / numOfElems
-			if indexExists[startIdx] {
-				continue
-			}
-			newIndices = append(newIndices, startIdx)
-			indexExists[startIdx] = true
-			newRoots = append(newRoots, fieldRoots[i])
-		}
-		fieldRoot, f.fieldLayers, err = stateutil.RecomputeFromLayerVariable(newRoots, newIndices, f.fieldLayers)
-		if err != nil {
-			return [32]byte{}, err
-		}
-		f.numOfElems = reflect.ValueOf(elements).Len()
-		return stateutil.AddInMixin(fieldRoot, uint64(f.numOfElems))
-	default:
-		return [32]byte{}, errors.Errorf("unrecognized data type in field map: %v", reflect.TypeOf(f.dataType).Name())
-	}
-}
-
-// CopyTrie copies the references to the elements the trie
-// is built on.
-func (f *FieldTrie) CopyTrie() *FieldTrie {
-	if f.fieldLayers == nil {
-		return &FieldTrie{
-			field:      f.field,
-			dataType:   f.dataType,
-			reference:  stateutil.NewRef(1),
-			RWMutex:    new(sync.RWMutex),
-			length:     f.length,
-			numOfElems: f.numOfElems,
-		}
-	}
-	dstFieldTrie := make([][]*[32]byte, len(f.fieldLayers))
-	for i, layer := range f.fieldLayers {
-		dstFieldTrie[i] = make([]*[32]byte, len(layer))
-		copy(dstFieldTrie[i], layer)
-	}
-	return &FieldTrie{
-		fieldLayers: dstFieldTrie,
-		field:       f.field,
-		dataType:    f.dataType,
-		reference:   stateutil.NewRef(1),
-		RWMutex:     new(sync.RWMutex),
-		length:      f.length,
-		numOfElems:  f.numOfElems,
-	}
-}
-
-// TrieRoot returns the corresponding root of the trie.
-func (f *FieldTrie) TrieRoot() ([32]byte, error) {
-	switch f.dataType {
-	case types.BasicArray:
-		return *f.fieldLayers[len(f.fieldLayers)-1][0], nil
-	case types.CompositeArray:
-		trieRoot := *f.fieldLayers[len(f.fieldLayers)-1][0]
-		return stateutil.AddInMixin(trieRoot, uint64(len(f.fieldLayers[0])))
-	case types.CompressedArray:
-		trieRoot := *f.fieldLayers[len(f.fieldLayers)-1][0]
-		return stateutil.AddInMixin(trieRoot, uint64(f.numOfElems))
-	default:
-		return [32]byte{}, errors.Errorf("unrecognized data type in field map: %v", reflect.TypeOf(f.dataType).Name())
-	}
-}
-
-// FieldReference returns the underlying field reference
-// object for the trie.
-func (f *FieldTrie) FieldReference() *stateutil.Reference {
-	return f.reference
-}
-
-// Empty checks whether the underlying field trie is
-// empty or not.
-func (f *FieldTrie) Empty() bool {
-	return f == nil || len(f.fieldLayers) == 0
-}
-
-// InsertFieldLayer manually inserts a field layer. This method
-// bypasses the normal method of field computation, it is only
-// meant to be used in tests.
-func (f *FieldTrie) InsertFieldLayer(layer [][]*[32]byte) {
-	f.fieldLayers = layer
-}

beacon-chain/state/state-native/fieldtrie/field_trie_helpers.go

@@ -1,295 +0,0 @@
-package fieldtrie
-
-import (
-	"encoding/binary"
-	"fmt"
-	"reflect"
-
-	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/types"
-	"github.com/prysmaticlabs/prysm/crypto/hash"
-	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
-	"github.com/prysmaticlabs/prysm/encoding/ssz"
-	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
-	"github.com/prysmaticlabs/prysm/runtime/version"
-)
-
-// ProofFromMerkleLayers creates a proof starting at the leaf index of the state Merkle layers.
-func ProofFromMerkleLayers(layers [][][]byte, startingLeafIndex types.FieldIndex) [][]byte {
-	// The merkle tree structure looks as follows:
-	// [[r1, r2, r3, r4], [parent1, parent2], [root]]
-	proof := make([][]byte, 0)
-	currentIndex := startingLeafIndex
-	for i := 0; i < len(layers)-1; i++ {
-		neighborIdx := currentIndex ^ 1
-		neighbor := layers[i][neighborIdx]
-		proof = append(proof, neighbor)
-		currentIndex = currentIndex / 2
-	}
-	return proof
-}
-
-func (f *FieldTrie) validateIndices(idxs []uint64) error {
-	length := f.length
-	if f.dataType == types.CompressedArray {
-		comLength, err := f.field.ElemsInChunk()
-		if err != nil {
-			return err
-		}
-		length *= comLength
-	}
-	for _, idx := range idxs {
-		if idx >= length {
-			return errors.Errorf("invalid index for field %s: %d >= length %d", f.field.String(version.Phase0), idx, length)
-		}
-	}
-	return nil
-}
-
-func validateElements(field types.FieldIndex, dataType types.DataType, elements interface{}, length uint64) error {
-	if dataType == types.CompressedArray {
-		comLength, err := field.ElemsInChunk()
-		if err != nil {
-			return err
-		}
-		length *= comLength
-	}
-	val := reflect.ValueOf(elements)
-	if val.Len() > int(length) {
-		return errors.Errorf("elements length is larger than expected for field %s: %d > %d", field.String(version.Phase0), val.Len(), length)
-	}
-	return nil
-}
-
-// fieldConverters converts the corresponding field and the provided elements to the appropriate roots.
-func fieldConverters(field types.FieldIndex, indices []uint64, elements interface{}, convertAll bool) ([][32]byte, error) {
-	switch field {
-	case types.BlockRoots, types.StateRoots, types.RandaoMixes:
-		val, ok := elements.([][]byte)
-		if !ok {
-			return nil, errors.Errorf("Wanted type of %v but got %v",
-				reflect.TypeOf([][]byte{}).Name(), reflect.TypeOf(elements).Name())
-		}
-		return handleByteArrays(val, indices, convertAll)
-	case types.Eth1DataVotes:
-		val, ok := elements.([]*ethpb.Eth1Data)
-		if !ok {
-			return nil, errors.Errorf("Wanted type of %v but got %v",
-				reflect.TypeOf([]*ethpb.Eth1Data{}).Name(), reflect.TypeOf(elements).Name())
-		}
-		return handleEth1DataSlice(val, indices, convertAll)
-	case types.Validators:
-		val, ok := elements.([]*ethpb.Validator)
-		if !ok {
-			return nil, errors.Errorf("Wanted type of %v but got %v",
-				reflect.TypeOf([]*ethpb.Validator{}).Name(), reflect.TypeOf(elements).Name())
-		}
-		return handleValidatorSlice(val, indices, convertAll)
-	case types.PreviousEpochAttestations, types.CurrentEpochAttestations:
-		val, ok := elements.([]*ethpb.PendingAttestation)
-		if !ok {
-			return nil, errors.Errorf("Wanted type of %v but got %v",
-				reflect.TypeOf([]*ethpb.PendingAttestation{}).Name(), reflect.TypeOf(elements).Name())
-		}
-		return handlePendingAttestation(val, indices, convertAll)
-	case types.Balances:
-		val, ok := elements.([]uint64)
-		if !ok {
-			return nil, errors.Errorf("Wanted type of %v but got %v",
-				reflect.TypeOf([]uint64{}).Name(), reflect.TypeOf(elements).Name())
-		}
-		return handleBalanceSlice(val, indices, convertAll)
-	default:
-		return [][32]byte{}, errors.Errorf("got unsupported type of %v", reflect.TypeOf(elements).Name())
-	}
-}
-
-// handleByteArrays computes and returns byte arrays in a slice of root format.
-func handleByteArrays(val [][]byte, indices []uint64, convertAll bool) ([][32]byte, error) {
-	length := len(indices)
-	if convertAll {
-		length = len(val)
-	}
-	roots := make([][32]byte, 0, length)
-	rootCreator := func(input []byte) {
-		newRoot := bytesutil.ToBytes32(input)
-		roots = append(roots, newRoot)
-	}
-	if convertAll {
-		for i := range val {
-			rootCreator(val[i])
-		}
-		return roots, nil
-	}
-	if len(val) > 0 {
-		for _, idx := range indices {
-			if idx > uint64(len(val))-1 {
-				return nil, fmt.Errorf("index %d greater than number of byte arrays %d", idx, len(val))
-			}
-			rootCreator(val[idx])
-		}
-	}
-	return roots, nil
-}
-
-// handleValidatorSlice returns the validator indices in a slice of root format.
-func handleValidatorSlice(val []*ethpb.Validator, indices []uint64, convertAll bool) ([][32]byte, error) {
-	length := len(indices)
-	if convertAll {
-		length = len(val)
-	}
-	roots := make([][32]byte, 0, length)
-	hasher := hash.CustomSHA256Hasher()
-	rootCreator := func(input *ethpb.Validator) error {
-		newRoot, err := stateutil.ValidatorRootWithHasher(hasher, input)
-		if err != nil {
-			return err
-		}
-		roots = append(roots, newRoot)
-		return nil
-	}
-	if convertAll {
-		for i := range val {
-			err := rootCreator(val[i])
-			if err != nil {
-				return nil, err
-			}
-		}
-		return roots, nil
-	}
-	if len(val) > 0 {
-		for _, idx := range indices {
-			if idx > uint64(len(val))-1 {
-				return nil, fmt.Errorf("index %d greater than number of validators %d", idx, len(val))
-			}
-			err := rootCreator(val[idx])
-			if err != nil {
-				return nil, err
-			}
-		}
-	}
-	return roots, nil
-}
-
-// handleEth1DataSlice processes a list of eth1data and indices into the appropriate roots.
-func handleEth1DataSlice(val []*ethpb.Eth1Data, indices []uint64, convertAll bool) ([][32]byte, error) {
-	length := len(indices)
-	if convertAll {
-		length = len(val)
-	}
-	roots := make([][32]byte, 0, length)
-	hasher := hash.CustomSHA256Hasher()
-	rootCreator := func(input *ethpb.Eth1Data) error {
-		newRoot, err := stateutil.Eth1DataRootWithHasher(hasher, input)
-		if err != nil {
-			return err
-		}
-		roots = append(roots, newRoot)
-		return nil
-	}
-	if convertAll {
-		for i := range val {
-			err := rootCreator(val[i])
-			if err != nil {
-				return nil, err
-			}
-		}
-		return roots, nil
-	}
-	if len(val) > 0 {
-		for _, idx := range indices {
-			if idx > uint64(len(val))-1 {
-				return nil, fmt.Errorf("index %d greater than number of items in eth1 data slice %d", idx, len(val))
-			}
-			err := rootCreator(val[idx])
-			if err != nil {
-				return nil, err
-			}
-		}
-	}
-	return roots, nil
-}
-
-func handlePendingAttestation(val []*ethpb.PendingAttestation, indices []uint64, convertAll bool) ([][32]byte, error) {
-	length := len(indices)
-	if convertAll {
-		length = len(val)
-	}
-	roots := make([][32]byte, 0, length)
-	hasher := hash.CustomSHA256Hasher()
-	rootCreator := func(input *ethpb.PendingAttestation) error {
-		newRoot, err := stateutil.PendingAttRootWithHasher(hasher, input)
-		if err != nil {
-			return err
-		}
-		roots = append(roots, newRoot)
-		return nil
-	}
-	if convertAll {
-		for i := range val {
-			err := rootCreator(val[i])
-			if err != nil {
-				return nil, err
-			}
-		}
-		return roots, nil
-	}
-	if len(val) > 0 {
-		for _, idx := range indices {
-			if idx > uint64(len(val))-1 {
-				return nil, fmt.Errorf("index %d greater than number of pending attestations %d", idx, len(val))
-			}
-			err := rootCreator(val[idx])
-			if err != nil {
-				return nil, err
-			}
-		}
-	}
-	return roots, nil
-}
-
-func handleBalanceSlice(val, indices []uint64, convertAll bool) ([][32]byte, error) {
-	if convertAll {
-		balancesMarshaling := make([][]byte, 0)
-		for _, b := range val {
-			balanceBuf := make([]byte, 8)
-			binary.LittleEndian.PutUint64(balanceBuf, b)
-			balancesMarshaling = append(balancesMarshaling, balanceBuf)
-		}
-		balancesChunks, err := ssz.PackByChunk(balancesMarshaling)
-		if err != nil {
-			return [][32]byte{}, errors.Wrap(err, "could not pack balances into chunks")
-		}
-		return balancesChunks, nil
-	}
-	if len(val) > 0 {
-		numOfElems, err := types.Balances.ElemsInChunk()
-		if err != nil {
-			return nil, err
-		}
-		roots := [][32]byte{}
-		for _, idx := range indices {
-			// We split the indexes into their relevant groups. Balances
-			// are compressed according to 4 values -> 1 chunk.
-			startIdx := idx / numOfElems
-			startGroup := startIdx * numOfElems
-			chunk := [32]byte{}
-			sizeOfElem := len(chunk) / int(numOfElems)
-			for i, j := 0, startGroup; j < startGroup+numOfElems; i, j = i+sizeOfElem, j+1 {
-				wantedVal := uint64(0)
-				// We are adding chunks in sets of 4, if the set is at the edge of the array
-				// then you will need to zero out the rest of the chunk. Ex : 41 indexes,
-				// so 41 % 4 = 1 . There are 3 indexes, which do not exist yet but we
-				// have to add in as a root. These 3 indexes are then given a 'zero' value.
-				if int(j) < len(val) {
-					wantedVal = val[j]
-				}
-				binary.LittleEndian.PutUint64(chunk[i:i+sizeOfElem], wantedVal)
-			}
-			roots = append(roots, chunk)
-		}
-		return roots, nil
-	}
-	return [][32]byte{}, nil
-}

beacon-chain/state/state-native/fieldtrie/field_trie_test.go

@@ -1,79 +0,0 @@
-package fieldtrie_test
-
-import (
-	"testing"
-
-	types "github.com/prysmaticlabs/eth2-types"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/fieldtrie"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
-	stateTypes "github.com/prysmaticlabs/prysm/beacon-chain/state/types"
-	"github.com/prysmaticlabs/prysm/config/params"
-	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
-	"github.com/prysmaticlabs/prysm/testing/assert"
-	"github.com/prysmaticlabs/prysm/testing/require"
-	"github.com/prysmaticlabs/prysm/testing/util"
-)
-
-func TestFieldTrie_NewTrie(t *testing.T) {
-	newState, _ := util.DeterministicGenesisState(t, 40)
-
-	// 5 represents the enum value of state roots
-	trie, err := fieldtrie.NewFieldTrie(5, stateTypes.BasicArray, newState.StateRoots(), uint64(params.BeaconConfig().SlotsPerHistoricalRoot))
-	require.NoError(t, err)
-	root, err := stateutil.RootsArrayHashTreeRoot(newState.StateRoots(), uint64(params.BeaconConfig().SlotsPerHistoricalRoot), "StateRoots")
-	require.NoError(t, err)
-	newRoot, err := trie.TrieRoot()
-	require.NoError(t, err)
-	assert.Equal(t, root, newRoot)
-}
-
-func TestFieldTrie_RecomputeTrie(t *testing.T) {
-	newState, _ := util.DeterministicGenesisState(t, 32)
-	// 10 represents the enum value of validators
-	trie, err := fieldtrie.NewFieldTrie(11, stateTypes.CompositeArray, newState.Validators(), params.BeaconConfig().ValidatorRegistryLimit)
-	require.NoError(t, err)
-
-	changedIdx := []uint64{2, 29}
-	val1, err := newState.ValidatorAtIndex(10)
-	require.NoError(t, err)
-	val2, err := newState.ValidatorAtIndex(11)
-	require.NoError(t, err)
-	val1.Slashed = true
-	val1.ExitEpoch = 20
-
-	val2.Slashed = true
-	val2.ExitEpoch = 40
-
-	changedVals := []*ethpb.Validator{val1, val2}
-	require.NoError(t, newState.UpdateValidatorAtIndex(types.ValidatorIndex(changedIdx[0]), changedVals[0]))
-	require.NoError(t, newState.UpdateValidatorAtIndex(types.ValidatorIndex(changedIdx[1]), changedVals[1]))
-
-	expectedRoot, err := stateutil.ValidatorRegistryRoot(newState.Validators())
-	require.NoError(t, err)
-	root, err := trie.RecomputeTrie(changedIdx, newState.Validators())
-	require.NoError(t, err)
-	assert.Equal(t, expectedRoot, root)
-}
-
-func TestFieldTrie_CopyTrieImmutable(t *testing.T) {
-	newState, _ := util.DeterministicGenesisState(t, 32)
-	// 12 represents the enum value of randao mixes.
-	trie, err := fieldtrie.NewFieldTrie(13, stateTypes.BasicArray, newState.RandaoMixes(), uint64(params.BeaconConfig().EpochsPerHistoricalVector))
-	require.NoError(t, err)
-
-	newTrie := trie.CopyTrie()
-
-	changedIdx := []uint64{2, 29}
-
-	changedVals := [][32]byte{{'A', 'B'}, {'C', 'D'}}
-	require.NoError(t, newState.UpdateRandaoMixesAtIndex(changedIdx[0], changedVals[0][:]))
-	require.NoError(t, newState.UpdateRandaoMixesAtIndex(changedIdx[1], changedVals[1][:]))
-
-	root, err := trie.RecomputeTrie(changedIdx, newState.RandaoMixes())
-	require.NoError(t, err)
-	newRoot, err := newTrie.TrieRoot()
-	require.NoError(t, err)
-	if root == newRoot {
-		t.Errorf("Wanted roots to be different, but they are the same: %#x", root)
-	}
-}

beacon-chain/state/state-native/fieldtrie/helpers_test.go

@@ -1,85 +0,0 @@
-package fieldtrie
-
-import (
-	"encoding/binary"
-	"sync"
-	"testing"
-
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/types"
-	"github.com/prysmaticlabs/prysm/config/params"
-	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
-	"github.com/prysmaticlabs/prysm/testing/assert"
-)
-
-func Test_handlePendingAttestation_OutOfRange(t *testing.T) {
-	items := make([]*ethpb.PendingAttestation, 1)
-	indices := []uint64{3}
-	_, err := handlePendingAttestation(items, indices, false)
-	assert.ErrorContains(t, "index 3 greater than number of pending attestations 1", err)
-}
-
-func Test_handleEth1DataSlice_OutOfRange(t *testing.T) {
-	items := make([]*ethpb.Eth1Data, 1)
-	indices := []uint64{3}
-	_, err := handleEth1DataSlice(items, indices, false)
-	assert.ErrorContains(t, "index 3 greater than number of items in eth1 data slice 1", err)
-
-}
-
-func Test_handleValidatorSlice_OutOfRange(t *testing.T) {
-	vals := make([]*ethpb.Validator, 1)
-	indices := []uint64{3}
-	_, err := handleValidatorSlice(vals, indices, false)
-	assert.ErrorContains(t, "index 3 greater than number of validators 1", err)
-}
-
-func TestBalancesSlice_CorrectRoots_All(t *testing.T) {
-	balances := []uint64{5, 2929, 34, 1291, 354305}
-	roots, err := handleBalanceSlice(balances, []uint64{}, true)
-	assert.NoError(t, err)
-
-	root1 := [32]byte{}
-	binary.LittleEndian.PutUint64(root1[:8], balances[0])
-	binary.LittleEndian.PutUint64(root1[8:16], balances[1])
-	binary.LittleEndian.PutUint64(root1[16:24], balances[2])
-	binary.LittleEndian.PutUint64(root1[24:32], balances[3])
-
-	root2 := [32]byte{}
-	binary.LittleEndian.PutUint64(root2[:8], balances[4])
-
-	assert.DeepEqual(t, roots, [][32]byte{root1, root2})
-}
-
-func TestBalancesSlice_CorrectRoots_Some(t *testing.T) {
-	balances := []uint64{5, 2929, 34, 1291, 354305}
-	roots, err := handleBalanceSlice(balances, []uint64{2, 3}, false)
-	assert.NoError(t, err)
-
-	root1 := [32]byte{}
-	binary.LittleEndian.PutUint64(root1[:8], balances[0])
-	binary.LittleEndian.PutUint64(root1[8:16], balances[1])
-	binary.LittleEndian.PutUint64(root1[16:24], balances[2])
-	binary.LittleEndian.PutUint64(root1[24:32], balances[3])
-
-	// Returns root for each indice(even if duplicated)
-	assert.DeepEqual(t, roots, [][32]byte{root1, root1})
-}
-
-func TestValidateIndices_CompressedField(t *testing.T) {
-	fakeTrie := &FieldTrie{
-		RWMutex:     new(sync.RWMutex),
-		reference:   stateutil.NewRef(0),
-		fieldLayers: nil,
-		field:       types.Balances,
-		dataType:    types.CompressedArray,
-		length:      params.BeaconConfig().ValidatorRegistryLimit / 4,
-		numOfElems:  0,
-	}
-	goodIdx := params.BeaconConfig().ValidatorRegistryLimit - 1
-	assert.NoError(t, fakeTrie.validateIndices([]uint64{goodIdx}))
-
-	badIdx := goodIdx + 1
-	assert.ErrorContains(t, "invalid index for field balances", fakeTrie.validateIndices([]uint64{badIdx}))
-
-}

beacon-chain/state/state-native/v1/BUILD.bazel

@@ -52,8 +52,8 @@ go_library(
     ],
     deps = [
         "//beacon-chain/state:go_default_library",
+        "//beacon-chain/state/fieldtrie:go_default_library",
         "//beacon-chain/state/state-native/custom-types:go_default_library",
-        "//beacon-chain/state/state-native/fieldtrie:go_default_library",
         "//beacon-chain/state/stateutil:go_default_library",
         "//beacon-chain/state/types:go_default_library",
         "//config/features:go_default_library",

beacon-chain/state/state-native/v1/beacon_state_mainnet.go

@@ -7,8 +7,8 @@ import (
 
 	eth2types "github.com/prysmaticlabs/eth2-types"
 	"github.com/prysmaticlabs/go-bitfield"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/fieldtrie"
 	customtypes "github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/custom-types"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/fieldtrie"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/types"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"

beacon-chain/state/state-native/v1/beacon_state_minimal.go

@@ -7,8 +7,8 @@ import (
 
 	eth2types "github.com/prysmaticlabs/eth2-types"
 	"github.com/prysmaticlabs/go-bitfield"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/fieldtrie"
 	customtypes "github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/custom-types"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/fieldtrie"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/types"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"

beacon-chain/state/state-native/v1/proofs.go

@@ -5,7 +5,7 @@ import (
 	"encoding/binary"
 
 	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/fieldtrie"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/fieldtrie"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
 )
 

beacon-chain/state/state-native/v1/state_trie.go

@@ -8,8 +8,8 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/fieldtrie"
 	customtypes "github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/custom-types"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/fieldtrie"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/types"
 	"github.com/prysmaticlabs/prysm/config/features"

beacon-chain/state/state-native/v2/BUILD.bazel

@@ -43,8 +43,8 @@ go_library(
     ],
     deps = [
         "//beacon-chain/state:go_default_library",
+        "//beacon-chain/state/fieldtrie:go_default_library",
         "//beacon-chain/state/state-native/custom-types:go_default_library",
-        "//beacon-chain/state/state-native/fieldtrie:go_default_library",
         "//beacon-chain/state/state-native/v1:go_default_library",
         "//beacon-chain/state/stateutil:go_default_library",
         "//beacon-chain/state/types:go_default_library",

beacon-chain/state/state-native/v2/beacon_state_mainnet.go

@@ -7,8 +7,8 @@ import (
 
 	eth2types "github.com/prysmaticlabs/eth2-types"
 	"github.com/prysmaticlabs/go-bitfield"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/fieldtrie"
 	customtypes "github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/custom-types"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/fieldtrie"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/types"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"

beacon-chain/state/state-native/v2/beacon_state_minimal.go

@@ -7,8 +7,8 @@ import (
 
 	eth2types "github.com/prysmaticlabs/eth2-types"
 	"github.com/prysmaticlabs/go-bitfield"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/fieldtrie"
 	customtypes "github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/custom-types"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/fieldtrie"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/types"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"

beacon-chain/state/state-native/v2/proofs.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"encoding/binary"
 
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/fieldtrie"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/fieldtrie"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
 )
 

beacon-chain/state/state-native/v2/state_trie.go

@@ -9,8 +9,8 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/fieldtrie"
 	customtypes "github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/custom-types"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/fieldtrie"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/types"
 	"github.com/prysmaticlabs/prysm/config/features"

beacon-chain/state/state-native/v3/BUILD.bazel

@@ -44,8 +44,8 @@ go_library(
     ],
     deps = [
         "//beacon-chain/state:go_default_library",
+        "//beacon-chain/state/fieldtrie:go_default_library",
         "//beacon-chain/state/state-native/custom-types:go_default_library",
-        "//beacon-chain/state/state-native/fieldtrie:go_default_library",
         "//beacon-chain/state/state-native/v1:go_default_library",
         "//beacon-chain/state/stateutil:go_default_library",
         "//beacon-chain/state/types:go_default_library",

beacon-chain/state/state-native/v3/beacon_state_mainnet.go

@@ -7,8 +7,8 @@ import (
 
 	eth2types "github.com/prysmaticlabs/eth2-types"
 	"github.com/prysmaticlabs/go-bitfield"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/fieldtrie"
 	customtypes "github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/custom-types"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/fieldtrie"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/types"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"

beacon-chain/state/state-native/v3/beacon_state_minimal.go

@@ -7,8 +7,8 @@ import (
 
 	eth2types "github.com/prysmaticlabs/eth2-types"
 	"github.com/prysmaticlabs/go-bitfield"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/fieldtrie"
 	customtypes "github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/custom-types"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/fieldtrie"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/types"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"

beacon-chain/state/state-native/v3/proofs.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"encoding/binary"
 
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/fieldtrie"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/fieldtrie"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
 )
 

beacon-chain/state/state-native/v3/state_trie.go

@@ -8,8 +8,8 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/fieldtrie"
 	customtypes "github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/custom-types"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/fieldtrie"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/types"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"