Merge branch 'develop' of https://github.com/prysmaticlabs/geth-sharding into copyOnWrite

beacon-chain/state/fieldtrie/field_trie.go

@@ -56,7 +56,7 @@ func NewFieldTrie(field types.FieldIndex, dataType types.DataType, elements inte
 			reference:   stateutil.NewRef(1),
 			RWMutex:     new(sync.RWMutex),
 			length:      length,
-			numOfElems:  reflect.Indirect(reflect.ValueOf(elements)).Len(),
+			numOfElems:  retrieveLength(elements),
 		}, nil
 	case types.CompositeArray, types.CompressedArray:
 		return &FieldTrie{
@@ -66,7 +66,7 @@ func NewFieldTrie(field types.FieldIndex, dataType types.DataType, elements inte
 			reference:   stateutil.NewRef(1),
 			RWMutex:     new(sync.RWMutex),
 			length:      length,
-			numOfElems:  reflect.Indirect(reflect.ValueOf(elements)).Len(),
+			numOfElems:  retrieveLength(elements),
 		}, nil
 	default:
 		return nil, errors.Errorf("unrecognized data type in field map: %v", reflect.TypeOf(dataType).Name())
@@ -97,14 +97,14 @@ func (f *FieldTrie) RecomputeTrie(indices []uint64, elements interface{}) ([32]b
 		if err != nil {
 			return [32]byte{}, err
 		}
-		f.numOfElems = reflect.Indirect(reflect.ValueOf(elements)).Len()
+		f.numOfElems = retrieveLength(elements)
 		return fieldRoot, nil
 	case types.CompositeArray:
 		fieldRoot, f.fieldLayers, err = stateutil.RecomputeFromLayerVariable(fieldRoots, indices, f.fieldLayers)
 		if err != nil {
 			return [32]byte{}, err
 		}
-		f.numOfElems = reflect.Indirect(reflect.ValueOf(elements)).Len()
+		f.numOfElems = retrieveLength(elements)
 		return stateutil.AddInMixin(fieldRoot, uint64(len(f.fieldLayers[0])))
 	case types.CompressedArray:
 		numOfElems, err := f.field.ElemsInChunk()
@@ -133,7 +133,7 @@ func (f *FieldTrie) RecomputeTrie(indices []uint64, elements interface{}) ([32]b
 		if err != nil {
 			return [32]byte{}, err
 		}
-		f.numOfElems = reflect.Indirect(reflect.ValueOf(elements)).Len()
+		f.numOfElems = retrieveLength(elements)
 		return stateutil.AddInMixin(fieldRoot, uint64(f.numOfElems))
 	default:
 		return [32]byte{}, errors.Errorf("unrecognized data type in field map: %v", reflect.TypeOf(f.dataType).Name())

beacon-chain/state/fieldtrie/field_trie_helpers.go

@@ -57,9 +57,11 @@ func validateElements(field types.FieldIndex, dataType types.DataType, elements
 		}
 		length *= comLength
 	}
-	val := reflect.Indirect(reflect.ValueOf(elements))
+	elemLen := retrieveLength(elements)
-	if uint64(val.Len()) > length {
+
-		return errors.Errorf("elements length is larger than expected for field %s: %d > %d", field.String(version.Phase0), val.Len(), length)
+	castedLen := int(length) // lint:ignore uintcast- ajhdjhd
+	if elemLen > castedLen {
+		return errors.Errorf("elements length is larger than expected for field %s: %d > %d", field.String(version.Phase0), elemLen, length)
 	}
 	return nil
 }
@@ -72,7 +74,7 @@ func fieldConverters(field types.FieldIndex, indices []uint64, elements interfac
 		case [][]byte:
 			return handleByteArrays(val, indices, convertAll)
 		case *customtypes.BlockRoots:
-			return handle32ByteArrays(val[:], indices, convertAll)
+			return handleIndexer(val, indices, convertAll)
 		default:
 			return nil, errors.Errorf("Incorrect type used for block roots")
 		}
@@ -90,7 +92,7 @@ func fieldConverters(field types.FieldIndex, indices []uint64, elements interfac
 		case [][]byte:
 			return handleByteArrays(val, indices, convertAll)
 		case *customtypes.RandaoMixes:
-			return handle32ByteArrays(val[:], indices, convertAll)
+			return handleIndexer(val, indices, convertAll)
 		default:
 			return nil, errors.Errorf("Incorrect type used for randao mixes")
 		}
@@ -182,6 +184,34 @@ func handle32ByteArrays(val [][32]byte, indices []uint64, convertAll bool) ([][3
 	return roots, nil
 }
 
+// handle32ByteArrays computes and returns 32 byte arrays in a slice of root format.
+func handleIndexer(indexer customtypes.Indexer, indices []uint64, convertAll bool) ([][32]byte, error) {
+	length := len(indices)
+	totalLength := indexer.TotalLength()
+	if convertAll {
+		length = int(totalLength) // lint:ignore uintcast- ajhdjhd
+	}
+	roots := make([][32]byte, 0, length)
+	rootCreator := func(input [32]byte) {
+		roots = append(roots, input)
+	}
+	if convertAll {
+		for i := uint64(0); i < uint64(length); i++ {
+			rootCreator(indexer.RootAtIndex(i))
+		}
+		return roots, nil
+	}
+	if totalLength > 0 {
+		for _, idx := range indices {
+			if idx > totalLength-1 {
+				return nil, fmt.Errorf("index %d greater than number of byte arrays %d", idx, totalLength)
+			}
+			rootCreator(indexer.RootAtIndex(idx))
+		}
+	}
+	return roots, nil
+}
+
 // handleValidatorSlice returns the validator indices in a slice of root format.
 func handleValidatorSlice(val []*ethpb.Validator, indices []uint64, convertAll bool) ([][32]byte, error) {
 	length := len(indices)
@@ -348,3 +378,17 @@ func handleBalanceSlice(val, indices []uint64, convertAll bool) ([][32]byte, err
 	}
 	return [][32]byte{}, nil
 }
+
+func retrieveLength(elements interface{}) int {
+	elemLen := int(0)
+	elemVal := reflect.ValueOf(elements)
+	if reflect.Indirect(elemVal).Kind() == reflect.Struct {
+		meth := elemVal.MethodByName("TotalLength")
+		ret := meth.Call([]reflect.Value{})
+		elemLen = int(ret[0].Uint()) // lint:ignore uintcast- ajhdjhd
+	} else {
+		val := reflect.Indirect(elemVal)
+		elemLen = val.Len()
+	}
+	return elemLen
+}

beacon-chain/state/state-native/custom-types/BUILD.bazel

@@ -12,6 +12,7 @@ go_library(
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/custom-types",
     visibility = ["//visibility:public"],
     deps = [
+        "//beacon-chain/state/stateutil:go_default_library",
         "//config/fieldparams:go_default_library",
         "@com_github_ferranbt_fastssz//:go_default_library",
     ],

beacon-chain/state/state-native/custom-types/block_roots.go

@@ -2,28 +2,162 @@ package customtypes
 
 import (
 	"fmt"
+	"reflect"
+	"runtime"
+	"sort"
+	"sync"
+	"unsafe"
 
 	fssz "github.com/ferranbt/fastssz"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 )
 
-var _ fssz.HashRoot = (BlockRoots)([fieldparams.BlockRootsLength][32]byte{})
+var _ fssz.HashRoot = (*BlockRoots)(nil)
 var _ fssz.Marshaler = (*BlockRoots)(nil)
 var _ fssz.Unmarshaler = (*BlockRoots)(nil)
 
+type Indexer interface {
+	RootAtIndex(idx uint64) [32]byte
+	TotalLength() uint64
+}
+
 // BlockRoots represents block roots of the beacon state.
-type BlockRoots [fieldparams.BlockRootsLength][32]byte
+type BlockRoots struct {
+	baseArray    *baseArrayBlockRoots
+	fieldJournal map[uint64][32]byte
+	generation   uint64
+	*stateutil.Reference
+}
+
+type baseArrayBlockRoots struct {
+	baseArray     *[fieldparams.BlockRootsLength][32]byte
+	descendantMap map[uint64][]uintptr
+	*sync.RWMutex
+	*stateutil.Reference
+}
+
+type sorter struct {
+	objs        [][]uintptr
+	generations []uint64
+}
+
+func (s sorter) Len() int {
+	return len(s.generations)
+}
+
+func (s sorter) Swap(i, j int) {
+	s.objs[i], s.objs[j] = s.objs[j], s.objs[i]
+	s.generations[i], s.generations[j] = s.generations[j], s.generations[i]
+}
+
+func (s sorter) Less(i, j int) bool {
+	return s.generations[i] < s.generations[j]
+}
+
+func (b *baseArrayBlockRoots) RootAtIndex(idx uint64) [32]byte {
+	b.RWMutex.RLock()
+	defer b.RWMutex.RUnlock()
+	return b.baseArray[idx]
+}
+
+func (b *baseArrayBlockRoots) TotalLength() uint64 {
+	return fieldparams.BlockRootsLength
+}
+
+func (b *baseArrayBlockRoots) addGeneration(generation uint64, descendant uintptr) {
+	b.RWMutex.Lock()
+	defer b.RWMutex.Unlock()
+	b.descendantMap[generation] = append(b.descendantMap[generation], descendant)
+}
+
+func (b *baseArrayBlockRoots) removeGeneration(generation uint64, descendant uintptr) {
+	b.RWMutex.Lock()
+	defer b.RWMutex.Unlock()
+	ptrVals := b.descendantMap[generation]
+	newVals := []uintptr{}
+	for _, v := range ptrVals {
+		if v == descendant {
+			continue
+		}
+		newVals = append(newVals, v)
+	}
+	b.descendantMap[generation] = newVals
+}
+
+func (b *baseArrayBlockRoots) numOfDescendants() uint64 {
+	b.RWMutex.RLock()
+	defer b.RWMutex.RUnlock()
+	return uint64(len(b.descendantMap))
+}
+
+func (b *baseArrayBlockRoots) cleanUp() {
+	b.RWMutex.Lock()
+	defer b.RWMutex.Unlock()
+	fmt.Printf("\n cleaning up block roots %d \n ", len(b.descendantMap))
+	listOfObjs := [][]uintptr{}
+	generations := []uint64{}
+	for g, objs := range b.descendantMap {
+		generations = append(generations, g)
+		listOfObjs = append(listOfObjs, objs)
+	}
+	sortedObj := sorter{
+		objs:        listOfObjs,
+		generations: generations,
+	}
+	sort.Sort(sortedObj)
+	lastReferencedGen := 0
+	lastRefrencedIdx := 0
+	lastRefPointer := 0
+	for i, g := range sortedObj.generations {
+		for j, o := range sortedObj.objs[i] {
+
+			x := (*BlockRoots)(unsafe.Pointer(o))
+			if x == nil {
+				continue
+			}
+
+			lastReferencedGen = int(g) // lint:ignore uintcast- ajhdjhd
+			lastRefrencedIdx = i
+			lastRefPointer = j
+			break
+		}
+		if lastReferencedGen != 0 {
+			break
+		}
+	}
+	fmt.Printf("\n block root map %d, %d, %d \n ", lastReferencedGen, lastRefrencedIdx, lastRefPointer)
+
+	br := (*BlockRoots)(unsafe.Pointer(sortedObj.objs[lastRefrencedIdx][lastRefPointer]))
+	for k, v := range br.fieldJournal {
+		b.baseArray[k] = v
+	}
+	sortedObj.generations = sortedObj.generations[lastRefrencedIdx:]
+	sortedObj.objs = sortedObj.objs[lastRefrencedIdx:]
+
+	newMap := make(map[uint64][]uintptr)
+	for i, g := range sortedObj.generations {
+		newMap[g] = sortedObj.objs[i]
+	}
+	b.descendantMap = newMap
+}
 
 // HashTreeRoot returns calculated hash root.
-func (r BlockRoots) HashTreeRoot() ([32]byte, error) {
+func (r *BlockRoots) HashTreeRoot() ([32]byte, error) {
 	return fssz.HashWithDefaultHasher(r)
 }
 
 // HashTreeRootWith hashes a BlockRoots object with a Hasher from the default HasherPool.
-func (r BlockRoots) HashTreeRootWith(hh *fssz.Hasher) error {
+func (r *BlockRoots) HashTreeRootWith(hh *fssz.Hasher) error {
 	index := hh.Index()
-	for _, sRoot := range r {
+
-		hh.Append(sRoot[:])
+	for i := uint64(0); i < r.baseArray.TotalLength(); i++ {
+		if val, ok := r.fieldJournal[i]; ok {
+			hh.Append(val[:])
+			continue
+		}
+		rt := r.baseArray.RootAtIndex(i)
+		hh.Append(rt[:])
 	}
 	hh.Merkleize(index)
 	return nil
@@ -34,12 +168,13 @@ func (r *BlockRoots) UnmarshalSSZ(buf []byte) error {
 	if len(buf) != r.SizeSSZ() {
 		return fmt.Errorf("expected buffer of length %d received %d", r.SizeSSZ(), len(buf))
 	}
+	r.baseArray.Lock()
+	defer r.baseArray.Unlock()
 
-	var roots BlockRoots
+	for i := range r.baseArray.baseArray {
-	for i := range roots {
+		copy(r.baseArray.baseArray[i][:], buf[i*32:(i+1)*32])
-		copy(roots[i][:], buf[i*32:(i+1)*32])
 	}
-	*r = roots
+
 	return nil
 }
 
@@ -55,10 +190,13 @@ func (r *BlockRoots) MarshalSSZTo(dst []byte) ([]byte, error) {
 // MarshalSSZ marshals BlockRoots into a serialized object.
 func (r *BlockRoots) MarshalSSZ() ([]byte, error) {
 	marshalled := make([]byte, fieldparams.BlockRootsLength*32)
-	for i, r32 := range r {
+	for i := uint64(0); i < r.baseArray.TotalLength(); i++ {
-		for j, rr := range r32 {
+		if val, ok := r.fieldJournal[i]; ok {
-			marshalled[i*32+j] = rr
+			copy(marshalled[i*32:], val[:])
+			continue
 		}
+		rt := r.baseArray.RootAtIndex(i)
+		copy(marshalled[i*32:], rt[:])
 	}
 	return marshalled, nil
 }
@@ -73,10 +211,152 @@ func (r *BlockRoots) Slice() [][]byte {
 	if r == nil {
 		return nil
 	}
-	bRoots := make([][]byte, len(r))
+	bRoots := make([][]byte, r.baseArray.TotalLength())
-	for i, root := range r {
+	for i := uint64(0); i < r.baseArray.TotalLength(); i++ {
-		tmp := root
+		if val, ok := r.fieldJournal[i]; ok {
-		bRoots[i] = tmp[:]
+			bRoots[i] = val[:]
+			continue
+		}
+		rt := r.baseArray.RootAtIndex(i)
+		bRoots[i] = rt[:]
 	}
 	return bRoots
 }
+
+// Slice converts a customtypes.BlockRoots object into a 2D byte slice.
+func (r *BlockRoots) Array() [fieldparams.BlockRootsLength][32]byte {
+	if r == nil {
+		return [fieldparams.BlockRootsLength][32]byte{}
+	}
+	bRoots := [fieldparams.BlockRootsLength][32]byte{}
+	for i := uint64(0); i < r.baseArray.TotalLength(); i++ {
+		if val, ok := r.fieldJournal[i]; ok {
+			bRoots[i] = val
+			continue
+		}
+		rt := r.baseArray.RootAtIndex(i)
+		bRoots[i] = rt
+	}
+	return bRoots
+}
+
+func SetFromSlice(slice [][]byte) *BlockRoots {
+	br := &BlockRoots{
+		baseArray: &baseArrayBlockRoots{
+			baseArray:     new([fieldparams.BlockRootsLength][32]byte),
+			descendantMap: map[uint64][]uintptr{},
+			RWMutex:       new(sync.RWMutex),
+			Reference:     stateutil.NewRef(1),
+		},
+		fieldJournal: map[uint64][32]byte{},
+		Reference:    stateutil.NewRef(1),
+	}
+	for i, rt := range slice {
+		copy(br.baseArray.baseArray[i][:], rt)
+	}
+	runtime.SetFinalizer(br, blockRootsFinalizer)
+	return br
+}
+
+func (r *BlockRoots) SetFromBaseField(field [fieldparams.BlockRootsLength][32]byte) {
+	r.baseArray = &baseArrayBlockRoots{
+		baseArray:     &field,
+		descendantMap: map[uint64][]uintptr{},
+		RWMutex:       new(sync.RWMutex),
+		Reference:     stateutil.NewRef(1),
+	}
+	r.fieldJournal = map[uint64][32]byte{}
+	r.Reference = stateutil.NewRef(1)
+	r.baseArray.addGeneration(0, reflect.ValueOf(r).Pointer())
+	runtime.SetFinalizer(r, blockRootsFinalizer)
+}
+
+func (r *BlockRoots) RootAtIndex(idx uint64) [32]byte {
+	if val, ok := r.fieldJournal[idx]; ok {
+		return val
+	}
+	return r.baseArray.RootAtIndex(idx)
+}
+
+func (r *BlockRoots) SetRootAtIndex(idx uint64, val [32]byte) {
+	if r.Refs() <= 1 && r.baseArray.Refs() <= 1 {
+		r.baseArray.Lock()
+		r.baseArray.baseArray[idx] = val
+		r.baseArray.Unlock()
+		return
+	}
+	if r.Refs() <= 1 {
+		r.fieldJournal[idx] = val
+		r.baseArray.removeGeneration(r.generation, reflect.ValueOf(r).Pointer())
+		r.generation++
+		r.baseArray.addGeneration(r.generation, reflect.ValueOf(r).Pointer())
+		return
+	}
+	newJournal := make(map[uint64][32]byte)
+	for k, val := range r.fieldJournal {
+		newJournal[k] = val
+	}
+
+	r.fieldJournal = newJournal
+	r.MinusRef()
+	r.Reference = stateutil.NewRef(1)
+	r.fieldJournal[idx] = val
+	r.baseArray.removeGeneration(r.generation, reflect.ValueOf(r).Pointer())
+	r.generation++
+	r.baseArray.addGeneration(r.generation, reflect.ValueOf(r).Pointer())
+}
+
+func (r *BlockRoots) Copy() *BlockRoots {
+	r.baseArray.AddRef()
+	r.Reference.AddRef()
+	br := &BlockRoots{
+		baseArray:    r.baseArray,
+		fieldJournal: r.fieldJournal,
+		Reference:    r.Reference,
+		generation:   r.generation,
+	}
+	r.baseArray.addGeneration(r.generation, reflect.ValueOf(br).Pointer())
+	if r.baseArray.numOfDescendants() > 20 {
+		r.baseArray.cleanUp()
+	}
+	runtime.SetFinalizer(br, blockRootsFinalizer)
+	return br
+}
+
+func (r *BlockRoots) TotalLength() uint64 {
+	return fieldparams.BlockRootsLength
+}
+
+func (r *BlockRoots) IncreaseRef() {
+	r.Reference.AddRef()
+	r.baseArray.Reference.AddRef()
+}
+
+func (r *BlockRoots) DecreaseRef() {
+	r.Reference.MinusRef()
+	r.baseArray.Reference.MinusRef()
+}
+
+func blockRootsFinalizer(br *BlockRoots) {
+	br.baseArray.Lock()
+	defer br.baseArray.Unlock()
+	ptrVal := reflect.ValueOf(br).Pointer()
+	vals, ok := br.baseArray.descendantMap[br.generation]
+	if !ok {
+		return
+	}
+	exists := false
+	wantedVals := []uintptr{}
+	for _, v := range vals {
+		if v == ptrVal {
+			exists = true
+			continue
+		}
+		newV := v
+		wantedVals = append(wantedVals, newV)
+	}
+	if !exists {
+		return
+	}
+	br.baseArray.descendantMap[br.generation] = wantedVals
+}

beacon-chain/state/state-native/custom-types/block_roots_test.go

@@ -1,24 +1,25 @@
 package customtypes
 
 import (
+	"bytes"
 	"reflect"
 	"testing"
 
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
-	"github.com/prysmaticlabs/prysm/testing/assert"
 )
 
 func TestBlockRoots_Casting(t *testing.T) {
 	var b [fieldparams.BlockRootsLength][32]byte
-	d := BlockRoots(b)
+	f := SetFromSlice([][]byte{})
-	if !reflect.DeepEqual([fieldparams.BlockRootsLength][32]byte(d), b) {
+	f.SetFromBaseField(b)
-		t.Errorf("Unequal: %v = %v", d, b)
+	if !reflect.DeepEqual(f.Array(), b) {
+		t.Errorf("Unequal: %v = %v", f.Array(), b)
 	}
 }
 
 func TestBlockRoots_UnmarshalSSZ(t *testing.T) {
 	t.Run("Ok", func(t *testing.T) {
-		d := BlockRoots{}
+		d := SetFromSlice([][]byte{})
 		var b [fieldparams.BlockRootsLength][32]byte
 		b[0] = [32]byte{'f', 'o', 'o'}
 		b[1] = [32]byte{'b', 'a', 'r'}
@@ -32,8 +33,8 @@ func TestBlockRoots_UnmarshalSSZ(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Unexpected error: %v", err)
 		}
-		if !reflect.DeepEqual(b, [fieldparams.BlockRootsLength][32]byte(d)) {
+		if !reflect.DeepEqual(b, d.Array()) {
-			t.Errorf("Unequal: %v = %v", b, [fieldparams.BlockRootsLength][32]byte(d))
+			t.Errorf("Unequal: %v = %v", b, d.Array())
 		}
 	})
 
@@ -70,28 +71,47 @@ func TestBlockRoots_MarshalSSZTo(t *testing.T) {
 }
 
 func TestBlockRoots_MarshalSSZ(t *testing.T) {
-	d := BlockRoots{}
+	d := SetFromSlice([][]byte{})
-	d[0] = [32]byte{'f', 'o', 'o'}
+	d.IncreaseRef()
-	d[1] = [32]byte{'b', 'a', 'r'}
+	d.SetRootAtIndex(0, [32]byte{'f', 'o', 'o'})
+	d.IncreaseRef()
+	d.IncreaseRef()
+	d.SetRootAtIndex(1, [32]byte{'b', 'a', 'r'})
 	b, err := d.MarshalSSZ()
 	if err != nil {
 		t.Fatalf("Unexpected error: %v", err)
 	}
-	if !reflect.DeepEqual(d[0][:], b[0:32]) {
+	rt := d.RootAtIndex(0)
-		t.Errorf("Unequal: %v = %v", d[0], b[0:32])
+	if !reflect.DeepEqual(rt[:], b[0:32]) {
+		t.Errorf("Unequal: %v = %v", rt, b[0:32])
 	}
-	if !reflect.DeepEqual(d[1][:], b[32:64]) {
+	rt = d.RootAtIndex(1)
-		t.Errorf("Unequal: %v = %v", d[0], b[32:64])
+	if !reflect.DeepEqual(rt[:], b[32:64]) {
+		t.Errorf("Unequal: %v = %v", rt, b[32:64])
+	}
+	d2 := SetFromSlice([][]byte{})
+	err = d2.UnmarshalSSZ(b)
+	if err != nil {
+		t.Error(err)
+	}
+	res, err := d2.MarshalSSZ()
+	if err != nil {
+		t.Error(err)
+	}
+	if !bytes.Equal(res, b) {
+		t.Error("unequal")
 	}
 }
 
 func TestBlockRoots_SizeSSZ(t *testing.T) {
-	d := BlockRoots{}
+	d := SetFromSlice([][]byte{})
 	if d.SizeSSZ() != fieldparams.BlockRootsLength*32 {
 		t.Errorf("Wrong SSZ size. Expected %v vs actual %v", fieldparams.BlockRootsLength*32, d.SizeSSZ())
 	}
 }
 
+/*
+
 func TestBlockRoots_Slice(t *testing.T) {
 	a, b, c := [32]byte{'a'}, [32]byte{'b'}, [32]byte{'c'}
 	roots := BlockRoots{}
@@ -103,3 +123,4 @@ func TestBlockRoots_Slice(t *testing.T) {
 	assert.DeepEqual(t, b[:], slice[10])
 	assert.DeepEqual(t, c[:], slice[100])
 }
+*/

beacon-chain/state/state-native/custom-types/randao_mixes.go

@@ -2,48 +2,77 @@ package customtypes
 
 import (
 	"fmt"
+	"sync"
 
 	fssz "github.com/ferranbt/fastssz"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 )
 
-var _ fssz.HashRoot = (RandaoMixes)([fieldparams.RandaoMixesLength][32]byte{})
+var _ fssz.HashRoot = (*RandaoMixes)(nil)
 var _ fssz.Marshaler = (*RandaoMixes)(nil)
 var _ fssz.Unmarshaler = (*RandaoMixes)(nil)
 
-// RandaoMixes represents RANDAO mixes of the beacon state.
+// BlockRoots represents block roots of the beacon state.
-type RandaoMixes [fieldparams.RandaoMixesLength][32]byte
+type RandaoMixes struct {
+	baseArray    *baseArrayRandaoMixes
+	fieldJournal map[uint64][32]byte
+	*stateutil.Reference
+}
+
+type baseArrayRandaoMixes struct {
+	baseArray *[fieldparams.RandaoMixesLength][32]byte
+	*sync.RWMutex
+	*stateutil.Reference
+}
+
+func (b *baseArrayRandaoMixes) RootAtIndex(idx uint64) [32]byte {
+	b.RWMutex.RLock()
+	defer b.RWMutex.RUnlock()
+	return b.baseArray[idx]
+}
+
+func (b *baseArrayRandaoMixes) TotalLength() uint64 {
+	return fieldparams.RandaoMixesLength
+}
 
 // HashTreeRoot returns calculated hash root.
-func (r RandaoMixes) HashTreeRoot() ([32]byte, error) {
+func (r *RandaoMixes) HashTreeRoot() ([32]byte, error) {
 	return fssz.HashWithDefaultHasher(r)
 }
 
-// HashTreeRootWith hashes a RandaoMixes object with a Hasher from the default HasherPool.
+// HashTreeRootWith hashes a BlockRoots object with a Hasher from the default HasherPool.
-func (r RandaoMixes) HashTreeRootWith(hh *fssz.Hasher) error {
+func (r *RandaoMixes) HashTreeRootWith(hh *fssz.Hasher) error {
 	index := hh.Index()
-	for _, sRoot := range r {
+
-		hh.Append(sRoot[:])
+	for i := uint64(0); i < r.baseArray.TotalLength(); i++ {
+		if val, ok := r.fieldJournal[i]; ok {
+			hh.Append(val[:])
+			continue
+		}
+		rt := r.baseArray.RootAtIndex(i)
+		hh.Append(rt[:])
 	}
 	hh.Merkleize(index)
 	return nil
 }
 
-// UnmarshalSSZ deserializes the provided bytes buffer into the RandaoMixes object.
+// UnmarshalSSZ deserializes the provided bytes buffer into the BlockRoots object.
 func (r *RandaoMixes) UnmarshalSSZ(buf []byte) error {
 	if len(buf) != r.SizeSSZ() {
 		return fmt.Errorf("expected buffer of length %d received %d", r.SizeSSZ(), len(buf))
 	}
+	r.baseArray.Lock()
+	defer r.baseArray.Unlock()
 
-	var roots RandaoMixes
+	for i := range r.baseArray.baseArray {
-	for i := range roots {
+		copy(r.baseArray.baseArray[i][:], buf[i*32:(i+1)*32])
-		copy(roots[i][:], buf[i*32:(i+1)*32])
 	}
-	*r = roots
+
 	return nil
 }
 
-// MarshalSSZTo marshals RandaoMixes with the provided byte slice.
+// MarshalSSZTo marshals BlockRoots with the provided byte slice.
 func (r *RandaoMixes) MarshalSSZTo(dst []byte) ([]byte, error) {
 	marshalled, err := r.MarshalSSZ()
 	if err != nil {
@@ -52,13 +81,16 @@ func (r *RandaoMixes) MarshalSSZTo(dst []byte) ([]byte, error) {
 	return append(dst, marshalled...), nil
 }
 
-// MarshalSSZ marshals RandaoMixes into a serialized object.
+// MarshalSSZ marshals BlockRoots into a serialized object.
 func (r *RandaoMixes) MarshalSSZ() ([]byte, error) {
 	marshalled := make([]byte, fieldparams.RandaoMixesLength*32)
-	for i, r32 := range r {
+	for i := uint64(0); i < r.baseArray.TotalLength(); i++ {
-		for j, rr := range r32 {
+		if val, ok := r.fieldJournal[i]; ok {
-			marshalled[i*32+j] = rr
+			copy(marshalled[i*32:], val[:])
+			continue
 		}
+		rt := r.baseArray.RootAtIndex(i)
+		copy(marshalled[i*32:], rt[:])
 	}
 	return marshalled, nil
 }
@@ -68,15 +100,90 @@ func (_ *RandaoMixes) SizeSSZ() int {
 	return fieldparams.RandaoMixesLength * 32
 }
 
-// Slice converts a customtypes.RandaoMixes object into a 2D byte slice.
+// Slice converts a customtypes.BlockRoots object into a 2D byte slice.
 func (r *RandaoMixes) Slice() [][]byte {
 	if r == nil {
 		return nil
 	}
-	mixes := make([][]byte, len(r))
+	bRoots := make([][]byte, r.baseArray.TotalLength())
-	for i, root := range r {
+	for i := uint64(0); i < r.baseArray.TotalLength(); i++ {
-		tmp := root
+		if val, ok := r.fieldJournal[i]; ok {
-		mixes[i] = tmp[:]
+			bRoots[i] = val[:]
+			continue
+		}
+		rt := r.baseArray.RootAtIndex(i)
+		bRoots[i] = rt[:]
 	}
-	return mixes
+	return bRoots
+}
+
+func SetFromSliceRandao(slice [][]byte) *RandaoMixes {
+	br := &RandaoMixes{
+		baseArray: &baseArrayRandaoMixes{
+			baseArray: new([fieldparams.RandaoMixesLength][32]byte),
+			RWMutex:   new(sync.RWMutex),
+			Reference: stateutil.NewRef(1),
+		},
+		fieldJournal: map[uint64][32]byte{},
+		Reference:    stateutil.NewRef(1),
+	}
+	for i, rt := range slice {
+		copy(br.baseArray.baseArray[i][:], rt)
+	}
+	return br
+}
+
+func (r *RandaoMixes) SetFromBaseField(field [fieldparams.RandaoMixesLength][32]byte) {
+	r.baseArray.baseArray = &field
+}
+
+func (r *RandaoMixes) RootAtIndex(idx uint64) [32]byte {
+	if val, ok := r.fieldJournal[idx]; ok {
+		return val
+	}
+	return r.baseArray.RootAtIndex(idx)
+}
+
+func (r *RandaoMixes) SetRootAtIndex(idx uint64, val [32]byte) {
+	if r.Refs() <= 1 && r.baseArray.Refs() <= 1 {
+		r.baseArray.baseArray[idx] = val
+		return
+	}
+	if r.Refs() <= 1 {
+		r.fieldJournal[idx] = val
+		return
+	}
+	newJournal := make(map[uint64][32]byte)
+	for k, val := range r.fieldJournal {
+		newJournal[k] = val
+	}
+	r.fieldJournal = newJournal
+	r.MinusRef()
+	r.Reference = stateutil.NewRef(1)
+	r.fieldJournal[idx] = val
+}
+
+func (r *RandaoMixes) Copy() *RandaoMixes {
+	r.baseArray.AddRef()
+	r.Reference.AddRef()
+	rm := &RandaoMixes{
+		baseArray:    r.baseArray,
+		fieldJournal: r.fieldJournal,
+		Reference:    r.Reference,
+	}
+	return rm
+}
+
+func (r *RandaoMixes) TotalLength() uint64 {
+	return fieldparams.RandaoMixesLength
+}
+
+func (r *RandaoMixes) IncreaseRef() {
+	r.Reference.AddRef()
+	r.baseArray.Reference.AddRef()
+}
+
+func (r *RandaoMixes) DecreaseRef() {
+	r.Reference.MinusRef()
+	r.baseArray.Reference.MinusRef()
 }

beacon-chain/state/state-native/v1/getters_block.go

@@ -76,8 +76,8 @@ func (b *BeaconState) BlockRootAtIndex(idx uint64) ([]byte, error) {
 // input index value.
 // This assumes that a lock is already held on BeaconState.
 func (b *BeaconState) blockRootAtIndex(idx uint64) ([32]byte, error) {
-	if uint64(len(b.blockRoots)) <= idx {
+	if b.blockRoots.TotalLength() <= idx {
 		return [32]byte{}, fmt.Errorf("index %d out of range", idx)
 	}
-	return b.blockRoots[idx], nil
+	return b.blockRoots.RootAtIndex(idx), nil
 }

beacon-chain/state/state-native/v1/getters_randao.go

@@ -37,10 +37,10 @@ func (b *BeaconState) RandaoMixAtIndex(idx uint64) ([]byte, error) {
 // input index value.
 // This assumes that a lock is already held on BeaconState.
 func (b *BeaconState) randaoMixAtIndex(idx uint64) ([32]byte, error) {
-	if uint64(len(b.randaoMixes)) <= idx {
+	if b.randaoMixes.TotalLength() <= idx {
 		return [32]byte{}, fmt.Errorf("index %d out of range", idx)
 	}
-	return b.randaoMixes[idx], nil
+	return b.randaoMixes.RootAtIndex(idx), nil
 }
 
 // RandaoMixesLength returns the length of the randao mixes slice.
@@ -62,5 +62,5 @@ func (b *BeaconState) randaoMixesLength() int {
 		return 0
 	}
 
-	return len(b.randaoMixes)
+	return int(b.randaoMixes.TotalLength()) // lint:ignore uintcast- ajhdjhd
 }

beacon-chain/state/state-native/v1/setters_block.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 
 	customtypes "github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/custom-types"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 )
@@ -25,14 +24,12 @@ func (b *BeaconState) SetBlockRoots(val [][]byte) error {
 	b.lock.Lock()
 	defer b.lock.Unlock()
 
-	b.sharedFieldReferences[blockRoots].MinusRef()
-	b.sharedFieldReferences[blockRoots] = stateutil.NewRef(1)
-
 	var rootsArr [fieldparams.BlockRootsLength][32]byte
 	for i := 0; i < len(rootsArr); i++ {
 		copy(rootsArr[i][:], val[i])
 	}
-	roots := customtypes.BlockRoots(rootsArr)
+	roots := customtypes.BlockRoots{}
+	roots.SetFromBaseField(rootsArr)
 	b.blockRoots = &roots
 	b.markFieldAsDirty(blockRoots)
 	b.rebuildTrie[blockRoots] = true
@@ -42,24 +39,13 @@ func (b *BeaconState) SetBlockRoots(val [][]byte) error {
 // UpdateBlockRootAtIndex for the beacon state. Updates the block root
 // at a specific index to a new value.
 func (b *BeaconState) UpdateBlockRootAtIndex(idx uint64, blockRoot [32]byte) error {
-	if uint64(len(b.blockRoots)) <= idx {
+	if b.blockRoots.TotalLength() <= idx {
 		return fmt.Errorf("invalid index provided %d", idx)
 	}
 	b.lock.Lock()
 	defer b.lock.Unlock()
 
-	r := b.blockRoots
+	b.blockRoots.SetRootAtIndex(idx, blockRoot)
-	if ref := b.sharedFieldReferences[blockRoots]; ref.Refs() > 1 {
-		// Copy elements in underlying array by reference.
-		roots := *b.blockRoots
-		rootsCopy := roots
-		r = &rootsCopy
-		ref.MinusRef()
-		b.sharedFieldReferences[blockRoots] = stateutil.NewRef(1)
-	}
-
-	r[idx] = blockRoot
-	b.blockRoots = r
 
 	b.markFieldAsDirty(blockRoots)
 	b.addDirtyIndices(blockRoots, []uint64{idx})

beacon-chain/state/state-native/v1/setters_randao.go

@@ -3,7 +3,6 @@ package v1
 import (
 	"github.com/pkg/errors"
 	customtypes "github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/custom-types"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
 )
@@ -14,14 +13,12 @@ func (b *BeaconState) SetRandaoMixes(val [][]byte) error {
 	b.lock.Lock()
 	defer b.lock.Unlock()
 
-	b.sharedFieldReferences[randaoMixes].MinusRef()
-	b.sharedFieldReferences[randaoMixes] = stateutil.NewRef(1)
-
 	var mixesArr [fieldparams.RandaoMixesLength][32]byte
 	for i := 0; i < len(mixesArr); i++ {
 		copy(mixesArr[i][:], val[i])
 	}
-	mixes := customtypes.RandaoMixes(mixesArr)
+	mixes := customtypes.RandaoMixes{}
+	mixes.SetFromBaseField(mixesArr)
 	b.randaoMixes = &mixes
 	b.markFieldAsDirty(randaoMixes)
 	b.rebuildTrie[randaoMixes] = true
@@ -31,24 +28,13 @@ func (b *BeaconState) SetRandaoMixes(val [][]byte) error {
 // UpdateRandaoMixesAtIndex for the beacon state. Updates the randao mixes
 // at a specific index to a new value.
 func (b *BeaconState) UpdateRandaoMixesAtIndex(idx uint64, val []byte) error {
-	if uint64(len(b.randaoMixes)) <= idx {
+	if b.randaoMixes.TotalLength() <= idx {
 		return errors.Errorf("invalid index provided %d", idx)
 	}
 	b.lock.Lock()
 	defer b.lock.Unlock()
 
-	mixes := b.randaoMixes
+	b.randaoMixes.SetRootAtIndex(idx, bytesutil.ToBytes32(val))
-	if refs := b.sharedFieldReferences[randaoMixes].Refs(); refs > 1 {
-		// Copy elements in underlying array by reference.
-		m := *b.randaoMixes
-		mCopy := m
-		mixes = &mCopy
-		b.sharedFieldReferences[randaoMixes].MinusRef()
-		b.sharedFieldReferences[randaoMixes] = stateutil.NewRef(1)
-	}
-
-	mixes[idx] = bytesutil.ToBytes32(val)
-	b.randaoMixes = mixes
 	b.markFieldAsDirty(randaoMixes)
 	b.addDirtyIndices(randaoMixes, []uint64{idx})
 

beacon-chain/state/state-native/v1/state_trie.go

@@ -36,10 +36,8 @@ func InitializeFromProtoUnsafe(st *ethpb.BeaconState) (state.BeaconState, error)
 		return nil, errors.New("received nil state")
 	}
 
-	var bRoots customtypes.BlockRoots
+	bRoots := customtypes.SetFromSlice(st.BlockRoots)
-	for i, r := range st.BlockRoots {
+
-		copy(bRoots[i][:], r)
-	}
 	var sRoots customtypes.StateRoots
 	for i, r := range st.StateRoots {
 		copy(sRoots[i][:], r)
@@ -48,10 +46,7 @@ func InitializeFromProtoUnsafe(st *ethpb.BeaconState) (state.BeaconState, error)
 	for i, r := range st.HistoricalRoots {
 		copy(hRoots[i][:], r)
 	}
-	var mixes customtypes.RandaoMixes
+	mixes := customtypes.SetFromSliceRandao(st.RandaoMixes)
-	for i, m := range st.RandaoMixes {
-		copy(mixes[i][:], m)
-	}
 
 	fieldCount := params.BeaconConfig().BeaconStateFieldCount
 	b := &BeaconState{
@@ -60,7 +55,7 @@ func InitializeFromProtoUnsafe(st *ethpb.BeaconState) (state.BeaconState, error)
 		slot:                        st.Slot,
 		fork:                        st.Fork,
 		latestBlockHeader:           st.LatestBlockHeader,
-		blockRoots:                  &bRoots,
+		blockRoots:                  bRoots,
 		stateRoots:                  &sRoots,
 		historicalRoots:             hRoots,
 		eth1Data:                    st.Eth1Data,
@@ -68,7 +63,7 @@ func InitializeFromProtoUnsafe(st *ethpb.BeaconState) (state.BeaconState, error)
 		eth1DepositIndex:            st.Eth1DepositIndex,
 		validators:                  st.Validators,
 		balances:                    st.Balances,
-		randaoMixes:                 &mixes,
+		randaoMixes:                 mixes,
 		slashings:                   st.Slashings,
 		previousEpochAttestations:   st.PreviousEpochAttestations,
 		currentEpochAttestations:    st.CurrentEpochAttestations,
@@ -99,7 +94,6 @@ func InitializeFromProtoUnsafe(st *ethpb.BeaconState) (state.BeaconState, error)
 	// Initialize field reference tracking for shared data.
 	b.sharedFieldReferences[randaoMixes] = stateutil.NewRef(1)
 	b.sharedFieldReferences[stateRoots] = stateutil.NewRef(1)
-	b.sharedFieldReferences[blockRoots] = stateutil.NewRef(1)
 	b.sharedFieldReferences[previousEpochAttestations] = stateutil.NewRef(1)
 	b.sharedFieldReferences[currentEpochAttestations] = stateutil.NewRef(1)
 	b.sharedFieldReferences[slashings] = stateutil.NewRef(1)
@@ -127,9 +121,9 @@ func (b *BeaconState) Copy() state.BeaconState {
 		slashings: b.slashings,
 
 		// Large arrays, infrequently changed, constant size.
-		blockRoots:                b.blockRoots,
+		blockRoots:                b.blockRoots.Copy(),
 		stateRoots:                b.stateRoots,
-		randaoMixes:               b.randaoMixes,
+		randaoMixes:               b.randaoMixes.Copy(),
 		previousEpochAttestations: b.previousEpochAttestations,
 		currentEpochAttestations:  b.currentEpochAttestations,
 		eth1DataVotes:             b.eth1DataVotes,
@@ -211,6 +205,9 @@ func (b *BeaconState) Copy() state.BeaconState {
 			}
 
 		}
+		b.blockRoots.MinusRef()
+		b.randaoMixes.MinusRef()
+
 		for i := 0; i < fieldCount; i++ {
 			field := types.FieldIndex(i)
 			delete(b.stateFieldLeaves, field)

beacon-chain/state/state-native/v2/getters_block.go

@@ -76,9 +76,8 @@ func (b *BeaconState) BlockRootAtIndex(idx uint64) ([]byte, error) {
 // input index value.
 // This assumes that a lock is already held on BeaconState.
 func (b *BeaconState) blockRootAtIndex(idx uint64) ([32]byte, error) {
-	if uint64(len(b.blockRoots)) <= idx {
+	if b.blockRoots.TotalLength() <= idx {
 		return [32]byte{}, fmt.Errorf("index %d out of range", idx)
 	}
-
+	return b.blockRoots.RootAtIndex(idx), nil
-	return b.blockRoots[idx], nil
 }

beacon-chain/state/state-native/v2/getters_randao.go

@@ -37,11 +37,10 @@ func (b *BeaconState) RandaoMixAtIndex(idx uint64) ([]byte, error) {
 // input index value.
 // This assumes that a lock is already held on BeaconState.
 func (b *BeaconState) randaoMixAtIndex(idx uint64) ([32]byte, error) {
-	if uint64(len(b.randaoMixes)) <= idx {
+	if b.randaoMixes.TotalLength() <= idx {
 		return [32]byte{}, fmt.Errorf("index %d out of range", idx)
 	}
-
+	return b.randaoMixes.RootAtIndex(idx), nil
-	return b.randaoMixes[idx], nil
 }
 
 // RandaoMixesLength returns the length of the randao mixes slice.
@@ -63,5 +62,5 @@ func (b *BeaconState) randaoMixesLength() int {
 		return 0
 	}
 
-	return len(b.randaoMixes)
+	return int(b.randaoMixes.TotalLength()) // lint:ignore uintcast- ajhdjhd
 }

beacon-chain/state/state-native/v2/setters_block.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 
 	customtypes "github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/custom-types"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 )
@@ -25,14 +24,12 @@ func (b *BeaconState) SetBlockRoots(val [][]byte) error {
 	b.lock.Lock()
 	defer b.lock.Unlock()
 
-	b.sharedFieldReferences[blockRoots].MinusRef()
-	b.sharedFieldReferences[blockRoots] = stateutil.NewRef(1)
-
 	var rootsArr [fieldparams.BlockRootsLength][32]byte
 	for i := 0; i < len(rootsArr); i++ {
 		copy(rootsArr[i][:], val[i])
 	}
-	roots := customtypes.BlockRoots(rootsArr)
+	roots := customtypes.BlockRoots{}
+	roots.SetFromBaseField(rootsArr)
 	b.blockRoots = &roots
 	b.markFieldAsDirty(blockRoots)
 	b.rebuildTrie[blockRoots] = true
@@ -42,24 +39,13 @@ func (b *BeaconState) SetBlockRoots(val [][]byte) error {
 // UpdateBlockRootAtIndex for the beacon state. Updates the block root
 // at a specific index to a new value.
 func (b *BeaconState) UpdateBlockRootAtIndex(idx uint64, blockRoot [32]byte) error {
-	if uint64(len(b.blockRoots)) <= idx {
+	if b.blockRoots.TotalLength() <= idx {
 		return fmt.Errorf("invalid index provided %d", idx)
 	}
 	b.lock.Lock()
 	defer b.lock.Unlock()
 
-	r := b.blockRoots
+	b.blockRoots.SetRootAtIndex(idx, blockRoot)
-	if ref := b.sharedFieldReferences[blockRoots]; ref.Refs() > 1 {
-		// Copy elements in underlying array by reference.
-		roots := *b.blockRoots
-		rootsCopy := roots
-		r = &rootsCopy
-		ref.MinusRef()
-		b.sharedFieldReferences[blockRoots] = stateutil.NewRef(1)
-	}
-
-	r[idx] = blockRoot
-	b.blockRoots = r
 
 	b.markFieldAsDirty(blockRoots)
 	b.addDirtyIndices(blockRoots, []uint64{idx})

beacon-chain/state/state-native/v2/setters_randao.go

@@ -3,7 +3,6 @@ package v2
 import (
 	"github.com/pkg/errors"
 	customtypes "github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/custom-types"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
 )
@@ -14,14 +13,12 @@ func (b *BeaconState) SetRandaoMixes(val [][]byte) error {
 	b.lock.Lock()
 	defer b.lock.Unlock()
 
-	b.sharedFieldReferences[randaoMixes].MinusRef()
-	b.sharedFieldReferences[randaoMixes] = stateutil.NewRef(1)
-
 	var mixesArr [fieldparams.RandaoMixesLength][32]byte
 	for i := 0; i < len(mixesArr); i++ {
 		copy(mixesArr[i][:], val[i])
 	}
-	mixes := customtypes.RandaoMixes(mixesArr)
+	mixes := customtypes.RandaoMixes{}
+	mixes.SetFromBaseField(mixesArr)
 	b.randaoMixes = &mixes
 	b.markFieldAsDirty(randaoMixes)
 	b.rebuildTrie[randaoMixes] = true
@@ -31,24 +28,13 @@ func (b *BeaconState) SetRandaoMixes(val [][]byte) error {
 // UpdateRandaoMixesAtIndex for the beacon state. Updates the randao mixes
 // at a specific index to a new value.
 func (b *BeaconState) UpdateRandaoMixesAtIndex(idx uint64, val []byte) error {
-	if uint64(len(b.randaoMixes)) <= idx {
+	if b.randaoMixes.TotalLength() <= idx {
 		return errors.Errorf("invalid index provided %d", idx)
 	}
 	b.lock.Lock()
 	defer b.lock.Unlock()
 
-	mixes := b.randaoMixes
+	b.randaoMixes.SetRootAtIndex(idx, bytesutil.ToBytes32(val))
-	if refs := b.sharedFieldReferences[randaoMixes].Refs(); refs > 1 {
-		// Copy elements in underlying array by reference.
-		m := *b.randaoMixes
-		mCopy := m
-		mixes = &mCopy
-		b.sharedFieldReferences[randaoMixes].MinusRef()
-		b.sharedFieldReferences[randaoMixes] = stateutil.NewRef(1)
-	}
-
-	mixes[idx] = bytesutil.ToBytes32(val)
-	b.randaoMixes = mixes
 	b.markFieldAsDirty(randaoMixes)
 	b.addDirtyIndices(randaoMixes, []uint64{idx})
 

beacon-chain/state/state-native/v2/state_trie.go

@@ -35,10 +35,8 @@ func InitializeFromProtoUnsafe(st *ethpb.BeaconStateAltair) (*BeaconState, error
 		return nil, errors.New("received nil state")
 	}
 
-	var bRoots customtypes.BlockRoots
+	bRoots := customtypes.SetFromSlice(st.BlockRoots)
-	for i, r := range st.BlockRoots {
+
-		bRoots[i] = bytesutil.ToBytes32(r)
-	}
 	var sRoots customtypes.StateRoots
 	for i, r := range st.StateRoots {
 		sRoots[i] = bytesutil.ToBytes32(r)
@@ -47,10 +45,7 @@ func InitializeFromProtoUnsafe(st *ethpb.BeaconStateAltair) (*BeaconState, error
 	for i, r := range st.HistoricalRoots {
 		hRoots[i] = bytesutil.ToBytes32(r)
 	}
-	var mixes customtypes.RandaoMixes
+	mixes := customtypes.SetFromSliceRandao(st.RandaoMixes)
-	for i, m := range st.RandaoMixes {
-		mixes[i] = bytesutil.ToBytes32(m)
-	}
 
 	fieldCount := params.BeaconConfig().BeaconStateAltairFieldCount
 	b := &BeaconState{
@@ -59,7 +54,7 @@ func InitializeFromProtoUnsafe(st *ethpb.BeaconStateAltair) (*BeaconState, error
 		slot:                        st.Slot,
 		fork:                        st.Fork,
 		latestBlockHeader:           st.LatestBlockHeader,
-		blockRoots:                  &bRoots,
+		blockRoots:                  bRoots,
 		stateRoots:                  &sRoots,
 		historicalRoots:             hRoots,
 		eth1Data:                    st.Eth1Data,
@@ -67,7 +62,7 @@ func InitializeFromProtoUnsafe(st *ethpb.BeaconStateAltair) (*BeaconState, error
 		eth1DepositIndex:            st.Eth1DepositIndex,
 		validators:                  st.Validators,
 		balances:                    st.Balances,
-		randaoMixes:                 &mixes,
+		randaoMixes:                 mixes,
 		slashings:                   st.Slashings,
 		previousEpochParticipation:  st.PreviousEpochParticipation,
 		currentEpochParticipation:   st.CurrentEpochParticipation,
@@ -101,7 +96,6 @@ func InitializeFromProtoUnsafe(st *ethpb.BeaconStateAltair) (*BeaconState, error
 	// Initialize field reference tracking for shared data.
 	b.sharedFieldReferences[randaoMixes] = stateutil.NewRef(1)
 	b.sharedFieldReferences[stateRoots] = stateutil.NewRef(1)
-	b.sharedFieldReferences[blockRoots] = stateutil.NewRef(1)
 	b.sharedFieldReferences[previousEpochParticipationBits] = stateutil.NewRef(1) // New in Altair.
 	b.sharedFieldReferences[currentEpochParticipationBits] = stateutil.NewRef(1)  // New in Altair.
 	b.sharedFieldReferences[slashings] = stateutil.NewRef(1)
@@ -128,9 +122,9 @@ func (b *BeaconState) Copy() state.BeaconState {
 		eth1DepositIndex: b.eth1DepositIndex,
 
 		// Large arrays, infrequently changed, constant size.
-		blockRoots:    b.blockRoots,
+		blockRoots:    b.blockRoots.Copy(),
 		stateRoots:    b.stateRoots,
-		randaoMixes:   b.randaoMixes,
+		randaoMixes:   b.randaoMixes.Copy(),
 		slashings:     b.slashings,
 		eth1DataVotes: b.eth1DataVotes,
 
@@ -215,6 +209,8 @@ func (b *BeaconState) Copy() state.BeaconState {
 				b.stateFieldLeaves[field].FieldReference().MinusRef()
 			}
 		}
+		b.blockRoots.DecreaseRef()
+		b.randaoMixes.DecreaseRef()
 		for i := 0; i < fieldCount; i++ {
 			field := types.FieldIndex(i)
 			delete(b.stateFieldLeaves, field)

beacon-chain/state/state-native/v3/getters_block.go

@@ -76,9 +76,8 @@ func (b *BeaconState) BlockRootAtIndex(idx uint64) ([]byte, error) {
 // input index value.
 // This assumes that a lock is already held on BeaconState.
 func (b *BeaconState) blockRootAtIndex(idx uint64) ([32]byte, error) {
-	if uint64(len(b.blockRoots)) <= idx {
+	if b.blockRoots.TotalLength() <= idx {
 		return [32]byte{}, fmt.Errorf("index %d out of range", idx)
 	}
-
+	return b.blockRoots.RootAtIndex(idx), nil
-	return b.blockRoots[idx], nil
 }

beacon-chain/state/state-native/v3/getters_randao.go

@@ -37,11 +37,10 @@ func (b *BeaconState) RandaoMixAtIndex(idx uint64) ([]byte, error) {
 // input index value.
 // This assumes that a lock is already held on BeaconState.
 func (b *BeaconState) randaoMixAtIndex(idx uint64) ([32]byte, error) {
-	if uint64(len(b.randaoMixes)) <= idx {
+	if b.randaoMixes.TotalLength() <= idx {
 		return [32]byte{}, fmt.Errorf("index %d out of range", idx)
 	}
-
+	return b.randaoMixes.RootAtIndex(idx), nil
-	return b.randaoMixes[idx], nil
 }
 
 // RandaoMixesLength returns the length of the randao mixes slice.
@@ -63,5 +62,5 @@ func (b *BeaconState) randaoMixesLength() int {
 		return 0
 	}
 
-	return len(b.randaoMixes)
+	return int(b.randaoMixes.TotalLength()) // lint:ignore uintcast- ajhdjhd
 }

beacon-chain/state/state-native/v3/setters_block.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 
 	customtypes "github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/custom-types"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 )
@@ -25,14 +24,12 @@ func (b *BeaconState) SetBlockRoots(val [][]byte) error {
 	b.lock.Lock()
 	defer b.lock.Unlock()
 
-	b.sharedFieldReferences[blockRoots].MinusRef()
-	b.sharedFieldReferences[blockRoots] = stateutil.NewRef(1)
-
 	var rootsArr [fieldparams.BlockRootsLength][fieldparams.RootLength]byte
 	for i := 0; i < len(rootsArr); i++ {
 		copy(rootsArr[i][:], val[i])
 	}
-	roots := customtypes.BlockRoots(rootsArr)
+	roots := customtypes.BlockRoots{}
+	roots.SetFromBaseField(rootsArr)
 	b.blockRoots = &roots
 	b.markFieldAsDirty(blockRoots)
 	b.rebuildTrie[blockRoots] = true
@@ -42,24 +39,13 @@ func (b *BeaconState) SetBlockRoots(val [][]byte) error {
 // UpdateBlockRootAtIndex for the beacon state. Updates the block root
 // at a specific index to a new value.
 func (b *BeaconState) UpdateBlockRootAtIndex(idx uint64, blockRoot [32]byte) error {
-	if uint64(len(b.blockRoots)) <= idx {
+	if b.blockRoots.TotalLength() <= idx {
 		return fmt.Errorf("invalid index provided %d", idx)
 	}
 	b.lock.Lock()
 	defer b.lock.Unlock()
 
-	r := b.blockRoots
+	b.blockRoots.SetRootAtIndex(idx, blockRoot)
-	if ref := b.sharedFieldReferences[blockRoots]; ref.Refs() > 1 {
-		// Copy elements in underlying array by reference.
-		roots := *b.blockRoots
-		rootsCopy := roots
-		r = &rootsCopy
-		ref.MinusRef()
-		b.sharedFieldReferences[blockRoots] = stateutil.NewRef(1)
-	}
-
-	r[idx] = blockRoot
-	b.blockRoots = r
 
 	b.markFieldAsDirty(blockRoots)
 	b.addDirtyIndices(blockRoots, []uint64{idx})

beacon-chain/state/state-native/v3/setters_randao.go

@@ -3,7 +3,6 @@ package v3
 import (
 	"github.com/pkg/errors"
 	customtypes "github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/custom-types"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
 )
@@ -14,14 +13,12 @@ func (b *BeaconState) SetRandaoMixes(val [][]byte) error {
 	b.lock.Lock()
 	defer b.lock.Unlock()
 
-	b.sharedFieldReferences[randaoMixes].MinusRef()
+	var mixesArr [fieldparams.RandaoMixesLength][32]byte
-	b.sharedFieldReferences[randaoMixes] = stateutil.NewRef(1)
-
-	var mixesArr [fieldparams.RandaoMixesLength][fieldparams.RootLength]byte
 	for i := 0; i < len(mixesArr); i++ {
 		copy(mixesArr[i][:], val[i])
 	}
-	mixes := customtypes.RandaoMixes(mixesArr)
+	mixes := customtypes.RandaoMixes{}
+	mixes.SetFromBaseField(mixesArr)
 	b.randaoMixes = &mixes
 	b.markFieldAsDirty(randaoMixes)
 	b.rebuildTrie[randaoMixes] = true
@@ -31,24 +28,13 @@ func (b *BeaconState) SetRandaoMixes(val [][]byte) error {
 // UpdateRandaoMixesAtIndex for the beacon state. Updates the randao mixes
 // at a specific index to a new value.
 func (b *BeaconState) UpdateRandaoMixesAtIndex(idx uint64, val []byte) error {
-	if uint64(len(b.randaoMixes)) <= idx {
+	if b.randaoMixes.TotalLength() <= idx {
 		return errors.Errorf("invalid index provided %d", idx)
 	}
 	b.lock.Lock()
 	defer b.lock.Unlock()
 
-	mixes := b.randaoMixes
+	b.randaoMixes.SetRootAtIndex(idx, bytesutil.ToBytes32(val))
-	if refs := b.sharedFieldReferences[randaoMixes].Refs(); refs > 1 {
-		// Copy elements in underlying array by reference.
-		m := *b.randaoMixes
-		mCopy := m
-		mixes = &mCopy
-		b.sharedFieldReferences[randaoMixes].MinusRef()
-		b.sharedFieldReferences[randaoMixes] = stateutil.NewRef(1)
-	}
-
-	mixes[idx] = bytesutil.ToBytes32(val)
-	b.randaoMixes = mixes
 	b.markFieldAsDirty(randaoMixes)
 	b.addDirtyIndices(randaoMixes, []uint64{idx})
 

beacon-chain/state/state-native/v3/state_trie.go

@@ -36,10 +36,8 @@ func InitializeFromProtoUnsafe(st *ethpb.BeaconStateBellatrix) (state.BeaconStat
 		return nil, errors.New("received nil state")
 	}
 
-	var bRoots customtypes.BlockRoots
+	bRoots := customtypes.SetFromSlice(st.BlockRoots)
-	for i, r := range st.BlockRoots {
+
-		bRoots[i] = bytesutil.ToBytes32(r)
-	}
 	var sRoots customtypes.StateRoots
 	for i, r := range st.StateRoots {
 		sRoots[i] = bytesutil.ToBytes32(r)
@@ -48,10 +46,7 @@ func InitializeFromProtoUnsafe(st *ethpb.BeaconStateBellatrix) (state.BeaconStat
 	for i, r := range st.HistoricalRoots {
 		hRoots[i] = bytesutil.ToBytes32(r)
 	}
-	var mixes customtypes.RandaoMixes
+	mixes := customtypes.SetFromSliceRandao(st.RandaoMixes)
-	for i, m := range st.RandaoMixes {
-		mixes[i] = bytesutil.ToBytes32(m)
-	}
 
 	fieldCount := params.BeaconConfig().BeaconStateBellatrixFieldCount
 	b := &BeaconState{
@@ -60,7 +55,7 @@ func InitializeFromProtoUnsafe(st *ethpb.BeaconStateBellatrix) (state.BeaconStat
 		slot:                         st.Slot,
 		fork:                         st.Fork,
 		latestBlockHeader:            st.LatestBlockHeader,
-		blockRoots:                   &bRoots,
+		blockRoots:                   bRoots,
 		stateRoots:                   &sRoots,
 		historicalRoots:              hRoots,
 		eth1Data:                     st.Eth1Data,
@@ -68,7 +63,7 @@ func InitializeFromProtoUnsafe(st *ethpb.BeaconStateBellatrix) (state.BeaconStat
 		eth1DepositIndex:             st.Eth1DepositIndex,
 		validators:                   st.Validators,
 		balances:                     st.Balances,
-		randaoMixes:                  &mixes,
+		randaoMixes:                  mixes,
 		slashings:                    st.Slashings,
 		previousEpochParticipation:   st.PreviousEpochParticipation,
 		currentEpochParticipation:    st.CurrentEpochParticipation,
@@ -101,9 +96,7 @@ func InitializeFromProtoUnsafe(st *ethpb.BeaconStateBellatrix) (state.BeaconStat
 	}
 
 	// Initialize field reference tracking for shared data.
-	b.sharedFieldReferences[randaoMixes] = stateutil.NewRef(1)
 	b.sharedFieldReferences[stateRoots] = stateutil.NewRef(1)
-	b.sharedFieldReferences[blockRoots] = stateutil.NewRef(1)
 	b.sharedFieldReferences[previousEpochParticipationBits] = stateutil.NewRef(1) // New in Altair.
 	b.sharedFieldReferences[currentEpochParticipationBits] = stateutil.NewRef(1)  // New in Altair.
 	b.sharedFieldReferences[slashings] = stateutil.NewRef(1)
@@ -130,9 +123,9 @@ func (b *BeaconState) Copy() state.BeaconState {
 		eth1DepositIndex: b.eth1DepositIndex,
 
 		// Large arrays, infrequently changed, constant size.
-		randaoMixes:   b.randaoMixes,
+		randaoMixes:   b.randaoMixes.Copy(),
 		stateRoots:    b.stateRoots,
-		blockRoots:    b.blockRoots,
+		blockRoots:    b.blockRoots.Copy(),
 		slashings:     b.slashings,
 		eth1DataVotes: b.eth1DataVotes,
 
@@ -208,6 +201,7 @@ func (b *BeaconState) Copy() state.BeaconState {
 			}
 		}
 	}
+
 	state.StateCount.Inc()
 	// Finalizer runs when dst is being destroyed in garbage collection.
 	runtime.SetFinalizer(dst, func(b *BeaconState) {
@@ -217,6 +211,9 @@ func (b *BeaconState) Copy() state.BeaconState {
 				b.stateFieldLeaves[field].FieldReference().MinusRef()
 			}
 		}
+		b.blockRoots.DecreaseRef()
+		b.randaoMixes.DecreaseRef()
+
 		for i := 0; i < fieldCount; i++ {
 			field := types.FieldIndex(i)
 			delete(b.stateFieldLeaves, field)

config/features/flags.go

@@ -144,6 +144,7 @@ var devModeFlags = []cli.Flag{
 	enablePeerScorer,
 	enableVecHTR,
 	enableForkChoiceDoublyLinkedTree,
+	enableNativeState,
 }
 
 // ValidatorFlags contains a list of all the feature flags that apply to the validator client.

nogo_config.json

@@ -1,6 +1,7 @@
 {
   "unsafeptr": {
     "exclude_files": {
+      "beacon-chain/state/state-native/custom-types/block_roots.go": "Needed for field management operations",
       "external/.*": "Unsafe third party code",
       "rules_go_work-.*": "Third party code"
     }