Optimize Multivalue Slice For Trie Recomputation (#13238)

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

container/multi-value-slice/BUILD.bazel

@@ -5,6 +5,7 @@ go_library(
     srcs = ["multi_value_slice.go"],
     importpath = "github.com/prysmaticlabs/prysm/v4/container/multi-value-slice",
     visibility = ["//visibility:public"],
+    deps = ["@com_github_pkg_errors//:go_default_library"],
 )
 
 go_test(

container/multi-value-slice/multi_value_slice.go

@@ -92,6 +92,8 @@ package mvslice
 import (
 	"fmt"
 	"sync"
+
+	"github.com/pkg/errors"
 )
 
 // Id is an object identifier.
@@ -103,8 +105,22 @@ type Identifiable interface {
 }
 
 // MultiValueSlice defines an abstraction over all concrete implementations of the generic Slice.
-type MultiValueSlice[O Identifiable] interface {
-	Len(obj O) int
+type MultiValueSlice[V comparable] interface {
+	Len(obj Identifiable) int
+	At(obj Identifiable, index uint64) (V, error)
+	Value(obj Identifiable) []V
+}
+
+// MultiValueSliceComposite describes a struct for which we have access to a multivalue
+// slice along with the desired state.
+type MultiValueSliceComposite[V comparable] struct {
+	Identifiable
+	MultiValueSlice[V]
+}
+
+// State returns the referenced state.
+func (m MultiValueSliceComposite[V]) State() Identifiable {
+	return m.Identifiable
 }
 
 // Value defines a single value along with one or more IDs that share this value.
@@ -124,7 +140,7 @@ type MultiValueItem[V any] struct {
 //   - O interfaces.Identifiable - the type of objects sharing the slice. The constraint is required
 //     because we need a way to compare objects against each other in order to know which objects
 //     values should be accessed.
-type Slice[V comparable, O Identifiable] struct {
+type Slice[V comparable] struct {
 	sharedItems     []V
 	individualItems map[uint64]*MultiValueItem[V]
 	appendedItems   []*MultiValueItem[V]
@@ -133,7 +149,7 @@ type Slice[V comparable, O Identifiable] struct {
 }
 
 // Init initializes the slice with sensible defaults. Input values are assigned to shared items.
-func (s *Slice[V, O]) Init(items []V) {
+func (s *Slice[V]) Init(items []V) {
 	s.sharedItems = items
 	s.individualItems = map[uint64]*MultiValueItem[V]{}
 	s.appendedItems = []*MultiValueItem[V]{}
@@ -141,7 +157,7 @@ func (s *Slice[V, O]) Init(items []V) {
 }
 
 // Len returns the number of items for the input object.
-func (s *Slice[V, O]) Len(obj O) int {
+func (s *Slice[V]) Len(obj Identifiable) int {
 	s.lock.RLock()
 	defer s.lock.RUnlock()
 
@@ -153,7 +169,7 @@ func (s *Slice[V, O]) Len(obj O) int {
 }
 
 // Copy copies items between the source and destination.
-func (s *Slice[V, O]) Copy(src O, dst O) {
+func (s *Slice[V]) Copy(src, dst Identifiable) {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
@@ -190,7 +206,7 @@ func (s *Slice[V, O]) Copy(src O, dst O) {
 }
 
 // Value returns all items for the input object.
-func (s *Slice[V, O]) Value(obj O) []V {
+func (s *Slice[V]) Value(obj Identifiable) []V {
 	s.lock.RLock()
 	defer s.lock.RUnlock()
 
@@ -228,7 +244,7 @@ func (s *Slice[V, O]) Value(obj O) []V {
 // We first check if the index is within the length of shared items.
 // If it is, then we return an individual value at that index - if it exists - or a shared value otherwise.
 // If the index is beyond the length of shared values, it is an appended item and that's what gets returned.
-func (s *Slice[V, O]) At(obj O, index uint64) (V, error) {
+func (s *Slice[V]) At(obj Identifiable, index uint64) (V, error) {
 	s.lock.RLock()
 	defer s.lock.RUnlock()
 
@@ -266,7 +282,7 @@ func (s *Slice[V, O]) At(obj O, index uint64) (V, error) {
 }
 
 // UpdateAt updates the item at the required index for the input object to the passed in value.
-func (s *Slice[V, O]) UpdateAt(obj O, index uint64, val V) error {
+func (s *Slice[V]) UpdateAt(obj Identifiable, index uint64, val V) error {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
@@ -283,7 +299,7 @@ func (s *Slice[V, O]) UpdateAt(obj O, index uint64, val V) error {
 }
 
 // Append adds a new item to the input object.
-func (s *Slice[V, O]) Append(obj O, val V) {
+func (s *Slice[V]) Append(obj Identifiable, val V) {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
@@ -332,7 +348,7 @@ func (s *Slice[V, O]) Append(obj O, val V) {
 
 // Detach removes the input object from the multi-value slice.
 // What this means in practice is that we remove all individual and appended values for that object and clear the cached length.
-func (s *Slice[V, O]) Detach(obj O) {
+func (s *Slice[V]) Detach(obj Identifiable) {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
@@ -378,7 +394,7 @@ func (s *Slice[V, O]) Detach(obj O) {
 	delete(s.cachedLengths, obj.Id())
 }
 
-func (s *Slice[V, O]) fillOriginalItems(obj O, items *[]V) {
+func (s *Slice[V]) fillOriginalItems(obj Identifiable, items *[]V) {
 	for i, item := range s.sharedItems {
 		ind, ok := s.individualItems[uint64(i)]
 		if !ok {
@@ -399,7 +415,7 @@ func (s *Slice[V, O]) fillOriginalItems(obj O, items *[]V) {
 	}
 }
 
-func (s *Slice[V, O]) updateOriginalItem(obj O, index uint64, val V) {
+func (s *Slice[V]) updateOriginalItem(obj Identifiable, index uint64, val V) {
 	ind, ok := s.individualItems[index]
 	if ok {
 		for mvi, v := range ind.Values {
@@ -440,7 +456,7 @@ func (s *Slice[V, O]) updateOriginalItem(obj O, index uint64, val V) {
 	}
 }
 
-func (s *Slice[V, O]) updateAppendedItem(obj O, index uint64, val V) error {
+func (s *Slice[V]) updateAppendedItem(obj Identifiable, index uint64, val V) error {
 	item := s.appendedItems[index-uint64(len(s.sharedItems))]
 	found := false
 	for vi, v := range item.Values {
@@ -491,3 +507,34 @@ func deleteElemFromSlice[T any](s []T, i int) []T {
 	s = s[:len(s)-1]   // Truncate slice.
 	return s
 }
+
+// EmptyMVSlice specifies a type which allows a normal slice to conform
+// to the multivalue slice interface.
+type EmptyMVSlice[V comparable] struct {
+	fullSlice []V
+}
+
+func (e EmptyMVSlice[V]) Len(_ Identifiable) int {
+	return len(e.fullSlice)
+}
+
+func (e EmptyMVSlice[V]) At(_ Identifiable, index uint64) (V, error) {
+	if index >= uint64(len(e.fullSlice)) {
+		var def V
+		return def, errors.Errorf("index %d out of bounds", index)
+	}
+	return e.fullSlice[index], nil
+}
+
+func (e EmptyMVSlice[V]) Value(_ Identifiable) []V {
+	return e.fullSlice
+}
+
+// BuildEmptyCompositeSlice builds a composite multivalue object with a native
+// slice.
+func BuildEmptyCompositeSlice[V comparable](values []V) MultiValueSliceComposite[V] {
+	return MultiValueSliceComposite[V]{
+		Identifiable:    nil,
+		MultiValueSlice: EmptyMVSlice[V]{fullSlice: values},
+	}
+}

container/multi-value-slice/multi_value_slice_test.go

@@ -21,7 +21,7 @@ func (o *testObject) SetId(id uint64) {
 }
 
 func TestLen(t *testing.T) {
-	s := &Slice[int, *testObject]{}
+	s := &Slice[int]{}
 	s.Init([]int{1, 2, 3})
 	s.cachedLengths[1] = 123
 	t.Run("cached", func(t *testing.T) {
@@ -93,7 +93,7 @@ func TestValue(t *testing.T) {
 	assert.Equal(t, 3, v[6])
 	assert.Equal(t, 2, v[7])
 
-	s = &Slice[int, *testObject]{}
+	s = &Slice[int]{}
 	s.Init([]int{1, 2, 3})
 
 	v = s.Value(&testObject{id: 999})
@@ -246,7 +246,7 @@ func TestAppend(t *testing.T) {
 	// - we also want to check that cached length is properly updated after every append
 
 	// we want to start with the simplest slice possible
-	s := &Slice[int, *testObject]{}
+	s := &Slice[int]{}
 	s.Init([]int{0})
 	first := &testObject{id: 1}
 	second := &testObject{id: 2}
@@ -335,8 +335,8 @@ func TestDetach(t *testing.T) {
 // Index 5: Different appended value
 // Index 6: Same appended value
 // Index 7: Appended value ONLY for the second object
-func setup() *Slice[int, *testObject] {
-	s := &Slice[int, *testObject]{}
+func setup() *Slice[int] {
+	s := &Slice[int]{}
 	s.Init([]int{123, 123, 123, 123, 123})
 	s.individualItems[1] = &MultiValueItem[int]{
 		Values: []*Value[int]{
@@ -410,7 +410,7 @@ func setup() *Slice[int, *testObject] {
 	return s
 }
 
-func assertIndividualFound(t *testing.T, slice *Slice[int, *testObject], id uint64, itemIndex uint64, expected int) {
+func assertIndividualFound(t *testing.T, slice *Slice[int], id uint64, itemIndex uint64, expected int) {
 	found := false
 	for _, v := range slice.individualItems[itemIndex].Values {
 		for _, o := range v.ids {
@@ -423,7 +423,7 @@ func assertIndividualFound(t *testing.T, slice *Slice[int, *testObject], id uint
 	assert.Equal(t, true, found)
 }
 
-func assertIndividualNotFound(t *testing.T, slice *Slice[int, *testObject], id uint64, itemIndex uint64) {
+func assertIndividualNotFound(t *testing.T, slice *Slice[int], id uint64, itemIndex uint64) {
 	found := false
 	for _, v := range slice.individualItems[itemIndex].Values {
 		for _, o := range v.ids {
@@ -435,7 +435,7 @@ func assertIndividualNotFound(t *testing.T, slice *Slice[int, *testObject], id u
 	assert.Equal(t, false, found)
 }
 
-func assertAppendedFound(t *testing.T, slice *Slice[int, *testObject], id uint64, itemIndex uint64, expected int) {
+func assertAppendedFound(t *testing.T, slice *Slice[int], id uint64, itemIndex uint64, expected int) {
 	found := false
 	for _, v := range slice.appendedItems[itemIndex].Values {
 		for _, o := range v.ids {
@@ -448,7 +448,7 @@ func assertAppendedFound(t *testing.T, slice *Slice[int, *testObject], id uint64
 	assert.Equal(t, true, found)
 }
 
-func assertAppendedNotFound(t *testing.T, slice *Slice[int, *testObject], id uint64, itemIndex uint64) {
+func assertAppendedNotFound(t *testing.T, slice *Slice[int], id uint64, itemIndex uint64) {
 	found := false
 	for _, v := range slice.appendedItems[itemIndex].Values {
 		for _, o := range v.ids {
@@ -466,14 +466,14 @@ func BenchmarkValue(b *testing.B) {
 	const _10m = 10000000
 
 	b.Run("100,000 shared items", func(b *testing.B) {
-		s := &Slice[int, *testObject]{}
+		s := &Slice[int]{}
 		s.Init(make([]int, _100k))
 		for i := 0; i < b.N; i++ {
 			s.Value(&testObject{})
 		}
 	})
 	b.Run("100,000 equal individual items", func(b *testing.B) {
-		s := &Slice[int, *testObject]{}
+		s := &Slice[int]{}
 		s.Init(make([]int, _100k))
 		s.individualItems[0] = &MultiValueItem[int]{Values: []*Value[int]{{val: 999, ids: []uint64{}}}}
 		objs := make([]*testObject, _100k)
@@ -486,7 +486,7 @@ func BenchmarkValue(b *testing.B) {
 		}
 	})
 	b.Run("100,000 different individual items", func(b *testing.B) {
-		s := &Slice[int, *testObject]{}
+		s := &Slice[int]{}
 		s.Init(make([]int, _100k))
 		objs := make([]*testObject, _100k)
 		for i := 0; i < len(objs); i++ {
@@ -498,7 +498,7 @@ func BenchmarkValue(b *testing.B) {
 		}
 	})
 	b.Run("100,000 shared items and 100,000 equal appended items", func(b *testing.B) {
-		s := &Slice[int, *testObject]{}
+		s := &Slice[int]{}
 		s.Init(make([]int, _100k))
 		s.appendedItems = []*MultiValueItem[int]{{Values: []*Value[int]{{val: 999, ids: []uint64{}}}}}
 		objs := make([]*testObject, _100k)
@@ -511,7 +511,7 @@ func BenchmarkValue(b *testing.B) {
 		}
 	})
 	b.Run("100,000 shared items and 100,000 different appended items", func(b *testing.B) {
-		s := &Slice[int, *testObject]{}
+		s := &Slice[int]{}
 		s.Init(make([]int, _100k))
 		s.appendedItems = []*MultiValueItem[int]{}
 		objs := make([]*testObject, _100k)
@@ -524,14 +524,14 @@ func BenchmarkValue(b *testing.B) {
 		}
 	})
 	b.Run("1,000,000 shared items", func(b *testing.B) {
-		s := &Slice[int, *testObject]{}
+		s := &Slice[int]{}
 		s.Init(make([]int, _1m))
 		for i := 0; i < b.N; i++ {
 			s.Value(&testObject{})
 		}
 	})
 	b.Run("1,000,000 equal individual items", func(b *testing.B) {
-		s := &Slice[int, *testObject]{}
+		s := &Slice[int]{}
 		s.Init(make([]int, _1m))
 		s.individualItems[0] = &MultiValueItem[int]{Values: []*Value[int]{{val: 999, ids: []uint64{}}}}
 		objs := make([]*testObject, _1m)
@@ -544,7 +544,7 @@ func BenchmarkValue(b *testing.B) {
 		}
 	})
 	b.Run("1,000,000 different individual items", func(b *testing.B) {
-		s := &Slice[int, *testObject]{}
+		s := &Slice[int]{}
 		s.Init(make([]int, _1m))
 		objs := make([]*testObject, _1m)
 		for i := 0; i < len(objs); i++ {
@@ -556,7 +556,7 @@ func BenchmarkValue(b *testing.B) {
 		}
 	})
 	b.Run("1,000,000 shared items and 1,000,000 equal appended items", func(b *testing.B) {
-		s := &Slice[int, *testObject]{}
+		s := &Slice[int]{}
 		s.Init(make([]int, _1m))
 		s.appendedItems = []*MultiValueItem[int]{{Values: []*Value[int]{{val: 999, ids: []uint64{}}}}}
 		objs := make([]*testObject, _1m)
@@ -569,7 +569,7 @@ func BenchmarkValue(b *testing.B) {
 		}
 	})
 	b.Run("1,000,000 shared items and 1,000,000 different appended items", func(b *testing.B) {
-		s := &Slice[int, *testObject]{}
+		s := &Slice[int]{}
 		s.Init(make([]int, _1m))
 		s.appendedItems = []*MultiValueItem[int]{}
 		objs := make([]*testObject, _1m)
@@ -582,14 +582,14 @@ func BenchmarkValue(b *testing.B) {
 		}
 	})
 	b.Run("10,000,000 shared items", func(b *testing.B) {
-		s := &Slice[int, *testObject]{}
+		s := &Slice[int]{}
 		s.Init(make([]int, _10m))
 		for i := 0; i < b.N; i++ {
 			s.Value(&testObject{})
 		}
 	})
 	b.Run("10,000,000 equal individual items", func(b *testing.B) {
-		s := &Slice[int, *testObject]{}
+		s := &Slice[int]{}
 		s.Init(make([]int, _10m))
 		s.individualItems[0] = &MultiValueItem[int]{Values: []*Value[int]{{val: 999, ids: []uint64{}}}}
 		objs := make([]*testObject, _10m)
@@ -602,7 +602,7 @@ func BenchmarkValue(b *testing.B) {
 		}
 	})
 	b.Run("10,000,000 different individual items", func(b *testing.B) {
-		s := &Slice[int, *testObject]{}
+		s := &Slice[int]{}
 		s.Init(make([]int, _10m))
 		objs := make([]*testObject, _10m)
 		for i := 0; i < len(objs); i++ {
@@ -614,7 +614,7 @@ func BenchmarkValue(b *testing.B) {
 		}
 	})
 	b.Run("10,000,000 shared items and 10,000,000 equal appended items", func(b *testing.B) {
-		s := &Slice[int, *testObject]{}
+		s := &Slice[int]{}
 		s.Init(make([]int, _10m))
 		s.appendedItems = []*MultiValueItem[int]{{Values: []*Value[int]{{val: 999, ids: []uint64{}}}}}
 		objs := make([]*testObject, _10m)
@@ -627,7 +627,7 @@ func BenchmarkValue(b *testing.B) {
 		}
 	})
 	b.Run("10,000,000 shared items and 10,000,000 different appended items", func(b *testing.B) {
-		s := &Slice[int, *testObject]{}
+		s := &Slice[int]{}
 		s.Init(make([]int, _10m))
 		s.appendedItems = []*MultiValueItem[int]{}
 		objs := make([]*testObject, _10m)