diff --git a/container/multi-value-slice/BUILD.bazel b/container/multi-value-slice/BUILD.bazel new file mode 100644 index 0000000000..b96cddf8d2 --- /dev/null +++ b/container/multi-value-slice/BUILD.bazel @@ -0,0 +1,23 @@ +load("@prysm//tools/go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["multi_value_slice.go"], + importpath = "github.com/prysmaticlabs/prysm/v4/container/multi-value-slice", + visibility = ["//visibility:public"], + deps = [ + "//container/multi-value-slice/interfaces:go_default_library", + "@com_github_google_uuid//:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["multi_value_slice_test.go"], + embed = [":go_default_library"], + deps = [ + "//testing/assert:go_default_library", + "//testing/require:go_default_library", + "@com_github_google_uuid//:go_default_library", + ], +) diff --git a/container/multi-value-slice/interfaces/BUILD.bazel b/container/multi-value-slice/interfaces/BUILD.bazel new file mode 100644 index 0000000000..f8be939929 --- /dev/null +++ b/container/multi-value-slice/interfaces/BUILD.bazel @@ -0,0 +1,9 @@ +load("@prysm//tools/go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["interfaces.go"], + importpath = "github.com/prysmaticlabs/prysm/v4/container/multi-value-slice/interfaces", + visibility = ["//visibility:public"], + deps = ["@com_github_google_uuid//:go_default_library"], +) diff --git a/container/multi-value-slice/interfaces/interfaces.go b/container/multi-value-slice/interfaces/interfaces.go new file mode 100644 index 0000000000..b22ab64de1 --- /dev/null +++ b/container/multi-value-slice/interfaces/interfaces.go @@ -0,0 +1,9 @@ +package interfaces + +import "github.com/google/uuid" + +// Identifiable represents an object that can be uniquely identified by its Id. +type Identifiable interface { + Id() uuid.UUID + SetId(id uuid.UUID) +} diff --git a/container/multi-value-slice/multi_value_slice.go b/container/multi-value-slice/multi_value_slice.go new file mode 100644 index 0000000000..a19837c2e1 --- /dev/null +++ b/container/multi-value-slice/multi_value_slice.go @@ -0,0 +1,488 @@ +// Package mvslice defines a multi value slice container. The purpose of the container is to be a replacement for a slice +// in scenarios where many objects of the same type share a copy of an identical or nearly identical slice. +// In such case using the multi value slice should result in less memory allocation because many values of the slice can be shared between objects. +// +// The multi value slice should be initialized by calling the Init function and passing the initial values of the slice. +// After initializing the slice, it can be shared between object by using the Copy function. +// Note that simply assigning the same multi value slice to several objects is not enough for it to work properly. +// Calling Copy is required in most circumstances (an exception is when the source object has only shared values). +// +// s := &Slice[int, *testObject]{} +// s.Init([]int{1, 2, 3}) +// src := &testObject{id: id1, slice: s} // id1 is some UUID +// dst := &testObject{id: id2, slice: s} // id2 is some UUID +// s.Copy(src, dst) +// +// Each Value stores a value of type V along with identifiers to objects that have this value. +// A MultiValueItem is a slice of Value elements. A Slice contains shared items, individual items and appended items. +// +// You can think of a shared value as the original value (i.e. the value at the point in time when the multi value slice was constructed), +// and of an individual value as a changed value. +// There is no notion of a shared appended value because appended values never have an original value (appended values are empty when the slice is created). +// +// Whenever any of the slice’s functions (apart from Init) is called, the function needs to know which object it is dealing with. +// This is because if an object has an individual/appended value, the function must get/set/change this particular value instead of the shared value +// or another individual/appended value. +// +// The way appended items are stored is as follows. Let’s say appended items were a regular slice that is initially empty, +// and we append an item for object0 and then append another item for object1. +// Now we have two items in the slice, but object1 only has an item in index 1. This makes things very confusing and hard to deal with. +// If we make appended items a []*Value, things don’t become much better. +// It is therefore easiest to make appended items a []*MultiValueItem, which allows each object to have its own values starting at index 0 +// and not having any “gaps”. +// +// The Detach function should be called when an object gets garbage collected. +// Its purpose is to clean up the slice from individual/appended values of the collected object. +// Otherwise the slice will get polluted with values for non-existing objects. +// +// Example diagram illustrating what happens after copying, updating and detaching: +// +// Create object o1 with value 10. At this point we only have a shared value. +// +// =================== +// shared | individual +// =================== +// 10 | +// +// Copy object o1 to object o2. o2 shares the value with o1, no individual value is created. +// +// =================== +// shared | individual +// =================== +// 10 | +// +// Update value of object o2 to 20. An individual value is created. +// +// =================== +// shared | individual +// =================== +// 10 | 20: [o2] +// +// Copy object o2 to object o3. The individual value's object list is updated. +// +// =================== +// shared | individual +// =================== +// 10 | 20: [o2,o3] +// +// Update value of object o3 to 30. There are two individual values now, one for o2 and one for o3. +// +// =================== +// shared | individual +// =================== +// 10 | 20: [o2] +// | 30: [o3] +// +// Update value of object o2 to 10. o2 no longer has an individual value +// because it got "reverted" to the original, shared value, +// +// =================== +// shared | individual +// =================== +// 10 | 30: [o3] +// +// Detach object o3. Individual value for o3 is removed. +// +// =================== +// shared | individual +// =================== +// 10 | +package mvslice + +import ( + "fmt" + "sync" + + "github.com/google/uuid" + "github.com/prysmaticlabs/prysm/v4/container/multi-value-slice/interfaces" +) + +// MultiValueSlice defines an abstraction over all concrete implementations of the generic Slice. +type MultiValueSlice[O interfaces.Identifiable] interface { + Len(obj O) uuid.UUID +} + +// Value defines a single value along with one or more IDs that share this value. +type Value[V any] struct { + val V + ids []uuid.UUID +} + +// MultiValueItem defines a collection of Value items. +type MultiValueItem[V any] struct { + Values []*Value[V] +} + +// Slice is the main component of the multi-value slice data structure. It has two type parameters: +// - V comparable - the type of values stored the slice. The constraint is required +// because certain operations (e.g. updating, appending) have to compare values against each other. +// - O interfaces.Identifiable - the type of objects sharing the slice. The constraint is required +// because we need a way to compare objects against each other in order to know which objects +// values should be accessed. +type Slice[V comparable, O interfaces.Identifiable] struct { + sharedItems []V + individualItems map[uint64]*MultiValueItem[V] + appendedItems []*MultiValueItem[V] + cachedLengths map[uuid.UUID]int + lock sync.RWMutex +} + +// Init initializes the slice with sensible defaults. Input values are assigned to shared items. +func (s *Slice[V, O]) Init(items []V) { + s.sharedItems = items + s.individualItems = map[uint64]*MultiValueItem[V]{} + s.appendedItems = []*MultiValueItem[V]{} + s.cachedLengths = map[uuid.UUID]int{} +} + +// Len returns the number of items for the input object. +func (s *Slice[V, O]) Len(obj O) int { + s.lock.RLock() + defer s.lock.RUnlock() + + l, ok := s.cachedLengths[obj.Id()] + if !ok { + return len(s.sharedItems) + } + return l +} + +// Copy copies items between the source and destination. +func (s *Slice[V, O]) Copy(src O, dst O) { + s.lock.Lock() + defer s.lock.Unlock() + + for _, item := range s.individualItems { + for _, v := range item.Values { + _, found := containsId(v.ids, src.Id()) + if found { + v.ids = append(v.ids, dst.Id()) + break + } + } + } + + for _, item := range s.appendedItems { + found := false + for _, v := range item.Values { + _, found = containsId(v.ids, src.Id()) + if found { + v.ids = append(v.ids, dst.Id()) + break + } + } + if !found { + // This is an optimization. If we didn't find an appended item at index i, + // then all larger indices don't have an appended item for the object either. + break + } + } + + srcLen, ok := s.cachedLengths[src.Id()] + if ok { + s.cachedLengths[dst.Id()] = srcLen + } +} + +// Value returns all items for the input object. +func (s *Slice[V, O]) Value(obj O) []V { + s.lock.RLock() + defer s.lock.RUnlock() + + l, ok := s.cachedLengths[obj.Id()] + if ok { + result := make([]V, l) + s.fillOriginalItems(obj, &result) + + sharedLen := len(s.sharedItems) + for i, item := range s.appendedItems { + found := false + for _, v := range item.Values { + _, found = containsId(v.ids, obj.Id()) + if found { + result[sharedLen+i] = v.val + break + } + } + if !found { + // This is an optimization. If we didn't find an appended item at index i, + // then all larger indices don't have an appended item for the object either. + return result + } + } + return result + } else { + result := make([]V, len(s.sharedItems)) + s.fillOriginalItems(obj, &result) + return result + } +} + +// At returns the item at the requested index for the input object. +// Appended items' indices are always larger than shared/individual items' indices. +// We first check if the index is within the length of shared items. +// If it is, then we return an individual value at that index - if it exists - or a shared value otherwise. +// If the index is beyond the length of shared values, it is an appended item and that's what gets returned. +func (s *Slice[V, O]) At(obj O, index uint64) (V, error) { + s.lock.RLock() + defer s.lock.RUnlock() + + if index >= uint64(len(s.sharedItems)+len(s.appendedItems)) { + var def V + return def, fmt.Errorf("index %d out of bounds", index) + } + + isOriginal := index < uint64(len(s.sharedItems)) + if isOriginal { + ind, ok := s.individualItems[index] + if !ok { + return s.sharedItems[index], nil + } + for _, v := range ind.Values { + for _, id := range v.ids { + if id == obj.Id() { + return v.val, nil + } + } + } + return s.sharedItems[index], nil + } else { + item := s.appendedItems[index-uint64(len(s.sharedItems))] + for _, v := range item.Values { + for _, id := range v.ids { + if id == obj.Id() { + return v.val, nil + } + } + } + var def V + return def, fmt.Errorf("index %d out of bounds", index) + } +} + +// UpdateAt updates the item at the required index for the input object to the passed in value. +func (s *Slice[V, O]) UpdateAt(obj O, index uint64, val V) error { + s.lock.Lock() + defer s.lock.Unlock() + + if index >= uint64(len(s.sharedItems)+len(s.appendedItems)) { + return fmt.Errorf("index %d out of bounds", index) + } + + isOriginal := index < uint64(len(s.sharedItems)) + if isOriginal { + s.updateOriginalItem(obj, index, val) + return nil + } + return s.updateAppendedItem(obj, index, val) +} + +// Append adds a new item to the input object. +func (s *Slice[V, O]) Append(obj O, val V) { + s.lock.Lock() + defer s.lock.Unlock() + + if len(s.appendedItems) == 0 { + s.appendedItems = append(s.appendedItems, &MultiValueItem[V]{Values: []*Value[V]{{val: val, ids: []uuid.UUID{obj.Id()}}}}) + s.cachedLengths[obj.Id()] = len(s.sharedItems) + 1 + return + } + + for _, item := range s.appendedItems { + found := false + for _, v := range item.Values { + _, found = containsId(v.ids, obj.Id()) + if found { + break + } + } + if !found { + newValue := true + for _, v := range item.Values { + if v.val == val { + v.ids = append(v.ids, obj.Id()) + newValue = false + break + } + } + if newValue { + item.Values = append(item.Values, &Value[V]{val: val, ids: []uuid.UUID{obj.Id()}}) + } + + l, ok := s.cachedLengths[obj.Id()] + if ok { + s.cachedLengths[obj.Id()] = l + 1 + } else { + s.cachedLengths[obj.Id()] = len(s.sharedItems) + 1 + } + + return + } + } + + s.appendedItems = append(s.appendedItems, &MultiValueItem[V]{Values: []*Value[V]{{val: val, ids: []uuid.UUID{obj.Id()}}}}) + + s.cachedLengths[obj.Id()] = s.cachedLengths[obj.Id()] + 1 +} + +// Detach removes the input object from the multi-value slice. +// What this means in practice is that we remove all individual and appended values for that object and clear the cached length. +func (s *Slice[V, O]) Detach(obj O) { + s.lock.Lock() + defer s.lock.Unlock() + + for i, ind := range s.individualItems { + for vi, v := range ind.Values { + foundIndex, found := containsId(v.ids, obj.Id()) + if found { + if len(v.ids) == 1 { + if len(ind.Values) == 1 { + delete(s.individualItems, i) + } else { + ind.Values = deleteElemFromSlice(ind.Values, vi) + } + } else { + v.ids = deleteElemFromSlice(v.ids, foundIndex) + } + break + } + } + } + + for _, item := range s.appendedItems { + found := false + for vi, v := range item.Values { + var foundIndex int + foundIndex, found = containsId(v.ids, obj.Id()) + if found { + if len(v.ids) == 1 { + item.Values = deleteElemFromSlice(item.Values, vi) + } else { + v.ids = deleteElemFromSlice(v.ids, foundIndex) + } + break + } + } + if !found { + // This is an optimization. If we didn't find an appended item at index i, + // then all larger indices don't have an appended item for the object either. + break + } + } + + delete(s.cachedLengths, obj.Id()) +} + +func (s *Slice[V, O]) fillOriginalItems(obj O, items *[]V) { + for i, item := range s.sharedItems { + ind, ok := s.individualItems[uint64(i)] + if !ok { + (*items)[i] = item + } else { + found := false + for _, v := range ind.Values { + _, found = containsId(v.ids, obj.Id()) + if found { + (*items)[i] = v.val + break + } + } + if !found { + (*items)[i] = item + } + } + } +} + +func (s *Slice[V, O]) updateOriginalItem(obj O, index uint64, val V) { + ind, ok := s.individualItems[index] + if ok { + for mvi, v := range ind.Values { + // if we find an existing value, we remove it + foundIndex, found := containsId(v.ids, obj.Id()) + if found { + if len(v.ids) == 1 { + // There is an improvement to be made here. If len(ind.Values) == 1, + // then after removing the item from the slice s.individualItems[i] + // will be a useless map entry whose value is an empty slice. + ind.Values = deleteElemFromSlice(ind.Values, mvi) + } else { + v.ids = deleteElemFromSlice(v.ids, foundIndex) + } + break + } + } + } + + if val == s.sharedItems[index] { + return + } + + if !ok { + s.individualItems[index] = &MultiValueItem[V]{Values: []*Value[V]{{val: val, ids: []uuid.UUID{obj.Id()}}}} + } else { + newValue := true + for _, v := range ind.Values { + if v.val == val { + v.ids = append(v.ids, obj.Id()) + newValue = false + break + } + } + if newValue { + ind.Values = append(ind.Values, &Value[V]{val: val, ids: []uuid.UUID{obj.Id()}}) + } + } +} + +func (s *Slice[V, O]) updateAppendedItem(obj O, index uint64, val V) error { + item := s.appendedItems[index-uint64(len(s.sharedItems))] + found := false + for vi, v := range item.Values { + var foundIndex int + // if we find an existing value, we remove it + foundIndex, found = containsId(v.ids, obj.Id()) + if found { + if len(v.ids) == 1 { + item.Values = deleteElemFromSlice(item.Values, vi) + } else { + v.ids = deleteElemFromSlice(v.ids, foundIndex) + } + break + } + } + if !found { + return fmt.Errorf("index %d out of bounds", index) + } + + newValue := true + for _, v := range item.Values { + if v.val == val { + v.ids = append(v.ids, obj.Id()) + newValue = false + break + } + } + if newValue { + item.Values = append(item.Values, &Value[V]{val: val, ids: []uuid.UUID{obj.Id()}}) + } + + return nil +} + +func containsId(ids []uuid.UUID, wanted uuid.UUID) (int, bool) { + for i, id := range ids { + if id == wanted { + return i, true + } + } + return 0, false +} + +// deleteElemFromSlice does not relocate the slice, but it also does not preserve the order of items. +// This is not a problem here because the order of values in a MultiValueItem and object IDs doesn't matter. +func deleteElemFromSlice[T any](s []T, i int) []T { + s[i] = s[len(s)-1] // Copy last element to index i. + s = s[:len(s)-1] // Truncate slice. + return s +} diff --git a/container/multi-value-slice/multi_value_slice_test.go b/container/multi-value-slice/multi_value_slice_test.go new file mode 100644 index 0000000000..6e03bde04d --- /dev/null +++ b/container/multi-value-slice/multi_value_slice_test.go @@ -0,0 +1,664 @@ +package mvslice + +import ( + "math/rand" + "testing" + + "github.com/google/uuid" + "github.com/prysmaticlabs/prysm/v4/testing/assert" + "github.com/prysmaticlabs/prysm/v4/testing/require" +) + +var ( + id1 = uuid.New() + id2 = uuid.New() + id999 = uuid.New() +) + +type testObject struct { + id uuid.UUID + slice *Slice[int, *testObject] +} + +func (o *testObject) Id() uuid.UUID { + return o.id +} + +func (o *testObject) SetId(id uuid.UUID) { + o.id = id +} + +func TestLen(t *testing.T) { + s := &Slice[int, *testObject]{} + s.Init([]int{1, 2, 3}) + id := uuid.New() + s.cachedLengths[id] = 123 + t.Run("cached", func(t *testing.T) { + assert.Equal(t, 123, s.Len(&testObject{id: id})) + }) + t.Run("not cached", func(t *testing.T) { + assert.Equal(t, 3, s.Len(&testObject{id: uuid.New()})) + }) +} + +func TestCopy(t *testing.T) { + // What we want to check: + // - shared value is copied + // - when the source object has an individual value, it is copied + // - when the source object does not have an individual value, the shared value is copied + // - when the source object has an appended value, it is copied + // - when the source object does not have an appended value, nothing is copied + // - length of destination object is cached + + s := setup() + src := &testObject{id: id1, slice: s} + dst := &testObject{id: id999, slice: s} + + s.Copy(src, dst) + + assert.Equal(t, (*MultiValueItem[int])(nil), dst.slice.individualItems[0]) + assertIndividualFound(t, s, dst.id, 1, 1) + assertIndividualFound(t, s, dst.id, 2, 3) + assertIndividualFound(t, s, dst.id, 3, 1) + assertIndividualNotFound(t, s, dst.id, 4) + assertAppendedFound(t, s, dst.id, 0, 1) + assertAppendedFound(t, s, dst.id, 1, 3) + assertAppendedNotFound(t, s, dst.id, 2) + l, ok := s.cachedLengths[id999] + require.Equal(t, true, ok) + assert.Equal(t, 7, l) +} + +func TestValue(t *testing.T) { + // What we want to check: + // - correct values are returned for first object + // - correct values are returned for second object + // - correct values are returned for an object without appended items + + s := setup() + first := &testObject{id: id1, slice: s} + second := &testObject{id: id2, slice: s} + + v := s.Value(first) + + require.Equal(t, 7, len(v)) + assert.Equal(t, 123, v[0]) + assert.Equal(t, 1, v[1]) + assert.Equal(t, 3, v[2]) + assert.Equal(t, 1, v[3]) + assert.Equal(t, 123, v[4]) + assert.Equal(t, 1, v[5]) + assert.Equal(t, 3, v[6]) + + v = s.Value(second) + + require.Equal(t, 8, len(v)) + assert.Equal(t, 123, v[0]) + assert.Equal(t, 2, v[1]) + assert.Equal(t, 3, v[2]) + assert.Equal(t, 123, v[3]) + assert.Equal(t, 2, v[4]) + assert.Equal(t, 2, v[5]) + assert.Equal(t, 3, v[6]) + assert.Equal(t, 2, v[7]) + + s = &Slice[int, *testObject]{} + s.Init([]int{1, 2, 3}) + id := uuid.New() + + v = s.Value(&testObject{id: id}) + + require.Equal(t, 3, len(v)) + assert.Equal(t, 1, v[0]) + assert.Equal(t, 2, v[1]) + assert.Equal(t, 3, v[2]) +} + +func TestAt(t *testing.T) { + // What we want to check: + // - correct values are returned for first object + // - correct values are returned for second object + // - ERROR when index too large in general + // - ERROR when index not too large in general, but too large for an object + + s := setup() + first := &testObject{id: id1, slice: s} + second := &testObject{id: id2, slice: s} + + v, err := s.At(first, 0) + require.NoError(t, err) + assert.Equal(t, 123, v) + v, err = s.At(first, 1) + require.NoError(t, err) + assert.Equal(t, 1, v) + v, err = s.At(first, 2) + require.NoError(t, err) + assert.Equal(t, 3, v) + v, err = s.At(first, 3) + require.NoError(t, err) + assert.Equal(t, 1, v) + v, err = s.At(first, 4) + require.NoError(t, err) + assert.Equal(t, 123, v) + v, err = s.At(first, 5) + require.NoError(t, err) + assert.Equal(t, 1, v) + v, err = s.At(first, 6) + require.NoError(t, err) + assert.Equal(t, 3, v) + _, err = s.At(first, 7) + assert.ErrorContains(t, "index 7 out of bounds", err) + + v, err = s.At(second, 0) + require.NoError(t, err) + assert.Equal(t, 123, v) + v, err = s.At(second, 1) + require.NoError(t, err) + assert.Equal(t, 2, v) + v, err = s.At(second, 2) + require.NoError(t, err) + assert.Equal(t, 3, v) + v, err = s.At(second, 3) + require.NoError(t, err) + assert.Equal(t, 123, v) + v, err = s.At(second, 4) + require.NoError(t, err) + assert.Equal(t, 2, v) + v, err = s.At(second, 5) + require.NoError(t, err) + assert.Equal(t, 2, v) + v, err = s.At(second, 6) + require.NoError(t, err) + assert.Equal(t, 3, v) + v, err = s.At(second, 7) + require.NoError(t, err) + assert.Equal(t, 2, v) + _, err = s.At(second, 8) + assert.ErrorContains(t, "index 8 out of bounds", err) +} + +func TestUpdateAt(t *testing.T) { + // What we want to check: + // - shared value is updated only for the updated object, creating a new individual value (shared value remains the same) + // - individual value (different for both objects) is updated to a third value + // - individual value (different for both objects) is updated to the other object's value + // - individual value (equal for both objects) is updated + // - individual value existing only for the updated object is updated + // - individual value existing only for the other-object appends an item to the individual value + // - individual value updated to the original shared value removes that individual value + // - appended value (different for both objects) is updated to a third value + // - appended value (different for both objects) is updated to the other object's value + // - appended value (equal for both objects) is updated + // - appended value existing for one object is updated + // - ERROR when index too large in general + // - ERROR when index not too large in general, but too large for an object + + s := setup() + first := &testObject{id: id1, slice: s} + second := &testObject{id: id2, slice: s} + + require.NoError(t, s.UpdateAt(first, 0, 999)) + assert.Equal(t, 123, s.sharedItems[0]) + assertIndividualFound(t, s, first.id, 0, 999) + assertIndividualNotFound(t, s, second.id, 0) + + require.NoError(t, s.UpdateAt(first, 1, 999)) + assertIndividualFound(t, s, first.id, 1, 999) + assertIndividualFound(t, s, second.id, 1, 2) + + require.NoError(t, s.UpdateAt(first, 1, 2)) + assertIndividualFound(t, s, first.id, 1, 2) + assertIndividualFound(t, s, second.id, 1, 2) + + require.NoError(t, s.UpdateAt(first, 2, 999)) + assertIndividualFound(t, s, first.id, 2, 999) + assertIndividualFound(t, s, second.id, 2, 3) + + require.NoError(t, s.UpdateAt(first, 3, 999)) + assertIndividualFound(t, s, first.id, 3, 999) + assertIndividualNotFound(t, s, second.id, 3) + + require.NoError(t, s.UpdateAt(first, 4, 999)) + assertIndividualFound(t, s, first.id, 4, 999) + assertIndividualFound(t, s, second.id, 4, 2) + + require.NoError(t, s.UpdateAt(first, 4, 123)) + assertIndividualNotFound(t, s, first.id, 4) + assertIndividualFound(t, s, second.id, 4, 2) + + require.NoError(t, s.UpdateAt(first, 5, 999)) + assertAppendedFound(t, s, first.id, 0, 999) + assertAppendedFound(t, s, second.id, 0, 2) + + require.NoError(t, s.UpdateAt(first, 5, 2)) + assertAppendedFound(t, s, first.id, 0, 2) + assertAppendedFound(t, s, second.id, 0, 2) + + require.NoError(t, s.UpdateAt(first, 6, 999)) + assertAppendedFound(t, s, first.id, 1, 999) + assertAppendedFound(t, s, second.id, 1, 3) + + // we update the second object because there are no more appended items for the first object + require.NoError(t, s.UpdateAt(second, 7, 999)) + assertAppendedNotFound(t, s, first.id, 2) + assertAppendedFound(t, s, second.id, 2, 999) + + assert.ErrorContains(t, "index 7 out of bounds", s.UpdateAt(first, 7, 999)) + assert.ErrorContains(t, "index 8 out of bounds", s.UpdateAt(second, 8, 999)) +} + +func TestAppend(t *testing.T) { + // What we want to check: + // - appending first item ever to the slice + // - appending an item to an object when there is no corresponding item for the other object + // - appending an item to an object when there is a corresponding item with same value for the other object + // - appending an item to an object when there is a corresponding item with different value for the other object + // - we also want to check that cached length is properly updated after every append + + // we want to start with the simplest slice possible + s := &Slice[int, *testObject]{} + s.Init([]int{0}) + first := &testObject{id: id1, slice: s} + second := &testObject{id: id2, slice: s} + + // append first value ever + s.Append(first, 1) + require.Equal(t, 1, len(s.appendedItems)) + assertAppendedFound(t, s, first.id, 0, 1) + assertAppendedNotFound(t, s, second.id, 0) + l, ok := s.cachedLengths[first.id] + require.Equal(t, true, ok) + assert.Equal(t, 2, l) + _, ok = s.cachedLengths[second.id] + assert.Equal(t, false, ok) + + // append one more value to the first object, so that we can test two append scenarios for the second object + s.Append(first, 1) + + // append the first value to the second object, equal to the value for the first object + s.Append(second, 1) + require.Equal(t, 2, len(s.appendedItems)) + assertAppendedFound(t, s, first.id, 0, 1) + assertAppendedFound(t, s, second.id, 0, 1) + l, ok = s.cachedLengths[first.id] + require.Equal(t, true, ok) + assert.Equal(t, 3, l) + l, ok = s.cachedLengths[second.id] + assert.Equal(t, true, ok) + assert.Equal(t, 2, l) + + // append the first value to the second object, different than the value for the first object + s.Append(second, 2) + require.Equal(t, 2, len(s.appendedItems)) + assertAppendedFound(t, s, first.id, 1, 1) + assertAppendedFound(t, s, second.id, 1, 2) + l, ok = s.cachedLengths[first.id] + require.Equal(t, true, ok) + assert.Equal(t, 3, l) + l, ok = s.cachedLengths[second.id] + assert.Equal(t, true, ok) + assert.Equal(t, 3, l) +} + +func TestDetach(t *testing.T) { + // What we want to check: + // - no individual or appended items left after detaching an object + // - length removed from cache + + s := setup() + obj := &testObject{id: id1, slice: s} + + s.Detach(obj) + + for _, item := range s.individualItems { + found := false + for _, v := range item.Values { + for _, o := range v.ids { + if o == obj.id { + found = true + } + } + } + assert.Equal(t, false, found) + } + for _, item := range s.appendedItems { + found := false + for _, v := range item.Values { + for _, o := range v.ids { + if o == obj.id { + found = true + } + } + } + assert.Equal(t, false, found) + } + _, ok := s.cachedLengths[obj.id] + assert.Equal(t, false, ok) +} + +// Share the slice between 2 objects. +// Index 0: Shared value +// Index 1: Different individual value +// Index 2: Same individual value +// Index 3: Individual value ONLY for the first object +// Index 4: Individual value ONLY for the second object +// Index 5: Different appended value +// Index 6: Same appended value +// Index 7: Appended value ONLY for the second object +func setup() *Slice[int, *testObject] { + s := &Slice[int, *testObject]{} + s.Init([]int{123, 123, 123, 123, 123}) + s.individualItems[1] = &MultiValueItem[int]{ + Values: []*Value[int]{ + { + val: 1, + ids: []uuid.UUID{id1}, + }, + { + val: 2, + ids: []uuid.UUID{id2}, + }, + }, + } + s.individualItems[2] = &MultiValueItem[int]{ + Values: []*Value[int]{ + { + val: 3, + ids: []uuid.UUID{id1, id2}, + }, + }, + } + s.individualItems[3] = &MultiValueItem[int]{ + Values: []*Value[int]{ + { + val: 1, + ids: []uuid.UUID{id1}, + }, + }, + } + s.individualItems[4] = &MultiValueItem[int]{ + Values: []*Value[int]{ + { + val: 2, + ids: []uuid.UUID{id2}, + }, + }, + } + s.appendedItems = []*MultiValueItem[int]{ + { + Values: []*Value[int]{ + { + val: 1, + ids: []uuid.UUID{id1}, + }, + { + val: 2, + ids: []uuid.UUID{id2}, + }, + }, + }, + { + Values: []*Value[int]{ + { + val: 3, + ids: []uuid.UUID{id1, id2}, + }, + }, + }, + { + Values: []*Value[int]{ + { + val: 2, + ids: []uuid.UUID{id2}, + }, + }, + }, + } + s.cachedLengths[id1] = 7 + s.cachedLengths[id2] = 8 + + return s +} + +func assertIndividualFound(t *testing.T, slice *Slice[int, *testObject], id uuid.UUID, itemIndex uint64, expected int) { + found := false + for _, v := range slice.individualItems[itemIndex].Values { + for _, o := range v.ids { + if o == id { + found = true + assert.Equal(t, expected, v.val) + } + } + } + assert.Equal(t, true, found) +} + +func assertIndividualNotFound(t *testing.T, slice *Slice[int, *testObject], id uuid.UUID, itemIndex uint64) { + found := false + for _, v := range slice.individualItems[itemIndex].Values { + for _, o := range v.ids { + if o == id { + found = true + } + } + } + assert.Equal(t, false, found) +} + +func assertAppendedFound(t *testing.T, slice *Slice[int, *testObject], id uuid.UUID, itemIndex uint64, expected int) { + found := false + for _, v := range slice.appendedItems[itemIndex].Values { + for _, o := range v.ids { + if o == id { + found = true + assert.Equal(t, expected, v.val) + } + } + } + assert.Equal(t, true, found) +} + +func assertAppendedNotFound(t *testing.T, slice *Slice[int, *testObject], id uuid.UUID, itemIndex uint64) { + found := false + for _, v := range slice.appendedItems[itemIndex].Values { + for _, o := range v.ids { + if o == id { + found = true + } + } + } + assert.Equal(t, false, found) +} + +func BenchmarkValue(b *testing.B) { + const _100k = 100000 + const _1m = 1000000 + const _10m = 10000000 + + b.Run("100,000 shared items", func(b *testing.B) { + s := &Slice[int, *testObject]{} + s.Init(make([]int, _100k)) + for i := 0; i < b.N; i++ { + s.Value(&testObject{}) + } + }) + b.Run("100,000 equal individual items", func(b *testing.B) { + s := &Slice[int, *testObject]{} + s.Init(make([]int, _100k)) + s.individualItems[0] = &MultiValueItem[int]{Values: []*Value[int]{{val: 999, ids: []uuid.UUID{}}}} + objs := make([]*testObject, _100k) + for i := 0; i < len(objs); i++ { + id := uuid.New() + objs[i] = &testObject{id: id, slice: s} + s.individualItems[0].Values[0].ids = append(s.individualItems[0].Values[0].ids, id) + } + for i := 0; i < b.N; i++ { + s.Value(objs[rand.Intn(_100k)]) + } + }) + b.Run("100,000 different individual items", func(b *testing.B) { + s := &Slice[int, *testObject]{} + s.Init(make([]int, _100k)) + objs := make([]*testObject, _100k) + for i := 0; i < len(objs); i++ { + id := uuid.New() + objs[i] = &testObject{id: id, slice: s} + s.individualItems[uint64(i)] = &MultiValueItem[int]{Values: []*Value[int]{{val: i, ids: []uuid.UUID{id}}}} + } + for i := 0; i < b.N; i++ { + s.Value(objs[rand.Intn(_100k)]) + } + }) + b.Run("100,000 shared items and 100,000 equal appended items", func(b *testing.B) { + s := &Slice[int, *testObject]{} + s.Init(make([]int, _100k)) + s.appendedItems = []*MultiValueItem[int]{{Values: []*Value[int]{{val: 999, ids: []uuid.UUID{}}}}} + objs := make([]*testObject, _100k) + for i := 0; i < len(objs); i++ { + id := uuid.New() + objs[i] = &testObject{id: id, slice: s} + s.appendedItems[0].Values[0].ids = append(s.appendedItems[0].Values[0].ids, id) + } + for i := 0; i < b.N; i++ { + s.Value(objs[rand.Intn(_100k)]) + } + }) + b.Run("100,000 shared items and 100,000 different appended items", func(b *testing.B) { + s := &Slice[int, *testObject]{} + s.Init(make([]int, _100k)) + s.appendedItems = []*MultiValueItem[int]{} + objs := make([]*testObject, _100k) + for i := 0; i < len(objs); i++ { + id := uuid.New() + objs[i] = &testObject{id: id, slice: s} + s.appendedItems = append(s.appendedItems, &MultiValueItem[int]{Values: []*Value[int]{{val: i, ids: []uuid.UUID{id}}}}) + } + for i := 0; i < b.N; i++ { + s.Value(objs[rand.Intn(_100k)]) + } + }) + b.Run("1,000,000 shared items", func(b *testing.B) { + s := &Slice[int, *testObject]{} + s.Init(make([]int, _1m)) + for i := 0; i < b.N; i++ { + s.Value(&testObject{}) + } + }) + b.Run("1,000,000 equal individual items", func(b *testing.B) { + s := &Slice[int, *testObject]{} + s.Init(make([]int, _1m)) + s.individualItems[0] = &MultiValueItem[int]{Values: []*Value[int]{{val: 999, ids: []uuid.UUID{}}}} + objs := make([]*testObject, _1m) + for i := 0; i < len(objs); i++ { + id := uuid.New() + objs[i] = &testObject{id: id, slice: s} + s.individualItems[0].Values[0].ids = append(s.individualItems[0].Values[0].ids, id) + } + for i := 0; i < b.N; i++ { + s.Value(objs[rand.Intn(_1m)]) + } + }) + b.Run("1,000,000 different individual items", func(b *testing.B) { + s := &Slice[int, *testObject]{} + s.Init(make([]int, _1m)) + objs := make([]*testObject, _1m) + for i := 0; i < len(objs); i++ { + id := uuid.New() + objs[i] = &testObject{id: id, slice: s} + s.individualItems[uint64(i)] = &MultiValueItem[int]{Values: []*Value[int]{{val: i, ids: []uuid.UUID{id}}}} + } + for i := 0; i < b.N; i++ { + s.Value(objs[rand.Intn(_1m)]) + } + }) + b.Run("1,000,000 shared items and 1,000,000 equal appended items", func(b *testing.B) { + s := &Slice[int, *testObject]{} + s.Init(make([]int, _1m)) + s.appendedItems = []*MultiValueItem[int]{{Values: []*Value[int]{{val: 999, ids: []uuid.UUID{}}}}} + objs := make([]*testObject, _1m) + for i := 0; i < len(objs); i++ { + id := uuid.New() + objs[i] = &testObject{id: id, slice: s} + s.appendedItems[0].Values[0].ids = append(s.appendedItems[0].Values[0].ids, id) + } + for i := 0; i < b.N; i++ { + s.Value(objs[rand.Intn(_1m)]) + } + }) + b.Run("1,000,000 shared items and 1,000,000 different appended items", func(b *testing.B) { + s := &Slice[int, *testObject]{} + s.Init(make([]int, _1m)) + s.appendedItems = []*MultiValueItem[int]{} + objs := make([]*testObject, _1m) + for i := 0; i < len(objs); i++ { + id := uuid.New() + objs[i] = &testObject{id: id, slice: s} + s.appendedItems = append(s.appendedItems, &MultiValueItem[int]{Values: []*Value[int]{{val: i, ids: []uuid.UUID{id}}}}) + } + for i := 0; i < b.N; i++ { + s.Value(objs[rand.Intn(_1m)]) + } + }) + b.Run("10,000,000 shared items", func(b *testing.B) { + s := &Slice[int, *testObject]{} + s.Init(make([]int, _10m)) + for i := 0; i < b.N; i++ { + s.Value(&testObject{}) + } + }) + b.Run("10,000,000 equal individual items", func(b *testing.B) { + s := &Slice[int, *testObject]{} + s.Init(make([]int, _10m)) + s.individualItems[0] = &MultiValueItem[int]{Values: []*Value[int]{{val: 999, ids: []uuid.UUID{}}}} + objs := make([]*testObject, _10m) + for i := 0; i < len(objs); i++ { + id := uuid.New() + objs[i] = &testObject{id: id, slice: s} + s.individualItems[0].Values[0].ids = append(s.individualItems[0].Values[0].ids, id) + } + for i := 0; i < b.N; i++ { + s.Value(objs[rand.Intn(_10m)]) + } + }) + b.Run("10,000,000 different individual items", func(b *testing.B) { + s := &Slice[int, *testObject]{} + s.Init(make([]int, _10m)) + objs := make([]*testObject, _10m) + for i := 0; i < len(objs); i++ { + id := uuid.New() + objs[i] = &testObject{id: id, slice: s} + s.individualItems[uint64(i)] = &MultiValueItem[int]{Values: []*Value[int]{{val: i, ids: []uuid.UUID{id}}}} + } + for i := 0; i < b.N; i++ { + s.Value(objs[rand.Intn(_10m)]) + } + }) + b.Run("10,000,000 shared items and 10,000,000 equal appended items", func(b *testing.B) { + s := &Slice[int, *testObject]{} + s.Init(make([]int, _10m)) + s.appendedItems = []*MultiValueItem[int]{{Values: []*Value[int]{{val: 999, ids: []uuid.UUID{}}}}} + objs := make([]*testObject, _10m) + for i := 0; i < len(objs); i++ { + id := uuid.New() + objs[i] = &testObject{id: id, slice: s} + s.appendedItems[0].Values[0].ids = append(s.appendedItems[0].Values[0].ids, id) + } + for i := 0; i < b.N; i++ { + s.Value(objs[rand.Intn(_10m)]) + } + }) + b.Run("10,000,000 shared items and 10,000,000 different appended items", func(b *testing.B) { + s := &Slice[int, *testObject]{} + s.Init(make([]int, _10m)) + s.appendedItems = []*MultiValueItem[int]{} + objs := make([]*testObject, _10m) + for i := 0; i < len(objs); i++ { + id := uuid.New() + objs[i] = &testObject{id: id, slice: s} + s.appendedItems = append(s.appendedItems, &MultiValueItem[int]{Values: []*Value[int]{{val: i, ids: []uuid.UUID{id}}}}) + } + for i := 0; i < b.N; i++ { + s.Value(objs[rand.Intn(_10m)]) + } + }) +}