Hardening Committee Cache for Runtime (#4270)

beacon-chain/blockchain/forkchoice/metrics.go

@@ -57,6 +57,10 @@ var (
 		Name: "beacon_current_validators",
 		Help: "Number of status=pending|active|exited|withdrawable validators in current epoch",
 	})
+	sigFailsToVerify = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "att_signature_failed_to_verify_with_cache",
+		Help: "Number of attestation signatures that failed to verify with cache on, but succeeded without cache",
+	})
 )
 
 func reportEpochMetrics(state *pb.BeaconState) {

beacon-chain/blockchain/forkchoice/process_attestation.go

@@ -179,7 +179,29 @@ func (s *Store) verifyAttestation(ctx context.Context, baseState *pb.BeaconState
 	if err != nil {
 		return nil, errors.Wrap(err, "could not convert attestation to indexed attestation")
 	}
+
 	if err := blocks.VerifyIndexedAttestation(ctx, baseState, indexedAtt); err != nil {
+
+		// TODO(3603): Delete the following signature verify fallback when issue 3603 closes.
+		// When signature fails to verify with committee cache enabled at run time,
+		// the following re-runs the same signature verify routine without cache in play.
+		// This provides extra assurance that committee cache can't break run time.
+		if err == blocks.ErrSigFailedToVerify {
+			committee, err = helpers.BeaconCommitteeWithoutCache(baseState, a.Data.Slot, a.Data.CommitteeIndex)
+			if err != nil {
+				return nil, errors.Wrap(err, "could not convert attestation to indexed attestation without cache")
+			}
+			indexedAtt, err = blocks.ConvertToIndexed(ctx, a, committee)
+			if err != nil {
+				return nil, errors.Wrap(err, "could not convert attestation to indexed attestation")
+			}
+			if err := blocks.VerifyIndexedAttestation(ctx, baseState, indexedAtt); err != nil {
+				return nil, errors.Wrap(err, "could not verify indexed attestation without cache")
+			}
+			sigFailsToVerify.Inc()
+			return indexedAtt, nil
+		}
+
 		return nil, errors.Wrap(err, "could not verify indexed attestation")
 	}
 	return indexedAtt, nil

beacon-chain/blockchain/forkchoice/process_block.go

@@ -136,7 +136,7 @@ func (s *Store) OnBlock(ctx context.Context, b *ethpb.BeaconBlock) error {
 		logEpochData(postState)
 		reportEpochMetrics(postState)
 
-		// Update committee shuffled indices at the end of every epoch
+		// Update committees cache at epoch boundary slot.
 		if featureconfig.Get().EnableNewCache {
 			if err := helpers.UpdateCommitteeCache(postState); err != nil {
 				return err
@@ -240,13 +240,6 @@ func (s *Store) OnBlockInitialSyncStateTransition(ctx context.Context, b *ethpb.
 	// Epoch boundary bookkeeping such as logging epoch summaries.
 	if helpers.IsEpochStart(postState.Slot) {
 		reportEpochMetrics(postState)
-
-		// Update committee shuffled indices at the end of every epoch
-		if featureconfig.Get().EnableNewCache {
-			if err := helpers.UpdateCommitteeCache(postState); err != nil {
-				return err
-			}
-		}
 	}
 
 	return nil

beacon-chain/cache/BUILD.bazel

@@ -41,6 +41,7 @@ go_test(
     deps = [
         "//proto/beacon/p2p/v1:go_default_library",
         "//proto/beacon/rpc/v1:go_default_library",
+        "//shared/bytesutil:go_default_library",
         "//shared/featureconfig:go_default_library",
         "//shared/hashutil:go_default_library",
         "//shared/params:go_default_library",

beacon-chain/cache/committee.go

@@ -2,7 +2,6 @@ package cache
 
 import (
 	"errors"
-	"strconv"
 	"sync"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -18,9 +17,10 @@ var (
 	// a Committee struct.
 	ErrNotCommittee = errors.New("object is not a committee struct")
 
-	// maxShuffledIndicesSize defines the max number of shuffled indices list can cache.
-	// 3 for previous, current epoch and next epoch.
-	maxShuffledIndicesSize = 3
+	// maxCommitteesCacheSize defines the max number of shuffled committees on per randao basis can cache.
+	// Due to reorgs, it's good to keep the old cache around for quickly switch over. 10 is a generous 
+	// cache size as it considers 3 concurrent branches over 3 epochs.
+	maxCommitteesCacheSize = 10
 
 	// CommitteeCacheMiss tracks the number of committee requests that aren't present in the cache.
 	CommitteeCacheMiss = promauto.NewCounter(prometheus.CounterOpts{
@@ -34,47 +34,47 @@ var (
 	})
 )
 
-// Committee defines the committee per epoch and index.
-type Committee struct {
-	CommitteeCount uint64
-	Epoch          uint64
-	Committee      []uint64
+// Committees defines the shuffled committees seed.
+type Committees struct {
+	CommitteeCount  uint64
+	Seed            [32]byte
+	ShuffledIndices []uint64
+	SortedIndices   []uint64
 }
 
-// CommitteeCache is a struct with 1 queue for looking up shuffled indices list by epoch and committee index.
+// CommitteeCache is a struct with 1 queue for looking up shuffled indices list by seed.
 type CommitteeCache struct {
 	CommitteeCache *cache.FIFO
 	lock           sync.RWMutex
 }
 
-// committeeKeyFn takes the epoch as the key to retrieve shuffled indices of a committee in a given epoch.
+// committeeKeyFn takes the seed as the key to retrieve shuffled indices of a committee in a given epoch.
 func committeeKeyFn(obj interface{}) (string, error) {
-	info, ok := obj.(*Committee)
+	info, ok := obj.(*Committees)
 	if !ok {
 		return "", ErrNotCommittee
 	}
 
-	return strconv.Itoa(int(info.Epoch)), nil
+	return key(info.Seed), nil
 }
 
-// NewCommitteeCache creates a new committee cache for storing/accessing shuffled indices of a committee.
-func NewCommitteeCache() *CommitteeCache {
+// NewCommitteesCache creates a new committee cache for storing/accessing shuffled indices of a committee.
+func NewCommitteesCache() *CommitteeCache {
 	return &CommitteeCache{
 		CommitteeCache: cache.NewFIFO(committeeKeyFn),
 	}
 }
 
-// ShuffledIndices fetches the shuffled indices by slot and committee index. Every list of indices
+// Committee fetches the shuffled indices by slot and committee index. Every list of indices
 // represent one committee. Returns true if the list exists with slot and committee index. Otherwise returns false, nil.
-func (c *CommitteeCache) ShuffledIndices(slot uint64, index uint64) ([]uint64, error) {
+func (c *CommitteeCache) Committee(slot uint64, seed [32]byte, index uint64) ([]uint64, error) {
 	if !featureconfig.Get().EnableShuffledIndexCache && !featureconfig.Get().EnableNewCache {
 		return nil, nil
 	}
 	c.lock.RLock()
 	defer c.lock.RUnlock()
 
-	epoch := int(slot / params.BeaconConfig().SlotsPerEpoch)
-	obj, exists, err := c.CommitteeCache.GetByKey(strconv.Itoa(epoch))
+	obj, exists, err := c.CommitteeCache.GetByKey(key(seed))
 	if err != nil {
 		return nil, err
 	}
@@ -86,7 +86,7 @@ func (c *CommitteeCache) ShuffledIndices(slot uint64, index uint64) ([]uint64, e
 		return nil, nil
 	}
 
-	item, ok := obj.(*Committee)
+	item, ok := obj.(*Committees)
 	if !ok {
 		return nil, ErrNotCommittee
 	}
@@ -98,100 +98,36 @@ func (c *CommitteeCache) ShuffledIndices(slot uint64, index uint64) ([]uint64, e
 
 	indexOffSet := index + (slot%params.BeaconConfig().SlotsPerEpoch)*committeeCountPerSlot
 	start, end := startEndIndices(item, indexOffSet)
-	return item.Committee[start:end], nil
+
+	return item.ShuffledIndices[start:end], nil
 }
 
 // AddCommitteeShuffledList adds Committee shuffled list object to the cache. T
 // his method also trims the least recently list if the cache size has ready the max cache size limit.
-func (c *CommitteeCache) AddCommitteeShuffledList(committee *Committee) error {
+func (c *CommitteeCache) AddCommitteeShuffledList(committees *Committees) error {
 	if !featureconfig.Get().EnableShuffledIndexCache && !featureconfig.Get().EnableNewCache {
 		return nil
 	}
 	c.lock.Lock()
 	defer c.lock.Unlock()
-	if err := c.CommitteeCache.AddIfNotPresent(committee); err != nil {
+
+	if err := c.CommitteeCache.AddIfNotPresent(committees); err != nil {
 		return err
 	}
-	trim(c.CommitteeCache, maxShuffledIndicesSize)
+
+	trim(c.CommitteeCache, maxCommitteesCacheSize)
 	return nil
 }
 
-// Epochs returns the epochs stored in the committee cache. These are the keys to the cache.
-func (c *CommitteeCache) Epochs() ([]uint64, error) {
-	if !featureconfig.Get().EnableShuffledIndexCache {
-		return nil, nil
-	}
-	c.lock.RLock()
-	defer c.lock.RUnlock()
-
-	epochs := make([]uint64, len(c.CommitteeCache.ListKeys()))
-	for i, s := range c.CommitteeCache.ListKeys() {
-		epoch, err := strconv.Atoi(s)
-		if err != nil {
-			return nil, err
-		}
-		epochs[i] = uint64(epoch)
-	}
-	return epochs, nil
-}
-
-// EpochInCache returns true if an input epoch is part of keys in cache.
-func (c *CommitteeCache) EpochInCache(wantedEpoch uint64) (bool, error) {
-	if !featureconfig.Get().EnableShuffledIndexCache && !featureconfig.Get().EnableNewCache {
-		return false, nil
-	}
-	c.lock.RLock()
-	defer c.lock.RUnlock()
-
-	for _, s := range c.CommitteeCache.ListKeys() {
-		epoch, err := strconv.Atoi(s)
-		if err != nil {
-			return false, err
-		}
-		if wantedEpoch == uint64(epoch) {
-			return true, nil
-		}
-	}
-	return false, nil
-}
-
-// CommitteeCountPerSlot returns the number of committees in a given slot as stored in cache.
-func (c *CommitteeCache) CommitteeCountPerSlot(slot uint64) (uint64, bool, error) {
-	if !featureconfig.Get().EnableShuffledIndexCache && !featureconfig.Get().EnableNewCache {
-		return 0, false, nil
-	}
-	c.lock.RLock()
-	defer c.lock.RUnlock()
-	epoch := int(slot / params.BeaconConfig().SlotsPerEpoch)
-	obj, exists, err := c.CommitteeCache.GetByKey(strconv.Itoa(int(epoch)))
-	if err != nil {
-		return 0, false, err
-	}
-
-	if exists {
-		CommitteeCacheHit.Inc()
-	} else {
-		CommitteeCacheMiss.Inc()
-		return 0, false, nil
-	}
-
-	item, ok := obj.(*Committee)
-	if !ok {
-		return 0, false, ErrNotCommittee
-	}
-
-	return item.CommitteeCount / params.BeaconConfig().SlotsPerEpoch, true, nil
-}
-
-// ActiveIndices returns the active indices of a given epoch stored in cache.
-func (c *CommitteeCache) ActiveIndices(epoch uint64) ([]uint64, error) {
+// ActiveIndices returns the active indices of a given seed stored in cache.
+func (c *CommitteeCache) ActiveIndices(seed [32]byte) ([]uint64, error) {
 	if !featureconfig.Get().EnableShuffledIndexCache && !featureconfig.Get().EnableNewCache {
 		return nil, nil
 	}
 
 	c.lock.RLock()
 	defer c.lock.RUnlock()
-	obj, exists, err := c.CommitteeCache.GetByKey(strconv.Itoa(int(epoch)))
+	obj, exists, err := c.CommitteeCache.GetByKey(key(seed))
 	if err != nil {
 		return nil, err
 	}
@@ -203,18 +139,25 @@ func (c *CommitteeCache) ActiveIndices(epoch uint64) ([]uint64, error) {
 		return nil, nil
 	}
 
-	item, ok := obj.(*Committee)
+	item, ok := obj.(*Committees)
 	if !ok {
 		return nil, ErrNotCommittee
 	}
 
-	return item.Committee, nil
+	return item.SortedIndices, nil
 }
 
-func startEndIndices(c *Committee, index uint64) (uint64, uint64) {
-	validatorCount := uint64(len(c.Committee))
+func startEndIndices(c *Committees, index uint64) (uint64, uint64) {
+	validatorCount := uint64(len(c.ShuffledIndices))
 	start := sliceutil.SplitOffset(validatorCount, c.CommitteeCount, index)
 	end := sliceutil.SplitOffset(validatorCount, c.CommitteeCount, index+1)
-
 	return start, end
 }
+
+// Using seed as source for key to handle reorgs in the same epoch.
+// The seed is derived from state's array of randao mixes and epoch value
+// hashed together. This avoids collisions on different validator set. Spec definition:
+// https://github.com/ethereum/eth2.0-specs/blob/v0.9.2/specs/core/0_beacon-chain.md#get_seed
+func key(seed [32]byte) string {
+	return string(seed[:])
+}

beacon-chain/cache/committee_test.go

@@ -2,25 +2,27 @@ package cache
 
 import (
 	"reflect"
+	"sort"
 	"strconv"
 	"testing"
 
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
 func TestCommitteeKeyFn_OK(t *testing.T) {
-	item := &Committee{
-		Epoch:          999,
-		CommitteeCount: 1,
-		Committee:      []uint64{1, 2, 3, 4, 5},
+	item := &Committees{
+		CommitteeCount:  1,
+		Seed:            [32]byte{'A'},
+		ShuffledIndices: []uint64{1, 2, 3, 4, 5},
 	}
 
-	key, err := committeeKeyFn(item)
+	k, err := committeeKeyFn(item)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if key != strconv.Itoa(int(item.Epoch)) {
-		t.Errorf("Incorrect hash key: %s, expected %s", key, strconv.Itoa(int(item.Epoch)))
+	if k != key(item.Seed) {
+		t.Errorf("Incorrect hash k: %s, expected %s", k, key(item.Seed))
 	}
 }
 
@@ -32,17 +34,17 @@ func TestCommitteeKeyFn_InvalidObj(t *testing.T) {
 }
 
 func TestCommitteeCache_CommitteesByEpoch(t *testing.T) {
-	cache := NewCommitteeCache()
+	cache := NewCommitteesCache()
 
-	item := &Committee{
-		Epoch:          1,
-		Committee:      []uint64{1, 2, 3, 4, 5, 6},
-		CommitteeCount: 3,
+	item := &Committees{
+		ShuffledIndices: []uint64{1, 2, 3, 4, 5, 6},
+		Seed:            [32]byte{'A'},
+		CommitteeCount:  3,
 	}
 
-	slot := uint64(item.Epoch * params.BeaconConfig().SlotsPerEpoch)
+	slot := params.BeaconConfig().SlotsPerEpoch
 	committeeIndex := uint64(1)
-	indices, err := cache.ShuffledIndices(slot, committeeIndex)
+	indices, err := cache.Committee(slot, item.Seed, committeeIndex)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -54,102 +56,26 @@ func TestCommitteeCache_CommitteesByEpoch(t *testing.T) {
 		t.Fatal(err)
 	}
 	wantedIndex := uint64(0)
-	indices, err = cache.ShuffledIndices(slot, wantedIndex)
+	indices, err = cache.Committee(slot, item.Seed, wantedIndex)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	start, end := startEndIndices(item, wantedIndex)
-	if !reflect.DeepEqual(indices, item.Committee[start:end]) {
+	if !reflect.DeepEqual(indices, item.ShuffledIndices[start:end]) {
 		t.Errorf(
 			"Expected fetched active indices to be %v, got %v",
 			indices,
-			item.Committee[start:end],
+			item.ShuffledIndices[start:end],
 		)
 	}
 }
 
-func TestCommitteeCache_CanRotate(t *testing.T) {
-	cache := NewCommitteeCache()
-	item1 := &Committee{Epoch: 1}
-	if err := cache.AddCommitteeShuffledList(item1); err != nil {
-		t.Fatal(err)
-	}
-	item2 := &Committee{Epoch: 2}
-	if err := cache.AddCommitteeShuffledList(item2); err != nil {
-		t.Fatal(err)
-	}
-	epochs, err := cache.Epochs()
-	if err != nil {
-		t.Fatal(err)
-	}
-	wanted := item1.Epoch + item2.Epoch
-	if sum(epochs) != wanted {
-		t.Errorf("Wanted: %v, got: %v", wanted, sum(epochs))
-	}
-
-	item3 := &Committee{Epoch: 4}
-	if err := cache.AddCommitteeShuffledList(item3); err != nil {
-		t.Fatal(err)
-	}
-	epochs, err = cache.Epochs()
-	if err != nil {
-		t.Fatal(err)
-	}
-	wanted = item1.Epoch + item2.Epoch + item3.Epoch
-	if sum(epochs) != wanted {
-		t.Errorf("Wanted: %v, got: %v", wanted, sum(epochs))
-	}
-
-	item4 := &Committee{Epoch: 6}
-	if err := cache.AddCommitteeShuffledList(item4); err != nil {
-		t.Fatal(err)
-	}
-	epochs, err = cache.Epochs()
-	if err != nil {
-		t.Fatal(err)
-	}
-	wanted = item2.Epoch + item3.Epoch + item4.Epoch
-	if sum(epochs) != wanted {
-		t.Errorf("Wanted: %v, got: %v", wanted, sum(epochs))
-	}
-}
-
-func TestCommitteeCache_EpochInCache(t *testing.T) {
-	cache := NewCommitteeCache()
-	if err := cache.AddCommitteeShuffledList(&Committee{Epoch: 1}); err != nil {
-		t.Fatal(err)
-	}
-	if err := cache.AddCommitteeShuffledList(&Committee{Epoch: 2}); err != nil {
-		t.Fatal(err)
-	}
-	if err := cache.AddCommitteeShuffledList(&Committee{Epoch: 99}); err != nil {
-		t.Fatal(err)
-	}
-	if err := cache.AddCommitteeShuffledList(&Committee{Epoch: 100}); err != nil {
-		t.Fatal(err)
-	}
-	inCache, err := cache.EpochInCache(1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if inCache {
-		t.Error("Epoch shouldn't be in cache")
-	}
-	inCache, err = cache.EpochInCache(100)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !inCache {
-		t.Error("Epoch should be in cache")
-	}
-}
-
 func TestCommitteeCache_ActiveIndices(t *testing.T) {
-	cache := NewCommitteeCache()
+	cache := NewCommitteesCache()
 
-	item := &Committee{Epoch: 1, Committee: []uint64{1, 2, 3, 4, 5, 6}}
-	indices, err := cache.ActiveIndices(1)
+	item := &Committees{Seed: [32]byte{'A'}, SortedIndices: []uint64{1, 2, 3, 4, 5, 6}}
+	indices, err := cache.ActiveIndices(item.Seed)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -161,19 +87,41 @@ func TestCommitteeCache_ActiveIndices(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	indices, err = cache.ActiveIndices(1)
+	indices, err = cache.ActiveIndices(item.Seed)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(indices, item.Committee) {
+	if !reflect.DeepEqual(indices, item.SortedIndices) {
 		t.Error("Did not receive correct active indices from cache")
 	}
 }
 
-func sum(values []uint64) uint64 {
-	sum := uint64(0)
-	for _, v := range values {
-		sum = v + sum
+func TestCommitteeCache_CanRotate(t *testing.T) {
+	cache := NewCommitteesCache()
+
+	// Should rotate out all the epochs except 190 through 199.
+	for i := 100; i < 200; i++ {
+		s := []byte(strconv.Itoa(i))
+		item := &Committees{Seed: bytesutil.ToBytes32(s)}
+		if err := cache.AddCommitteeShuffledList(item); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	k := cache.CommitteeCache.ListKeys()
+	if len(k) != maxCommitteesCacheSize {
+		t.Errorf("wanted: %d, got: %d", maxCommitteesCacheSize, len(k))
+	}
+
+	sort.Slice(k, func(i, j int) bool {
+		return k[i] < k[j]
+	})
+	s := bytesutil.ToBytes32([]byte(strconv.Itoa(190)))
+	if k[0] != key(s) {
+		t.Error("incorrect key received for slot 190")
+	}
+	s = bytesutil.ToBytes32([]byte(strconv.Itoa(199)))
+	if k[len(k)-1] != key(s) {
+		t.Error("incorrect key received for slot 199")
 	}
-	return sum
 }

beacon-chain/core/blocks/block_operations_fuzz_test.go

@@ -34,4 +34,3 @@ func TestFuzzProcessBlockHeader_10000(t *testing.T) {
 		_, _ = blocks.ProcessBlockHeader(state, block)
 	}
 }
-

beacon-chain/core/helpers/committee.go

@@ -3,6 +3,7 @@ package helpers
 
 import (
 	"fmt"
+	"sort"
 
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
@@ -14,7 +15,7 @@ import (
 	"github.com/prysmaticlabs/prysm/shared/sliceutil"
 )
 
-var committeeCache = cache.NewCommitteeCache()
+var committeeCache = cache.NewCommitteesCache()
 
 // CommitteeCountAtSlot returns the number of crosslink committees of a slot.
 //
@@ -44,7 +45,7 @@ func CommitteeCountAtSlot(state *pb.BeaconState, slot uint64) (uint64, error) {
 	return committeePerSlot, nil
 }
 
-// BeaconCommittee returns the crosslink committee of a given epoch.
+// BeaconCommittee returns the crosslink committee of a given slot and committee index.
 //
 // Spec pseudocode definition:
 //   def get_beacon_committee(state: BeaconState, slot: Slot, index: CommitteeIndex) -> Sequence[ValidatorIndex]:
@@ -63,7 +64,12 @@ func CommitteeCountAtSlot(state *pb.BeaconState, slot uint64) (uint64, error) {
 func BeaconCommittee(state *pb.BeaconState, slot uint64, index uint64) ([]uint64, error) {
 	epoch := SlotToEpoch(slot)
 	if featureconfig.Get().EnableNewCache {
-		indices, err := committeeCache.ShuffledIndices(slot, index)
+		seed, err := Seed(state, epoch, params.BeaconConfig().DomainBeaconAttester)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not get seed")
+		}
+
+		indices, err := committeeCache.Committee(slot, seed, index)
 		if err != nil {
 			return nil, errors.Wrap(err, "could not interface with committee cache")
 		}
@@ -88,6 +94,39 @@ func BeaconCommittee(state *pb.BeaconState, slot uint64, index uint64) ([]uint64
 	if err != nil {
 		return nil, errors.Wrap(err, "could not get active indices")
 	}
+
+	if featureconfig.Get().EnableNewCache {
+		if err := UpdateCommitteeCache(state); err != nil {
+			return nil, errors.Wrap(err, "could not update committee cache")
+		}
+	}
+
+	return ComputeCommittee(indices, seed, epochOffset, count)
+}
+
+// BeaconCommitteeWithoutCache returns the crosslink committee of a given slot and committee index without the
+// usage of committee cache.
+// TODO(3603): Delete this function when issue 3603 closes.
+func BeaconCommitteeWithoutCache(state *pb.BeaconState, slot uint64, index uint64) ([]uint64, error) {
+	epoch := SlotToEpoch(slot)
+
+	committeesPerSlot, err := CommitteeCountAtSlot(state, slot)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get committee count at slot")
+	}
+	epochOffset := index + (slot%params.BeaconConfig().SlotsPerEpoch)*committeesPerSlot
+	count := committeesPerSlot * params.BeaconConfig().SlotsPerEpoch
+
+	seed, err := Seed(state, epoch, params.BeaconConfig().DomainBeaconAttester)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get seed")
+	}
+
+	indices, err := ActiveValidatorIndices(state, epoch)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get active indices")
+	}
+
 	return ComputeCommittee(indices, seed, epochOffset, count)
 }
 
@@ -266,9 +305,11 @@ func ShuffledIndices(state *pb.BeaconState, epoch uint64) ([]uint64, error) {
 		return nil, errors.Wrapf(err, "could not get seed for epoch %d", epoch)
 	}
 
-	indices, err := ActiveValidatorIndices(state, epoch)
-	if err != nil {
-		return nil, errors.Wrapf(err, "could not get active indices %d", epoch)
+	indices := make([]uint64, 0, len(state.Validators))
+	for i, v := range state.Validators {
+		if IsActiveValidator(v, epoch) {
+			indices = append(indices, uint64(i))
+		}
 	}
 
 	validatorCount := uint64(len(indices))
@@ -289,7 +330,7 @@ func ShuffledIndices(state *pb.BeaconState, epoch uint64) ([]uint64, error) {
 func UpdateCommitteeCache(state *pb.BeaconState) error {
 	currentEpoch := CurrentEpoch(state)
 	for _, epoch := range []uint64{currentEpoch, currentEpoch + 1} {
-		committees, err := ShuffledIndices(state, epoch)
+		shuffledIndices, err := ShuffledIndices(state, epoch)
 		if err != nil {
 			return err
 		}
@@ -297,13 +338,29 @@ func UpdateCommitteeCache(state *pb.BeaconState) error {
 		if err != nil {
 			return err
 		}
-		if err := committeeCache.AddCommitteeShuffledList(&cache.Committee{
-			Epoch:          epoch,
-			Committee:      committees,
-			CommitteeCount: count * params.BeaconConfig().SlotsPerEpoch,
+		seed, err := Seed(state, epoch, params.BeaconConfig().DomainBeaconAttester)
+		if err != nil {
+			return err
+		}
+
+		// Store the sorted indices as well as shuffled indices. In current spec,
+		// sorted indices is required to retrieve proposer index. This is also
+		// used for failing verify signature fallback.
+		sortedIndices := make([]uint64, len(shuffledIndices))
+		copy(sortedIndices, shuffledIndices)
+		sort.Slice(sortedIndices, func(i, j int) bool {
+			return sortedIndices[i] < sortedIndices[j]
+		})
+
+		if err := committeeCache.AddCommitteeShuffledList(&cache.Committees{
+			ShuffledIndices: shuffledIndices,
+			CommitteeCount:  count * params.BeaconConfig().SlotsPerEpoch,
+			Seed:            seed,
+			SortedIndices:   sortedIndices,
 		}); err != nil {
 			return err
 		}
 	}
+
 	return nil
 }

beacon-chain/core/helpers/committee_test.go

@@ -134,6 +134,67 @@ func TestAttestationParticipants_NoCommitteeCache(t *testing.T) {
 	}
 }
 
+func TestAttestingIndicesWithBeaconCommitteeWithoutCache_Ok(t *testing.T) {
+	committeeSize := uint64(16)
+	validators := make([]*ethpb.Validator, committeeSize*params.BeaconConfig().SlotsPerEpoch)
+	for i := 0; i < len(validators); i++ {
+		validators[i] = &ethpb.Validator{
+			ExitEpoch: params.BeaconConfig().FarFutureEpoch,
+		}
+	}
+
+	state := &pb.BeaconState{
+		Slot:        params.BeaconConfig().SlotsPerEpoch,
+		Validators:  validators,
+		RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
+	}
+
+	attestationData := &ethpb.AttestationData{}
+
+	tests := []struct {
+		attestationSlot uint64
+		bitfield        bitfield.Bitlist
+		wanted          []uint64
+	}{
+		{
+			attestationSlot: 3,
+			bitfield:        bitfield.Bitlist{0x07},
+			wanted:          []uint64{355, 416},
+		},
+		{
+			attestationSlot: 2,
+			bitfield:        bitfield.Bitlist{0x05},
+			wanted:          []uint64{447},
+		},
+		{
+			attestationSlot: 11,
+			bitfield:        bitfield.Bitlist{0x07},
+			wanted:          []uint64{67, 508},
+		},
+	}
+
+	for _, tt := range tests {
+		attestationData.Target = &ethpb.Checkpoint{Epoch: 0}
+		attestationData.Slot = tt.attestationSlot
+		committee, err := BeaconCommitteeWithoutCache(state, tt.attestationSlot, 0 /* committee index */)
+		if err != nil {
+			t.Error(err)
+		}
+		result, err := AttestingIndices(tt.bitfield, committee)
+		if err != nil {
+			t.Errorf("Failed to get attestation participants: %v", err)
+		}
+
+		if !reflect.DeepEqual(tt.wanted, result) {
+			t.Errorf(
+				"Result indices was an unexpected value. Wanted %d, got %d",
+				tt.wanted,
+				result,
+			)
+		}
+	}
+}
+
 func TestAttestationParticipants_EmptyBitfield(t *testing.T) {
 	validators := make([]*ethpb.Validator, params.BeaconConfig().MinGenesisActiveValidatorCount)
 	for i := 0; i < len(validators); i++ {
@@ -429,7 +490,7 @@ func TestShuffledIndices_ShuffleRightLength(t *testing.T) {
 
 func TestUpdateCommitteeCache_CanUpdate(t *testing.T) {
 	c := featureconfig.Get()
-	c.EnableShuffledIndexCache = true
+	c.EnableNewCache = true
 	featureconfig.Init(c)
 	defer featureconfig.Init(nil)
 
@@ -450,16 +511,15 @@ func TestUpdateCommitteeCache_CanUpdate(t *testing.T) {
 	if err := UpdateCommitteeCache(state); err != nil {
 		t.Fatal(err)
 	}
-	savedEpochs, err := committeeCache.Epochs()
+
+	epoch := uint64(1)
+	idx := uint64(1)
+	seed, err := Seed(state, epoch, params.BeaconConfig().DomainBeaconAttester)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if len(savedEpochs) != 2 {
-		t.Error("Did not save correct epoch lengths")
-	}
-	epoch := uint64(1)
-	idx := uint64(1)
-	indices, err = committeeCache.ShuffledIndices(epoch, idx)
+
+	indices, err = committeeCache.Committee(StartSlot(epoch), seed, idx)
 	if err != nil {
 		t.Fatal(err)
 	}

beacon-chain/core/helpers/validators.go

@@ -58,7 +58,11 @@ func IsSlashableValidator(validator *ethpb.Validator, epoch uint64) bool {
 //    return [ValidatorIndex(i) for i, v in enumerate(state.validators) if is_active_validator(v, epoch)]
 func ActiveValidatorIndices(state *pb.BeaconState, epoch uint64) ([]uint64, error) {
 	if featureconfig.Get().EnableNewCache {
-		activeIndices, err := committeeCache.ActiveIndices(epoch)
+		seed, err := Seed(state, epoch, params.BeaconConfig().DomainBeaconAttester)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not get seed")
+		}
+		activeIndices, err := committeeCache.ActiveIndices(seed)
 		if err != nil {
 			return nil, errors.Wrap(err, "could not interface with committee cache")
 		}
@@ -74,6 +78,12 @@ func ActiveValidatorIndices(state *pb.BeaconState, epoch uint64) ([]uint64, erro
 		}
 	}
 
+	if featureconfig.Get().EnableNewCache {
+		if err := UpdateCommitteeCache(state); err != nil {
+			return nil, errors.Wrap(err, "could not update committee cache")
+		}
+	}
+
 	return indices, nil
 }