Add Custom Deadline for Slot Progression (#8388)

* use custom deadline

* revert

* preston's comment

Co-authored-by: Preston Van Loon <preston@prysmaticlabs.com>
Co-authored-by: prylabs-bulldozer[bot] <58059840+prylabs-bulldozer[bot]@users.noreply.github.com>

beacon-chain/blockchain/process_block.go

@@ -3,6 +3,7 @@ package blockchain
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
@@ -18,6 +19,9 @@ import (
 	"go.opencensus.io/trace"
 )
 
+// A custom slot deadline for processing state slots in our cache.
+const slotDeadline = 5 * time.Second
+
 // This defines size of the upper bound for initial sync block cache.
 var initialSyncBlockCacheSize = 2 * params.BeaconConfig().SlotsPerEpoch
 
@@ -104,7 +108,12 @@ func (s *Service) onBlock(ctx context.Context, signed *ethpb.SignedBeaconBlock,
 	// Updating next slot state cache can happen in the background. It shouldn't block rest of the process.
 	if featureconfig.Get().EnableNextSlotStateCache {
 		go func() {
-			if err := state.UpdateNextSlotCache(ctx, blockRoot[:], postState); err != nil {
+			// Use a custom deadline here, since this method runs asynchronously.
+			// We ignore the parent method's context and instead create a new one
+			// with a custom deadline, therefore using the background context instead.
+			slotCtx, cancel := context.WithTimeout(context.Background(), slotDeadline)
+			defer cancel()
+			if err := state.UpdateNextSlotCache(slotCtx, blockRoot[:], postState); err != nil {
 				log.WithError(err).Debug("could not update next slot state cache")
 			}
 		}()