Add Check for Goroutines Count (#2608)

* changes

* revert ide

* goimports

* Update shared/cmd/flags.go

beacon-chain/blockchain/service.go

@@ -7,6 +7,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"runtime"
 	"sync"
 	"time"
 
@@ -49,6 +50,7 @@ type ChainService struct {
 	canonicalBlocks      map[uint64][]byte
 	canonicalBlocksLock  sync.RWMutex
 	receiveBlockLock     sync.Mutex
+	maxRoutines          int64
 }
 
 // Config options for the service.
@@ -60,6 +62,7 @@ type Config struct {
 	OpsPoolService operations.OperationFeeds
 	DevMode        bool
 	P2p            p2p.Broadcaster
+	MaxRoutines    int64
 }
 
 // NewChainService instantiates a new service instance that will
@@ -78,6 +81,7 @@ func NewChainService(ctx context.Context, cfg *Config) (*ChainService, error) {
 		stateInitializedFeed: new(event.Feed),
 		p2p:                  cfg.P2p,
 		canonicalBlocks:      make(map[uint64][]byte),
+		maxRoutines:          cfg.MaxRoutines,
 	}, nil
 }
 
@@ -195,6 +199,9 @@ func (c *ChainService) Stop() error {
 // Status always returns nil.
 // TODO(1202): Add service health checks.
 func (c *ChainService) Status() error {
+	if runtime.NumGoroutine() > int(c.maxRoutines) {
+		return fmt.Errorf("too many goroutines %d", runtime.NumGoroutine())
+	}
 	return nil
 }
 

beacon-chain/main.go

@@ -15,7 +15,7 @@ import (
 	"github.com/prysmaticlabs/prysm/shared/version"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
-	"github.com/x-cray/logrus-prefixed-formatter"
+	prefixed "github.com/x-cray/logrus-prefixed-formatter"
 )
 
 func startNode(ctx *cli.Context) error {
@@ -68,6 +68,7 @@ func main() {
 		cmd.DisableMonitoringFlag,
 		cmd.ClearDB,
 		cmd.LogFormat,
+		cmd.MaxGoroutines,
 		debug.PProfFlag,
 		debug.PProfAddrFlag,
 		debug.PProfPortFlag,

beacon-chain/node/node.go

@@ -194,7 +194,7 @@ func (b *BeaconNode) registerP2P(ctx *cli.Context) error {
 	return b.services.RegisterService(beaconp2p)
 }
 
-func (b *BeaconNode) registerBlockchainService(_ *cli.Context) error {
+func (b *BeaconNode) registerBlockchainService(ctx *cli.Context) error {
 	var web3Service *powchain.Web3Service
 	if err := b.services.FetchService(&web3Service); err != nil {
 		return err
@@ -211,6 +211,7 @@ func (b *BeaconNode) registerBlockchainService(_ *cli.Context) error {
 	if err := b.services.FetchService(&p2pService); err != nil {
 		return err
 	}
+	maxRoutines := ctx.GlobalInt64(cmd.MaxGoroutines.Name)
 
 	blockchainService, err := blockchain.NewChainService(context.Background(), &blockchain.Config{
 		BeaconDB:       b.db,
@@ -218,6 +219,7 @@ func (b *BeaconNode) registerBlockchainService(_ *cli.Context) error {
 		OpsPoolService: opsService,
 		AttsService:    attsService,
 		P2p:            p2pService,
+		MaxRoutines:    maxRoutines,
 	})
 	if err != nil {
 		return fmt.Errorf("could not register blockchain service: %v", err)

beacon-chain/usage.go

@@ -56,6 +56,7 @@ var appHelpFlagGroups = []flagGroup{
 			cmd.TraceSampleFractionFlag,
 			cmd.MonitoringPortFlag,
 			cmd.DisableMonitoringFlag,
+			cmd.MaxGoroutines,
 		},
 	},
 	{

shared/cmd/flags.go

@@ -105,4 +105,10 @@ var (
 		Usage: "Specify log formatting. Supports: text, json, fluentd.",
 		Value: "text",
 	}
+	// MaxGoroutines specifies the maximum amount of goroutines tolerated, before a status check fails.
+	MaxGoroutines = cli.Int64Flag{
+		Name:  "max-goroutines",
+		Usage: "Specifies the upper limit of goroutines running before a status check fails",
+		Value: 5000,
+	}
 )