refactor genesis state flag handling, support url (#10449)

* refactor genesis state flag handling, support url

* lint fix

Co-authored-by: Kasey Kirkham <kasey@users.noreply.github.com>
Co-authored-by: Raul Jordan <raul@prysmaticlabs.com>

api/client/beacon/client.go

@@ -132,6 +132,15 @@ func validHostname(h string) (string, error) {
 	return fmt.Sprintf("%s:%s", host, port), nil
 }
 
+// NodeURL returns a human-readable string representation of the beacon node base url.
+func (c *Client) NodeURL() string {
+	u := &url.URL{
+		Scheme: c.scheme,
+		Host:   c.host,
+	}
+	return u.String()
+}
+
 func (c *Client) urlForPath(methodPath string) *url.URL {
 	u := &url.URL{
 		Scheme: c.scheme,

beacon-chain/db/iface/interface.go

@@ -99,7 +99,7 @@ type HeadAccessDatabase interface {
 	SaveHeadBlockRoot(ctx context.Context, blockRoot [32]byte) error
 
 	// Genesis operations.
-	LoadGenesis(ctx context.Context, r io.Reader) error
+	LoadGenesis(ctx context.Context, stateBytes []byte) error
 	SaveGenesisData(ctx context.Context, state state.BeaconState) error
 	EnsureEmbeddedGenesis(ctx context.Context) error
 

beacon-chain/db/kv/genesis.go

@@ -4,8 +4,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io"
-	"io/ioutil"
 
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
@@ -54,14 +52,10 @@ func (s *Store) SaveGenesisData(ctx context.Context, genesisState state.BeaconSt
 	return nil
 }
 
-// LoadGenesis loads a genesis state from a given file path, if no genesis exists already.
-func (s *Store) LoadGenesis(ctx context.Context, r io.Reader) error {
-	b, err := ioutil.ReadAll(r)
-	if err != nil {
-		return err
-	}
+// LoadGenesis loads a genesis state from a ssz-serialized byte slice, if no genesis exists already.
+func (s *Store) LoadGenesis(ctx context.Context, sb []byte) error {
 	st := &ethpb.BeaconState{}
-	if err := st.UnmarshalSSZ(b); err != nil {
+	if err := st.UnmarshalSSZ(sb); err != nil {
 		return err
 	}
 	gs, err := statev1.InitializeFromProtoUnsafe(st)

beacon-chain/db/kv/genesis_test.go

@@ -53,21 +53,16 @@ func TestLoadGenesisFromFile(t *testing.T) {
 	if err == nil {
 		fp = rfp
 	}
-
-	r, err := os.Open(fp)
+	sb, err := os.ReadFile(fp)
 	assert.NoError(t, err)
-	defer func() {
-		assert.NoError(t, r.Close())
-	}()
 
 	db := setupDB(t)
-	assert.NoError(t, db.LoadGenesis(context.Background(), r))
+	assert.NoError(t, db.LoadGenesis(context.Background(), sb))
 	testGenesisDataSaved(t, db)
 
 	// Loading the same genesis again should not throw an error
-	_, err = r.Seek(0, 0)
 	assert.NoError(t, err)
-	assert.NoError(t, db.LoadGenesis(context.Background(), r))
+	assert.NoError(t, db.LoadGenesis(context.Background(), sb))
 }
 
 func TestLoadGenesisFromFile_mismatchedForkVersion(t *testing.T) {
@@ -76,15 +71,12 @@ func TestLoadGenesisFromFile_mismatchedForkVersion(t *testing.T) {
 	if err == nil {
 		fp = rfp
 	}
-	r, err := os.Open(fp)
+	sb, err := os.ReadFile(fp)
 	assert.NoError(t, err)
-	defer func() {
-		assert.NoError(t, r.Close())
-	}()
 
 	// Loading a genesis with the wrong fork version as beacon config should throw an error.
 	db := setupDB(t)
-	assert.ErrorContains(t, "does not match config genesis fork version", db.LoadGenesis(context.Background(), r))
+	assert.ErrorContains(t, "does not match config genesis fork version", db.LoadGenesis(context.Background(), sb))
 }
 
 func TestEnsureEmbeddedGenesis(t *testing.T) {

beacon-chain/node/BUILD.bazel

@@ -43,6 +43,7 @@ go_library(
         "//beacon-chain/sync:go_default_library",
         "//beacon-chain/sync/backfill:go_default_library",
         "//beacon-chain/sync/checkpoint:go_default_library",
+        "//beacon-chain/sync/genesis:go_default_library",
         "//beacon-chain/sync/initial-sync:go_default_library",
         "//cmd:go_default_library",
         "//cmd/beacon-chain/flags:go_default_library",

beacon-chain/node/node.go

@@ -46,6 +46,7 @@ import (
 	regularsync "github.com/prysmaticlabs/prysm/beacon-chain/sync"
 	"github.com/prysmaticlabs/prysm/beacon-chain/sync/backfill"
 	"github.com/prysmaticlabs/prysm/beacon-chain/sync/checkpoint"
+	"github.com/prysmaticlabs/prysm/beacon-chain/sync/genesis"
 	initialsync "github.com/prysmaticlabs/prysm/beacon-chain/sync/initial-sync"
 	"github.com/prysmaticlabs/prysm/cmd"
 	"github.com/prysmaticlabs/prysm/cmd/beacon-chain/flags"
@@ -104,6 +105,7 @@ type BeaconNode struct {
 	finalizedStateAtStartUp state.BeaconState
 	serviceFlagOpts         *serviceFlagOpts
 	blockchainFlagOpts      []blockchain.Option
+	GenesisInitializer      genesis.Initializer
 	CheckpointInitializer   checkpoint.Initializer
 }
 
@@ -381,20 +383,10 @@ func (b *BeaconNode) startDB(cliCtx *cli.Context, depositAddress string) error {
 	if err != nil {
 		return errors.Wrap(err, "could not create deposit cache")
 	}
-
 	b.depositCache = depositCache
 
-	if cliCtx.IsSet(flags.GenesisStatePath.Name) {
-		r, err := os.Open(cliCtx.String(flags.GenesisStatePath.Name))
-		if err != nil {
-			return err
-		}
-		defer func() {
-			if err := r.Close(); err != nil {
-				log.WithError(err).Error("Failed to close genesis file")
-			}
-		}()
-		if err := b.db.LoadGenesis(b.ctx, r); err != nil {
+	if b.GenesisInitializer != nil {
+		if err := b.GenesisInitializer.Initialize(b.ctx, d); err != nil {
 			if err == db.ErrExistingGenesisState {
 				return errors.New("Genesis state flag specified but a genesis state " +
 					"exists already. Run again with --clear-db and/or ensure you are using the " +
@@ -403,6 +395,7 @@ func (b *BeaconNode) startDB(cliCtx *cli.Context, depositAddress string) error {
 			return errors.Wrap(err, "could not load genesis from file")
 		}
 	}
+
 	if err := b.db.EnsureEmbeddedGenesis(b.ctx); err != nil {
 		return err
 	}

beacon-chain/sync/genesis/BUILD.bazel

@@ -0,0 +1,17 @@
+load("@prysm//tools/go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "api.go",
+        "file.go",
+    ],
+    importpath = "github.com/prysmaticlabs/prysm/beacon-chain/sync/genesis",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//api/client/beacon:go_default_library",
+        "//beacon-chain/db:go_default_library",
+        "//io/file:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
+    ],
+)

beacon-chain/sync/genesis/api.go

@@ -0,0 +1,35 @@
+package genesis
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/api/client/beacon"
+	"github.com/prysmaticlabs/prysm/beacon-chain/db"
+)
+
+// APIInitializer manages initializing the genesis state and block to prepare the beacon node for syncing.
+// The genesis state is retrieved from the remote beacon node api, using the debug state retrieval endpoint.
+type APIInitializer struct {
+	c *beacon.Client
+}
+
+// NewAPIInitializer creates an APIInitializer, handling the set up of a beacon node api client
+// using the provided host string.
+func NewAPIInitializer(beaconNodeHost string) (*APIInitializer, error) {
+	c, err := beacon.NewClient(beaconNodeHost)
+	if err != nil {
+		return nil, errors.Wrapf(err, "unable to parse beacon node url or hostname - %s", beaconNodeHost)
+	}
+	return &APIInitializer{c: c}, nil
+}
+
+// Initialize downloads origin state and block for checkpoint sync and initializes database records to
+// prepare the node to begin syncing from that point.
+func (dl *APIInitializer) Initialize(ctx context.Context, d db.Database) error {
+	sb, err := dl.c.GetState(ctx, beacon.IdGenesis)
+	if err != nil {
+		return errors.Wrapf(err, "Error retrieving genesis state from %s", dl.c.NodeURL())
+	}
+	return d.LoadGenesis(ctx, sb)
+}

beacon-chain/sync/genesis/file.go

@@ -0,0 +1,59 @@
+package genesis
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/prysmaticlabs/prysm/io/file"
+
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/beacon-chain/db"
+)
+
+// Initializer describes a type that is able to obtain the checkpoint sync data (BeaconState and SignedBeaconBlock)
+// in some way and perform database setup to prepare the beacon node for syncing from the given checkpoint.
+// See FileInitializer and APIInitializer.
+type Initializer interface {
+	Initialize(ctx context.Context, d db.Database) error
+}
+
+// NewFileInitializer validates the given path information and creates an Initializer which will
+// use the provided state and block files to prepare the node for checkpoint sync.
+func NewFileInitializer(statePath string) (*FileInitializer, error) {
+	var err error
+	if err = existsAndIsFile(statePath); err != nil {
+		return nil, err
+	}
+	// stat just to make sure it actually exists and is a file
+	return &FileInitializer{statePath: statePath}, nil
+}
+
+// FileInitializer initializes a beacon-node database genesis state and block
+// using ssz-encoded state data stored in files on the local filesystem.
+type FileInitializer struct {
+	statePath string
+}
+
+// Initialize is called in the BeaconNode db startup code if an Initializer is present.
+// Initialize prepares the beacondb using the provided genesis state.
+func (fi *FileInitializer) Initialize(ctx context.Context, d db.Database) error {
+	serState, err := file.ReadFileAsBytes(fi.statePath)
+	if err != nil {
+		return errors.Wrapf(err, "error reading state file %s for checkpoint sync init", fi.statePath)
+	}
+	return d.LoadGenesis(ctx, serState)
+}
+
+var _ Initializer = &FileInitializer{}
+
+func existsAndIsFile(path string) error {
+	info, err := os.Stat(path)
+	if err != nil {
+		return errors.Wrapf(err, "error checking existence of ssz-encoded file %s for genesis state init", path)
+	}
+	if info.IsDir() {
+		return fmt.Errorf("%s is a directory, please specify full path to file", path)
+	}
+	return nil
+}

cmd/beacon-chain/BUILD.bazel

@@ -22,6 +22,7 @@ go_library(
         "//cmd/beacon-chain/flags:go_default_library",
         "//cmd/beacon-chain/powchain:go_default_library",
         "//cmd/beacon-chain/sync/checkpoint:go_default_library",
+        "//cmd/beacon-chain/sync/genesis:go_default_library",
         "//config/features:go_default_library",
         "//io/file:go_default_library",
         "//io/logs:go_default_library",

cmd/beacon-chain/flags/base.go

@@ -191,12 +191,6 @@ var (
 		Usage: "Sets the maximum number of headers that a deposit log query can fetch.",
 		Value: uint64(1000),
 	}
-	// GenesisStatePath defines a flag to start the beacon chain from a give genesis state file.
-	GenesisStatePath = &cli.StringFlag{
-		Name: "genesis-state",
-		Usage: "Load a genesis state from ssz file. Testnet genesis files can be found in the " +
-			"eth2-clients/eth2-testnets repository on github.",
-	}
 	// WeakSubjectivityCheckpoint defines the weak subjectivity checkpoint the node must sync through to defend against long range attacks.
 	WeakSubjectivityCheckpoint = &cli.StringFlag{
 		Name: "weak-subjectivity-checkpoint",

cmd/beacon-chain/main.go

@@ -18,6 +18,7 @@ import (
 	"github.com/prysmaticlabs/prysm/cmd/beacon-chain/flags"
 	powchaincmd "github.com/prysmaticlabs/prysm/cmd/beacon-chain/powchain"
 	"github.com/prysmaticlabs/prysm/cmd/beacon-chain/sync/checkpoint"
+	"github.com/prysmaticlabs/prysm/cmd/beacon-chain/sync/genesis"
 	"github.com/prysmaticlabs/prysm/config/features"
 	"github.com/prysmaticlabs/prysm/io/file"
 	"github.com/prysmaticlabs/prysm/io/logs"
@@ -65,7 +66,6 @@ var appFlags = []cli.Flag{
 	flags.NetworkID,
 	flags.WeakSubjectivityCheckpoint,
 	flags.Eth1HeaderReqLimit,
-	flags.GenesisStatePath,
 	flags.MinPeersPerSubnet,
 	flags.SuggestedFeeRecipient,
 	cmd.EnableBackupWebhookFlag,
@@ -122,6 +122,8 @@ var appFlags = []cli.Flag{
 	checkpoint.BlockPath,
 	checkpoint.StatePath,
 	checkpoint.RemoteURL,
+	genesis.StatePath,
+	genesis.BeaconAPIURL,
 }
 
 func init() {
@@ -246,13 +248,21 @@ func startNode(ctx *cli.Context) error {
 		node.WithBlockchainFlagOptions(blockchainFlagOpts),
 		node.WithPowchainFlagOptions(powchainFlagOpts),
 	}
-	cptOpts, err := checkpoint.BeaconNodeOptions(ctx)
-	if err != nil {
-		return err
+
+	optFuncs := []func(*cli.Context) (node.Option, error){
+		genesis.BeaconNodeOptions,
+		checkpoint.BeaconNodeOptions,
 	}
-	if cptOpts != nil {
-		opts = append(opts, cptOpts)
+	for _, of := range optFuncs {
+		ofo, err := of(ctx)
+		if err != nil {
+			return err
+		}
+		if ofo != nil {
+			opts = append(opts, ofo)
+		}
 	}
+
 	beacon, err := node.New(ctx, opts...)
 	if err != nil {
 		return err

cmd/beacon-chain/sync/genesis/BUILD.bazel

@@ -0,0 +1,14 @@
+load("@prysm//tools/go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["options.go"],
+    importpath = "github.com/prysmaticlabs/prysm/cmd/beacon-chain/sync/genesis",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//beacon-chain/node:go_default_library",
+        "//beacon-chain/sync/genesis:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
+        "@com_github_urfave_cli_v2//:go_default_library",
+    ],
+)

cmd/beacon-chain/sync/genesis/options.go

@@ -0,0 +1,53 @@
+package genesis
+
+import (
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/beacon-chain/node"
+	"github.com/prysmaticlabs/prysm/beacon-chain/sync/genesis"
+	"github.com/urfave/cli/v2"
+)
+
+var (
+	// StatePath defines a flag to start the beacon chain from a give genesis state file.
+	StatePath = &cli.PathFlag{
+		Name: "genesis-state",
+		Usage: "Load a genesis state from ssz file. Testnet genesis files can be found in the " +
+			"eth2-clients/eth2-testnets repository on github.",
+	}
+	BeaconAPIURL = &cli.StringFlag{
+		Name: "genesis-beacon-api-url",
+		Usage: "URL of a synced beacon node to trust for obtaining genesis state. " +
+			"As an additional safety measure, it is strongly recommended to only use this option in conjunction with " +
+			"--weak-subjectivity-checkpoint flag",
+	}
+)
+
+// BeaconNodeOptions is responsible for determining if the checkpoint sync options have been used, and if so,
+// reading the block and state ssz-serialized values from the filesystem locations specified and preparing a
+// checkpoint.Initializer, which uses the provided io.ReadClosers to initialize the beacon node database.
+func BeaconNodeOptions(c *cli.Context) (node.Option, error) {
+	statePath := c.Path(StatePath.Name)
+	remoteURL := c.String(BeaconAPIURL.Name)
+	if remoteURL != "" {
+		return func(node *node.BeaconNode) error {
+			var err error
+			node.GenesisInitializer, err = genesis.NewAPIInitializer(remoteURL)
+			if err != nil {
+				return errors.Wrap(err, "error constructing beacon node api client for genesis state init")
+			}
+			return nil
+		}, nil
+	}
+
+	if statePath == "" {
+		return nil, nil
+	}
+
+	return func(node *node.BeaconNode) (err error) {
+		node.GenesisInitializer, err = genesis.NewFileInitializer(statePath)
+		if err != nil {
+			return errors.Wrap(err, "error preparing to initialize genesis db state from local ssz files")
+		}
+		return nil
+	}, nil
+}

cmd/beacon-chain/usage.go

@@ -8,6 +8,7 @@ import (
 	"github.com/prysmaticlabs/prysm/cmd"
 	"github.com/prysmaticlabs/prysm/cmd/beacon-chain/flags"
 	"github.com/prysmaticlabs/prysm/cmd/beacon-chain/sync/checkpoint"
+	"github.com/prysmaticlabs/prysm/cmd/beacon-chain/sync/genesis"
 	"github.com/prysmaticlabs/prysm/config/features"
 	"github.com/prysmaticlabs/prysm/runtime/debug"
 	"github.com/urfave/cli/v2"
@@ -76,9 +77,6 @@ var appHelpFlagGroups = []flagGroup{
 			cmd.BoltMMapInitialSizeFlag,
 			cmd.ValidatorMonitorIndicesFlag,
 			cmd.ApiTimeoutFlag,
-			checkpoint.BlockPath,
-			checkpoint.StatePath,
-			checkpoint.RemoteURL,
 		},
 	},
 	{
@@ -127,8 +125,12 @@ var appHelpFlagGroups = []flagGroup{
 			flags.NetworkID,
 			flags.WeakSubjectivityCheckpoint,
 			flags.Eth1HeaderReqLimit,
-			flags.GenesisStatePath,
 			flags.MinPeersPerSubnet,
+			checkpoint.BlockPath,
+			checkpoint.StatePath,
+			checkpoint.RemoteURL,
+			genesis.StatePath,
+			genesis.BeaconAPIURL,
 		},
 	},
 	{