From bfbf693660e01909d31ef5b02068218635ae1c69 Mon Sep 17 00:00:00 2001 From: kasey <489222+kasey@users.noreply.github.com> Date: Fri, 25 Mar 2022 12:18:03 -0500 Subject: [PATCH] Checkpoint Sync 3/5 - beacon node api client lib and prysmctl cli tool (#10385) Co-authored-by: Kasey Kirkham --- api/client/beacon/BUILD.bazel | 54 +++ api/client/beacon/checkpoint.go | 262 +++++++++++ api/client/beacon/checkpoint_test.go | 407 ++++++++++++++++ api/client/beacon/client.go | 437 ++++++++++++++++++ api/client/beacon/client_test.go | 58 +++ api/client/beacon/doc.go | 6 + api/client/beacon/errors.go | 13 + .../core/helpers/weak_subjectivity.go | 20 +- .../core/helpers/weak_subjectivity_test.go | 4 +- beacon-chain/rpc/eth/beacon/blocks.go | 3 +- cmd/prysmctl/BUILD.bazel | 20 + cmd/prysmctl/checkpoint/BUILD.bazel | 17 + cmd/prysmctl/checkpoint/checkpoint.go | 15 + cmd/prysmctl/checkpoint/latest.go | 55 +++ cmd/prysmctl/checkpoint/save.go | 71 +++ cmd/prysmctl/main.go | 25 + go.mod | 2 +- network/forks/errors.go | 4 + network/forks/ordered.go | 13 + network/forks/ordered_test.go | 20 + 20 files changed, 1492 insertions(+), 14 deletions(-) create mode 100644 api/client/beacon/BUILD.bazel create mode 100644 api/client/beacon/checkpoint.go create mode 100644 api/client/beacon/checkpoint_test.go create mode 100644 api/client/beacon/client.go create mode 100644 api/client/beacon/client_test.go create mode 100644 api/client/beacon/doc.go create mode 100644 api/client/beacon/errors.go create mode 100644 cmd/prysmctl/BUILD.bazel create mode 100644 cmd/prysmctl/checkpoint/BUILD.bazel create mode 100644 cmd/prysmctl/checkpoint/checkpoint.go create mode 100644 cmd/prysmctl/checkpoint/latest.go create mode 100644 cmd/prysmctl/checkpoint/save.go create mode 100644 cmd/prysmctl/main.go diff --git a/api/client/beacon/BUILD.bazel b/api/client/beacon/BUILD.bazel new file mode 100644 index 0000000000..2159d68ec0 --- /dev/null +++ b/api/client/beacon/BUILD.bazel @@ -0,0 +1,54 @@ +load("@prysm//tools/go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "checkpoint.go", + "client.go", + "doc.go", + "errors.go", + ], + importpath = "github.com/prysmaticlabs/prysm/api/client/beacon", + visibility = ["//visibility:public"], + deps = [ + "//beacon-chain/core/helpers:go_default_library", + "//beacon-chain/rpc/apimiddleware:go_default_library", + "//beacon-chain/state:go_default_library", + "//encoding/bytesutil:go_default_library", + "//encoding/ssz/detect:go_default_library", + "//io/file:go_default_library", + "//network/forks:go_default_library", + "//proto/prysm/v1alpha1:go_default_library", + "//proto/prysm/v1alpha1/block:go_default_library", + "//runtime/version:go_default_library", + "//time/slots:go_default_library", + "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library", + "@com_github_pkg_errors//:go_default_library", + "@com_github_prysmaticlabs_eth2_types//:go_default_library", + "@com_github_sirupsen_logrus//:go_default_library", + "@org_golang_x_mod//semver:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = [ + "checkpoint_test.go", + "client_test.go", + ], + embed = [":go_default_library"], + deps = [ + "//beacon-chain/state:go_default_library", + "//config/params:go_default_library", + "//encoding/ssz/detect:go_default_library", + "//network/forks:go_default_library", + "//proto/prysm/v1alpha1:go_default_library", + "//proto/prysm/v1alpha1/wrapper:go_default_library", + "//runtime/version:go_default_library", + "//testing/require:go_default_library", + "//testing/util:go_default_library", + "//time/slots:go_default_library", + "@com_github_pkg_errors//:go_default_library", + "@com_github_prysmaticlabs_eth2_types//:go_default_library", + ], +) diff --git a/api/client/beacon/checkpoint.go b/api/client/beacon/checkpoint.go new file mode 100644 index 0000000000..004bbc46bc --- /dev/null +++ b/api/client/beacon/checkpoint.go @@ -0,0 +1,262 @@ +package beacon + +import ( + "context" + "fmt" + "path" + + "github.com/pkg/errors" + types "github.com/prysmaticlabs/eth2-types" + "github.com/prysmaticlabs/prysm/beacon-chain/core/helpers" + "github.com/prysmaticlabs/prysm/beacon-chain/state" + "github.com/prysmaticlabs/prysm/encoding/ssz/detect" + "github.com/prysmaticlabs/prysm/io/file" + "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/block" + "github.com/prysmaticlabs/prysm/runtime/version" + "github.com/prysmaticlabs/prysm/time/slots" + log "github.com/sirupsen/logrus" + "golang.org/x/mod/semver" +) + +// OriginData represents the BeaconState and SignedBeaconBlock necessary to start an empty Beacon Node +// using Checkpoint Sync. +type OriginData struct { + wsd *WeakSubjectivityData + sb []byte + bb []byte + st state.BeaconState + b block.SignedBeaconBlock + cf *detect.VersionedUnmarshaler +} + +// CheckpointString returns the standard string representation of a Checkpoint for the block root and epoch for the +// SignedBeaconBlock value found by DownloadOriginData. +// The format is a a hex-encoded block root, followed by the epoch of the block, separated by a colon. For example: +// "0x1c35540cac127315fabb6bf29181f2ae0de1a3fc909d2e76ba771e61312cc49a:74888" +func (od *OriginData) CheckpointString() string { + return fmt.Sprintf("%#x:%d", od.wsd.BlockRoot, od.wsd.Epoch) +} + +// SaveBlock saves the downloaded block to a unique file in the given path. +// For readability and collision avoidance, the file name includes: type, config name, slot and root +func (od *OriginData) SaveBlock(dir string) (string, error) { + blockPath := path.Join(dir, fname("state", od.cf, od.st.Slot(), od.wsd.BlockRoot)) + return blockPath, file.WriteFile(blockPath, od.sb) +} + +// SaveState saves the downloaded state to a unique file in the given path. +// For readability and collision avoidance, the file name includes: type, config name, slot and root +func (od *OriginData) SaveState(dir string) (string, error) { + statePath := path.Join(dir, fname("state", od.cf, od.st.Slot(), od.wsd.StateRoot)) + return statePath, file.WriteFile(statePath, od.sb) +} + +// StateBytes returns the ssz-encoded bytes of the downloaded BeaconState value. +func (od *OriginData) StateBytes() []byte { + return od.sb +} + +// BlockBytes returns the ssz-encoded bytes of the downloaded SignedBeaconBlock value. +func (od *OriginData) BlockBytes() []byte { + return od.bb +} + +func fname(prefix string, cf *detect.VersionedUnmarshaler, slot types.Slot, root [32]byte) string { + return fmt.Sprintf("%s_%s_%s_%d-%#x.ssz", prefix, cf.Config.ConfigName, version.String(cf.Fork), slot, root) +} + +// this method downloads the head state, which can be used to find the correct chain config +// and use prysm's helper methods to compute the latest weak subjectivity epoch. +func getWeakSubjectivityEpochFromHead(ctx context.Context, client *Client) (types.Epoch, error) { + headBytes, err := client.GetState(ctx, IdHead) + if err != nil { + return 0, err + } + cf, err := detect.FromState(headBytes) + if err != nil { + return 0, errors.Wrap(err, "error detecting chain config for beacon state") + } + log.Printf("detected supported config in remote head state, name=%s, fork=%s", cf.Config.ConfigName, version.String(cf.Fork)) + headState, err := cf.UnmarshalBeaconState(headBytes) + if err != nil { + return 0, errors.Wrap(err, "error unmarshaling state to correct version") + } + + epoch, err := helpers.LatestWeakSubjectivityEpoch(ctx, headState, cf.Config) + if err != nil { + return 0, errors.Wrap(err, "error computing the weak subjectivity epoch from head state") + } + + log.Printf("(computed client-side) weak subjectivity epoch = %d", epoch) + return epoch, nil +} + +const ( + prysmMinimumVersion = "v2.0.7" + prysmImplementationName = "Prysm" +) + +// ErrUnsupportedPrysmCheckpointVersion indicates remote beacon node can't be used for checkpoint retrieval. +var ErrUnsupportedPrysmCheckpointVersion = errors.New("node does not meet minimum version requirements for checkpoint retrieval") + +// for older endpoints or clients that do not support the weak_subjectivity api method +// we gather the necessary data for a checkpoint sync by: +// - inspecting the remote server's head state and computing the weak subjectivity epoch locally +// - requesting the state at the first slot of the epoch +// - using hash_tree_root(state.latest_block_header) to compute the block the state integrates +// - requesting that block by its root +func downloadBackwardsCompatible(ctx context.Context, client *Client) (*OriginData, error) { + log.Print("falling back to generic checkpoint derivation, weak_subjectivity API not supported by server") + nv, err := client.GetNodeVersion(ctx) + if err != nil { + return nil, errors.Wrap(err, "unable to proceed with fallback method without confirming node version") + } + if nv.implementation == prysmImplementationName && semver.Compare(nv.semver, prysmMinimumVersion) < 0 { + return nil, errors.Wrapf(ErrUnsupportedPrysmCheckpointVersion, "%s < minimum (%s)", nv.semver, prysmMinimumVersion) + } + epoch, err := getWeakSubjectivityEpochFromHead(ctx, client) + if err != nil { + return nil, errors.Wrap(err, "error computing weak subjectivity epoch via head state inspection") + } + + // use first slot of the epoch for the state slot + slot, err := slots.EpochStart(epoch) + if err != nil { + return nil, errors.Wrapf(err, "error computing first slot of epoch=%d", epoch) + } + + log.Printf("requesting checkpoint state at slot %d", slot) + // get the state at the first slot of the epoch + stateBytes, err := client.GetState(ctx, IdFromSlot(slot)) + if err != nil { + return nil, errors.Wrapf(err, "failed to request state by slot from api, slot=%d", slot) + } + + // ConfigFork is used to unmarshal the BeaconState so we can read the block root in latest_block_header + cf, err := detect.FromState(stateBytes) + if err != nil { + return nil, errors.Wrap(err, "error detecting chain config for beacon state") + } + log.Printf("detected supported config in checkpoint state, name=%s, fork=%s", cf.Config.ConfigName, version.String(cf.Fork)) + + st, err := cf.UnmarshalBeaconState(stateBytes) + if err != nil { + return nil, errors.Wrap(err, "error using detected config fork to unmarshal state bytes") + } + + // compute state and block roots + stateRoot, err := st.HashTreeRoot(ctx) + if err != nil { + return nil, errors.Wrap(err, "error computing hash_tree_root of state") + } + + header := st.LatestBlockHeader() + header.StateRoot = stateRoot[:] + computedBlockRoot, err := header.HashTreeRoot() + if err != nil { + return nil, errors.Wrap(err, "error while computing block root using state data") + } + + blockBytes, err := client.GetBlock(ctx, IdFromRoot(computedBlockRoot)) + if err != nil { + return nil, errors.Wrapf(err, "error requesting block by root = %d", computedBlockRoot) + } + block, err := cf.UnmarshalBeaconBlock(blockBytes) + if err != nil { + return nil, errors.Wrap(err, "unable to unmarshal block to a supported type using the detected fork schedule") + } + blockRoot, err := block.Block().HashTreeRoot() + if err != nil { + return nil, errors.Wrap(err, "error computing hash_tree_root for block obtained via root") + } + + log.Printf("BeaconState slot=%d, Block slot=%d", st.Slot(), block.Block().Slot()) + log.Printf("BeaconState htr=%#xd, Block state_root=%#x", stateRoot, block.Block().StateRoot()) + log.Printf("BeaconBlock root computed from state=%#x, Block htr=%#x", computedBlockRoot, blockRoot) + + return &OriginData{ + wsd: &WeakSubjectivityData{ + BlockRoot: blockRoot, + StateRoot: stateRoot, + Epoch: epoch, + }, + st: st, + sb: stateBytes, + b: block, + bb: blockBytes, + cf: cf, + }, nil +} + +// DownloadOriginData attempts to use the proposed weak_subjectivity beacon node api +// to obtain the weak_subjectivity metadata (epoch, block_root, state_root) needed to sync +// a beacon node from the canonical weak subjectivity checkpoint. As this is a proposed API +// that will only be supported by prysm at first, in the event of a 404 we fallback to using a +// different technique where we first download the head state which can be used to compute the +// weak subjectivity epoch on the client side. +func DownloadOriginData(ctx context.Context, client *Client) (*OriginData, error) { + ws, err := client.GetWeakSubjectivity(ctx) + if err != nil { + // a 404/405 is expected if querying an endpoint that doesn't support the weak subjectivity checkpoint api + if !errors.Is(err, ErrNotOK) { + return nil, errors.Wrap(err, "unexpected API response for prysm-only weak subjectivity checkpoint API") + } + // fall back to vanilla Beacon Node API method + return downloadBackwardsCompatible(ctx, client) + } + log.Printf("server weak subjectivity checkpoint response - epoch=%d, block_root=%#x, state_root=%#x", ws.Epoch, ws.BlockRoot, ws.StateRoot) + + // use first slot of the epoch for the block slot + slot, err := slots.EpochStart(ws.Epoch) + if err != nil { + return nil, errors.Wrapf(err, "error computing first slot of epoch=%d", ws.Epoch) + } + log.Printf("requesting checkpoint state at slot %d", slot) + + stateBytes, err := client.GetState(ctx, IdFromSlot(slot)) + if err != nil { + return nil, errors.Wrapf(err, "failed to request state by slot from api, slot=%d", slot) + } + cf, err := detect.FromState(stateBytes) + if err != nil { + return nil, errors.Wrap(err, "error detecting chain config for beacon state") + } + log.Printf("detected supported config in checkpoint state, name=%s, fork=%s", cf.Config.ConfigName, version.String(cf.Fork)) + + state, err := cf.UnmarshalBeaconState(stateBytes) + if err != nil { + return nil, errors.Wrap(err, "error using detected config fork to unmarshal state bytes") + } + stateRoot, err := state.HashTreeRoot(ctx) + if err != nil { + return nil, errors.Wrapf(err, "failed to compute htr for state at slot=%d", slot) + } + + blockRoot, err := state.LatestBlockHeader().HashTreeRoot() + if err != nil { + return nil, errors.Wrap(err, "error computing hash_tree_root of latest_block_header") + } + blockBytes, err := client.GetBlock(ctx, IdFromRoot(ws.BlockRoot)) + if err != nil { + return nil, errors.Wrapf(err, "error requesting block by slot = %d", slot) + } + block, err := cf.UnmarshalBeaconBlock(blockBytes) + if err != nil { + return nil, errors.Wrap(err, "unable to unmarshal block to a supported type using the detected fork schedule") + } + realBlockRoot, err := block.Block().HashTreeRoot() + if err != nil { + return nil, errors.Wrap(err, "error computing hash_tree_root of retrieved block") + } + log.Printf("BeaconState slot=%d, Block slot=%d", state.Slot(), block.Block().Slot()) + log.Printf("BeaconState htr=%#xd, Block state_root=%#x", stateRoot, block.Block().StateRoot()) + log.Printf("BeaconState latest_block_header htr=%#xd, block htr=%#x", blockRoot, realBlockRoot) + return &OriginData{ + wsd: ws, + st: state, + b: block, + sb: stateBytes, + bb: blockBytes, + cf: cf, + }, nil +} diff --git a/api/client/beacon/checkpoint_test.go b/api/client/beacon/checkpoint_test.go new file mode 100644 index 0000000000..a4e0026d02 --- /dev/null +++ b/api/client/beacon/checkpoint_test.go @@ -0,0 +1,407 @@ +package beacon + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "testing" + + "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/wrapper" + + "github.com/prysmaticlabs/prysm/beacon-chain/state" + "github.com/prysmaticlabs/prysm/network/forks" + ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1" + "github.com/prysmaticlabs/prysm/testing/util" + "github.com/prysmaticlabs/prysm/time/slots" + + types "github.com/prysmaticlabs/eth2-types" + "github.com/prysmaticlabs/prysm/config/params" + "github.com/prysmaticlabs/prysm/encoding/ssz/detect" + "github.com/prysmaticlabs/prysm/runtime/version" + + "github.com/pkg/errors" + "github.com/prysmaticlabs/prysm/testing/require" +) + +type testRT struct { + rt func(*http.Request) (*http.Response, error) +} + +func (rt *testRT) RoundTrip(req *http.Request) (*http.Response, error) { + if rt.rt != nil { + return rt.rt(req) + } + return nil, errors.New("RoundTripper not implemented") +} + +var _ http.RoundTripper = &testRT{} + +func marshalToEnvelope(val interface{}) ([]byte, error) { + raw, err := json.Marshal(val) + if err != nil { + return nil, errors.Wrap(err, "error marshaling value to place in data envelope") + } + env := struct { + Data json.RawMessage `json:"data"` + }{ + Data: raw, + } + return json.Marshal(env) +} + +func TestMarshalToEnvelope(t *testing.T) { + d := struct { + Version string `json:"version"` + }{ + Version: "Prysm/v2.0.5 (linux amd64)", + } + encoded, err := marshalToEnvelope(d) + require.NoError(t, err) + expected := `{"data":{"version":"Prysm/v2.0.5 (linux amd64)"}}` + require.Equal(t, expected, string(encoded)) +} + +func TestFallbackVersionCheck(t *testing.T) { + c := &Client{ + hc: &http.Client{}, + host: "localhost:3500", + scheme: "http", + } + c.hc.Transport = &testRT{rt: func(req *http.Request) (*http.Response, error) { + res := &http.Response{Request: req} + switch req.URL.Path { + case getNodeVersionPath: + res.StatusCode = http.StatusOK + b := bytes.NewBuffer(nil) + d := struct { + Version string `json:"version"` + }{ + Version: "Prysm/v2.0.5 (linux amd64)", + } + encoded, err := marshalToEnvelope(d) + require.NoError(t, err) + b.Write(encoded) + res.Body = io.NopCloser(b) + case getWeakSubjectivityPath: + res.StatusCode = http.StatusNotFound + } + + return res, nil + }} + + ctx := context.Background() + _, err := DownloadOriginData(ctx, c) + require.ErrorIs(t, err, ErrUnsupportedPrysmCheckpointVersion) +} + +func TestFname(t *testing.T) { + vu := &detect.VersionedUnmarshaler{ + Config: params.MainnetConfig(), + Fork: version.Phase0, + } + slot := types.Slot(23) + prefix := "block" + var root [32]byte + copy(root[:], []byte{0x23, 0x23, 0x23}) + expected := "block_mainnet_phase0_23-0x2323230000000000000000000000000000000000000000000000000000000000.ssz" + actual := fname(prefix, vu, slot, root) + require.Equal(t, expected, actual) + + vu.Config = params.MinimalSpecConfig() + vu.Fork = version.Altair + slot = 17 + prefix = "state" + copy(root[29:], []byte{0x17, 0x17, 0x17}) + expected = "state_minimal_altair_17-0x2323230000000000000000000000000000000000000000000000000000171717.ssz" + actual = fname(prefix, vu, slot, root) + require.Equal(t, expected, actual) +} + +func TestDownloadOriginData(t *testing.T) { + ctx := context.Background() + cfg := params.MainnetConfig() + + epoch := cfg.AltairForkEpoch - 1 + // set up checkpoint state, using the epoch that will be computed as the ws checkpoint state based on the head state + wSlot, err := slots.EpochStart(epoch) + require.NoError(t, err) + wst, err := util.NewBeaconState() + require.NoError(t, err) + fork, err := forkForEpoch(cfg, epoch) + require.NoError(t, wst.SetFork(fork)) + + // set up checkpoint block + b, err := wrapper.WrappedSignedBeaconBlock(util.NewBeaconBlock()) + require.NoError(t, wrapper.SetBlockParentRoot(b, cfg.ZeroHash)) + require.NoError(t, wrapper.SetBlockSlot(b, wSlot)) + require.NoError(t, wrapper.SetProposerIndex(b, 0)) + + // set up state header pointing at checkpoint block - this is how the block is downloaded by root + header, err := b.Header() + require.NoError(t, err) + require.NoError(t, wst.SetLatestBlockHeader(header.Header)) + + // order of operations can be confusing here: + // - when computing the state root, make sure block header is complete, EXCEPT the state root should be zero-value + // - before computing the block root (to match the request route), the block should include the state root + // *computed from the state with a header that does not have a state root set yet* + wRoot, err := wst.HashTreeRoot(ctx) + require.NoError(t, err) + + require.NoError(t, wrapper.SetBlockStateRoot(b, wRoot)) + serBlock, err := b.MarshalSSZ() + require.NoError(t, err) + bRoot, err := b.Block().HashTreeRoot() + require.NoError(t, err) + + wsSerialized, err := wst.MarshalSSZ() + require.NoError(t, err) + expectedWSD := WeakSubjectivityData{ + BlockRoot: bRoot, + StateRoot: wRoot, + Epoch: epoch, + } + + hc := &http.Client{ + Transport: &testRT{rt: func(req *http.Request) (*http.Response, error) { + res := &http.Response{Request: req} + switch req.URL.Path { + case getWeakSubjectivityPath: + res.StatusCode = http.StatusOK + cp := struct { + Epoch string `json:"epoch"` + Root string `json:"root"` + }{ + Epoch: fmt.Sprintf("%d", slots.ToEpoch(b.Block().Slot())), + Root: fmt.Sprintf("%#x", bRoot), + } + wsr := struct { + Checkpoint interface{} `json:"ws_checkpoint"` + StateRoot string `json:"state_root"` + }{ + Checkpoint: cp, + StateRoot: fmt.Sprintf("%#x", wRoot), + } + rb, err := marshalToEnvelope(wsr) + require.NoError(t, err) + res.Body = io.NopCloser(bytes.NewBuffer(rb)) + case renderGetStatePath(IdFromSlot(wSlot)): + res.StatusCode = http.StatusOK + res.Body = io.NopCloser(bytes.NewBuffer(wsSerialized)) + case renderGetBlockPath(IdFromRoot(bRoot)): + res.StatusCode = http.StatusOK + res.Body = io.NopCloser(bytes.NewBuffer(serBlock)) + } + + return res, nil + }}, + } + c := &Client{ + hc: hc, + host: "localhost:3500", + scheme: "http", + } + + od, err := DownloadOriginData(ctx, c) + require.NoError(t, err) + require.Equal(t, expectedWSD.Epoch, od.wsd.Epoch) + require.Equal(t, expectedWSD.StateRoot, od.wsd.StateRoot) + require.Equal(t, expectedWSD.BlockRoot, od.wsd.BlockRoot) + require.DeepEqual(t, wsSerialized, od.sb) + require.DeepEqual(t, serBlock, od.bb) + require.DeepEqual(t, wst.Fork().CurrentVersion, od.cf.Version[:]) + require.DeepEqual(t, version.Phase0, od.cf.Fork) +} + +// runs downloadBackwardsCompatible directly +// and via DownloadOriginData with a round tripper that triggers the backwards compatible code path +func TestDownloadBackwardsCompatibleCombined(t *testing.T) { + ctx := context.Background() + cfg := params.MainnetConfig() + + st, expectedEpoch := defaultTestHeadState(t, cfg) + serialized, err := st.MarshalSSZ() + require.NoError(t, err) + + // set up checkpoint state, using the epoch that will be computed as the ws checkpoint state based on the head state + wSlot, err := slots.EpochStart(expectedEpoch) + require.NoError(t, err) + wst, err := util.NewBeaconState() + require.NoError(t, err) + fork, err := forkForEpoch(cfg, cfg.GenesisEpoch) + require.NoError(t, wst.SetFork(fork)) + + // set up checkpoint block + b, err := wrapper.WrappedSignedBeaconBlock(util.NewBeaconBlock()) + require.NoError(t, wrapper.SetBlockParentRoot(b, cfg.ZeroHash)) + require.NoError(t, wrapper.SetBlockSlot(b, wSlot)) + require.NoError(t, wrapper.SetProposerIndex(b, 0)) + + // set up state header pointing at checkpoint block - this is how the block is downloaded by root + header, err := b.Header() + require.NoError(t, err) + require.NoError(t, wst.SetLatestBlockHeader(header.Header)) + + // order of operations can be confusing here: + // - when computing the state root, make sure block header is complete, EXCEPT the state root should be zero-value + // - before computing the block root (to match the request route), the block should include the state root + // *computed from the state with a header that does not have a state root set yet* + wRoot, err := wst.HashTreeRoot(ctx) + require.NoError(t, err) + + require.NoError(t, wrapper.SetBlockStateRoot(b, wRoot)) + serBlock, err := b.MarshalSSZ() + require.NoError(t, err) + bRoot, err := b.Block().HashTreeRoot() + require.NoError(t, err) + + wsSerialized, err := wst.MarshalSSZ() + require.NoError(t, err) + + hc := &http.Client{ + Transport: &testRT{rt: func(req *http.Request) (*http.Response, error) { + res := &http.Response{Request: req} + switch req.URL.Path { + case getNodeVersionPath: + res.StatusCode = http.StatusOK + b := bytes.NewBuffer(nil) + d := struct { + Version string `json:"version"` + }{ + Version: "Lighthouse/v0.1.5 (Linux x86_64)", + } + encoded, err := marshalToEnvelope(d) + require.NoError(t, err) + b.Write(encoded) + res.Body = io.NopCloser(b) + case getWeakSubjectivityPath: + res.StatusCode = http.StatusNotFound + case renderGetStatePath(IdHead): + res.StatusCode = http.StatusOK + res.Body = io.NopCloser(bytes.NewBuffer(serialized)) + case renderGetStatePath(IdFromSlot(wSlot)): + res.StatusCode = http.StatusOK + res.Body = io.NopCloser(bytes.NewBuffer(wsSerialized)) + case renderGetBlockPath(IdFromRoot(bRoot)): + res.StatusCode = http.StatusOK + res.Body = io.NopCloser(bytes.NewBuffer(serBlock)) + } + + return res, nil + }}, + } + c := &Client{ + hc: hc, + host: "localhost:3500", + scheme: "http", + } + + odPub, err := DownloadOriginData(ctx, c) + require.NoError(t, err) + + odPriv, err := downloadBackwardsCompatible(ctx, c) + require.NoError(t, err) + require.DeepEqual(t, odPriv.wsd, odPub.wsd) + require.DeepEqual(t, odPriv.sb, odPub.sb) + require.DeepEqual(t, odPriv.bb, odPub.bb) + require.DeepEqual(t, odPriv.cf.Fork, odPub.cf.Fork) + require.DeepEqual(t, odPriv.cf.Version, odPub.cf.Version) +} + +func TestGetWeakSubjectivityEpochFromHead(t *testing.T) { + st, expectedEpoch := defaultTestHeadState(t, params.MainnetConfig()) + serialized, err := st.MarshalSSZ() + require.NoError(t, err) + hc := &http.Client{ + Transport: &testRT{rt: func(req *http.Request) (*http.Response, error) { + res := &http.Response{Request: req} + switch req.URL.Path { + case renderGetStatePath(IdHead): + res.StatusCode = http.StatusOK + res.Body = io.NopCloser(bytes.NewBuffer(serialized)) + } + return res, nil + }}, + } + c := &Client{ + hc: hc, + host: "localhost:3500", + scheme: "http", + } + actualEpoch, err := getWeakSubjectivityEpochFromHead(context.Background(), c) + require.NoError(t, err) + require.Equal(t, expectedEpoch, actualEpoch) +} + +func forkForEpoch(cfg *params.BeaconChainConfig, epoch types.Epoch) (*ethpb.Fork, error) { + os := forks.NewOrderedSchedule(cfg) + currentVersion, err := os.VersionForEpoch(epoch) + if err != nil { + return nil, err + } + prevVersion, err := os.Previous(currentVersion) + if err != nil { + if !errors.Is(err, forks.ErrNoPreviousVersion) { + return nil, err + } + // use same version for both in the case of genesis + prevVersion = currentVersion + } + forkEpoch := cfg.ForkVersionSchedule[currentVersion] + return ðpb.Fork{ + PreviousVersion: prevVersion[:], + CurrentVersion: currentVersion[:], + Epoch: forkEpoch, + }, nil +} + +func defaultTestHeadState(t *testing.T, cfg *params.BeaconChainConfig) (state.BeaconState, types.Epoch) { + st, err := util.NewBeaconStateAltair() + require.NoError(t, err) + + fork, err := forkForEpoch(cfg, cfg.AltairForkEpoch) + require.NoError(t, err) + require.NoError(t, st.SetFork(fork)) + + slot, err := slots.EpochStart(cfg.AltairForkEpoch) + require.NoError(t, err) + require.NoError(t, st.SetSlot(slot)) + + var validatorCount, avgBalance uint64 = 100, 35 + require.NoError(t, populateValidators(cfg, st, validatorCount, avgBalance)) + require.NoError(t, st.SetFinalizedCheckpoint(ðpb.Checkpoint{ + Epoch: fork.Epoch - 10, + Root: make([]byte, 32), + })) + // to see the math for this, look at helpers.LatestWeakSubjectivityEpoch + // and for the values use mainnet config values, the validatorCount and avgBalance above, and altair fork epoch + expectedEpoch := slots.ToEpoch(st.Slot()) - 224 + return st, expectedEpoch +} + +// TODO(10429): refactor beacon state options in testing/util to take a state.BeaconState so this can become an option +func populateValidators(cfg *params.BeaconChainConfig, st state.BeaconState, valCount, avgBalance uint64) error { + validators := make([]*ethpb.Validator, valCount) + balances := make([]uint64, len(validators)) + for i := uint64(0); i < valCount; i++ { + validators[i] = ðpb.Validator{ + PublicKey: make([]byte, cfg.BLSPubkeyLength), + WithdrawalCredentials: make([]byte, 32), + EffectiveBalance: avgBalance * 1e9, + ExitEpoch: cfg.FarFutureEpoch, + } + balances[i] = validators[i].EffectiveBalance + } + + if err := st.SetValidators(validators); err != nil { + return err + } + if err := st.SetBalances(balances); err != nil { + return err + } + + return nil +} diff --git a/api/client/beacon/client.go b/api/client/beacon/client.go new file mode 100644 index 0000000000..0d40781557 --- /dev/null +++ b/api/client/beacon/client.go @@ -0,0 +1,437 @@ +package beacon + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/url" + "path" + "regexp" + "sort" + "strconv" + "text/template" + "time" + + "github.com/prysmaticlabs/prysm/network/forks" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/pkg/errors" + types "github.com/prysmaticlabs/eth2-types" + "github.com/prysmaticlabs/prysm/beacon-chain/rpc/apimiddleware" + "github.com/prysmaticlabs/prysm/encoding/bytesutil" + ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1" + log "github.com/sirupsen/logrus" +) + +const ( + getSignedBlockPath = "/eth/v2/beacon/blocks" + getBlockRootPath = "/eth/v1/beacon/blocks/{{.Id}}/root" + getForkForStatePath = "/eth/v1/beacon/states/{{.Id}}/fork" + getWeakSubjectivityPath = "/eth/v1/beacon/weak_subjectivity" + getForkSchedulePath = "/eth/v1/config/fork_schedule" + getStatePath = "/eth/v2/debug/beacon/states" + getNodeVersionPath = "/eth/v1/node/version" +) + +// StateOrBlockId represents the block_id / state_id parameters that several of the Eth Beacon API methods accept. +// StateOrBlockId constants are defined for named identifiers, and helper methods are provided +// for slot and root identifiers. Example text from the Eth Beacon Node API documentation: +// +// "Block identifier can be one of: "head" (canonical head in node's view), "genesis", "finalized", +// , ." +type StateOrBlockId string + +const ( + IdFinalized StateOrBlockId = "finalized" + IdGenesis StateOrBlockId = "genesis" + IdHead StateOrBlockId = "head" + IdJustified StateOrBlockId = "justified" +) + +// IdFromRoot encodes a block root in the format expected by the API in places where a root can be used to identify +// a BeaconState or SignedBeaconBlock. +func IdFromRoot(r [32]byte) StateOrBlockId { + return StateOrBlockId(fmt.Sprintf("%#x", r)) +} + +// IdFromRoot encodes a Slot in the format expected by the API in places where a slot can be used to identify +// a BeaconState or SignedBeaconBlock. +func IdFromSlot(s types.Slot) StateOrBlockId { + return StateOrBlockId(strconv.FormatUint(uint64(s), 10)) +} + +// idTemplate is used to create template functions that can interpolate StateOrBlockId values. +func idTemplate(ts string) func(StateOrBlockId) string { + t := template.Must(template.New("").Parse(ts)) + f := func(id StateOrBlockId) string { + b := bytes.NewBuffer(nil) + err := t.Execute(b, struct{ Id string }{Id: string(id)}) + if err != nil { + panic(fmt.Sprintf("invalid idTemplate: %s", ts)) + } + return b.String() + } + // run the template to ensure that it is valid + // this should happen load time (using package scoped vars) to ensure runtime errors aren't possible + _ = f(IdGenesis) + return f +} + +// ClientOpt is a functional option for the Client type (http.Client wrapper) +type ClientOpt func(*Client) + +// WithTimeout sets the .Timeout attribute of the wrapped http.Client. +func WithTimeout(timeout time.Duration) ClientOpt { + return func(c *Client) { + c.hc.Timeout = timeout + } +} + +// Client provides a collection of helper methods for calling the Eth Beacon Node API endpoints. +type Client struct { + hc *http.Client + host string + scheme string +} + +// NewClient constructs a new client with the provided options (ex WithTimeout). +// `host` is the base host + port used to construct request urls. This value can be +// a URL string, or NewClient will assume an http endpoint if just `host:port` is used. +func NewClient(host string, opts ...ClientOpt) (*Client, error) { + host, err := validHostname(host) + if err != nil { + return nil, err + } + c := &Client{ + hc: &http.Client{}, + scheme: "http", + host: host, + } + for _, o := range opts { + o(c) + } + return c, nil +} + +func validHostname(h string) (string, error) { + // try to parse as url (being permissive) + u, err := url.Parse(h) + if err == nil && u.Host != "" { + return u.Host, nil + } + // try to parse as host:port + host, port, err := net.SplitHostPort(h) + if err != nil { + return "", err + } + return fmt.Sprintf("%s:%s", host, port), nil +} + +func (c *Client) urlForPath(methodPath string) *url.URL { + u := &url.URL{ + Scheme: c.scheme, + Host: c.host, + } + u.Path = path.Join(u.Path, methodPath) + return u +} + +type reqOption func(*http.Request) + +func withSSZEncoding() reqOption { + return func(req *http.Request) { + req.Header.Set("Accept", "application/octet-stream") + } +} + +// get is a generic, opinionated GET function to reduce boilerplate amongst the getters in this package. +func (c *Client) get(ctx context.Context, path string, opts ...reqOption) ([]byte, error) { + u := c.urlForPath(path) + log.Printf("requesting %s", u.String()) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + for _, o := range opts { + o(req) + } + r, err := c.hc.Do(req) + if err != nil { + return nil, err + } + defer func() { + err = r.Body.Close() + }() + if r.StatusCode != http.StatusOK { + return nil, non200Err(r) + } + b, err := io.ReadAll(r.Body) + if err != nil { + return nil, errors.Wrap(err, "error reading http response body from GetBlock") + } + return b, nil +} + +func renderGetBlockPath(id StateOrBlockId) string { + return path.Join(getSignedBlockPath, string(id)) +} + +// GetBlock retrieves the SignedBeaconBlock for the given block id. +// Block identifier can be one of: "head" (canonical head in node's view), "genesis", "finalized", +// , . Variables of type StateOrBlockId are exported by this package +// for the named identifiers. +// The return value contains the ssz-encoded bytes. +func (c *Client) GetBlock(ctx context.Context, blockId StateOrBlockId) ([]byte, error) { + blockPath := renderGetBlockPath(blockId) + b, err := c.get(ctx, blockPath, withSSZEncoding()) + if err != nil { + return nil, errors.Wrapf(err, "error requesting state by id = %s", blockId) + } + return b, nil +} + +var getBlockRootTpl = idTemplate(getBlockRootPath) + +// GetBlockRoot retrieves the hash_tree_root of the BeaconBlock for the given block id. +// Block identifier can be one of: "head" (canonical head in node's view), "genesis", "finalized", +// , . Variables of type StateOrBlockId are exported by this package +// for the named identifiers. +func (c *Client) GetBlockRoot(ctx context.Context, blockId StateOrBlockId) ([32]byte, error) { + rootPath := getBlockRootTpl(blockId) + b, err := c.get(ctx, rootPath) + if err != nil { + return [32]byte{}, errors.Wrapf(err, "error requesting block root by id = %s", blockId) + } + jsonr := &struct{ Data struct{ Root string } }{} + err = json.Unmarshal(b, jsonr) + if err != nil { + return [32]byte{}, errors.Wrap(err, "error decoding json data from get block root response") + } + rs, err := hexutil.Decode(jsonr.Data.Root) + if err != nil { + return [32]byte{}, errors.Wrap(err, fmt.Sprintf("error decoding hex-encoded value %s", jsonr.Data.Root)) + } + return bytesutil.ToBytes32(rs), nil +} + +var getForkTpl = idTemplate(getForkForStatePath) + +// GetFork queries the Beacon Node API for the Fork from the state identified by stateId. +// Block identifier can be one of: "head" (canonical head in node's view), "genesis", "finalized", +// , . Variables of type StateOrBlockId are exported by this package +// for the named identifiers. +func (c *Client) GetFork(ctx context.Context, stateId StateOrBlockId) (*ethpb.Fork, error) { + body, err := c.get(ctx, getForkTpl(stateId)) + if err != nil { + return nil, errors.Wrapf(err, "error requesting fork by state id = %s", stateId) + } + fr := &forkResponse{} + dataWrapper := &struct{ Data *forkResponse }{Data: fr} + err = json.Unmarshal(body, dataWrapper) + if err != nil { + return nil, errors.Wrap(err, "error decoding json response in GetFork") + } + + return fr.Fork() +} + +// GetForkSchedule retrieve all forks, past present and future, of which this node is aware. +func (c *Client) GetForkSchedule(ctx context.Context) (forks.OrderedSchedule, error) { + body, err := c.get(ctx, getForkSchedulePath) + if err != nil { + return nil, errors.Wrap(err, "error requesting fork schedule") + } + fsr := &forkScheduleResponse{} + err = json.Unmarshal(body, fsr) + if err != nil { + return nil, err + } + ofs, err := fsr.OrderedForkSchedule() + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("problem unmarshaling %s response", getForkSchedulePath)) + } + return ofs, nil +} + +type NodeVersion struct { + implementation string + semver string + systemInfo string +} + +var versionRE = regexp.MustCompile(`^(\w+)\/(v\d+\.\d+\.\d+) \((.*)\)$`) + +func parseNodeVersion(v string) (*NodeVersion, error) { + groups := versionRE.FindStringSubmatch(v) + if len(groups) != 4 { + return nil, errors.Wrapf(ErrInvalidNodeVersion, "could not be parsed: %s", v) + } + return &NodeVersion{ + implementation: groups[1], + semver: groups[2], + systemInfo: groups[3], + }, nil +} + +// GetNodeVersion requests that the beacon node identify information about its implementation in a format +// similar to a HTTP User-Agent field. ex: Lighthouse/v0.1.5 (Linux x86_64) +func (c *Client) GetNodeVersion(ctx context.Context) (*NodeVersion, error) { + b, err := c.get(ctx, getNodeVersionPath) + if err != nil { + return nil, errors.Wrap(err, "error requesting node version") + } + d := struct { + Data struct { + Version string `json:"version"` + } `json:"data"` + }{} + err = json.Unmarshal(b, &d) + if err != nil { + return nil, errors.Wrapf(err, "error unmarshaling response body: %s", string(b)) + } + return parseNodeVersion(d.Data.Version) +} + +func renderGetStatePath(id StateOrBlockId) string { + return path.Join(getStatePath, string(id)) +} + +// GetState retrieves the BeaconState for the given state id. +// State identifier can be one of: "head" (canonical head in node's view), "genesis", "finalized", +// , . Variables of type StateOrBlockId are exported by this package +// for the named identifiers. +// The return value contains the ssz-encoded bytes. +func (c *Client) GetState(ctx context.Context, stateId StateOrBlockId) ([]byte, error) { + statePath := path.Join(getStatePath, string(stateId)) + b, err := c.get(ctx, statePath, withSSZEncoding()) + if err != nil { + return nil, errors.Wrapf(err, "error requesting state by id = %s", stateId) + } + return b, nil +} + +// GetWeakSubjectivity calls a proposed API endpoint that is unique to prysm +// This api method does the following: +// - computes weak subjectivity epoch +// - finds the highest non-skipped block preceding the epoch +// - returns the htr of the found block and returns this + the value of state_root from the block +func (c *Client) GetWeakSubjectivity(ctx context.Context) (*WeakSubjectivityData, error) { + body, err := c.get(ctx, getWeakSubjectivityPath) + if err != nil { + return nil, err + } + v := &apimiddleware.WeakSubjectivityResponse{} + err = json.Unmarshal(body, v) + if err != nil { + return nil, err + } + epoch, err := strconv.ParseUint(v.Data.Checkpoint.Epoch, 10, 64) + if err != nil { + return nil, err + } + blockRoot, err := hexutil.Decode(v.Data.Checkpoint.Root) + if err != nil { + return nil, err + } + stateRoot, err := hexutil.Decode(v.Data.StateRoot) + if err != nil { + return nil, err + } + return &WeakSubjectivityData{ + Epoch: types.Epoch(epoch), + BlockRoot: bytesutil.ToBytes32(blockRoot), + StateRoot: bytesutil.ToBytes32(stateRoot), + }, nil +} + +// WeakSubjectivityData represents the state root, block root and epoch of the BeaconState + SignedBeaconBlock +// that falls at the beginning of the current weak subjectivity period. These values can be used to construct +// a weak subjectivity checkpoint, or to download a BeaconState+SignedBeaconBlock pair that can be used to bootstrap +// a new Beacon Node using Checkpoint Sync. +type WeakSubjectivityData struct { + BlockRoot [32]byte + StateRoot [32]byte + Epoch types.Epoch +} + +func non200Err(response *http.Response) error { + bodyBytes, err := ioutil.ReadAll(response.Body) + var body string + if err != nil { + body = "(Unable to read response body.)" + } else { + body = "response body:\n" + string(bodyBytes) + } + msg := fmt.Sprintf("code=%d, url=%s, body=%s", response.StatusCode, response.Request.URL, body) + switch response.StatusCode { + case 404: + return errors.Wrap(ErrNotFound, msg) + default: + return errors.Wrap(ErrNotOK, msg) + } +} + +type forkResponse struct { + PreviousVersion string `json:"previous_version"` + CurrentVersion string `json:"current_version"` + Epoch string `json:"epoch"` +} + +func (f *forkResponse) Fork() (*ethpb.Fork, error) { + epoch, err := strconv.ParseUint(f.Epoch, 10, 64) + if err != nil { + return nil, err + } + cSlice, err := hexutil.Decode(f.CurrentVersion) + if err != nil { + return nil, err + } + if len(cSlice) != 4 { + return nil, fmt.Errorf("got %d byte version for CurrentVersion, expected 4 bytes. hex=%s", len(cSlice), f.CurrentVersion) + } + pSlice, err := hexutil.Decode(f.PreviousVersion) + if err != nil { + return nil, err + } + if len(pSlice) != 4 { + return nil, fmt.Errorf("got %d byte version, expected 4 bytes. version hex=%s", len(pSlice), f.PreviousVersion) + } + return ðpb.Fork{ + CurrentVersion: cSlice, + PreviousVersion: pSlice, + Epoch: types.Epoch(epoch), + }, nil +} + +type forkScheduleResponse struct { + Data []forkResponse +} + +func (fsr *forkScheduleResponse) OrderedForkSchedule() (forks.OrderedSchedule, error) { + ofs := make(forks.OrderedSchedule, 0) + for _, d := range fsr.Data { + epoch, err := strconv.Atoi(d.Epoch) + if err != nil { + return nil, err + } + vSlice, err := hexutil.Decode(d.CurrentVersion) + if err != nil { + return nil, err + } + if len(vSlice) != 4 { + return nil, fmt.Errorf("got %d byte version, expected 4 bytes. version hex=%s", len(vSlice), d.CurrentVersion) + } + version := bytesutil.ToBytes4(vSlice) + ofs = append(ofs, forks.ForkScheduleEntry{ + Version: version, + Epoch: types.Epoch(uint64(epoch)), + }) + } + sort.Sort(ofs) + return ofs, nil +} diff --git a/api/client/beacon/client_test.go b/api/client/beacon/client_test.go new file mode 100644 index 0000000000..76a6c86ccd --- /dev/null +++ b/api/client/beacon/client_test.go @@ -0,0 +1,58 @@ +package beacon + +import ( + "testing" + + "github.com/prysmaticlabs/prysm/testing/require" +) + +func TestParseNodeVersion(t *testing.T) { + cases := []struct { + name string + v string + err error + nv *NodeVersion + }{ + { + name: "empty string", + v: "", + err: ErrInvalidNodeVersion, + }, + { + name: "Prysm as the version string", + v: "Prysm", + err: ErrInvalidNodeVersion, + }, + { + name: "semver only", + v: "v2.0.6", + err: ErrInvalidNodeVersion, + }, + { + name: "implementation and semver only", + v: "Prysm/v2.0.6", + err: ErrInvalidNodeVersion, + }, + { + name: "complete version", + v: "Prysm/v2.0.6 (linux amd64)", + nv: &NodeVersion{ + implementation: "Prysm", + semver: "v2.0.6", + systemInfo: "linux amd64", + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + nv, err := parseNodeVersion(c.v) + if c.err != nil { + require.ErrorIs(t, err, c.err) + } else { + require.NoError(t, err) + require.DeepEqual(t, c.nv, nv) + } + }) + } +} diff --git a/api/client/beacon/doc.go b/api/client/beacon/doc.go new file mode 100644 index 0000000000..ed7def487d --- /dev/null +++ b/api/client/beacon/doc.go @@ -0,0 +1,6 @@ +/* +Package beacon provides a client for interacting with the standard Eth Beacon Node API. +Interactive swagger documentation for the API is available here: https://ethereum.github.io/beacon-APIs/ + +*/ +package beacon diff --git a/api/client/beacon/errors.go b/api/client/beacon/errors.go new file mode 100644 index 0000000000..8ca4a292bc --- /dev/null +++ b/api/client/beacon/errors.go @@ -0,0 +1,13 @@ +package beacon + +import "github.com/pkg/errors" + +// ErrNotOK is used to indicate when an HTTP request to the Beacon Node API failed with any non-2xx response code. +// More specific errors may be returned, but an error in reaction to a non-2xx response will always wrap ErrNotOK. +var ErrNotOK = errors.New("did not receive 2xx response from API") + +// ErrNotFound specifically means that a '404 - NOT FOUND' response was received from the API. +var ErrNotFound = errors.Wrap(ErrNotOK, "recv 404 NotFound response from API") + +// ErrInvalidNodeVersion indicates that the /eth/v1/node/version api response format was not recognized. +var ErrInvalidNodeVersion = errors.New("invalid node version response") diff --git a/beacon-chain/core/helpers/weak_subjectivity.go b/beacon-chain/core/helpers/weak_subjectivity.go index 8be8bdc1a6..2a1af44a61 100644 --- a/beacon-chain/core/helpers/weak_subjectivity.go +++ b/beacon-chain/core/helpers/weak_subjectivity.go @@ -55,9 +55,9 @@ import ( // ) // // return ws_period -func ComputeWeakSubjectivityPeriod(ctx context.Context, st state.ReadOnlyBeaconState) (types.Epoch, error) { +func ComputeWeakSubjectivityPeriod(ctx context.Context, st state.ReadOnlyBeaconState, cfg *params.BeaconChainConfig) (types.Epoch, error) { // Weak subjectivity period cannot be smaller than withdrawal delay. - wsp := uint64(params.BeaconConfig().MinValidatorWithdrawabilityDelay) + wsp := uint64(cfg.MinValidatorWithdrawabilityDelay) // Cardinality of active validator set. N, err := ActiveValidatorCount(ctx, st, time.CurrentEpoch(st)) @@ -73,10 +73,10 @@ func ComputeWeakSubjectivityPeriod(ctx context.Context, st state.ReadOnlyBeaconS if err != nil { return 0, fmt.Errorf("cannot find total active balance of validators: %w", err) } - t = t / N / params.BeaconConfig().GweiPerEth + t = t / N / cfg.GweiPerEth // Maximum effective balance per validator. - T := params.BeaconConfig().MaxEffectiveBalance / params.BeaconConfig().GweiPerEth + T := cfg.MaxEffectiveBalance / cfg.GweiPerEth // Validator churn limit. delta, err := ValidatorChurnLimit(N) @@ -85,14 +85,14 @@ func ComputeWeakSubjectivityPeriod(ctx context.Context, st state.ReadOnlyBeaconS } // Balance top-ups. - Delta := uint64(params.BeaconConfig().SlotsPerEpoch.Mul(params.BeaconConfig().MaxDeposits)) + Delta := uint64(cfg.SlotsPerEpoch.Mul(cfg.MaxDeposits)) if delta == 0 || Delta == 0 { return 0, errors.New("either validator churn limit or balance top-ups is zero") } // Safety decay, maximum tolerable loss of safety margin of FFG finality. - D := params.BeaconConfig().SafetyDecay + D := cfg.SafetyDecay if T*(200+3*D) < t*(200+12*D) { epochsForValidatorSetChurn := N * (t*(200+12*D) - T*(200+3*D)) / (600 * delta * (2*t + T)) @@ -123,7 +123,7 @@ func ComputeWeakSubjectivityPeriod(ctx context.Context, st state.ReadOnlyBeaconS // current_epoch = compute_epoch_at_slot(get_current_slot(store)) // return current_epoch <= ws_state_epoch + ws_period func IsWithinWeakSubjectivityPeriod( - ctx context.Context, currentEpoch types.Epoch, wsState state.ReadOnlyBeaconState, wsStateRoot [fieldparams.RootLength]byte, wsEpoch types.Epoch) (bool, error) { + ctx context.Context, currentEpoch types.Epoch, wsState state.ReadOnlyBeaconState, wsStateRoot [fieldparams.RootLength]byte, wsEpoch types.Epoch, cfg *params.BeaconChainConfig) (bool, error) { // Make sure that incoming objects are not nil. if wsState == nil || wsState.IsNil() || wsState.LatestBlockHeader() == nil { return false, errors.New("invalid weak subjectivity state or checkpoint") @@ -140,7 +140,7 @@ func IsWithinWeakSubjectivityPeriod( } // Compare given epoch to state epoch + weak subjectivity period. - wsPeriod, err := ComputeWeakSubjectivityPeriod(ctx, wsState) + wsPeriod, err := ComputeWeakSubjectivityPeriod(ctx, wsState, cfg) if err != nil { return false, fmt.Errorf("cannot compute weak subjectivity period: %w", err) } @@ -154,8 +154,8 @@ func IsWithinWeakSubjectivityPeriod( // Within the weak subjectivity period, if two conflicting blocks are finalized, 1/3 - D (D := safety decay) // of validators will get slashed. Therefore, it is safe to assume that any finalized checkpoint within that // period is protected by this safety margin. -func LatestWeakSubjectivityEpoch(ctx context.Context, st state.ReadOnlyBeaconState) (types.Epoch, error) { - wsPeriod, err := ComputeWeakSubjectivityPeriod(ctx, st) +func LatestWeakSubjectivityEpoch(ctx context.Context, st state.ReadOnlyBeaconState, cfg *params.BeaconChainConfig) (types.Epoch, error) { + wsPeriod, err := ComputeWeakSubjectivityPeriod(ctx, st, cfg) if err != nil { return 0, err } diff --git a/beacon-chain/core/helpers/weak_subjectivity_test.go b/beacon-chain/core/helpers/weak_subjectivity_test.go index 0c30c2f160..3c0230740b 100644 --- a/beacon-chain/core/helpers/weak_subjectivity_test.go +++ b/beacon-chain/core/helpers/weak_subjectivity_test.go @@ -48,7 +48,7 @@ func TestWeakSubjectivity_ComputeWeakSubjectivityPeriod(t *testing.T) { t.Run(fmt.Sprintf("valCount: %d, avgBalance: %d", tt.valCount, tt.avgBalance), func(t *testing.T) { // Reset committee cache - as we need to recalculate active validator set for each test. helpers.ClearCache() - got, err := helpers.ComputeWeakSubjectivityPeriod(context.Background(), genState(t, tt.valCount, tt.avgBalance)) + got, err := helpers.ComputeWeakSubjectivityPeriod(context.Background(), genState(t, tt.valCount, tt.avgBalance), params.BeaconConfig()) require.NoError(t, err) assert.Equal(t, tt.want, got, "valCount: %v, avgBalance: %v", tt.valCount, tt.avgBalance) }) @@ -178,7 +178,7 @@ func TestWeakSubjectivity_IsWithinWeakSubjectivityPeriod(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { sr, _, e := tt.genWsCheckpoint() - got, err := helpers.IsWithinWeakSubjectivityPeriod(context.Background(), tt.epoch, tt.genWsState(), sr, e) + got, err := helpers.IsWithinWeakSubjectivityPeriod(context.Background(), tt.epoch, tt.genWsState(), sr, e, params.BeaconConfig()) if tt.wantedErr != "" { assert.Equal(t, false, got) assert.ErrorContains(t, tt.wantedErr, err) diff --git a/beacon-chain/rpc/eth/beacon/blocks.go b/beacon-chain/rpc/eth/beacon/blocks.go index 17b96958ab..1322db8e0e 100644 --- a/beacon-chain/rpc/eth/beacon/blocks.go +++ b/beacon-chain/rpc/eth/beacon/blocks.go @@ -13,6 +13,7 @@ import ( "github.com/prysmaticlabs/prysm/beacon-chain/core/helpers" "github.com/prysmaticlabs/prysm/beacon-chain/db/filters" rpchelpers "github.com/prysmaticlabs/prysm/beacon-chain/rpc/eth/helpers" + "github.com/prysmaticlabs/prysm/config/params" "github.com/prysmaticlabs/prysm/encoding/bytesutil" ethpbv1 "github.com/prysmaticlabs/prysm/proto/eth/v1" ethpbv2 "github.com/prysmaticlabs/prysm/proto/eth/v2" @@ -55,7 +56,7 @@ func (bs *Server) GetWeakSubjectivity(ctx context.Context, _ *empty.Empty) (*eth if err != nil { return nil, status.Error(codes.Internal, "could not get head state") } - wsEpoch, err := helpers.LatestWeakSubjectivityEpoch(ctx, hs) + wsEpoch, err := helpers.LatestWeakSubjectivityEpoch(ctx, hs, params.BeaconConfig()) if err != nil { return nil, status.Errorf(codes.Internal, "could not get weak subjectivity epoch: %v", err) } diff --git a/cmd/prysmctl/BUILD.bazel b/cmd/prysmctl/BUILD.bazel new file mode 100644 index 0000000000..fd0ebd7ed1 --- /dev/null +++ b/cmd/prysmctl/BUILD.bazel @@ -0,0 +1,20 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary") +load("@prysm//tools/go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["main.go"], + importpath = "github.com/prysmaticlabs/prysm/cmd/prysmctl", + visibility = ["//visibility:private"], + deps = [ + "//cmd/prysmctl/checkpoint:go_default_library", + "@com_github_sirupsen_logrus//:go_default_library", + "@com_github_urfave_cli_v2//:go_default_library", + ], +) + +go_binary( + name = "prysmctl", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) diff --git a/cmd/prysmctl/checkpoint/BUILD.bazel b/cmd/prysmctl/checkpoint/BUILD.bazel new file mode 100644 index 0000000000..c21b9062ba --- /dev/null +++ b/cmd/prysmctl/checkpoint/BUILD.bazel @@ -0,0 +1,17 @@ +load("@prysm//tools/go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "checkpoint.go", + "latest.go", + "save.go", + ], + importpath = "github.com/prysmaticlabs/prysm/cmd/prysmctl/checkpoint", + visibility = ["//visibility:public"], + deps = [ + "//api/client/beacon:go_default_library", + "@com_github_sirupsen_logrus//:go_default_library", + "@com_github_urfave_cli_v2//:go_default_library", + ], +) diff --git a/cmd/prysmctl/checkpoint/checkpoint.go b/cmd/prysmctl/checkpoint/checkpoint.go new file mode 100644 index 0000000000..2367a8f825 --- /dev/null +++ b/cmd/prysmctl/checkpoint/checkpoint.go @@ -0,0 +1,15 @@ +package checkpoint + +import "github.com/urfave/cli/v2" + +var Commands = []*cli.Command{ + { + Name: "checkpoint", + Aliases: []string{"cpt"}, + Usage: "commands for managing checkpoint syncing", + Subcommands: []*cli.Command{ + latestCmd, + saveCmd, + }, + }, +} diff --git a/cmd/prysmctl/checkpoint/latest.go b/cmd/prysmctl/checkpoint/latest.go new file mode 100644 index 0000000000..731b8a2132 --- /dev/null +++ b/cmd/prysmctl/checkpoint/latest.go @@ -0,0 +1,55 @@ +package checkpoint + +import ( + "context" + "fmt" + "time" + + "github.com/prysmaticlabs/prysm/api/client/beacon" + "github.com/urfave/cli/v2" +) + +var latestFlags = struct { + BeaconNodeHost string + Timeout time.Duration +}{} + +var latestCmd = &cli.Command{ + Name: "latest", + Usage: "Compute the latest weak subjectivity checkpoint (block_root:epoch) using trusted server data.", + Action: cliActionLatest, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "beacon-node-host", + Usage: "host:port for beacon node to query", + Destination: &latestFlags.BeaconNodeHost, + Value: "http://localhost:3500", + }, + &cli.DurationFlag{ + Name: "http-timeout", + Usage: "timeout for http requests made to beacon-node-url (uses duration format, ex: 2m31s). default: 2m", + Destination: &latestFlags.Timeout, + Value: time.Minute * 2, + }, + }, +} + +func cliActionLatest(_ *cli.Context) error { + ctx := context.Background() + f := latestFlags + + opts := []beacon.ClientOpt{beacon.WithTimeout(f.Timeout)} + client, err := beacon.NewClient(latestFlags.BeaconNodeHost, opts...) + if err != nil { + return err + } + + od, err := beacon.DownloadOriginData(ctx, client) + if err != nil { + return err + } + fmt.Println("\nUse the following flag when starting a prysm Beacon Node to ensure the chain history " + + "includes the Weak Subjectivity Checkpoint: ") + fmt.Printf("--weak-subjectivity-checkpoint=%s\n\n", od.CheckpointString()) + return nil +} diff --git a/cmd/prysmctl/checkpoint/save.go b/cmd/prysmctl/checkpoint/save.go new file mode 100644 index 0000000000..906bd8b732 --- /dev/null +++ b/cmd/prysmctl/checkpoint/save.go @@ -0,0 +1,71 @@ +package checkpoint + +import ( + "context" + "os" + "time" + + "github.com/prysmaticlabs/prysm/api/client/beacon" + log "github.com/sirupsen/logrus" + "github.com/urfave/cli/v2" +) + +var saveFlags = struct { + BeaconNodeHost string + Timeout time.Duration +}{} + +var saveCmd = &cli.Command{ + Name: "save", + Usage: "Query for the current weak subjectivity period epoch, then download the corresponding state and block. To be used for checkpoint sync.", + Action: cliActionSave, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "beacon-node-host", + Usage: "host:port for beacon node connection", + Destination: &saveFlags.BeaconNodeHost, + Value: "localhost:3500", + }, + &cli.DurationFlag{ + Name: "http-timeout", + Usage: "timeout for http requests made to beacon-node-url (uses duration format, ex: 2m31s). default: 4m", + Destination: &saveFlags.Timeout, + Value: time.Minute * 4, + }, + }, +} + +func cliActionSave(_ *cli.Context) error { + ctx := context.Background() + f := saveFlags + + opts := []beacon.ClientOpt{beacon.WithTimeout(f.Timeout)} + client, err := beacon.NewClient(saveFlags.BeaconNodeHost, opts...) + if err != nil { + return err + } + + cwd, err := os.Getwd() + if err != nil { + return err + } + + od, err := beacon.DownloadOriginData(ctx, client) + if err != nil { + return err + } + + blockPath, err := od.SaveBlock(cwd) + if err != nil { + return err + } + log.Printf("saved ssz-encoded block to to %s", blockPath) + + statePath, err := od.SaveState(cwd) + if err != nil { + return err + } + log.Printf("saved ssz-encoded state to to %s", statePath) + + return nil +} diff --git a/cmd/prysmctl/main.go b/cmd/prysmctl/main.go new file mode 100644 index 0000000000..d50cf1f524 --- /dev/null +++ b/cmd/prysmctl/main.go @@ -0,0 +1,25 @@ +package main + +import ( + "os" + + "github.com/prysmaticlabs/prysm/cmd/prysmctl/checkpoint" + log "github.com/sirupsen/logrus" + "github.com/urfave/cli/v2" +) + +var prysmctlCommands []*cli.Command + +func main() { + app := &cli.App{ + Commands: prysmctlCommands, + } + err := app.Run(os.Args) + if err != nil { + log.Fatal(err) + } +} + +func init() { + prysmctlCommands = append(prysmctlCommands, checkpoint.Commands...) +} diff --git a/go.mod b/go.mod index 676878f404..76c90f4d27 100644 --- a/go.mod +++ b/go.mod @@ -223,7 +223,6 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.19.0 // indirect - golang.org/x/mod v0.5.1 // indirect golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c // indirect golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect @@ -257,6 +256,7 @@ require ( github.com/peterh/liner v1.2.0 // indirect github.com/prometheus/tsdb v0.10.0 // indirect github.com/prysmaticlabs/gohashtree v0.0.1-alpha.0.20220303211031-f753e083138c + golang.org/x/mod v0.5.1 golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 // indirect google.golang.org/api v0.34.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/network/forks/errors.go b/network/forks/errors.go index 5a9422413b..29d19ac001 100644 --- a/network/forks/errors.go +++ b/network/forks/errors.go @@ -4,3 +4,7 @@ import "github.com/pkg/errors" // ErrVersionNotFound indicates the config package couldn't determine the version for an epoch using the fork schedule. var ErrVersionNotFound = errors.New("could not find an entry in the fork schedule") + +// ErrNoPreviousVersion indicates that a version prior to the given version could not be found, because the given version +// is the first one in the list +var ErrNoPreviousVersion = errors.New("no previous version") diff --git a/network/forks/ordered.go b/network/forks/ordered.go index ae8bdbb434..c11b8be2b0 100644 --- a/network/forks/ordered.go +++ b/network/forks/ordered.go @@ -38,6 +38,19 @@ func (o OrderedSchedule) VersionForEpoch(epoch types.Epoch) ([fieldparams.Versio return [fieldparams.VersionLength]byte{}, errors.Wrapf(ErrVersionNotFound, "no epoch in list <= %d", epoch) } +func (o OrderedSchedule) Previous(version [fieldparams.VersionLength]byte) ([fieldparams.VersionLength]byte, error) { + for i := len(o) - 1; i >= 0; i-- { + if o[i].Version == version { + if i-1 >= 0 { + return o[i-1].Version, nil + } else { + return [fieldparams.VersionLength]byte{}, errors.Wrapf(ErrNoPreviousVersion, "%#x is the first version", version) + } + } + } + return [fieldparams.VersionLength]byte{}, errors.Wrapf(ErrVersionNotFound, "no version in list == %#x", version) +} + // Converts the ForkVersionSchedule map into a list of Version+Epoch values, ordered by Epoch from lowest to highest. // See docs for OrderedSchedule for more detail on what you can do with this type. func NewOrderedSchedule(b *params.BeaconChainConfig) OrderedSchedule { diff --git a/network/forks/ordered_test.go b/network/forks/ordered_test.go index 6dbe9eef2b..b540b4a729 100644 --- a/network/forks/ordered_test.go +++ b/network/forks/ordered_test.go @@ -103,3 +103,23 @@ func testForkVersionScheduleBCC() *params.BeaconChainConfig { }, } } + +func TestPrevious(t *testing.T) { + cfg := testForkVersionScheduleBCC() + os := NewOrderedSchedule(cfg) + unreal := [4]byte{255, 255, 255, 255} + _, err := os.Previous(unreal) + require.ErrorIs(t, err, ErrVersionNotFound) + // first element has no previous, should return appropriate error + _, err = os.Previous(os[0].Version) + require.ErrorIs(t, err, ErrNoPreviousVersion) + // work up the list from the second element to the last, make sure each result matches the previous element + // this test of course relies on TestOrderedConfigSchedule to be correct! + prev := os[0].Version + for i := 1; i < len(os); i++ { + p, err := os.Previous(os[i].Version) + require.NoError(t, err) + require.Equal(t, prev, p) + prev = os[i].Version + } +}