Load graffiti from file (#8041)

* Pass graffati file and   use it

* Visibility

* Parse test

* More proposal tests

* Gazelle

* Add sequential functionality

* fix length check

* Update priorities. Specified -> random -> default

* Log warn instead return err

* Comment

* E2e test

* Comment

Co-authored-by: Preston Van Loon <preston@prysmaticlabs.com>
This commit is contained in:
terence tsao
2020-12-04 15:15:12 -08:00
committed by GitHub
parent 8ad328d9b3
commit b4437e6cec
18 changed files with 391 additions and 1 deletions

View File

@@ -60,9 +60,14 @@ func StartNewValidatorClient(t *testing.T, config *types.E2EConfig, validatorNum
if err != nil {
t.Fatal(err)
}
gFile, err := helpers.GraffitiYamlFile(e2e.TestParams.TestPath)
if err != nil {
t.Fatal(err)
}
args := []string{
fmt.Sprintf("--datadir=%s/eth2-val-%d", e2e.TestParams.TestPath, index),
fmt.Sprintf("--log-file=%s", file.Name()),
fmt.Sprintf("--graffiti-file=%s", gFile),
fmt.Sprintf("--interop-num-validators=%d", validatorNum),
fmt.Sprintf("--interop-start-index=%d", offset),
fmt.Sprintf("--monitoring-port=%d", e2e.TestParams.ValidatorMetricsPort+index),

View File

@@ -17,6 +17,7 @@ go_library(
deps = [
"//beacon-chain/core/helpers:go_default_library",
"//beacon-chain/p2p:go_default_library",
"//endtoend/helpers:go_default_library",
"//endtoend/params:go_default_library",
"//endtoend/policies:go_default_library",
"//endtoend/types:go_default_library",

View File

@@ -10,9 +10,11 @@ import (
"github.com/pkg/errors"
eth "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
corehelpers "github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
"github.com/prysmaticlabs/prysm/endtoend/helpers"
e2e "github.com/prysmaticlabs/prysm/endtoend/params"
"github.com/prysmaticlabs/prysm/endtoend/policies"
"github.com/prysmaticlabs/prysm/endtoend/types"
"github.com/prysmaticlabs/prysm/shared/bytesutil"
"github.com/prysmaticlabs/prysm/shared/params"
"github.com/prysmaticlabs/prysm/shared/testutil"
"golang.org/x/exp/rand"
@@ -44,6 +46,13 @@ var ProcessesDepositsInBlocks = types.Evaluator{
Evaluation: processesDepositsInBlocks,
}
// VerifyBlockGraffiti ensures the block graffiti is one of the random list.
var VerifyBlockGraffiti = types.Evaluator{
Name: "verify_graffiti_in_blocks_epoch_%d",
Policy: policies.AfterNthEpoch(0),
Evaluation: verifyGraffitiInBlocks,
}
// ActivatesDepositedValidators ensures the expected amount of validator deposits are activated into the state.
var ActivatesDepositedValidators = types.Evaluator{
Name: "processes_deposit_validators_epoch_%d",
@@ -109,6 +118,36 @@ func processesDepositsInBlocks(conns ...*grpc.ClientConn) error {
return nil
}
func verifyGraffitiInBlocks(conns ...*grpc.ClientConn) error {
conn := conns[0]
client := eth.NewBeaconChainClient(conn)
chainHead, err := client.GetChainHead(context.Background(), &ptypes.Empty{})
if err != nil {
return errors.Wrap(err, "failed to get chain head")
}
req := &eth.ListBlocksRequest{QueryFilter: &eth.ListBlocksRequest_Epoch{Epoch: chainHead.HeadEpoch - 1}}
blks, err := client.ListBlocks(context.Background(), req)
if err != nil {
return errors.Wrap(err, "failed to get blocks from beacon-chain")
}
for _, blk := range blks.BlockContainers {
var e bool
for _, graffiti := range helpers.Graffiti {
if bytes.Equal(bytesutil.PadTo([]byte(graffiti), 32), blk.Block.Block.Body.Graffiti) {
e = true
break
}
}
if !e && blk.Block.Block.Slot != 0 {
return errors.New("could not get graffiti from the list")
}
}
return nil
}
func activatesDepositedValidators(conns ...*grpc.ClientConn) error {
conn := conns[0]
client := eth.NewBeaconChainClient(conn)

View File

@@ -29,6 +29,8 @@ const (
maxFileBufferSize = 1024 * 1024
)
var Graffiti = []string{"Sushi", "Ramen", "Takoyaki"}
// DeleteAndCreateFile checks if the file path given exists, if it does, it deletes it and creates a new file.
// If not, it just creates the requested file.
func DeleteAndCreateFile(tmpPath, fileName string) (*os.File, error) {
@@ -83,6 +85,20 @@ func WaitForTextInFile(file *os.File, text string) error {
}
}
func GraffitiYamlFile(testDir string) (string, error) {
b := []byte(`default: "Rice"
random:
- "Sushi"
- "Ramen"
- "Takoyaki"
`)
f := filepath.Join(testDir, "graffiti.yaml")
if err := ioutil.WriteFile(f, b, os.ModePerm); err != nil {
return "", err
}
return f, nil
}
// LogOutput logs the output of all log files made.
func LogOutput(t *testing.T, config *types.E2EConfig) {
// Log out errors from beacon chain nodes.

View File

@@ -46,6 +46,7 @@ func TestEndToEnd_MinimalConfig(t *testing.T) {
ev.ValidatorsParticipating,
ev.FinalizationOccurs,
ev.ProcessesDepositsInBlocks,
ev.VerifyBlockGraffiti,
ev.ActivatesDepositedValidators,
ev.DepositedValidatorsAreActive,
ev.ProposeVoluntaryExit,

View File

@@ -31,11 +31,13 @@ go_library(
"//shared/grpcutils:go_default_library",
"//shared/hashutil:go_default_library",
"//shared/params:go_default_library",
"//shared/rand:go_default_library",
"//shared/slotutil:go_default_library",
"//shared/timeutils:go_default_library",
"//validator/accounts/wallet:go_default_library",
"//validator/db:go_default_library",
"//validator/db/kv:go_default_library",
"//validator/graffiti:go_default_library",
"//validator/keymanager:go_default_library",
"//validator/keymanager/imported:go_default_library",
"//validator/slashing-protection:go_default_library",
@@ -96,6 +98,7 @@ go_test(
"//shared/timeutils:go_default_library",
"//validator/db/kv:go_default_library",
"//validator/db/testing:go_default_library",
"//validator/graffiti:go_default_library",
"//validator/testing:go_default_library",
"@com_github_gogo_protobuf//types:go_default_library",
"@com_github_golang_mock//gomock:go_default_library",

View File

@@ -4,6 +4,7 @@ package client
import (
"context"
"fmt"
"time"
"github.com/gogo/protobuf/types"
"github.com/pkg/errors"
@@ -13,6 +14,7 @@ import (
"github.com/prysmaticlabs/prysm/shared/bls"
"github.com/prysmaticlabs/prysm/shared/bytesutil"
"github.com/prysmaticlabs/prysm/shared/params"
"github.com/prysmaticlabs/prysm/shared/rand"
"github.com/prysmaticlabs/prysm/shared/timeutils"
"github.com/sirupsen/logrus"
"go.opencensus.io/trace"
@@ -52,11 +54,19 @@ func (v *validator) ProposeBlock(ctx context.Context, slot uint64, pubKey [48]by
return
}
g, err := v.getGraffiti(ctx, pubKey)
if err != nil {
// Graffiti is not a critical enough to fail block production and cause
// validator to miss block reward. When failed, validator should continue
// to produce the block.
log.WithError(err).Warn("Could not get graffiti")
}
// Request block from beacon node
b, err := v.validatorClient.GetBlock(ctx, &ethpb.BlockRequest{
Slot: slot,
RandaoReveal: randaoReveal,
Graffiti: v.graffiti,
Graffiti: g,
})
if err != nil {
log.WithField("blockSlot", slot).WithError(err).Error("Failed to request block from beacon node")
@@ -257,3 +267,40 @@ func signVoluntaryExit(
}
return sig.Marshal(), nil
}
// Gets the graffiti from cli or file for the validator public key.
func (v *validator) getGraffiti(ctx context.Context, pubKey [48]byte) ([]byte, error) {
// When specified, default graffiti from the command line takes the first priority.
if len(v.graffiti) != 0 {
return v.graffiti, nil
}
if v.graffitiStruct == nil {
return nil, errors.New("graffitiStruct can't be nil")
}
// When specified, individual validator specified graffiti takes the second priority.
idx, err := v.validatorClient.ValidatorIndex(ctx, &ethpb.ValidatorIndexRequest{PublicKey: pubKey[:]})
if err != nil {
return []byte{}, err
}
g, ok := v.graffitiStruct.Specific[idx.Index]
if ok {
return []byte(g), nil
}
// When specified, a graffiti from the random list in the file take third priority.
if len(v.graffitiStruct.Random) != 0 {
r := rand.NewGenerator()
r.Seed(time.Now().Unix())
i := r.Uint64() % uint64(len(v.graffitiStruct.Random))
return []byte(v.graffitiStruct.Random[i]), nil
}
// Finally, default graffiti if specified in the file will be used.
if v.graffitiStruct.Default != "" {
return []byte(v.graffitiStruct.Default), nil
}
return []byte{}, nil
}

View File

@@ -4,6 +4,7 @@ import (
"context"
"encoding/hex"
"errors"
"strings"
"testing"
"time"
@@ -21,6 +22,7 @@ import (
"github.com/prysmaticlabs/prysm/shared/testutil/require"
"github.com/prysmaticlabs/prysm/validator/db/kv"
testing2 "github.com/prysmaticlabs/prysm/validator/db/testing"
"github.com/prysmaticlabs/prysm/validator/graffiti"
logTest "github.com/sirupsen/logrus/hooks/test"
)
@@ -622,3 +624,84 @@ func TestSignBlock(t *testing.T) {
// proposer domain
require.DeepEqual(t, proposerDomain, domain.SignatureDomain)
}
func TestGetGraffiti_Ok(t *testing.T) {
ctrl := gomock.NewController(t)
m := &mocks{
validatorClient: mock.NewMockBeaconNodeValidatorClient(ctrl),
}
pubKey := [48]byte{'a'}
tests := []struct {
name string
v *validator
want []byte
}{
{name: "use default cli graffiti",
v: &validator{
graffiti: []byte{'b'},
graffitiStruct: &graffiti.Graffiti{
Default: "c",
Random: []string{"d", "e"},
Specific: map[uint64]string{
1: "f",
2: "g",
},
},
},
want: []byte{'b'},
},
{name: "use default file graffiti",
v: &validator{
validatorClient: m.validatorClient,
graffitiStruct: &graffiti.Graffiti{
Default: "c",
},
},
want: []byte{'c'},
},
{name: "use random file graffiti",
v: &validator{
validatorClient: m.validatorClient,
graffitiStruct: &graffiti.Graffiti{
Random: []string{"d"},
Default: "c",
},
},
want: []byte{'d'},
},
{name: "use validator file graffiti, has validator",
v: &validator{
validatorClient: m.validatorClient,
graffitiStruct: &graffiti.Graffiti{
Random: []string{"d"},
Default: "c",
Specific: map[uint64]string{
1: "f",
2: "g",
},
},
},
want: []byte{'g'},
},
{name: "use validator file graffiti, none specified",
v: &validator{
validatorClient: m.validatorClient,
graffitiStruct: &graffiti.Graffiti{},
},
want: []byte{},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if !strings.Contains(tt.name, "use default cli graffiti") {
m.validatorClient.EXPECT().
ValidatorIndex(gomock.Any(), &ethpb.ValidatorIndexRequest{PublicKey: pubKey[:]}).
Return(&ethpb.ValidatorIndexResponse{Index: 2}, nil)
}
got, err := tt.v.getGraffiti(context.Background(), pubKey)
require.NoError(t, err)
require.DeepEqual(t, tt.want, got)
})
}
}

View File

@@ -22,6 +22,7 @@ import (
"github.com/prysmaticlabs/prysm/shared/params"
"github.com/prysmaticlabs/prysm/validator/accounts/wallet"
"github.com/prysmaticlabs/prysm/validator/db"
"github.com/prysmaticlabs/prysm/validator/graffiti"
"github.com/prysmaticlabs/prysm/validator/keymanager"
"github.com/prysmaticlabs/prysm/validator/keymanager/imported"
slashingprotection "github.com/prysmaticlabs/prysm/validator/slashing-protection"
@@ -74,6 +75,7 @@ type ValidatorService struct {
keyManager keymanager.IKeymanager
grpcHeaders []string
graffiti []byte
graffitiStruct *graffiti.Graffiti
}
// Config for the validator service.
@@ -94,6 +96,7 @@ type Config struct {
CertFlag string
DataDir string
GrpcHeadersFlag string
GraffitiStruct *graffiti.Graffiti
}
// NewValidatorService creates a new validator service for the service
@@ -119,6 +122,7 @@ func NewValidatorService(ctx context.Context, cfg *Config) (*ValidatorService, e
db: cfg.ValDB,
walletInitializedFeed: cfg.WalletInitializedFeed,
useWeb: cfg.UseWeb,
graffitiStruct: cfg.GraffitiStruct,
}, nil
}
@@ -195,6 +199,7 @@ func (v *ValidatorService) Start() {
voteStats: voteStats{startEpoch: ^uint64(0)},
useWeb: v.useWeb,
walletInitializedFeed: v.walletInitializedFeed,
graffitiStruct: v.graffitiStruct,
}
go run(v.ctx, v.validator)
go v.recheckKeys(v.ctx)

View File

@@ -30,6 +30,7 @@ import (
"github.com/prysmaticlabs/prysm/validator/accounts/wallet"
vdb "github.com/prysmaticlabs/prysm/validator/db"
"github.com/prysmaticlabs/prysm/validator/db/kv"
"github.com/prysmaticlabs/prysm/validator/graffiti"
"github.com/prysmaticlabs/prysm/validator/keymanager"
slashingprotection "github.com/prysmaticlabs/prysm/validator/slashing-protection"
"github.com/sirupsen/logrus"
@@ -77,6 +78,7 @@ type validator struct {
db vdb.Database
graffiti []byte
voteStats voteStats
graffitiStruct *graffiti.Graffiti
}
// Done cleans up the validator.

View File

@@ -282,6 +282,11 @@ var (
Usage: "Enables the web portal for the validator client (work in progress)",
Value: false,
}
// GraffitiFileFlag specifies the file path to load graffiti values.
GraffitiFileFlag = &cli.StringFlag{
Name: "graffiti-file",
Usage: "The path to a YAML file with graffiti values",
}
)
// DefaultValidatorDir returns OS-specific default validator directory.

View File

@@ -0,0 +1,17 @@
load("@io_bazel_rules_go//go:def.bzl", "go_test")
load("@prysm//tools/go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = ["parse_graffiti.go"],
importpath = "github.com/prysmaticlabs/prysm/validator/graffiti",
visibility = ["//validator:__subpackages__"],
deps = ["@in_gopkg_yaml_v2//:go_default_library"],
)
go_test(
name = "go_default_test",
srcs = ["parse_graffiti_test.go"],
embed = [":go_default_library"],
deps = ["//shared/testutil/require:go_default_library"],
)

View File

@@ -0,0 +1,26 @@
package graffiti
import (
"io/ioutil"
"gopkg.in/yaml.v2"
)
type Graffiti struct {
Default string `yaml:"default,omitempty"`
Random []string `yaml:"random,omitempty"`
Specific map[uint64]string `yaml:"specific,omitempty"`
}
// ParseGraffitiFile parses the graffiti file and returns the graffiti struct.
func ParseGraffitiFile(f string) (*Graffiti, error) {
yamlFile, err := ioutil.ReadFile(f)
if err != nil {
return nil, err
}
g := &Graffiti{}
if err := yaml.Unmarshal(yamlFile, g); err != nil {
return nil, err
}
return g, nil
}

View File

@@ -0,0 +1,117 @@
package graffiti
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/prysmaticlabs/prysm/shared/testutil/require"
)
func TestParseGraffitiFile_Default(t *testing.T) {
input := []byte(`default: "Mr T was here"`)
dirName := t.TempDir() + "somedir"
err := os.MkdirAll(dirName, os.ModePerm)
require.NoError(t, err)
someFileName := filepath.Join(dirName, "somefile.txt")
require.NoError(t, ioutil.WriteFile(someFileName, input, os.ModePerm))
got, err := ParseGraffitiFile(someFileName)
require.NoError(t, err)
wanted := &Graffiti{
Default: "Mr T was here",
}
require.DeepEqual(t, wanted, got)
}
func TestParseGraffitiFile_Random(t *testing.T) {
input := []byte(`random:
- "Mr A was here"
- "Mr B was here"
- "Mr C was here"`)
dirName := t.TempDir() + "somedir"
err := os.MkdirAll(dirName, os.ModePerm)
require.NoError(t, err)
someFileName := filepath.Join(dirName, "somefile.txt")
require.NoError(t, ioutil.WriteFile(someFileName, input, os.ModePerm))
got, err := ParseGraffitiFile(someFileName)
require.NoError(t, err)
wanted := &Graffiti{
Random: []string{
"Mr A was here",
"Mr B was here",
"Mr C was here",
},
}
require.DeepEqual(t, wanted, got)
}
func TestParseGraffitiFile_Validators(t *testing.T) {
input := []byte(`
specific:
1234: Yolo
555: "What's up"
703727: Meow`)
dirName := t.TempDir() + "somedir"
err := os.MkdirAll(dirName, os.ModePerm)
require.NoError(t, err)
someFileName := filepath.Join(dirName, "somefile.txt")
require.NoError(t, ioutil.WriteFile(someFileName, input, os.ModePerm))
got, err := ParseGraffitiFile(someFileName)
require.NoError(t, err)
wanted := &Graffiti{
Specific: map[uint64]string{
1234: "Yolo",
555: "What's up",
703727: "Meow",
},
}
require.DeepEqual(t, wanted, got)
}
func TestParseGraffitiFile_AllFields(t *testing.T) {
input := []byte(`default: "Mr T was here"
random:
- "Mr A was here"
- "Mr B was here"
- "Mr C was here"
specific:
1234: Yolo
555: "What's up"
703727: Meow`)
dirName := t.TempDir() + "somedir"
err := os.MkdirAll(dirName, os.ModePerm)
require.NoError(t, err)
someFileName := filepath.Join(dirName, "somefile.txt")
require.NoError(t, ioutil.WriteFile(someFileName, input, os.ModePerm))
got, err := ParseGraffitiFile(someFileName)
require.NoError(t, err)
wanted := &Graffiti{
Default: "Mr T was here",
Random: []string{
"Mr A was here",
"Mr B was here",
"Mr C was here",
},
Specific: map[uint64]string{
1234: "Yolo",
555: "What's up",
703727: "Meow",
},
}
require.DeepEqual(t, wanted, got)
}

View File

@@ -68,6 +68,7 @@ var appFlags = []cli.Flag{
flags.WalletPasswordFileFlag,
flags.WalletDirFlag,
flags.EnableWebFlag,
flags.GraffitiFileFlag,
cmd.BackupWebhookOutputDir,
cmd.EnableBackupWebhookFlag,
cmd.MinimalConfigFlag,

View File

@@ -39,6 +39,7 @@ go_library(
"//validator/client:go_default_library",
"//validator/db/kv:go_default_library",
"//validator/flags:go_default_library",
"//validator/graffiti:go_default_library",
"//validator/keymanager:go_default_library",
"//validator/keymanager/imported:go_default_library",
"//validator/rpc:go_default_library",

View File

@@ -29,6 +29,7 @@ import (
"github.com/prysmaticlabs/prysm/validator/client"
"github.com/prysmaticlabs/prysm/validator/db/kv"
"github.com/prysmaticlabs/prysm/validator/flags"
g "github.com/prysmaticlabs/prysm/validator/graffiti"
"github.com/prysmaticlabs/prysm/validator/keymanager"
"github.com/prysmaticlabs/prysm/validator/keymanager/imported"
"github.com/prysmaticlabs/prysm/validator/rpc"
@@ -99,9 +100,16 @@ func NewValidatorClient(cliCtx *cli.Context) (*ValidatorClient, error) {
}
return ValidatorClient, nil
}
if cliCtx.IsSet(cmd.ChainConfigFileFlag.Name) {
chainConfigFileName := cliCtx.String(cmd.ChainConfigFileFlag.Name)
params.LoadChainConfigFile(chainConfigFileName)
}
if err := ValidatorClient.initializeFromCLI(cliCtx); err != nil {
return nil, err
}
return ValidatorClient, nil
}
@@ -363,6 +371,17 @@ func (s *ValidatorClient) registerClientService(
if err := s.services.FetchService(&sp); err == nil {
protector = sp
}
gStruct := &g.Graffiti{}
var err error
if s.cliCtx.IsSet(flags.GraffitiFileFlag.Name) {
n := s.cliCtx.String(flags.GraffitiFileFlag.Name)
gStruct, err = g.ParseGraffitiFile(n)
if err != nil {
log.WithError(err).Warn("Could not parse graffiti file")
}
}
v, err := client.NewValidatorService(s.cliCtx.Context, &client.Config{
Endpoint: endpoint,
DataDir: dataDir,
@@ -379,6 +398,7 @@ func (s *ValidatorClient) registerClientService(
ValDB: s.db,
UseWeb: s.cliCtx.Bool(flags.EnableWebFlag.Name),
WalletInitializedFeed: s.walletInitialized,
GraffitiStruct: gStruct,
})
if err != nil {

View File

@@ -101,6 +101,7 @@ var appHelpFlagGroups = []flagGroup{
flags.DisableAccountMetricsFlag,
flags.WalletDirFlag,
flags.WalletPasswordFileFlag,
flags.GraffitiFileFlag,
},
},
{