diff --git a/WORKSPACE b/WORKSPACE index b46331002c..0998eaf541 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -292,6 +292,10 @@ go_repository( name = "com_github_prysmaticlabs_go_ssz", commit = "e24db4d9e9637cf88ee9e4a779e339a1686a84ee", importpath = "github.com/prysmaticlabs/go-ssz", + patch_args = ["-p1"], + patches = [ + "//third_party:com_github_prysmaticlabs_go_ssz.patch", + ], ) go_repository( @@ -1598,7 +1602,6 @@ go_repository( go_repository( name = "com_github_ferranbt_fastssz", + commit = "06015a5d84f9e4eefe2c21377ca678fa8f1a1b09", importpath = "github.com/ferranbt/fastssz", - sum = "h1:oUQredbOIzWIMmeGR9dTLzSi4DqRVwxrPzSDiLJBp4Q=", - version = "v0.0.0-20200310214500-3283b9706406", ) diff --git a/beacon-chain/rpc/beacon/BUILD.bazel b/beacon-chain/rpc/beacon/BUILD.bazel index d8035a1f33..1ce18b8098 100644 --- a/beacon-chain/rpc/beacon/BUILD.bazel +++ b/beacon-chain/rpc/beacon/BUILD.bazel @@ -89,6 +89,7 @@ go_test( "//beacon-chain/state:go_default_library", "//proto/beacon/p2p/v1:go_default_library", "//shared/attestationutil:go_default_library", + "//shared/bytesutil:go_default_library", "//shared/featureconfig:go_default_library", "//shared/params:go_default_library", "//shared/testutil:go_default_library", diff --git a/beacon-chain/rpc/beacon/attestations_test.go b/beacon-chain/rpc/beacon/attestations_test.go index a338822f69..fbf9a09b86 100644 --- a/beacon-chain/rpc/beacon/attestations_test.go +++ b/beacon-chain/rpc/beacon/attestations_test.go @@ -26,6 +26,7 @@ import ( stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state" pbp2p "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" "github.com/prysmaticlabs/prysm/shared/attestationutil" + "github.com/prysmaticlabs/prysm/shared/bytesutil" "github.com/prysmaticlabs/prysm/shared/params" "github.com/prysmaticlabs/prysm/shared/testutil" ) @@ -937,7 +938,7 @@ func TestServer_StreamIndexedAttestations_OK(t *testing.T) { for j := 0; j < numValidators; j++ { attExample := ðpb.Attestation{ Data: ðpb.AttestationData{ - BeaconBlockRoot: []byte("root"), + BeaconBlockRoot: bytesutil.PadTo([]byte("root"), 32), Slot: i, Target: ðpb.Checkpoint{ Epoch: 0, diff --git a/beacon-chain/sync/validate_beacon_blocks_test.go b/beacon-chain/sync/validate_beacon_blocks_test.go index a05d70be6a..b0296dc356 100644 --- a/beacon-chain/sync/validate_beacon_blocks_test.go +++ b/beacon-chain/sync/validate_beacon_blocks_test.go @@ -33,7 +33,7 @@ func TestValidateBeaconBlockPubSub_InvalidSignature(t *testing.T) { Slot: 1, ParentRoot: testutil.Random32Bytes(t), }, - Signature: []byte("fake"), + Signature: bytesutil.PadTo([]byte("fake"), 96), } p := p2ptest.NewTestP2P(t) diff --git a/beacon-chain/sync/validate_committee_index_beacon_attestation_test.go b/beacon-chain/sync/validate_committee_index_beacon_attestation_test.go index c4474a21d2..67f8cc8340 100644 --- a/beacon-chain/sync/validate_committee_index_beacon_attestation_test.go +++ b/beacon-chain/sync/validate_committee_index_beacon_attestation_test.go @@ -17,6 +17,7 @@ import ( beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state" mockSync "github.com/prysmaticlabs/prysm/beacon-chain/sync/initial-sync/testing" pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" + "github.com/prysmaticlabs/prysm/shared/bytesutil" "github.com/prysmaticlabs/prysm/shared/params" ) @@ -108,7 +109,7 @@ func TestService_validateCommitteeIndexBeaconAttestation(t *testing.T) { msg: ðpb.Attestation{ AggregationBits: bitfield.Bitlist{0b1010}, Data: ðpb.AttestationData{ - BeaconBlockRoot: []byte("missing"), + BeaconBlockRoot: bytesutil.PadTo([]byte("missing"), 32), CommitteeIndex: 1, Slot: 63, }, diff --git a/proto/beacon/p2p/v1/BUILD.bazel b/proto/beacon/p2p/v1/BUILD.bazel index 62afae55cc..8f96cb49a4 100644 --- a/proto/beacon/p2p/v1/BUILD.bazel +++ b/proto/beacon/p2p/v1/BUILD.bazel @@ -4,6 +4,7 @@ load("@rules_proto//proto:defs.bzl", "proto_library") load("@io_bazel_rules_go//go:def.bzl", "go_library") load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") load("//proto:ssz_proto_library.bzl", "ssz_proto_files") +load("//tools:ssz.bzl", "SSZ_DEPS", "ssz_gen_marshal") go_proto_library( name = "v1_go_proto", @@ -17,11 +18,27 @@ go_proto_library( ], ) +ssz_gen_marshal( + name = "ssz_generated_files", + go_proto = ":v1_go_proto", + includes = [ + "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library", + ], + objs = [ + "BeaconBlocksByRangeRequest", + "Fork", + "HistoricalBatch", + "Status", + "BeaconState", + ], +) + go_library( name = "go_default_library", + srcs = [":ssz_generated_files"], embed = [":v1_go_proto"], - importpath = "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1", visibility = ["//visibility:public"], + deps = SSZ_DEPS, ) ssz_proto_files( diff --git a/shared/bytesutil/bytes.go b/shared/bytesutil/bytes.go index 98cf2d7cf6..845a6fa09e 100644 --- a/shared/bytesutil/bytes.go +++ b/shared/bytesutil/bytes.go @@ -232,3 +232,13 @@ func ReverseBytes32Slice(arr [][32]byte) [][32]byte { } return arr } + + +// PadTo pads a byte slice to the given size. If the byte slice is larger than the given size, the +// original slice is returned. +func PadTo(b []byte, size int) []byte { + if len(b) > size { + return b + } + return append(b, make([]byte, size-len(b))...) +} \ No newline at end of file diff --git a/shared/testutil/block.go b/shared/testutil/block.go index fa6bff7ae9..fe75d9fe92 100644 --- a/shared/testutil/block.go +++ b/shared/testutil/block.go @@ -15,6 +15,7 @@ import ( "github.com/prysmaticlabs/prysm/beacon-chain/core/state" stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state" "github.com/prysmaticlabs/prysm/shared/bls" + "github.com/prysmaticlabs/prysm/shared/bytesutil" "github.com/prysmaticlabs/prysm/shared/params" ) @@ -163,7 +164,7 @@ func GenerateProposerSlashingForValidator( header1 := ðpb.SignedBeaconBlockHeader{ Header: ðpb.BeaconBlockHeader{ Slot: bState.Slot(), - BodyRoot: []byte{0, 1, 0}, + BodyRoot: bytesutil.PadTo([]byte{0, 1, 0}, 32), }, } root, err := ssz.HashTreeRoot(header1.Header) @@ -180,7 +181,7 @@ func GenerateProposerSlashingForValidator( header2 := ðpb.SignedBeaconBlockHeader{ Header: ðpb.BeaconBlockHeader{ Slot: bState.Slot(), - BodyRoot: []byte{0, 2, 0}, + BodyRoot: bytesutil.PadTo([]byte{0, 2, 0}, 32), }, } root, err = ssz.HashTreeRoot(header2.Header) diff --git a/third_party/com_github_prysmaticlabs_ethereumapis-tags.patch b/third_party/com_github_prysmaticlabs_ethereumapis-tags.patch index 18529a9e62..627605f483 100644 --- a/third_party/com_github_prysmaticlabs_ethereumapis-tags.patch +++ b/third_party/com_github_prysmaticlabs_ethereumapis-tags.patch @@ -1,8 +1,16 @@ diff --git a/eth/v1alpha1/BUILD.bazel b/eth/v1alpha1/BUILD.bazel -index c0fbe31..ae4ff87 100644 +index c0fbe31..1211829 100644 --- a/eth/v1alpha1/BUILD.bazel +++ b/eth/v1alpha1/BUILD.bazel -@@ -25,7 +25,6 @@ proto_library( +@@ -4,6 +4,7 @@ + + load("@rules_proto//proto:defs.bzl", "proto_library") + load("@rules_java//java:defs.bzl", "java_proto_library") ++load("@prysm//tools:ssz.bzl", "SSZ_DEPS", "ssz_gen_marshal") + + ############################################################################## + # Go +@@ -25,7 +26,6 @@ proto_library( "beacon_chain.proto", "node.proto", "validator.proto", @@ -10,7 +18,7 @@ index c0fbe31..ae4ff87 100644 ], visibility = ["//visibility:public"], deps = [ -@@ -33,6 +32,7 @@ proto_library( +@@ -33,6 +33,7 @@ proto_library( "@com_google_protobuf//:any_proto", "@com_google_protobuf//:timestamp_proto", "@go_googleapis//google/api:annotations_proto", @@ -18,7 +26,7 @@ index c0fbe31..ae4ff87 100644 "@grpc_ecosystem_grpc_gateway//protoc-gen-swagger/options:options_proto", ], ) -@@ -48,11 +48,30 @@ java_proto_library( +@@ -48,11 +49,46 @@ java_proto_library( go_proto_library( name = "go_proto", @@ -33,6 +41,22 @@ index c0fbe31..ae4ff87 100644 + ], +) + ++ssz_gen_marshal( ++ name = "ssz_generated_files", ++ go_proto = ":go_proto", ++ objs = [ ++ "Attestation", ++ "BeaconBlock", ++ "SignedBeaconBlock", ++ "Validator", ++ "BeaconBlockHeader", ++ "AttesterSlashing", ++ "VoluntaryExit", ++ "Deposit", ++ "ProposerSlashing", ++ ], ++) ++ +go_proto_library( + name = "go_grpc_gateway_library", + compilers = [ @@ -50,6 +74,18 @@ index c0fbe31..ae4ff87 100644 "@go_googleapis//google/api:annotations_go_proto", "@grpc_ecosystem_grpc_gateway//protoc-gen-swagger/options:options_go_proto", ], +@@ -60,9 +96,11 @@ go_proto_library( + + go_library( + name = "go_default_library", ++ srcs = [":ssz_generated_files"], + embed = [":go_proto"], + importpath = "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1", + visibility = ["//visibility:public"], ++ deps = SSZ_DEPS, + ) + + protoc_gen_swagger( diff --git a/eth/v1alpha1/attestation.proto b/eth/v1alpha1/attestation.proto index b177b76..28b4b46 100644 --- a/eth/v1alpha1/attestation.proto diff --git a/third_party/com_github_prysmaticlabs_go_ssz.patch b/third_party/com_github_prysmaticlabs_go_ssz.patch new file mode 100644 index 0000000000..f11969174b --- /dev/null +++ b/third_party/com_github_prysmaticlabs_go_ssz.patch @@ -0,0 +1,46 @@ +diff --git a/BUILD.bazel b/BUILD.bazel +index e502174..1c29b7a 100644 +--- a/BUILD.bazel ++++ b/BUILD.bazel +@@ -61,6 +61,7 @@ go_library( + visibility = ["//visibility:public"], + deps = [ + "//types:go_default_library", ++ "@com_github_ferranbt_fastssz//:go_default_library", + "@com_github_pkg_errors//:go_default_library", + "@com_github_prysmaticlabs_go_bitfield//:go_default_library", + ], +diff --git a/ssz.go b/ssz.go +index 61fb9ef..24755fc 100644 +--- a/ssz.go ++++ b/ssz.go +@@ -5,6 +5,7 @@ import ( + "reflect" + "strings" + ++ fssz "github.com/ferranbt/fastssz" + "github.com/pkg/errors" + "github.com/prysmaticlabs/go-bitfield" + "github.com/prysmaticlabs/go-ssz/types" +@@ -48,6 +49,11 @@ func Marshal(val interface{}) ([]byte, error) { + if val == nil { + return nil, errors.New("untyped-value nil cannot be marshaled") + } ++ ++ if v, ok := val.(fssz.Marshaler); ok { ++ return v.MarshalSSZ() ++ } ++ + rval := reflect.ValueOf(val) + + // We pre-allocate a buffer-size depending on the value's calculated total byte size. +@@ -87,6 +93,9 @@ func Unmarshal(input []byte, val interface{}) error { + if val == nil { + return errors.New("cannot unmarshal into untyped, nil value") + } ++ if v, ok := val.(fssz.Unmarshaler); ok { ++ return v.UnmarshalSSZ(input) ++ } + if len(input) == 0 { + return errors.New("no data to unmarshal from, input is an empty byte slice []byte{}") + } diff --git a/tools/ssz.bzl b/tools/ssz.bzl index 783f7447f0..9a6d4b86ce 100644 --- a/tools/ssz.bzl +++ b/tools/ssz.bzl @@ -1,26 +1,40 @@ load( "@io_bazel_rules_go//go:def.bzl", "GoLibrary", + "GoSource", ) def _ssz_go_proto_library_impl(ctx): - go_proto = ctx.attr.go_proto + if ctx.attr.go_proto != None: + go_proto = ctx.attr.go_proto + input_files = go_proto[OutputGroupInfo].go_generated_srcs.to_list() + package_path = input_files[0].dirname + elif hasattr(ctx.attr, "srcs") and len(ctx.attr.srcs) > 0: + package_path = ctx.attr.srcs[0].files.to_list()[0].dirname + input_files = ctx.attr.srcs[0].files.to_list() + else: + fail("Must have go_proto or srcs") - generated_pb_go_files = go_proto[OutputGroupInfo].go_generated_srcs - - # Run the tool on the generated files - package_path = generated_pb_go_files.to_list()[0].dirname + # Run the tool. output = ctx.outputs.out args = [ "--output=%s" % output.path, "--path=%s" % package_path, ] + if hasattr(ctx.attr, "includes") and len(ctx.attr.includes) > 0: + incs = [] + for include in ctx.attr.includes: + incs.append(include[GoSource].srcs[0].dirname) + input_files += include[GoSource].srcs + args.append("--include=%s" % ",".join(incs)) + if len(ctx.attr.objs) > 0: args += ["--objs=%s" % ",".join(ctx.attr.objs)] + ctx.actions.run( executable = ctx.executable.sszgen, progress_message = "Generating ssz marshal and unmarshal functions", - inputs = generated_pb_go_files, + inputs = input_files, arguments = args, outputs = [output], ) @@ -56,6 +70,7 @@ go_library( ssz_gen_marshal = rule( implementation = _ssz_go_proto_library_impl, attrs = { + "srcs": attr.label_list(allow_files = True), "go_proto": attr.label(providers = [GoLibrary]), "sszgen": attr.label( default = Label("@com_github_ferranbt_fastssz//sszgen:sszgen"), @@ -63,6 +78,7 @@ ssz_gen_marshal = rule( cfg = "host", ), "objs": attr.string_list(), + "includes": attr.label_list(providers = [GoLibrary]), }, outputs = {"out": "generated.ssz.go"}, )