Add ssz marshal and unmarshal for most data structures (#5121)

* Add ssz marshal and unmarshal for most data structures
* Merge refs/heads/master into ssz-stuff
* Merge refs/heads/master into ssz-stuff
* Merge refs/heads/master into ssz-stuff
* Merge refs/heads/master into ssz-stuff
* Merge refs/heads/master into ssz-stuff
* Merge refs/heads/master into ssz-stuff
* Update ferran SSZ
* Update ferran's SSZ
* Merge refs/heads/master into ssz-stuff
* fix tests
* Merge branch 'ssz-stuff' of github.com:prysmaticlabs/prysm into ssz-stuff
* gaz

WORKSPACE

@@ -292,6 +292,10 @@ go_repository(
     name = "com_github_prysmaticlabs_go_ssz",
     commit = "e24db4d9e9637cf88ee9e4a779e339a1686a84ee",
     importpath = "github.com/prysmaticlabs/go-ssz",
+    patch_args = ["-p1"],
+    patches = [
+        "//third_party:com_github_prysmaticlabs_go_ssz.patch",
+    ],
 )
 
 go_repository(
@@ -1598,7 +1602,6 @@ go_repository(
 
 go_repository(
     name = "com_github_ferranbt_fastssz",
+    commit = "06015a5d84f9e4eefe2c21377ca678fa8f1a1b09",
     importpath = "github.com/ferranbt/fastssz",
-    sum = "h1:oUQredbOIzWIMmeGR9dTLzSi4DqRVwxrPzSDiLJBp4Q=",
-    version = "v0.0.0-20200310214500-3283b9706406",
 )

beacon-chain/rpc/beacon/BUILD.bazel

@@ -89,6 +89,7 @@ go_test(
         "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/attestationutil:go_default_library",
+        "//shared/bytesutil:go_default_library",
         "//shared/featureconfig:go_default_library",
         "//shared/params:go_default_library",
         "//shared/testutil:go_default_library",

beacon-chain/rpc/beacon/attestations_test.go

@@ -26,6 +26,7 @@ import (
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pbp2p "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/attestationutil"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
 )
@@ -937,7 +938,7 @@ func TestServer_StreamIndexedAttestations_OK(t *testing.T) {
 		for j := 0; j < numValidators; j++ {
 			attExample := &ethpb.Attestation{
 				Data: &ethpb.AttestationData{
-					BeaconBlockRoot: []byte("root"),
+					BeaconBlockRoot: bytesutil.PadTo([]byte("root"), 32),
 					Slot:            i,
 					Target: &ethpb.Checkpoint{
 						Epoch: 0,

beacon-chain/sync/validate_beacon_blocks_test.go

@@ -33,7 +33,7 @@ func TestValidateBeaconBlockPubSub_InvalidSignature(t *testing.T) {
 			Slot:       1,
 			ParentRoot: testutil.Random32Bytes(t),
 		},
-		Signature: []byte("fake"),
+		Signature: bytesutil.PadTo([]byte("fake"), 96),
 	}
 
 	p := p2ptest.NewTestP2P(t)

beacon-chain/sync/validate_committee_index_beacon_attestation_test.go

@@ -17,6 +17,7 @@ import (
 	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	mockSync "github.com/prysmaticlabs/prysm/beacon-chain/sync/initial-sync/testing"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
@@ -108,7 +109,7 @@ func TestService_validateCommitteeIndexBeaconAttestation(t *testing.T) {
 			msg: &ethpb.Attestation{
 				AggregationBits: bitfield.Bitlist{0b1010},
 				Data: &ethpb.AttestationData{
-					BeaconBlockRoot: []byte("missing"),
+					BeaconBlockRoot: bytesutil.PadTo([]byte("missing"), 32),
 					CommitteeIndex:  1,
 					Slot:            63,
 				},

proto/beacon/p2p/v1/BUILD.bazel

@@ -4,6 +4,7 @@ load("@rules_proto//proto:defs.bzl", "proto_library")
 load("@io_bazel_rules_go//go:def.bzl", "go_library")
 load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library")
 load("//proto:ssz_proto_library.bzl", "ssz_proto_files")
+load("//tools:ssz.bzl", "SSZ_DEPS", "ssz_gen_marshal")
 
 go_proto_library(
     name = "v1_go_proto",
@@ -17,11 +18,27 @@ go_proto_library(
     ],
 )
 
+ssz_gen_marshal(
+    name = "ssz_generated_files",
+    go_proto = ":v1_go_proto",
+    includes = [
+        "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
+    ],
+    objs = [
+        "BeaconBlocksByRangeRequest",
+        "Fork",
+        "HistoricalBatch",
+        "Status",
+        "BeaconState",
+    ],
+)
+
 go_library(
     name = "go_default_library",
+    srcs = [":ssz_generated_files"],
     embed = [":v1_go_proto"],
-    importpath = "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1",
     visibility = ["//visibility:public"],
+    deps = SSZ_DEPS,
 )
 
 ssz_proto_files(

shared/bytesutil/bytes.go

@@ -232,3 +232,13 @@ func ReverseBytes32Slice(arr [][32]byte) [][32]byte {
 	}
 	return arr
 }
+
+
+// PadTo pads a byte slice to the given size. If the byte slice is larger than the given size, the
+// original slice is returned.
+func PadTo(b []byte, size int) []byte {
+	if len(b) > size {
+		return b
+	}
+	return append(b, make([]byte, size-len(b))...)
+}

shared/testutil/block.go

@@ -15,6 +15,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/bls"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
@@ -163,7 +164,7 @@ func GenerateProposerSlashingForValidator(
 	header1 := &ethpb.SignedBeaconBlockHeader{
 		Header: &ethpb.BeaconBlockHeader{
 			Slot:     bState.Slot(),
-			BodyRoot: []byte{0, 1, 0},
+			BodyRoot: bytesutil.PadTo([]byte{0, 1, 0}, 32),
 		},
 	}
 	root, err := ssz.HashTreeRoot(header1.Header)
@@ -180,7 +181,7 @@ func GenerateProposerSlashingForValidator(
 	header2 := &ethpb.SignedBeaconBlockHeader{
 		Header: &ethpb.BeaconBlockHeader{
 			Slot:     bState.Slot(),
-			BodyRoot: []byte{0, 2, 0},
+			BodyRoot: bytesutil.PadTo([]byte{0, 2, 0}, 32),
 		},
 	}
 	root, err = ssz.HashTreeRoot(header2.Header)

third_party/com_github_prysmaticlabs_ethereumapis-tags.patch

@@ -1,8 +1,16 @@
 diff --git a/eth/v1alpha1/BUILD.bazel b/eth/v1alpha1/BUILD.bazel
-index c0fbe31..ae4ff87 100644
+index c0fbe31..1211829 100644
 --- a/eth/v1alpha1/BUILD.bazel
 +++ b/eth/v1alpha1/BUILD.bazel
-@@ -25,7 +25,6 @@ proto_library(
+@@ -4,6 +4,7 @@
+ 
+ load("@rules_proto//proto:defs.bzl", "proto_library")
+ load("@rules_java//java:defs.bzl", "java_proto_library")
++load("@prysm//tools:ssz.bzl", "SSZ_DEPS", "ssz_gen_marshal")
+ 
+ ##############################################################################
+ # Go
+@@ -25,7 +26,6 @@ proto_library(
          "beacon_chain.proto",
          "node.proto",
          "validator.proto",
@@ -10,7 +18,7 @@ index c0fbe31..ae4ff87 100644
      ],
      visibility = ["//visibility:public"],
      deps = [
-@@ -33,6 +32,7 @@ proto_library(
+@@ -33,6 +33,7 @@ proto_library(
          "@com_google_protobuf//:any_proto",
          "@com_google_protobuf//:timestamp_proto",
          "@go_googleapis//google/api:annotations_proto",
@@ -18,7 +26,7 @@ index c0fbe31..ae4ff87 100644
          "@grpc_ecosystem_grpc_gateway//protoc-gen-swagger/options:options_proto",
      ],
  )
-@@ -48,11 +48,30 @@ java_proto_library(
+@@ -48,11 +49,46 @@ java_proto_library(
  
  go_proto_library(
      name = "go_proto",
@@ -33,6 +41,22 @@ index c0fbe31..ae4ff87 100644
 +    ],
 +)
 +
++ssz_gen_marshal(
++    name = "ssz_generated_files",
++    go_proto = ":go_proto",
++    objs = [
++        "Attestation",
++        "BeaconBlock",
++        "SignedBeaconBlock",
++        "Validator",
++        "BeaconBlockHeader",
++        "AttesterSlashing",
++        "VoluntaryExit",
++        "Deposit",
++        "ProposerSlashing",
++    ],
++)
++
 +go_proto_library(
 +    name = "go_grpc_gateway_library",
 +    compilers = [
@@ -50,6 +74,18 @@ index c0fbe31..ae4ff87 100644
          "@go_googleapis//google/api:annotations_go_proto",
          "@grpc_ecosystem_grpc_gateway//protoc-gen-swagger/options:options_go_proto",
      ],
+@@ -60,9 +96,11 @@ go_proto_library(
+ 
+ go_library(
+     name = "go_default_library",
++    srcs = [":ssz_generated_files"],
+     embed = [":go_proto"],
+     importpath = "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1",
+     visibility = ["//visibility:public"],
++    deps = SSZ_DEPS,
+ )
+ 
+ protoc_gen_swagger(
 diff --git a/eth/v1alpha1/attestation.proto b/eth/v1alpha1/attestation.proto
 index b177b76..28b4b46 100644
 --- a/eth/v1alpha1/attestation.proto

third_party/com_github_prysmaticlabs_go_ssz.patch

@@ -0,0 +1,46 @@
+diff --git a/BUILD.bazel b/BUILD.bazel
+index e502174..1c29b7a 100644
+--- a/BUILD.bazel
++++ b/BUILD.bazel
+@@ -61,6 +61,7 @@ go_library(
+     visibility = ["//visibility:public"],
+     deps = [
+         "//types:go_default_library",
++        "@com_github_ferranbt_fastssz//:go_default_library",
+         "@com_github_pkg_errors//:go_default_library",
+         "@com_github_prysmaticlabs_go_bitfield//:go_default_library",
+     ],
+diff --git a/ssz.go b/ssz.go
+index 61fb9ef..24755fc 100644
+--- a/ssz.go
++++ b/ssz.go
+@@ -5,6 +5,7 @@ import (
+ 	"reflect"
+ 	"strings"
+ 
++	fssz "github.com/ferranbt/fastssz"
+ 	"github.com/pkg/errors"
+ 	"github.com/prysmaticlabs/go-bitfield"
+ 	"github.com/prysmaticlabs/go-ssz/types"
+@@ -48,6 +49,11 @@ func Marshal(val interface{}) ([]byte, error) {
+ 	if val == nil {
+ 		return nil, errors.New("untyped-value nil cannot be marshaled")
+ 	}
++
++	if v, ok := val.(fssz.Marshaler); ok {
++		return v.MarshalSSZ()
++	}
++
+ 	rval := reflect.ValueOf(val)
+ 
+ 	// We pre-allocate a buffer-size depending on the value's calculated total byte size.
+@@ -87,6 +93,9 @@ func Unmarshal(input []byte, val interface{}) error {
+ 	if val == nil {
+ 		return errors.New("cannot unmarshal into untyped, nil value")
+ 	}
++	if v, ok := val.(fssz.Unmarshaler); ok {
++		return v.UnmarshalSSZ(input)
++	}
+ 	if len(input) == 0 {
+ 		return errors.New("no data to unmarshal from, input is an empty byte slice []byte{}")
+ 	}

tools/ssz.bzl

@@ -1,26 +1,40 @@
 load(
     "@io_bazel_rules_go//go:def.bzl",
     "GoLibrary",
+    "GoSource",
 )
 
 def _ssz_go_proto_library_impl(ctx):
-    go_proto = ctx.attr.go_proto
+    if ctx.attr.go_proto != None:
+        go_proto = ctx.attr.go_proto
+        input_files = go_proto[OutputGroupInfo].go_generated_srcs.to_list()
+        package_path = input_files[0].dirname
+    elif hasattr(ctx.attr, "srcs") and len(ctx.attr.srcs) > 0:
+        package_path = ctx.attr.srcs[0].files.to_list()[0].dirname
+        input_files = ctx.attr.srcs[0].files.to_list()
+    else:
+        fail("Must have go_proto or srcs")
 
-    generated_pb_go_files = go_proto[OutputGroupInfo].go_generated_srcs
+    # Run the tool.
-
-    # Run the tool on the generated files
-    package_path = generated_pb_go_files.to_list()[0].dirname
     output = ctx.outputs.out
     args = [
         "--output=%s" % output.path,
         "--path=%s" % package_path,
     ]
+    if hasattr(ctx.attr, "includes") and len(ctx.attr.includes) > 0:
+        incs = []
+        for include in ctx.attr.includes:
+            incs.append(include[GoSource].srcs[0].dirname)
+            input_files += include[GoSource].srcs
+        args.append("--include=%s" % ",".join(incs))
+
     if len(ctx.attr.objs) > 0:
         args += ["--objs=%s" % ",".join(ctx.attr.objs)]
+
     ctx.actions.run(
         executable = ctx.executable.sszgen,
         progress_message = "Generating ssz marshal and unmarshal functions",
-        inputs = generated_pb_go_files,
+        inputs = input_files,
         arguments = args,
         outputs = [output],
     )
@@ -56,6 +70,7 @@ go_library(
 ssz_gen_marshal = rule(
     implementation = _ssz_go_proto_library_impl,
     attrs = {
+        "srcs": attr.label_list(allow_files = True),
         "go_proto": attr.label(providers = [GoLibrary]),
         "sszgen": attr.label(
             default = Label("@com_github_ferranbt_fastssz//sszgen:sszgen"),
@@ -63,6 +78,7 @@ ssz_gen_marshal = rule(
             cfg = "host",
         ),
         "objs": attr.string_list(),
+        "includes": attr.label_list(providers = [GoLibrary]),
     },
     outputs = {"out": "generated.ssz.go"},
 )