Refactor dependencies, make Prysm "go gettable" (#6053)

* Fix a few deps to work with go.mod, check in generated files

* Update Gossipsub to 1.1 (#5998)

* update libs

* add new validators

* add new deps

* new set of deps

* tls

* further fix gossip update

* get everything to build

* clean up

* gaz

* fix build

* fix all tests

* add deps to images

* imports

Co-authored-by: rauljordan <raul@prysmaticlabs.com>

* Beacon chain builds with go build

* fix bazel

* fix dep

* lint

* Add github action for testing go

* on PR for any branch

* fix libp2p test failure

* Fix TestProcessBlock_PassesProcessingConditions by updating the proposer index in test

* Revert "Fix TestProcessBlock_PassesProcessingConditions by updating the proposer index in test"

This reverts commit 43676894ab.

* Compute and set proposer index instead of hard code

* Add back go mod/sum, fix deps

* go build ./...

* Temporarily skip two tests

* Fix kafka confluent patch

* Fix kafka confluent patch

* fix kafka build

* fix kafka

* Add info in DEPENDENCIES. Added a stub link for Why Bazel? until https://github.com/prysmaticlabs/documentation/issues/138

* Update fuzz ssz files as well

* Update fuzz ssz files as well

* getting closer

* rollback rules_go and gazelle

* fix gogo protobuf

* install librdkafka-dev as part of github actions

* Update kafka to a recent version where librkafkfa is not required for go modules

* clarify comment

* fix kafka build

* disable go tests

* comment

* Fix geth dependencies for end to end

* rename word

* lint

* fix docker

Co-authored-by: Nishant Das <nishdas93@gmail.com>
Co-authored-by: rauljordan <raul@prysmaticlabs.com>
Co-authored-by: terence tsao <terence@prysmaticlabs.com>

tools/unencrypted-keys-gen/BUILD.bazel

@@ -10,6 +10,7 @@ go_library(
     ],
     deps = [
         "//shared/interop:go_default_library",
+        "//tools/unencrypted-keys-gen/keygen:go_default_library",
     ],
 )
 
@@ -23,4 +24,5 @@ go_test(
     name = "go_default_test",
     srcs = ["main_test.go"],
     embed = [":go_default_library"],
+    deps = ["//tools/unencrypted-keys-gen/keygen:go_default_library"],
 )

tools/unencrypted-keys-gen/keygen/BUILD.bazel

@@ -0,0 +1,9 @@
+load("@prysm//tools/go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["keygen.go"],
+    importpath = "github.com/prysmaticlabs/prysm/tools/unencrypted-keys-gen/keygen",
+    visibility = ["//visibility:public"],
+    deps = ["@com_github_sirupsen_logrus//:go_default_library"],
+)

tools/unencrypted-keys-gen/keygen/keygen.go

@@ -0,0 +1,36 @@
+package keygen
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// UnencryptedKeysContainer defines the structure of the unecrypted key JSON file.
+type UnencryptedKeysContainer struct {
+	Keys []*UnencryptedKeys `json:"keys"`
+}
+
+// UnencryptedKeys is the inner struct of the JSON file.
+type UnencryptedKeys struct {
+	ValidatorKey  []byte `json:"validator_key"`
+	WithdrawalKey []byte `json:"withdrawal_key"`
+}
+
+// SaveUnencryptedKeysToFile JSON encodes the container and writes to the writer.
+func SaveUnencryptedKeysToFile(w io.Writer, ctnr *UnencryptedKeysContainer) error {
+	enc, err := json.Marshal(ctnr)
+	if err != nil {
+		log.Fatal(err)
+	}
+	n, err := w.Write(enc)
+	if err != nil {
+		return err
+	}
+	if n != len(enc) {
+		return fmt.Errorf("failed to write %d bytes to file, wrote %d", len(enc), n)
+	}
+	return nil
+}

tools/unencrypted-keys-gen/main.go

@@ -1,10 +1,8 @@
 package main
 
 import (
-	"encoding/json"
 	"flag"
-	"fmt"
-	"io"
+	"github.com/prysmaticlabs/prysm/tools/unencrypted-keys-gen/keygen"
 	"log"
 	"os"
 
@@ -18,17 +16,6 @@ var (
 	overwrite  = flag.Bool("overwrite", false, "If the key file exists, it will be overwritten")
 )
 
-// UnencryptedKeysContainer defines the structure of the unecrypted key JSON file.
-type UnencryptedKeysContainer struct {
-	Keys []*UnencryptedKeys `json:"keys"`
-}
-
-// UnencryptedKeys is the inner struct of the JSON file.
-type UnencryptedKeys struct {
-	ValidatorKey  []byte `json:"validator_key"`
-	WithdrawalKey []byte `json:"withdrawal_key"`
-}
-
 func main() {
 	flag.Parse()
 	if *numKeys == 0 {
@@ -55,14 +42,14 @@ func main() {
 	}()
 
 	ctnr := generateUnencryptedKeys(*startIndex)
-	if err := SaveUnencryptedKeysToFile(file, ctnr); err != nil {
+	if err := keygen.SaveUnencryptedKeysToFile(file, ctnr); err != nil {
 		log.Fatal(err)
 	}
 }
 
-func generateUnencryptedKeys(startIndex uint64) *UnencryptedKeysContainer {
-	ctnr := &UnencryptedKeysContainer{
-		Keys: make([]*UnencryptedKeys, *numKeys),
+func generateUnencryptedKeys(startIndex uint64) *keygen.UnencryptedKeysContainer {
+	ctnr := &keygen.UnencryptedKeysContainer{
+		Keys: make([]*keygen.UnencryptedKeys, *numKeys),
 	}
 
 	sks, _, err := interop.DeterministicallyGenerateKeys(startIndex, uint64(*numKeys))
@@ -72,7 +59,7 @@ func generateUnencryptedKeys(startIndex uint64) *UnencryptedKeysContainer {
 	}
 
 	for i, sk := range sks {
-		ctnr.Keys[i] = &UnencryptedKeys{
+		ctnr.Keys[i] = &keygen.UnencryptedKeys{
 			ValidatorKey:  sk.Marshal(),
 			WithdrawalKey: sk.Marshal(),
 		}
@@ -80,18 +67,3 @@ func generateUnencryptedKeys(startIndex uint64) *UnencryptedKeysContainer {
 	return ctnr
 }
 
-// SaveUnencryptedKeysToFile JSON encodes the container and writes to the writer.
-func SaveUnencryptedKeysToFile(w io.Writer, ctnr *UnencryptedKeysContainer) error {
-	enc, err := json.Marshal(ctnr)
-	if err != nil {
-		log.Fatal(err)
-	}
-	n, err := w.Write(enc)
-	if err != nil {
-		return err
-	}
-	if n != len(enc) {
-		return fmt.Errorf("failed to write %d bytes to file, wrote %d", len(enc), n)
-	}
-	return nil
-}

tools/unencrypted-keys-gen/main_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"encoding/json"
+	"github.com/prysmaticlabs/prysm/tools/unencrypted-keys-gen/keygen"
 	"reflect"
 	"testing"
 )
@@ -12,11 +13,11 @@ func TestSavesUnencryptedKeys(t *testing.T) {
 	numKeys = &keys
 	ctnr := generateUnencryptedKeys(0 /* start index */)
 	buf := new(bytes.Buffer)
-	if err := SaveUnencryptedKeysToFile(buf, ctnr); err != nil {
+	if err := keygen.SaveUnencryptedKeysToFile(buf, ctnr); err != nil {
 		t.Fatal(err)
 	}
 	enc := buf.Bytes()
-	dec := &UnencryptedKeysContainer{}
+	dec := &keygen.UnencryptedKeysContainer{}
 	if err := json.Unmarshal(enc, dec); err != nil {
 		t.Fatal(err)
 	}