Web3signer: persistent public keys (#13682)

* WIP

* broken and still wip

* more wip improving saving

* wip

* removing cyclic dependency

* gaz

* fixes

* fixing more tests and how files load

* fixing wallet tests

* fixing test

* updating keymanager tests

* improving how the web3signer keymanager works

* WIP

* updated keymanager to read from file

* gaz

* reuse readkeyfile function and add in duplicate keys check

* adding in locks to increase safety

* refactored how saving keys work, more tests needed:

* fix test

* fix tests

* adding unit tests and cleaning up locks

* fixing tests

* tests were not fixed properly

* removing unneeded files

* Update cmd/validator/accounts/wallet_utils.go

Co-authored-by: Sammy Rosso <15244892+saolyn@users.noreply.github.com>

* Update validator/accounts/wallet/wallet.go

Co-authored-by: Sammy Rosso <15244892+saolyn@users.noreply.github.com>

* review feedback

* updating flags and e2e

* deepsource fix

* resolving feedback

* removing fatal test for now

* addressing manu's feedback

* gofmt

* fixing tests

* fixing unit tests

* more idomatic feedback

* updating log files

* updating based on preston's suggestion

* improving logs and event triggers

* addressing comments from manu

* truncating was not triggering key file reload

* fixing unit test

* removing wrong dependency

* fix another broken unit test

* fixing bad pathing on file

* handle errors in test

* fixing testdata dependency

* resolving deepsource and comment around logs

* removing unneeded buffer

* reworking ux of web3signer file, unit tests to come

* adding unit tests for file change retries

* Update validator/keymanager/remote-web3signer/keymanager.go

Co-authored-by: Sammy Rosso <15244892+saolyn@users.noreply.github.com>

* Update validator/keymanager/remote-web3signer/keymanager.go

Co-authored-by: Sammy Rosso <15244892+saolyn@users.noreply.github.com>

* Update validator/keymanager/remote-web3signer/keymanager.go

Co-authored-by: Sammy Rosso <15244892+saolyn@users.noreply.github.com>

* updating based on review feedback

* missed err check

* adding some aliases to make running easier

* Update validator/keymanager/remote-web3signer/log.go

Co-authored-by: Manu NALEPA <enalepa@offchainlabs.com>

* Update validator/keymanager/remote-web3signer/keymanager.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update validator/keymanager/remote-web3signer/keymanager.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update validator/keymanager/remote-web3signer/keymanager.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update validator/keymanager/remote-web3signer/keymanager.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update validator/keymanager/remote-web3signer/keymanager.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update validator/keymanager/remote-web3signer/keymanager.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update validator/keymanager/remote-web3signer/keymanager.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update validator/keymanager/remote-web3signer/keymanager.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update validator/keymanager/remote-web3signer/keymanager.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* radek's review

* Update validator/keymanager/remote-web3signer/internal/client.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update validator/keymanager/remote-web3signer/keymanager.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update validator/keymanager/remote-web3signer/keymanager.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update validator/keymanager/remote-web3signer/keymanager.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update validator/keymanager/remote-web3signer/keymanager.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update validator/keymanager/remote-web3signer/keymanager.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update validator/keymanager/remote-web3signer/keymanager.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update validator/keymanager/remote-web3signer/keymanager.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* addressing more review feedback and linting

* fixing tests

* adding log

* adding 1 more test

* improving logs

---------

Co-authored-by: Sammy Rosso <15244892+saolyn@users.noreply.github.com>
Co-authored-by: Manu NALEPA <enalepa@offchainlabs.com>
Co-authored-by: Radosław Kapka <rkapka@wp.pl>

io/file/BUILD.bazel

@@ -2,12 +2,16 @@ load("@prysm//tools/go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
-    srcs = ["fileutil.go"],
+    srcs = [
+        "fileutil.go",
+        "log.go",
+    ],
     importpath = "github.com/prysmaticlabs/prysm/v5/io/file",
     visibility = ["//visibility:public"],
     deps = [
         "//config/params:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
     ],
 )
 

io/file/fileutil.go

@@ -382,3 +382,27 @@ func DirFiles(dir string) ([]string, error) {
 	}
 	return files, nil
 }
+
+// WriteLinesToFile writes a slice of strings to a file, each string on a new line.
+func WriteLinesToFile(lines []string, filename string) error {
+	// Open the file for writing. If the file does not exist, create it, or truncate it if it does.
+	f, err := os.Create(filepath.Clean(filename))
+	if err != nil {
+		return fmt.Errorf("error creating file: %w", err)
+	}
+	defer func(file *os.File) {
+		err := file.Close()
+		if err != nil {
+			log.Error(err.Error())
+		}
+	}(f)
+
+	// Iterate through all lines in the slice and write them to the file
+	for _, line := range lines {
+		if _, err := f.WriteString(line + "\n"); err != nil {
+			return fmt.Errorf("error writing line to file: %w", err)
+		}
+	}
+
+	return nil
+}

io/file/fileutil_test.go

@@ -24,6 +24,7 @@ import (
 	"os/user"
 	"path/filepath"
 	"sort"
+	"strings"
 	"testing"
 
 	"github.com/prysmaticlabs/prysm/v5/config/params"
@@ -567,3 +568,37 @@ func TestHasReadWritePermissions(t *testing.T) {
 		})
 	}
 }
+
+func TestWriteLinesToFile(t *testing.T) {
+	filename := filepath.Join(t.TempDir(), "testfile.txt")
+	t.Run("write to a new file", func(t *testing.T) {
+		lines := []string{"line1", "line2", "line3"}
+		require.NoError(t, file.WriteLinesToFile(lines, filename))
+		// Check file content
+		content, err := os.ReadFile(filepath.Clean(filename))
+		if err != nil {
+			t.Fatalf("failed to read file: %v", err)
+		}
+
+		// Join lines with newline for comparison
+		expectedContent := strings.Join(lines, "\n") + "\n"
+		if string(content) != expectedContent {
+			t.Errorf("file content = %q, want %q", string(content), expectedContent)
+		}
+	})
+	t.Run("overwrite existing file", func(t *testing.T) {
+		lines := []string{"line4", "line5"}
+		require.NoError(t, file.WriteLinesToFile(lines, filename))
+		// Check file content
+		content, err := os.ReadFile(filepath.Clean(filename))
+		if err != nil {
+			t.Fatalf("failed to read file: %v", err)
+		}
+
+		// Join lines with newline for comparison
+		expectedContent := strings.Join(lines, "\n") + "\n"
+		if string(content) != expectedContent {
+			t.Errorf("file content = %q, want %q", string(content), expectedContent)
+		}
+	})
+}

io/file/log.go

@@ -0,0 +1,5 @@
+package file
+
+import "github.com/sirupsen/logrus"
+
+var log = logrus.WithField("prefix", "fileutil")