Ephemeral debug logfile (#16108)

**What type of PR is this?**
Feature

**What does this PR do? Why is it needed?**
This PR introduces an ephemeral log file that captures debug logs for 24
hours.

- it captures debug logs regardless of the user provided (or
non-provided) `--verbosity` flag.
- it allows a maximum of 250MB for each log file. 
- it keeps 1 backup logfile in case of size-based rotations. (as opposed
to time-based)
- this is enabled by default for beacon and validator nodes.
- the log files live in `datadir/logs/` directory under the names of
`beacon-chain.log` and `validator.log`. backups have a timestamp in
their name as well.
- the feature can be disabled using the `--disable-ephemeral-log-file`
flag.

changelog/bastin_ephemeral-debug-logfile.md

@@ -0,0 +1,5 @@
+### Added
+
+- Added an ephemeral debug logfile that for beacon and validator nodes that captures debug-level logs for 24 hours. It
+  also keeps 1 backup of in case of size-based rotation. The logfiles are stored in `datadir/logs/`. This feature is
+  enabled by default and can be disabled by setting the `--disable-ephemeral-log-file` flag.

cmd/beacon-chain/flags/base.go

@@ -356,6 +356,12 @@ var (
 		Usage: "A comma-separated list of exponents (of 2) in decreasing order, defining the state diff hierarchy levels. The last exponent must be greater than or equal to 5.",
 		Value: cli.NewIntSlice(21, 18, 16, 13, 11, 9, 5),
 	}
+	// DisableEphemeralLogFile disables the 24 hour debug log file.
+	DisableEphemeralLogFile = &cli.BoolFlag{
+		Name:  "disable-ephemeral-log-file",
+		Usage: "Disables the creation of a debug log file that keeps 24 hours of logs.",
+		Value: false,
+	}
 	// DisableGetBlobsV2 disables the engine_getBlobsV2 usage.
 	DisableGetBlobsV2 = &cli.BoolFlag{
 		Name:  "disable-get-blobs-v2",

cmd/beacon-chain/main.go

@@ -158,6 +158,7 @@ var appFlags = []cli.Flag{
 	dasFlags.BackfillOldestSlot,
 	dasFlags.BlobRetentionEpochFlag,
 	flags.BatchVerifierLimit,
+	flags.DisableEphemeralLogFile,
 }
 
 func init() {
@@ -223,6 +224,12 @@ func before(ctx *cli.Context) error {
 		}
 	}
 
+	if !ctx.Bool(flags.DisableEphemeralLogFile.Name) {
+		if err := logs.ConfigureEphemeralLogFile(ctx.String(cmd.DataDirFlag.Name), ctx.App.Name); err != nil {
+			log.WithError(err).Error("Failed to configure debug log file")
+		}
+	}
+
 	if err := cmd.ExpandSingleEndpointIfFile(ctx, flags.ExecutionEngineEndpoint); err != nil {
 		return errors.Wrap(err, "failed to expand single endpoint")
 	}

cmd/beacon-chain/usage.go

@@ -200,6 +200,7 @@ var appHelpFlagGroups = []flagGroup{
 			cmd.LogFormat,
 			cmd.LogFileName,
 			cmd.VerbosityFlag,
+			flags.DisableEphemeralLogFile,
 		},
 	},
 	{ // Feature flags.

cmd/validator/flags/flags.go

@@ -410,6 +410,12 @@ var (
 		Usage: "Maximum number of health checks to perform before exiting if not healthy. Set to 0 or a negative number for indefinite checks.",
 		Value: DefaultMaxHealthChecks,
 	}
+	// DisableEphemeralLogFile disables the 24 hour debug log file.
+	DisableEphemeralLogFile = &cli.BoolFlag{
+		Name:  "disable-ephemeral-log-file",
+		Usage: "Disables the creation of a debug log file that keeps 24 hours of logs.",
+		Value: false,
+	}
 )
 
 // DefaultValidatorDir returns OS-specific default validator directory.

cmd/validator/main.go

@@ -115,6 +115,7 @@ var appFlags = []cli.Flag{
 	debug.BlockProfileRateFlag,
 	debug.MutexProfileFractionFlag,
 	cmd.AcceptTosFlag,
+	flags.DisableEphemeralLogFile,
 }
 
 func init() {
@@ -199,6 +200,12 @@ func main() {
 				}
 			}
 
+			if !ctx.Bool(flags.DisableEphemeralLogFile.Name) {
+				if err := logs.ConfigureEphemeralLogFile(ctx.String(cmd.DataDirFlag.Name), ctx.App.Name); err != nil {
+					log.WithError(err).Error("Failed to configure debug log file")
+				}
+			}
+
 			// Fix data dir for Windows users.
 			outdatedDataDir := filepath.Join(file.HomeDir(), "AppData", "Roaming", "Eth2Validators")
 			currentDataDir := flags.DefaultValidatorDir()

cmd/validator/usage.go

@@ -75,6 +75,7 @@ var appHelpFlagGroups = []flagGroup{
 			cmd.GrpcMaxCallRecvMsgSizeFlag,
 			cmd.AcceptTosFlag,
 			cmd.ApiTimeoutFlag,
+			flags.DisableEphemeralLogFile,
 		},
 	},
 	{

go.mod

@@ -96,6 +96,7 @@ require (
 	google.golang.org/grpc v1.71.0
 	google.golang.org/protobuf v1.36.5
 	gopkg.in/d4l3k/messagediff.v1 v1.2.1
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
 	honnef.co/go/tools v0.6.0
@@ -273,7 +274,6 @@ require (
 	golang.org/x/tools/go/expect v0.1.1-deprecated // indirect
 	gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	lukechampine.com/blake3 v1.3.0 // indirect
 	rsc.io/tmplfunc v0.0.3 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect

io/logs/BUILD.bazel

@@ -17,7 +17,9 @@ go_library(
         "//io/file:go_default_library",
         "//runtime/logging/logrus-prefixed-formatter:go_default_library",
         "@com_github_hashicorp_golang_lru//:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
+        "@in_gopkg_natefinch_lumberjack_v2//:go_default_library",
     ],
 )
 

io/logs/logutil.go

@@ -12,12 +12,16 @@ import (
 	"github.com/OffchainLabs/prysm/v7/config/params"
 	"github.com/OffchainLabs/prysm/v7/io/file"
 	prefixed "github.com/OffchainLabs/prysm/v7/runtime/logging/logrus-prefixed-formatter"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"gopkg.in/natefinch/lumberjack.v2"
 )
 
+var ephemeralLogFileVerbosity = logrus.DebugLevel
+
 // SetLoggingLevel sets the base logging level for logrus.
 func SetLoggingLevel(lvl logrus.Level) {
-	logrus.SetLevel(lvl)
+	logrus.SetLevel(max(lvl, ephemeralLogFileVerbosity))
 }
 
 func addLogWriter(w io.Writer) {
@@ -61,6 +65,39 @@ func ConfigurePersistentLogging(logFileName string, format string, lvl logrus.Le
 	return nil
 }
 
+// ConfigureEphemeralLogFile adds a log file that keeps 24 hours of logs with >debug verbosity.
+func ConfigureEphemeralLogFile(datadirPath string, app string) error {
+	logFilePath := filepath.Join(datadirPath, "logs", app+".log")
+	if err := file.MkdirAll(filepath.Dir(logFilePath)); err != nil {
+		return errors.Wrap(err, "failed to create directory")
+	}
+
+	// Create formatter and writer hook
+	formatter := new(prefixed.TextFormatter)
+	formatter.TimestampFormat = "2006-01-02 15:04:05.00"
+	formatter.FullTimestamp = true
+	// If persistent log files are written - we disable the log messages coloring because
+	// the colors are ANSI codes and seen as gibberish in the log files.
+	formatter.DisableColors = true
+
+	// configure the lumberjack log writer to rotate logs every ~24 hours
+	debugWriter := &lumberjack.Logger{
+		Filename:   logFilePath,
+		MaxSize:    250, // MB, to avoid unbounded growth
+		MaxBackups: 1,   // one backup in case of size-based rotations
+		MaxAge:     1,   // days; files older than this are removed
+	}
+
+	logrus.AddHook(&WriterHook{
+		Formatter:     formatter,
+		Writer:        debugWriter,
+		AllowedLevels: logrus.AllLevels[:ephemeralLogFileVerbosity+1],
+	})
+
+	logrus.Info("Ephemeral log file initialized")
+	return nil
+}
+
 // MaskCredentialsLogging masks the url credentials before logging for security purpose
 // [scheme:][//[userinfo@]host][/]path[?query][#fragment] -->  [scheme:][//[***]host][/***][#***]
 // if the format is not matched nothing is done, string is returned as is.

io/logs/logutil_test.go

@@ -62,3 +62,38 @@ func TestConfigurePersistantLogging(t *testing.T) {
 		return
 	}
 }
+
+func TestConfigureEphemeralLogFile(t *testing.T) {
+	testParentDir := t.TempDir()
+
+	// 1. Test creation of file in an existing parent directory
+	existingDirectory := "test-1-existing-testing-dir"
+
+	err := ConfigureEphemeralLogFile(fmt.Sprintf("%s/%s", testParentDir, existingDirectory), "beacon-chain")
+	require.NoError(t, err)
+
+	// 2. Test creation of file along with parent directory
+	nonExistingDirectory := "test-2-non-existing-testing-dir"
+
+	err = ConfigureEphemeralLogFile(fmt.Sprintf("%s/%s", testParentDir, nonExistingDirectory), "beacon-chain")
+	require.NoError(t, err)
+
+	// 3. Test creation of file in an existing parent directory with a non-existing sub-directory
+	existingDirectory = "test-3-existing-testing-dir"
+	nonExistingSubDirectory := "test-3-non-existing-sub-dir"
+	err = os.Mkdir(fmt.Sprintf("%s/%s", testParentDir, existingDirectory), 0700)
+	if err != nil {
+		return
+	}
+	err = ConfigureEphemeralLogFile(fmt.Sprintf("%s/%s/%s", testParentDir, existingDirectory, nonExistingSubDirectory), "beacon-chain")
+	require.NoError(t, err)
+
+	//4. Create log file in a directory without 700 permissions
+	existingDirectory = "test-4-existing-testing-dir"
+	err = os.Mkdir(fmt.Sprintf("%s/%s", testParentDir, existingDirectory), 0750)
+	if err != nil {
+		return
+	}
+	err = ConfigureEphemeralLogFile(fmt.Sprintf("%s/%s/%s", testParentDir, existingDirectory, nonExistingSubDirectory), "beacon-chain")
+	require.NoError(t, err)
+}