From c11fa183b9684e983d7783c1af691f649fa0ea1d Mon Sep 17 00:00:00 2001 From: Bastin <43618253+Inspector-Butters@users.noreply.github.com> Date: Thu, 8 Jan 2026 15:16:22 +0100 Subject: [PATCH] Ephemeral debug logfile (#16108) **What type of PR is this?** Feature **What does this PR do? Why is it needed?** This PR introduces an ephemeral log file that captures debug logs for 24 hours. - it captures debug logs regardless of the user provided (or non-provided) `--verbosity` flag. - it allows a maximum of 250MB for each log file. - it keeps 1 backup logfile in case of size-based rotations. (as opposed to time-based) - this is enabled by default for beacon and validator nodes. - the log files live in `datadir/logs/` directory under the names of `beacon-chain.log` and `validator.log`. backups have a timestamp in their name as well. - the feature can be disabled using the `--disable-ephemeral-log-file` flag. --- changelog/bastin_ephemeral-debug-logfile.md | 5 +++ cmd/beacon-chain/flags/base.go | 6 ++++ cmd/beacon-chain/main.go | 7 ++++ cmd/beacon-chain/usage.go | 1 + cmd/validator/flags/flags.go | 6 ++++ cmd/validator/main.go | 7 ++++ cmd/validator/usage.go | 1 + go.mod | 2 +- io/logs/BUILD.bazel | 2 ++ io/logs/logutil.go | 39 ++++++++++++++++++++- io/logs/logutil_test.go | 35 ++++++++++++++++++ 11 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 changelog/bastin_ephemeral-debug-logfile.md diff --git a/changelog/bastin_ephemeral-debug-logfile.md b/changelog/bastin_ephemeral-debug-logfile.md new file mode 100644 index 0000000000..bb954bcccb --- /dev/null +++ b/changelog/bastin_ephemeral-debug-logfile.md @@ -0,0 +1,5 @@ +### Added + +- Added an ephemeral debug logfile that for beacon and validator nodes that captures debug-level logs for 24 hours. It + also keeps 1 backup of in case of size-based rotation. The logfiles are stored in `datadir/logs/`. This feature is + enabled by default and can be disabled by setting the `--disable-ephemeral-log-file` flag. \ No newline at end of file diff --git a/cmd/beacon-chain/flags/base.go b/cmd/beacon-chain/flags/base.go index 789200cda5..1cb8bf4ba1 100644 --- a/cmd/beacon-chain/flags/base.go +++ b/cmd/beacon-chain/flags/base.go @@ -356,6 +356,12 @@ var ( Usage: "A comma-separated list of exponents (of 2) in decreasing order, defining the state diff hierarchy levels. The last exponent must be greater than or equal to 5.", Value: cli.NewIntSlice(21, 18, 16, 13, 11, 9, 5), } + // DisableEphemeralLogFile disables the 24 hour debug log file. + DisableEphemeralLogFile = &cli.BoolFlag{ + Name: "disable-ephemeral-log-file", + Usage: "Disables the creation of a debug log file that keeps 24 hours of logs.", + Value: false, + } // DisableGetBlobsV2 disables the engine_getBlobsV2 usage. DisableGetBlobsV2 = &cli.BoolFlag{ Name: "disable-get-blobs-v2", diff --git a/cmd/beacon-chain/main.go b/cmd/beacon-chain/main.go index ce839899c0..3af661b9cd 100644 --- a/cmd/beacon-chain/main.go +++ b/cmd/beacon-chain/main.go @@ -158,6 +158,7 @@ var appFlags = []cli.Flag{ dasFlags.BackfillOldestSlot, dasFlags.BlobRetentionEpochFlag, flags.BatchVerifierLimit, + flags.DisableEphemeralLogFile, } func init() { @@ -223,6 +224,12 @@ func before(ctx *cli.Context) error { } } + if !ctx.Bool(flags.DisableEphemeralLogFile.Name) { + if err := logs.ConfigureEphemeralLogFile(ctx.String(cmd.DataDirFlag.Name), ctx.App.Name); err != nil { + log.WithError(err).Error("Failed to configure debug log file") + } + } + if err := cmd.ExpandSingleEndpointIfFile(ctx, flags.ExecutionEngineEndpoint); err != nil { return errors.Wrap(err, "failed to expand single endpoint") } diff --git a/cmd/beacon-chain/usage.go b/cmd/beacon-chain/usage.go index 303a47b433..fb71d6f18e 100644 --- a/cmd/beacon-chain/usage.go +++ b/cmd/beacon-chain/usage.go @@ -200,6 +200,7 @@ var appHelpFlagGroups = []flagGroup{ cmd.LogFormat, cmd.LogFileName, cmd.VerbosityFlag, + flags.DisableEphemeralLogFile, }, }, { // Feature flags. diff --git a/cmd/validator/flags/flags.go b/cmd/validator/flags/flags.go index 3580effef0..16f4161100 100644 --- a/cmd/validator/flags/flags.go +++ b/cmd/validator/flags/flags.go @@ -410,6 +410,12 @@ var ( Usage: "Maximum number of health checks to perform before exiting if not healthy. Set to 0 or a negative number for indefinite checks.", Value: DefaultMaxHealthChecks, } + // DisableEphemeralLogFile disables the 24 hour debug log file. + DisableEphemeralLogFile = &cli.BoolFlag{ + Name: "disable-ephemeral-log-file", + Usage: "Disables the creation of a debug log file that keeps 24 hours of logs.", + Value: false, + } ) // DefaultValidatorDir returns OS-specific default validator directory. diff --git a/cmd/validator/main.go b/cmd/validator/main.go index 60176a48b4..7b4d8bea61 100644 --- a/cmd/validator/main.go +++ b/cmd/validator/main.go @@ -115,6 +115,7 @@ var appFlags = []cli.Flag{ debug.BlockProfileRateFlag, debug.MutexProfileFractionFlag, cmd.AcceptTosFlag, + flags.DisableEphemeralLogFile, } func init() { @@ -199,6 +200,12 @@ func main() { } } + if !ctx.Bool(flags.DisableEphemeralLogFile.Name) { + if err := logs.ConfigureEphemeralLogFile(ctx.String(cmd.DataDirFlag.Name), ctx.App.Name); err != nil { + log.WithError(err).Error("Failed to configure debug log file") + } + } + // Fix data dir for Windows users. outdatedDataDir := filepath.Join(file.HomeDir(), "AppData", "Roaming", "Eth2Validators") currentDataDir := flags.DefaultValidatorDir() diff --git a/cmd/validator/usage.go b/cmd/validator/usage.go index 8e2f4fed1b..0ae3c286ba 100644 --- a/cmd/validator/usage.go +++ b/cmd/validator/usage.go @@ -75,6 +75,7 @@ var appHelpFlagGroups = []flagGroup{ cmd.GrpcMaxCallRecvMsgSizeFlag, cmd.AcceptTosFlag, cmd.ApiTimeoutFlag, + flags.DisableEphemeralLogFile, }, }, { diff --git a/go.mod b/go.mod index 8b0952af28..ccb798f714 100644 --- a/go.mod +++ b/go.mod @@ -96,6 +96,7 @@ require ( google.golang.org/grpc v1.71.0 google.golang.org/protobuf v1.36.5 gopkg.in/d4l3k/messagediff.v1 v1.2.1 + gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 honnef.co/go/tools v0.6.0 @@ -273,7 +274,6 @@ require ( golang.org/x/tools/go/expect v0.1.1-deprecated // indirect gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect lukechampine.com/blake3 v1.3.0 // indirect rsc.io/tmplfunc v0.0.3 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/io/logs/BUILD.bazel b/io/logs/BUILD.bazel index 84fb131d61..b543826795 100644 --- a/io/logs/BUILD.bazel +++ b/io/logs/BUILD.bazel @@ -17,7 +17,9 @@ go_library( "//io/file:go_default_library", "//runtime/logging/logrus-prefixed-formatter:go_default_library", "@com_github_hashicorp_golang_lru//:go_default_library", + "@com_github_pkg_errors//:go_default_library", "@com_github_sirupsen_logrus//:go_default_library", + "@in_gopkg_natefinch_lumberjack_v2//:go_default_library", ], ) diff --git a/io/logs/logutil.go b/io/logs/logutil.go index a7831b7603..868d8d5e44 100644 --- a/io/logs/logutil.go +++ b/io/logs/logutil.go @@ -12,12 +12,16 @@ import ( "github.com/OffchainLabs/prysm/v7/config/params" "github.com/OffchainLabs/prysm/v7/io/file" prefixed "github.com/OffchainLabs/prysm/v7/runtime/logging/logrus-prefixed-formatter" + "github.com/pkg/errors" "github.com/sirupsen/logrus" + "gopkg.in/natefinch/lumberjack.v2" ) +var ephemeralLogFileVerbosity = logrus.DebugLevel + // SetLoggingLevel sets the base logging level for logrus. func SetLoggingLevel(lvl logrus.Level) { - logrus.SetLevel(lvl) + logrus.SetLevel(max(lvl, ephemeralLogFileVerbosity)) } func addLogWriter(w io.Writer) { @@ -61,6 +65,39 @@ func ConfigurePersistentLogging(logFileName string, format string, lvl logrus.Le return nil } +// ConfigureEphemeralLogFile adds a log file that keeps 24 hours of logs with >debug verbosity. +func ConfigureEphemeralLogFile(datadirPath string, app string) error { + logFilePath := filepath.Join(datadirPath, "logs", app+".log") + if err := file.MkdirAll(filepath.Dir(logFilePath)); err != nil { + return errors.Wrap(err, "failed to create directory") + } + + // Create formatter and writer hook + formatter := new(prefixed.TextFormatter) + formatter.TimestampFormat = "2006-01-02 15:04:05.00" + formatter.FullTimestamp = true + // If persistent log files are written - we disable the log messages coloring because + // the colors are ANSI codes and seen as gibberish in the log files. + formatter.DisableColors = true + + // configure the lumberjack log writer to rotate logs every ~24 hours + debugWriter := &lumberjack.Logger{ + Filename: logFilePath, + MaxSize: 250, // MB, to avoid unbounded growth + MaxBackups: 1, // one backup in case of size-based rotations + MaxAge: 1, // days; files older than this are removed + } + + logrus.AddHook(&WriterHook{ + Formatter: formatter, + Writer: debugWriter, + AllowedLevels: logrus.AllLevels[:ephemeralLogFileVerbosity+1], + }) + + logrus.Info("Ephemeral log file initialized") + return nil +} + // MaskCredentialsLogging masks the url credentials before logging for security purpose // [scheme:][//[userinfo@]host][/]path[?query][#fragment] --> [scheme:][//[***]host][/***][#***] // if the format is not matched nothing is done, string is returned as is. diff --git a/io/logs/logutil_test.go b/io/logs/logutil_test.go index d65361b745..8289ff077c 100644 --- a/io/logs/logutil_test.go +++ b/io/logs/logutil_test.go @@ -62,3 +62,38 @@ func TestConfigurePersistantLogging(t *testing.T) { return } } + +func TestConfigureEphemeralLogFile(t *testing.T) { + testParentDir := t.TempDir() + + // 1. Test creation of file in an existing parent directory + existingDirectory := "test-1-existing-testing-dir" + + err := ConfigureEphemeralLogFile(fmt.Sprintf("%s/%s", testParentDir, existingDirectory), "beacon-chain") + require.NoError(t, err) + + // 2. Test creation of file along with parent directory + nonExistingDirectory := "test-2-non-existing-testing-dir" + + err = ConfigureEphemeralLogFile(fmt.Sprintf("%s/%s", testParentDir, nonExistingDirectory), "beacon-chain") + require.NoError(t, err) + + // 3. Test creation of file in an existing parent directory with a non-existing sub-directory + existingDirectory = "test-3-existing-testing-dir" + nonExistingSubDirectory := "test-3-non-existing-sub-dir" + err = os.Mkdir(fmt.Sprintf("%s/%s", testParentDir, existingDirectory), 0700) + if err != nil { + return + } + err = ConfigureEphemeralLogFile(fmt.Sprintf("%s/%s/%s", testParentDir, existingDirectory, nonExistingSubDirectory), "beacon-chain") + require.NoError(t, err) + + //4. Create log file in a directory without 700 permissions + existingDirectory = "test-4-existing-testing-dir" + err = os.Mkdir(fmt.Sprintf("%s/%s", testParentDir, existingDirectory), 0750) + if err != nil { + return + } + err = ConfigureEphemeralLogFile(fmt.Sprintf("%s/%s/%s", testParentDir, existingDirectory, nonExistingSubDirectory), "beacon-chain") + require.NoError(t, err) +}