mirror of
https://github.com/OffchainLabs/prysm.git
synced 2026-01-09 15:37:56 -05:00
#### This PR sets the foundation for the new logging features. --- The goal of this big PR is the following: 1. Adding a log.go file to every package: [_commit_](54f6396d4c) - Writing a bash script that adds the log.go file to every package that imports logrus, except the excluded packages, configured at the top of the bash script. - the log.go file creates a log variable and sets a field called `package` to the full path of that package. - I have tried to fix every error/problem that came from mass generation of this file. (duplicate declarations, different prefix names, etc...) - some packages had the log.go file from before, and had some helper functions in there as well. I've moved all of them to a `log_helpers.go` file within each package. 2. Create a CI rule which verifies that: [_commit_](b799c3a0ef) - every package which imports logrus, also has a log.go file, except the excluded packages. - the `package` field of each log.go variable, has the correct path. (to detect when we move a package or change it's name) - I pushed a commit with a manually changed log.go file to trigger the ci check failure and it worked. 3. Alter the logging system to read the prefix from this `package` field for every log while outputing: [_commit_](b0c7f1146c) - some packages have/want/need a different log prefix than their package name (like `kv`). This can be solved by keeping a map of package paths to prefix names somewhere. --- **Some notes:** - Please review everything carefully. - I created the `prefixReplacement` map and populated the data that I deemed necessary. Please check it and complain if something doesn't make sense or is missing. I attached at the bottom, the list of all the packages that used to use a different name than their package name as their prefix. - I have chosen to mark some packages to be excluded from this whole process. They will either not log anything, or log without a prefix, or log using their previously defined prefix. See the list of exclusions in the bottom. - I fixed all the tests that failed because of this change. These were failing because they were expecting the old prefix to be in the generated logs. I have changed those to expect the new `package` field instead. This might not be a great solution. Ideally we might want to remove this from the tests so they only test for relevant fields in the logs. but this is a problem for another day. - Please run the node with this config, and mention if you see something weird in the logs. (use different verbosities) - The CI workflow uses a script that basically runs the `hack/gen-logs.sh` and checks that the git diff is zero. that script is `hack/check-logs.sh`. This means that if one runs this script locally, it will not actually _check_ anything, rather than just regenerate the log.go files and fix any mistake. This might be confusing. Please suggest solutions if you think it's a problem. --- **A list of packages that used a different prefix than their package names for their logs:** - beacon-chain/cache/depositsnapshot/ package depositsnapshot, prefix "cache" - beacon-chain/core/transition/log.go — package transition, prefix "state" - beacon-chain/db/kv/log.go — package kv, prefix "db" - beacon-chain/db/slasherkv/log.go — package slasherkv, prefix "slasherdb" - beacon-chain/db/pruner/pruner.go — package pruner, prefix "db-pruner" - beacon-chain/light-client/log.go — package light_client, prefix "light-client" - beacon-chain/operations/attestations/log.go — package attestations, prefix "pool/attestations" - beacon-chain/operations/slashings/log.go — package slashings, prefix "pool/slashings" - beacon-chain/rpc/core/log.go — package core, prefix "rpc/core" - beacon-chain/rpc/eth/beacon/log.go — package beacon, prefix "rpc/beaconv1" - beacon-chain/rpc/eth/validator/log.go — package validator, prefix "beacon-api" - beacon-chain/rpc/prysm/v1alpha1/beacon/log.go — package beacon, prefix "rpc" - beacon-chain/rpc/prysm/v1alpha1/validator/log.go — package validator, prefix "rpc/validator" - beacon-chain/state/stategen/log.go — package stategen, prefix "state-gen" - beacon-chain/sync/checkpoint/log.go — package checkpoint, prefix "checkpoint-sync" - beacon-chain/sync/initial-sync/log.go — package initialsync, prefix "initial-sync" - cmd/prysmctl/p2p/log.go — package p2p, prefix "prysmctl-p2p" - config/features/log.go -- package features, prefix "flags" - io/file/log.go — package file, prefix "fileutil" - proto/prysm/v1alpha1/log.go — package eth, prefix "protobuf" - validator/client/beacon-api/log.go — package beacon_api, prefix "beacon-api" - validator/db/kv/log.go — package kv, prefix "db" - validator/db/filesystem/db.go — package filesystem, prefix "db" - validator/keymanager/derived/log.go — package derived, prefix "derived-keymanager" - validator/keymanager/local/log.go — package local, prefix "local-keymanager" - validator/keymanager/remote-web3signer/log.go — package remote_web3signer, prefix "remote-keymanager" - validator/keymanager/remote-web3signer/internal/log.go — package internal, prefix "remote-web3signer- internal" - beacon-chain/forkchoice/doubly... prefix is "forkchoice-doublylinkedtree" **List of excluded directories (their subdirectories are also excluded):** ``` EXCLUDED_PATH_PREFIXES=( "testing" "validator/client/testutil" "beacon-chain/p2p/testing" "beacon-chain/rpc/eth/config" "beacon-chain/rpc/prysm/v1alpha1/debug" "tools" "runtime" "monitoring" "io" "cmd" ".well-known" "changelog" "hack" "specrefs" "third_party" "bazel-out" "bazel-bin" "bazel-prysm" "bazel-testlogs" "build" ".github" ".jj" ".idea" ".vscode" ) ```
428 lines
11 KiB
Go
428 lines
11 KiB
Go
package prefixed
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
"unicode"
|
|
|
|
"github.com/mgutz/ansi"
|
|
"github.com/sirupsen/logrus"
|
|
"golang.org/x/crypto/ssh/terminal"
|
|
)
|
|
|
|
const defaultTimestampFormat = time.RFC3339
|
|
|
|
var regex = regexp.MustCompile(`^\\[(.*?)\\]`)
|
|
|
|
var (
|
|
baseTimestamp time.Time = time.Now()
|
|
defaultColorScheme *ColorScheme = &ColorScheme{
|
|
InfoLevelStyle: "green",
|
|
WarnLevelStyle: "yellow",
|
|
ErrorLevelStyle: "red",
|
|
FatalLevelStyle: "red",
|
|
PanicLevelStyle: "red",
|
|
DebugLevelStyle: "blue",
|
|
PrefixStyle: "cyan",
|
|
TimestampStyle: "black+h",
|
|
}
|
|
noColorsColorScheme *compiledColorScheme = &compiledColorScheme{
|
|
InfoLevelColor: ansi.ColorFunc(""),
|
|
WarnLevelColor: ansi.ColorFunc(""),
|
|
ErrorLevelColor: ansi.ColorFunc(""),
|
|
FatalLevelColor: ansi.ColorFunc(""),
|
|
PanicLevelColor: ansi.ColorFunc(""),
|
|
DebugLevelColor: ansi.ColorFunc(""),
|
|
PrefixColor: ansi.ColorFunc(""),
|
|
TimestampColor: ansi.ColorFunc(""),
|
|
}
|
|
defaultCompiledColorScheme *compiledColorScheme = compileColorScheme(defaultColorScheme)
|
|
)
|
|
|
|
func miniTS() int {
|
|
return int(time.Since(baseTimestamp) / time.Second)
|
|
}
|
|
|
|
type ColorScheme struct {
|
|
InfoLevelStyle string
|
|
WarnLevelStyle string
|
|
ErrorLevelStyle string
|
|
FatalLevelStyle string
|
|
PanicLevelStyle string
|
|
DebugLevelStyle string
|
|
PrefixStyle string
|
|
TimestampStyle string
|
|
}
|
|
|
|
type compiledColorScheme struct {
|
|
InfoLevelColor func(string) string
|
|
WarnLevelColor func(string) string
|
|
ErrorLevelColor func(string) string
|
|
FatalLevelColor func(string) string
|
|
PanicLevelColor func(string) string
|
|
DebugLevelColor func(string) string
|
|
PrefixColor func(string) string
|
|
TimestampColor func(string) string
|
|
}
|
|
|
|
type TextFormatter struct {
|
|
// Set to true to bypass checking for a TTY before outputting colors.
|
|
ForceColors bool
|
|
|
|
// Whether the logger's out is to a terminal.
|
|
isTerminal bool
|
|
|
|
// Force disabling colors. For a TTY colors are enabled by default.
|
|
DisableColors bool
|
|
|
|
// Force formatted layout, even for non-TTY output.
|
|
ForceFormatting bool
|
|
|
|
// Disable timestamp logging. useful when output is redirected to logging
|
|
// system that already adds timestamps.
|
|
DisableTimestamp bool
|
|
|
|
// Disable the conversion of the log levels to uppercase
|
|
DisableUppercase bool
|
|
|
|
// Enable logging the full timestamp when a TTY is attached instead of just
|
|
// the time passed since beginning of execution.
|
|
FullTimestamp bool
|
|
|
|
// The fields are sorted by default for a consistent output. For applications
|
|
// that log extremely frequently and don't use the JSON formatter this may not
|
|
// be desired.
|
|
DisableSorting bool
|
|
|
|
// Wrap empty fields in quotes if true.
|
|
QuoteEmptyFields bool
|
|
|
|
sync.Once
|
|
|
|
// Pad msg field with spaces on the right for display.
|
|
// The value for this parameter will be the size of padding.
|
|
// Its default value is zero, which means no padding will be applied for msg.
|
|
SpacePadding int
|
|
|
|
// Color scheme to use.
|
|
colorScheme *compiledColorScheme
|
|
|
|
// Can be set to the override the default quoting character "
|
|
// with something else. For example: ', or `.
|
|
QuoteCharacter string
|
|
|
|
// Timestamp format to use for display when a full timestamp is printed.
|
|
TimestampFormat string
|
|
}
|
|
|
|
func getCompiledColor(main string, fallback string) func(string) string {
|
|
var style string
|
|
if main != "" {
|
|
style = main
|
|
} else {
|
|
style = fallback
|
|
}
|
|
return ansi.ColorFunc(style)
|
|
}
|
|
|
|
func compileColorScheme(s *ColorScheme) *compiledColorScheme {
|
|
return &compiledColorScheme{
|
|
InfoLevelColor: getCompiledColor(s.InfoLevelStyle, defaultColorScheme.InfoLevelStyle),
|
|
WarnLevelColor: getCompiledColor(s.WarnLevelStyle, defaultColorScheme.WarnLevelStyle),
|
|
ErrorLevelColor: getCompiledColor(s.ErrorLevelStyle, defaultColorScheme.ErrorLevelStyle),
|
|
FatalLevelColor: getCompiledColor(s.FatalLevelStyle, defaultColorScheme.FatalLevelStyle),
|
|
PanicLevelColor: getCompiledColor(s.PanicLevelStyle, defaultColorScheme.PanicLevelStyle),
|
|
DebugLevelColor: getCompiledColor(s.DebugLevelStyle, defaultColorScheme.DebugLevelStyle),
|
|
PrefixColor: getCompiledColor(s.PrefixStyle, defaultColorScheme.PrefixStyle),
|
|
TimestampColor: getCompiledColor(s.TimestampStyle, defaultColorScheme.TimestampStyle),
|
|
}
|
|
}
|
|
|
|
func (f *TextFormatter) init(entry *logrus.Entry) {
|
|
if len(f.QuoteCharacter) == 0 {
|
|
f.QuoteCharacter = "\""
|
|
}
|
|
if entry.Logger != nil {
|
|
f.isTerminal = f.checkIfTerminal(entry.Logger.Out)
|
|
}
|
|
}
|
|
|
|
func (f *TextFormatter) checkIfTerminal(w io.Writer) bool {
|
|
switch v := w.(type) {
|
|
case *os.File:
|
|
return terminal.IsTerminal(int(v.Fd()))
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func (f *TextFormatter) SetColorScheme(colorScheme *ColorScheme) {
|
|
f.colorScheme = compileColorScheme(colorScheme)
|
|
}
|
|
|
|
func (f *TextFormatter) Format(entry *logrus.Entry) ([]byte, error) {
|
|
var b *bytes.Buffer
|
|
keys := make([]string, 0, len(entry.Data))
|
|
for k := range entry.Data {
|
|
keys = append(keys, k)
|
|
}
|
|
lastKeyIdx := len(keys) - 1
|
|
|
|
if !f.DisableSorting {
|
|
sort.Strings(keys)
|
|
}
|
|
if entry.Buffer != nil {
|
|
b = entry.Buffer
|
|
} else {
|
|
b = &bytes.Buffer{}
|
|
}
|
|
|
|
prefixFieldClashes(entry.Data)
|
|
|
|
f.Do(func() { f.init(entry) })
|
|
|
|
isFormatted := f.ForceFormatting || f.isTerminal
|
|
|
|
timestampFormat := f.TimestampFormat
|
|
if timestampFormat == "" {
|
|
timestampFormat = defaultTimestampFormat
|
|
}
|
|
if isFormatted {
|
|
isColored := (f.ForceColors || f.isTerminal) && !f.DisableColors
|
|
var colorScheme *compiledColorScheme
|
|
if isColored {
|
|
if f.colorScheme == nil {
|
|
colorScheme = defaultCompiledColorScheme
|
|
} else {
|
|
colorScheme = f.colorScheme
|
|
}
|
|
} else {
|
|
colorScheme = noColorsColorScheme
|
|
}
|
|
if err := f.printColored(b, entry, keys, timestampFormat, colorScheme); err != nil {
|
|
return nil, err
|
|
}
|
|
} else {
|
|
if !f.DisableTimestamp {
|
|
if err := f.appendKeyValue(b, "time", entry.Time.Format(timestampFormat), true); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
if err := f.appendKeyValue(b, "level", entry.Level.String(), true); err != nil {
|
|
return nil, err
|
|
}
|
|
if entry.Message != "" {
|
|
if err := f.appendKeyValue(b, "msg", entry.Message, lastKeyIdx >= 0); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
for i, key := range keys {
|
|
if err := f.appendKeyValue(b, key, entry.Data[key], lastKeyIdx != i); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
}
|
|
|
|
if err := b.WriteByte('\n'); err != nil {
|
|
return nil, err
|
|
}
|
|
return b.Bytes(), nil
|
|
}
|
|
|
|
func (f *TextFormatter) printColored(b *bytes.Buffer, entry *logrus.Entry, keys []string, timestampFormat string, colorScheme *compiledColorScheme) (err error) {
|
|
var levelColor func(string) string
|
|
var levelText string
|
|
switch entry.Level {
|
|
case logrus.InfoLevel:
|
|
levelColor = colorScheme.InfoLevelColor
|
|
case logrus.WarnLevel:
|
|
levelColor = colorScheme.WarnLevelColor
|
|
case logrus.ErrorLevel:
|
|
levelColor = colorScheme.ErrorLevelColor
|
|
case logrus.FatalLevel:
|
|
levelColor = colorScheme.FatalLevelColor
|
|
case logrus.PanicLevel:
|
|
levelColor = colorScheme.PanicLevelColor
|
|
default:
|
|
levelColor = colorScheme.DebugLevelColor
|
|
}
|
|
|
|
if entry.Level != logrus.WarnLevel {
|
|
levelText = entry.Level.String()
|
|
} else {
|
|
levelText = "warn"
|
|
}
|
|
|
|
if !f.DisableUppercase {
|
|
levelText = strings.ToUpper(levelText)
|
|
}
|
|
|
|
level := levelColor(fmt.Sprintf("%5s", levelText))
|
|
|
|
prefix := ""
|
|
prefixValue, message := getPrefixAndMessage(entry)
|
|
if len(prefixValue) > 0 {
|
|
prefix = colorScheme.PrefixColor(" " + prefixValue + ":")
|
|
}
|
|
|
|
messageFormat := "%s"
|
|
if f.SpacePadding != 0 {
|
|
messageFormat = fmt.Sprintf("%%-%ds", f.SpacePadding)
|
|
}
|
|
|
|
if f.DisableTimestamp {
|
|
_, err = fmt.Fprintf(b, "%s%s "+messageFormat, level, prefix, message)
|
|
} else {
|
|
var timestamp string
|
|
if !f.FullTimestamp {
|
|
timestamp = fmt.Sprintf("[%04d]", miniTS())
|
|
} else {
|
|
timestamp = fmt.Sprintf("[%s]", entry.Time.Format(timestampFormat))
|
|
}
|
|
_, err = fmt.Fprintf(b, "%s %s%s "+messageFormat, colorScheme.TimestampColor(timestamp), level, prefix, message)
|
|
}
|
|
for _, k := range keys {
|
|
if k != "package" {
|
|
v := entry.Data[k]
|
|
|
|
format := "%+v"
|
|
if k == logrus.ErrorKey {
|
|
format = "%v" // To avoid printing stack traces for errors
|
|
}
|
|
|
|
// Sanitize field values to remove new lines and other control characters.
|
|
s := sanitize(fmt.Sprintf(format, v))
|
|
_, err = fmt.Fprintf(b, " %s=%s", levelColor(k), s)
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
func sanitize(s string) string {
|
|
return strings.Map(func(r rune) rune {
|
|
if unicode.IsControl(r) {
|
|
return -1
|
|
}
|
|
return r
|
|
}, s)
|
|
}
|
|
|
|
func (f *TextFormatter) needsQuoting(text string) bool {
|
|
if f.QuoteEmptyFields && len(text) == 0 {
|
|
return true
|
|
}
|
|
for _, ch := range text {
|
|
if !((ch >= 'a' && ch <= 'z') ||
|
|
(ch >= 'A' && ch <= 'Z') ||
|
|
(ch >= '0' && ch <= '9') ||
|
|
ch == '-' || ch == '.') {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// getPrefixAndMessage extracts the prefix and the message from the entry by the following order:
|
|
// 1. If the "prefix" field is set in entry.Data, use that as prefix.
|
|
// 2. If the "package" field is set in entry.Data, use that to determine the prefix.
|
|
// If a replacement is found in prefixReplacements, use that. Otherwise, use the package name.
|
|
// 3. Try to extract the prefix from the message itself by looking for a pattern like "[prefix] message".
|
|
// 4. If none of the above methods yield a prefix, return an empty prefix and the original message.
|
|
func getPrefixAndMessage(entry *logrus.Entry) (string, string) {
|
|
prefix := ""
|
|
msg := entry.Message
|
|
|
|
if prefixOldMethod, ok := entry.Data["prefix"]; ok {
|
|
return prefixOldMethod.(string), msg
|
|
}
|
|
|
|
if packagePath, ok := entry.Data["package"]; ok {
|
|
if prefixReplacement, ok := prefixReplacements[packagePath.(string)]; ok {
|
|
return prefixReplacement, msg
|
|
}
|
|
pathSplit := strings.Split(packagePath.(string), "/")
|
|
prefix = pathSplit[len(pathSplit)-1]
|
|
return prefix, msg
|
|
}
|
|
|
|
if prefixExtracted, trimmedMsg := extractPrefix(msg); len(prefixExtracted) > 0 {
|
|
return prefixExtracted, trimmedMsg
|
|
}
|
|
|
|
return "", msg
|
|
}
|
|
|
|
func extractPrefix(msg string) (string, string) {
|
|
prefix := ""
|
|
|
|
if regex.MatchString(msg) {
|
|
match := regex.FindString(msg)
|
|
prefix, msg = match[1:len(match)-1], strings.TrimSpace(msg[len(match):])
|
|
}
|
|
return prefix, msg
|
|
}
|
|
|
|
func (f *TextFormatter) appendKeyValue(b *bytes.Buffer, key string, value any, appendSpace bool) error {
|
|
b.WriteString(key)
|
|
b.WriteByte('=')
|
|
if err := f.appendValue(b, value); err != nil {
|
|
return err
|
|
}
|
|
|
|
if appendSpace {
|
|
return b.WriteByte(' ')
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (f *TextFormatter) appendValue(b *bytes.Buffer, value any) (err error) {
|
|
switch value := value.(type) {
|
|
case string:
|
|
if !f.needsQuoting(value) {
|
|
_, err = b.WriteString(value)
|
|
} else {
|
|
_, err = fmt.Fprintf(b, "%s%v%s", f.QuoteCharacter, value, f.QuoteCharacter)
|
|
}
|
|
case error:
|
|
errmsg := value.Error()
|
|
if !f.needsQuoting(errmsg) {
|
|
_, err = b.WriteString(errmsg)
|
|
} else {
|
|
_, err = fmt.Fprintf(b, "%s%v%s", f.QuoteCharacter, errmsg, f.QuoteCharacter)
|
|
}
|
|
default:
|
|
_, err = fmt.Fprint(b, value)
|
|
}
|
|
return
|
|
}
|
|
|
|
// This is to not silently overwrite `time`, `msg` and `level` fields when
|
|
// dumping it. If this code wasn't there doing:
|
|
//
|
|
// logrus.WithField("level", 1).Info("hello")
|
|
//
|
|
// would just silently drop the user provided level. Instead with this code
|
|
// it'll be logged as:
|
|
//
|
|
// {"level": "info", "fields.level": 1, "msg": "hello", "time": "..."}
|
|
func prefixFieldClashes(data logrus.Fields) {
|
|
if t, ok := data["time"]; ok {
|
|
data["fields.time"] = t
|
|
}
|
|
|
|
if m, ok := data["msg"]; ok {
|
|
data["fields.msg"] = m
|
|
}
|
|
|
|
if l, ok := data["level"]; ok {
|
|
data["fields.level"] = l
|
|
}
|
|
}
|