Compare commits

...

6 Commits

Author SHA1 Message Date
Preston Van Loon
22ed41484a The implementation is working as expected, but tests are failing 2025-10-01 12:07:54 -05:00
Preston Van Loon
0750be615f Mostly working 2025-10-01 11:51:12 -05:00
Preston Van Loon
503d96396d Working implementation for the most part 2025-10-01 11:26:05 -05:00
Preston Van Loon
0af23cc47b Add TODOs for the prysmctl log command. 2025-10-01 11:22:07 -05:00
Preston Van Loon
aa68477881 TranslateFluentdtoUnstructuredLog implemented, tests pass. The timestamp needs work though... 2025-10-01 10:51:23 -05:00
Preston Van Loon
5e43d940cc Test case for translating fluentd logs to text logs 2025-10-01 10:46:07 -05:00
6 changed files with 311 additions and 0 deletions

View File

@@ -10,6 +10,7 @@ go_library(
deps = [
"//cmd/prysmctl/checkpointsync:go_default_library",
"//cmd/prysmctl/db:go_default_library",
"//cmd/prysmctl/logging:go_default_library",
"//cmd/prysmctl/p2p:go_default_library",
"//cmd/prysmctl/testnet:go_default_library",
"//cmd/prysmctl/validator:go_default_library",

View File

@@ -0,0 +1,29 @@
load("@prysm//tools/go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = [
"commands.go",
"json_to_text.go",
],
importpath = "github.com/OffchainLabs/prysm/v6/cmd/prysmctl/logging",
visibility = ["//visibility:public"],
deps = [
"//runtime/logging/logrus-prefixed-formatter:go_default_library",
"@com_github_sirupsen_logrus//:go_default_library",
"@com_github_urfave_cli_v2//:go_default_library",
],
)
go_test(
name = "go_default_test",
srcs = ["json_to_text_test.go"],
deps = [
":go_default_library",
"//runtime/logging/logrus-prefixed-formatter:go_default_library",
"//testing/assert:go_default_library",
"//testing/require:go_default_library",
"@com_github_joonix_log//:go_default_library",
"@com_github_sirupsen_logrus//:go_default_library",
],
)

View File

@@ -0,0 +1,76 @@
package logging
import (
"bufio"
"fmt"
"io"
"os"
"github.com/urfave/cli/v2"
)
var Commands = []*cli.Command{
{
Name: "logs",
Aliases: []string{"l", "logging"},
Usage: "Translate logs from fluentd or json to unstructured text logs",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "from",
Usage: "Input log format (fluentd, text, json)",
Value: "fluentd",
},
&cli.StringFlag{
Name: "to",
Usage: "Output log format (fluentd, text, json)",
Value: "text",
},
},
Action: func(ctx *cli.Context) error {
from := ctx.String("from")
to := ctx.String("to")
// Validate flags
validFormats := map[string]bool{"fluentd": true, "text": true, "json": true}
if !validFormats[from] {
return fmt.Errorf("invalid --from format: %s. Must be one of: fluentd, text, json", from)
}
if !validFormats[to] {
return fmt.Errorf("invalid --to format: %s. Must be one of: fluentd, text, json", to)
}
// Only fluentd to text is currently implemented
if from != "fluentd" || to != "text" {
return fmt.Errorf("only fluentd to text translation is currently supported")
}
// Read from stdin line by line
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
line := scanner.Text()
if line == "" {
continue
}
// Translate the log line
translated, err := TranslateFluentdtoUnstructuredLog(line)
if err != nil {
// Write error to stderr and continue processing
fmt.Fprintf(os.Stderr, "Error translating line: %v\n", err)
continue
}
// Write to stdout (without extra newline as TranslateFluentdtoUnstructuredLog adds one)
if _, err := io.WriteString(os.Stdout, translated); err != nil {
return fmt.Errorf("failed to write output: %w", err)
}
}
if err := scanner.Err(); err != nil {
return fmt.Errorf("error reading input: %w", err)
}
return nil
},
},
}

View File

@@ -0,0 +1,108 @@
package logging
import (
"encoding/json"
"strings"
"time"
prefixed "github.com/OffchainLabs/prysm/v6/runtime/logging/logrus-prefixed-formatter"
"github.com/sirupsen/logrus"
)
// TranslateFluentdtoUnstructuredLog accepts a JSON object as a string and converts it to Prysm's
// default unstructured text logger.
func TranslateFluentdtoUnstructuredLog(s string) (string, error) {
// Parse the JSON input
var data map[string]interface{}
if err := json.Unmarshal([]byte(s), &data); err != nil {
return "", err
}
// Create a logrus entry
entry := &logrus.Entry{
Data: make(logrus.Fields),
}
// Extract timestamp if present, otherwise use zero time
// This matches the test expectations and is fine since we'll only
// use this for translating existing logs that don't have timestamps
if ts, ok := data["timestamp"].(string); ok {
// Try to parse the timestamp
if parsedTime, err := time.Parse(time.RFC3339, ts); err == nil {
entry.Time = parsedTime
} else {
entry.Time = time.Time{} // Zero time if parse fails
}
delete(data, "timestamp")
} else if ts, ok := data["time"].(string); ok {
// Alternative field name
if parsedTime, err := time.Parse(time.RFC3339, ts); err == nil {
entry.Time = parsedTime
} else {
entry.Time = time.Time{} // Zero time if parse fails
}
delete(data, "time")
} else {
// No timestamp in JSON, use zero time (will show as 0001-01-01)
entry.Time = time.Time{}
}
// Extract message and severity
if msg, ok := data["message"].(string); ok {
entry.Message = msg
delete(data, "message")
}
if severity, ok := data["severity"].(string); ok {
// Convert severity to logrus level
level, err := logrus.ParseLevel(strings.ToLower(severity))
if err != nil {
// Default to info if we can't parse the level
entry.Level = logrus.InfoLevel
} else {
entry.Level = level
}
delete(data, "severity")
} else {
entry.Level = logrus.InfoLevel
}
// All remaining fields go into Data
// Convert float64 to int64 if they're whole numbers to avoid scientific notation
for k, v := range data {
switch val := v.(type) {
case float64:
// Check if it's a whole number
if val == float64(int64(val)) {
entry.Data[k] = int64(val)
} else {
entry.Data[k] = val
}
case float32:
// Check if it's a whole number
if val == float32(int64(val)) {
entry.Data[k] = int64(val)
} else {
entry.Data[k] = val
}
default:
entry.Data[k] = v
}
}
// Use the prefixed formatter to format the entry.
formatter := &prefixed.TextFormatter{
FullTimestamp: true,
TimestampFormat: "2006-01-02 15:04:05.00", // Match beacon-chain format
DisableColors: false,
ForceColors: true, // Force colors even when not a TTY
ForceFormatting: true, // Force formatted output even when not a TTY
}
formatted, err := formatter.Format(entry)
if err != nil {
return "", err
}
return string(formatted), nil
}

View File

@@ -0,0 +1,95 @@
package logging_test
import (
"fmt"
"testing"
"github.com/OffchainLabs/prysm/v6/cmd/prysmctl/logging"
prefixed "github.com/OffchainLabs/prysm/v6/runtime/logging/logrus-prefixed-formatter"
"github.com/OffchainLabs/prysm/v6/testing/assert"
"github.com/OffchainLabs/prysm/v6/testing/require"
joonix "github.com/joonix/log"
"github.com/sirupsen/logrus"
)
type testCase struct {
input string
output string
}
func TestTranslateFluentdtoUnstructuredLog(t *testing.T) {
tests := []testCase{
createTestCaseFluentdToText(t, &logrus.Entry{
Data: logrus.Fields{
"prefix": "p2p",
"error": "something really bad happened",
"slot": 529,
},
Level: logrus.DebugLevel,
Message: "Failed to process something not very important",
}),
createTestCaseFluentdToText(t, &logrus.Entry{
Data: logrus.Fields{
"prefix": "core",
"error": "something really really bad happened",
"slot": 530,
},
Level: logrus.ErrorLevel,
Message: "Failed to process something very important",
}),
createTestCaseFluentdToText(t, &logrus.Entry{
Data: logrus.Fields{
"prefix": "core",
"slot": 100_000_000,
"hash": "0xabcdef",
},
Level: logrus.InfoLevel,
Message: "Processed something successfully",
}),
}
for i, tt := range tests {
t.Run(fmt.Sprintf("scenario_%d", i), func(t *testing.T) {
t.Logf("Input was %v", tt.input)
got, err := logging.TranslateFluentdtoUnstructuredLog(tt.input)
assert.NoError(t, err)
require.Equal(t, tt.output, got, "Did not get expected output")
})
}
}
func createTestCaseFluentdToText(t *testing.T, e *logrus.Entry) testCase {
return testCase{
input: logToString(t, fluentdFormat(t), e),
output: logToString(t, textFormat(), e),
}
}
type formatter interface {
Format(entry *logrus.Entry) ([]byte, error)
}
func logToString(t *testing.T, f formatter, e *logrus.Entry) string {
b, err := f.Format(e)
require.NoError(t, err)
return string(b)
}
func fluentdFormat(t *testing.T) formatter {
f := joonix.NewFormatter()
require.NoError(t, joonix.DisableTimestampFormat(f))
return f
}
func textFormat() formatter {
formatter := new(prefixed.TextFormatter)
formatter.FullTimestamp = true
formatter.TimestampFormat = "2006-01-02 15:04:05.00"
formatter.DisableColors = false
formatter.ForceColors = true // Force colors to match the implementation
formatter.ForceFormatting = true // Force formatted output
return formatter
}

View File

@@ -5,6 +5,7 @@ import (
"github.com/OffchainLabs/prysm/v6/cmd/prysmctl/checkpointsync"
"github.com/OffchainLabs/prysm/v6/cmd/prysmctl/db"
"github.com/OffchainLabs/prysm/v6/cmd/prysmctl/logging"
"github.com/OffchainLabs/prysm/v6/cmd/prysmctl/p2p"
"github.com/OffchainLabs/prysm/v6/cmd/prysmctl/testnet"
"github.com/OffchainLabs/prysm/v6/cmd/prysmctl/validator"
@@ -32,4 +33,5 @@ func init() {
prysmctlCommands = append(prysmctlCommands, testnet.Commands...)
prysmctlCommands = append(prysmctlCommands, weaksubjectivity.Commands...)
prysmctlCommands = append(prysmctlCommands, validator.Commands...)
prysmctlCommands = append(prysmctlCommands, logging.Commands...)
}