mirror of
https://github.com/OffchainLabs/prysm.git
synced 2026-01-09 15:37:56 -05:00
#### This PR sets the foundation for the new logging features. --- The goal of this big PR is the following: 1. Adding a log.go file to every package: [_commit_](54f6396d4c) - Writing a bash script that adds the log.go file to every package that imports logrus, except the excluded packages, configured at the top of the bash script. - the log.go file creates a log variable and sets a field called `package` to the full path of that package. - I have tried to fix every error/problem that came from mass generation of this file. (duplicate declarations, different prefix names, etc...) - some packages had the log.go file from before, and had some helper functions in there as well. I've moved all of them to a `log_helpers.go` file within each package. 2. Create a CI rule which verifies that: [_commit_](b799c3a0ef) - every package which imports logrus, also has a log.go file, except the excluded packages. - the `package` field of each log.go variable, has the correct path. (to detect when we move a package or change it's name) - I pushed a commit with a manually changed log.go file to trigger the ci check failure and it worked. 3. Alter the logging system to read the prefix from this `package` field for every log while outputing: [_commit_](b0c7f1146c) - some packages have/want/need a different log prefix than their package name (like `kv`). This can be solved by keeping a map of package paths to prefix names somewhere. --- **Some notes:** - Please review everything carefully. - I created the `prefixReplacement` map and populated the data that I deemed necessary. Please check it and complain if something doesn't make sense or is missing. I attached at the bottom, the list of all the packages that used to use a different name than their package name as their prefix. - I have chosen to mark some packages to be excluded from this whole process. They will either not log anything, or log without a prefix, or log using their previously defined prefix. See the list of exclusions in the bottom. - I fixed all the tests that failed because of this change. These were failing because they were expecting the old prefix to be in the generated logs. I have changed those to expect the new `package` field instead. This might not be a great solution. Ideally we might want to remove this from the tests so they only test for relevant fields in the logs. but this is a problem for another day. - Please run the node with this config, and mention if you see something weird in the logs. (use different verbosities) - The CI workflow uses a script that basically runs the `hack/gen-logs.sh` and checks that the git diff is zero. that script is `hack/check-logs.sh`. This means that if one runs this script locally, it will not actually _check_ anything, rather than just regenerate the log.go files and fix any mistake. This might be confusing. Please suggest solutions if you think it's a problem. --- **A list of packages that used a different prefix than their package names for their logs:** - beacon-chain/cache/depositsnapshot/ package depositsnapshot, prefix "cache" - beacon-chain/core/transition/log.go — package transition, prefix "state" - beacon-chain/db/kv/log.go — package kv, prefix "db" - beacon-chain/db/slasherkv/log.go — package slasherkv, prefix "slasherdb" - beacon-chain/db/pruner/pruner.go — package pruner, prefix "db-pruner" - beacon-chain/light-client/log.go — package light_client, prefix "light-client" - beacon-chain/operations/attestations/log.go — package attestations, prefix "pool/attestations" - beacon-chain/operations/slashings/log.go — package slashings, prefix "pool/slashings" - beacon-chain/rpc/core/log.go — package core, prefix "rpc/core" - beacon-chain/rpc/eth/beacon/log.go — package beacon, prefix "rpc/beaconv1" - beacon-chain/rpc/eth/validator/log.go — package validator, prefix "beacon-api" - beacon-chain/rpc/prysm/v1alpha1/beacon/log.go — package beacon, prefix "rpc" - beacon-chain/rpc/prysm/v1alpha1/validator/log.go — package validator, prefix "rpc/validator" - beacon-chain/state/stategen/log.go — package stategen, prefix "state-gen" - beacon-chain/sync/checkpoint/log.go — package checkpoint, prefix "checkpoint-sync" - beacon-chain/sync/initial-sync/log.go — package initialsync, prefix "initial-sync" - cmd/prysmctl/p2p/log.go — package p2p, prefix "prysmctl-p2p" - config/features/log.go -- package features, prefix "flags" - io/file/log.go — package file, prefix "fileutil" - proto/prysm/v1alpha1/log.go — package eth, prefix "protobuf" - validator/client/beacon-api/log.go — package beacon_api, prefix "beacon-api" - validator/db/kv/log.go — package kv, prefix "db" - validator/db/filesystem/db.go — package filesystem, prefix "db" - validator/keymanager/derived/log.go — package derived, prefix "derived-keymanager" - validator/keymanager/local/log.go — package local, prefix "local-keymanager" - validator/keymanager/remote-web3signer/log.go — package remote_web3signer, prefix "remote-keymanager" - validator/keymanager/remote-web3signer/internal/log.go — package internal, prefix "remote-web3signer- internal" - beacon-chain/forkchoice/doubly... prefix is "forkchoice-doublylinkedtree" **List of excluded directories (their subdirectories are also excluded):** ``` EXCLUDED_PATH_PREFIXES=( "testing" "validator/client/testutil" "beacon-chain/p2p/testing" "beacon-chain/rpc/eth/config" "beacon-chain/rpc/prysm/v1alpha1/debug" "tools" "runtime" "monitoring" "io" "cmd" ".well-known" "changelog" "hack" "specrefs" "third_party" "bazel-out" "bazel-bin" "bazel-prysm" "bazel-testlogs" "build" ".github" ".jj" ".idea" ".vscode" ) ```
309 lines
8.4 KiB
Go
309 lines
8.4 KiB
Go
package middleware
|
|
|
|
import (
|
|
"bytes"
|
|
"compress/gzip"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
|
|
"github.com/OffchainLabs/prysm/v7/api"
|
|
"github.com/OffchainLabs/prysm/v7/testing/require"
|
|
)
|
|
|
|
// frozenHeaderRecorder allows asserting that response headers were not modified
|
|
// after the call to WriteHeader.
|
|
//
|
|
// Its purpose is to have a regression test for https://github.com/OffchainLabs/prysm/pull/15499.
|
|
type frozenHeaderRecorder struct {
|
|
*httptest.ResponseRecorder
|
|
frozenHeader http.Header
|
|
}
|
|
|
|
func (r *frozenHeaderRecorder) WriteHeader(code int) {
|
|
if r.frozenHeader != nil {
|
|
return
|
|
}
|
|
r.ResponseRecorder.WriteHeader(code)
|
|
r.frozenHeader = r.ResponseRecorder.Header().Clone()
|
|
}
|
|
|
|
func TestNormalizeQueryValuesHandler(t *testing.T) {
|
|
nextHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
_, err := w.Write([]byte("next handler"))
|
|
require.NoError(t, err)
|
|
})
|
|
|
|
handler := NormalizeQueryValuesHandler(nextHandler)
|
|
|
|
tests := []struct {
|
|
name string
|
|
inputQuery string
|
|
expectedQuery string
|
|
}{
|
|
{
|
|
name: "3 values",
|
|
inputQuery: "key=value1,value2,value3",
|
|
expectedQuery: "key=value1&key=value2&key=value3", // replace with expected normalized value
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
req, err := http.NewRequest("GET", "/test?"+test.inputQuery, http.NoBody)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
rr := httptest.NewRecorder()
|
|
handler.ServeHTTP(rr, req)
|
|
|
|
if rr.Code != http.StatusOK {
|
|
t.Errorf("handler returned wrong status code: got %v want %v", rr.Code, http.StatusOK)
|
|
}
|
|
|
|
if req.URL.RawQuery != test.expectedQuery {
|
|
t.Errorf("query not normalized: got %v want %v", req.URL.RawQuery, test.expectedQuery)
|
|
}
|
|
|
|
if rr.Body.String() != "next handler" {
|
|
t.Errorf("next handler was not executed")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestContentTypeHandler(t *testing.T) {
|
|
acceptedMediaTypes := []string{api.JsonMediaType, api.OctetStreamMediaType}
|
|
|
|
nextHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
_, err := w.Write([]byte("next handler"))
|
|
require.NoError(t, err)
|
|
})
|
|
|
|
handler := ContentTypeHandler(acceptedMediaTypes)(nextHandler)
|
|
|
|
tests := []struct {
|
|
name string
|
|
contentType string
|
|
expectedStatusCode int
|
|
isGet bool
|
|
}{
|
|
{
|
|
name: "Accepted Content-Type - application/json",
|
|
contentType: api.JsonMediaType,
|
|
expectedStatusCode: http.StatusOK,
|
|
},
|
|
{
|
|
name: "Accepted Content-Type - ssz format",
|
|
contentType: api.OctetStreamMediaType,
|
|
expectedStatusCode: http.StatusOK,
|
|
},
|
|
{
|
|
name: "Unsupported Content-Type - text/plain",
|
|
contentType: "text/plain",
|
|
expectedStatusCode: http.StatusUnsupportedMediaType,
|
|
},
|
|
{
|
|
name: "Missing Content-Type",
|
|
contentType: "",
|
|
expectedStatusCode: http.StatusUnsupportedMediaType,
|
|
},
|
|
{
|
|
name: "GET request skips content type check",
|
|
contentType: "",
|
|
expectedStatusCode: http.StatusOK,
|
|
isGet: true,
|
|
},
|
|
{
|
|
name: "Content type contains charset is ok",
|
|
contentType: "application/json; charset=utf-8",
|
|
expectedStatusCode: http.StatusOK,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
httpMethod := http.MethodPost
|
|
if tt.isGet {
|
|
httpMethod = http.MethodGet
|
|
}
|
|
req := httptest.NewRequest(httpMethod, "/", nil)
|
|
if tt.contentType != "" {
|
|
req.Header.Set("Content-Type", tt.contentType)
|
|
}
|
|
rr := httptest.NewRecorder()
|
|
|
|
handler.ServeHTTP(rr, req)
|
|
|
|
if status := rr.Code; status != tt.expectedStatusCode {
|
|
t.Errorf("handler returned wrong status code: got %v want %v", status, tt.expectedStatusCode)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestAcceptEncodingHeaderHandler(t *testing.T) {
|
|
dummyContent := "Test gzip middleware content"
|
|
nextHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", r.Header.Get("Accept"))
|
|
w.WriteHeader(http.StatusOK)
|
|
_, err := w.Write([]byte(dummyContent))
|
|
require.NoError(t, err)
|
|
})
|
|
|
|
handler := AcceptEncodingHeaderHandler()(nextHandler)
|
|
|
|
tests := []struct {
|
|
name string
|
|
accept string
|
|
acceptEncoding string
|
|
expectCompressed bool
|
|
}{
|
|
{
|
|
name: "Accept gzip",
|
|
accept: api.JsonMediaType,
|
|
acceptEncoding: "gzip",
|
|
expectCompressed: true,
|
|
},
|
|
{
|
|
name: "Accept multiple encodings",
|
|
accept: api.JsonMediaType,
|
|
acceptEncoding: "deflate, gzip",
|
|
expectCompressed: true,
|
|
},
|
|
{
|
|
name: "Accept unsupported encoding",
|
|
accept: api.JsonMediaType,
|
|
acceptEncoding: "deflate",
|
|
expectCompressed: false,
|
|
},
|
|
{
|
|
name: "No accept encoding header",
|
|
accept: api.JsonMediaType,
|
|
acceptEncoding: "",
|
|
expectCompressed: false,
|
|
},
|
|
{
|
|
name: "SSZ",
|
|
accept: api.OctetStreamMediaType,
|
|
acceptEncoding: "gzip",
|
|
expectCompressed: false,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
req := httptest.NewRequest("GET", "/", nil)
|
|
req.Header.Set("Accept", tt.accept)
|
|
if tt.acceptEncoding != "" {
|
|
req.Header.Set("Accept-Encoding", tt.acceptEncoding)
|
|
}
|
|
rr := &frozenHeaderRecorder{ResponseRecorder: httptest.NewRecorder()}
|
|
|
|
handler.ServeHTTP(rr, req)
|
|
|
|
if tt.expectCompressed {
|
|
require.Equal(t, "gzip", rr.frozenHeader.Get("Content-Encoding"), "Expected Content-Encoding header to be 'gzip'")
|
|
|
|
compressedBody := rr.Body.Bytes()
|
|
require.NotEqual(t, dummyContent, string(compressedBody), "Response body should be compressed and differ from the original")
|
|
|
|
gzReader, err := gzip.NewReader(bytes.NewReader(compressedBody))
|
|
require.NoError(t, err, "Failed to create gzipReader")
|
|
defer func() {
|
|
if err := gzReader.Close(); err != nil {
|
|
log.WithError(err).Error("Failed to close gzip reader")
|
|
}
|
|
}()
|
|
|
|
decompressedBody, err := io.ReadAll(gzReader)
|
|
require.NoError(t, err, "Failed to decompress response body")
|
|
require.Equal(t, dummyContent, string(decompressedBody), "Decompressed content should match the original")
|
|
} else {
|
|
require.Equal(t, dummyContent, rr.Body.String(), "Response body should be uncompressed and match the original")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestAcceptHeaderHandler(t *testing.T) {
|
|
acceptedTypes := []string{"application/json", "application/octet-stream"}
|
|
|
|
nextHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
_, err := w.Write([]byte("next handler"))
|
|
require.NoError(t, err)
|
|
})
|
|
|
|
handler := AcceptHeaderHandler(acceptedTypes)(nextHandler)
|
|
|
|
tests := []struct {
|
|
name string
|
|
acceptHeader string
|
|
expectedStatusCode int
|
|
}{
|
|
{
|
|
name: "Accepted Accept-Type - application/json",
|
|
acceptHeader: "application/json",
|
|
expectedStatusCode: http.StatusOK,
|
|
},
|
|
{
|
|
name: "Accepted Accept-Type - application/octet-stream",
|
|
acceptHeader: "application/octet-stream",
|
|
expectedStatusCode: http.StatusOK,
|
|
},
|
|
{
|
|
name: "Accepted Accept-Type with parameters",
|
|
acceptHeader: "application/json;q=0.9, application/octet-stream;q=0.8",
|
|
expectedStatusCode: http.StatusOK,
|
|
},
|
|
{
|
|
name: "Unsupported Accept-Type - text/plain",
|
|
acceptHeader: "text/plain",
|
|
expectedStatusCode: http.StatusNotAcceptable,
|
|
},
|
|
{
|
|
name: "Missing Accept header",
|
|
acceptHeader: "",
|
|
expectedStatusCode: http.StatusOK,
|
|
},
|
|
{
|
|
name: "*/* is accepted",
|
|
acceptHeader: "*/*",
|
|
expectedStatusCode: http.StatusOK,
|
|
},
|
|
{
|
|
name: "application/* is accepted",
|
|
acceptHeader: "application/*",
|
|
expectedStatusCode: http.StatusOK,
|
|
},
|
|
{
|
|
name: "/* is unsupported",
|
|
acceptHeader: "/*",
|
|
expectedStatusCode: http.StatusNotAcceptable,
|
|
},
|
|
{
|
|
name: "application/ is unsupported",
|
|
acceptHeader: "application/",
|
|
expectedStatusCode: http.StatusNotAcceptable,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
req := httptest.NewRequest("GET", "/", nil)
|
|
if tt.acceptHeader != "" {
|
|
req.Header.Set("Accept", tt.acceptHeader)
|
|
}
|
|
rr := httptest.NewRecorder()
|
|
|
|
handler.ServeHTTP(rr, req)
|
|
|
|
if status := rr.Code; status != tt.expectedStatusCode {
|
|
t.Errorf("handler returned wrong status code: got %v want %v", status, tt.expectedStatusCode)
|
|
}
|
|
})
|
|
}
|
|
}
|