Prevent Usage of Stdlib File/Dir Writing With Static Analysis (#7685)

* write file and mkdirall analyzers

* include analyzer in build bazel

* comments to the single entrypoint and fix validator references

* enforce 600 for files, 700 for dirs

* pass validator tests

* add to nogo

* remove references

* beaconfuzz

* docker img

* fix up kv issue

* mkdir if not exists

* radek comments

* final comments

* Try to fix file problem

Co-authored-by: Ivan Martinez <ivanthegreatdev@gmail.com>

tools/analyzers/properpermissions/BUILD.bazel

@@ -0,0 +1,28 @@
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_tool_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["analyzer.go"],
+    importpath = "github.com/prysmaticlabs/prysm/tools/analyzers/properpermissions",
+    visibility = ["//visibility:public"],
+    deps = [
+        "@org_golang_x_tools//go/analysis:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/inspect:go_default_library",
+        "@org_golang_x_tools//go/ast/inspector:go_default_library",
+    ],
+)
+
+go_tool_library(
+    name = "go_tool_library",
+    srcs = ["analyzer.go"],
+    importpath = "github.com/prysmaticlabs/prysm/tools/analyzers/properpermissions",
+    visibility = ["//visibility:public"],
+    deps = [
+        "@org_golang_x_tools//go/analysis:go_tool_library",
+        "@org_golang_x_tools//go/analysis/passes/inspect:go_tool_library",
+        "@org_golang_x_tools//go/ast/inspector:go_tool_library",
+    ],
+)
+
+# gazelle:exclude analyzer_test.go

tools/analyzers/properpermissions/analyzer.go

@@ -0,0 +1,104 @@
+// Package properpermissions implements a static analyzer to ensure that Prysm does not
+// use ioutil.MkdirAll or os.WriteFile as they are unsafe when it comes to guaranteeing
+// file permissions and not overriding existing permissions. Instead, users are warned
+// to utilize shared/fileutil as the canonical solution.
+package properpermissions
+
+import (
+	"errors"
+	"fmt"
+	"go/ast"
+
+	"golang.org/x/tools/go/analysis"
+	"golang.org/x/tools/go/analysis/passes/inspect"
+	"golang.org/x/tools/go/ast/inspector"
+)
+
+// Doc explaining the tool.
+const Doc = "Tool to enforce usage of Prysm's internal file-writing utils instead of os.MkdirAll or ioutil.WriteFile"
+
+var (
+	errUnsafePackage = errors.New(
+		"os and ioutil dir and file writing functions are not permissions-safe, use shared/fileutil",
+	)
+	disallowedFns = []string{"MkdirAll", "WriteFile"}
+)
+
+// Analyzer runs static analysis.
+var Analyzer = &analysis.Analyzer{
+	Name:     "properpermissions",
+	Doc:      Doc,
+	Requires: []*analysis.Analyzer{inspect.Analyzer},
+	Run:      run,
+}
+
+func run(pass *analysis.Pass) (interface{}, error) {
+	inspect, ok := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector)
+	if !ok {
+		return nil, errors.New("analyzer is not type *inspector.Inspector")
+	}
+
+	nodeFilter := []ast.Node{
+		(*ast.File)(nil),
+		(*ast.ImportSpec)(nil),
+		(*ast.CallExpr)(nil),
+	}
+
+	aliases := make(map[string]string)
+
+	inspect.Preorder(nodeFilter, func(node ast.Node) {
+		switch stmt := node.(type) {
+		case *ast.File:
+			// Reset aliases (per file).
+			aliases = make(map[string]string)
+		case *ast.ImportSpec:
+			// Collect aliases.
+			pkg := stmt.Path.Value
+			if pkg == "\"os\"" {
+				if stmt.Name != nil {
+					aliases[stmt.Name.Name] = pkg
+				} else {
+					aliases["os"] = pkg
+				}
+			}
+			if pkg == "\"io/ioutil\"" {
+				if stmt.Name != nil {
+					aliases[stmt.Name.Name] = pkg
+				} else {
+					aliases["ioutil"] = pkg
+				}
+			}
+		case *ast.CallExpr:
+			// Check if any of disallowed functions have been used.
+			for alias, pkg := range aliases {
+				for _, fn := range disallowedFns {
+					if isPkgDot(stmt.Fun, alias, fn) {
+						pass.Reportf(
+							node.Pos(),
+							fmt.Sprintf(
+								"%v: %s.%s() (from %s)",
+								errUnsafePackage,
+								alias,
+								fn,
+								pkg,
+							),
+						)
+					}
+				}
+			}
+		}
+	})
+
+	return nil, nil
+}
+
+func isPkgDot(expr ast.Expr, pkg, name string) bool {
+	sel, ok := expr.(*ast.SelectorExpr)
+	res := ok && isIdent(sel.X, pkg) && isIdent(sel.Sel, name)
+	return res
+}
+
+func isIdent(expr ast.Expr, ident string) bool {
+	id, ok := expr.(*ast.Ident)
+	return ok && id.Name == ident
+}

tools/analyzers/properpermissions/analyzer_test.go

@@ -0,0 +1,11 @@
+package properpermissions
+
+import (
+	"testing"
+
+	"golang.org/x/tools/go/analysis/analysistest"
+)
+
+func TestAnalyzer(t *testing.T) {
+	analysistest.Run(t, analysistest.TestData(), Analyzer)
+}

tools/analyzers/properpermissions/testdata/BUILD.bazel

@@ -0,0 +1,11 @@
+load("@prysm//tools/go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "custom_imports.go",
+        "regular_imports.go",
+    ],
+    importpath = "github.com/prysmaticlabs/prysm/tools/analyzers/properpermissions/testdata",
+    visibility = ["//visibility:public"],
+)

tools/analyzers/properpermissions/testdata/custom_imports.go

@@ -0,0 +1,18 @@
+package testdata
+
+import (
+	"crypto/rand"
+	"fmt"
+	ioAlias "io/ioutil"
+	"math/big"
+	osAlias "os"
+	"path/filepath"
+)
+
+func UseAliasedPackages() {
+	randPath, _ := rand.Int(rand.Reader, big.NewInt(1000000))
+	p := filepath.Join(tempDir(), fmt.Sprintf("/%d", randPath))
+	_ = osAlias.MkdirAll(p, osAlias.ModePerm) // want "os and ioutil dir and file writing functions are not permissions-safe, use shared/fileutil"
+	someFile := filepath.Join(p, "some.txt")
+	_ = ioAlias.WriteFile(someFile, []byte("hello"), osAlias.ModePerm) // want "os and ioutil dir and file writing functions are not permissions-safe, use shared/fileutil"
+}

tools/analyzers/properpermissions/testdata/regular_imports.go

@@ -0,0 +1,26 @@
+package testdata
+
+import (
+	"crypto/rand"
+	"fmt"
+	"io/ioutil"
+	"math/big"
+	"os"
+	"path/filepath"
+)
+
+func tempDir() string {
+	d := os.Getenv("TEST_TMPDIR")
+	if d == "" {
+		return os.TempDir()
+	}
+	return d
+}
+
+func UseOsMkdirAllAndWriteFile() {
+	randPath, _ := rand.Int(rand.Reader, big.NewInt(1000000))
+	p := filepath.Join(tempDir(), fmt.Sprintf("/%d", randPath))
+	_ = os.MkdirAll(p, os.ModePerm) // want "os and ioutil dir and file writing functions are not permissions-safe, use shared/fileutil"
+	someFile := filepath.Join(p, "some.txt")
+	_ = ioutil.WriteFile(someFile, []byte("hello"), os.ModePerm) // want "os and ioutil dir and file writing functions are not permissions-safe, use shared/fileutil"
+}

tools/beacon-fuzz/BUILD.bazel

@@ -5,6 +5,9 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
 go_library(
     name = "go_default_library",
     srcs = ["main.go"],
+    deps = [
+        "//shared/fileutil:go_default_library",
+    ],
     importpath = "github.com/prysmaticlabs/prysm/tools/beacon-fuzz",
     visibility = ["//visibility:private"],
 )

tools/beacon-fuzz/main.go

@@ -9,6 +9,8 @@ import (
 	"path/filepath"
 	"strconv"
 	"text/template"
+
+	"github.com/prysmaticlabs/prysm/shared/fileutil"
 )
 
 var (
@@ -54,7 +56,7 @@ func main() {
 	}
 
 	res := execTmpl(tpl, input{Package: "testing", MapStr: sszBytesToMapStr(m)})
-	if err := ioutil.WriteFile(*output, res.Bytes(), 0644); err != nil {
+	if err := fileutil.WriteFile(*output, res.Bytes()); err != nil {
 		panic(err)
 	}
 }

tools/benchmark-files-gen/BUILD.bazel

@@ -13,6 +13,7 @@ go_library(
         "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/benchutil:go_default_library",
+        "//shared/fileutil:go_default_library",
         "//shared/interop:go_default_library",
         "//shared/params:go_default_library",
         "//shared/testutil:go_default_library",

tools/benchmark-files-gen/main.go

@@ -9,6 +9,7 @@ import (
 	"path"
 
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/fileutil"
 
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
@@ -44,7 +45,7 @@ func main() {
 		}
 	}
 
-	if err := os.MkdirAll(*outputDir, os.ModePerm); err != nil {
+	if err := fileutil.MkdirAll(*outputDir); err != nil {
 		log.Fatal(err)
 	}
 
@@ -77,7 +78,7 @@ func generateGenesisBeaconState() error {
 	if err != nil {
 		return err
 	}
-	return ioutil.WriteFile(path.Join(*outputDir, benchutil.GenesisFileName), beaconBytes, 0644)
+	return fileutil.WriteFile(path.Join(*outputDir, benchutil.GenesisFileName), beaconBytes)
 }
 
 func generateMarshalledFullStateAndBlock() error {
@@ -150,7 +151,7 @@ func generateMarshalledFullStateAndBlock() error {
 	if err != nil {
 		return err
 	}
-	if err := ioutil.WriteFile(path.Join(*outputDir, benchutil.BState1EpochFileName), beaconBytes, 0644); err != nil {
+	if err := fileutil.WriteFile(path.Join(*outputDir, benchutil.BState1EpochFileName), beaconBytes); err != nil {
 		return err
 	}
 
@@ -165,7 +166,7 @@ func generateMarshalledFullStateAndBlock() error {
 		return err
 	}
 
-	return ioutil.WriteFile(path.Join(*outputDir, benchutil.FullBlockFileName), blockBytes, 0644)
+	return fileutil.WriteFile(path.Join(*outputDir, benchutil.FullBlockFileName), blockBytes)
 }
 
 func generate2FullEpochState() error {
@@ -200,7 +201,7 @@ func generate2FullEpochState() error {
 		return err
 	}
 
-	return ioutil.WriteFile(path.Join(*outputDir, benchutil.BState2EpochFileName), beaconBytes, 0644)
+	return fileutil.WriteFile(path.Join(*outputDir, benchutil.BState2EpochFileName), beaconBytes)
 }
 
 func genesisBeaconState() (*stateTrie.BeaconState, error) {

tools/enr-calculator/BUILD.bazel

@@ -10,6 +10,7 @@ go_library(
     importpath = "github.com/prysmaticlabs/prysm/tools/enr-calculator",
     visibility = ["//visibility:private"],
     deps = [
+        "//shared/fileutil:go_default_library",
         "//shared/maxprocs:go_default_library",
         "@com_github_ethereum_go_ethereum//p2p/enode:go_default_library",
         "@com_github_ethereum_go_ethereum//p2p/enr:go_default_library",
@@ -41,6 +42,7 @@ go_image(
         "@com_github_ethereum_go_ethereum//p2p/enr:go_default_library",
         "@com_github_libp2p_go_libp2p_core//crypto:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
+        "//shared/fileutil:go_default_library",
         "//shared/maxprocs:go_default_library",
     ],
 )

tools/enr-calculator/main.go

@@ -6,12 +6,12 @@ import (
 	"crypto/ecdsa"
 	"encoding/hex"
 	"flag"
-	"io/ioutil"
 	"net"
 
 	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/enr"
 	"github.com/libp2p/go-libp2p-core/crypto"
+	"github.com/prysmaticlabs/prysm/shared/fileutil"
 	_ "github.com/prysmaticlabs/prysm/shared/maxprocs"
 	log "github.com/sirupsen/logrus"
 )
@@ -68,7 +68,7 @@ func main() {
 	log.Info(localNode.Node().String())
 
 	if *outfile != "" {
-		err := ioutil.WriteFile(*outfile, []byte(localNode.Node().String()), 0644)
+		err := fileutil.WriteFile(*outfile, []byte(localNode.Node().String()))
 		if err != nil {
 			panic(err)
 		}

tools/genesis-state-gen/main.go

@@ -96,7 +96,7 @@ func main() {
 			log.Printf("Could not ssz marshal the genesis beacon state: %v", err)
 			return
 		}
-		if err := ioutil.WriteFile(*sszOutputFile, encodedState, 0644); err != nil {
+		if err := fileutil.WriteFile(*sszOutputFile, encodedState); err != nil {
 			log.Printf("Could not write encoded genesis beacon state to file: %v", err)
 			return
 		}
@@ -108,7 +108,7 @@ func main() {
 			log.Printf("Could not yaml marshal the genesis beacon state: %v", err)
 			return
 		}
-		if err := ioutil.WriteFile(*yamlOutputFile, encodedState, 0644); err != nil {
+		if err := fileutil.WriteFile(*yamlOutputFile, encodedState); err != nil {
 			log.Printf("Could not write encoded genesis beacon state to file: %v", err)
 			return
 		}
@@ -120,7 +120,7 @@ func main() {
 			log.Printf("Could not json marshal the genesis beacon state: %v", err)
 			return
 		}
-		if err := ioutil.WriteFile(*jsonOutputFile, encodedState, 0644); err != nil {
+		if err := fileutil.WriteFile(*jsonOutputFile, encodedState); err != nil {
 			log.Printf("Could not write encoded genesis beacon state to file: %v", err)
 			return
 		}

tools/interop/export-genesis/BUILD.bazel

@@ -9,6 +9,7 @@ go_library(
     deps = [
         "//beacon-chain/cache:go_default_library",
         "//beacon-chain/db:go_default_library",
+        "//shared/fileutil:go_default_library",
     ],
 )
 

tools/interop/export-genesis/main.go

@@ -3,11 +3,11 @@ package main
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"os"
 
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/shared/fileutil"
 )
 
 // A basic tool to extract genesis.ssz from existing beaconchain.db.
@@ -41,7 +41,7 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
-	if err := ioutil.WriteFile(os.Args[2], b, 0644); err != nil {
+	if err := fileutil.WriteFile(os.Args[2], b); err != nil {
 		panic(err)
 	}
 	fmt.Println("done")

tools/keystores/BUILD.bazel

@@ -9,7 +9,6 @@ go_library(
     deps = [
         "//shared/bls:go_default_library",
         "//shared/fileutil:go_default_library",
-        "//shared/params:go_default_library",
         "//shared/promptutil:go_default_library",
         "//validator/keymanager:go_default_library",
         "@com_github_google_uuid//:go_default_library",

tools/keystores/main.go

@@ -19,7 +19,6 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/shared/bls"
 	"github.com/prysmaticlabs/prysm/shared/fileutil"
-	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/promptutil"
 	"github.com/prysmaticlabs/prysm/validator/keymanager"
 	"github.com/urfave/cli/v2"
@@ -209,7 +208,7 @@ func encrypt(cliCtx *cli.Context) error {
 	if err != nil {
 		return errors.Wrap(err, "could not json marshal keystore")
 	}
-	if err := ioutil.WriteFile(fullPath, encodedFile, params.BeaconIoConfig().ReadWritePermissions); err != nil {
+	if err := fileutil.WriteFile(fullPath, encodedFile); err != nil {
 		return errors.Wrapf(err, "could not write file at path: %s", fullPath)
 	}
 	fmt.Printf(

tools/update-genesis-time/BUILD.bazel

@@ -8,6 +8,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//proto/beacon/p2p/v1:go_default_library",
+        "//shared/fileutil:go_default_library",
         "//shared/timeutils:go_default_library",
         "@com_github_prysmaticlabs_go_ssz//:go_default_library",
     ],

tools/update-genesis-time/main.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/prysmaticlabs/go-ssz"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/fileutil"
 	"github.com/prysmaticlabs/prysm/shared/timeutils"
 )
 
@@ -36,7 +37,7 @@ func main() {
 	if err != nil {
 		log.Fatalf("Could not ssz marshal the beacon state: %v", err)
 	}
-	if err := ioutil.WriteFile(*inputSSZState, encodedState, 0644); err != nil {
+	if err := fileutil.WriteFile(*inputSSZState, encodedState); err != nil {
 		log.Fatalf("Could not write encoded beacon state to file: %v", err)
 	}
 	log.Printf("Done writing to %s", *inputSSZState)