add gopkgdriver implementation (genception)

deps.bzl

@@ -582,6 +582,12 @@ def prysm_deps():
         sum = "h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ=",
         version = "v1.5.5",
     )
+    go_repository(
+        name = "com_github_dave_jennifer",
+        importpath = "github.com/dave/jennifer",
+        sum = "h1:MQ/6emI2xM7wt0tJzJzyUik2Q3Tcn2eE0vtYgh4GPVI=",
+        version = "v1.6.0",
+    )
     go_repository(
         name = "com_github_davecgh_go_spew",
         importpath = "github.com/davecgh/go-spew",

tools/genception/BUILD.bazel

@@ -0,0 +1,5 @@
+alias(
+    name = "methodicalgen",
+    actual = "@com_github_offchainlabs_methodical_ssz//cmd/ssz:ssz",
+    visibility = ["//visibility:public"],
+)

tools/genception/cmd/BUILD.bazel

@@ -0,0 +1,16 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary")
+load("@prysm//tools/go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    importpath = "github.com/prysmaticlabs/prysm/v5/tools/genception/cmd",
+    visibility = ["//visibility:private"],
+    deps = ["//tools/genception/driver:go_default_library"],
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)

tools/genception/cmd/main.go

@@ -0,0 +1,104 @@
+// Copyright 2021 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"strings"
+
+	"github.com/prysmaticlabs/prysm/v5/tools/genception/driver"
+)
+
+var log = driver.Logger
+
+func run(_ context.Context, in io.Reader, out io.Writer, args []string) error {
+	rec, err := driver.NewRecorder()
+	if err != nil {
+		return fmt.Errorf("unable to initialize recorder: %w", err)
+	}
+	resolver, err := driver.NewPathResolver()
+	if err != nil {
+		return fmt.Errorf("unable to initialize path resolver: %w", err)
+	}
+	jsonFiles, err := driver.LoadJsonListing()
+	if err != nil {
+		return fmt.Errorf("unable to lookup package: %w", err)
+	}
+	pd, err := driver.NewJSONPackagesDriver(jsonFiles, resolver.Resolve)
+	if err != nil {
+		return fmt.Errorf("unable to load JSON files: %w", err)
+	}
+
+	request, err := driver.ReadDriverRequest(in)
+	if err != nil {
+		return fmt.Errorf("unable to read request: %w", err)
+	}
+	if err := rec.RecordRequest(args, request); err != nil {
+		return fmt.Errorf("unable to record request: %w", err)
+	}
+	// Note: we are returning all files required to build a specific package.
+	// For file queries (`file=`), this means that the CompiledGoFiles will
+	// include more than the only file being specified.
+	resp := pd.Handle(request, args)
+	if err := rec.RecordResponse(resp); err != nil {
+		return fmt.Errorf("unable to record response: %w", err)
+	}
+	data, err := json.Marshal(resp)
+	if err != nil {
+		return fmt.Errorf("unable to marshal response: %v", err)
+	}
+	_, err = out.Write(data)
+	recErr := recordResponse(data)
+	if recErr != nil {
+		log.WithError(recErr).Error("unable to record response")
+	}
+	return err
+}
+
+func recordResponse(data []byte) error {
+	path := os.Getenv("GOPACKAGESDRIVER_RESPONSE_PATH")
+	if path == "" {
+		log.Info("GOPACKAGESDRIVER_RESPONSE_PATH not set; set this to a writable directory path if you want to record record the driver response for debugging")
+		return nil
+	}
+	f, err := os.CreateTemp(path, "*_response.json")
+	if err != nil {
+		return err
+	}
+	_, err = f.Write(data)
+	return err
+}
+
+func main() {
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
+	defer stop()
+
+	log.WithField("args", strings.Join(os.Args[1:], " ")).Info("genception lookup")
+	if err := run(ctx, os.Stdin, os.Stdout, os.Args[1:]); err != nil {
+		_, err := fmt.Fprintf(os.Stderr, "error: %v", err)
+		if err != nil {
+			log.WithError(err).Error("unhandled error in package resolution")
+		}
+		// gopls will check the packages driver exit code, and if there is an
+		// error, it will fall back to go list. Obviously we don't want that,
+		// so force a 0 exit code.
+		os.Exit(0)
+	}
+}

tools/genception/driver/BUILD.bazel

@@ -0,0 +1,33 @@
+load("@prysm//tools/go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "bazel_json_builder.go",
+        "build_context.go",
+        "driver_request.go",
+        "flatpackage.go",
+        "index.go",
+        "json_packages_driver.go",
+        "logger.go",
+        "packageregistry.go",
+        "recorder.go",
+    ],
+    importpath = "github.com/prysmaticlabs/prysm/v5/tools/genception/driver",
+    visibility = ["//visibility:public"],
+    deps = [
+        "@com_github_pkg_errors//:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "index_test.go",
+        "packageregistry_test.go",
+    ],
+    data = glob(["testdata/**"]),
+    embed = [":go_default_library"],
+    deps = ["//testing/require:go_default_library"],
+)

tools/genception/driver/bazel_json_builder.go

@@ -0,0 +1,156 @@
+// Copyright 2021 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package driver
+
+import (
+	"os"
+	"strings"
+)
+
+var RulesGoStdlibLabel = "@io_bazel_rules_go//:stdlib"
+
+/*
+type BazelJSONBuilder struct {
+	packagesBaseDir string
+	includeTests    bool
+}
+
+
+var _defaultKinds = []string{"go_library", "go_test", "go_binary"}
+
+var externalRe = regexp.MustCompile(`.*\/external\/([^\/]+)(\/(.*))?\/([^\/]+.go)`)
+
+func (b *BazelJSONBuilder) fileQuery(filename string) string {
+	label := filename
+
+	if strings.HasPrefix(filename, "./") {
+		label = strings.TrimPrefix(filename, "./")
+	}
+
+	if matches := externalRe.FindStringSubmatch(filename); len(matches) == 5 {
+		// if filepath is for a third party lib, we need to know, what external
+		// library this file is part of.
+		matches = append(matches[:2], matches[3:]...)
+		label = fmt.Sprintf("@%s//%s", matches[1], strings.Join(matches[2:], ":"))
+	}
+
+	relToBin, err := filepath.Rel(b.bazel.info["output_path"], filename)
+	if err == nil && !strings.HasPrefix(relToBin, "../") {
+		parts := strings.SplitN(relToBin, string(filepath.Separator), 3)
+		relToBin = parts[2]
+		// We've effectively converted filename from bazel-bin/some/path.go to some/path.go;
+		// Check if a BUILD.bazel files exists under this dir, if not walk up and repeat.
+		relToBin = filepath.Dir(relToBin)
+		_, err = os.Stat(filepath.Join(b.bazel.WorkspaceRoot(), relToBin, "BUILD.bazel"))
+		for errors.Is(err, os.ErrNotExist) && relToBin != "." {
+			relToBin = filepath.Dir(relToBin)
+			_, err = os.Stat(filepath.Join(b.bazel.WorkspaceRoot(), relToBin, "BUILD.bazel"))
+		}
+
+		if err == nil {
+			// return package path found and build all targets (codegen doesn't fall under go_library)
+			// Otherwise fallback to default
+			if relToBin == "." {
+				relToBin = ""
+			}
+			label = fmt.Sprintf("//%s:all", relToBin)
+		}
+	}
+
+	return label
+}
+
+func isLocalImport(path string) bool {
+	return path == "." || path == ".." ||
+		strings.HasPrefix(path, "./") || strings.HasPrefix(path, "../") ||
+		filepath.IsAbs(path)
+}
+
+func NewBazelJSONBuilder(includeTests bool) (*BazelJSONBuilder, error) {
+	return &BazelJSONBuilder{
+		includeTests:    includeTests,
+	}, nil
+}
+
+func (b *BazelJSONBuilder) Labels(ctx context.Context, requests []string) ([]string, error) {
+	ret := make([]string, 0, len(requests))
+	for _, request := range requests {
+		result := ""
+		if strings.HasSuffix(request, ".go") {
+			f := strings.TrimPrefix(request, "file=")
+			result = b.fileQuery(f)
+		} else if request == "builtin" || request == "std" {
+			result = fmt.Sprintf(RulesGoStdlibLabel)
+		}
+
+		if result != "" {
+			ret = append(ret, result)
+		}
+	}
+	if len(ret) == 0 {
+		return []string{RulesGoStdlibLabel}, nil
+	}
+	return ret, nil
+}
+
+func (b *BazelJSONBuilder) PathResolver() PathResolverFunc {
+	return func(p string) string {
+		p = strings.Replace(p, "__BAZEL_EXECROOT__", os.Getenv("PWD"), 1)
+		p = strings.Replace(p, "__BAZEL_OUTPUT_BASE__", b.packagesBaseDir, 1)
+		return p
+	}
+}
+*/
+
+func NewPathResolver() (*PathResolver, error) {
+	outBase, err := PackagesBaseFromEnv()
+	if err != nil {
+		return nil, err
+	}
+	return &PathResolver{
+		execRoot:   os.Getenv("PWD"),
+		outputBase: outBase,
+	}, nil
+}
+
+type PathResolver struct {
+	outputBase string
+	execRoot   string
+}
+
+const (
+	prefixExecRoot   = "__BAZEL_EXECROOT__"
+	prefixOutputBase = "__BAZEL_OUTPUT_BASE__"
+	prefixWorkspace  = "__BAZEL_WORKSPACE__"
+)
+
+var prefixes = []string{prefixExecRoot, prefixOutputBase, prefixWorkspace}
+
+func (r PathResolver) Resolve(path string) string {
+	for _, prefix := range prefixes {
+		if strings.HasPrefix(path, prefix) {
+			for _, rpl := range []string{r.execRoot, r.outputBase} {
+				rp := strings.Replace(path, prefix, rpl, 1)
+				_, err := os.Stat(rp)
+				if err == nil {
+					return rp
+				}
+			}
+			return path
+		}
+	}
+	log.WithField("path", path).Warn("unrecognized path prefix when resolving source paths in json import metadata")
+	return path
+}

tools/genception/driver/build_context.go

@@ -0,0 +1,44 @@
+package driver
+
+import (
+	"go/build"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+var buildContext = makeBuildContext()
+
+func makeBuildContext() *build.Context {
+	bctx := build.Default
+	bctx.BuildTags = strings.Split(getenvDefault("GOTAGS", ""), ",")
+
+	return &bctx
+}
+
+func filterSourceFilesForTags(files []string) []string {
+	ret := make([]string, 0, len(files))
+
+	for _, f := range files {
+		dir, filename := filepath.Split(f)
+		ext := filepath.Ext(f)
+
+		match, err := buildContext.MatchFile(dir, filename)
+		if err != nil {
+			log.WithError(err).WithField("file", f).Warn("error matching file")
+		}
+		// MatchFile filters out anything without a file extension. In the
+		// case of CompiledGoFiles (in particular gco processed files from
+		// the cache), we want them.
+		if match || ext == "" {
+			ret = append(ret, f)
+		}
+	}
+	return ret
+}
+func getenvDefault(key, defaultValue string) string {
+	if v, ok := os.LookupEnv(key); ok {
+		return v
+	}
+	return defaultValue
+}

tools/genception/driver/driver_request.go

@@ -0,0 +1,91 @@
+// Copyright 2021 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package driver
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+)
+
+// From https://pkg.go.dev/golang.org/x/tools/go/packages#LoadMode
+type LoadMode int
+
+// Only NeedExportsFile is needed in our case
+const (
+	// NeedName adds Name and PkgPath.
+	NeedName LoadMode = 1 << iota
+
+	// NeedFiles adds GoFiles and OtherFiles.
+	NeedFiles
+
+	// NeedCompiledGoFiles adds CompiledGoFiles.
+	NeedCompiledGoFiles
+
+	// NeedImports adds Imports. If NeedDeps is not set, the Imports field will contain
+	// "placeholder" Packages with only the ID set.
+	NeedImports
+
+	// NeedDeps adds the fields requested by the LoadMode in the packages in Imports.
+	NeedDeps
+
+	// NeedExportsFile adds ExportFile.
+	NeedExportFile
+
+	// NeedTypes adds Types, Fset, and IllTyped.
+	NeedTypes
+
+	// NeedSyntax adds Syntax.
+	NeedSyntax
+
+	// NeedTypesInfo adds TypesInfo.
+	NeedTypesInfo
+
+	// NeedTypesSizes adds TypesSizes.
+	NeedTypesSizes
+
+	// typecheckCgo enables full support for type checking cgo. Requires Go 1.15+.
+	// Modifies CompiledGoFiles and Types, and has no effect on its own.
+	typecheckCgo
+
+	// NeedModule adds Module.
+	NeedModule
+)
+
+// Deprecated: NeedExportsFile is a historical misspelling of NeedExportFile.
+const NeedExportsFile = NeedExportFile
+
+// From https://github.com/golang/tools/blob/v0.1.0/go/packages/external.go#L32
+// Most fields are disabled since there is no need for them
+type DriverRequest struct {
+	Mode LoadMode `json:"mode"`
+	// Env specifies the environment the underlying build system should be run in.
+	// Env []string `json:"env"`
+	// BuildFlags are flags that should be passed to the underlying build system.
+	// BuildFlags []string `json:"build_flags"`
+	// Tests specifies whether the patterns should also return test packages.
+	Tests bool `json:"tests"`
+	// Overlay maps file paths (relative to the driver's working directory) to the byte contents
+	// of overlay files.
+	// Overlay map[string][]byte `json:"overlay"`
+}
+
+func ReadDriverRequest(r io.Reader) (*DriverRequest, error) {
+	req := &DriverRequest{}
+	if err := json.NewDecoder(r).Decode(&req); err != nil {
+		return nil, fmt.Errorf("unable to decode driver request: %w", err)
+	}
+	return req, nil
+}

tools/genception/driver/flatpackage.go

@@ -0,0 +1,217 @@
+// Copyright 2021 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package driver
+
+import (
+	"encoding/json"
+	"fmt"
+	"go/parser"
+	"go/token"
+	"os"
+	"strconv"
+	"strings"
+)
+
+type ResolvePkgFunc func(importPath string) string
+
+// Copy and pasted from golang.org/x/tools/go/packages
+type FlatPackagesError struct {
+	Pos  string // "file:line:col" or "file:line" or "" or "-"
+	Msg  string
+	Kind FlatPackagesErrorKind
+}
+
+type FlatPackagesErrorKind int
+
+const (
+	UnknownError FlatPackagesErrorKind = iota
+	ListError
+	ParseError
+	TypeError
+)
+
+func (err FlatPackagesError) Error() string {
+	pos := err.Pos
+	if pos == "" {
+		pos = "-" // like token.Position{}.String()
+	}
+	return pos + ": " + err.Msg
+}
+
+// FlatPackage is the JSON form of Package
+// It drops all the type and syntax fields, and transforms the Imports
+type FlatPackage struct {
+	ID              string
+	Name            string              `json:",omitempty"`
+	PkgPath         string              `json:",omitempty"`
+	Errors          []FlatPackagesError `json:",omitempty"`
+	GoFiles         []string            `json:",omitempty"`
+	CompiledGoFiles []string            `json:",omitempty"`
+	OtherFiles      []string            `json:",omitempty"`
+	ExportFile      string              `json:",omitempty"`
+	Imports         map[string]string   `json:",omitempty"`
+	Standard        bool                `json:",omitempty"`
+}
+
+type (
+	PackageFunc      func(pkg *FlatPackage)
+	PathResolverFunc func(path string) string
+)
+
+func resolvePathsInPlace(prf PathResolverFunc, paths []string) {
+	for i, path := range paths {
+		paths[i] = prf(path)
+	}
+}
+
+func WalkFlatPackagesFromJSON(jsonFile string, onPkg PackageFunc) error {
+	f, err := os.Open(jsonFile)
+	if err != nil {
+		return fmt.Errorf("unable to open package JSON file: %w", err)
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			log.WithError(err).WithField("file", f.Name()).Error("unable to close file")
+		}
+	}()
+
+	decoder := json.NewDecoder(f)
+	for decoder.More() {
+		pkg := &FlatPackage{}
+		if err := decoder.Decode(&pkg); err != nil {
+			return fmt.Errorf("unable to decode package in %s: %w", f.Name(), err)
+		}
+
+		onPkg(pkg)
+	}
+	return nil
+}
+
+func (fp *FlatPackage) ResolvePaths(prf PathResolverFunc) {
+	resolvePathsInPlace(prf, fp.CompiledGoFiles)
+	resolvePathsInPlace(prf, fp.GoFiles)
+	resolvePathsInPlace(prf, fp.OtherFiles)
+	fp.ExportFile = prf(fp.ExportFile)
+}
+
+// FilterFilesForBuildTags filters the source files given the current build
+// tags.
+func (fp *FlatPackage) FilterFilesForBuildTags() {
+	fp.GoFiles = filterSourceFilesForTags(fp.GoFiles)
+	fp.CompiledGoFiles = filterSourceFilesForTags(fp.CompiledGoFiles)
+}
+
+func (fp *FlatPackage) filterTestSuffix(files []string) (err error, testFiles []string, xTestFiles, nonTestFiles []string) {
+	for _, filename := range files {
+		if strings.HasSuffix(filename, "_test.go") {
+			fset := token.NewFileSet()
+			f, err := parser.ParseFile(fset, filename, nil, parser.PackageClauseOnly)
+			if err != nil {
+				return err, nil, nil, nil
+			}
+			if f.Name.Name == fp.Name {
+				testFiles = append(testFiles, filename)
+			} else {
+				xTestFiles = append(xTestFiles, filename)
+			}
+		} else {
+			nonTestFiles = append(nonTestFiles, filename)
+		}
+	}
+	return
+}
+
+func (fp *FlatPackage) MoveTestFiles() *FlatPackage {
+	err, tgf, xtgf, gf := fp.filterTestSuffix(fp.GoFiles)
+
+	if err != nil {
+		return nil
+	}
+	fp.GoFiles = append(gf, tgf...)
+	fp.CompiledGoFiles = append(gf, tgf...)
+
+	if len(xtgf) == 0 {
+		return nil
+	}
+
+	newImports := make(map[string]string, len(fp.Imports))
+	for k, v := range fp.Imports {
+		newImports[k] = v
+	}
+
+	newImports[fp.PkgPath] = fp.ID
+
+	// Clone package, only xtgf files
+	return &FlatPackage{
+		ID:              fp.ID + "_xtest",
+		Name:            fp.Name + "_test",
+		PkgPath:         fp.PkgPath + "_test",
+		Imports:         newImports,
+		Errors:          fp.Errors,
+		GoFiles:         append([]string{}, xtgf...),
+		CompiledGoFiles: append([]string{}, xtgf...),
+		OtherFiles:      fp.OtherFiles,
+		ExportFile:      fp.ExportFile,
+		Standard:        fp.Standard,
+	}
+}
+
+func (fp *FlatPackage) IsStdlib() bool {
+	return fp.Standard
+}
+
+func (fp *FlatPackage) ResolveImports(resolve ResolvePkgFunc) error {
+	// Stdlib packages are already complete import wise
+	if fp.IsStdlib() {
+		return nil
+	}
+
+	fset := token.NewFileSet()
+
+	for _, file := range fp.CompiledGoFiles {
+		f, err := parser.ParseFile(fset, file, nil, parser.ImportsOnly)
+		if err != nil {
+			return err
+		}
+		// If the name is not provided, fetch it from the sources
+		if fp.Name == "" {
+			fp.Name = f.Name.Name
+		}
+
+		for _, rawImport := range f.Imports {
+			imp, err := strconv.Unquote(rawImport.Path.Value)
+			if err != nil {
+				continue
+			}
+			// We don't handle CGo for now
+			if imp == "C" {
+				continue
+			}
+			if _, ok := fp.Imports[imp]; ok {
+				continue
+			}
+
+			if pkgID := resolve(imp); pkgID != "" {
+				fp.Imports[imp] = pkgID
+			}
+		}
+	}
+
+	return nil
+}
+
+func (fp *FlatPackage) IsRoot() bool {
+	return strings.HasPrefix(fp.ID, "//")
+}

tools/genception/driver/index.go

@@ -0,0 +1,57 @@
+package driver
+
+import (
+	"encoding/json"
+	"os"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	ENV_JSON_INDEX_PATH = "PACKAGE_JSON_INVENTORY"
+	ENV_PACKAGES_BASE   = "PACKAGES_BASE"
+)
+
+var ErrUnsetEnvVar = errors.New("required env var not set")
+
+// LoadJsonListing reads the list of json package index files created by the bazel gopackagesdriver aspect:
+// https://github.com/bazelbuild/rules_go/blob/master/go/tools/gopackagesdriver/aspect.bzl
+// This list is serialized as a []string paths, relative to the bazel exec root.
+func LoadJsonListing() ([]string, error) {
+	path, err := JsonIndexPathFromEnv()
+	if err != nil {
+		return nil, err
+	}
+	return ReadJsonIndex(path)
+}
+
+func ReadJsonIndex(path string) ([]string, error) {
+	um := make([]string, 0)
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	log.WithField("path", path).Info("Read json index file")
+	if err := json.Unmarshal(b, &um); err != nil {
+		return nil, err
+	}
+
+	return um, nil
+}
+
+// JsonIndexPathFromEnv reads the path to the json index file from the environment.
+func JsonIndexPathFromEnv() (string, error) {
+	p := os.Getenv(ENV_JSON_INDEX_PATH)
+	if p == "" {
+		return "", errors.Wrap(ErrUnsetEnvVar, ENV_JSON_INDEX_PATH)
+	}
+	return p, nil
+}
+
+func PackagesBaseFromEnv() (string, error) {
+	p := os.Getenv(ENV_PACKAGES_BASE)
+	if p == "" {
+		return "", errors.Wrap(ErrUnsetEnvVar, ENV_PACKAGES_BASE)
+	}
+	return p, nil
+}

tools/genception/driver/index_test.go

@@ -0,0 +1,58 @@
+package driver
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/v5/testing/require"
+)
+
+func TestJsonList(t *testing.T) {
+	path := "testdata/json-list.json"
+	files, err := ReadJsonIndex(path)
+	require.NoError(t, err)
+	require.Equal(t, 4, len(files))
+}
+
+func TestJsonIndexPathFromEnv(t *testing.T) {
+	cases := []struct {
+		val     string
+		err     error
+		envname string
+		getter  func() (string, error)
+	}{
+		{
+			getter: JsonIndexPathFromEnv,
+			err:    ErrUnsetEnvVar,
+		},
+		{
+			getter:  JsonIndexPathFromEnv,
+			envname: ENV_JSON_INDEX_PATH,
+			val:     "/path/to/file",
+		},
+		{
+			getter: PackagesBaseFromEnv,
+			err:    ErrUnsetEnvVar,
+		},
+		{
+			getter:  PackagesBaseFromEnv,
+			envname: ENV_PACKAGES_BASE,
+			val:     "/path/to/base",
+		},
+	}
+
+	for i, c := range cases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			if c.envname != "" {
+				t.Setenv(c.envname, c.val)
+			}
+			v, err := c.getter()
+			if c.err != nil {
+				require.ErrorIs(t, err, c.err)
+				return
+			}
+			require.NoError(t, err)
+			require.Equal(t, c.val, v)
+		})
+	}
+}

tools/genception/driver/json_packages_driver.go

@@ -0,0 +1,98 @@
+// Copyright 2021 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package driver
+
+import (
+	"fmt"
+	"runtime"
+)
+
+type JSONPackagesDriver struct {
+	registry *PackageRegistry
+}
+
+func NewJSONPackagesDriver(jsonFiles []string, prf PathResolverFunc) (*JSONPackagesDriver, error) {
+	jpd := &JSONPackagesDriver{
+		registry: NewPackageRegistry(),
+	}
+
+	for _, f := range jsonFiles {
+		if err := WalkFlatPackagesFromJSON(f, func(pkg *FlatPackage) {
+			jpd.registry.Add(pkg)
+		}); err != nil {
+			return nil, fmt.Errorf("unable to walk json: %w", err)
+		}
+	}
+
+	if err := jpd.registry.ResolvePaths(prf); err != nil {
+		return nil, fmt.Errorf("unable to resolve paths: %w", err)
+	}
+
+	if err := jpd.registry.ResolveImports(); err != nil {
+		return nil, fmt.Errorf("unable to resolve paths: %w", err)
+	}
+
+	return jpd, nil
+}
+
+func (b *JSONPackagesDriver) Handle(req *DriverRequest, queries []string) *driverResponse {
+	r, p := b.registry.Query(req, queries)
+	return &driverResponse{
+		NotHandled: false,
+		Compiler:   "gc",
+		Arch:       runtime.GOARCH,
+		Roots:      r,
+		Packages:   p,
+	}
+}
+
+func (b *JSONPackagesDriver) GetResponse(labels []string) *driverResponse {
+	rootPkgs, packages := b.registry.Match(labels)
+
+	return &driverResponse{
+		NotHandled: false,
+		Compiler:   "gc",
+		Arch:       runtime.GOARCH,
+		Roots:      rootPkgs,
+		Packages:   packages,
+	}
+}
+
+type driverResponse struct {
+	// NotHandled is returned if the request can't be handled by the current
+	// driver. If an external driver returns a response with NotHandled, the
+	// rest of the driverResponse is ignored, and go/packages will fallback
+	// to the next driver. If go/packages is extended in the future to support
+	// lists of multiple drivers, go/packages will fall back to the next driver.
+	NotHandled bool
+
+	// Compiler and Arch are the arguments pass of types.SizesFor
+	// to get a types.Sizes to use when type checking.
+	Compiler string
+	Arch     string
+
+	// Roots is the set of package IDs that make up the root packages.
+	// We have to encode this separately because when we encode a single package
+	// we cannot know if it is one of the roots as that requires knowledge of the
+	// graph it is part of.
+	Roots []string `json:",omitempty"`
+
+	// Packages is the full set of packages in the graph.
+	// The packages are not connected into a graph.
+	// The Imports if populated will be stubs that only have their ID set.
+	// Imports will be connected and then type and syntax information added in a
+	// later pass (see refine).
+	Packages []*FlatPackage
+}

tools/genception/driver/logger.go

@@ -0,0 +1,26 @@
+package driver
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/sirupsen/logrus"
+)
+
+// Create a new instance of the logger. You can have any number of instances.
+var log = logrus.New()
+var Logger *logrus.Logger
+
+func init() {
+	path := os.Getenv("GOPACKAGESDRIVER_LOG_PATH")
+	if path == "" {
+		path = filepath.Join(os.Getenv("PWD"), "genception.log")
+	}
+	file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
+	if err == nil {
+		log.Out = file
+	} else {
+		log.Info("Failed to log to file, using default stderr")
+	}
+	Logger = log
+}

tools/genception/driver/packageregistry.go

@@ -0,0 +1,205 @@
+// Copyright 2021 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package driver
+
+import (
+	"fmt"
+	"strings"
+)
+
+type PackageRegistry struct {
+	packages map[string]*FlatPackage
+	stdlib   map[string]string
+}
+
+func NewPackageRegistry(pkgs ...*FlatPackage) *PackageRegistry {
+	pr := &PackageRegistry{
+		packages: map[string]*FlatPackage{},
+		stdlib:   map[string]string{},
+	}
+	pr.Add(pkgs...)
+	return pr
+}
+
+const stdlibPrefix = "@@io_bazel_rules_go//stdlib:"
+
+func canonicalizeId(path, id string, pkg *FlatPackage) string {
+	if strings.HasPrefix(id, stdlibPrefix) {
+		return id[len(stdlibPrefix):]
+	}
+	if pkg.IsStdlib() {
+		return id
+	}
+	return path
+}
+
+func rewritePackage(pkg *FlatPackage) {
+	pkg.ID = pkg.PkgPath
+	for k, v := range pkg.Imports {
+		// rewrite package ID mapping to be the same as the path
+		pkg.Imports[k] = canonicalizeId(k, v, pkg)
+	}
+}
+
+// returns true if a is a superset of b
+func isSuperset(a, b []string) bool {
+	if len(a) < len(b) {
+		return false
+	}
+	bi := 0
+	for i := range a {
+		if a[i] == b[bi] {
+			bi++
+			if bi == len(b) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// Update merges the contents of 2 packages together in the instance where they have the same package path.
+// This can happen when the gopackages aspect traverses to a child label and generates separate json files transitive targets.
+// For example, in //proto/prysm/v1alpha1 we see both `:go_default_library` and `:go_proto` from `//proto/engine/v1`.
+// Without the merge, `:go_proto` can overwrite `:go_default_library`, leaving sources files out of the final graph.
+func (pr *PackageRegistry) Update(pkg *FlatPackage) {
+	existing, ok := pr.packages[pkg.PkgPath]
+	if !ok {
+		pr.packages[pkg.PkgPath] = pkg
+		return
+	}
+	if isSuperset(pkg.GoFiles, existing.GoFiles) {
+		existing.GoFiles = pkg.GoFiles
+	}
+}
+
+func (pr *PackageRegistry) Add(pkgs ...*FlatPackage) *PackageRegistry {
+	for _, pkg := range pkgs {
+		rewritePackage(pkg)
+		pr.packages[pkg.PkgPath] = pkg
+
+		if pkg.IsStdlib() {
+			pr.stdlib[pkg.PkgPath] = pkg.ID
+		}
+	}
+	return pr
+}
+
+func (pr *PackageRegistry) ResolvePaths(prf PathResolverFunc) error {
+	for _, pkg := range pr.packages {
+		pkg.ResolvePaths(prf)
+		pkg.FilterFilesForBuildTags()
+	}
+	return nil
+}
+
+// ResolveImports adds stdlib imports to packages. This is required because
+// stdlib packages are not part of the JSON file exports as bazel is unaware of
+// them.
+func (pr *PackageRegistry) ResolveImports() error {
+	resolve := func(importPath string) string {
+		if pkgID, ok := pr.stdlib[importPath]; ok {
+			return pkgID
+		}
+
+		return ""
+	}
+
+	for _, pkg := range pr.packages {
+		if err := pkg.ResolveImports(resolve); err != nil {
+			return err
+		}
+		testFp := pkg.MoveTestFiles()
+		if testFp != nil {
+			pr.packages[testFp.ID] = testFp
+		}
+	}
+
+	return nil
+}
+
+func (pr *PackageRegistry) walk(acc map[string]*FlatPackage, root string) {
+	pkg := pr.packages[root]
+
+	if pkg == nil {
+		log.WithField("root", root).Error("package ID not found")
+		return
+	}
+
+	acc[pkg.ID] = pkg
+	for _, pkgID := range pkg.Imports {
+		if _, ok := acc[pkgID]; !ok {
+			pr.walk(acc, pkgID)
+		}
+	}
+}
+
+func (pr *PackageRegistry) Query(req *DriverRequest, queries []string) ([]string, []*FlatPackage) {
+	walkedPackages := map[string]*FlatPackage{}
+	retRoots := make([]string, 0, len(queries))
+	for _, rootPkg := range queries {
+		retRoots = append(retRoots, rootPkg)
+		pr.walk(walkedPackages, rootPkg)
+	}
+
+	retPkgs := make([]*FlatPackage, 0, len(walkedPackages))
+	for _, pkg := range walkedPackages {
+		retPkgs = append(retPkgs, pkg)
+	}
+
+	return retRoots, retPkgs
+}
+
+func (pr *PackageRegistry) Match(labels []string) ([]string, []*FlatPackage) {
+	roots := map[string]struct{}{}
+
+	for _, label := range labels {
+		// When packagesdriver is ran from rules go, rulesGoRepositoryName will just be @
+		if !strings.HasPrefix(label, "@") {
+			// Canonical labels is only since Bazel 6.0.0
+			label = fmt.Sprintf("@%s", label)
+		}
+
+		if label == RulesGoStdlibLabel {
+			// For stdlib, we need to append all the subpackages as roots
+			// since RulesGoStdLibLabel doesn't actually show up in the stdlib pkg.json
+			for _, pkg := range pr.packages {
+				if pkg.Standard {
+					roots[pkg.ID] = struct{}{}
+				}
+			}
+		} else {
+			roots[label] = struct{}{}
+			// If an xtest package exists for this package add it to the roots
+			if _, ok := pr.packages[label+"_xtest"]; ok {
+				roots[label+"_xtest"] = struct{}{}
+			}
+		}
+	}
+
+	walkedPackages := map[string]*FlatPackage{}
+	retRoots := make([]string, 0, len(roots))
+	for rootPkg := range roots {
+		retRoots = append(retRoots, rootPkg)
+		pr.walk(walkedPackages, rootPkg)
+	}
+
+	retPkgs := make([]*FlatPackage, 0, len(walkedPackages))
+	for _, pkg := range walkedPackages {
+		retPkgs = append(retPkgs, pkg)
+	}
+
+	return retRoots, retPkgs
+}

tools/genception/driver/packageregistry_test.go

@@ -0,0 +1,27 @@
+package driver
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestIsSuperset(t *testing.T) {
+	cases := []struct {
+		a        []string
+		b        []string
+		expected bool
+	}{
+		{[]string{"a", "b", "c", "d"}, []string{"a", "b"}, true},
+		{[]string{"a", "b", "c", "d"}, []string{"a", "b", "c", "d"}, true},
+		{[]string{"a", "b", "c", "d"}, []string{"a", "b", "c", "d", "e"}, false},
+		{[]string{"a", "b", "c", "d"}, []string{"a", "b", "c"}, true},
+		{[]string{}, []string{"a"}, false},
+	}
+	for _, c := range cases {
+		t.Run(strings.Join(c.a, "_")+"__"+strings.Join(c.b, "_"), func(t *testing.T) {
+			if isSuperset(c.a, c.b) != c.expected {
+				t.Errorf("isSuperset(%v, %v) != %v", c.a, c.b, c.expected)
+			}
+		})
+	}
+}

tools/genception/driver/recorder.go

@@ -0,0 +1,52 @@
+package driver
+
+import (
+	"encoding/json"
+	"os"
+	"path"
+	"strconv"
+	"time"
+)
+
+type Recorder struct {
+	base string
+	t    time.Time
+}
+
+func NewRecorder() (*Recorder, error) {
+	base := os.Getenv("PWD")
+	r := &Recorder{base: base, t: time.Now()}
+	if err := r.Mkdir(); err != nil {
+		return nil, err
+	}
+	return r, nil
+}
+func (r *Recorder) Dir() string {
+	return path.Join(r.base, strconv.FormatInt(r.t.UTC().UnixNano(), 10))
+}
+
+func (r *Recorder) Mkdir() error {
+	return os.MkdirAll(r.Dir(), 0755)
+}
+
+func (r *Recorder) RecordRequest(args []string, req *DriverRequest) error {
+	b, err := json.Marshal(struct {
+		Args    []string
+		Request *DriverRequest
+	}{
+		Args:    args,
+		Request: req,
+	})
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(path.Join(r.Dir(), "request.json"), b, 0644)
+}
+
+func (r *Recorder) RecordResponse(resp *driverResponse) error {
+	b, err := json.Marshal(resp)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(path.Join(r.Dir(), "response.json"), b, 0644)
+}

tools/genception/driver/testdata/json-list.json

@@ -0,0 +1,6 @@
+[
+  "bazel-out/darwin_arm64-fastbuild/bin/external/io_bazel_rules_go/stdlib_/stdlib.pkg.json",
+  "bazel-out/darwin_arm64-fastbuild/bin/external/com_github_thomaso_mirodin_intmath/constants/c64/c64.pkg.json",
+  "bazel-out/darwin_arm64-fastbuild/bin/external/com_github_thomaso_mirodin_intmath/u64/u64.pkg.json",
+  "bazel-out/darwin_arm64-fastbuild/bin/proto/prysm/v1alpha1/go_proto.pkg.json"
+]