feat: support go corset optimisation and metadata (#713)

This adds the option to run the checker in "bench" mode, and also adds support for passing a go-corset optimisation level via the bin/checker.  The purpose of this is to enable testing at different optimisation levels to see what effect this has.  Finally, it exposes the metadata stored in the zkevm.bin file, such that it can be used for cross-checking purposes.  For example, the metadata includes the git commit of the enclosing repository when it was built.

---------

Co-authored-by: AlexandreBelling <alexandrebelling8@gmail.com>

prover/cmd/dev-tools/corset-checker/main.go

@@ -4,21 +4,36 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/consensys/linea-monorepo/prover/config"
+	"github.com/consensys/linea-monorepo/prover/crypto/ringsis"
+	"github.com/consensys/linea-monorepo/prover/protocol/compiler"
+	"github.com/consensys/linea-monorepo/prover/protocol/compiler/cleanup"
 	"github.com/consensys/linea-monorepo/prover/protocol/compiler/dummy"
+	"github.com/consensys/linea-monorepo/prover/protocol/compiler/mimc"
+	"github.com/consensys/linea-monorepo/prover/protocol/compiler/selfrecursion"
+	"github.com/consensys/linea-monorepo/prover/protocol/compiler/vortex"
 	"github.com/consensys/linea-monorepo/prover/protocol/wizard"
 )
 
 func main() {
 
-	cfg, traceFile, pErr := getParamsFromCLI()
+	cfg, optConfig, traceFile, pErr := getParamsFromCLI()
 	if pErr != nil {
 		fmt.Printf("FATAL\n")
 		fmt.Printf("err = %v\n", pErr)
 		os.Exit(1)
 	}
 
+	suite := []func(*wizard.CompiledIOP){
+		dummy.Compile,
+	}
+
+	if cfg.Execution.ProverMode == config.ProverModeBench {
+		suite = fullSuite()
+	}
+
 	var (
-		comp  = wizard.Compile(MakeDefine(cfg), dummy.Compile)
+		comp  = wizard.Compile(MakeDefine(cfg, optConfig), suite...)
 		proof = wizard.Prove(comp, MakeProver(traceFile))
 		vErr  = wizard.Verify(comp, proof)
 	)
@@ -34,3 +49,48 @@ func main() {
 	}
 
 }
+
+func fullSuite() []func(*wizard.CompiledIOP) {
+
+	sisInstance := ringsis.Params{LogTwoBound: 16, LogTwoDegree: 6}
+
+	return []func(*wizard.CompiledIOP){
+		mimc.CompileMiMC,
+		compiler.Arcane(1<<10, 1<<19, false),
+		vortex.Compile(
+			2,
+			vortex.ForceNumOpenedColumns(256),
+			vortex.WithSISParams(&sisInstance),
+		),
+
+		selfrecursion.SelfRecurse,
+		cleanup.CleanUp,
+		mimc.CompileMiMC,
+		compiler.Arcane(1<<10, 1<<18, false),
+		vortex.Compile(
+			2,
+			vortex.ForceNumOpenedColumns(256),
+			vortex.WithSISParams(&sisInstance),
+		),
+
+		selfrecursion.SelfRecurse,
+		cleanup.CleanUp,
+		mimc.CompileMiMC,
+		compiler.Arcane(1<<10, 1<<16, false),
+		vortex.Compile(
+			8,
+			vortex.ForceNumOpenedColumns(64),
+			vortex.WithSISParams(&sisInstance),
+		),
+
+		selfrecursion.SelfRecurse,
+		cleanup.CleanUp,
+		mimc.CompileMiMC,
+		compiler.Arcane(1<<10, 1<<13, false),
+		vortex.Compile(
+			8,
+			vortex.ForceNumOpenedColumns(64),
+			vortex.ReplaceSisByMimc(),
+		),
+	}
+}

prover/cmd/dev-tools/corset-checker/params.go

@@ -4,35 +4,44 @@ import (
 	"flag"
 	"fmt"
 
+	"github.com/consensys/go-corset/pkg/mir"
 	"github.com/consensys/linea-monorepo/prover/config"
 )
 
 var (
-	configFPathCLI string
-	traceFPathCLI  string
+	configFPathCLI       string
+	traceFPathCLI        string
+	optimisationLevelCLI uint
 )
 
 func init() {
 	flag.StringVar(&configFPathCLI, "config", "", "path to the config file. Only the trace limits are read")
 	flag.StringVar(&traceFPathCLI, "trace-file", "", "path to the `.lt` trace file")
+	flag.UintVar(&optimisationLevelCLI, "opt", 1, "set go-corset optimisation level to apply")
 	flag.Parse()
 }
 
-func getParamsFromCLI() (cfg *config.Config, traceFPath string, err error) {
+func getParamsFromCLI() (cfg *config.Config, optConfig *mir.OptimisationConfig, traceFPath string, err error) {
 
 	flag.Parse()
 
 	if len(configFPathCLI) == 0 {
-		return nil, "", fmt.Errorf("could not find the config path, got %++v", configFPathCLI)
+		return nil, nil, "", fmt.Errorf("could not find the config path, got %++v", configFPathCLI)
 	}
 
 	if len(traceFPathCLI) == 0 {
-		return nil, "", fmt.Errorf("could not find the trace file path, got %++v", traceFPathCLI)
+		return nil, nil, "", fmt.Errorf("could not find the trace file path, got %++v", traceFPathCLI)
 	}
 
 	if cfg, err = config.NewConfigFromFile(configFPathCLI); err != nil {
-		return nil, "", fmt.Errorf("could not parse the config: %w", err)
+		return nil, nil, "", fmt.Errorf("could not parse the config: %w", err)
 	}
-
-	return cfg, traceFPathCLI, nil
+	// Sanity check specified optimisation level makes sense.
+	if optimisationLevelCLI >= uint(len(mir.OPTIMISATION_LEVELS)) {
+		return nil, nil, "", fmt.Errorf("invalid optimisation level: %d", optimisationLevelCLI)
+	}
+	// Set optimisation config
+	optConfig = &mir.OPTIMISATION_LEVELS[optimisationLevelCLI]
+	// Done
+	return cfg, optConfig, traceFPathCLI, nil
 }

prover/cmd/dev-tools/corset-checker/zkevm.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"github.com/consensys/go-corset/pkg/mir"
 	"github.com/consensys/linea-monorepo/prover/config"
 	"github.com/consensys/linea-monorepo/prover/protocol/wizard"
 	"github.com/consensys/linea-monorepo/prover/zkevm/arithmetization"
@@ -8,9 +9,12 @@ import (
 
 var globalArith *arithmetization.Arithmetization
 
-func MakeDefine(cfg *config.Config) wizard.DefineFunc {
+func MakeDefine(cfg *config.Config, optConfig *mir.OptimisationConfig) wizard.DefineFunc {
 	return func(build *wizard.Builder) {
-		globalArith = arithmetization.NewArithmetization(build, arithmetization.Settings{Limits: &cfg.TracesLimits})
+		globalArith = arithmetization.NewArithmetization(build, arithmetization.Settings{
+			Limits:            &cfg.TracesLimits,
+			OptimisationLevel: optConfig,
+		})
 
 	}
 }

prover/go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/consensys/compress v0.2.5
 	github.com/consensys/gnark v0.11.1-0.20250107100237-2cb190338a01
 	github.com/consensys/gnark-crypto v0.14.1-0.20250117145449-0493a37cc361
-	github.com/consensys/go-corset v0.0.0-20250217020957-ab7f2d548fa8
+	github.com/consensys/go-corset v0.0.0-20250219023519-757eed939609
 	github.com/crate-crypto/go-kzg-4844 v1.1.0
 	github.com/dlclark/regexp2 v1.11.2
 	github.com/fxamacker/cbor/v2 v2.7.0

prover/go.sum

@@ -102,6 +102,8 @@ github.com/consensys/gnark-crypto v0.14.1-0.20250117145449-0493a37cc361 h1:7HGt1
 github.com/consensys/gnark-crypto v0.14.1-0.20250117145449-0493a37cc361/go.mod h1:q9s22Y0WIHd9UCBfD+xGeW8wDJ7WAGZZpMrLFqzBzrQ=
 github.com/consensys/go-corset v0.0.0-20250217020957-ab7f2d548fa8 h1:q6LG3JTvcx9OfKKWDS/5xcT0+mO2gHJnKKpy5mdMN+g=
 github.com/consensys/go-corset v0.0.0-20250217020957-ab7f2d548fa8/go.mod h1:rNP3hMR2Sjy5EdQNTHINwaM5kD08E3CSw8CCKhljjO8=
+github.com/consensys/go-corset v0.0.0-20250219023519-757eed939609 h1:UW+f/2XpsFc3FahhhDfK3bvmzpS0LriEm/1NzPctkDU=
+github.com/consensys/go-corset v0.0.0-20250219023519-757eed939609/go.mod h1:rNP3hMR2Sjy5EdQNTHINwaM5kD08E3CSw8CCKhljjO8=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=

prover/zkevm/arithmetization/arithmetization.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	"github.com/consensys/go-corset/pkg/air"
+	"github.com/consensys/go-corset/pkg/mir"
 	"github.com/consensys/linea-monorepo/prover/backend/files"
 	"github.com/consensys/linea-monorepo/prover/config"
 	"github.com/consensys/linea-monorepo/prover/protocol/wizard"
@@ -12,6 +13,10 @@ import (
 // Settings specifies the parameters for the arithmetization part of the zkEVM.
 type Settings struct {
 	Limits *config.TracesLimits
+	// OptimisationLevel determines the optimisation level which go-corset will
+	// apply when compiling the zkevm.bin file to AIR constraints.  If in doubt,
+	// use mir.DEFAULT_OPTIMISATION_LEVEL.
+	OptimisationLevel *mir.OptimisationConfig
 }
 
 // Arithmetization exposes all the methods relevant for the user to interact
@@ -26,8 +31,7 @@ type Arithmetization struct {
 // NewArithmetization is the function that declares all the columns and the constraints of
 // the zkEVM in the input builder object.
 func NewArithmetization(builder *wizard.Builder, settings Settings) *Arithmetization {
-
-	schema, errS := ReadZkevmBin()
+	schema, _, errS := ReadZkevmBin(settings.OptimisationLevel)
 	if errS != nil {
 		panic(errS)
 	}

prover/zkevm/arithmetization/definition_test.go

@@ -4,6 +4,7 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/consensys/go-corset/pkg/mir"
 	"github.com/consensys/linea-monorepo/prover/config"
 	"github.com/consensys/linea-monorepo/prover/protocol/coin"
 	"github.com/consensys/linea-monorepo/prover/protocol/column"
@@ -23,9 +24,9 @@ func TestDefine(t *testing.T) {
 			Coins:           wizard.NewRegister[coin.Name, coin.Info](),
 			Precomputed:     collection.NewMapping[ifaces.ColID, ifaces.ColAssignment](),
 		}
-		schema, errBin = ReadZkevmBin()
-		limits         = &config.TracesLimits{}
-		limitRefl      = reflect.ValueOf(limits).Elem()
+		schema, _, errBin = ReadZkevmBin(&mir.DEFAULT_OPTIMISATION_LEVEL)
+		limits            = &config.TracesLimits{}
+		limitRefl         = reflect.ValueOf(limits).Elem()
 	)
 
 	for i := 0; i < limitRefl.NumField(); i++ {

prover/zkevm/arithmetization/files.go

@@ -10,6 +10,7 @@ import (
 	"github.com/consensys/go-corset/pkg/air"
 	"github.com/consensys/go-corset/pkg/binfile"
 	"github.com/consensys/go-corset/pkg/corset"
+	"github.com/consensys/go-corset/pkg/mir"
 	"github.com/consensys/go-corset/pkg/schema"
 	"github.com/consensys/go-corset/pkg/trace"
 	"github.com/consensys/go-corset/pkg/trace/lt"
@@ -24,9 +25,20 @@ const TraceOverflowExitCode = 77
 //go:embed zkevm.bin
 var zkevmStr string
 
-// ReadZkEvmBin parses and compiles a "zkevm.bin" into an air.Schema. f is closed
-// at the end of the function call.
-func ReadZkevmBin() (*air.Schema, error) {
+// ReadZkevmBin parses and compiles a "zkevm.bin" file into an air.Schema,
+// whilst applying whatever optimisations are requested. Optimisations can
+// impact the size of the generated schema and, consequently, the size of the
+// expanded trace.  For example, certain optimisations eliminate unnecessary
+// columns creates for multiplicative inverses.  However, optimisations do not
+// always improve overall performance, as they can increase the complexity of
+// other constraints.  The DEFAULT_OPTIMISATION_LEVEL is the recommended level
+// to use in general, whilst others are intended for testing purposes (i.e. to
+// try out new optimisations to see whether they help or hinder, etc).
+//
+// This additionally extracts the metadata map from the zkevm.bin file.  This
+// contains information which can be used to cross-check the zkevm.bin file,
+// such as the git commit of the enclosing repository when it was built.
+func ReadZkevmBin(optConfig *mir.OptimisationConfig) (schema *air.Schema, metadata map[string]string, err error) {
 	var (
 		binf binfile.BinaryFile
 		buf  []byte = []byte(zkevmStr)
@@ -34,15 +46,17 @@ func ReadZkevmBin() (*air.Schema, error) {
 	// TODO: why is only this one needed??
 	gob.Register(binfile.Attribute(&corset.SourceMap{}))
 	// Parse zkbinary file
-	err := binf.UnmarshalBinary(buf)
+	err = binf.UnmarshalBinary(buf)
 	// Sanity check for errors
 	if err != nil {
-		return nil, fmt.Errorf("could not parse the read bytes of the 'zkevm.bin' file into an hir.Schema: %w", err)
+		return nil, nil, fmt.Errorf("could not parse the read bytes of the 'zkevm.bin' file into an hir.Schema: %w", err)
 	}
 	// Extract schema
 	hirSchema := &binf.Schema
+	// Attempt to extract metadata from bin file.
+	metadata, err = binf.Header.GetMetaData()
 	// This performs the corset compilation
-	return hirSchema.LowerToMir().LowerToAir(), nil
+	return hirSchema.LowerToMir().LowerToAir(*optConfig), metadata, err
 }
 
 func ReadLtTraces(f io.ReadCloser, sch *air.Schema) (trace.Trace, error) {

prover/zkevm/checker.go

@@ -3,6 +3,7 @@ package zkevm
 import (
 	"sync"
 
+	"github.com/consensys/go-corset/pkg/mir"
 	"github.com/consensys/linea-monorepo/prover/config"
 	"github.com/consensys/linea-monorepo/prover/protocol/compiler/dummy"
 	"github.com/consensys/linea-monorepo/prover/protocol/wizard"
@@ -29,7 +30,8 @@ func CheckerZkEvm(tl *config.TracesLimits) *ZkEvm {
 	onceCheckerZkEvm.Do(func() {
 		settings := Settings{
 			Arithmetization: arithmetization.Settings{
-				Limits: tl,
+				Limits:            tl,
+				OptimisationLevel: &mir.DEFAULT_OPTIMISATION_LEVEL,
 			},
 			CompilationSuite: checkerCompilationSuite,
 			Metadata: wizard.VersionMetadata{

prover/zkevm/full.go

@@ -3,6 +3,7 @@ package zkevm
 import (
 	"sync"
 
+	"github.com/consensys/go-corset/pkg/mir"
 	"github.com/consensys/linea-monorepo/prover/config"
 	"github.com/consensys/linea-monorepo/prover/crypto/ringsis"
 	"github.com/consensys/linea-monorepo/prover/protocol/compiler"
@@ -127,7 +128,8 @@ func fullZKEVMWithSuite(tl *config.TracesLimits, suite compilationSuite) *ZkEvm
 	settings := Settings{
 		CompilationSuite: suite,
 		Arithmetization: arithmetization.Settings{
-			Limits: tl,
+			Limits:            tl,
+			OptimisationLevel: &mir.DEFAULT_OPTIMISATION_LEVEL,
 		},
 		Statemanager: statemanager.Settings{
 			AccSettings: accumulator.Settings{

prover/zkevm/partial.go

@@ -3,6 +3,7 @@ package zkevm
 import (
 	"sync"
 
+	"github.com/consensys/go-corset/pkg/mir"
 	"github.com/consensys/linea-monorepo/prover/config"
 	"github.com/consensys/linea-monorepo/prover/protocol/compiler"
 	"github.com/consensys/linea-monorepo/prover/protocol/compiler/vortex"
@@ -47,7 +48,8 @@ func PartialZkEvm(tl *config.TracesLimits) *ZkEvm {
 		// modules to verify keccak or the state-manager traces.
 		settings := Settings{
 			Arithmetization: arithmetization.Settings{
-				Limits: tl,
+				Limits:            tl,
+				OptimisationLevel: &mir.DEFAULT_OPTIMISATION_LEVEL,
 			},
 			CompilationSuite: partialCompilationSuite,
 			Metadata: wizard.VersionMetadata{