feat: add roller (#6)

Co-authored-by: Steven Gu <asongala@163.com>
This commit is contained in:
HAOYUatHZ
2022-09-27 23:08:08 +08:00
committed by GitHub
parent 3492ab11b3
commit 45d7405692
28 changed files with 48069 additions and 0 deletions

71
.github/workflows/roller.yml vendored Normal file
View File

@@ -0,0 +1,71 @@
name: Roller
on:
push:
branches:
- main
- staging
paths:
- 'roller/**'
pull_request:
branches:
- main
- staging
paths:
- 'roller/**'
defaults:
run:
working-directory: 'roller'
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2022-08-23
override: true
components: rustfmt, clippy
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.17.x
- name: Checkout code
uses: actions/checkout@v2
- name: Test
run: |
make roller
go test -v ./...
check:
runs-on: ubuntu-latest
steps:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.17.x
- name: Checkout code
uses: actions/checkout@v2
- name: Lint
run: |
rm -rf $HOME/.cache/golangci-lint
make lint
goimports-lint:
runs-on: ubuntu-latest
steps:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.17.x
- name: Checkout code
uses: actions/checkout@v2
- name: Install goimports
run: go get golang.org/x/tools/cmd/goimports
- run: goimports -local scroll-tech/go-roller/ -w .
- run: go mod tidy
# If there are any diffs from goimports or go mod tidy, fail.
- name: Verify no changes from goimports and go mod tidy
run: |
if [ -n "$(git status --porcelain)" ]; then
exit 1
fi

14
roller/.gitignore vendored Normal file
View File

@@ -0,0 +1,14 @@
.idea
stack/test_stack
mock/stack
build/bin/
# ignore db file
bbolt_db
# ignore cgo in macOS
roller/prover/lib/libprover.dylib
roller/prover/rust/target
params/
seed

View File

273
roller/.golangci.yml Normal file
View File

@@ -0,0 +1,273 @@
# Source: https://github.com/golangci/golangci-lint/blob/master/.golangci.example.yml
# options for analysis running
run:
# default concurrency is a available CPU number
concurrency: 4
# timeout for analysis, e.g. 30s, 5m, default is 1m
deadline: 5m
# exit code when at least one issue was found, default is 1
issues-exit-code: 1
# include test files or not, default is true
tests: true
# list of build tags, all linters use it. Default is empty list.
#build-tags:
# which dirs to skip: they won't be analyzed;
# can use regexp here: generated.*, regexp is applied on full path;
# default value is empty list, but next dirs are always skipped independently
# from this option's value:
# vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
#skip-dirs:
# which files to skip: they will be analyzed, but issues from them
# won't be reported. Default value is empty list, but there is
# no need to include all autogenerated files, we confidently recognize
# autogenerated files. If it's not please let us know.
#skip-files:
# by default isn't set. If set we pass it to "go list -mod={option}". From "go help modules":
# If invoked with -mod=readonly, the go command is disallowed from the implicit
# automatic updating of go.mod described above. Instead, it fails when any changes
# to go.mod are needed. This setting is most useful to check that go.mod does
# not need updates, such as in a continuous integration and testing system.
# If invoked with -mod=vendor, the go command assumes that the vendor
# directory holds the correct copies of dependencies and ignores
# the dependency descriptions in go.mod.
#modules-download-mode: (release|readonly|vendor)
# output configuration options
output:
# colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
format: colored-line-number
# print lines of code with issue, default is true
print-issued-lines: true
# print linter name in the end of issue text, default is true
print-linter-name: true
# all available settings of specific linters
linters-settings:
errcheck:
# report about not checking of errors in type assetions: `a := b.(MyStruct)`;
# default is false: such cases aren't reported by default.
check-type-assertions: false
# report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`;
# default is false: such cases aren't reported by default.
check-blank: false
# [deprecated] comma-separated list of pairs of the form pkg:regex
# the regex is used to ignore names within pkg. (default "fmt:.*").
# see https://github.com/kisielk/errcheck#the-deprecated-method for details
ignore: fmt:.*,io/ioutil:^Read.*
# path to a file containing a list of functions to exclude from checking
# see https://github.com/kisielk/errcheck#excluding-functions for details
#exclude: /path/to/file.txt
govet:
# report about shadowed variables
check-shadowing: true
golint:
# minimal confidence for issues, default is 0.8
min-confidence: 0.8
gofmt:
# simplify code: gofmt with `-s` option, true by default
simplify: true
goimports:
# put imports beginning with prefix after 3rd-party packages;
# it's a comma-separated list of prefixes
#local-prefixes: github.com/org/project
gocyclo:
# minimal code complexity to report, 30 by default (but we recommend 10-20)
min-complexity: 30
maligned:
# print struct with more effective memory layout or not, false by default
suggest-new: true
dupl:
# tokens count to trigger issue, 150 by default
threshold: 100
goconst:
# minimal length of string constant, 3 by default
min-len: 3
# minimal occurrences count to trigger, 3 by default
min-occurrences: 3
depguard:
list-type: blacklist
include-go-root: false
packages:
- github.com/davecgh/go-spew/spew
misspell:
# Correct spellings using locale preferences for US or UK.
# Default is to use a neutral variety of English.
# Setting locale to US will correct the British spelling of 'colour' to 'color'.
locale: US
ignore-words:
- gossamer
lll:
# max line length, lines longer will be reported. Default is 120.
# '\t' is counted as 1 character by default, and can be changed with the tab-width option
line-length: 120
# tab width in spaces. Default to 1.
tab-width: 1
unused:
# treat code as a program (not a library) and report unused exported identifiers; default is false.
# XXX: if you enable this setting, unused will report a lot of false-positives in text editors:
# if it's called for subdir of a project it can't find funcs usages. All text editor integrations
# with golangci-lint call it on a directory with the changed file.
check-exported: false
unparam:
# Inspect exported functions, default is false. Set to true if no external program/library imports your code.
# XXX: if you enable this setting, unparam will report a lot of false-positives in text editors:
# if it's called for subdir of a project it can't find external interfaces. All text editor integrations
# with golangci-lint call it on a directory with the changed file.
check-exported: false
nakedret:
# make an issue if func has more lines of code than this setting and it has naked returns; default is 30
max-func-lines: 30
prealloc:
# XXX: we don't recommend using this linter before doing performance profiling.
# For most programs usage of prealloc will be a premature optimization.
# Report preallocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them.
# True by default.
simple: true
range-loops: true # Report preallocation suggestions on range loops, true by default
for-loops: false # Report preallocation suggestions on for loops, false by default
gocritic:
# Which checks should be disabled; can't be combined with 'enabled-checks'; default is empty
disabled-checks:
- regexpMust
# Enable multiple checks by tags, run `GL_DEBUG=gocritic golangci-lint` run to see all tags and checks.
# Empty list by default. See https://github.com/go-critic/go-critic#usage -> section "Tags".
enabled-tags:
- performance
settings: # settings passed to gocritic
captLocal: # must be valid enabled check name
paramsOnly: true
rangeValCopy:
sizeThreshold: 32
linters:
enable:
- megacheck
- govet
- gofmt
- goimports
- varcheck
- misspell
- ineffassign
- gosimple
- unconvert
- goconst
- errcheck
- govet
- staticcheck
- gosec
- bodyclose
- goprintffuncname
- golint
- depguard
- gocyclo
- unparam
enable-all: false
disable:
disable-all: false
presets:
fast: false
issues:
# List of regexps of issue texts to exclude, empty list by default.
# But independently from this option we use default exclude patterns,
# it can be disabled by `exclude-use-default: false`. To list all
# excluded by default patterns execute `golangci-lint run --help`
#exclude:
# Excluding configuration per-path, per-linter, per-text and per-source
exclude-rules:
# Exclude some linters from running on tests files.
- path: _test\.go
linters:
- gocyclo
- errcheck
- dupl
- gosec
# Exclude known linters from partially hard-vendored code,
# which is impossible to exclude via "nolint" comments.
- path: internal/hmac/
text: "weak cryptographic primitive"
linters:
- gosec
# Exclude some staticcheck messages
- linters:
- staticcheck
text: "SA9003:"
- linters:
- golint
text: "package comment should be of the form"
- linters:
- golint
text: "don't use ALL_CAPS in Go names;"
- linters:
- golint
text: "don't use underscores in Go names;"
# Exclude lll issues for long lines with go:generate
- linters:
- lll
source: "^//go:generate "
text: "long-lines"
- linters:
- wsl
text: "return statements should not be cuddled if block has more than two lines"
- linters:
- wsl
text: "branch statements should not be cuddled if block has more than two lines"
- linters:
- wsl
text: "declarations should never be cuddled"
- linters:
- wsl
text: "expressions should not be cuddled with declarations or returns"
# Independently from option `exclude` we use default exclude patterns,
# it can be disabled by this option. To list all
# excluded by default patterns execute `golangci-lint run --help`.
# Default value for this option is true.
exclude-use-default: false
# Maximum issues count per one linter. Set to 0 to disable. Default is 50.
max-per-linter: 0
# Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
max-same-issues: 0
# Show only new issues: if there are unstaged changes or untracked files,
# only those changes are analyzed, else only changes in HEAD~ are analyzed.
# It's a super-useful option for integration of golangci-lint into existing
# large codebase. It's not practical to fix all existing issues at the moment
# of integration: much better don't allow issues in new code.
# Default is false.
new: false

15
roller/Dockerfile Normal file
View File

@@ -0,0 +1,15 @@
# Build roller in a stock Go builder container
FROM golang:1.17-alpine as builder
ENV GOPROXY https://goproxy.io,direct
ADD . /go-roller
RUN apk add --no-cache gcc musl-dev linux-headers git ca-certificates \
&& cd /go-roller/cmd/roller/ && go build -v -p 4
# Pull roller into a second stage deploy alpine container
FROM alpine:latest
COPY --from=builder /go-roller/cmd/roller/roller /bin/
ENTRYPOINT ["roller"]

30
roller/Makefile Normal file
View File

@@ -0,0 +1,30 @@
.PHONY: lint docker clean roller
IMAGE_NAME=roller-go
IMAGE_VERSION=latest
prepare-libprover:
cd roller/prover/rust && cargo build --release && cp target/release/libprover.a ../lib/
roller: ## Builds the Roller instance.
cd roller/prover/rust && cargo build --release && cp target/release/libprover.a ../lib/
GOBIN=$(PWD)/build/bin go install ./cmd/roller
gpu-roller: ## Builds the GPU Roller instance.
cd roller/prover/rust && cargo build --release && cp target/release/libprover.a ../lib/
GOBIN=$(PWD)/build/bin go install -tags gpu ./cmd/roller
test-prover:
go test -timeout 0 -v ./roller/prover
test-gpu-prover:
go test -tags gpu -timeout 0 -v ./roller/prover
lint: ## Lint the files - used for CI
GOBIN=$(PWD)/build/bin go run build/lint.go
clean: ## Empty out the bin folder
@rm -rf build/bin
docker:
docker build -t scrolltech/${IMAGE_NAME}:${IMAGE_VERSION} ./

12
roller/README.md Normal file
View File

@@ -0,0 +1,12 @@
# Roller
## Build
```shell
make clean && make roller
```
## Start
- use config.toml
```shell
./build/bin/roller
```

41848
roller/assets/trace.json Normal file

File diff suppressed because one or more lines are too long

66
roller/build/lint.go Normal file
View File

@@ -0,0 +1,66 @@
//go:build none
// +build none
package main
import (
"flag"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"runtime"
)
const (
// GolangCIVersion to be used for linting.
GolangCIVersion = "github.com/golangci/golangci-lint/cmd/golangci-lint@v1.43.0"
)
// GOBIN environment variable.
func goBin() string {
if os.Getenv("GOBIN") == "" {
log.Fatal("GOBIN not set")
}
return os.Getenv("GOBIN")
}
func main() {
log.SetFlags(log.Lshortfile)
if _, err := os.Stat(filepath.Join("build", "lint.go")); os.IsNotExist(err) {
log.Fatal("should run build from root dir")
}
lint()
}
//nolint:gosec
func lint() {
v := flag.Bool("v", false, "log verbosely")
// Make sure GOLANGCI is downloaded and available.
argsGet := []string{"get", GolangCIVersion}
cmd := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), argsGet...)
out, err := cmd.CombinedOutput()
if err != nil {
log.Fatalf("could not list pkgs: %v\n%s", err, string(out))
}
cmd = exec.Command(filepath.Join(goBin(), "golangci-lint"))
cmd.Args = append(cmd.Args, "run", "--config", ".golangci.yml")
if *v {
cmd.Args = append(cmd.Args, "-v")
}
fmt.Println("Linting...")
cmd.Stderr, cmd.Stdout = os.Stderr, os.Stdout
if err := cmd.Run(); err != nil {
log.Fatal("Error: Could not Lint ", "error: ", err, ", cmd: ", cmd)
}
}

View File

@@ -0,0 +1,36 @@
package main
import (
"io"
"os"
"path/filepath"
"github.com/mattn/go-colorable"
"github.com/mattn/go-isatty"
"github.com/scroll-tech/go-ethereum/cmd/utils"
"github.com/scroll-tech/go-ethereum/log"
"github.com/urfave/cli/v2"
)
func setup(ctx *cli.Context) error {
var ostream log.Handler
if logFile := ctx.String(logFileFlag.Name); len(logFile) > 0 {
fp, err := os.OpenFile(filepath.Clean(logFile), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
utils.Fatalf("Failed to open log file", "err", err)
}
ostream = log.StreamHandler(io.Writer(fp), log.TerminalFormat(true))
} else {
output := io.Writer(os.Stderr)
usecolor := (isatty.IsTerminal(os.Stderr.Fd()) || isatty.IsCygwinTerminal(os.Stderr.Fd())) && os.Getenv("TERM") != "dumb"
if usecolor {
output = colorable.NewColorableStderr()
}
ostream = log.StreamHandler(output, log.TerminalFormat(usecolor))
}
glogger := log.NewGlogHandler(ostream)
// Set log level
glogger.Verbosity(log.Lvl(ctx.Int(verbosityFlag.Name)))
log.Root().SetHandler(glogger)
return nil
}

76
roller/cmd/roller/main.go Normal file
View File

@@ -0,0 +1,76 @@
package main
import (
"fmt"
"os"
"github.com/scroll-tech/go-ethereum/log"
"github.com/urfave/cli/v2"
"scroll-tech/go-roller/config"
"scroll-tech/go-roller/roller"
)
var (
// cfgFileFlag load json type config file.
cfgFileFlag = cli.StringFlag{
Name: "config",
Usage: "TOML configuration file",
Value: "./config.toml",
}
// logFileFlag decides where the logger output is sent. If this flag is left
// empty, it will log to stdout.
logFileFlag = cli.StringFlag{
Name: "logfile",
Usage: "Tells the sequencer where to write log entries",
}
// verbosityFlag log level.
verbosityFlag = cli.IntFlag{
Name: "verbosity",
Usage: "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail",
Value: 3,
}
)
func main() {
app := cli.NewApp()
app.Action = action
app.Name = "Roller"
app.Usage = "The Scroll L2 Roller"
app.Version = "v0.0.1"
app.Flags = append(app.Flags, []cli.Flag{
&cfgFileFlag,
&logFileFlag,
&verbosityFlag,
}...)
app.Before = func(ctx *cli.Context) error {
return setup(ctx)
}
if err := app.Run(os.Args); err != nil {
_, _ = fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func action(ctx *cli.Context) error {
// Get config
cfg, err := config.InitConfig(ctx.String(cfgFileFlag.Name))
if err != nil {
return err
}
// Create roller
r, err := roller.NewRoller(cfg)
if err != nil {
return err
}
defer r.Close()
log.Info("go-roller start successful", "name", cfg.RollerName)
return r.Run()
}

9
roller/config.toml Normal file
View File

@@ -0,0 +1,9 @@
roller_name = "my_roller"
secret_key = "dcf2cbdd171a21c480aa7f53d77f31bb102282b3ff099c78e3118b37348c72f7"
scroll_url = "ws://localhost:9000"
db_path = "bbolt_db"
[prover]
mock_mode = false
params_path = "params"
seed_path = "seed"

41
roller/config/config.go Normal file
View File

@@ -0,0 +1,41 @@
package config
import (
"path/filepath"
"github.com/BurntSushi/toml"
"github.com/scroll-tech/go-ethereum/log"
)
// Config loads roller configuration items.
type Config struct {
RollerName string `toml:"roller_name"`
SecretKey string `toml:"secret_key"`
ScrollURL string `toml:"scroll_url"`
Prover *ProverConfig `toml:"prover"`
DBPath string `toml:"db_path"`
}
// ProverConfig loads zk roller configuration items.
type ProverConfig struct {
MockMode bool `toml:"mock_mode"`
ParamsPath string `toml:"params_path"`
SeedPath string `toml:"seed_path"`
}
// InitConfig inits config from file.
func InitConfig(path string) (*Config, error) {
cfg := &Config{}
_, err := toml.DecodeFile(path, cfg)
if err != nil {
log.Error("init config failed", "error", err)
return nil, err
}
if !filepath.IsAbs(cfg.DBPath) {
if cfg.DBPath, err = filepath.Abs(cfg.DBPath); err != nil {
log.Error("Failed to get abs path", "error", err)
return nil, err
}
}
return cfg, nil
}

81
roller/go.mod Normal file
View File

@@ -0,0 +1,81 @@
module scroll-tech/go-roller
go 1.17
require (
github.com/BurntSushi/toml v1.1.0
github.com/ethereum/go-ethereum v1.10.23
github.com/gorilla/websocket v1.5.0
github.com/mattn/go-colorable v0.1.12
github.com/mattn/go-isatty v0.0.14
github.com/scroll-tech/go-ethereum v1.10.14-0.20220825064909-ff9405b2fa8c
github.com/stretchr/testify v1.7.2
github.com/urfave/cli/v2 v2.10.2
go.etcd.io/bbolt v1.3.6
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90
)
require (
github.com/StackExchange/wmi v1.2.1 // indirect
github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
github.com/btcsuite/btcd v0.22.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/deckarep/golang-set v1.8.0 // indirect
github.com/deepmap/oapi-codegen v1.8.2 // indirect
github.com/edsrzf/mmap-go v1.0.0 // indirect
github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect
github.com/fsnotify/fsnotify v1.5.4 // indirect
github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
github.com/go-kit/kit v0.12.0 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-stack/stack v1.8.0 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/go-cmp v0.5.8 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/graph-gophers/graphql-go v1.3.0 // indirect
github.com/hashicorp/go-bexpr v0.1.10 // indirect
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
github.com/holiman/uint256 v1.2.0 // indirect
github.com/huin/goupnp v1.0.3 // indirect
github.com/iden3/go-iden3-crypto v0.0.13 // indirect
github.com/influxdata/influxdb v1.8.3 // indirect
github.com/influxdata/influxdb-client-go/v2 v2.4.0 // indirect
github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
github.com/julienschmidt/httprouter v1.3.0 // indirect
github.com/kr/pretty v0.3.0 // indirect
github.com/mattn/go-runewidth v0.0.9 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mitchellh/pointerstructure v1.2.0 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/onsi/ginkgo v1.16.4 // indirect
github.com/onsi/gomega v1.18.1 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/tsdb v0.7.1 // indirect
github.com/rjeczalik/notify v0.9.1 // indirect
github.com/rogpeppe/go-internal v1.8.1 // indirect
github.com/rs/cors v1.8.2 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 // indirect
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
github.com/tklauser/go-sysconf v0.3.10 // indirect
github.com/tklauser/numcpus v0.4.0 // indirect
github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
gopkg.in/urfave/cli.v1 v1.20.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

1127
roller/go.sum Normal file

File diff suppressed because it is too large Load Diff

110
roller/mock/mock_test.go Normal file
View File

@@ -0,0 +1,110 @@
package mock
import (
"encoding/json"
"fmt"
"math/rand"
"net/http"
"os"
"path/filepath"
"testing"
"time"
"github.com/ethereum/go-ethereum/crypto/secp256k1"
"github.com/gorilla/websocket"
"github.com/scroll-tech/go-ethereum/common"
"github.com/stretchr/testify/assert"
"scroll-tech/go-roller/config"
"scroll-tech/go-roller/roller"
. "scroll-tech/go-roller/types"
)
var (
cfg *config.Config
scrollPort = 9020
mockPath string
)
func TestMain(m *testing.M) {
mockPath = "/tmp/roller_mock_test"
_ = os.RemoveAll(mockPath)
if err := os.Mkdir(mockPath, os.ModePerm); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
scrollPort = rand.Intn(9000)
cfg = &config.Config{
RollerName: "test-roller",
SecretKey: "dcf2cbdd171a21c480aa7f53d77f31bb102282b3ff099c78e3118b37348c72f7",
ScrollURL: fmt.Sprintf("ws://localhost:%d", scrollPort),
Prover: &config.ProverConfig{MockMode: true},
DBPath: filepath.Join(mockPath, "stack_db"),
}
os.Exit(m.Run())
}
func TestRoller(t *testing.T) {
go mockScroll(t)
r, err := roller.NewRoller(cfg)
assert.NoError(t, err)
go r.Run()
<-time.NewTimer(2 * time.Second).C
r.Close()
}
func mockScroll(t *testing.T) {
http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
up := websocket.Upgrader{}
c, err := up.Upgrade(w, req, nil)
assert.NoError(t, err)
var payload []byte
payload, err = func(c *websocket.Conn) ([]byte, error) {
for {
var mt int
mt, payload, err = c.ReadMessage()
if err != nil {
return nil, err
}
if mt == websocket.BinaryMessage {
return payload, nil
}
}
}(c)
assert.NoError(t, err)
msg := &Msg{}
err = json.Unmarshal(payload, msg)
assert.NoError(t, err)
authMsg := &AuthMessage{}
err = json.Unmarshal(msg.Payload, authMsg)
assert.NoError(t, err)
// Verify signature
hash, err := authMsg.Identity.Hash()
assert.NoError(t, err)
if !secp256k1.VerifySignature(common.FromHex(authMsg.Identity.PublicKey), hash, common.FromHex(authMsg.Signature)[:64]) {
assert.NoError(t, err)
}
t.Log("signature verification successful. Roller: ", authMsg.Identity.Name)
assert.Equal(t, cfg.RollerName, authMsg.Identity.Name)
traces := &BlockTraces{
ID: 16,
Traces: nil,
}
msgByt, err := roller.MakeMsgByt(BlockTrace, traces)
assert.NoError(t, err)
err = c.WriteMessage(websocket.BinaryMessage, msgByt)
assert.NoError(t, err)
})
http.ListenAndServe(fmt.Sprintf(":%d", scrollPort), nil)
}

View File

@@ -0,0 +1,2 @@
init_prover(char *params_path, char *seed_path);
char* create_agg_proof(char *trace);

View File

@@ -0,0 +1,71 @@
//nolint:typecheck
package prover
/*
#cgo LDFLAGS: ./roller/prover/lib/libprover.a -lm -ldl
#cgo gpu LDFLAGS: ./roller/prover/lib/libprover.a -lm -ldl -lgmp -lstdc++ -lprocps -L/usr/local/cuda/lib64/ -lcudart
#include <stdlib.h>
#include "./lib/prover.h"
*/
import "C"
import (
"encoding/json"
"unsafe"
"scroll-tech/go-roller/config"
"github.com/scroll-tech/go-ethereum/core/types"
. "scroll-tech/go-roller/types"
)
// Prover sends block-traces to rust-prover through socket and get back the zk-proof.
type Prover struct {
cfg *config.ProverConfig
}
// NewProver inits a Prover object.
func NewProver(cfg *config.ProverConfig) (*Prover, error) {
if cfg.MockMode {
return &Prover{cfg: cfg}, nil
}
paramsPathStr := C.CString(cfg.ParamsPath)
seedPathStr := C.CString(cfg.SeedPath)
defer func() {
C.free(unsafe.Pointer(paramsPathStr))
C.free(unsafe.Pointer(seedPathStr))
}()
C.init_prover(paramsPathStr, seedPathStr)
return &Prover{cfg: cfg}, nil
}
// Prove call rust ffi to generate proof, if first failed, try again.
func (p *Prover) Prove(traces *types.BlockResult) (*AggProof, error) {
proof, err := p.prove(traces)
if err != nil {
return p.prove(traces)
}
return proof, nil
}
func (p *Prover) prove(traces *types.BlockResult) (*AggProof, error) {
if p.cfg.MockMode {
return &AggProof{}, nil
}
tracesByt, err := json.Marshal(traces)
if err != nil {
return nil, err
}
tracesStr := C.CString(string(tracesByt))
defer func() {
C.free(unsafe.Pointer(tracesStr))
}()
cProof := C.create_agg_proof(tracesStr)
proof := C.GoString(cProof)
zkProof := &AggProof{}
err = json.Unmarshal([]byte(proof), zkProof)
return zkProof, err
}

View File

@@ -0,0 +1,47 @@
package prover_test
import (
"encoding/json"
"io/ioutil"
"os"
"testing"
"scroll-tech/go-roller/config"
"scroll-tech/go-roller/roller/prover"
"github.com/scroll-tech/go-ethereum/core/types"
"github.com/stretchr/testify/assert"
)
type RpcTrace struct {
Jsonrpc string `json:"jsonrpc"`
ID int64 `json:"id"`
Result *types.BlockResult `json:"result"`
}
func TestFFI(t *testing.T) {
if os.Getenv("TEST_FFI") != "true" {
return
}
as := assert.New(t)
cfg := &config.ProverConfig{
MockMode: false,
ParamsPath: "../../assets/test_params",
SeedPath: "../../assets/test_seed",
}
prover, err := prover.NewProver(cfg)
as.NoError(err)
f, err := os.Open("../../assets/trace.json")
as.NoError(err)
byt, err := ioutil.ReadAll(f)
as.NoError(err)
rpcTrace := &RpcTrace{}
as.NoError(json.Unmarshal(byt, rpcTrace))
_, err = prover.Prove(rpcTrace.Result)
as.NoError(err)
t.Log("prove success")
}

3535
roller/roller/prover/rust/Cargo.lock generated Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,35 @@
[package]
name = "prover"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[patch.crates-io]
# This fork makes bitvec 0.20.x work with funty 1.1 and funty 1.2. Without
# this fork, bitvec 0.20.x is incompatible with funty 1.2, which we depend on,
# and leads to a compilation error. This can be removed once the upstream PR
# is resolved: https://github.com/bitvecto-rs/bitvec/pull/141
bitvec = { git = "https://github.com/ed255/bitvec.git", rev = "5cfc5fa8496c66872d21905e677120fc3e79693c" }
[lib]
crate-type = ["staticlib", "cdylib"]
[dependencies]
zkevm = { git = "https://github.com/scroll-tech/common-rs", rev = "4f76d52004dd87a" }
types = { git = "https://github.com/scroll-tech/common-rs", rev = "4f76d52004dd87a" }
log = "0.4"
serde = "1.0"
serde_derive = "1.0"
serde_json = "1.0.66"
libc = "0.2"
once_cell = "1.8.0"
[profile.test]
opt-level = 3
debug-assertions = true
[profile.release]
opt-level = 3
debug-assertions = true

View File

@@ -0,0 +1 @@
nightly-2022-08-23

View File

@@ -0,0 +1,22 @@
#![feature(once_cell)]
pub mod prove;
pub(crate) mod utils {
use std::ffi::{CStr, CString};
use std::os::raw::c_char;
pub(crate) fn c_char_to_str(c: *const c_char) -> &'static str {
let cstr = unsafe { CStr::from_ptr(c) };
cstr.to_str().unwrap()
}
pub(crate) fn c_char_to_vec(c: *const c_char) -> Vec<u8> {
let cstr = unsafe { CStr::from_ptr(c) };
cstr.to_bytes().to_vec()
}
pub(crate) fn vec_to_c_char(bytes: Vec<u8>) -> *const c_char {
CString::new(bytes).unwrap().into_raw()
}
}

View File

@@ -0,0 +1,35 @@
use crate::utils::{c_char_to_str, c_char_to_vec, vec_to_c_char};
use libc::c_char;
use std::cell::OnceCell;
use types::eth::BlockResult;
use zkevm::circuit::AGG_DEGREE;
use zkevm::utils::{load_or_create_params, load_or_create_seed};
use zkevm::{circuit::DEGREE, prover::Prover};
static mut PROVER: OnceCell<Prover> = OnceCell::new();
/// # Safety
#[no_mangle]
pub unsafe extern "C" fn init_prover(params_path: *const c_char, seed_path: *const c_char) {
let params_path = c_char_to_str(params_path);
let seed_path = c_char_to_str(seed_path);
let params = load_or_create_params(params_path, *DEGREE).unwrap();
let agg_params = load_or_create_params(params_path, *AGG_DEGREE).unwrap();
let seed = load_or_create_seed(seed_path).unwrap();
let p = Prover::from_params_and_seed(params, agg_params, seed);
PROVER.set(p).unwrap();
}
/// # Safety
#[no_mangle]
pub unsafe extern "C" fn create_agg_proof(trace_char: *const c_char) -> *const c_char {
let trace_vec = c_char_to_vec(trace_char);
let trace = serde_json::from_slice::<BlockResult>(&trace_vec).unwrap();
let proof = PROVER
.get_mut()
.unwrap()
.create_agg_circuit_proof(&trace)
.unwrap();
let proof_bytes = serde_json::to_vec(&proof).unwrap();
vec_to_c_char(proof_bytes)
}

277
roller/roller/roller.go Normal file
View File

@@ -0,0 +1,277 @@
package roller
import (
"encoding/json"
"errors"
"fmt"
"strings"
"sync/atomic"
"time"
"github.com/gorilla/websocket"
"github.com/scroll-tech/go-ethereum/common"
"github.com/scroll-tech/go-ethereum/crypto"
"github.com/scroll-tech/go-ethereum/log"
"scroll-tech/go-roller/config"
"scroll-tech/go-roller/roller/prover"
"scroll-tech/go-roller/store"
. "scroll-tech/go-roller/types"
)
var (
writeWait = time.Second + readWait
// consider ping message
readWait = time.Minute * 30
// retry scroll
retryWait = time.Second * 10
// net normal close
errNormalClose = errors.New("use of closed network connection")
)
// Roller contains websocket conn to Scroll, Stack, unix-socket to ipc-prover.
type Roller struct {
cfg *config.Config
conn *websocket.Conn
stack *store.Stack
prover *prover.Prover
isClosed int64
stopChan chan struct{}
}
// NewRoller new a Roller object.
func NewRoller(cfg *config.Config) (*Roller, error) {
// Get stack db handler
stackDb, err := store.NewStack(cfg.DBPath)
if err != nil {
return nil, err
}
// Create prover instance
log.Info("init prover")
pver, err := prover.NewProver(cfg.Prover)
if err != nil {
return nil, err
}
log.Info("init prover successfully!")
conn, _, err := websocket.DefaultDialer.Dial(cfg.ScrollURL, nil)
if err != nil {
return nil, err
}
return &Roller{
cfg: cfg,
conn: conn,
stack: stackDb,
prover: pver,
stopChan: make(chan struct{}),
}, nil
}
// Run runs Roller.
func (r *Roller) Run() error {
log.Info("start to register to scroll")
if err := r.Register(); err != nil {
log.Crit("register to scroll failed", "error", err)
}
log.Info("register to scroll successfully!")
go func() {
r.HandleScroll()
r.Close()
}()
return r.ProveLoop()
}
// Register registers Roller to the Scroll through Websocket.
func (r *Roller) Register() error {
priv, err := crypto.HexToECDSA(r.cfg.SecretKey)
if err != nil {
return fmt.Errorf("generate private-key failed %v", err)
}
authMsg := &AuthMessage{
Identity: Identity{
Name: r.cfg.RollerName,
Timestamp: time.Now().UnixMilli(),
PublicKey: common.Bytes2Hex(crypto.FromECDSAPub(&priv.PublicKey)),
},
Signature: "",
}
// Sign auth message
if err = authMsg.Sign(priv); err != nil {
return fmt.Errorf("Sign auth message failed %v", err)
}
msgByt, err := MakeMsgByt(Register, authMsg)
if err != nil {
return err
}
return r.conn.WriteMessage(websocket.BinaryMessage, msgByt)
}
// HandleScroll accepts block-traces from Scroll through the Websocket and store it into Stack.
func (r *Roller) HandleScroll() {
for {
select {
case <-r.stopChan:
return
default:
_ = r.conn.SetWriteDeadline(time.Now().Add(writeWait))
_ = r.conn.SetReadDeadline(time.Now().Add(readWait))
if err := r.handMessage(); err != nil && !strings.Contains(err.Error(), errNormalClose.Error()) {
log.Error("handle scroll failed", "error", err)
r.mustRetryScroll()
continue
}
}
}
}
func (r *Roller) mustRetryScroll() {
for {
log.Info("retry to connect to scroll...")
conn, _, err := websocket.DefaultDialer.Dial(r.cfg.ScrollURL, nil)
if err != nil {
log.Error("failed to connect scroll: ", "error", err)
time.Sleep(retryWait)
} else {
r.conn = conn
log.Info("re-connect to scroll successfully!")
break
}
}
for {
log.Info("retry to register to scroll...")
err := r.Register()
if err != nil {
log.Error("register to scroll failed", "error", err)
time.Sleep(retryWait)
} else {
log.Info("re-register to scroll successfully!")
break
}
}
}
// ProveLoop keep popping the block-traces from Stack and sends it to rust-prover for loop.
func (r *Roller) ProveLoop() (err error) {
for {
select {
case <-r.stopChan:
return nil
default:
_ = r.conn.SetWriteDeadline(time.Now().Add(writeWait))
if err = r.prove(); err != nil {
if errors.Is(err, store.ErrEmpty) {
log.Debug("get empty trace", "error", err)
time.Sleep(time.Second * 3)
continue
}
if strings.Contains(err.Error(), errNormalClose.Error()) {
return nil
}
log.Error("prove failed", "error", err)
}
}
}
}
func (r *Roller) handMessage() error {
mt, msg, err := r.conn.ReadMessage()
if err != nil {
return err
}
switch mt {
case websocket.BinaryMessage:
if err = r.persistTrace(msg); err != nil {
return err
}
}
return nil
}
func (r *Roller) prove() error {
traces, err := r.stack.Pop()
if err != nil {
return err
}
log.Info("start to prove block", "block-id", traces.ID)
var proofMsg *ProofMsg
proof, err := r.prover.Prove(traces.Traces)
if err != nil {
proofMsg = &ProofMsg{
Status: StatusProofError,
Error: err.Error(),
ID: traces.ID,
Proof: &AggProof{},
}
log.Error("prove block failed!", "block-id", traces.ID)
} else {
proofMsg = &ProofMsg{
Status: StatusOk,
ID: traces.ID,
Proof: proof,
}
log.Info("prove block successfully!", "block-id", traces.ID)
}
msgByt, err := MakeMsgByt(Proof, proofMsg)
if err != nil {
return err
}
return r.conn.WriteMessage(websocket.BinaryMessage, msgByt)
}
// Close closes the websocket connection.
func (r *Roller) Close() {
if atomic.LoadInt64(&r.isClosed) == 1 {
return
}
atomic.StoreInt64(&r.isClosed, 1)
close(r.stopChan)
// Close scroll's ws
_ = r.conn.Close()
// Close db
if err := r.stack.Close(); err != nil {
log.Error("failed to close bbolt db", "error", err)
}
}
func (r *Roller) persistTrace(byt []byte) error {
var msg = &Msg{}
err := json.Unmarshal(byt, msg)
if err != nil {
return err
}
if msg.Type != BlockTrace {
log.Error("message from Scroll illegal")
return nil
}
var traces = &BlockTraces{}
if err := json.Unmarshal(msg.Payload, traces); err != nil {
return err
}
log.Info("Accept BlockTrace from Scroll", "ID", traces.ID)
return r.stack.Push(traces)
}
// MakeMsgByt Marshals Msg to bytes.
func MakeMsgByt(msgTyp MsgType, payloadVal interface{}) ([]byte, error) {
payload, err := json.Marshal(payloadVal)
if err != nil {
return nil, err
}
msg := &Msg{
Type: msgTyp,
Payload: payload,
}
return json.Marshal(msg)
}

73
roller/store/stack.go Normal file
View File

@@ -0,0 +1,73 @@
package store
import (
"encoding/binary"
"encoding/json"
"errors"
"github.com/scroll-tech/go-ethereum/log"
"go.etcd.io/bbolt"
rollertypes "scroll-tech/go-roller/types"
)
var (
// ErrEmpty empty error message
ErrEmpty = errors.New("content is empty")
)
// Stack is a first-input last-output db.
type Stack struct {
*bbolt.DB
}
var bucket = []byte("stack")
// NewStack new a Stack object.
func NewStack(path string) (*Stack, error) {
kvdb, err := bbolt.Open(path, 0666, nil)
if err != nil {
return nil, err
}
err = kvdb.Update(func(tx *bbolt.Tx) error {
_, err = tx.CreateBucketIfNotExists(bucket)
return err
})
if err != nil {
log.Crit("init stack failed", "error", err)
}
return &Stack{DB: kvdb}, nil
}
// Push appends the block-traces on the top of Stack.
func (s *Stack) Push(traces *rollertypes.BlockTraces) error {
byt, err := json.Marshal(traces)
if err != nil {
return err
}
key := make([]byte, 8)
binary.BigEndian.PutUint64(key, traces.ID)
return s.Update(func(tx *bbolt.Tx) error {
return tx.Bucket(bucket).Put(key, byt)
})
}
// Pop pops the block-traces on the top of Stack.
func (s *Stack) Pop() (*rollertypes.BlockTraces, error) {
var value []byte
if err := s.Update(func(tx *bbolt.Tx) error {
var key []byte
bu := tx.Bucket(bucket)
c := bu.Cursor()
key, value = c.Last()
return bu.Delete(key)
}); err != nil {
return nil, err
}
if len(value) == 0 {
return nil, ErrEmpty
}
traces := &rollertypes.BlockTraces{}
return traces, json.Unmarshal(value, traces)
}

View File

@@ -0,0 +1,39 @@
package store
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
. "scroll-tech/go-roller/types"
)
func TestStack(t *testing.T) {
// Create temp path
path, err := ioutil.TempDir("/tmp/", "stack_db_test-")
assert.NoError(t, err)
defer os.RemoveAll(path)
// Create stack db instance
s, err := NewStack(filepath.Join(path, "test-stack"))
assert.NoError(t, err)
defer s.Close()
for i := 0; i < 3; i++ {
trace := &BlockTraces{
ID: uint64(i),
Traces: nil,
}
err := s.Push(trace)
assert.NoError(t, err)
}
for i := 2; i >= 0; i-- {
trace, err := s.Pop()
assert.NoError(t, err)
assert.Equal(t, uint64(i), trace.ID)
}
}

123
roller/types/types.go Normal file
View File

@@ -0,0 +1,123 @@
package message
import (
"crypto/ecdsa"
"encoding/json"
"github.com/scroll-tech/go-ethereum/common"
"github.com/scroll-tech/go-ethereum/core/types"
"github.com/scroll-tech/go-ethereum/crypto"
"golang.org/x/crypto/blake2s"
)
// MsgType denotes the type of message being sent or received.
type MsgType uint16
const (
// Error message.
Error MsgType = iota
// Register message, sent by a roller when a connection is established.
Register
// BlockTrace message, sent by a sequencer to a roller to notify them
// they need to generate a proof.
BlockTrace
// Proof message, sent by a roller to a sequencer when they have finished
// proof generation of a given set of block traces.
Proof
)
// RespStatus represents status code from roller to scroll
type RespStatus uint32
const (
// StatusOk means generate proof success
StatusOk RespStatus = iota
// StatusProofError means generate proof failed
StatusProofError
)
// Msg is the top-level message container which contains the payload and the
// message identifier.
type Msg struct {
// Message type
Type MsgType `json:"type"`
// Message payload
Payload []byte `json:"payload"`
}
// AuthMessage is the first message exchanged from the Roller to the Sequencer.
// It effectively acts as a registration, and makes the Roller identification
// known to the Sequencer.
type AuthMessage struct {
// Message fields
Identity Identity `json:"message"`
// Roller signature
Signature string `json:"signature"`
}
// Sign auth message
func (a *AuthMessage) Sign(priv *ecdsa.PrivateKey) error {
// Hash identity content
hash, err := a.Identity.Hash()
if err != nil {
return err
}
// Sign register message
sig, err := crypto.Sign(hash, priv)
if err != nil {
return err
}
a.Signature = common.Bytes2Hex(sig)
return nil
}
// Identity contains all the fields to be signed by the roller.
type Identity struct {
// Roller name
Name string `json:"name"`
// Time of message creation
Timestamp int64 `json:"timestamp"`
// Roller public key
PublicKey string `json:"publicKey"`
}
// Hash returns the hash of the auth message, which should be the message used
// to construct the Signature.
func (i *Identity) Hash() ([]byte, error) {
bs, err := json.Marshal(i)
if err != nil {
return nil, err
}
hash := blake2s.Sum256(bs)
return hash[:], nil
}
// BlockTraces is a wrapper type around types.BlockResult which adds an ID
// that identifies which proof generation session these block traces are
// associated to. This then allows the roller to add the ID back to their
// proof message once generated, and in turn helps the sequencer understand
// where to handle the proof.
type BlockTraces struct {
ID uint64 `json:"id"`
Traces *types.BlockResult `json:"blockTraces"`
}
// ProofMsg is the message received from rollers that contains zk proof, the status of
// the proof generation succeeded, and an error message if proof generation failed.
type ProofMsg struct {
Status RespStatus `json:"status"`
Error string `json:"error,omitempty"`
ID uint64 `json:"id"`
// FIXME: Maybe we need to allow Proof omitempty? Also in scroll.
Proof *AggProof `json:"proof"`
}
// AggProof includes the proof and public input that are required to verification and rollup.
type AggProof struct {
Proof []byte `json:"proof"`
Instance []byte `json:"instance"`
FinalPair []byte `json:"final_pair"`
Vk []byte `json:"vk"`
}