From 11fa61c45cdcd221521d59363a505a06d65b2335 Mon Sep 17 00:00:00 2001 From: ChuhanJin <60994121+ChuhanJin@users.noreply.github.com> Date: Fri, 9 Dec 2022 17:37:22 +0800 Subject: [PATCH] refact(orm): parallel insert and change gas_cost compute logic (#133) Co-authored-by: vincent <419436363@qq.com> --- Jenkinsfile | 2 +- bridge/l2/watcher_test.go | 2 +- common/go.mod | 2 +- common/utils/trace_gas.go | 30 ++++++++++++++++++++++++++++++ common/utils/trace_gas_test.go | 31 +++++++++++++++++++++++++++++++ database/orm/block_result.go | 6 ++++-- go.work.sum | 10 +++------- 7 files changed, 71 insertions(+), 12 deletions(-) create mode 100644 common/utils/trace_gas.go create mode 100644 common/utils/trace_gas_test.go diff --git a/Jenkinsfile b/Jenkinsfile index 9264bb2b8..6e5122367 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -29,7 +29,7 @@ pipeline { } } steps { - //start to build project + // start to build project sh '''#!/bin/bash export PATH=/home/ubuntu/go/bin:$PATH make dev_docker diff --git a/bridge/l2/watcher_test.go b/bridge/l2/watcher_test.go index 7daed6885..827a014c6 100644 --- a/bridge/l2/watcher_test.go +++ b/bridge/l2/watcher_test.go @@ -88,7 +88,7 @@ func testMonitorBridgeContract(t *testing.T) { t.Fatalf("Call failed") } - //extra block mined + // extra block mined toAddress = common.HexToAddress("0x4592d8f8d7b001e72cb26a73e4fa1806a51ac79d") message = []byte("testbridgecontract") tx, err = instance.SendMessage(auth, toAddress, message, auth.GasPrice) diff --git a/common/go.mod b/common/go.mod index 67534a3af..70ac5d82f 100644 --- a/common/go.mod +++ b/common/go.mod @@ -11,6 +11,7 @@ require ( github.com/scroll-tech/go-ethereum v1.10.14-0.20221125025612-4ea77a7577c6 github.com/stretchr/testify v1.8.0 github.com/urfave/cli/v2 v2.10.2 + golang.org/x/sync v0.1.0 ) require ( @@ -79,7 +80,6 @@ require ( golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect golang.org/x/mod v0.7.0 // indirect golang.org/x/net v0.2.0 // indirect - golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.2.0 // indirect golang.org/x/text v0.4.0 // indirect golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect diff --git a/common/utils/trace_gas.go b/common/utils/trace_gas.go new file mode 100644 index 000000000..6765b9910 --- /dev/null +++ b/common/utils/trace_gas.go @@ -0,0 +1,30 @@ +package utils + +import ( + "sync/atomic" + + "github.com/scroll-tech/go-ethereum/core/types" + "golang.org/x/sync/errgroup" +) + +// ComputeTraceGasCost computes gascost based on ExecutionResults.StructLogs.GasCost +func ComputeTraceGasCost(trace *types.BlockTrace) uint64 { + var ( + gasCost uint64 + eg errgroup.Group + ) + for idx := range trace.ExecutionResults { + i := idx + eg.Go(func() error { + var sum uint64 + for _, log := range trace.ExecutionResults[i].StructLogs { + sum += log.GasCost + } + atomic.AddUint64(&gasCost, sum) + return nil + }) + } + _ = eg.Wait() + + return gasCost +} diff --git a/common/utils/trace_gas_test.go b/common/utils/trace_gas_test.go new file mode 100644 index 000000000..aed1c434c --- /dev/null +++ b/common/utils/trace_gas_test.go @@ -0,0 +1,31 @@ +package utils_test + +import ( + "encoding/json" + "os" + "testing" + + "github.com/scroll-tech/go-ethereum/core/types" + "github.com/stretchr/testify/assert" + + "scroll-tech/common/utils" +) + +// TestComputeTraceCost test ComputeTraceGasCost function +func TestComputeTraceCost(t *testing.T) { + templateBlockTrace, err := os.ReadFile("../testdata/blockTrace_03.json") + assert.NoError(t, err) + // unmarshal blockTrace + blockTrace := &types.BlockTrace{} + err = json.Unmarshal(templateBlockTrace, blockTrace) + assert.NoError(t, err) + var sum uint64 + for _, v := range blockTrace.ExecutionResults { + for _, sv := range v.StructLogs { + sum += sv.GasCost + } + } + + res := utils.ComputeTraceGasCost(blockTrace) + assert.Equal(t, sum, res) +} diff --git a/database/orm/block_result.go b/database/orm/block_result.go index 8211bd885..1fb6d6f72 100644 --- a/database/orm/block_result.go +++ b/database/orm/block_result.go @@ -12,6 +12,8 @@ import ( "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/core/types" "github.com/scroll-tech/go-ethereum/log" + + "scroll-tech/common/utils" ) type blockTraceOrm struct { @@ -163,17 +165,17 @@ func (o *blockTraceOrm) InsertBlockTraces(ctx context.Context, blockTraces []*ty log.Error("failed to marshal blockTrace", "hash", hash, "err", err) return err } + gas_cost := utils.ComputeTraceGasCost(trace) traceMaps[i] = map[string]interface{}{ "number": number, "hash": hash, "parent_hash": trace.Header.ParentHash.String(), "trace": string(data), "tx_num": tx_num, - "gas_used": trace.Header.GasUsed, + "gas_used": gas_cost, "block_timestamp": mtime, } } - _, err := o.db.NamedExec(`INSERT INTO public.block_trace (number, hash, parent_hash, trace, tx_num, gas_used, block_timestamp) VALUES (:number, :hash, :parent_hash, :trace, :tx_num, :gas_used, :block_timestamp);`, traceMaps) if err != nil { log.Error("failed to insert blockTraces", "err", err) diff --git a/go.work.sum b/go.work.sum index 7e29ac0c9..8a9dfeec4 100644 --- a/go.work.sum +++ b/go.work.sum @@ -20,6 +20,7 @@ github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSW github.com/Azure/go-autorest/autorest/mocks v0.3.0 h1:qJumjCaCudz+OcqE9/XtEPfvtOjOmKaui4EOpFI6zZc= github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= +github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= github.com/ClickHouse/clickhouse-go/v2 v2.2.0 h1:dj00TDKY+xwuTJdbpspCSmTLFyWzRJerTHwaBxut1C0= github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08= @@ -188,16 +189,11 @@ golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20191227195350-da58074b4299 h1:zQpM52jfKHG6II1ISZY1ZcpygvuSFZpLwfluuF89XOg= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= -golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=