staterecovery: fix cli options regression (#567)

* staterecovery: fix cli options regression & add CI jobs
This commit is contained in:
Pedro Novais
2025-01-20 11:56:25 +00:00
committed by GitHub
parent 6c8a282743
commit d01ef99a5a
61 changed files with 430 additions and 505 deletions

View File

@@ -15,6 +15,7 @@ jobs:
name: Filter commit changes
outputs:
coordinator: ${{ steps.filter.outputs.coordinator }}
staterecovery: ${{ steps.filter.outputs.staterecovery }}
postman: ${{ steps.filter.outputs.postman }}
prover: ${{ steps.filter.outputs.prover }}
traces-api-facade: ${{ steps.filter.outputs.traces-api-facade }}
@@ -50,6 +51,18 @@ jobs:
- 'docker/compose.yml'
- 'docker/compose-local-dev.overrides.yml'
- 'docker/compose-local-dev-traces-v2.overrides.yml'
staterecovery:
- 'state-recovery/**'
- 'buildSrc/**'
- 'jvm-libs/**'
- 'gradle/**'
- 'build.gradle'
- 'gradle.properties'
- 'settings.gradle'
- '.github/workflows/staterecovery-*.yml'
- '.github/workflows/main.yml'
- '.github/workflows/reuse-*.yml'
- 'e2e/**'
postman:
- 'postman/**'
- 'sdk/**'
@@ -172,6 +185,7 @@ jobs:
with:
commit_tag: ${{ needs.store-image-name-and-tags.outputs.commit_tag }}
coordinator_changed: ${{ needs.filter-commit-changes.outputs.coordinator }}
staterecovery_changed: ${{ needs.filter-commit-changes.outputs.staterecovery }}
postman_changed: ${{ needs.filter-commit-changes.outputs.postman }}
prover_changed: ${{ needs.filter-commit-changes.outputs.prover }}
traces_api_facade_changed: ${{ needs.filter-commit-changes.outputs.traces-api-facade }}

View File

@@ -0,0 +1,87 @@
name: staterecovery-testing
on:
workflow_call:
inputs:
commit_tag:
required: true
type: string
secrets:
DOCKERHUB_USERNAME:
required: false
DOCKERHUB_TOKEN:
required: false
concurrency:
group: staterecovery-testing-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
jobs:
cache-docker-images:
uses: ./.github/workflows/cache-docker-images.yml
secrets: inherit
run-tests:
env:
COMMIT_TAG: ${{ inputs.commit_tag }}
GITHUB_TOKEN: ${{ secrets._GITHUB_TOKEN_RELEASE_ACCESS }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-large
name: Staterecovery tests
steps:
- name: Checkout
uses: actions/checkout@v4
- uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b #v4.5.0
with:
distribution: temurin
java-version: 21
- name: Setup Gradle
# Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies.
# See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md
uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1
- name: Restore cached images
id: restore-cached-images
uses: actions/cache/restore@v4.0.2
with:
path: ~/docker-images
key: cached-images
restore-keys: |
cached-images
- name: Staterecovery - Build and Unit tests
run: |
./gradlew state-recovery:besu-plugin:buildNeeded
# Install pnpm to deploy smart contracts
# FIXME: use web3j to deploy contracts and remove this.
- name: Setup nodejs environment
uses: ./.github/actions/setup-nodejs
with:
pnpm-install-options: '--frozen-lockfile --prefer-offline --filter contracts --ignore-scripts'
- name: Staterecovery - Build plugin shadowJar
run: |
./gradlew state-recovery:besu-plugin:shadowJar
- name: Run integration tests
timeout-minutes: 15
run: |
./gradlew state-recovery:test-cases:integrationTest
- name: Run Jacoco
run: |
./gradlew jacocoRootReport
- name: Upload Jacoco test coverage report
uses: actions/upload-artifact@v4
with:
name: jacocoRootReport-${{ env.COMMIT_TAG }}.xml
if-no-files-found: error
path: |
${{ github.workspace }}/build/reports/jacoco/jacocoRootReport/jacocoRootReport.xml
- name: Upload coverage to Codecov
if: ${{ env.CODECOV_TOKEN != '' }}
uses: codecov/codecov-action@v5
with:
fail_ci_if_error: true
files: ${{ github.workspace }}/build/reports/jacoco/jacocoRootReport/jacocoRootReport.xml
flags: kotlin
os: linux
name: codecov-staterecovery
verbose: true
token: ${{ secrets.CODECOV_TOKEN }}

View File

@@ -9,6 +9,9 @@ on:
coordinator_changed:
required: true
type: string
staterecovery_changed:
required: true
type: string
postman_changed:
required: true
type: string
@@ -47,11 +50,18 @@ jobs:
transaction-exclusion-api:
uses: ./.github/workflows/transaction-exclusion-api-testing.yml
if: ${{ inputs.transaction_exclusion_api_changed == 'true' }}
staterecovery:
uses: ./.github/workflows/staterecovery-testing.yml
if: ${{ inputs.staterecovery_changed == 'true' }}
with:
commit_tag: ${{ inputs.commit_tag }}
secrets: inherit
# If all jobs are skipped, the workflow will still succeed.
always_succeed:
runs-on: ubuntu-24.04
if: ${{ inputs.coordinator_changed == 'false' && inputs.prover_changed == 'false' && inputs.postman_changed == 'false' && inputs.traces_api_facade_changed == 'false' && inputs.transaction_exclusion_api_changed == 'false' }}
steps:
- name: Ensure Workflow Success
run: echo "All jobs were skipped, but workflow succeeds."
run: echo "All jobs were skipped, but workflow succeeds."

View File

@@ -28,7 +28,7 @@ clean-testnet-folders:
rm -rf tmp/testnet/*
clean-environment:
docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 --profile debug --profile staterecover down || true
docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 --profile debug --profile staterecovery kill -s 9 || true
make clean-local-folders
docker network prune -f
docker volume rm linea-local-dev linea-logs || true # ignore failure if volumes do not exist already
@@ -47,7 +47,7 @@ start-l2-blockchain-only:
start-whole-environment: COMPOSE_PROFILES:=l1,l2
start-whole-environment:
# docker compose -f docker/compose.yml -f docker/compose-local-dev.overrides.yml build prover
COMPOSE_PROFILES=$(COMPOSE_PROFILES) docker compose -f docker/compose.yml -f docker/compose-local-dev.overrides.yml up -d
L1_GENESIS_TIME=$(get_future_time) COMPOSE_PROFILES=$(COMPOSE_PROFILES) docker compose -f docker/compose.yml -f docker/compose-local-dev.overrides.yml up -d
start-whole-environment-traces-v2: COMPOSE_PROFILES:=l1,l2
@@ -218,17 +218,22 @@ deploy-contracts-minimal:
cd .. && \
$(MAKE) -j6 deploy-linea-rollup-v$(L1_CONTRACT_VERSION) deploy-l2messageservice
fresh-start-all-staterecover: COMPOSE_PROFILES:=l1,l2,staterecover
fresh-start-all-staterecover: L1_CONTRACT_VERSION:=6
fresh-start-all-staterecover:
fresh-start-all-staterecovery: COMPOSE_PROFILES:=l1,l2,staterecovery
fresh-start-all-staterecovery: L1_CONTRACT_VERSION:=6
fresh-start-all-staterecovery:
make clean-environment
L1_GENESIS_TIME=$(get_future_time) make start-whole-environment-traces-v2 COMPOSE_PROFILES=$(COMPOSE_PROFILES)
$(MAKE) deploy-contracts-minimal L1_CONTRACT_VERSION=$(L1_CONTRACT_VERSION)
fresh-start-staterecover-for-replay-only: COMPOSE_PROFILES:=l1,staterecover
fresh-start-staterecover-for-replay-only:
make clean-environment
L1_GENESIS_TIME=$(get_future_time) make start-whole-environment-traces-v2 COMPOSE_PROFILES=$(COMPOSE_PROFILES)
fresh-start-staterecovery-for-replay-only: COMPOSE_PROFILES:=l1,staterecovery
fresh-start-staterecovery-for-replay-only:
make clean-environment
L1_GENESIS_TIME=$(get_future_time) make start-whole-environment-traces-v2 COMPOSE_PROFILES=$(COMPOSE_PROFILES)
staterecovery-replay-from-genesis: L1_ROLLUP_CONTRACT_ADDRESS:=0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9
staterecovery-replay-from-genesis:
docker compose -f docker/compose.yml down zkbesu-shomei-sr shomei-sr
L1_ROLLUP_CONTRACT_ADDRESS=$(L1_ROLLUP_CONTRACT_ADDRESS) docker compose -f docker/compose.yml up zkbesu-shomei-sr shomei-sr -d
testnet-start-l2:
docker compose -f docker/compose.yml -f docker/compose-testnet-sync.overrides.yml --profile l2 up -d
@@ -291,14 +296,3 @@ restart-coordinator:
make stop-coordinator
make start-coordinator
start-traces-api:
mkdir -p tmp/local/logs
mkdir -p tmp/local/traces/raw
./gradlew traces-api:app:run > tmp/local/logs/traces-app.log & echo "$$!" > tmp/local/traces-app.pid
stop-traces-api:
make stop_pid PID_FILE=tmp/local/traces-app.pid
restart-traces-api:
make stop-traces-api
make start-traces-api

View File

@@ -186,7 +186,6 @@ dockerCompose {
"l1-node-genesis-generator",
"l1-el-node",
"l1-cl-node",
"l2-node",
"blobscan-api",
"blobscan-indexer",
"redis",
@@ -200,7 +199,7 @@ dockerCompose {
"--profile",
"l2",
"--profile",
"staterecover"
"staterecovery"
]
useComposeFiles = [
"${project.rootDir.path}/docker/compose.yml",

View File

@@ -11,8 +11,11 @@ import java.math.BigInteger
// this class is mainly intended to be used for testing purposes
class StaticGasProvider(
private val _chainId: Long,
// setting default high values because
// tests suite sends loads of Tx and blobs, causes spikes in gas prices
private val maxFeePerGas: ULong = 22uL.gwei,
private val maxPriorityFeePerGas: ULong = 20uL.gwei,
private val maxFeePerBlobGas: ULong = 1000uL.gwei,
private val gasLimit: ULong = 30_000_000uL
) : AtomicContractEIP1559GasProvider, EIP4844GasProvider {
override fun getEIP1559GasFees(): EIP1559GasFees {
@@ -54,6 +57,6 @@ class StaticGasProvider(
}
override fun getEIP4844GasFees(): EIP4844GasFees {
return EIP4844GasFees(getEIP1559GasFees(), maxFeePerGas)
return EIP4844GasFees(getEIP1559GasFees(), maxFeePerBlobGas)
}
}

View File

@@ -0,0 +1,78 @@
package linea.testing
import net.consensys.linea.async.toSafeFuture
import net.consensys.linea.testing.filesystem.getPathTo
import org.apache.logging.log4j.LogManager
import tech.pegasys.teku.infrastructure.async.SafeFuture
import java.io.BufferedReader
import java.io.File
import java.io.InputStreamReader
import kotlin.time.Duration
import kotlin.time.Duration.Companion.minutes
data class CommandResult(
val exitCode: Int,
val stdOut: List<String>,
val stdErr: List<String>
)
object Runner {
fun executeCommand(
command: String,
envVars: Map<String, String> = emptyMap(),
executionDir: File = getPathTo("Makefile").parent.toFile(),
timeout: Duration = 1.minutes
): SafeFuture<CommandResult> {
val log = LogManager.getLogger("net.consensys.zkevm.ethereum.CommandExecutor")
val processBuilder = ProcessBuilder("/bin/sh", "-c", command)
processBuilder.directory(executionDir)
// Set environment variables
val env = processBuilder.environment()
for ((key, value) in envVars) {
env[key] = value
}
val process = processBuilder.start()
val stdOutReader = BufferedReader(InputStreamReader(process.inputStream))
val stdErrorReader = BufferedReader(InputStreamReader(process.errorStream))
// Read the standard output
log.debug(
"going to execute command: dir='{}', command='{}', envVars={} commandProcessId={} processInfo={}",
executionDir,
command,
envVars,
process.pid(),
process.info()
)
process.waitFor(timeout.inWholeMilliseconds, java.util.concurrent.TimeUnit.MILLISECONDS)
val futureResult = process
.onExit()
.thenApply { processResult ->
val stdOutLines = stdOutReader.lines().toList()
val stdErrLines = stdErrorReader.lines().toList()
log.debug(
"command finished: dir='{}', command='{}', exitCode={} envVars={} processId={} threadId={}",
executionDir,
command,
processResult.exitValue(),
envVars,
ProcessHandle.current().pid(),
Thread.currentThread().threadId()
)
log.debug(
"stdout: {}",
stdOutLines.joinToString("\n")
)
log.debug(
"stderr: {}",
stdErrLines.joinToString("\n")
)
CommandResult(processResult.exitValue(), stdOutLines, stdErrLines)
}
return futureResult.toSafeFuture()
}
}

View File

@@ -3,6 +3,7 @@ package net.consensys.zkevm.ethereum
import build.linea.contract.l1.LineaContractVersion
import com.sksamuel.hoplite.ConfigLoaderBuilder
import com.sksamuel.hoplite.addFileSource
import net.consensys.gwei
import net.consensys.linea.contract.AsyncFriendlyTransactionManager
import net.consensys.linea.contract.EIP1559GasProvider
import net.consensys.linea.contract.LineaRollupAsyncFriendly
@@ -64,8 +65,9 @@ interface ContractsManager {
transactionManager: AsyncFriendlyTransactionManager,
gasProvider: ContractEIP1559GasProvider = StaticGasProvider(
L1AccountManager.chainId,
maxFeePerGas = 11_000uL,
maxPriorityFeePerGas = 10_000uL,
maxFeePerGas = 55UL.gwei,
maxPriorityFeePerGas = 50UL.gwei,
maxFeePerBlobGas = 1_000UL.gwei,
gasLimit = 1_000_000uL
),
smartContractErrors: SmartContractErrors? = null
@@ -93,8 +95,9 @@ interface ContractsManager {
transactionManager: AsyncFriendlyTransactionManager,
gasProvider: ContractEIP1559GasProvider = StaticGasProvider(
L1AccountManager.chainId,
maxFeePerGas = 11_000uL,
maxPriorityFeePerGas = 10_000uL,
maxFeePerGas = 55UL.gwei,
maxPriorityFeePerGas = 50UL.gwei,
maxFeePerBlobGas = 1_000UL.gwei,
gasLimit = 1_000_000uL
)
): LineaRollupAsyncFriendly

View File

@@ -1,79 +1,12 @@
package net.consensys.zkevm.ethereum
import build.linea.contract.l1.LineaContractVersion
import net.consensys.linea.async.toSafeFuture
import net.consensys.linea.testing.filesystem.getPathTo
import org.apache.logging.log4j.LogManager
import linea.testing.CommandResult
import linea.testing.Runner
import tech.pegasys.teku.infrastructure.async.SafeFuture
import java.io.BufferedReader
import java.io.File
import java.io.InputStreamReader
import java.util.regex.Matcher
import java.util.regex.Pattern
data class CommandResult(
val exitCode: Int,
val stdOut: List<String>,
val stdErr: List<String>
)
fun executeCommand(
command: String,
envVars: Map<String, String> = emptyMap(),
executionDir: File = getPathTo("Makefile").parent.toFile()
): SafeFuture<CommandResult> {
val log = LogManager.getLogger("net.consensys.zkevm.ethereum.CommandExecutor")
val processBuilder = ProcessBuilder("/bin/sh", "-c", command)
processBuilder.directory(executionDir)
// Set environment variables
val env = processBuilder.environment()
for ((key, value) in envVars) {
env[key] = value
}
val process = processBuilder.start()
val stdOutReader = BufferedReader(InputStreamReader(process.inputStream))
val stdErrorReader = BufferedReader(InputStreamReader(process.errorStream))
// Read the standard output
log.debug(
"going to execute command: dir='{}', command='{}', envVars={} commandProcessId={} processInfo={}",
executionDir,
command,
envVars,
process.pid(),
process.info()
)
process.waitFor(60, java.util.concurrent.TimeUnit.SECONDS)
val futureResult = process
.onExit()
.thenApply { processResult ->
val stdOutLines = stdOutReader.lines().toList()
val stdErrLines = stdErrorReader.lines().toList()
log.debug(
"command finished: dir='{}', command='{}', exitCode={} envVars={} processId={} threadId={}",
executionDir,
command,
processResult.exitValue(),
envVars,
ProcessHandle.current().pid(),
Thread.currentThread().threadId()
)
log.debug(
"stdout: {}",
stdOutLines.joinToString("\n")
)
log.debug(
"stderr: {}",
stdErrLines.joinToString("\n")
)
CommandResult(processResult.exitValue(), stdOutLines, stdErrLines)
}
return futureResult.toSafeFuture()
}
internal val lineaRollupAddressPattern = Pattern.compile(
"^contract=LineaRollup(?:.*)? deployed: address=(0x[0-9a-fA-F]{40}) blockNumber=(\\d+)"
)
@@ -112,7 +45,7 @@ private fun deployContract(
env: Map<String, String> = emptyMap(),
addressPattern: Pattern
): SafeFuture<DeployedContract> {
return executeCommand(
return Runner.executeCommand(
command = command,
envVars = env
)

View File

@@ -334,7 +334,7 @@ services:
image: postgres:16.0
hostname: postgres
container_name: postgres
profiles: [ "l2", "debug", "external-to-monorepo", "staterecover" ]
profiles: [ "l2", "debug", "external-to-monorepo", "staterecovery" ]
environment:
POSTGRES_USER: ${POSTGRES_USER:-postgres}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres}
@@ -400,7 +400,6 @@ services:
l1-cl-node:
container_name: l1-cl-node
hostname: l1-cl-node
# image: consensys/teku:24.2.0
image: consensys/teku:24.10.3
profiles: [ "l1", "debug", "external-to-monorepo" ]
depends_on:
@@ -628,7 +627,7 @@ services:
hostname: blobscan-api
image: blossomlabs/blobscan-api:1.1.0
platform: linux/amd64 # only linux available
profiles: [ "staterecover" ]
profiles: [ "staterecovery" ]
ports:
- "4001:4001"
env_file: "./config/blobscan/env"
@@ -659,7 +658,7 @@ services:
hostname: blobscan-indexer
image: blossomlabs/blobscan-indexer:0.2.1
platform: linux/amd64 # only linux available
profiles: [ "staterecover" ]
profiles: [ "staterecovery" ]
env_file: "./config/blobscan/env"
networks:
linea:
@@ -680,7 +679,7 @@ services:
container_name: redis
hostname: redis
image: "redis:7.4.1-alpine"
profiles: [ "staterecover" ]
profiles: [ "staterecovery" ]
ports:
- "6379:6379"
environment:
@@ -701,7 +700,7 @@ services:
image: consensys/linea-besu-package:${SEQUENCER_TAG:-devnet-811f30b}
hostname: zkbesu-shomei-sr
container_name: zkbesu-shomei-sr
profiles: [ "external-to-monorepo", "staterecover" ]
profiles: [ "external-to-monorepo", "staterecovery" ]
privileged: true
# restart: none
user: root
@@ -744,19 +743,19 @@ services:
--plugin-staterecovery-blobscan-endpoint=http://blobscan-api:4001 \
--plugin-staterecovery-linea-sequencer-beneficiary-address=0x6d976c9b8ceee705d4fe8699b44e5eb58242f484 \
--plugin-staterecovery-overriding-recovery-start-block-number=1 \
--plugin-staterecovery-l1-polling-interval=PT1S
--plugin-staterecovery-l1-polling-interval=PT0.1S
volumes:
- ./config/zkbesu-shomei/zkbesu-config.toml:/var/lib/besu/zkbesu-config.toml:ro
- ./config/zkbesu-shomei/log4j-staterecovery.xml:/var/lib/besu/log4j.xml:ro
- ./config/linea-local-dev-genesis-PoA-besu.json/:/var/lib/besu/genesis.json:ro
- ../state-recover/besu-plugin/build/libs/linea-staterecover-plugin-0.0.1-rc2.jar:/opt/besu/lib/linea-staterecover-plugin-0.0.1-rc2.jar
- ../state-recover/besu-plugin/build/libs/linea-staterecover-plugin-0.0.1-rc2.jar:/opt/besu/plugins/linea-staterecover-plugin-0.0.1-rc2.jar
- ../state-recovery/besu-plugin/build/libs/linea-staterecovery-plugin-0.0.1-rc2.jar:/opt/besu/lib/linea-staterecovery-plugin-0.0.1-rc2.jar
- ../state-recovery/besu-plugin/build/libs/linea-staterecovery-plugin-0.0.1-rc2.jar:/opt/besu/plugins/linea-staterecovery-plugin-0.0.1-rc2.jar
shomei-sr:
image: consensys/linea-shomei:2.3.0
hostname: shomei-sr
container_name: shomei-sr
profiles: [ "external-to-monorepo", "staterecover" ]
profiles: [ "external-to-monorepo", "staterecovery" ]
depends_on:
zkbesu-shomei-sr:
condition: service_started

View File

@@ -34,16 +34,16 @@
<Logger name="io.opentelemetry" level="WARN" additivity="false">
<AppenderRef ref="Console"/>
</Logger>
<Logger name="linea.staterecover" level="INFO" additivity="false">
<Logger name="linea.staterecovery" level="INFO" additivity="false">
<AppenderRef ref="Console"/>
</Logger>
<Logger name="linea.staterecover.clients.ExecutionLayerInProcessClient" level="TRACE" additivity="false">
<Logger name="linea.staterecovery.clients.ExecutionLayerInProcessClient" level="DEBUG" additivity="false">
<AppenderRef ref="Console"/>
</Logger>
<Logger name="linea.plugin.staterecover.clients.l1" level="DEBUG" additivity="false">
<Logger name="linea.plugin.staterecovery.clients.l1" level="DEBUG" additivity="false">
<AppenderRef ref="Console"/>
</Logger>
<Logger name="linea.plugin.staterecover.clients.l1.transaction-details" level="DEBUG" additivity="false">
<Logger name="linea.plugin.staterecovery.clients.l1.transaction-details" level="DEBUG" additivity="false">
<AppenderRef ref="Console"/>
</Logger>
<Root level="${sys:root.log.level}">

View File

@@ -38,17 +38,18 @@ data class SubmissionTxHashes(
)
fun submitBlobsAndAggregations(
contractClient: LineaRollupSmartContractClient,
contractClientForBlobSubmission: LineaRollupSmartContractClient,
contractClientForAggregationSubmission: LineaRollupSmartContractClient = contractClientForBlobSubmission,
aggregationsAndBlobs: List<AggregationAndBlobs>,
blobChunksSize: Int = 6
): SubmissionTxHashes {
val blobSubmissionTxHashes = submitBlobs(contractClient, aggregationsAndBlobs, blobChunksSize)
val blobSubmissionTxHashes = submitBlobs(contractClientForBlobSubmission, aggregationsAndBlobs, blobChunksSize)
return aggregationsAndBlobs
.filter { it.aggregation != null }
.mapIndexed { index, (aggregation, aggBlobs) ->
aggregation as Aggregation
val parentAgg = aggregationsAndBlobs.getOrNull(index - 1)?.aggregation
contractClient.finalizeBlocks(
contractClientForAggregationSubmission.finalizeBlocks(
aggregation = aggregation.aggregationProof!!,
aggregationLastBlob = aggBlobs.last(),
parentShnarf = aggBlobs.first().blobCompressionProof!!.prevShnarf,
@@ -61,14 +62,16 @@ fun submitBlobsAndAggregations(
}
fun submitBlobsAndAggregationsAndWaitExecution(
contractClient: LineaRollupSmartContractClient,
contractClientForBlobSubmission: LineaRollupSmartContractClient,
contractClientForAggregationSubmission: LineaRollupSmartContractClient = contractClientForBlobSubmission,
aggregationsAndBlobs: List<AggregationAndBlobs>,
blobChunksSize: Int = 6,
l1Web3jClient: Web3j,
waitTimeout: Duration = 2.minutes
) {
val submissionTxHashes = submitBlobsAndAggregations(
contractClient = contractClient,
contractClientForBlobSubmission = contractClientForAggregationSubmission,
contractClientForAggregationSubmission = contractClientForAggregationSubmission,
aggregationsAndBlobs = aggregationsAndBlobs,
blobChunksSize = blobChunksSize
)

View File

@@ -66,11 +66,11 @@ include 'transaction-exclusion-api:app'
include 'transaction-exclusion-api:core'
include 'transaction-exclusion-api:persistence:rejectedtransaction'
include 'state-recover:appcore:clients-interfaces'
include 'state-recover:appcore:domain-models'
include 'state-recover:appcore:logic'
include 'state-recover:besu-plugin'
include 'state-recover:clients:blobscan-client'
include 'state-recover:clients:execution-layer-json-rpc-client'
include 'state-recover:clients:eth-api'
include 'state-recover:test-cases'
include 'state-recovery:appcore:clients-interfaces'
include 'state-recovery:appcore:domain-models'
include 'state-recovery:appcore:logic'
include 'state-recovery:besu-plugin'
include 'state-recovery:clients:blobscan-client'
include 'state-recovery:clients:execution-layer-json-rpc-client'
include 'state-recovery:clients:eth-api'
include 'state-recovery:test-cases'

View File

@@ -1 +0,0 @@
linea.staterecover.plugin.LineaStateRecoverPlugin

View File

@@ -1,197 +0,0 @@
package linea.staterecover
import build.linea.clients.StateManagerClientV1
import build.linea.clients.StateManagerV1JsonRpcClient
import build.linea.contract.l1.LineaContractVersion
import build.linea.contract.l1.LineaRollupSmartContractClientReadOnly
import build.linea.contract.l1.Web3JLineaRollupSmartContractClientReadOnly
import build.linea.staterecover.clients.el.ExecutionLayerJsonRpcClient
import io.micrometer.core.instrument.simple.SimpleMeterRegistry
import io.vertx.core.Vertx
import io.vertx.junit5.VertxExtension
import linea.build.staterecover.clients.VertxTransactionDetailsClient
import linea.domain.RetryConfig
import linea.log4j.configureLoggers
import linea.staterecover.clients.blobscan.BlobScanClient
import linea.web3j.Web3JLogsSearcher
import net.consensys.linea.BlockParameter
import net.consensys.linea.jsonrpc.client.RequestRetryConfig
import net.consensys.linea.jsonrpc.client.VertxHttpJsonRpcClientFactory
import net.consensys.linea.metrics.micrometer.MicrometerMetricsFacade
import net.consensys.linea.testing.submission.AggregationAndBlobs
import net.consensys.linea.testing.submission.loadBlobsAndAggregationsSortedAndGrouped
import net.consensys.linea.testing.submission.submitBlobsAndAggregationsAndWaitExecution
import net.consensys.zkevm.coordinator.clients.smartcontract.LineaRollupSmartContractClient
import net.consensys.zkevm.ethereum.ContractsManager
import net.consensys.zkevm.ethereum.Web3jClientManager
import org.apache.logging.log4j.Level
import org.apache.logging.log4j.LogManager
import org.assertj.core.api.Assertions.assertThat
import org.awaitility.Awaitility.await
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.extension.ExtendWith
import java.net.URI
import kotlin.time.Duration.Companion.milliseconds
import kotlin.time.Duration.Companion.minutes
import kotlin.time.Duration.Companion.seconds
import kotlin.time.toJavaDuration
@ExtendWith(VertxExtension::class)
class StateRecoverAppIntTest {
private val log = LogManager.getLogger("test.case.StateRecoverAppIntTest")
private lateinit var stateRecoverApp: StateRecoverApp
private lateinit var aggregationsAndBlobs: List<AggregationAndBlobs>
private lateinit var executionLayerClient: ExecutionLayerClient
private lateinit var stateManagerClient: StateManagerClientV1
private lateinit var transactionDetailsClient: TransactionDetailsClient
private lateinit var lineaContractClient: LineaRollupSmartContractClientReadOnly
private lateinit var contractClientForSubmissions: LineaRollupSmartContractClient
private val testDataDir = "testdata/coordinator/prover/v3/"
private val l1RpcUrl = "http://localhost:8445"
private val blobScanUrl = "http://localhost:4001"
private val executionClientUrl = "http://localhost:9145"
private val stateManagerUrl = "http://localhost:8890"
@BeforeEach
fun beforeEach(vertx: Vertx) {
val jsonRpcFactory = VertxHttpJsonRpcClientFactory(
vertx = vertx,
metricsFacade = MicrometerMetricsFacade(SimpleMeterRegistry())
)
aggregationsAndBlobs = loadBlobsAndAggregationsSortedAndGrouped(
blobsResponsesDir = "$testDataDir/compression/responses",
aggregationsResponsesDir = "$testDataDir/aggregation/responses"
)
executionLayerClient = ExecutionLayerJsonRpcClient.create(
rpcClientFactory = jsonRpcFactory,
endpoint = URI(executionClientUrl),
requestRetryConfig = RequestRetryConfig(
backoffDelay = 10.milliseconds,
timeout = 2.seconds,
maxRetries = 4u,
failuresWarningThreshold = 1U
),
logger = LogManager.getLogger("test.clients.l1.executionlayer")
)
stateManagerClient = StateManagerV1JsonRpcClient.create(
rpcClientFactory = jsonRpcFactory,
endpoints = listOf(URI(stateManagerUrl)),
maxInflightRequestsPerClient = 1U,
requestRetry = RequestRetryConfig(
backoffDelay = 10.milliseconds,
timeout = 2.seconds
),
zkStateManagerVersion = "2.3.0",
logger = LogManager.getLogger("test.clients.l1.state-manager")
)
transactionDetailsClient = VertxTransactionDetailsClient.create(
jsonRpcClientFactory = jsonRpcFactory,
endpoint = URI(l1RpcUrl),
retryConfig = RequestRetryConfig(
backoffDelay = 10.milliseconds,
timeout = 2.seconds
),
logger = LogManager.getLogger("test.clients.l1.transaction-details")
)
val rollupDeploymentResult = ContractsManager.get()
.deployLineaRollup(numberOfOperators = 2, contractVersion = LineaContractVersion.V6).get()
lineaContractClient = Web3JLineaRollupSmartContractClientReadOnly(
web3j = Web3jClientManager.buildL1Client(
log = LogManager.getLogger("test.clients.l1.linea-contract"),
requestResponseLogLevel = Level.INFO,
failuresLogLevel = Level.WARN
),
contractAddress = rollupDeploymentResult.contractAddress
)
val logsSearcher = run {
val log = LogManager.getLogger("test.clients.l1.events-fetcher")
Web3JLogsSearcher(
vertx = vertx,
web3jClient = Web3jClientManager.buildL1Client(
log = log,
requestResponseLogLevel = Level.TRACE,
failuresLogLevel = Level.WARN
),
Web3JLogsSearcher.Config(
backoffDelay = 1.milliseconds,
requestRetryConfig = RetryConfig.noRetries
),
log = log
)
}
contractClientForSubmissions = rollupDeploymentResult.rollupOperatorClient
val blobScanClient = BlobScanClient.create(
vertx = vertx,
endpoint = URI(blobScanUrl),
requestRetryConfig = RequestRetryConfig(
backoffDelay = 10.milliseconds,
timeout = 2.seconds
),
logger = LogManager.getLogger("test.clients.l1.blobscan")
)
configureLoggers(
rootLevel = Level.INFO,
"test.clients.l1.executionlayer" to Level.INFO,
"test.clients.l1.web3j-default" to Level.INFO,
"test.clients.l1.state-manager" to Level.INFO,
"test.clients.l1.transaction-details" to Level.INFO,
"test.clients.l1.linea-contract" to Level.INFO,
"test.clients.l1.events-fetcher" to Level.INFO,
"test.clients.l1.blobscan" to Level.INFO,
"net.consensys.linea.contract.l1" to Level.INFO
)
stateRecoverApp = StateRecoverApp(
vertx = vertx,
elClient = executionLayerClient,
blobFetcher = blobScanClient,
ethLogsSearcher = logsSearcher,
stateManagerClient = stateManagerClient,
transactionDetailsClient = transactionDetailsClient,
blockHeaderStaticFields = BlockHeaderStaticFields.localDev,
lineaContractClient = lineaContractClient,
config = StateRecoverApp.Config(
l1LatestSearchBlock = BlockParameter.Tag.LATEST,
l1PollingInterval = 5.seconds,
executionClientPollingInterval = 1.seconds,
smartContractAddress = lineaContractClient.getAddress(),
logsBlockChunkSize = 100_000u
)
)
}
@Test
fun `state recovery from genesis`() {
stateRecoverApp.start().get()
submitBlobsAndAggregationsAndWaitExecution(
contractClient = contractClientForSubmissions,
aggregationsAndBlobs = aggregationsAndBlobs,
l1Web3jClient = Web3jClientManager.l1Client
)
val lastAggregation = aggregationsAndBlobs.findLast { it.aggregation != null }!!.aggregation!!
await()
.atMost(4.minutes.toJavaDuration())
.untilAsserted {
assertThat(stateRecoverApp.lastSuccessfullyRecoveredFinalization?.event?.endBlockNumber)
.isEqualTo(lastAggregation.endBlockNumber)
}
assertThat(executionLayerClient.lineaGetStateRecoveryStatus().get())
.isEqualTo(
StateRecoveryStatus(
headBlockNumber = lastAggregation.endBlockNumber,
stateRecoverStartBlockNumber = 1UL
)
)
}
}

View File

@@ -1,30 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="warn">
<Appenders>
<Console name="console" target="SYSTEM_OUT">
<PatternLayout pattern="[%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} %c{1} - %msg%n"/>
<!-- <PatternLayout pattern="%msg%n"/>-->
</Console>
</Appenders>
<Loggers>
<!-- Set level to DEBUG to log Web3J request/responses -->
<!-- <Logger name="test.clients.l1.execution-layer" level="trace" additivity="false">-->
<Logger name="test.fake.clients" level="trace" additivity="false">
<AppenderRef ref="console"/>
</Logger>
<Logger name="test.clients.l1.events-fetcher" level="DEBUG" additivity="false">
<!-- <Logger name="test.clients" level="trace" additivity="false">-->
<AppenderRef ref="console"/>
</Logger>
<Logger name="test.clients.l1.blobscan" level="DEBUG" additivity="false">
<AppenderRef ref="console"/>
</Logger>
<Logger name="test.clients.l1.executionlayer" level="TRACE" additivity="false">
<AppenderRef ref="console"/>
</Logger>
<Root level="info" additivity="false">
<appender-ref ref="console"/>
</Root>
</Loggers>
</Configuration>

View File

@@ -8,5 +8,5 @@ dependencies {
api(project(':jvm-libs:generic:extensions:kotlin'))
api(project(':jvm-libs:generic:serialization:jackson'))
api(project(':jvm-libs:linea:core:domain-models'))
api(project(':state-recover:appcore:domain-models'))
api(project(':state-recovery:appcore:domain-models'))
}

View File

@@ -1,4 +1,4 @@
package linea.staterecover
package linea.staterecovery
import tech.pegasys.teku.infrastructure.async.SafeFuture

View File

@@ -1,4 +1,4 @@
package linea.staterecover
package linea.staterecovery
import net.consensys.linea.BlockNumberAndHash
import net.consensys.linea.BlockParameter

View File

@@ -1,4 +1,4 @@
package linea.staterecover
package linea.staterecovery
import com.fasterxml.jackson.core.JacksonException
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper

View File

@@ -1,4 +1,4 @@
package linea.staterecover
package linea.staterecovery
import tech.pegasys.teku.infrastructure.async.SafeFuture

View File

@@ -1,4 +1,4 @@
package linea.staterecover
package linea.staterecovery
import org.assertj.core.api.Assertions.assertThat
import org.assertj.core.api.Assertions.assertThatThrownBy

View File

@@ -1,4 +1,4 @@
package linea.staterecover
package linea.staterecovery
import kotlinx.datetime.Instant
import net.consensys.encodeHex

View File

@@ -1,4 +1,4 @@
package linea.staterecover
package linea.staterecovery
import net.consensys.encodeHex
import java.math.BigInteger

View File

@@ -13,8 +13,8 @@ dependencies {
api(project(':jvm-libs:linea:clients:interfaces'))
api(project(':jvm-libs:linea:clients:linea-state-manager'))
api(project(':jvm-libs:linea:blob-decompressor'))
api(project(':state-recover:appcore:clients-interfaces'))
api(project(':state-recover:appcore:domain-models'))
api(project(':state-recovery:appcore:clients-interfaces'))
api(project(':state-recovery:appcore:domain-models'))
api project(':jvm-libs:linea:besu-rlp-and-mappers')
api project(':jvm-libs:linea:besu-libs')

View File

@@ -1,4 +1,4 @@
package linea.staterecover
package linea.staterecovery
import io.vertx.core.Vertx
import kotlinx.datetime.Clock

View File

@@ -1,4 +1,4 @@
package linea.staterecover
package linea.staterecovery
import build.linea.clients.StateManagerClientV1
import build.linea.domain.BlockInterval

View File

@@ -1,4 +1,4 @@
package linea.staterecover
package linea.staterecovery
import build.linea.domain.BlockInterval
import build.linea.domain.EthLog

View File

@@ -1,4 +1,4 @@
package linea.staterecover
package linea.staterecovery
import build.linea.domain.EthLogEvent
import linea.EthLogsSearcher

View File

@@ -1,4 +1,4 @@
package linea.staterecover
package linea.staterecovery
import build.linea.clients.StateManagerClientV1
import build.linea.contract.l1.LineaRollupSmartContractClientReadOnly

View File

@@ -1,4 +1,4 @@
package linea.staterecover
package linea.staterecovery
import build.linea.domain.EthLogEvent
import io.vertx.core.Vertx

View File

@@ -1,4 +1,4 @@
package linea.staterecover
package linea.staterecovery
import io.vertx.core.Vertx
import kotlinx.datetime.Instant

View File

@@ -3,8 +3,8 @@ plugins {
id 'com.gradleup.shadow' version '8.3.5'
}
group = 'build.linea.staterecover'
archivesBaseName = 'linea-staterecover-plugin'
group = 'build.linea.staterecovery'
archivesBaseName = 'linea-staterecovery-plugin'
version = '0.0.1-rc2'
dependencies {
@@ -16,10 +16,10 @@ dependencies {
api(project(":jvm-libs:generic:serialization:jackson"))
api(project(":jvm-libs:linea:clients:linea-l1-contract-client"))
api(project(":jvm-libs:linea:web3j-extensions"))
api(project(":state-recover:appcore:logic"))
api(project(":state-recover:clients:blobscan-client"))
api(project(":state-recover:clients:eth-api"))
api(project(":state-recover:clients:execution-layer-json-rpc-client"))
api(project(":state-recovery:appcore:logic"))
api(project(":state-recovery:clients:blobscan-client"))
api(project(":state-recovery:clients:eth-api"))
api(project(":state-recovery:clients:execution-layer-json-rpc-client"))
}
ext.groupsToIncludeInShadow = [

View File

@@ -1,11 +1,11 @@
package linea.staterecover.clients
package linea.staterecovery.clients
import linea.staterecover.BlockFromL1RecoveredData
import linea.staterecover.ExecutionLayerClient
import linea.staterecover.RecoveryStatusPersistence
import linea.staterecover.StateRecoveryStatus
import linea.staterecover.plugin.BlockImporter
import linea.staterecover.plugin.RecoveryModeManager
import linea.staterecovery.BlockFromL1RecoveredData
import linea.staterecovery.ExecutionLayerClient
import linea.staterecovery.RecoveryStatusPersistence
import linea.staterecovery.StateRecoveryStatus
import linea.staterecovery.plugin.BlockImporter
import linea.staterecovery.plugin.RecoveryModeManager
import net.consensys.linea.BlockNumberAndHash
import net.consensys.linea.BlockParameter
import org.apache.logging.log4j.LogManager

View File

@@ -1,4 +1,4 @@
package linea.staterecover.plugin
package linea.staterecovery.plugin
import build.linea.clients.StateManagerClientV1
import build.linea.clients.StateManagerV1JsonRpcClient
@@ -6,12 +6,12 @@ import build.linea.contract.l1.Web3JLineaRollupSmartContractClientReadOnly
import io.micrometer.core.instrument.MeterRegistry
import io.vertx.core.Vertx
import io.vertx.micrometer.backends.BackendRegistries
import linea.build.staterecover.clients.VertxTransactionDetailsClient
import linea.staterecover.BlockHeaderStaticFields
import linea.staterecover.ExecutionLayerClient
import linea.staterecover.StateRecoverApp
import linea.staterecover.TransactionDetailsClient
import linea.staterecover.clients.blobscan.BlobScanClient
import linea.staterecovery.BlockHeaderStaticFields
import linea.staterecovery.ExecutionLayerClient
import linea.staterecovery.StateRecoverApp
import linea.staterecovery.TransactionDetailsClient
import linea.staterecovery.clients.VertxTransactionDetailsClient
import linea.staterecovery.clients.blobscan.BlobScanClient
import linea.web3j.Web3JLogsSearcher
import linea.web3j.createWeb3jHttpClient
import net.consensys.linea.jsonrpc.client.RequestRetryConfig

View File

@@ -1,4 +1,4 @@
package linea.staterecover.plugin
package linea.staterecovery.plugin
import org.hyperledger.besu.plugin.data.BlockBody
import org.hyperledger.besu.plugin.data.BlockContext

View File

@@ -1,6 +1,6 @@
package linea.staterecover.plugin
package linea.staterecovery.plugin
import linea.staterecover.BlockFromL1RecoveredData
import linea.staterecovery.BlockFromL1RecoveredData
import net.consensys.encodeHex
import net.consensys.toBigInteger
import net.consensys.toULong

View File

@@ -1,12 +1,12 @@
package linea.staterecover.plugin
package linea.staterecovery.plugin
import io.micrometer.core.instrument.simple.SimpleMeterRegistry
import io.vertx.core.Vertx
import linea.staterecover.BlockHeaderStaticFields
import linea.staterecover.FileBasedRecoveryStatusPersistence
import linea.staterecover.RecoveryStatusPersistence
import linea.staterecover.StateRecoverApp
import linea.staterecover.clients.ExecutionLayerInProcessClient
import linea.staterecovery.BlockHeaderStaticFields
import linea.staterecovery.FileBasedRecoveryStatusPersistence
import linea.staterecovery.RecoveryStatusPersistence
import linea.staterecovery.StateRecoverApp
import linea.staterecovery.clients.ExecutionLayerInProcessClient
import net.consensys.linea.async.get
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
@@ -41,7 +41,7 @@ open class LineaStateRecoverPlugin : BesuPlugin {
this.serviceManager = serviceManager
serviceManager
.getServiceOrThrow(PicoCLIOptions::class.java)
.addPicoCLIOptions(PluginCliOptions.cliOptionsPrefix, cliOptions)
.addPicoCLIOptions(PluginCliOptions.cliPluginPrefixName, cliOptions)
log.debug("registered")
}

View File

@@ -1,10 +1,10 @@
package linea.staterecover.plugin
package linea.staterecovery.plugin
import org.hyperledger.besu.datatypes.Address
import picocli.CommandLine
import java.net.URI
import java.time.Duration
import kotlin.time.Duration.Companion.seconds
import kotlin.time.Duration.Companion.milliseconds
import kotlin.time.toKotlinDuration
data class PluginConfig(
@@ -17,13 +17,14 @@ data class PluginConfig(
val overridingRecoveryStartBlockNumber: ULong? = null
) {
init {
require(l1PollingInterval >= 1.seconds) { "Polling interval=$l1PollingInterval must be greater that 1s." }
require(l1PollingInterval >= 1.milliseconds) { "Polling interval=$l1PollingInterval must be greater than 1ms." }
}
}
class PluginCliOptions {
companion object {
const val cliOptionsPrefix = "plugin-staterecovery"
const val cliPluginPrefixName = "staterecovery"
private const val cliOptionsPrefix = "plugin-$cliPluginPrefixName"
}
@CommandLine.Option(
@@ -99,7 +100,7 @@ class PluginCliOptions {
class AddressConverter : CommandLine.ITypeConverter<Address> {
override fun convert(value: String): Address {
return Address.fromHexString(value) ?: throw CommandLine.TypeConversionException(
return Address.fromHexStringStrict(value) ?: throw CommandLine.TypeConversionException(
"Invalid address: $value"
)
}

View File

@@ -1,6 +1,6 @@
package linea.staterecover.plugin
package linea.staterecovery.plugin
import linea.staterecover.RecoveryStatusPersistence
import linea.staterecovery.RecoveryStatusPersistence
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
import org.hyperledger.besu.plugin.data.AddedBlockContext

View File

@@ -1,7 +1,7 @@
package linea.staterecover.plugin
package linea.staterecovery.plugin
import linea.staterecover.TransactionFromL1RecoveredData
import linea.staterecover.TransactionFromL1RecoveredData.AccessTuple
import linea.staterecovery.TransactionFromL1RecoveredData
import linea.staterecovery.TransactionFromL1RecoveredData.AccessTuple
import net.consensys.encodeHex
import net.consensys.toBigInteger
import org.apache.tuweni.bytes.Bytes

View File

@@ -0,0 +1 @@
linea.staterecovery.plugin.LineaStateRecoverPlugin

View File

@@ -17,7 +17,7 @@ dependencies {
implementation(project(':jvm-libs:linea:clients:linea-state-manager'))
implementation(project(':jvm-libs:linea:core:domain-models'))
implementation(project(':jvm-libs:linea:core:long-running-service'))
implementation(project(':state-recover:appcore:clients-interfaces'))
implementation(project(':state-recovery:appcore:clients-interfaces'))
implementation("io.vertx:vertx-web-client:${libs.versions.vertx}")
testImplementation "com.github.tomakehurst:wiremock-jre8:${libs.versions.wiremock.get()}"

View File

@@ -1,10 +1,10 @@
package linea.staterecover.clients.blobscan
package linea.staterecovery.clients.blobscan
import io.vertx.core.Vertx
import io.vertx.core.json.JsonObject
import io.vertx.ext.web.client.WebClient
import io.vertx.ext.web.client.WebClientOptions
import linea.staterecover.BlobFetcher
import linea.staterecovery.BlobFetcher
import net.consensys.decodeHex
import net.consensys.encodeHex
import net.consensys.linea.jsonrpc.client.RequestRetryConfig

View File

@@ -1,4 +1,4 @@
package linea.staterecover.clients.blobscan
package linea.staterecovery.clients.blobscan
import io.vertx.core.Vertx
import io.vertx.core.buffer.Buffer

View File

@@ -1,4 +1,4 @@
package linea.staterecover.clients.blobscan
package linea.staterecovery.clients.blobscan
import com.github.tomakehurst.wiremock.WireMockServer
import com.github.tomakehurst.wiremock.client.WireMock

View File

@@ -7,5 +7,5 @@ dependencies {
implementation(project(':jvm-libs:linea:web3j-extensions'))
implementation(project(':jvm-libs:generic:serialization:jackson'))
api(project(':jvm-libs:generic:json-rpc'))
implementation(project(':state-recover:appcore:clients-interfaces'))
implementation(project(':state-recovery:appcore:clients-interfaces'))
}

View File

@@ -1,8 +1,8 @@
package linea.build.staterecover.clients
package linea.staterecovery.clients
import com.fasterxml.jackson.databind.JsonNode
import com.github.michaelbull.result.Err
import linea.staterecover.TransactionDetailsClient
import linea.staterecovery.TransactionDetailsClient
import net.consensys.decodeHex
import net.consensys.linea.jsonrpc.client.JsonRpcClientFactory
import net.consensys.linea.jsonrpc.client.JsonRpcV2Client

View File

@@ -10,8 +10,8 @@ dependencies {
implementation(project(':jvm-libs:generic:vertx-helper'))
implementation(project(':jvm-libs:generic:serialization:jackson'))
implementation(project(':jvm-libs:linea:core:domain-models'))
implementation(project(':state-recover:appcore:clients-interfaces'))
implementation(project(':state-recover:appcore:domain-models'))
implementation(project(':state-recovery:appcore:clients-interfaces'))
implementation(project(':state-recovery:appcore:domain-models'))
testImplementation project(':jvm-libs:linea:core:metrics')

View File

@@ -1,4 +1,4 @@
package build.linea.staterecover.clients.el
package linea.staterecovery.clients.el
import build.linea.s11n.jackson.InstantAsHexNumberDeserializer
import build.linea.s11n.jackson.InstantAsHexNumberSerializer
@@ -7,9 +7,9 @@ import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.databind.JsonNode
import com.fasterxml.jackson.databind.module.SimpleModule
import kotlinx.datetime.Instant
import linea.staterecover.BlockFromL1RecoveredData
import linea.staterecover.ExecutionLayerClient
import linea.staterecover.StateRecoveryStatus
import linea.staterecovery.BlockFromL1RecoveredData
import linea.staterecovery.ExecutionLayerClient
import linea.staterecovery.StateRecoveryStatus
import net.consensys.decodeHex
import net.consensys.fromHexString
import net.consensys.linea.BlockNumberAndHash
@@ -54,7 +54,7 @@ class ExecutionLayerJsonRpcClient internal constructor(
.makeRequest(
method = "linea_getStateRecoveryStatus",
params = emptyList<Unit>(),
resultMapper = ::stateRecoveryStatusFromJsonNode
resultMapper = Companion::stateRecoveryStatusFromJsonNode
)
}
@@ -63,7 +63,7 @@ class ExecutionLayerJsonRpcClient internal constructor(
.makeRequest(
method = "linea_enableStateRecovery",
params = listOf(stateRecoverStartBlockNumber),
resultMapper = ::stateRecoveryStatusFromJsonNode
resultMapper = Companion::stateRecoveryStatusFromJsonNode
)
}

View File

@@ -1,4 +1,4 @@
package build.linea.staterecover.clients.el
package linea.staterecovery.clients.el
import com.github.tomakehurst.wiremock.WireMockServer
import com.github.tomakehurst.wiremock.client.WireMock.containing
@@ -8,11 +8,11 @@ import com.github.tomakehurst.wiremock.core.WireMockConfiguration
import io.micrometer.core.instrument.simple.SimpleMeterRegistry
import io.vertx.junit5.VertxExtension
import kotlinx.datetime.Instant
import linea.staterecover.BlockFromL1RecoveredData
import linea.staterecover.BlockHeaderFromL1RecoveredData
import linea.staterecover.ExecutionLayerClient
import linea.staterecover.StateRecoveryStatus
import linea.staterecover.TransactionFromL1RecoveredData
import linea.staterecovery.BlockFromL1RecoveredData
import linea.staterecovery.BlockHeaderFromL1RecoveredData
import linea.staterecovery.ExecutionLayerClient
import linea.staterecovery.StateRecoveryStatus
import linea.staterecovery.TransactionFromL1RecoveredData
import net.consensys.decodeHex
import net.consensys.linea.BlockNumberAndHash
import net.consensys.linea.BlockParameter

View File

@@ -16,15 +16,15 @@ dependencies {
api(project(':jvm-libs:linea:clients:linea-l1-contract-client'))
api(project(':jvm-libs:linea:clients:linea-state-manager'))
api(project(':jvm-libs:linea:blob-decompressor'))
api(project(':state-recover:appcore:clients-interfaces'))
api(project(':state-recover:appcore:domain-models'))
api(project(':state-recover:appcore:logic'))
api(project(':state-recovery:appcore:clients-interfaces'))
api(project(':state-recovery:appcore:domain-models'))
api(project(':state-recovery:appcore:logic'))
implementation project(':jvm-libs:linea:besu-libs')
implementation(testFixtures(project(':jvm-libs:generic:json-rpc')))
implementation(project(':state-recover:clients:eth-api'))
implementation(project(':state-recover:clients:blobscan-client'))
implementation(project(':state-recover:clients:execution-layer-json-rpc-client'))
implementation(project(':state-recovery:clients:eth-api'))
implementation(project(':state-recovery:clients:blobscan-client'))
implementation(project(':state-recovery:clients:execution-layer-json-rpc-client'))
implementation(project(':coordinator:clients:smart-contract-client'))
implementation(project(':jvm-libs:linea:linea-contracts:l1-rollup'))
implementation('build.linea:l1-rollup-contract-client:6.0.0')
@@ -65,7 +65,7 @@ task integrationTest(type: Test) { test ->
classpath = sourceSets.integrationTest.runtimeClasspath
testClassesDirs = sourceSets.integrationTest.output.classesDirs
// dependsOn(":localStackForStateRecoverComposeUp")
dependsOn(":localStackForStateRecoverComposeUp")
testLogging {
events TestLogEvent.FAILED,

View File

@@ -1,4 +1,4 @@
package linea.staterecover
package linea.staterecovery
import build.linea.contract.l1.LineaContractVersion
import io.vertx.core.Vertx
@@ -6,16 +6,18 @@ import io.vertx.junit5.Timeout
import io.vertx.junit5.VertxExtension
import io.vertx.junit5.VertxTestContext
import linea.domain.RetryConfig
import linea.log4j.configureLoggers
import linea.web3j.Web3JLogsSearcher
import net.consensys.linea.BlockParameter
import net.consensys.linea.testing.submission.AggregationAndBlobs
import net.consensys.linea.testing.submission.loadBlobsAndAggregationsSortedAndGrouped
import net.consensys.linea.testing.submission.submitBlobsAndAggregations
import net.consensys.zkevm.coordinator.clients.smartcontract.LineaRollupSmartContractClient
import net.consensys.linea.testing.submission.submitBlobsAndAggregationsAndWaitExecution
import net.consensys.zkevm.domain.Aggregation
import net.consensys.zkevm.ethereum.ContractsManager
import net.consensys.zkevm.ethereum.LineaRollupDeploymentResult
import net.consensys.zkevm.ethereum.MakeFileDelegatedContractsManager.connectToLineaRollupContract
import net.consensys.zkevm.ethereum.MakeFileDelegatedContractsManager.lineaRollupContractErrors
import net.consensys.zkevm.ethereum.Web3jClientManager
import net.consensys.zkevm.ethereum.waitForTxReceipt
import org.apache.logging.log4j.Level
import org.apache.logging.log4j.LogManager
import org.assertj.core.api.Assertions.assertThat
@@ -28,8 +30,8 @@ import kotlin.time.toJavaDuration
@ExtendWith(VertxExtension::class)
class LineaSubmissionEventsClientIntTest {
private lateinit var contractClient: LineaRollupSmartContractClient
private val testDataDir = "testdata/coordinator/prover/v2/"
private lateinit var rollupDeploymentResult: LineaRollupDeploymentResult
// 1-block-per-blob test data has 3 aggregations: 1..7, 8..14, 15..21.
// We will upgrade the contract in the middle of 2nd aggregation: 12
@@ -41,6 +43,14 @@ class LineaSubmissionEventsClientIntTest {
private fun setupTest(
vertx: Vertx
) {
configureLoggers(
rootLevel = Level.INFO,
"net.consensys.linea.contract.Web3JContractAsyncHelper" to Level.WARN,
"test.clients.l1.executionlayer" to Level.INFO,
"test.clients.l1.web3j-default" to Level.INFO,
"test.clients.l1.linea-contract" to Level.INFO,
"test.clients.l1.events-fetcher" to Level.INFO
)
val rollupDeploymentFuture = ContractsManager.get()
.deployLineaRollup(numberOfOperators = 2, contractVersion = LineaContractVersion.V6)
// load files from FS while smc deploy
@@ -49,11 +59,10 @@ class LineaSubmissionEventsClientIntTest {
aggregationsResponsesDir = "$testDataDir/aggregation/responses"
)
// wait smc deployment finishes
val rollupDeploymentResult = rollupDeploymentFuture.get()
contractClient = rollupDeploymentResult.rollupOperatorClient
rollupDeploymentResult = rollupDeploymentFuture.get()
val eventsFetcherWeb3jClient = Web3jClientManager.buildL1Client(
log = LogManager.getLogger("test.clients.l1.events-fetcher"),
requestResponseLogLevel = Level.INFO,
requestResponseLogLevel = Level.DEBUG,
failuresLogLevel = Level.WARN
)
submissionEventsFetcher = LineaSubmissionEventsClientImpl(
@@ -87,16 +96,17 @@ class LineaSubmissionEventsClientIntTest {
) {
setupTest(vertx)
val submissionTxHashes = submitBlobsAndAggregations(
contractClient = contractClient,
submitBlobsAndAggregationsAndWaitExecution(
contractClientForBlobSubmission = rollupDeploymentResult.rollupOperatorClient,
contractClientForAggregationSubmission = connectToLineaRollupContract(
contractAddress = rollupDeploymentResult.contractAddress,
transactionManager = rollupDeploymentResult.rollupOperators[1].txManager,
smartContractErrors = lineaRollupContractErrors
),
aggregationsAndBlobs = aggregationsAndBlobs,
blobChunksSize = 6
)
// wait for all finalizations Txs to be mined
Web3jClientManager.l1Client.waitForTxReceipt(
txHash = submissionTxHashes.aggregationTxHashes.last(),
timeout = 2.minutes
blobChunksSize = 6,
l1Web3jClient = Web3jClientManager.l1Client,
waitTimeout = 4.minutes
)
val expectedSubmissionEventsToFind: List<Pair<DataFinalizedV3, List<DataSubmittedV3>>> =

View File

@@ -1,4 +1,4 @@
package linea.staterecover
package linea.staterecovery
import build.linea.contract.l1.LineaContractVersion
import build.linea.contract.l1.LineaRollupSmartContractClientReadOnly
@@ -6,13 +6,13 @@ import build.linea.contract.l1.Web3JLineaRollupSmartContractClientReadOnly
import io.micrometer.core.instrument.simple.SimpleMeterRegistry
import io.vertx.core.Vertx
import io.vertx.junit5.VertxExtension
import linea.build.staterecover.clients.VertxTransactionDetailsClient
import linea.domain.RetryConfig
import linea.log4j.configureLoggers
import linea.staterecover.clients.blobscan.BlobScanClient
import linea.staterecover.test.FakeExecutionLayerClient
import linea.staterecover.test.FakeStateManagerClient
import linea.staterecover.test.FakeStateManagerClientBasedOnBlobsRecords
import linea.staterecovery.clients.VertxTransactionDetailsClient
import linea.staterecovery.clients.blobscan.BlobScanClient
import linea.staterecovery.test.FakeExecutionLayerClient
import linea.staterecovery.test.FakeStateManagerClient
import linea.staterecovery.test.FakeStateManagerClientBasedOnBlobsRecords
import linea.web3j.Web3JLogsSearcher
import net.consensys.linea.BlockNumberAndHash
import net.consensys.linea.BlockParameter
@@ -24,6 +24,8 @@ import net.consensys.linea.testing.submission.loadBlobsAndAggregationsSortedAndG
import net.consensys.linea.testing.submission.submitBlobsAndAggregationsAndWaitExecution
import net.consensys.zkevm.coordinator.clients.smartcontract.LineaRollupSmartContractClient
import net.consensys.zkevm.ethereum.ContractsManager
import net.consensys.zkevm.ethereum.MakeFileDelegatedContractsManager.connectToLineaRollupContract
import net.consensys.zkevm.ethereum.MakeFileDelegatedContractsManager.lineaRollupContractErrors
import net.consensys.zkevm.ethereum.Web3jClientManager
import org.apache.logging.log4j.Level
import org.apache.logging.log4j.LogManager
@@ -40,7 +42,7 @@ import kotlin.time.Duration.Companion.seconds
import kotlin.time.toJavaDuration
@ExtendWith(VertxExtension::class)
class StateRecoverAppWithFakeExecutionClientIntTest {
class StateRecoveryAppWithFakeExecutionClientIntTest {
private val log = LogManager.getLogger("test.case.StateRecoverAppWithFakeExecutionClientIntTest")
private lateinit var stateRecoverApp: StateRecoverApp
private lateinit var aggregationsAndBlobs: List<AggregationAndBlobs>
@@ -49,7 +51,8 @@ class StateRecoverAppWithFakeExecutionClientIntTest {
private lateinit var transactionDetailsClient: TransactionDetailsClient
private lateinit var lineaContractClient: LineaRollupSmartContractClientReadOnly
private lateinit var contractClientForSubmissions: LineaRollupSmartContractClient
private lateinit var contractClientForBlobSubmissions: LineaRollupSmartContractClient
private lateinit var contractClientForAggregationSubmissions: LineaRollupSmartContractClient
private val testDataDir = run {
"testdata/coordinator/prover/v3"
}
@@ -109,7 +112,12 @@ class StateRecoverAppWithFakeExecutionClientIntTest {
log = LogManager.getLogger("test.clients.l1.events-fetcher")
)
contractClientForSubmissions = rollupDeploymentResult.rollupOperatorClient
contractClientForBlobSubmissions = rollupDeploymentResult.rollupOperatorClient
contractClientForAggregationSubmissions = connectToLineaRollupContract(
rollupDeploymentResult.contractAddress,
rollupDeploymentResult.rollupOperators[1].txManager,
smartContractErrors = lineaRollupContractErrors
)
val blobScanClient = BlobScanClient.create(
vertx = vertx,
endpoint = URI(blobScanUrl),
@@ -155,13 +163,13 @@ class StateRecoverAppWithFakeExecutionClientIntTest {
}
private fun submitDataToL1ContactAndWaitExecution(
contractClient: LineaRollupSmartContractClient = contractClientForSubmissions,
aggregationsAndBlobs: List<AggregationAndBlobs> = this.aggregationsAndBlobs,
blobChunksSize: Int = 6,
waitTimeout: Duration = 2.minutes
waitTimeout: Duration = 4.minutes
) {
submitBlobsAndAggregationsAndWaitExecution(
contractClient = contractClient,
contractClientForBlobSubmission = contractClientForBlobSubmissions,
contractClientForAggregationSubmission = contractClientForAggregationSubmissions,
aggregationsAndBlobs = aggregationsAndBlobs,
blobChunksSize = blobChunksSize,
waitTimeout = waitTimeout,

View File

@@ -1,4 +1,4 @@
package linea.staterecover
package linea.staterecovery
import build.linea.clients.StateManagerClientV1
import build.linea.clients.StateManagerV1JsonRpcClient
@@ -8,6 +8,7 @@ import io.micrometer.core.instrument.simple.SimpleMeterRegistry
import io.vertx.core.Vertx
import io.vertx.junit5.VertxExtension
import linea.log4j.configureLoggers
import linea.testing.Runner
import linea.web3j.createWeb3jHttpClient
import net.consensys.linea.BlockParameter
import net.consensys.linea.jsonrpc.client.RequestRetryConfig
@@ -19,6 +20,8 @@ import net.consensys.linea.testing.submission.submitBlobsAndAggregationsAndWaitE
import net.consensys.toULong
import net.consensys.zkevm.ethereum.ContractsManager
import net.consensys.zkevm.ethereum.LineaRollupDeploymentResult
import net.consensys.zkevm.ethereum.MakeFileDelegatedContractsManager.connectToLineaRollupContract
import net.consensys.zkevm.ethereum.MakeFileDelegatedContractsManager.lineaRollupContractErrors
import net.consensys.zkevm.ethereum.Web3jClientManager
import org.apache.logging.log4j.Level
import org.apache.logging.log4j.LogManager
@@ -27,6 +30,7 @@ import org.awaitility.Awaitility.await
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.extension.ExtendWith
import tech.pegasys.teku.infrastructure.async.SafeFuture
import java.net.URI
import kotlin.time.Duration.Companion.milliseconds
import kotlin.time.Duration.Companion.minutes
@@ -34,11 +38,12 @@ import kotlin.time.Duration.Companion.seconds
import kotlin.time.toJavaDuration
@ExtendWith(VertxExtension::class)
class StateRecoveryManualReplayToLocalStackIntTest {
class StateRecoveryWithRealBesuAndStateManagerIntTest {
private val log = LogManager.getLogger("test.case.StateRecoverAppWithLocalStackIntTest")
private lateinit var stateManagerClient: StateManagerClientV1
private val testDataDir = "testdata/coordinator/prover/v3"
private val executionLayerUrl = "http://localhost:9145"
private val stateManagerUrl = "http://localhost:8890"
@BeforeEach
@@ -67,6 +72,7 @@ class StateRecoveryManualReplayToLocalStackIntTest {
fun setupDeployContractForL2L1StateReplay() {
configureLoggers(
rootLevel = Level.INFO,
"net.consensys.linea.contract.Web3JContractAsyncHelper" to Level.WARN,
"test.clients.l1.executionlayer" to Level.INFO,
"test.clients.l1.web3j-default" to Level.INFO,
"test.clients.l1.state-manager" to Level.DEBUG,
@@ -84,20 +90,32 @@ class StateRecoveryManualReplayToLocalStackIntTest {
this.rollupDeploymentResult = ContractsManager.get()
.deployLineaRollup(numberOfOperators = 2, contractVersion = LineaContractVersion.V6).get()
log.info("""LineaRollup address=${rollupDeploymentResult.contractAddress}""")
log.info(
"""
Start state recovery besu and shomei with the following configuration:
log.info("starting stack for recovery of state pushed to L1")
val staterecoveryNodesStartFuture = SafeFuture.supplyAsync {
Runner.executeCommand(
"make staterecovery-replay-from-genesis L1_ROLLUP_CONTRACT_ADDRESS=${rollupDeploymentResult.contractAddress}"
)
}
val blobsSubmissionFuture = SafeFuture.supplyAsync {
submitBlobsAndAggregationsAndWaitExecution(
contractClientForBlobSubmission = rollupDeploymentResult.rollupOperatorClient,
contractClientForAggregationSubmission = connectToLineaRollupContract(
rollupDeploymentResult.contractAddress,
// index 0 is the first operator in rollupOperatorClient
rollupDeploymentResult.rollupOperators[1].txManager,
smartContractErrors = lineaRollupContractErrors
),
aggregationsAndBlobs = aggregationsAndBlobs,
blobChunksSize = 6,
l1Web3jClient = Web3jClientManager.l1Client,
waitTimeout = 4.minutes
)
}
SafeFuture.allOf(staterecoveryNodesStartFuture, blobsSubmissionFuture).get()
./gradlew state-recover:besu-plugin:shadowJar \
&& docker compose -f docker/compose.yml down zkbesu-shomei-sr shomei-sr \
&& L1_ROLLUP_CONTRACT_ADDRESS=${rollupDeploymentResult.contractAddress} docker compose -f docker/compose.yml up zkbesu-shomei-sr shomei-sr
val web3jElClient = createWeb3jHttpClient(executionLayerUrl)
""".trimIndent()
)
val web3jElClient = createWeb3jHttpClient("http://localhost:9145")
// wait for statemanager to be up and running
// wait for state-manager to be up and running
await()
.pollInterval(1.seconds.toJavaDuration())
.atMost(5.minutes.toJavaDuration())
@@ -105,22 +123,13 @@ class StateRecoveryManualReplayToLocalStackIntTest {
kotlin.runCatching {
assertThat(web3jElClient.ethBlockNumber().send().blockNumber.toLong()).isGreaterThanOrEqualTo(0L)
}.getOrElse {
log.info("could not connect to stateManager $stateManagerUrl")
throw AssertionError("could not connect to stateManager $stateManagerUrl", it)
log.info("waiting for Besu to start, trying to connect to $executionLayerUrl")
throw AssertionError("could not connect to $executionLayerUrl", it)
}
}
val lastAggregationAndBlobs = aggregationsAndBlobs.findLast { it.aggregation != null }!!
val lastAggregation = lastAggregationAndBlobs.aggregation!!
// wait until state recovery Besu and Shomei are up
submitBlobsAndAggregationsAndWaitExecution(
contractClient = rollupDeploymentResult.rollupOperatorClient,
aggregationsAndBlobs = aggregationsAndBlobs,
blobChunksSize = 6,
l1Web3jClient = Web3jClientManager.l1Client
)
log.info("finalization={} executed on l1", lastAggregation.intervalString())
await()
.untilAsserted {
assertThat(
@@ -128,6 +137,7 @@ class StateRecoveryManualReplayToLocalStackIntTest {
.finalizedL2BlockNumber(blockParameter = BlockParameter.Tag.LATEST).get()
).isGreaterThanOrEqualTo(lastAggregation.endBlockNumber)
}
log.info("finalization={} executed on l1", lastAggregation.intervalString())
val expectedZkEndStateRootHash = lastAggregationAndBlobs.blobs.last().blobCompressionProof!!.finalStateRootHash
await()

View File

@@ -1,8 +1,8 @@
package linea.staterecover.test
package linea.staterecovery.test
import linea.staterecover.BlockFromL1RecoveredData
import linea.staterecover.ExecutionLayerClient
import linea.staterecover.StateRecoveryStatus
import linea.staterecovery.BlockFromL1RecoveredData
import linea.staterecovery.ExecutionLayerClient
import linea.staterecovery.StateRecoveryStatus
import net.consensys.linea.BlockNumberAndHash
import net.consensys.linea.BlockParameter
import net.consensys.linea.CommonDomainFunctions

View File

@@ -1,4 +1,4 @@
package linea.staterecover.test
package linea.staterecovery.test
import build.linea.clients.GetZkEVMStateMerkleProofResponse
import build.linea.clients.StateManagerClientV1
@@ -8,7 +8,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode
import com.github.michaelbull.result.Ok
import com.github.michaelbull.result.Result
import linea.EthLogsSearcher
import linea.staterecover.DataFinalizedV3
import linea.staterecovery.DataFinalizedV3
import net.consensys.linea.BlockParameter
import net.consensys.linea.errors.ErrorResponse
import net.consensys.toHexStringUInt256

View File

@@ -1,4 +1,4 @@
package linea.staterecover
package linea.staterecovery
import build.linea.clients.StateManagerClientV1
import build.linea.contract.l1.LineaRollupSmartContractClientReadOnly
@@ -7,12 +7,12 @@ import io.micrometer.core.instrument.simple.SimpleMeterRegistry
import io.vertx.core.Vertx
import io.vertx.junit5.VertxExtension
import linea.EthLogsSearcher
import linea.build.staterecover.clients.VertxTransactionDetailsClient
import linea.domain.RetryConfig
import linea.log4j.configureLoggers
import linea.staterecover.clients.blobscan.BlobScanClient
import linea.staterecover.test.FakeExecutionLayerClient
import linea.staterecover.test.FakeStateManagerClientReadFromL1
import linea.staterecovery.clients.VertxTransactionDetailsClient
import linea.staterecovery.clients.blobscan.BlobScanClient
import linea.staterecovery.test.FakeExecutionLayerClient
import linea.staterecovery.test.FakeStateManagerClientReadFromL1
import linea.web3j.Web3JLogsSearcher
import net.consensys.linea.BlockNumberAndHash
import net.consensys.linea.BlockParameter