Coordinator: replace teku ExecutionPlayloadV1 by Domain Block class - fix block encoding (#406)

* coordinator: replace Teku ExecutionPayloadV1.kt by our own Domain Block
* update GHA runners and CI fixes
---------
Signed-off-by: Pedro Novais <1478752+jpnovais@users.noreply.github.com>
Co-authored-by: Roman Vaseev <4833306+Filter94@users.noreply.github.com>
This commit is contained in:
Pedro Novais
2024-12-16 17:04:38 +00:00
committed by GitHub
parent eafe6870a9
commit 829630b0a1
97 changed files with 3055 additions and 1203 deletions

View File

@@ -51,7 +51,7 @@ concurrency:
jobs:
build-and-publish:
runs-on: [self-hosted, ubuntu-20.04, X64, small]
runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med
name: Coordinator build
env:
COMMIT_TAG: ${{ inputs.commit_tag }}
@@ -68,12 +68,14 @@ jobs:
echo "TAGS=${{ env.IMAGE_NAME }}:${{ env.COMMIT_TAG }},${{ env.IMAGE_NAME }}:${{ env.DEVELOP_TAG }}" >> $GITHUB_ENV
- name: Checkout
uses: actions/checkout@v4
- uses: actions/setup-java@v4
- uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b #v4.5.0
with:
distribution: temurin
java-version: 21
- name: Setup Gradle
uses: gradle/actions/setup-gradle@v4
# Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies.
# See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md
uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1
- name: Build dist
run: |
./gradlew coordinator:app:installDist --no-daemon

View File

@@ -26,17 +26,19 @@ jobs:
GITHUB_TOKEN: ${{ secrets._GITHUB_TOKEN_RELEASE_ACCESS }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
runs-on: [self-hosted, ubuntu-22.04, X64, medium]
runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-large
name: Coordinator tests
steps:
- name: Checkout
uses: actions/checkout@v4
- uses: actions/setup-java@v4
- uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b #v4.5.0
with:
distribution: temurin
java-version: 21
- name: Setup Gradle
uses: gradle/actions/setup-gradle@v4
# Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies.
# See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md
uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1
- name: Restore cached images
id: restore-cached-images
uses: actions/cache/restore@v4.0.2

View File

@@ -13,7 +13,7 @@ on:
jobs:
release:
runs-on: [self-hosted, ubuntu-20.04, X64, small]
runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med
steps:
- name: Checkout code
uses: actions/checkout@v4

View File

@@ -76,7 +76,7 @@ jobs:
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
outputs:
tests_outcome: ${{ steps.run_e2e_tests.outcome }}
runs-on: [self-hosted, ubuntu-20.04, X64, large]
runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-large
steps:
- name: Setup upterm session
if: ${{ inputs.e2e-tests-with-ssh }}
@@ -116,13 +116,16 @@ jobs:
make pull-images-external-to-monorepo
- name: Download local docker image artifacts
uses: actions/download-artifact@v4
with:
pattern: linea-*
- name: Load Docker images
run: |
gunzip -c /runner/_work/linea-monorepo/linea-monorepo/linea-coordinator/linea-coordinator-docker-image.tar.gz | docker load &&
gunzip -c /runner/_work/linea-monorepo/linea-monorepo/linea-postman/linea-postman-docker-image.tar.gz | docker load &&
gunzip -c /runner/_work/linea-monorepo/linea-monorepo/linea-prover/linea-prover-docker-image.tar.gz | docker load &&
gunzip -c /runner/_work/linea-monorepo/linea-monorepo/linea-traces-api-facade/linea-traces-api-facade-docker-image.tar.gz | docker load &&
gunzip -c /runner/_work/linea-monorepo/linea-monorepo/linea-transaction-exclusion-api/linea-transaction-exclusion-api-docker-image.tar.gz | docker load
pwd && ls -la && echo "GITHUB_WORKSPACE=$GITHUB_WORKSPACE" &&
gunzip -c $GITHUB_WORKSPACE/linea-coordinator/linea-coordinator-docker-image.tar.gz | docker load &&
gunzip -c $GITHUB_WORKSPACE/linea-postman/linea-postman-docker-image.tar.gz | docker load &&
gunzip -c $GITHUB_WORKSPACE/linea-prover/linea-prover-docker-image.tar.gz | docker load &&
gunzip -c $GITHUB_WORKSPACE/linea-traces-api-facade/linea-traces-api-facade-docker-image.tar.gz | docker load &&
gunzip -c $GITHUB_WORKSPACE/linea-transaction-exclusion-api/linea-transaction-exclusion-api-docker-image.tar.gz | docker load
shell: bash
- name: Spin up fresh environment with geth tracing with retry
if: ${{ inputs.tracing-engine == 'geth' }}

View File

@@ -51,7 +51,7 @@ concurrency:
jobs:
build-and-publish:
runs-on: [self-hosted, ubuntu-20.04, X64, small]
runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med
name: Traces api facade build
env:
COMMIT_TAG: ${{ inputs.commit_tag }}

View File

@@ -51,7 +51,7 @@ concurrency:
jobs:
build-and-publish:
runs-on: [self-hosted, ubuntu-20.04, X64, small]
runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med
name: Transaction exclusion api build
env:
COMMIT_TAG: ${{ inputs.commit_tag }}

View File

@@ -41,8 +41,6 @@ dependencies {
implementation project(':coordinator:persistence:batch')
implementation project(':coordinator:persistence:feehistory')
implementation project(':coordinator:persistence:db-common')
implementation project(":jvm-libs:linea:teku-execution-client")
implementation "tech.pegasys.teku.internal:bytes:${libs.versions.teku.get()}"
implementation project(':coordinator:ethereum:gas-pricing:static-cap')
implementation project(':coordinator:ethereum:gas-pricing:dynamic-cap')
@@ -66,8 +64,10 @@ dependencies {
implementation "com.fasterxml.jackson.module:jackson-module-kotlin:${libs.versions.jackson.get()}"
implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${libs.versions.jackson.get()}")
testImplementation "org.apache.logging.log4j:log4j-slf4j2-impl:${libs.versions.log4j.get()}"
testImplementation project(':jvm-libs:generic:serialization:jackson')
testImplementation testFixtures(project(':jvm-libs:linea:core:domain-models'))
testImplementation testFixtures(project(':jvm-libs:generic:json-rpc'))
testImplementation project(':coordinator:ethereum:test-utils')
testImplementation project(':jvm-libs:linea:testing:teku-helper')
testImplementation "io.vertx:vertx-junit5"
}

View File

@@ -1,9 +1,7 @@
package net.consensys.zkevm.coordinator.app
import com.fasterxml.jackson.databind.module.SimpleModule
import io.micrometer.core.instrument.MeterRegistry
import io.vertx.core.Vertx
import io.vertx.core.json.jackson.DatabindCodec
import io.vertx.micrometer.backends.BackendRegistries
import io.vertx.sqlclient.SqlClient
import net.consensys.linea.async.toSafeFuture
@@ -32,11 +30,9 @@ import net.consensys.zkevm.persistence.db.PersistenceRetryer
import org.apache.logging.log4j.Level
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
import org.apache.tuweni.bytes.Bytes
import org.web3j.protocol.Web3j
import org.web3j.protocol.http.HttpService
import org.web3j.utils.Async
import tech.pegasys.teku.ethereum.executionclient.serialization.BytesSerializer
import tech.pegasys.teku.infrastructure.async.SafeFuture
import kotlin.time.toKotlinDuration
@@ -48,12 +44,6 @@ class CoordinatorApp(private val configs: CoordinatorConfig) {
log.debug("Vertx full configs: {}", vertxConfig)
log.info("App configs: {}", configs)
// TODO: adapt JsonMessageProcessor to use custom ObjectMapper
// this is just dark magic.
val module = SimpleModule()
module.addSerializer(Bytes::class.java, BytesSerializer())
DatabindCodec.mapper().registerModule(module)
// .enable(SerializationFeature.INDENT_OUTPUT)
Vertx.vertx(vertxConfig)
}
private val meterRegistry: MeterRegistry = BackendRegistries.getDefaultNow()

View File

@@ -6,6 +6,7 @@ import build.linea.contract.l1.LineaRollupSmartContractClientReadOnly
import build.linea.contract.l1.Web3JLineaRollupSmartContractClientReadOnly
import io.vertx.core.Vertx
import kotlinx.datetime.Clock
import linea.encoding.BlockRLPEncoder
import net.consensys.linea.BlockNumberAndHash
import net.consensys.linea.blob.ShnarfCalculatorVersion
import net.consensys.linea.contract.LineaRollupAsyncFriendly
@@ -56,7 +57,6 @@ import net.consensys.zkevm.coordinator.clients.smartcontract.LineaRollupSmartCon
import net.consensys.zkevm.domain.BlobSubmittedEvent
import net.consensys.zkevm.domain.BlocksConflation
import net.consensys.zkevm.domain.FinalizationSubmittedEvent
import net.consensys.zkevm.encoding.ExecutionPayloadV1RLPEncoderByBesuImplementation
import net.consensys.zkevm.ethereum.coordination.EventDispatcher
import net.consensys.zkevm.ethereum.coordination.HighestConflationTracker
import net.consensys.zkevm.ethereum.coordination.HighestProvenBatchTracker
@@ -323,7 +323,7 @@ class L1DependentApp(
lastBlockNumber = lastProcessedBlockNumber,
clock = Clock.System,
latestBlockProvider = GethCliqueSafeBlockProvider(
l2ExtendedWeb3j,
l2ExtendedWeb3j.web3jClient,
GethCliqueSafeBlockProvider.Config(configs.l2.blocksToFinalization.toLong())
)
)
@@ -608,7 +608,7 @@ class L1DependentApp(
deadlineCheckInterval = configs.proofAggregation.deadlineCheckInterval.toKotlinDuration(),
aggregationDeadline = configs.proofAggregation.aggregationDeadline.toKotlinDuration(),
latestBlockProvider = GethCliqueSafeBlockProvider(
l2ExtendedWeb3j,
l2ExtendedWeb3j.web3jClient,
GethCliqueSafeBlockProvider.Config(configs.l2.blocksToFinalization.toLong())
),
maxProofsPerAggregation = configs.proofAggregation.aggregationProofsLimit.toUInt(),
@@ -868,7 +868,7 @@ class L1DependentApp(
conflationService = conflationService,
tracesCountersClient = tracesCountersClient,
vertx = vertx,
payloadEncoder = ExecutionPayloadV1RLPEncoderByBesuImplementation
encoder = BlockRLPEncoder
)
}
@@ -904,7 +904,7 @@ class L1DependentApp(
log.info("Resuming conflation from block={} inclusive", lastProcessedBlockNumber + 1UL)
val blockCreationMonitor = BlockCreationMonitor(
vertx = vertx,
extendedWeb3j = l2ExtendedWeb3j,
web3j = l2ExtendedWeb3j,
startingBlockNumberExclusive = lastProcessedBlockNumber.toLong(),
blockCreationListener = block2BatchCoordinator,
lastProvenBlockNumberProviderAsync = lastProvenBlockNumberProvider,

View File

@@ -1,6 +1,9 @@
package net.consensys.zkevm.coordinator.blockcreation
import io.vertx.core.Vertx
import linea.domain.Block
import net.consensys.encodeHex
import net.consensys.linea.BlockParameter.Companion.toBlockParameter
import net.consensys.linea.async.AsyncRetryer
import net.consensys.linea.web3j.ExtendedWeb3J
import net.consensys.zkevm.PeriodicPollingService
@@ -8,10 +11,6 @@ import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockCreated
import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockCreationListener
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
import org.apache.tuweni.bytes.Bytes32
import org.web3j.protocol.core.DefaultBlockParameter
import org.web3j.protocol.core.methods.response.EthBlock
import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1
import tech.pegasys.teku.infrastructure.async.SafeFuture
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.atomic.AtomicLong
@@ -21,7 +20,7 @@ import kotlin.time.Duration.Companion.days
class BlockCreationMonitor(
private val vertx: Vertx,
private val extendedWeb3j: ExtendedWeb3J,
private val web3j: ExtendedWeb3J,
private val startingBlockNumberExclusive: Long,
private val blockCreationListener: BlockCreationListener,
private val lastProvenBlockNumberProviderAsync: LastProvenBlockNumberProviderAsync,
@@ -37,11 +36,11 @@ class BlockCreationMonitor(
val blocksToFinalization: Long,
val blocksFetchLimit: Long,
val startingBlockWaitTimeout: Duration = 14.days,
val lastL2BlockNumberToProcessInclusive: ULong?
val lastL2BlockNumberToProcessInclusive: ULong? = null
)
private val _nexBlockNumberToFetch: AtomicLong = AtomicLong(startingBlockNumberExclusive + 1)
private val expectedParentBlockHash: AtomicReference<Bytes32> = AtomicReference(null)
private val expectedParentBlockHash: AtomicReference<ByteArray> = AtomicReference(null)
private val reorgDetected: AtomicBoolean = AtomicBoolean(false)
private var statingBlockAvailabilityFuture: SafeFuture<*>? = null
@@ -72,8 +71,8 @@ class BlockCreationMonitor(
vertx,
backoffDelay = config.pollingInterval,
timeout = config.startingBlockWaitTimeout,
stopRetriesPredicate = { block: EthBlock ->
if (block.block == null) {
stopRetriesPredicate = { block: Block? ->
if (block == null) {
log.warn(
"Block {} not found yet. Retrying in {}",
startingBlockNumberExclusive,
@@ -82,19 +81,12 @@ class BlockCreationMonitor(
false
} else {
log.info("Block {} found. Resuming block monitor", startingBlockNumberExclusive)
expectedParentBlockHash.set(Bytes32.fromHexString(block.block.hash))
expectedParentBlockHash.set(block.hash)
true
}
}
) {
SafeFuture.of(
extendedWeb3j.web3jClient
.ethGetBlockByNumber(
DefaultBlockParameter.valueOf(startingBlockNumberExclusive.toBigInteger()),
false
)
.sendAsync()
)
web3j.ethGetBlock(startingBlockNumberExclusive.toBlockParameter())
}
}
@@ -102,7 +94,7 @@ class BlockCreationMonitor(
}
override fun action(): SafeFuture<*> {
log.trace("tick start")
log.trace("tick start: nexBlockNumberToFetch={}", nexBlockNumberToFetch)
return lastProvenBlockNumberProviderAsync.getLastProvenBlockNumber()
.thenCompose { lastProvenBlockNumber ->
if (!nextBlockNumberWithinLimit(lastProvenBlockNumber)) {
@@ -119,8 +111,8 @@ class BlockCreationMonitor(
nexBlockNumberToFetch.toULong() > config.lastL2BlockNumberToProcessInclusive
) {
log.warn(
"Stopping Conflation, Blob and Aggregation at lastL2BlockNumberInclusiveToProcess - 1. " +
"All blocks unto and including lastL2BlockNumberInclusiveToProcess={} have been processed. " +
"stopping conflation at lastL2BlockNumberInclusiveToProcess - 1. " +
"All blocks upto and including lastL2BlockNumberInclusiveToProcess={} have been processed. " +
"nextBlockNumberToFetch={}",
config.lastL2BlockNumberToProcessInclusive,
nexBlockNumberToFetch
@@ -128,29 +120,29 @@ class BlockCreationMonitor(
SafeFuture.COMPLETE
} else {
getNetNextSafeBlock()
.thenCompose { payload ->
if (payload != null) {
if (payload.parentHash == expectedParentBlockHash.get()) {
notifyListener(payload)
.thenCompose { block ->
if (block != null) {
if (block.parentHash.contentEquals(expectedParentBlockHash.get())) {
notifyListener(block)
.whenSuccess {
log.debug(
"updating nexBlockNumberToFetch from {} --> {}",
_nexBlockNumberToFetch.get(),
_nexBlockNumberToFetch.incrementAndGet()
)
expectedParentBlockHash.set(payload.blockHash)
expectedParentBlockHash.set(block.hash)
}
} else {
reorgDetected.set(true)
log.error(
"Shooting down conflation poller, " +
"chain reorg detected: block { blockNumber={} hash={} parentHash={} } should have parentHash={}",
payload.blockNumber.longValue(),
payload.blockHash.toHexString().subSequence(0, 8),
payload.parentHash.toHexString().subSequence(0, 8),
expectedParentBlockHash.get().toHexString().subSequence(0, 8)
block.number,
block.hash.encodeHex(),
block.parentHash.encodeHex(),
expectedParentBlockHash.get().encodeHex()
)
SafeFuture.failedFuture(IllegalStateException("Reorg detected on block ${payload.blockNumber}"))
SafeFuture.failedFuture(IllegalStateException("Reorg detected on block ${block.number}"))
}
} else {
SafeFuture.completedFuture(Unit)
@@ -165,26 +157,28 @@ class BlockCreationMonitor(
}
}
private fun notifyListener(payload: ExecutionPayloadV1): SafeFuture<Unit> {
return blockCreationListener.acceptBlock(BlockCreated(payload))
private fun notifyListener(payload: Block): SafeFuture<Unit> {
log.trace("notifying blockCreationListener: block={}", payload.number)
return blockCreationListener
.acceptBlock(BlockCreated(payload))
.thenApply {
log.debug(
"blockCreationListener blockNumber={} resolved with success",
payload.blockNumber
payload.number
)
}
.whenException { throwable ->
log.warn(
"Failed to notify blockCreationListener: blockNumber={} errorMessage={}",
payload.blockNumber.bigIntegerValue(),
payload.number,
throwable.message,
throwable
)
}
}
private fun getNetNextSafeBlock(): SafeFuture<ExecutionPayloadV1?> {
return extendedWeb3j
private fun getNetNextSafeBlock(): SafeFuture<Block?> {
return web3j
.ethBlockNumber()
.thenCompose { latestBlockNumber ->
// Check if is safe to fetch nextWaitingBlockNumber
@@ -192,9 +186,12 @@ class BlockCreationMonitor(
_nexBlockNumberToFetch.get() + config.blocksToFinalization
) {
val blockNumber = _nexBlockNumberToFetch.get()
extendedWeb3j.ethGetExecutionPayloadByNumber(blockNumber)
web3j.ethGetBlock(blockNumber.toBlockParameter())
.thenPeek { block ->
log.trace("requestedBock={} responselock={}", blockNumber, block?.number)
}
.whenException {
log.error(
log.warn(
"eth_getBlockByNumber({}) failed: errorMessage={}",
blockNumber,
it.message,

View File

@@ -1,27 +1,31 @@
package net.consensys.zkevm.coordinator.blockcreation
import net.consensys.linea.web3j.ExtendedWeb3J
import build.linea.web3j.domain.toWeb3j
import linea.domain.Block
import linea.web3j.toDomain
import net.consensys.linea.BlockParameter.Companion.toBlockParameter
import net.consensys.linea.async.toSafeFuture
import net.consensys.zkevm.ethereum.coordination.blockcreation.SafeBlockProvider
import org.web3j.protocol.Web3j
import org.web3j.protocol.core.DefaultBlockParameterName
import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1
import tech.pegasys.teku.infrastructure.async.SafeFuture
class GethCliqueSafeBlockProvider(
private val extendedWeb3j: ExtendedWeb3J,
private val web3j: Web3j,
private val config: Config
) : SafeBlockProvider {
data class Config(
val blocksToFinalization: Long
)
override fun getLatestSafeBlock(): SafeFuture<ExecutionPayloadV1> {
return SafeFuture.of(
extendedWeb3j.web3jClient
.ethGetBlockByNumber(DefaultBlockParameterName.LATEST, false).sendAsync()
)
override fun getLatestSafeBlock(): SafeFuture<Block> {
return web3j
.ethGetBlockByNumber(DefaultBlockParameterName.LATEST, false).sendAsync()
.toSafeFuture()
.thenCompose { block ->
val safeBlockNumber = (block.block.number.toLong() - config.blocksToFinalization).coerceAtLeast(0)
extendedWeb3j.ethGetExecutionPayloadByNumber(safeBlockNumber)
web3j.ethGetBlockByNumber(safeBlockNumber.toBlockParameter().toWeb3j(), true).sendAsync().toSafeFuture()
}
.thenApply { it.block.toDomain() }
}
}

View File

@@ -34,7 +34,7 @@ class HighestProvenBatchTracker(initialProvenBlockNumber: ULong) :
class HighestConflationTracker(initialProvenBlockNumber: ULong) :
MaxLongTracker<BlocksConflation>(initialProvenBlockNumber.toLong()) {
override fun convertToLong(trackable: BlocksConflation): Long {
return trackable.blocks.last().blockNumber.longValue()
return trackable.blocks.last().number.toLong()
}
}

View File

@@ -1,14 +1,24 @@
package net.consensys.zkevm.coordinator.blockcreation
import build.linea.s11n.jackson.ethApiObjectMapper
import io.vertx.core.Vertx
import io.vertx.junit5.VertxExtension
import io.vertx.junit5.VertxTestContext
import linea.domain.Block
import linea.domain.createBlock
import linea.domain.toEthGetBlockResponse
import linea.jsonrpc.TestingJsonRpcServer
import linea.log4j.configureLoggers
import linea.web3j.createWeb3jHttpClient
import net.consensys.ByteArrayExt
import net.consensys.decodeHex
import net.consensys.linea.async.get
import net.consensys.linea.web3j.ExtendedWeb3J
import net.consensys.linea.web3j.ExtendedWeb3JImpl
import net.consensys.toHexString
import net.consensys.toULongFromHex
import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockCreated
import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockCreationListener
import org.apache.logging.log4j.Level
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
import org.assertj.core.api.Assertions.assertThat
import org.awaitility.Awaitility.await
@@ -16,93 +26,86 @@ import org.junit.jupiter.api.AfterEach
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.extension.ExtendWith
import org.mockito.ArgumentMatchers.eq
import org.mockito.Mockito.atLeastOnce
import org.mockito.kotlin.any
import org.mockito.kotlin.atLeast
import org.mockito.kotlin.atMost
import org.mockito.kotlin.doReturn
import org.mockito.kotlin.mock
import org.mockito.kotlin.never
import org.mockito.kotlin.times
import org.mockito.kotlin.verify
import org.mockito.kotlin.whenever
import org.web3j.protocol.Web3j
import org.web3j.protocol.core.Request
import org.web3j.protocol.core.methods.response.EthBlock
import tech.pegasys.teku.ethereum.executionclient.schema.executionPayloadV1
import tech.pegasys.teku.infrastructure.async.SafeFuture
import java.math.BigInteger
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import kotlin.time.Duration
import java.util.concurrent.atomic.AtomicLong
import kotlin.time.Duration.Companion.milliseconds
import kotlin.time.Duration.Companion.seconds
import kotlin.time.toJavaDuration
@ExtendWith(VertxExtension::class)
class BlockCreationMonitorTest {
private val parentHash = "0x1000000000000000000000000000000000000000000000000000000000000000".decodeHex()
private val startingBlockNumberInclusive: Long = 100
private val blocksToFetch: Long = 5L
private val lastBlockNumberInclusiveToProcess: ULong = startingBlockNumberInclusive.toULong() + 10uL
private lateinit var log: Logger
private lateinit var web3jNativeClientMock: Web3j
private lateinit var web3jClient: ExtendedWeb3J
private lateinit var blockCreationListener: BlockCreationListener
private var lastProvenBlock: Long = startingBlockNumberInclusive
private lateinit var blockCreationListener: BlockCreationListenerDouble
private var config: BlockCreationMonitor.Config =
BlockCreationMonitor.Config(
pollingInterval = 100.milliseconds,
blocksToFinalization = 2L,
blocksFetchLimit = blocksToFetch,
lastL2BlockNumberToProcessInclusive = lastBlockNumberInclusiveToProcess
blocksFetchLimit = 500,
lastL2BlockNumberToProcessInclusive = null
)
private lateinit var vertx: Vertx
private val executor = Executors.newSingleThreadScheduledExecutor()
private lateinit var lastProvenBlockNumberProvider: LastProvenBlockNumberProviderDouble
private lateinit var monitor: BlockCreationMonitor
private lateinit var fakeL2RpcNode: TestingJsonRpcServer
private class BlockCreationListenerDouble() : BlockCreationListener {
val blocksReceived: MutableList<Block> = CopyOnWriteArrayList()
override fun acceptBlock(blockEvent: BlockCreated): SafeFuture<Unit> {
blocksReceived.add(blockEvent.block)
return SafeFuture.completedFuture(Unit)
}
}
private class LastProvenBlockNumberProviderDouble(
initialValue: ULong
) : LastProvenBlockNumberProviderAsync {
var lastProvenBlock: AtomicLong = AtomicLong(initialValue.toLong())
override fun getLastProvenBlockNumber(): SafeFuture<Long> {
return SafeFuture.completedFuture(lastProvenBlock.get())
}
}
fun createBlockCreationMonitor(
startingBlockNumberExclusive: Long = 99,
blockCreationListener: BlockCreationListener = this.blockCreationListener,
config: BlockCreationMonitor.Config = this.config
): BlockCreationMonitor {
return BlockCreationMonitor(
this.vertx,
web3jClient,
startingBlockNumberExclusive = startingBlockNumberExclusive,
blockCreationListener,
lastProvenBlockNumberProvider,
config
)
}
@BeforeEach
fun beforeEach(vertx: Vertx) {
val ethGetBlockByNumberMock: Request<Any, EthBlock> = mock {
on { sendAsync() } doReturn SafeFuture.completedFuture(EthBlock())
on { sendAsync() } doReturn SafeFuture.completedFuture(EthBlock())
on { sendAsync() } doReturn SafeFuture.completedFuture(
EthBlock().apply {
result = EthBlock.Block()
.apply {
setNumber("0x63")
hash = "0x1000000000000000000000000000000000000000000000000000000000000000"
}
}
)
}
web3jNativeClientMock = mock {
on { ethGetBlockByNumber(any(), any()) } doReturn ethGetBlockByNumberMock
}
web3jClient = mock {
on { web3jClient } doReturn web3jNativeClientMock
}
blockCreationListener =
mock { on { acceptBlock(any()) } doReturn SafeFuture.completedFuture(Unit) }
configureLoggers(Level.INFO, "test.client.l2.web3j" to Level.TRACE)
this.vertx = vertx
log = mock()
val lastProvenBlockNumberProviderAsync = object : LastProvenBlockNumberProviderAsync {
override fun getLastProvenBlockNumber(): SafeFuture<Long> {
return SafeFuture.completedFuture(lastProvenBlock)
}
}
monitor =
BlockCreationMonitor(
vertx,
web3jClient,
startingBlockNumberExclusive = startingBlockNumberInclusive - 1,
blockCreationListener,
lastProvenBlockNumberProviderAsync,
config
fakeL2RpcNode = TestingJsonRpcServer(
vertx = vertx,
recordRequestsResponses = true,
responseObjectMapper = ethApiObjectMapper
)
blockCreationListener = BlockCreationListenerDouble()
web3jClient = ExtendedWeb3JImpl(
createWeb3jHttpClient(
rpcUrl = "http://localhost:${fakeL2RpcNode.boundPort}",
log = LogManager.getLogger("test.client.l2.web3j")
)
)
lastProvenBlockNumberProvider = LastProvenBlockNumberProviderDouble(99u)
}
@AfterEach
@@ -111,427 +114,259 @@ class BlockCreationMonitorTest {
vertx.close().get()
}
@Test
fun `skip blocks after lastBlockNumberInclusiveToProcess`(vertx: Vertx, testContext: VertxTestContext) {
val lastProvenBlockNumberProviderAsync = mock<LastProvenBlockNumberProviderAsync>()
whenever(lastProvenBlockNumberProviderAsync.getLastProvenBlockNumber()).thenAnswer {
SafeFuture.completedFuture((lastBlockNumberInclusiveToProcess - 2uL).toLong())
fun createBlocks(
startBlockNumber: ULong,
numberOfBlocks: Int,
startBlockHash: ByteArray = ByteArrayExt.random32(),
startBlockParentHash: ByteArray = ByteArrayExt.random32()
): List<Block> {
var blockHash = startBlockHash
var parentHash = startBlockParentHash
return (0..numberOfBlocks).map { i ->
createBlock(
number = startBlockNumber + i.toULong(),
hash = blockHash,
parentHash = parentHash
).also {
blockHash = ByteArrayExt.random32()
parentHash = it.hash
}
}
}
private fun setupFakeExecutionLayerWithBlocks(blocks: List<Block>) {
fakeL2RpcNode.handle("eth_getBlockByNumber") { request ->
val blockNumber = ((request.params as List<Any?>)[0] as String).toULongFromHex()
blocks.find { it.number == blockNumber }?.toEthGetBlockResponse()
}
monitor =
BlockCreationMonitor(
vertx,
web3jClient,
startingBlockNumberExclusive = (lastBlockNumberInclusiveToProcess - 2uL).toLong(),
blockCreationListener,
lastProvenBlockNumberProviderAsync,
config
)
val payload =
executionPayloadV1(blockNumber = lastBlockNumberInclusiveToProcess.toLong() - 1, parentHash = parentHash)
val payload2 =
executionPayloadV1(
blockNumber = lastBlockNumberInclusiveToProcess.toLong(),
parentHash = payload.blockHash.toArray()
)
val payload3 =
executionPayloadV1(
blockNumber = lastBlockNumberInclusiveToProcess.toLong() + 1,
parentHash = payload2.blockHash.toArray()
)
val headBlockNumber = lastBlockNumberInclusiveToProcess.toLong() + config.blocksToFinalization
whenever(web3jClient.ethBlockNumber())
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber)))
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 1)))
whenever(web3jClient.ethGetExecutionPayloadByNumber(any()))
.thenReturn(SafeFuture.completedFuture(payload))
.thenReturn(SafeFuture.completedFuture(payload2))
.thenReturn(SafeFuture.completedFuture(payload3))
whenever(blockCreationListener.acceptBlock(any())).thenReturn(SafeFuture.completedFuture(Unit))
monitor.start().thenApply {
await()
.untilAsserted {
verify(lastProvenBlockNumberProviderAsync, atLeast(3)).getLastProvenBlockNumber()
verify(web3jClient).ethGetExecutionPayloadByNumber(eq(lastBlockNumberInclusiveToProcess.toLong() - 1))
verify(web3jClient).ethGetExecutionPayloadByNumber(eq(lastBlockNumberInclusiveToProcess.toLong()))
verify(web3jClient, never()).ethGetExecutionPayloadByNumber(
eq(lastBlockNumberInclusiveToProcess.toLong() + 1)
)
verify(blockCreationListener, times(1)).acceptBlock(BlockCreated(payload))
verify(blockCreationListener, times(1)).acceptBlock(BlockCreated(payload2))
verify(blockCreationListener, never()).acceptBlock(BlockCreated(payload3))
assertThat(monitor.nexBlockNumberToFetch).isEqualTo(lastBlockNumberInclusiveToProcess.toLong() + 1)
}
testContext.completeNow()
fakeL2RpcNode.handle("eth_blockNumber") { _ ->
blocks.last().number.toHexString()
}
.whenException(testContext::failNow)
}
@Test
fun `notifies listener after block is finalized sync`(vertx: Vertx, testContext: VertxTestContext) {
val payload =
executionPayloadV1(blockNumber = startingBlockNumberInclusive, parentHash = parentHash)
val payload2 =
executionPayloadV1(
blockNumber = startingBlockNumberInclusive + 1,
parentHash = payload.blockHash.toArray()
)
val headBlockNumber = startingBlockNumberInclusive + config.blocksToFinalization
whenever(web3jClient.ethBlockNumber())
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber)))
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 1)))
whenever(web3jClient.ethGetExecutionPayloadByNumber(any()))
.thenReturn(SafeFuture.completedFuture(payload))
.thenReturn(SafeFuture.completedFuture(payload2))
whenever(blockCreationListener.acceptBlock(any())).thenReturn(SafeFuture.completedFuture(Unit))
val lastProvenBlockNumberProviderAsync = mock<LastProvenBlockNumberProviderAsync>()
val monitor =
BlockCreationMonitor(
vertx,
web3jClient,
startingBlockNumberExclusive = startingBlockNumberInclusive - 1,
blockCreationListener,
lastProvenBlockNumberProviderAsync,
config
)
whenever(lastProvenBlockNumberProviderAsync.getLastProvenBlockNumber()).thenAnswer {
SafeFuture.completedFuture(lastProvenBlock)
}
monitor.start().thenApply {
await()
.untilAsserted {
verify(lastProvenBlockNumberProviderAsync, atLeast(3)).getLastProvenBlockNumber()
verify(web3jClient).ethGetExecutionPayloadByNumber(eq(startingBlockNumberInclusive))
verify(web3jClient).ethGetExecutionPayloadByNumber(eq(startingBlockNumberInclusive + 1))
verify(blockCreationListener).acceptBlock(BlockCreated(payload))
verify(blockCreationListener).acceptBlock(BlockCreated(payload2))
assertThat(monitor.nexBlockNumberToFetch).isEqualTo(startingBlockNumberInclusive + 2)
}
testContext.completeNow()
}
.whenException(testContext::failNow)
}
@Test
fun `does not notify listener when block is not safely finalized`(testContext: VertxTestContext) {
val payload =
executionPayloadV1(blockNumber = startingBlockNumberInclusive, parentHash = parentHash)
val headBlockNumber = startingBlockNumberInclusive + config.blocksToFinalization - 1
whenever(web3jClient.ethBlockNumber())
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber)))
whenever(web3jClient.ethGetExecutionPayloadByNumber(any()))
.thenReturn(SafeFuture.completedFuture(payload))
whenever(blockCreationListener.acceptBlock(any())).thenReturn(SafeFuture.completedFuture(Unit))
monitor.start().thenApply {
await()
.timeout(1.seconds.toJavaDuration())
.untilAsserted {
verify(web3jClient, never()).ethGetExecutionPayloadByNumber(any())
verify(blockCreationListener, never()).acceptBlock(BlockCreated(payload))
assertThat(monitor.nexBlockNumberToFetch).isEqualTo(startingBlockNumberInclusive)
}
testContext.completeNow()
}
.whenException(testContext::failNow)
}
@Test
fun `when listener throws retries on the next tick and moves on`(testContext: VertxTestContext) {
val payload =
executionPayloadV1(blockNumber = startingBlockNumberInclusive, parentHash = parentHash)
val headBlockNumber = startingBlockNumberInclusive + config.blocksToFinalization + 1
whenever(web3jClient.ethBlockNumber())
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber)))
whenever(web3jClient.ethGetExecutionPayloadByNumber(any()))
.thenReturn(SafeFuture.completedFuture(payload))
whenever(blockCreationListener.acceptBlock(any()))
.thenReturn(SafeFuture.failedFuture(Exception("Notification 1 Error")))
.thenThrow(RuntimeException("Notification 2 Error"))
.thenReturn(SafeFuture.failedFuture(Exception("Notification 3 Error")))
.thenReturn(SafeFuture.completedFuture(Unit))
monitor.start().thenApply {
await()
.timeout(5.seconds.toJavaDuration())
.untilAsserted {
verify(blockCreationListener, atLeast(4)).acceptBlock(BlockCreated(payload))
assertThat(monitor.nexBlockNumberToFetch).isEqualTo(startingBlockNumberInclusive + 1)
}
testContext.completeNow()
}
.whenException(testContext::failNow)
}
@Test
fun `is resilient to connection failures`(testContext: VertxTestContext) {
val payload =
executionPayloadV1(blockNumber = startingBlockNumberInclusive, parentHash = parentHash)
val headBlockNumber = startingBlockNumberInclusive + config.blocksToFinalization
whenever(web3jClient.ethBlockNumber())
.thenReturn(SafeFuture.failedFuture(Exception("ethBlockNumber Error 1")))
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber)))
whenever(web3jClient.ethGetExecutionPayloadByNumber(any()))
.thenReturn(SafeFuture.failedFuture(Exception("ethGetExecutionPayloadByNumber Error 1")))
.thenReturn(SafeFuture.completedFuture(payload))
whenever(blockCreationListener.acceptBlock(any())).thenReturn(SafeFuture.completedFuture(Unit))
monitor.start().thenApply {
await()
.timeout(1.seconds.toJavaDuration())
.untilAsserted {
verify(blockCreationListener, times(1)).acceptBlock(BlockCreated(payload))
assertThat(monitor.nexBlockNumberToFetch).isEqualTo(startingBlockNumberInclusive + 1)
}
testContext.completeNow()
}
.whenException(testContext::failNow)
}
@Test
fun `should stop when reorg is detected above blocksToFinalization limit - manual intervention necessary`(
testContext: VertxTestContext
) {
val payload =
executionPayloadV1(blockNumber = startingBlockNumberInclusive, parentHash = parentHash)
val payload2 =
executionPayloadV1(
blockNumber = startingBlockNumberInclusive + 1,
parentHash = ByteArrayExt.random32()
)
val headBlockNumber = startingBlockNumberInclusive + config.blocksToFinalization
whenever(web3jClient.ethBlockNumber())
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber)))
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 1)))
whenever(web3jClient.ethGetExecutionPayloadByNumber(any()))
.thenReturn(SafeFuture.completedFuture(payload))
.thenReturn(SafeFuture.completedFuture(payload2))
whenever(blockCreationListener.acceptBlock(any())).thenReturn(SafeFuture.completedFuture(Unit))
monitor.start().thenApply {
await()
.timeout(1.seconds.toJavaDuration())
.untilAsserted {
verify(blockCreationListener, times(1)).acceptBlock(BlockCreated(payload))
verify(blockCreationListener, never()).acceptBlock(BlockCreated(payload2))
assertThat(monitor.nexBlockNumberToFetch).isEqualTo(startingBlockNumberInclusive + 1)
}
testContext.completeNow()
}
.whenException(testContext::failNow)
}
private fun <V> delay(delay: Duration, action: () -> SafeFuture<V>): SafeFuture<V> {
val future = SafeFuture<V>()
executor.schedule(
{ action().propagateTo(future) },
delay.inWholeMilliseconds,
TimeUnit.MILLISECONDS
fun `should stop fetching blocks after lastBlockNumberInclusiveToProcess`() {
monitor = createBlockCreationMonitor(
startingBlockNumberExclusive = 99,
config = config.copy(lastL2BlockNumberToProcessInclusive = 103u)
)
return future
setupFakeExecutionLayerWithBlocks(createBlocks(startBlockNumber = 99u, numberOfBlocks = 20))
monitor.start()
await()
.atMost(20.seconds.toJavaDuration())
.untilAsserted {
assertThat(blockCreationListener.blocksReceived).isNotEmpty
assertThat(blockCreationListener.blocksReceived.last().number).isGreaterThanOrEqualTo(103u)
}
// Wait for a while to make sure no more blocks are fetched
await().atLeast(config.pollingInterval.times(3).toJavaDuration())
assertThat(blockCreationListener.blocksReceived.last().number).isEqualTo(103UL)
}
@Test
fun `should poll in order when response takes longer that polling interval`(testContext: VertxTestContext) {
val payload =
executionPayloadV1(blockNumber = startingBlockNumberInclusive, parentHash = parentHash)
val payload2 =
executionPayloadV1(
blockNumber = startingBlockNumberInclusive + 1,
parentHash = payload.blockHash.toArray()
)
val headBlockNumber = startingBlockNumberInclusive + config.blocksToFinalization
fun `should notify lister only after block is considered final on L2`() {
monitor = createBlockCreationMonitor(
startingBlockNumberExclusive = 99,
config = config.copy(blocksToFinalization = 2, blocksFetchLimit = 500)
)
whenever(web3jClient.ethBlockNumber())
.then {
delay(config.pollingInterval.times(2)) {
SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber))
setupFakeExecutionLayerWithBlocks(createBlocks(startBlockNumber = 99u, numberOfBlocks = 200))
fakeL2RpcNode.handle("eth_blockNumber") { _ -> 105UL.toHexString() }
// latest eligible conflation is: 105 - 2 = 103, inclusive
monitor.start()
await()
.atMost(20.seconds.toJavaDuration())
.untilAsserted {
assertThat(blockCreationListener.blocksReceived).isNotEmpty
assertThat(blockCreationListener.blocksReceived.last().number).isGreaterThanOrEqualTo(103u)
}
// Wait for a while to make sure no more blocks are fetched
await().atLeast(config.pollingInterval.times(3).toJavaDuration())
// assert that no more block were sent to the listener
assertThat(blockCreationListener.blocksReceived.last().number).isEqualTo(103UL)
// move chain head forward
fakeL2RpcNode.handle("eth_blockNumber") { _ -> 120UL.toHexString() }
// assert it resumes conflation
await()
.atMost(20.seconds.toJavaDuration())
.untilAsserted {
assertThat(blockCreationListener.blocksReceived.last().number).isEqualTo(118UL)
}
}
@Test
fun `shall retry notify the listener when it throws and keeps block order`() {
val fakeBuggyLister = object : BlockCreationListener {
var errorCount = 0
override fun acceptBlock(blockEvent: BlockCreated): SafeFuture<Unit> {
return if (blockEvent.block.number == 105UL && errorCount < 3) {
errorCount++
throw RuntimeException("Error on block 105")
} else {
blockCreationListener.acceptBlock(blockEvent)
}
}
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 1)))
whenever(web3jClient.ethGetExecutionPayloadByNumber(any()))
.then { delay(config.pollingInterval.times(2)) { SafeFuture.completedFuture(payload) } }
.thenReturn(SafeFuture.completedFuture(payload2))
whenever(blockCreationListener.acceptBlock(any())).thenReturn(SafeFuture.completedFuture(Unit))
monitor.start().thenApply {
await()
.untilAsserted {
verify(blockCreationListener).acceptBlock(BlockCreated(payload))
verify(blockCreationListener).acceptBlock(BlockCreated(payload2))
assertThat(monitor.nexBlockNumberToFetch).isEqualTo(startingBlockNumberInclusive + 2)
}
testContext.completeNow()
}
.whenException(testContext::failNow)
monitor = createBlockCreationMonitor(
startingBlockNumberExclusive = 99,
blockCreationListener = fakeBuggyLister,
config = config.copy(blocksToFinalization = 2, lastL2BlockNumberToProcessInclusive = 112u)
)
setupFakeExecutionLayerWithBlocks(createBlocks(startBlockNumber = 99u, numberOfBlocks = 20))
monitor.start()
await()
.atMost(20.seconds.toJavaDuration())
.untilAsserted {
assertThat(blockCreationListener.blocksReceived).isNotEmpty
assertThat(blockCreationListener.blocksReceived.last().number).isGreaterThanOrEqualTo(110u)
}
// assert it got block only once and in order
assertThat(blockCreationListener.blocksReceived.map { it.number }).containsExactly(
100UL, 101UL, 102UL, 103UL, 104UL, 105UL, 106UL, 107UL, 108UL, 109UL, 110UL
)
}
@Test
fun `should be resilient to connection failures`() {
monitor = createBlockCreationMonitor(
startingBlockNumberExclusive = 99
)
setupFakeExecutionLayerWithBlocks(createBlocks(startBlockNumber = 99u, numberOfBlocks = 200))
monitor.start()
await()
.atMost(20.seconds.toJavaDuration())
.untilAsserted {
assertThat(blockCreationListener.blocksReceived).isNotEmpty
assertThat(blockCreationListener.blocksReceived.last().number).isGreaterThanOrEqualTo(103u)
}
fakeL2RpcNode.stopHttpServer()
val lastBlockReceived = blockCreationListener.blocksReceived.last().number
// Wait for a while to make sure no more blocks are fetched
await().atLeast(config.pollingInterval.times(2).toJavaDuration())
fakeL2RpcNode.resumeHttpServer()
await()
.atMost(20.seconds.toJavaDuration())
.untilAsserted {
assertThat(blockCreationListener.blocksReceived).isNotEmpty
assertThat(blockCreationListener.blocksReceived.last().number).isGreaterThan(lastBlockReceived)
}
}
@Test
fun `should stop when reorg is detected above blocksToFinalization limit - manual intervention necessary`() {
monitor = createBlockCreationMonitor(
startingBlockNumberExclusive = 99
)
// simulate reorg by changing parent hash of block 105
val blocks = createBlocks(startBlockNumber = 99u, numberOfBlocks = 20).map { block: Block ->
if (block.number == 105UL) {
block.copy(parentHash = ByteArrayExt.random32())
} else {
block
}
}
setupFakeExecutionLayerWithBlocks(blocks)
monitor.start()
await()
.atMost(20.seconds.toJavaDuration())
.untilAsserted {
assertThat(blockCreationListener.blocksReceived).isNotEmpty
assertThat(blockCreationListener.blocksReceived.last().number).isGreaterThanOrEqualTo(104UL)
}
// Wait for a while to make sure no more blocks are fetched
await().atLeast(config.pollingInterval.times(3).toJavaDuration())
assertThat(blockCreationListener.blocksReceived.last().number).isEqualTo(104UL)
}
@Test
fun `should poll in order when response takes longer that polling interval`() {
monitor = createBlockCreationMonitor(
startingBlockNumberExclusive = 99,
config = config.copy(pollingInterval = 100.milliseconds)
)
val blocks = createBlocks(startBlockNumber = 99u, numberOfBlocks = 20)
setupFakeExecutionLayerWithBlocks(blocks)
fakeL2RpcNode.responsesArtificialDelay = 600.milliseconds
monitor.start()
await()
.atMost(20.seconds.toJavaDuration())
.untilAsserted {
assertThat(blockCreationListener.blocksReceived).isNotEmpty
assertThat(blockCreationListener.blocksReceived.map { it.number }).containsExactly(
100UL,
101UL,
102UL,
103UL,
104UL,
105UL
)
}
}
@Test
fun `start allow 2nd call when already started`() {
monitor = createBlockCreationMonitor(
startingBlockNumberExclusive = 99
)
setupFakeExecutionLayerWithBlocks(createBlocks(startBlockNumber = 99u, numberOfBlocks = 5))
monitor.start().get()
monitor.start().get()
}
@Test
fun `stop should be idempotent`(testContext: VertxTestContext) {
val payload =
executionPayloadV1(blockNumber = startingBlockNumberInclusive, parentHash = parentHash)
val payload2 =
executionPayloadV1(
blockNumber = startingBlockNumberInclusive + 1,
parentHash = payload.blockHash.toArray()
)
val headBlockNumber = startingBlockNumberInclusive + config.blocksToFinalization
whenever(web3jClient.ethBlockNumber())
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber)))
.then {
delay(config.pollingInterval.times(30)) {
SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 1))
}
fun `should stop fetching blocks when gap is greater than fetch limit and resume upon catchup`() {
monitor = createBlockCreationMonitor(
startingBlockNumberExclusive = 99,
config = config.copy(blocksToFinalization = 0, blocksFetchLimit = 5)
)
setupFakeExecutionLayerWithBlocks(createBlocks(startBlockNumber = 99u, numberOfBlocks = 30))
lastProvenBlockNumberProvider.lastProvenBlock.set(105)
monitor.start()
await()
.atMost(20.seconds.toJavaDuration())
.untilAsserted {
assertThat(blockCreationListener.blocksReceived).isNotEmpty
assertThat(blockCreationListener.blocksReceived.last().number).isGreaterThanOrEqualTo(110UL)
}
whenever(web3jClient.ethGetExecutionPayloadByNumber(any()))
.thenReturn(SafeFuture.completedFuture(payload))
.then { delay(config.pollingInterval.times(30)) { SafeFuture.completedFuture(payload2) } }
whenever(blockCreationListener.acceptBlock(any())).thenReturn(SafeFuture.completedFuture(Unit))
monitor.start().thenApply {
await()
.timeout(1.seconds.toJavaDuration())
.untilAsserted {
verify(blockCreationListener, times(1)).acceptBlock(any())
}
}
.whenException(testContext::failNow)
// Wait for a while to make sure no more blocks are fetched
await().atLeast(config.pollingInterval.times(3).toJavaDuration())
monitor.stop().thenApply {
await()
.timeout(1.seconds.toJavaDuration())
.untilAsserted {
verify(blockCreationListener, times(1)).acceptBlock(any())
}
testContext.completeNow()
}
.whenException(testContext::failNow)
}
// it shall remain at 110
assertThat(blockCreationListener.blocksReceived.last().number).isEqualTo(110UL)
@Test
fun `block shouldn't be fetched when block gap is greater than fetch limit`(testContext: VertxTestContext) {
val payload = executionPayloadV1(blockNumber = startingBlockNumberInclusive, parentHash = parentHash)
val payload2 =
executionPayloadV1(blockNumber = startingBlockNumberInclusive + 1, parentHash = payload.blockHash.toArray())
val payload3 =
executionPayloadV1(blockNumber = startingBlockNumberInclusive + 2, parentHash = payload2.blockHash.toArray())
val payload4 =
executionPayloadV1(blockNumber = startingBlockNumberInclusive + 3, parentHash = payload3.blockHash.toArray())
val payload5 =
executionPayloadV1(blockNumber = startingBlockNumberInclusive + 4, parentHash = payload4.blockHash.toArray())
val payload6 =
executionPayloadV1(blockNumber = startingBlockNumberInclusive + 5, parentHash = payload5.blockHash.toArray())
val payload7 =
executionPayloadV1(blockNumber = startingBlockNumberInclusive + 6, parentHash = payload6.blockHash.toArray())
// simulate prover catchup
lastProvenBlockNumberProvider.lastProvenBlock.set(120)
val headBlockNumber = startingBlockNumberInclusive + config.blocksToFinalization
whenever(web3jClient.ethGetExecutionPayloadByNumber(any()))
.thenReturn(SafeFuture.completedFuture(payload))
.then { SafeFuture.completedFuture(payload2) }
.then { SafeFuture.completedFuture(payload3) }
.then { SafeFuture.completedFuture(payload4) }
.then { SafeFuture.completedFuture(payload5) }
.then { SafeFuture.completedFuture(payload6) }
.then { SafeFuture.completedFuture(payload7) }
whenever(web3jClient.ethBlockNumber())
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber)))
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 2)))
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 3)))
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 4)))
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 5)))
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 6)))
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 7)))
whenever(blockCreationListener.acceptBlock(any())).thenReturn(SafeFuture.completedFuture(Unit))
monitor.start().thenApply {
await()
.timeout(4.seconds.toJavaDuration())
.untilAsserted {
verify(blockCreationListener, atLeastOnce()).acceptBlock(any())
// Number of invocations should remain at 5 as the blocks are now above the limit
verify(blockCreationListener, atMost(6)).acceptBlock(any())
}
testContext.completeNow()
}
.whenException(testContext::failNow)
}
@Test
fun `last block not fetched until finalization catches up to limit`(vertx: Vertx, testContext: VertxTestContext) {
val payload = executionPayloadV1(blockNumber = startingBlockNumberInclusive, parentHash = parentHash)
val payload2 = executionPayloadV1(blockNumber = startingBlockNumberInclusive + 1, parentHash = payload.blockHash)
val payload3 = executionPayloadV1(blockNumber = startingBlockNumberInclusive + 2, parentHash = payload2.blockHash)
val payload4 = executionPayloadV1(blockNumber = startingBlockNumberInclusive + 3, parentHash = payload3.blockHash)
val payload5 = executionPayloadV1(blockNumber = startingBlockNumberInclusive + 4, parentHash = payload4.blockHash)
val payload6 = executionPayloadV1(blockNumber = startingBlockNumberInclusive + 5, parentHash = payload5.blockHash)
val payload7 = executionPayloadV1(blockNumber = startingBlockNumberInclusive + 6, parentHash = payload6.blockHash)
val headBlockNumber = startingBlockNumberInclusive + config.blocksToFinalization + config.blocksFetchLimit
whenever(web3jClient.ethGetExecutionPayloadByNumber(any()))
.thenReturn(SafeFuture.completedFuture(payload))
.thenReturn(SafeFuture.completedFuture(payload2))
.thenReturn(SafeFuture.completedFuture(payload3))
.thenReturn(SafeFuture.completedFuture(payload4))
.thenReturn(SafeFuture.completedFuture(payload5))
.thenReturn(SafeFuture.completedFuture(payload6))
.thenReturn(SafeFuture.completedFuture(payload7))
whenever(web3jClient.ethBlockNumber())
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber)))
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 1)))
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 2)))
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 3)))
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 4)))
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 5)))
.thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 6)))
whenever(blockCreationListener.acceptBlock(any())).thenReturn(SafeFuture.completedFuture(Unit))
val lastProvenBlockNumberProviderAsync = mock<LastProvenBlockNumberProviderAsync>()
val monitor =
BlockCreationMonitor(
vertx,
web3jClient,
startingBlockNumberExclusive = startingBlockNumberInclusive - 1,
blockCreationListener,
lastProvenBlockNumberProviderAsync,
config
)
whenever(lastProvenBlockNumberProviderAsync.getLastProvenBlockNumber()).thenAnswer {
SafeFuture.completedFuture(lastProvenBlock)
}
monitor.start().thenApply {
await()
.timeout(4.seconds.toJavaDuration())
.untilAsserted {
verify(blockCreationListener, atLeastOnce()).acceptBlock(any())
verify(blockCreationListener, times(6)).acceptBlock(any())
}
}.thenApply {
whenever(lastProvenBlockNumberProviderAsync.getLastProvenBlockNumber()).thenAnswer {
SafeFuture.completedFuture(lastProvenBlock + 1)
// assert it resumes conflation
await()
.atMost(20.seconds.toJavaDuration())
.untilAsserted {
assertThat(blockCreationListener.blocksReceived.last().number).isGreaterThanOrEqualTo(125UL)
}
await()
.timeout(2.seconds.toJavaDuration())
.untilAsserted {
verify(blockCreationListener, times(7)).acceptBlock(any())
}
testContext.completeNow()
}
.whenException(testContext::failNow)
}
}

View File

@@ -16,16 +16,13 @@ dependencies {
implementation "io.vertx:vertx-core"
// Block dependencies
implementation "org.hyperledger.besu:besu-datatypes:${libs.versions.besu.get()}"
implementation "org.hyperledger.besu.internal:rlp:${libs.versions.besu.get()}"
implementation project(':jvm-libs:linea:besu-libs')
implementation "com.fasterxml.jackson.core:jackson-annotations:${libs.versions.jackson.get()}"
implementation "com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson.get()}"
implementation "com.fasterxml.jackson.module:jackson-module-kotlin:${libs.versions.jackson.get()}"
implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${libs.versions.jackson.get()}")
testImplementation project(':jvm-libs:linea:testing:teku-helper')
testImplementation testFixtures(project(':jvm-libs:linea:core:domain-models'))
testImplementation "io.vertx:vertx-junit5"
testImplementation "tech.pegasys.teku.internal:spec:${libs.versions.teku.get()}"
testImplementation "tech.pegasys.teku.internal:spec:${libs.versions.teku.get()}:test-fixtures"
}

View File

@@ -3,6 +3,7 @@ package net.consensys.zkevm.coordinator.clients.prover
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.node.ArrayNode
import io.vertx.core.Vertx
import linea.encoding.BlockRLPEncoder
import net.consensys.encodeHex
import net.consensys.linea.async.toSafeFuture
import net.consensys.toBigInteger
@@ -13,11 +14,9 @@ import net.consensys.zkevm.coordinator.clients.L2MessageServiceLogsClient
import net.consensys.zkevm.coordinator.clients.prover.serialization.JsonSerialization
import net.consensys.zkevm.domain.ProofIndex
import net.consensys.zkevm.domain.RlpBridgeLogsData
import net.consensys.zkevm.encoding.ExecutionPayloadV1Encoder
import net.consensys.zkevm.encoding.ExecutionPayloadV1RLPEncoderByBesuImplementation
import net.consensys.zkevm.encoding.BlockEncoder
import net.consensys.zkevm.fileio.FileReader
import net.consensys.zkevm.fileio.FileWriter
import net.consensys.zkevm.toULong
import org.apache.logging.log4j.LogManager
import org.web3j.protocol.Web3j
import org.web3j.protocol.core.DefaultBlockParameter
@@ -37,7 +36,7 @@ data class BatchExecutionProofRequestDto(
internal class ExecutionProofRequestDataDecorator(
private val l2MessageServiceLogsClient: L2MessageServiceLogsClient,
private val l2Web3jClient: Web3j,
private val encoder: ExecutionPayloadV1Encoder = ExecutionPayloadV1RLPEncoderByBesuImplementation
private val encoder: BlockEncoder = BlockRLPEncoder
) : (BatchExecutionProofRequestV1) -> SafeFuture<BatchExecutionProofRequestDto> {
private fun getBlockStateRootHash(blockNumber: ULong): SafeFuture<String> {
return l2Web3jClient
@@ -52,13 +51,13 @@ internal class ExecutionProofRequestDataDecorator(
override fun invoke(request: BatchExecutionProofRequestV1): SafeFuture<BatchExecutionProofRequestDto> {
val bridgeLogsSfList = request.blocks.map { block ->
l2MessageServiceLogsClient.getBridgeLogs(blockNumber = block.blockNumber.longValue())
l2MessageServiceLogsClient.getBridgeLogs(blockNumber = block.number.toLong())
.thenApply { block to it }
}
return SafeFuture.collectAll(bridgeLogsSfList.stream())
.thenCombine(
getBlockStateRootHash(request.blocks.first().blockNumber.toULong() - 1UL)
getBlockStateRootHash(request.blocks.first().number.toULong() - 1UL)
) { blocksAndBridgeLogs, previousKeccakStateRootHash ->
BatchExecutionProofRequestDto(
zkParentStateRootHash = request.type2StateData.zkParentStateRootHash.encodeHex(),

View File

@@ -2,13 +2,15 @@ package net.consensys.zkevm.coordinator.clients.prover
import build.linea.clients.GetZkEVMStateMerkleProofResponse
import com.fasterxml.jackson.databind.node.ArrayNode
import linea.domain.Block
import linea.domain.createBlock
import net.consensys.ByteArrayExt
import net.consensys.encodeHex
import net.consensys.zkevm.coordinator.clients.BatchExecutionProofRequestV1
import net.consensys.zkevm.coordinator.clients.GenerateTracesResponse
import net.consensys.zkevm.coordinator.clients.L2MessageServiceLogsClient
import net.consensys.zkevm.domain.RlpBridgeLogsData
import net.consensys.zkevm.encoding.ExecutionPayloadV1Encoder
import net.consensys.zkevm.encoding.BlockEncoder
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
@@ -21,8 +23,6 @@ import org.mockito.kotlin.spy
import org.mockito.kotlin.whenever
import org.web3j.protocol.Web3j
import org.web3j.protocol.core.methods.response.EthBlock
import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1
import tech.pegasys.teku.ethereum.executionclient.schema.executionPayloadV1
import tech.pegasys.teku.infrastructure.async.SafeFuture
import kotlin.random.Random
@@ -30,11 +30,11 @@ class ExecutionProofRequestDataDecoratorTest {
private lateinit var l2MessageServiceLogsClient: L2MessageServiceLogsClient
private lateinit var l2Web3jClient: Web3j
private lateinit var encoder: ExecutionPayloadV1Encoder
private lateinit var encoder: BlockEncoder
private lateinit var requestDatDecorator: ExecutionProofRequestDataDecorator
private val fakeEncoder: ExecutionPayloadV1Encoder = object : ExecutionPayloadV1Encoder {
override fun encode(payload: ExecutionPayloadV1): ByteArray {
return payload.blockNumber.toString().toByteArray()
private val fakeEncoder: BlockEncoder = object : BlockEncoder {
override fun encode(block: Block): ByteArray {
return block.number.toString().toByteArray()
}
}
@@ -48,8 +48,8 @@ class ExecutionProofRequestDataDecoratorTest {
@Test
fun `should decorate data with bridge logs and parent stateRootHash`() {
val executionPayload1 = executionPayloadV1(blockNumber = 123, gasLimit = 20_000_000UL)
val executionPayload2 = executionPayloadV1(blockNumber = 124, gasLimit = 20_000_000UL)
val block1 = createBlock(number = 123UL)
val block2 = createBlock(number = 124UL)
val type2StateResponse = GetZkEVMStateMerkleProofResponse(
zkStateMerkleProof = ArrayNode(null),
zkParentStateRootHash = ByteArrayExt.random32(),
@@ -61,7 +61,7 @@ class ExecutionProofRequestDataDecoratorTest {
tracesEngineVersion = "1.0.0"
)
val request = BatchExecutionProofRequestV1(
blocks = listOf(executionPayload1, executionPayload2),
blocks = listOf(block1, block2),
tracesResponse = generateTracesResponse,
type2StateData = type2StateResponse
)
@@ -74,9 +74,9 @@ class ExecutionProofRequestDataDecoratorTest {
SafeFuture.completedFuture(mockedEthBlock)
}
whenever(l2MessageServiceLogsClient.getBridgeLogs(eq(executionPayload1.blockNumber.longValue())))
whenever(l2MessageServiceLogsClient.getBridgeLogs(eq(block1.number.toLong())))
.thenReturn(SafeFuture.completedFuture(listOf(CommonTestData.bridgeLogs[0])))
whenever(l2MessageServiceLogsClient.getBridgeLogs(eq(executionPayload2.blockNumber.longValue())))
whenever(l2MessageServiceLogsClient.getBridgeLogs(eq(block2.number.toLong())))
.thenReturn(SafeFuture.completedFuture(listOf(CommonTestData.bridgeLogs[1])))
val requestDto = requestDatDecorator.invoke(request).get()

View File

@@ -7,9 +7,7 @@ dependencies {
implementation project(':jvm-libs:generic:extensions:futures')
implementation project(':jvm-libs:generic:json-rpc')
implementation project(':jvm-libs:linea:metrics:micrometer')
implementation project(":jvm-libs:linea:teku-execution-client")
implementation "tech.pegasys.teku.internal:unsigned:${libs.versions.teku.get()}"
implementation project(':jvm-libs:linea:core:traces')
api "io.vertx:vertx-core"
testImplementation 'org.junit.jupiter:junit-jupiter'

View File

@@ -17,7 +17,6 @@ dependencies {
api project(':jvm-libs:generic:extensions:futures')
api "tech.pegasys.teku.internal:unsigned:${libs.versions.teku.get()}"
api "org.jetbrains.kotlinx:kotlinx-datetime:${libs.versions.kotlinxDatetime.get()}"
implementation project(":jvm-libs:linea:teku-execution-client")
implementation "io.vertx:vertx-core"
// jackson shall never be used in the core module
// however, it is used already :( but was as transitive through Teku Execution Client
@@ -30,10 +29,10 @@ dependencies {
}
testFixturesApi "org.jetbrains.kotlinx:kotlinx-datetime:${libs.versions.kotlinxDatetime.get()}"
testFixturesApi testFixtures(project(':jvm-libs:linea:core:domain-models'))
testFixturesImplementation("org.web3j:core:${libs.versions.web3j.get()}") {
exclude group: 'org.slf4j', module: 'slf4j-nop'
}
testImplementation project(":jvm-libs:linea:testing:teku-helper")
testImplementation project(':jvm-libs:linea:metrics:micrometer')
testImplementation(testFixtures(project(':jvm-libs:linea:core:traces')))
testImplementation(testFixtures(project(':jvm-libs:generic:extensions:kotlin')))

View File

@@ -2,18 +2,17 @@ package net.consensys.zkevm.coordinator.clients
import build.linea.clients.GetZkEVMStateMerkleProofResponse
import build.linea.domain.BlockInterval
import net.consensys.zkevm.toULong
import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1
import linea.domain.Block
data class BatchExecutionProofRequestV1(
val blocks: List<ExecutionPayloadV1>,
val blocks: List<Block>,
val tracesResponse: GenerateTracesResponse,
val type2StateData: GetZkEVMStateMerkleProofResponse
) : BlockInterval {
override val startBlockNumber: ULong
get() = blocks.first().blockNumber.toULong()
get() = blocks.first().number
override val endBlockNumber: ULong
get() = blocks.last().blockNumber.toULong()
get() = blocks.last().number
}
data class BatchExecutionProofResponse(

View File

@@ -2,24 +2,23 @@ package net.consensys.zkevm.domain
import build.linea.domain.BlockInterval
import kotlinx.datetime.Instant
import linea.domain.Block
import net.consensys.isSortedBy
import net.consensys.linea.CommonDomainFunctions
import net.consensys.linea.traces.TracesCounters
import net.consensys.zkevm.toULong
import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1
data class BlocksConflation(
val blocks: List<ExecutionPayloadV1>,
val blocks: List<Block>,
val conflationResult: ConflationCalculationResult
) : BlockInterval {
init {
require(blocks.isSortedBy { it.blockNumber }) { "Blocks list must be sorted by blockNumber" }
require(blocks.isSortedBy { it.number }) { "Blocks list must be sorted by blockNumber" }
}
override val startBlockNumber: ULong
get() = blocks.first().blockNumber.toULong()
get() = blocks.first().number.toULong()
override val endBlockNumber: ULong
get() = blocks.last().blockNumber.toULong()
get() = blocks.last().number.toULong()
}
data class Batch(

View File

@@ -0,0 +1,7 @@
package net.consensys.zkevm.encoding
import linea.domain.Block
fun interface BlockEncoder {
fun encode(block: Block): ByteArray
}

View File

@@ -1,7 +0,0 @@
package net.consensys.zkevm.encoding
import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1
fun interface ExecutionPayloadV1Encoder {
fun encode(payload: ExecutionPayloadV1): ByteArray
}

View File

@@ -1,10 +1,10 @@
package net.consensys.zkevm.ethereum.coordination.blockcreation
import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1
import linea.domain.Block
import tech.pegasys.teku.infrastructure.async.SafeFuture
data class BlockCreated(
val executionPayload: ExecutionPayloadV1
val block: Block
)
fun interface BlockCreationListener {

View File

@@ -1,49 +1,12 @@
package net.consensys.zkevm.ethereum.coordination.blockcreation
import kotlinx.datetime.Instant
import net.consensys.zkevm.toULong
import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1
import linea.domain.Block
import linea.domain.BlockHeaderSummary
import tech.pegasys.teku.infrastructure.async.SafeFuture
data class BlockHeaderSummary(
val number: ULong,
val hash: ByteArray,
val timestamp: Instant
) {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as BlockHeaderSummary
if (number != other.number) return false
if (!hash.contentEquals(other.hash)) return false
if (timestamp != other.timestamp) return false
return true
}
override fun hashCode(): Int {
var result = number.hashCode()
result = 31 * result + hash.contentHashCode()
result = 31 * result + timestamp.hashCode()
return result
}
override fun toString(): String {
return "BlockHeaderSummary(number=$number, hash=${hash.contentToString()}, timestamp=$timestamp)"
}
}
interface SafeBlockProvider {
fun getLatestSafeBlock(): SafeFuture<ExecutionPayloadV1>
fun getLatestSafeBlock(): SafeFuture<Block>
fun getLatestSafeBlockHeader(): SafeFuture<BlockHeaderSummary> {
return getLatestSafeBlock().thenApply {
BlockHeaderSummary(
it.blockNumber.toULong(),
it.blockHash.toArray(),
Instant.fromEpochSeconds(it.timestamp.longValue())
)
}
return getLatestSafeBlock().thenApply { it.headerSummary }
}
}

View File

@@ -4,17 +4,16 @@ import com.github.michaelbull.result.Err
import com.github.michaelbull.result.Ok
import io.vertx.core.Vertx
import kotlinx.datetime.Instant
import net.consensys.linea.BlockNumberAndHash
import linea.domain.Block
import net.consensys.linea.async.toSafeFuture
import net.consensys.linea.errors.ErrorResponse
import net.consensys.zkevm.coordinator.clients.GetTracesCountersResponse
import net.consensys.zkevm.coordinator.clients.TracesCountersClientV1
import net.consensys.zkevm.coordinator.clients.TracesServiceErrorType
import net.consensys.zkevm.domain.BlockCounters
import net.consensys.zkevm.encoding.ExecutionPayloadV1Encoder
import net.consensys.zkevm.encoding.BlockEncoder
import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockCreated
import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockCreationListener
import net.consensys.zkevm.toULong
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
import tech.pegasys.teku.infrastructure.async.SafeFuture
@@ -24,19 +23,14 @@ class BlockToBatchSubmissionCoordinator(
private val conflationService: ConflationService,
private val tracesCountersClient: TracesCountersClientV1,
private val vertx: Vertx,
private val payloadEncoder: ExecutionPayloadV1Encoder,
private val encoder: BlockEncoder,
private val log: Logger = LogManager.getLogger(BlockToBatchSubmissionCoordinator::class.java)
) : BlockCreationListener {
private fun getTracesCounters(
blockEvent: BlockCreated
block: Block
): SafeFuture<GetTracesCountersResponse> {
return tracesCountersClient
.rollupGetTracesCounters(
BlockNumberAndHash(
blockEvent.executionPayload.blockNumber.toULong(),
blockEvent.executionPayload.blockHash.toArray()
)
)
.rollupGetTracesCounters(block.numberAndHash)
.thenCompose { result ->
when (result) {
is Err<ErrorResponse<TracesServiceErrorType>> -> {
@@ -51,31 +45,37 @@ class BlockToBatchSubmissionCoordinator(
}
override fun acceptBlock(blockEvent: BlockCreated): SafeFuture<Unit> {
log.debug("Accepting new block={}", blockEvent.executionPayload.blockNumber)
vertx.executeBlocking(
Callable {
payloadEncoder.encode(blockEvent.executionPayload)
}
).toSafeFuture().thenCombine(getTracesCounters(blockEvent)) { blockRLPEncoded, traces ->
conflationService.newBlock(
blockEvent.executionPayload,
BlockCounters(
blockNumber = blockEvent.executionPayload.blockNumber.toULong(),
blockTimestamp = Instant.fromEpochSeconds(blockEvent.executionPayload.timestamp.longValue()),
tracesCounters = traces.tracesCounters,
blockRLPEncoded = blockRLPEncoded
log.debug("accepting new block={}", blockEvent.block.number)
encodeBlock(blockEvent.block)
.thenCombine(getTracesCounters(blockEvent.block)) { blockRLPEncoded, traces ->
conflationService.newBlock(
blockEvent.block,
BlockCounters(
blockNumber = blockEvent.block.number,
blockTimestamp = Instant.fromEpochSeconds(blockEvent.block.timestamp.toLong()),
tracesCounters = traces.tracesCounters,
blockRLPEncoded = blockRLPEncoded
)
)
)
}.whenException { th ->
log.error(
"Failed to conflate block={} errorMessage={}",
blockEvent.executionPayload.blockNumber,
th.message,
th
)
}
}.whenException { th ->
log.error(
"Failed to conflate block={} errorMessage={}",
blockEvent.block.number,
th.message,
th
)
}
// This is to parallelize `getTracesCounters` requests which would otherwise be sent sequentially
return SafeFuture.completedFuture(Unit)
}
private fun encodeBlock(block: Block): SafeFuture<ByteArray> {
return vertx.executeBlocking(
Callable {
encoder.encode(block)
}
)
.toSafeFuture()
}
}

View File

@@ -1,14 +1,13 @@
package net.consensys.zkevm.ethereum.coordination.conflation
import linea.domain.Block
import net.consensys.linea.metrics.LineaMetricsCategory
import net.consensys.linea.metrics.MetricsFacade
import net.consensys.zkevm.domain.BlockCounters
import net.consensys.zkevm.domain.BlocksConflation
import net.consensys.zkevm.domain.ConflationCalculationResult
import net.consensys.zkevm.toULong
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1
import tech.pegasys.teku.infrastructure.async.SafeFuture
import java.util.concurrent.PriorityBlockingQueue
import java.util.concurrent.TimeUnit
@@ -20,14 +19,14 @@ class ConflationServiceImpl(
ConflationService {
private val log: Logger = LogManager.getLogger(this::class.java)
private var listener: ConflationHandler = ConflationHandler { SafeFuture.completedFuture<Unit>(null) }
private val blocksInProgress: MutableList<ExecutionPayloadV1> = mutableListOf()
private val blocksInProgress: MutableList<Block> = mutableListOf()
data class PayloadAndBlockCounters(
val executionPayload: ExecutionPayloadV1,
val block: Block,
val blockCounters: BlockCounters
) : Comparable<PayloadAndBlockCounters> {
override fun compareTo(other: PayloadAndBlockCounters): Int {
return this.executionPayload.blockNumber.compareTo(other.executionPayload.blockNumber)
return this.block.number.compareTo(other.block.number)
}
}
@@ -66,8 +65,8 @@ class ConflationServiceImpl(
)
val blocksToConflate =
blocksInProgress
.filter { it.blockNumber.toULong() in conflation.blocksRange }
.sortedBy { it.blockNumber }
.filter { it.number in conflation.blocksRange }
.sortedBy { it.number }
blocksInProgress.removeAll(blocksToConflate)
return listener.handleConflatedBatch(BlocksConflation(blocksToConflate, conflation))
@@ -82,21 +81,21 @@ class ConflationServiceImpl(
}
@Synchronized
override fun newBlock(block: ExecutionPayloadV1, blockCounters: BlockCounters) {
require(block.blockNumber.toULong() == blockCounters.blockNumber) {
"Payload blockNumber ${block.blockNumber} does not match blockCounters.blockNumber=${blockCounters.blockNumber}"
override fun newBlock(block: Block, blockCounters: BlockCounters) {
require(block.number == blockCounters.blockNumber) {
"block=${block.number} does not match blockCounters.blockNumber=${blockCounters.blockNumber}"
}
blocksCounter.increment()
log.trace(
"newBlock={} calculatorLastBlockNumber={} blocksToConflateSize={} blocksInProgressSize={}",
block.blockNumber,
block.number,
calculator.lastBlockNumber,
blocksToConflate.size,
blocksInProgress.size
)
blocksToConflate.add(PayloadAndBlockCounters(block, blockCounters))
blocksInProgress.add(block)
log.trace("block {} added to conflation queue", block.blockNumber)
log.trace("block {} added to conflation queue", block.number)
sendBlocksInOrderToTracesCounter()
}
@@ -104,14 +103,14 @@ class ConflationServiceImpl(
var nextBlockNumberToConflate = calculator.lastBlockNumber + 1u
var nextAvailableBlock = blocksToConflate.peek()
while (nextAvailableBlock?.executionPayload?.blockNumber?.toULong() == nextBlockNumberToConflate) {
while (nextAvailableBlock?.block?.number == nextBlockNumberToConflate) {
nextAvailableBlock = blocksToConflate.poll(100, TimeUnit.MILLISECONDS)
log.trace(
"block {} removed from conflation queue and sent to calculator",
nextAvailableBlock?.executionPayload?.blockNumber
nextAvailableBlock?.block?.number
)
calculator.newBlock(nextAvailableBlock.blockCounters)
nextBlockNumberToConflate = nextAvailableBlock.executionPayload.blockNumber.toULong() + 1u
nextBlockNumberToConflate = nextAvailableBlock.block.number + 1u
nextAvailableBlock = blocksToConflate.peek()
}
}

View File

@@ -3,12 +3,10 @@ package net.consensys.zkevm.ethereum.coordination.conflation
import com.github.michaelbull.result.getOrElse
import com.github.michaelbull.result.runCatching
import io.vertx.core.Vertx
import net.consensys.linea.BlockNumberAndHash
import net.consensys.linea.async.AsyncRetryer
import net.consensys.zkevm.domain.BlocksConflation
import net.consensys.zkevm.ethereum.coordination.proofcreation.BatchProofHandler
import net.consensys.zkevm.ethereum.coordination.proofcreation.ZkProofCreationCoordinator
import net.consensys.zkevm.toULong
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
import tech.pegasys.teku.infrastructure.async.SafeFuture
@@ -57,9 +55,7 @@ class ProofGeneratingConflationHandlerImpl(
}
private fun conflationToProofCreation(conflation: BlocksConflation): SafeFuture<*> {
val blockNumbersAndHash = conflation.blocks.map {
BlockNumberAndHash(it.blockNumber.toULong(), it.blockHash.toArray())
}
val blockNumbersAndHash = conflation.blocks.map { it.numberAndHash }
val blockIntervalString = conflation.conflationResult.intervalString()
return tracesProductionCoordinator
.conflateExecutionTraces(blockNumbersAndHash)

View File

@@ -1,10 +1,10 @@
package net.consensys.zkevm.ethereum.coordination.conflation
import linea.domain.Block
import net.consensys.zkevm.domain.Blob
import net.consensys.zkevm.domain.BlockCounters
import net.consensys.zkevm.domain.BlocksConflation
import net.consensys.zkevm.domain.ConflationCalculationResult
import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1
import tech.pegasys.teku.infrastructure.async.SafeFuture
fun interface BlobCreationHandler {
@@ -23,6 +23,6 @@ interface TracesConflationCalculator {
}
interface ConflationService {
fun newBlock(block: ExecutionPayloadV1, blockCounters: BlockCounters)
fun newBlock(block: Block, blockCounters: BlockCounters)
fun onConflatedBatch(consumer: ConflationHandler)
}

View File

@@ -2,7 +2,6 @@ package net.consensys.zkevm.ethereum.coordination.conflation.upgrade
import net.consensys.zkevm.domain.BlocksConflation
import net.consensys.zkevm.ethereum.coordination.conflation.ConflationHandler
import net.consensys.zkevm.toULong
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
import tech.pegasys.teku.infrastructure.async.SafeFuture
@@ -20,8 +19,8 @@ class SwitchAwareConflationHandler(
override fun handleConflatedBatch(conflation: BlocksConflation): SafeFuture<*> {
return switchProvider.getSwitch(newVersion).thenCompose {
switchBlock ->
val conflationStartBlockNumber = conflation.blocks.first().blockNumber.toULong()
val conflationEndBlockNumber = conflation.blocks.last().blockNumber.toULong()
val conflationStartBlockNumber = conflation.blocks.first().number.toULong()
val conflationEndBlockNumber = conflation.blocks.last().number.toULong()
if (switchBlock == null || conflationStartBlockNumber < switchBlock) {
log.debug("Handing conflation [$conflationStartBlockNumber, $conflationEndBlockNumber] over to old handler")
oldHandler.handleConflatedBatch(conflation)

View File

@@ -5,7 +5,6 @@ import net.consensys.zkevm.coordinator.clients.ExecutionProverClientV2
import net.consensys.zkevm.domain.Batch
import net.consensys.zkevm.domain.BlocksConflation
import net.consensys.zkevm.ethereum.coordination.conflation.BlocksTracesConflated
import net.consensys.zkevm.toULong
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
import tech.pegasys.teku.infrastructure.async.SafeFuture
@@ -19,8 +18,8 @@ class ZkProofCreationCoordinatorImpl(
blocksConflation: BlocksConflation,
traces: BlocksTracesConflated
): SafeFuture<Batch> {
val startBlockNumber = blocksConflation.blocks.first().blockNumber.toULong()
val endBlockNumber = blocksConflation.blocks.last().blockNumber.toULong()
val startBlockNumber = blocksConflation.blocks.first().number.toULong()
val endBlockNumber = blocksConflation.blocks.last().number.toULong()
val blocksConflationInterval = blocksConflation.intervalString()
return executionProverClient

View File

@@ -2,10 +2,10 @@ package net.consensys.zkevm.ethereum.coordination.aggregation
import kotlinx.datetime.Clock
import kotlinx.datetime.Instant
import linea.domain.BlockHeaderSummary
import net.consensys.ByteArrayExt
import net.consensys.zkevm.domain.BlobCounters
import net.consensys.zkevm.domain.BlobsToAggregate
import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockHeaderSummary
import net.consensys.zkevm.ethereum.coordination.blockcreation.SafeBlockProvider
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test

View File

@@ -2,13 +2,13 @@ package net.consensys.zkevm.ethereum.coordination.aggregation
import io.micrometer.core.instrument.simple.SimpleMeterRegistry
import kotlinx.datetime.Instant
import linea.domain.BlockHeaderSummary
import net.consensys.ByteArrayExt
import net.consensys.FakeFixedClock
import net.consensys.linea.metrics.MetricsFacade
import net.consensys.linea.metrics.micrometer.MicrometerMetricsFacade
import net.consensys.zkevm.domain.BlobCounters
import net.consensys.zkevm.domain.BlobsToAggregate
import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockHeaderSummary
import net.consensys.zkevm.ethereum.coordination.blockcreation.SafeBlockProvider
import org.assertj.core.api.Assertions
import org.assertj.core.api.Assertions.assertThat

View File

@@ -3,12 +3,11 @@ package net.consensys.zkevm.ethereum.coordination.conflation
import com.github.michaelbull.result.Ok
import io.vertx.core.Vertx
import io.vertx.junit5.VertxExtension
import net.consensys.linea.BlockNumberAndHash
import linea.domain.createBlock
import net.consensys.linea.traces.TracesCountersV1
import net.consensys.zkevm.coordinator.clients.GetTracesCountersResponse
import net.consensys.zkevm.coordinator.clients.TracesCountersClientV1
import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockCreated
import net.consensys.zkevm.toULong
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
import org.assertj.core.api.Assertions
@@ -22,7 +21,6 @@ import org.mockito.kotlin.eq
import org.mockito.kotlin.times
import org.mockito.kotlin.verify
import org.mockito.kotlin.whenever
import tech.pegasys.teku.ethereum.executionclient.schema.randomExecutionPayload
import tech.pegasys.teku.infrastructure.async.SafeFuture
import kotlin.time.Duration.Companion.seconds
import kotlin.time.toJavaDuration
@@ -31,9 +29,8 @@ import kotlin.time.toJavaDuration
class BlockToBatchSubmissionCoordinatorTest {
companion object {
private val defaultConflationService = ConflationServiceImpl(mock(), mock())
private const val ARBITRARY_BLOCK_NUMBER = 100L
private val randomExecutionPayload = randomExecutionPayload(blockNumber = ARBITRARY_BLOCK_NUMBER)
private val baseBlock = BlockCreated(randomExecutionPayload)
private val randomBlock = createBlock(number = 100UL)
private val baseBlock = BlockCreated(randomBlock)
private val blockRlpEncoded = ByteArray(0)
private val tracesCounters = TracesCountersV1.EMPTY_TRACES_COUNT
}
@@ -45,22 +42,14 @@ class BlockToBatchSubmissionCoordinatorTest {
): BlockToBatchSubmissionCoordinator {
val tracesCountersClient =
mock<TracesCountersClientV1>().also {
whenever(
it.rollupGetTracesCounters(
BlockNumberAndHash(
randomExecutionPayload.blockNumber.toULong(),
randomExecutionPayload.blockHash.toArray()
)
)
).thenReturn(
SafeFuture.completedFuture(Ok(GetTracesCountersResponse(tracesCounters, "")))
)
whenever(it.rollupGetTracesCounters(randomBlock.numberAndHash))
.thenReturn(SafeFuture.completedFuture(Ok(GetTracesCountersResponse(tracesCounters, ""))))
}
return BlockToBatchSubmissionCoordinator(
conflationService = conflationService,
tracesCountersClient = tracesCountersClient,
vertx = vertx,
payloadEncoder = { blockRlpEncoded },
encoder = { blockRlpEncoded },
log = log
)
}

View File

@@ -2,10 +2,10 @@ package net.consensys.zkevm.ethereum.coordination.conflation
import kotlinx.datetime.Clock
import kotlinx.datetime.Instant
import linea.domain.BlockHeaderSummary
import net.consensys.ByteArrayExt
import net.consensys.linea.traces.fakeTracesCountersV1
import net.consensys.zkevm.domain.BlockCounters
import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockHeaderSummary
import net.consensys.zkevm.ethereum.coordination.blockcreation.SafeBlockProvider
import org.apache.logging.log4j.Logger
import org.assertj.core.api.Assertions.assertThat

View File

@@ -1,13 +1,13 @@
package net.consensys.zkevm.ethereum.coordination.conflation
import kotlinx.datetime.Instant
import linea.domain.createBlock
import net.consensys.linea.traces.TracesCountersV1
import net.consensys.linea.traces.fakeTracesCountersV1
import net.consensys.zkevm.domain.BlockCounters
import net.consensys.zkevm.domain.BlocksConflation
import net.consensys.zkevm.domain.ConflationCalculationResult
import net.consensys.zkevm.domain.ConflationTrigger
import net.consensys.zkevm.toULong
import org.assertj.core.api.Assertions.assertThat
import org.assertj.core.api.Assertions.assertThatThrownBy
import org.awaitility.Awaitility
@@ -17,7 +17,6 @@ import org.mockito.Mockito.RETURNS_DEEP_STUBS
import org.mockito.kotlin.any
import org.mockito.kotlin.mock
import org.mockito.kotlin.whenever
import tech.pegasys.teku.ethereum.executionclient.schema.executionPayloadV1
import tech.pegasys.teku.infrastructure.async.SafeFuture
import java.time.Duration
import java.util.concurrent.Executors
@@ -43,9 +42,9 @@ class ConflationServiceImplTest {
@Test
fun `emits event with blocks when calculator emits conflation`() {
val payload1 = executionPayloadV1(blockNumber = 1, gasLimit = 20_000_000UL)
val payload2 = executionPayloadV1(blockNumber = 2, gasLimit = 20_000_000UL)
val payload3 = executionPayloadV1(blockNumber = 3, gasLimit = 20_000_000UL)
val payload1 = createBlock(number = 1UL, gasLimit = 20_000_000UL)
val payload2 = createBlock(number = 2UL, gasLimit = 20_000_000UL)
val payload3 = createBlock(number = 3UL, gasLimit = 20_000_000UL)
val payload1Time = Instant.parse("2021-01-01T00:00:00Z")
val payloadCounters1 = BlockCounters(
blockNumber = 1UL,
@@ -100,7 +99,7 @@ class ConflationServiceImplTest {
val moduleTracesCounter = 10u
assertThat(numberOfBlocks % numberOfThreads).isEqualTo(0)
val expectedConflations = numberOfBlocks / conflationBlockLimit.toInt() - 1
val blocks = (1..numberOfBlocks).map { executionPayloadV1(blockNumber = it.toLong(), gasLimit = 20_000_000UL) }
val blocks = (1UL..numberOfBlocks.toULong()).map { createBlock(number = it, gasLimit = 20_000_000UL) }
val fixedTracesCounters = fakeTracesCountersV1(moduleTracesCounter)
val blockTime = Instant.parse("2021-01-01T00:00:00Z")
val conflationEvents = mutableListOf<BlocksConflation>()
@@ -118,7 +117,7 @@ class ConflationServiceImplTest {
conflationService.newBlock(
it,
BlockCounters(
blockNumber = it.blockNumber.toULong(),
blockNumber = it.number.toULong(),
blockTimestamp = blockTime,
tracesCounters = fixedTracesCounters,
blockRLPEncoded = ByteArray(0)
@@ -152,13 +151,13 @@ class ConflationServiceImplTest {
val failingConflationCalculator: TracesConflationCalculator = mock()
whenever(failingConflationCalculator.newBlock(any())).thenThrow(expectedException)
conflationService = ConflationServiceImpl(failingConflationCalculator, mock(defaultAnswer = RETURNS_DEEP_STUBS))
val block = executionPayloadV1(blockNumber = 1, gasLimit = 20_000_000UL)
val block = createBlock(number = 1UL, gasLimit = 20_000_000UL)
assertThatThrownBy {
conflationService.newBlock(
block,
BlockCounters(
blockNumber = block.blockNumber.toULong(),
blockNumber = block.number.toULong(),
blockTimestamp = blockTime,
tracesCounters = fixedTracesCounters,
blockRLPEncoded = ByteArray(0)

View File

@@ -1,6 +1,7 @@
package net.consensys.zkevm.ethereum.coordination.conflation
import kotlinx.datetime.Instant
import linea.domain.BlockHeaderSummary
import net.consensys.ByteArrayExt
import net.consensys.FakeFixedClock
import net.consensys.linea.traces.TracesCountersV1
@@ -11,7 +12,6 @@ import net.consensys.zkevm.domain.ConflationCalculationResult
import net.consensys.zkevm.domain.ConflationTrigger
import net.consensys.zkevm.ethereum.coordination.blob.BlobCompressor
import net.consensys.zkevm.ethereum.coordination.blob.FakeBlobCompressor
import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockHeaderSummary
import net.consensys.zkevm.ethereum.coordination.blockcreation.SafeBlockProvider
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.BeforeEach

View File

@@ -1,6 +1,7 @@
package net.consensys.zkevm.ethereum.coordination.conflation
import kotlinx.datetime.Instant
import linea.domain.BlockHeaderSummary
import net.consensys.ByteArrayExt
import net.consensys.FakeFixedClock
import net.consensys.linea.metrics.MetricsFacade
@@ -10,7 +11,6 @@ import net.consensys.zkevm.domain.BlockCounters
import net.consensys.zkevm.domain.ConflationCalculationResult
import net.consensys.zkevm.domain.ConflationTrigger
import net.consensys.zkevm.ethereum.coordination.blob.FakeBlobCompressor
import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockHeaderSummary
import net.consensys.zkevm.ethereum.coordination.blockcreation.SafeBlockProvider
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.BeforeEach

View File

@@ -1,11 +1,11 @@
package net.consensys.zkevm.ethereum.coordination.conflation.upgrade
import linea.domain.createBlock
import net.consensys.linea.traces.TracesCountersV1
import net.consensys.zkevm.domain.BlocksConflation
import net.consensys.zkevm.domain.ConflationCalculationResult
import net.consensys.zkevm.domain.ConflationTrigger
import net.consensys.zkevm.ethereum.coordination.conflation.ConflationHandler
import net.consensys.zkevm.toULong
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
import org.mockito.kotlin.any
@@ -15,7 +15,6 @@ import org.mockito.kotlin.never
import org.mockito.kotlin.times
import org.mockito.kotlin.verify
import org.mockito.kotlin.whenever
import tech.pegasys.teku.ethereum.executionclient.schema.executionPayloadV1
import tech.pegasys.teku.infrastructure.async.SafeFuture
@Suppress("DEPRECATION")
@@ -28,13 +27,12 @@ class SwitchAwareConflationHandlerTest {
private val switchBlock = 100UL
private fun generateArbitraryConflation(startBlockNumber: ULong, blocksLong: UInt): BlocksConflation {
val executionPayloads = (startBlockNumber..startBlockNumber + blocksLong).map {
executionPayloadV1(blockNumber = it.toLong(), gasLimit = 20_000_000UL)
}
val executionPayloads = (startBlockNumber..startBlockNumber + blocksLong)
.map { createBlock(number = it) }
val conflationCalculationResult = ConflationCalculationResult(
startBlockNumber = executionPayloads.first().blockNumber.toULong(),
endBlockNumber = executionPayloads.last().blockNumber.toULong(),
startBlockNumber = executionPayloads.first().number.toULong(),
endBlockNumber = executionPayloads.last().number.toULong(),
conflationTrigger = ConflationTrigger.TRACES_LIMIT,
tracesCounters = TracesCountersV1.EMPTY_TRACES_COUNT
)

View File

@@ -20,7 +20,6 @@ dependencies {
implementation("org.web3j:core:${libs.versions.web3j.get()}") {
exclude group: "org.slf4j", module: "slf4j-nop"
}
implementation project(":jvm-libs:linea:teku-execution-client")
testImplementation(project(":jvm-libs:linea:testing:l1-blob-and-proof-submission"))
testImplementation(project(":coordinator:persistence:aggregation"))

View File

@@ -37,8 +37,6 @@ class GasPriceCapProviderImplTest {
private val adjustmentConstant = 25U
private val finalizationTargetMaxDelay = 6.hours
private val gasPriceCapsCoefficient = 1.0.div(1.1)
private val historicBaseFeePerBlobGasLowerBound = 200000000uL // 0.2GWei
private val initialFixedAvgReward = 100000000uL // 0.1GWei
private val gasPriceCapCalculator = GasPriceCapCalculatorImpl()
private lateinit var targetBlockTime: Instant

View File

@@ -4,13 +4,6 @@ plugins {
dependencies {
api (project(":coordinator:core"))
api project(":jvm-libs:linea:teku-execution-client")
implementation "tech.pegasys.teku.internal:bytes:${libs.versions.teku.get()}"
implementation "org.hyperledger.besu:besu-datatypes:${libs.versions.besu.get()}"
implementation "org.hyperledger.besu:evm:${libs.versions.besu.get()}"
implementation "org.hyperledger.besu.internal:rlp:${libs.versions.besu.get()}"
implementation "org.hyperledger.besu.internal:core:${libs.versions.besu.get()}"
implementation "org.hyperledger.besu:plugin-api:${libs.versions.besu.get()}"
testImplementation project(":jvm-libs:linea:testing:teku-helper")
api project(":jvm-libs:linea:besu-libs")
api project(":jvm-libs:linea:besu-rlp-and-mappers")
}

View File

@@ -0,0 +1,9 @@
package linea.encoding
import linea.domain.toBesu
import linea.rlp.RLP
import net.consensys.zkevm.encoding.BlockEncoder
object BlockRLPEncoder : BlockEncoder {
override fun encode(block: linea.domain.Block): ByteArray = RLP.encodeBlock(block.toBesu())
}

View File

@@ -1,44 +0,0 @@
package net.consensys.zkevm.encoding
import org.hyperledger.besu.datatypes.Address
import org.hyperledger.besu.datatypes.Hash
import org.hyperledger.besu.datatypes.Wei
import org.hyperledger.besu.ethereum.core.Block
import org.hyperledger.besu.ethereum.core.BlockBody
import org.hyperledger.besu.ethereum.core.BlockHeaderBuilder
import org.hyperledger.besu.ethereum.core.Difficulty
import org.hyperledger.besu.ethereum.core.encoding.EncodingContext
import org.hyperledger.besu.ethereum.core.encoding.TransactionDecoder
import org.hyperledger.besu.ethereum.mainnet.BodyValidation
import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions
import org.hyperledger.besu.evm.log.LogsBloomFilter
import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1
object ExecutionPayloadV1RLPEncoderByBesuImplementation : ExecutionPayloadV1Encoder {
override fun encode(payload: ExecutionPayloadV1): ByteArray {
val parsedTransactions = payload.transactions
.map { TransactionDecoder.decodeOpaqueBytes(it, EncodingContext.BLOCK_BODY) }
val parsedBody = BlockBody(parsedTransactions, emptyList())
val blockHeader =
BlockHeaderBuilder.create()
.parentHash(Hash.wrap(payload.parentHash))
.ommersHash(Hash.EMPTY_LIST_HASH)
.coinbase(Address.wrap(payload.feeRecipient.wrappedBytes))
.stateRoot(Hash.wrap(payload.stateRoot))
.transactionsRoot(BodyValidation.transactionsRoot(parsedBody.transactions))
.receiptsRoot(Hash.wrap(payload.receiptsRoot))
.logsBloom(LogsBloomFilter(payload.logsBloom))
.difficulty(Difficulty.ZERO)
.number(payload.blockNumber.longValue())
.gasLimit(payload.gasLimit.longValue())
.gasUsed(payload.gasLimit.longValue())
.timestamp(payload.timestamp.longValue())
.extraData(payload.extraData)
.baseFee(Wei.wrap(payload.baseFeePerGas.toBytes()))
.mixHash(Hash.wrap(payload.prevRandao))
.nonce(0) // this works because Linea is not using PoW
.blockHeaderFunctions(MainnetBlockHeaderFunctions())
.buildBlockHeader()
return Block(blockHeader, parsedBody).toRlp().toArray()
}
}

View File

@@ -1,33 +0,0 @@
package net.consensys.zkevm.encoding
import net.consensys.zkevm.toULong
import org.apache.tuweni.bytes.Bytes
import org.assertj.core.api.Assertions.assertThat
import org.hyperledger.besu.ethereum.core.Block
import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions
import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput
import org.junit.jupiter.api.Test
import tech.pegasys.teku.ethereum.executionclient.schema.randomExecutionPayload
class ExecutionPayloadV1RLPEncoderByBesuImplementationTest {
@Test
fun encode() {
val payload = randomExecutionPayload()
val rlpEncodedPayload = ExecutionPayloadV1RLPEncoderByBesuImplementation.encode(payload)
val block = Block.readFrom(BytesValueRLPInput(Bytes.wrap(rlpEncodedPayload), false), MainnetBlockHeaderFunctions())
assertThat(block.header.number.toULong()).isEqualTo(payload.blockNumber.toULong())
// we cannot assert oh block hash because Besu will calculate real Hash whereas random payload has random bytes
// assertThat(block.header.blockHash.toHexString()).isEqualTo(payload.blockHash.toHexString())
assertThat(block.header.gasLimit.toULong()).isEqualTo(payload.gasLimit.toULong())
assertThat(block.header.logsBloom.toArray()).isEqualTo(payload.logsBloom.toArray())
assertThat(block.header.parentHash.toArray()).isEqualTo(payload.parentHash.toArray())
assertThat(block.header.prevRandao.get().toArray()).isEqualTo(payload.prevRandao.toArray())
assertThat(block.header.stateRoot.toArray())
.isEqualTo(payload.stateRoot.toArray())
assertThat(payload.transactions).isEmpty()
// FIXME: add remaining fields assertions
}
}

View File

@@ -14,7 +14,6 @@ dependencies {
testImplementation("com.fasterxml.jackson.core:jackson-annotations:${libs.versions.jackson.get()}")
testImplementation("com.fasterxml.jackson.module:jackson-module-kotlin:${libs.versions.jackson.get()}")
testImplementation "io.tmio:tuweni-units:${libs.versions.tuweni.get()}"
testImplementation("tech.pegasys.teku.internal:executionclient:${libs.versions.teku.get()}")
testImplementation(project(":coordinator:persistence:db-common"))
testImplementation(testFixtures(project(":coordinator:core")))
testImplementation(testFixtures(project(":jvm-libs:generic:extensions:kotlin")))

View File

@@ -16,7 +16,6 @@ dependencies {
testImplementation("com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson.get()}")
testImplementation("com.fasterxml.jackson.core:jackson-annotations:${libs.versions.jackson.get()}")
testImplementation("com.fasterxml.jackson.module:jackson-module-kotlin:${libs.versions.jackson.get()}")
testImplementation("tech.pegasys.teku.internal:executionclient:${libs.versions.teku.get()}")
testImplementation(testFixtures(project(":jvm-libs:generic:persistence:db")))
testImplementation(testFixtures(project(":jvm-libs:generic:extensions:kotlin")))
testImplementation("io.vertx:vertx-junit5")

View File

@@ -6,7 +6,7 @@ plugins {
description = "Utilities related to futures used in Linea"
dependencies {
implementation "io.vertx:vertx-core"
api "io.vertx:vertx-core"
testImplementation("io.vertx:vertx-junit5")
}

View File

@@ -1,5 +1,6 @@
package net.consensys
import java.math.BigInteger
import java.util.HexFormat
fun String.decodeHex(): ByteArray {
@@ -10,3 +11,8 @@ fun String.decodeHex(): ByteArray {
fun String.containsAny(strings: List<String>, ignoreCase: Boolean): Boolean {
return strings.any { this.contains(it, ignoreCase) }
}
fun String.toIntFromHex(): Int = removePrefix("0x").toInt(16)
fun String.toLongFromHex(): Long = removePrefix("0x").toLong(16)
fun String.toULongFromHex(): ULong = BigInteger(removePrefix("0x"), 16).toULong()
fun String.toBigIntegerFromHex(): BigInteger = BigInteger(removePrefix("0x"), 16)

View File

@@ -64,7 +64,7 @@ fun ULong.toGWei(): Double = this.toDouble().toGWei()
* Parses an hexadecimal string as [ULong] number and returns the result.
* @throws NumberFormatException if the string is not a valid hexadecimal representation of a number.
*/
fun ULong.Companion.fromHexString(value: String): ULong = value.replace("0x", "").toULong(16)
fun ULong.Companion.fromHexString(value: String): ULong = value.removePrefix("0x").toULong(16)
fun <T : Comparable<T>> ClosedRange<T>.toIntervalString(): String {
val size = if (start <= endInclusive) {

View File

@@ -27,4 +27,28 @@ class StringExtensionsTest {
assertThat("this includes lorem ipsum".containsAny(stringList, ignoreCase = true)).isTrue()
assertThat("this string won't match".containsAny(stringList, ignoreCase = true)).isFalse()
}
@Test
fun `String#toIntFromHex`() {
assertThat("0x00".toIntFromHex()).isEqualTo(0)
assertThat("0x01".toIntFromHex()).isEqualTo(1)
assertThat("0x123456".toIntFromHex()).isEqualTo(1193046)
assertThat("0x7FFFFFFF".toIntFromHex()).isEqualTo(Int.MAX_VALUE)
}
@Test
fun `String#toLongFromHex`() {
assertThat("0x00".toLongFromHex()).isEqualTo(0L)
assertThat("0x01".toLongFromHex()).isEqualTo(1L)
assertThat("0x123456".toLongFromHex()).isEqualTo(1193046L)
assertThat("0x7FFFFFFFFFFFFFFF".toLongFromHex()).isEqualTo(Long.MAX_VALUE)
}
@Test
fun `String#toULongFromHex`() {
assertThat("0x00".toULongFromHex()).isEqualTo(0UL)
assertThat("0x01".toULongFromHex()).isEqualTo(1UL)
assertThat("0x123456".toULongFromHex()).isEqualTo(1193046UL)
assertThat("0xffffffffffffffff".toULongFromHex()).isEqualTo(ULong.MAX_VALUE)
}
}

View File

@@ -1,6 +1,7 @@
plugins {
id 'net.consensys.zkevm.kotlin-library-conventions'
id 'java-library'
id 'java-test-fixtures'
}
description = "JSON RPC 2.0 utilities"

View File

@@ -1,5 +1,8 @@
package net.consensys.linea.jsonrpc
import com.fasterxml.jackson.databind.JsonNode
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.fasterxml.jackson.module.kotlin.registerKotlinModule
import com.github.michaelbull.result.Err
import com.github.michaelbull.result.Ok
@@ -21,6 +24,7 @@ import io.vertx.core.json.Json
import io.vertx.core.json.JsonArray
import io.vertx.core.json.JsonObject
import io.vertx.core.json.jackson.DatabindCodec
import io.vertx.core.json.jackson.VertxModule
import io.vertx.ext.auth.User
import net.consensys.linea.metrics.micrometer.DynamicTagTimerCapture
import net.consensys.linea.metrics.micrometer.SimpleTimerCapture
@@ -51,12 +55,15 @@ private data class RequestContext(
class JsonRpcMessageProcessor(
private val requestsHandler: JsonRpcRequestHandler,
private val meterRegistry: MeterRegistry,
private val requestParser: JsonRpcRequestParser = Companion::parseRequest
private val requestParser: JsonRpcRequestParser = Companion::parseRequest,
private val log: Logger = LogManager.getLogger(JsonRpcMessageProcessor::class.java),
private val responseResultObjectMapper: ObjectMapper = jacksonObjectMapper().registerModules(VertxModule()),
private val rpcEnvelopeObjectMapper: ObjectMapper = jacksonObjectMapper()
) : JsonRpcMessageHandler {
init {
DatabindCodec.mapper().registerKotlinModule()
}
private val log: Logger = LogManager.getLogger(this.javaClass)
private val counterBuilder = Counter.builder("jsonrpc.counter")
override fun invoke(user: User?, messageJsonStr: String): Future<String> =
handleMessage(user, messageJsonStr)
@@ -174,7 +181,13 @@ class JsonRpcMessageProcessor(
return SimpleTimerCapture<String>(meterRegistry, "jsonrpc.serialization.response")
.setDescription("Time of json response serialization")
.setTag("method", requestContext.method)
.captureTime { Json.encode(requestContext.result.merge()) }
.captureTime {
val result = requestContext.result.map { successResponse ->
val resultJsonNode = responseResultObjectMapper.valueToTree<JsonNode>(successResponse.result)
successResponse.copy(result = resultJsonNode)
}
rpcEnvelopeObjectMapper.writeValueAsString(result.merge())
}
}
private fun handleRequest(

View File

@@ -32,6 +32,7 @@ data class JsonRpcSuccessResponse(
val result: Any?
) : JsonRpcResponse(jsonrpc, id) {
constructor(id: Any, result: Any?) : this("2.0", id, result)
constructor(request: JsonRpcRequest, result: Any?) : this(request.jsonrpc, id = request.id, result)
}
@JsonPropertyOrder("jsonrpc", "id", "error")
@@ -112,4 +113,6 @@ class JsonRpcErrorResponseException(
val rpcErrorCode: Int,
val rpcErrorMessage: String,
val rpcErrorData: Any? = null
) : RuntimeException("code=$rpcErrorCode message=$rpcErrorMessage errorData=$rpcErrorData")
) : RuntimeException("code=$rpcErrorCode message=$rpcErrorMessage errorData=$rpcErrorData") {
fun asJsonRpcError(): JsonRpcError = JsonRpcError(rpcErrorCode, rpcErrorMessage, rpcErrorData)
}

View File

@@ -14,12 +14,12 @@ import org.apache.logging.log4j.Logger
class HttpJsonRpcServer(
private val port: UInt,
private val path: String,
private val requestHandler: Handler<RoutingContext>
private val requestHandler: Handler<RoutingContext>,
val serverName: String = ""
) : AbstractVerticle() {
private val log: Logger = LogManager.getLogger(this.javaClass)
private lateinit var httpServer: HttpServer
val bindedPort: Int
val boundPort: Int
get() = if (this::httpServer.isInitialized) {
httpServer.actualPort()
} else {
@@ -28,15 +28,19 @@ class HttpJsonRpcServer(
override fun start(startPromise: Promise<Void>) {
val options = HttpServerOptions().setPort(port.toInt()).setReusePort(true)
log.debug("Creating Http server on port {}", port)
log.debug("creating {} Http server on port {}", port)
httpServer = vertx.createHttpServer(options)
httpServer.requestHandler(buildRouter())
httpServer.listen { res: AsyncResult<HttpServer> ->
if (res.succeeded()) {
log.info("Http server started and listening on port {}", res.result().actualPort())
log.info(
"{} http server started and listening on port {}",
serverName,
res.result().actualPort()
)
startPromise.complete()
} else {
log.error("Creating Http server: {}", res.cause())
log.error("error creating {} http server: {}", serverName, res.cause())
startPromise.fail(res.cause())
}
}
@@ -50,5 +54,6 @@ class HttpJsonRpcServer(
override fun stop(endFuture: Promise<Void>) {
httpServer.close(endFuture)
super.stop(endFuture)
}
}

View File

@@ -0,0 +1,123 @@
package linea.jsonrpc
import com.github.michaelbull.result.Err
import com.github.michaelbull.result.Ok
import io.micrometer.core.instrument.simple.SimpleMeterRegistry
import io.vertx.junit5.VertxExtension
import net.consensys.linea.async.get
import net.consensys.linea.jsonrpc.JsonRpcErrorResponse
import net.consensys.linea.jsonrpc.JsonRpcErrorResponseException
import net.consensys.linea.jsonrpc.JsonRpcSuccessResponse
import net.consensys.linea.jsonrpc.client.JsonRpcV2Client
import net.consensys.linea.jsonrpc.client.RequestRetryConfig
import net.consensys.linea.jsonrpc.client.VertxHttpJsonRpcClientFactory
import org.assertj.core.api.Assertions.assertThat
import org.assertj.core.api.Assertions.assertThatThrownBy
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.extension.ExtendWith
import java.net.URI
import kotlin.time.Duration.Companion.milliseconds
import kotlin.time.Duration.Companion.minutes
@ExtendWith(VertxExtension::class)
class TestingJsonRpcServerTest {
private lateinit var jsonRpcServer: TestingJsonRpcServer
private lateinit var client: JsonRpcV2Client
@BeforeEach
fun beforeEach(vertx: io.vertx.core.Vertx) {
jsonRpcServer = TestingJsonRpcServer(
vertx = vertx,
recordRequestsResponses = true
)
val rpcClientFactory = VertxHttpJsonRpcClientFactory(
vertx = vertx,
meterRegistry = SimpleMeterRegistry()
)
client = rpcClientFactory.createJsonRpcV2Client(
endpoints = listOf(URI.create("http://localhost:${jsonRpcServer.boundPort}")),
retryConfig = RequestRetryConfig(
maxRetries = 10u,
backoffDelay = 10.milliseconds,
timeout = 2.minutes
),
shallRetryRequestsClientBasePredicate = {
false
} // disable retry
)
}
@Test
fun `when no method handler is defined returns method not found`() {
assertThatThrownBy {
client.makeRequest(
method = "not_existing_method",
params = mapOf("k1" to "v1", "k2" to 100),
resultMapper = { it },
shallRetryRequestPredicate = { false }
).get()
}.hasCauseInstanceOf(JsonRpcErrorResponseException::class.java)
.hasMessageContaining("Method not found")
// check recorded request
jsonRpcServer.recordedRequests().also {
assertThat(it).hasSize(1)
val (request, responseFuture) = it[0]
assertThat(request.method).isEqualTo("not_existing_method")
assertThat(request.params).isEqualTo(mapOf("k1" to "v1", "k2" to 100))
assertThat(responseFuture.get()).isEqualTo(
Err(JsonRpcErrorResponse.methodNotFound(request.id, data = "not_existing_method"))
)
}
}
@Test
fun `when handlers are provided shall forward to correct one`() {
jsonRpcServer.handle("add") { request ->
@Suppress("UNCHECKED_CAST")
val params = request.params as List<Int>
params.sumOf { it }
}
jsonRpcServer.handle("addUser") { request ->
@Suppress("UNCHECKED_CAST")
val params = request.params as Map<String, Any?>
"user=${params["name"]} email=${params["email"]}"
}
jsonRpcServer.handle("multiply") { _ -> "not expected" }
assertThat(
client.makeRequest(
method = "add",
params = listOf(1, 2, 3),
resultMapper = { it }
).get()
)
.isEqualTo(6)
assertThat(
client.makeRequest(
method = "addUser",
params = mapOf("name" to "John", "email" to "john@email.com"),
resultMapper = { it }
).get()
)
.isEqualTo("user=John email=john@email.com")
// check recorded request
jsonRpcServer.recordedRequests().also {
assertThat(it).hasSize(2)
it[0].also { (request, responseFuture) ->
assertThat(request.method).isEqualTo("add")
assertThat(request.params).isEqualTo(listOf(1, 2, 3))
assertThat(responseFuture.get()).isEqualTo(Ok(JsonRpcSuccessResponse(id = request.id, result = 6)))
}
it[1].also { (request, responseFuture) ->
assertThat(request.method).isEqualTo("addUser")
assertThat(request.params).isEqualTo(mapOf("name" to "John", "email" to "john@email.com"))
assertThat(responseFuture.get())
.isEqualTo(Ok(JsonRpcSuccessResponse(id = request.id, result = "user=John email=john@email.com")))
}
}
}
}

View File

@@ -0,0 +1,175 @@
package linea.jsonrpc
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.github.michaelbull.result.Err
import com.github.michaelbull.result.Ok
import com.github.michaelbull.result.Result
import io.micrometer.core.instrument.simple.SimpleMeterRegistry
import io.vertx.core.DeploymentOptions
import io.vertx.core.Future
import io.vertx.core.Promise
import io.vertx.core.Vertx
import io.vertx.core.json.JsonObject
import io.vertx.ext.auth.User
import net.consensys.linea.async.get
import net.consensys.linea.jsonrpc.HttpRequestHandler
import net.consensys.linea.jsonrpc.JsonRpcErrorResponse
import net.consensys.linea.jsonrpc.JsonRpcErrorResponseException
import net.consensys.linea.jsonrpc.JsonRpcMessageProcessor
import net.consensys.linea.jsonrpc.JsonRpcRequest
import net.consensys.linea.jsonrpc.JsonRpcSuccessResponse
import net.consensys.linea.jsonrpc.httpserver.HttpJsonRpcServer
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
import tech.pegasys.teku.infrastructure.async.SafeFuture
import java.util.concurrent.ConcurrentHashMap
import kotlin.time.Duration
import kotlin.time.Duration.Companion.milliseconds
open class TestingJsonRpcServer(
port: Int = 0,
val apiPath: String = "/",
val recordRequestsResponses: Boolean = false,
val serverName: String = "FakeJsonRpcServer",
loggerName: String = serverName,
val vertx: Vertx = Vertx.vertx(),
val responseObjectMapper: ObjectMapper = jacksonObjectMapper(),
responsesArtificialDelay: Duration? = null
) {
val log: Logger = LogManager.getLogger(loggerName)
private var httpServer: HttpJsonRpcServer = createHttpServer(port)
val boundPort: Int
get() = httpServer.boundPort
private var verticleId: String? = null
private val handlers: MutableMap<String, (JsonRpcRequest) -> Any?> = ConcurrentHashMap()
private var requests: MutableList<
Pair<JsonRpcRequest, Future<Result<JsonRpcSuccessResponse, JsonRpcErrorResponse>>>
> = mutableListOf()
var responsesArtificialDelay: Duration? = responsesArtificialDelay
set(value) {
require(value == null || value > 0.milliseconds) { "artificialDelay=$value must be greater than 0ms" }
field = value
}
private fun createHttpServer(port: Int?): HttpJsonRpcServer {
return HttpJsonRpcServer(
port = port?.toUInt() ?: 0u,
path = apiPath,
requestHandler = HttpRequestHandler(
JsonRpcMessageProcessor(
requestsHandler = this::handleRequest,
meterRegistry = SimpleMeterRegistry(),
log = log,
responseResultObjectMapper = responseObjectMapper
)
),
serverName = serverName
)
}
init {
vertx
.deployVerticle(httpServer, DeploymentOptions().setInstances(1))
.onSuccess { verticleId: String -> this.verticleId = verticleId }
.get()
}
fun stopHttpServer(): Future<Unit> {
return vertx.undeploy(verticleId).map { }
}
fun resumeHttpServer(): Future<Unit> {
// reuse the same port
httpServer = createHttpServer(boundPort)
return vertx
.deployVerticle(httpServer, DeploymentOptions().setInstances(1))
.onSuccess { verticleId: String ->
log.info("Http server resumed at port {}", httpServer.boundPort)
this.verticleId = verticleId
}
.onFailure { th ->
log.error("Error resuming http server", th)
}
.map { }
}
@Suppress("UNUSED_PARAMETER")
private fun handleRequest(
user: User?,
jsonRpcRequest: JsonRpcRequest,
requestJson: JsonObject
): Future<Result<JsonRpcSuccessResponse, JsonRpcErrorResponse>> {
// need this otherwise kotlin compiler/IDE struggle to infer the type
val result: Future<Result<JsonRpcSuccessResponse, JsonRpcErrorResponse>> = (
handlers[jsonRpcRequest.method]
?.let { handler ->
try {
val result = handler(jsonRpcRequest)
Future.succeededFuture(
Ok(
JsonRpcSuccessResponse(
request = jsonRpcRequest,
result = result
)
)
)
} catch (e: JsonRpcErrorResponseException) {
Future.succeededFuture(Err(JsonRpcErrorResponse(jsonRpcRequest.id, e.asJsonRpcError())))
} catch (e: Exception) {
Future.succeededFuture(Err(JsonRpcErrorResponse.internalError(jsonRpcRequest.id, data = e.message)))
}
}
?: Future.succeededFuture(Err(JsonRpcErrorResponse.methodNotFound(jsonRpcRequest.id, jsonRpcRequest.method)))
)
return result
.let { future ->
responsesArtificialDelay?.let { future.delayed(it) } ?: future
}
.also {
if (recordRequestsResponses) {
requests.add(jsonRpcRequest to it)
}
}
}
/**
* Handler shall return response result or throw [JsonRpcErrorResponseException] if error
*/
fun handle(
method: String,
methodHandler: (jsonRpcRequest: JsonRpcRequest) -> Any?
) {
handlers[method] = methodHandler
}
fun recordedRequests(): List<Pair<JsonRpcRequest, Future<Result<JsonRpcSuccessResponse, JsonRpcErrorResponse>>>> {
return requests.toList()
}
fun cleanRecordedRequests() {
requests.clear()
}
fun callCountByMethod(method: String): Int {
return requests.count { it.first.method == method }
}
private fun <T> Future<T>.delayed(delay: Duration): Future<T> {
val promise = Promise.promise<T>()
vertx.setTimer(delay.inWholeMilliseconds) {
this.onComplete(promise)
}
return promise.future()
}
private fun <T> SafeFuture<T>.delayed(delay: Duration): SafeFuture<T> {
val promise = SafeFuture<T>()
vertx.setTimer(delay.inWholeMilliseconds) {
this.thenAccept(promise::complete).exceptionally { promise.completeExceptionally(it); null }
}
return promise
}
}

View File

@@ -0,0 +1,14 @@
package linea.log4j
import org.apache.logging.log4j.Level
import org.apache.logging.log4j.core.config.Configurator
fun configureLoggers(
rootLevel: Level = Level.INFO,
vararg loggerConfigs: Pair<String, Level>
) {
Configurator.setRootLevel(rootLevel)
loggerConfigs.forEach { (loggerName, level) ->
Configurator.setLevel(loggerName, level)
}
}

View File

@@ -20,4 +20,16 @@ dependencies {
api("org.hyperledger.besu:plugin-api:${libs.versions.besu.get()}") {
transitive = false
}
api("org.hyperledger.besu.internal:rlp:${libs.versions.besu.get()}") {
transitive = false
}
api("io.tmio:tuweni-bytes:${libs.versions.tuweni.get()}") {
transitive = false
}
api("io.tmio:tuweni-units:${libs.versions.tuweni.get()}") {
transitive = false
}
}

View File

@@ -0,0 +1,11 @@
plugins {
id 'net.consensys.zkevm.kotlin-library-conventions'
}
dependencies {
api(project(':jvm-libs:generic:extensions:kotlin'))
api(project(':jvm-libs:generic:extensions:futures'))
api(project(':jvm-libs:linea:core:domain-models'))
api(project(':jvm-libs:linea:besu-libs'))
api "io.vertx:vertx-core"
}

View File

@@ -0,0 +1,63 @@
package linea.domain
import linea.domain.MapperBesuToLineaDomain.mapToDomain
import net.consensys.toULong
import org.hyperledger.besu.ethereum.core.Transaction
import kotlin.jvm.optionals.getOrNull
fun org.hyperledger.besu.ethereum.core.Block.toDomain(): Block {
return mapToDomain(this)
}
object MapperBesuToLineaDomain {
fun mapToDomain(besuBlock: org.hyperledger.besu.ethereum.core.Block): Block {
val block = Block(
number = besuBlock.header.getNumber().toULong(),
hash = besuBlock.header.hash.toArray(),
parentHash = besuBlock.header.parentHash.toArray(),
ommersHash = besuBlock.header.ommersHash.toArray(),
miner = besuBlock.header.coinbase.toArray(),
stateRoot = besuBlock.header.stateRoot.toArray(),
transactionsRoot = besuBlock.header.transactionsRoot.toArray(),
receiptsRoot = besuBlock.header.receiptsRoot.toArray(),
logsBloom = besuBlock.header.logsBloom.toArray(),
difficulty = besuBlock.header.difficulty.toBigInteger().toULong(),
gasLimit = besuBlock.header.gasLimit.toULong(),
gasUsed = besuBlock.header.gasUsed.toULong(),
timestamp = besuBlock.header.timestamp.toULong(),
extraData = besuBlock.header.extraData.toArray(),
mixHash = besuBlock.header.mixHash.toArray(),
nonce = besuBlock.header.nonce.toULong(),
baseFeePerGas = besuBlock.header.baseFee.getOrNull()?.toBigInteger()?.toULong(),
ommers = besuBlock.body.ommers.map { it.hash.toArray() },
transactions = besuBlock.body.transactions.map(MapperBesuToLineaDomain::mapToDomain)
)
return block
}
fun mapToDomain(transaction: Transaction): linea.domain.Transaction {
return Transaction(
nonce = transaction.nonce.toULong(),
gasPrice = transaction.getGasPrice().getOrNull()?.toBigInteger()?.toULong(),
gasLimit = transaction.gasLimit.toULong(),
to = transaction.to.getOrNull()?.toArray(),
value = transaction.value.toBigInteger(),
input = transaction.payload.toArray(),
r = transaction.signature.getR(),
s = transaction.signature.getS(),
v = transaction.getV().toULong(),
yParity = transaction.yParity?.toULong(),
type = transaction.type.toDomain(),
chainId = transaction.chainId.getOrNull()?.toULong(),
maxFeePerGas = transaction.maxFeePerGas.getOrNull()?.toBigInteger()?.toULong(),
maxPriorityFeePerGas = transaction.maxPriorityFeePerGas.getOrNull()?.toBigInteger()?.toULong(),
accessList = transaction.accessList.getOrNull()?.map { accessListEntry ->
AccessListEntry(
accessListEntry.address.toArray(),
accessListEntry.storageKeys.map { it.toArray() }
)
}
)
}
}

View File

@@ -0,0 +1,142 @@
package linea.domain
import linea.domain.MapperLineaDomainToBesu.mapToBesu
import net.consensys.encodeHex
import net.consensys.toBigInteger
import org.apache.tuweni.bytes.Bytes
import org.apache.tuweni.bytes.Bytes32
import org.hyperledger.besu.crypto.SECP256K1
import org.hyperledger.besu.datatypes.AccessListEntry
import org.hyperledger.besu.datatypes.Address
import org.hyperledger.besu.datatypes.Hash
import org.hyperledger.besu.datatypes.Wei
import org.hyperledger.besu.ethereum.core.BlockBody
import org.hyperledger.besu.ethereum.core.BlockHeaderBuilder
import org.hyperledger.besu.ethereum.core.Difficulty
import org.hyperledger.besu.ethereum.core.Transaction
import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions
import org.hyperledger.besu.evm.log.LogsBloomFilter
import java.math.BigInteger
fun Block.toBesu(): org.hyperledger.besu.ethereum.core.Block = mapToBesu(this)
fun linea.domain.Transaction.toBesu(): Transaction = mapToBesu(this)
object MapperLineaDomainToBesu {
private val secp256k1 = SECP256K1()
private val blockHeaderFunctions = MainnetBlockHeaderFunctions()
fun recIdFromV(v: BigInteger): Pair<Byte, BigInteger?> {
val recId: Byte
var chainId: BigInteger? = null
if (v == Transaction.REPLAY_UNPROTECTED_V_BASE || v == Transaction.REPLAY_UNPROTECTED_V_BASE_PLUS_1) {
recId = v.subtract(Transaction.REPLAY_UNPROTECTED_V_BASE).byteValueExact()
} else if (v > Transaction.REPLAY_PROTECTED_V_MIN) {
chainId = v.subtract(Transaction.REPLAY_PROTECTED_V_BASE).divide(Transaction.TWO)
recId = v.subtract(Transaction.TWO.multiply(chainId).add(Transaction.REPLAY_PROTECTED_V_BASE)).byteValueExact()
} else {
throw RuntimeException("An unsupported encoded `v` value of $v was found")
}
return Pair(recId, chainId)
}
fun getRecIdAndChainId(tx: linea.domain.Transaction): Pair<Byte, BigInteger?> {
if (tx.type == TransactionType.FRONTIER) {
return recIdFromV(tx.v.toBigInteger())
} else {
return tx.v.toByte() to tx.chainId?.toBigInteger()
}
}
fun mapToBesu(block: Block): org.hyperledger.besu.ethereum.core.Block {
runCatching {
val header = BlockHeaderBuilder.create()
.parentHash(Hash.wrap(Bytes32.wrap(block.parentHash)))
.ommersHash(Hash.wrap(Bytes32.wrap(block.ommersHash)))
.coinbase(Address.wrap(Bytes.wrap(block.miner)))
.stateRoot(Hash.wrap(Bytes32.wrap(block.stateRoot)))
.transactionsRoot(Hash.wrap(Bytes32.wrap(block.transactionsRoot)))
.receiptsRoot(Hash.wrap(Bytes32.wrap(block.receiptsRoot)))
.logsBloom(LogsBloomFilter.fromHexString(block.logsBloom.encodeHex()))
.difficulty(Difficulty.fromHexOrDecimalString(block.difficulty.toString()))
.number(block.number.toLong())
.gasLimit(block.gasLimit.toLong())
.gasUsed(block.gasUsed.toLong())
.timestamp(block.timestamp.toLong())
.extraData(Bytes.wrap(block.extraData))
.mixHash(Hash.wrap(Bytes32.wrap(block.mixHash)))
.nonce(block.nonce.toLong())
.baseFee(block.baseFeePerGas?.toWei())
.blockHeaderFunctions(blockHeaderFunctions)
.buildBlockHeader()
val transactions =
block.transactions.mapIndexed { index, transaction ->
mapToBesu(block.number, index, transaction)
}
// linea does not support uncles, so we are not converting them
// throwing an exception just in case we get one and we can fix it
if (block.ommers.isNotEmpty()) {
throw IllegalStateException("Uncles are not supported: block=${block.number}")
}
val body = BlockBody(transactions, emptyList())
return org.hyperledger.besu.ethereum.core.Block(header, body)
}.getOrElse { th ->
if (th.message?.startsWith("Error mapping transaction to Besu") ?: false) {
throw th
} else {
throw RuntimeException("Error mapping block to Besu: block=${block.number}", th)
}
}
}
fun mapToBesu(blockNumber: ULong, txIndex: Int, tx: linea.domain.Transaction): Transaction {
return runCatching { mapToBesu(tx) }
.getOrElse { th ->
throw RuntimeException(
"Error mapping transaction to Besu: block=$blockNumber txIndex=$txIndex transaction=$tx",
th
)
}
}
fun mapToBesu(tx: linea.domain.Transaction): Transaction {
val (recId, chainId) = getRecIdAndChainId(tx)
val signature = secp256k1.createSignature(
tx.r,
tx.s,
recId
)
val besuType = tx.type.toBesu()
return Transaction.builder()
.type(tx.type.toBesu())
.nonce(tx.nonce.toLong())
.apply { tx.gasPrice?.let { gasPrice(it.toWei()) } }
.gasLimit(tx.gasLimit.toLong())
.to(tx.to?.let { Address.wrap(Bytes.wrap(it)) })
.value(tx.value.toWei())
.payload(Bytes.wrap(tx.input))
.chainId(tx.chainId?.toBigInteger() ?: chainId)
.maxPriorityFeePerGas(tx.maxPriorityFeePerGas?.toWei())
.maxFeePerGas(tx.maxFeePerGas?.toWei())
.apply {
if (besuType.supportsAccessList()) {
val accList = tx.accessList?.map { entry ->
AccessListEntry(
Address.wrap(Bytes.wrap(entry.address)),
entry.storageKeys.map { Bytes32.wrap(it) }
)
} ?: emptyList()
accessList(accList)
}
}
.signature(signature)
.build()
}
fun ULong.toWei(): Wei = Wei.of(this.toBigInteger())
fun BigInteger.toWei(): Wei = Wei.of(this)
}

View File

@@ -0,0 +1,23 @@
package linea.domain
import org.hyperledger.besu.datatypes.TransactionType
fun TransactionType.toDomain(): linea.domain.TransactionType {
return when (this) {
TransactionType.FRONTIER -> linea.domain.TransactionType.FRONTIER
TransactionType.EIP1559 -> linea.domain.TransactionType.EIP1559
TransactionType.ACCESS_LIST -> linea.domain.TransactionType.ACCESS_LIST
TransactionType.BLOB -> linea.domain.TransactionType.BLOB
TransactionType.DELEGATE_CODE -> linea.domain.TransactionType.DELEGATE_CODE
}
}
fun linea.domain.TransactionType.toBesu(): TransactionType {
return when (this) {
linea.domain.TransactionType.FRONTIER -> TransactionType.FRONTIER
linea.domain.TransactionType.EIP1559 -> TransactionType.EIP1559
linea.domain.TransactionType.ACCESS_LIST -> TransactionType.ACCESS_LIST
linea.domain.TransactionType.BLOB -> TransactionType.BLOB
linea.domain.TransactionType.DELEGATE_CODE -> TransactionType.DELEGATE_CODE
}
}

View File

@@ -0,0 +1,56 @@
package linea.rlp
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
import org.apache.tuweni.bytes.Bytes
import org.hyperledger.besu.datatypes.Hash
import org.hyperledger.besu.ethereum.core.Block
import org.hyperledger.besu.ethereum.core.BlockBody
import org.hyperledger.besu.ethereum.core.BlockHeader
import org.hyperledger.besu.ethereum.core.BlockHeaderFunctions
import org.hyperledger.besu.ethereum.core.ParsedExtraData
import org.hyperledger.besu.ethereum.core.Transaction
import org.hyperledger.besu.ethereum.core.Withdrawal
import org.hyperledger.besu.ethereum.rlp.RLPInput
object BesuRlpBlobDecoder : BesuBlockRlpDecoder {
val log: Logger = LogManager.getLogger(BesuRlpBlobDecoder::class.java)
val transactionDecoder: NoSignatureTransactionDecoder = NoSignatureTransactionDecoder()
// 1.Decompressor places Block's hash in parentHash
// Because we are reusing Geth/Besu rlp encoding that recalculate the hashes.
// so here we override the hash function to use the parentHash as the hash
// 2. we don't compresse extraData, so just returning null
val hashFunction: BlockHeaderFunctions = object : BlockHeaderFunctions {
override fun hash(blockHeader: BlockHeader): Hash = blockHeader.parentHash
override fun parseExtraData(blockHeader: BlockHeader): ParsedExtraData? = null
}
override fun decode(block: ByteArray): Block {
log.trace("Decoding block from RLP blob: rawRlpSize={}", block.size)
return decode(org.hyperledger.besu.ethereum.rlp.RLP.input(Bytes.wrap(block)), hashFunction)
}
fun decode(rlpInput: RLPInput, hashFunction: BlockHeaderFunctions): Block {
rlpInput.enterList()
// Read the header
val header: BlockHeader = BlockHeader.readFrom(rlpInput, hashFunction)
// Use NoSignatureTransactionDecoder to decode transactions
val transactions: List<Transaction> = rlpInput.readList(transactionDecoder::decode)
// Read the ommers
val ommers: List<BlockHeader> = rlpInput.readList { rlp: RLPInput ->
BlockHeader.readFrom(rlp, hashFunction)
}
// Read the withdrawals
if (!rlpInput.isEndOfCurrentList) {
rlpInput.readList<Any>(Withdrawal::readFrom)
}
rlpInput.leaveList()
return Block(header, BlockBody(transactions, ommers))
}
}

View File

@@ -0,0 +1,60 @@
package linea.rlp
import io.vertx.core.Vertx
import net.consensys.linea.async.toSafeFuture
import org.hyperledger.besu.ethereum.core.Block
import tech.pegasys.teku.infrastructure.async.SafeFuture
import java.util.concurrent.Callable
object BesuMainnetBlockRlpEncoder : BesuBlockRlpEncoder {
override fun encode(block: Block): ByteArray = RLP.encodeBlock(block)
}
object BesuMainnetBlockRlpDecoder : BesuBlockRlpDecoder {
override fun decode(block: ByteArray): Block = RLP.decodeBlockWithMainnetFunctions(block)
}
class BesuRlpMainnetEncoderAsyncVertxImpl(
val vertx: Vertx,
val encoder: BesuBlockRlpEncoder = BesuMainnetBlockRlpEncoder
) : BesuBlockRlpEncoderAsync {
override fun encodeAsync(block: Block): SafeFuture<ByteArray> {
return vertx.executeBlocking(
Callable {
encoder.encode(block)
},
false
)
.toSafeFuture()
}
}
/**
* We can decode with Mainnet full functionality or
* with custom decoder for blob decompressed transactions without signature and blocks without header
* used for state reconstruction
*/
class BesuRlpDecoderAsyncVertxImpl(
private val vertx: Vertx,
private val decoder: BesuBlockRlpDecoder
) : BesuBlockRlpDecoderAsync {
companion object {
fun mainnetDecoder(vertx: Vertx): BesuBlockRlpDecoderAsync {
return BesuRlpDecoderAsyncVertxImpl(vertx, BesuMainnetBlockRlpDecoder)
}
fun blobDecoder(vertx: Vertx): BesuBlockRlpDecoderAsync {
return BesuRlpDecoderAsyncVertxImpl(vertx, BesuRlpBlobDecoder)
}
}
override fun decodeAsync(block: ByteArray): SafeFuture<Block> {
return vertx.executeBlocking(
Callable {
decoder.decode(block)
},
false
)
.toSafeFuture()
}
}

View File

@@ -0,0 +1,12 @@
package linea.rlp
import linea.domain.BinaryDecoder
import linea.domain.BinaryDecoderAsync
import linea.domain.BinaryEncoder
import linea.domain.BinaryEncoderAsync
import org.hyperledger.besu.ethereum.core.Block
interface BesuBlockRlpEncoder : BinaryEncoder<Block>
interface BesuBlockRlpEncoderAsync : BinaryEncoderAsync<Block>
interface BesuBlockRlpDecoder : BinaryDecoder<Block>
interface BesuBlockRlpDecoderAsync : BinaryDecoderAsync<Block>

View File

@@ -0,0 +1,141 @@
package linea.rlp
import org.apache.tuweni.bytes.Bytes
import org.hyperledger.besu.crypto.SECPSignature
import org.hyperledger.besu.datatypes.AccessListEntry
import org.hyperledger.besu.datatypes.Address
import org.hyperledger.besu.datatypes.TransactionType
import org.hyperledger.besu.datatypes.Wei
import org.hyperledger.besu.ethereum.core.Transaction
import org.hyperledger.besu.ethereum.rlp.RLP
import org.hyperledger.besu.ethereum.rlp.RLPInput
import java.math.BigInteger
class NoSignatureTransactionDecoder {
fun decode(input: RLPInput): Transaction {
if (!input.nextIsList()) {
val typedTransactionBytes = input.readBytes()
val transactionInput = RLP.input(typedTransactionBytes.slice(1))
val transactionType = typedTransactionBytes[0]
if (transactionType.toInt() == 0x01) {
return decodeAccessList(transactionInput)
}
if (transactionType.toInt() == 0x02) {
return decode1559(transactionInput)
}
throw IllegalArgumentException("Unsupported transaction type")
} else { // Frontier transaction
return decodeFrontier(input)
}
}
private fun decodeAccessList(transactionInput: RLPInput): Transaction {
val builder = Transaction.builder()
transactionInput.enterList()
builder
.type(TransactionType.ACCESS_LIST)
.chainId(BigInteger.valueOf(transactionInput.readLongScalar()))
.nonce(transactionInput.readLongScalar())
.gasPrice(Wei.of(transactionInput.readUInt256Scalar()))
.gasLimit(transactionInput.readLongScalar())
.to(
transactionInput
.readBytes { addressBytes: Bytes ->
if (addressBytes.isEmpty) null else Address.wrap(addressBytes)
}
)
.value(Wei.of(transactionInput.readUInt256Scalar()))
.payload(transactionInput.readBytes())
.accessList(
transactionInput.readList { accessListEntryRLPInput: RLPInput ->
accessListEntryRLPInput.enterList()
val accessListEntry =
AccessListEntry(
Address.wrap(accessListEntryRLPInput.readBytes()),
accessListEntryRLPInput.readList { obj: RLPInput -> obj.readBytes32() }
)
accessListEntryRLPInput.leaveList()
accessListEntry
}
)
transactionInput.readUnsignedByteScalar()
builder.sender(Address.extract(transactionInput.readUInt256Scalar()))
transactionInput.readUInt256Scalar()
transactionInput.leaveList()
return builder.signature(SECPSignature(BigInteger.ZERO, BigInteger.ZERO, 0.toByte())).build()
}
private fun decode1559(transactionInput: RLPInput): Transaction {
val builder = Transaction.builder()
transactionInput.enterList()
val chainId = transactionInput.readBigIntegerScalar()
builder
.type(TransactionType.EIP1559)
.chainId(chainId)
.nonce(transactionInput.readLongScalar())
.maxPriorityFeePerGas(Wei.of(transactionInput.readUInt256Scalar()))
.maxFeePerGas(Wei.of(transactionInput.readUInt256Scalar()))
.gasLimit(transactionInput.readLongScalar())
.to(
transactionInput.readBytes { v: Bytes ->
if (v.isEmpty) {
null
} else {
Address.wrap(
v
)
}
}
)
.value(Wei.of(transactionInput.readUInt256Scalar()))
.payload(transactionInput.readBytes())
.accessList(
transactionInput.readList { accessListEntryRLPInput: RLPInput ->
accessListEntryRLPInput.enterList()
val accessListEntry =
AccessListEntry(
Address.wrap(accessListEntryRLPInput.readBytes()),
accessListEntryRLPInput.readList { obj: RLPInput -> obj.readBytes32() }
)
accessListEntryRLPInput.leaveList()
accessListEntry
}
)
transactionInput.readUnsignedByteScalar()
builder.sender(Address.extract(transactionInput.readUInt256Scalar()))
transactionInput.readUInt256Scalar()
transactionInput.leaveList()
return builder.signature(SECPSignature(BigInteger.ZERO, BigInteger.ZERO, 0.toByte())).build()
}
private fun decodeFrontier(input: RLPInput): Transaction {
val builder = Transaction.builder()
input.enterList()
builder
.type(TransactionType.FRONTIER)
.nonce(input.readLongScalar())
.gasPrice(Wei.of(input.readUInt256Scalar()))
.gasLimit(input.readLongScalar())
.to(
input.readBytes { v: Bytes ->
if (v.isEmpty) {
null
} else {
Address.wrap(
v
)
}
}
)
.value(Wei.of(input.readUInt256Scalar()))
.payload(input.readBytes())
input.readBigIntegerScalar()
builder.sender(Address.extract(input.readUInt256Scalar()))
input.readUInt256Scalar()
val signature = SECPSignature(BigInteger.ZERO, BigInteger.ZERO, 0.toByte())
input.leaveList()
return builder.signature(signature).build()
}
}

View File

@@ -0,0 +1,43 @@
package linea.rlp
import org.apache.tuweni.bytes.Bytes
import org.hyperledger.besu.ethereum.core.Block
import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions
import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput
import org.hyperledger.besu.ethereum.rlp.RLP
object RLP {
fun encodeBlock(besuBlock: org.hyperledger.besu.ethereum.core.Block): ByteArray {
return besuBlock.toRlp().toArray()
}
fun decodeBlockWithMainnetFunctions(block: ByteArray): org.hyperledger.besu.ethereum.core.Block {
return Block.readFrom(
RLP.input(Bytes.wrap(block)),
MainnetBlockHeaderFunctions()
)
}
fun encodeList(list: List<ByteArray>): ByteArray {
val encoder = BytesValueRLPOutput()
encoder.startList()
list.forEach {
encoder.writeBytes(Bytes.wrap(it))
}
encoder.endList()
return encoder.encoded().toArray()
}
fun decodeList(
bytes: ByteArray
): List<ByteArray> {
val items = mutableListOf<ByteArray>()
val rlpInput = RLP.input(Bytes.wrap(bytes), false)
rlpInput.enterList()
while (!rlpInput.isEndOfCurrentList) {
items.add(rlpInput.readBytes().toArray())
}
rlpInput.leaveList()
return items
}
}

View File

@@ -9,6 +9,8 @@ description = 'Java JNA wrapper for Linea Blob Compressor Library implemented in
dependencies {
implementation "net.java.dev.jna:jna:${libs.versions.jna.get()}"
implementation project(":jvm-libs:generic:extensions:kotlin")
implementation "org.apache.logging.log4j:log4j-api:${libs.versions.log4j.get()}"
implementation "org.apache.logging.log4j:log4j-core:${libs.versions.log4j.get()}"
testImplementation project(":jvm-libs:linea:blob-shnarf-calculator")
}

View File

@@ -0,0 +1,113 @@
package linea.blob
import net.consensys.encodeHex
import net.consensys.linea.blob.BlobCompressorVersion
import net.consensys.linea.blob.GoNativeBlobCompressor
import net.consensys.linea.blob.GoNativeBlobCompressorFactory
import org.apache.logging.log4j.LogManager
class BlobCompressionException(message: String) : RuntimeException(message)
interface BlobCompressor {
/**
* @Throws(BlobCompressionException::class) when blockRLPEncoded is invalid
*/
fun canAppendBlock(blockRLPEncoded: ByteArray): Boolean
/**
* @Throws(BlobCompressionException::class) when blockRLPEncoded is invalid
*/
fun appendBlock(blockRLPEncoded: ByteArray): AppendResult
fun startNewBatch()
fun getCompressedData(): ByteArray
fun reset()
data class AppendResult(
// returns false if last chunk would go over dataLimit. Does not append last block.
val blockAppended: Boolean,
val compressedSizeBefore: Int,
// even when block is not appended, compressedSizeAfter should as if it was appended
val compressedSizeAfter: Int
)
}
class GoBackedBlobCompressor private constructor(
internal val goNativeBlobCompressor: GoNativeBlobCompressor
) : BlobCompressor {
companion object {
@Volatile
private var instance: GoBackedBlobCompressor? = null
fun getInstance(
compressorVersion: BlobCompressorVersion = BlobCompressorVersion.V0_1_0,
dataLimit: UInt
): GoBackedBlobCompressor {
if (instance == null) {
synchronized(this) {
if (instance == null) {
val goNativeBlobCompressor = GoNativeBlobCompressorFactory.getInstance(compressorVersion)
val initialized = goNativeBlobCompressor.Init(
dataLimit.toInt(),
GoNativeBlobCompressorFactory.dictionaryPath.toString()
)
if (!initialized) {
throw InstantiationException(goNativeBlobCompressor.Error())
}
instance = GoBackedBlobCompressor(goNativeBlobCompressor)
} else {
throw IllegalStateException("Compressor singleton instance already created")
}
}
} else {
throw IllegalStateException("Compressor singleton instance already created")
}
return instance!!
}
}
private val log = LogManager.getLogger(GoBackedBlobCompressor::class.java)
override fun canAppendBlock(blockRLPEncoded: ByteArray): Boolean {
return goNativeBlobCompressor.CanWrite(blockRLPEncoded, blockRLPEncoded.size)
}
fun inflightBlobSize(): Int {
return goNativeBlobCompressor.Len()
}
override fun appendBlock(blockRLPEncoded: ByteArray): BlobCompressor.AppendResult {
val compressionSizeBefore = goNativeBlobCompressor.Len()
val appended = goNativeBlobCompressor.Write(blockRLPEncoded, blockRLPEncoded.size)
val compressedSizeAfter = goNativeBlobCompressor.Len()
log.trace(
"block compressed: blockRlpSize={} compressionDataBefore={} compressionDataAfter={} compressionRatio={}",
blockRLPEncoded.size,
compressionSizeBefore,
compressedSizeAfter,
1.0 - ((compressedSizeAfter - compressionSizeBefore).toDouble() / blockRLPEncoded.size)
)
val error = goNativeBlobCompressor.Error()
if (error != null) {
log.error("Failure while writing the following RLP encoded block: {}", blockRLPEncoded.encodeHex())
throw BlobCompressionException(error)
}
return BlobCompressor.AppendResult(appended, compressionSizeBefore, compressedSizeAfter)
}
override fun startNewBatch() {
goNativeBlobCompressor.StartNewBatch()
}
override fun getCompressedData(): ByteArray {
val compressedData = ByteArray(goNativeBlobCompressor.Len())
goNativeBlobCompressor.Bytes(compressedData)
return compressedData
}
override fun reset() {
goNativeBlobCompressor.Reset()
}
}

View File

@@ -0,0 +1,90 @@
package linea.blob
import net.consensys.linea.blob.BlobCompressorVersion
import net.consensys.linea.nativecompressor.CompressorTestData
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.TestInstance
import org.junit.jupiter.api.assertThrows
import kotlin.random.Random
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class GoBackedBlobCompressorTest {
companion object {
private const val DATA_LIMIT = 16 * 1024
private val TEST_DATA = CompressorTestData.blocksRlpEncoded
private val compressor = GoBackedBlobCompressor.getInstance(BlobCompressorVersion.V0_1_0, DATA_LIMIT.toUInt())
}
@BeforeEach
fun before() {
compressor.reset()
}
@Test
fun `test appendBlock with data within limit`() {
val blocks = TEST_DATA
val result = compressor.appendBlock(blocks.first())
assertThat(result.blockAppended).isTrue
assertThat(result.compressedSizeBefore).isZero()
assertThat(result.compressedSizeAfter).isGreaterThan(0)
}
@Test
fun `test invalid rlp block`() {
val block = Random.nextBytes(100)
assertThrows<BlobCompressionException>("rlp: expected input list for types.extblock") {
compressor.appendBlock(block)
}
}
@Test
fun `test compression data limit exceeded`() {
val blocks = TEST_DATA.iterator()
var result = compressor.appendBlock(blocks.next())
while (result.blockAppended && blocks.hasNext()) {
val blockRlp = blocks.next()
val canAppend = compressor.canAppendBlock(blockRlp)
result = compressor.appendBlock(blockRlp)
// assert consistency between canAppendBlock and appendBlock
assertThat(canAppend).isEqualTo(result.blockAppended)
}
assertThat(result.blockAppended).isFalse()
assertThat(result.compressedSizeBefore).isGreaterThan(0)
assertThat(result.compressedSizeAfter).isEqualTo(result.compressedSizeBefore)
}
@Test
fun `test reset`() {
val blocks = TEST_DATA.iterator()
assertThat(compressor.goNativeBlobCompressor.Len()).isZero()
var res = compressor.appendBlock(blocks.next())
assertThat(res.blockAppended).isTrue()
assertThat(res.compressedSizeBefore).isZero()
assertThat(res.compressedSizeAfter).isGreaterThan(0)
assertThat(res.compressedSizeAfter).isEqualTo(compressor.goNativeBlobCompressor.Len())
compressor.reset()
assertThat(compressor.goNativeBlobCompressor.Len()).isZero()
res = compressor.appendBlock(blocks.next())
assertThat(res.blockAppended).isTrue()
assertThat(res.compressedSizeBefore).isZero()
assertThat(res.compressedSizeAfter).isGreaterThan(0)
assertThat(res.compressedSizeAfter).isEqualTo(compressor.goNativeBlobCompressor.Len())
}
@Test
fun `test batches`() {
val blocks = TEST_DATA.iterator()
var res = compressor.appendBlock(blocks.next())
assertThat(res.blockAppended).isTrue()
compressor.startNewBatch()
res = compressor.appendBlock(blocks.next())
assertThat(res.blockAppended).isTrue()
assertThat(compressor.getCompressedData().size).isGreaterThan(0)
}
}

View File

@@ -36,7 +36,7 @@ def libsZipDownloadOutputDir = project.parent.layout.buildDirectory.asFile.get()
task downloadNativeLibs {
doLast {
fetchLibFromZip("https://github.com/Consensys/linea-monorepo/releases/download/blob-libs-v1.1.0-test8/linea-blob-libs-v1.1.0-test8.zip", "blob_decompressor", libsZipDownloadOutputDir)
fetchLibFromZip("https://github.com/Consensys/linea-monorepo/releases/download/blob-libs-v1.1.0-test9/linea-blob-libs-v1.1.0-test9.zip", "blob_decompressor", libsZipDownloadOutputDir)
}
}

View File

@@ -13,7 +13,7 @@ interface BlobDecompressor {
internal class Adapter(
private val delegate: GoNativeBlobDecompressorJnaBinding,
private val maxExpectedCompressionRatio: Int = 10,
private val maxExpectedCompressionRatio: Int = 20,
dictionaries: List<Path>
) : BlobDecompressor {
init {

View File

@@ -1,11 +1,13 @@
plugins {
id 'net.consensys.zkevm.kotlin-common-conventions'
id 'java-test-fixtures'
}
description="Linea domain models"
dependencies {
implementation project(":jvm-libs:generic:extensions:kotlin")
testFixturesApi "org.jetbrains.kotlinx:kotlinx-datetime:${libs.versions.kotlinxDatetime.get()}"
}
jar {

View File

@@ -0,0 +1,25 @@
package linea.domain
import tech.pegasys.teku.infrastructure.async.SafeFuture
interface BinaryEncoder<T> {
fun encode(block: T): ByteArray
fun encode(blocks: List<T>): List<ByteArray> = blocks.map { encode(it) }
}
interface BinaryDecoder<T> {
fun decode(block: ByteArray): T
fun decode(blocks: List<ByteArray>): List<T> = blocks.map { decode(it) }
}
interface BinaryEncoderAsync<T> {
fun encodeAsync(block: T): SafeFuture<ByteArray>
fun encodeAsync(blocks: List<T>): SafeFuture<List<ByteArray>> =
SafeFuture.collectAll(blocks.map { encodeAsync(it) }.stream())
}
interface BinaryDecoderAsync<T> {
fun decodeAsync(block: ByteArray): SafeFuture<T>
fun decodeAsync(blocks: List<ByteArray>): SafeFuture<List<T>> =
SafeFuture.collectAll(blocks.map { decodeAsync(it) }.stream())
}

View File

@@ -0,0 +1,146 @@
package linea.domain
import kotlinx.datetime.Instant
import net.consensys.encodeHex
import net.consensys.linea.BlockNumberAndHash
data class Block(
val number: ULong,
val hash: ByteArray,
val parentHash: ByteArray,
val ommersHash: ByteArray,
val miner: ByteArray,
val stateRoot: ByteArray,
val transactionsRoot: ByteArray,
val receiptsRoot: ByteArray,
val logsBloom: ByteArray,
val difficulty: ULong,
val gasLimit: ULong,
val gasUsed: ULong,
val timestamp: ULong,
val extraData: ByteArray,
val mixHash: ByteArray,
val nonce: ULong,
val baseFeePerGas: ULong? = null, // Optional field for EIP-1559 blocks
val transactions: List<Transaction> = emptyList(), // List of transaction hashes
val ommers: List<ByteArray> = emptyList() // List of uncle block hashes
) {
companion object {
// companion object to allow static extension functions
}
val numberAndHash = BlockNumberAndHash(this.number, this.hash)
val headerSummary = BlockHeaderSummary(this.number, this.hash, Instant.fromEpochSeconds(this.timestamp.toLong()))
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as Block
if (number != other.number) return false
if (!hash.contentEquals(other.hash)) return false
if (!parentHash.contentEquals(other.parentHash)) return false
if (!ommersHash.contentEquals(other.ommersHash)) return false
if (!miner.contentEquals(other.miner)) return false
if (!stateRoot.contentEquals(other.stateRoot)) return false
if (!transactionsRoot.contentEquals(other.transactionsRoot)) return false
if (!receiptsRoot.contentEquals(other.receiptsRoot)) return false
if (!logsBloom.contentEquals(other.logsBloom)) return false
if (difficulty != other.difficulty) return false
if (gasLimit != other.gasLimit) return false
if (gasUsed != other.gasUsed) return false
if (timestamp != other.timestamp) return false
if (!extraData.contentEquals(other.extraData)) return false
if (!mixHash.contentEquals(other.mixHash)) return false
if (nonce != other.nonce) return false
if (baseFeePerGas != other.baseFeePerGas) return false
if (transactions != other.transactions) return false
if (ommers != other.ommers) return false
if (numberAndHash != other.numberAndHash) return false
if (headerSummary != other.headerSummary) return false
return true
}
override fun hashCode(): Int {
var result = number.hashCode()
result = 31 * result + hash.contentHashCode()
result = 31 * result + parentHash.contentHashCode()
result = 31 * result + ommersHash.contentHashCode()
result = 31 * result + miner.contentHashCode()
result = 31 * result + stateRoot.contentHashCode()
result = 31 * result + transactionsRoot.contentHashCode()
result = 31 * result + receiptsRoot.contentHashCode()
result = 31 * result + logsBloom.contentHashCode()
result = 31 * result + difficulty.hashCode()
result = 31 * result + gasLimit.hashCode()
result = 31 * result + gasUsed.hashCode()
result = 31 * result + timestamp.hashCode()
result = 31 * result + extraData.contentHashCode()
result = 31 * result + mixHash.contentHashCode()
result = 31 * result + nonce.hashCode()
result = 31 * result + (baseFeePerGas?.hashCode() ?: 0)
result = 31 * result + transactions.hashCode()
result = 31 * result + ommers.hashCode()
result = 31 * result + numberAndHash.hashCode()
result = 31 * result + headerSummary.hashCode()
return result
}
override fun toString(): String {
return "Block(" +
"number=$number, " +
"hash=${hash.encodeHex()}, " +
"parentHash=${parentHash.encodeHex()}, " +
"ommersHash=${ommersHash.encodeHex()}, " +
"miner=${miner.encodeHex()}, " +
"stateRoot=${stateRoot.encodeHex()}, " +
"transactionsRoot=${transactionsRoot.encodeHex()}, " +
"receiptsRoot=${receiptsRoot.encodeHex()}, " +
"logsBloom=${logsBloom.encodeHex()}, " +
"difficulty=$difficulty, " +
"gasLimit=$gasLimit, " +
"gasUsed=$gasUsed, " +
"timestamp=$timestamp, " +
"extraData=${extraData.encodeHex()}, " +
"mixHash=${mixHash.encodeHex()}, " +
"nonce=$nonce, " +
"baseFeePerGas=$baseFeePerGas, " +
"transactions=$transactions, " +
"ommers=$ommers" + ")"
}
}
data class BlockHeaderSummary(
val number: ULong,
val hash: ByteArray,
val timestamp: Instant
) {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as BlockHeaderSummary
if (number != other.number) return false
if (!hash.contentEquals(other.hash)) return false
if (timestamp != other.timestamp) return false
return true
}
override fun hashCode(): Int {
var result = number.hashCode()
result = 31 * result + hash.contentHashCode()
result = 31 * result + timestamp.hashCode()
return result
}
override fun toString(): String {
return "BlockHeaderSummary(" +
"number=$number, " +
"hash=${hash.contentToString()}, " +
"timestamp=$timestamp)"
}
}

View File

@@ -0,0 +1,173 @@
package linea.domain
import net.consensys.encodeHex
import java.math.BigInteger
import java.util.EnumSet
enum class TransactionType(private val typeValue: Int) {
FRONTIER(248),
ACCESS_LIST(1),
EIP1559(2),
BLOB(3), // Not supported by Linea atm, but here for completeness
DELEGATE_CODE(4); // Not supported by Linea atm, but here for completeness
val serializedType: Byte
get() = typeValue.toByte()
val ethSerializedType: Byte
get() = if (this == FRONTIER) 0 else serializedType
fun compareTo(b: Byte?): Int {
return serializedType.compareTo(b!!)
}
fun supports1559FeeMarket(): Boolean {
return !TransactionType.LEGACY_FEE_MARKET_TRANSACTION_TYPES.contains(this)
}
companion object {
private val ACCESS_LIST_SUPPORTED_TRANSACTION_TYPES: Set<TransactionType> =
EnumSet.of(ACCESS_LIST, EIP1559, BLOB, DELEGATE_CODE)
private val LEGACY_FEE_MARKET_TRANSACTION_TYPES: Set<TransactionType> = EnumSet.of(FRONTIER, ACCESS_LIST)
fun fromSerializedValue(serializedTypeValue: Int): TransactionType {
return entries
.firstOrNull { type: TransactionType -> type.typeValue == serializedTypeValue }
?: throw IllegalArgumentException(
String.format(
"Unsupported transaction type %x",
serializedTypeValue
)
)
}
fun fromEthApiSerializedValue(serializedTypeValue: Int): TransactionType {
if (serializedTypeValue == 0) {
return FRONTIER
}
return fromSerializedValue(serializedTypeValue)
}
}
}
data class Transaction(
val type: TransactionType,
val nonce: ULong,
val gasLimit: ULong,
val to: ByteArray?, // Nullable for contract creation transactions
val value: BigInteger,
val input: ByteArray,
val r: BigInteger,
val s: BigInteger,
val v: ULong,
val yParity: ULong?,
val chainId: ULong? = null, // Optional field for EIP-155 transactions
val gasPrice: ULong?, // null for EIP-1559 transactions
val maxFeePerGas: ULong? = null, // null for EIP-1559 transactions
val maxPriorityFeePerGas: ULong? = null, // null for non EIP-1559 transactions
val accessList: List<AccessListEntry>? // null non for EIP-2930 transactions
) {
companion object {
// companion object to allow static extension functions
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as Transaction
if (nonce != other.nonce) return false
if (gasPrice != other.gasPrice) return false
if (gasLimit != other.gasLimit) return false
if (to != null) {
if (other.to == null) return false
if (!to.contentEquals(other.to)) return false
} else if (other.to != null) return false
if (value != other.value) return false
if (!input.contentEquals(other.input)) return false
if (r != other.r) return false
if (s != other.s) return false
if (v != other.v) return false
if (yParity != other.yParity) return false
if (type != other.type) return false
if (chainId != other.chainId) return false
if (maxPriorityFeePerGas != other.maxPriorityFeePerGas) return false
if (maxFeePerGas != other.maxFeePerGas) return false
if (accessList != other.accessList) return false
return true
}
override fun hashCode(): Int {
var result = nonce.hashCode()
result = 31 * result + gasPrice.hashCode()
result = 31 * result + gasLimit.hashCode()
result = 31 * result + (to?.contentHashCode() ?: 0)
result = 31 * result + value.hashCode()
result = 31 * result + input.contentHashCode()
result = 31 * result + r.hashCode()
result = 31 * result + s.hashCode()
result = 31 * result + v.hashCode()
result = 31 * result + yParity.hashCode()
result = 31 * result + type.hashCode()
result = 31 * result + (chainId?.hashCode() ?: 0)
result = 31 * result + (maxPriorityFeePerGas?.hashCode() ?: 0)
result = 31 * result + (maxFeePerGas?.hashCode() ?: 0)
result = 31 * result + accessList.hashCode()
return result
}
override fun toString(): String {
return "Transaction(" +
"type=$type, " +
"nonce=$nonce, " +
"gasLimit=$gasLimit, " +
"to=${to?.encodeHex()}, " +
"value=$value, " +
"input=${input.encodeHex()}, " +
"r=$r, " +
"s=$s, " +
"v=$v, " +
"yParity=$yParity, " +
"chainId=$chainId, " +
"gasPrice=$gasPrice, " +
"maxFeePerGas=$maxFeePerGas, " +
"maxPriorityFeePerGas=$maxPriorityFeePerGas, " +
"accessList=$accessList)"
}
}
data class AccessListEntry(
val address: ByteArray,
val storageKeys: List<ByteArray>
) {
override fun toString(): String {
return "AccessListEntry(" +
"address=${address.encodeHex()}, " +
"storageKeys=[${storageKeys.joinToString(",") { it.encodeHex() }}]" +
")"
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as AccessListEntry
if (!address.contentEquals(other.address)) return false
if (storageKeys.size != other.storageKeys.size) return false
storageKeys.zip(other.storageKeys).forEach { (a, b) ->
if (!a.contentEquals(b)) return false
}
return true
}
override fun hashCode(): Int {
var result = address.contentHashCode()
result = 31 * result + storageKeys.hashCode()
return result
}
}

View File

@@ -7,7 +7,7 @@ sealed interface BlockParameter {
companion object {
fun fromNumber(blockNumber: Number): BlockNumber {
require(blockNumber.toLong() > 0) { "block number must be greater than 0, value=$blockNumber" }
require(blockNumber.toLong() >= 0) { "block number must be greater or equal than 0, value=$blockNumber" }
return BlockNumber(blockNumber.toLong().toULong())
}

View File

@@ -0,0 +1,107 @@
package linea.domain
import kotlinx.datetime.Clock
import kotlinx.datetime.Instant
import net.consensys.ByteArrayExt
val zeroHash = ByteArray(32) { 0 }
fun createBlock(
number: ULong = 0UL,
hash: ByteArray = ByteArrayExt.random32(),
gasLimit: ULong = 60_000_000UL,
gasUsed: ULong = 30_000_000UL,
difficulty: ULong = 2UL,
parentHash: ByteArray = ByteArrayExt.random32(),
stateRoot: ByteArray = ByteArrayExt.random32(),
receiptsRoot: ByteArray = ByteArrayExt.random32(),
logsBloom: ByteArray = ByteArrayExt.random32(),
ommersHash: ByteArray = ByteArrayExt.random32(),
timestamp: Instant = Clock.System.now(),
extraData: ByteArray = ByteArrayExt.random32(),
baseFeePerGas: ULong = 7UL,
transactionsRoot: ByteArray = ByteArrayExt.random32(),
transactions: List<Transaction> = emptyList()
): Block {
return Block(
number = number,
hash = hash,
parentHash = parentHash,
ommersHash = ommersHash,
miner = zeroHash,
stateRoot = stateRoot,
transactionsRoot = transactionsRoot,
receiptsRoot = receiptsRoot,
logsBloom = logsBloom,
difficulty = difficulty,
gasLimit = gasLimit,
gasUsed = gasUsed,
timestamp = timestamp.epochSeconds.toULong(),
extraData = extraData,
mixHash = zeroHash,
nonce = 0UL,
baseFeePerGas = baseFeePerGas,
transactions = transactions,
ommers = emptyList()
)
}
/**
* This is very similar to Block class,
* but creating DTO to avoid coupling with domain model,
* some fields are not present in domain model, e.g uncles
*
* This is meant to help creating fake JSON-RPC server
*/
class EthGetBlockResponseDTO(
val number: ULong,
val hash: ByteArray,
val parentHash: ByteArray,
val miner: ByteArray,
val stateRoot: ByteArray,
val transactionsRoot: ByteArray,
val receiptsRoot: ByteArray,
val logsBloom: ByteArray,
val difficulty: ULong,
val gasLimit: ULong,
val gasUsed: ULong,
val timestamp: ULong,
val extraData: ByteArray,
val mixHash: ByteArray,
val nonce: ULong,
val baseFeePerGas: ULong?,
val sha3Uncles: ByteArray, // ommersHash
val size: ULong,
val totalDifficulty: ULong,
val transactions: List<ByteArray>,
val uncles: List<ByteArray> = emptyList()
)
fun Block?.toEthGetBlockResponse(
size: ULong = 10UL * 1024UL,
totalDifficulty: ULong = this?.difficulty ?: 0UL
): EthGetBlockResponseDTO? {
if (this == null) return null
return EthGetBlockResponseDTO(
number = this.number,
hash = this.hash,
parentHash = this.parentHash,
miner = this.miner,
stateRoot = this.stateRoot,
transactionsRoot = this.transactionsRoot,
receiptsRoot = this.receiptsRoot,
logsBloom = this.logsBloom,
difficulty = this.difficulty,
gasLimit = this.gasLimit,
gasUsed = this.gasUsed,
timestamp = this.timestamp,
extraData = this.extraData,
mixHash = this.mixHash,
nonce = this.nonce,
baseFeePerGas = this.baseFeePerGas,
sha3Uncles = this.ommersHash,
size = size,
totalDifficulty = totalDifficulty,
transactions = emptyList<ByteArray>()
)
}

View File

@@ -1,19 +0,0 @@
plugins {
id 'net.consensys.zkevm.kotlin-library-conventions'
id 'java-library'
}
dependencies {
api("tech.pegasys.teku.internal:executionclient:${libs.versions.teku.get()}") {
exclude group: 'org.hyperledger.besu'
exclude group: 'org.web3j'
exclude group: 'com.github.jnr'
exclude group: 'com.squareup.okhttp3'
exclude group: 'io.reactivex.rxjava2'
exclude group: 'org.java-websocket'
exclude group: 'com.fasterxml.jackson.core:jackson-databind'
exclude group: 'org.slf4j'
exclude group: 'tech.pegasys.teku.internal'
exclude group: 'io.jsonwebtoken'
}
}

View File

@@ -1,16 +0,0 @@
plugins {
id 'net.consensys.zkevm.kotlin-library-conventions'
}
description="Linea test utilities for interaction with Engine API by Teku client"
dependencies {
api project(":jvm-libs:linea:teku-execution-client")
api "tech.pegasys.teku.internal:unsigned:${libs.versions.teku.get()}"
api "tech.pegasys.teku.internal:bytes:${libs.versions.teku.get()}"
implementation "tech.pegasys.teku.internal:spec:${libs.versions.teku.get()}"
implementation "tech.pegasys.teku.internal:spec:${libs.versions.teku.get()}:test-fixtures"
implementation "io.tmio:tuweni-units:${libs.versions.tuweni.get()}"
implementation project(':jvm-libs:generic:extensions:kotlin')
}

View File

@@ -1,131 +0,0 @@
package tech.pegasys.teku.ethereum.executionclient.schema
import kotlinx.datetime.Clock
import kotlinx.datetime.Instant
import net.consensys.ByteArrayExt
import net.consensys.toBigInteger
import org.apache.tuweni.bytes.Bytes
import org.apache.tuweni.bytes.Bytes32
import org.apache.tuweni.units.bigints.UInt256
import tech.pegasys.teku.infrastructure.bytes.Bytes20
import tech.pegasys.teku.infrastructure.unsigned.UInt64
import tech.pegasys.teku.spec.TestSpecFactory
import tech.pegasys.teku.spec.util.DataStructureUtil
import java.math.BigInteger
fun executionPayloadV1(
blockNumber: Long = 0,
parentHash: ByteArray = ByteArrayExt.random32(),
feeRecipient: ByteArray = ByteArrayExt.random(20),
stateRoot: ByteArray = ByteArrayExt.random32(),
receiptsRoot: ByteArray = ByteArrayExt.random32(),
logsBloom: ByteArray = ByteArrayExt.random32(),
prevRandao: ByteArray = ByteArrayExt.random32(),
gasLimit: ULong = 0UL,
gasUsed: ULong = 0UL,
timestamp: Instant = Clock.System.now(),
extraData: ByteArray = ByteArrayExt.random32(),
baseFeePerGas: BigInteger = BigInteger.valueOf(256),
blockHash: ByteArray = ByteArrayExt.random32(),
transactions: List<ByteArray> = emptyList()
): ExecutionPayloadV1 {
return ExecutionPayloadV1(
Bytes32.wrap(parentHash),
Bytes20(Bytes.wrap(feeRecipient)),
Bytes32.wrap(stateRoot),
Bytes32.wrap(receiptsRoot),
Bytes.wrap(logsBloom),
Bytes32.wrap(prevRandao),
UInt64.valueOf(blockNumber),
UInt64.valueOf(gasLimit.toBigInteger()),
UInt64.valueOf(gasUsed.toBigInteger()),
UInt64.valueOf(timestamp.epochSeconds),
Bytes.wrap(extraData),
UInt256.valueOf(baseFeePerGas),
Bytes32.wrap(blockHash),
transactions.map { Bytes.wrap(it) }
)
}
fun executionPayloadV1(
blockNumber: Long = 0,
parentHash: Bytes32 = Bytes32.random(),
feeRecipient: Bytes20 = Bytes20(Bytes.random(20)),
stateRoot: Bytes32 = Bytes32.random(),
receiptsRoot: Bytes32 = Bytes32.random(),
logsBloom: Bytes = Bytes32.random(),
prevRandao: Bytes32 = Bytes32.random(),
gasLimit: UInt64 = UInt64.valueOf(0),
gasUsed: UInt64 = UInt64.valueOf(0),
timestamp: UInt64 = UInt64.valueOf(0),
extraData: Bytes = Bytes32.random(),
baseFeePerGas: UInt256 = UInt256.valueOf(256),
blockHash: Bytes32 = Bytes32.random(),
transactions: List<Bytes> = emptyList()
): ExecutionPayloadV1 {
return ExecutionPayloadV1(
parentHash,
feeRecipient,
stateRoot,
receiptsRoot,
logsBloom,
prevRandao,
UInt64.valueOf(blockNumber),
gasLimit,
gasUsed,
timestamp,
extraData,
baseFeePerGas,
blockHash,
transactions
)
}
fun randomExecutionPayload(
transactionsRlp: List<Bytes> = emptyList(),
blockNumber: Long? = null
): ExecutionPayloadV1 {
val executionPayload = dataStructureUtil.randomExecutionPayload()
return ExecutionPayloadV1(
/* parentHash = */ executionPayload.parentHash,
/* feeRecipient = */
executionPayload.feeRecipient,
/* stateRoot = */
executionPayload.stateRoot,
/* receiptsRoot = */
executionPayload.receiptsRoot,
/* logsBloom = */
executionPayload.logsBloom,
/* prevRandao = */
executionPayload.prevRandao,
/* blockNumber = */
blockNumber?.let(UInt64::valueOf) ?: executionPayload.blockNumber.cropToPositiveSignedLong(),
/* gasLimit = */
executionPayload.gasLimit.cropToPositiveSignedLong(),
/* gasUsed = */
executionPayload.gasUsed.cropToPositiveSignedLong(),
/* timestamp = */
executionPayload.timestamp.cropToPositiveSignedLong(),
/* extraData = */
executionPayload.extraData,
/* baseFeePerGas = */
executionPayload.baseFeePerGas,
/* blockHash = */
executionPayload.blockHash,
/* transactions = */
transactionsRlp
)
}
val dataStructureUtil: DataStructureUtil = DataStructureUtil(TestSpecFactory.createMinimalBellatrix())
// Teku UInt64 has a bug allow negative number to be created
// random test payload creates such cases we need to fix it
private fun UInt64.cropToPositiveSignedLong(): UInt64 {
val longValue = this.longValue()
return if (longValue < 0) {
return UInt64.valueOf(-longValue)
} else {
this
}
}

View File

@@ -10,19 +10,17 @@ dependencies {
api "org.web3j:core:${libs.versions.web3j.get()}"
api project(':jvm-libs:linea:core:domain-models')
api project(':jvm-libs:generic:logging')
// For domain mappers
api project(':jvm-libs:linea:besu-libs')
implementation project(":jvm-libs:generic:extensions:kotlin")
implementation project(":jvm-libs:generic:extensions:futures")
implementation "tech.pegasys.teku.internal:bytes:${libs.versions.teku.get()}"
implementation "tech.pegasys.teku.internal:jackson:${libs.versions.teku.get()}"
// Returned by domain mapper
api project(":jvm-libs:linea:teku-execution-client")
implementation "tech.pegasys.teku.internal:unsigned:${libs.versions.teku.get()}"
implementation "org.hyperledger.besu:besu-datatypes:${libs.versions.besu.get()}"
testImplementation "org.apache.logging.log4j:log4j-slf4j2-impl:${libs.versions.log4j.get()}"
testImplementation "com.fasterxml.jackson.core:jackson-annotations:${libs.versions.jackson.get()}"
testImplementation "com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson.get()}"
testImplementation project(":jvm-libs:linea:besu-rlp-and-mappers")
}
jar {

View File

@@ -0,0 +1,92 @@
package linea.web3j
import linea.domain.AccessListEntry
import linea.domain.Block
import linea.domain.Transaction
import linea.domain.TransactionType
import net.consensys.decodeHex
import net.consensys.toBigIntegerFromHex
import net.consensys.toIntFromHex
import net.consensys.toULong
import net.consensys.toULongFromHex
import org.web3j.protocol.core.methods.response.EthBlock
fun EthBlock.Block.toDomain(): Block = mapToDomain(this)
fun mapToDomain(web3jBlock: EthBlock.Block): Block {
val block = Block(
number = web3jBlock.number.toULong(),
hash = web3jBlock.hash.decodeHex(),
parentHash = web3jBlock.parentHash.decodeHex(),
ommersHash = web3jBlock.sha3Uncles.decodeHex(),
miner = web3jBlock.miner.decodeHex(),
nonce = web3jBlock.nonce.toULong(),
stateRoot = web3jBlock.stateRoot.decodeHex(),
transactionsRoot = web3jBlock.transactionsRoot.decodeHex(),
receiptsRoot = web3jBlock.receiptsRoot.decodeHex(),
logsBloom = web3jBlock.logsBloom.decodeHex(),
difficulty = web3jBlock.difficulty.toULong(),
gasLimit = web3jBlock.gasLimit.toULong(),
gasUsed = web3jBlock.gasUsed.toULong(),
timestamp = web3jBlock.timestamp.toULong(),
extraData = web3jBlock.extraData.decodeHex(),
mixHash = web3jBlock.mixHash.decodeHex(),
baseFeePerGas = web3jBlock.baseFeePerGas?.toULong(), // Optional field for EIP-1559 blocks
ommers = web3jBlock.uncles.map { it.decodeHex() }, // List of uncle block hashes
transactions = run {
if (web3jBlock.transactions.isNotEmpty() && web3jBlock.transactions[0] !is EthBlock.TransactionObject) {
throw IllegalArgumentException(
"Expected to be have full EthBlock.TransactionObject." +
"Got just transaction hashes."
)
}
web3jBlock.transactions.map { (it as EthBlock.TransactionObject).toDomain() }
}
)
return block
}
fun EthBlock.TransactionObject.toDomain(): Transaction {
val txType = mapType(this.type)
var gasPrice: ULong? = null
var maxFeePerGas: ULong? = null
var maxPriorityFeePerGas: ULong? = null
if (txType.supports1559FeeMarket()) {
maxFeePerGas = this.maxFeePerGas?.toULong()
maxPriorityFeePerGas = this.maxPriorityFeePerGas?.toULong()
} else {
gasPrice = this.gasPrice.toULong()
}
val accessList = this.accessList?.map { accessListEntry ->
AccessListEntry(
accessListEntry.address.decodeHex(),
accessListEntry.storageKeys.map { it.decodeHex() }
)
}
val domainTx = Transaction(
nonce = this.nonce.toULong(),
gasLimit = this.gas.toULong(),
to = this.to?.decodeHex(),
value = this.value,
input = this.input.decodeHex(),
r = this.r.toBigIntegerFromHex(),
s = this.s.toBigIntegerFromHex(),
v = this.v.toULong(),
yParity = this.getyParity()?.toULongFromHex(),
type = mapType(this.type), // Optional field for EIP-2718 typed transactions
chainId = this.chainId?.toULong(), // Optional field for EIP-155 transactions
gasPrice = gasPrice, // Optional field for EIP-1559 transactions
maxFeePerGas = maxFeePerGas, // Optional field for EIP-1559 transactions
maxPriorityFeePerGas = maxPriorityFeePerGas, // Optional field for EIP-1559 transactions,
accessList = accessList
)
return domainTx
}
fun mapType(type: String?): TransactionType {
return type
?.let { TransactionType.fromEthApiSerializedValue(it.toIntFromHex()) }
?: TransactionType.FRONTIER
}

View File

@@ -0,0 +1,33 @@
package linea.web3j
import net.consensys.linea.web3j.okHttpClientBuilder
import org.apache.logging.log4j.Level
import org.apache.logging.log4j.Logger
import org.web3j.protocol.Web3j
import org.web3j.protocol.http.HttpService
import org.web3j.utils.Async
import java.util.concurrent.ScheduledExecutorService
import kotlin.time.Duration
import kotlin.time.Duration.Companion.milliseconds
fun createWeb3jHttpClient(
rpcUrl: String,
log: Logger = org.apache.logging.log4j.LogManager.getLogger(Web3j::class.java),
pollingInterval: Duration = 500.milliseconds,
executorService: ScheduledExecutorService = Async.defaultExecutorService(),
requestResponseLogLevel: Level = Level.TRACE,
failuresLogLevel: Level = Level.DEBUG
): Web3j {
return Web3j.build(
HttpService(
rpcUrl,
okHttpClientBuilder(
logger = log,
requestResponseLogLevel = requestResponseLogLevel,
failuresLogLevel = failuresLogLevel
).build()
),
pollingInterval.inWholeMilliseconds,
executorService
)
}

View File

@@ -1,138 +0,0 @@
package net.consensys.linea.web3j
import net.consensys.linea.bigIntFromPrefixedHex
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
import org.apache.tuweni.bytes.Bytes
import org.apache.tuweni.bytes.Bytes32
import org.apache.tuweni.units.bigints.UInt256
import org.bouncycastle.math.ec.custom.sec.SecP256K1Curve
import org.hyperledger.besu.crypto.SECPSignature
import org.hyperledger.besu.datatypes.AccessListEntry
import org.hyperledger.besu.datatypes.Address
import org.hyperledger.besu.datatypes.Wei
import org.hyperledger.besu.ethereum.core.Transaction
import org.hyperledger.besu.ethereum.core.encoding.EncodingContext
import org.hyperledger.besu.ethereum.core.encoding.TransactionEncoder
import org.web3j.protocol.core.methods.response.EthBlock
import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1
import tech.pegasys.teku.infrastructure.bytes.Bytes20
import tech.pegasys.teku.infrastructure.unsigned.UInt64
import java.math.BigInteger
private val log: Logger = LogManager.getLogger("DomainObjectMappers")
fun EthBlock.Block.toExecutionPayloadV1(): ExecutionPayloadV1 {
/**
* @JsonProperty("parentHash") Bytes32 parentHash,
* @JsonProperty("feeRecipient") Bytes20 feeRecipient,
* @JsonProperty("stateRoot") Bytes32 stateRoot,
* @JsonProperty("receiptsRoot") Bytes32 receiptsRoot,
* @JsonProperty("logsBloom") Bytes logsBloom,
* @JsonProperty("prevRandao") Bytes32 prevRandao,
* @JsonProperty("blockNumber") UInt64 blockNumber,
* @JsonProperty("gasLimit") UInt64 gasLimit,
* @JsonProperty("gasUsed") UInt64 gasUsed,
* @JsonProperty("timestamp") UInt64 timestamp,
* @JsonProperty("extraData") Bytes extraData,
* @JsonProperty("baseFeePerGas") UInt256 baseFeePerGas,
* @JsonProperty("blockHash") Bytes32 blockHash,
* @JsonProperty("transactions") List<Bytes> transactions)
*/
return ExecutionPayloadV1(
Bytes32.fromHexString(this.parentHash),
Bytes20.fromHexString(this.miner),
Bytes32.fromHexString(this.stateRoot),
Bytes32.fromHexString(this.receiptsRoot),
Bytes.fromHexString(this.logsBloom),
Bytes32.fromHexString(this.mixHash),
UInt64.valueOf(this.number),
UInt64.valueOf(this.gasLimit),
UInt64.valueOf(this.gasUsed),
UInt64.valueOf(this.timestamp),
Bytes.fromHexString(this.extraData),
UInt256.valueOf(this.baseFeePerGas),
Bytes32.fromHexString(this.hash),
this.transactions.map {
val transaction = it.get() as EthBlock.TransactionObject
kotlin.runCatching {
transaction.toBytes()
}.onFailure { th ->
log.error(
"Failed to encode transaction! blockNumber={} tx={} errorMessage={}",
this.number,
transaction.hash.toString(),
th.message,
th
)
}
.getOrThrow()
}
)
}
fun recIdFromV(v: BigInteger): Pair<Byte, BigInteger?> {
val recId: Byte
var chainId: BigInteger? = null
if (v == Transaction.REPLAY_UNPROTECTED_V_BASE || v == Transaction.REPLAY_UNPROTECTED_V_BASE_PLUS_1) {
recId = v.subtract(Transaction.REPLAY_UNPROTECTED_V_BASE).byteValueExact()
} else if (v > Transaction.REPLAY_PROTECTED_V_MIN) {
chainId = v.subtract(Transaction.REPLAY_PROTECTED_V_BASE).divide(Transaction.TWO)
recId = v.subtract(Transaction.TWO.multiply(chainId).add(Transaction.REPLAY_PROTECTED_V_BASE)).byteValueExact()
} else {
throw RuntimeException("An unsupported encoded `v` value of $v was found")
}
return Pair(recId, chainId)
}
// TODO: Test
fun EthBlock.TransactionObject.toBytes(): Bytes {
val isFrontier = this.type == "0x0"
val (recId, chainId) = if (isFrontier) {
recIdFromV(this.v.toBigInteger())
} else {
Pair(this.v.toByte(), BigInteger.valueOf(this.chainId))
}
val signature = SECPSignature.create(
this.r.bigIntFromPrefixedHex(),
this.s.bigIntFromPrefixedHex(),
recId,
SecP256K1Curve().order
)
val transaction = Transaction.builder()
.nonce(this.nonce.toLong())
.also { builder ->
if (isFrontier || this.type == "0x1") {
builder.gasPrice(Wei.of(this.gasPrice))
} else {
builder.maxPriorityFeePerGas(Wei.of(this.maxPriorityFeePerGas))
builder.maxFeePerGas(Wei.of(this.maxFeePerGas))
}
}
.gasLimit(this.gas.toLong())
.to(Address.fromHexString(this.to))
.value(Wei.of(this.value))
.signature(signature)
.payload(Bytes.fromHexString(this.input))
.also { builder ->
this.accessList?.also { accessList ->
builder.accessList(
accessList.map { entry ->
AccessListEntry.createAccessListEntry(
Address.fromHexString(entry.address),
entry.storageKeys
)
}
)
}
}
.sender(Address.fromHexString(this.from))
.apply {
if (chainId != null) {
chainId(chainId)
}
}
.build()
return TransactionEncoder.encodeOpaqueBytes(transaction, EncodingContext.BLOCK_BODY)
}

View File

@@ -1,9 +1,13 @@
package net.consensys.linea.web3j
import build.linea.web3j.domain.toWeb3j
import linea.domain.Block
import linea.web3j.toDomain
import net.consensys.linea.BlockParameter
import net.consensys.linea.async.toSafeFuture
import org.web3j.protocol.Web3j
import org.web3j.protocol.core.DefaultBlockParameter
import org.web3j.protocol.core.Response
import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1
import tech.pegasys.teku.infrastructure.async.SafeFuture
import java.math.BigInteger
@@ -14,14 +18,14 @@ import java.math.BigInteger
interface ExtendedWeb3J {
val web3jClient: Web3j
fun ethBlockNumber(): SafeFuture<BigInteger>
fun ethGetExecutionPayloadByNumber(blockNumber: Long): SafeFuture<ExecutionPayloadV1>
fun ethGetBlock(blockParameter: BlockParameter): SafeFuture<Block?>
fun ethGetBlockTimestampByNumber(blockNumber: Long): SafeFuture<BigInteger>
}
class ExtendedWeb3JImpl(override val web3jClient: Web3j) : ExtendedWeb3J {
private fun buildException(error: Response.Error): Exception =
Exception("${error.code}: ${error.message}")
RuntimeException("${error.code}: ${error.message}")
override fun ethBlockNumber(): SafeFuture<BigInteger> {
return SafeFuture.of(web3jClient.ethBlockNumber().sendAsync()).thenCompose { response ->
@@ -33,22 +37,21 @@ class ExtendedWeb3JImpl(override val web3jClient: Web3j) : ExtendedWeb3J {
}
}
override fun ethGetExecutionPayloadByNumber(blockNumber: Long): SafeFuture<ExecutionPayloadV1> {
return SafeFuture.of(
web3jClient
.ethGetBlockByNumber(
DefaultBlockParameter.valueOf(BigInteger.valueOf(blockNumber)),
true
)
.sendAsync()
)
override fun ethGetBlock(blockParameter: BlockParameter): SafeFuture<Block?> {
return web3jClient
.ethGetBlockByNumber(
blockParameter.toWeb3j(),
true
)
.sendAsync()
.toSafeFuture()
.thenCompose { response ->
if (response.hasError()) {
SafeFuture.failedFuture(buildException(response.error))
} else {
response.block?.let {
SafeFuture.completedFuture(response.block.toExecutionPayloadV1())
} ?: SafeFuture.failedFuture(Exception("Block $blockNumber not found!"))
SafeFuture.completedFuture(response.block.toDomain())
} ?: SafeFuture.failedFuture(RuntimeException("Block $blockParameter not found!"))
}
}
}

View File

@@ -0,0 +1,411 @@
package linea.web3j
import linea.domain.AccessListEntry
import linea.domain.Transaction
import linea.domain.TransactionType
import linea.domain.toBesu
import net.consensys.decodeHex
import net.consensys.toBigInteger
import net.consensys.toBigIntegerFromHex
import org.apache.tuweni.bytes.Bytes
import org.apache.tuweni.bytes.Bytes32
import org.assertj.core.api.Assertions.assertThat
import org.assertj.core.api.Assertions.fail
import org.hyperledger.besu.datatypes.Address
import org.hyperledger.besu.datatypes.Wei
import org.junit.jupiter.api.Test
import org.web3j.protocol.ObjectMapperFactory
import org.web3j.protocol.core.methods.response.EthBlock
import kotlin.jvm.optionals.getOrElse
import kotlin.jvm.optionals.getOrNull
class EthGetBlockToLineaBlockMapperTest {
// using raw JSON from eth_getBlockByNumber responses because realistically represens our use case
// Also it's very easy create new test cases
private fun serialize(json: String): EthBlock.TransactionObject {
return ObjectMapperFactory.getObjectMapper().readValue(json, EthBlock.TransactionObject::class.java)
}
@Test
fun `should map frontier transactions`() {
val txWeb3j = serialize(
"""
{
"blockHash": "0x004257e560a5f82595dddb73f752b904efef4b73cb3ece1469f5e5091e3c9665",
"blockNumber": "0xe1d30",
"chainId": "0xe705",
"from": "0x228466f2c715cbec05deabfac040ce3619d7cf0b",
"gas": "0x5208",
"gasPrice": "0xee2d984",
"hash": "0x5d3b5e1ae3e4ea5612e6907cb09c4e0e5482171b4c2af794e17b77314547bb79",
"input": "0x",
"nonce": "0x97411",
"r": "0xdf28597129341d5d345c9043c7d0b0a22be82cac13988cfc1d8cbdaf3ab3f35b",
"s": "0x3189b2ff80d8f728d6fb7503b46734ee77a60a42db01d0b09db10bdc9d5caa44",
"to": "0x228466f2c715cbec05deabfac040ce3619d7cf0b",
"transactionIndex": "0x0",
"type": "0x0",
"v": "0x1ce2e",
"value": "0x186a0"
}
""".trimIndent()
)
val domainTx = txWeb3j.toDomain()
assertThat(domainTx).isEqualTo(
Transaction(
nonce = 0x97411UL,
gasPrice = 0xee2d984UL,
gasLimit = 0x5208UL,
to = "0x228466f2c715cbec05deabfac040ce3619d7cf0b".decodeHex(),
value = 0x186a0UL.toBigInteger(),
input = "0x".decodeHex(),
r = "0xdf28597129341d5d345c9043c7d0b0a22be82cac13988cfc1d8cbdaf3ab3f35b".toBigIntegerFromHex(),
s = "0x3189b2ff80d8f728d6fb7503b46734ee77a60a42db01d0b09db10bdc9d5caa44".toBigIntegerFromHex(),
v = 118318UL,
yParity = null,
type = TransactionType.FRONTIER,
chainId = 0xe705UL,
maxFeePerGas = null,
maxPriorityFeePerGas = null,
accessList = null
)
)
domainTx.toBesu().also { besuTx ->
assertThat(besuTx.type).isEqualTo(org.hyperledger.besu.datatypes.TransactionType.FRONTIER)
assertThat(besuTx.nonce).isEqualTo(0x97411L)
assertThat(besuTx.gasPrice.getOrNull()).isEqualTo(Wei.of(0xee2d984L))
assertThat(besuTx.gasLimit).isEqualTo(0x5208L)
assertThat(besuTx.to.getOrNull()).isEqualTo(Address.fromHexString("0x228466f2c715cbec05deabfac040ce3619d7cf0b"))
assertThat(besuTx.value).isEqualTo(Wei.of(0x186a0L))
assertThat(besuTx.payload).isEqualTo(Bytes.EMPTY)
assertThat(besuTx.signature.r).isEqualTo(
"0xdf28597129341d5d345c9043c7d0b0a22be82cac13988cfc1d8cbdaf3ab3f35b".toBigIntegerFromHex()
)
assertThat(besuTx.signature.s).isEqualTo(
"0x3189b2ff80d8f728d6fb7503b46734ee77a60a42db01d0b09db10bdc9d5caa44".toBigIntegerFromHex()
)
assertThat(besuTx.signature.recId).isEqualTo(1)
assertThat(besuTx.chainId.getOrNull()).isEqualTo(0xe705L)
assertThat(besuTx.maxFeePerGas).isEmpty()
assertThat(besuTx.maxPriorityFeePerGas).isEmpty()
}
}
@Test
fun `should map transaction with AccessList`() {
val txWeb3j = serialize(
"""
{
"accessList": [
{
"address": "0x8d97689c9818892b700e27f316cc3e41e17fbeb9",
"storageKeys": [
"0x0000000000000000000000000000000000000000000000000000000000000000",
"0x0000000000000000000000000000000000000000000000000000000000000001"
]
}
],
"blockHash": "0x7480ae911853c1fba10145401a21ddca3943b5894d74cbbf7a6beec526d1f9c2",
"blockNumber": "0xa",
"chainId": "0x539",
"from": "0xce3b7d471fd1fdd10d788ae64e48a9c2f2361179",
"gas": "0x30d40",
"gasPrice": "0x1017df87",
"hash": "0x8ef620582ed8ba98c8496a42b27a30ff7b1de901b1ff7e65b22ea59a2d0668ce",
"input": "0x",
"nonce": "0x0",
"to": "0x8d97689c9818892b700e27f316cc3e41e17fbeb9",
"transactionIndex": "0x2",
"type": "0x1",
"value": "0x2386f26fc10000",
"yParity": "0x1",
"v": "0x1",
"r": "0x4f24ed24207bec8591c8172584dc3b57cdf3ee96afbd5e63905a90a704ff33f0",
"s": "0x6277bb9d2614843a4791ff2c192e70876438ec940c39d92deb504591b83dfeb3"
}
""".trimIndent()
)
val domainTx = txWeb3j.toDomain()
assertThat(domainTx).isEqualTo(
Transaction(
nonce = 0UL,
gasPrice = 0x1017df87UL,
gasLimit = 0x30d40UL,
to = "0x8d97689c9818892b700e27f316cc3e41e17fbeb9".decodeHex(),
value = 0x2386f26fc10000UL.toBigInteger(),
input = "0x".decodeHex(),
r = "0x4f24ed24207bec8591c8172584dc3b57cdf3ee96afbd5e63905a90a704ff33f0".toBigIntegerFromHex(),
s = "0x6277bb9d2614843a4791ff2c192e70876438ec940c39d92deb504591b83dfeb3".toBigIntegerFromHex(),
v = 1UL,
yParity = 1UL,
type = TransactionType.ACCESS_LIST,
chainId = 0x539UL,
maxFeePerGas = null,
maxPriorityFeePerGas = null,
accessList = listOf(
AccessListEntry(
address = "0x8d97689c9818892b700e27f316cc3e41e17fbeb9".decodeHex(),
listOf(
"0x0000000000000000000000000000000000000000000000000000000000000000".decodeHex(),
"0x0000000000000000000000000000000000000000000000000000000000000001".decodeHex()
)
)
)
)
)
domainTx.toBesu().also { besuTx ->
assertThat(besuTx.nonce).isEqualTo(0L)
assertThat(besuTx.gasPrice.getOrNull()).isEqualTo(Wei.of(0x1017df87L))
assertThat(besuTx.gasLimit).isEqualTo(0x30d40L)
assertThat(besuTx.to.getOrNull()).isEqualTo(Address.fromHexString("0x8d97689c9818892b700e27f316cc3e41e17fbeb9"))
assertThat(besuTx.value).isEqualTo(Wei.of(0x2386f26fc10000L))
assertThat(besuTx.payload).isEqualTo(Bytes.EMPTY)
assertThat(besuTx.signature.r).isEqualTo(
"0x4f24ed24207bec8591c8172584dc3b57cdf3ee96afbd5e63905a90a704ff33f0".toBigIntegerFromHex()
)
assertThat(besuTx.signature.s).isEqualTo(
"0x6277bb9d2614843a4791ff2c192e70876438ec940c39d92deb504591b83dfeb3".toBigIntegerFromHex()
)
assertThat(besuTx.signature.recId).isEqualTo(1)
assertThat(besuTx.type).isEqualTo(org.hyperledger.besu.datatypes.TransactionType.ACCESS_LIST)
assertThat(besuTx.chainId.getOrNull()).isEqualTo(0x539L)
assertThat(besuTx.maxFeePerGas).isEmpty()
assertThat(besuTx.maxPriorityFeePerGas).isEmpty()
val accessList = besuTx.accessList.getOrElse { fail("AccessList is empty") }
assertThat(accessList.get(0).address)
.isEqualTo(Address.fromHexString("0x8d97689c9818892b700e27f316cc3e41e17fbeb9"))
assertThat(accessList.get(0).storageKeys)
.containsExactly(
Bytes32.fromHexString("0x0000000000000000000000000000000000000000000000000000000000000000"),
Bytes32.fromHexString("0x0000000000000000000000000000000000000000000000000000000000000001")
)
}
}
@Test
fun `should map type accessList with empty list`() {
val txWeb3j = serialize(
"""
{
"accessList": [],
"blockHash": "0x7480ae911853c1fba10145401a21ddca3943b5894d74cbbf7a6beec526d1f9c2",
"blockNumber": "0xa",
"chainId": "0x539",
"from": "0x5007b0259849a673d0d780611f9a2ed8821d9ebe",
"gas": "0x5208",
"gasPrice": "0x1017df87",
"hash": "0xa2334c8858bb44ef3e9ef7f3523ec058ab24a869cfad7333fdf7bf3bb76deec4",
"input": "0x",
"nonce": "0x0",
"to": "0x8d97689c9818892b700e27f316cc3e41e17fbeb9",
"transactionIndex": "0x3",
"type": "0x1",
"value": "0x2386f26fc10000",
"yParity": "0x1",
"v": "0x1",
"r": "0xc57273f9ba15320937d5d9dfd1dc0b18d1e678b34bd3a4bfd29a63e11a856292",
"s": "0x7aa875a64835ecc5f9ac1a9fe3ab38d2a62bb3643a2597ab585a5607641a0c57"
}
""".trimIndent()
)
val domainTx = txWeb3j.toDomain()
assertThat(domainTx).isEqualTo(
Transaction(
nonce = 0UL,
gasPrice = 0x1017df87UL,
gasLimit = 0x5208UL,
to = "0x8d97689c9818892b700e27f316cc3e41e17fbeb9".decodeHex(),
value = 0x2386f26fc10000UL.toBigInteger(),
input = "0x".decodeHex(),
r = "0xc57273f9ba15320937d5d9dfd1dc0b18d1e678b34bd3a4bfd29a63e11a856292".toBigIntegerFromHex(),
s = "0x7aa875a64835ecc5f9ac1a9fe3ab38d2a62bb3643a2597ab585a5607641a0c57".toBigIntegerFromHex(),
v = 1UL,
yParity = 1UL,
type = TransactionType.ACCESS_LIST,
chainId = 0x539UL,
maxFeePerGas = null,
maxPriorityFeePerGas = null,
accessList = emptyList()
)
)
domainTx.toBesu().also { besuTx ->
assertThat(besuTx.type).isEqualTo(org.hyperledger.besu.datatypes.TransactionType.ACCESS_LIST)
assertThat(besuTx.nonce).isEqualTo(0L)
assertThat(besuTx.gasPrice.getOrNull()).isEqualTo(Wei.of(0x1017df87L))
assertThat(besuTx.gasLimit).isEqualTo(0x5208L)
assertThat(besuTx.to.getOrNull()).isEqualTo(Address.fromHexString("0x8d97689c9818892b700e27f316cc3e41e17fbeb9"))
assertThat(besuTx.value).isEqualTo(Wei.of(0x2386f26fc10000L))
assertThat(besuTx.payload).isEqualTo(Bytes.EMPTY)
assertThat(besuTx.signature.r).isEqualTo(
"0xc57273f9ba15320937d5d9dfd1dc0b18d1e678b34bd3a4bfd29a63e11a856292".toBigIntegerFromHex()
)
assertThat(besuTx.signature.s).isEqualTo(
"0x7aa875a64835ecc5f9ac1a9fe3ab38d2a62bb3643a2597ab585a5607641a0c57".toBigIntegerFromHex()
)
assertThat(besuTx.signature.recId).isEqualTo(1)
assertThat(besuTx.chainId.getOrNull()).isEqualTo(0x539L)
assertThat(besuTx.maxFeePerGas).isEmpty()
assertThat(besuTx.maxPriorityFeePerGas).isEmpty()
// it shall have an empty accessList
assertThat(besuTx.accessList.getOrNull()).isEmpty()
}
}
@Test
fun `it should map EIP1559 tx`() {
val input =
"""
0xdeb3cdf2000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000003e84d538bf9753309729adf92867d75bc2e58b566cb204b0d1b018fbf311e4b49bc52c1c450b2f412f5d9a44e01990d6127bc805ba8ef079ed2a897070378d706fbd2f5cf52b0e172541b7f11b9c2f0e0b91a67c5caf5908ddfdd2d340349e7398b698b3c336876c88e232f0e8f3197f2683e54a4439abb7d210d84cc1d3ad1bd48d0ba5dc30714d253743a734f17e88354eea550f7945df35d4c6316fcfaad09846f81f59b8127b037dbce6e5ffa45120fc69e6852f0c8ac2fcc9fd5e72503dd1c8d114ee079ffed84ddd6851e438cdd2a0ed1df9f0255481dc2a61a4808d856525619c948fbbd063bfd3db42504547db68c29990540eb7a36a1a8a0e483544eb634ae33f43f5bac2d991b9f6b36e23a7a299ade5b30ab96ad6dae27a9c374ae5f702fc689f596450c467722f24b7621ba5663ed6e08b620f04bc524338cac50e9ebc302d0b33dd9e2e563f05ce26303666a6c8c0f8dcd0b475f2219398ff4552533d28660c8d0d2843ce238ffb856dee06bc28e1ec0b92c3cb7b91378c07b049f3af20017dbdfdf48320ea7cf5f331bee27ba33d6a41351b3f044612a45f51451c068c23d6aec6784f623c6855acb95f07f213ad8605861fd8601ffa9a0282508a4d859769cb61247389020587a570cba1eac8b05576bd5b7a81b166f3c7b0f0ae0a8117f642d3fd0957e1cface4d10ebe6475a9a1f3bf6e3b1b7c16e50e529adcf0cf278aa64b9fecedcb0d894ab7ba6589e96ff56dff7b8636413f46cdc073c22521f6b89d7b68ca6f8af1ecc4e453137c801a9b35b5c4869883f59aaeaa7ee637d71c7e02f08894cffdb51a368b225fa1ab00e3ad2d91d1275d048aad5eb5d34438622c7aea1759b3fe747c2b0fddd62159de1d7cabbccde9c1e3511a34432e0c4e6dede019e38493fc29292ea321621629c1ffc62160747ee136171c96f55af7af6a29b8ca94f12d12b7c706974b1e586b3674a6aec7510f1025ba399f7a97f5911187b040b7a494e191bd761ad2a78daf427f5ee19cf24bc45fab34d32747de0f0a2c6bd33d2440d9f5d20da22da34e418d54d6894d42edd6d0c5a4f9b02d510a23db40cc455b7c423bd43b6fcd0f3655285e16ba8d9bdaf3f2147de572c33568353b5f5dd820f49dbddbc63297aed5e2f342b383a83319f9beed9d3d358a3dc7c0a010b85954fec3b34c3227a9b4447bd5d30b8b78c0ef36cd8197e867d37778b24e8ebfaadf08c42f3db6e5e46cb025bb4e98334ce0a7a59ba155eaf3968621f353d075e0d68f0787259e344a72e8938ebe3a81458ad20df917dc1392fe759210f045f7d87177ad39a13ec704301f1f0845b8c6cbc52f8f77c043bcc80adb513470d0e5a6b02df65259bcc3198efe01c555a5d28bf89f818ea1b984a64db220f487e230652000000000000000000000000000000000000000000000000
""".trimIndent().trim()
val txWeb3j = serialize(
"""
{
"accessList": [],
"blockHash": "0x7480ae911853c1fba10145401a21ddca3943b5894d74cbbf7a6beec526d1f9c2",
"blockNumber": "0xa",
"chainId": "0x539",
"from": "0x5007b0259849a673d0d780611f9a2ed8821d9ebe",
"gas": "0x5208",
"gasPrice": "0x1017df87",
"hash": "0xa2334c8858bb44ef3e9ef7f3523ec058ab24a869cfad7333fdf7bf3bb76deec4",
"input": "$input",
"nonce": "0x0",
"to": "0xe4392c8ecc46b304c83cdb5edaf742899b1bda93",
"transactionIndex": "0x3",
"type": "0x2",
"value": "0x2386f26fc10000",
"yParity": "0x1",
"v": "0x1",
"r": "0xeb4f70991ea4f14d23efb32591da3621d551406fd32bdfdd78bb677dec13160a",
"s": "0x783aaa89f73ef7535924da8fd5f12e15cae1a0811c4c4746d1c23abff1eacddf",
"maxFeePerGas": "0x1017dff7",
"maxPriorityFeePerGas": "0x1017df87"
}
""".trimIndent()
)
val txDomain = txWeb3j.toDomain()
assertThat(txDomain).isEqualTo(
Transaction(
nonce = 0UL,
// when type is EIP1559 gasPrice is null,
// eth_getBlock returns effectiveGasPrice but we will place as null here
gasPrice = null,
gasLimit = 0x5208UL,
to = "0xe4392c8ecc46b304c83cdb5edaf742899b1bda93".decodeHex(),
value = 0x2386f26fc10000UL.toBigInteger(),
input = input.decodeHex(),
r = "0xeb4f70991ea4f14d23efb32591da3621d551406fd32bdfdd78bb677dec13160a".toBigIntegerFromHex(),
s = "0x783aaa89f73ef7535924da8fd5f12e15cae1a0811c4c4746d1c23abff1eacddf".toBigIntegerFromHex(),
v = 1UL,
yParity = 1UL,
type = TransactionType.EIP1559,
chainId = 0x539UL,
maxFeePerGas = 0x1017dff7UL,
maxPriorityFeePerGas = 0x1017df87UL,
accessList = emptyList()
)
)
txDomain.toBesu().also { txBesu ->
assertThat(txBesu.type).isEqualTo(org.hyperledger.besu.datatypes.TransactionType.EIP1559)
assertThat(txBesu.nonce).isEqualTo(0L)
assertThat(txBesu.gasPrice.getOrNull()).isNull()
assertThat(txBesu.gasLimit).isEqualTo(0x5208L)
assertThat(txBesu.to.getOrNull()).isEqualTo(Address.fromHexString("0xe4392c8ecc46b304c83cdb5edaf742899b1bda93"))
assertThat(txBesu.value).isEqualTo(Wei.of(0x2386f26fc10000L))
assertThat(txBesu.payload).isEqualTo(Bytes.fromHexString(input))
assertThat(txBesu.signature.r).isEqualTo(
"0xeb4f70991ea4f14d23efb32591da3621d551406fd32bdfdd78bb677dec13160a".toBigIntegerFromHex()
)
assertThat(txBesu.signature.s).isEqualTo(
"0x783aaa89f73ef7535924da8fd5f12e15cae1a0811c4c4746d1c23abff1eacddf".toBigIntegerFromHex()
)
assertThat(txBesu.signature.recId).isEqualTo(1)
assertThat(txBesu.chainId.getOrNull()).isEqualTo(0x539L)
assertThat(txBesu.maxFeePerGas.getOrNull()).isEqualTo(Wei.of(0x1017dff7L))
assertThat(txBesu.maxPriorityFeePerGas.getOrNull()).isEqualTo(Wei.of(0x1017df87L))
assertThat(txBesu.accessList.getOrNull()).isEmpty()
}
}
@Test
fun `shall decode tx with to=null`() {
val input = """
0x608060405234801561001057600080fd5b5061001a3361001f565b61006f565b600080546001600160a01b038381166001600160a01b0319831681178455604051919092169283917f8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e09190a35050565b6108658061007e6000396000f3fe60806040526004361061007b5760003560e01c80639623609d1161004e5780639623609d1461012b57806399a88ec41461013e578063f2fde38b1461015e578063f3b7dead1461017e57600080fd5b8063204e1c7a14610080578063715018a6146100c95780637eff275e146100e05780638da5cb5b14610100575b600080fd5b34801561008c57600080fd5b506100a061009b366004610608565b61019e565b60405173ffffffffffffffffffffffffffffffffffffffff909116815260200160405180910390f35b3480156100d557600080fd5b506100de610255565b005b3480156100ec57600080fd5b506100de6100fb36600461062c565b610269565b34801561010c57600080fd5b5060005473ffffffffffffffffffffffffffffffffffffffff166100a0565b6100de610139366004610694565b6102f7565b34801561014a57600080fd5b506100de61015936600461062c565b61038c565b34801561016a57600080fd5b506100de610179366004610608565b6103e8565b34801561018a57600080fd5b506100a0610199366004610608565b6104a4565b60008060008373ffffffffffffffffffffffffffffffffffffffff166040516101ea907f5c60da1b00000000000000000000000000000000000000000000000000000000815260040190565b600060405180830381855afa9150503d8060008114610225576040519150601f19603f3d011682016040523d82523d6000602084013e61022a565b606091505b50915091508161023957600080fd5b8080602001905181019061024d9190610788565b949350505050565b61025d6104f0565b6102676000610571565b565b6102716104f0565b6040517f8f28397000000000000000000000000000000000000000000000000000000000815273ffffffffffffffffffffffffffffffffffffffff8281166004830152831690638f283970906024015b600060405180830381600087803b1580156102db57600080fd5b505af11580156102ef573d6000803e3d6000fd5b505050505050565b6102ff6104f0565b6040517f4f1ef28600000000000000000000000000000000000000000000000000000000815273ffffffffffffffffffffffffffffffffffffffff841690634f1ef28690349061035590869086906004016107a5565b6000604051808303818588803b15801561036e57600080fd5b505af1158015610382573d6000803e3d6000fd5b5050505050505050565b6103946104f0565b6040517f3659cfe600000000000000000000000000000000000000000000000000000000815273ffffffffffffffffffffffffffffffffffffffff8281166004830152831690633659cfe6906024016102c1565b6103f06104f0565b73ffffffffffffffffffffffffffffffffffffffff8116610498576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602660248201527f4f776e61626c653a206e6577206f776e657220697320746865207a65726f206160448201527f646472657373000000000000000000000000000000000000000000000000000060648201526084015b60405180910390fd5b6104a181610571565b50565b60008060008373ffffffffffffffffffffffffffffffffffffffff166040516101ea907ff851a44000000000000000000000000000000000000000000000000000000000815260040190565b60005473ffffffffffffffffffffffffffffffffffffffff163314610267576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820181905260248201527f4f776e61626c653a2063616c6c6572206973206e6f7420746865206f776e6572604482015260640161048f565b6000805473ffffffffffffffffffffffffffffffffffffffff8381167fffffffffffffffffffffffff0000000000000000000000000000000000000000831681178455604051919092169283917f8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e09190a35050565b73ffffffffffffffffffffffffffffffffffffffff811681146104a157600080fd5b60006020828403121561061a57600080fd5b8135610625816105e6565b9392505050565b6000806040838503121561063f57600080fd5b823561064a816105e6565b9150602083013561065a816105e6565b809150509250929050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b6000806000606084860312156106a957600080fd5b83356106b4816105e6565b925060208401356106c4816105e6565b9150604084013567ffffffffffffffff808211156106e157600080fd5b818601915086601f8301126106f557600080fd5b81358181111561070757610707610665565b604051601f82017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0908116603f0116810190838211818310171561074d5761074d610665565b8160405282815289602084870101111561076657600080fd5b8260208601602083013760006020848301015280955050505050509250925092565b60006020828403121561079a57600080fd5b8151610625816105e6565b73ffffffffffffffffffffffffffffffffffffffff8316815260006020604081840152835180604085015260005b818110156107ef578581018301518582016060015282016107d3565b5060006060828601015260607fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0601f83011685010192505050939250505056fea2646970667358221220688ab5dd8d9528556ea321a4b4ef35edd0288c19274db8bf4057c8b61d9e438764736f6c63430008130033
""".trimIndent().trim()
val txWeb3j = serialize(
"""
{
"accessList": [],
"blockHash": "0xf9bf74ade4a723a5527badeb62ce58d478f1022df0effc2a091898ef068563b6",
"blockNumber": "0x1",
"chainId": "0x539",
"from": "0x1b9abeec3215d8ade8a33607f2cf0f4f60e5f0d0",
"gas": "0x83a3d",
"gasPrice": "0x7",
"maxPriorityFeePerGas": "0x0",
"maxFeePerGas": "0xe",
"hash": "0xc9647251765f5d679e024dd0e5c0f4700c431f129e50847c3f73e2aa2262e593",
"input": "$input",
"nonce": "0x1",
"to": null,
"transactionIndex": "0x1",
"type": "0x2",
"value": "0x0",
"yParity": "0x1",
"v": "0x1",
"r": "0xf7afccb560d0c52bea021ba522a27dbd6c3aba3512dd2d3b2f476ed8dd87d5f7",
"s": "0x5f47f6ddcf1c216eb33eb69db553d682de34c78f5a5ab97905a428c2182f32e"
}
""".trimIndent()
)
val txDomain = txWeb3j.toDomain()
assertThat(txDomain).isEqualTo(
Transaction(
nonce = 1UL,
gasLimit = 0x83a3dUL,
to = null,
value = 0UL.toBigInteger(),
input = input.decodeHex(),
r = "0xf7afccb560d0c52bea021ba522a27dbd6c3aba3512dd2d3b2f476ed8dd87d5f7".toBigIntegerFromHex(),
s = "0x5f47f6ddcf1c216eb33eb69db553d682de34c78f5a5ab97905a428c2182f32e".toBigIntegerFromHex(),
v = 1UL,
yParity = 1UL,
type = TransactionType.EIP1559,
chainId = 0x539UL,
gasPrice = null,
maxFeePerGas = 0xeUL,
maxPriorityFeePerGas = 0UL,
accessList = emptyList()
)
)
txDomain.toBesu().let { txBesu ->
assertThat(txBesu.type).isEqualTo(org.hyperledger.besu.datatypes.TransactionType.EIP1559)
assertThat(txBesu.nonce).isEqualTo(1L)
assertThat(txBesu.gasPrice.getOrNull()).isNull()
assertThat(txBesu.gasLimit).isEqualTo(0x83a3dL)
assertThat(txBesu.to.getOrNull()).isNull()
assertThat(txBesu.value).isEqualTo(Wei.ZERO)
assertThat(txBesu.payload).isEqualTo(Bytes.fromHexString(input))
assertThat(txBesu.signature.r).isEqualTo(
"0xf7afccb560d0c52bea021ba522a27dbd6c3aba3512dd2d3b2f476ed8dd87d5f7".toBigIntegerFromHex()
)
assertThat(txBesu.signature.s).isEqualTo(
"0x5f47f6ddcf1c216eb33eb69db553d682de34c78f5a5ab97905a428c2182f32e".toBigIntegerFromHex()
)
assertThat(txBesu.signature.recId).isEqualTo(1)
assertThat(txBesu.chainId.getOrNull()).isEqualTo(0x539L)
assertThat(txBesu.maxFeePerGas.getOrNull()).isEqualTo(Wei.of(0xeL))
assertThat(txBesu.maxPriorityFeePerGas.getOrNull()).isEqualTo(Wei.ZERO)
assertThat(txBesu.accessList.getOrNull()).isEmpty()
}
}
}

View File

@@ -1,5 +1,6 @@
package net.consensys.linea.web3j
/*
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import org.web3j.protocol.core.methods.response.AccessListObject
@@ -219,3 +220,4 @@ class DomainObjectMappersTest {
assertThat(encodedTransaction.toString()).isEqualTo(signedContractCreation)
}
}
*/

View File

@@ -26,10 +26,9 @@ include 'jvm-libs:linea:core:traces'
include 'jvm-libs:linea:linea-contracts:l1-rollup'
include 'jvm-libs:linea:linea-contracts:l2-message-service'
include 'jvm-libs:linea:metrics:micrometer'
include 'jvm-libs:linea:teku-execution-client'
include 'jvm-libs:linea:besu-rlp-and-mappers'
include 'jvm-libs:linea:testing:file-system'
include 'jvm-libs:linea:testing:l1-blob-and-proof-submission'
include 'jvm-libs:linea:testing:teku-helper'
include 'jvm-libs:linea:web3j-extensions'
include 'coordinator:app'

View File

@@ -3,6 +3,11 @@ plugins {
}
dependencies {
implementation project(":jvm-libs:linea:teku-execution-client")
implementation project(':jvm-libs:generic:extensions:futures')
implementation project(':jvm-libs:linea:core:domain-models')
implementation project(':jvm-libs:linea:core:long-running-service')
implementation project(':jvm-libs:linea:web3j-extensions')
implementation project(':jvm-libs:linea:blob-compressor')
implementation project(':jvm-libs:linea:blob-decompressor')
implementation project(':jvm-libs:linea:besu-rlp-and-mappers')
}

View File

@@ -1,23 +0,0 @@
package net.consensys.linea
import net.consensys.linea.web3j.ExtendedWeb3JImpl
import org.web3j.protocol.Web3j
import org.web3j.protocol.http.HttpService
import org.web3j.utils.Async
import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1
import tech.pegasys.teku.infrastructure.async.SafeFuture
class BlockReader {
private val web3jClient: Web3j = Web3j.build(
HttpService("https://linea-sepolia.infura.io/v3/"),
1000,
Async.defaultExecutorService()
)
private val asyncWeb3J = ExtendedWeb3JImpl(web3jClient)
fun getBlockPayload(blockNumber: Long): SafeFuture<ExecutionPayloadV1> {
val encodedPayload = asyncWeb3J.ethGetExecutionPayloadByNumber(blockNumber)
return encodedPayload
}
}

View File

@@ -1,20 +0,0 @@
package net.consensys.linea
import org.apache.logging.log4j.LogManager
class TransactionEncodingToolMain {
companion object {
private val log = LogManager.getLogger(TransactionEncodingToolMain::class)
@JvmStatic
fun main(args: Array<String>) {
startApp()
}
private fun startApp() {
val app = BlockReader()
app.getBlockPayload(924973)
}
}
}

View File

@@ -0,0 +1,180 @@
package linea.test
import io.vertx.core.Vertx
import linea.blob.GoBackedBlobCompressor
import linea.domain.Block
import linea.domain.toBesu
import linea.rlp.BesuRlpDecoderAsyncVertxImpl
import linea.rlp.BesuRlpMainnetEncoderAsyncVertxImpl
import linea.rlp.RLP
import net.consensys.linea.CommonDomainFunctions
import net.consensys.linea.blob.BlobCompressorVersion
import net.consensys.linea.blob.BlobDecompressorVersion
import net.consensys.linea.blob.GoNativeBlobDecompressorFactory
import net.consensys.zkevm.PeriodicPollingService
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
import org.assertj.core.api.Assertions.assertThat
import org.assertj.core.api.Assertions.fail
import tech.pegasys.teku.infrastructure.async.SafeFuture
import java.util.concurrent.ConcurrentLinkedQueue
import java.util.concurrent.atomic.AtomicReference
import kotlin.time.Duration.Companion.milliseconds
// 100MB, much larger than a real blob, but just for testing to allow faster testing by compressing more blocks
val BLOB_COMPRESSOR_SIZE: UInt = 100u * 1024u * 1024U
class BlockEncodingValidator(
val vertx: Vertx,
val compressorVersion: BlobCompressorVersion = BlobCompressorVersion.V1_0_1,
val decompressorVersion: BlobDecompressorVersion = BlobDecompressorVersion.V1_1_0,
val blobSizeLimitBytes: UInt = BLOB_COMPRESSOR_SIZE,
val log: Logger = LogManager.getLogger(BlockEncodingValidator::class.java)
) : PeriodicPollingService(vertx, pollingIntervalMs = 1.milliseconds.inWholeMilliseconds, log = log) {
val compressor = GoBackedBlobCompressor.getInstance(compressorVersion, blobSizeLimitBytes)
val decompressor = GoNativeBlobDecompressorFactory.getInstance(decompressorVersion)
val rlpEncoder = BesuRlpMainnetEncoderAsyncVertxImpl(vertx)
val rlpMainnetDecoder = BesuRlpDecoderAsyncVertxImpl.mainnetDecoder(vertx)
val rlpBlobDecoder = BesuRlpDecoderAsyncVertxImpl.blobDecoder(vertx)
val queueOfBlocksToValidate = ConcurrentLinkedQueue<Block>()
var highestValidatedBlockNumber = AtomicReference<ULong>(ULong.MIN_VALUE)
override fun action(): SafeFuture<*> {
return validateCycle()
}
fun validateRlpEncodingDecoding(blocks: List<Block>): SafeFuture<Unit> {
val besuBlocks = blocks.map { it.toBesu() }
return rlpEncoder.encodeAsync(besuBlocks)
.thenCompose { encodedBlocks ->
rlpMainnetDecoder.decodeAsync(encodedBlocks)
}
.thenApply { decodedBlocks ->
val unMatchingBlocks = besuBlocks.zip(decodedBlocks).filter { (expected, actual) -> expected != actual }
if (unMatchingBlocks.isEmpty()) {
log.info(
"all blocks encoding/decoding match: blocks={}",
CommonDomainFunctions.blockIntervalString(blocks.first().number, blocks.last().number)
)
} else {
unMatchingBlocks.forEach { (expected, actual) ->
log.error(
"block encoding/decoding mismatch: block={} \nexpected={} \nactual={}",
expected.header.number,
expected,
actual
)
}
}
}
}
fun validateCompression(blocks: List<Block>): SafeFuture<Unit> {
queueOfBlocksToValidate.addAll(blocks)
return SafeFuture.completedFuture(Unit)
}
fun validateCycle(): SafeFuture<Unit> {
val blocks = queueOfBlocksToValidate.pull(300)
if (blocks.isEmpty()) {
return SafeFuture.completedFuture(Unit)
}
log.info(
"compression validation blocks={} started",
CommonDomainFunctions.blockIntervalString(blocks.first().number, blocks.last().number)
)
val besuBlocks = blocks.map { it.toBesu() }
val originalBlockInterval = CommonDomainFunctions.blockIntervalString(blocks.first().number, blocks.last().number)
return rlpEncoder.encodeAsync(besuBlocks)
.thenCompose { encodedBlocks ->
encodedBlocks.forEach { compressor.appendBlock(it) }
val compressedData = compressor.getCompressedData()
compressor.reset()
val decompressedData = decompressor.decompress(compressedData)
val decompressedBlocksList = RLP.decodeList(decompressedData)
rlpBlobDecoder.decodeAsync(decompressedBlocksList)
}.thenApply { decompressedBlocks ->
assertThat(decompressedBlocks.size).isEqualTo(besuBlocks.size)
.withFailMessage(
// this can happen if not all blocks fit into compressor limit
"originalBlocks=$originalBlockInterval decompressedBlocks.size=${decompressedBlocks.size} != "
)
decompressedBlocks.zip(besuBlocks).forEach { (decompressed, original) ->
runCatching {
assertBlock(decompressed, original)
}.getOrElse {
log.error(
"Decompressed block={} does not match: error={}",
original.header.number,
it.message,
it
)
}
}
}
.thenPeek {
highestValidatedBlockNumber.set(highestValidatedBlockNumber.get().coerceAtLeast(blocks.last().number))
log.info(
"compression validation blocks={} finished",
originalBlockInterval
)
}
}
}
fun <T> ConcurrentLinkedQueue<T>.pull(elementsLimit: Int): List<T> {
val elements = mutableListOf<T>()
var element = poll()
while (element != null && elements.size < elementsLimit) {
elements.add(element)
element = poll()
}
return elements
}
fun assertBlock(
decompressedBlock: org.hyperledger.besu.ethereum.core.Block,
originalBlock: org.hyperledger.besu.ethereum.core.Block,
log: Logger = LogManager.getLogger("test.assert.Block")
) {
// on decompression, the hash is placed as parentHash because besu recomputes the hash
// but custom decoder overrides hash calculation to use parentHash
assertThat(decompressedBlock.header.timestamp).isEqualTo(originalBlock.header.timestamp)
assertThat(decompressedBlock.header.hash).isEqualTo(originalBlock.header.hash)
decompressedBlock.body.transactions.forEachIndexed { index, decompressedTx ->
val originalTx = originalBlock.body.transactions[index]
log.trace(
"block={} txIndex={} \n originalTx={} \n decodedTx={} \n originalTxRlp={}",
originalBlock.header.number,
index,
originalTx,
decompressedTx,
originalTx.encoded()
)
runCatching {
assertThat(decompressedTx.type).isEqualTo(originalTx.type)
assertThat(decompressedTx.sender).isEqualTo(originalTx.sender)
assertThat(decompressedTx.nonce).isEqualTo(originalTx.nonce)
assertThat(decompressedTx.gasLimit).isEqualTo(originalTx.gasLimit)
if (originalTx.type.supports1559FeeMarket()) {
assertThat(decompressedTx.maxFeePerGas).isEqualTo(originalTx.maxFeePerGas)
assertThat(decompressedTx.maxPriorityFeePerGas).isEqualTo(originalTx.maxPriorityFeePerGas)
} else {
assertThat(decompressedTx.gasPrice).isEqualTo(originalTx.gasPrice)
}
assertThat(decompressedTx.to).isEqualTo(originalTx.to)
assertThat(decompressedTx.value).isEqualTo(originalTx.value)
assertThat(decompressedTx.accessList).isEqualTo(originalTx.accessList)
assertThat(decompressedTx.payload).isEqualTo(originalTx.payload)
}.getOrElse { th ->
fail(
"Transaction does not match: block=${originalBlock.header.number} " +
"txIndex=$index error=${th.message} origTxRlp=${originalTx.encoded()}",
th
)
}
}
}

View File

@@ -0,0 +1,77 @@
package linea.test
import build.linea.web3j.domain.toWeb3j
import io.vertx.core.Vertx
import linea.domain.Block
import linea.web3j.toDomain
import net.consensys.linea.BlockParameter.Companion.toBlockParameter
import net.consensys.linea.async.AsyncRetryer
import net.consensys.linea.async.toSafeFuture
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
import org.web3j.protocol.Web3j
import tech.pegasys.teku.infrastructure.async.SafeFuture
import java.util.concurrent.atomic.AtomicLong
import kotlin.time.Duration.Companion.milliseconds
class BlocksFetcher(
val web3j: Web3j,
val vertx: Vertx = Vertx.vertx(),
val pollingChuckSize: UInt = 100U,
val log: Logger = LogManager.getLogger(BlocksFetcher::class.java)
) {
fun fetchBlocks(
startBlockNumber: ULong,
endBlockNumber: ULong
): SafeFuture<List<Block>> {
return (startBlockNumber..endBlockNumber).toList()
.map { blockNumber ->
web3j.ethGetBlockByNumber(blockNumber.toBlockParameter().toWeb3j(), true)
.sendAsync()
.toSafeFuture()
.thenApply {
if (it.hasError()) {
log.error("Error fetching block={} errorMessage={}", blockNumber, it.error.message)
}
runCatching {
it.block.toDomain()
}
.getOrElse {
log.error("Error parsing block=$blockNumber", it)
null
}
}
}
.let { SafeFuture.collectAll(it.stream()) }
.thenApply { blocks: List<Block?> ->
blocks.filterNotNull().sortedBy { it.number }
}
}
fun consumeBlocks(
startBlockNumber: ULong,
endBlockNumber: ULong? = null,
chunkSize: UInt = pollingChuckSize,
consumer: (List<Block>) -> SafeFuture<*>
): SafeFuture<*> {
val lastBlockFetched = AtomicLong(startBlockNumber.toLong() - 1)
return AsyncRetryer.retry(
vertx,
backoffDelay = 1000.milliseconds,
stopRetriesPredicate = {
endBlockNumber?.let { lastBlockFetched.get().toULong() >= it } ?: false
},
stopRetriesOnErrorPredicate = {
it is Exception
}
) {
val start = (lastBlockFetched.get() + 1).toULong()
val end = (start + chunkSize - 1U).coerceAtMost(endBlockNumber ?: ULong.MAX_VALUE)
fetchBlocks(start, end)
.thenCompose { blocks ->
lastBlockFetched.set(blocks.last().number.toLong())
consumer(blocks)
}
}
}
}

View File

@@ -0,0 +1,67 @@
package linea.test
import io.vertx.core.Vertx
import linea.web3j.createWeb3jHttpClient
import net.consensys.linea.CommonDomainFunctions
import org.apache.logging.log4j.Level
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
import org.web3j.protocol.Web3j
import tech.pegasys.teku.infrastructure.async.SafeFuture
import java.util.concurrent.atomic.AtomicReference
class FetchAndValidationRunner(
val vertx: Vertx = Vertx.vertx(),
val rpcUrl: String,
val log: Logger = LogManager.getLogger(FetchAndValidationRunner::class.java)
) {
val web3j: Web3j = createWeb3jHttpClient(
rpcUrl = rpcUrl,
// executorService = vertx.nettyEventLoopGroup(),
log = LogManager.getLogger("test.client.web3j"),
requestResponseLogLevel = Level.DEBUG,
failuresLogLevel = Level.ERROR
)
val validator = BlockEncodingValidator(vertx = vertx, log = log).also { it.start() }
val blocksFetcher = BlocksFetcher(web3j, log = log)
val targetEndBlockNumber = AtomicReference<ULong?>()
fun awaitValidationFinishes(): SafeFuture<Unit> {
val result = SafeFuture<Unit>()
vertx.setPeriodic(2000) { timerId ->
if (targetEndBlockNumber.get() != null &&
validator.highestValidatedBlockNumber.get() >= targetEndBlockNumber.get()!!
) {
vertx.cancelTimer(timerId)
validator.stop()
web3j.shutdown()
result.complete(Unit)
}
}
return result
}
fun fetchAndValidateBlocks(
startBlockNumber: ULong,
endBlockNumber: ULong? = null,
chuckSize: UInt = 100U,
rlpEncodingDecodingOnly: Boolean = false
): SafeFuture<*> {
targetEndBlockNumber.set(endBlockNumber)
return blocksFetcher.consumeBlocks(
startBlockNumber = startBlockNumber,
endBlockNumber = endBlockNumber,
chunkSize = chuckSize
) { blocks ->
log.info(
"got blocks: {}",
CommonDomainFunctions.blockIntervalString(blocks.first().number, blocks.last().number)
)
if (rlpEncodingDecodingOnly) {
validator.validateRlpEncodingDecoding(blocks)
} else {
validator.validateCompression(blocks)
}
}
}
}

View File

@@ -0,0 +1,61 @@
package linea.test
import io.vertx.core.Vertx
import net.consensys.linea.async.get
import org.apache.logging.log4j.Level
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.core.config.Configurator
import java.util.concurrent.TimeUnit
fun configureLoggers(loggerConfigs: List<Pair<String, Level>>) {
loggerConfigs.forEach { (loggerName, level) ->
Configurator.setLevel(loggerName, level)
}
}
fun main() {
val rpcUrl = run {
"https://linea-sepolia.infura.io/v3/${System.getenv("INFURA_PROJECT_ID")}"
// "https://linea-mainnet.infura.io/v3/${System.getenv("INFURA_PROJECT_ID")}"
}
val vertx = Vertx.vertx()
vertx.exceptionHandler { error ->
println("Unhandled exception: message=${error.message}")
LogManager.getLogger("vertx").error("Unhandled exception: message={}", error.message, error)
}
val fetcherAndValidate =
FetchAndValidationRunner(
rpcUrl = rpcUrl,
vertx = vertx,
log = LogManager.getLogger("test.validator")
)
configureLoggers(
listOf(
"linea.rlp" to Level.INFO,
"test.client.web3j" to Level.INFO,
"test.validator" to Level.INFO
)
)
// Sepolia Blocks
val startBlockNumber = 930_973UL
// val startBlockNumber = 5_099_599UL
// Mainnet Blocks
// val startBlockNumber = 10_000_308UL
runCatching {
fetcherAndValidate.fetchAndValidateBlocks(
startBlockNumber = startBlockNumber,
endBlockNumber = startBlockNumber + 100_000U,
// endBlockNumber = startBlockNumber + 0u,
chuckSize = 1_000U,
rlpEncodingDecodingOnly = false
).get(2, TimeUnit.MINUTES)
}.onFailure { error ->
fetcherAndValidate.log.error("Error fetching and validating blocks", error)
}
fetcherAndValidate.awaitValidationFinishes().get()
println("waited validation finishes")
// fetcherAndValidate.validator.stop()
vertx.close().get()
println("closed vertx")
}

View File

@@ -78,7 +78,7 @@ class Api(
)
.compose { verticleId: String ->
jsonRpcServerId = verticleId
serverPort = httpServer!!.bindedPort
serverPort = httpServer!!.boundPort
vertx.deployVerticle(observabilityServer).onSuccess { monitorVerticleId ->
this.observabilityServerId = monitorVerticleId
}