From 013d1c955365ca5aa826e0ccd1d01e83d59f509c Mon Sep 17 00:00:00 2001 From: Thabokani <149070269+Thabokani@users.noreply.github.com> Date: Sat, 13 Jan 2024 00:58:54 +0800 Subject: [PATCH 01/20] Fix typo in SUPPORT.md (#6395) Signed-off-by: Thabokani <149070269+Thabokani@users.noreply.github.com> Co-authored-by: Fabio Di Fabio --- SUPPORT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SUPPORT.md b/SUPPORT.md index 16a9bccd6..a9eb54acb 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -7,7 +7,7 @@ Welcome to the Besu repository! The following links are a set of guidelines for Having Github, Discord, and Linux Foundation accounts is necessary for obtaining support for Besu through the community channels, wiki and issue management. * If you want to raise an issue, you can do so [on the github issue tab](https://github.com/hyperledger/besu/issues). * Hyperledger Discord requires a [Discord account]. -* The Hyperlegder wiki also requires a [Linux Foundation (LF) account] in order to edit pages. +* The Hyperledger wiki also requires a [Linux Foundation (LF) account] in order to edit pages. ### Useful support links From b65c633860c8f7f0fa2dc34c0bae98e5ef00f1d1 Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Sat, 13 Jan 2024 03:47:30 +1000 Subject: [PATCH 02/20] reduce machine size (#6392) Signed-off-by: Sally MacFarlane Co-authored-by: Fabio Di Fabio --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2f008df25..6d08e515a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -233,7 +233,7 @@ jobs: - capture_test_logs acceptanceTestsCliqueBft: - executor: besu_executor_xl + executor: besu_executor_med steps: - prepare - attach_workspace: @@ -242,7 +242,7 @@ jobs: name: AcceptanceTests (Non-Mainnet) no_output_timeout: 20m command: | - ./gradlew --no-daemon acceptanceTestCliqueBft + ./gradlew --no-daemon --max-workers=1 acceptanceTestCliqueBft - capture_test_results - capture_test_logs From 9fbe4fbbb2e82671ddc8b7fe127b6d8706b2fc7c Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Sun, 14 Jan 2024 22:50:45 +0100 Subject: [PATCH 03/20] Upgrade Guava dependency (#6396) * Bump com.google.guava:guava to 33.0.0 Signed-off-by: Fabio Di Fabio * Update pending tx estimated memory size after Guava update Signed-off-by: Fabio Di Fabio * Rebuilt Gradle validation metadata Signed-off-by: Fabio Di Fabio * Update CHANGELOG Signed-off-by: Fabio Di Fabio --------- Signed-off-by: Fabio Di Fabio --- CHANGELOG.md | 1 + ...ingTransactionEstimatedMemorySizeTest.java | 2 +- gradle/verification-metadata.xml | 96 +++++++++---------- gradle/versions.gradle | 2 +- 4 files changed, 47 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abcc2ab3c..bb8d3cb9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Introduce TransactionEvaluationContext to pass data between transaction selectors and plugin, during block creation [#6381](https://github.com/hyperledger/besu/pull/6381) - Upgrade dependencies [#6377](https://github.com/hyperledger/besu/pull/6377) - Upgrade `com.fasterxml.jackson` dependencies [#6378](https://github.com/hyperledger/besu/pull/6378) +- Upgrade Guava dependency [#6396](https://github.com/hyperledger/besu/pull/6396) ### Bug fixes - INTERNAL_ERROR from `eth_estimateGas` JSON/RPC calls [#6344](https://github.com/hyperledger/besu/issues/6344) diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java index c9c81543a..20b94f5bf 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java @@ -58,7 +58,7 @@ public class PendingTransactionEstimatedMemorySizeTest extends BaseTransactionPo private static final Set> SHARED_CLASSES = Set.of(SignatureAlgorithm.class, TransactionType.class); private static final Set COMMON_CONSTANT_FIELD_PATHS = - Set.of(".value.ctor", ".hashNoSignature"); + Set.of(".value.ctor", ".hashNoSignature", ".signature.encoded.delegate"); private static final Set EIP1559_EIP4844_CONSTANT_FIELD_PATHS = Sets.union(COMMON_CONSTANT_FIELD_PATHS, Set.of(".gasPrice")); private static final Set FRONTIER_ACCESS_LIST_CONSTANT_FIELD_PATHS = diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 30d55eb68..58a562fff 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -760,14 +760,6 @@ - - - - - - - - @@ -777,9 +769,6 @@ - - - @@ -792,6 +781,14 @@ + + + + + + + + @@ -808,11 +805,6 @@ - - - - - @@ -828,6 +820,11 @@ + + + + + @@ -920,12 +917,12 @@ - - - + + + - - + + @@ -944,16 +941,22 @@ + + + + + + + + + + + - - - - - @@ -964,6 +967,11 @@ + + + + + @@ -998,14 +1006,6 @@ - - - - - - - - @@ -3492,22 +3492,6 @@ - - - - - - - - - - - - - - - - @@ -3523,8 +3507,16 @@ - - + + + + + + + + + + diff --git a/gradle/versions.gradle b/gradle/versions.gradle index b497109d7..c27af91f0 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -46,7 +46,7 @@ dependencyManagement { entry 'error_prone_test_helpers' } - dependency 'com.google.guava:guava:31.1-jre' + dependency 'com.google.guava:guava:33.0.0-jre' dependency 'com.graphql-java:graphql-java:21.3' From 39d7042ca3f41091a7d24d12bfca6c85fe5b4dd8 Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Mon, 15 Jan 2024 10:09:37 +1000 Subject: [PATCH 04/20] log a different warning if pruning is enabled with BONSAI (#6401) Signed-off-by: Sally MacFarlane --- CHANGELOG.md | 2 +- .../org/hyperledger/besu/cli/BesuCommand.java | 12 ++++++++++-- .../org/hyperledger/besu/cli/BesuCommandTest.java | 15 +++++++++++++-- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb8d3cb9b..13f26a257 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,7 +30,7 @@ ### Breaking Changes ### Deprecations -- Forest pruning (`pruning-enabled` options) is deprecated and will be removed soon. To save disk space consider switching to Bonsai data storage format [#6230](https://github.com/hyperledger/besu/pull/6230) +- Forest pruning (`pruning-enabled` option) is deprecated and will be removed soon. To save disk space consider switching to Bonsai data storage format [#6230](https://github.com/hyperledger/besu/pull/6230) ### Additions and Improvements - Add error messages on authentication failures with username and password [#6212](https://github.com/hyperledger/besu/pull/6212) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 9842d52f8..d2fb52433 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -148,6 +148,7 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder; import org.hyperledger.besu.ethereum.trie.forest.pruner.PrunerConfiguration; +import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.evm.precompile.AbstractAltBnPrecompiledContract; import org.hyperledger.besu.evm.precompile.BigIntegerModularExponentiationPrecompiledContract; import org.hyperledger.besu.evm.precompile.KZGPointEvalPrecompiledContract; @@ -2063,8 +2064,15 @@ public class BesuCommand implements DefaultCommandValues, Runnable { } if (isPruningEnabled()) { - logger.warn( - "Forest pruning is deprecated and will be removed soon. To save disk space consider switching to Bonsai data storage format."); + if (dataStorageOptions + .toDomainObject() + .getDataStorageFormat() + .equals(DataStorageFormat.BONSAI)) { + logger.warn("Forest pruning is ignored with Bonsai data storage format."); + } else { + logger.warn( + "Forest pruning is deprecated and will be removed soon. To save disk space consider switching to Bonsai data storage format."); + } } } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index ba0e5b275..dc56c3484 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -3840,8 +3840,8 @@ public class BesuCommandTest extends CommandTestAbstract { } @Test - public void pruningLogsDeprecationWarning() { - parseCommand("--pruning-enabled"); + public void pruningLogsDeprecationWarningWithForest() { + parseCommand("--pruning-enabled", "--data-storage-format=FOREST"); verify(mockControllerBuilder).isPruningEnabled(true); @@ -3854,6 +3854,17 @@ public class BesuCommandTest extends CommandTestAbstract { + " To save disk space consider switching to Bonsai data storage format.")); } + @Test + public void pruningLogsIgnoredWarningWithBonsai() { + parseCommand("--pruning-enabled", "--data-storage-format=BONSAI"); + + verify(mockControllerBuilder).isPruningEnabled(true); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + verify(mockLogger).warn(contains("Forest pruning is ignored with Bonsai data storage format.")); + } + @Test public void devModeOptionMustBeUsed() throws Exception { parseCommand("--network", "dev"); From 724e3d040e74204c9ace7844a0651cbdebd671f9 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 16 Jan 2024 00:27:57 +0100 Subject: [PATCH 05/20] Upgrade Mockito (#6397) Signed-off-by: Fabio Di Fabio --- CHANGELOG.md | 1 + .../ethereum/api/handlers/HandlerFactory.java | 4 +- .../internal/methods/DebugTraceBlock.java | 23 ++-- .../JsonRpcHttpServiceHostAllowlistTest.java | 64 +++++---- .../jsonrpc/JsonRpcHttpServiceLoginTest.java | 62 ++++----- .../JsonRpcHttpServiceRpcApisTest.java | 129 +++++++++--------- .../api/jsonrpc/JsonRpcHttpServiceTest.java | 84 ++++++------ .../jsonrpc/JsonRpcHttpServiceTestBase.java | 88 +++++++----- .../JsonRpcHttpServiceTlsClientAuthTest.java | 64 +++++---- ...RpcHttpServiceTlsMisconfigurationTest.java | 64 +++++---- .../jsonrpc/JsonRpcHttpServiceTlsTest.java | 64 +++++---- .../DebugStandardTraceBlockToFileTest.java | 28 ++-- .../internal/methods/DebugTraceBlockTest.java | 45 +++--- .../backwardsync/BackwardSyncAlgSpec.java | 4 +- .../rlpx/connections/netty/DeFramerTest.java | 13 +- .../evm/operations/ChainIdOperationTest.java | 3 +- gradle/verification-metadata.xml | 46 +++---- gradle/versions.gradle | 2 +- 18 files changed, 395 insertions(+), 393 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13f26a257..d520eba54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - Upgrade dependencies [#6377](https://github.com/hyperledger/besu/pull/6377) - Upgrade `com.fasterxml.jackson` dependencies [#6378](https://github.com/hyperledger/besu/pull/6378) - Upgrade Guava dependency [#6396](https://github.com/hyperledger/besu/pull/6396) +- Upgrade Mockito [#6397](https://github.com/hyperledger/besu/pull/6397) ### Bug fixes - INTERNAL_ERROR from `eth_estimateGas` JSON/RPC calls [#6344](https://github.com/hyperledger/besu/issues/6344) diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/handlers/HandlerFactory.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/handlers/HandlerFactory.java index 2d3639d52..03465f70f 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/handlers/HandlerFactory.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/handlers/HandlerFactory.java @@ -22,6 +22,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.JsonRpcMethod; import java.util.Collection; import java.util.Map; import java.util.Optional; +import java.util.function.Function; import java.util.stream.Collectors; import io.opentelemetry.api.trace.Tracer; @@ -35,7 +36,8 @@ public class HandlerFactory { assert methods != null && globalOptions != null; return TimeoutHandler.handler( Optional.of(globalOptions), - methods.keySet().stream().collect(Collectors.toMap(String::new, ignored -> globalOptions))); + methods.keySet().stream() + .collect(Collectors.toMap(Function.identity(), ignored -> globalOptions))); } public static Handler authentication( diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlock.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlock.java index 270f176c7..6dab0436a 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlock.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlock.java @@ -46,15 +46,15 @@ public class DebugTraceBlock implements JsonRpcMethod { private static final Logger LOG = LoggerFactory.getLogger(DebugTraceBlock.class); private final Supplier blockTracerSupplier; private final BlockHeaderFunctions blockHeaderFunctions; - private final BlockchainQueries blockchain; + private final BlockchainQueries blockchainQueries; public DebugTraceBlock( final Supplier blockTracerSupplier, final BlockHeaderFunctions blockHeaderFunctions, - final BlockchainQueries blockchain) { + final BlockchainQueries blockchainQueries) { this.blockTracerSupplier = blockTracerSupplier; this.blockHeaderFunctions = blockHeaderFunctions; - this.blockchain = blockchain; + this.blockchainQueries = blockchainQueries; } @Override @@ -79,18 +79,17 @@ public class DebugTraceBlock implements JsonRpcMethod { .map(TransactionTraceParams::traceOptions) .orElse(TraceOptions.DEFAULT); - if (this.blockchain.blockByHash(block.getHeader().getParentHash()).isPresent()) { + if (this.blockchainQueries.blockByHash(block.getHeader().getParentHash()).isPresent()) { final Collection results = Tracer.processTracing( - blockchain, + blockchainQueries, Optional.of(block.getHeader()), - mutableWorldState -> { - return blockTracerSupplier - .get() - .trace(mutableWorldState, block, new DebugOperationTracer(traceOptions)) - .map(BlockTrace::getTransactionTraces) - .map(DebugTraceTransactionResult::of); - }) + mutableWorldState -> + blockTracerSupplier + .get() + .trace(mutableWorldState, block, new DebugOperationTracer(traceOptions)) + .map(BlockTrace::getTransactionTraces) + .map(DebugTraceTransactionResult::of)) .orElse(null); return new JsonRpcSuccessResponse(requestContext.getRequest().getId(), results); } else { diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceHostAllowlistTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceHostAllowlistTest.java index da7319282..f433bcdc9 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceHostAllowlistTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceHostAllowlistTest.java @@ -17,7 +17,6 @@ package org.hyperledger.besu.ethereum.api.jsonrpc; import static org.assertj.core.api.Assertions.assertThat; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.DEFAULT_RPC_APIS; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import org.hyperledger.besu.config.StubGenesisConfigOptions; import org.hyperledger.besu.ethereum.ProtocolContext; @@ -98,38 +97,37 @@ public class JsonRpcHttpServiceHostAllowlistTest { supportedCapabilities.add(EthProtocol.ETH63); rpcMethods = - spy( - new JsonRpcMethodsFactory() - .methods( - CLIENT_VERSION, - CHAIN_ID, - new StubGenesisConfigOptions(), - peerDiscoveryMock, - blockchainQueries, - synchronizer, - MainnetProtocolSchedule.fromConfig( - new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID)), - mock(ProtocolContext.class), - mock(FilterManager.class), - mock(TransactionPool.class), - mock(MiningParameters.class), - mock(PoWMiningCoordinator.class), - new NoOpMetricsSystem(), - supportedCapabilities, - Optional.of(mock(AccountLocalConfigPermissioningController.class)), - Optional.of(mock(NodeLocalConfigPermissioningController.class)), - DEFAULT_RPC_APIS, - mock(PrivacyParameters.class), - mock(JsonRpcConfiguration.class), - mock(WebSocketConfiguration.class), - mock(MetricsConfiguration.class), - natService, - new HashMap<>(), - folder, - mock(EthPeers.class), - vertx, - mock(ApiConfiguration.class), - Optional.empty())); + new JsonRpcMethodsFactory() + .methods( + CLIENT_VERSION, + CHAIN_ID, + new StubGenesisConfigOptions(), + peerDiscoveryMock, + blockchainQueries, + synchronizer, + MainnetProtocolSchedule.fromConfig( + new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID)), + mock(ProtocolContext.class), + mock(FilterManager.class), + mock(TransactionPool.class), + mock(MiningParameters.class), + mock(PoWMiningCoordinator.class), + new NoOpMetricsSystem(), + supportedCapabilities, + Optional.of(mock(AccountLocalConfigPermissioningController.class)), + Optional.of(mock(NodeLocalConfigPermissioningController.class)), + DEFAULT_RPC_APIS, + mock(PrivacyParameters.class), + mock(JsonRpcConfiguration.class), + mock(WebSocketConfiguration.class), + mock(MetricsConfiguration.class), + natService, + new HashMap<>(), + folder, + mock(EthPeers.class), + vertx, + mock(ApiConfiguration.class), + Optional.empty()); service = createJsonRpcHttpService(); service.start().join(); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceLoginTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceLoginTest.java index 7b9eac247..aa1b58217 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceLoginTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceLoginTest.java @@ -19,7 +19,6 @@ import static java.util.concurrent.TimeUnit.MINUTES; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.util.Lists.list; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import org.hyperledger.besu.config.StubGenesisConfigOptions; import org.hyperledger.besu.ethereum.ProtocolContext; @@ -129,37 +128,36 @@ public class JsonRpcHttpServiceLoginTest { new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID); rpcMethods = - spy( - new JsonRpcMethodsFactory() - .methods( - CLIENT_VERSION, - CHAIN_ID, - genesisConfigOptions, - peerDiscoveryMock, - blockchainQueries, - synchronizer, - MainnetProtocolSchedule.fromConfig(genesisConfigOptions), - mock(ProtocolContext.class), - mock(FilterManager.class), - mock(TransactionPool.class), - mock(MiningParameters.class), - mock(PoWMiningCoordinator.class), - new NoOpMetricsSystem(), - supportedCapabilities, - Optional.empty(), - Optional.empty(), - JSON_RPC_APIS, - mock(PrivacyParameters.class), - mock(JsonRpcConfiguration.class), - mock(WebSocketConfiguration.class), - mock(MetricsConfiguration.class), - natService, - new HashMap<>(), - folder, - mock(EthPeers.class), - vertx, - mock(ApiConfiguration.class), - Optional.empty())); + new JsonRpcMethodsFactory() + .methods( + CLIENT_VERSION, + CHAIN_ID, + genesisConfigOptions, + peerDiscoveryMock, + blockchainQueries, + synchronizer, + MainnetProtocolSchedule.fromConfig(genesisConfigOptions), + mock(ProtocolContext.class), + mock(FilterManager.class), + mock(TransactionPool.class), + mock(MiningParameters.class), + mock(PoWMiningCoordinator.class), + new NoOpMetricsSystem(), + supportedCapabilities, + Optional.empty(), + Optional.empty(), + JSON_RPC_APIS, + mock(PrivacyParameters.class), + mock(JsonRpcConfiguration.class), + mock(WebSocketConfiguration.class), + mock(MetricsConfiguration.class), + natService, + new HashMap<>(), + folder, + mock(EthPeers.class), + vertx, + mock(ApiConfiguration.class), + Optional.empty()); service = createJsonRpcHttpService(); jwtAuth = service.authenticationService.get().getJwtAuthProvider(); service.start().join(); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceRpcApisTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceRpcApisTest.java index 6870c4fed..8323f61b2 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceRpcApisTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceRpcApisTest.java @@ -17,7 +17,6 @@ package org.hyperledger.besu.ethereum.api.jsonrpc; import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import org.hyperledger.besu.config.StubGenesisConfigOptions; @@ -201,37 +200,36 @@ public class JsonRpcHttpServiceRpcApisTest { supportedCapabilities.add(EthProtocol.ETH63); final Map rpcMethods = - spy( - new JsonRpcMethodsFactory() - .methods( - CLIENT_VERSION, - NETWORK_ID, - new StubGenesisConfigOptions(), - mock(P2PNetwork.class), - blockchainQueries, - mock(Synchronizer.class), - ProtocolScheduleFixture.MAINNET, - mock(ProtocolContext.class), - mock(FilterManager.class), - mock(TransactionPool.class), - mock(MiningParameters.class), - mock(PoWMiningCoordinator.class), - new NoOpMetricsSystem(), - supportedCapabilities, - Optional.of(mock(AccountLocalConfigPermissioningController.class)), - Optional.of(mock(NodeLocalConfigPermissioningController.class)), - config.getRpcApis(), - mock(PrivacyParameters.class), - mock(JsonRpcConfiguration.class), - mock(WebSocketConfiguration.class), - mock(MetricsConfiguration.class), - natService, - new HashMap<>(), - folder, - mock(EthPeers.class), - vertx, - mock(ApiConfiguration.class), - Optional.empty())); + new JsonRpcMethodsFactory() + .methods( + CLIENT_VERSION, + NETWORK_ID, + new StubGenesisConfigOptions(), + mock(P2PNetwork.class), + blockchainQueries, + mock(Synchronizer.class), + ProtocolScheduleFixture.MAINNET, + mock(ProtocolContext.class), + mock(FilterManager.class), + mock(TransactionPool.class), + mock(MiningParameters.class), + mock(PoWMiningCoordinator.class), + new NoOpMetricsSystem(), + supportedCapabilities, + Optional.of(mock(AccountLocalConfigPermissioningController.class)), + Optional.of(mock(NodeLocalConfigPermissioningController.class)), + config.getRpcApis(), + mock(PrivacyParameters.class), + mock(JsonRpcConfiguration.class), + mock(WebSocketConfiguration.class), + mock(MetricsConfiguration.class), + natService, + new HashMap<>(), + folder, + mock(EthPeers.class), + vertx, + mock(ApiConfiguration.class), + Optional.empty()); final JsonRpcHttpService jsonRpcHttpService = new JsonRpcHttpService( vertx, @@ -302,8 +300,7 @@ public class JsonRpcHttpServiceRpcApisTest { final WebSocketConfiguration webSocketConfiguration, final P2PNetwork p2pNetwork, final MetricsConfiguration metricsConfiguration, - final NatService natService) - throws Exception { + final NatService natService) { final Set supportedCapabilities = new HashSet<>(); supportedCapabilities.add(EthProtocol.ETH62); supportedCapabilities.add(EthProtocol.ETH63); @@ -311,37 +308,36 @@ public class JsonRpcHttpServiceRpcApisTest { webSocketConfiguration.setPort(0); final Map rpcMethods = - spy( - new JsonRpcMethodsFactory() - .methods( - CLIENT_VERSION, - NETWORK_ID, - new StubGenesisConfigOptions(), - p2pNetwork, - blockchainQueries, - mock(Synchronizer.class), - ProtocolScheduleFixture.MAINNET, - mock(ProtocolContext.class), - mock(FilterManager.class), - mock(TransactionPool.class), - mock(MiningParameters.class), - mock(PoWMiningCoordinator.class), - new NoOpMetricsSystem(), - supportedCapabilities, - Optional.of(mock(AccountLocalConfigPermissioningController.class)), - Optional.of(mock(NodeLocalConfigPermissioningController.class)), - jsonRpcConfiguration.getRpcApis(), - mock(PrivacyParameters.class), - jsonRpcConfiguration, - webSocketConfiguration, - metricsConfiguration, - natService, - new HashMap<>(), - folder, - mock(EthPeers.class), - vertx, - mock(ApiConfiguration.class), - Optional.empty())); + new JsonRpcMethodsFactory() + .methods( + CLIENT_VERSION, + NETWORK_ID, + new StubGenesisConfigOptions(), + p2pNetwork, + blockchainQueries, + mock(Synchronizer.class), + ProtocolScheduleFixture.MAINNET, + mock(ProtocolContext.class), + mock(FilterManager.class), + mock(TransactionPool.class), + mock(MiningParameters.class), + mock(PoWMiningCoordinator.class), + new NoOpMetricsSystem(), + supportedCapabilities, + Optional.of(mock(AccountLocalConfigPermissioningController.class)), + Optional.of(mock(NodeLocalConfigPermissioningController.class)), + jsonRpcConfiguration.getRpcApis(), + mock(PrivacyParameters.class), + jsonRpcConfiguration, + webSocketConfiguration, + metricsConfiguration, + natService, + new HashMap<>(), + folder, + mock(EthPeers.class), + vertx, + mock(ApiConfiguration.class), + Optional.empty()); final JsonRpcHttpService jsonRpcHttpService = new JsonRpcHttpService( vertx, @@ -425,8 +421,7 @@ public class JsonRpcHttpServiceRpcApisTest { "{\"jsonrpc\":\"2.0\",\"id\":" + Json.encode(id) + ",\"method\":\"net_services\"}", JSON); } - public JsonRpcHttpService getJsonRpcHttpService(final boolean[] enabledNetServices) - throws Exception { + public JsonRpcHttpService getJsonRpcHttpService(final boolean[] enabledNetServices) { JsonRpcConfiguration jsonRpcConfiguration = JsonRpcConfiguration.createDefault(); WebSocketConfiguration webSocketConfiguration = WebSocketConfiguration.createDefault(); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTest.java index f9c9b14cc..901f20ff7 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTest.java @@ -17,10 +17,7 @@ package org.hyperledger.besu.ethereum.api.jsonrpc; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.Address; @@ -1389,65 +1386,68 @@ public class JsonRpcHttpServiceTest extends JsonRpcHttpServiceTestBase { + "\"}", JSON); - when(rpcMethods.get(any(String.class))).thenReturn(null); - when(rpcMethods.containsKey(any(String.class))).thenReturn(false); + try (var unused = disableRpcMethod(methodName)) { - try (final Response resp = client.newCall(buildPostRequest(body)).execute()) { - assertThat(resp.code()).isEqualTo(200); - final JsonObject json = new JsonObject(resp.body().string()); - final RpcErrorType expectedError = RpcErrorType.METHOD_NOT_ENABLED; - testHelper.assertValidJsonRpcError( - json, id, expectedError.getCode(), expectedError.getMessage()); + try (final Response resp = client.newCall(buildPostRequest(body)).execute()) { + assertThat(resp.code()).isEqualTo(200); + final JsonObject json = new JsonObject(resp.body().string()); + final RpcErrorType expectedError = RpcErrorType.METHOD_NOT_ENABLED; + testHelper.assertValidJsonRpcError( + json, id, expectedError.getCode(), expectedError.getMessage()); + } } - - verify(rpcMethods).containsKey(methodName); - verify(rpcMethods).get(methodName); - - reset(rpcMethods); } @Test public void exceptionallyHandleJsonSingleRequest() throws Exception { + final String methodName = "foo"; final JsonRpcMethod jsonRpcMethod = mock(JsonRpcMethod.class); - when(jsonRpcMethod.getName()).thenReturn("foo"); + when(jsonRpcMethod.getName()).thenReturn(methodName); when(jsonRpcMethod.response(any())).thenThrow(new RuntimeException("test exception")); - doReturn(jsonRpcMethod).when(rpcMethods).get("foo"); + try (var unused = addRpcMethod(methodName, jsonRpcMethod)) { - final RequestBody body = - RequestBody.create("{\"jsonrpc\":\"2.0\",\"id\":\"666\",\"method\":\"foo\"}", JSON); + final RequestBody body = + RequestBody.create( + "{\"jsonrpc\":\"2.0\",\"id\":\"666\",\"method\":\"" + methodName + "\"}", JSON); - try (final Response resp = client.newCall(buildPostRequest(body)).execute()) { - assertThat(resp.code()).isEqualTo(200); - final JsonObject json = new JsonObject(resp.body().string()); - final RpcErrorType expectedError = RpcErrorType.INTERNAL_ERROR; - testHelper.assertValidJsonRpcError( - json, "666", expectedError.getCode(), expectedError.getMessage()); + try (final Response resp = client.newCall(buildPostRequest(body)).execute()) { + assertThat(resp.code()).isEqualTo(200); + final JsonObject json = new JsonObject(resp.body().string()); + final RpcErrorType expectedError = RpcErrorType.INTERNAL_ERROR; + testHelper.assertValidJsonRpcError( + json, "666", expectedError.getCode(), expectedError.getMessage()); + } } } @Test public void exceptionallyHandleJsonBatchRequest() throws Exception { + final String methodName = "foo"; final JsonRpcMethod jsonRpcMethod = mock(JsonRpcMethod.class); - when(jsonRpcMethod.getName()).thenReturn("foo"); + when(jsonRpcMethod.getName()).thenReturn(methodName); when(jsonRpcMethod.response(any())).thenThrow(new RuntimeException("test exception")); - doReturn(jsonRpcMethod).when(rpcMethods).get("foo"); - final RequestBody body = - RequestBody.create( - "[{\"jsonrpc\":\"2.0\",\"id\":\"000\",\"method\":\"web3_clientVersion\"}," - + "{\"jsonrpc\":\"2.0\",\"id\":\"111\",\"method\":\"foo\"}," - + "{\"jsonrpc\":\"2.0\",\"id\":\"222\",\"method\":\"net_version\"}]", - JSON); + try (var unused = addRpcMethod(methodName, jsonRpcMethod)) { - try (final Response resp = client.newCall(buildPostRequest(body)).execute()) { - assertThat(resp.code()).isEqualTo(200); - final JsonArray array = new JsonArray(resp.body().string()); - testHelper.assertValidJsonRpcResult(array.getJsonObject(0), "000"); - final RpcErrorType expectedError = RpcErrorType.INTERNAL_ERROR; - testHelper.assertValidJsonRpcError( - array.getJsonObject(1), "111", expectedError.getCode(), expectedError.getMessage()); - testHelper.assertValidJsonRpcResult(array.getJsonObject(2), "222"); + final RequestBody body = + RequestBody.create( + "[{\"jsonrpc\":\"2.0\",\"id\":\"000\",\"method\":\"web3_clientVersion\"}," + + "{\"jsonrpc\":\"2.0\",\"id\":\"111\",\"method\":\"" + + methodName + + "\"}," + + "{\"jsonrpc\":\"2.0\",\"id\":\"222\",\"method\":\"net_version\"}]", + JSON); + + try (final Response resp = client.newCall(buildPostRequest(body)).execute()) { + assertThat(resp.code()).isEqualTo(200); + final JsonArray array = new JsonArray(resp.body().string()); + testHelper.assertValidJsonRpcResult(array.getJsonObject(0), "000"); + final RpcErrorType expectedError = RpcErrorType.INTERNAL_ERROR; + testHelper.assertValidJsonRpcError( + array.getJsonObject(1), "111", expectedError.getCode(), expectedError.getMessage()); + testHelper.assertValidJsonRpcResult(array.getJsonObject(2), "222"); + } } } diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTestBase.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTestBase.java index 580af7b74..257dbf166 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTestBase.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTestBase.java @@ -16,7 +16,6 @@ package org.hyperledger.besu.ethereum.api.jsonrpc; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import org.hyperledger.besu.config.StubGenesisConfigOptions; import org.hyperledger.besu.ethereum.ProtocolContext; @@ -72,8 +71,9 @@ public class JsonRpcHttpServiceTestBase { protected final JsonRpcTestHelper testHelper = new JsonRpcTestHelper(); private static final Vertx vertx = Vertx.vertx(); - protected static Map rpcMethods; + private static Map disabledRpcMethods; + private static Set addedRpcMethods; protected static JsonRpcHttpService service; protected static OkHttpClient client; protected static String baseUrl; @@ -106,39 +106,41 @@ public class JsonRpcHttpServiceTestBase { supportedCapabilities.add(EthProtocol.ETH63); rpcMethods = - spy( - new JsonRpcMethodsFactory() - .methods( - CLIENT_VERSION, - CHAIN_ID, - new StubGenesisConfigOptions(), - peerDiscoveryMock, - blockchainQueries, - synchronizer, - MainnetProtocolSchedule.fromConfig( - new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID), - EvmConfiguration.DEFAULT), - mock(ProtocolContext.class), - mock(FilterManager.class), - mock(TransactionPool.class), - mock(MiningParameters.class), - mock(PoWMiningCoordinator.class), - new NoOpMetricsSystem(), - supportedCapabilities, - Optional.of(mock(AccountLocalConfigPermissioningController.class)), - Optional.of(mock(NodeLocalConfigPermissioningController.class)), - JSON_RPC_APIS, - mock(PrivacyParameters.class), - mock(JsonRpcConfiguration.class), - mock(WebSocketConfiguration.class), - mock(MetricsConfiguration.class), - natService, - new HashMap<>(), - folder, - ethPeersMock, - vertx, - mock(ApiConfiguration.class), - Optional.empty())); + new JsonRpcMethodsFactory() + .methods( + CLIENT_VERSION, + CHAIN_ID, + new StubGenesisConfigOptions(), + peerDiscoveryMock, + blockchainQueries, + synchronizer, + MainnetProtocolSchedule.fromConfig( + new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID), + EvmConfiguration.DEFAULT), + mock(ProtocolContext.class), + mock(FilterManager.class), + mock(TransactionPool.class), + mock(MiningParameters.class), + mock(PoWMiningCoordinator.class), + new NoOpMetricsSystem(), + supportedCapabilities, + Optional.of(mock(AccountLocalConfigPermissioningController.class)), + Optional.of(mock(NodeLocalConfigPermissioningController.class)), + JSON_RPC_APIS, + mock(PrivacyParameters.class), + mock(JsonRpcConfiguration.class), + mock(WebSocketConfiguration.class), + mock(MetricsConfiguration.class), + natService, + new HashMap<>(), + folder, + ethPeersMock, + vertx, + mock(ApiConfiguration.class), + Optional.empty()); + disabledRpcMethods = new HashMap<>(); + addedRpcMethods = new HashSet<>(); + service = createJsonRpcHttpService(createLimitedJsonRpcConfig()); service.start().join(); @@ -189,6 +191,22 @@ public class JsonRpcHttpServiceTestBase { return new Request.Builder().get().url(baseUrl + path).build(); } + protected AutoCloseable disableRpcMethod(final String methodName) { + disabledRpcMethods.put(methodName, rpcMethods.remove(methodName)); + return () -> resetRpcMethods(); + } + + protected AutoCloseable addRpcMethod(final String methodName, final JsonRpcMethod method) { + rpcMethods.put(methodName, method); + addedRpcMethods.add(methodName); + return () -> resetRpcMethods(); + } + + protected void resetRpcMethods() { + disabledRpcMethods.forEach(rpcMethods::put); + addedRpcMethods.forEach(rpcMethods::remove); + } + /** Tears down the HTTP server. */ @AfterAll public static void shutdownServer() { diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsClientAuthTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsClientAuthTest.java index 3a9a5577a..70cc3d58f 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsClientAuthTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsClientAuthTest.java @@ -21,7 +21,6 @@ import static org.hyperledger.besu.ethereum.api.tls.KnownClientFileUtil.writeToK import static org.hyperledger.besu.ethereum.api.tls.TlsClientAuthConfiguration.Builder.aTlsClientAuthConfiguration; import static org.hyperledger.besu.ethereum.api.tls.TlsConfiguration.Builder.aTlsConfiguration; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import org.hyperledger.besu.config.StubGenesisConfigOptions; import org.hyperledger.besu.ethereum.ProtocolContext; @@ -112,38 +111,37 @@ public class JsonRpcHttpServiceTlsClientAuthTest { supportedCapabilities.add(EthProtocol.ETH63); rpcMethods = - spy( - new JsonRpcMethodsFactory() - .methods( - CLIENT_VERSION, - CHAIN_ID, - new StubGenesisConfigOptions(), - peerDiscoveryMock, - blockchainQueries, - synchronizer, - MainnetProtocolSchedule.fromConfig( - new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID)), - mock(ProtocolContext.class), - mock(FilterManager.class), - mock(TransactionPool.class), - mock(MiningParameters.class), - mock(PoWMiningCoordinator.class), - new NoOpMetricsSystem(), - supportedCapabilities, - Optional.of(mock(AccountLocalConfigPermissioningController.class)), - Optional.of(mock(NodeLocalConfigPermissioningController.class)), - DEFAULT_RPC_APIS, - mock(PrivacyParameters.class), - mock(JsonRpcConfiguration.class), - mock(WebSocketConfiguration.class), - mock(MetricsConfiguration.class), - natService, - Collections.emptyMap(), - folder, - mock(EthPeers.class), - vertx, - mock(ApiConfiguration.class), - Optional.empty())); + new JsonRpcMethodsFactory() + .methods( + CLIENT_VERSION, + CHAIN_ID, + new StubGenesisConfigOptions(), + peerDiscoveryMock, + blockchainQueries, + synchronizer, + MainnetProtocolSchedule.fromConfig( + new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID)), + mock(ProtocolContext.class), + mock(FilterManager.class), + mock(TransactionPool.class), + mock(MiningParameters.class), + mock(PoWMiningCoordinator.class), + new NoOpMetricsSystem(), + supportedCapabilities, + Optional.of(mock(AccountLocalConfigPermissioningController.class)), + Optional.of(mock(NodeLocalConfigPermissioningController.class)), + DEFAULT_RPC_APIS, + mock(PrivacyParameters.class), + mock(JsonRpcConfiguration.class), + mock(WebSocketConfiguration.class), + mock(MetricsConfiguration.class), + natService, + Collections.emptyMap(), + folder, + mock(EthPeers.class), + vertx, + mock(ApiConfiguration.class), + Optional.empty()); System.setProperty("javax.net.ssl.trustStore", CLIENT_AS_CA_CERT.getKeyStoreFile().toString()); System.setProperty( diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsMisconfigurationTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsMisconfigurationTest.java index 695dec932..ee069f032 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsMisconfigurationTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsMisconfigurationTest.java @@ -20,7 +20,6 @@ import static org.hyperledger.besu.ethereum.api.tls.KnownClientFileUtil.writeToK import static org.hyperledger.besu.ethereum.api.tls.TlsClientAuthConfiguration.Builder.aTlsClientAuthConfiguration; import static org.hyperledger.besu.ethereum.api.tls.TlsConfiguration.Builder.aTlsConfiguration; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import org.hyperledger.besu.config.StubGenesisConfigOptions; import org.hyperledger.besu.ethereum.ProtocolContext; @@ -100,38 +99,37 @@ class JsonRpcHttpServiceTlsMisconfigurationTest { supportedCapabilities.add(EthProtocol.ETH63); rpcMethods = - spy( - new JsonRpcMethodsFactory() - .methods( - CLIENT_VERSION, - CHAIN_ID, - new StubGenesisConfigOptions(), - peerDiscoveryMock, - blockchainQueries, - synchronizer, - MainnetProtocolSchedule.fromConfig( - new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID)), - mock(ProtocolContext.class), - mock(FilterManager.class), - mock(TransactionPool.class), - mock(MiningParameters.class), - mock(PoWMiningCoordinator.class), - new NoOpMetricsSystem(), - supportedCapabilities, - Optional.of(mock(AccountLocalConfigPermissioningController.class)), - Optional.of(mock(NodeLocalConfigPermissioningController.class)), - DEFAULT_RPC_APIS, - mock(PrivacyParameters.class), - mock(JsonRpcConfiguration.class), - mock(WebSocketConfiguration.class), - mock(MetricsConfiguration.class), - natService, - Collections.emptyMap(), - tempDir.getRoot(), - mock(EthPeers.class), - vertx, - mock(ApiConfiguration.class), - Optional.empty())); + new JsonRpcMethodsFactory() + .methods( + CLIENT_VERSION, + CHAIN_ID, + new StubGenesisConfigOptions(), + peerDiscoveryMock, + blockchainQueries, + synchronizer, + MainnetProtocolSchedule.fromConfig( + new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID)), + mock(ProtocolContext.class), + mock(FilterManager.class), + mock(TransactionPool.class), + mock(MiningParameters.class), + mock(PoWMiningCoordinator.class), + new NoOpMetricsSystem(), + supportedCapabilities, + Optional.of(mock(AccountLocalConfigPermissioningController.class)), + Optional.of(mock(NodeLocalConfigPermissioningController.class)), + DEFAULT_RPC_APIS, + mock(PrivacyParameters.class), + mock(JsonRpcConfiguration.class), + mock(WebSocketConfiguration.class), + mock(MetricsConfiguration.class), + natService, + Collections.emptyMap(), + tempDir.getRoot(), + mock(EthPeers.class), + vertx, + mock(ApiConfiguration.class), + Optional.empty()); } @AfterEach diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsTest.java index 587d142b7..295c7b91b 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsTest.java @@ -20,7 +20,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.DEFAULT_RPC_APIS; import static org.hyperledger.besu.ethereum.api.tls.TlsConfiguration.Builder.aTlsConfiguration; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import org.hyperledger.besu.config.StubGenesisConfigOptions; import org.hyperledger.besu.ethereum.ProtocolContext; @@ -101,38 +100,37 @@ public class JsonRpcHttpServiceTlsTest { supportedCapabilities.add(EthProtocol.ETH63); rpcMethods = - spy( - new JsonRpcMethodsFactory() - .methods( - CLIENT_VERSION, - CHAIN_ID, - new StubGenesisConfigOptions(), - peerDiscoveryMock, - blockchainQueries, - synchronizer, - MainnetProtocolSchedule.fromConfig( - new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID)), - mock(ProtocolContext.class), - mock(FilterManager.class), - mock(TransactionPool.class), - mock(MiningParameters.class), - mock(PoWMiningCoordinator.class), - new NoOpMetricsSystem(), - supportedCapabilities, - Optional.of(mock(AccountLocalConfigPermissioningController.class)), - Optional.of(mock(NodeLocalConfigPermissioningController.class)), - DEFAULT_RPC_APIS, - mock(PrivacyParameters.class), - mock(JsonRpcConfiguration.class), - mock(WebSocketConfiguration.class), - mock(MetricsConfiguration.class), - natService, - Collections.emptyMap(), - folder, - mock(EthPeers.class), - vertx, - mock(ApiConfiguration.class), - Optional.empty())); + new JsonRpcMethodsFactory() + .methods( + CLIENT_VERSION, + CHAIN_ID, + new StubGenesisConfigOptions(), + peerDiscoveryMock, + blockchainQueries, + synchronizer, + MainnetProtocolSchedule.fromConfig( + new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID)), + mock(ProtocolContext.class), + mock(FilterManager.class), + mock(TransactionPool.class), + mock(MiningParameters.class), + mock(PoWMiningCoordinator.class), + new NoOpMetricsSystem(), + supportedCapabilities, + Optional.of(mock(AccountLocalConfigPermissioningController.class)), + Optional.of(mock(NodeLocalConfigPermissioningController.class)), + DEFAULT_RPC_APIS, + mock(PrivacyParameters.class), + mock(JsonRpcConfiguration.class), + mock(WebSocketConfiguration.class), + mock(MetricsConfiguration.class), + natService, + Collections.emptyMap(), + folder, + mock(EthPeers.class), + vertx, + mock(ApiConfiguration.class), + Optional.empty()); service = createJsonRpcHttpService(createJsonRpcConfig()); service.start().join(); baseUrl = service.url(); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBlockToFileTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBlockToFileTest.java index 42ec4c697..2cf7dcbfc 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBlockToFileTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBlockToFileTest.java @@ -18,11 +18,12 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTracer; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; @@ -30,28 +31,23 @@ import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockDataGenerator; import org.hyperledger.besu.ethereum.core.MutableWorldState; -import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import java.nio.file.Path; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Optional; +import java.util.function.Function; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import org.mockito.Answers; public class DebugStandardTraceBlockToFileTest { // this tempDir is deliberately static @TempDir private static Path folder; - private final WorldStateArchive archive = - mock(WorldStateArchive.class, Answers.RETURNS_DEEP_STUBS); private final Blockchain blockchain = mock(Blockchain.class); - private final BlockchainQueries blockchainQueries = - spy(new BlockchainQueries(blockchain, archive)); + private final BlockchainQueries blockchainQueries = mock(BlockchainQueries.class); private final TransactionTracer transactionTracer = mock(TransactionTracer.class); private final DebugStandardTraceBlockToFile debugStandardTraceBlockToFile = new DebugStandardTraceBlockToFile(() -> transactionTracer, blockchainQueries, folder); @@ -76,20 +72,26 @@ public class DebugStandardTraceBlockToFileTest { new JsonRpcRequestContext( new JsonRpcRequest("2.0", "debug_standardTraceBlockToFile", params)); - final List paths = new ArrayList<>(); - paths.add("path-1"); - - when(blockchainQueries.getBlockchain()).thenReturn(blockchain); + final List paths = List.of("path-1"); when(blockchain.getBlockByHash(block.getHash())).thenReturn(Optional.of(block)); when(blockchain.getBlockHeader(genesis.getHash())).thenReturn(Optional.of(genesis.getHeader())); + when(blockchainQueries.getBlockchain()).thenReturn(blockchain); + + when(blockchainQueries.getAndMapWorldState(any(), any())) + .thenAnswer( + invocationOnMock -> { + Function> mapper = + invocationOnMock.getArgument(1); + return mapper.apply(mock(Tracer.TraceableState.class)); + }); when(transactionTracer.traceTransactionToFile( any(MutableWorldState.class), eq(block.getHash()), any(), any())) .thenReturn(paths); final JsonRpcSuccessResponse response = (JsonRpcSuccessResponse) debugStandardTraceBlockToFile.response(request); - final List result = (ArrayList) response.getResult(); + final List result = (List) response.getResult(); assertThat(result.size()).isEqualTo(1); } diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockTest.java index 60841c24e..5da556182 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockTest.java @@ -18,9 +18,8 @@ import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doAnswer; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.Wei; @@ -35,32 +34,25 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSucces import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.RpcErrorType; import org.hyperledger.besu.ethereum.api.query.BlockWithMetadata; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; -import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockDataGenerator; import org.hyperledger.besu.ethereum.debug.TraceFrame; import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; -import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import java.util.Collection; import java.util.Collections; import java.util.Optional; import java.util.OptionalLong; +import java.util.function.Function; import org.apache.tuweni.bytes.Bytes; import org.junit.jupiter.api.Test; -import org.mockito.Answers; -import org.mockito.Mockito; public class DebugTraceBlockTest { private final BlockTracer blockTracer = mock(BlockTracer.class); - private final WorldStateArchive archive = - mock(WorldStateArchive.class, Answers.RETURNS_DEEP_STUBS); - private final Blockchain blockchain = mock(Blockchain.class); - private final BlockchainQueries blockchainQueries = - spy(new BlockchainQueries(blockchain, archive)); + private final BlockchainQueries blockchainQueries = mock(BlockchainQueries.class); private final DebugTraceBlock debugTraceBlock = new DebugTraceBlock(() -> blockTracer, new MainnetBlockHeaderFunctions(), blockchainQueries); @@ -127,22 +119,25 @@ public class DebugTraceBlockTest { when(transaction2Trace.getResult()).thenReturn(transaction2Result); when(transaction1Result.getOutput()).thenReturn(Bytes.fromHexString("1234")); when(transaction2Result.getOutput()).thenReturn(Bytes.fromHexString("1234")); - when(blockTracer.trace(any(Tracer.TraceableState.class), Mockito.eq(block), any())) + when(blockTracer.trace(any(Tracer.TraceableState.class), eq(block), any())) .thenReturn(Optional.of(blockTrace)); - when(blockchain.getBlockHeader(parentBlock.getHash())) - .thenReturn(Optional.of(parentBlock.getHeader())); - doAnswer( - invocation -> - Optional.of( - new BlockWithMetadata<>( - parentBlock.getHeader(), - Collections.emptyList(), - Collections.emptyList(), - parentBlock.getHeader().getDifficulty(), - parentBlock.calculateSize()))) - .when(blockchainQueries) - .blockByHash(parentBlock.getHash()); + when(blockchainQueries.blockByHash(parentBlock.getHash())) + .thenReturn( + Optional.of( + new BlockWithMetadata<>( + parentBlock.getHeader(), + Collections.emptyList(), + Collections.emptyList(), + parentBlock.getHeader().getDifficulty(), + parentBlock.calculateSize()))); + when(blockchainQueries.getAndMapWorldState(eq(parentBlock.getHash()), any())) + .thenAnswer( + invocationOnMock -> { + Function> mapper = + invocationOnMock.getArgument(1); + return mapper.apply(mock(Tracer.TraceableState.class)); + }); final JsonRpcSuccessResponse response = (JsonRpcSuccessResponse) debugTraceBlock.response(request); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java index 8a1a0a226..83355ecc2 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java @@ -165,7 +165,7 @@ public class BackwardSyncAlgSpec { ttdCaptor.getValue().onTTDReached(true); - voidCompletableFuture.get(100, TimeUnit.MILLISECONDS); + voidCompletableFuture.get(200, TimeUnit.MILLISECONDS); assertThat(voidCompletableFuture).isCompleted(); verify(context.getSyncState()).unsubscribeTTDReached(88L); @@ -192,7 +192,7 @@ public class BackwardSyncAlgSpec { completionCaptor.getValue().onInitialSyncCompleted(); - voidCompletableFuture.get(100, TimeUnit.MILLISECONDS); + voidCompletableFuture.get(200, TimeUnit.MILLISECONDS); assertThat(voidCompletableFuture).isCompleted(); verify(context.getSyncState()).unsubscribeTTDReached(88L); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java index dfafb002f..f5030fd9a 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java @@ -65,6 +65,7 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelId; import io.netty.channel.ChannelPipeline; @@ -196,7 +197,7 @@ public class DeFramerTest { assertThat(out).isEmpty(); // Next phase of pipeline should be setup - verify(pipeline, times(1)).addLast(any()); + verify(pipeline, times(1)).addLast(any(ChannelHandler[].class)); // Next message should be pushed out final PingMessage nextMessage = PingMessage.get(); @@ -204,7 +205,7 @@ public class DeFramerTest { when(framer.deframe(eq(nextData))) .thenReturn(new RawMessage(nextMessage.getCode(), nextMessage.getData())) .thenReturn(null); - verify(pipeline, times(1)).addLast(any()); + verify(pipeline, times(1)).addLast(any(ChannelHandler[].class)); deFramer.decode(ctx, nextData, out); assertThat(out.size()).isEqualTo(1); } @@ -246,7 +247,7 @@ public class DeFramerTest { assertThat(peerConnection.getPeer().getEnodeURL()).isEqualTo(expectedEnode); // Next phase of pipeline should be setup - verify(pipeline, times(1)).addLast(any()); + verify(pipeline, times(1)).addLast(any(ChannelHandler[].class)); // Next message should be pushed out final PingMessage nextMessage = PingMessage.get(); @@ -254,7 +255,7 @@ public class DeFramerTest { when(framer.deframe(eq(nextData))) .thenReturn(new RawMessage(nextMessage.getCode(), nextMessage.getData())) .thenReturn(null); - verify(pipeline, times(1)).addLast(any()); + verify(pipeline, times(1)).addLast(any(ChannelHandler[].class)); deFramer.decode(ctx, nextData, out); assertThat(out.size()).isEqualTo(1); } @@ -292,7 +293,7 @@ public class DeFramerTest { assertThat(out).isEmpty(); // Next phase of pipeline should be setup - verify(pipeline, times(1)).addLast(any()); + verify(pipeline, times(1)).addLast(any(ChannelHandler[].class)); } @Test @@ -321,7 +322,7 @@ public class DeFramerTest { assertThat(out).isEmpty(); // Next phase of pipeline should be setup - verify(pipeline, times(1)).addLast(any()); + verify(pipeline, times(1)).addLast(any(ChannelHandler[].class)); } @Test diff --git a/evm/src/test/java/org/hyperledger/besu/evm/operations/ChainIdOperationTest.java b/evm/src/test/java/org/hyperledger/besu/evm/operations/ChainIdOperationTest.java index 7202be03c..e8d2c3d02 100644 --- a/evm/src/test/java/org/hyperledger/besu/evm/operations/ChainIdOperationTest.java +++ b/evm/src/test/java/org/hyperledger/besu/evm/operations/ChainIdOperationTest.java @@ -27,7 +27,6 @@ import java.util.List; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.units.bigints.UInt256; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -51,7 +50,7 @@ class ChainIdOperationTest { void shouldReturnChainId(final String chainIdString, final int expectedGas) { Bytes32 chainId = Bytes32.fromHexString(chainIdString); ChainIdOperation operation = new ChainIdOperation(new ConstantinopleGasCalculator(), chainId); - final ArgumentCaptor arg = ArgumentCaptor.forClass(UInt256.class); + final ArgumentCaptor arg = ArgumentCaptor.forClass(Bytes.class); when(messageFrame.getRemainingGas()).thenReturn(100L); operation.execute(messageFrame, null); Mockito.verify(messageFrame).getRemainingGas(); diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 58a562fff..906d4a2ee 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2858,12 +2858,12 @@ - - - + + + - - + + @@ -2874,17 +2874,17 @@ - - - + + + - - + + - - - + + + @@ -5316,20 +5316,20 @@ - - - + + + - - + + - - - + + + - - + + diff --git a/gradle/versions.gradle b/gradle/versions.gradle index c27af91f0..747a386cb 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -192,7 +192,7 @@ dependencyManagement { entry 'org.jupnp' } - dependencySet(group: 'org.mockito', version:'4.11.0') { + dependencySet(group: 'org.mockito', version:'5.8.0') { entry 'mockito-core' entry 'mockito-junit-jupiter' } From 2c1d3d28410d6b0f3f55b67e0f2ad8cb7e111d5b Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Tue, 16 Jan 2024 10:53:41 +1000 Subject: [PATCH 06/20] cliqueBft AT task flakiness - extend no_output_timeout (#6406) * revert machine change and extend timeout instead Signed-off-by: Sally MacFarlane --------- Signed-off-by: Sally MacFarlane --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6d08e515a..065a0280d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -240,7 +240,7 @@ jobs: at: ~/project - run: name: AcceptanceTests (Non-Mainnet) - no_output_timeout: 20m + no_output_timeout: 30m command: | ./gradlew --no-daemon --max-workers=1 acceptanceTestCliqueBft - capture_test_results From c316a6dfb5ea354bad50194840362c6622910b0e Mon Sep 17 00:00:00 2001 From: Gabriel-Trintinalia Date: Tue, 16 Jan 2024 13:53:55 +1100 Subject: [PATCH 07/20] Remove deprecated `--privacy-onchain-groups-enabled` option (#6411) Signed-off-by: Gabriel-Trintinalia --- CHANGELOG.md | 1 + .../org/hyperledger/besu/cli/BesuCommand.java | 21 +------- .../util/ConfigOptionSearchAndRunHandler.java | 11 ---- .../hyperledger/besu/cli/BesuCommandTest.java | 51 ------------------- .../src/test/resources/everything_config.toml | 1 - 5 files changed, 3 insertions(+), 82 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d520eba54..504d120a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Breaking Changes - New `EXECUTION_HALTED` error returned if there is an error executing or simulating a transaction, with the reason for execution being halted. Replaces the generic `INTERNAL_ERROR` return code in certain cases which some applications may be checking for [#6343](https://github.com/hyperledger/besu/pull/6343) - The Besu Docker images with `openjdk-latest` tags since 23.10.3 were incorrectly using UID 1001 instead of 1000 for the container's `besu` user. The user now uses 1000 again. Containers created from or migrated to images using UID 1001 will need to chown their persistent database files to UID 1000 [#6360](https://github.com/hyperledger/besu/pull/6360) +- The deprecated `--privacy-onchain-groups-enabled` option has now been removed. Use the `--privacy-flexible-groups-enabled` option instead. [#6411](https://github.com/hyperledger/besu/pull/6411) ### Deprecations diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index d2fb52433..c51d03d43 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -22,7 +22,6 @@ import static java.util.Collections.singletonList; import static org.hyperledger.besu.cli.DefaultCommandValues.getDefaultBesuDataPath; import static org.hyperledger.besu.cli.config.NetworkName.MAINNET; import static org.hyperledger.besu.cli.util.CommandLineUtils.DEPENDENCY_WARNING_MSG; -import static org.hyperledger.besu.cli.util.CommandLineUtils.DEPRECATION_WARNING_MSG; import static org.hyperledger.besu.cli.util.CommandLineUtils.isOptionSet; import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH; import static org.hyperledger.besu.ethereum.api.graphql.GraphQLConfiguration.DEFAULT_GRAPHQL_HTTP_PORT; @@ -953,13 +952,6 @@ public class BesuCommand implements DefaultCommandValues, Runnable { names = {"--privacy-flexible-groups-enabled"}, description = "Enable flexible privacy groups (default: ${DEFAULT-VALUE})") private final Boolean isFlexiblePrivacyGroupsEnabled = false; - - @Option( - hidden = true, - names = {"--privacy-onchain-groups-enabled"}, - description = - "!!DEPRECATED!! Use `--privacy-flexible-groups-enabled` instead. Enable flexible (onchain) privacy groups (default: ${DEFAULT-VALUE})") - private final Boolean isOnchainPrivacyGroupsEnabled = false; } // Metrics Option Group @@ -1716,8 +1708,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable { } if (unstablePrivacyPluginOptions.isPrivacyPluginEnabled() - && (privacyOptionGroup.isFlexiblePrivacyGroupsEnabled - || privacyOptionGroup.isOnchainPrivacyGroupsEnabled)) { + && privacyOptionGroup.isFlexiblePrivacyGroupsEnabled) { throw new ParameterException( commandLine, "Privacy Plugin can not be used with flexible privacy groups"); } @@ -2056,13 +2047,6 @@ public class BesuCommand implements DefaultCommandValues, Runnable { "--security-module=" + DEFAULT_SECURITY_MODULE); } - if (Boolean.TRUE.equals(privacyOptionGroup.isOnchainPrivacyGroupsEnabled)) { - logger.warn( - DEPRECATION_WARNING_MSG, - "--privacy-onchain-groups-enabled", - "--privacy-flexible-groups-enabled"); - } - if (isPruningEnabled()) { if (dataStorageOptions .toDomainObject() @@ -2750,8 +2734,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable { privacyParametersBuilder.setMultiTenancyEnabled( privacyOptionGroup.isPrivacyMultiTenancyEnabled); privacyParametersBuilder.setFlexiblePrivacyGroupsEnabled( - privacyOptionGroup.isFlexiblePrivacyGroupsEnabled - || privacyOptionGroup.isOnchainPrivacyGroupsEnabled); + privacyOptionGroup.isFlexiblePrivacyGroupsEnabled); privacyParametersBuilder.setPrivacyPluginEnabled( unstablePrivacyPluginOptions.isPrivacyPluginEnabled()); diff --git a/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandler.java b/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandler.java index 17cf2e649..5866d43cd 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandler.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandler.java @@ -55,7 +55,6 @@ public class ConfigOptionSearchAndRunHandler extends CommandLine.RunLast { public List handle(final ParseResult parseResult) throws ParameterException { final CommandLine commandLine = parseResult.commandSpec().commandLine(); final Optional configFile = findConfigFile(parseResult, commandLine); - validatePrivacyOptions(parseResult, commandLine); commandLine.setDefaultValueProvider(createDefaultValueProvider(commandLine, configFile)); commandLine.setExecutionStrategy(resultHandler); commandLine.setParameterExceptionHandler(parameterExceptionHandler); @@ -64,16 +63,6 @@ public class ConfigOptionSearchAndRunHandler extends CommandLine.RunLast { return new ArrayList<>(); } - private void validatePrivacyOptions( - final ParseResult parseResult, final CommandLine commandLine) { - if (parseResult.hasMatchedOption("--privacy-onchain-groups-enabled") - && parseResult.hasMatchedOption("--privacy-flexible-groups-enabled")) { - throw new ParameterException( - commandLine, - "The `--privacy-onchain-groups-enabled` option is deprecated and you should only use `--privacy-flexible-groups-enabled`"); - } - } - private Optional findConfigFile( final ParseResult parseResult, final CommandLine commandLine) { if (parseResult.hasMatchedOption("--config-file") diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index dc56c3484..262965563 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -29,7 +29,6 @@ import static org.hyperledger.besu.cli.config.NetworkName.MAINNET; import static org.hyperledger.besu.cli.config.NetworkName.MORDOR; import static org.hyperledger.besu.cli.config.NetworkName.SEPOLIA; import static org.hyperledger.besu.cli.util.CommandLineUtils.DEPENDENCY_WARNING_MSG; -import static org.hyperledger.besu.cli.util.CommandLineUtils.DEPRECATION_WARNING_MSG; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.ENGINE; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.ETH; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.NET; @@ -1977,16 +1976,6 @@ public class BesuCommandTest extends CommandTestAbstract { "The `--ethstats-contact` requires ethstats server URL to be provided. Either remove --ethstats-contact or provide a URL (via --ethstats=nodename:secret@host:port)"); } - @Test - public void privacyOnchainGroupsEnabledCannotBeUsedWithPrivacyFlexibleGroupsEnabled() { - parseCommand("--privacy-onchain-groups-enabled", "--privacy-flexible-groups-enabled"); - Mockito.verifyNoInteractions(mockRunnerBuilder); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains( - "The `--privacy-onchain-groups-enabled` option is deprecated and you should only use `--privacy-flexible-groups-enabled`"); - } - @Test public void parsesValidBonsaiTrieLimitBackLayersOption() { parseCommand("--data-storage-format", "BONSAI", "--bonsai-historical-block-limit", "11"); @@ -4203,46 +4192,6 @@ public class BesuCommandTest extends CommandTestAbstract { assertThat(privacyParameters.isFlexiblePrivacyGroupsEnabled()).isEqualTo(false); } - @Test - public void onchainPrivacyGroupEnabledFlagValueIsSet() { - parseCommand( - "--privacy-enabled", - "--privacy-public-key-file", - ENCLAVE_PUBLIC_KEY_PATH, - "--privacy-onchain-groups-enabled", - "--min-gas-price", - "0"); - - final ArgumentCaptor privacyParametersArgumentCaptor = - ArgumentCaptor.forClass(PrivacyParameters.class); - - verify(mockControllerBuilder).privacyParameters(privacyParametersArgumentCaptor.capture()); - verify(mockControllerBuilder).build(); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - - final PrivacyParameters privacyParameters = privacyParametersArgumentCaptor.getValue(); - assertThat(privacyParameters.isFlexiblePrivacyGroupsEnabled()).isEqualTo(true); - } - - @Test - public void onchainPrivacyGroupEnabledOptionIsDeprecated() { - parseCommand( - "--privacy-enabled", - "--privacy-public-key-file", - ENCLAVE_PUBLIC_KEY_PATH, - "--privacy-onchain-groups-enabled", - "--min-gas-price", - "0"); - - verify(mockLogger) - .warn( - DEPRECATION_WARNING_MSG, - "--privacy-onchain-groups-enabled", - "--privacy-flexible-groups-enabled"); - } - @Test public void flexiblePrivacyGroupEnabledFlagValueIsSet() { parseCommand( diff --git a/besu/src/test/resources/everything_config.toml b/besu/src/test/resources/everything_config.toml index e516060da..c653f3f60 100644 --- a/besu/src/test/resources/everything_config.toml +++ b/besu/src/test/resources/everything_config.toml @@ -169,7 +169,6 @@ privacy-enabled=false privacy-multi-tenancy-enabled=true privacy-marker-transaction-signing-key-file="./signerKey" privacy-enable-database-migration=false -privacy-onchain-groups-enabled=false privacy-flexible-groups-enabled=false # Transaction Pool From 25f8e57a36b673a44953f63cc2e0f400e08be444 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Wed, 17 Jan 2024 10:06:06 +0100 Subject: [PATCH 08/20] Upgrade tech.pegasys.discovery:discovery (#6414) Signed-off-by: Fabio Di Fabio --- CHANGELOG.md | 1 + .../internal/ENRResponsePacketDataTest.java | 48 +++++++---- gradle/allowed-licenses.json | 5 ++ gradle/verification-metadata.xml | 86 ++++++++++--------- gradle/versions.gradle | 2 +- 5 files changed, 87 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 504d120a9..e09db4de5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - Upgrade `com.fasterxml.jackson` dependencies [#6378](https://github.com/hyperledger/besu/pull/6378) - Upgrade Guava dependency [#6396](https://github.com/hyperledger/besu/pull/6396) - Upgrade Mockito [#6397](https://github.com/hyperledger/besu/pull/6397) +- Upgrade `tech.pegasys.discovery:discovery` [#6414](https://github.com/hyperledger/besu/pull/6414) ### Bug fixes - INTERNAL_ERROR from `eth_estimateGas` JSON/RPC calls [#6344](https://github.com/hyperledger/besu/issues/6344) diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/ENRResponsePacketDataTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/ENRResponsePacketDataTest.java index 72bb407d1..7ba47bd97 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/ENRResponsePacketDataTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/ENRResponsePacketDataTest.java @@ -20,6 +20,8 @@ import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.rlp.RLP; import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.crypto.SECP256K1; import org.apache.tuweni.units.bigints.UInt64; import org.ethereum.beacon.discovery.schema.EnrField; import org.ethereum.beacon.discovery.schema.IdentitySchema; @@ -34,8 +36,10 @@ public class ENRResponsePacketDataTest { final Bytes requestHash = Bytes.fromHexStringLenient("0x1234"); final Bytes nodeId = Bytes.fromHexString("a448f24c6d18e575453db13171562b71999873db5b286df957af199ec94617f7"); - final Bytes privateKey = - Bytes.fromHexString("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291"); + final SECP256K1.SecretKey privateKey = + SECP256K1.SecretKey.fromBytes( + Bytes32.fromHexString( + "b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")); NodeRecord nodeRecord = NodeRecordFactory.DEFAULT.createFromValues( @@ -48,7 +52,8 @@ public class ENRResponsePacketDataTest { new EnrField(EnrField.TCP, 8080), new EnrField(EnrField.TCP_V6, 8080), new EnrField( - EnrField.PKEY_SECP256K1, Functions.derivePublicKeyFromPrivate(privateKey))); + EnrField.PKEY_SECP256K1, + Functions.deriveCompressedPublicKeyFromPrivate(privateKey))); nodeRecord.sign(privateKey); assertThat(nodeRecord.getNodeId()).isEqualTo(nodeId); @@ -72,8 +77,10 @@ public class ENRResponsePacketDataTest { final Bytes requestHash = Bytes.fromHexStringLenient("0x1234"); final Bytes nodeId = Bytes.fromHexString("a448f24c6d18e575453db13171562b71999873db5b286df957af199ec94617f7"); - final Bytes privateKey = - Bytes.fromHexString("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291"); + final SECP256K1.SecretKey privateKey = + SECP256K1.SecretKey.fromBytes( + Bytes32.fromHexString( + "b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")); NodeRecord nodeRecord = NodeRecordFactory.DEFAULT.createFromValues( @@ -82,7 +89,8 @@ public class ENRResponsePacketDataTest { new EnrField(EnrField.IP_V4, Bytes.fromHexString("0x7F000001")), new EnrField(EnrField.UDP, 30303), new EnrField( - EnrField.PKEY_SECP256K1, Functions.derivePublicKeyFromPrivate(privateKey))); + EnrField.PKEY_SECP256K1, + Functions.deriveCompressedPublicKeyFromPrivate(privateKey))); nodeRecord.sign(privateKey); assertThat(nodeRecord.getNodeId()).isEqualTo(nodeId); @@ -109,8 +117,10 @@ public class ENRResponsePacketDataTest { final Bytes requestHash = Bytes.fromHexStringLenient("0x1234"); final Bytes nodeId = Bytes.fromHexString("a448f24c6d18e575453db13171562b71999873db5b286df957af199ec94617f7"); - final Bytes privateKey = - Bytes.fromHexString("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291"); + final SECP256K1.SecretKey privateKey = + SECP256K1.SecretKey.fromBytes( + Bytes32.fromHexString( + "b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")); NodeRecord nodeRecord = NodeRecordFactory.DEFAULT.createFromValues( @@ -119,7 +129,8 @@ public class ENRResponsePacketDataTest { new EnrField(EnrField.IP_V4, Bytes.fromHexString("0x7F000001")), new EnrField(EnrField.UDP, 30303), new EnrField( - EnrField.PKEY_SECP256K1, Functions.derivePublicKeyFromPrivate(privateKey))); + EnrField.PKEY_SECP256K1, + Functions.deriveCompressedPublicKeyFromPrivate(privateKey))); nodeRecord.sign(privateKey); assertThat(nodeRecord.getNodeId()).isEqualTo(nodeId); @@ -144,8 +155,10 @@ public class ENRResponsePacketDataTest { final Bytes requestHash = Bytes.fromHexStringLenient("0x1234"); final Bytes nodeId = Bytes.fromHexString("a448f24c6d18e575453db13171562b71999873db5b286df957af199ec94617f7"); - final Bytes privateKey = - Bytes.fromHexString("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291"); + final SECP256K1.SecretKey privateKey = + SECP256K1.SecretKey.fromBytes( + Bytes32.fromHexString( + "b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")); NodeRecord nodeRecord = NodeRecordFactory.DEFAULT.createFromValues( @@ -153,7 +166,9 @@ public class ENRResponsePacketDataTest { new EnrField(EnrField.ID, IdentitySchema.V4), new EnrField(EnrField.IP_V4, Bytes.fromHexString("0x7F000001")), new EnrField(EnrField.UDP, 30303), - new EnrField(EnrField.PKEY_SECP256K1, Functions.derivePublicKeyFromPrivate(privateKey)), + new EnrField( + EnrField.PKEY_SECP256K1, + Functions.deriveCompressedPublicKeyFromPrivate(privateKey)), new EnrField("foo", Bytes.fromHexString("0x1234"))); nodeRecord.sign(privateKey); @@ -181,8 +196,10 @@ public class ENRResponsePacketDataTest { @Test public void readFrom_invalidSignature() { final Bytes requestHash = Bytes.fromHexStringLenient("0x1234"); - final Bytes privateKey = - Bytes.fromHexString("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f292"); + final SECP256K1.SecretKey privateKey = + SECP256K1.SecretKey.fromBytes( + Bytes32.fromHexString( + "b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f292")); NodeRecord nodeRecord = NodeRecordFactory.DEFAULT.createFromValues( @@ -191,7 +208,8 @@ public class ENRResponsePacketDataTest { new EnrField(EnrField.IP_V4, Bytes.fromHexString("0x7F000001")), new EnrField(EnrField.UDP, 30303), new EnrField( - EnrField.PKEY_SECP256K1, Functions.derivePublicKeyFromPrivate(privateKey))); + EnrField.PKEY_SECP256K1, + Functions.deriveCompressedPublicKeyFromPrivate(privateKey))); nodeRecord.sign(privateKey); nodeRecord.set(EnrField.UDP, 1234); diff --git a/gradle/allowed-licenses.json b/gradle/allowed-licenses.json index 252701908..f7a3d8be8 100644 --- a/gradle/allowed-licenses.json +++ b/gradle/allowed-licenses.json @@ -56,6 +56,11 @@ "moduleVersion": "1.0.3", "moduleName": "org.reactivestreams:reactive-streams" }, + { + "moduleLicense": "MIT-0", + "moduleVersion": "1.0.4", + "moduleName": "org.reactivestreams:reactive-streams" + }, { "moduleLicense": "Eclipse Public License - v 1.0", "moduleVersion": "4.13.2", diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 906d4a2ee..eb7949b75 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2402,15 +2402,15 @@ - - - + + + - - + + - - + + @@ -3359,44 +3359,44 @@ - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + @@ -5551,6 +5551,14 @@ + + + + + + + + @@ -5958,15 +5966,15 @@ - - - + + + - - + + - - + + diff --git a/gradle/versions.gradle b/gradle/versions.gradle index 747a386cb..27fd97fb4 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -231,6 +231,6 @@ dependencyManagement { dependency 'org.yaml:snakeyaml:2.0' - dependency 'tech.pegasys.discovery:discovery:22.2.0' + dependency 'tech.pegasys.discovery:discovery:22.12.0' } } From 5a3ed755c341abb7b40db4478e73a53371330b8d Mon Sep 17 00:00:00 2001 From: Simon Dudley Date: Thu, 18 Jan 2024 07:46:49 +1000 Subject: [PATCH 09/20] Downgrade rocksdbjni to 8.3.2 following FOREST bug (#6419) Signed-off-by: Simon Dudley --- gradle/verification-metadata.xml | 10 +++++----- gradle/versions.gradle | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index eb7949b75..673ac2df4 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -5567,12 +5567,12 @@ - - - + + + - - + + diff --git a/gradle/versions.gradle b/gradle/versions.gradle index 27fd97fb4..43fe774c4 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -204,7 +204,7 @@ dependencyManagement { dependency 'org.owasp.encoder:encoder:1.2.3' - dependency 'org.rocksdb:rocksdbjni:8.9.1' + dependency 'org.rocksdb:rocksdbjni:8.3.2' // 8.9.1 causes a bug with a FOREST canary dependencySet(group: 'org.slf4j', version:'2.0.10') { entry 'slf4j-api' From 2ba692e322e821023e13e224c9cc6f5f34042a2b Mon Sep 17 00:00:00 2001 From: Karim TAAM Date: Thu, 18 Jan 2024 10:01:28 +0100 Subject: [PATCH 10/20] Generate genesis root hash with the used data storage format (#6306) * add logic to use bonsai for genesis root hash calculation Signed-off-by: Karim Taam --- .../controller/BesuControllerBuilder.java | 4 +- .../besu/ethereum/chain/GenesisState.java | 59 +++++++++--- .../cache/NoOpCachedWorldStorageManager.java | 66 +++++++++++++ .../bonsai/trielog/NoOpTrieLogManager.java | 51 ++++++++++ .../bonsai/worldview/BonsaiWorldState.java | 2 +- .../common/GenesisWorldStateProvider.java | 89 ++++++++++++++++++ .../besu/ethereum/chain/GenesisStateTest.java | 67 +++++++++---- .../BonsaiReferenceTestWorldState.java | 93 ++----------------- 8 files changed, 311 insertions(+), 120 deletions(-) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 6bcfea3ca..f2d3fce98 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -591,7 +591,9 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides prepForBuild(); final ProtocolSchedule protocolSchedule = createProtocolSchedule(); - final GenesisState genesisState = GenesisState.fromConfig(genesisConfig, protocolSchedule); + final GenesisState genesisState = + GenesisState.fromConfig( + dataStorageConfiguration.getDataStorageFormat(), genesisConfig, protocolSchedule); final VariablesStorage variablesStorage = storageProvider.createVariablesStorage(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java index 87d45be81..14def049f 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.ethereum.chain; import static java.util.Collections.emptyList; +import static org.hyperledger.besu.ethereum.trie.common.GenesisWorldStateProvider.createGenesisWorldState; import org.hyperledger.besu.config.GenesisAllocation; import org.hyperledger.besu.config.GenesisConfigFile; @@ -32,14 +33,10 @@ import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.core.Withdrawal; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState; +import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.evm.account.MutableAccount; -import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.log.LogsBloomFilter; import org.hyperledger.besu.evm.worldstate.WorldUpdater; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.math.BigInteger; import java.util.HashMap; @@ -77,6 +74,21 @@ public final class GenesisState { return fromConfig(GenesisConfigFile.fromConfig(json), protocolSchedule); } + /** + * Construct a {@link GenesisState} from a JSON string. + * + * @param dataStorageFormat A {@link DataStorageFormat} describing the storage format to use + * @param json A JSON string describing the genesis block + * @param protocolSchedule A protocol Schedule associated with + * @return A new {@link GenesisState}. + */ + public static GenesisState fromJson( + final DataStorageFormat dataStorageFormat, + final String json, + final ProtocolSchedule protocolSchedule) { + return fromConfig(dataStorageFormat, GenesisConfigFile.fromConfig(json), protocolSchedule); + } + /** * Construct a {@link GenesisState} from a JSON object. * @@ -86,10 +98,28 @@ public final class GenesisState { */ public static GenesisState fromConfig( final GenesisConfigFile config, final ProtocolSchedule protocolSchedule) { + return fromConfig(DataStorageFormat.FOREST, config, protocolSchedule); + } + + /** + * Construct a {@link GenesisState} from a JSON object. + * + * @param dataStorageFormat A {@link DataStorageFormat} describing the storage format to use + * @param config A {@link GenesisConfigFile} describing the genesis block. + * @param protocolSchedule A protocol Schedule associated with + * @return A new {@link GenesisState}. + */ + public static GenesisState fromConfig( + final DataStorageFormat dataStorageFormat, + final GenesisConfigFile config, + final ProtocolSchedule protocolSchedule) { final List genesisAccounts = parseAllocations(config).toList(); final Block block = new Block( - buildHeader(config, calculateGenesisStateHash(genesisAccounts), protocolSchedule), + buildHeader( + config, + calculateGenesisStateHash(dataStorageFormat, genesisAccounts), + protocolSchedule), buildBody(config)); return new GenesisState(block, genesisAccounts); } @@ -133,15 +163,14 @@ public final class GenesisState { target.persist(rootHeader); } - private static Hash calculateGenesisStateHash(final List genesisAccounts) { - final ForestWorldStateKeyValueStorage stateStorage = - new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); - final WorldStatePreimageKeyValueStorage preimageStorage = - new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()); - final MutableWorldState worldState = - new ForestMutableWorldState(stateStorage, preimageStorage, EvmConfiguration.DEFAULT); - writeAccountsTo(worldState, genesisAccounts, null); - return worldState.rootHash(); + private static Hash calculateGenesisStateHash( + final DataStorageFormat dataStorageFormat, final List genesisAccounts) { + try (var worldState = createGenesisWorldState(dataStorageFormat)) { + writeAccountsTo(worldState, genesisAccounts, null); + return worldState.rootHash(); + } catch (Exception e) { + throw new RuntimeException(e); + } } private static BlockHeader buildHeader( diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java new file mode 100644 index 000000000..0359c4642 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java @@ -0,0 +1,66 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.bonsai.cache; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; + +import java.util.Optional; +import java.util.function.Function; + +public class NoOpCachedWorldStorageManager extends CachedWorldStorageManager { + + public NoOpCachedWorldStorageManager( + final BonsaiWorldStateKeyValueStorage bonsaiWorldStateKeyValueStorage) { + super(null, bonsaiWorldStateKeyValueStorage, new NoOpMetricsSystem()); + } + + @Override + public synchronized void addCachedLayer( + final BlockHeader blockHeader, + final Hash worldStateRootHash, + final BonsaiWorldState forWorldState) { + // no cache + } + + @Override + public boolean containWorldStateStorage(final Hash blockHash) { + return false; + } + + @Override + public Optional getWorldState(final Hash blockHash) { + return Optional.empty(); + } + + @Override + public Optional getNearestWorldState(final BlockHeader blockHeader) { + return Optional.empty(); + } + + @Override + public Optional getHeadWorldState( + final Function> hashBlockHeaderFunction) { + return Optional.empty(); + } + + @Override + public void reset() { + // world states are not re-used + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java new file mode 100644 index 000000000..7cb024a25 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java @@ -0,0 +1,51 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.bonsai.trielog; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.plugin.services.trielogs.TrieLog; + +import java.util.Optional; + +public class NoOpTrieLogManager extends TrieLogManager { + + public NoOpTrieLogManager() { + super(null, null, 0, null, TrieLogPruner.noOpTrieLogPruner()); + } + + @Override + public synchronized void saveTrieLog( + final BonsaiWorldStateUpdateAccumulator localUpdater, + final Hash forWorldStateRootHash, + final BlockHeader forBlockHeader, + final BonsaiWorldState forWorldState) { + // notify trie log added observers, synchronously + TrieLog trieLog = trieLogFactory.create(localUpdater, forBlockHeader); + trieLogObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog))); + } + + @Override + public long getMaxLayersToLoad() { + return 0; + } + + @Override + public Optional getTrieLogLayer(final Hash blockHash) { + return Optional.empty(); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java index 548a3a1c4..4544a1c1d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java @@ -92,7 +92,7 @@ public class BonsaiWorldState evmConfiguration); } - protected BonsaiWorldState( + public BonsaiWorldState( final BonsaiWorldStateKeyValueStorage worldStateStorage, final CachedMerkleTrieLoader cachedMerkleTrieLoader, final CachedWorldStorageManager cachedWorldStorageManager, diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java new file mode 100644 index 000000000..ad1e920b7 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java @@ -0,0 +1,89 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.trie.common; + +import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider; +import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.bonsai.cache.NoOpCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.NoOpTrieLogManager; +import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState; +import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.evm.internal.EvmConfiguration; +import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; +import org.hyperledger.besu.services.kvstore.SegmentedInMemoryKeyValueStorage; + +import java.util.Objects; + +public class GenesisWorldStateProvider { + + /** + * Creates a Genesis world state based on the provided data storage format. + * + * @param dataStorageFormat the data storage format to use + * @return a mutable world state for the Genesis block + */ + public static MutableWorldState createGenesisWorldState( + final DataStorageFormat dataStorageFormat) { + if (Objects.requireNonNull(dataStorageFormat) == DataStorageFormat.BONSAI) { + return createGenesisBonsaiWorldState(); + } else { + return createGenesisForestWorldState(); + } + } + + /** + * Creates a Genesis world state using the Bonsai data storage format. + * + * @return a mutable world state for the Genesis block + */ + private static MutableWorldState createGenesisBonsaiWorldState() { + final CachedMerkleTrieLoader cachedMerkleTrieLoader = + new CachedMerkleTrieLoader(new NoOpMetricsSystem()); + final BonsaiWorldStateKeyValueStorage bonsaiWorldStateKeyValueStorage = + new BonsaiWorldStateKeyValueStorage( + new KeyValueStorageProvider( + segmentIdentifiers -> new SegmentedInMemoryKeyValueStorage(), + new InMemoryKeyValueStorage(), + new NoOpMetricsSystem()), + new NoOpMetricsSystem()); + return new BonsaiWorldState( + bonsaiWorldStateKeyValueStorage, + cachedMerkleTrieLoader, + new NoOpCachedWorldStorageManager(bonsaiWorldStateKeyValueStorage), + new NoOpTrieLogManager(), + EvmConfiguration.DEFAULT); + } + + /** + * Creates a Genesis world state using the Forest data storage format. + * + * @return a mutable world state for the Genesis block + */ + private static MutableWorldState createGenesisForestWorldState() { + final ForestWorldStateKeyValueStorage stateStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final WorldStatePreimageKeyValueStorage preimageStorage = + new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()); + return new ForestMutableWorldState(stateStorage, preimageStorage, EvmConfiguration.DEFAULT); + } +} diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/GenesisStateTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/GenesisStateTest.java index 1a0695426..0a8fc12b6 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/GenesisStateTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/GenesisStateTest.java @@ -24,14 +24,21 @@ import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.core.ProtocolScheduleFixture; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.evm.account.Account; +import java.util.stream.Stream; + import com.google.common.base.Charsets; import com.google.common.io.Resources; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.units.bigints.UInt256; import org.bouncycastle.util.encoders.Hex; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.params.provider.ArgumentsSource; public final class GenesisStateTest { @@ -46,10 +53,20 @@ public final class GenesisStateTest { private static final String EXPECTED_CODE = "0x608060405260043610610116577c01000000000000000000000000000000000000000000000000000000006000350463025e7c278114610158578063173825d91461019e57806320ea8d86146101d15780632f54bf6e146101fb5780633411c81c14610242578063547415251461027b5780637065cb48146102c1578063784547a7146102f45780638b51d13f1461031e5780639ace38c214610348578063a0e67e2b14610415578063a8abe69a1461047a578063b5dc40c3146104ba578063b77bf600146104e4578063ba51a6df146104f9578063c01a8c8414610523578063c64274741461054d578063d74f8edd14610615578063dc8452cd1461062a578063e20056e61461063f578063ee22610b1461067a575b60003411156101565760408051348152905133917fe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c919081900360200190a25b005b34801561016457600080fd5b506101826004803603602081101561017b57600080fd5b50356106a4565b60408051600160a060020a039092168252519081900360200190f35b3480156101aa57600080fd5b50610156600480360360208110156101c157600080fd5b5035600160a060020a03166106cc565b3480156101dd57600080fd5b50610156600480360360208110156101f457600080fd5b503561083c565b34801561020757600080fd5b5061022e6004803603602081101561021e57600080fd5b5035600160a060020a03166108f6565b604080519115158252519081900360200190f35b34801561024e57600080fd5b5061022e6004803603604081101561026557600080fd5b5080359060200135600160a060020a031661090b565b34801561028757600080fd5b506102af6004803603604081101561029e57600080fd5b50803515159060200135151561092b565b60408051918252519081900360200190f35b3480156102cd57600080fd5b50610156600480360360208110156102e457600080fd5b5035600160a060020a0316610997565b34801561030057600080fd5b5061022e6004803603602081101561031757600080fd5b5035610abc565b34801561032a57600080fd5b506102af6004803603602081101561034157600080fd5b5035610b43565b34801561035457600080fd5b506103726004803603602081101561036b57600080fd5b5035610bb2565b6040518085600160a060020a0316600160a060020a031681526020018481526020018060200183151515158152602001828103825284818151815260200191508051906020019080838360005b838110156103d75781810151838201526020016103bf565b50505050905090810190601f1680156104045780820380516001836020036101000a031916815260200191505b509550505050505060405180910390f35b34801561042157600080fd5b5061042a610c70565b60408051602080825283518183015283519192839290830191858101910280838360005b8381101561046657818101518382015260200161044e565b505050509050019250505060405180910390f35b34801561048657600080fd5b5061042a6004803603608081101561049d57600080fd5b508035906020810135906040810135151590606001351515610cd3565b3480156104c657600080fd5b5061042a600480360360208110156104dd57600080fd5b5035610e04565b3480156104f057600080fd5b506102af610f75565b34801561050557600080fd5b506101566004803603602081101561051c57600080fd5b5035610f7b565b34801561052f57600080fd5b506101566004803603602081101561054657600080fd5b5035610ffa565b34801561055957600080fd5b506102af6004803603606081101561057057600080fd5b600160a060020a03823516916020810135918101906060810160408201356401000000008111156105a057600080fd5b8201836020820111156105b257600080fd5b803590602001918460018302840111640100000000831117156105d457600080fd5b91908080601f0160208091040260200160405190810160405280939291908181526020018383808284376000920191909152509295506110c5945050505050565b34801561062157600080fd5b506102af6110e4565b34801561063657600080fd5b506102af6110e9565b34801561064b57600080fd5b506101566004803603604081101561066257600080fd5b50600160a060020a03813581169160200135166110ef565b34801561068657600080fd5b506101566004803603602081101561069d57600080fd5b5035611289565b60038054829081106106b257fe5b600091825260209091200154600160a060020a0316905081565b3330146106d857600080fd5b600160a060020a038116600090815260026020526040902054819060ff16151561070157600080fd5b600160a060020a0382166000908152600260205260408120805460ff191690555b600354600019018110156107d75782600160a060020a031660038281548110151561074957fe5b600091825260209091200154600160a060020a031614156107cf5760038054600019810190811061077657fe5b60009182526020909120015460038054600160a060020a03909216918390811061079c57fe5b9060005260206000200160006101000a815481600160a060020a030219169083600160a060020a031602179055506107d7565b600101610722565b506003805460001901906107eb9082611557565b5060035460045411156108045760035461080490610f7b565b604051600160a060020a038316907f8001553a916ef2f495d26a907cc54d96ed840d7bda71e73194bf5a9df7a76b9090600090a25050565b3360008181526002602052604090205460ff16151561085a57600080fd5b60008281526001602090815260408083203380855292529091205483919060ff16151561088657600080fd5b600084815260208190526040902060030154849060ff16156108a757600080fd5b6000858152600160209081526040808320338085529252808320805460ff191690555187927ff6a317157440607f36269043eb55f1287a5a19ba2216afeab88cd46cbcfb88e991a35050505050565b60026020526000908152604090205460ff1681565b600160209081526000928352604080842090915290825290205460ff1681565b6000805b60055481101561099057838015610958575060008181526020819052604090206003015460ff16155b8061097c575082801561097c575060008181526020819052604090206003015460ff165b15610988576001820191505b60010161092f565b5092915050565b3330146109a357600080fd5b600160a060020a038116600090815260026020526040902054819060ff16156109cb57600080fd5b81600160a060020a03811615156109e157600080fd5b600380549050600101600454603282111580156109fe5750818111155b8015610a0957508015155b8015610a1457508115155b1515610a1f57600080fd5b600160a060020a038516600081815260026020526040808220805460ff1916600190811790915560038054918201815583527fc2575a0e9e593c00f959f8c92f12db2869c3395a3b0502d05e2516446f71f85b01805473ffffffffffffffffffffffffffffffffffffffff191684179055517ff39e6e1eb0edcf53c221607b54b00cd28f3196fed0a24994dc308b8f611b682d9190a25050505050565b600080805b600354811015610b3b5760008481526001602052604081206003805491929184908110610aea57fe5b6000918252602080832090910154600160a060020a0316835282019290925260400190205460ff1615610b1e576001820191505b600454821415610b3357600192505050610b3e565b600101610ac1565b50505b919050565b6000805b600354811015610bac5760008381526001602052604081206003805491929184908110610b7057fe5b6000918252602080832090910154600160a060020a0316835282019290925260400190205460ff1615610ba4576001820191505b600101610b47565b50919050565b6000602081815291815260409081902080546001808301546002808501805487516101009582161595909502600019011691909104601f8101889004880284018801909652858352600160a060020a0390931695909491929190830182828015610c5d5780601f10610c3257610100808354040283529160200191610c5d565b820191906000526020600020905b815481529060010190602001808311610c4057829003601f168201915b5050506003909301549192505060ff1684565b60606003805480602002602001604051908101604052809291908181526020018280548015610cc857602002820191906000526020600020905b8154600160a060020a03168152600190910190602001808311610caa575b505050505090505b90565b606080600554604051908082528060200260200182016040528015610d02578160200160208202803883390190505b5090506000805b600554811015610d8457858015610d32575060008181526020819052604090206003015460ff16155b80610d565750848015610d56575060008181526020819052604090206003015460ff165b15610d7c57808383815181101515610d6a57fe5b60209081029091010152600191909101905b600101610d09565b878703604051908082528060200260200182016040528015610db0578160200160208202803883390190505b5093508790505b86811015610df9578281815181101515610dcd57fe5b9060200190602002015184898303815181101515610de757fe5b60209081029091010152600101610db7565b505050949350505050565b606080600380549050604051908082528060200260200182016040528015610e36578160200160208202803883390190505b5090506000805b600354811015610eee5760008581526001602052604081206003805491929184908110610e6657fe5b6000918252602080832090910154600160a060020a0316835282019290925260400190205460ff1615610ee6576003805482908110610ea157fe5b6000918252602090912001548351600160a060020a0390911690849084908110610ec757fe5b600160a060020a03909216602092830290910190910152600191909101905b600101610e3d565b81604051908082528060200260200182016040528015610f18578160200160208202803883390190505b509350600090505b81811015610f6d578281815181101515610f3657fe5b906020019060200201518482815181101515610f4e57fe5b600160a060020a03909216602092830290910190910152600101610f20565b505050919050565b60055481565b333014610f8757600080fd5b6003548160328211801590610f9c5750818111155b8015610fa757508015155b8015610fb257508115155b1515610fbd57600080fd5b60048390556040805184815290517fa3f1ee9126a074d9326c682f561767f710e927faa811f7a99829d49dc421797a9181900360200190a1505050565b3360008181526002602052604090205460ff16151561101857600080fd5b6000828152602081905260409020548290600160a060020a0316151561103d57600080fd5b60008381526001602090815260408083203380855292529091205484919060ff161561106857600080fd5b6000858152600160208181526040808420338086529252808420805460ff1916909317909255905187927f4a504a94899432a9846e1aa406dceb1bcfd538bb839071d49d1e5e23f5be30ef91a36110be85611289565b5050505050565b60006110d2848484611444565b90506110dd81610ffa565b9392505050565b603281565b60045481565b3330146110fb57600080fd5b600160a060020a038216600090815260026020526040902054829060ff16151561112457600080fd5b600160a060020a038216600090815260026020526040902054829060ff161561114c57600080fd5b82600160a060020a038116151561116257600080fd5b60005b6003548110156111ee5785600160a060020a031660038281548110151561118857fe5b600091825260209091200154600160a060020a031614156111e657846003828154811015156111b357fe5b9060005260206000200160006101000a815481600160a060020a030219169083600160a060020a031602179055506111ee565b600101611165565b50600160a060020a03808616600081815260026020526040808220805460ff1990811690915593881682528082208054909416600117909355915190917f8001553a916ef2f495d26a907cc54d96ed840d7bda71e73194bf5a9df7a76b9091a2604051600160a060020a038516907ff39e6e1eb0edcf53c221607b54b00cd28f3196fed0a24994dc308b8f611b682d90600090a25050505050565b3360008181526002602052604090205460ff1615156112a757600080fd5b60008281526001602090815260408083203380855292529091205483919060ff1615156112d357600080fd5b600084815260208190526040902060030154849060ff16156112f457600080fd5b6112fd85610abc565b156110be576000858152602081815260409182902060038101805460ff19166001908117909155815481830154600280850180548851601f6000199783161561010002979097019091169290920494850187900487028201870190975283815293956113cf95600160a060020a039093169491939283908301828280156113c55780601f1061139a576101008083540402835291602001916113c5565b820191906000526020600020905b8154815290600101906020018083116113a857829003601f168201915b5050505050611534565b156114045760405186907f33e13ecb54c3076d8e8bb8c2881800a4d972b792045ffae98fdf46df365fed7590600090a261143c565b60405186907f526441bb6c1aba3c9a4a6ca1d6545da9c2333c8c48343ef398eb858d72b7923690600090a260038101805460ff191690555b505050505050565b600083600160a060020a038116151561145c57600080fd5b60055460408051608081018252600160a060020a0388811682526020808301898152838501898152600060608601819052878152808452959095208451815473ffffffffffffffffffffffffffffffffffffffff1916941693909317835551600183015592518051949650919390926114dc926002850192910190611580565b50606091909101516003909101805460ff191691151591909117905560058054600101905560405182907fc0ba8fe4b176c1714197d43b9cc6bcf797a4a7461c5fe8d0ef6e184ae7601e5190600090a2509392505050565b6000806040516020840160008287838a8c6187965a03f198975050505050505050565b81548183558181111561157b5760008381526020902061157b9181019083016115fe565b505050565b828054600181600116156101000203166002900490600052602060002090601f016020900481019282601f106115c157805160ff19168380011785556115ee565b828001600101855582156115ee579182015b828111156115ee5782518255916020019190600101906115d3565b506115fa9291506115fe565b5090565b610cd091905b808211156115fa576000815560010161160456fea165627a7a7230582070d3c680a2cf749f81772e7fffa2883f27a13c65fcfff32190d7585b0c6f0ce40029"; - @Test - public void createFromJsonWithAllocs() throws Exception { + static class GenesisStateTestArguments implements ArgumentsProvider { + @Override + public Stream provideArguments(final ExtensionContext context) { + return Stream.of( + Arguments.of(DataStorageFormat.BONSAI), Arguments.of(DataStorageFormat.FOREST)); + } + } + + @ParameterizedTest + @ArgumentsSource(GenesisStateTestArguments.class) + public void createFromJsonWithAllocs(final DataStorageFormat dataStorageFormat) throws Exception { final GenesisState genesisState = GenesisState.fromJson( + dataStorageFormat, Resources.toString(GenesisStateTest.class.getResource("genesis1.json"), Charsets.UTF_8), ProtocolScheduleFixture.MAINNET); final BlockHeader header = genesisState.getBlock().getHeader(); @@ -74,10 +91,12 @@ public final class GenesisStateTest { assertThat(second.getBalance().toLong()).isEqualTo(222222222); } - @Test - public void createFromJsonNoAllocs() throws Exception { + @ParameterizedTest + @ArgumentsSource(GenesisStateTestArguments.class) + public void createFromJsonNoAllocs(final DataStorageFormat dataStorageFormat) throws Exception { final GenesisState genesisState = GenesisState.fromJson( + dataStorageFormat, Resources.toString(GenesisStateTest.class.getResource("genesis2.json"), Charsets.UTF_8), ProtocolScheduleFixture.MAINNET); final BlockHeader header = genesisState.getBlock().getHeader(); @@ -89,10 +108,12 @@ public final class GenesisStateTest { assertThat(header.getParentHash()).isEqualTo(Hash.ZERO); } - private void assertContractInvariants(final String sourceFile, final String blockHash) + private void assertContractInvariants( + final DataStorageFormat dataStorageFormat, final String sourceFile, final String blockHash) throws Exception { final GenesisState genesisState = GenesisState.fromJson( + dataStorageFormat, Resources.toString(GenesisStateTest.class.getResource(sourceFile), Charsets.UTF_8), ProtocolScheduleFixture.MAINNET); final BlockHeader header = genesisState.getBlock().getHeader(); @@ -113,16 +134,22 @@ public final class GenesisStateTest { "000000000000000000000000385ef55e292fa39cf5ffbad99f534294565519ba"); } - @Test - public void createFromJsonWithContract() throws Exception { + @ParameterizedTest + @ArgumentsSource(GenesisStateTestArguments.class) + public void createFromJsonWithContract(final DataStorageFormat dataStorageFormat) + throws Exception { assertContractInvariants( - "genesis3.json", "0xe7fd8db206dcaf066b7c97b8a42a0abc18653613560748557ab44868652a78b6"); + dataStorageFormat, + "genesis3.json", + "0xe7fd8db206dcaf066b7c97b8a42a0abc18653613560748557ab44868652a78b6"); } - @Test - public void createFromJsonWithNonce() throws Exception { + @ParameterizedTest + @ArgumentsSource(GenesisStateTestArguments.class) + public void createFromJsonWithNonce(final DataStorageFormat dataStorageFormat) throws Exception { final GenesisState genesisState = GenesisState.fromJson( + dataStorageFormat, Resources.toString( GenesisStateTest.class.getResource("genesisNonce.json"), Charsets.UTF_8), ProtocolScheduleFixture.MAINNET); @@ -133,10 +160,12 @@ public final class GenesisStateTest { "0x36750291f1a8429aeb553a790dc2d149d04dbba0ca4cfc7fd5eb12d478117c9f")); } - @Test - public void encodeOlympicBlock() throws Exception { + @ParameterizedTest + @ArgumentsSource(GenesisStateTestArguments.class) + public void encodeOlympicBlock(final DataStorageFormat dataStorageFormat) throws Exception { final GenesisState genesisState = GenesisState.fromJson( + dataStorageFormat, Resources.toString( GenesisStateTest.class.getResource("genesis-olympic.json"), Charsets.UTF_8), ProtocolScheduleFixture.MAINNET); @@ -152,10 +181,12 @@ public final class GenesisStateTest { .isEqualTo(UInt256.fromHexString(value)); } - @Test - public void genesisFromShanghai() throws Exception { + @ParameterizedTest + @ArgumentsSource(GenesisStateTestArguments.class) + public void genesisFromShanghai(final DataStorageFormat dataStorageFormat) throws Exception { final GenesisState genesisState = GenesisState.fromJson( + dataStorageFormat, Resources.toString( GenesisStateTest.class.getResource("genesis_shanghai.json"), Charsets.UTF_8), ProtocolScheduleFixture.MAINNET); @@ -200,10 +231,12 @@ public final class GenesisStateTest { assertThat(lastBalance).isEqualTo(Wei.fromHexString("0x123450000000000000000")); } - @Test - public void genesisFromCancun() throws Exception { + @ParameterizedTest + @ArgumentsSource(GenesisStateTestArguments.class) + public void genesisFromCancun(final DataStorageFormat dataStorageFormat) throws Exception { final GenesisState genesisState = GenesisState.fromJson( + dataStorageFormat, Resources.toString( GenesisStateTest.class.getResource("genesis_cancun.json"), Charsets.UTF_8), ProtocolScheduleFixture.MAINNET); diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java index 26fd811bf..034f8b9de 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java @@ -20,22 +20,18 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.bonsai.cache.NoOpCachedWorldStorageManager; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiPreImageProxy; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogAddedEvent; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.NoOpTrieLogManager; import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import java.util.Map; -import java.util.Optional; -import java.util.function.Function; import java.util.stream.Stream; import com.fasterxml.jackson.annotation.JsonCreator; @@ -117,14 +113,14 @@ public class BonsaiReferenceTestWorldState extends BonsaiWorldState final BonsaiPreImageProxy preImageProxy = new BonsaiPreImageProxy.BonsaiReferenceTestPreImageProxy(); + final BonsaiWorldStateKeyValueStorage bonsaiWorldStateKeyValueStorage = + new BonsaiWorldStateKeyValueStorage(new InMemoryKeyValueStorageProvider(), metricsSystem); + final BonsaiReferenceTestWorldStateStorage worldStateStorage = - new BonsaiReferenceTestWorldStateStorage( - new BonsaiWorldStateKeyValueStorage( - new InMemoryKeyValueStorageProvider(), metricsSystem), - preImageProxy); + new BonsaiReferenceTestWorldStateStorage(bonsaiWorldStateKeyValueStorage, preImageProxy); final NoOpCachedWorldStorageManager noOpCachedWorldStorageManager = - new NoOpCachedWorldStorageManager(); + new NoOpCachedWorldStorageManager(bonsaiWorldStateKeyValueStorage); final BonsaiReferenceTestWorldState worldState = new BonsaiReferenceTestWorldState( @@ -149,81 +145,6 @@ public class BonsaiReferenceTestWorldState extends BonsaiWorldState return this.refTestStorage.streamAccounts(this, startKeyHash, limit); } - static class NoOpCachedWorldStorageManager extends CachedWorldStorageManager { - - public NoOpCachedWorldStorageManager() { - super( - null, - new BonsaiWorldStateKeyValueStorage( - new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem()), - new NoOpMetricsSystem()); - } - - @SuppressWarnings({"UnsynchronizedOverridesSynchronized", "squid:S3551"}) - @Override - public void addCachedLayer( - final BlockHeader blockHeader, - final Hash worldStateRootHash, - final BonsaiWorldState forWorldState) { - // reference test world states are not cached - } - - @Override - public boolean containWorldStateStorage(final Hash blockHash) { - return false; - } - - @Override - public Optional getWorldState(final Hash blockHash) { - return Optional.empty(); - } - - @Override - public Optional getNearestWorldState(final BlockHeader blockHeader) { - return Optional.empty(); - } - - @Override - public Optional getHeadWorldState( - final Function> hashBlockHeaderFunction) { - return Optional.empty(); - } - - @Override - public void reset() { - // reference test world states are not re-used - } - } - - static class NoOpTrieLogManager extends TrieLogManager { - - public NoOpTrieLogManager() { - super(null, null, 0, null, TrieLogPruner.noOpTrieLogPruner()); - } - - @SuppressWarnings({"UnsynchronizedOverridesSynchronized", "squid:S3551"}) - @Override - public void saveTrieLog( - final BonsaiWorldStateUpdateAccumulator localUpdater, - final Hash forWorldStateRootHash, - final BlockHeader forBlockHeader, - final BonsaiWorldState forWorldState) { - // notify trie log added observers, synchronously - TrieLog trieLog = trieLogFactory.create(localUpdater, forBlockHeader); - trieLogObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog))); - } - - @Override - public long getMaxLayersToLoad() { - return 0; - } - - @Override - public Optional getTrieLogLayer(final Hash blockHash) { - return Optional.empty(); - } - } - @Override protected Hash hashAndSavePreImage(final Bytes value) { // by default do not save has preImages From 2d7941f8b84eb311a42b52270daf32e0465429ca Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Thu, 18 Jan 2024 11:05:31 +0100 Subject: [PATCH 11/20] Promote block txs selection max time options to stable (#6423) Signed-off-by: Fabio Di Fabio --- CHANGELOG.md | 6 +- .../org/hyperledger/besu/cli/BesuCommand.java | 8 +- .../converter/PositiveNumberConverter.java | 33 ++++++++ .../PositiveNumberConversionException.java | 30 +++++++ .../besu/cli/options/MiningOptions.java | 78 ++++++++----------- .../hyperledger/besu/cli/BesuCommandTest.java | 3 + .../besu/cli/options/MiningOptionsTest.java | 52 ++++++------- .../src/test/resources/everything_config.toml | 2 + .../txselection/BlockTransactionSelector.java | 2 +- .../AbstractBlockTransactionSelectorTest.java | 35 +++++---- ...FeeMarketBlockTransactionSelectorTest.java | 2 +- .../besu/ethereum/core/MiningParameters.java | 52 +++++++------ 12 files changed, 181 insertions(+), 122 deletions(-) create mode 100644 besu/src/main/java/org/hyperledger/besu/cli/converter/PositiveNumberConverter.java create mode 100644 besu/src/main/java/org/hyperledger/besu/cli/converter/exception/PositiveNumberConversionException.java diff --git a/CHANGELOG.md b/CHANGELOG.md index e09db4de5..4ab78beb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,18 +6,20 @@ - New `EXECUTION_HALTED` error returned if there is an error executing or simulating a transaction, with the reason for execution being halted. Replaces the generic `INTERNAL_ERROR` return code in certain cases which some applications may be checking for [#6343](https://github.com/hyperledger/besu/pull/6343) - The Besu Docker images with `openjdk-latest` tags since 23.10.3 were incorrectly using UID 1001 instead of 1000 for the container's `besu` user. The user now uses 1000 again. Containers created from or migrated to images using UID 1001 will need to chown their persistent database files to UID 1000 [#6360](https://github.com/hyperledger/besu/pull/6360) - The deprecated `--privacy-onchain-groups-enabled` option has now been removed. Use the `--privacy-flexible-groups-enabled` option instead. [#6411](https://github.com/hyperledger/besu/pull/6411) +- The time that can be spent selecting transactions during block creation is not capped at 5 seconds for PoS and PoW networks, and for PoA networks, at 75% of the block period specified in the genesis, this to prevent possible DoS in case a single transaction is taking too long to execute, and to have a stable block production rate, but it could be a breaking change if an existing network used to have transactions that takes more time to executed that the newly introduced limit, if it is mandatory for these network to keep processing these long processing transaction, then the default value of `block-txs-selection-max-time` or `poa-block-txs-selection-max-time` needs to be tuned accordingly. ### Deprecations ### Additions and Improvements - Optimize RocksDB WAL files, allows for faster restart and a more linear disk space utilization [#6328](https://github.com/hyperledger/besu/pull/6328) - Disable transaction handling when the node is not in sync, to avoid unnecessary transaction validation work [#6302](https://github.com/hyperledger/besu/pull/6302) -- Introduce TransactionEvaluationContext to pass data between transaction selectors and plugin, during block creation [#6381](https://github.com/hyperledger/besu/pull/6381) +- Introduce TransactionEvaluationContext to pass data between transaction selectors and plugin, during block creation [#6381](https://github.com/hyperledger/besu/pull/6381) - Upgrade dependencies [#6377](https://github.com/hyperledger/besu/pull/6377) -- Upgrade `com.fasterxml.jackson` dependencies [#6378](https://github.com/hyperledger/besu/pull/6378) +- Upgrade `com.fasterxml.jackson` dependencies [#6378](https://github.com/hyperledger/besu/pull/6378) - Upgrade Guava dependency [#6396](https://github.com/hyperledger/besu/pull/6396) - Upgrade Mockito [#6397](https://github.com/hyperledger/besu/pull/6397) - Upgrade `tech.pegasys.discovery:discovery` [#6414](https://github.com/hyperledger/besu/pull/6414) +- Options to tune the max allowed time that can be spent selecting transactions during block creation are now stable [#6423](https://github.com/hyperledger/besu/pull/6423) ### Bug fixes - INTERNAL_ERROR from `eth_estimateGas` JSON/RPC calls [#6344](https://github.com/hyperledger/besu/issues/6344) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index c51d03d43..7f276c704 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -2907,17 +2907,15 @@ public class BesuCommand implements DefaultCommandValues, Runnable { ImmutableMiningParameters.builder().from(miningOptions.toDomainObject()); final var actualGenesisOptions = getActualGenesisConfigOptions(); if (actualGenesisOptions.isPoa()) { - miningParametersBuilder.unstable( - ImmutableMiningParameters.Unstable.builder() - .minBlockTime(getMinBlockTime(actualGenesisOptions)) - .build()); + miningParametersBuilder.genesisBlockPeriodSeconds( + getGenesisBlockPeriodSeconds(actualGenesisOptions)); } miningParameters = miningParametersBuilder.build(); } return miningParameters; } - private int getMinBlockTime(final GenesisConfigOptions genesisConfigOptions) { + private int getGenesisBlockPeriodSeconds(final GenesisConfigOptions genesisConfigOptions) { if (genesisConfigOptions.isClique()) { return genesisConfigOptions.getCliqueConfigOptions().getBlockPeriodSeconds(); } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/converter/PositiveNumberConverter.java b/besu/src/main/java/org/hyperledger/besu/cli/converter/PositiveNumberConverter.java new file mode 100644 index 000000000..2b45f35ff --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/converter/PositiveNumberConverter.java @@ -0,0 +1,33 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.converter; + +import org.hyperledger.besu.cli.converter.exception.PercentageConversionException; +import org.hyperledger.besu.util.number.PositiveNumber; + +import picocli.CommandLine; + +/** The PositiveNumber Cli type converter. */ +public class PositiveNumberConverter implements CommandLine.ITypeConverter { + + @Override + public PositiveNumber convert(final String value) throws PercentageConversionException { + try { + return PositiveNumber.fromString(value); + } catch (NullPointerException | IllegalArgumentException e) { + throw new PercentageConversionException(value); + } + } +} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/converter/exception/PositiveNumberConversionException.java b/besu/src/main/java/org/hyperledger/besu/cli/converter/exception/PositiveNumberConversionException.java new file mode 100644 index 000000000..c9f0d9e93 --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/converter/exception/PositiveNumberConversionException.java @@ -0,0 +1,30 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.converter.exception; + +import static java.lang.String.format; + +/** The custom PositiveNumber conversion exception. */ +public final class PositiveNumberConversionException extends Exception { + + /** + * Instantiates a new PositiveNumber conversion exception. + * + * @param value the invalid value to add in exception message + */ + public PositiveNumberConversionException(final String value) { + super(format("Invalid value: %s, should be a positive number >0.", value)); + } +} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java index 78e2032e4..55197b8e8 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java @@ -16,20 +16,20 @@ package org.hyperledger.besu.cli.options; import static java.util.Arrays.asList; import static java.util.Collections.singletonList; +import static org.hyperledger.besu.ethereum.core.MiningParameters.DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; +import static org.hyperledger.besu.ethereum.core.MiningParameters.DEFAULT_POA_BLOCK_TXS_SELECTION_MAX_TIME; import static org.hyperledger.besu.ethereum.core.MiningParameters.MutableInitValues.DEFAULT_EXTRA_DATA; import static org.hyperledger.besu.ethereum.core.MiningParameters.MutableInitValues.DEFAULT_MIN_BLOCK_OCCUPANCY_RATIO; import static org.hyperledger.besu.ethereum.core.MiningParameters.MutableInitValues.DEFAULT_MIN_PRIORITY_FEE_PER_GAS; import static org.hyperledger.besu.ethereum.core.MiningParameters.MutableInitValues.DEFAULT_MIN_TRANSACTION_GAS_PRICE; import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_MAX_OMMERS_DEPTH; -import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; -import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_POA_BLOCK_TXS_SELECTION_MAX_TIME; import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_POS_BLOCK_CREATION_MAX_TIME; import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_POS_BLOCK_CREATION_REPETITION_MIN_DURATION; import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_POW_JOB_TTL; import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_REMOTE_SEALERS_LIMIT; import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_REMOTE_SEALERS_TTL; -import org.hyperledger.besu.cli.converter.PercentageConverter; +import org.hyperledger.besu.cli.converter.PositiveNumberConverter; import org.hyperledger.besu.cli.util.CommandLineUtils; import org.hyperledger.besu.config.GenesisConfigOptions; import org.hyperledger.besu.datatypes.Address; @@ -37,7 +37,7 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters; import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters.MutableInitValues; import org.hyperledger.besu.ethereum.core.MiningParameters; -import org.hyperledger.besu.util.number.Percentage; +import org.hyperledger.besu.util.number.PositiveNumber; import java.util.List; @@ -115,6 +115,24 @@ public class MiningOptions implements CLIOptions { + " If set, each block's gas limit will approach this setting over time.") private Long targetGasLimit = null; + @Option( + names = {"--block-txs-selection-max-time"}, + converter = PositiveNumberConverter.class, + description = + "Specifies the maximum time, in milliseconds, that could be spent selecting transactions to be included in the block." + + " Not compatible with PoA networks, see poa-block-txs-selection-max-time. (default: ${DEFAULT-VALUE})") + private PositiveNumber nonPoaBlockTxsSelectionMaxTime = + DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; + + @Option( + names = {"--poa-block-txs-selection-max-time"}, + converter = PositiveNumberConverter.class, + description = + "Specifies the maximum time that could be spent selecting transactions to be included in the block, as a percentage of the fixed block time of the PoA network." + + " To be only used on PoA networks, for other networks see block-txs-selection-max-time." + + " (default: ${DEFAULT-VALUE})") + private PositiveNumber poaBlockTxsSelectionMaxTime = DEFAULT_POA_BLOCK_TXS_SELECTION_MAX_TIME; + @CommandLine.ArgGroup(validate = false) private final Unstable unstableOptions = new Unstable(); @@ -168,25 +186,6 @@ public class MiningOptions implements CLIOptions { + " then it waits before next repetition. Must be positive and ≤ 2000 (default: ${DEFAULT-VALUE} milliseconds)") private Long posBlockCreationRepetitionMinDuration = DEFAULT_POS_BLOCK_CREATION_REPETITION_MIN_DURATION; - - @CommandLine.Option( - hidden = true, - names = {"--Xblock-txs-selection-max-time"}, - description = - "Specifies the maximum time, in milliseconds, that could be spent selecting transactions to be included in the block." - + " Not compatible with PoA networks, see Xpoa-block-txs-selection-max-time." - + " Must be positive and ≤ (default: ${DEFAULT-VALUE})") - private Long nonPoaBlockTxsSelectionMaxTime = DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; - - @CommandLine.Option( - hidden = true, - names = {"--Xpoa-block-txs-selection-max-time"}, - converter = PercentageConverter.class, - description = - "Specifies the maximum time that could be spent selecting transactions to be included in the block, as a percentage of the fixed block time of the PoA network." - + " To be only used on PoA networks, for other networks see Xblock-txs-selection-max-time." - + " (default: ${DEFAULT-VALUE})") - private Percentage poaBlockTxsSelectionMaxTime = DEFAULT_POA_BLOCK_TXS_SELECTION_MAX_TIME; } private MiningOptions() {} @@ -270,26 +269,17 @@ public class MiningOptions implements CLIOptions { if (genesisConfigOptions.isPoa()) { CommandLineUtils.failIfOptionDoesntMeetRequirement( commandLine, - "--Xblock-txs-selection-max-time can't be used with PoA networks," - + " see Xpoa-block-txs-selection-max-time instead", + "--block-txs-selection-max-time can't be used with PoA networks," + + " see poa-block-txs-selection-max-time instead", false, - singletonList("--Xblock-txs-selection-max-time")); + singletonList("--block-txs-selection-max-time")); } else { CommandLineUtils.failIfOptionDoesntMeetRequirement( commandLine, - "--Xpoa-block-txs-selection-max-time can be only used with PoA networks," - + " see --Xblock-txs-selection-max-time instead", + "--poa-block-txs-selection-max-time can be only used with PoA networks," + + " see --block-txs-selection-max-time instead", false, - singletonList("--Xpoa-block-txs-selection-max-time")); - - if (unstableOptions.nonPoaBlockTxsSelectionMaxTime <= 0 - || unstableOptions.nonPoaBlockTxsSelectionMaxTime - > DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME) { - throw new ParameterException( - commandLine, - "--Xblock-txs-selection-max-time must be positive and ≤ " - + DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME); - } + singletonList("--poa-block-txs-selection-max-time")); } } @@ -303,6 +293,10 @@ public class MiningOptions implements CLIOptions { miningOptions.minTransactionGasPrice = miningParameters.getMinTransactionGasPrice(); miningOptions.minPriorityFeePerGas = miningParameters.getMinPriorityFeePerGas(); miningOptions.minBlockOccupancyRatio = miningParameters.getMinBlockOccupancyRatio(); + miningOptions.nonPoaBlockTxsSelectionMaxTime = + miningParameters.getNonPoaBlockTxsSelectionMaxTime(); + miningOptions.poaBlockTxsSelectionMaxTime = miningParameters.getPoaBlockTxsSelectionMaxTime(); + miningOptions.unstableOptions.remoteSealersLimit = miningParameters.getUnstable().getRemoteSealersLimit(); miningOptions.unstableOptions.remoteSealersTimeToLive = @@ -317,10 +311,6 @@ public class MiningOptions implements CLIOptions { miningParameters.getUnstable().getPosBlockCreationMaxTime(); miningOptions.unstableOptions.posBlockCreationRepetitionMinDuration = miningParameters.getUnstable().getPosBlockCreationRepetitionMinDuration(); - miningOptions.unstableOptions.nonPoaBlockTxsSelectionMaxTime = - miningParameters.getUnstable().getBlockTxsSelectionMaxTime(); - miningOptions.unstableOptions.poaBlockTxsSelectionMaxTime = - miningParameters.getUnstable().getPoaBlockTxsSelectionMaxTime(); miningParameters.getCoinbase().ifPresent(coinbase -> miningOptions.coinbase = coinbase); miningParameters.getTargetGasLimit().ifPresent(tgl -> miningOptions.targetGasLimit = tgl); @@ -350,6 +340,8 @@ public class MiningOptions implements CLIOptions { .isStratumMiningEnabled(iStratumMiningEnabled) .stratumNetworkInterface(stratumNetworkInterface) .stratumPort(stratumPort) + .nonPoaBlockTxsSelectionMaxTime(nonPoaBlockTxsSelectionMaxTime) + .poaBlockTxsSelectionMaxTime(poaBlockTxsSelectionMaxTime) .unstable( ImmutableMiningParameters.Unstable.builder() .remoteSealersLimit(unstableOptions.remoteSealersLimit) @@ -360,8 +352,6 @@ public class MiningOptions implements CLIOptions { .posBlockCreationMaxTime(unstableOptions.posBlockCreationMaxTime) .posBlockCreationRepetitionMinDuration( unstableOptions.posBlockCreationRepetitionMinDuration) - .nonPoaBlockTxsSelectionMaxTime(unstableOptions.nonPoaBlockTxsSelectionMaxTime) - .poaBlockTxsSelectionMaxTime(unstableOptions.poaBlockTxsSelectionMaxTime) .build()); return miningParametersBuilder.build(); diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index 262965563..04d9f0155 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -95,6 +95,7 @@ import org.hyperledger.besu.plugin.services.privacy.PrivateMarkerTransactionFact import org.hyperledger.besu.plugin.services.rpc.PluginRpcRequest; import org.hyperledger.besu.util.number.Fraction; import org.hyperledger.besu.util.number.Percentage; +import org.hyperledger.besu.util.number.PositiveNumber; import org.hyperledger.besu.util.platform.PlatformDetector; import java.io.File; @@ -846,6 +847,8 @@ public class BesuCommandTest extends CommandTestAbstract { tomlResult.getDouble(tomlKey); } else if (Percentage.class.isAssignableFrom(optionSpec.type())) { tomlResult.getLong(tomlKey); + } else if (PositiveNumber.class.isAssignableFrom(optionSpec.type())) { + tomlResult.getLong(tomlKey); } else { tomlResult.getString(tomlKey); } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/MiningOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/MiningOptionsTest.java index 404c021e3..4b1fdddb5 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/MiningOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/MiningOptionsTest.java @@ -15,8 +15,8 @@ package org.hyperledger.besu.cli.options; import static org.assertj.core.api.Assertions.assertThat; -import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; -import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_POA_BLOCK_TXS_SELECTION_MAX_TIME; +import static org.hyperledger.besu.ethereum.core.MiningParameters.DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; +import static org.hyperledger.besu.ethereum.core.MiningParameters.DEFAULT_POA_BLOCK_TXS_SELECTION_MAX_TIME; import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_POS_BLOCK_CREATION_MAX_TIME; import static org.mockito.Mockito.atMost; import static org.mockito.Mockito.verify; @@ -28,7 +28,7 @@ import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters; import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters.MutableInitValues; import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters.Unstable; import org.hyperledger.besu.ethereum.core.MiningParameters; -import org.hyperledger.besu.util.number.Percentage; +import org.hyperledger.besu.util.number.PositiveNumber; import java.io.IOException; import java.nio.file.Path; @@ -315,35 +315,26 @@ public class MiningOptionsTest extends AbstractCLIOptionsTest - assertThat(miningParams.getUnstable().getBlockTxsSelectionMaxTime()) + assertThat(miningParams.getNonPoaBlockTxsSelectionMaxTime()) .isEqualTo(DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME)); } @Test public void blockTxsSelectionMaxTimeOption() { internalTestSuccess( - miningParams -> - assertThat(miningParams.getUnstable().getBlockTxsSelectionMaxTime()).isEqualTo(1700L), - "--Xblock-txs-selection-max-time", + miningParams -> assertThat(miningParams.getBlockTxsSelectionMaxTime()).isEqualTo(1700L), + "--block-txs-selection-max-time", "1700"); } - @Test - public void blockTxsSelectionMaxTimeOutOfAllowedRange() { - internalTestFailure( - "--Xblock-txs-selection-max-time must be positive and ≤ 5000", - "--Xblock-txs-selection-max-time", - "6000"); - } - @Test public void blockTxsSelectionMaxTimeIncompatibleWithPoaNetworks() throws IOException { final Path genesisFileIBFT2 = createFakeGenesisFile(VALID_GENESIS_IBFT2_POST_LONDON); internalTestFailure( - "--Xblock-txs-selection-max-time can't be used with PoA networks, see Xpoa-block-txs-selection-max-time instead", + "--block-txs-selection-max-time can't be used with PoA networks, see poa-block-txs-selection-max-time instead", "--genesis-file", genesisFileIBFT2.toString(), - "--Xblock-txs-selection-max-time", + "--block-txs-selection-max-time", "2"); } @@ -351,7 +342,7 @@ public class MiningOptionsTest extends AbstractCLIOptionsTest - assertThat(miningParams.getUnstable().getPoaBlockTxsSelectionMaxTime()) + assertThat(miningParams.getPoaBlockTxsSelectionMaxTime()) .isEqualTo(DEFAULT_POA_BLOCK_TXS_SELECTION_MAX_TIME)); } @@ -360,27 +351,32 @@ public class MiningOptionsTest extends AbstractCLIOptionsTest - assertThat(miningParams.getUnstable().getPoaBlockTxsSelectionMaxTime()) - .isEqualTo(Percentage.fromInt(80)), + assertThat(miningParams.getPoaBlockTxsSelectionMaxTime()) + .isEqualTo(PositiveNumber.fromInt(80)), "--genesis-file", genesisFileIBFT2.toString(), - "--Xpoa-block-txs-selection-max-time", + "--poa-block-txs-selection-max-time", "80"); } @Test - public void poaBlockTxsSelectionMaxTimeOutOfAllowedRange() { - internalTestFailure( - "Invalid value for option '--Xpoa-block-txs-selection-max-time': cannot convert '110' to Percentage", - "--Xpoa-block-txs-selection-max-time", - "110"); + public void poaBlockTxsSelectionMaxTimeOptionOver100Percent() throws IOException { + final Path genesisFileIBFT2 = createFakeGenesisFile(VALID_GENESIS_IBFT2_POST_LONDON); + internalTestSuccess( + miningParams -> + assertThat(miningParams.getPoaBlockTxsSelectionMaxTime()) + .isEqualTo(PositiveNumber.fromInt(200)), + "--genesis-file", + genesisFileIBFT2.toString(), + "--poa-block-txs-selection-max-time", + "200"); } @Test public void poaBlockTxsSelectionMaxTimeOnlyCompatibleWithPoaNetworks() { internalTestFailure( - "--Xpoa-block-txs-selection-max-time can be only used with PoA networks, see --Xblock-txs-selection-max-time instead", - "--Xpoa-block-txs-selection-max-time", + "--poa-block-txs-selection-max-time can be only used with PoA networks, see --block-txs-selection-max-time instead", + "--poa-block-txs-selection-max-time", "90"); } diff --git a/besu/src/test/resources/everything_config.toml b/besu/src/test/resources/everything_config.toml index c653f3f60..b8dae6c18 100644 --- a/besu/src/test/resources/everything_config.toml +++ b/besu/src/test/resources/everything_config.toml @@ -142,6 +142,8 @@ min-priority-fee=0 min-block-occupancy-ratio=0.7 miner-stratum-host="0.0.0.0" miner-stratum-port=8008 +block-txs-selection-max-time=5000 +poa-block-txs-selection-max-time=75 Xminer-remote-sealers-limit=1000 Xminer-remote-sealers-hashrate-ttl=10 Xpos-block-creation-max-time=5 diff --git a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java index 944cf426d..26ac79ef3 100644 --- a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java +++ b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java @@ -136,7 +136,7 @@ public class BlockTransactionSelector { this.pluginTransactionSelector = pluginTransactionSelector; this.pluginOperationTracer = pluginTransactionSelector.getOperationTracer(); blockWorldStateUpdater = worldState.updater(); - blockTxsSelectionMaxTime = miningParameters.getUnstable().getBlockTxsSelectionMaxTime(); + blockTxsSelectionMaxTime = miningParameters.getBlockTxsSelectionMaxTime(); } private List createTransactionSelectors( diff --git a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java index 0b0b2cd78..e593569ff 100644 --- a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java +++ b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java @@ -17,7 +17,7 @@ package org.hyperledger.besu.ethereum.blockcreation; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.entry; import static org.awaitility.Awaitility.await; -import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; +import static org.hyperledger.besu.ethereum.core.MiningParameters.DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; import static org.hyperledger.besu.plugin.data.TransactionSelectionResult.BLOCK_SELECTION_TIMEOUT; import static org.hyperledger.besu.plugin.data.TransactionSelectionResult.PRIORITY_FEE_PER_GAS_BELOW_CURRENT_MIN; import static org.hyperledger.besu.plugin.data.TransactionSelectionResult.SELECTED; @@ -54,7 +54,6 @@ import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.core.Difficulty; import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters; import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters.MutableInitValues; -import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters.Unstable; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.MutableWorldState; @@ -85,7 +84,7 @@ import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelecto import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelectorFactory; import org.hyperledger.besu.plugin.services.txselection.TransactionEvaluationContext; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; -import org.hyperledger.besu.util.number.Percentage; +import org.hyperledger.besu.util.number.PositiveNumber; import java.math.BigInteger; import java.time.Instant; @@ -956,8 +955,8 @@ public abstract class AbstractBlockTransactionSelectorTest { final ProcessableBlockHeader blockHeader = createBlock(301_000); final Address miningBeneficiary = AddressHelpers.ofValue(1); - final int poaMinBlockTime = 1; - final long blockTxsSelectionMaxTime = 750; + final int poaGenesisBlockPeriod = 1; + final int blockTxsSelectionMaxTime = 750; final List transactionsToInject = new ArrayList<>(3); for (int i = 0; i < 2; i++) { @@ -987,9 +986,14 @@ public abstract class AbstractBlockTransactionSelectorTest { createBlockSelectorAndSetupTxPool( isPoa ? createMiningParameters( - Wei.ZERO, MIN_OCCUPANCY_100_PERCENT, poaMinBlockTime, Percentage.fromInt(75)) + Wei.ZERO, + MIN_OCCUPANCY_100_PERCENT, + poaGenesisBlockPeriod, + PositiveNumber.fromInt(75)) : createMiningParameters( - Wei.ZERO, MIN_OCCUPANCY_100_PERCENT, blockTxsSelectionMaxTime), + Wei.ZERO, + MIN_OCCUPANCY_100_PERCENT, + PositiveNumber.fromInt(blockTxsSelectionMaxTime)), transactionProcessor, blockHeader, miningBeneficiary, @@ -1176,33 +1180,32 @@ public abstract class AbstractBlockTransactionSelectorTest { } protected MiningParameters createMiningParameters( - final Wei minGasPrice, final double minBlockOccupancyRatio, final long txsSelectionMaxTime) { + final Wei minGasPrice, + final double minBlockOccupancyRatio, + final PositiveNumber txsSelectionMaxTime) { return ImmutableMiningParameters.builder() .mutableInitValues( MutableInitValues.builder() .minTransactionGasPrice(minGasPrice) .minBlockOccupancyRatio(minBlockOccupancyRatio) .build()) - .unstable(Unstable.builder().nonPoaBlockTxsSelectionMaxTime(txsSelectionMaxTime).build()) + .nonPoaBlockTxsSelectionMaxTime(txsSelectionMaxTime) .build(); } protected MiningParameters createMiningParameters( final Wei minGasPrice, final double minBlockOccupancyRatio, - final int minBlockTime, - final Percentage minBlockTimePercentage) { + final int genesisBlockPeriodSeconds, + final PositiveNumber minBlockTimePercentage) { return ImmutableMiningParameters.builder() .mutableInitValues( MutableInitValues.builder() .minTransactionGasPrice(minGasPrice) .minBlockOccupancyRatio(minBlockOccupancyRatio) .build()) - .unstable( - Unstable.builder() - .minBlockTime(minBlockTime) - .poaBlockTxsSelectionMaxTime(minBlockTimePercentage) - .build()) + .genesisBlockPeriodSeconds(genesisBlockPeriodSeconds) + .poaBlockTxsSelectionMaxTime(minBlockTimePercentage) .build(); } diff --git a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/LondonFeeMarketBlockTransactionSelectorTest.java b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/LondonFeeMarketBlockTransactionSelectorTest.java index 63e0d54ec..eb1a98cbd 100644 --- a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/LondonFeeMarketBlockTransactionSelectorTest.java +++ b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/LondonFeeMarketBlockTransactionSelectorTest.java @@ -16,7 +16,7 @@ package org.hyperledger.besu.ethereum.blockcreation; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.entry; -import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; +import static org.hyperledger.besu.ethereum.core.MiningParameters.DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; import static org.mockito.Mockito.mock; import org.hyperledger.besu.config.GenesisConfigFile; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MiningParameters.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MiningParameters.java index 32ac5ee92..7f543f370 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MiningParameters.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MiningParameters.java @@ -16,7 +16,7 @@ package org.hyperledger.besu.ethereum.core; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Wei; -import org.hyperledger.besu.util.number.Percentage; +import org.hyperledger.besu.util.number.PositiveNumber; import java.time.Duration; import java.util.Objects; @@ -32,6 +32,10 @@ import org.immutables.value.Value; @Value.Immutable @Value.Enclosing public abstract class MiningParameters { + public static final PositiveNumber DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME = + PositiveNumber.fromInt((int) Duration.ofSeconds(5).toMillis()); + public static final PositiveNumber DEFAULT_POA_BLOCK_TXS_SELECTION_MAX_TIME = + PositiveNumber.fromInt(75); public static final MiningParameters MINING_DISABLED = ImmutableMiningParameters.builder() .mutableInitValues( @@ -130,6 +134,28 @@ public abstract class MiningParameters { return 8008; } + @Value.Default + public PositiveNumber getNonPoaBlockTxsSelectionMaxTime() { + return DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; + } + + @Value.Default + public PositiveNumber getPoaBlockTxsSelectionMaxTime() { + return DEFAULT_POA_BLOCK_TXS_SELECTION_MAX_TIME; + } + + public abstract OptionalInt getGenesisBlockPeriodSeconds(); + + @Value.Derived + public long getBlockTxsSelectionMaxTime() { + if (getGenesisBlockPeriodSeconds().isPresent()) { + return (TimeUnit.SECONDS.toMillis(getGenesisBlockPeriodSeconds().getAsInt()) + * getPoaBlockTxsSelectionMaxTime().getValue()) + / 100; + } + return getNonPoaBlockTxsSelectionMaxTime().getValue(); + } + @Value.Default protected MutableRuntimeValues getMutableRuntimeValues() { return new MutableRuntimeValues(getMutableInitValues()); @@ -266,8 +292,6 @@ public abstract class MiningParameters { int DEFAULT_MAX_OMMERS_DEPTH = 8; long DEFAULT_POS_BLOCK_CREATION_MAX_TIME = Duration.ofSeconds(12).toMillis(); long DEFAULT_POS_BLOCK_CREATION_REPETITION_MIN_DURATION = Duration.ofMillis(500).toMillis(); - long DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME = Duration.ofSeconds(5).toMillis(); - Percentage DEFAULT_POA_BLOCK_TXS_SELECTION_MAX_TIME = Percentage.fromInt(75); MiningParameters.Unstable DEFAULT = ImmutableMiningParameters.Unstable.builder().build(); @@ -305,27 +329,5 @@ public abstract class MiningParameters { default String getStratumExtranonce() { return "080c"; } - - @Value.Default - default long getNonPoaBlockTxsSelectionMaxTime() { - return DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; - } - - @Value.Default - default Percentage getPoaBlockTxsSelectionMaxTime() { - return DEFAULT_POA_BLOCK_TXS_SELECTION_MAX_TIME; - } - - OptionalInt getMinBlockTime(); - - @Value.Derived - default long getBlockTxsSelectionMaxTime() { - if (getMinBlockTime().isPresent()) { - return (TimeUnit.SECONDS.toMillis(getMinBlockTime().getAsInt()) - * getPoaBlockTxsSelectionMaxTime().getValue()) - / 100; - } - return getNonPoaBlockTxsSelectionMaxTime(); - } } } From f81d5445f1032128ad3e3e06a56d7e32b5011222 Mon Sep 17 00:00:00 2001 From: Matt Nelson <85905982+non-fungible-nelson@users.noreply.github.com> Date: Thu, 18 Jan 2024 16:02:58 -0700 Subject: [PATCH 12/20] Fix to increment/decrement gas-limit in block production (#6425) Signed-off-by: Matt Nelson <85905982+non-fungible-nelson@users.noreply.github.com> Co-authored-by: Stefan Pingel <16143240+pinges@users.noreply.github.com> Co-authored-by: Simon Dudley Co-authored-by: Sally MacFarlane Signed-off-by: Simon Dudley --- CHANGELOG.md | 1 + .../12_cancun_get_built_block.json | 4 +- .../AbstractGasLimitSpecification.java | 1 - .../FrontierTargetingGasLimitCalculator.java | 10 ++--- .../LondonTargetingGasLimitCalculator.java | 10 +---- .../TargetingGasLimitCalculatorTest.java | 43 ++++++++++++------- 6 files changed, 35 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ab78beb7..eecedac55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ - Fix Besu Docker images with `openjdk-latest` tags since 23.10.3 using UID 1001 instead of 1000 for the `besu` user [#6360](https://github.com/hyperledger/besu/pull/6360) - Fluent EVM API definition for Tangerine Whistle had incorrect code size validation configured [#6382](https://github.com/hyperledger/besu/pull/6382) - Correct mining beneficiary for Clique networks in TraceServiceImpl [#6390](https://github.com/hyperledger/besu/pull/6390) +- Fix to gas limit delta calculations used in block production. Besu should now increment or decrement the block gas limit towards its target correctly (thanks @arbora) #6425 ### Download Links diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/cancun/test-cases/block-production/12_cancun_get_built_block.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/cancun/test-cases/block-production/12_cancun_get_built_block.json index 753001192..8cbdd6264 100644 --- a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/cancun/test-cases/block-production/12_cancun_get_built_block.json +++ b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/cancun/test-cases/block-production/12_cancun_get_built_block.json @@ -10,7 +10,7 @@ "stateRoot" : "0x8d9115d9211932d4a3a1f068fb8fe262b0b2ab0bfd74eaece1a572efe6336677", "logsBloom" : "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "prevRandao" : "0xc13da06dc53836ca0766057413b9683eb9a8773bbb8fcc5691e41c25b56dda1d", - "gasLimit" : "0x2ff3d8", + "gasLimit" : "0x2ffbd2", "gasUsed" : "0xf618", "timestamp" : "0x1236", "extraData" : "0x", @@ -70,7 +70,7 @@ "amount" : "0x64" } ], "blockNumber" : "0x1", - "blockHash" : "0xf1e35607932349e87f29e1053a4fb2666782e09fde21ded74c1f7e4a57d3fa2b", + "blockHash" : "0x736bdddc2eca36fe8ed4ed515e5d295a08d7eaddc0d0fda2a35408127eb890d0", "receiptsRoot" : "0x9af165447e5b3193e9ac8389418648ee6d6cb1d37459fe65cfc245fc358721bd", "blobGasUsed" : "0x60000" }, diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractGasLimitSpecification.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractGasLimitSpecification.java index 70e80527b..b1a21e54b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractGasLimitSpecification.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractGasLimitSpecification.java @@ -19,7 +19,6 @@ import static com.google.common.base.Preconditions.checkArgument; /** Specification for the block gasLimit. */ public abstract class AbstractGasLimitSpecification { - public static final long DEFAULT_MAX_CONSTANT_ADMUSTMENT_INCREMENT = 1024L; public static final long DEFAULT_MIN_GAS_LIMIT = 5000L; public static final long DEFAULT_MAX_GAS_LIMIT = Long.MAX_VALUE; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/FrontierTargetingGasLimitCalculator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/FrontierTargetingGasLimitCalculator.java index 49295e394..ed067731b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/FrontierTargetingGasLimitCalculator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/FrontierTargetingGasLimitCalculator.java @@ -23,16 +23,13 @@ public class FrontierTargetingGasLimitCalculator extends AbstractGasLimitSpecifi implements GasLimitCalculator { private static final Logger LOG = LoggerFactory.getLogger(FrontierTargetingGasLimitCalculator.class); - private final long maxConstantAdjustmentIncrement; public FrontierTargetingGasLimitCalculator() { - this(DEFAULT_MAX_CONSTANT_ADMUSTMENT_INCREMENT, DEFAULT_MIN_GAS_LIMIT, DEFAULT_MAX_GAS_LIMIT); + this(DEFAULT_MIN_GAS_LIMIT, DEFAULT_MAX_GAS_LIMIT); } - public FrontierTargetingGasLimitCalculator( - final long maxConstantAdjustmentIncrement, final long minGasLimit, final long maxGasLimit) { + public FrontierTargetingGasLimitCalculator(final long minGasLimit, final long maxGasLimit) { super(minGasLimit, maxGasLimit); - this.maxConstantAdjustmentIncrement = maxConstantAdjustmentIncrement; } @Override @@ -55,8 +52,7 @@ public class FrontierTargetingGasLimitCalculator extends AbstractGasLimitSpecifi } private long adjustAmount(final long currentGasLimit) { - final long maxProportionalAdjustmentLimit = Math.max(deltaBound(currentGasLimit) - 1, 0); - return Math.min(maxConstantAdjustmentIncrement, maxProportionalAdjustmentLimit); + return Math.max(deltaBound(currentGasLimit) - 1, 0); } protected long safeAddAtMost(final long gasLimit) { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/LondonTargetingGasLimitCalculator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/LondonTargetingGasLimitCalculator.java index 6e03a2523..3c86f7707 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/LondonTargetingGasLimitCalculator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/LondonTargetingGasLimitCalculator.java @@ -27,21 +27,15 @@ public class LondonTargetingGasLimitCalculator extends FrontierTargetingGasLimit public LondonTargetingGasLimitCalculator( final long londonForkBlock, final BaseFeeMarket feeMarket) { - this( - DEFAULT_MAX_CONSTANT_ADMUSTMENT_INCREMENT, - DEFAULT_MIN_GAS_LIMIT, - DEFAULT_MAX_GAS_LIMIT, - londonForkBlock, - feeMarket); + this(DEFAULT_MIN_GAS_LIMIT, DEFAULT_MAX_GAS_LIMIT, londonForkBlock, feeMarket); } public LondonTargetingGasLimitCalculator( - final long maxConstantAdjustmentIncrement, final long minGasLimit, final long maxGasLimit, final long londonForkBlock, final BaseFeeMarket feeMarket) { - super(maxConstantAdjustmentIncrement, minGasLimit, maxGasLimit); + super(minGasLimit, maxGasLimit); this.londonForkBlock = londonForkBlock; this.feeMarket = feeMarket; } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/TargetingGasLimitCalculatorTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/TargetingGasLimitCalculatorTest.java index b19d2310c..ef197143f 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/TargetingGasLimitCalculatorTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/TargetingGasLimitCalculatorTest.java @@ -27,22 +27,6 @@ import org.junit.jupiter.api.Test; public class TargetingGasLimitCalculatorTest { private static final long ADJUSTMENT_FACTOR = 1024L; - @Test - public void verifyGasLimitIsIncreasedWithinLimits() { - FrontierTargetingGasLimitCalculator targetingGasLimitCalculator = - new FrontierTargetingGasLimitCalculator(); - assertThat(targetingGasLimitCalculator.nextGasLimit(8_000_000L, 10_000_000L, 1L)) - .isEqualTo(8_000_000L + ADJUSTMENT_FACTOR); - } - - @Test - public void verifyGasLimitIsDecreasedWithinLimits() { - FrontierTargetingGasLimitCalculator targetingGasLimitCalculator = - new FrontierTargetingGasLimitCalculator(); - assertThat(targetingGasLimitCalculator.nextGasLimit(12_000_000L, 10_000_000L, 1L)) - .isEqualTo(12_000_000L - ADJUSTMENT_FACTOR); - } - @Test public void verifyGasLimitReachesTarget() { final long target = 10_000_000L; @@ -55,6 +39,33 @@ public class TargetingGasLimitCalculatorTest { .isEqualTo(target); } + @Test + public void verifyAdjustmentDeltas() { + assertDeltas(20000000L, 20019530L, 19980470L); + assertDeltas(40000000L, 40039061L, 39960939L); + } + + private void assertDeltas( + final long gasLimit, final long expectedIncrease, final long expectedDecrease) { + FrontierTargetingGasLimitCalculator targetingGasLimitCalculator = + new FrontierTargetingGasLimitCalculator(); + // increase + assertThat(targetingGasLimitCalculator.nextGasLimit(gasLimit, gasLimit * 2, 1L)) + .isEqualTo(expectedIncrease); + // decrease + assertThat(targetingGasLimitCalculator.nextGasLimit(gasLimit, 0, 1L)) + .isEqualTo(expectedDecrease); + // small decrease + assertThat(targetingGasLimitCalculator.nextGasLimit(gasLimit, gasLimit - 1, 1L)) + .isEqualTo(gasLimit - 1); + // small increase + assertThat(targetingGasLimitCalculator.nextGasLimit(gasLimit, gasLimit + 1, 1L)) + .isEqualTo(gasLimit + 1); + // no change + assertThat(targetingGasLimitCalculator.nextGasLimit(gasLimit, gasLimit, 1L)) + .isEqualTo(gasLimit); + } + @Test public void verifyMinGasLimit() { assertThat(AbstractGasLimitSpecification.isValidTargetGasLimit(DEFAULT_MIN_GAS_LIMIT - 1)) From 94d86afb40a6a59d86117c93a647719fd342991b Mon Sep 17 00:00:00 2001 From: Jason Frame Date: Fri, 19 Jan 2024 14:21:44 +1000 Subject: [PATCH 13/20] Bonsai keyvalue refactor (#6404) Refactor BonsaiWorldStateKeyValueStorage to use a FlatDbStrategyProvider Signed-off-by: Jason Frame --- .../storage/TrieLogSubCommand.java | 3 +- .../controller/BesuControllerBuilder.java | 3 +- .../storage/TrieLogHelperTest.java | 3 +- .../controller/BesuControllerBuilderTest.java | 17 ++- .../MergeBesuControllerBuilderTest.java | 4 +- .../QbftBesuControllerBuilderTest.java | 4 +- .../ethereum/storage/StorageProvider.java | 4 +- .../keyvalue/KeyValueStorageProvider.java | 8 +- .../trie/bonsai/BonsaiWorldStateProvider.java | 5 +- .../cache/CachedWorldStorageManager.java | 20 +-- .../cache/NoOpCachedWorldStorageManager.java | 3 +- ...nsaiSnapshotWorldStateKeyValueStorage.java | 18 +-- .../BonsaiWorldStateKeyValueStorage.java | 117 ++++++------------ .../storage/BonsaiWorldStateLayerStorage.java | 12 +- .../storage/flat/FlatDbStrategyProvider.java | 105 ++++++++++++++++ .../common/GenesisWorldStateProvider.java | 4 +- .../core/InMemoryKeyValueStorageProvider.java | 15 ++- .../BlockImportExceptionHandlingTest.java | 4 +- .../trie/bonsai/AbstractIsolationTests.java | 11 +- .../bonsai/BonsaiWorldStateArchiveTest.java | 25 ++-- .../bonsai/CachedMerkleTrieLoaderTest.java | 13 +- .../ethereum/trie/bonsai/LogRollingTests.java | 19 ++- .../ethereum/trie/bonsai/RollingImport.java | 4 +- .../BonsaiWorldStateKeyValueStorageTest.java | 8 +- .../flat/FlatDbStrategyProviderTest.java | 89 +++++++++++++ .../WorldStateDownloaderBenchmark.java | 5 +- .../FastWorldDownloadStateTest.java | 5 +- .../worldstate/PersistDataStepTest.java | 5 +- .../snapsync/AccountHealingTrackingTest.java | 5 +- .../sync/snapsync/PersistDataStepTest.java | 5 +- .../snapsync/SnapWorldDownloadStateTest.java | 5 +- .../eth/sync/snapsync/TaskGenerator.java | 5 +- ...ntFlatDatabaseHealingRangeRequestTest.java | 7 +- ...geFlatDatabaseHealingRangeRequestTest.java | 4 +- .../StorageTrieNodeHealingRequestTest.java | 4 +- .../BonsaiReferenceTestWorldState.java | 6 +- 36 files changed, 388 insertions(+), 186 deletions(-) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProvider.java create mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index bf75fd6eb..74e00197f 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -139,8 +139,7 @@ public class TrieLogSubCommand implements Runnable { final StorageProvider storageProvider = besuController.getStorageProvider(); final BonsaiWorldStateKeyValueStorage rootWorldStateStorage = - (BonsaiWorldStateKeyValueStorage) - storageProvider.createWorldStateStorage(DataStorageFormat.BONSAI); + (BonsaiWorldStateKeyValueStorage) storageProvider.createWorldStateStorage(config); final MutableBlockchain blockchain = besuController.getProtocolContext().getBlockchain(); return new TrieLogContext(config, rootWorldStateStorage, blockchain); } diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index f2d3fce98..f2ecbe85c 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -598,7 +598,7 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides final VariablesStorage variablesStorage = storageProvider.createVariablesStorage(); final WorldStateStorage worldStateStorage = - storageProvider.createWorldStateStorage(dataStorageConfiguration.getDataStorageFormat()); + storageProvider.createWorldStateStorage(dataStorageConfiguration); final BlockchainStorage blockchainStorage = storageProvider.createBlockchainStorage(protocolSchedule, variablesStorage); @@ -1088,7 +1088,6 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides blockchain, Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()), cachedMerkleTrieLoader, - metricsSystem, besuComponent.map(BesuComponent::getBesuPluginContext).orElse(null), evmConfiguration, trieLogPruner); diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java index 82659701b..0ba575a9e 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -75,7 +75,8 @@ class TrieLogHelperTest { blockHeader5 = new BlockHeaderTestFixture().number(5).buildHeader(); inMemoryWorldState = - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); var updater = inMemoryWorldState.updater(); updater diff --git a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java index dd9452e78..9adf8da23 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java @@ -131,7 +131,7 @@ public class BesuControllerBuilderTest { when(synchronizerConfiguration.getBlockPropagationRange()).thenReturn(Range.closed(1L, 2L)); lenient() - .when(storageProvider.createWorldStateStorage(DataStorageFormat.FOREST)) + .when(storageProvider.createWorldStateStorage(DataStorageConfiguration.DEFAULT_CONFIG)) .thenReturn(worldStateStorage); lenient() .when(storageProvider.createWorldStatePreimageStorage()) @@ -166,6 +166,11 @@ public class BesuControllerBuilderTest { @Test public void shouldDisablePruningIfBonsaiIsEnabled() { + DataStorageConfiguration dataStorageConfiguration = + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(DataStorageFormat.BONSAI) + .bonsaiMaxLayersToLoad(DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD) + .build(); BonsaiWorldState mockWorldState = mock(BonsaiWorldState.class, Answers.RETURNS_DEEP_STUBS); doReturn(worldStateArchive) .when(besuControllerBuilder) @@ -173,15 +178,9 @@ public class BesuControllerBuilderTest { any(WorldStateStorage.class), any(Blockchain.class), any(CachedMerkleTrieLoader.class)); doReturn(mockWorldState).when(worldStateArchive).getMutable(); - when(storageProvider.createWorldStateStorage(DataStorageFormat.BONSAI)) + when(storageProvider.createWorldStateStorage(dataStorageConfiguration)) .thenReturn(bonsaiWorldStateStorage); - besuControllerBuilder - .isPruningEnabled(true) - .dataStorageConfiguration( - ImmutableDataStorageConfiguration.builder() - .dataStorageFormat(DataStorageFormat.BONSAI) - .bonsaiMaxLayersToLoad(DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD) - .build()); + besuControllerBuilder.isPruningEnabled(true).dataStorageConfiguration(dataStorageConfiguration); besuControllerBuilder.build(); verify(storageProvider, never()) diff --git a/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java index 4368e8313..9506a35f4 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java @@ -52,7 +52,7 @@ import org.hyperledger.besu.ethereum.p2p.config.NetworkingConfiguration; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStoragePrefixedKeyBlockchainStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; @@ -145,7 +145,7 @@ public class MergeBesuControllerBuilderTest { .thenReturn(Range.closed(1L, 2L)); lenient() - .when(storageProvider.createWorldStateStorage(DataStorageFormat.FOREST)) + .when(storageProvider.createWorldStateStorage(DataStorageConfiguration.DEFAULT_CONFIG)) .thenReturn(worldStateStorage); lenient() .when(storageProvider.createWorldStatePreimageStorage()) diff --git a/besu/src/test/java/org/hyperledger/besu/controller/QbftBesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/QbftBesuControllerBuilderTest.java index 75fecf3f9..fc1b2f55f 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/QbftBesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/QbftBesuControllerBuilderTest.java @@ -48,7 +48,7 @@ import org.hyperledger.besu.ethereum.p2p.config.NetworkingConfiguration; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStoragePrefixedKeyBlockchainStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.evm.internal.EvmConfiguration; @@ -114,7 +114,7 @@ public class QbftBesuControllerBuilderTest { new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), new MainnetBlockHeaderFunctions())); lenient() - .when(storageProvider.createWorldStateStorage(DataStorageFormat.FOREST)) + .when(storageProvider.createWorldStateStorage(DataStorageConfiguration.DEFAULT_CONFIG)) .thenReturn(worldStateStorage); lenient().when(worldStateStorage.isWorldStateAvailable(any(), any())).thenReturn(true); lenient().when(worldStateStorage.updater()).thenReturn(mock(WorldStateStorage.Updater.class)); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/StorageProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/StorageProvider.java index 6cc9741cb..2f3cd3dff 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/StorageProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/StorageProvider.java @@ -17,7 +17,7 @@ package org.hyperledger.besu.ethereum.storage; import org.hyperledger.besu.ethereum.chain.BlockchainStorage; import org.hyperledger.besu.ethereum.chain.VariablesStorage; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; @@ -34,7 +34,7 @@ public interface StorageProvider extends Closeable { BlockchainStorage createBlockchainStorage( ProtocolSchedule protocolSchedule, VariablesStorage variablesStorage); - WorldStateStorage createWorldStateStorage(DataStorageFormat dataStorageFormat); + WorldStateStorage createWorldStateStorage(DataStorageConfiguration dataStorageFormat); WorldStatePreimageStorage createWorldStatePreimageStorage(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java index dea6b932c..44c15e815 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java @@ -21,6 +21,7 @@ import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; @@ -75,9 +76,10 @@ public class KeyValueStorageProvider implements StorageProvider { } @Override - public WorldStateStorage createWorldStateStorage(final DataStorageFormat dataStorageFormat) { - if (dataStorageFormat.equals(DataStorageFormat.BONSAI)) { - return new BonsaiWorldStateKeyValueStorage(this, metricsSystem); + public WorldStateStorage createWorldStateStorage( + final DataStorageConfiguration dataStorageConfiguration) { + if (dataStorageConfiguration.getDataStorageFormat().equals(DataStorageFormat.BONSAI)) { + return new BonsaiWorldStateKeyValueStorage(this, metricsSystem, dataStorageConfiguration); } else { return new ForestWorldStateKeyValueStorage( getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.WORLD_STATE)); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java index 4c783888c..4f1b6d65b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java @@ -39,7 +39,6 @@ import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.worldstate.WorldState; -import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; @@ -73,13 +72,11 @@ public class BonsaiWorldStateProvider implements WorldStateArchive { final Blockchain blockchain, final Optional maxLayersToLoad, final CachedMerkleTrieLoader cachedMerkleTrieLoader, - final ObservableMetricsSystem metricsSystem, final BesuContext pluginContext, final EvmConfiguration evmConfiguration, final TrieLogPruner trieLogPruner) { - this.cachedWorldStorageManager = - new CachedWorldStorageManager(this, worldStateStorage, metricsSystem); + this.cachedWorldStorageManager = new CachedWorldStorageManager(this, worldStateStorage); // TODO: de-dup constructors this.trieLogManager = new TrieLogManager( diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedWorldStorageManager.java index 8a09c136a..6ca38a27a 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedWorldStorageManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedWorldStorageManager.java @@ -22,7 +22,6 @@ import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValu import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateLayerStorage; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.evm.internal.EvmConfiguration; -import org.hyperledger.besu.metrics.ObservableMetricsSystem; import java.util.ArrayList; import java.util.Comparator; @@ -41,7 +40,6 @@ public class CachedWorldStorageManager public static final long RETAINED_LAYERS = 512; // at least 256 + typical rollbacks private static final Logger LOG = LoggerFactory.getLogger(CachedWorldStorageManager.class); private final BonsaiWorldStateProvider archive; - private final ObservableMetricsSystem metricsSystem; private final EvmConfiguration evmConfiguration; private final BonsaiWorldStateKeyValueStorage rootWorldStateStorage; @@ -51,26 +49,18 @@ public class CachedWorldStorageManager final BonsaiWorldStateProvider archive, final BonsaiWorldStateKeyValueStorage worldStateStorage, final Map cachedWorldStatesByHash, - final ObservableMetricsSystem metricsSystem, final EvmConfiguration evmConfiguration) { worldStateStorage.subscribe(this); this.rootWorldStateStorage = worldStateStorage; this.cachedWorldStatesByHash = cachedWorldStatesByHash; this.archive = archive; - this.metricsSystem = metricsSystem; this.evmConfiguration = evmConfiguration; } public CachedWorldStorageManager( final BonsaiWorldStateProvider archive, - final BonsaiWorldStateKeyValueStorage worldStateStorage, - final ObservableMetricsSystem metricsSystem) { - this( - archive, - worldStateStorage, - new ConcurrentHashMap<>(), - metricsSystem, - EvmConfiguration.DEFAULT); + final BonsaiWorldStateKeyValueStorage worldStateStorage) { + this(archive, worldStateStorage, new ConcurrentHashMap<>(), EvmConfiguration.DEFAULT); } public synchronized void addCachedLayer( @@ -92,8 +82,7 @@ public class CachedWorldStorageManager cachedBonsaiWorldView .get() .updateWorldStateStorage( - new BonsaiSnapshotWorldStateKeyValueStorage( - forWorldState.getWorldStateStorage(), metricsSystem)); + new BonsaiSnapshotWorldStateKeyValueStorage(forWorldState.getWorldStateStorage())); } } else { LOG.atDebug() @@ -106,8 +95,7 @@ public class CachedWorldStorageManager blockHeader.getHash(), new CachedBonsaiWorldView( blockHeader, - new BonsaiSnapshotWorldStateKeyValueStorage( - forWorldState.getWorldStateStorage(), metricsSystem))); + new BonsaiSnapshotWorldStateKeyValueStorage(forWorldState.getWorldStateStorage()))); } else { // otherwise, add the layer to the cache cachedWorldStatesByHash.put( diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java index 0359c4642..d86128b13 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java @@ -18,7 +18,6 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import java.util.Optional; import java.util.function.Function; @@ -27,7 +26,7 @@ public class NoOpCachedWorldStorageManager extends CachedWorldStorageManager { public NoOpCachedWorldStorageManager( final BonsaiWorldStateKeyValueStorage bonsaiWorldStateKeyValueStorage) { - super(null, bonsaiWorldStateKeyValueStorage, new NoOpMetricsSystem()); + super(null, bonsaiWorldStateKeyValueStorage); } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java index 69c74ec27..b9643cdc4 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java @@ -18,7 +18,6 @@ package org.hyperledger.besu.ethereum.trie.bonsai.storage; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; -import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.services.exception.StorageException; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.SnappableKeyValueStorage; @@ -43,26 +42,19 @@ public class BonsaiSnapshotWorldStateKeyValueStorage extends BonsaiWorldStateKey public BonsaiSnapshotWorldStateKeyValueStorage( final BonsaiWorldStateKeyValueStorage parentWorldStateStorage, final SnappedKeyValueStorage segmentedWorldStateStorage, - final KeyValueStorage trieLogStorage, - final ObservableMetricsSystem metricsSystem) { + final KeyValueStorage trieLogStorage) { super( - parentWorldStateStorage.flatDbMode, - parentWorldStateStorage.flatDbStrategy, - segmentedWorldStateStorage, - trieLogStorage, - metricsSystem); + parentWorldStateStorage.flatDbStrategyProvider, segmentedWorldStateStorage, trieLogStorage); this.parentWorldStateStorage = parentWorldStateStorage; this.subscribeParentId = parentWorldStateStorage.subscribe(this); } public BonsaiSnapshotWorldStateKeyValueStorage( - final BonsaiWorldStateKeyValueStorage worldStateStorage, - final ObservableMetricsSystem metricsSystem) { + final BonsaiWorldStateKeyValueStorage worldStateStorage) { this( worldStateStorage, ((SnappableKeyValueStorage) worldStateStorage.composedWorldStateStorage).takeSnapshot(), - worldStateStorage.trieLogStorage, - metricsSystem); + worldStateStorage.trieLogStorage); } private boolean isClosedGet() { @@ -78,7 +70,7 @@ public class BonsaiSnapshotWorldStateKeyValueStorage extends BonsaiWorldStateKey return new Updater( ((SnappedKeyValueStorage) composedWorldStateStorage).getSnapshotTransaction(), trieLogStorage.startTransaction(), - flatDbStrategy); + getFlatDbStrategy()); } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java index 97384fd6c..efe97f32c 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java @@ -25,14 +25,14 @@ import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.bonsai.storage.flat.FlatDbStrategy; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.flat.FullFlatDbStrategy; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.flat.PartialFlatDbStrategy; +import org.hyperledger.besu.ethereum.trie.bonsai.storage.flat.FlatDbStrategyProvider; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.evm.account.AccountStorageEntry; -import org.hyperledger.besu.metrics.ObservableMetricsSystem; +import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; @@ -64,17 +64,11 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC public static final byte[] WORLD_BLOCK_HASH_KEY = "worldBlockHash".getBytes(StandardCharsets.UTF_8); - // 0x666C61744462537461747573 - public static final byte[] FLAT_DB_MODE = "flatDbStatus".getBytes(StandardCharsets.UTF_8); - - protected FlatDbMode flatDbMode; - protected FlatDbStrategy flatDbStrategy; + protected final FlatDbStrategyProvider flatDbStrategyProvider; protected final SegmentedKeyValueStorage composedWorldStateStorage; protected final KeyValueStorage trieLogStorage; - protected final ObservableMetricsSystem metricsSystem; - private final AtomicBoolean shouldClose = new AtomicBoolean(false); protected final AtomicBoolean isClosed = new AtomicBoolean(false); @@ -82,62 +76,27 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC protected final Subscribers subscribers = Subscribers.create(); public BonsaiWorldStateKeyValueStorage( - final StorageProvider provider, final ObservableMetricsSystem metricsSystem) { + final StorageProvider provider, + final MetricsSystem metricsSystem, + final DataStorageConfiguration dataStorageConfiguration) { this.composedWorldStateStorage = provider.getStorageBySegmentIdentifiers( List.of( ACCOUNT_INFO_STATE, CODE_STORAGE, ACCOUNT_STORAGE_STORAGE, TRIE_BRANCH_STORAGE)); this.trieLogStorage = provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE); - this.metricsSystem = metricsSystem; - loadFlatDbStrategy(); + this.flatDbStrategyProvider = + new FlatDbStrategyProvider(metricsSystem, dataStorageConfiguration); + flatDbStrategyProvider.loadFlatDbStrategy(composedWorldStateStorage); } public BonsaiWorldStateKeyValueStorage( - final FlatDbMode flatDbMode, - final FlatDbStrategy flatDbStrategy, + final FlatDbStrategyProvider flatDbStrategyProvider, final SegmentedKeyValueStorage composedWorldStateStorage, - final KeyValueStorage trieLogStorage, - final ObservableMetricsSystem metricsSystem) { - this.flatDbMode = flatDbMode; - this.flatDbStrategy = flatDbStrategy; + final KeyValueStorage trieLogStorage) { + this.flatDbStrategyProvider = flatDbStrategyProvider; this.composedWorldStateStorage = composedWorldStateStorage; this.trieLogStorage = trieLogStorage; - this.metricsSystem = metricsSystem; - } - - private void loadFlatDbStrategy() { - // derive our flatdb strategy from db or default: - var newFlatDbMode = deriveFlatDbStrategy(); - - // if flatDbMode is not loaded or has changed, reload flatDbStrategy - if (this.flatDbMode == null || !this.flatDbMode.equals(newFlatDbMode)) { - this.flatDbMode = newFlatDbMode; - if (flatDbMode == FlatDbMode.FULL) { - this.flatDbStrategy = new FullFlatDbStrategy(metricsSystem); - } else { - this.flatDbStrategy = new PartialFlatDbStrategy(metricsSystem); - } - } - } - - public FlatDbMode deriveFlatDbStrategy() { - var flatDbMode = - FlatDbMode.fromVersion( - composedWorldStateStorage - .get(TRIE_BRANCH_STORAGE, FLAT_DB_MODE) - .map(Bytes::wrap) - .orElse(FlatDbMode.PARTIAL.getVersion())); - LOG.info("Bonsai flat db mode found {}", flatDbMode); - - return flatDbMode; - } - - public FlatDbStrategy getFlatDbStrategy() { - if (flatDbStrategy == null) { - loadFlatDbStrategy(); - } - return flatDbStrategy; } @Override @@ -147,7 +106,7 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC @Override public FlatDbMode getFlatDbMode() { - return flatDbMode; + return flatDbStrategyProvider.getFlatDbMode(); } @Override @@ -155,12 +114,15 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC if (codeHash.equals(Hash.EMPTY)) { return Optional.of(Bytes.EMPTY); } else { - return getFlatDbStrategy().getFlatCode(codeHash, accountHash, composedWorldStateStorage); + return flatDbStrategyProvider + .getFlatDbStrategy(composedWorldStateStorage) + .getFlatCode(codeHash, accountHash, composedWorldStateStorage); } } public Optional getAccount(final Hash accountHash) { - return getFlatDbStrategy() + return flatDbStrategyProvider + .getFlatDbStrategy(composedWorldStateStorage) .getFlatAccount( this::getWorldStateRootHash, this::getAccountStateTrieNode, @@ -243,7 +205,8 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC final Supplier> storageRootSupplier, final Hash accountHash, final StorageSlotKey storageSlotKey) { - return getFlatDbStrategy() + return flatDbStrategyProvider + .getFlatDbStrategy(composedWorldStateStorage) .getFlatStorageValueByStorageSlotKey( this::getWorldStateRootHash, storageRootSupplier, @@ -256,14 +219,16 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC @Override public Map streamFlatAccounts( final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { - return getFlatDbStrategy() + return flatDbStrategyProvider + .getFlatDbStrategy(composedWorldStateStorage) .streamAccountFlatDatabase(composedWorldStateStorage, startKeyHash, endKeyHash, max); } @Override public Map streamFlatStorages( final Hash accountHash, final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { - return getFlatDbStrategy() + return flatDbStrategyProvider + .getFlatDbStrategy(composedWorldStateStorage) .streamStorageFlatDatabase( composedWorldStateStorage, accountHash, startKeyHash, endKeyHash, max); } @@ -288,31 +253,23 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC } public void upgradeToFullFlatDbMode() { - final SegmentedKeyValueStorageTransaction transaction = - composedWorldStateStorage.startTransaction(); - // TODO: consider ARCHIVE mode - transaction.put( - TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.FULL.getVersion().toArrayUnsafe()); - transaction.commit(); - loadFlatDbStrategy(); // force reload of flat db reader strategy + flatDbStrategyProvider.upgradeToFullFlatDbMode(composedWorldStateStorage); } public void downgradeToPartialFlatDbMode() { - final SegmentedKeyValueStorageTransaction transaction = - composedWorldStateStorage.startTransaction(); - transaction.put( - TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.PARTIAL.getVersion().toArrayUnsafe()); - transaction.commit(); - loadFlatDbStrategy(); // force reload of flat db reader strategy + flatDbStrategyProvider.downgradeToPartialFlatDbMode(composedWorldStateStorage); } @Override public void clear() { subscribers.forEach(BonsaiStorageSubscriber::onClearStorage); - getFlatDbStrategy().clearAll(composedWorldStateStorage); + flatDbStrategyProvider + .getFlatDbStrategy(composedWorldStateStorage) + .clearAll(composedWorldStateStorage); composedWorldStateStorage.clear(TRIE_BRANCH_STORAGE); trieLogStorage.clear(); - loadFlatDbStrategy(); // force reload of flat db reader strategy + flatDbStrategyProvider.loadFlatDbStrategy( + composedWorldStateStorage); // force reload of flat db reader strategy } @Override @@ -324,7 +281,9 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC @Override public void clearFlatDatabase() { subscribers.forEach(BonsaiStorageSubscriber::onClearFlatDatabaseStorage); - getFlatDbStrategy().resetOnResync(composedWorldStateStorage); + flatDbStrategyProvider + .getFlatDbStrategy(composedWorldStateStorage) + .resetOnResync(composedWorldStateStorage); } @Override @@ -332,7 +291,7 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC return new Updater( composedWorldStateStorage.startTransaction(), trieLogStorage.startTransaction(), - flatDbStrategy); + flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage)); } @Override @@ -359,6 +318,10 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC throw new RuntimeException("removeNodeAddedListener not available"); } + public FlatDbStrategy getFlatDbStrategy() { + return flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage); + } + public interface BonsaiUpdater extends WorldStateStorage.Updater { BonsaiUpdater removeCode(final Hash accountHash); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateLayerStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateLayerStorage.java index 0dd61b8d7..92d4f54fe 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateLayerStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateLayerStorage.java @@ -17,7 +17,6 @@ package org.hyperledger.besu.ethereum.trie.bonsai.storage; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; -import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.SnappedKeyValueStorage; import org.hyperledger.besu.services.kvstore.LayeredKeyValueStorage; @@ -29,16 +28,14 @@ public class BonsaiWorldStateLayerStorage extends BonsaiSnapshotWorldStateKeyVal this( new LayeredKeyValueStorage(parent.composedWorldStateStorage), parent.trieLogStorage, - parent, - parent.metricsSystem); + parent); } public BonsaiWorldStateLayerStorage( final SnappedKeyValueStorage composedWorldStateStorage, final KeyValueStorage trieLogStorage, - final BonsaiWorldStateKeyValueStorage parent, - final ObservableMetricsSystem metricsSystem) { - super(parent, composedWorldStateStorage, trieLogStorage, metricsSystem); + final BonsaiWorldStateKeyValueStorage parent) { + super(parent, composedWorldStateStorage, trieLogStorage); } @Override @@ -51,7 +48,6 @@ public class BonsaiWorldStateLayerStorage extends BonsaiSnapshotWorldStateKeyVal return new BonsaiWorldStateLayerStorage( ((LayeredKeyValueStorage) composedWorldStateStorage).clone(), trieLogStorage, - parentWorldStateStorage, - metricsSystem); + parentWorldStateStorage); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProvider.java new file mode 100644 index 000000000..55f89fc66 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProvider.java @@ -0,0 +1,105 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; + +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; + +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; +import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; +import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; + +import java.nio.charset.StandardCharsets; + +import org.apache.tuweni.bytes.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FlatDbStrategyProvider { + private static final Logger LOG = LoggerFactory.getLogger(FlatDbStrategyProvider.class); + + // 0x666C61744462537461747573 + public static final byte[] FLAT_DB_MODE = "flatDbStatus".getBytes(StandardCharsets.UTF_8); + private final MetricsSystem metricsSystem; + protected FlatDbMode flatDbMode; + protected FlatDbStrategy flatDbStrategy; + + public FlatDbStrategyProvider( + final MetricsSystem metricsSystem, final DataStorageConfiguration dataStorageConfiguration) { + this.metricsSystem = metricsSystem; + } + + public void loadFlatDbStrategy(final SegmentedKeyValueStorage composedWorldStateStorage) { + // derive our flatdb strategy from db or default: + var newFlatDbMode = deriveFlatDbStrategy(composedWorldStateStorage); + + // if flatDbMode is not loaded or has changed, reload flatDbStrategy + if (this.flatDbMode == null || !this.flatDbMode.equals(newFlatDbMode)) { + this.flatDbMode = newFlatDbMode; + if (flatDbMode == FlatDbMode.FULL) { + this.flatDbStrategy = new FullFlatDbStrategy(metricsSystem); + } else { + this.flatDbStrategy = new PartialFlatDbStrategy(metricsSystem); + } + } + } + + private FlatDbMode deriveFlatDbStrategy( + final SegmentedKeyValueStorage composedWorldStateStorage) { + var flatDbMode = + FlatDbMode.fromVersion( + composedWorldStateStorage + .get(TRIE_BRANCH_STORAGE, FLAT_DB_MODE) + .map(Bytes::wrap) + .orElse(FlatDbMode.PARTIAL.getVersion())); + LOG.info("Bonsai flat db mode found {}", flatDbMode); + + return flatDbMode; + } + + public FlatDbStrategy getFlatDbStrategy( + final SegmentedKeyValueStorage composedWorldStateStorage) { + if (flatDbStrategy == null) { + loadFlatDbStrategy(composedWorldStateStorage); + } + return flatDbStrategy; + } + + public void upgradeToFullFlatDbMode(final SegmentedKeyValueStorage composedWorldStateStorage) { + final SegmentedKeyValueStorageTransaction transaction = + composedWorldStateStorage.startTransaction(); + // TODO: consider ARCHIVE mode + transaction.put( + TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.FULL.getVersion().toArrayUnsafe()); + transaction.commit(); + loadFlatDbStrategy(composedWorldStateStorage); // force reload of flat db reader strategy + } + + public void downgradeToPartialFlatDbMode( + final SegmentedKeyValueStorage composedWorldStateStorage) { + final SegmentedKeyValueStorageTransaction transaction = + composedWorldStateStorage.startTransaction(); + transaction.put( + TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.PARTIAL.getVersion().toArrayUnsafe()); + transaction.commit(); + loadFlatDbStrategy(composedWorldStateStorage); // force reload of flat db reader strategy + } + + public FlatDbMode getFlatDbMode() { + return flatDbMode; + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java index ad1e920b7..6e61b5595 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java @@ -26,6 +26,7 @@ import org.hyperledger.besu.ethereum.trie.bonsai.trielog.NoOpTrieLogManager; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -65,7 +66,8 @@ public class GenesisWorldStateProvider { segmentIdentifiers -> new SegmentedInMemoryKeyValueStorage(), new InMemoryKeyValueStorage(), new NoOpMetricsSystem()), - new NoOpMetricsSystem()); + new NoOpMetricsSystem(), + DataStorageConfiguration.DEFAULT_CONFIG); return new BonsaiWorldState( bonsaiWorldStateKeyValueStorage, cachedMerkleTrieLoader, diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java index 514c88210..21f32441b 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java @@ -14,6 +14,8 @@ */ package org.hyperledger.besu.ethereum.core; +import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; + import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.DefaultBlockchain; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; @@ -32,7 +34,9 @@ import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.trie.forest.ForestWorldStateArchive; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; @@ -96,13 +100,18 @@ public class InMemoryKeyValueStorageProvider extends KeyValueStorageProvider { new InMemoryKeyValueStorageProvider(); final CachedMerkleTrieLoader cachedMerkleTrieLoader = new CachedMerkleTrieLoader(new NoOpMetricsSystem()); + final DataStorageConfiguration bonsaiDataStorageConfig = + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(DataStorageFormat.BONSAI) + .bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD) + .unstable(DataStorageConfiguration.Unstable.DEFAULT) + .build(); return new BonsaiWorldStateProvider( (BonsaiWorldStateKeyValueStorage) - inMemoryKeyValueStorageProvider.createWorldStateStorage(DataStorageFormat.BONSAI), + inMemoryKeyValueStorageProvider.createWorldStateStorage(bonsaiDataStorageConfig), blockchain, Optional.empty(), cachedMerkleTrieLoader, - new NoOpMetricsSystem(), null, evmConfiguration, TrieLogPruner.noOpTrieLogPruner()); @@ -111,7 +120,7 @@ public class InMemoryKeyValueStorageProvider extends KeyValueStorageProvider { public static MutableWorldState createInMemoryWorldState() { final InMemoryKeyValueStorageProvider provider = new InMemoryKeyValueStorageProvider(); return new ForestMutableWorldState( - provider.createWorldStateStorage(DataStorageFormat.FOREST), + provider.createWorldStateStorage(DataStorageConfiguration.DEFAULT_CONFIG), provider.createWorldStatePreimageStorage(), EvmConfiguration.DEFAULT); } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java index f8ac492c9..ecf17da90 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java @@ -44,6 +44,7 @@ import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.evm.internal.EvmConfiguration; @@ -80,7 +81,8 @@ class BlockImportExceptionHandlingTest { private final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); private final WorldStateStorage worldStateStorage = - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); private final WorldStateArchive worldStateArchive = // contains a BonsaiWorldState which we need to spy on. diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java index a77c51269..eba09987d 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java @@ -68,7 +68,9 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBui import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.BesuConfiguration; @@ -147,14 +149,19 @@ public abstract class AbstractIsolationTests { public void createStorage() { bonsaiWorldStateStorage = (BonsaiWorldStateKeyValueStorage) - createKeyValueStorageProvider().createWorldStateStorage(DataStorageFormat.BONSAI); + createKeyValueStorageProvider() + .createWorldStateStorage( + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(DataStorageFormat.BONSAI) + .bonsaiMaxLayersToLoad( + DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD) + .build()); archive = new BonsaiWorldStateProvider( bonsaiWorldStateStorage, blockchain, Optional.of(16L), new CachedMerkleTrieLoader(new NoOpMetricsSystem()), - new NoOpMetricsSystem(), null, EvmConfiguration.DEFAULT, TrieLogPruner.noOpTrieLogPruner()); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateArchiveTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateArchiveTest.java index 6afde3c94..7c1391029 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateArchiveTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateArchiveTest.java @@ -44,6 +44,7 @@ import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; @@ -106,7 +107,8 @@ class BonsaiWorldStateArchiveTest { new BonsaiWorldStateProvider( cachedWorldStorageManager, trieLogManager, - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()), + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG), blockchain, new CachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT); @@ -119,11 +121,11 @@ class BonsaiWorldStateArchiveTest { void testGetMutableReturnEmptyWhenLoadMoreThanLimitLayersBack() { bonsaiWorldStateArchive = new BonsaiWorldStateProvider( - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()), + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG), blockchain, Optional.of(512L), new CachedMerkleTrieLoader(new NoOpMetricsSystem()), - new NoOpMetricsSystem(), null, EvmConfiguration.DEFAULT, TrieLogPruner.noOpTrieLogPruner()); @@ -141,7 +143,8 @@ class BonsaiWorldStateArchiveTest { new BonsaiWorldStateProvider( cachedWorldStorageManager, trieLogManager, - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()), + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG), blockchain, new CachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT); @@ -167,7 +170,8 @@ class BonsaiWorldStateArchiveTest { .getTrieLogLayer(any(Hash.class)); var worldStateStorage = - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); bonsaiWorldStateArchive = spy( new BonsaiWorldStateProvider( @@ -193,7 +197,8 @@ class BonsaiWorldStateArchiveTest { void testGetMutableWithStorageConsistencyNotRollbackTheState() { var worldStateStorage = - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); bonsaiWorldStateArchive = spy( new BonsaiWorldStateProvider( @@ -229,7 +234,8 @@ class BonsaiWorldStateArchiveTest { .getTrieLogLayer(any(Hash.class)); var worldStateStorage = - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); bonsaiWorldStateArchive = spy( @@ -276,7 +282,10 @@ class BonsaiWorldStateArchiveTest { new BonsaiWorldStateProvider( cachedWorldStorageManager, trieLogManager, - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()), + new BonsaiWorldStateKeyValueStorage( + storageProvider, + new NoOpMetricsSystem(), + DataStorageConfiguration.DEFAULT_CONFIG), blockchain, new CachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT)); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/CachedMerkleTrieLoaderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/CachedMerkleTrieLoaderTest.java index 758d70881..59dfacb5f 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/CachedMerkleTrieLoaderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/CachedMerkleTrieLoaderTest.java @@ -29,6 +29,7 @@ import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -48,7 +49,9 @@ class CachedMerkleTrieLoaderTest { private CachedMerkleTrieLoader merkleTrieLoader; private final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); private final BonsaiWorldStateKeyValueStorage inMemoryWorldState = - Mockito.spy(new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem())); + Mockito.spy( + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG)); final List
accounts = List.of(Address.fromHexString("0xdeadbeef"), Address.fromHexString("0xdeadbeee")); @@ -71,7 +74,9 @@ class CachedMerkleTrieLoaderTest { final BonsaiWorldStateKeyValueStorage emptyStorage = new BonsaiWorldStateKeyValueStorage( - new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem()); + new InMemoryKeyValueStorageProvider(), + new NoOpMetricsSystem(), + DataStorageConfiguration.DEFAULT_CONFIG); StoredMerklePatriciaTrie cachedTrie = new StoredMerklePatriciaTrie<>( (location, hash) -> @@ -110,7 +115,9 @@ class CachedMerkleTrieLoaderTest { final List cachedSlots = new ArrayList<>(); final BonsaiWorldStateKeyValueStorage emptyStorage = new BonsaiWorldStateKeyValueStorage( - new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem()); + new InMemoryKeyValueStorageProvider(), + new NoOpMetricsSystem(), + DataStorageConfiguration.DEFAULT_CONFIG); final StoredMerklePatriciaTrie cachedTrie = new StoredMerklePatriciaTrie<>( (location, hash) -> diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/LogRollingTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/LogRollingTests.java index f2dad3bc6..26d0d1302 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/LogRollingTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/LogRollingTests.java @@ -34,6 +34,7 @@ import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.log.LogsBloomFilter; @@ -161,7 +162,8 @@ class LogRollingTests { final BonsaiWorldState worldState = new BonsaiWorldState( archive, - new BonsaiWorldStateKeyValueStorage(provider, new NoOpMetricsSystem()), + new BonsaiWorldStateKeyValueStorage( + provider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG), EvmConfiguration.DEFAULT); final WorldUpdater updater = worldState.updater(); @@ -174,7 +176,8 @@ class LogRollingTests { final BonsaiWorldState secondWorldState = new BonsaiWorldState( secondArchive, - new BonsaiWorldStateKeyValueStorage(secondProvider, new NoOpMetricsSystem()), + new BonsaiWorldStateKeyValueStorage( + secondProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG), EvmConfiguration.DEFAULT); final BonsaiWorldStateUpdateAccumulator secondUpdater = (BonsaiWorldStateUpdateAccumulator) secondWorldState.updater(); @@ -205,7 +208,8 @@ class LogRollingTests { final BonsaiWorldState worldState = new BonsaiWorldState( archive, - new BonsaiWorldStateKeyValueStorage(provider, new NoOpMetricsSystem()), + new BonsaiWorldStateKeyValueStorage( + provider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG), EvmConfiguration.DEFAULT); final WorldUpdater updater = worldState.updater(); @@ -226,7 +230,8 @@ class LogRollingTests { final BonsaiWorldState secondWorldState = new BonsaiWorldState( secondArchive, - new BonsaiWorldStateKeyValueStorage(secondProvider, new NoOpMetricsSystem()), + new BonsaiWorldStateKeyValueStorage( + secondProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG), EvmConfiguration.DEFAULT); final BonsaiWorldStateUpdateAccumulator secondUpdater = (BonsaiWorldStateUpdateAccumulator) secondWorldState.updater(); @@ -258,7 +263,8 @@ class LogRollingTests { final BonsaiWorldState worldState = new BonsaiWorldState( archive, - new BonsaiWorldStateKeyValueStorage(provider, new NoOpMetricsSystem()), + new BonsaiWorldStateKeyValueStorage( + provider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG), EvmConfiguration.DEFAULT); final WorldUpdater updater = worldState.updater(); @@ -286,7 +292,8 @@ class LogRollingTests { final BonsaiWorldState secondWorldState = new BonsaiWorldState( secondArchive, - new BonsaiWorldStateKeyValueStorage(secondProvider, new NoOpMetricsSystem()), + new BonsaiWorldStateKeyValueStorage( + secondProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG), EvmConfiguration.DEFAULT); final WorldUpdater secondUpdater = secondWorldState.updater(); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/RollingImport.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/RollingImport.java index af230e436..301ddbc8e 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/RollingImport.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/RollingImport.java @@ -30,6 +30,7 @@ import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; @@ -56,7 +57,8 @@ public class RollingImport { final BonsaiWorldState bonsaiState = new BonsaiWorldState( archive, - new BonsaiWorldStateKeyValueStorage(provider, new NoOpMetricsSystem()), + new BonsaiWorldStateKeyValueStorage( + provider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG), EvmConfiguration.DEFAULT); final SegmentedInMemoryKeyValueStorage worldStateStorage = (SegmentedInMemoryKeyValueStorage) diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java index f6bb51197..239332d6a 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java @@ -36,6 +36,7 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.StorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -452,7 +453,9 @@ public class BonsaiWorldStateKeyValueStorageTest { private BonsaiWorldStateKeyValueStorage emptyStorage() { return new BonsaiWorldStateKeyValueStorage( - new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem()); + new InMemoryKeyValueStorageProvider(), + new NoOpMetricsSystem(), + DataStorageConfiguration.DEFAULT_CONFIG); } @Test @@ -487,6 +490,7 @@ public class BonsaiWorldStateKeyValueStorageTest { .thenReturn(mockTrieLogStorage); when(mockStorageProvider.getStorageBySegmentIdentifiers(any())) .thenReturn(mock(SegmentedKeyValueStorage.class)); - return new BonsaiWorldStateKeyValueStorage(mockStorageProvider, new NoOpMetricsSystem()); + return new BonsaiWorldStateKeyValueStorage( + mockStorageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); } } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java new file mode 100644 index 000000000..8d2984879 --- /dev/null +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java @@ -0,0 +1,89 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; +import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; +import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; +import org.hyperledger.besu.services.kvstore.SegmentedInMemoryKeyValueStorage; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class FlatDbStrategyProviderTest { + private final FlatDbStrategyProvider flatDbStrategyProvider = + new FlatDbStrategyProvider(new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); + private final SegmentedKeyValueStorage composedWorldStateStorage = + new SegmentedInMemoryKeyValueStorage(List.of(KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE)); + + @ParameterizedTest + @EnumSource(FlatDbMode.class) + void loadsFlatDbStrategyForStoredFlatDbMode(final FlatDbMode flatDbMode) { + updateFlatDbMode(flatDbMode); + + flatDbStrategyProvider.loadFlatDbStrategy(composedWorldStateStorage); + assertThat(flatDbStrategyProvider.getFlatDbMode()).isEqualTo(flatDbMode); + } + + @Test + void loadsPartialFlatDbStrategyWhenNoFlatDbModeStored() { + flatDbStrategyProvider.loadFlatDbStrategy(composedWorldStateStorage); + assertThat(flatDbStrategyProvider.getFlatDbMode()).isEqualTo(FlatDbMode.PARTIAL); + } + + @Test + void upgradesFlatDbStrategyToFullFlatDbMode() { + updateFlatDbMode(FlatDbMode.PARTIAL); + + flatDbStrategyProvider.upgradeToFullFlatDbMode(composedWorldStateStorage); + assertThat(flatDbStrategyProvider.flatDbMode).isEqualTo(FlatDbMode.FULL); + assertThat(flatDbStrategyProvider.flatDbStrategy).isNotNull(); + assertThat(flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage)) + .isInstanceOf(FullFlatDbStrategy.class); + } + + @Test + void downgradesFlatDbStrategyToPartiallyFlatDbMode() { + updateFlatDbMode(FlatDbMode.FULL); + + flatDbStrategyProvider.downgradeToPartialFlatDbMode(composedWorldStateStorage); + assertThat(flatDbStrategyProvider.flatDbMode).isEqualTo(FlatDbMode.PARTIAL); + assertThat(flatDbStrategyProvider.flatDbStrategy).isNotNull(); + assertThat(flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage)) + .isInstanceOf(PartialFlatDbStrategy.class); + } + + private void updateFlatDbMode(final FlatDbMode flatDbMode) { + final SegmentedKeyValueStorageTransaction transaction = + composedWorldStateStorage.startTransaction(); + transaction.put( + KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE, + FlatDbStrategyProvider.FLAT_DB_MODE, + flatDbMode.getVersion().toArrayUnsafe()); + transaction.commit(); + } +} diff --git a/ethereum/eth/src/jmh/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderBenchmark.java b/ethereum/eth/src/jmh/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderBenchmark.java index 5f7cbcdc8..71ba9b3ec 100644 --- a/ethereum/eth/src/jmh/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderBenchmark.java +++ b/ethereum/eth/src/jmh/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderBenchmark.java @@ -38,7 +38,7 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate.NodeDataReques import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.metrics.ObservableMetricsSystem; @@ -105,7 +105,8 @@ public class WorldStateDownloaderBenchmark { final StorageProvider storageProvider = createKeyValueStorageProvider(tempDir, tempDir.resolve("database")); - worldStateStorage = storageProvider.createWorldStateStorage(DataStorageFormat.FOREST); + worldStateStorage = + storageProvider.createWorldStateStorage(DataStorageConfiguration.DEFAULT_CONFIG); pendingRequests = new InMemoryTasksPriorityQueues<>(); worldStateDownloader = diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java index 91c770d47..4b4c73f3f 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java @@ -28,6 +28,7 @@ import org.hyperledger.besu.ethereum.eth.sync.worldstate.StalledDownloadExceptio import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloadProcess; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -80,7 +81,9 @@ public class FastWorldDownloadStateTest { if (storageFormat == DataStorageFormat.BONSAI) { worldStateStorage = new BonsaiWorldStateKeyValueStorage( - new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem()); + new InMemoryKeyValueStorageProvider(), + new NoOpMetricsSystem(), + DataStorageConfiguration.DEFAULT_CONFIG); } else { worldStateStorage = new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/PersistDataStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/PersistDataStepTest.java index e857648f7..7306cf301 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/PersistDataStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/PersistDataStepTest.java @@ -26,7 +26,7 @@ import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.patricia.SimpleMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.services.tasks.Task; @@ -40,7 +40,8 @@ import org.junit.jupiter.api.Test; public class PersistDataStepTest { private final WorldStateStorage worldStateStorage = - new InMemoryKeyValueStorageProvider().createWorldStateStorage(DataStorageFormat.FOREST); + new InMemoryKeyValueStorageProvider() + .createWorldStateStorage(DataStorageConfiguration.DEFAULT_CONFIG); private final FastWorldDownloadState downloadState = mock(FastWorldDownloadState.class); private final Bytes rootNodeData = Bytes.of(1, 1, 1, 1); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java index ead7a1ca6..565441eaa 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java @@ -34,6 +34,7 @@ import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.trie.patricia.StoredNodeFactory; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -58,7 +59,9 @@ public class AccountHealingTrackingTest { private final List
accounts = List.of(Address.fromHexString("0xdeadbeef")); private final WorldStateStorage worldStateStorage = new BonsaiWorldStateKeyValueStorage( - new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem()); + new InMemoryKeyValueStorageProvider(), + new NoOpMetricsSystem(), + DataStorageConfiguration.DEFAULT_CONFIG); private WorldStateProofProvider worldStateProofProvider; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java index 69047bc6e..dedce974c 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java @@ -26,7 +26,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.BytecodeRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.StorageRangeDataRequest; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.services.tasks.Task; @@ -39,7 +39,8 @@ import org.junit.jupiter.api.Test; public class PersistDataStepTest { private final WorldStateStorage worldStateStorage = - new InMemoryKeyValueStorageProvider().createWorldStateStorage(DataStorageFormat.FOREST); + new InMemoryKeyValueStorageProvider() + .createWorldStateStorage(DataStorageConfiguration.DEFAULT_CONFIG); private final SnapSyncProcessState snapSyncState = mock(SnapSyncProcessState.class); private final SnapWorldDownloadState downloadState = mock(SnapWorldDownloadState.class); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java index a8dc4a9b0..b275e60e2 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java @@ -40,6 +40,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloadProcess; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -108,7 +109,9 @@ public class SnapWorldDownloadStateTest { if (storageFormat == DataStorageFormat.BONSAI) { worldStateStorage = new BonsaiWorldStateKeyValueStorage( - new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem()); + new InMemoryKeyValueStorageProvider(), + new NoOpMetricsSystem(), + DataStorageConfiguration.DEFAULT_CONFIG); } else { worldStateStorage = new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java index 64bda37ae..7d1a4f3db 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java @@ -27,7 +27,7 @@ import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.services.tasks.Task; @@ -44,7 +44,8 @@ public class TaskGenerator { public static List> createAccountRequest(final boolean withData) { final WorldStateStorage worldStateStorage = - new InMemoryKeyValueStorageProvider().createWorldStateStorage(DataStorageFormat.FOREST); + new InMemoryKeyValueStorageProvider() + .createWorldStateStorage(DataStorageConfiguration.DEFAULT_CONFIG); final WorldStateProofProvider worldStateProofProvider = new WorldStateProofProvider(worldStateStorage); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java index a61b5c8ea..40db311c4 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java @@ -31,6 +31,7 @@ import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; @@ -179,7 +180,8 @@ public class AccountFlatDatabaseHealingRangeRequestTest { final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); final WorldStateStorage worldStateStorage = - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); final WorldStateProofProvider proofProvider = new WorldStateProofProvider(worldStateStorage); final MerkleTrie accountStateTrie = TrieGenerator.generateTrie(worldStateStorage, 15); @@ -233,7 +235,8 @@ public class AccountFlatDatabaseHealingRangeRequestTest { final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); final WorldStateStorage worldStateStorage = - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); final WorldStateProofProvider proofProvider = new WorldStateProofProvider(worldStateStorage); final MerkleTrie accountStateTrie = TrieGenerator.generateTrie(worldStateStorage, 15); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java index 7fe386257..cfaf422db 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java @@ -33,6 +33,7 @@ import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -78,7 +79,8 @@ class StorageFlatDatabaseHealingRangeRequestTest { public void setup() { final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); worldStateStorage = - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); proofProvider = new WorldStateProofProvider(worldStateStorage); trie = TrieGenerator.generateTrie( diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java index 5295f4e3b..6f98a51ad 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java @@ -24,6 +24,7 @@ import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; @@ -74,7 +75,8 @@ class StorageTrieNodeHealingRequestTest { } else { final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); worldStateStorage = - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); } final MerkleTrie trie = TrieGenerator.generateTrie( diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java index 034f8b9de..91c768a55 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java @@ -26,6 +26,7 @@ import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValu import org.hyperledger.besu.ethereum.trie.bonsai.trielog.NoOpTrieLogManager; import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.metrics.ObservableMetricsSystem; @@ -114,7 +115,10 @@ public class BonsaiReferenceTestWorldState extends BonsaiWorldState new BonsaiPreImageProxy.BonsaiReferenceTestPreImageProxy(); final BonsaiWorldStateKeyValueStorage bonsaiWorldStateKeyValueStorage = - new BonsaiWorldStateKeyValueStorage(new InMemoryKeyValueStorageProvider(), metricsSystem); + new BonsaiWorldStateKeyValueStorage( + new InMemoryKeyValueStorageProvider(), + metricsSystem, + DataStorageConfiguration.DEFAULT_CONFIG); final BonsaiReferenceTestWorldStateStorage worldStateStorage = new BonsaiReferenceTestWorldStateStorage(bonsaiWorldStateKeyValueStorage, preImageProxy); From e76bcf0babb5957b789587a20a5f024b2432464d Mon Sep 17 00:00:00 2001 From: Danno Ferrin Date: Thu, 18 Jan 2024 23:24:12 -0700 Subject: [PATCH 14/20] EOF Spec Updates (#6431) Update a few EOF features based on updated spec * Add a prague reference test target * Run evmtool from prestate when no code specified * RETF and dangling immediate arg fixes Signed-off-by: Danno Ferrin --- .../java/org/hyperledger/besu/evmtool/EvmToolCommand.java | 3 +++ .../referencetests/ReferenceTestProtocolSchedules.java | 2 +- .../org/hyperledger/besu/evm/code/CodeV1Validation.java | 5 +++++ .../java/org/hyperledger/besu/evm/code/EOFLayout.java | 4 ++-- .../besu/evm/operation/AbstractCreateOperation.java | 1 - .../org/hyperledger/besu/evm/operation/RetFOperation.java | 2 +- .../besu/evm/processor/ContractCreationProcessor.java | 1 + .../hyperledger/besu/evm/tracing/StandardJsonTracer.java | 7 ++++--- .../java/org/hyperledger/besu/evm/code/EOFLayoutTest.java | 8 ++++---- 9 files changed, 21 insertions(+), 12 deletions(-) diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommand.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommand.java index 743c215ec..900fcca70 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommand.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommand.java @@ -371,6 +371,9 @@ public class EvmToolCommand implements Runnable { long txGas = gas - intrinsicGasCost - accessListCost; final EVM evm = protocolSpec.getEvm(); + if (codeBytes.isEmpty()) { + codeBytes = component.getWorldState().get(receiver).getCode(); + } Code code = evm.getCode(Hash.hash(codeBytes), codeBytes); if (!code.isValid()) { out.println(((CodeInvalid) code).getInvalidReason()); diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestProtocolSchedules.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestProtocolSchedules.java index 3c5d84567..31cdfaca1 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestProtocolSchedules.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestProtocolSchedules.java @@ -79,7 +79,7 @@ public class ReferenceTestProtocolSchedules { createSchedule(genesisStub.clone().shanghaiTime(0).cancunTime(15000))); builder.put("Cancun", createSchedule(genesisStub.clone().cancunTime(0))); // TODO remove this after execution-test-specs finalize - builder.put("Shanghai+6780", createSchedule(genesisStub.clone().cancunTime(0))); + builder.put("Prague", createSchedule(genesisStub.clone().futureEipsTime(0))); builder.put("Future_EIPs", createSchedule(genesisStub.clone().futureEipsTime(0))); builder.put("Experimental_EIPs", createSchedule(genesisStub.clone().experimentalEipsTime(0))); return new ReferenceTestProtocolSchedules(builder.build()); diff --git a/evm/src/main/java/org/hyperledger/besu/evm/code/CodeV1Validation.java b/evm/src/main/java/org/hyperledger/besu/evm/code/CodeV1Validation.java index 6174a2734..e16498c78 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/code/CodeV1Validation.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/code/CodeV1Validation.java @@ -779,6 +779,11 @@ public final class CodeV1Validation { } currentPC += pcAdvance; + if (currentPC >= stackHeights.length) { + return String.format( + "Dangling immediate argument for opcode 0x%x at PC %d in code section %d.", + currentStackHeight, codeLength - pcAdvance, codeSectionToValidate); + } stackHeights[currentPC] = currentStackHeight; unusedBytes -= pcAdvance; } diff --git a/evm/src/main/java/org/hyperledger/besu/evm/code/EOFLayout.java b/evm/src/main/java/org/hyperledger/besu/evm/code/EOFLayout.java index f92022ca4..92016829a 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/code/EOFLayout.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/code/EOFLayout.java @@ -158,7 +158,7 @@ public class EOFLayout { if (typeData[codeSectionCount - 1][2] == -1) { return invalidLayout(container, version, "Incomplete type section"); } - if (typeData[0][0] != 0 || typeData[0][1] != 0) { + if (typeData[0][0] != 0 || (typeData[0][1] & 0x7f) != 0) { return invalidLayout( container, version, "Code section does not have zero inputs and outputs"); } @@ -182,7 +182,7 @@ public class EOFLayout { version, "Type data input stack too large - 0x" + Integer.toHexString(typeData[i][0])); } - if (typeData[i][1] > 0x7f) { + if (typeData[i][1] > 0x80) { return invalidLayout( container, version, diff --git a/evm/src/main/java/org/hyperledger/besu/evm/operation/AbstractCreateOperation.java b/evm/src/main/java/org/hyperledger/besu/evm/operation/AbstractCreateOperation.java index d695edf1c..e4fec2dbf 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/operation/AbstractCreateOperation.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/operation/AbstractCreateOperation.java @@ -141,7 +141,6 @@ public abstract class AbstractCreateOperation extends AbstractOperation { final Wei value = Wei.wrap(parent.getStackItem(0)); final Address contractAddress = targetContractAddress(parent); - parent.addCreate(contractAddress); final long childGasStipend = gasCalculator().gasAvailableForChildCreate(parent.getRemainingGas()); diff --git a/evm/src/main/java/org/hyperledger/besu/evm/operation/RetFOperation.java b/evm/src/main/java/org/hyperledger/besu/evm/operation/RetFOperation.java index 327dac1cb..3f40081bc 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/operation/RetFOperation.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/operation/RetFOperation.java @@ -24,7 +24,7 @@ public class RetFOperation extends AbstractOperation { /** The Opcode. */ public static final int OPCODE = 0xe4; /** The Ret F success. */ - static final OperationResult retfSuccess = new OperationResult(4, null); + static final OperationResult retfSuccess = new OperationResult(3, null); /** * Instantiates a new Ret F operation. diff --git a/evm/src/main/java/org/hyperledger/besu/evm/processor/ContractCreationProcessor.java b/evm/src/main/java/org/hyperledger/besu/evm/processor/ContractCreationProcessor.java index 7acea8937..20daf3ab2 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/processor/ContractCreationProcessor.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/processor/ContractCreationProcessor.java @@ -122,6 +122,7 @@ public class ContractCreationProcessor extends AbstractMessageProcessor { operationTracer.traceAccountCreationResult( frame, Optional.of(ExceptionalHaltReason.INSUFFICIENT_GAS)); } else { + frame.addCreate(contractAddress); contract.incrementBalance(frame.getValue()); contract.setNonce(initialContractNonce); contract.clearStorage(); diff --git a/evm/src/main/java/org/hyperledger/besu/evm/tracing/StandardJsonTracer.java b/evm/src/main/java/org/hyperledger/besu/evm/tracing/StandardJsonTracer.java index 5d90107a6..04254a718 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/tracing/StandardJsonTracer.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/tracing/StandardJsonTracer.java @@ -126,9 +126,7 @@ public class StandardJsonTracer implements OperationTracer { for (int i = messageFrame.stackSize() - 1; i >= 0; i--) { stack.add("\"" + shortBytes(messageFrame.getStackItem(i)) + "\""); } - pc = - messageFrame.getPC() - - messageFrame.getCode().getCodeSection(messageFrame.getSection()).getEntryPoint(); + pc = messageFrame.getPC() - messageFrame.getCode().getCodeSection(0).getEntryPoint(); section = messageFrame.getSection(); gas = shortNumber(messageFrame.getRemainingGas()); memorySize = messageFrame.memoryWordSize() * 32; @@ -171,6 +169,9 @@ public class StandardJsonTracer implements OperationTracer { public void tracePostExecution( final MessageFrame messageFrame, final Operation.OperationResult executeResult) { final Operation currentOp = messageFrame.getCurrentOperation(); + if (currentOp.isVirtualOperation()) { + return; + } final int opcode = currentOp.getOpcode(); final Bytes returnData = messageFrame.getReturnData(); diff --git a/evm/src/test/java/org/hyperledger/besu/evm/code/EOFLayoutTest.java b/evm/src/test/java/org/hyperledger/besu/evm/code/EOFLayoutTest.java index 3e8e17f2a..a0f83ddd9 100644 --- a/evm/src/test/java/org/hyperledger/besu/evm/code/EOFLayoutTest.java +++ b/evm/src/test/java/org/hyperledger/besu/evm/code/EOFLayoutTest.java @@ -276,15 +276,15 @@ public class EOFLayoutTest { 1 }, { - "EF0001 010010 0200040001000200020002 030000 00 00000000 80000000 00010000 02030000 FE 5000 3000 8000", + "EF0001 010010 0200040001000200020002 030000 00 00000000 F0000000 00010000 02030000 FE 5000 3000 8000", "inputs too large", - "Type data input stack too large - 0x80", + "Type data input stack too large - 0xf0", 1 }, { - "EF0001 010010 0200040001000200020002 030000 00 00000000 01000000 00800000 02030000 FE 5000 3000 8000", + "EF0001 010010 0200040001000200020002 030000 00 00000000 01000000 00F00000 02030000 FE 5000 3000 8000", "outputs too large", - "Type data output stack too large - 0x80", + "Type data output stack too large - 0xf0", 1 }, { From 958a07237848fdd1bba7fe7675876c299b7f4a3d Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Fri, 19 Jan 2024 17:20:20 +1000 Subject: [PATCH 15/20] 24.1.2-SNAPSHOT (#6432) Signed-off-by: Sally MacFarlane --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 75ee1d04e..8fe780b08 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=24.1.1-SNAPSHOT +version=24.1.2-SNAPSHOT org.gradle.welcome=never # Set exports/opens flags required by Google Java Format and ErrorProne plugins. (JEP-396) From 925f4946b8cd62bbb047558c857e47b38a4957f1 Mon Sep 17 00:00:00 2001 From: Matt Whitehead Date: Fri, 19 Jan 2024 10:34:02 +0000 Subject: [PATCH 16/20] Use `From` field in a `PING` packet when creating a peer table entry (#6225) * Use P2P 'from' host when parsing incoming P2P packets, if it is present Signed-off-by: Matthew Whitehead * Use UDP source address if PING 'from' address is 127.0.0.1 and add a unit test. Signed-off-by: Matthew Whitehead * Spotless Java, address PR comment Signed-off-by: Matthew Whitehead * Refactor handleIncomingPacket to allow for specific trace logs to show how selection is being done Signed-off-by: Matthew Whitehead * Add change log entry Signed-off-by: Matthew Whitehead * Refactor handleIncomingPacket Signed-off-by: Matthew Whitehead --------- Signed-off-by: Matthew Whitehead --- CHANGELOG.md | 1 + .../p2p/discovery/PeerDiscoveryAgent.java | 23 +++++++++++++++++- .../p2p/discovery/PeerDiscoveryAgentTest.java | 24 +++++++++++++++++++ .../discovery/PeerDiscoveryTestHelper.java | 8 +++++++ 4 files changed, 55 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eecedac55..7f84aa88b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,7 @@ https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.10.3-hotfix/besu- ### Bug fixes - Fix Docker image name clash between Besu and evmtool [#6194](https://github.com/hyperledger/besu/pull/6194) - Fix `logIndex` in `eth_getTransactionReceipt` JSON RPC method [#6206](https://github.com/hyperledger/besu/pull/6206) +- Fix the way an advertised host configured with `--p2p-host` is treated when communicating with the originator of a PING packet [#6225](https://github.com/hyperledger/besu/pull/6225) ### Download Links https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.10.3/besu-23.10.3.zip / sha256 da7ef8a6ceb88d3e327cacddcdb32218d1750b464c14165a74068f6dc6e0871a diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java index 8af596018..2324630d8 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java @@ -282,8 +282,29 @@ public abstract class PeerDiscoveryAgent { .flatMap(Endpoint::getTcpPort) .orElse(udpPort); + // If the host is present in the P2P PING packet itself, use that as the endpoint. If the P2P + // PING packet specifies 127.0.0.1 (the default if a custom value is not specified with + // --p2p-host or via a suitable --nat-method) we ignore it in favour of the UDP source address. + // The likelihood is that the UDP source will be 127.0.0.1 anyway, but this reduces the chance + // of an unexpected change in behaviour as a result of + // https://github.com/hyperledger/besu/issues/6224 being fixed. + final String host = + packet + .getPacketData(PingPacketData.class) + .flatMap(PingPacketData::getFrom) + .map(Endpoint::getHost) + .filter(abc -> !abc.equals("127.0.0.1")) + .stream() + .peek( + h -> + LOG.trace( + "Using \"From\" endpoint {} specified in ping packet. Ignoring UDP source host {}", + h, + sourceEndpoint.getHost())) + .findFirst() + .orElseGet(sourceEndpoint::getHost); + // Notify the peer controller. - final String host = sourceEndpoint.getHost(); final DiscoveryPeer peer = DiscoveryPeer.fromEnode( EnodeURLImpl.builder() diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgentTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgentTest.java index 97d167a26..240f4673e 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgentTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgentTest.java @@ -244,6 +244,30 @@ public class PeerDiscoveryAgentTest { } } + @Test + public void endpointHonoursCustomAdvertisedAddressInPingPacket() { + + // Start a peer with the default advertised host + final MockPeerDiscoveryAgent agent1 = helper.startDiscoveryAgent(); + + // Start another peer with its advertised host set to a custom value + final MockPeerDiscoveryAgent agent2 = helper.startDiscoveryAgent("192.168.0.1"); + + // Send a PING so we can exchange messages + Packet packet = helper.createPingPacket(agent2, agent1); + helper.sendMessageBetweenAgents(agent2, agent1, packet); + + // Agent 1's peers should have endpoints that match the custom advertised value... + agent1 + .streamDiscoveredPeers() + .forEach(peer -> assertThat(peer.getEndpoint().getHost()).isEqualTo("192.168.0.1")); + + // ...but agent 2's peers should have endpoints that match the default + agent2 + .streamDiscoveredPeers() + .forEach(peer -> assertThat(peer.getEndpoint().getHost()).isEqualTo("127.0.0.1")); + } + @Test public void shouldEvictPeerWhenPermissionsRevoked() { final PeerPermissionsDenylist denylist = PeerPermissionsDenylist.create(); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTestHelper.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTestHelper.java index e51b6320a..464273243 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTestHelper.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTestHelper.java @@ -165,6 +165,14 @@ public class PeerDiscoveryTestHelper { return startDiscoveryAgent(agentBuilder); } + public MockPeerDiscoveryAgent startDiscoveryAgent( + final String advertisedHost, final DiscoveryPeer... bootstrapPeers) { + final AgentBuilder agentBuilder = + agentBuilder().bootstrapPeers(bootstrapPeers).advertisedHost(advertisedHost); + + return startDiscoveryAgent(agentBuilder); + } + /** * Start a single discovery agent with the provided bootstrap peers. * From 98718ae270657cd26fb8f525eb9e9e506524c6c9 Mon Sep 17 00:00:00 2001 From: Matt Whitehead Date: Fri, 19 Jan 2024 14:08:53 +0000 Subject: [PATCH 17/20] Only accept a address from a peer if it is a valid IP address (#6439) Signed-off-by: Matthew Whitehead --- .../besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java index 2324630d8..30272413d 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java @@ -293,7 +293,9 @@ public abstract class PeerDiscoveryAgent { .getPacketData(PingPacketData.class) .flatMap(PingPacketData::getFrom) .map(Endpoint::getHost) - .filter(abc -> !abc.equals("127.0.0.1")) + .filter( + fromAddr -> + (!fromAddr.equals("127.0.0.1") && InetAddresses.isInetAddress(fromAddr))) .stream() .peek( h -> From 24718e391adf1bae94408560ce859b7185dd24ac Mon Sep 17 00:00:00 2001 From: Matt Whitehead Date: Fri, 19 Jan 2024 16:52:48 +0000 Subject: [PATCH 18/20] Fix changelog after incorrect merge of 6225 (#6438) * Fix changelog after incorrect merge of 6225 Signed-off-by: Matthew Whitehead * Create 24.1.2-snapshot changelog entry Signed-off-by: Matthew Whitehead * Remove entry from previous version in changelog Signed-off-by: Matthew Whitehead --------- Signed-off-by: Matthew Whitehead --- CHANGELOG.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f84aa88b..772efa5af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,19 @@ # Changelog -## 24.1.1-SNAPSHOT +## 24.1.2-SNAPSHOT + +### Breaking Changes + +### Deprecations + +### Additions and Improvements + +### Bug fixes +- Fix the way an advertised host configured with `--p2p-host` is treated when communicating with the originator of a PING packet [#6225](https://github.com/hyperledger/besu/pull/6225) + +### Download Links + +## 24.1.1 ### Breaking Changes - New `EXECUTION_HALTED` error returned if there is an error executing or simulating a transaction, with the reason for execution being halted. Replaces the generic `INTERNAL_ERROR` return code in certain cases which some applications may be checking for [#6343](https://github.com/hyperledger/besu/pull/6343) @@ -30,7 +43,6 @@ ### Download Links - ## 24.1.0 ### Breaking Changes @@ -87,7 +99,6 @@ https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.10.3-hotfix/besu- ### Bug fixes - Fix Docker image name clash between Besu and evmtool [#6194](https://github.com/hyperledger/besu/pull/6194) - Fix `logIndex` in `eth_getTransactionReceipt` JSON RPC method [#6206](https://github.com/hyperledger/besu/pull/6206) -- Fix the way an advertised host configured with `--p2p-host` is treated when communicating with the originator of a PING packet [#6225](https://github.com/hyperledger/besu/pull/6225) ### Download Links https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.10.3/besu-23.10.3.zip / sha256 da7ef8a6ceb88d3e327cacddcdb32218d1750b464c14165a74068f6dc6e0871a From cfea3ab2fd590827669c1cfddc121fa9b244d4de Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Mon, 22 Jan 2024 11:11:10 +1000 Subject: [PATCH 19/20] Import export trie log (#6363) * Import and export trie log subcommands * change option name and fix descriptions Signed-off-by: Gabriel Fukushima Co-authored-by: Jason Frame --- .../options/stable/DataStorageOptions.java | 11 +- .../subcommands/storage/TrieLogHelper.java | 103 ++++++++-- .../storage/TrieLogSubCommand.java | 106 +++++++++- .../stable/DataStorageOptionsTest.java | 20 +- .../storage/TrieLogHelperTest.java | 192 +++++++++++++----- 5 files changed, 349 insertions(+), 83 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java index e0b197356..5b4cf43eb 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java @@ -62,23 +62,28 @@ public class DataStorageOptions implements CLIOptions private final DataStorageOptions.Unstable unstableOptions = new Unstable(); static class Unstable { + private static final String BONSAI_LIMIT_TRIE_LOGS_ENABLED = + "--Xbonsai-limit-trie-logs-enabled"; + private static final String BONSAI_TRIE_LOGS_RETENTION_THRESHOLD = + "--Xbonsai-trie-logs-retention-threshold"; + private static final String BONSAI_TRIE_LOG_PRUNING_LIMIT = "--Xbonsai-trie-logs-pruning-limit"; @CommandLine.Option( hidden = true, - names = {"--Xbonsai-trie-log-pruning-enabled"}, + names = {BONSAI_LIMIT_TRIE_LOGS_ENABLED}, description = "Enable trie log pruning. (default: ${DEFAULT-VALUE})") private boolean bonsaiTrieLogPruningEnabled = DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED; @CommandLine.Option( hidden = true, - names = {"--Xbonsai-trie-log-retention-threshold"}, + names = {BONSAI_TRIE_LOGS_RETENTION_THRESHOLD}, description = "The number of blocks for which to retain trie logs. (default: ${DEFAULT-VALUE})") private long bonsaiTrieLogRetentionThreshold = DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; @CommandLine.Option( hidden = true, - names = {"--Xbonsai-trie-log-pruning-limit"}, + names = {BONSAI_TRIE_LOG_PRUNING_LIMIT}, description = "The max number of blocks to load and prune trie logs for at startup. (default: ${DEFAULT-VALUE})") private int bonsaiTrieLogPruningLimit = DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT; diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index 52dbe5590..c4e924a83 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -22,7 +22,11 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; +import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import java.io.File; @@ -32,6 +36,7 @@ import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.PrintWriter; +import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.IdentityHashMap; @@ -39,6 +44,7 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -97,16 +103,15 @@ public class TrieLogHelper { final String batchFileNameBase) { for (long batchNumber = 1; batchNumber <= numberOfBatches; batchNumber++) { - + final String batchFileName = batchFileNameBase + "-" + batchNumber; final long firstBlockOfBatch = chainHeight - ((batchNumber - 1) * BATCH_SIZE); - final long lastBlockOfBatch = Math.max(chainHeight - (batchNumber * BATCH_SIZE), lastBlockNumberToRetainTrieLogsFor); - final List trieLogKeys = getTrieLogKeysForBlocks(blockchain, firstBlockOfBatch, lastBlockOfBatch); - saveTrieLogBatches(batchFileNameBase, rootWorldStateStorage, batchNumber, trieLogKeys); + LOG.info("Saving trie logs to retain in file (batch {})...", batchNumber); + saveTrieLogBatches(batchFileName, rootWorldStateStorage, trieLogKeys); } LOG.info("Clear trie logs..."); @@ -118,15 +123,12 @@ public class TrieLogHelper { } private static void saveTrieLogBatches( - final String batchFileNameBase, + final String batchFileName, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, - final long batchNumber, final List trieLogKeys) { - LOG.info("Saving trie logs to retain in file (batch {})...", batchNumber); - try { - saveTrieLogsInFile(trieLogKeys, rootWorldStateStorage, batchNumber, batchFileNameBase); + saveTrieLogsInFile(trieLogKeys, rootWorldStateStorage, batchFileName); } catch (IOException e) { LOG.error("Error saving trie logs to file: {}", e.getMessage()); throw new RuntimeException(e); @@ -210,9 +212,8 @@ public class TrieLogHelper { final String batchFileNameBase) throws IOException { // process in chunk to avoid OOM - - IdentityHashMap trieLogsToRetain = - readTrieLogsFromFile(batchFileNameBase, batchNumber); + final String batchFileName = batchFileNameBase + "-" + batchNumber; + IdentityHashMap trieLogsToRetain = readTrieLogsFromFile(batchFileName); final int chunkSize = ROCKSDB_MAX_INSERTS_PER_TRANSACTION; List keys = new ArrayList<>(trieLogsToRetain.keySet()); @@ -265,11 +266,10 @@ public class TrieLogHelper { private static void saveTrieLogsInFile( final List trieLogsKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, - final long batchNumber, - final String batchFileNameBase) + final String batchFileName) throws IOException { - File file = new File(batchFileNameBase + "-" + batchNumber); + File file = new File(batchFileName); if (file.exists()) { LOG.error("File already exists, skipping file creation"); return; @@ -285,17 +285,14 @@ public class TrieLogHelper { } @SuppressWarnings("unchecked") - private static IdentityHashMap readTrieLogsFromFile( - final String batchFileNameBase, final long batchNumber) { + static IdentityHashMap readTrieLogsFromFile(final String batchFileName) { IdentityHashMap trieLogs; - try (FileInputStream fis = new FileInputStream(batchFileNameBase + "-" + batchNumber); + try (FileInputStream fis = new FileInputStream(batchFileName); ObjectInputStream ois = new ObjectInputStream(fis)) { trieLogs = (IdentityHashMap) ois.readObject(); - } catch (IOException | ClassNotFoundException e) { - LOG.error(e.getMessage()); throw new RuntimeException(e); } @@ -303,6 +300,52 @@ public class TrieLogHelper { return trieLogs; } + private static void saveTrieLogsAsRlpInFile( + final List trieLogsKeys, + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final String batchFileName) { + File file = new File(batchFileName); + if (file.exists()) { + LOG.error("File already exists, skipping file creation"); + return; + } + + final IdentityHashMap trieLogs = + getTrieLogs(trieLogsKeys, rootWorldStateStorage); + final Bytes rlp = + RLP.encode( + o -> + o.writeList( + trieLogs.entrySet(), (val, out) -> out.writeRaw(Bytes.wrap(val.getValue())))); + try { + Files.write(file.toPath(), rlp.toArrayUnsafe()); + } catch (IOException e) { + LOG.error(e.getMessage()); + throw new RuntimeException(e); + } + } + + static IdentityHashMap readTrieLogsAsRlpFromFile(final String batchFileName) { + try { + final Bytes file = Bytes.wrap(Files.readAllBytes(Path.of(batchFileName))); + final BytesValueRLPInput input = new BytesValueRLPInput(file, false); + + input.enterList(); + final IdentityHashMap trieLogs = new IdentityHashMap<>(); + while (!input.isEndOfCurrentList()) { + final Bytes trieLogBytes = input.currentListAsBytes(); + TrieLogLayer trieLogLayer = + TrieLogFactoryImpl.readFrom(new BytesValueRLPInput(Bytes.wrap(trieLogBytes), false)); + trieLogs.put(trieLogLayer.getBlockHash().toArrayUnsafe(), trieLogBytes.toArrayUnsafe()); + } + input.leaveList(); + + return trieLogs; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + private static IdentityHashMap getTrieLogs( final List trieLogKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage) { IdentityHashMap trieLogsToRetain = new IdentityHashMap<>(); @@ -357,5 +400,25 @@ public class TrieLogHelper { count.total, count.canonicalCount, count.forkCount, count.orphanCount); } + static void importTrieLog( + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final Path trieLogFilePath) { + + var trieLog = readTrieLogsAsRlpFromFile(trieLogFilePath.toString()); + + var updater = rootWorldStateStorage.updater(); + trieLog.forEach((key, value) -> updater.getTrieLogStorageTransaction().put(key, value)); + updater.getTrieLogStorageTransaction().commit(); + } + + static void exportTrieLog( + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final List trieLogHash, + final Path directoryPath) + throws IOException { + final String trieLogFile = directoryPath.toString(); + + saveTrieLogsAsRlpInFile(trieLogHash, rootWorldStateStorage, trieLogFile); + } + record TrieLogCount(int total, int canonicalCount, int forkCount, int orphanCount) {} } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index 74e00197f..e624b5f38 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -19,6 +19,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import org.hyperledger.besu.cli.util.VersionProvider; import org.hyperledger.besu.controller.BesuController; +import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; @@ -26,9 +27,11 @@ import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import java.io.IOException; import java.io.PrintWriter; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.List; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; @@ -43,7 +46,12 @@ import picocli.CommandLine.ParentCommand; description = "Manipulate trie logs", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class, - subcommands = {TrieLogSubCommand.CountTrieLog.class, TrieLogSubCommand.PruneTrieLog.class}) + subcommands = { + TrieLogSubCommand.CountTrieLog.class, + TrieLogSubCommand.PruneTrieLog.class, + TrieLogSubCommand.ExportTrieLog.class, + TrieLogSubCommand.ImportTrieLog.class + }) public class TrieLogSubCommand implements Runnable { @SuppressWarnings("UnusedVariable") @@ -123,6 +131,102 @@ public class TrieLogSubCommand implements Runnable { } } + @Command( + name = "export", + description = "This command exports the trie log of a determined block to a binary file", + mixinStandardHelpOptions = true, + versionProvider = VersionProvider.class) + static class ExportTrieLog implements Runnable { + + @SuppressWarnings("unused") + @ParentCommand + private TrieLogSubCommand parentCommand; + + @SuppressWarnings("unused") + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec + + @CommandLine.Option( + names = "--trie-log-block-hash", + description = + "Comma separated list of hashes from the blocks you want to export the trie logs of", + split = " {0,1}, {0,1}", + arity = "1..*") + private List trieLogBlockHashList; + + @CommandLine.Option( + names = "--trie-log-file-path", + description = "The file you want to export the trie logs to", + arity = "1..1") + private Path trieLogFilePath = null; + + @Override + public void run() { + if (trieLogFilePath == null) { + trieLogFilePath = + Paths.get( + TrieLogSubCommand.parentCommand + .parentCommand + .dataDir() + .resolve("trie-logs.bin") + .toAbsolutePath() + .toString()); + } + + TrieLogContext context = getTrieLogContext(); + + final List listOfBlockHashes = + trieLogBlockHashList.stream().map(Hash::fromHexString).toList(); + + try { + TrieLogHelper.exportTrieLog( + context.rootWorldStateStorage(), listOfBlockHashes, trieLogFilePath); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + @Command( + name = "import", + description = "This command imports a trie log exported by another besu node", + mixinStandardHelpOptions = true, + versionProvider = VersionProvider.class) + static class ImportTrieLog implements Runnable { + + @SuppressWarnings("unused") + @ParentCommand + private TrieLogSubCommand parentCommand; + + @SuppressWarnings("unused") + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec + + @CommandLine.Option( + names = "--trie-log-file-path", + description = "The file you want to import the trie logs from", + arity = "1..1") + private Path trieLogFilePath = null; + + @Override + public void run() { + if (trieLogFilePath == null) { + trieLogFilePath = + Paths.get( + TrieLogSubCommand.parentCommand + .parentCommand + .dataDir() + .resolve("trie-logs.bin") + .toAbsolutePath() + .toString()); + } + + TrieLogContext context = getTrieLogContext(); + + TrieLogHelper.importTrieLog(context.rootWorldStateStorage(), trieLogFilePath); + } + } + record TrieLogContext( DataStorageConfiguration config, BonsaiWorldStateKeyValueStorage rootWorldStateStorage, diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java index 437053afd..2a6390197 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java @@ -34,8 +34,8 @@ public class DataStorageOptionsTest dataStorageConfiguration -> assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningLimit()) .isEqualTo(1), - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-pruning-limit", + "--Xbonsai-limit-trie-logs-enabled", + "--Xbonsai-trie-logs-pruning-limit", "1"); } @@ -43,8 +43,8 @@ public class DataStorageOptionsTest public void bonsaiTrieLogPruningLimitShouldBePositive() { internalTestFailure( "--Xbonsai-trie-log-pruning-limit=0 must be greater than 0", - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-pruning-limit", + "--Xbonsai-limit-trie-logs-enabled", + "--Xbonsai-trie-logs-pruning-limit", "0"); } @@ -54,8 +54,8 @@ public class DataStorageOptionsTest dataStorageConfiguration -> assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold()) .isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD + 1), - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-retention-threshold", + "--Xbonsai-limit-trie-logs-enabled", + "--Xbonsai-trie-logs-retention-threshold", "513"); } @@ -65,8 +65,8 @@ public class DataStorageOptionsTest dataStorageConfiguration -> assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold()) .isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD), - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-retention-threshold", + "--Xbonsai-limit-trie-logs-enabled", + "--Xbonsai-trie-logs-retention-threshold", "512"); } @@ -74,8 +74,8 @@ public class DataStorageOptionsTest public void bonsaiTrieLogRetentionThresholdShouldBeAboveMinimum() { internalTestFailure( "--Xbonsai-trie-log-retention-threshold minimum value is 512", - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-retention-threshold", + "--Xbonsai-limit-trie-logs-enabled", + "--Xbonsai-trie-logs-retention-threshold", "511"); } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java index 0ba575a9e..5d4ede7f3 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.cli.subcommands.storage; +import static java.util.Collections.singletonList; import static org.hyperledger.besu.ethereum.worldstate.DataStorageFormat.BONSAI; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -27,8 +28,11 @@ import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -36,11 +40,12 @@ import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import org.apache.tuweni.bytes.Bytes; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -56,17 +61,14 @@ class TrieLogHelperTest { @Mock private MutableBlockchain blockchain; - @TempDir static Path dataDir; - - Path test; static BlockHeader blockHeader1; static BlockHeader blockHeader2; static BlockHeader blockHeader3; static BlockHeader blockHeader4; static BlockHeader blockHeader5; - @BeforeAll - public static void setup() throws IOException { + @BeforeEach + public void setup() throws IOException { blockHeader1 = new BlockHeaderTestFixture().number(1).buildHeader(); blockHeader2 = new BlockHeaderTestFixture().number(2).buildHeader(); @@ -78,33 +80,33 @@ class TrieLogHelperTest { new BonsaiWorldStateKeyValueStorage( storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); + createTrieLog(blockHeader1); + var updater = inMemoryWorldState.updater(); updater .getTrieLogStorageTransaction() - .put(blockHeader1.getHash().toArrayUnsafe(), Bytes.fromHexString("0x01").toArrayUnsafe()); + .put(blockHeader1.getHash().toArrayUnsafe(), createTrieLog(blockHeader1)); updater .getTrieLogStorageTransaction() - .put(blockHeader2.getHash().toArrayUnsafe(), Bytes.fromHexString("0x02").toArrayUnsafe()); + .put(blockHeader2.getHash().toArrayUnsafe(), createTrieLog(blockHeader2)); updater .getTrieLogStorageTransaction() - .put(blockHeader3.getHash().toArrayUnsafe(), Bytes.fromHexString("0x03").toArrayUnsafe()); + .put(blockHeader3.getHash().toArrayUnsafe(), createTrieLog(blockHeader3)); updater .getTrieLogStorageTransaction() - .put(blockHeader4.getHash().toArrayUnsafe(), Bytes.fromHexString("0x04").toArrayUnsafe()); + .put(blockHeader4.getHash().toArrayUnsafe(), createTrieLog(blockHeader4)); updater .getTrieLogStorageTransaction() - .put(blockHeader5.getHash().toArrayUnsafe(), Bytes.fromHexString("0x05").toArrayUnsafe()); + .put(blockHeader5.getHash().toArrayUnsafe(), createTrieLog(blockHeader5)); updater.getTrieLogStorageTransaction().commit(); } - @BeforeEach - void createDirectory() throws IOException { - Files.createDirectories(dataDir.resolve("database")); - } - - @AfterEach - void deleteDirectory() throws IOException { - Files.deleteIfExists(dataDir.resolve("database")); + private static byte[] createTrieLog(final BlockHeader blockHeader) { + TrieLogLayer trieLogLayer = new TrieLogLayer(); + trieLogLayer.setBlockHash(blockHeader.getBlockHash()); + final BytesValueRLPOutput rlpLog = new BytesValueRLPOutput(); + TrieLogFactoryImpl.writeTo(trieLogLayer, rlpLog); + return rlpLog.encoded().toArrayUnsafe(); } void mockBlockchainBase() { @@ -114,7 +116,8 @@ class TrieLogHelperTest { } @Test - public void prune() { + public void prune(final @TempDir Path dataDir) throws IOException { + Files.createDirectories(dataDir.resolve("database")); DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() @@ -134,14 +137,11 @@ class TrieLogHelperTest { // assert trie logs that will be pruned exist before prune call assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), - Bytes.fromHexString("0x01").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), createTrieLog(blockHeader1)); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), - Bytes.fromHexString("0x02").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), createTrieLog(blockHeader2)); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), - Bytes.fromHexString("0x03").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), createTrieLog(blockHeader3)); TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir); @@ -151,18 +151,15 @@ class TrieLogHelperTest { // assert retained trie logs are in the DB assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), - Bytes.fromHexString("0x03").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), createTrieLog(blockHeader3)); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), - Bytes.fromHexString("0x04").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), createTrieLog(blockHeader4)); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), - Bytes.fromHexString("0x05").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), createTrieLog(blockHeader5)); } @Test - public void cantPruneIfNoFinalizedIsFound() { + public void cantPruneIfNoFinalizedIsFound(final @TempDir Path dataDir) { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) @@ -184,7 +181,7 @@ class TrieLogHelperTest { } @Test - public void cantPruneIfUserRetainsMoreLayerThanExistingChainLength() { + public void cantPruneIfUserRetainsMoreLayerThanExistingChainLength(final @TempDir Path dataDir) { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) @@ -205,7 +202,7 @@ class TrieLogHelperTest { } @Test - public void cantPruneIfUserRequiredFurtherThanFinalized() { + public void cantPruneIfUserRequiredFurtherThanFinalized(final @TempDir Path dataDir) { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() @@ -227,8 +224,7 @@ class TrieLogHelperTest { } @Test - public void exceptionWhileSavingFileStopsPruneProcess() throws IOException { - Files.delete(dataDir.resolve("database")); + public void exceptionWhileSavingFileStopsPruneProcess(final @TempDir Path dataDir) { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() @@ -244,23 +240,121 @@ class TrieLogHelperTest { assertThrows( RuntimeException.class, () -> - TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); + TrieLogHelper.prune( + dataStorageConfiguration, + inMemoryWorldState, + blockchain, + dataDir.resolve("unknownPath"))); // assert all trie logs are still in the DB assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), - Bytes.fromHexString("0x01").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), createTrieLog(blockHeader1)); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), - Bytes.fromHexString("0x02").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), createTrieLog(blockHeader2)); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), - Bytes.fromHexString("0x03").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), createTrieLog(blockHeader3)); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), - Bytes.fromHexString("0x04").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), createTrieLog(blockHeader4)); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), - Bytes.fromHexString("0x05").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), createTrieLog(blockHeader5)); + } + + @Test + public void exportedTrieMatchesDbTrieLog(final @TempDir Path dataDir) throws IOException { + TrieLogHelper.exportTrieLog( + inMemoryWorldState, + singletonList(blockHeader1.getHash()), + dataDir.resolve("trie-log-dump")); + + var trieLog = + TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()) + .entrySet() + .stream() + .findFirst() + .get(); + + assertArrayEquals(trieLog.getKey(), blockHeader1.getHash().toArrayUnsafe()); + assertArrayEquals( + trieLog.getValue(), inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); + } + + @Test + public void exportedMultipleTriesMatchDbTrieLogs(final @TempDir Path dataDir) throws IOException { + TrieLogHelper.exportTrieLog( + inMemoryWorldState, + List.of(blockHeader1.getHash(), blockHeader2.getHash(), blockHeader3.getHash()), + dataDir.resolve("trie-log-dump")); + + var trieLogs = + TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()) + .entrySet() + .stream() + .collect(Collectors.toMap(e -> Bytes.wrap(e.getKey()), Map.Entry::getValue)); + + assertArrayEquals( + trieLogs.get(blockHeader1.getHash()), + inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); + assertArrayEquals( + trieLogs.get(blockHeader2.getHash()), + inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get()); + assertArrayEquals( + trieLogs.get(blockHeader3.getHash()), + inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get()); + } + + @Test + public void importedTrieLogMatchesDbTrieLog(final @TempDir Path dataDir) throws IOException { + StorageProvider tempStorageProvider = new InMemoryKeyValueStorageProvider(); + BonsaiWorldStateKeyValueStorage inMemoryWorldState2 = + new BonsaiWorldStateKeyValueStorage( + tempStorageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); + + TrieLogHelper.exportTrieLog( + inMemoryWorldState, + singletonList(blockHeader1.getHash()), + dataDir.resolve("trie-log-dump")); + + var trieLog = + TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()); + var updater = inMemoryWorldState2.updater(); + + trieLog.forEach((k, v) -> updater.getTrieLogStorageTransaction().put(k, v)); + + updater.getTrieLogStorageTransaction().commit(); + + assertArrayEquals( + inMemoryWorldState2.getTrieLog(blockHeader1.getHash()).get(), + inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); + } + + @Test + public void importedMultipleTriesMatchDbTrieLogs(final @TempDir Path dataDir) throws IOException { + StorageProvider tempStorageProvider = new InMemoryKeyValueStorageProvider(); + BonsaiWorldStateKeyValueStorage inMemoryWorldState2 = + new BonsaiWorldStateKeyValueStorage( + tempStorageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); + + TrieLogHelper.exportTrieLog( + inMemoryWorldState, + List.of(blockHeader1.getHash(), blockHeader2.getHash(), blockHeader3.getHash()), + dataDir.resolve("trie-log-dump")); + + var trieLog = + TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()); + var updater = inMemoryWorldState2.updater(); + + trieLog.forEach((k, v) -> updater.getTrieLogStorageTransaction().put(k, v)); + + updater.getTrieLogStorageTransaction().commit(); + + assertArrayEquals( + inMemoryWorldState2.getTrieLog(blockHeader1.getHash()).get(), + inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); + assertArrayEquals( + inMemoryWorldState2.getTrieLog(blockHeader2.getHash()).get(), + inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get()); + assertArrayEquals( + inMemoryWorldState2.getTrieLog(blockHeader3.getHash()).get(), + inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get()); } } From 921bc175c8af626c62ae4982f0ec09538bdc2322 Mon Sep 17 00:00:00 2001 From: Stefan Pingel <16143240+pinges@users.noreply.github.com> Date: Mon, 22 Jan 2024 14:56:43 +1000 Subject: [PATCH 20/20] Make fork id the default and try to recover the DiscoveryPeer for incoming connections from the PeerTable (#5628) * make the request for the ENR the default and try to recover the DiscoveryPeer for incoming connections from the PeerTable Signed-off-by: Stefan Signed-off-by: stefan.pingel@consensys.net --- CHANGELOG.md | 4 +- .../options/unstable/NetworkingOptions.java | 6 +- .../cli/options/NetworkingOptionsTest.java | 2 +- .../besu/ethereum/eth/manager/EthPeers.java | 1 + .../eth/manager/EthProtocolManager.java | 124 +++++++++++------- .../p2p/config/DiscoveryConfiguration.java | 2 +- .../p2p/config/NetworkingConfiguration.java | 1 + .../p2p/discovery/PeerDiscoveryAgent.java | 8 +- .../discovery/VertxPeerDiscoveryAgent.java | 7 +- .../internal/PeerDiscoveryController.java | 55 ++------ .../p2p/discovery/internal/PeerTable.java | 14 +- .../p2p/network/DefaultP2PNetwork.java | 12 +- .../besu/ethereum/p2p/rlpx/RlpxAgent.java | 20 +-- .../netty/AbstractHandshakeHandler.java | 87 ++++++------ .../p2p/rlpx/connections/netty/DeFramer.java | 40 +++++- .../netty/HandshakeHandlerInbound.java | 7 +- .../netty/HandshakeHandlerOutbound.java | 7 +- .../netty/NettyConnectionInitializer.java | 12 +- .../netty/NettyTLSConnectionInitializer.java | 12 +- .../discovery/PeerDiscoveryTestHelper.java | 1 + .../internal/MockPeerDiscoveryAgent.java | 3 +- .../internal/PeerDiscoveryControllerTest.java | 46 +------ .../PeerDiscoveryTableRefreshTest.java | 2 - .../p2p/discovery/internal/PeerTableTest.java | 20 +-- .../RecursivePeerRefreshStateTest.java | 6 +- .../p2p/network/DefaultP2PNetworkTest.java | 26 +--- .../besu/ethereum/p2p/rlpx/RlpxAgentTest.java | 2 +- .../rlpx/connections/netty/DeFramerTest.java | 50 ++++++- .../NettyTLSConnectionInitializerTest.java | 5 +- 29 files changed, 309 insertions(+), 273 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 772efa5af..a3c84b37b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,9 @@ - New `EXECUTION_HALTED` error returned if there is an error executing or simulating a transaction, with the reason for execution being halted. Replaces the generic `INTERNAL_ERROR` return code in certain cases which some applications may be checking for [#6343](https://github.com/hyperledger/besu/pull/6343) - The Besu Docker images with `openjdk-latest` tags since 23.10.3 were incorrectly using UID 1001 instead of 1000 for the container's `besu` user. The user now uses 1000 again. Containers created from or migrated to images using UID 1001 will need to chown their persistent database files to UID 1000 [#6360](https://github.com/hyperledger/besu/pull/6360) - The deprecated `--privacy-onchain-groups-enabled` option has now been removed. Use the `--privacy-flexible-groups-enabled` option instead. [#6411](https://github.com/hyperledger/besu/pull/6411) -- The time that can be spent selecting transactions during block creation is not capped at 5 seconds for PoS and PoW networks, and for PoA networks, at 75% of the block period specified in the genesis, this to prevent possible DoS in case a single transaction is taking too long to execute, and to have a stable block production rate, but it could be a breaking change if an existing network used to have transactions that takes more time to executed that the newly introduced limit, if it is mandatory for these network to keep processing these long processing transaction, then the default value of `block-txs-selection-max-time` or `poa-block-txs-selection-max-time` needs to be tuned accordingly. +- Requesting the Ethereum Node Record (ENR) to acquire the fork id from bonded peers is now enabled by default, so the following change has been made [#5628](https://github.com/hyperledger/besu/pull/5628): + - `--Xfilter-on-enr-fork-id` has been removed. To disable the feature use `--filter-on-enr-fork-id=false`. +- The time that can be spent selecting transactions during block creation is not capped at 5 seconds for PoS and PoW networks, and for PoA networks, at 75% of the block period specified in the genesis, this to prevent possible DoS in case a single transaction is taking too long to execute, and to have a stable block production rate, but it could be a breaking change if an existing network used to have transactions that takes more time to executed that the newly introduced limit, if it is mandatory for these network to keep processing these long processing transaction, then the default value of `block-txs-selection-max-time` or `poa-block-txs-selection-max-time` needs to be tuned accordingly. ### Deprecations diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/NetworkingOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/NetworkingOptions.java index 0ad68e362..69e62edfc 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/NetworkingOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/NetworkingOptions.java @@ -37,7 +37,7 @@ public class NetworkingOptions implements CLIOptions { private final String DNS_DISCOVERY_SERVER_OVERRIDE_FLAG = "--Xp2p-dns-discovery-server"; private final String DISCOVERY_PROTOCOL_V5_ENABLED = "--Xv5-discovery-enabled"; /** The constant FILTER_ON_ENR_FORK_ID. */ - public static final String FILTER_ON_ENR_FORK_ID = "--Xfilter-on-enr-fork-id"; + public static final String FILTER_ON_ENR_FORK_ID = "--filter-on-enr-fork-id"; @CommandLine.Option( names = INITIATE_CONNECTIONS_FREQUENCY_FLAG, @@ -76,9 +76,9 @@ public class NetworkingOptions implements CLIOptions { @CommandLine.Option( names = FILTER_ON_ENR_FORK_ID, hidden = true, - defaultValue = "false", + defaultValue = "true", description = "Whether to enable filtering of peers based on the ENR field ForkId)") - private final Boolean filterOnEnrForkId = false; + private final Boolean filterOnEnrForkId = NetworkingConfiguration.DEFAULT_FILTER_ON_ENR_FORK_ID; @CommandLine.Option( hidden = true, diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/NetworkingOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/NetworkingOptionsTest.java index 4b4601bb9..c96b9035e 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/NetworkingOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/NetworkingOptionsTest.java @@ -134,7 +134,7 @@ public class NetworkingOptionsTest final NetworkingOptions options = cmd.getNetworkingOptions(); final NetworkingConfiguration networkingConfig = options.toDomainObject(); - assertThat(networkingConfig.getDiscovery().isFilterOnEnrForkIdEnabled()).isEqualTo(false); + assertThat(networkingConfig.getDiscovery().isFilterOnEnrForkIdEnabled()).isEqualTo(true); assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); assertThat(commandOutput.toString(UTF_8)).isEmpty(); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java index e28b0a24a..c27b473d3 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java @@ -139,6 +139,7 @@ public class EthPeers { "peer_limit", "The maximum number of peers this node allows to connect", () -> peerUpperBound); + connectedPeersCounter = metricsSystem.createCounter( BesuMetricCategory.PEERS, "connected_total", "Total number of peers connected"); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManager.java index 6925df9d4..774d35cf1 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManager.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManager.java @@ -110,7 +110,7 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { this.blockBroadcaster = new BlockBroadcaster(ethContext); - supportedCapabilities = + this.supportedCapabilities = calculateCapabilities(synchronizerConfiguration, ethereumWireProtocolConfiguration); // Run validators @@ -252,11 +252,14 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { @Override public void stop() { if (stopped.compareAndSet(false, true)) { - LOG.info("Stopping {} Subprotocol.", getSupportedProtocol()); + LOG.atInfo().setMessage("Stopping {} Subprotocol.").addArgument(getSupportedProtocol()).log(); scheduler.stop(); shutdown.countDown(); } else { - LOG.error("Attempted to stop already stopped {} Subprotocol.", getSupportedProtocol()); + LOG.atInfo() + .setMessage("Attempted to stop already stopped {} Subprotocol.") + .addArgument(this::getSupportedProtocol) + .log(); } } @@ -264,7 +267,10 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { public void awaitStop() throws InterruptedException { shutdown.await(); scheduler.awaitStop(); - LOG.info("{} Subprotocol stopped.", getSupportedProtocol()); + LOG.atInfo() + .setMessage("{} Subprotocol stopped.") + .addArgument(this::getSupportedProtocol) + .log(); } @Override @@ -277,8 +283,10 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { EthProtocolLogger.logProcessMessage(cap, code); final EthPeer ethPeer = ethPeers.peer(message.getConnection()); if (ethPeer == null) { - LOG.debug( - "Ignoring message received from unknown peer connection: {}", message.getConnection()); + LOG.atDebug() + .setMessage("Ignoring message received from unknown peer connection: {}") + .addArgument(message::getConnection) + .log(); return; } @@ -288,19 +296,24 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { return; } else if (!ethPeer.statusHasBeenReceived()) { // Peers are required to send status messages before any other message type - LOG.debug( - "{} requires a Status ({}) message to be sent first. Instead, received message {} (BREACH_OF_PROTOCOL). Disconnecting from {}.", - this.getClass().getSimpleName(), - EthPV62.STATUS, - code, - ethPeer); + LOG.atDebug() + .setMessage( + "{} requires a Status ({}) message to be sent first. Instead, received message {} (BREACH_OF_PROTOCOL). Disconnecting from {}.") + .addArgument(() -> this.getClass().getSimpleName()) + .addArgument(EthPV62.STATUS) + .addArgument(code) + .addArgument(ethPeer::toString) + .log(); ethPeer.disconnect(DisconnectReason.BREACH_OF_PROTOCOL); return; } if (this.mergePeerFilter.isPresent()) { if (this.mergePeerFilter.get().disconnectIfGossipingBlocks(message, ethPeer)) { - LOG.debug("Post-merge disconnect: peer still gossiping blocks {}", ethPeer); + LOG.atDebug() + .setMessage("Post-merge disconnect: peer still gossiping blocks {}") + .addArgument(ethPeer::toString) + .log(); handleDisconnect(ethPeer.getConnection(), DisconnectReason.SUBPROTOCOL_TRIGGERED, false); return; } @@ -333,11 +346,12 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { maybeResponseData = ethMessages.dispatch(ethMessage); } } catch (final RLPException e) { - LOG.debug( - "Received malformed message {} (BREACH_OF_PROTOCOL), disconnecting: {}", - messageData.getData(), - ethPeer, - e); + LOG.atDebug() + .setMessage("Received malformed message {} (BREACH_OF_PROTOCOL), disconnecting: {}, {}") + .addArgument(messageData::getData) + .addArgument(ethPeer::toString) + .addArgument(e::toString) + .log(); ethPeer.disconnect(DisconnectMessage.DisconnectReason.BREACH_OF_PROTOCOL); } @@ -368,23 +382,31 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { genesisHash, latestForkId); try { - LOG.trace("Sending status message to {} for connection {}.", peer.getId(), connection); + LOG.atTrace() + .setMessage("Sending status message to {} for connection {}.") + .addArgument(peer::getId) + .addArgument(connection::toString) + .log(); peer.send(status, getSupportedProtocol(), connection); peer.registerStatusSent(connection); } catch (final PeerNotConnected peerNotConnected) { // Nothing to do. } - LOG.trace("{}", ethPeers); + LOG.atTrace().setMessage("{}").addArgument(ethPeers::toString).log(); } @Override public boolean shouldConnect(final Peer peer, final boolean incoming) { - if (peer.getForkId().map(forkId -> forkIdManager.peerCheck(forkId)).orElse(true)) { - LOG.trace("ForkId OK or not available"); + if (peer.getForkId().map(forkIdManager::peerCheck).orElse(true)) { + LOG.atDebug() + .setMessage("ForkId OK or not available for peer {}") + .addArgument(peer::getId) + .log(); if (ethPeers.shouldConnect(peer, incoming)) { return true; } } + LOG.atDebug().setMessage("ForkId check failed for peer {}").addArgument(peer::getId).log(); return false; } @@ -397,11 +419,11 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { LOG.atDebug() .setMessage("Disconnect - {} - {} - {}... - {} peers left") .addArgument(initiatedByPeer ? "Inbound" : "Outbound") - .addArgument(reason) - .addArgument(connection.getPeer().getId().slice(0, 8)) - .addArgument(ethPeers.peerCount()) + .addArgument(reason::toString) + .addArgument(() -> connection.getPeer().getId().slice(0, 8)) + .addArgument(ethPeers::peerCount) .log(); - LOG.trace("{}", ethPeers); + LOG.atTrace().setMessage("{}").addArgument(ethPeers::toString).log(); } } @@ -412,43 +434,41 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { try { if (!status.networkId().equals(networkId)) { LOG.atDebug() - .setMessage("Mismatched network id: {}, EthPeer {}...") - .addArgument(status.networkId()) - .addArgument(peer.getShortNodeId()) - .log(); - LOG.atTrace() - .setMessage("Mismatched network id: {}, EthPeer {}") - .addArgument(status.networkId()) - .addArgument(peer) + .setMessage("Mismatched network id: {}, peer {}") + .addArgument(status::networkId) + .addArgument(() -> getPeerOrPeerId(peer)) .log(); peer.disconnect(DisconnectReason.SUBPROTOCOL_TRIGGERED); } else if (!forkIdManager.peerCheck(forkId) && status.protocolVersion() > 63) { - LOG.debug( - "{} has matching network id ({}), but non-matching fork id: {}", - peer, - networkId, - forkId); + LOG.atDebug() + .setMessage("{} has matching network id ({}), but non-matching fork id: {}") + .addArgument(() -> getPeerOrPeerId(peer)) + .addArgument(networkId::toString) + .addArgument(forkId) + .log(); peer.disconnect(DisconnectReason.SUBPROTOCOL_TRIGGERED); } else if (forkIdManager.peerCheck(status.genesisHash())) { - LOG.debug( - "{} has matching network id ({}), but non-matching genesis hash: {}", - peer, - networkId, - status.genesisHash()); + LOG.atDebug() + .setMessage("{} has matching network id ({}), but non-matching genesis hash: {}") + .addArgument(() -> getPeerOrPeerId(peer)) + .addArgument(networkId::toString) + .addArgument(status::genesisHash) + .log(); peer.disconnect(DisconnectReason.SUBPROTOCOL_TRIGGERED); } else if (mergePeerFilter.isPresent() && mergePeerFilter.get().disconnectIfPoW(status, peer)) { LOG.atDebug() .setMessage("Post-merge disconnect: peer still PoW {}") - .addArgument(peer.getShortNodeId()) + .addArgument(() -> getPeerOrPeerId(peer)) .log(); handleDisconnect(peer.getConnection(), DisconnectReason.SUBPROTOCOL_TRIGGERED, false); } else { - LOG.debug( - "Received status message from {}: {} with connection {}", - peer, - status, - message.getConnection()); + LOG.atDebug() + .setMessage("Received status message from {}: {} with connection {}") + .addArgument(peer::toString) + .addArgument(status::toString) + .addArgument(message::getConnection) + .log(); peer.registerStatusReceived( status.bestHash(), status.totalDifficulty(), @@ -467,6 +487,10 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { } } + private Object getPeerOrPeerId(final EthPeer peer) { + return LOG.isTraceEnabled() ? peer : peer.getShortNodeId(); + } + @Override public void blockMined(final Block block) { // This assumes the block has already been included in the chain diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/DiscoveryConfiguration.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/DiscoveryConfiguration.java index 036e592e1..86bb079a2 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/DiscoveryConfiguration.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/DiscoveryConfiguration.java @@ -32,7 +32,7 @@ public class DiscoveryConfiguration { private List bootnodes = new ArrayList<>(); private String dnsDiscoveryURL; private boolean discoveryV5Enabled = false; - private boolean filterOnEnrForkId = false; + private boolean filterOnEnrForkId = NetworkingConfiguration.DEFAULT_FILTER_ON_ENR_FORK_ID; public static DiscoveryConfiguration create() { return new DiscoveryConfiguration(); diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/NetworkingConfiguration.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/NetworkingConfiguration.java index 0de53cfd7..478e36173 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/NetworkingConfiguration.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/NetworkingConfiguration.java @@ -23,6 +23,7 @@ public class NetworkingConfiguration { public static final int DEFAULT_INITIATE_CONNECTIONS_FREQUENCY_SEC = 30; public static final int DEFAULT_CHECK_MAINTAINED_CONNECTIONS_FREQUENCY_SEC = 60; public static final int DEFAULT_PEER_LOWER_BOUND = 25; + public static final boolean DEFAULT_FILTER_ON_ENR_FORK_ID = true; private DiscoveryConfiguration discovery = new DiscoveryConfiguration(); private RlpxConfiguration rlpx = new RlpxConfiguration(); diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java index 30272413d..de7d04718 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java @@ -26,6 +26,7 @@ import org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration; import org.hyperledger.besu.ethereum.p2p.discovery.internal.Packet; import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerDiscoveryController; import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerRequirement; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.discovery.internal.PingPacketData; import org.hyperledger.besu.ethereum.p2p.discovery.internal.TimerUtil; import org.hyperledger.besu.ethereum.p2p.peers.EnodeURLImpl; @@ -81,6 +82,7 @@ public abstract class PeerDiscoveryAgent { private final MetricsSystem metricsSystem; private final RlpxAgent rlpxAgent; private final ForkIdManager forkIdManager; + private final PeerTable peerTable; /* The peer controller, which takes care of the state machine of peers. */ protected Optional controller = Optional.empty(); @@ -109,7 +111,8 @@ public abstract class PeerDiscoveryAgent { final MetricsSystem metricsSystem, final StorageProvider storageProvider, final ForkIdManager forkIdManager, - final RlpxAgent rlpxAgent) { + final RlpxAgent rlpxAgent, + final PeerTable peerTable) { this.metricsSystem = metricsSystem; checkArgument(nodeKey != null, "nodeKey cannot be null"); checkArgument(config != null, "provided configuration cannot be null"); @@ -130,6 +133,7 @@ public abstract class PeerDiscoveryAgent { this.forkIdManager = forkIdManager; this.forkIdSupplier = () -> forkIdManager.getForkIdForChainHead().getForkIdAsBytesList(); this.rlpxAgent = rlpxAgent; + this.peerTable = peerTable; } protected abstract TimerUtil createTimer(); @@ -263,9 +267,9 @@ public abstract class PeerDiscoveryAgent { .peerRequirement(PeerRequirement.combine(peerRequirements)) .peerPermissions(peerPermissions) .metricsSystem(metricsSystem) - .forkIdManager(forkIdManager) .filterOnEnrForkId((config.isFilterOnEnrForkIdEnabled())) .rlpxAgent(rlpxAgent) + .peerTable(peerTable) .build(); } diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/VertxPeerDiscoveryAgent.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/VertxPeerDiscoveryAgent.java index 27a2be8be..ef098896a 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/VertxPeerDiscoveryAgent.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/VertxPeerDiscoveryAgent.java @@ -23,6 +23,7 @@ import org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration; import org.hyperledger.besu.ethereum.p2p.discovery.internal.Packet; import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerDiscoveryController; import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerDiscoveryController.AsyncExecutor; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.discovery.internal.TimerUtil; import org.hyperledger.besu.ethereum.p2p.discovery.internal.VertxTimerUtil; import org.hyperledger.besu.ethereum.p2p.permissions.PeerPermissions; @@ -73,7 +74,8 @@ public class VertxPeerDiscoveryAgent extends PeerDiscoveryAgent { final MetricsSystem metricsSystem, final StorageProvider storageProvider, final ForkIdManager forkIdManager, - final RlpxAgent rlpxAgent) { + final RlpxAgent rlpxAgent, + final PeerTable peerTable) { super( nodeKey, config, @@ -82,7 +84,8 @@ public class VertxPeerDiscoveryAgent extends PeerDiscoveryAgent { metricsSystem, storageProvider, forkIdManager, - rlpxAgent); + rlpxAgent, + peerTable); checkArgument(vertx != null, "vertx instance cannot be null"); this.vertx = vertx; diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryController.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryController.java index ec829d207..af3790def 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryController.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryController.java @@ -21,8 +21,6 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import org.hyperledger.besu.cryptoservices.NodeKey; -import org.hyperledger.besu.ethereum.forkid.ForkId; -import org.hyperledger.besu.ethereum.forkid.ForkIdManager; import org.hyperledger.besu.ethereum.p2p.discovery.DiscoveryPeer; import org.hyperledger.besu.ethereum.p2p.discovery.PeerDiscoveryStatus; import org.hyperledger.besu.ethereum.p2p.peers.Peer; @@ -129,7 +127,6 @@ public class PeerDiscoveryController { private final DiscoveryProtocolLogger discoveryProtocolLogger; private final LabelledMetric interactionCounter; private final LabelledMetric interactionRetryCounter; - private final ForkIdManager forkIdManager; private final boolean filterOnEnrForkId; private final RlpxAgent rlpxAgent; @@ -161,7 +158,6 @@ public class PeerDiscoveryController { final PeerPermissions peerPermissions, final MetricsSystem metricsSystem, final Optional> maybeCacheForEnrRequests, - final ForkIdManager forkIdManager, final boolean filterOnEnrForkId, final RlpxAgent rlpxAgent) { this.timerUtil = timerUtil; @@ -197,11 +193,11 @@ public class PeerDiscoveryController { "discovery_interaction_retry_count", "Total number of interaction retries performed", "type"); + this.cachedEnrRequests = maybeCacheForEnrRequests.orElse( CacheBuilder.newBuilder().maximumSize(50).expireAfterWrite(10, SECONDS).build()); - this.forkIdManager = forkIdManager; this.filterOnEnrForkId = filterOnEnrForkId; } @@ -314,6 +310,7 @@ public class PeerDiscoveryController { } final DiscoveryPeer peer = resolvePeer(sender); + final Bytes peerId = peer.getId(); switch (packet.getType()) { case PING: if (peerPermissions.allowInboundBonding(peer)) { @@ -333,10 +330,10 @@ public class PeerDiscoveryController { if (filterOnEnrForkId) { requestENR(peer); } - bondingPeers.invalidate(peer.getId()); + bondingPeers.invalidate(peerId); addToPeerTable(peer); recursivePeerRefreshState.onBondingComplete(peer); - Optional.ofNullable(cachedEnrRequests.getIfPresent(peer.getId())) + Optional.ofNullable(cachedEnrRequests.getIfPresent(peerId)) .ifPresent(cachedEnrRequest -> processEnrRequest(peer, cachedEnrRequest)); }); break; @@ -360,12 +357,12 @@ public class PeerDiscoveryController { if (PeerDiscoveryStatus.BONDED.equals(peer.getStatus())) { processEnrRequest(peer, packet); } else if (PeerDiscoveryStatus.BONDING.equals(peer.getStatus())) { - LOG.trace("ENR_REQUEST cached for bonding peer Id: {}", peer.getId()); + LOG.trace("ENR_REQUEST cached for bonding peer Id: {}", peerId); // Due to UDP, it may happen that we receive the ENR_REQUEST just before the PONG. // Because peers want to send the ENR_REQUEST directly after the pong. // If this happens we don't want to ignore the request but process when bonded. // this cache allows to keep the request and to respond after having processed the PONG - cachedEnrRequests.put(peer.getId(), packet); + cachedEnrRequests.put(peerId, packet); } break; case ENR_RESPONSE: @@ -376,26 +373,6 @@ public class PeerDiscoveryController { packet.getPacketData(ENRResponsePacketData.class); final NodeRecord enr = packetData.get().getEnr(); peer.setNodeRecord(enr); - - final Optional maybeForkId = peer.getForkId(); - if (maybeForkId.isPresent()) { - if (forkIdManager.peerCheck(maybeForkId.get())) { - connectOnRlpxLayer(peer); - LOG.debug( - "Peer {} PASSED fork id check. ForkId received: {}", - sender.getId(), - maybeForkId.get()); - } else { - LOG.debug( - "Peer {} FAILED fork id check. ForkId received: {}", - sender.getId(), - maybeForkId.get()); - } - } else { - // if the peer hasn't sent the ForkId try to connect to it anyways - connectOnRlpxLayer(peer); - LOG.debug("No fork id sent by peer: {}", peer.getId()); - } }); break; } @@ -431,9 +408,7 @@ public class PeerDiscoveryController { if (peer.getStatus() != PeerDiscoveryStatus.BONDED) { peer.setStatus(PeerDiscoveryStatus.BONDED); - if (!filterOnEnrForkId) { - connectOnRlpxLayer(peer); - } + connectOnRlpxLayer(peer); } final PeerTable.AddResult result = peerTable.tryAdd(peer); @@ -560,8 +535,6 @@ public class PeerDiscoveryController { */ @VisibleForTesting void requestENR(final DiscoveryPeer peer) { - peer.setStatus(PeerDiscoveryStatus.ENR_REQUESTED); - final Consumer action = interaction -> { final ENRRequestPacketData data = ENRRequestPacketData.create(); @@ -838,7 +811,6 @@ public class PeerDiscoveryController { private Cache cachedEnrRequests = CacheBuilder.newBuilder().maximumSize(50).expireAfterWrite(10, SECONDS).build(); - private ForkIdManager forkIdManager; private RlpxAgent rlpxAgent; private Builder() {} @@ -846,10 +818,6 @@ public class PeerDiscoveryController { public PeerDiscoveryController build() { validate(); - if (peerTable == null) { - peerTable = new PeerTable(this.nodeKey.getPublicKey().getEncodedBytes(), 16); - } - return new PeerDiscoveryController( nodeKey, localPeer, @@ -864,7 +832,6 @@ public class PeerDiscoveryController { peerPermissions, metricsSystem, Optional.of(cachedEnrRequests), - forkIdManager, filterOnEnrForkId, rlpxAgent); } @@ -875,8 +842,8 @@ public class PeerDiscoveryController { validateRequiredDependency(timerUtil, "TimerUtil"); validateRequiredDependency(workerExecutor, "AsyncExecutor"); validateRequiredDependency(metricsSystem, "MetricsSystem"); - validateRequiredDependency(forkIdManager, "ForkIdManager"); validateRequiredDependency(rlpxAgent, "RlpxAgent"); + validateRequiredDependency(peerTable, "PeerTable"); } private void validateRequiredDependency(final Object object, final String name) { @@ -970,11 +937,5 @@ public class PeerDiscoveryController { this.rlpxAgent = rlpxAgent; return this; } - - public Builder forkIdManager(final ForkIdManager forkIdManager) { - checkNotNull(forkIdManager); - this.forkIdManager = forkIdManager; - return this; - } } } diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerTable.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerTable.java index e153acbfc..f0e0be1fe 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerTable.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerTable.java @@ -56,26 +56,21 @@ public class PeerTable { * Builds a new peer table, where distance is calculated using the provided nodeId as a baseline. * * @param nodeId The ID of the node where this peer table is stored. - * @param bucketSize The maximum length of each k-bucket. */ - public PeerTable(final Bytes nodeId, final int bucketSize) { + public PeerTable(final Bytes nodeId) { this.keccak256 = Hash.keccak256(nodeId); this.table = Stream.generate(() -> new Bucket(DEFAULT_BUCKET_SIZE)) .limit(N_BUCKETS + 1) .toArray(Bucket[]::new); this.distanceCache = new ConcurrentHashMap<>(); - this.maxEntriesCnt = N_BUCKETS * bucketSize; + this.maxEntriesCnt = N_BUCKETS * DEFAULT_BUCKET_SIZE; // A bloom filter with 4096 expected insertions of 64-byte keys with a 0.1% false positive // probability yields a memory footprint of ~7.5kb. buildBloomFilter(); } - public PeerTable(final Bytes nodeId) { - this(nodeId, DEFAULT_BUCKET_SIZE); - } - /** * Returns the table's representation of a peer, if it exists. * @@ -83,11 +78,12 @@ public class PeerTable { * @return The stored representation. */ public Optional get(final PeerId peer) { - if (!idBloom.mightContain(peer.getId())) { + final Bytes peerId = peer.getId(); + if (!idBloom.mightContain(peerId)) { return Optional.empty(); } final int distance = distanceFrom(peer); - return table[distance].getAndTouch(peer.getId()); + return table[distance].getAndTouch(peerId); } /** diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetwork.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetwork.java index ec65934b2..11352b38c 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetwork.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetwork.java @@ -27,6 +27,7 @@ import org.hyperledger.besu.ethereum.p2p.discovery.DiscoveryPeer; import org.hyperledger.besu.ethereum.p2p.discovery.PeerDiscoveryAgent; import org.hyperledger.besu.ethereum.p2p.discovery.PeerDiscoveryStatus; import org.hyperledger.besu.ethereum.p2p.discovery.VertxPeerDiscoveryAgent; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.peers.DefaultPeerPrivileges; import org.hyperledger.besu.ethereum.p2p.peers.EnodeURLImpl; import org.hyperledger.besu.ethereum.p2p.peers.LocalNode; @@ -383,11 +384,12 @@ public class DefaultP2PNetwork implements P2PNetwork { @VisibleForTesting void attemptPeerConnections() { LOG.trace("Initiating connections to discovered peers."); - rlpxAgent.connect( + final Stream toTry = streamDiscoveredPeers() .filter(peer -> peer.getStatus() == PeerDiscoveryStatus.BONDED) .filter(peerDiscoveryAgent::checkForkId) - .sorted(Comparator.comparing(DiscoveryPeer::getLastAttemptedConnection))); + .sorted(Comparator.comparing(DiscoveryPeer::getLastAttemptedConnection)); + toTry.forEach(rlpxAgent::connect); } @Override @@ -511,6 +513,7 @@ public class DefaultP2PNetwork implements P2PNetwork { private Supplier> allConnectionsSupplier; private Supplier> allActiveConnectionsSupplier; private int peersLowerBound; + private PeerTable peerTable; public P2PNetwork build() { validate(); @@ -528,6 +531,7 @@ public class DefaultP2PNetwork implements P2PNetwork { final MutableLocalNode localNode = MutableLocalNode.create(config.getRlpx().getClientId(), 5, supportedCapabilities); final PeerPrivileges peerPrivileges = new DefaultPeerPrivileges(maintainedPeers); + peerTable = new PeerTable(nodeKey.getPublicKey().getEncodedBytes()); rlpxAgent = rlpxAgent == null ? createRlpxAgent(localNode, peerPrivileges) : rlpxAgent; peerDiscoveryAgent = peerDiscoveryAgent == null ? createDiscoveryAgent() : peerDiscoveryAgent; @@ -572,7 +576,8 @@ public class DefaultP2PNetwork implements P2PNetwork { metricsSystem, storageProvider, forkIdManager, - rlpxAgent); + rlpxAgent, + peerTable); } private RlpxAgent createRlpxAgent( @@ -589,6 +594,7 @@ public class DefaultP2PNetwork implements P2PNetwork { .allConnectionsSupplier(allConnectionsSupplier) .allActiveConnectionsSupplier(allActiveConnectionsSupplier) .peersLowerBound(peersLowerBound) + .peerTable(peerTable) .build(); } diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/RlpxAgent.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/RlpxAgent.java index 98a1f60df..4a8e227d3 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/RlpxAgent.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/RlpxAgent.java @@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkState; import org.hyperledger.besu.cryptoservices.NodeKey; import org.hyperledger.besu.ethereum.p2p.config.RlpxConfiguration; import org.hyperledger.besu.ethereum.p2p.discovery.DiscoveryPeer; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.peers.LocalNode; import org.hyperledger.besu.ethereum.p2p.peers.Peer; import org.hyperledger.besu.ethereum.p2p.peers.PeerPrivileges; @@ -162,13 +163,6 @@ public class RlpxAgent { } } - public void connect(final Stream peerStream) { - if (!localNode.isReady()) { - return; - } - peerStream.forEach(this::connect); - } - public void disconnect(final Bytes peerId, final DisconnectReason reason) { try { allActiveConnectionsSupplier @@ -206,6 +200,7 @@ public class RlpxAgent { + this.getClass().getSimpleName() + " has finished starting")); } + // Check peer is valid final EnodeURL enode = peer.getEnodeURL(); if (!enode.isListening()) { @@ -380,6 +375,7 @@ public class RlpxAgent { private Supplier> allConnectionsSupplier; private Supplier> allActiveConnectionsSupplier; private int peersLowerBound; + private PeerTable peerTable; private Builder() {} @@ -399,12 +395,13 @@ public class RlpxAgent { localNode, connectionEvents, metricsSystem, - p2pTLSConfiguration.get()); + p2pTLSConfiguration.get(), + peerTable); } else { LOG.debug("Using default NettyConnectionInitializer"); connectionInitializer = new NettyConnectionInitializer( - nodeKey, config, localNode, connectionEvents, metricsSystem); + nodeKey, config, localNode, connectionEvents, metricsSystem, peerTable); } } @@ -499,5 +496,10 @@ public class RlpxAgent { this.peersLowerBound = peersLowerBound; return this; } + + public Builder peerTable(final PeerTable peerTable) { + this.peerTable = peerTable; + return this; + } } } diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/AbstractHandshakeHandler.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/AbstractHandshakeHandler.java index 003a6ab1d..80be0a673 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/AbstractHandshakeHandler.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/AbstractHandshakeHandler.java @@ -14,6 +14,7 @@ */ package org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.peers.LocalNode; import org.hyperledger.besu.ethereum.p2p.peers.Peer; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; @@ -60,6 +61,7 @@ abstract class AbstractHandshakeHandler extends SimpleChannelInboundHandler subProtocols, @@ -70,7 +72,8 @@ abstract class AbstractHandshakeHandler extends SimpleChannelInboundHandler { + if (ff.isSuccess()) { + LOG.trace("Successfully wrote hello message"); + } + }); + msg.retain(); + ctx.fireChannelRead(msg); } - - final Bytes nodeId = handshaker.partyPubKey().getEncodedBytes(); - if (!localNode.isReady()) { - // If we're handling a connection before the node is fully up, just disconnect - LOG.debug("Rejecting connection because local node is not ready {}", nodeId); - disconnect(ctx, DisconnectMessage.DisconnectReason.UNKNOWN); - return; - } - - LOG.trace("Sending framed hello"); - - // Exchange keys done - final Framer framer = this.framerProvider.buildFramer(handshaker.secrets()); - - final ByteToMessageDecoder deFramer = - new DeFramer( - framer, - subProtocols, - localNode, - expectedPeer, - connectionEventDispatcher, - connectionFuture, - metricsSystem, - inboundInitiated); - - ctx.channel() - .pipeline() - .replace(this, "DeFramer", deFramer) - .addBefore("DeFramer", "validate", new ValidateFirstOutboundMessage(framer)); - - ctx.writeAndFlush(new OutboundMessage(null, HelloMessage.create(localNode.getPeerInfo()))) - .addListener( - ff -> { - if (ff.isSuccess()) { - LOG.trace("Successfully wrote hello message"); - } - }); - msg.retain(); - ctx.fireChannelRead(msg); } private void disconnect( diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramer.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramer.java index b39a6b821..c7c600d3f 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramer.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramer.java @@ -14,6 +14,8 @@ */ package org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty; +import org.hyperledger.besu.ethereum.p2p.discovery.DiscoveryPeer; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.network.exceptions.BreachOfProtocolException; import org.hyperledger.besu.ethereum.p2p.network.exceptions.IncompatiblePeerException; import org.hyperledger.besu.ethereum.p2p.network.exceptions.PeerChannelClosedException; @@ -70,6 +72,7 @@ final class DeFramer extends ByteToMessageDecoder { private final Optional expectedPeer; private final List subProtocols; private final boolean inboundInitiated; + private final PeerTable peerTable; private boolean hellosExchanged; private final LabelledMetric outboundMessagesCounter; @@ -81,7 +84,8 @@ final class DeFramer extends ByteToMessageDecoder { final PeerConnectionEventDispatcher connectionEventDispatcher, final CompletableFuture connectFuture, final MetricsSystem metricsSystem, - final boolean inboundInitiated) { + final boolean inboundInitiated, + final PeerTable peerTable) { this.framer = framer; this.subProtocols = subProtocols; this.localNode = localNode; @@ -89,6 +93,7 @@ final class DeFramer extends ByteToMessageDecoder { this.connectFuture = connectFuture; this.connectionEventDispatcher = connectionEventDispatcher; this.inboundInitiated = inboundInitiated; + this.peerTable = peerTable; this.outboundMessagesCounter = metricsSystem.createLabelledCounter( BesuMetricCategory.NETWORK, @@ -105,8 +110,11 @@ final class DeFramer extends ByteToMessageDecoder { while ((message = framer.deframe(in)) != null) { if (hellosExchanged) { + out.add(message); + } else if (message.getCode() == WireMessageCodes.HELLO) { + hellosExchanged = true; // Decode first hello and use the payload to modify pipeline final PeerInfo peerInfo; @@ -129,13 +137,27 @@ final class DeFramer extends ByteToMessageDecoder { subProtocols, localNode.getPeerInfo().getCapabilities(), peerInfo.getCapabilities()); - final Optional peer = expectedPeer.or(() -> createPeer(peerInfo, ctx)); - if (peer.isEmpty()) { - LOG.debug("Failed to create connection for peer {}", peerInfo); - connectFuture.completeExceptionally(new PeerChannelClosedException(peerInfo)); - ctx.close(); - return; + + Optional peer; + if (expectedPeer.isPresent()) { + peer = expectedPeer; + } else { + // This is an inbound "Hello" message. Create peer from information from the Hello message + peer = createPeer(peerInfo, ctx); + if (peer.isEmpty()) { + LOG.debug("Failed to create connection for peer {}", peerInfo); + connectFuture.completeExceptionally(new PeerChannelClosedException(peerInfo)); + ctx.close(); + return; + } + // If we can find the DiscoveryPeer for the peer in the PeerTable we use it, because + // it could contains additional information, like the fork id. + final Optional discoveryPeer = peerTable.get(peer.get()); + if (discoveryPeer.isPresent()) { + peer = Optional.of(discoveryPeer.get()); + } } + final PeerConnection connection = new NettyPeerConnection( ctx, @@ -176,7 +198,9 @@ final class DeFramer extends ByteToMessageDecoder { capabilityMultiplexer, connection, connectionEventDispatcher, waitingForPong), new MessageFramer(capabilityMultiplexer, framer)); connectFuture.complete(connection); + } else if (message.getCode() == WireMessageCodes.DISCONNECT) { + final DisconnectMessage disconnectMessage = DisconnectMessage.readFrom(message); LOG.debug( "Peer {} disconnected before sending HELLO. Reason: {}", @@ -185,8 +209,10 @@ final class DeFramer extends ByteToMessageDecoder { ctx.close(); connectFuture.completeExceptionally( new PeerDisconnectedException(disconnectMessage.getReason())); + } else { // Unexpected message - disconnect + LOG.debug( "Message received before HELLO's exchanged (BREACH_OF_PROTOCOL), disconnecting. Peer: {}, Code: {}, Data: {}", expectedPeer.map(Peer::getEnodeURLString).orElse("unknown"), diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/HandshakeHandlerInbound.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/HandshakeHandlerInbound.java index 184cf5cf8..962de68f9 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/HandshakeHandlerInbound.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/HandshakeHandlerInbound.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty; import org.hyperledger.besu.cryptoservices.NodeKey; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.peers.LocalNode; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnectionEventDispatcher; @@ -40,7 +41,8 @@ final class HandshakeHandlerInbound extends AbstractHandshakeHandler { final PeerConnectionEventDispatcher connectionEventDispatcher, final MetricsSystem metricsSystem, final HandshakerProvider handshakerProvider, - final FramerProvider framerProvider) { + final FramerProvider framerProvider, + final PeerTable peerTable) { super( subProtocols, localNode, @@ -50,7 +52,8 @@ final class HandshakeHandlerInbound extends AbstractHandshakeHandler { metricsSystem, handshakerProvider, framerProvider, - true); + true, + peerTable); handshaker.prepareResponder(nodeKey); } diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/HandshakeHandlerOutbound.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/HandshakeHandlerOutbound.java index 205b6f655..46e600d74 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/HandshakeHandlerOutbound.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/HandshakeHandlerOutbound.java @@ -16,6 +16,7 @@ package org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty; import org.hyperledger.besu.crypto.SignatureAlgorithmFactory; import org.hyperledger.besu.cryptoservices.NodeKey; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.peers.LocalNode; import org.hyperledger.besu.ethereum.p2p.peers.Peer; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; @@ -50,7 +51,8 @@ final class HandshakeHandlerOutbound extends AbstractHandshakeHandler { final PeerConnectionEventDispatcher connectionEventDispatcher, final MetricsSystem metricsSystem, final HandshakerProvider handshakerProvider, - final FramerProvider framerProvider) { + final FramerProvider framerProvider, + final PeerTable peerTable) { super( subProtocols, localNode, @@ -60,7 +62,8 @@ final class HandshakeHandlerOutbound extends AbstractHandshakeHandler { metricsSystem, handshakerProvider, framerProvider, - false); + false, + peerTable); handshaker.prepareInitiator( nodeKey, SignatureAlgorithmFactory.getInstance().createPublicKey(peer.getId())); this.first = handshaker.firstMessage(); diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyConnectionInitializer.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyConnectionInitializer.java index c20e511df..f386c59a3 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyConnectionInitializer.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyConnectionInitializer.java @@ -17,6 +17,7 @@ package org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty; import org.hyperledger.besu.cryptoservices.NodeKey; import org.hyperledger.besu.ethereum.p2p.config.RlpxConfiguration; import org.hyperledger.besu.ethereum.p2p.discovery.DiscoveryPeer; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.peers.LocalNode; import org.hyperledger.besu.ethereum.p2p.peers.Peer; import org.hyperledger.besu.ethereum.p2p.rlpx.ConnectCallback; @@ -68,6 +69,7 @@ public class NettyConnectionInitializer private final PeerConnectionEventDispatcher eventDispatcher; private final MetricsSystem metricsSystem; private final Subscribers connectSubscribers = Subscribers.create(); + private final PeerTable peerTable; private ChannelFuture server; private final EventLoopGroup boss = new NioEventLoopGroup(1); @@ -80,12 +82,14 @@ public class NettyConnectionInitializer final RlpxConfiguration config, final LocalNode localNode, final PeerConnectionEventDispatcher eventDispatcher, - final MetricsSystem metricsSystem) { + final MetricsSystem metricsSystem, + final PeerTable peerTable) { this.nodeKey = nodeKey; this.config = config; this.localNode = localNode; this.eventDispatcher = eventDispatcher; this.metricsSystem = metricsSystem; + this.peerTable = peerTable; metricsSystem.createIntegerGauge( BesuMetricCategory.NETWORK, @@ -244,7 +248,8 @@ public class NettyConnectionInitializer eventDispatcher, metricsSystem, this, - this); + this, + peerTable); } @Nonnull @@ -259,7 +264,8 @@ public class NettyConnectionInitializer eventDispatcher, metricsSystem, this, - this); + this, + peerTable); } @Nonnull diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyTLSConnectionInitializer.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyTLSConnectionInitializer.java index 4e6010771..db41c1574 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyTLSConnectionInitializer.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyTLSConnectionInitializer.java @@ -19,6 +19,7 @@ import static org.hyperledger.besu.ethereum.p2p.rlpx.RlpxFrameConstants.LENGTH_M import org.hyperledger.besu.cryptoservices.NodeKey; import org.hyperledger.besu.ethereum.p2p.config.RlpxConfiguration; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.peers.LocalNode; import org.hyperledger.besu.ethereum.p2p.peers.Peer; import org.hyperledger.besu.ethereum.p2p.plain.PlainFramer; @@ -55,7 +56,8 @@ public class NettyTLSConnectionInitializer extends NettyConnectionInitializer { final LocalNode localNode, final PeerConnectionEventDispatcher eventDispatcher, final MetricsSystem metricsSystem, - final TLSConfiguration p2pTLSConfiguration) { + final TLSConfiguration p2pTLSConfiguration, + final PeerTable peerTable) { this( nodeKey, config, @@ -63,7 +65,8 @@ public class NettyTLSConnectionInitializer extends NettyConnectionInitializer { eventDispatcher, metricsSystem, defaultTlsContextFactorySupplier(p2pTLSConfiguration), - p2pTLSConfiguration.getClientHelloSniHeaderEnabled()); + p2pTLSConfiguration.getClientHelloSniHeaderEnabled(), + peerTable); } @VisibleForTesting @@ -74,8 +77,9 @@ public class NettyTLSConnectionInitializer extends NettyConnectionInitializer { final PeerConnectionEventDispatcher eventDispatcher, final MetricsSystem metricsSystem, final Supplier tlsContextFactorySupplier, - final Boolean clientHelloSniHeaderEnabled) { - super(nodeKey, config, localNode, eventDispatcher, metricsSystem); + final Boolean clientHelloSniHeaderEnabled, + final PeerTable peerTable) { + super(nodeKey, config, localNode, eventDispatcher, metricsSystem, peerTable); if (tlsContextFactorySupplier != null) { this.tlsContextFactorySupplier = Optional.of(Suppliers.memoize(tlsContextFactorySupplier::get)); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTestHelper.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTestHelper.java index 464273243..ffc9fb1c3 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTestHelper.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTestHelper.java @@ -295,6 +295,7 @@ public class PeerDiscoveryTestHelper { config.setAdvertisedHost(advertisedHost); config.setBindPort(port); config.setActive(active); + config.setFilterOnEnrForkId(false); final ForkIdManager mockForkIdManager = mock(ForkIdManager.class); final ForkId forkId = new ForkId(Bytes.EMPTY, Bytes.EMPTY); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/MockPeerDiscoveryAgent.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/MockPeerDiscoveryAgent.java index ea6e1593b..88196f18b 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/MockPeerDiscoveryAgent.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/MockPeerDiscoveryAgent.java @@ -63,7 +63,8 @@ public class MockPeerDiscoveryAgent extends PeerDiscoveryAgent { new NoOpMetricsSystem(), new InMemoryKeyValueStorageProvider(), forkIdManager, - rlpxAgent); + rlpxAgent, + new PeerTable(nodeKey.getPublicKey().getEncodedBytes())); this.agentNetwork = agentNetwork; } diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryControllerTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryControllerTest.java index 182366ac0..ca0d7c643 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryControllerTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryControllerTest.java @@ -35,8 +35,6 @@ import org.hyperledger.besu.crypto.Hash; import org.hyperledger.besu.crypto.SignatureAlgorithm; import org.hyperledger.besu.crypto.SignatureAlgorithmFactory; import org.hyperledger.besu.cryptoservices.NodeKey; -import org.hyperledger.besu.ethereum.forkid.ForkId; -import org.hyperledger.besu.ethereum.forkid.ForkIdManager; import org.hyperledger.besu.ethereum.p2p.discovery.DiscoveryPeer; import org.hyperledger.besu.ethereum.p2p.discovery.Endpoint; import org.hyperledger.besu.ethereum.p2p.discovery.PeerDiscoveryStatus; @@ -1480,14 +1478,12 @@ public class PeerDiscoveryControllerTest { } @Test - public void shouldFiltersOnForkIdSuccess() { + public void forkIdShouldBeAvailableIfEnrPacketContainsForkId() { final List nodeKeys = PeerDiscoveryTestHelper.generateNodeKeys(1); final List peers = helper.createDiscoveryPeers(nodeKeys); - final ForkIdManager forkIdManager = mock(ForkIdManager.class); final DiscoveryPeer sender = peers.get(0); - final Packet enrPacket = prepareForForkIdCheck(forkIdManager, nodeKeys, sender, true); + final Packet enrPacket = prepareForForkIdCheck(nodeKeys, sender, true); - when(forkIdManager.peerCheck(any(ForkId.class))).thenReturn(true); controller.onMessage(enrPacket, sender); final Optional maybePeer = @@ -1501,35 +1497,12 @@ public class PeerDiscoveryControllerTest { verify(controller, times(1)).connectOnRlpxLayer(eq(maybePeer.get())); } - @Test - public void shouldFiltersOnForkIdFailure() { - final List nodeKeys = PeerDiscoveryTestHelper.generateNodeKeys(1); - final List peers = helper.createDiscoveryPeers(nodeKeys); - final ForkIdManager forkIdManager = mock(ForkIdManager.class); - final DiscoveryPeer sender = peers.get(0); - final Packet enrPacket = prepareForForkIdCheck(forkIdManager, nodeKeys, sender, true); - - when(forkIdManager.peerCheck(any(ForkId.class))).thenReturn(false); - controller.onMessage(enrPacket, sender); - - final Optional maybePeer = - controller - .streamDiscoveredPeers() - .filter(p -> p.getId().equals(sender.getId())) - .findFirst(); - - assertThat(maybePeer.isPresent()).isTrue(); - assertThat(maybePeer.get().getForkId().isPresent()).isTrue(); - verify(controller, never()).connectOnRlpxLayer(eq(maybePeer.get())); - } - @Test public void shouldStillCallConnectIfNoForkIdSent() { final List nodeKeys = PeerDiscoveryTestHelper.generateNodeKeys(1); final List peers = helper.createDiscoveryPeers(nodeKeys); final DiscoveryPeer sender = peers.get(0); - final Packet enrPacket = - prepareForForkIdCheck(mock(ForkIdManager.class), nodeKeys, sender, false); + final Packet enrPacket = prepareForForkIdCheck(nodeKeys, sender, false); controller.onMessage(enrPacket, sender); @@ -1546,10 +1519,7 @@ public class PeerDiscoveryControllerTest { @NotNull private Packet prepareForForkIdCheck( - final ForkIdManager forkIdManager, - final List nodeKeys, - final DiscoveryPeer sender, - final boolean sendForkId) { + final List nodeKeys, final DiscoveryPeer sender, final boolean sendForkId) { final HashMap packetTypeBytesHashMap = new HashMap<>(); final OutboundMessageHandler outboundMessageHandler = (dp, pa) -> packetTypeBytesHashMap.put(pa.getType(), pa.getHash()); @@ -1573,7 +1543,6 @@ public class PeerDiscoveryControllerTest { .outboundMessageHandler(outboundMessageHandler) .enrCache(enrs) .filterOnForkId(true) - .forkIdManager(forkIdManager) .build(); // Mock the creation of the PING packet, so that we can control the hash, which gets validated @@ -1720,7 +1689,6 @@ public class PeerDiscoveryControllerTest { private Cache enrs = CacheBuilder.newBuilder().maximumSize(50).expireAfterWrite(10, TimeUnit.SECONDS).build(); private boolean filterOnForkId = false; - private ForkIdManager forkIdManager; public static ControllerBuilder create() { return new ControllerBuilder(); @@ -1776,11 +1744,6 @@ public class PeerDiscoveryControllerTest { return this; } - public ControllerBuilder forkIdManager(final ForkIdManager forkIdManager) { - this.forkIdManager = forkIdManager; - return this; - } - PeerDiscoveryController build() { checkNotNull(nodeKey); if (localPeer == null) { @@ -1803,7 +1766,6 @@ public class PeerDiscoveryControllerTest { .peerPermissions(peerPermissions) .metricsSystem(new NoOpMetricsSystem()) .cacheForEnrRequests(enrs) - .forkIdManager(forkIdManager == null ? mock(ForkIdManager.class) : forkIdManager) .filterOnEnrForkId(filterOnForkId) .rlpxAgent(mock(RlpxAgent.class)) .build()); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryTableRefreshTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryTableRefreshTest.java index 949b31890..6320c9096 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryTableRefreshTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryTableRefreshTest.java @@ -24,7 +24,6 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import org.hyperledger.besu.cryptoservices.NodeKey; -import org.hyperledger.besu.ethereum.forkid.ForkIdManager; import org.hyperledger.besu.ethereum.p2p.discovery.DiscoveryPeer; import org.hyperledger.besu.ethereum.p2p.discovery.PeerDiscoveryStatus; import org.hyperledger.besu.ethereum.p2p.discovery.PeerDiscoveryTestHelper; @@ -72,7 +71,6 @@ public class PeerDiscoveryTableRefreshTest { .tableRefreshIntervalMs(0) .metricsSystem(new NoOpMetricsSystem()) .rlpxAgent(mock(RlpxAgent.class)) - .forkIdManager(mock(ForkIdManager.class)) .build()); controller.start(); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerTableTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerTableTest.java index c0909a9b8..dff9d2316 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerTableTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerTableTest.java @@ -43,7 +43,7 @@ public class PeerTableTest { @Test public void addPeer() { - final PeerTable table = new PeerTable(Peer.randomId(), 16); + final PeerTable table = new PeerTable(Peer.randomId()); final List peers = helper.createDiscoveryPeers(5); for (final DiscoveryPeer peer : peers) { @@ -63,7 +63,7 @@ public class PeerTableTest { .ipAddress("127.0.0.1") .discoveryAndListeningPorts(12345) .build()); - final PeerTable table = new PeerTable(localPeer.getId(), 16); + final PeerTable table = new PeerTable(localPeer.getId()); final PeerTable.AddResult result = table.tryAdd(localPeer); assertThat(result.getOutcome()).isEqualTo(AddOutcome.SELF); @@ -72,7 +72,7 @@ public class PeerTableTest { @Test public void peerExists() { - final PeerTable table = new PeerTable(Peer.randomId(), 16); + final PeerTable table = new PeerTable(Peer.randomId()); final DiscoveryPeer peer = helper.createDiscoveryPeer(); assertThat(table.tryAdd(peer).getOutcome()).isEqualTo(AddOutcome.ADDED); @@ -87,7 +87,7 @@ public class PeerTableTest { @Test public void peerExists_withDifferentIp() { - final PeerTable table = new PeerTable(Peer.randomId(), 16); + final PeerTable table = new PeerTable(Peer.randomId()); final Bytes peerId = SIGNATURE_ALGORITHM.get().generateKeyPair().getPublicKey().getEncodedBytes(); final DiscoveryPeer peer = @@ -107,7 +107,7 @@ public class PeerTableTest { @Test public void peerExists_withDifferentUdpPort() { - final PeerTable table = new PeerTable(Peer.randomId(), 16); + final PeerTable table = new PeerTable(Peer.randomId()); final Bytes peerId = SIGNATURE_ALGORITHM.get().generateKeyPair().getPublicKey().getEncodedBytes(); final DiscoveryPeer peer = @@ -127,7 +127,7 @@ public class PeerTableTest { @Test public void peerExists_withDifferentIdAndUdpPort() { - final PeerTable table = new PeerTable(Peer.randomId(), 16); + final PeerTable table = new PeerTable(Peer.randomId()); final Bytes peerId = SIGNATURE_ALGORITHM.get().generateKeyPair().getPublicKey().getEncodedBytes(); final DiscoveryPeer peer = @@ -147,7 +147,7 @@ public class PeerTableTest { @Test public void evictExistingPeerShouldEvict() { - final PeerTable table = new PeerTable(Peer.randomId(), 16); + final PeerTable table = new PeerTable(Peer.randomId()); final DiscoveryPeer peer = helper.createDiscoveryPeer(); table.tryAdd(peer); @@ -158,7 +158,7 @@ public class PeerTableTest { @Test public void evictPeerFromEmptyTableShouldNotEvict() { - final PeerTable table = new PeerTable(Peer.randomId(), 16); + final PeerTable table = new PeerTable(Peer.randomId()); final DiscoveryPeer peer = helper.createDiscoveryPeer(); final EvictResult evictResult = table.tryEvict(peer); @@ -167,7 +167,7 @@ public class PeerTableTest { @Test public void evictAbsentPeerShouldNotEvict() { - final PeerTable table = new PeerTable(Peer.randomId(), 16); + final PeerTable table = new PeerTable(Peer.randomId()); final DiscoveryPeer peer = helper.createDiscoveryPeer(); final List otherPeers = helper.createDiscoveryPeers(5); otherPeers.forEach(table::tryAdd); @@ -179,7 +179,7 @@ public class PeerTableTest { @Test public void evictSelfPeerShouldReturnSelfOutcome() { final DiscoveryPeer peer = helper.createDiscoveryPeer(); - final PeerTable table = new PeerTable(peer.getId(), 16); + final PeerTable table = new PeerTable(peer.getId()); final EvictResult evictResult = table.tryEvict(peer); assertThat(evictResult.getOutcome()).isEqualTo(EvictOutcome.SELF); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/RecursivePeerRefreshStateTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/RecursivePeerRefreshStateTest.java index e79abb883..5d26f8cc6 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/RecursivePeerRefreshStateTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/RecursivePeerRefreshStateTest.java @@ -57,7 +57,7 @@ public class RecursivePeerRefreshStateTest { neighborFinder, timerUtil, localPeer, - new PeerTable(createId(999), 16), + new PeerTable(createId(999)), peerPermissions, 5, 100); @@ -180,7 +180,7 @@ public class RecursivePeerRefreshStateTest { neighborFinder, timerUtil, localPeer, - new PeerTable(createId(999), 16), + new PeerTable(createId(999)), peerPermissions, 5, 1); @@ -466,7 +466,7 @@ public class RecursivePeerRefreshStateTest { neighborFinder, timerUtil, localPeer, - new PeerTable(createId(999), 16), + new PeerTable(createId(999)), peerPermissions, 5, 100); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetworkTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetworkTest.java index a54c9038b..68a4f76b5 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetworkTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetworkTest.java @@ -55,7 +55,6 @@ import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.stream.Collectors; import java.util.stream.Stream; import io.vertx.core.Context; @@ -82,7 +81,7 @@ public final class DefaultP2PNetworkTest { @Mock PeerDiscoveryAgent discoveryAgent; @Mock RlpxAgent rlpxAgent; - @Captor private ArgumentCaptor> peerStreamCaptor; + @Captor private ArgumentCaptor peerCaptor; private final NetworkingConfiguration config = NetworkingConfiguration.create() @@ -276,12 +275,9 @@ public final class DefaultP2PNetworkTest { final DefaultP2PNetwork network = network(); network.attemptPeerConnections(); - verify(rlpxAgent, times(1)).connect(peerStreamCaptor.capture()); + verify(rlpxAgent, times(1)).connect(peerCaptor.capture()); - final List capturedPeers = - peerStreamCaptor.getValue().collect(Collectors.toList()); - assertThat(capturedPeers.contains(discoPeer)).isTrue(); - assertThat(capturedPeers.size()).isEqualTo(1); + assertThat(peerCaptor.getValue()).isEqualTo(discoPeer); } @Test @@ -293,12 +289,7 @@ public final class DefaultP2PNetworkTest { final DefaultP2PNetwork network = network(); network.attemptPeerConnections(); - verify(rlpxAgent, times(1)).connect(peerStreamCaptor.capture()); - - final List capturedPeers = - peerStreamCaptor.getValue().collect(Collectors.toList()); - assertThat(capturedPeers.contains(discoPeer)).isFalse(); - assertThat(capturedPeers.size()).isEqualTo(0); + verify(rlpxAgent, times(0)).connect(any()); } @Test @@ -314,14 +305,7 @@ public final class DefaultP2PNetworkTest { final DefaultP2PNetwork network = network(); network.attemptPeerConnections(); - verify(rlpxAgent, times(1)).connect(peerStreamCaptor.capture()); - - final List capturedPeers = - peerStreamCaptor.getValue().collect(Collectors.toList()); - assertThat(capturedPeers.size()).isEqualTo(3); - assertThat(capturedPeers.get(0)).isEqualTo(discoPeers.get(1)); - assertThat(capturedPeers.get(1)).isEqualTo(discoPeers.get(0)); - assertThat(capturedPeers.get(2)).isEqualTo(discoPeers.get(2)); + verify(rlpxAgent, times(3)).connect(any()); } @Test diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/RlpxAgentTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/RlpxAgentTest.java index d2fdf8c76..c639579ee 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/RlpxAgentTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/RlpxAgentTest.java @@ -296,7 +296,7 @@ public class RlpxAgentTest { Stream.generate(PeerTestHelper::createPeer).limit(peerNo); agent = spy(agent); - agent.connect(peerStream); + peerStream.forEach(agent::connect); assertThat(agent.getMapOfCompletableFutures().size()).isEqualTo(peerNo); verify(agent, times(peerNo)).connect(any(Peer.class)); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java index f5030fd9a..952ebf7c2 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java @@ -24,6 +24,9 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import org.hyperledger.besu.ethereum.forkid.ForkId; +import org.hyperledger.besu.ethereum.p2p.discovery.DiscoveryPeer; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.network.exceptions.BreachOfProtocolException; import org.hyperledger.besu.ethereum.p2p.network.exceptions.IncompatiblePeerException; import org.hyperledger.besu.ethereum.p2p.network.exceptions.PeerChannelClosedException; @@ -104,7 +107,7 @@ public class DeFramerTest { private final LocalNode localNode = LocalNode.create(clientId, p2pVersion, capabilities, localEnode); - private final DeFramer deFramer = createDeFramer(null); + private final DeFramer deFramer = createDeFramer(null, Optional.empty()); @BeforeEach @SuppressWarnings("unchecked") @@ -219,7 +222,7 @@ public class DeFramerTest { final Peer peer = createRemotePeer(); final PeerInfo remotePeerInfo = new PeerInfo(p2pVersion, clientId, capabilities, 0, peer.getId()); - final DeFramer deFramer = createDeFramer(null); + final DeFramer deFramer = createDeFramer(null, Optional.empty()); final HelloMessage helloMessage = HelloMessage.create(remotePeerInfo); final ByteBuf data = Unpooled.wrappedBuffer(helloMessage.getData().toArray()); @@ -260,6 +263,39 @@ public class DeFramerTest { assertThat(out.size()).isEqualTo(1); } + @Test + public void decode_duringHandshakeFindsPeerInPeerTable() + throws ExecutionException, InterruptedException { + final ChannelFuture future = NettyMocks.channelFuture(false); + when(channel.closeFuture()).thenReturn(future); + + final Peer peer = createRemotePeer(); + final PeerInfo remotePeerInfo = + new PeerInfo(p2pVersion, clientId, capabilities, 0, peer.getId()); + + final HelloMessage helloMessage = HelloMessage.create(remotePeerInfo); + final Bytes nodeId = helloMessage.getPeerInfo().getNodeId(); + final String enodeURLString = + "enode://" + nodeId.toString().substring(2) + "@" + "12.13.14.15:30303?discport=30301"; + final Optional discoveryPeer = + DiscoveryPeer.from(DefaultPeer.fromURI(enodeURLString)); + final ForkId forkId = new ForkId(Bytes.fromHexString("0x190a55ad"), 4L); + discoveryPeer.orElseThrow().setForkId(forkId); + final DeFramer deFramer = createDeFramer(null, discoveryPeer); + final ByteBuf data = Unpooled.wrappedBuffer(helloMessage.getData().toArray()); + when(framer.deframe(eq(data))) + .thenReturn(new RawMessage(helloMessage.getCode(), helloMessage.getData())) + .thenReturn(null); + final List out = new ArrayList<>(); + deFramer.decode(ctx, data, out); + + assertThat(connectFuture).isDone(); + assertThat(connectFuture).isNotCompletedExceptionally(); + final PeerConnection peerConnection = connectFuture.get(); + assertThat(peerConnection.getPeerInfo()).isEqualTo(remotePeerInfo); + assertThat(peerConnection.getPeer().getForkId().orElseThrow()).isEqualTo(forkId); + } + @Test public void decode_handlesUnexpectedPeerId() { final ChannelFuture future = NettyMocks.channelFuture(false); @@ -274,7 +310,7 @@ public class DeFramerTest { capabilities, peer.getEnodeURL().getListeningPortOrZero(), mismatchedId); - final DeFramer deFramer = createDeFramer(peer); + final DeFramer deFramer = createDeFramer(peer, Optional.empty()); final HelloMessage helloMessage = HelloMessage.create(remotePeerInfo); final ByteBuf data = Unpooled.wrappedBuffer(helloMessage.getData().toArray()); @@ -414,7 +450,10 @@ public class DeFramerTest { forPeer.getId()); } - private DeFramer createDeFramer(final Peer expectedPeer) { + private DeFramer createDeFramer( + final Peer expectedPeer, final Optional peerInPeerTable) { + final PeerTable peerTable = new PeerTable(localNode.getPeerInfo().getNodeId()); + peerInPeerTable.ifPresent(peerTable::tryAdd); return new DeFramer( framer, Arrays.asList(MockSubProtocol.create("eth")), @@ -423,6 +462,7 @@ public class DeFramerTest { connectionEventDispatcher, connectFuture, new NoOpMetricsSystem(), - true); + true, + peerTable); } } diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyTLSConnectionInitializerTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyTLSConnectionInitializerTest.java index d4637952b..f7f3dbc51 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyTLSConnectionInitializerTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyTLSConnectionInitializerTest.java @@ -25,6 +25,7 @@ import static org.mockito.Mockito.when; import org.hyperledger.besu.cryptoservices.NodeKey; import org.hyperledger.besu.ethereum.p2p.config.RlpxConfiguration; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.peers.LocalNode; import org.hyperledger.besu.ethereum.p2p.peers.Peer; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnectionEventDispatcher; @@ -44,6 +45,7 @@ import io.netty.handler.codec.compression.SnappyFrameDecoder; import io.netty.handler.codec.compression.SnappyFrameEncoder; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslHandler; +import org.apache.tuweni.bytes.Bytes; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -95,7 +97,8 @@ public class NettyTLSConnectionInitializerTest { eventDispatcher, new NoOpMetricsSystem(), () -> tlsContextFactory, - clientHelloSniHeaderEnabled); + clientHelloSniHeaderEnabled, + new PeerTable(Bytes.random(64))); } @Test