diff --git a/CHANGELOG.md b/CHANGELOG.md index 48955bf2b..f5b24a6f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - BFT networks won't start with SNAP or CHECKPOINT sync (previously Besu would start with this config but quietly fail to sync, so it's now more obvious that it won't work) [#6625](https://github.com/hyperledger/besu/pull/6625), [#6667](https://github.com/hyperledger/besu/pull/6667) ### Upcoming Breaking Changes +- Receipt compaction will be enabled by default in a future version of Besu. After this change it will not be possible to downgrade to the previous Besu version. ### Deprecations @@ -25,11 +26,13 @@ - `eth_call` for blob tx allows for empty `maxFeePerBlobGas` [#6731](https://github.com/hyperledger/besu/pull/6731) - Extend error handling of plugin RPC methods [#6759](https://github.com/hyperledger/besu/pull/6759) - Added engine_newPayloadV4 and engine_getPayloadV4 methods [#6783](https://github.com/hyperledger/besu/pull/6783) +- Reduce storage size of receipts [#6602](https://github.com/hyperledger/besu/pull/6602) ### Bug fixes - Fix txpool dump/restore race condition [#6665](https://github.com/hyperledger/besu/pull/6665) - Make block transaction selection max time aware of PoA transitions [#6676](https://github.com/hyperledger/besu/pull/6676) - Don't enable the BFT mining coordinator when running sub commands such as `blocks export` [#6675](https://github.com/hyperledger/besu/pull/6675) +- In JSON-RPC return optional `v` fields for type 1 and type 2 transactions [#6762](https://github.com/hyperledger/besu/pull/6762) ### Download Links diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/AcceptanceTestBase.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/AcceptanceTestBase.java index cac4deb9d..0e1f6d65d 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/AcceptanceTestBase.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/AcceptanceTestBase.java @@ -56,15 +56,12 @@ import java.math.BigInteger; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import org.junit.After; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.extension.ExtendWith; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Superclass for acceptance tests. For now (transition to junit5 is ongoing) this class supports - * junit4 format. - */ +/** Superclass for acceptance tests. */ @ExtendWith(AcceptanceTestBaseTestWatcher.class) public class AcceptanceTestBase { @@ -131,7 +128,7 @@ public class AcceptanceTestBase { exitedSuccessfully = new ExitedWithCode(0); } - @After + @AfterEach public void tearDownAcceptanceTestBase() { reportMemory(); cluster.close(); diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java index c822ce899..b2f9bc0ab 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java @@ -62,6 +62,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Properties; @@ -431,7 +432,9 @@ public class BesuNode implements NodeConfiguration, RunnableNode, AutoCloseable getGenesisConfig() .map( gc -> - gc.toLowerCase().contains("ibft") ? ConsensusType.IBFT2 : ConsensusType.QBFT) + gc.toLowerCase(Locale.ROOT).contains("ibft") + ? ConsensusType.IBFT2 + : ConsensusType.QBFT) .orElse(ConsensusType.IBFT2); nodeRequests = @@ -786,6 +789,21 @@ public class BesuNode implements NodeConfiguration, RunnableNode, AutoCloseable nodeRequests.shutdown(); nodeRequests = null; } + + deleteRuntimeFiles(); + } + + private void deleteRuntimeFiles() { + try { + Files.deleteIfExists(homeDirectory.resolve("besu.networks")); + } catch (IOException e) { + LOG.error("Failed to clean up besu.networks file in {}", homeDirectory, e); + } + try { + Files.deleteIfExists(homeDirectory.resolve("besu.ports")); + } catch (IOException e) { + LOG.error("Failed to clean up besu.ports file in {}", homeDirectory, e); + } } @Override diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java index 2b80ebea5..adf872e19 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java @@ -52,6 +52,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import org.apache.commons.lang3.SystemUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; @@ -77,8 +78,15 @@ public class ProcessBesuNodeRunner implements BesuNodeRunner { final Path dataDir = node.homeDirectory(); + final var workingDir = + new File(System.getProperty("user.dir")).getParentFile().getParentFile().toPath(); + final List params = new ArrayList<>(); - params.add("build/install/besu/bin/besu"); + if (SystemUtils.IS_OS_WINDOWS) { + params.add(workingDir.resolve("build\\install\\besu\\bin\\besu.bat").toString()); + } else { + params.add("build/install/besu/bin/besu"); + } params.add("--data-path"); params.add(dataDir.toAbsolutePath().toString()); @@ -422,15 +430,13 @@ public class ProcessBesuNodeRunner implements BesuNodeRunner { LOG.info("Creating besu process with params {}", params); final ProcessBuilder processBuilder = new ProcessBuilder(params) - .directory(new File(System.getProperty("user.dir")).getParentFile().getParentFile()) + .directory(workingDir.toFile()) .redirectErrorStream(true) .redirectInput(Redirect.INHERIT); if (!node.getPlugins().isEmpty()) { processBuilder .environment() - .put( - "BESU_OPTS", - "-Dbesu.plugins.dir=" + dataDir.resolve("plugins").toAbsolutePath().toString()); + .put("BESU_OPTS", "-Dbesu.plugins.dir=" + dataDir.resolve("plugins").toAbsolutePath()); } // Use non-blocking randomness for acceptance tests processBuilder @@ -572,7 +578,7 @@ public class ProcessBesuNodeRunner implements BesuNodeRunner { LOG.info("Killing {} process, pid {}", name, process.pid()); - process.destroy(); + process.descendants().forEach(ProcessHandle::destroy); try { process.waitFor(30, TimeUnit.SECONDS); } catch (final InterruptedException e) { diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/DeploySmartContractTransaction.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/DeploySmartContractTransaction.java index 14bc9a550..97e5c9d38 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/DeploySmartContractTransaction.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/DeploySmartContractTransaction.java @@ -20,6 +20,7 @@ import java.lang.reflect.Method; import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.stream.Collectors; import org.web3j.crypto.Credentials; @@ -83,7 +84,7 @@ public class DeploySmartContractTransaction implements Trans @SuppressWarnings("rawtypes") private boolean parameterTypesAreEqual( - final Class[] expectedTypes, final ArrayList actualObjects) { + final Class[] expectedTypes, final List actualObjects) { if (expectedTypes.length != actualObjects.size()) { return false; } diff --git a/besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java b/besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java index 8d0a693e2..6379e2c1a 100644 --- a/besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java @@ -133,6 +133,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -800,7 +801,7 @@ public class RunnerBuilder { metricsSystem, supportedCapabilities, jsonRpcConfiguration.getRpcApis().stream() - .filter(apiGroup -> !apiGroup.toLowerCase().startsWith("engine")) + .filter(apiGroup -> !apiGroup.toLowerCase(Locale.ROOT).startsWith("engine")) .collect(Collectors.toList()), filterManager, accountLocalConfigPermissioningController, @@ -938,7 +939,7 @@ public class RunnerBuilder { metricsSystem, supportedCapabilities, webSocketConfiguration.getRpcApis().stream() - .filter(apiGroup -> !apiGroup.toLowerCase().startsWith("engine")) + .filter(apiGroup -> !apiGroup.toLowerCase(Locale.ROOT).startsWith("engine")) .collect(Collectors.toList()), filterManager, accountLocalConfigPermissioningController, @@ -1021,7 +1022,7 @@ public class RunnerBuilder { metricsSystem, supportedCapabilities, jsonRpcIpcConfiguration.getEnabledApis().stream() - .filter(apiGroup -> !apiGroup.toLowerCase().startsWith("engine")) + .filter(apiGroup -> !apiGroup.toLowerCase(Locale.ROOT).startsWith("engine")) .collect(Collectors.toList()), filterManager, accountLocalConfigPermissioningController, diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index c993dda3d..901c2cf26 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -2623,10 +2623,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable { SignatureAlgorithmFactory.setInstance(SignatureAlgorithmType.create(ecCurve.get())); } catch (final IllegalArgumentException e) { throw new CommandLine.InitializationException( - new StringBuilder() - .append("Invalid genesis file configuration for ecCurve. ") - .append(e.getMessage()) - .toString()); + "Invalid genesis file configuration for ecCurve. " + e.getMessage()); } } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/config/NetworkName.java b/besu/src/main/java/org/hyperledger/besu/cli/config/NetworkName.java index 554d0645a..868f1396d 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/config/NetworkName.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/config/NetworkName.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.cli.config; import java.math.BigInteger; +import java.util.Locale; import java.util.Optional; import org.apache.commons.lang3.StringUtils; @@ -95,7 +96,7 @@ public enum NetworkName { * @return the string */ public String normalize() { - return StringUtils.capitalize(name().toLowerCase()); + return StringUtils.capitalize(name().toLowerCase(Locale.ROOT)); } /** diff --git a/besu/src/main/java/org/hyperledger/besu/cli/config/ProfileName.java b/besu/src/main/java/org/hyperledger/besu/cli/config/ProfileName.java index 70b3d83ed..111261d5c 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/config/ProfileName.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/config/ProfileName.java @@ -14,6 +14,8 @@ */ package org.hyperledger.besu.cli.config; +import java.util.Locale; + import org.apache.commons.lang3.StringUtils; /** Enum for profile names. Each profile corresponds to a configuration file. */ @@ -51,6 +53,6 @@ public enum ProfileName { @Override public String toString() { - return StringUtils.capitalize(name().replaceAll("_", " ").toLowerCase()); + return StringUtils.capitalize(name().replaceAll("_", " ").toLowerCase(Locale.ROOT)); } } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/converter/MetricCategoryConverter.java b/besu/src/main/java/org/hyperledger/besu/cli/converter/MetricCategoryConverter.java index 684c3ce30..91ca381b5 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/converter/MetricCategoryConverter.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/converter/MetricCategoryConverter.java @@ -18,6 +18,7 @@ import org.hyperledger.besu.plugin.services.metrics.MetricCategory; import java.util.EnumSet; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import com.google.common.annotations.VisibleForTesting; @@ -54,7 +55,7 @@ public class MetricCategoryConverter implements CommandLine.ITypeConverter arity = "1") private Long bonsaiMaxLayersToLoad = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; + @Option( + names = "--receipt-compaction-enabled", + description = "Enables compact storing of receipts (default: ${DEFAULT-VALUE}).", + arity = "1") + private Boolean receiptCompactionEnabled = DEFAULT_RECEIPT_COMPACTION_ENABLED; + @CommandLine.ArgGroup(validate = false) private final DataStorageOptions.Unstable unstableOptions = new Unstable(); @@ -149,6 +157,7 @@ public class DataStorageOptions implements CLIOptions final DataStorageOptions dataStorageOptions = DataStorageOptions.create(); dataStorageOptions.dataStorageFormat = domainObject.getDataStorageFormat(); dataStorageOptions.bonsaiMaxLayersToLoad = domainObject.getBonsaiMaxLayersToLoad(); + dataStorageOptions.receiptCompactionEnabled = domainObject.getReceiptCompactionEnabled(); dataStorageOptions.unstableOptions.bonsaiLimitTrieLogsEnabled = domainObject.getUnstable().getBonsaiLimitTrieLogsEnabled(); dataStorageOptions.unstableOptions.bonsaiTrieLogPruningWindowSize = @@ -164,6 +173,7 @@ public class DataStorageOptions implements CLIOptions return ImmutableDataStorageConfiguration.builder() .dataStorageFormat(dataStorageFormat) .bonsaiMaxLayersToLoad(bonsaiMaxLayersToLoad) + .receiptCompactionEnabled(receiptCompactionEnabled) .unstable( ImmutableDataStorageConfiguration.Unstable.builder() .bonsaiLimitTrieLogsEnabled(unstableOptions.bonsaiLimitTrieLogsEnabled) @@ -184,6 +194,6 @@ public class DataStorageOptions implements CLIOptions * @return the normalized string */ public String normalizeDataStorageFormat() { - return StringUtils.capitalize(dataStorageFormat.toString().toLowerCase()); + return StringUtils.capitalize(dataStorageFormat.toString().toLowerCase(Locale.ROOT)); } } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/LoggingLevelOption.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/LoggingLevelOption.java index ad119ea61..ce5198084 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/LoggingLevelOption.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/LoggingLevelOption.java @@ -14,6 +14,7 @@ */ package org.hyperledger.besu.cli.options.stable; +import java.util.Locale; import java.util.Set; import picocli.CommandLine; @@ -52,8 +53,8 @@ public class LoggingLevelOption { if ("FATAL".equalsIgnoreCase(logLevel)) { System.out.println("FATAL level is deprecated"); this.logLevel = "ERROR"; - } else if (ACCEPTED_VALUES.contains(logLevel.toUpperCase())) { - this.logLevel = logLevel.toUpperCase(); + } else if (ACCEPTED_VALUES.contains(logLevel.toUpperCase(Locale.ROOT))) { + this.logLevel = logLevel.toUpperCase(Locale.ROOT); } else { throw new CommandLine.ParameterException( spec.commandLine(), "Unknown logging value: " + logLevel); diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/GenerateBlockchainConfig.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/GenerateBlockchainConfig.java index 3dc02f0f2..f53f3a805 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/GenerateBlockchainConfig.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/GenerateBlockchainConfig.java @@ -178,11 +178,9 @@ class GenerateBlockchainConfig implements Runnable { if (!SIGNATURE_ALGORITHM.get().isValidPublicKey(publicKey)) { throw new IllegalArgumentException( - new StringBuilder() - .append(publicKeyText) - .append(" is not a valid public key for elliptic curve ") - .append(SIGNATURE_ALGORITHM.get().getCurveName()) - .toString()); + publicKeyText + + " is not a valid public key for elliptic curve " + + SIGNATURE_ALGORITHM.get().getCurveName()); } writeKeypair(publicKey, null); @@ -297,10 +295,7 @@ class GenerateBlockchainConfig implements Runnable { SignatureAlgorithmFactory.setInstance(SignatureAlgorithmType.create(ecCurve.get())); } catch (IllegalArgumentException e) { throw new IllegalArgumentException( - new StringBuilder() - .append("Invalid parameter for ecCurve in genesis config: ") - .append(e.getMessage()) - .toString()); + "Invalid parameter for ecCurve in genesis config: " + e.getMessage()); } } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index 8cebe7761..d94c19d8b 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -27,9 +27,9 @@ import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.rlp.RLP; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import java.io.File; diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index a47271e24..0ee2e4532 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -25,8 +25,8 @@ import org.hyperledger.besu.controller.BesuController; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; diff --git a/besu/src/main/java/org/hyperledger/besu/components/BesuComponent.java b/besu/src/main/java/org/hyperledger/besu/components/BesuComponent.java index f0093243a..65ba2d931 100644 --- a/besu/src/main/java/org/hyperledger/besu/components/BesuComponent.java +++ b/besu/src/main/java/org/hyperledger/besu/components/BesuComponent.java @@ -19,8 +19,8 @@ package org.hyperledger.besu.components; import org.hyperledger.besu.cli.BesuCommand; import org.hyperledger.besu.ethereum.eth.transactions.BlobCache; import org.hyperledger.besu.ethereum.eth.transactions.BlobCacheModule; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoaderModule; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoaderModule; import org.hyperledger.besu.metrics.MetricsSystemModule; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.services.BesuPluginContextImpl; @@ -37,7 +37,7 @@ import org.slf4j.Logger; modules = { BesuCommandModule.class, MetricsSystemModule.class, - CachedMerkleTrieLoaderModule.class, + BonsaiCachedMerkleTrieLoaderModule.class, BesuPluginContextModule.class, BlobCacheModule.class }) @@ -55,7 +55,7 @@ public interface BesuComponent { * * @return CachedMerkleTrieLoader */ - CachedMerkleTrieLoader getCachedMerkleTrieLoader(); + BonsaiCachedMerkleTrieLoader getCachedMerkleTrieLoader(); /** * a metrics system that is observable by a Prometheus or OTEL metrics collection subsystem diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 422980490..8990ba9e4 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -81,11 +81,11 @@ import org.hyperledger.besu.ethereum.p2p.config.NetworkingConfiguration; import org.hyperledger.besu.ethereum.p2p.config.SubProtocolConfiguration; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.trie.forest.ForestWorldStateArchive; import org.hyperledger.besu.ethereum.trie.forest.pruner.MarkSweepPruner; import org.hyperledger.besu.ethereum.trie.forest.pruner.Pruner; @@ -564,7 +564,8 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides storageProvider.createWorldStateStorageCoordinator(dataStorageConfiguration); final BlockchainStorage blockchainStorage = - storageProvider.createBlockchainStorage(protocolSchedule, variablesStorage); + storageProvider.createBlockchainStorage( + protocolSchedule, variablesStorage, dataStorageConfiguration); final MutableBlockchain blockchain = DefaultBlockchain.createMutable( @@ -575,13 +576,14 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides dataDirectory.toString(), numberOfBlocksToCache); - final CachedMerkleTrieLoader cachedMerkleTrieLoader = + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader = besuComponent .map(BesuComponent::getCachedMerkleTrieLoader) - .orElseGet(() -> new CachedMerkleTrieLoader(metricsSystem)); + .orElseGet(() -> new BonsaiCachedMerkleTrieLoader(metricsSystem)); final WorldStateArchive worldStateArchive = - createWorldStateArchive(worldStateStorageCoordinator, blockchain, cachedMerkleTrieLoader); + createWorldStateArchive( + worldStateStorageCoordinator, blockchain, bonsaiCachedMerkleTrieLoader); if (blockchain.getChainHeadBlockNumber() < 1) { genesisState.writeStateTo(worldStateArchive.getMutable()); @@ -1047,7 +1049,7 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides WorldStateArchive createWorldStateArchive( final WorldStateStorageCoordinator worldStateStorageCoordinator, final Blockchain blockchain, - final CachedMerkleTrieLoader cachedMerkleTrieLoader) { + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader) { return switch (dataStorageConfiguration.getDataStorageFormat()) { case BONSAI -> { final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage = @@ -1056,7 +1058,7 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides worldStateKeyValueStorage, blockchain, Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()), - cachedMerkleTrieLoader, + bonsaiCachedMerkleTrieLoader, besuComponent.map(BesuComponent::getBesuPluginContext).orElse(null), evmConfiguration); } @@ -1066,6 +1068,8 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides yield new ForestWorldStateArchive( worldStateStorageCoordinator, preimageStorage, evmConfiguration); } + default -> throw new IllegalStateException( + "Unexpected value: " + dataStorageConfiguration.getDataStorageFormat()); }; } diff --git a/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java index 426c15aed..090a9a9e3 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java @@ -249,6 +249,7 @@ public class TransitionBesuControllerBuilder extends BesuControllerBuilder { return sync; } + @SuppressWarnings("UnusedVariable") private void initTransitionWatcher( final ProtocolContext protocolContext, final TransitionCoordinator composedCoordinator) { diff --git a/besu/src/main/java/org/hyperledger/besu/services/BesuConfigurationImpl.java b/besu/src/main/java/org/hyperledger/besu/services/BesuConfigurationImpl.java index e7fb8202b..ed2e481d7 100644 --- a/besu/src/main/java/org/hyperledger/besu/services/BesuConfigurationImpl.java +++ b/besu/src/main/java/org/hyperledger/besu/services/BesuConfigurationImpl.java @@ -67,4 +67,38 @@ public class BesuConfigurationImpl implements BesuConfiguration { public Wei getMinGasPrice() { return miningParameters.getMinTransactionGasPrice(); } + + @Override + public org.hyperledger.besu.plugin.services.storage.DataStorageConfiguration + getDataStorageConfiguration() { + return new DataStoreConfigurationImpl(dataStorageConfiguration); + } + + /** + * A concrete implementation of DataStorageConfiguration which is used in Besu plugin framework. + */ + public static class DataStoreConfigurationImpl + implements org.hyperledger.besu.plugin.services.storage.DataStorageConfiguration { + + private final DataStorageConfiguration dataStorageConfiguration; + + /** + * Instantiate the concrete implementation of the plugin DataStorageConfiguration. + * + * @param dataStorageConfiguration The Ethereum core module data storage configuration + */ + public DataStoreConfigurationImpl(final DataStorageConfiguration dataStorageConfiguration) { + this.dataStorageConfiguration = dataStorageConfiguration; + } + + @Override + public DataStorageFormat getDatabaseFormat() { + return dataStorageConfiguration.getDataStorageFormat(); + } + + @Override + public boolean getReceiptCompactionEnabled() { + return dataStorageConfiguration.getReceiptCompactionEnabled(); + } + } } diff --git a/besu/src/main/java/org/hyperledger/besu/services/RpcEndpointServiceImpl.java b/besu/src/main/java/org/hyperledger/besu/services/RpcEndpointServiceImpl.java index 94bb55608..34e5b78a6 100644 --- a/besu/src/main/java/org/hyperledger/besu/services/RpcEndpointServiceImpl.java +++ b/besu/src/main/java/org/hyperledger/besu/services/RpcEndpointServiceImpl.java @@ -24,6 +24,7 @@ import org.hyperledger.besu.plugin.services.rpc.PluginRpcRequest; import java.util.Collection; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; @@ -58,7 +59,10 @@ public class RpcEndpointServiceImpl implements RpcEndpointService { namespaces.stream() .anyMatch( namespace -> - entry.getKey().toUpperCase().startsWith(namespace.toUpperCase()))) + entry + .getKey() + .toUpperCase(Locale.ROOT) + .startsWith(namespace.toUpperCase(Locale.ROOT)))) .map(entry -> new PluginJsonRpcMethod(entry.getKey(), entry.getValue())) .collect(Collectors.toMap(PluginJsonRpcMethod::getName, e -> e)); } @@ -71,6 +75,7 @@ public class RpcEndpointServiceImpl implements RpcEndpointService { */ public boolean hasNamespace(final String namespace) { return rpcMethods.keySet().stream() - .anyMatch(key -> key.toUpperCase().startsWith(namespace.toUpperCase())); + .anyMatch( + key -> key.toUpperCase(Locale.ROOT).startsWith(namespace.toUpperCase(Locale.ROOT))); } } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java index bd6985df5..94eba18ad 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java @@ -110,6 +110,24 @@ public class DataStorageOptionsTest "false"); } + @Test + public void receiptCompactionCanBeEnabled() { + internalTestSuccess( + dataStorageConfiguration -> + assertThat(dataStorageConfiguration.getReceiptCompactionEnabled()).isEqualTo(true), + "--receipt-compaction-enabled", + "true"); + } + + @Test + public void receiptCompactionCanBeDisabled() { + internalTestSuccess( + dataStorageConfiguration -> + assertThat(dataStorageConfiguration.getReceiptCompactionEnabled()).isEqualTo(false), + "--receipt-compaction-enabled", + "false"); + } + @Override protected DataStorageConfiguration createDefaultDomainObject() { return DataStorageConfiguration.DEFAULT_CONFIG; diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java index c2789c680..d4a6dd2dc 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -31,9 +31,9 @@ import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; diff --git a/besu/src/test/java/org/hyperledger/besu/controller/AbstractBftBesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/AbstractBftBesuControllerBuilderTest.java index bcfe8a2e1..2f6924112 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/AbstractBftBesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/AbstractBftBesuControllerBuilderTest.java @@ -108,12 +108,13 @@ public abstract class AbstractBftBesuControllerBuilderTest { lenient().when(genesisConfigFile.getConfigOptions()).thenReturn(genesisConfigOptions); lenient().when(genesisConfigOptions.getCheckpointOptions()).thenReturn(checkpointConfigOptions); lenient() - .when(storageProvider.createBlockchainStorage(any(), any())) + .when(storageProvider.createBlockchainStorage(any(), any(), any())) .thenReturn( new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions())); + new MainnetBlockHeaderFunctions(), + false)); lenient() .when( storageProvider.createWorldStateStorageCoordinator( diff --git a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java index b7429ea7b..5eb14201c 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java @@ -43,9 +43,9 @@ import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStoragePrefixedKeyBlockchainStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.trie.forest.pruner.PrunerConfiguration; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; @@ -124,12 +124,13 @@ public class BesuControllerBuilderTest { when(ethashConfigOptions.getFixedDifficulty()).thenReturn(OptionalLong.empty()); when(storageProvider.getStorageBySegmentIdentifier(any())) .thenReturn(new InMemoryKeyValueStorage()); - when(storageProvider.createBlockchainStorage(any(), any())) + when(storageProvider.createBlockchainStorage(any(), any(), any())) .thenReturn( new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions())); + new MainnetBlockHeaderFunctions(), + false)); when(synchronizerConfiguration.getDownloaderParallelism()).thenReturn(1); when(synchronizerConfiguration.getTransactionsParallelism()).thenReturn(1); when(synchronizerConfiguration.getComputationParallelism()).thenReturn(1); @@ -188,7 +189,7 @@ public class BesuControllerBuilderTest { .createWorldStateArchive( any(WorldStateStorageCoordinator.class), any(Blockchain.class), - any(CachedMerkleTrieLoader.class)); + any(BonsaiCachedMerkleTrieLoader.class)); doReturn(mockWorldState).when(worldStateArchive).getMutable(); when(storageProvider.createWorldStateStorageCoordinator(dataStorageConfiguration)) .thenReturn(new WorldStateStorageCoordinator(bonsaiWorldStateStorage)); diff --git a/besu/src/test/java/org/hyperledger/besu/controller/CliqueBesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/CliqueBesuControllerBuilderTest.java index b96a052c6..631149516 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/CliqueBesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/CliqueBesuControllerBuilderTest.java @@ -117,12 +117,13 @@ public class CliqueBesuControllerBuilderTest { lenient().when(genesisConfigFile.getConfigOptions()).thenReturn(genesisConfigOptions); lenient().when(genesisConfigOptions.getCheckpointOptions()).thenReturn(checkpointConfigOptions); lenient() - .when(storageProvider.createBlockchainStorage(any(), any())) + .when(storageProvider.createBlockchainStorage(any(), any(), any())) .thenReturn( new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions())); + new MainnetBlockHeaderFunctions(), + false)); lenient() .when( storageProvider.createWorldStateStorageCoordinator( diff --git a/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java index f10b50401..e9c22a82b 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java @@ -133,12 +133,13 @@ public class MergeBesuControllerBuilderTest { when(genesisConfigOptions.getTerminalBlockHash()).thenReturn(Optional.of(Hash.ZERO)); lenient().when(genesisConfigOptions.getTerminalBlockNumber()).thenReturn(OptionalLong.of(1L)); lenient() - .when(storageProvider.createBlockchainStorage(any(), any())) + .when(storageProvider.createBlockchainStorage(any(), any(), any())) .thenReturn( new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions())); + new MainnetBlockHeaderFunctions(), + false)); lenient() .when(storageProvider.getStorageBySegmentIdentifier(any())) .thenReturn(new InMemoryKeyValueStorage()); diff --git a/besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java b/besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java index 2aa5b7c01..321ef8a3b 100644 --- a/besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java +++ b/besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java @@ -122,7 +122,8 @@ public class BesuEventsImplTest { new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions()), + new MainnetBlockHeaderFunctions(), + false), new NoOpMetricsSystem(), 0); diff --git a/besu/src/test/resources/everything_config.toml b/besu/src/test/resources/everything_config.toml index 2bcbd21b4..87fc5986d 100644 --- a/besu/src/test/resources/everything_config.toml +++ b/besu/src/test/resources/everything_config.toml @@ -215,6 +215,7 @@ ethstats-cacert-file="./root.cert" # Data storage data-storage-format="BONSAI" bonsai-historical-block-limit=512 +receipt-compaction-enabled=true # feature flags Xsecp256k1-native-enabled=false diff --git a/build.gradle b/build.gradle index f63c3f1e7..cc26f1656 100644 --- a/build.gradle +++ b/build.gradle @@ -29,7 +29,7 @@ plugins { id 'com.jfrog.artifactory' version '5.1.11' id 'io.spring.dependency-management' version '1.1.4' id 'me.champeau.jmh' version '0.7.2' apply false - id 'net.ltgt.errorprone' version '3.0.1' + id 'net.ltgt.errorprone' version '3.1.0' id 'maven-publish' } @@ -75,6 +75,9 @@ def _strListCmdArg(name) { return _strListCmdArg(name, null) } +// set the shell command to use according to os +def shell = org.gradle.internal.os.OperatingSystem.current().isWindows() ? "${projectDir}\\wslsh.bat" : '/bin/bash' + licenseReport { // This is for the allowed-licenses-file in checkLicense Task // Accepts File, URL or String path to local or remote file @@ -115,12 +118,12 @@ allprojects { } task sourcesJar(type: Jar, dependsOn: classes) { - classifier = 'sources' + archiveClassifier = 'sources' from sourceSets.main.allSource } task javadocJar(type: Jar, dependsOn: javadoc) { - classifier = 'javadoc' + archiveClassifier = 'javadoc' from javadoc.outputDirectory } @@ -143,6 +146,7 @@ allprojects { url 'https://splunk.jfrog.io/splunk/ext-releases-local' content { includeGroupByRegex('com\\.splunk\\..*') } } + mavenCentral() // ethereum execution spec tests fixtures. Exclusively for ethereum submodule to run ref tests @@ -164,6 +168,8 @@ allprojects { dependencies { components.all(BouncyCastleCapability) errorprone 'com.google.errorprone:error_prone_core' + // https://github.com/hyperledger/besu-errorprone-checks/ + errorprone "org.hyperledger.besu:besu-errorprone-checks" } configurations.all { @@ -199,7 +205,7 @@ allprojects { format 'sol', { target '**/*.sol' } } - tasks.withType(JavaCompile) { + tasks.withType(JavaCompile).configureEach { options.compilerArgs += [ '-Xlint:unchecked', '-Xlint:cast', @@ -212,8 +218,8 @@ allprojects { ] options.errorprone { - excludedPaths = '.*/(generated/*.*|.*ReferenceTest_.*|build/.*/annotation-output/.*)' - + excludedPaths = '.*/generated/*.*' + disableWarningsInGeneratedCode = true // Our equals need to be symmetric, this checker doesn't respect that. check('EqualsGetClass', CheckSeverity.OFF) // We like to use futures with no return values. @@ -275,7 +281,7 @@ allprojects { * */ test { - jvmArgs = [ + jvmArgs += [ '-Xmx4g', '-XX:-UseGCOverheadLimit', // Mockito and jackson-databind do some strange reflection during tests. @@ -380,7 +386,7 @@ subprojects { task testSupportJar(type: Jar) { archiveBaseName = "${project.name}-support-test" - classifier = 'test-support' + archiveClassifier = 'test-support' from sourceSets.testSupport.output } } @@ -739,7 +745,7 @@ task distDocker { println "Building for platform ${project.getProperty('docker-platform')}" } def gitDetails = getGitCommitDetails(10) - executable "sh" + executable shell workingDir dockerBuildDir args "-c", "docker build ${dockerPlatform} --build-arg BUILD_DATE=${buildTime()} --build-arg VERSION=${dockerBuildVersion} --build-arg VCS_REF=${gitDetails.hash} -t ${image} ." } @@ -747,7 +753,7 @@ task distDocker { // tag the "default" (which is the variant in the zero position) exec { - executable "sh" + executable shell args "-c", "docker tag '${dockerImageName}:${dockerBuildVersion}-${dockerVariants[0]}' '${dockerImageName}:${dockerBuildVersion}'" } } @@ -766,8 +772,8 @@ task testDocker { exec { def image = project.hasProperty('release.releaseVersion') ? "${dockerImageName}:" + project.property('release.releaseVersion') : "${dockerImageName}:${project.version}" workingDir "${projectDir}/docker/${variant}" - executable "sh" - args "-c", "bash ../test.sh ${image}-${variant}" + executable shell + args "-c", "../test.sh ${image}-${variant}" } } } @@ -795,7 +801,7 @@ task dockerUpload { def cmd = "docker tag '${variantImage}' '${archVariantImage}' && docker push '${archVariantImage}'" additionalTags.each { tag -> cmd += " && docker tag '${variantImage}' '${dockerImageName}:${tag.trim()}-${variant}-${architecture}' && docker push '${dockerImageName}:${tag.trim()}-${variant}-${architecture}'" } println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } } @@ -805,7 +811,7 @@ task dockerUpload { def cmd = "docker tag ${image} ${archImage} && docker push '${archImage}'" additionalTags.each { tag -> cmd += " && docker tag '${image}' '${dockerImageName}:${tag.trim()}-${architecture}' && docker push '${dockerImageName}:${tag.trim()}-${architecture}'" } println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } } @@ -822,13 +828,13 @@ task dockerUploadRelease { exec { def cmd = "docker pull '${variantImage}-${architecture}' && docker tag '${variantImage}-${architecture}' '${dockerImageName}:latest-${variant}-${architecture}'" println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } exec { def cmd = "docker push '${dockerImageName}:latest-${variant}-${architecture}'" println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } } @@ -837,13 +843,13 @@ task dockerUploadRelease { def archImage = "${image}-${architecture}" def cmd = "docker pull '${archImage}' && docker tag ${archImage} '${dockerImageName}:latest-${architecture}'" println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } exec { def cmd = "docker push '${dockerImageName}:latest-${architecture}'" println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } } @@ -873,13 +879,13 @@ task manifestDocker { exec { def cmd = "docker manifest create '${variantImage}' ${targets}" println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } exec { def cmd = "docker manifest push '${variantImage}'" println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } } @@ -889,13 +895,13 @@ task manifestDocker { archs.forEach { arch -> targets += "'${baseTag}-${arch}' " } def cmd = "docker manifest create '${baseTag}' ${targets}" println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } exec { def cmd = "docker manifest push '${baseTag}'" println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } } @@ -915,13 +921,13 @@ task manifestDockerRelease { exec { def cmd = "docker manifest create '${variantImage}' ${targets} --amend" println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } exec { def cmd = "docker manifest push '${variantImage}'" println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } } @@ -931,13 +937,13 @@ task manifestDockerRelease { archs.forEach { arch -> targets += "'${baseTag}-${arch}' " } def cmd = "docker manifest create '${baseTag}' ${targets} --amend" println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } exec { def cmd = "docker manifest push '${baseTag}'" println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } } @@ -968,7 +974,7 @@ task checkSpdxHeader(type: CheckSpdxHeader) { jacocoTestReport { reports { - xml.enabled true + xml.required = true } } @@ -979,25 +985,12 @@ task jacocoRootReport(type: org.gradle.testing.jacoco.tasks.JacocoReport) { executionData.from fileTree(dir: '.', includes: ['**/jacoco/*.exec']) reports { xml.required = true - xml.enabled = true csv.required = true html.destination file("build/reports/jacocoHtml") } onlyIf = { true } } -configurations { annotationProcessor } - -// Prevent errorprone-checks being dependent upon errorprone-checks! -// However, ensure all subprojects comply with the custom rules. -configure(subprojects.findAll { it.name != 'errorprone-checks' }) { - dependencies { annotationProcessor project(":errorprone-checks") } - - tasks.withType(JavaCompile) { - options.annotationProcessorPath = configurations.annotationProcessor - } -} - // http://label-schema.org/rc1/ // using the RFC3339 format "2016-04-12T23:20:50.52Z" def buildTime() { @@ -1078,9 +1071,11 @@ tasks.register("verifyDistributions") { } dependencies { + errorprone 'com.google.errorprone:error_prone_core' + // https://github.com/hyperledger/besu-errorprone-checks/ + errorprone 'org.hyperledger.besu:besu-errorprone-checks' implementation project(':besu') implementation project(':ethereum:evmtool') - errorprone 'com.google.errorprone:error_prone_core' } @CompileStatic diff --git a/config/src/main/java/org/hyperledger/besu/config/BftFork.java b/config/src/main/java/org/hyperledger/besu/config/BftFork.java index 7e9c408ba..de188744b 100644 --- a/config/src/main/java/org/hyperledger/besu/config/BftFork.java +++ b/config/src/main/java/org/hyperledger/besu/config/BftFork.java @@ -18,6 +18,7 @@ import org.hyperledger.besu.datatypes.Address; import java.math.BigInteger; import java.util.List; +import java.util.Locale; import java.util.Optional; import java.util.OptionalInt; @@ -89,7 +90,7 @@ public class BftFork implements Fork { return Optional.empty(); } final String weiStr = configFileContent.get(); - if (weiStr.toLowerCase().startsWith("0x")) { + if (weiStr.toLowerCase(Locale.ROOT).startsWith("0x")) { return Optional.of(new BigInteger(1, Bytes.fromHexStringLenient(weiStr).toArrayUnsafe())); } return Optional.of(new BigInteger(weiStr)); diff --git a/config/src/main/java/org/hyperledger/besu/config/JsonBftConfigOptions.java b/config/src/main/java/org/hyperledger/besu/config/JsonBftConfigOptions.java index 4dfa49fa3..a3791a6e5 100644 --- a/config/src/main/java/org/hyperledger/besu/config/JsonBftConfigOptions.java +++ b/config/src/main/java/org/hyperledger/besu/config/JsonBftConfigOptions.java @@ -17,6 +17,7 @@ package org.hyperledger.besu.config; import org.hyperledger.besu.datatypes.Address; import java.math.BigInteger; +import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -117,7 +118,7 @@ public class JsonBftConfigOptions implements BftConfigOptions { return BigInteger.ZERO; } final String weiStr = configFileContent.get(); - if (weiStr.toLowerCase().startsWith("0x")) { + if (weiStr.toLowerCase(Locale.ROOT).startsWith("0x")) { return new BigInteger(1, Bytes.fromHexStringLenient(weiStr).toArrayUnsafe()); } return new BigInteger(weiStr); diff --git a/config/src/main/java/org/hyperledger/besu/config/JsonUtil.java b/config/src/main/java/org/hyperledger/besu/config/JsonUtil.java index 45c933d09..ef19bc7b8 100644 --- a/config/src/main/java/org/hyperledger/besu/config/JsonUtil.java +++ b/config/src/main/java/org/hyperledger/besu/config/JsonUtil.java @@ -446,7 +446,8 @@ public class JsonUtil { final String errorMessage = String.format( "Expected %s value but got %s", - expectedType.toString().toLowerCase(), node.getNodeType().toString().toLowerCase()); + expectedType.toString().toLowerCase(Locale.ROOT), + node.getNodeType().toString().toLowerCase(Locale.ROOT)); throw new IllegalArgumentException(errorMessage); } return true; diff --git a/config/src/main/resources/holesky.json b/config/src/main/resources/holesky.json index 63f2b53a0..af552a372 100644 --- a/config/src/main/resources/holesky.json +++ b/config/src/main/resources/holesky.json @@ -17,6 +17,7 @@ "cancunTime": 1707305664, "ethash": {}, "discovery": { + "dns": "AKA3AM6LPBYEUDMVNU3BSVQJ5AD45Y7YPOHJLEF6W26QOE4VTUDPE@all.holesky.ethdisco.net", "bootnodes": [ "enode://ac906289e4b7f12df423d654c5a962b6ebe5b3a74cc9e06292a85221f9a64a6f1cfdd6b714ed6dacef51578f92b34c60ee91e9ede9c7f8fadc4d347326d95e2b@146.190.13.128:30303", "enode://a3435a0155a3e837c02f5e7f5662a2f1fbc25b48e4dc232016e1c51b544cb5b4510ef633ea3278c0e970fa8ad8141e2d4d0f9f95456c537ff05fdf9b31c15072@178.128.136.233:30303" diff --git a/consensus/common/src/test-support/java/org/hyperledger/besu/consensus/common/bft/inttest/NetworkLayout.java b/consensus/common/src/test-support/java/org/hyperledger/besu/consensus/common/bft/inttest/NetworkLayout.java index 4fbeefcc8..bb84888c2 100644 --- a/consensus/common/src/test-support/java/org/hyperledger/besu/consensus/common/bft/inttest/NetworkLayout.java +++ b/consensus/common/src/test-support/java/org/hyperledger/besu/consensus/common/bft/inttest/NetworkLayout.java @@ -21,6 +21,7 @@ import org.hyperledger.besu.ethereum.core.Util; import java.util.ArrayList; import java.util.List; +import java.util.NavigableMap; import java.util.Set; import java.util.TreeMap; @@ -29,11 +30,11 @@ import com.google.common.collect.Iterables; public class NetworkLayout { private final NodeParams localNode; - private final TreeMap addressKeyMap; + private final NavigableMap addressKeyMap; private final List remotePeers; public NetworkLayout( - final NodeParams localNode, final TreeMap addressKeyMap) { + final NodeParams localNode, final NavigableMap addressKeyMap) { this.localNode = localNode; this.addressKeyMap = addressKeyMap; this.remotePeers = new ArrayList<>(addressKeyMap.values()); @@ -42,14 +43,14 @@ public class NetworkLayout { public static NetworkLayout createNetworkLayout( final int validatorCount, final int firstLocalNodeBlockNum) { - final TreeMap addressKeyMap = createValidators(validatorCount); + final NavigableMap addressKeyMap = createValidators(validatorCount); final NodeParams localNode = Iterables.get(addressKeyMap.values(), firstLocalNodeBlockNum); return new NetworkLayout(localNode, addressKeyMap); } - private static TreeMap createValidators(final int validatorCount) { + private static NavigableMap createValidators(final int validatorCount) { // Map is required to be sorted by address final TreeMap addressKeyMap = new TreeMap<>(); diff --git a/crypto/algorithms/src/main/java/org/hyperledger/besu/crypto/SignatureAlgorithmFactory.java b/crypto/algorithms/src/main/java/org/hyperledger/besu/crypto/SignatureAlgorithmFactory.java index 0aaf6d70f..823561b52 100644 --- a/crypto/algorithms/src/main/java/org/hyperledger/besu/crypto/SignatureAlgorithmFactory.java +++ b/crypto/algorithms/src/main/java/org/hyperledger/besu/crypto/SignatureAlgorithmFactory.java @@ -49,10 +49,8 @@ public class SignatureAlgorithmFactory { if (!SignatureAlgorithmType.isDefault(instance)) { LOG.info( - new StringBuilder("The signature algorithm uses the elliptic curve ") - .append(instance.getCurveName()) - .append(". The usage of alternative elliptic curves is still experimental.") - .toString()); + "The signature algorithm uses the elliptic curve {}. The usage of alternative elliptic curves is still experimental.", + instance.getCurveName()); } } diff --git a/crypto/algorithms/src/main/java/org/hyperledger/besu/crypto/SignatureAlgorithmType.java b/crypto/algorithms/src/main/java/org/hyperledger/besu/crypto/SignatureAlgorithmType.java index 26e877ab9..8f2521038 100644 --- a/crypto/algorithms/src/main/java/org/hyperledger/besu/crypto/SignatureAlgorithmType.java +++ b/crypto/algorithms/src/main/java/org/hyperledger/besu/crypto/SignatureAlgorithmType.java @@ -94,11 +94,9 @@ public class SignatureAlgorithmType { } private static String invalidTypeErrorMessage(final String invalidEcCurve) { - return new StringBuilder() - .append(invalidEcCurve) - .append(" is not in the list of valid elliptic curves ") - .append(getEcCurvesListAsString()) - .toString(); + return invalidEcCurve + + " is not in the list of valid elliptic curves " + + getEcCurvesListAsString(); } private static String getEcCurvesListAsString() { diff --git a/crypto/algorithms/src/test/java/org/hyperledger/besu/crypto/SECP256R1Test.java b/crypto/algorithms/src/test/java/org/hyperledger/besu/crypto/SECP256R1Test.java index 26e03e5bd..7112d3629 100644 --- a/crypto/algorithms/src/test/java/org/hyperledger/besu/crypto/SECP256R1Test.java +++ b/crypto/algorithms/src/test/java/org/hyperledger/besu/crypto/SECP256R1Test.java @@ -160,7 +160,7 @@ public class SECP256R1Test { final BigInteger recoveredPubKeyBigInt = secp256R1.recoverFromSignature( signature.getRecId(), signature.getR(), signature.getS(), dataHash); - assertThat(recoveredPubKeyBigInt).isEqualTo(recoveredPubKeyBigInt); + assertThat(recoveredPubKeyBigInt).isEqualTo(publicKeyBigInt); }); } diff --git a/datatypes/src/main/java/org/hyperledger/besu/datatypes/Wei.java b/datatypes/src/main/java/org/hyperledger/besu/datatypes/Wei.java index 2e2a8ac4f..ee8d5ef6f 100644 --- a/datatypes/src/main/java/org/hyperledger/besu/datatypes/Wei.java +++ b/datatypes/src/main/java/org/hyperledger/besu/datatypes/Wei.java @@ -16,6 +16,7 @@ package org.hyperledger.besu.datatypes; import java.math.BigInteger; import java.util.Arrays; +import java.util.Locale; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.units.bigints.BaseUInt256Value; @@ -224,7 +225,7 @@ public final class Wei extends BaseUInt256Value implements Quantity { @Override public String toString() { - return name().toLowerCase(); + return name().toLowerCase(Locale.ROOT); } } } diff --git a/enclave/src/integration-test/java/org/hyperledger/besu/enclave/TlsHelpers.java b/enclave/src/integration-test/java/org/hyperledger/besu/enclave/TlsHelpers.java index 7eec56153..89ecddba2 100644 --- a/enclave/src/integration-test/java/org/hyperledger/besu/enclave/TlsHelpers.java +++ b/enclave/src/integration-test/java/org/hyperledger/besu/enclave/TlsHelpers.java @@ -32,6 +32,7 @@ import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.util.Enumeration; import java.util.List; +import java.util.Locale; import java.util.Optional; import java.util.StringJoiner; @@ -93,6 +94,6 @@ public class TlsHelpers { joiner.add(String.format("%02X", b)); } - return joiner.toString().toLowerCase(); + return joiner.toString().toLowerCase(Locale.ROOT); } } diff --git a/errorprone-checks/README.md b/errorprone-checks/README.md deleted file mode 100644 index 34f336cbc..000000000 --- a/errorprone-checks/README.md +++ /dev/null @@ -1,9 +0,0 @@ -The creation of custom errorprone checkers was largely derived from: -* https://github.com/tbroyer/gradle-errorprone-plugin -* https://errorprone.info/docs/installation -* https://github.com/google/error-prone/wiki/Writing-a-check - -To allow for debugging from within intellij, the following must be added to the VM args -in the run/debug configuration (this assumes your gradle cache is at the default location under -your home): --Xbootclasspath/p:${HOME}/.gradle/caches/./modules-2/files-2.1/com.google.errorprone/javac/9+181-r4173-1/bdf4c0aa7d540ee1f7bf14d47447aea4bbf450c5/javac-9+181-r4173-1.jar diff --git a/errorprone-checks/build.gradle b/errorprone-checks/build.gradle deleted file mode 100644 index fdb773b75..000000000 --- a/errorprone-checks/build.gradle +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -// we use this config to get the path of the JDK 9 javac jar, to -// stick it in the bootclasspath when running tests -configurations.maybeCreate("epJavac") - - -apply plugin: 'java' -apply plugin: 'net.ltgt.errorprone' - -sourceCompatibility = 17 -targetCompatibility = 17 - -dependencies { - api 'org.slf4j:slf4j-api' - - annotationProcessor 'com.google.auto.service:auto-service' - - implementation 'com.google.auto.service:auto-service' - implementation 'com.google.errorprone:error_prone_annotation' - implementation 'com.google.errorprone:error_prone_core' - implementation 'info.picocli:picocli' - - testImplementation 'com.google.errorprone:error_prone_test_helpers' - testImplementation 'org.assertj:assertj-core' - testImplementation 'org.junit.jupiter:junit-jupiter' - // imported to get org.jetbrains.annotations.NotNull - testImplementation 'org.jetbrains.kotlin:kotlin-stdlib' - - epJavac 'com.google.errorprone:error_prone_check_api' -} - -test { testLogging { showStandardStreams = true } } - - -tasks.withType(JavaCompile) { - options.compilerArgs += [ - '--add-exports', - 'jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED', - '--add-exports', - 'jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED', - '--add-exports', - 'jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED', - '--add-exports', - 'jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED', - '--add-exports', - 'jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED', - '--add-exports', - 'jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED', - '--add-exports', - 'jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED', - '--add-exports', - 'jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED' - ] -} - -javadoc { enabled = false } diff --git a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/BannedMethod.java b/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/BannedMethod.java deleted file mode 100644 index 8f71df921..000000000 --- a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/BannedMethod.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import static com.google.errorprone.BugPattern.SeverityLevel.WARNING; -import static com.google.errorprone.bugpatterns.BugChecker.MethodInvocationTreeMatcher; -import static com.google.errorprone.matchers.Description.NO_MATCH; -import static com.google.errorprone.matchers.method.MethodMatchers.staticMethod; - -import java.util.Map; - -import com.google.auto.service.AutoService; -import com.google.common.collect.ImmutableMap; -import com.google.errorprone.BugPattern; -import com.google.errorprone.VisitorState; -import com.google.errorprone.bugpatterns.BugChecker; -import com.google.errorprone.matchers.Description; -import com.google.errorprone.matchers.Matcher; -import com.sun.source.tree.ExpressionTree; -import com.sun.source.tree.MethodInvocationTree; - -@AutoService(BugChecker.class) -@BugPattern( - summary = "Some methods should not be used, make sure that doesn't happen.", - severity = WARNING, - linkType = BugPattern.LinkType.NONE) -public class BannedMethod extends BugChecker implements MethodInvocationTreeMatcher { - - private static final ImmutableMap, String> BANNED_METHOD_LIST = - ImmutableMap.of( - staticMethod().onClass("com.google.common.base.Objects").withAnyName(), - "Do not use com.google.common.base.Objects methods, use java.util.Objects methods instead.", - staticMethod().onClass("org.junit.Assert"), - "Do not use junit assertions. Use assertj assertions instead.", - staticMethod().onClass("org.apache.logging.log4j.LogManager"), - "Do not use org.apache.logging.log4j.LogManager, use org.slf4j.LoggerFactory methods instead."); - - @Override - public Description matchMethodInvocation( - final MethodInvocationTree tree, final VisitorState state) { - for (final Map.Entry, String> entry : BANNED_METHOD_LIST.entrySet()) { - if (entry.getKey().matches(tree, state)) { - return buildDescription(tree).setMessage(entry.getValue()).build(); - } - } - return NO_MATCH; - } -} diff --git a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectly.java b/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectly.java deleted file mode 100644 index 95c7d1803..000000000 --- a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectly.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import static com.google.errorprone.BugPattern.SeverityLevel.WARNING; - -import com.google.auto.service.AutoService; -import com.google.errorprone.BugPattern; -import com.google.errorprone.VisitorState; -import com.google.errorprone.bugpatterns.BugChecker; -import com.google.errorprone.bugpatterns.BugChecker.MethodInvocationTreeMatcher; -import com.google.errorprone.bugpatterns.BugChecker.NewClassTreeMatcher; -import com.google.errorprone.matchers.Description; -import com.google.errorprone.util.ASTHelpers; -import com.sun.source.tree.MethodInvocationTree; -import com.sun.source.tree.NewClassTree; -import com.sun.tools.javac.code.Symbol; - -@AutoService(BugChecker.class) -@BugPattern( - summary = "Do not create SecureRandom directly.", - severity = WARNING, - linkType = BugPattern.LinkType.NONE) -public class DoNotCreateSecureRandomDirectly extends BugChecker - implements MethodInvocationTreeMatcher, NewClassTreeMatcher { - - @SuppressWarnings("TreeToString") - @Override - public Description matchMethodInvocation( - final MethodInvocationTree tree, final VisitorState state) { - if (tree.getMethodSelect().toString().equals("SecureRandom.getInstance")) { - return describeMatch(tree); - } - - return Description.NO_MATCH; - } - - @Override - public Description matchNewClass(final NewClassTree tree, final VisitorState state) { - final Symbol sym = ASTHelpers.getSymbol(tree.getIdentifier()); - if (sym != null && sym.toString().equals("java.security.SecureRandom")) { - return describeMatch(tree); - } - - return Description.NO_MATCH; - } -} diff --git a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectly.java b/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectly.java deleted file mode 100644 index 7ce0c81f0..000000000 --- a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectly.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import static com.google.errorprone.BugPattern.SeverityLevel.WARNING; - -import com.google.auto.service.AutoService; -import com.google.errorprone.BugPattern; -import com.google.errorprone.VisitorState; -import com.google.errorprone.bugpatterns.BugChecker; -import com.google.errorprone.bugpatterns.BugChecker.MethodInvocationTreeMatcher; -import com.google.errorprone.matchers.Description; -import com.sun.source.tree.MethodInvocationTree; - -@AutoService(BugChecker.class) -@BugPattern( - summary = "Do not invoke MessageDigest.getInstance directly.", - severity = WARNING, - linkType = BugPattern.LinkType.NONE) -public class DoNotInvokeMessageDigestDirectly extends BugChecker - implements MethodInvocationTreeMatcher { - - @SuppressWarnings("TreeToString") - @Override - public Description matchMethodInvocation( - final MethodInvocationTree tree, final VisitorState state) { - if (tree.getMethodSelect().toString().equals("MessageDigest.getInstance")) { - return describeMatch(tree); - } - return Description.NO_MATCH; - } -} diff --git a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/DoNotReturnNullOptionals.java b/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/DoNotReturnNullOptionals.java deleted file mode 100644 index 7310058e5..000000000 --- a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/DoNotReturnNullOptionals.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import static com.google.errorprone.BugPattern.SeverityLevel.SUGGESTION; -import static com.google.errorprone.matchers.Matchers.contains; -import static com.sun.source.tree.Tree.Kind.NULL_LITERAL; - -import com.google.auto.service.AutoService; -import com.google.errorprone.BugPattern; -import com.google.errorprone.VisitorState; -import com.google.errorprone.bugpatterns.BugChecker; -import com.google.errorprone.bugpatterns.BugChecker.MethodTreeMatcher; -import com.google.errorprone.matchers.Description; -import com.google.errorprone.matchers.Matcher; -import com.sun.source.tree.MethodTree; -import com.sun.source.tree.ReturnTree; -import com.sun.source.tree.Tree; - -/* - * This is reworked from an example found at: - * https://github.com/google/error-prone/wiki/Writing-a-check - */ - -@AutoService(BugChecker.class) // the service descriptor -@BugPattern( - summary = "Do not return null optionals.", - severity = SUGGESTION, - linkType = BugPattern.LinkType.NONE) -public class DoNotReturnNullOptionals extends BugChecker implements MethodTreeMatcher { - - private static class ReturnNullMatcher implements Matcher { - - @Override - public boolean matches(final Tree tree, final VisitorState state) { - if ((tree instanceof ReturnTree) && (((ReturnTree) tree).getExpression() != null)) { - return ((ReturnTree) tree).getExpression().getKind() == NULL_LITERAL; - } - return false; - } - } - - private static final Matcher RETURN_NULL = new ReturnNullMatcher(); - private static final Matcher CONTAINS_RETURN_NULL = contains(RETURN_NULL); - - @SuppressWarnings("TreeToString") - @Override - public Description matchMethod(final MethodTree tree, final VisitorState state) { - if ((tree.getReturnType() == null) - || !tree.getReturnType().toString().startsWith("Optional<") - || (tree.getBody() == null) - || (!CONTAINS_RETURN_NULL.matches(tree.getBody(), state))) { - return Description.NO_MATCH; - } - return describeMatch(tree); - } -} diff --git a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayed.java b/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayed.java deleted file mode 100644 index 3cda88479..000000000 --- a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayed.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import static com.google.errorprone.BugPattern.SeverityLevel.WARNING; - -import java.util.Map; -import java.util.Optional; -import javax.lang.model.element.AnnotationMirror; -import javax.lang.model.element.AnnotationValue; -import javax.lang.model.element.ExecutableElement; - -import com.google.auto.service.AutoService; -import com.google.errorprone.BugPattern; -import com.google.errorprone.VisitorState; -import com.google.errorprone.bugpatterns.BugChecker; -import com.google.errorprone.bugpatterns.BugChecker.AnnotationTreeMatcher; -import com.google.errorprone.matchers.Description; -import com.google.errorprone.util.ASTHelpers; -import com.sun.source.tree.AnnotationTree; -import com.sun.tools.javac.tree.JCTree; - -@AutoService(BugChecker.class) -@BugPattern( - summary = "Experimental options must be hidden and not present in the BesuCommand class.", - severity = WARNING, - linkType = BugPattern.LinkType.NONE) -public class ExperimentalCliOptionMustBeCorrectlyDisplayed extends BugChecker - implements AnnotationTreeMatcher { - - @Override - public Description matchAnnotation(AnnotationTree tree, VisitorState state) { - final AnnotationMirror annotationMirror = ASTHelpers.getAnnotationMirror(tree); - if (annotationMirror.getAnnotationType().toString().equals("picocli.CommandLine.Option")) { - final Optional names = - getAnnotationValue(annotationMirror, "names"); - if (names.isPresent() && names.get().getValue().toString().contains("--X")) { - final JCTree.JCCompilationUnit compilation = - (JCTree.JCCompilationUnit) state.getPath().getCompilationUnit(); - if (compilation.getSourceFile().getName().endsWith("BesuCommand.java")) { - return describeMatch(tree); - } - final Optional isHidden = - getAnnotationValue(annotationMirror, "hidden"); - if (isHidden.isEmpty() || !((boolean) isHidden.get().getValue())) { - return describeMatch(tree); - } - } - } - return Description.NO_MATCH; - } - - private Optional getAnnotationValue( - final AnnotationMirror annotationMirror, final String name) { - final Map elementValues = - annotationMirror.getElementValues(); - final Optional retValue = - elementValues.keySet().stream() - .filter(k -> k.getSimpleName().toString().equals(name)) - .map(elementValues::get) - .findAny(); - return retValue; - } -} diff --git a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinal.java b/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinal.java deleted file mode 100644 index abd6f3b58..000000000 --- a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinal.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import static com.google.errorprone.BugPattern.SeverityLevel.WARNING; - -import javax.lang.model.element.Modifier; - -import com.google.auto.service.AutoService; -import com.google.errorprone.BugPattern; -import com.google.errorprone.VisitorState; -import com.google.errorprone.bugpatterns.BugChecker; -import com.google.errorprone.bugpatterns.BugChecker.ClassTreeMatcher; -import com.google.errorprone.bugpatterns.BugChecker.MethodTreeMatcher; -import com.google.errorprone.matchers.Description; -import com.google.errorprone.util.ASTHelpers; -import com.sun.source.tree.ClassTree; -import com.sun.source.tree.MethodTree; -import com.sun.source.tree.ModifiersTree; -import com.sun.source.tree.VariableTree; - -@AutoService(BugChecker.class) -@BugPattern( - summary = "Method input parameters must be final.", - severity = WARNING, - linkType = BugPattern.LinkType.NONE) -public class MethodInputParametersMustBeFinal extends BugChecker - implements MethodTreeMatcher, ClassTreeMatcher { - - private boolean isAbstraction = false; - private boolean isGenerated = false; - - @Override - public Description matchClass(final ClassTree tree, final VisitorState state) { - isAbstraction = - isInterface(tree.getModifiers()) - || isAnonymousClassInAbstraction(tree) - || isEnumInAbstraction(tree); - isGenerated = ASTHelpers.hasDirectAnnotationWithSimpleName(tree, "Generated"); - return Description.NO_MATCH; - } - - @Override - public Description matchMethod(final MethodTree tree, final VisitorState state) { - if (isGenerated) { - return Description.NO_MATCH; - } - - final ModifiersTree mods = tree.getModifiers(); - - if (isAbstraction) { - if (isConcreteMethod(mods)) { - return matchParameters(tree); - } - } else if (isNotAbstract(mods)) { - return matchParameters(tree); - } - - return Description.NO_MATCH; - } - - private Description matchParameters(final MethodTree tree) { - for (final VariableTree inputParameter : tree.getParameters()) { - if (isMissingFinalModifier(inputParameter)) { - return describeMatch(tree); - } - } - - return Description.NO_MATCH; - } - - private boolean isMissingFinalModifier(final VariableTree inputParameter) { - return !inputParameter.getModifiers().getFlags().contains(Modifier.FINAL); - } - - private boolean isNotAbstract(final ModifiersTree mods) { - return !mods.getFlags().contains(Modifier.ABSTRACT); - } - - @SuppressWarnings("TreeToString") - private boolean isInterface(final ModifiersTree mods) { - return mods.toString().contains("interface"); - } - - private boolean isConcreteMethod(final ModifiersTree mods) { - return mods.getFlags().contains(Modifier.DEFAULT) || mods.getFlags().contains(Modifier.STATIC); - } - - private boolean isAnonymousClassInAbstraction(final ClassTree tree) { - return isAbstraction && isAnonymousClass(tree); - } - - private boolean isAnonymousClass(final ClassTree tree) { - return tree.getSimpleName().contentEquals(""); - } - - private boolean isEnumInAbstraction(final ClassTree tree) { - return isAbstraction && isEnum(tree); - } - - @SuppressWarnings("TreeToString") - private boolean isEnum(final ClassTree tree) { - return tree.toString().contains("enum"); - } - - @Override - public boolean equals(Object o) { - // isAbstract and isGenerated are transient calculations, not relevant to equality checks - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - return super.equals(o); - } - - @Override - public int hashCode() { - // isAbstract and isGenerated are transient calculations, not relevant to equality checks - return super.hashCode(); - } -} diff --git a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/PreferCommonAnnotations.java b/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/PreferCommonAnnotations.java deleted file mode 100644 index acc46c0f7..000000000 --- a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/PreferCommonAnnotations.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * (c) Copyright 2023 Palantir Technologies Inc. All rights reserved. - * Copyright Hyperledger Besu contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -/* Derived from https://github.com/palantir/gradle-baseline/blob/6fe385a80291473e7fc1441f176454bec4184d6b/baseline-error-prone/src/main/java/com/palantir/baseline/errorprone/PreferCommonAnnotations.java */ - -package org.hyperledger.errorpronechecks; - -import java.util.Map; -import java.util.Objects; - -import com.google.auto.service.AutoService; -import com.google.errorprone.BugPattern; -import com.google.errorprone.BugPattern.SeverityLevel; -import com.google.errorprone.VisitorState; -import com.google.errorprone.bugpatterns.BugChecker; -import com.google.errorprone.bugpatterns.BugChecker.ImportTreeMatcher; -import com.google.errorprone.fixes.SuggestedFix; -import com.google.errorprone.matchers.Description; -import com.google.errorprone.util.ASTHelpers; -import com.sun.source.tree.ImportTree; -import com.sun.tools.javac.code.Type; - -/** - * Checker that recommends using the common version of an annotation. - * - *

Examples: - Guava's version of {@code @VisibleForTesting} over other copies. - */ -@AutoService(BugChecker.class) -@BugPattern( - summary = "Prefer the common version of annotations over other copies.", - severity = SeverityLevel.WARNING) -public final class PreferCommonAnnotations extends BugChecker implements ImportTreeMatcher { - - /** ClassName -> preferred import. */ - private static final Map PREFERRED_IMPORTS = - Map.of("org.jetbrains.annotations.NotNull", "javax.annotation.Nonnull"); - - @Override - public Description matchImport(ImportTree tree, VisitorState state) { - Type importType = ASTHelpers.getType(tree.getQualifiedIdentifier()); - if (importType == null) { - return Description.NO_MATCH; - } - String importName = importType.toString(); - for (Map.Entry entry : PREFERRED_IMPORTS.entrySet()) { - String affectedClassName = entry.getKey(); - String preferredType = entry.getValue(); - if (importName.endsWith(affectedClassName) && !Objects.equals(importName, preferredType)) { - SuggestedFix fix = - SuggestedFix.builder().removeImport(importName).addImport(preferredType).build(); - return this.buildDescription(tree) - .setMessage("Do not use " + importName + " use " + preferredType + " instead.") - .addFix(fix) - .build(); - } - } - return Description.NO_MATCH; - } -} diff --git a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggers.java b/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggers.java deleted file mode 100644 index 445f3226d..000000000 --- a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggers.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import static com.google.errorprone.BugPattern.SeverityLevel.WARNING; -import static com.google.errorprone.fixes.SuggestedFixes.addModifiers; -import static com.google.errorprone.matchers.Description.NO_MATCH; -import static com.google.errorprone.util.ASTHelpers.getType; -import static com.google.errorprone.util.ASTHelpers.isSubtype; - -import java.util.List; -import java.util.Optional; -import javax.lang.model.element.ElementKind; -import javax.lang.model.element.Modifier; - -import com.google.auto.service.AutoService; -import com.google.errorprone.BugPattern; -import com.google.errorprone.VisitorState; -import com.google.errorprone.bugpatterns.BugChecker; -import com.google.errorprone.bugpatterns.BugChecker.VariableTreeMatcher; -import com.google.errorprone.fixes.SuggestedFix; -import com.google.errorprone.matchers.Description; -import com.google.errorprone.suppliers.Supplier; -import com.google.errorprone.suppliers.Suppliers; -import com.google.errorprone.util.ASTHelpers; -import com.sun.source.tree.VariableTree; -import com.sun.tools.javac.code.Symbol; -import com.sun.tools.javac.code.Type; - -@AutoService(BugChecker.class) -@BugPattern( - summary = "Logger classes should be private, static, and final.", - severity = WARNING, - linkType = BugPattern.LinkType.NONE) -public class PrivateStaticFinalLoggers extends BugChecker implements VariableTreeMatcher { - - static final Supplier ORG_SLF4J_LOGGER = Suppliers.typeFromString("org.slf4j.Logger"); - - @Override - public Description matchVariable(final VariableTree tree, final VisitorState state) { - final Symbol.VarSymbol sym = ASTHelpers.getSymbol(tree); - if (sym == null || sym.getKind() != ElementKind.FIELD) { - return NO_MATCH; - } - if (sym.getModifiers() - .containsAll(List.of(Modifier.PRIVATE, Modifier.STATIC, Modifier.FINAL))) { - return NO_MATCH; - } - if (!isSubtype(getType(tree), ORG_SLF4J_LOGGER.get(state), state)) { - return NO_MATCH; - } - Optional fixes = - addModifiers(tree, state, Modifier.PRIVATE, Modifier.STATIC, Modifier.FINAL); - return buildDescription(tree) - .addFix(fixes.isPresent() ? fixes.get() : SuggestedFix.emptyFix()) - .build(); - } -} diff --git a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/BannedMethodTest.java b/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/BannedMethodTest.java deleted file mode 100644 index 48f382946..000000000 --- a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/BannedMethodTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import com.google.errorprone.CompilationTestHelper; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class BannedMethodTest { - - private CompilationTestHelper compilationHelper; - - @BeforeEach - public void setup() { - compilationHelper = CompilationTestHelper.newInstance(BannedMethod.class, getClass()); - } - - @Test - public void bannedMethodsPositiveCases() { - compilationHelper.addSourceFile("BannedMethodPositiveCases.java").doTest(); - } - - @Test - public void bannedMethodsNegativeCases() { - compilationHelper.addSourceFile("BannedMethodNegativeCases.java").doTest(); - } -} diff --git a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectlyTest.java b/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectlyTest.java deleted file mode 100644 index 525f5a868..000000000 --- a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectlyTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import com.google.errorprone.CompilationTestHelper; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class DoNotCreateSecureRandomDirectlyTest { - - private CompilationTestHelper compilationHelper; - - @BeforeEach - public void setup() { - compilationHelper = - CompilationTestHelper.newInstance(DoNotCreateSecureRandomDirectly.class, getClass()); - } - - @Test - public void doNotCreateSecureRandomDirectlyPositiveCases() { - compilationHelper.addSourceFile("DoNotCreateSecureRandomDirectlyPositiveCases.java").doTest(); - } - - @Test - public void doNotCreateSecureRandomDirectlyNegativeCases() { - compilationHelper.addSourceFile("DoNotCreateSecureRandomDirectlyNegativeCases.java").doTest(); - } -} diff --git a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectlyTest.java b/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectlyTest.java deleted file mode 100644 index 986aedb63..000000000 --- a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectlyTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import com.google.errorprone.CompilationTestHelper; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class DoNotInvokeMessageDigestDirectlyTest { - - private CompilationTestHelper compilationHelper; - - @BeforeEach - public void setup() { - compilationHelper = - CompilationTestHelper.newInstance(DoNotInvokeMessageDigestDirectly.class, getClass()); - } - - @Test - public void doNotInvokeMessageDigestDirectlyPositiveCases() { - compilationHelper.addSourceFile("DoNotInvokeMessageDigestDirectlyPositiveCases.java").doTest(); - } - - @Test - public void doNotInvokeMessageDigestDirectlyNegativeCases() { - compilationHelper.addSourceFile("DoNotInvokeMessageDigestDirectlyNegativeCases.java").doTest(); - } -} diff --git a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/DoNotReturnNullOptionalsTest.java b/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/DoNotReturnNullOptionalsTest.java deleted file mode 100644 index 0055c9fc1..000000000 --- a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/DoNotReturnNullOptionalsTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import com.google.errorprone.CompilationTestHelper; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class DoNotReturnNullOptionalsTest { - - private CompilationTestHelper compilationHelper; - - @BeforeEach - public void setup() { - compilationHelper = - CompilationTestHelper.newInstance(DoNotReturnNullOptionals.class, getClass()); - } - - @Test - public void doNotReturnNullPositiveCases() { - compilationHelper.addSourceFile("DoNotReturnNullOptionalsPositiveCases.java").doTest(); - } - - @Test - public void doNotReturnNullNegativeCases() { - compilationHelper.addSourceFile("DoNotReturnNullOptionalsNegativeCases.java").doTest(); - } -} diff --git a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayedTest.java b/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayedTest.java deleted file mode 100644 index 159c3905e..000000000 --- a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayedTest.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import com.google.errorprone.CompilationTestHelper; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class ExperimentalCliOptionMustBeCorrectlyDisplayedTest { - - private CompilationTestHelper compilationHelper; - - @BeforeEach - public void setup() { - compilationHelper = - CompilationTestHelper.newInstance( - ExperimentalCliOptionMustBeCorrectlyDisplayed.class, getClass()); - } - - @Test - public void experimentalCliOptionMustBeHiddenPositiveCases() { - compilationHelper - .addSourceFile("ExperimentalCliOptionMustBeCorrectlyDisplayedPositiveCases.java") - .doTest(); - } - - @Test - public void experimentalCliOptionMustBeHiddenNegativeCases() { - compilationHelper - .addSourceFile("ExperimentalCliOptionMustBeCorrectlyDisplayedNegativeCases.java") - .doTest(); - } -} diff --git a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalTest.java b/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalTest.java deleted file mode 100644 index f94fa2d60..000000000 --- a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalTest.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import com.google.errorprone.CompilationTestHelper; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class MethodInputParametersMustBeFinalTest { - - private CompilationTestHelper compilationHelper; - - @BeforeEach - public void setup() { - compilationHelper = - CompilationTestHelper.newInstance(MethodInputParametersMustBeFinal.class, getClass()); - } - - @Test - public void methodInputParametersMustBeFinalPositiveCases() { - compilationHelper.addSourceFile("MethodInputParametersMustBeFinalPositiveCases.java").doTest(); - } - - @Test - public void methodInputParametersMustBeFinalInterfacePositiveCases() { - compilationHelper - .addSourceFile("MethodInputParametersMustBeFinalInterfacePositiveCases.java") - .doTest(); - } - - @Test - public void methodInputParametersMustBeFinalNegativeCases() { - compilationHelper.addSourceFile("MethodInputParametersMustBeFinalNegativeCases.java").doTest(); - } - - @Test - public void methodInputParametersMustBeFinalInterfaceNegativeCases() { - compilationHelper - .addSourceFile("MethodInputParametersMustBeFinalInterfaceNegativeCases.java") - .doTest(); - } -} diff --git a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/PreferCommonAnnotationsTest.java b/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/PreferCommonAnnotationsTest.java deleted file mode 100644 index b8a95f7d8..000000000 --- a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/PreferCommonAnnotationsTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright Hyperledger Besu contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import com.google.errorprone.CompilationTestHelper; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class PreferCommonAnnotationsTest { - - private CompilationTestHelper compilationHelper; - - @BeforeEach - public void setup() { - compilationHelper = - CompilationTestHelper.newInstance(PreferCommonAnnotations.class, getClass()); - } - - @Test - void preferCommonAnnotationsPositiveCases() { - compilationHelper.addSourceFile("PreferCommonAnnotationsPositiveCases.java").doTest(); - } - - @Test - void preferCommonAnnotationsNegativeCases() { - compilationHelper.addSourceFile("PreferCommonAnnotationsNegativeCases.java").doTest(); - } -} diff --git a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggersTest.java b/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggersTest.java deleted file mode 100644 index f15dabca1..000000000 --- a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggersTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import com.google.errorprone.CompilationTestHelper; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class PrivateStaticFinalLoggersTest { - - private CompilationTestHelper compilationHelper; - - @BeforeEach - public void setup() { - compilationHelper = - CompilationTestHelper.newInstance(PrivateStaticFinalLoggers.class, getClass()); - } - - @Test - public void privateStaticFinalLoggersPositiveCases() { - compilationHelper.addSourceFile("PrivateStaticFinalLoggersPositiveCases.java").doTest(); - } - - @Test - public void privateStaticFinalLoggersNegativeCases() { - compilationHelper.addSourceFile("PrivateStaticFinalLoggersNegativeCases.java").doTest(); - } -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/BannedMethodPositiveCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/BannedMethodPositiveCases.java deleted file mode 100644 index 0b2208a42..000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/BannedMethodPositiveCases.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.hyperledger.errorpronechecks; - -import com.google.common.base.Objects; - -public class BannedMethodPositiveCases { - - public void callsObjectsEquals() throws Exception { - // BUG: Diagnostic contains: Do not use com.google.common.base.Objects methods, use - // java.util.Objects methods instead. - Objects.equal("1", "1"); - } - - public void callsObjectsHashCode() throws Exception { - // BUG: Diagnostic contains: Do not use com.google.common.base.Objects methods, use - // java.util.Objects methods instead. - Objects.hashCode("1", "1"); - } - - public void usesJUnitAssertions() throws Exception { - // BUG: Diagnostic contains: Do not use junit assertions. Use assertj assertions instead. - org.junit.Assert.assertEquals(1, 1); - // BUG: Diagnostic contains: Do not use junit assertions. Use assertj assertions instead. - org.junit.Assert.assertNotEquals(1, 2); - // BUG: Diagnostic contains: Do not use junit assertions. Use assertj assertions instead. - org.junit.Assert.assertTrue(true); - // BUG: Diagnostic contains: Do not use junit assertions. Use assertj assertions instead. - org.junit.Assert.assertFalse(false); - // BUG: Diagnostic contains: Do not use junit assertions. Use assertj assertions instead. - org.junit.Assert.assertNull(null); - // BUG: Diagnostic contains: Do not use junit assertions. Use assertj assertions instead. - org.junit.Assert.assertNotNull("foo"); - // BUG: Diagnostic contains: Do not use junit assertions. Use assertj assertions instead. - org.junit.Assert.assertArrayEquals(new int[] {1}, new int[] {1}); - } -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectlyNegativeCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectlyNegativeCases.java deleted file mode 100644 index 704eeffa1..000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectlyNegativeCases.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.hyperledger.errorpronechecks; - -import java.security.Provider; -import java.security.SecureRandom; - -public class DoNotCreateSecureRandomDirectlyNegativeCases { - - public void callsNonJRESecureRandomGetInstance() throws Exception { - TestSecureRandom.getInstance(""); - TestSecureRandom.getInstance("", ""); - TestSecureRandom.getInstance("", new Provider("", 0, "") {}); - } - - public void invokesNonJRESecureRandomConstructor() throws Exception { - new TestSecureRandom(); - } - - private class TestSecureRandom extends SecureRandom {} -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectlyPositiveCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectlyPositiveCases.java deleted file mode 100644 index 8fb932cbf..000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectlyPositiveCases.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.hyperledger.errorpronechecks; - -import java.security.Provider; -import java.security.SecureRandom; - -public class DoNotCreateSecureRandomDirectlyPositiveCases { - - public void callsSecureRandomGetInstance() throws Exception { - // BUG: Diagnostic contains: Do not create SecureRandom directly. - SecureRandom.getInstance(""); - - // BUG: Diagnostic contains: Do not create SecureRandom directly. - SecureRandom.getInstance("", ""); - - // BUG: Diagnostic contains: Do not create SecureRandom directly. - SecureRandom.getInstance("", new Provider("", 0, "") {}); - } - - public void invokesSecureRandomConstructor() throws Exception { - // BUG: Diagnostic contains: Do not create SecureRandom directly. - new SecureRandom(); - - // BUG: Diagnostic contains: Do not create SecureRandom directly. - new SecureRandom(new byte[] {}); - } -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectlyPositiveCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectlyPositiveCases.java deleted file mode 100644 index 5aac65e16..000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectlyPositiveCases.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; - -public class DoNotInvokeMessageDigestDirectlyPositiveCases { - - public void callsMessageDigestGetInstance() throws NoSuchAlgorithmException { - // BUG: Diagnostic contains: Do not invoke MessageDigest.getInstance directly. - MessageDigest dig = MessageDigest.getInstance("SHA-256"); - } -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotReturnNullOptionalsNegativeCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotReturnNullOptionalsNegativeCases.java deleted file mode 100644 index aa9ac8139..000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotReturnNullOptionalsNegativeCases.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import java.util.Optional; -import javax.annotation.Nullable; - -public class DoNotReturnNullOptionalsNegativeCases { - - public interface allInterfacesAreValid { - public Optional ExpectToBeOverridden(); - } - - public DoNotReturnNullOptionalsNegativeCases() {} - - public Optional doesNotReturnNull() { - return Optional.of(3L); - } - - @Nullable - public Optional returnsNullButAnnotatedWithNullable() { - return Optional.empty(); - } -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotReturnNullOptionalsPositiveCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotReturnNullOptionalsPositiveCases.java deleted file mode 100644 index cd814d9cf..000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotReturnNullOptionalsPositiveCases.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.hyperledger.errorpronechecks; - -import java.util.Optional; - -public class DoNotReturnNullOptionalsPositiveCases { - - // BUG: Diagnostic contains: Do not return null optionals. - public Optional returnsNull() { - return null; - } - - // BUG: Diagnostic contains: Do not return null optionals. - public Optional sometimesReturnsNull(boolean random) { - if (random) { - - return null; - } - return Optional.of(2L); - } -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayedNegativeCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayedNegativeCases.java deleted file mode 100644 index 5362bba79..000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayedNegativeCases.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.hyperledger.errorpronechecks; - -import picocli.CommandLine; - -public class ExperimentalCliOptionMustBeCorrectlyDisplayedNegativeCases { - - @CommandLine.Option( - hidden = true, - names = {"--Xexperimental"}) - private String experimental = ""; - - @CommandLine.Option( - hidden = false, - names = {"--notExperimental"}) - private String notExperimental = ""; - - @CommandLine.Option(names = {"--notExperimental2"}) - private String notExperimental2 = ""; - - private class AnotherClass { - @CommandLine.Option(names = {"--notExperimentalInAnotherClass"}) - private String notExperimentalInAnotherClass = ""; - - @CommandLine.Option( - hidden = true, - names = {"--XexperimentalInAnotherClass"}) - private String experimentalInAnotherClass = ""; - } - - private class BesuCommand { - - @CommandLine.Option(names = {"--notExperimentalInBesuCommandClass"}) - private String notExperimentalInBesuCommandClass = ""; - } -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayedPositiveCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayedPositiveCases.java deleted file mode 100644 index c1721b4eb..000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayedPositiveCases.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.hyperledger.errorpronechecks; - -import picocli.CommandLine; - -public class ExperimentalCliOptionMustBeCorrectlyDisplayedPositiveCases { - - // BUG: Diagnostic contains: Experimental options must be hidden and not present in the - // BesuCommand class. - @CommandLine.Option( - hidden = false, - names = {"--Xexperimental"}) - private String experimental = ""; - - // BUG: Diagnostic contains: Experimental options must be hidden and not present in the - // BesuCommand class. - @CommandLine.Option(names = {"--Xexperimental2"}) - private String experimental2 = ""; - - private class BesuCommand { - - // BUG: Diagnostic contains: Experimental options must be hidden and not present in the - // BesuCommand class. - @CommandLine.Option(names = {"--XexperimentalInBesuCommandClass"}) - private String experimentalInBesuCommandClass = ""; - } -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalInterfaceNegativeCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalInterfaceNegativeCases.java deleted file mode 100644 index 237e3c173..000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalInterfaceNegativeCases.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import java.util.Observable; -import java.util.Observer; - -public interface MethodInputParametersMustBeFinalInterfaceNegativeCases { - - void parameterCannotBeFinal(int value); - - default void concreteMethod(final long value) {} - - static void anotherConcreteMethod(final double value) {} - - static Observer annonymousClass() { - return new Observer() { - @Override - public void update(final Observable o, final Object arg) {} - }; - } - - void methodAfterAnnonymousClass(int value); - - enum Status {} - - void methodAfterEnum(int value); -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalInterfacePositiveCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalInterfacePositiveCases.java deleted file mode 100644 index 759c1859d..000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalInterfacePositiveCases.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -public interface MethodInputParametersMustBeFinalInterfacePositiveCases { - - // BUG: Diagnostic contains: Method input parameters must be final. - default void concreteMethod(int value) {} - - // BUG: Diagnostic contains: Method input parameters must be final. - static void concreteStaticMethodsAreIncluded(int value) {} -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalNegativeCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalNegativeCases.java deleted file mode 100644 index a86056cf9..000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalNegativeCases.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import javax.annotation.processing.Generated; - -public class MethodInputParametersMustBeFinalNegativeCases { - - public void noInputParameters() {} - - public void onlyPrimativeInputParameters(final long value) {} - - public void onlyObjectInputParameters(final Object value) {} - - public void mixedInputParameters(final Object value, final int anotherValue) {} - - public interface allInterfacesAreValid { - void parameterCannotBeFinal(int value); - } -} - -@Generated( - value = "test", - comments = "Every method is buggy, but ignored because the class has been tagged generated") -class MethodInputParametersMustBeFinalPositiveCasesBugGenerated1 { - - public void primativeInputMethod(int value) {} - - public void objectInputMethod(Object value) {} - - public void mixedInputMethod(Object value, int anotherValue) {} - - @Generated( - value = "test", - comments = "Every method is buggy, but ignored because the class has been tagged generated") - public abstract class abstractClassDefinition { - public void concreteMethodsAreIncluded(int value) {} - } - - public void varArgsInputMethod(String... value) {} -} - -@Generated( - value = "test", - comments = "Every method is buggy, but ignored because the class has been tagged generated") -class MethodInputParametersMustBeFinalPositiveCasesBugGenerated2 { - - public void mixedInputMethodFirstFinal(final Object value, int anotherValue) {} - - public void mixedInputMethodSecondFinal(Object value, final int anotherValue) {} -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalPositiveCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalPositiveCases.java deleted file mode 100644 index f18fad651..000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalPositiveCases.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -public class MethodInputParametersMustBeFinalPositiveCases { - - // BUG: Diagnostic contains: Method input parameters must be final. - public void primativeInputMethod(int value) {} - - // BUG: Diagnostic contains: Method input parameters must be final. - public void objectInputMethod(Object value) {} - - // BUG: Diagnostic contains: Method input parameters must be final. - public void mixedInputMethod(Object value, int anotherValue) {} - - // BUG: Diagnostic contains: Method input parameters must be final. - public void mixedInputMethodFirstFinal(final Object value, int anotherValue) {} - - // BUG: Diagnostic contains: Method input parameters must be final. - public void mixedInputMethodSecondFinal(Object value, final int anotherValue) {} - - // BUG: Diagnostic contains: Method input parameters must be final. - public void varArgsInputMethod(String... value) {} - - public abstract class abstractClassDefinition { - // BUG: Diagnostic contains: Method input parameters must be final. - public void concreteMethodsAreIncluded(int value) {} - } -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PreferCommonAnnotationsNegativeCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PreferCommonAnnotationsNegativeCases.java deleted file mode 100644 index ec7cc634f..000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PreferCommonAnnotationsNegativeCases.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright Hyperledger Besu contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.hyperledger.errorpronechecks; - -import javax.annotation.Nonnull; - -public class PreferCommonAnnotationsNegativeCases { - - @Nonnull - public String getFoo() { - return "Foo"; - } - - // Fully Qualified Name is the "escape hatch" - @org.jetbrains.annotations.NotNull - public String getBar() { - return "Bar"; - } -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggersPositiveCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggersPositiveCases.java deleted file mode 100644 index cbe6c772d..000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggersPositiveCases.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.hyperledger.errorpronechecks; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class PrivateStaticFinalLoggersPositiveCases { - - // BUG: Diagnostic contains: Logger classes should be private, static, and final. - private final Logger LOG = LoggerFactory.getLogger(PrivateStaticFinalLoggersPositiveCases.class); -} diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/GraphQLHttpService.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/GraphQLHttpService.java index 03cf1cb7a..a9a12d617 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/GraphQLHttpService.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/GraphQLHttpService.java @@ -235,8 +235,7 @@ public class GraphQLHttpService { private boolean hostIsInAllowlist(final String hostHeader) { if (config.getHostsAllowlist().stream() - .anyMatch( - allowlistEntry -> allowlistEntry.toLowerCase().equals(hostHeader.toLowerCase()))) { + .anyMatch(allowlistEntry -> allowlistEntry.equalsIgnoreCase(hostHeader))) { return true; } else { LOG.trace("Host not in allowlist: '{}'", hostHeader); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/Scalars.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/Scalars.java index 5b69b0708..5762215bc 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/Scalars.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/Scalars.java @@ -330,7 +330,7 @@ public class Scalars { if (input instanceof Number number) { return number; } else if (input instanceof String string) { - final String value = string.toLowerCase(); + final String value = string.toLowerCase(Locale.ROOT); if (value.startsWith("0x")) { return Bytes.fromHexStringLenient(value).toLong(); } else { @@ -352,7 +352,7 @@ public class Scalars { if (input instanceof IntValue intValue) { return intValue.getValue().longValue(); } else if (input instanceof StringValue stringValue) { - final String value = stringValue.getValue().toLowerCase(); + final String value = stringValue.getValue().toLowerCase(Locale.ROOT); if (value.startsWith("0x")) { return Bytes.fromHexStringLenient(value).toLong(); } else { diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/AccountAdapter.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/AccountAdapter.java index 13caadab6..faf42aafc 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/AccountAdapter.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/AccountAdapter.java @@ -17,7 +17,7 @@ package org.hyperledger.besu.ethereum.api.graphql.internal.pojoadapter; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount; import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.account.AccountState; diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/TransactionAdapter.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/TransactionAdapter.java index f3ead13da..b0854e761 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/TransactionAdapter.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/TransactionAdapter.java @@ -274,7 +274,7 @@ public class TransactionAdapter extends AdapterBase { .map( receipt -> { final BytesValueRLPOutput rlpOutput = new BytesValueRLPOutput(); - receipt.getReceipt().writeTo(rlpOutput); + receipt.getReceipt().writeToForNetwork(rlpOutput); return rlpOutput.encoded(); }); } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpService.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpService.java index c2a247974..e86bbfdb8 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpService.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpService.java @@ -524,8 +524,7 @@ public class JsonRpcHttpService { private boolean hostIsInAllowlist(final String hostHeader) { if (config.getHostsAllowlist().stream() - .anyMatch( - allowlistEntry -> allowlistEntry.toLowerCase().equals(hostHeader.toLowerCase()))) { + .anyMatch(allowlistEntry -> allowlistEntry.equalsIgnoreCase(hostHeader))) { return true; } else { LOG.trace("Host not in allowlist: '{}'", hostHeader); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugGetRawReceipts.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugGetRawReceipts.java index 9e879de51..ba71ae0b3 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugGetRawReceipts.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugGetRawReceipts.java @@ -55,7 +55,7 @@ public class DebugGetRawReceipts extends AbstractBlockParameterOrBlockHashMethod private String[] toRLP(final List receipts) { return receipts.stream() - .map(receipt -> RLP.encode(receipt::writeTo).toHexString()) + .map(receipt -> RLP.encode(receipt::writeToForNetwork).toHexString()) .toArray(String[]::new); } } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/BlockParameter.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/BlockParameter.java index ceed60cbc..c90222ff3 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/BlockParameter.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/BlockParameter.java @@ -18,6 +18,7 @@ import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader; +import java.util.Locale; import java.util.Objects; import java.util.Optional; @@ -38,7 +39,7 @@ public class BlockParameter { @JsonCreator public BlockParameter(final String value) { - final String normalizedValue = value.toLowerCase(); + final String normalizedValue = value.toLowerCase(Locale.ROOT); switch (normalizedValue) { case "earliest": diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/BlockParameterOrBlockHash.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/BlockParameterOrBlockHash.java index 172a34d56..6cc537e69 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/BlockParameterOrBlockHash.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/BlockParameterOrBlockHash.java @@ -18,6 +18,7 @@ import org.hyperledger.besu.config.JsonUtil; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; +import java.util.Locale; import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; @@ -44,7 +45,7 @@ public class BlockParameterOrBlockHash { @JsonCreator public BlockParameterOrBlockHash(final Object value) throws JsonProcessingException { if (value instanceof String) { - final String normalizedValue = String.valueOf(value).toLowerCase(); + final String normalizedValue = String.valueOf(value).toLowerCase(Locale.ROOT); if (Objects.equals(normalizedValue, "earliest")) { type = BlockParameterType.EARLIEST; diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/PendingTransactionsParams.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/PendingTransactionsParams.java index 546fcec97..1bfe0e09a 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/PendingTransactionsParams.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/PendingTransactionsParams.java @@ -29,6 +29,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.transaction.po import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -82,7 +83,7 @@ public class PendingTransactionsParams { } else if (!map.isEmpty()) { final Map.Entry foundEntry = map.entrySet().stream().findFirst().get(); final Predicate predicate = - Predicate.fromValue(foundEntry.getKey().toUpperCase()) + Predicate.fromValue(foundEntry.getKey().toUpperCase(Locale.ROOT)) .orElseThrow( () -> new InvalidJsonRpcParameters( diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/TransactionCompleteResult.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/TransactionCompleteResult.java index 78c106980..003ee07ce 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/TransactionCompleteResult.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/TransactionCompleteResult.java @@ -126,7 +126,7 @@ public class TransactionCompleteResult implements TransactionResult { this.v = (transactionType == TransactionType.ACCESS_LIST || transactionType == TransactionType.EIP1559) - ? this.yParity + ? Quantity.create(transaction.getV()) : null; } this.value = Quantity.create(transaction.getValue()); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/TransactionPendingResult.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/TransactionPendingResult.java index 694284d22..62302808a 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/TransactionPendingResult.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/TransactionPendingResult.java @@ -116,7 +116,7 @@ public class TransactionPendingResult implements TransactionResult { this.v = (transactionType == TransactionType.ACCESS_LIST || transactionType == TransactionType.EIP1559) - ? this.yParity + ? Quantity.create(transaction.getV()) : null; } this.value = Quantity.create(transaction.getValue()); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/WebSocketService.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/WebSocketService.java index 80bb7d4fb..c040464dd 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/WebSocketService.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/WebSocketService.java @@ -331,9 +331,7 @@ public class WebSocketService { .map( header -> configuration.getHostsAllowlist().stream() - .anyMatch( - allowlistEntry -> - allowlistEntry.toLowerCase().equals(header.toLowerCase()))) + .anyMatch(allowListEntry -> allowListEntry.equalsIgnoreCase(header))) .orElse(false); } } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/pending/PendingTransactionDetailResult.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/pending/PendingTransactionDetailResult.java index 6634ded91..acbcb85ff 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/pending/PendingTransactionDetailResult.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/pending/PendingTransactionDetailResult.java @@ -73,7 +73,7 @@ public class PendingTransactionDetailResult implements JsonRpcResult { this.v = (transactionType == TransactionType.ACCESS_LIST || transactionType == TransactionType.EIP1559) - ? this.yParity + ? Quantity.create(tx.getV()) : null; } this.value = Quantity.create(tx.getValue()); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/StateBackupService.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/StateBackupService.java index 50c589c49..95a4923d4 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/StateBackupService.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/StateBackupService.java @@ -311,7 +311,7 @@ public class StateBackupService { bodyWriter.writeBytes(bodyOutput.encoded().toArrayUnsafe()); final BytesValueRLPOutput receiptsOutput = new BytesValueRLPOutput(); - receiptsOutput.writeList(receipts.get(), TransactionReceipt::writeToWithRevertReason); + receiptsOutput.writeList(receipts.get(), (r, rlpOut) -> r.writeToForStorage(rlpOut, false)); receiptsWriter.writeBytes(receiptsOutput.encoded().toArrayUnsafe()); backupStatus.storedBlock = blockNumber; diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/cache/TransactionLogBloomCacher.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/cache/TransactionLogBloomCacher.java index 31132a465..69a9edc82 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/cache/TransactionLogBloomCacher.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/cache/TransactionLogBloomCacher.java @@ -159,7 +159,7 @@ public class TransactionLogBloomCacher { return; } final long blockNumber = blockHeader.getNumber(); - LOG.atDebug() + LOG.atTrace() .setMessage("Caching logs bloom for block {}") .addArgument(() -> "0x" + Long.toHexString(blockNumber)) .log(); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTest.java index 901f20ff7..7c3157d22 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; @@ -1742,7 +1743,7 @@ public class JsonRpcHttpServiceTest extends JsonRpcHttpServiceTestBase { assertThat(Long.decode(result.getString("timestamp"))).isEqualTo(header.getTimestamp()); assertThat(Long.decode(result.getString("number"))).isEqualTo(header.getNumber()); // Nonce is a data field and should represent 8 bytes exactly - final String nonceResult = result.getString("nonce").toLowerCase(); + final String nonceResult = result.getString("nonce").toLowerCase(Locale.ROOT); assertThat(nonceResult.length() == 18 && nonceResult.startsWith("0x")).isTrue(); assertThat(Long.parseUnsignedLong(nonceResult.substring(2), 16)).isEqualTo(header.getNonce()); assertThat(Hash.fromHexString(result.getString("hash"))).isEqualTo(header.getHash()); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/blockheaders/NewBlockHeadersSubscriptionServiceTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/blockheaders/NewBlockHeadersSubscriptionServiceTest.java index 4902eb314..ecf3e5cc6 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/blockheaders/NewBlockHeadersSubscriptionServiceTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/blockheaders/NewBlockHeadersSubscriptionServiceTest.java @@ -63,7 +63,8 @@ public class NewBlockHeadersSubscriptionServiceTest { new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions()); + new MainnetBlockHeaderFunctions(), + false); private final Block genesisBlock = gen.genesisBlock(); private final MutableBlockchain blockchain = DefaultBlockchain.createMutable(genesisBlock, blockchainStorage, new NoOpMetricsSystem(), 0); diff --git a/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/graphql/eth_getTransaction_type2.json b/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/graphql/eth_getTransaction_type2.json index d88f5f515..c2c00330f 100644 --- a/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/graphql/eth_getTransaction_type2.json +++ b/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/graphql/eth_getTransaction_type2.json @@ -1,5 +1,5 @@ { - "request": "{transaction (hash : \"0x3ecd2ca6cf26c864d0ea5f038a58d4cd4a46a3e242fe92f446f392fdc232dd98\") { accessList { address storageKeys } maxFeePerGas maxPriorityFeePerGas nonce type status } } ", + "request": "{transaction (hash : \"0x3ecd2ca6cf26c864d0ea5f038a58d4cd4a46a3e242fe92f446f392fdc232dd98\") { accessList { address storageKeys } maxFeePerGas maxPriorityFeePerGas nonce type status yParity v} } ", "response": { "data": { "transaction": { @@ -15,7 +15,9 @@ "maxPriorityFeePerGas": "0x3b9aca00", "nonce": "0x20", "type": "0x2", - "status": "0x1" + "status": "0x1", + "yParity": "0x0", + "v": "0x25" } } }, diff --git a/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_getBlockByNumber_complete_shanghai.json b/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_getBlockByNumber_complete_shanghai.json index fb77fccd5..481dda711 100644 --- a/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_getBlockByNumber_complete_shanghai.json +++ b/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_getBlockByNumber_complete_shanghai.json @@ -58,7 +58,7 @@ "type": "0x2", "value": "0x0", "yParity": "0x0", - "v" : "0x0", + "v" : "0x25", "r": "0x8abbfbd4c5f2a13a8d5ed394ac50bac7d678f83a23f645818492f76e8ee17ab3", "s": "0x7bd38c6929235f775d68b45bd7dea7981264f9a265b6bea97b070e15be88389c" } diff --git a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java index b37cff7e4..c12061f4c 100644 --- a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java +++ b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java @@ -192,7 +192,7 @@ public class BlockTransactionSelector { isTimeout.set(true); } LOG.warn( - "Interrupting transaction selection since it is taking more than the max configured time of " + "Interrupting the selection of transactions for block inclusion as it exceeds the maximum configured duration of " + blockTxsSelectionMaxTime + "ms", e); diff --git a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java index a7dab2f12..26c3eb74c 100644 --- a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java +++ b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java @@ -167,7 +167,8 @@ public abstract class AbstractBlockTransactionSelectorTest { new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions()), + new MainnetBlockHeaderFunctions(), + false), new NoOpMetricsSystem(), 0); diff --git a/ethereum/core/build.gradle b/ethereum/core/build.gradle index 8e9aed30a..9744ea008 100644 --- a/ethereum/core/build.gradle +++ b/ethereum/core/build.gradle @@ -33,6 +33,7 @@ dependencies { api 'org.web3j:core' annotationProcessor 'org.openjdk.jmh:jmh-generator-annprocess' + annotationProcessor 'org.hyperledger.besu:besu-errorprone-checks' implementation project(':config') implementation project(':crypto:algorithms') diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java index 6aec8e664..2d7037528 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java @@ -495,10 +495,14 @@ public class Transaction @Override public BigInteger getV() { - if (transactionType != null && transactionType != TransactionType.FRONTIER) { - // EIP-2718 typed transaction, use yParity: + if (transactionType != null + && transactionType != TransactionType.FRONTIER + && transactionType != TransactionType.ACCESS_LIST + && transactionType != TransactionType.EIP1559) { + // Newer transaction type lacks V, so return null return null; } else { + // Mandatory for legacy, optional for EIP-2930 and EIP-1559 TXes, prohibited for all others. final BigInteger recId = BigInteger.valueOf(signature.getRecId()); return chainId .map(bigInteger -> recId.add(REPLAY_PROTECTED_V_BASE).add(TWO.multiply(bigInteger))) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/TransactionReceipt.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/TransactionReceipt.java index b7419b157..79fb1a501 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/TransactionReceipt.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/TransactionReceipt.java @@ -30,6 +30,7 @@ import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; import org.apache.tuweni.bytes.Bytes; @@ -169,23 +170,26 @@ public class TransactionReceipt implements org.hyperledger.besu.plugin.data.Tran * * @param out The RLP output to write to */ - public void writeTo(final RLPOutput out) { - writeTo(out, false); + public void writeToForNetwork(final RLPOutput out) { + writeTo(out, false, false); } - public void writeToWithRevertReason(final RLPOutput out) { - writeTo(out, true); + public void writeToForStorage(final RLPOutput out, final boolean compacted) { + writeTo(out, true, compacted); } - private void writeTo(final RLPOutput rlpOutput, final boolean withRevertReason) { + @VisibleForTesting + void writeTo(final RLPOutput rlpOutput, final boolean withRevertReason, final boolean compacted) { if (transactionType.equals(TransactionType.FRONTIER)) { - writeToForReceiptTrie(rlpOutput, withRevertReason); + writeToForReceiptTrie(rlpOutput, withRevertReason, compacted); } else { - rlpOutput.writeBytes(RLP.encode(out -> writeToForReceiptTrie(out, withRevertReason))); + rlpOutput.writeBytes( + RLP.encode(out -> writeToForReceiptTrie(out, withRevertReason, compacted))); } } - public void writeToForReceiptTrie(final RLPOutput rlpOutput, final boolean withRevertReason) { + public void writeToForReceiptTrie( + final RLPOutput rlpOutput, final boolean withRevertReason, final boolean compacted) { if (!transactionType.equals(TransactionType.FRONTIER)) { rlpOutput.writeIntScalar(transactionType.getSerializedType()); } @@ -200,8 +204,10 @@ public class TransactionReceipt implements org.hyperledger.besu.plugin.data.Tran rlpOutput.writeLongScalar(status); } rlpOutput.writeLongScalar(cumulativeGasUsed); - rlpOutput.writeBytes(bloomFilter); - rlpOutput.writeList(logs, Log::writeTo); + if (!compacted) { + rlpOutput.writeBytes(bloomFilter); + } + rlpOutput.writeList(logs, (log, logOutput) -> log.writeTo(logOutput, compacted)); if (withRevertReason && revertReason.isPresent()) { rlpOutput.writeBytes(revertReason.get()); } @@ -240,10 +246,21 @@ public class TransactionReceipt implements org.hyperledger.besu.plugin.data.Tran // correct transaction receipt encoding to use. final RLPInput firstElement = input.readAsRlp(); final long cumulativeGas = input.readLongScalar(); - // The logs below will populate the bloom filter upon construction. + + LogsBloomFilter bloomFilter = null; + + final boolean hasLogs = !input.nextIsList() && input.nextSize() == LogsBloomFilter.BYTE_SIZE; + if (hasLogs) { + // The logs below will populate the bloom filter upon construction. + bloomFilter = LogsBloomFilter.readFrom(input); + } // TODO consider validating that the logs and bloom filter match. - final LogsBloomFilter bloomFilter = LogsBloomFilter.readFrom(input); - final List logs = input.readList(Log::readFrom); + final boolean compacted = !hasLogs; + final List logs = input.readList(logInput -> Log.readFrom(logInput, compacted)); + if (compacted) { + bloomFilter = LogsBloomFilter.builder().insertLogs(logs).build(); + } + final Optional revertReason; if (input.isEndOfCurrentList()) { revertReason = Optional.empty(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java index 1a70ce493..c6cf6898e 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java @@ -31,8 +31,8 @@ import org.hyperledger.besu.ethereum.core.Withdrawal; import org.hyperledger.besu.ethereum.privacy.storage.PrivateMetadataUpdater; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.vm.BlockHashLookup; import org.hyperledger.besu.ethereum.vm.CachingBlockHashLookup; import org.hyperledger.besu.evm.gascalculator.CancunGasCalculator; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/BodyValidation.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/BodyValidation.java index f806c8d4c..ecf206a54 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/BodyValidation.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/BodyValidation.java @@ -118,7 +118,8 @@ public final class BodyValidation { trie.put( indexKey(i), RLP.encode( - rlpOutput -> receipts.get(i).writeToForReceiptTrie(rlpOutput, false)))); + rlpOutput -> + receipts.get(i).writeToForReceiptTrie(rlpOutput, false, false)))); return Hash.wrap(trie.getRootHash()); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ProtocolScheduleBuilder.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ProtocolScheduleBuilder.java index fbbb3aa8b..d70968891 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ProtocolScheduleBuilder.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ProtocolScheduleBuilder.java @@ -22,6 +22,7 @@ import org.hyperledger.besu.ethereum.privacy.PrivateTransactionValidator; import org.hyperledger.besu.evm.internal.EvmConfiguration; import java.math.BigInteger; +import java.util.NavigableMap; import java.util.Optional; import java.util.OptionalLong; import java.util.TreeMap; @@ -126,7 +127,7 @@ public class ProtocolScheduleBuilder { validateForkOrdering(); - final TreeMap builders = buildMilestoneMap(specFactory); + final NavigableMap builders = buildMilestoneMap(specFactory); // At this stage, all milestones are flagged with correct modifier, but ProtocolSpecs must be // inserted _AT_ the modifier block entry. @@ -320,7 +321,7 @@ public class ProtocolScheduleBuilder { return referenceForkBlock; } - private TreeMap buildMilestoneMap( + private NavigableMap buildMilestoneMap( final MainnetProtocolSpecFactory specFactory) { return createMilestones(specFactory) .flatMap(Optional::stream) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionReceipt.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionReceipt.java index 34ae107af..443c5077c 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionReceipt.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionReceipt.java @@ -168,16 +168,14 @@ public class PrivateTransactionReceipt { if (obj == this) { return true; } - if (!(obj instanceof PrivateTransactionReceipt)) { + if (!(obj instanceof PrivateTransactionReceipt other)) { return false; } - final PrivateTransactionReceipt other = (PrivateTransactionReceipt) obj; - return logs.equals(other.getLogs()) - && status == other.status - && output.equals(other.output) - && revertReason.isPresent() - ? revertReason.get().equals(other.revertReason.get()) - : true; + return !logs.equals(other.getLogs()) + || status != other.status + || !output.equals(other.output) + || revertReason.isEmpty() + || revertReason.get().equals(other.revertReason.get()); } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/StorageProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/StorageProvider.java index d927165b1..68c71156c 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/StorageProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/StorageProvider.java @@ -33,7 +33,9 @@ public interface StorageProvider extends Closeable { VariablesStorage createVariablesStorage(); BlockchainStorage createBlockchainStorage( - ProtocolSchedule protocolSchedule, VariablesStorage variablesStorage); + ProtocolSchedule protocolSchedule, + VariablesStorage variablesStorage, + DataStorageConfiguration storageConfiguration); WorldStateKeyValueStorage createWorldStateStorage( DataStorageConfiguration dataStorageConfiguration); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStoragePrefixedKeyBlockchainStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStoragePrefixedKeyBlockchainStorage.java index 9134ca91c..e6f3555f6 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStoragePrefixedKeyBlockchainStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStoragePrefixedKeyBlockchainStorage.java @@ -59,14 +59,17 @@ public class KeyValueStoragePrefixedKeyBlockchainStorage implements BlockchainSt final KeyValueStorage blockchainStorage; final VariablesStorage variablesStorage; final BlockHeaderFunctions blockHeaderFunctions; + final boolean receiptCompaction; public KeyValueStoragePrefixedKeyBlockchainStorage( final KeyValueStorage blockchainStorage, final VariablesStorage variablesStorage, - final BlockHeaderFunctions blockHeaderFunctions) { + final BlockHeaderFunctions blockHeaderFunctions, + final boolean receiptCompaction) { this.blockchainStorage = blockchainStorage; this.variablesStorage = variablesStorage; this.blockHeaderFunctions = blockHeaderFunctions; + this.receiptCompaction = receiptCompaction; migrateVariables(); } @@ -125,7 +128,8 @@ public class KeyValueStoragePrefixedKeyBlockchainStorage implements BlockchainSt @Override public Updater updater() { - return new Updater(blockchainStorage.startTransaction(), variablesStorage.updater()); + return new Updater( + blockchainStorage.startTransaction(), variablesStorage.updater(), receiptCompaction); } private List rlpDecodeTransactionReceipts(final Bytes bytes) { @@ -253,12 +257,15 @@ public class KeyValueStoragePrefixedKeyBlockchainStorage implements BlockchainSt private final KeyValueStorageTransaction blockchainTransaction; private final VariablesStorage.Updater variablesUpdater; + private final boolean receiptCompaction; Updater( final KeyValueStorageTransaction blockchainTransaction, - final VariablesStorage.Updater variablesUpdater) { + final VariablesStorage.Updater variablesUpdater, + final boolean receiptCompaction) { this.blockchainTransaction = blockchainTransaction; this.variablesUpdater = variablesUpdater; + this.receiptCompaction = receiptCompaction; } @Override @@ -365,7 +372,10 @@ public class KeyValueStoragePrefixedKeyBlockchainStorage implements BlockchainSt } private Bytes rlpEncode(final List receipts) { - return RLP.encode(o -> o.writeList(receipts, TransactionReceipt::writeToWithRevertReason)); + return RLP.encode( + o -> + o.writeList( + receipts, (r, rlpOutput) -> r.writeToForStorage(rlpOutput, receiptCompaction))); } private void removeVariables() { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java index 53a3f4546..944f668cd 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java @@ -19,7 +19,7 @@ import org.hyperledger.besu.ethereum.chain.VariablesStorage; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; @@ -69,11 +69,14 @@ public class KeyValueStorageProvider implements StorageProvider { @Override public BlockchainStorage createBlockchainStorage( - final ProtocolSchedule protocolSchedule, final VariablesStorage variablesStorage) { + final ProtocolSchedule protocolSchedule, + final VariablesStorage variablesStorage, + final DataStorageConfiguration dataStorageConfiguration) { return new KeyValueStoragePrefixedKeyBlockchainStorage( getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.BLOCKCHAIN), variablesStorage, - ScheduleBasedBlockHeaderFunctions.create(protocolSchedule)); + ScheduleBasedBlockHeaderFunctions.create(protocolSchedule), + dataStorageConfiguration.getReceiptCompactionEnabled()); } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java index 121fec3f4..7f89ff718 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java @@ -19,11 +19,11 @@ package org.hyperledger.besu.ethereum.trie.common; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.NoOpCachedWorldStorageManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.NoOpTrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.NoOpBonsaiCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.NoOpTrieLogManager; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; @@ -59,8 +59,8 @@ public class GenesisWorldStateProvider { * @return a mutable world state for the Genesis block */ private static MutableWorldState createGenesisBonsaiWorldState() { - final CachedMerkleTrieLoader cachedMerkleTrieLoader = - new CachedMerkleTrieLoader(new NoOpMetricsSystem()); + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader = + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()); final BonsaiWorldStateKeyValueStorage bonsaiWorldStateKeyValueStorage = new BonsaiWorldStateKeyValueStorage( new KeyValueStorageProvider( @@ -71,8 +71,8 @@ public class GenesisWorldStateProvider { DataStorageConfiguration.DEFAULT_BONSAI_CONFIG); return new BonsaiWorldState( bonsaiWorldStateKeyValueStorage, - cachedMerkleTrieLoader, - new NoOpCachedWorldStorageManager(bonsaiWorldStateKeyValueStorage), + bonsaiCachedMerkleTrieLoader, + new NoOpBonsaiCachedWorldStorageManager(bonsaiWorldStateKeyValueStorage), new NoOpTrieLogManager(), EvmConfiguration.DEFAULT); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiAccount.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiAccount.java similarity index 55% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiAccount.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiAccount.java index d93f550ca..6eea664e0 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiAccount.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiAccount.java @@ -14,48 +14,34 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import org.hyperledger.besu.datatypes.AccountValue; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; -import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.rlp.RLPException; import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.rlp.RLPOutput; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; import org.hyperledger.besu.evm.ModificationNotAllowedException; import org.hyperledger.besu.evm.account.AccountStorageEntry; -import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.worldstate.UpdateTrackingAccount; -import java.util.HashMap; -import java.util.Map; import java.util.NavigableMap; import java.util.Objects; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.units.bigints.UInt256; -public class BonsaiAccount implements MutableAccount, AccountValue { - private final BonsaiWorldView context; - private boolean immutable; - - private final Address address; - private final Hash addressHash; - private Hash codeHash; - private long nonce; - private Wei balance; +public class BonsaiAccount extends DiffBasedAccount { private Hash storageRoot; - private Bytes code; - - private final Map updatedStorage = new HashMap<>(); public BonsaiAccount( - final BonsaiWorldView context, + final DiffBasedWorldView context, final Address address, final Hash addressHash, final long nonce, @@ -63,31 +49,24 @@ public class BonsaiAccount implements MutableAccount, AccountValue { final Hash storageRoot, final Hash codeHash, final boolean mutable) { - this.context = context; - this.address = address; - this.addressHash = addressHash; - this.nonce = nonce; - this.balance = balance; + super(context, address, addressHash, nonce, balance, codeHash, mutable); this.storageRoot = storageRoot; - this.codeHash = codeHash; - - this.immutable = !mutable; } public BonsaiAccount( - final BonsaiWorldView context, + final DiffBasedWorldView context, final Address address, final AccountValue stateTrieAccount, final boolean mutable) { - this( + super( context, address, address.addressHash(), stateTrieAccount.getNonce(), stateTrieAccount.getBalance(), - stateTrieAccount.getStorageRoot(), stateTrieAccount.getCodeHash(), mutable); + this.storageRoot = stateTrieAccount.getStorageRoot(); } public BonsaiAccount(final BonsaiAccount toCopy) { @@ -95,37 +74,35 @@ public class BonsaiAccount implements MutableAccount, AccountValue { } public BonsaiAccount( - final BonsaiAccount toCopy, final BonsaiWorldView context, final boolean mutable) { - this.context = context; - this.address = toCopy.address; - this.addressHash = toCopy.addressHash; - this.nonce = toCopy.nonce; - this.balance = toCopy.balance; + final BonsaiAccount toCopy, final DiffBasedWorldView context, final boolean mutable) { + super( + context, + toCopy.address, + toCopy.addressHash, + toCopy.nonce, + toCopy.balance, + toCopy.codeHash, + mutable); this.storageRoot = toCopy.storageRoot; - this.codeHash = toCopy.codeHash; - this.code = toCopy.code; updatedStorage.putAll(toCopy.updatedStorage); - - this.immutable = !mutable; } public BonsaiAccount( - final BonsaiWorldView context, final UpdateTrackingAccount tracked) { - this.context = context; - this.address = tracked.getAddress(); - this.addressHash = tracked.getAddressHash(); - this.nonce = tracked.getNonce(); - this.balance = tracked.getBalance(); + final DiffBasedWorldView context, final UpdateTrackingAccount tracked) { + super( + context, + tracked.getAddress(), + tracked.getAddressHash(), + tracked.getNonce(), + tracked.getBalance(), + tracked.getCodeHash(), + true); this.storageRoot = Hash.EMPTY_TRIE_HASH; - this.codeHash = tracked.getCodeHash(); - this.code = tracked.getCode(); updatedStorage.putAll(tracked.getUpdatedStorage()); - - this.immutable = false; } public static BonsaiAccount fromRLP( - final BonsaiWorldView context, + final DiffBasedWorldView context, final Address address, final Bytes encoded, final boolean mutable) @@ -144,88 +121,11 @@ public class BonsaiAccount implements MutableAccount, AccountValue { context, address, address.addressHash(), nonce, balance, storageRoot, codeHash, mutable); } - @Override - public Address getAddress() { - return address; - } - - @Override - public Hash getAddressHash() { - return addressHash; - } - - @Override - public long getNonce() { - return nonce; - } - - @Override - public void setNonce(final long value) { - if (immutable) { - throw new ModificationNotAllowedException(); - } - nonce = value; - } - - @Override - public Wei getBalance() { - return balance; - } - - @Override - public void setBalance(final Wei value) { - if (immutable) { - throw new ModificationNotAllowedException(); - } - balance = value; - } - - @Override - public Bytes getCode() { - if (code == null) { - code = context.getCode(address, codeHash).orElse(Bytes.EMPTY); - } - return code; - } - - @Override - public void setCode(final Bytes code) { - if (immutable) { - throw new ModificationNotAllowedException(); - } - this.code = code; - if (code == null || code.isEmpty()) { - this.codeHash = Hash.EMPTY; - } else { - this.codeHash = Hash.hash(code); - } - } - - @Override - public Hash getCodeHash() { - return codeHash; - } - - @Override - public UInt256 getStorageValue(final UInt256 key) { - return context.getStorageValue(address, key); - } - - @Override - public UInt256 getOriginalStorageValue(final UInt256 key) { - return context.getPriorStorageValue(address, key); - } - @Override public NavigableMap storageEntriesFrom( final Bytes32 startKeyHash, final int limit) { - return context.getWorldStateStorage().storageEntriesFrom(this.addressHash, startKeyHash, limit); - } - - public Bytes serializeAccount() { - final BytesValueRLPOutput out = new BytesValueRLPOutput(); - writeTo(out); - return out.encoded(); + return ((BonsaiWorldStateKeyValueStorage) context.getWorldStateStorage()) + .storageEntriesFrom(this.addressHash, startKeyHash, limit); } @Override @@ -240,24 +140,6 @@ public class BonsaiAccount implements MutableAccount, AccountValue { out.endList(); } - @Override - public void setStorageValue(final UInt256 key, final UInt256 value) { - if (immutable) { - throw new ModificationNotAllowedException(); - } - updatedStorage.put(key, value); - } - - @Override - public void clearStorage() { - updatedStorage.clear(); - } - - @Override - public Map getUpdatedStorage() { - return updatedStorage; - } - @Override public Hash getStorageRoot() { return storageRoot; @@ -270,11 +152,6 @@ public class BonsaiAccount implements MutableAccount, AccountValue { this.storageRoot = storageRoot; } - @Override - public void becomeImmutable() { - immutable = true; - } - @Override public String toString() { return "AccountState{" diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProvider.java new file mode 100644 index 000000000..bfd295091 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProvider.java @@ -0,0 +1,145 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.rlp.RLP; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; +import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; +import org.hyperledger.besu.evm.internal.EvmConfiguration; +import org.hyperledger.besu.plugin.BesuContext; + +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.tuweni.bytes.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BonsaiWorldStateProvider extends DiffBasedWorldStateProvider { + + private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateProvider.class); + private final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader; + + public BonsaiWorldStateProvider( + final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, + final Blockchain blockchain, + final Optional maxLayersToLoad, + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader, + final BesuContext pluginContext, + final EvmConfiguration evmConfiguration) { + super(worldStateKeyValueStorage, blockchain, maxLayersToLoad, pluginContext); + this.bonsaiCachedMerkleTrieLoader = bonsaiCachedMerkleTrieLoader; + provideCachedWorldStorageManager( + new BonsaiCachedWorldStorageManager(this, worldStateKeyValueStorage)); + loadPersistedState(new BonsaiWorldState(this, worldStateKeyValueStorage, evmConfiguration)); + } + + @VisibleForTesting + BonsaiWorldStateProvider( + final BonsaiCachedWorldStorageManager bonsaiCachedWorldStorageManager, + final TrieLogManager trieLogManager, + final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, + final Blockchain blockchain, + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader, + final EvmConfiguration evmConfiguration) { + super(worldStateKeyValueStorage, blockchain, trieLogManager); + this.bonsaiCachedMerkleTrieLoader = bonsaiCachedMerkleTrieLoader; + provideCachedWorldStorageManager(bonsaiCachedWorldStorageManager); + loadPersistedState(new BonsaiWorldState(this, worldStateKeyValueStorage, evmConfiguration)); + } + + public BonsaiCachedMerkleTrieLoader getCachedMerkleTrieLoader() { + return bonsaiCachedMerkleTrieLoader; + } + + private BonsaiWorldStateKeyValueStorage getWorldStateKeyValueStorage() { + return (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage; + } + /** + * Prepares the state healing process for a given address and location. It prepares the state + * healing, including retrieving data from storage, identifying invalid slots or nodes, removing + * account and slot from the state trie, and committing the changes. Finally, it downgrades the + * world state storage to partial flat database mode. + */ + public void prepareStateHealing(final Address address, final Bytes location) { + final Set keysToDelete = new HashSet<>(); + final BonsaiWorldStateKeyValueStorage.Updater updater = + getWorldStateKeyValueStorage().updater(); + final Hash accountHash = address.addressHash(); + final StoredMerklePatriciaTrie accountTrie = + new StoredMerklePatriciaTrie<>( + (l, h) -> { + final Optional node = + getWorldStateKeyValueStorage().getAccountStateTrieNode(l, h); + if (node.isPresent()) { + keysToDelete.add(l); + } + return node; + }, + persistedState.getWorldStateRootHash(), + Function.identity(), + Function.identity()); + try { + accountTrie + .get(accountHash) + .map(RLP::input) + .map(StateTrieAccountValue::readFrom) + .ifPresent( + account -> { + final StoredMerklePatriciaTrie storageTrie = + new StoredMerklePatriciaTrie<>( + (l, h) -> { + Optional node = + getWorldStateKeyValueStorage() + .getAccountStorageTrieNode(accountHash, l, h); + if (node.isPresent()) { + keysToDelete.add(Bytes.concatenate(accountHash, l)); + } + return node; + }, + account.getStorageRoot(), + Function.identity(), + Function.identity()); + try { + storageTrie.getPath(location); + } catch (Exception eA) { + LOG.warn("Invalid slot found for account {} at location {}", address, location); + // ignore + } + }); + } catch (Exception eA) { + LOG.warn("Invalid node for account {} at location {}", address, location); + // ignore + } + keysToDelete.forEach(updater::removeAccountStateTrieNode); + updater.commit(); + + getWorldStateKeyValueStorage().downgradeToPartialFlatDbMode(); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedMerkleTrieLoader.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoader.java similarity index 94% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedMerkleTrieLoader.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoader.java index cbd1fc820..9f3251660 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedMerkleTrieLoader.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoader.java @@ -13,14 +13,15 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.cache; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.metrics.ObservableMetricsSystem; @@ -37,8 +38,7 @@ import io.prometheus.client.guava.cache.CacheMetricsCollector; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -public class CachedMerkleTrieLoader - implements BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber { +public class BonsaiCachedMerkleTrieLoader implements StorageSubscriber { private static final int ACCOUNT_CACHE_SIZE = 100_000; private static final int STORAGE_CACHE_SIZE = 200_000; @@ -47,7 +47,7 @@ public class CachedMerkleTrieLoader private final Cache storageNodes = CacheBuilder.newBuilder().recordStats().maximumSize(STORAGE_CACHE_SIZE).build(); - public CachedMerkleTrieLoader(final ObservableMetricsSystem metricsSystem) { + public BonsaiCachedMerkleTrieLoader(final ObservableMetricsSystem metricsSystem) { CacheMetricsCollector cacheMetrics = new CacheMetricsCollector(); cacheMetrics.addCache("accountsNodes", accountNodes); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedMerkleTrieLoaderModule.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoaderModule.java similarity index 77% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedMerkleTrieLoaderModule.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoaderModule.java index 63c8051f2..04a8efc45 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedMerkleTrieLoaderModule.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoaderModule.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.cache; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache; import org.hyperledger.besu.metrics.ObservableMetricsSystem; @@ -21,11 +21,11 @@ import dagger.Module; import dagger.Provides; @Module -public class CachedMerkleTrieLoaderModule { +public class BonsaiCachedMerkleTrieLoaderModule { @Provides - CachedMerkleTrieLoader provideCachedMerkleTrieLoaderModule( + BonsaiCachedMerkleTrieLoader provideCachedMerkleTrieLoaderModule( final ObservableMetricsSystem metricsSystem) { - return new CachedMerkleTrieLoader(metricsSystem); + return new BonsaiCachedMerkleTrieLoader(metricsSystem); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedWorldStorageManager.java new file mode 100644 index 000000000..a7bee477c --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedWorldStorageManager.java @@ -0,0 +1,60 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache; + +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateLayerStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.evm.internal.EvmConfiguration; + +public class BonsaiCachedWorldStorageManager extends DiffBasedCachedWorldStorageManager { + + public BonsaiCachedWorldStorageManager( + final BonsaiWorldStateProvider archive, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage) { + super(archive, worldStateKeyValueStorage); + } + + @Override + public DiffBasedWorldState createWorldState( + final DiffBasedWorldStateProvider archive, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, + final EvmConfiguration evmConfiguration) { + return new BonsaiWorldState( + (BonsaiWorldStateProvider) archive, + (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage, + evmConfiguration); + } + + @Override + public DiffBasedWorldStateKeyValueStorage createLayeredKeyValueStorage( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage) { + return new BonsaiWorldStateLayerStorage( + (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage); + } + + @Override + public DiffBasedWorldStateKeyValueStorage createSnapshotKeyValueStorage( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage) { + return new BonsaiSnapshotWorldStateKeyValueStorage( + (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/NoOpBonsaiCachedWorldStorageManager.java similarity index 68% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/NoOpBonsaiCachedWorldStorageManager.java index d86128b13..0e4ee9cbf 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/NoOpBonsaiCachedWorldStorageManager.java @@ -12,19 +12,19 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.cache; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; import java.util.Optional; import java.util.function.Function; -public class NoOpCachedWorldStorageManager extends CachedWorldStorageManager { +public class NoOpBonsaiCachedWorldStorageManager extends BonsaiCachedWorldStorageManager { - public NoOpCachedWorldStorageManager( + public NoOpBonsaiCachedWorldStorageManager( final BonsaiWorldStateKeyValueStorage bonsaiWorldStateKeyValueStorage) { super(null, bonsaiWorldStateKeyValueStorage); } @@ -33,7 +33,7 @@ public class NoOpCachedWorldStorageManager extends CachedWorldStorageManager { public synchronized void addCachedLayer( final BlockHeader blockHeader, final Hash worldStateRootHash, - final BonsaiWorldState forWorldState) { + final DiffBasedWorldState forWorldState) { // no cache } @@ -43,17 +43,17 @@ public class NoOpCachedWorldStorageManager extends CachedWorldStorageManager { } @Override - public Optional getWorldState(final Hash blockHash) { + public Optional getWorldState(final Hash blockHash) { return Optional.empty(); } @Override - public Optional getNearestWorldState(final BlockHeader blockHeader) { + public Optional getNearestWorldState(final BlockHeader blockHeader) { return Optional.empty(); } @Override - public Optional getHeadWorldState( + public Optional getHeadWorldState( final Function> hashBlockHeaderFunction) { return Optional.empty(); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiPreImageProxy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiPreImageProxy.java similarity index 97% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiPreImageProxy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiPreImageProxy.java index 16cda3104..4bae322a8 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiPreImageProxy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiPreImageProxy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java similarity index 93% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java index e20d84473..c2972e045 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java @@ -13,11 +13,12 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedSnapshotWorldStateKeyValueStorage; import org.hyperledger.besu.plugin.services.exception.StorageException; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.SnappableKeyValueStorage; @@ -32,7 +33,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class BonsaiSnapshotWorldStateKeyValueStorage extends BonsaiWorldStateKeyValueStorage - implements BonsaiStorageSubscriber { + implements DiffBasedSnapshotWorldStateKeyValueStorage, StorageSubscriber { protected final BonsaiWorldStateKeyValueStorage parentWorldStateStorage; private static final Logger LOG = @@ -53,9 +54,9 @@ public class BonsaiSnapshotWorldStateKeyValueStorage extends BonsaiWorldStateKey final BonsaiWorldStateKeyValueStorage worldStateStorageKeyValueStorage) { this( worldStateStorageKeyValueStorage, - ((SnappableKeyValueStorage) worldStateStorageKeyValueStorage.composedWorldStateStorage) + ((SnappableKeyValueStorage) worldStateStorageKeyValueStorage.getComposedWorldStateStorage()) .takeSnapshot(), - worldStateStorageKeyValueStorage.trieLogStorage); + worldStateStorageKeyValueStorage.getTrieLogStorage()); } private boolean isClosedGet() { @@ -207,7 +208,7 @@ public class BonsaiSnapshotWorldStateKeyValueStorage extends BonsaiWorldStateKey protected synchronized void doClose() throws Exception { if (!isClosedGet()) { // alert any subscribers we are closing: - subscribers.forEach(BonsaiStorageSubscriber::onCloseStorage); + subscribers.forEach(StorageSubscriber::onCloseStorage); // close all of the SnappedKeyValueStorages: composedWorldStateStorage.close(); @@ -220,6 +221,7 @@ public class BonsaiSnapshotWorldStateKeyValueStorage extends BonsaiWorldStateKey } } + @Override public BonsaiWorldStateKeyValueStorage getParentWorldStateStorage() { return parentWorldStateStorage; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorage.java similarity index 67% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorage.java index 5f477afeb..a58316fc6 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorage.java @@ -12,7 +12,7 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; @@ -24,8 +24,9 @@ import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.trie.MerkleTrie; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.flat.FlatDbStrategy; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.flat.FlatDbStrategyProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategyProvider; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; @@ -37,52 +38,28 @@ import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; -import org.hyperledger.besu.util.Subscribers; -import java.nio.charset.StandardCharsets; import java.util.List; -import java.util.Map; import java.util.NavigableMap; import java.util.Optional; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; -import java.util.stream.Stream; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class BonsaiWorldStateKeyValueStorage implements WorldStateKeyValueStorage, AutoCloseable { - private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateKeyValueStorage.class); - - // 0x776f726c64526f6f74 - public static final byte[] WORLD_ROOT_HASH_KEY = "worldRoot".getBytes(StandardCharsets.UTF_8); - // 0x776f726c64426c6f636b48617368 - public static final byte[] WORLD_BLOCK_HASH_KEY = - "worldBlockHash".getBytes(StandardCharsets.UTF_8); +public class BonsaiWorldStateKeyValueStorage extends DiffBasedWorldStateKeyValueStorage + implements WorldStateKeyValueStorage { protected final FlatDbStrategyProvider flatDbStrategyProvider; - protected final SegmentedKeyValueStorage composedWorldStateStorage; - protected final KeyValueStorage trieLogStorage; - - private final AtomicBoolean shouldClose = new AtomicBoolean(false); - - protected final AtomicBoolean isClosed = new AtomicBoolean(false); - - protected final Subscribers subscribers = Subscribers.create(); - public BonsaiWorldStateKeyValueStorage( final StorageProvider provider, final MetricsSystem metricsSystem, final DataStorageConfiguration dataStorageConfiguration) { - this.composedWorldStateStorage = + super( provider.getStorageBySegmentIdentifiers( List.of( - ACCOUNT_INFO_STATE, CODE_STORAGE, ACCOUNT_STORAGE_STORAGE, TRIE_BRANCH_STORAGE)); - this.trieLogStorage = - provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE); + ACCOUNT_INFO_STATE, CODE_STORAGE, ACCOUNT_STORAGE_STORAGE, TRIE_BRANCH_STORAGE)), + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE)); this.flatDbStrategyProvider = new FlatDbStrategyProvider(metricsSystem, dataStorageConfiguration); flatDbStrategyProvider.loadFlatDbStrategy(composedWorldStateStorage); @@ -92,9 +69,8 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateKeyValueStorag final FlatDbStrategyProvider flatDbStrategyProvider, final SegmentedKeyValueStorage composedWorldStateStorage, final KeyValueStorage trieLogStorage) { + super(composedWorldStateStorage, trieLogStorage); this.flatDbStrategyProvider = flatDbStrategyProvider; - this.composedWorldStateStorage = composedWorldStateStorage; - this.trieLogStorage = trieLogStorage; } @Override @@ -102,6 +78,7 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateKeyValueStorag return DataStorageFormat.BONSAI; } + @Override public FlatDbMode getFlatDbMode() { return flatDbStrategyProvider.getFlatDbMode(); } @@ -155,31 +132,6 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateKeyValueStorag .map(Bytes::wrap); } - public Optional getTrieLog(final Hash blockHash) { - return trieLogStorage.get(blockHash.toArrayUnsafe()); - } - - public Stream streamTrieLogKeys(final long limit) { - return trieLogStorage.streamKeys().limit(limit); - } - - public Optional getStateTrieNode(final Bytes location) { - return composedWorldStateStorage - .get(TRIE_BRANCH_STORAGE, location.toArrayUnsafe()) - .map(Bytes::wrap); - } - - public Optional getWorldStateRootHash() { - return composedWorldStateStorage.get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY).map(Bytes::wrap); - } - - public Optional getWorldStateBlockHash() { - return composedWorldStateStorage - .get(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY) - .map(Bytes32::wrap) - .map(Hash::wrap); - } - public Optional getStorageValueByStorageSlotKey( final Hash accountHash, final StorageSlotKey storageSlotKey) { return getStorageValueByStorageSlotKey( @@ -209,34 +161,11 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateKeyValueStorag composedWorldStateStorage); } - public Map streamFlatAccounts( - final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { - return flatDbStrategyProvider - .getFlatDbStrategy(composedWorldStateStorage) - .streamAccountFlatDatabase(composedWorldStateStorage, startKeyHash, endKeyHash, max); - } - - public Map streamFlatStorages( - final Hash accountHash, final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { - return flatDbStrategyProvider - .getFlatDbStrategy(composedWorldStateStorage) - .streamStorageFlatDatabase( - composedWorldStateStorage, accountHash, startKeyHash, endKeyHash, max); - } - public NavigableMap storageEntriesFrom( final Hash addressHash, final Bytes32 startKeyHash, final int limit) { throw new RuntimeException("Bonsai Tries does not currently support enumerating storage"); } - public boolean isWorldStateAvailable(final Bytes32 rootHash, final Hash blockHash) { - return composedWorldStateStorage - .get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY) - .map(Bytes32::wrap) - .map(hash -> hash.equals(rootHash) || trieLogStorage.containsKey(blockHash.toArrayUnsafe())) - .orElse(false); - } - public void upgradeToFullFlatDbMode() { flatDbStrategyProvider.upgradeToFullFlatDbMode(composedWorldStateStorage); } @@ -247,26 +176,14 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateKeyValueStorag @Override public void clear() { - subscribers.forEach(BonsaiStorageSubscriber::onClearStorage); - flatDbStrategyProvider - .getFlatDbStrategy(composedWorldStateStorage) - .clearAll(composedWorldStateStorage); - composedWorldStateStorage.clear(TRIE_BRANCH_STORAGE); - trieLogStorage.clear(); + super.clear(); flatDbStrategyProvider.loadFlatDbStrategy( composedWorldStateStorage); // force reload of flat db reader strategy } - public void clearTrieLog() { - subscribers.forEach(BonsaiStorageSubscriber::onClearTrieLog); - trieLogStorage.clear(); - } - - public void clearFlatDatabase() { - subscribers.forEach(BonsaiStorageSubscriber::onClearFlatDatabaseStorage); - flatDbStrategyProvider - .getFlatDbStrategy(composedWorldStateStorage) - .resetOnResync(composedWorldStateStorage); + @Override + public FlatDbStrategy getFlatDbStrategy() { + return flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage); } @Override @@ -277,20 +194,7 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateKeyValueStorag flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage)); } - public boolean pruneTrieLog(final Hash blockHash) { - try { - return trieLogStorage.tryDelete(blockHash.toArrayUnsafe()); - } catch (Exception e) { - LOG.error("Error pruning trie log for block hash {}", blockHash, e); - return false; - } - } - - public FlatDbStrategy getFlatDbStrategy() { - return flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage); - } - - public static class Updater implements WorldStateKeyValueStorage.Updater { + public static class Updater implements DiffBasedWorldStateKeyValueStorage.Updater { private final SegmentedKeyValueStorageTransaction composedWorldStateTransaction; private final KeyValueStorageTransaction trieLogStorageTransaction; @@ -340,6 +244,7 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateKeyValueStorag return this; } + @Override public Updater saveWorldState(final Bytes blockHash, final Bytes32 nodeHash, final Bytes node) { composedWorldStateTransaction.put( TRIE_BRANCH_STORAGE, Bytes.EMPTY.toArrayUnsafe(), node.toArrayUnsafe()); @@ -392,10 +297,12 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateKeyValueStorag composedWorldStateTransaction, accountHash, slotHash); } + @Override public SegmentedKeyValueStorageTransaction getWorldStateTransaction() { return composedWorldStateTransaction; } + @Override public KeyValueStorageTransaction getTrieLogStorageTransaction() { return trieLogStorageTransaction; } @@ -407,65 +314,10 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateKeyValueStorag composedWorldStateTransaction.commit(); } + @Override public void rollback() { composedWorldStateTransaction.rollback(); trieLogStorageTransaction.rollback(); } } - - @Override - public synchronized void close() throws Exception { - // when the storage clears, close - shouldClose.set(true); - tryClose(); - } - - public synchronized long subscribe(final BonsaiStorageSubscriber sub) { - if (isClosed.get()) { - throw new RuntimeException("Storage is marked to close or has already closed"); - } - return subscribers.subscribe(sub); - } - - public synchronized void unSubscribe(final long id) { - subscribers.unsubscribe(id); - try { - tryClose(); - } catch (Exception e) { - LOG.atWarn() - .setMessage("exception while trying to close : {}") - .addArgument(e::getMessage) - .log(); - } - } - - protected synchronized void tryClose() throws Exception { - if (shouldClose.get() && subscribers.getSubscriberCount() < 1) { - doClose(); - } - } - - protected synchronized void doClose() throws Exception { - if (!isClosed.get()) { - // alert any subscribers we are closing: - subscribers.forEach(BonsaiStorageSubscriber::onCloseStorage); - - // close all of the KeyValueStorages: - composedWorldStateStorage.close(); - trieLogStorage.close(); - - // set storage closed - isClosed.set(true); - } - } - - public interface BonsaiStorageSubscriber { - default void onClearStorage() {} - - default void onClearFlatDatabaseStorage() {} - - default void onClearTrieLog() {} - - default void onCloseStorage() {} - } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateLayerStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateLayerStorage.java similarity index 79% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateLayerStorage.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateLayerStorage.java index 92d4f54fe..045376f61 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateLayerStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateLayerStorage.java @@ -13,21 +13,22 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedLayeredWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.SnappedKeyValueStorage; import org.hyperledger.besu.services.kvstore.LayeredKeyValueStorage; public class BonsaiWorldStateLayerStorage extends BonsaiSnapshotWorldStateKeyValueStorage - implements BonsaiStorageSubscriber { + implements DiffBasedLayeredWorldStateKeyValueStorage, StorageSubscriber { public BonsaiWorldStateLayerStorage(final BonsaiWorldStateKeyValueStorage parent) { this( - new LayeredKeyValueStorage(parent.composedWorldStateStorage), - parent.trieLogStorage, + new LayeredKeyValueStorage(parent.getComposedWorldStateStorage()), + parent.getTrieLogStorage(), parent); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FullFlatDbStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/FullFlatDbStrategy.java similarity index 93% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FullFlatDbStrategy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/FullFlatDbStrategy.java index 83f7f030c..f8cbf2c3d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FullFlatDbStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/FullFlatDbStrategy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; @@ -21,6 +21,8 @@ import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIden import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.trie.NodeLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.CodeStorageStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategy; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/PartialFlatDbStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/PartialFlatDbStrategy.java similarity index 96% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/PartialFlatDbStrategy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/PartialFlatDbStrategy.java index cfe18dbe2..9e693d63e 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/PartialFlatDbStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/PartialFlatDbStrategy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; @@ -21,6 +21,8 @@ import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIden import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.trie.NodeLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.CodeStorageStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategy; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.trie.patricia.StoredNodeFactory; import org.hyperledger.besu.metrics.BesuMetricCategory; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryImpl.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryImpl.java similarity index 91% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryImpl.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryImpl.java index 2915a6718..6ebda666d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryImpl.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryImpl.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog; import org.hyperledger.besu.datatypes.AccountValue; import org.hyperledger.besu.datatypes.Address; @@ -23,7 +23,8 @@ import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.rlp.RLPOutput; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.plugin.data.BlockHeader; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; @@ -150,7 +151,7 @@ public class TrieLogFactoryImpl implements TrieLogFactory { final TrieLogLayer newLayer = new TrieLogLayer(); input.enterList(); - newLayer.blockHash = Hash.wrap(input.readBytes32()); + newLayer.setBlockHash(Hash.wrap(input.readBytes32())); while (!input.isEndOfCurrentList()) { input.enterList(); @@ -164,7 +165,9 @@ public class TrieLogFactoryImpl implements TrieLogFactory { final StateTrieAccountValue newValue = nullOrValue(input, StateTrieAccountValue::readFrom); final boolean isCleared = getOptionalIsCleared(input); input.leaveList(); - newLayer.accounts.put(address, new BonsaiValue<>(oldValue, newValue, isCleared)); + newLayer + .getAccountChanges() + .put(address, new DiffBasedValue<>(oldValue, newValue, isCleared)); } if (input.nextIsNull()) { @@ -175,13 +178,13 @@ public class TrieLogFactoryImpl implements TrieLogFactory { final Bytes newCode = nullOrValue(input, RLPInput::readBytes); final boolean isCleared = getOptionalIsCleared(input); input.leaveList(); - newLayer.code.put(address, new BonsaiValue<>(oldCode, newCode, isCleared)); + newLayer.getCodeChanges().put(address, new DiffBasedValue<>(oldCode, newCode, isCleared)); } if (input.nextIsNull()) { input.skipNext(); } else { - final Map> storageChanges = new TreeMap<>(); + final Map> storageChanges = new TreeMap<>(); input.enterList(); while (!input.isEndOfCurrentList()) { input.enterList(); @@ -190,11 +193,11 @@ public class TrieLogFactoryImpl implements TrieLogFactory { final UInt256 oldValue = nullOrValue(input, RLPInput::readUInt256Scalar); final UInt256 newValue = nullOrValue(input, RLPInput::readUInt256Scalar); final boolean isCleared = getOptionalIsCleared(input); - storageChanges.put(storageSlotKey, new BonsaiValue<>(oldValue, newValue, isCleared)); + storageChanges.put(storageSlotKey, new DiffBasedValue<>(oldValue, newValue, isCleared)); input.leaveList(); } input.leaveList(); - newLayer.storage.put(address, storageChanges); + newLayer.getStorageChanges().put(address, storageChanges); } // TODO add trie nodes diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java similarity index 59% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java index 9e5472f57..3b3af540d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java @@ -14,39 +14,33 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai.worldview; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; -import static org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; -import static org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; -import static org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldView.encodeTrieValue; +import static org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView.encodeTrieValue; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; import org.hyperledger.besu.ethereum.trie.NodeLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiAccount; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiValue; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateLayerStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator.StorageConsumingMap; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateLayerStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.StorageConsumingMap; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.internal.EvmConfiguration; -import org.hyperledger.besu.evm.worldstate.WorldUpdater; -import org.hyperledger.besu.plugin.services.exception.StorageException; -import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; @@ -63,24 +57,10 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.rlp.RLP; import org.apache.tuweni.units.bigints.UInt256; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public class BonsaiWorldState - implements MutableWorldState, BonsaiWorldView, BonsaiStorageSubscriber { +public class BonsaiWorldState extends DiffBasedWorldState { - private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldState.class); - - protected BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage; - - protected final CachedMerkleTrieLoader cachedMerkleTrieLoader; - protected final CachedWorldStorageManager cachedWorldStorageManager; - protected final TrieLogManager trieLogManager; - private BonsaiWorldStateUpdateAccumulator accumulator; - - protected Hash worldStateRootHash; - Hash worldStateBlockHash; - private boolean isFrozen; + protected final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader; public BonsaiWorldState( final BonsaiWorldStateProvider archive, @@ -96,90 +76,39 @@ public class BonsaiWorldState public BonsaiWorldState( final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, - final CachedMerkleTrieLoader cachedMerkleTrieLoader, - final CachedWorldStorageManager cachedWorldStorageManager, + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader, + final DiffBasedCachedWorldStorageManager cachedWorldStorageManager, final TrieLogManager trieLogManager, final EvmConfiguration evmConfiguration) { - this.worldStateKeyValueStorage = worldStateKeyValueStorage; - this.worldStateRootHash = - Hash.wrap( - Bytes32.wrap( - worldStateKeyValueStorage.getWorldStateRootHash().orElse(getEmptyTrieHash()))); - this.worldStateBlockHash = - Hash.wrap( - Bytes32.wrap(worldStateKeyValueStorage.getWorldStateBlockHash().orElse(Hash.ZERO))); - this.accumulator = + super(worldStateKeyValueStorage, cachedWorldStorageManager, trieLogManager); + this.bonsaiCachedMerkleTrieLoader = bonsaiCachedMerkleTrieLoader; + this.setAccumulator( new BonsaiWorldStateUpdateAccumulator( this, (addr, value) -> - cachedMerkleTrieLoader.preLoadAccount( - getWorldStateStorage(), worldStateRootHash, addr), + bonsaiCachedMerkleTrieLoader.preLoadAccount( + worldStateKeyValueStorage, worldStateRootHash, addr), (addr, value) -> - cachedMerkleTrieLoader.preLoadStorageSlot(getWorldStateStorage(), addr, value), - evmConfiguration); - this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; - this.cachedWorldStorageManager = cachedWorldStorageManager; - this.trieLogManager = trieLogManager; - } - - /** - * Override the accumulator solves the chicken-egg problem of needing a worldstate reference - * (this) when construction the Accumulator. - * - * @param accumulator accumulator to use. - */ - public void setAccumulator(final BonsaiWorldStateUpdateAccumulator accumulator) { - this.accumulator = accumulator; - } - - /** - * Returns the world state block hash of this world state - * - * @return the world state block hash. - */ - public Hash getWorldStateBlockHash() { - return worldStateBlockHash; - } - - /** - * Returns the world state root hash of this world state - * - * @return the world state root hash. - */ - public Hash getWorldStateRootHash() { - return worldStateRootHash; - } - - @Override - public boolean isPersisted() { - return isPersisted(worldStateKeyValueStorage); - } - - private boolean isPersisted(final WorldStateKeyValueStorage worldStateKeyValueStorage) { - return !(worldStateKeyValueStorage instanceof BonsaiSnapshotWorldStateKeyValueStorage); - } - - @Override - public Optional getCode(@Nonnull final Address address, final Hash codeHash) { - return worldStateKeyValueStorage.getCode(codeHash, address.addressHash()); - } - - /** - * Reset the worldState to this block header - * - * @param blockHeader block to use - */ - public void resetWorldStateTo(final BlockHeader blockHeader) { - worldStateBlockHash = blockHeader.getBlockHash(); - worldStateRootHash = blockHeader.getStateRoot(); + bonsaiCachedMerkleTrieLoader.preLoadStorageSlot( + getWorldStateStorage(), addr, value), + evmConfiguration)); } @Override public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { - return worldStateKeyValueStorage; + return (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage; } - private Hash calculateRootHash( + @Override + protected Hash calculateRootHash( + final Optional maybeStateUpdater, + final DiffBasedWorldStateUpdateAccumulator worldStateUpdater) { + return internalCalculateRootHash( + maybeStateUpdater.map(BonsaiWorldStateKeyValueStorage.Updater.class::cast), + (BonsaiWorldStateUpdateAccumulator) worldStateUpdater); + } + + private Hash internalCalculateRootHash( final Optional maybeStateUpdater, final BonsaiWorldStateUpdateAccumulator worldStateUpdater) { @@ -187,7 +116,7 @@ public class BonsaiWorldState // This must be done before updating the accounts so // that we can get the storage state hash - Stream>>> + Stream>>> storageStream = worldStateUpdater.getStorageToUpdate().entrySet().stream(); if (maybeStateUpdater.isEmpty()) { storageStream = @@ -205,8 +134,8 @@ public class BonsaiWorldState final StoredMerklePatriciaTrie accountTrie = createTrie( (location, hash) -> - cachedMerkleTrieLoader.getAccountStateTrieNode( - worldStateKeyValueStorage, location, hash), + bonsaiCachedMerkleTrieLoader.getAccountStateTrieNode( + getWorldStateStorage(), location, hash), worldStateRootHash); // for manicured tries and composting, collect branches here (not implemented) @@ -231,10 +160,10 @@ public class BonsaiWorldState final Optional maybeStateUpdater, final BonsaiWorldStateUpdateAccumulator worldStateUpdater, final StoredMerklePatriciaTrie accountTrie) { - for (final Map.Entry> accountUpdate : + for (final Map.Entry> accountUpdate : worldStateUpdater.getAccountsToUpdate().entrySet()) { final Bytes accountKey = accountUpdate.getKey(); - final BonsaiValue bonsaiValue = accountUpdate.getValue(); + final DiffBasedValue bonsaiValue = accountUpdate.getValue(); final BonsaiAccount updatedAccount = bonsaiValue.getUpdated(); try { if (updatedAccount == null) { @@ -264,7 +193,7 @@ public class BonsaiWorldState final BonsaiWorldStateUpdateAccumulator worldStateUpdater) { maybeStateUpdater.ifPresent( bonsaiUpdater -> { - for (final Map.Entry> codeUpdate : + for (final Map.Entry> codeUpdate : worldStateUpdater.getCodeToUpdate().entrySet()) { final Bytes updatedCode = codeUpdate.getValue().getUpdated(); final Hash accountHash = codeUpdate.getKey().addressHash(); @@ -294,12 +223,12 @@ public class BonsaiWorldState private void updateAccountStorageState( final Optional maybeStateUpdater, final BonsaiWorldStateUpdateAccumulator worldStateUpdater, - final Map.Entry>> + final Map.Entry>> storageAccountUpdate) { final Address updatedAddress = storageAccountUpdate.getKey(); final Hash updatedAddressHash = updatedAddress.addressHash(); if (worldStateUpdater.getAccountsToUpdate().containsKey(updatedAddress)) { - final BonsaiValue accountValue = + final DiffBasedValue accountValue = worldStateUpdater.getAccountsToUpdate().get(updatedAddress); final BonsaiAccount accountOriginal = accountValue.getPrior(); final Hash storageRoot = @@ -310,12 +239,12 @@ public class BonsaiWorldState final StoredMerklePatriciaTrie storageTrie = createTrie( (location, key) -> - cachedMerkleTrieLoader.getAccountStorageTrieNode( - worldStateKeyValueStorage, updatedAddressHash, location, key), + bonsaiCachedMerkleTrieLoader.getAccountStorageTrieNode( + getWorldStateStorage(), updatedAddressHash, location, key), storageRoot); // for manicured tries and composting, collect branches here (not implemented) - for (final Map.Entry> storageUpdate : + for (final Map.Entry> storageUpdate : storageAccountUpdate.getValue().entrySet()) { final Hash slotHash = storageUpdate.getKey().getSlotHash(); final UInt256 updatedStorage = storageUpdate.getValue().getUpdated(); @@ -360,11 +289,10 @@ public class BonsaiWorldState private void clearStorage( final Optional maybeStateUpdater, final BonsaiWorldStateUpdateAccumulator worldStateUpdater) { - for (final Address address : worldStateUpdater.getStorageToClear()) { // because we are clearing persisted values we need the account root as persisted final BonsaiAccount oldAccount = - worldStateKeyValueStorage + getWorldStateStorage() .getAccount(address.addressHash()) .map(bytes -> BonsaiAccount.fromRLP(BonsaiWorldState.this, address, bytes, true)) .orElse(null); @@ -379,7 +307,7 @@ public class BonsaiWorldState (location, key) -> getStorageTrieNode(addressHash, location, key), oldAccount.getStorageRoot()); try { - final StorageConsumingMap> storageToDelete = + final StorageConsumingMap> storageToDelete = worldStateUpdater.getStorageToUpdate().get(address); Map entriesToDelete = storageTrie.entriesFrom(Bytes32.ZERO, 256); while (!entriesToDelete.isEmpty()) { @@ -394,7 +322,7 @@ public class BonsaiWorldState address.addressHash(), storageSlotKey.getSlotHash())); storageToDelete .computeIfAbsent( - storageSlotKey, key -> new BonsaiValue<>(slotValue, null, true)) + storageSlotKey, key -> new DiffBasedValue<>(slotValue, null, true)) .setPrior(slotValue); }); entriesToDelete.keySet().forEach(storageTrie::remove); @@ -412,138 +340,6 @@ public class BonsaiWorldState } } - @Override - public void persist(final BlockHeader blockHeader) { - final Optional maybeBlockHeader = Optional.ofNullable(blockHeader); - LOG.atDebug() - .setMessage("Persist world state for block {}") - .addArgument(maybeBlockHeader) - .log(); - - final BonsaiWorldStateUpdateAccumulator localCopy = accumulator.copy(); - - boolean success = false; - - final BonsaiWorldStateKeyValueStorage.Updater stateUpdater = - worldStateKeyValueStorage.updater(); - Runnable saveTrieLog = () -> {}; - - try { - final Hash newWorldStateRootHash = - calculateRootHash(isFrozen ? Optional.empty() : Optional.of(stateUpdater), accumulator); - // if we are persisted with a block header, and the prior state is the parent - // then persist the TrieLog for that transition. - // If specified but not a direct descendant simply store the new block hash. - if (blockHeader != null) { - verifyWorldStateRoot(newWorldStateRootHash, blockHeader); - saveTrieLog = - () -> { - trieLogManager.saveTrieLog(localCopy, newWorldStateRootHash, blockHeader, this); - // not save a frozen state in the cache - if (!isFrozen) { - cachedWorldStorageManager.addCachedLayer(blockHeader, newWorldStateRootHash, this); - } - }; - - stateUpdater - .getWorldStateTransaction() - .put(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY, blockHeader.getHash().toArrayUnsafe()); - worldStateBlockHash = blockHeader.getHash(); - } else { - stateUpdater.getWorldStateTransaction().remove(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY); - worldStateBlockHash = null; - } - - stateUpdater - .getWorldStateTransaction() - .put(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY, newWorldStateRootHash.toArrayUnsafe()); - worldStateRootHash = newWorldStateRootHash; - success = true; - } finally { - if (success) { - stateUpdater.commit(); - accumulator.reset(); - saveTrieLog.run(); - } else { - stateUpdater.rollback(); - accumulator.reset(); - } - } - } - - protected void verifyWorldStateRoot(final Hash calculatedStateRoot, final BlockHeader header) { - if (!calculatedStateRoot.equals(header.getStateRoot())) { - throw new RuntimeException( - "World State Root does not match expected value, header " - + header.getStateRoot().toHexString() - + " calculated " - + calculatedStateRoot.toHexString()); - } - } - - @Override - public WorldUpdater updater() { - return accumulator; - } - - @Override - public Hash rootHash() { - if (isFrozen && accumulator.isAccumulatorStateChanged()) { - worldStateRootHash = calculateRootHash(Optional.empty(), accumulator.copy()); - accumulator.resetAccumulatorStateChanged(); - } - return Hash.wrap(worldStateRootHash); - } - - static final KeyValueStorageTransaction noOpTx = - new KeyValueStorageTransaction() { - - @Override - public void put(final byte[] key, final byte[] value) { - // no-op - } - - @Override - public void remove(final byte[] key) { - // no-op - } - - @Override - public void commit() throws StorageException { - // no-op - } - - @Override - public void rollback() { - // no-op - } - }; - - static final SegmentedKeyValueStorageTransaction noOpSegmentedTx = - new SegmentedKeyValueStorageTransaction() { - - @Override - public void put( - final SegmentIdentifier segmentIdentifier, final byte[] key, final byte[] value) { - // no-op - } - - @Override - public void remove(final SegmentIdentifier segmentIdentifier, final byte[] key) { - // no-op - } - - @Override - public void commit() throws StorageException { - // no-op - } - - @Override - public void rollback() { - // no-op - } - }; - @Override public Hash frontierRootHash() { return calculateRootHash( @@ -553,25 +349,28 @@ public class BonsaiWorldState accumulator.copy()); } - public Hash blockHash() { - return worldStateBlockHash; - } - @Override - public Stream streamAccounts(final Bytes32 startKeyHash, final int limit) { - throw new RuntimeException("Bonsai Tries do not provide account streaming."); + public MutableWorldState freeze() { + this.isFrozen = true; + this.worldStateKeyValueStorage = new BonsaiWorldStateLayerStorage(getWorldStateStorage()); + return this; } @Override public Account get(final Address address) { - return worldStateKeyValueStorage + return getWorldStateStorage() .getAccount(address.addressHash()) .map(bytes -> BonsaiAccount.fromRLP(accumulator, address, bytes, true)) .orElse(null); } + @Override + public Optional getCode(@Nonnull final Address address, final Hash codeHash) { + return getWorldStateStorage().getCode(codeHash, address.addressHash()); + } + protected Optional getAccountStateTrieNode(final Bytes location, final Bytes32 nodeHash) { - return worldStateKeyValueStorage.getAccountStateTrieNode(location, nodeHash); + return getWorldStateStorage().getAccountStateTrieNode(location, nodeHash); } private void writeTrieNode( @@ -584,7 +383,7 @@ public class BonsaiWorldState protected Optional getStorageTrieNode( final Hash accountHash, final Bytes location, final Bytes32 nodeHash) { - return worldStateKeyValueStorage.getAccountStorageTrieNode(accountHash, location, nodeHash); + return getWorldStateStorage().getAccountStorageTrieNode(accountHash, location, nodeHash); } private void writeStorageTrieNode( @@ -605,7 +404,7 @@ public class BonsaiWorldState @Override public Optional getStorageValueByStorageSlotKey( final Address address, final StorageSlotKey storageSlotKey) { - return worldStateKeyValueStorage + return getWorldStateStorage() .getStorageValueByStorageSlotKey(address.addressHash(), storageSlotKey) .map(UInt256::fromBytes); } @@ -614,7 +413,7 @@ public class BonsaiWorldState final Supplier> storageRootSupplier, final Address address, final StorageSlotKey storageSlotKey) { - return worldStateKeyValueStorage + return getWorldStateStorage() .getStorageValueByStorageSlotKey(storageRootSupplier, address.addressHash(), storageSlotKey) .map(UInt256::fromBytes); } @@ -632,50 +431,18 @@ public class BonsaiWorldState return storageTrie.entriesFrom(Bytes32.ZERO, Integer.MAX_VALUE); } - @Override - public MutableWorldState freeze() { - this.isFrozen = true; - this.worldStateKeyValueStorage = new BonsaiWorldStateLayerStorage(worldStateKeyValueStorage); - return this; - } - private StoredMerklePatriciaTrie createTrie( final NodeLoader nodeLoader, final Bytes32 rootHash) { return new StoredMerklePatriciaTrie<>( nodeLoader, rootHash, Function.identity(), Function.identity()); } - @Override - public void close() { - try { - if (!isPersisted()) { - this.worldStateKeyValueStorage.close(); - if (isFrozen) { - closeFrozenStorage(); - } - } - } catch (Exception e) { - // no op - } - } - - private void closeFrozenStorage() { - try { - final BonsaiWorldStateLayerStorage worldStateLayerStorage = - (BonsaiWorldStateLayerStorage) worldStateKeyValueStorage; - if (!isPersisted(worldStateLayerStorage.getParentWorldStateStorage())) { - worldStateLayerStorage.getParentWorldStateStorage().close(); - } - } catch (Exception e) { - // no op - } - } - protected Hash hashAndSavePreImage(final Bytes value) { // by default do not save has preImages return Hash.hash(value); } + @Override protected Hash getEmptyTrieHash() { return Hash.EMPTY_TRIE_HASH; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java new file mode 100644 index 000000000..ff2e696e1 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java @@ -0,0 +1,98 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview; + +import org.hyperledger.besu.datatypes.AccountValue; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.Consumer; +import org.hyperledger.besu.evm.internal.EvmConfiguration; +import org.hyperledger.besu.evm.worldstate.UpdateTrackingAccount; + +public class BonsaiWorldStateUpdateAccumulator + extends DiffBasedWorldStateUpdateAccumulator { + public BonsaiWorldStateUpdateAccumulator( + final DiffBasedWorldView world, + final Consumer> accountPreloader, + final Consumer storagePreloader, + final EvmConfiguration evmConfiguration) { + super(world, accountPreloader, storagePreloader, evmConfiguration); + } + + @Override + public DiffBasedWorldStateUpdateAccumulator copy() { + final BonsaiWorldStateUpdateAccumulator copy = + new BonsaiWorldStateUpdateAccumulator( + wrappedWorldView(), + getAccountPreloader(), + getStoragePreloader(), + getEvmConfiguration()); + copy.cloneFromUpdater(this); + return copy; + } + + @Override + protected BonsaiAccount copyAccount(final BonsaiAccount account) { + return new BonsaiAccount(account); + } + + @Override + protected BonsaiAccount copyAccount( + final BonsaiAccount toCopy, final DiffBasedWorldView context, final boolean mutable) { + return new BonsaiAccount(toCopy, context, mutable); + } + + @Override + protected BonsaiAccount createAccount( + final DiffBasedWorldView context, + final Address address, + final AccountValue stateTrieAccount, + final boolean mutable) { + return new BonsaiAccount(context, address, stateTrieAccount, mutable); + } + + @Override + protected BonsaiAccount createAccount( + final DiffBasedWorldView context, + final Address address, + final Hash addressHash, + final long nonce, + final Wei balance, + final Hash storageRoot, + final Hash codeHash, + final boolean mutable) { + return new BonsaiAccount( + context, address, addressHash, nonce, balance, storageRoot, codeHash, mutable); + } + + @Override + protected BonsaiAccount createAccount( + final DiffBasedWorldView context, final UpdateTrackingAccount tracked) { + return new BonsaiAccount(context, tracked); + } + + @Override + protected void assertCloseEnoughForDiffing( + final BonsaiAccount source, final AccountValue account, final String context) { + BonsaiAccount.assertCloseEnoughForDiffing(source, account, context); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedAccount.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedAccount.java new file mode 100644 index 000000000..044deb45a --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedAccount.java @@ -0,0 +1,206 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.hyperledger.besu.ethereum.trie.diffbased.common; + +import org.hyperledger.besu.datatypes.AccountValue; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; +import org.hyperledger.besu.evm.ModificationNotAllowedException; +import org.hyperledger.besu.evm.account.MutableAccount; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.units.bigints.UInt256; + +public abstract class DiffBasedAccount implements MutableAccount, AccountValue { + protected final DiffBasedWorldView context; + protected boolean immutable; + protected final Address address; + protected final Hash addressHash; + protected Hash codeHash; + protected long nonce; + protected Wei balance; + protected Bytes code; + + protected final Map updatedStorage = new HashMap<>(); + + public DiffBasedAccount( + final DiffBasedWorldView context, + final Address address, + final Hash addressHash, + final long nonce, + final Wei balance, + final Hash codeHash, + final boolean mutable) { + this.context = context; + this.address = address; + this.addressHash = addressHash; + this.nonce = nonce; + this.balance = balance; + this.codeHash = codeHash; + + this.immutable = !mutable; + } + + public DiffBasedAccount( + final DiffBasedWorldView context, + final Address address, + final AccountValue stateTrieAccount, + final boolean mutable) { + this( + context, + address, + address.addressHash(), + stateTrieAccount.getNonce(), + stateTrieAccount.getBalance(), + stateTrieAccount.getCodeHash(), + mutable); + } + + public DiffBasedAccount( + final DiffBasedAccount toCopy, final DiffBasedWorldView context, final boolean mutable) { + this.context = context; + this.address = toCopy.address; + this.addressHash = toCopy.addressHash; + this.nonce = toCopy.nonce; + this.balance = toCopy.balance; + this.codeHash = toCopy.codeHash; + this.code = toCopy.code; + updatedStorage.putAll(toCopy.updatedStorage); + + this.immutable = !mutable; + } + + @Override + public Address getAddress() { + return address; + } + + @Override + public Hash getAddressHash() { + return addressHash; + } + + @Override + public long getNonce() { + return nonce; + } + + @Override + public void setNonce(final long value) { + if (immutable) { + throw new ModificationNotAllowedException(); + } + nonce = value; + } + + @Override + public Wei getBalance() { + return balance; + } + + @Override + public void setBalance(final Wei value) { + if (immutable) { + throw new ModificationNotAllowedException(); + } + balance = value; + } + + @Override + public Bytes getCode() { + if (code == null) { + code = context.getCode(address, codeHash).orElse(Bytes.EMPTY); + } + return code; + } + + @Override + public void setCode(final Bytes code) { + if (immutable) { + throw new ModificationNotAllowedException(); + } + this.code = code; + if (code == null || code.isEmpty()) { + this.codeHash = Hash.EMPTY; + } else { + this.codeHash = Hash.hash(code); + } + } + + @Override + public Hash getCodeHash() { + return codeHash; + } + + @Override + public UInt256 getStorageValue(final UInt256 key) { + return context.getStorageValue(address, key); + } + + @Override + public UInt256 getOriginalStorageValue(final UInt256 key) { + return context.getPriorStorageValue(address, key); + } + + public Bytes serializeAccount() { + final BytesValueRLPOutput out = new BytesValueRLPOutput(); + writeTo(out); + return out.encoded(); + } + + @Override + public void setStorageValue(final UInt256 key, final UInt256 value) { + if (immutable) { + throw new ModificationNotAllowedException(); + } + updatedStorage.put(key, value); + } + + @Override + public void clearStorage() { + updatedStorage.clear(); + } + + @Override + public Map getUpdatedStorage() { + return updatedStorage; + } + + @Override + public void becomeImmutable() { + immutable = true; + } + + @Override + public String toString() { + return "AccountState{" + + "address=" + + address + + ", nonce=" + + nonce + + ", balance=" + + balance + + ", codeHash=" + + codeHash + + '}'; + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiValue.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedValue.java similarity index 80% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiValue.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedValue.java index c99da3fe2..bd9c3e3f5 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiValue.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedValue.java @@ -1,5 +1,5 @@ /* - * Copyright ConsenSys AG. + * Copyright Hyperledger Besu Contributors. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -11,38 +11,37 @@ * specific language governing permissions and limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.common; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; -public class BonsaiValue implements TrieLog.LogTuple { +public class DiffBasedValue implements TrieLog.LogTuple { private T prior; private T updated; private boolean lastStepCleared; private boolean clearedAtLeastOnce; - public BonsaiValue(final T prior, final T updated) { + public DiffBasedValue(final T prior, final T updated) { this.prior = prior; this.updated = updated; this.lastStepCleared = false; this.clearedAtLeastOnce = false; } - public BonsaiValue(final T prior, final T updated, final boolean lastStepCleared) { + public DiffBasedValue(final T prior, final T updated, final boolean lastStepCleared) { this.prior = prior; this.updated = updated; this.lastStepCleared = lastStepCleared; this.clearedAtLeastOnce = lastStepCleared; } - public BonsaiValue( + public DiffBasedValue( final T prior, final T updated, final boolean lastStepCleared, @@ -63,12 +62,12 @@ public class BonsaiValue implements TrieLog.LogTuple { return updated; } - public BonsaiValue setPrior(final T prior) { + public DiffBasedValue setPrior(final T prior) { this.prior = prior; return this; } - public BonsaiValue setUpdated(final T updated) { + public DiffBasedValue setUpdated(final T updated) { this.lastStepCleared = updated == null; if (lastStepCleared) { this.clearedAtLeastOnce = true; @@ -94,7 +93,7 @@ public class BonsaiValue implements TrieLog.LogTuple { @Override public String toString() { - return "BonsaiValue{" + return "DiffBasedValue{" + "prior=" + prior + ", updated=" @@ -112,7 +111,7 @@ public class BonsaiValue implements TrieLog.LogTuple { if (o == null || getClass() != o.getClass()) { return false; } - BonsaiValue that = (BonsaiValue) o; + DiffBasedValue that = (DiffBasedValue) o; return new EqualsBuilder() .append(lastStepCleared, that.lastStepCleared) .append(prior, that.prior) @@ -129,7 +128,7 @@ public class BonsaiValue implements TrieLog.LogTuple { .toHashCode(); } - public BonsaiValue copy() { - return new BonsaiValue(prior, updated, lastStepCleared, clearedAtLeastOnce); + public DiffBasedValue copy() { + return new DiffBasedValue(prior, updated, lastStepCleared, clearedAtLeastOnce); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedWorldStateProvider.java similarity index 62% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedWorldStateProvider.java index 0230a2258..18bf33382 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedWorldStateProvider.java @@ -14,9 +14,7 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai; - -import static org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager.RETAINED_LAYERS; +package org.hyperledger.besu.ethereum.trie.diffbased.common; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; @@ -25,91 +23,75 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.proof.WorldStateProof; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; -import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; -import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; -import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.worldstate.WorldState; import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.function.Function; -import com.google.common.annotations.VisibleForTesting; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.units.bigints.UInt256; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class BonsaiWorldStateProvider implements WorldStateArchive { +public abstract class DiffBasedWorldStateProvider implements WorldStateArchive { - private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateProvider.class); + private static final Logger LOG = LoggerFactory.getLogger(DiffBasedWorldStateProvider.class); - private final Blockchain blockchain; + protected final Blockchain blockchain; - private final CachedWorldStorageManager cachedWorldStorageManager; - private final TrieLogManager trieLogManager; - private final BonsaiWorldState persistedState; - private final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage; - private final CachedMerkleTrieLoader cachedMerkleTrieLoader; + protected final TrieLogManager trieLogManager; + protected DiffBasedCachedWorldStorageManager cachedWorldStorageManager; + protected DiffBasedWorldState persistedState; - public BonsaiWorldStateProvider( - final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, + protected final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage; + + public DiffBasedWorldStateProvider( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, final Blockchain blockchain, final Optional maxLayersToLoad, - final CachedMerkleTrieLoader cachedMerkleTrieLoader, - final BesuContext pluginContext, - final EvmConfiguration evmConfiguration) { + final BesuContext pluginContext) { this.worldStateKeyValueStorage = worldStateKeyValueStorage; - this.cachedWorldStorageManager = new CachedWorldStorageManager(this, worldStateKeyValueStorage); - // TODO: de-dup constructors this.trieLogManager = new TrieLogManager( blockchain, worldStateKeyValueStorage, - maxLayersToLoad.orElse(RETAINED_LAYERS), + maxLayersToLoad.orElse(DiffBasedCachedWorldStorageManager.RETAINED_LAYERS), pluginContext); this.blockchain = blockchain; - this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; - this.persistedState = new BonsaiWorldState(this, worldStateKeyValueStorage, evmConfiguration); - blockchain - .getBlockHeader(persistedState.getWorldStateBlockHash()) - .ifPresent( - blockHeader -> - this.cachedWorldStorageManager.addCachedLayer( - blockHeader, persistedState.getWorldStateRootHash(), persistedState)); } - @VisibleForTesting - BonsaiWorldStateProvider( - final CachedWorldStorageManager cachedWorldStorageManager, - final TrieLogManager trieLogManager, - final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, + public DiffBasedWorldStateProvider( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, final Blockchain blockchain, - final CachedMerkleTrieLoader cachedMerkleTrieLoader, - final EvmConfiguration evmConfiguration) { - this.cachedWorldStorageManager = cachedWorldStorageManager; + final TrieLogManager trieLogManager) { + + this.worldStateKeyValueStorage = worldStateKeyValueStorage; + // TODO: de-dup constructors this.trieLogManager = trieLogManager; this.blockchain = blockchain; - this.worldStateKeyValueStorage = worldStateKeyValueStorage; - this.persistedState = new BonsaiWorldState(this, worldStateKeyValueStorage, evmConfiguration); - this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; + } + + protected void provideCachedWorldStorageManager( + final DiffBasedCachedWorldStorageManager cachedWorldStorageManager) { + this.cachedWorldStorageManager = cachedWorldStorageManager; + } + + protected void loadPersistedState(final DiffBasedWorldState persistedState) { + this.persistedState = persistedState; blockchain .getBlockHeader(persistedState.getWorldStateBlockHash()) .ifPresent( @@ -158,9 +140,7 @@ public class BonsaiWorldStateProvider implements WorldStateArchive { .getWorldState(blockHeader.getHash()) .or(() -> cachedWorldStorageManager.getNearestWorldState(blockHeader)) .or(() -> cachedWorldStorageManager.getHeadWorldState(blockchain::getBlockHeader)) - .flatMap( - bonsaiWorldState -> - rollMutableStateToBlockHash(bonsaiWorldState, blockHeader.getHash())) + .flatMap(worldState -> rollMutableStateToBlockHash(worldState, blockHeader.getHash())) .map(MutableWorldState::freeze); } } @@ -172,7 +152,7 @@ public class BonsaiWorldStateProvider implements WorldStateArchive { } Optional rollMutableStateToBlockHash( - final BonsaiWorldState mutableState, final Hash blockHash) { + final DiffBasedWorldState mutableState, final Hash blockHash) { if (blockHash.equals(mutableState.blockHash())) { return Optional.of(mutableState); } else { @@ -221,19 +201,19 @@ public class BonsaiWorldStateProvider implements WorldStateArchive { } // attempt the state rolling - final BonsaiWorldStateUpdateAccumulator bonsaiUpdater = - (BonsaiWorldStateUpdateAccumulator) mutableState.updater(); + final DiffBasedWorldStateUpdateAccumulator diffBasedUpdater = + (DiffBasedWorldStateUpdateAccumulator) mutableState.updater(); try { for (final TrieLog rollBack : rollBacks) { LOG.debug("Attempting Rollback of {}", rollBack.getBlockHash()); - bonsaiUpdater.rollBack(rollBack); + diffBasedUpdater.rollBack(rollBack); } for (int i = rollForwards.size() - 1; i >= 0; i--) { final var forward = rollForwards.get(i); LOG.debug("Attempting Rollforward of {}", rollForwards.get(i).getBlockHash()); - bonsaiUpdater.rollForward(forward); + diffBasedUpdater.rollForward(forward); } - bonsaiUpdater.commit(); + diffBasedUpdater.commit(); mutableState.persist(blockchain.getBlockHeader(blockHash).get()); @@ -247,7 +227,7 @@ public class BonsaiWorldStateProvider implements WorldStateArchive { throw re; } catch (final Exception e) { // if we fail we must clean up the updater - bonsaiUpdater.reset(); + diffBasedUpdater.reset(); LOG.debug( "State rolling failed on " + mutableState.getWorldStateStorage().getClass().getSimpleName() @@ -269,80 +249,16 @@ public class BonsaiWorldStateProvider implements WorldStateArchive { } } - public CachedMerkleTrieLoader getCachedMerkleTrieLoader() { - return cachedMerkleTrieLoader; - } - @Override public MutableWorldState getMutable() { return persistedState; } - /** - * Prepares the state healing process for a given address and location. It prepares the state - * healing, including retrieving data from storage, identifying invalid slots or nodes, removing - * account and slot from the state trie, and committing the changes. Finally, it downgrades the - * world state storage to partial flat database mode. - */ - public void prepareStateHealing(final Address address, final Bytes location) { - final Set keysToDelete = new HashSet<>(); - final BonsaiWorldStateKeyValueStorage.Updater updater = worldStateKeyValueStorage.updater(); - final Hash accountHash = address.addressHash(); - final StoredMerklePatriciaTrie accountTrie = - new StoredMerklePatriciaTrie<>( - (l, h) -> { - final Optional node = worldStateKeyValueStorage.getAccountStateTrieNode(l, h); - if (node.isPresent()) { - keysToDelete.add(l); - } - return node; - }, - persistedState.getWorldStateRootHash(), - Function.identity(), - Function.identity()); - try { - accountTrie - .get(accountHash) - .map(RLP::input) - .map(StateTrieAccountValue::readFrom) - .ifPresent( - account -> { - final StoredMerklePatriciaTrie storageTrie = - new StoredMerklePatriciaTrie<>( - (l, h) -> { - Optional node = - worldStateKeyValueStorage.getAccountStorageTrieNode( - accountHash, l, h); - if (node.isPresent()) { - keysToDelete.add(Bytes.concatenate(accountHash, l)); - } - return node; - }, - account.getStorageRoot(), - Function.identity(), - Function.identity()); - try { - storageTrie.getPath(location); - } catch (Exception eA) { - LOG.warn("Invalid slot found for account {} at location {}", address, location); - // ignore - } - }); - } catch (Exception eA) { - LOG.warn("Invalid node for account {} at location {}", address, location); - // ignore - } - keysToDelete.forEach(bytes -> updater.removeAccountStateTrieNode(bytes)); - updater.commit(); - - worldStateKeyValueStorage.downgradeToPartialFlatDbMode(); - } - public TrieLogManager getTrieLogManager() { return trieLogManager; } - public CachedWorldStorageManager getCachedWorldStorageManager() { + public DiffBasedCachedWorldStorageManager getCachedWorldStorageManager() { return cachedWorldStorageManager; } @@ -360,7 +276,8 @@ public class BonsaiWorldStateProvider implements WorldStateArchive { final Address accountAddress, final List accountStorageKeys, final Function, ? extends Optional> mapper) { - try (BonsaiWorldState ws = (BonsaiWorldState) getMutable(blockHeader, false).orElse(null)) { + try (DiffBasedWorldState ws = + (DiffBasedWorldState) getMutable(blockHeader, false).orElse(null)) { if (ws != null) { final WorldStateProofProvider worldStateProofProvider = new WorldStateProofProvider( diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggersNegativeCases.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/StorageSubscriber.java similarity index 67% rename from errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggersNegativeCases.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/StorageSubscriber.java index 99d8986ac..fe16cf381 100644 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggersNegativeCases.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/StorageSubscriber.java @@ -1,5 +1,5 @@ /* - * Copyright ConsenSys AG. + * Copyright Hyperledger Besu Contributors. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -12,14 +12,14 @@ * * SPDX-License-Identifier: Apache-2.0 */ +package org.hyperledger.besu.ethereum.trie.diffbased.common; -package org.hyperledger.errorpronechecks; +public interface StorageSubscriber { + default void onClearStorage() {} -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + default void onClearFlatDatabaseStorage() {} -public class PrivateStaticFinalLoggersNegativeCases { + default void onClearTrieLog() {} - private static final Logger LOG = - LoggerFactory.getLogger(PrivateStaticFinalLoggersNegativeCases.class); + default void onCloseStorage() {} } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldStorageManager.java similarity index 62% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedWorldStorageManager.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldStorageManager.java index 3944b72a7..287213aa5 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedWorldStorageManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldStorageManager.java @@ -12,15 +12,15 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.cache; +package org.hyperledger.besu.ethereum.trie.diffbased.common.cache; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateLayerStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedLayeredWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; import org.hyperledger.besu.evm.internal.EvmConfiguration; import java.util.ArrayList; @@ -35,20 +35,20 @@ import org.apache.tuweni.bytes.Bytes32; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class CachedWorldStorageManager - implements BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber { +public abstract class DiffBasedCachedWorldStorageManager implements StorageSubscriber { public static final long RETAINED_LAYERS = 512; // at least 256 + typical rollbacks - private static final Logger LOG = LoggerFactory.getLogger(CachedWorldStorageManager.class); - private final BonsaiWorldStateProvider archive; + private static final Logger LOG = + LoggerFactory.getLogger(DiffBasedCachedWorldStorageManager.class); + private final DiffBasedWorldStateProvider archive; private final EvmConfiguration evmConfiguration; - private final BonsaiWorldStateKeyValueStorage rootWorldStateStorage; - private final Map cachedWorldStatesByHash; + private final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage; + private final Map cachedWorldStatesByHash; - private CachedWorldStorageManager( - final BonsaiWorldStateProvider archive, - final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, - final Map cachedWorldStatesByHash, + private DiffBasedCachedWorldStorageManager( + final DiffBasedWorldStateProvider archive, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, + final Map cachedWorldStatesByHash, final EvmConfiguration evmConfiguration) { worldStateKeyValueStorage.subscribe(this); this.rootWorldStateStorage = worldStateKeyValueStorage; @@ -57,32 +57,32 @@ public class CachedWorldStorageManager this.evmConfiguration = evmConfiguration; } - public CachedWorldStorageManager( - final BonsaiWorldStateProvider archive, - final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage) { + public DiffBasedCachedWorldStorageManager( + final DiffBasedWorldStateProvider archive, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage) { this(archive, worldStateKeyValueStorage, new ConcurrentHashMap<>(), EvmConfiguration.DEFAULT); } public synchronized void addCachedLayer( final BlockHeader blockHeader, final Hash worldStateRootHash, - final BonsaiWorldState forWorldState) { - final Optional cachedBonsaiWorldView = + final DiffBasedWorldState forWorldState) { + final Optional cachedDiffBasedWorldView = Optional.ofNullable(this.cachedWorldStatesByHash.get(blockHeader.getBlockHash())); - if (cachedBonsaiWorldView.isPresent()) { + if (cachedDiffBasedWorldView.isPresent()) { // only replace if it is a layered storage if (forWorldState.isPersisted() - && cachedBonsaiWorldView.get().getWorldStateStorage() - instanceof BonsaiWorldStateLayerStorage) { + && cachedDiffBasedWorldView.get().getWorldStateStorage() + instanceof DiffBasedLayeredWorldStateKeyValueStorage) { LOG.atDebug() .setMessage("updating layered world state for block {}, state root hash {}") .addArgument(blockHeader::toLogString) .addArgument(worldStateRootHash::toShortHexString) .log(); - cachedBonsaiWorldView + cachedDiffBasedWorldView .get() .updateWorldStateStorage( - new BonsaiSnapshotWorldStateKeyValueStorage(forWorldState.getWorldStateStorage())); + createSnapshotKeyValueStorage(forWorldState.getWorldStateStorage())); } } else { LOG.atDebug() @@ -93,16 +93,16 @@ public class CachedWorldStorageManager if (forWorldState.isPersisted()) { cachedWorldStatesByHash.put( blockHeader.getHash(), - new CachedBonsaiWorldView( - blockHeader, - new BonsaiSnapshotWorldStateKeyValueStorage(forWorldState.getWorldStateStorage()))); + new DiffBasedCachedWorldView( + blockHeader, createSnapshotKeyValueStorage(forWorldState.getWorldStateStorage()))); } else { // otherwise, add the layer to the cache cachedWorldStatesByHash.put( blockHeader.getHash(), - new CachedBonsaiWorldView( + new DiffBasedCachedWorldView( blockHeader, - ((BonsaiWorldStateLayerStorage) forWorldState.getWorldStateStorage()).clone())); + ((DiffBasedLayeredWorldStateKeyValueStorage) forWorldState.getWorldStateStorage()) + .clone())); } } scrubCachedLayers(blockHeader.getNumber()); @@ -122,15 +122,15 @@ public class CachedWorldStorageManager } } - public Optional getWorldState(final Hash blockHash) { + public Optional getWorldState(final Hash blockHash) { if (cachedWorldStatesByHash.containsKey(blockHash)) { // return a new worldstate using worldstate storage and an isolated copy of the updater return Optional.ofNullable(cachedWorldStatesByHash.get(blockHash)) .map( cached -> - new BonsaiWorldState( + createWorldState( archive, - new BonsaiWorldStateLayerStorage(cached.getWorldStateStorage()), + createLayeredKeyValueStorage(cached.getWorldStateStorage()), evmConfiguration)); } LOG.atDebug() @@ -141,7 +141,7 @@ public class CachedWorldStorageManager return Optional.empty(); } - public Optional getNearestWorldState(final BlockHeader blockHeader) { + public Optional getNearestWorldState(final BlockHeader blockHeader) { LOG.atDebug() .setMessage("getting nearest worldstate for {}") .addArgument(blockHeader.toLogString()) @@ -149,7 +149,7 @@ public class CachedWorldStorageManager return Optional.ofNullable( cachedWorldStatesByHash.get(blockHeader.getParentHash())) // search parent block - .map(CachedBonsaiWorldView::getWorldStateStorage) + .map(DiffBasedCachedWorldView::getWorldStateStorage) .or( () -> { // or else search the nearest state in the cache @@ -158,22 +158,22 @@ public class CachedWorldStorageManager .addArgument(blockHeader.toLogString()) .log(); - final List cachedBonsaiWorldViews = + final List cachedDiffBasedWorldViews = new ArrayList<>(cachedWorldStatesByHash.values()); - return cachedBonsaiWorldViews.stream() + return cachedDiffBasedWorldViews.stream() .sorted( Comparator.comparingLong( view -> Math.abs(blockHeader.getNumber() - view.getBlockNumber()))) - .map(CachedBonsaiWorldView::getWorldStateStorage) + .map(DiffBasedCachedWorldView::getWorldStateStorage) .findFirst(); }) .map( storage -> - new BonsaiWorldState( // wrap the state in a layered worldstate - archive, new BonsaiWorldStateLayerStorage(storage), evmConfiguration)); + createWorldState( // wrap the state in a layered worldstate + archive, createLayeredKeyValueStorage(storage), evmConfiguration)); } - public Optional getHeadWorldState( + public Optional getHeadWorldState( final Function> hashBlockHeaderFunction) { LOG.atDebug().setMessage("getting head worldstate").log(); @@ -187,7 +187,7 @@ public class CachedWorldStorageManager addCachedLayer( blockHeader, blockHeader.getStateRoot(), - new BonsaiWorldState(archive, rootWorldStateStorage, evmConfiguration)); + createWorldState(archive, rootWorldStateStorage, evmConfiguration)); return getWorldState(blockHeader.getHash()); }); } @@ -219,4 +219,15 @@ public class CachedWorldStorageManager public void onCloseStorage() { this.cachedWorldStatesByHash.clear(); } + + public abstract DiffBasedWorldState createWorldState( + final DiffBasedWorldStateProvider archive, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, + final EvmConfiguration evmConfiguration); + + public abstract DiffBasedWorldStateKeyValueStorage createLayeredKeyValueStorage( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage); + + public abstract DiffBasedWorldStateKeyValueStorage createSnapshotKeyValueStorage( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedBonsaiWorldView.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldView.java similarity index 71% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedBonsaiWorldView.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldView.java index 2b8fd69d1..951941f0c 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedBonsaiWorldView.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldView.java @@ -13,30 +13,30 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.cache; +package org.hyperledger.besu.ethereum.trie.diffbased.common.cache; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class CachedBonsaiWorldView - implements BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber { - private BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage; +public class DiffBasedCachedWorldView implements StorageSubscriber { + private DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage; private final BlockHeader blockHeader; private long worldViewSubscriberId; - private static final Logger LOG = LoggerFactory.getLogger(CachedBonsaiWorldView.class); + private static final Logger LOG = LoggerFactory.getLogger(DiffBasedCachedWorldView.class); - public CachedBonsaiWorldView( - final BlockHeader blockHeader, final BonsaiWorldStateKeyValueStorage worldView) { + public DiffBasedCachedWorldView( + final BlockHeader blockHeader, final DiffBasedWorldStateKeyValueStorage worldView) { this.blockHeader = blockHeader; this.worldStateKeyValueStorage = worldView; this.worldViewSubscriberId = worldStateKeyValueStorage.subscribe(this); } - public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { + public DiffBasedWorldStateKeyValueStorage getWorldStateStorage() { return worldStateKeyValueStorage; } @@ -58,10 +58,10 @@ public class CachedBonsaiWorldView } public synchronized void updateWorldStateStorage( - final BonsaiWorldStateKeyValueStorage newWorldStateStorage) { + final DiffBasedWorldStateKeyValueStorage newWorldStateStorage) { long newSubscriberId = newWorldStateStorage.subscribe(this); this.worldStateKeyValueStorage.unSubscribe(this.worldViewSubscriberId); - BonsaiWorldStateKeyValueStorage oldWorldStateStorage = this.worldStateKeyValueStorage; + final DiffBasedWorldStateKeyValueStorage oldWorldStateStorage = this.worldStateKeyValueStorage; this.worldStateKeyValueStorage = newWorldStateStorage; this.worldViewSubscriberId = newSubscriberId; try { diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/BannedMethodNegativeCases.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedLayeredWorldStateKeyValueStorage.java similarity index 65% rename from errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/BannedMethodNegativeCases.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedLayeredWorldStateKeyValueStorage.java index 5b105322e..a561072dc 100644 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/BannedMethodNegativeCases.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedLayeredWorldStateKeyValueStorage.java @@ -1,5 +1,5 @@ /* - * Copyright ConsenSys AG. + * Copyright Hyperledger Besu Contributors. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -11,19 +11,12 @@ * specific language governing permissions and limitations under the License. * * SPDX-License-Identifier: Apache-2.0 + * */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage; -package org.hyperledger.errorpronechecks; +public interface DiffBasedLayeredWorldStateKeyValueStorage + extends DiffBasedSnapshotWorldStateKeyValueStorage { -import java.util.Objects; - -public class BannedMethodNegativeCases { - - public void callsObjectsEquals() throws Exception { - Objects.equals("1", "1"); - } - - public void callsObjectsHashCode() throws Exception { - Objects.hash("1", "1"); - } + DiffBasedWorldStateKeyValueStorage clone(); } diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PreferCommonAnnotationsPositiveCases.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedSnapshotWorldStateKeyValueStorage.java similarity index 65% rename from errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PreferCommonAnnotationsPositiveCases.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedSnapshotWorldStateKeyValueStorage.java index 55760df18..9c1a691cf 100644 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PreferCommonAnnotationsPositiveCases.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedSnapshotWorldStateKeyValueStorage.java @@ -1,5 +1,5 @@ /* - * Copyright Hyperledger Besu contributors. + * Copyright Hyperledger Besu Contributors. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -11,17 +11,11 @@ * specific language governing permissions and limitations under the License. * * SPDX-License-Identifier: Apache-2.0 + * */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage; -package org.hyperledger.errorpronechecks; +public interface DiffBasedSnapshotWorldStateKeyValueStorage { -// BUG: Diagnostic contains: Do not use org.jetbrains.annotations.NotNull use -import org.jetbrains.annotations.NotNull; - -public class PreferCommonAnnotationsPositiveCases { - - @NotNull - public String getFoo() { - return "Foo"; - } + DiffBasedWorldStateKeyValueStorage getParentWorldStateStorage(); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedWorldStateKeyValueStorage.java new file mode 100644 index 000000000..2424828ee --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedWorldStateKeyValueStorage.java @@ -0,0 +1,234 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage; + +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.storage.StorageProvider; +import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategy; +import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; +import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; +import org.hyperledger.besu.util.Subscribers; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Stream; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class DiffBasedWorldStateKeyValueStorage + implements WorldStateKeyValueStorage, AutoCloseable { + private static final Logger LOG = + LoggerFactory.getLogger(DiffBasedWorldStateKeyValueStorage.class); + + // 0x776f726c64526f6f74 + public static final byte[] WORLD_ROOT_HASH_KEY = "worldRoot".getBytes(StandardCharsets.UTF_8); + // 0x776f726c64426c6f636b48617368 + public static final byte[] WORLD_BLOCK_HASH_KEY = + "worldBlockHash".getBytes(StandardCharsets.UTF_8); + + private final AtomicBoolean shouldClose = new AtomicBoolean(false); + + protected final AtomicBoolean isClosed = new AtomicBoolean(false); + + protected final Subscribers subscribers = Subscribers.create(); + protected final SegmentedKeyValueStorage composedWorldStateStorage; + protected final KeyValueStorage trieLogStorage; + + public DiffBasedWorldStateKeyValueStorage(final StorageProvider provider) { + this.composedWorldStateStorage = + provider.getStorageBySegmentIdentifiers( + List.of( + ACCOUNT_INFO_STATE, CODE_STORAGE, ACCOUNT_STORAGE_STORAGE, TRIE_BRANCH_STORAGE)); + this.trieLogStorage = + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE); + } + + public DiffBasedWorldStateKeyValueStorage( + final SegmentedKeyValueStorage composedWorldStateStorage, + final KeyValueStorage trieLogStorage) { + this.composedWorldStateStorage = composedWorldStateStorage; + this.trieLogStorage = trieLogStorage; + } + + public abstract FlatDbMode getFlatDbMode(); + + public abstract FlatDbStrategy getFlatDbStrategy(); + + @Override + public abstract DataStorageFormat getDataStorageFormat(); + + public SegmentedKeyValueStorage getComposedWorldStateStorage() { + return composedWorldStateStorage; + } + + public KeyValueStorage getTrieLogStorage() { + return trieLogStorage; + } + + public Optional getTrieLog(final Hash blockHash) { + return trieLogStorage.get(blockHash.toArrayUnsafe()); + } + + public Stream streamTrieLogKeys(final long limit) { + return trieLogStorage.streamKeys().limit(limit); + } + + public Optional getStateTrieNode(final Bytes location) { + return composedWorldStateStorage + .get(TRIE_BRANCH_STORAGE, location.toArrayUnsafe()) + .map(Bytes::wrap); + } + + public Optional getWorldStateRootHash() { + return composedWorldStateStorage.get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY).map(Bytes::wrap); + } + + public Optional getWorldStateBlockHash() { + return composedWorldStateStorage + .get(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY) + .map(Bytes32::wrap) + .map(Hash::wrap); + } + + public Map streamFlatAccounts( + final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { + return getFlatDbStrategy() + .streamAccountFlatDatabase(composedWorldStateStorage, startKeyHash, endKeyHash, max); + } + + public Map streamFlatStorages( + final Hash accountHash, final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { + return getFlatDbStrategy() + .streamStorageFlatDatabase( + composedWorldStateStorage, accountHash, startKeyHash, endKeyHash, max); + } + + public boolean isWorldStateAvailable(final Bytes32 rootHash, final Hash blockHash) { + return composedWorldStateStorage + .get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY) + .map(Bytes32::wrap) + .map(hash -> hash.equals(rootHash) || trieLogStorage.containsKey(blockHash.toArrayUnsafe())) + .orElse(false); + } + + @Override + public void clear() { + subscribers.forEach(StorageSubscriber::onClearStorage); + getFlatDbStrategy().clearAll(composedWorldStateStorage); + composedWorldStateStorage.clear(TRIE_BRANCH_STORAGE); + trieLogStorage.clear(); + } + + public void clearTrieLog() { + subscribers.forEach(StorageSubscriber::onClearTrieLog); + trieLogStorage.clear(); + } + + public void clearFlatDatabase() { + subscribers.forEach(StorageSubscriber::onClearFlatDatabaseStorage); + getFlatDbStrategy().resetOnResync(composedWorldStateStorage); + } + + @Override + public abstract Updater updater(); + + public boolean pruneTrieLog(final Hash blockHash) { + try { + return trieLogStorage.tryDelete(blockHash.toArrayUnsafe()); + } catch (Exception e) { + LOG.error("Error pruning trie log for block hash {}", blockHash, e); + return false; + } + } + + @Override + public synchronized void close() throws Exception { + // when the storage clears, close + shouldClose.set(true); + tryClose(); + } + + public synchronized long subscribe(final StorageSubscriber sub) { + if (isClosed.get()) { + throw new RuntimeException("Storage is marked to close or has already closed"); + } + return subscribers.subscribe(sub); + } + + public synchronized void unSubscribe(final long id) { + subscribers.unsubscribe(id); + try { + tryClose(); + } catch (Exception e) { + LOG.atWarn() + .setMessage("exception while trying to close : {}") + .addArgument(e::getMessage) + .log(); + } + } + + protected synchronized void tryClose() throws Exception { + if (shouldClose.get() && subscribers.getSubscriberCount() < 1) { + doClose(); + } + } + + protected synchronized void doClose() throws Exception { + if (!isClosed.get()) { + // alert any subscribers we are closing: + subscribers.forEach(StorageSubscriber::onCloseStorage); + + // close all of the KeyValueStorages: + composedWorldStateStorage.close(); + trieLogStorage.close(); + + // set storage closed + isClosed.set(true); + } + } + + public interface Updater extends WorldStateKeyValueStorage.Updater { + + DiffBasedWorldStateKeyValueStorage.Updater saveWorldState( + final Bytes blockHash, final Bytes32 nodeHash, final Bytes node); + + SegmentedKeyValueStorageTransaction getWorldStateTransaction(); + + KeyValueStorageTransaction getTrieLogStorageTransaction(); + + @Override + void commit(); + + void rollback(); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/AccountHashCodeStorageStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/AccountHashCodeStorageStrategy.java similarity index 96% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/AccountHashCodeStorageStrategy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/AccountHashCodeStorageStrategy.java index 7d77ff1a6..446ed3262 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/AccountHashCodeStorageStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/AccountHashCodeStorageStrategy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/CodeHashCodeStorageStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/CodeHashCodeStorageStrategy.java similarity index 96% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/CodeHashCodeStorageStrategy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/CodeHashCodeStorageStrategy.java index 6f4d8bb81..4263e423b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/CodeHashCodeStorageStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/CodeHashCodeStorageStrategy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/CodeStorageStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/CodeStorageStrategy.java similarity index 94% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/CodeStorageStrategy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/CodeStorageStrategy.java index 8767be247..91da4eff0 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/CodeStorageStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/CodeStorageStrategy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategy.java similarity index 99% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategy.java index d85f49325..37c2877b1 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProvider.java similarity index 95% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProvider.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProvider.java index e251f28c1..7614c1499 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProvider.java @@ -13,11 +13,13 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.FullFlatDbStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.PartialFlatDbStrategy; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.plugin.services.MetricsSystem; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/NoOpTrieLogManager.java similarity index 79% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/NoOpTrieLogManager.java index 4c9a520bd..d74a6804d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/NoOpTrieLogManager.java @@ -12,12 +12,12 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import java.util.Optional; @@ -30,10 +30,10 @@ public class NoOpTrieLogManager extends TrieLogManager { @Override public synchronized void saveTrieLog( - final BonsaiWorldStateUpdateAccumulator localUpdater, + final DiffBasedWorldStateUpdateAccumulator localUpdater, final Hash forWorldStateRootHash, final BlockHeader forBlockHeader, - final BonsaiWorldState forWorldState) { + final DiffBasedWorldState forWorldState) { // notify trie log added observers, synchronously TrieLog trieLog = trieLogFactory.create(localUpdater, forBlockHeader); trieLogObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog))); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogAddedEvent.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogAddedEvent.java similarity index 92% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogAddedEvent.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogAddedEvent.java index 3e1387997..28edc5cc6 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogAddedEvent.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogAddedEvent.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayer.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogLayer.java similarity index 79% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayer.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogLayer.java index ff1e29c41..ff6ef226e 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayer.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogLayer.java @@ -14,7 +14,7 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import static com.google.common.base.Preconditions.checkState; @@ -22,7 +22,7 @@ import org.hyperledger.besu.datatypes.AccountValue; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import java.util.HashMap; @@ -49,21 +49,21 @@ public class TrieLogLayer implements TrieLog { protected Hash blockHash; protected Optional blockNumber = Optional.empty(); - Map> getAccounts() { + Map> getAccounts() { return accounts; } - Map> getCode() { + Map> getCode() { return code; } - Map>> getStorage() { + Map>> getStorage() { return storage; } - protected final Map> accounts; - protected final Map> code; - protected final Map>> storage; + protected final Map> accounts; + protected final Map> code; + protected final Map>> storage; protected boolean frozen = false; public TrieLogLayer() { @@ -104,14 +104,14 @@ public class TrieLogLayer implements TrieLog { public TrieLogLayer addAccountChange( final Address address, final AccountValue oldValue, final AccountValue newValue) { checkState(!frozen, "Layer is Frozen"); - accounts.put(address, new BonsaiValue<>(oldValue, newValue)); + accounts.put(address, new DiffBasedValue<>(oldValue, newValue)); return this; } public TrieLogLayer addCodeChange( final Address address, final Bytes oldValue, final Bytes newValue, final Hash blockHash) { checkState(!frozen, "Layer is Frozen"); - code.put(address, new BonsaiValue<>(oldValue, newValue, newValue == null)); + code.put(address, new DiffBasedValue<>(oldValue, newValue, newValue == null)); return this; } @@ -123,22 +123,22 @@ public class TrieLogLayer implements TrieLog { checkState(!frozen, "Layer is Frozen"); storage .computeIfAbsent(address, a -> new TreeMap<>()) - .put(slot, new BonsaiValue<>(oldValue, newValue)); + .put(slot, new DiffBasedValue<>(oldValue, newValue)); return this; } @Override - public Map> getAccountChanges() { + public Map> getAccountChanges() { return accounts; } @Override - public Map> getCodeChanges() { + public Map> getCodeChanges() { return code; } @Override - public Map>> getStorageChanges() { + public Map>> getStorageChanges() { return storage; } @@ -147,18 +147,18 @@ public class TrieLogLayer implements TrieLog { } @Override - public Map> getStorageChanges(final Address address) { + public Map> getStorageChanges(final Address address) { return storage.getOrDefault(address, Map.of()); } @Override public Optional getPriorCode(final Address address) { - return Optional.ofNullable(code.get(address)).map(BonsaiValue::getPrior); + return Optional.ofNullable(code.get(address)).map(DiffBasedValue::getPrior); } @Override public Optional getCode(final Address address) { - return Optional.ofNullable(code.get(address)).map(BonsaiValue::getUpdated); + return Optional.ofNullable(code.get(address)).map(DiffBasedValue::getUpdated); } @Override @@ -166,7 +166,7 @@ public class TrieLogLayer implements TrieLog { final Address address, final StorageSlotKey storageSlotKey) { return Optional.ofNullable(storage.get(address)) .map(i -> i.get(storageSlotKey)) - .map(BonsaiValue::getPrior); + .map(DiffBasedValue::getPrior); } @Override @@ -174,24 +174,24 @@ public class TrieLogLayer implements TrieLog { final Address address, final StorageSlotKey storageSlotKey) { return Optional.ofNullable(storage.get(address)) .map(i -> i.get(storageSlotKey)) - .map(BonsaiValue::getUpdated); + .map(DiffBasedValue::getUpdated); } @Override public Optional getPriorAccount(final Address address) { - return Optional.ofNullable(accounts.get(address)).map(BonsaiValue::getPrior); + return Optional.ofNullable(accounts.get(address)).map(DiffBasedValue::getPrior); } @Override public Optional getAccount(final Address address) { - return Optional.ofNullable(accounts.get(address)).map(BonsaiValue::getUpdated); + return Optional.ofNullable(accounts.get(address)).map(DiffBasedValue::getUpdated); } public String dump() { final StringBuilder sb = new StringBuilder(); sb.append("TrieLog{" + "blockHash=").append(blockHash).append(frozen).append('}'); sb.append("accounts\n"); - for (final Map.Entry> account : accounts.entrySet()) { + for (final Map.Entry> account : accounts.entrySet()) { sb.append(" : ").append(account.getKey()).append("\n"); if (Objects.equals(account.getValue().getPrior(), account.getValue().getUpdated())) { sb.append(" = ").append(account.getValue().getUpdated()).append("\n"); @@ -201,7 +201,7 @@ public class TrieLogLayer implements TrieLog { } } sb.append("code").append("\n"); - for (final Map.Entry> code : code.entrySet()) { + for (final Map.Entry> code : code.entrySet()) { sb.append(" : ").append(code.getKey()).append("\n"); if (Objects.equals(code.getValue().getPrior(), code.getValue().getUpdated())) { sb.append(" = ").append(code.getValue().getPrior()).append("\n"); @@ -211,10 +211,10 @@ public class TrieLogLayer implements TrieLog { } } sb.append("Storage").append("\n"); - for (final Map.Entry>> storage : + for (final Map.Entry>> storage : storage.entrySet()) { sb.append(" : ").append(storage.getKey()).append("\n"); - for (final Map.Entry> slot : + for (final Map.Entry> slot : storage.getValue().entrySet()) { final UInt256 originalValue = slot.getValue().getPrior(); final UInt256 updatedValue = slot.getValue().getUpdated(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogManager.java similarity index 87% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManager.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogManager.java index 19ac6285f..5233cda6f 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogManager.java @@ -13,14 +13,15 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.services.TrieLogService; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; @@ -41,7 +42,7 @@ public class TrieLogManager { private static final Logger LOG = LoggerFactory.getLogger(TrieLogManager.class); public static final long LOG_RANGE_LIMIT = 1000; // restrict trielog range queries to 1k logs protected final Blockchain blockchain; - protected final BonsaiWorldStateKeyValueStorage rootWorldStateStorage; + protected final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage; protected final long maxLayersToLoad; protected final Subscribers trieLogObservers = Subscribers.create(); @@ -50,7 +51,7 @@ public class TrieLogManager { public TrieLogManager( final Blockchain blockchain, - final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, final long maxLayersToLoad, final BesuContext pluginContext) { this.blockchain = blockchain; @@ -60,15 +61,15 @@ public class TrieLogManager { } public synchronized void saveTrieLog( - final BonsaiWorldStateUpdateAccumulator localUpdater, + final DiffBasedWorldStateUpdateAccumulator localUpdater, final Hash forWorldStateRootHash, final BlockHeader forBlockHeader, - final BonsaiWorldState forWorldState) { + final DiffBasedWorldState forWorldState) { // do not overwrite a trielog layer that already exists in the database. // if it's only in memory we need to save it // for example, in case of reorg we don't replace a trielog layer if (rootWorldStateStorage.getTrieLog(forBlockHeader.getHash()).isEmpty()) { - final BonsaiWorldStateKeyValueStorage.Updater stateUpdater = + final DiffBasedWorldStateKeyValueStorage.Updater stateUpdater = forWorldState.getWorldStateStorage().updater(); boolean success = false; try { @@ -90,7 +91,7 @@ public class TrieLogManager { } private TrieLog prepareTrieLog( - final BlockHeader blockHeader, final BonsaiWorldStateUpdateAccumulator localUpdater) { + final BlockHeader blockHeader, final DiffBasedWorldStateUpdateAccumulator localUpdater) { LOG.atDebug() .setMessage("Adding layered world state for {}") .addArgument(blockHeader::toLogString) @@ -104,7 +105,7 @@ public class TrieLogManager { final BlockHeader blockHeader, final Hash worldStateRootHash, final TrieLog trieLog, - final BonsaiWorldStateKeyValueStorage.Updater stateUpdater) { + final DiffBasedWorldStateKeyValueStorage.Updater stateUpdater) { LOG.atDebug() .setMessage("Persisting trie log for block hash {} and world state root {}") .addArgument(blockHeader::toLogString) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java similarity index 94% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java index b72796f22..60a728b3f 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java @@ -13,13 +13,13 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent; import java.util.Comparator; @@ -41,7 +41,7 @@ public class TrieLogPruner implements TrieLogEvent.TrieLogObserver { private final int pruningLimit; private final int loadingLimit; - private final BonsaiWorldStateKeyValueStorage rootWorldStateStorage; + private final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage; private final Blockchain blockchain; private final Consumer executeAsync; private final long numBlocksToRetain; @@ -51,7 +51,7 @@ public class TrieLogPruner implements TrieLogEvent.TrieLogObserver { TreeMultimap.create(Comparator.reverseOrder(), Comparator.naturalOrder()); public TrieLogPruner( - final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage, final Blockchain blockchain, final Consumer executeAsync, final long numBlocksToRetain, @@ -100,7 +100,7 @@ public class TrieLogPruner implements TrieLogEvent.TrieLogObserver { } } - void addToPruneQueue(final long blockNumber, final Hash blockHash) { + public void addToPruneQueue(final long blockNumber, final Hash blockHash) { LOG.atTrace() .setMessage("adding trie log to queue for later pruning blockNumber {}; blockHash {}") .addArgument(blockNumber) @@ -109,7 +109,7 @@ public class TrieLogPruner implements TrieLogEvent.TrieLogObserver { trieLogBlocksAndForksByDescendingBlockNumber.put(blockNumber, blockHash); } - int pruneFromQueue() { + public int pruneFromQueue() { final long retainAboveThisBlock = blockchain.getChainHeadBlockNumber() - numBlocksToRetain; final Optional finalized = blockchain.getFinalized(); if (requireFinalizedBlock && finalized.isEmpty()) { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java new file mode 100644 index 000000000..7a909cbf8 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java @@ -0,0 +1,334 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview; + +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; +import static org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; +import static org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedLayeredWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedSnapshotWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; +import org.hyperledger.besu.evm.account.Account; +import org.hyperledger.besu.evm.worldstate.WorldUpdater; +import org.hyperledger.besu.plugin.services.exception.StorageException; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; +import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; + +import java.util.Optional; +import java.util.stream.Stream; +import javax.annotation.Nonnull; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class DiffBasedWorldState + implements MutableWorldState, DiffBasedWorldView, StorageSubscriber { + + private static final Logger LOG = LoggerFactory.getLogger(DiffBasedWorldState.class); + + protected DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage; + protected final DiffBasedCachedWorldStorageManager cachedWorldStorageManager; + protected final TrieLogManager trieLogManager; + protected DiffBasedWorldStateUpdateAccumulator accumulator; + + protected Hash worldStateRootHash; + protected Hash worldStateBlockHash; + protected boolean isFrozen; + + protected DiffBasedWorldState( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, + final DiffBasedCachedWorldStorageManager cachedWorldStorageManager, + final TrieLogManager trieLogManager) { + this.worldStateKeyValueStorage = worldStateKeyValueStorage; + this.worldStateRootHash = + Hash.wrap( + Bytes32.wrap( + worldStateKeyValueStorage.getWorldStateRootHash().orElse(getEmptyTrieHash()))); + this.worldStateBlockHash = + Hash.wrap( + Bytes32.wrap(worldStateKeyValueStorage.getWorldStateBlockHash().orElse(Hash.ZERO))); + this.cachedWorldStorageManager = cachedWorldStorageManager; + this.trieLogManager = trieLogManager; + } + + /** + * Having a protected method to override the accumulator solves the chicken-egg problem of needing + * a worldstate reference (this) when construction the Accumulator. + * + * @param accumulator accumulator to use. + */ + public void setAccumulator(final DiffBasedWorldStateUpdateAccumulator accumulator) { + this.accumulator = accumulator; + } + + /** + * Returns the world state block hash of this world state + * + * @return the world state block hash. + */ + public Hash getWorldStateBlockHash() { + return worldStateBlockHash; + } + + /** + * Returns the world state root hash of this world state + * + * @return the world state root hash. + */ + public Hash getWorldStateRootHash() { + return worldStateRootHash; + } + + @Override + public boolean isPersisted() { + return isPersisted(worldStateKeyValueStorage); + } + + private boolean isPersisted(final WorldStateKeyValueStorage worldStateKeyValueStorage) { + return !(worldStateKeyValueStorage instanceof DiffBasedSnapshotWorldStateKeyValueStorage); + } + + /** + * Reset the worldState to this block header + * + * @param blockHeader block to use + */ + public void resetWorldStateTo(final BlockHeader blockHeader) { + worldStateBlockHash = blockHeader.getBlockHash(); + worldStateRootHash = blockHeader.getStateRoot(); + } + + @Override + public DiffBasedWorldStateKeyValueStorage getWorldStateStorage() { + return worldStateKeyValueStorage; + } + + public DiffBasedWorldStateUpdateAccumulator getAccumulator() { + return accumulator; + } + + @Override + public void persist(final BlockHeader blockHeader) { + final Optional maybeBlockHeader = Optional.ofNullable(blockHeader); + LOG.atDebug() + .setMessage("Persist world state for block {}") + .addArgument(maybeBlockHeader) + .log(); + + final DiffBasedWorldStateUpdateAccumulator localCopy = accumulator.copy(); + + boolean success = false; + + final DiffBasedWorldStateKeyValueStorage.Updater stateUpdater = + worldStateKeyValueStorage.updater(); + Runnable saveTrieLog = () -> {}; + + try { + final Hash newWorldStateRootHash = + calculateRootHash(isFrozen ? Optional.empty() : Optional.of(stateUpdater), accumulator); + // if we are persisted with a block header, and the prior state is the parent + // then persist the TrieLog for that transition. + // If specified but not a direct descendant simply store the new block hash. + if (blockHeader != null) { + verifyWorldStateRoot(newWorldStateRootHash, blockHeader); + saveTrieLog = + () -> { + trieLogManager.saveTrieLog(localCopy, newWorldStateRootHash, blockHeader, this); + // not save a frozen state in the cache + if (!isFrozen) { + cachedWorldStorageManager.addCachedLayer(blockHeader, newWorldStateRootHash, this); + } + }; + + stateUpdater + .getWorldStateTransaction() + .put(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY, blockHeader.getHash().toArrayUnsafe()); + worldStateBlockHash = blockHeader.getHash(); + } else { + stateUpdater.getWorldStateTransaction().remove(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY); + worldStateBlockHash = null; + } + + stateUpdater + .getWorldStateTransaction() + .put(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY, newWorldStateRootHash.toArrayUnsafe()); + worldStateRootHash = newWorldStateRootHash; + success = true; + } finally { + if (success) { + stateUpdater.commit(); + accumulator.reset(); + saveTrieLog.run(); + } else { + stateUpdater.rollback(); + accumulator.reset(); + } + } + } + + protected void verifyWorldStateRoot(final Hash calculatedStateRoot, final BlockHeader header) { + if (!calculatedStateRoot.equals(header.getStateRoot())) { + throw new RuntimeException( + "World State Root does not match expected value, header " + + header.getStateRoot().toHexString() + + " calculated " + + calculatedStateRoot.toHexString()); + } + } + + @Override + public WorldUpdater updater() { + return accumulator; + } + + @Override + public Hash rootHash() { + if (isFrozen && accumulator.isAccumulatorStateChanged()) { + worldStateRootHash = calculateRootHash(Optional.empty(), accumulator.copy()); + accumulator.resetAccumulatorStateChanged(); + } + return Hash.wrap(worldStateRootHash); + } + + protected static final KeyValueStorageTransaction noOpTx = + new KeyValueStorageTransaction() { + + @Override + public void put(final byte[] key, final byte[] value) { + // no-op + } + + @Override + public void remove(final byte[] key) { + // no-op + } + + @Override + public void commit() throws StorageException { + // no-op + } + + @Override + public void rollback() { + // no-op + } + }; + + protected static final SegmentedKeyValueStorageTransaction noOpSegmentedTx = + new SegmentedKeyValueStorageTransaction() { + + @Override + public void put( + final SegmentIdentifier segmentIdentifier, final byte[] key, final byte[] value) { + // no-op + } + + @Override + public void remove(final SegmentIdentifier segmentIdentifier, final byte[] key) { + // no-op + } + + @Override + public void commit() throws StorageException { + // no-op + } + + @Override + public void rollback() { + // no-op + } + }; + + public Hash blockHash() { + return worldStateBlockHash; + } + + @Override + public Stream streamAccounts(final Bytes32 startKeyHash, final int limit) { + throw new RuntimeException("storage format do not provide account streaming."); + } + + @Override + public UInt256 getPriorStorageValue(final Address address, final UInt256 storageKey) { + return getStorageValue(address, storageKey); + } + + @Override + public void close() { + try { + if (!isPersisted()) { + this.worldStateKeyValueStorage.close(); + if (isFrozen) { + closeFrozenStorage(); + } + } + } catch (Exception e) { + // no op + } + } + + private void closeFrozenStorage() { + try { + final DiffBasedLayeredWorldStateKeyValueStorage worldStateLayerStorage = + (DiffBasedLayeredWorldStateKeyValueStorage) worldStateKeyValueStorage; + if (!isPersisted(worldStateLayerStorage.getParentWorldStateStorage())) { + worldStateLayerStorage.getParentWorldStateStorage().close(); + } + } catch (Exception e) { + // no op + } + } + + @Override + public abstract Hash frontierRootHash(); + + @Override + public abstract MutableWorldState freeze(); + + @Override + public abstract Account get(final Address address); + + @Override + public abstract UInt256 getStorageValue(final Address address, final UInt256 storageKey); + + @Override + public abstract Optional getStorageValueByStorageSlotKey( + final Address address, final StorageSlotKey storageSlotKey); + + @Override + public abstract Optional getCode(@Nonnull final Address address, final Hash codeHash); + + protected abstract Hash calculateRootHash( + final Optional maybeStateUpdater, + final DiffBasedWorldStateUpdateAccumulator worldStateUpdater); + + protected abstract Hash getEmptyTrieHash(); +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldView.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldView.java similarity index 85% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldView.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldView.java index cd16e7dec..626584aa2 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldView.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldView.java @@ -1,5 +1,5 @@ /* - * Copyright ConsenSys AG. + * Copyright Hyperledger Besu Contributors. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -11,16 +11,15 @@ * specific language governing permissions and limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - * */ -package org.hyperledger.besu.ethereum.trie.bonsai.worldview; +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.evm.worldstate.WorldView; @@ -31,7 +30,7 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.units.bigints.UInt256; -public interface BonsaiWorldView extends WorldView { +public interface DiffBasedWorldView extends WorldView { Optional getCode(Address address, final Hash codeHash); @@ -59,7 +58,7 @@ public interface BonsaiWorldView extends WorldView { boolean isPersisted(); - BonsaiWorldStateKeyValueStorage getWorldStateStorage(); + DiffBasedWorldStateKeyValueStorage getWorldStateStorage(); WorldUpdater updater(); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java similarity index 71% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java index 44c18176b..66169ee85 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java @@ -1,5 +1,5 @@ /* - * Copyright ConsenSys AG. + * Copyright Hyperledger Besu Contributors. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -11,10 +11,9 @@ * specific language governing permissions and limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - * */ -package org.hyperledger.besu.ethereum.trie.bonsai.worldview; +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator; import org.hyperledger.besu.datatypes.AccountValue; import org.hyperledger.besu.datatypes.Address; @@ -23,9 +22,14 @@ import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiAccount; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiValue; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.AccountConsumingMap; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.Consumer; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.StorageConsumingMap; import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.internal.EvmConfiguration; @@ -44,42 +48,40 @@ import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.function.Function; -import javax.annotation.Nonnull; -import com.google.common.collect.ForwardingMap; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.units.bigints.UInt256; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class BonsaiWorldStateUpdateAccumulator - extends AbstractWorldUpdater - implements BonsaiWorldView, TrieLogAccumulator { +@SuppressWarnings("unchecked") +public abstract class DiffBasedWorldStateUpdateAccumulator + extends AbstractWorldUpdater + implements DiffBasedWorldView, TrieLogAccumulator { private static final Logger LOG = - LoggerFactory.getLogger(BonsaiWorldStateUpdateAccumulator.class); - protected final Consumer> accountPreloader; + LoggerFactory.getLogger(DiffBasedWorldStateUpdateAccumulator.class); + protected final Consumer> accountPreloader; protected final Consumer storagePreloader; - private final AccountConsumingMap> accountsToUpdate; - private final Map> codeToUpdate = new ConcurrentHashMap<>(); + private final AccountConsumingMap> accountsToUpdate; + private final Map> codeToUpdate = new ConcurrentHashMap<>(); private final Set

storageToClear = Collections.synchronizedSet(new HashSet<>()); protected final EvmConfiguration evmConfiguration; // storage sub mapped by _hashed_ key. This is because in self_destruct calls we need to // enumerate the old storage and delete it. Those are trie stored by hashed key by spec and the // alternative was to keep a giant pre-image cache of the entire trie. - private final Map>> + private final Map>> storageToUpdate = new ConcurrentHashMap<>(); private final Map storageKeyHashLookup = new ConcurrentHashMap<>(); protected boolean isAccumulatorStateChanged; - public BonsaiWorldStateUpdateAccumulator( - final BonsaiWorldView world, - final Consumer> accountPreloader, + public DiffBasedWorldStateUpdateAccumulator( + final DiffBasedWorldView world, + final Consumer> accountPreloader, final Consumer storagePreloader, final EvmConfiguration evmConfiguration) { super(world, evmConfiguration); @@ -90,15 +92,7 @@ public class BonsaiWorldStateUpdateAccumulator this.evmConfiguration = evmConfiguration; } - public BonsaiWorldStateUpdateAccumulator copy() { - final BonsaiWorldStateUpdateAccumulator copy = - new BonsaiWorldStateUpdateAccumulator( - wrappedWorldView(), accountPreloader, storagePreloader, evmConfiguration); - copy.cloneFromUpdater(this); - return copy; - } - - void cloneFromUpdater(final BonsaiWorldStateUpdateAccumulator source) { + protected void cloneFromUpdater(final DiffBasedWorldStateUpdateAccumulator source) { accountsToUpdate.putAll(source.getAccountsToUpdate()); codeToUpdate.putAll(source.codeToUpdate); storageToClear.addAll(source.storageToClear); @@ -108,14 +102,25 @@ public class BonsaiWorldStateUpdateAccumulator this.isAccumulatorStateChanged = true; } + protected Consumer> getAccountPreloader() { + return accountPreloader; + } + + protected Consumer getStoragePreloader() { + return storagePreloader; + } + + protected EvmConfiguration getEvmConfiguration() { + return evmConfiguration; + } + @Override public Account get(final Address address) { return super.get(address); } @Override - protected UpdateTrackingAccount track( - final UpdateTrackingAccount account) { + protected UpdateTrackingAccount track(final UpdateTrackingAccount account) { return super.track(account); } @@ -126,21 +131,21 @@ public class BonsaiWorldStateUpdateAccumulator @Override public MutableAccount createAccount(final Address address, final long nonce, final Wei balance) { - BonsaiValue bonsaiValue = accountsToUpdate.get(address); + DiffBasedValue diffBasedValue = accountsToUpdate.get(address); - if (bonsaiValue == null) { - bonsaiValue = new BonsaiValue<>(null, null); - accountsToUpdate.put(address, bonsaiValue); - } else if (bonsaiValue.getUpdated() != null) { - if (bonsaiValue.getUpdated().isEmpty()) { - return track(new UpdateTrackingAccount<>(bonsaiValue.getUpdated())); + if (diffBasedValue == null) { + diffBasedValue = new DiffBasedValue<>(null, null); + accountsToUpdate.put(address, diffBasedValue); + } else if (diffBasedValue.getUpdated() != null) { + if (diffBasedValue.getUpdated().isEmpty()) { + return track(new UpdateTrackingAccount<>(diffBasedValue.getUpdated())); } else { throw new IllegalStateException("Cannot create an account when one already exists"); } } - final BonsaiAccount newAccount = - new BonsaiAccount( + final ACCOUNT newAccount = + createAccount( this, address, hashAndSaveAccountPreImage(address), @@ -149,17 +154,17 @@ public class BonsaiWorldStateUpdateAccumulator Hash.EMPTY_TRIE_HASH, Hash.EMPTY, true); - bonsaiValue.setUpdated(newAccount); + diffBasedValue.setUpdated(newAccount); return track(new UpdateTrackingAccount<>(newAccount)); } @Override - public Map> getAccountsToUpdate() { + public Map> getAccountsToUpdate() { return accountsToUpdate; } @Override - public Map> getCodeToUpdate() { + public Map> getCodeToUpdate() { return codeToUpdate; } @@ -168,40 +173,41 @@ public class BonsaiWorldStateUpdateAccumulator } @Override - public Map>> + public Map>> getStorageToUpdate() { return storageToUpdate; } @Override - protected BonsaiAccount getForMutation(final Address address) { - return loadAccount(address, BonsaiValue::getUpdated); + protected ACCOUNT getForMutation(final Address address) { + return loadAccount(address, DiffBasedValue::getUpdated); } - protected BonsaiAccount loadAccount( - final Address address, - final Function, BonsaiAccount> bonsaiAccountFunction) { + protected ACCOUNT loadAccount( + final Address address, final Function, ACCOUNT> accountFunction) { try { - final BonsaiValue bonsaiValue = accountsToUpdate.get(address); - if (bonsaiValue == null) { + final DiffBasedValue diffBasedValue = accountsToUpdate.get(address); + if (diffBasedValue == null) { final Account account; - if (wrappedWorldView() - instanceof BonsaiWorldStateUpdateAccumulator bonsaiWorldStateUpdateAccumulator) { - account = bonsaiWorldStateUpdateAccumulator.loadAccount(address, bonsaiAccountFunction); + if (wrappedWorldView() instanceof DiffBasedWorldStateUpdateAccumulator) { + final DiffBasedWorldStateUpdateAccumulator worldStateUpdateAccumulator = + (DiffBasedWorldStateUpdateAccumulator) wrappedWorldView(); + account = worldStateUpdateAccumulator.loadAccount(address, accountFunction); } else { account = wrappedWorldView().get(address); } - if (account instanceof BonsaiAccount bonsaiAccount) { - BonsaiAccount mutableAccount = new BonsaiAccount(bonsaiAccount, this, true); - accountsToUpdate.put(address, new BonsaiValue<>(bonsaiAccount, mutableAccount)); + if (account instanceof DiffBasedAccount diffBasedAccount) { + ACCOUNT mutableAccount = copyAccount((ACCOUNT) diffBasedAccount, this, true); + accountsToUpdate.put( + address, new DiffBasedValue<>((ACCOUNT) diffBasedAccount, mutableAccount)); return mutableAccount; } else { // add the empty read in accountsToUpdate - accountsToUpdate.put(address, new BonsaiValue<>(null, null)); + accountsToUpdate.put(address, new DiffBasedValue<>(null, null)); return null; } } else { - return bonsaiAccountFunction.apply(bonsaiValue); + return accountFunction.apply(diffBasedValue); } } catch (MerkleTrieException e) { // need to throw to trigger the heal @@ -229,12 +235,12 @@ public class BonsaiWorldStateUpdateAccumulator public void commit() { this.isAccumulatorStateChanged = true; for (final Address deletedAddress : getDeletedAccounts()) { - final BonsaiValue accountValue = + final DiffBasedValue accountValue = accountsToUpdate.computeIfAbsent( deletedAddress, - __ -> loadAccountFromParent(deletedAddress, new BonsaiValue<>(null, null, true))); + __ -> loadAccountFromParent(deletedAddress, new DiffBasedValue<>(null, null, true))); storageToClear.add(deletedAddress); - final BonsaiValue codeValue = codeToUpdate.get(deletedAddress); + final DiffBasedValue codeValue = codeToUpdate.get(deletedAddress); if (codeValue != null) { codeValue.setUpdated(null).setCleared(); } else { @@ -242,26 +248,27 @@ public class BonsaiWorldStateUpdateAccumulator .getCode( deletedAddress, Optional.ofNullable(accountValue) - .map(BonsaiValue::getPrior) - .map(BonsaiAccount::getCodeHash) + .map(DiffBasedValue::getPrior) + .map(DiffBasedAccount::getCodeHash) .orElse(Hash.EMPTY)) .ifPresent( deletedCode -> - codeToUpdate.put(deletedAddress, new BonsaiValue<>(deletedCode, null, true))); + codeToUpdate.put( + deletedAddress, new DiffBasedValue<>(deletedCode, null, true))); } // mark all updated storage as to be cleared - final Map> deletedStorageUpdates = + final Map> deletedStorageUpdates = storageToUpdate.computeIfAbsent( deletedAddress, k -> new StorageConsumingMap<>( deletedAddress, new ConcurrentHashMap<>(), storagePreloader)); - final Iterator>> iter = + final Iterator>> iter = deletedStorageUpdates.entrySet().iterator(); while (iter.hasNext()) { - final Map.Entry> updateEntry = iter.next(); - final BonsaiValue updatedSlot = updateEntry.getValue(); + final Map.Entry> updateEntry = iter.next(); + final DiffBasedValue updatedSlot = updateEntry.getValue(); if (updatedSlot.getPrior() == null || updatedSlot.getPrior().isZero()) { iter.remove(); } else { @@ -269,7 +276,7 @@ public class BonsaiWorldStateUpdateAccumulator } } - final BonsaiAccount originalValue = accountValue.getPrior(); + final ACCOUNT originalValue = accountValue.getPrior(); if (originalValue != null) { // Enumerate and delete addresses not updated wrappedWorldView() @@ -280,7 +287,8 @@ public class BonsaiWorldStateUpdateAccumulator new StorageSlotKey(Hash.wrap(keyHash), Optional.empty()); if (!deletedStorageUpdates.containsKey(storageSlotKey)) { final UInt256 value = UInt256.fromBytes(RLP.decodeOne(entryValue)); - deletedStorageUpdates.put(storageSlotKey, new BonsaiValue<>(value, null, true)); + deletedStorageUpdates.put( + storageSlotKey, new DiffBasedValue<>(value, null, true)); } }); } @@ -294,11 +302,11 @@ public class BonsaiWorldStateUpdateAccumulator .forEach( tracked -> { final Address updatedAddress = tracked.getAddress(); - final BonsaiAccount updatedAccount; - final BonsaiValue updatedAccountValue = + final ACCOUNT updatedAccount; + final DiffBasedValue updatedAccountValue = accountsToUpdate.get(updatedAddress); - final Map> pendingStorageUpdates = + final Map> pendingStorageUpdates = storageToUpdate.computeIfAbsent( updatedAddress, k -> @@ -311,12 +319,12 @@ public class BonsaiWorldStateUpdateAccumulator } if (tracked.getWrappedAccount() == null) { - updatedAccount = new BonsaiAccount(this, tracked); + updatedAccount = createAccount(this, tracked); tracked.setWrappedAccount(updatedAccount); if (updatedAccountValue == null) { - accountsToUpdate.put(updatedAddress, new BonsaiValue<>(null, updatedAccount)); + accountsToUpdate.put(updatedAddress, new DiffBasedValue<>(null, updatedAccount)); codeToUpdate.put( - updatedAddress, new BonsaiValue<>(null, updatedAccount.getCode())); + updatedAddress, new DiffBasedValue<>(null, updatedAccount.getCode())); } else { updatedAccountValue.setUpdated(updatedAccount); } @@ -334,17 +342,17 @@ public class BonsaiWorldStateUpdateAccumulator } if (tracked.codeWasUpdated()) { - final BonsaiValue pendingCode = + final DiffBasedValue pendingCode = codeToUpdate.computeIfAbsent( updatedAddress, addr -> - new BonsaiValue<>( + new DiffBasedValue<>( wrappedWorldView() .getCode( addr, Optional.ofNullable(updatedAccountValue) - .map(BonsaiValue::getPrior) - .map(BonsaiAccount::getCodeHash) + .map(DiffBasedValue::getPrior) + .map(DiffBasedAccount::getCodeHash) .orElse(Hash.EMPTY)) .orElse(null), null)); @@ -368,12 +376,11 @@ public class BonsaiWorldStateUpdateAccumulator final StorageSlotKey slotKey = new StorageSlotKey(hashAndSaveSlotPreImage(keyUInt), Optional.of(keyUInt)); final UInt256 value = storageUpdate.getValue(); - final BonsaiValue pendingValue = pendingStorageUpdates.get(slotKey); - + final DiffBasedValue pendingValue = pendingStorageUpdates.get(slotKey); if (pendingValue == null) { pendingStorageUpdates.put( slotKey, - new BonsaiValue<>( + new DiffBasedValue<>( updatedAccount.getOriginalStorageValue(keyUInt), value)); } else { pendingValue.setUpdated(value); @@ -394,7 +401,7 @@ public class BonsaiWorldStateUpdateAccumulator @Override public Optional getCode(final Address address, final Hash codeHash) { - final BonsaiValue localCode = codeToUpdate.get(address); + final DiffBasedValue localCode = codeToUpdate.get(address); if (localCode == null) { final Optional code = wrappedWorldView().getCode(address, codeHash); if (code.isEmpty() && !codeHash.equals(Hash.EMPTY)) { @@ -417,31 +424,26 @@ public class BonsaiWorldStateUpdateAccumulator @Override public Optional getStorageValueByStorageSlotKey( final Address address, final StorageSlotKey storageSlotKey) { - final Map> localAccountStorage = + final Map> localAccountStorage = storageToUpdate.get(address); if (localAccountStorage != null) { - final BonsaiValue value = localAccountStorage.get(storageSlotKey); + final DiffBasedValue value = localAccountStorage.get(storageSlotKey); if (value != null) { return Optional.ofNullable(value.getUpdated()); } } try { final Optional valueUInt = - (wrappedWorldView() instanceof BonsaiWorldState bonsaiWorldState) - ? bonsaiWorldState.getStorageValueByStorageSlotKey( - () -> - Optional.ofNullable(loadAccount(address, BonsaiValue::getPrior)) - .map(BonsaiAccount::getStorageRoot), - address, - storageSlotKey) + (wrappedWorldView() instanceof DiffBasedWorldState worldState) + ? worldState.getStorageValueByStorageSlotKey(address, storageSlotKey) : wrappedWorldView().getStorageValueByStorageSlotKey(address, storageSlotKey); storageToUpdate .computeIfAbsent( address, key -> new StorageConsumingMap<>(address, new ConcurrentHashMap<>(), storagePreloader)) - .put(storageSlotKey, new BonsaiValue<>(valueUInt.orElse(null), valueUInt.orElse(null))); - + .put( + storageSlotKey, new DiffBasedValue<>(valueUInt.orElse(null), valueUInt.orElse(null))); return valueUInt; } catch (MerkleTrieException e) { // need to throw to trigger the heal @@ -455,10 +457,10 @@ public class BonsaiWorldStateUpdateAccumulator // TODO maybe log the read into the trie layer? StorageSlotKey storageSlotKey = new StorageSlotKey(hashAndSaveSlotPreImage(storageKey), Optional.of(storageKey)); - final Map> localAccountStorage = + final Map> localAccountStorage = storageToUpdate.get(address); if (localAccountStorage != null) { - final BonsaiValue value = localAccountStorage.get(storageSlotKey); + final DiffBasedValue value = localAccountStorage.get(storageSlotKey); if (value != null) { if (value.isLastStepCleared()) { return UInt256.ZERO; @@ -482,11 +484,11 @@ public class BonsaiWorldStateUpdateAccumulator @Override public Map getAllAccountStorage(final Address address, final Hash rootHash) { final Map results = wrappedWorldView().getAllAccountStorage(address, rootHash); - final StorageConsumingMap> bonsaiValueStorage = + final StorageConsumingMap> diffBasedValueStorage = storageToUpdate.get(address); - if (bonsaiValueStorage != null) { + if (diffBasedValueStorage != null) { // hash the key to match the implied storage interface of hashed slotKey - bonsaiValueStorage.forEach( + diffBasedValueStorage.forEach( (key, value) -> results.put(key.getSlotHash(), value.getUpdated())); } return results; @@ -498,7 +500,7 @@ public class BonsaiWorldStateUpdateAccumulator } @Override - public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { + public DiffBasedWorldStateKeyValueStorage getWorldStateStorage() { return wrappedWorldView().getWorldStateStorage(); } @@ -550,7 +552,7 @@ public class BonsaiWorldStateUpdateAccumulator // non-change, a cached read. return; } - BonsaiValue accountValue = accountsToUpdate.get(address); + DiffBasedValue accountValue = accountsToUpdate.get(address); if (accountValue == null) { accountValue = loadAccountFromParent(address, accountValue); } @@ -558,7 +560,7 @@ public class BonsaiWorldStateUpdateAccumulator if (expectedValue == null && replacementValue != null) { accountsToUpdate.put( address, - new BonsaiValue<>(null, new BonsaiAccount(this, address, replacementValue, true))); + new DiffBasedValue<>(null, createAccount(this, address, replacementValue, true))); } else { throw new IllegalStateException( String.format( @@ -572,7 +574,7 @@ public class BonsaiWorldStateUpdateAccumulator "Expected to create account, but the account exists. Address=%s", address)); } } else { - BonsaiAccount.assertCloseEnoughForDiffing( + assertCloseEnoughForDiffing( accountValue.getUpdated(), expectedValue, "Address=" + address + " Prior Value in Rolling Change"); @@ -586,19 +588,18 @@ public class BonsaiWorldStateUpdateAccumulator accountValue.setUpdated(null); } } else { - accountValue.setUpdated( - new BonsaiAccount(wrappedWorldView(), address, replacementValue, true)); + accountValue.setUpdated(createAccount(wrappedWorldView(), address, replacementValue, true)); } } } - private BonsaiValue loadAccountFromParent( - final Address address, final BonsaiValue defaultValue) { + private DiffBasedValue loadAccountFromParent( + final Address address, final DiffBasedValue defaultValue) { try { final Account parentAccount = wrappedWorldView().get(address); - if (parentAccount instanceof BonsaiAccount account) { - final BonsaiValue loadedAccountValue = - new BonsaiValue<>(new BonsaiAccount(account), account); + if (parentAccount instanceof DiffBasedAccount account) { + final DiffBasedValue loadedAccountValue = + new DiffBasedValue<>(copyAccount((ACCOUNT) account), ((ACCOUNT) account)); accountsToUpdate.put(address, loadedAccountValue); return loadedAccountValue; } else { @@ -617,7 +618,7 @@ public class BonsaiWorldStateUpdateAccumulator // non-change, a cached read. return; } - BonsaiValue codeValue = codeToUpdate.get(address); + DiffBasedValue codeValue = codeToUpdate.get(address); if (codeValue == null) { final Bytes storedCode = wrappedWorldView() @@ -625,14 +626,14 @@ public class BonsaiWorldStateUpdateAccumulator address, Optional.ofNullable(expectedCode).map(Hash::hash).orElse(Hash.EMPTY)) .orElse(Bytes.EMPTY); if (!storedCode.isEmpty()) { - codeValue = new BonsaiValue<>(storedCode, storedCode); + codeValue = new DiffBasedValue<>(storedCode, storedCode); codeToUpdate.put(address, codeValue); } } if (codeValue == null) { if ((expectedCode == null || expectedCode.isEmpty()) && replacementCode != null) { - codeToUpdate.put(address, new BonsaiValue<>(null, replacementCode)); + codeToUpdate.put(address, new DiffBasedValue<>(null, replacementCode)); } else { throw new IllegalStateException( String.format( @@ -660,10 +661,10 @@ public class BonsaiWorldStateUpdateAccumulator } } - private Map> maybeCreateStorageMap( - final Map> storageMap, final Address address) { + private Map> maybeCreateStorageMap( + final Map> storageMap, final Address address) { if (storageMap == null) { - final StorageConsumingMap> newMap = + final StorageConsumingMap> newMap = new StorageConsumingMap<>(address, new ConcurrentHashMap<>(), storagePreloader); storageToUpdate.put(address, newMap); return newMap; @@ -685,13 +686,13 @@ public class BonsaiWorldStateUpdateAccumulator // corner case on deletes, non-change return; } - final Map> storageMap = storageToUpdate.get(address); - BonsaiValue slotValue = storageMap == null ? null : storageMap.get(storageSlotKey); + final Map> storageMap = storageToUpdate.get(address); + DiffBasedValue slotValue = storageMap == null ? null : storageMap.get(storageSlotKey); if (slotValue == null) { final Optional storageValue = wrappedWorldView().getStorageValueByStorageSlotKey(address, storageSlotKey); if (storageValue.isPresent()) { - slotValue = new BonsaiValue<>(storageValue.get(), storageValue.get()); + slotValue = new DiffBasedValue<>(storageValue.get(), storageValue.get()); storageToUpdate .computeIfAbsent( address, @@ -703,7 +704,7 @@ public class BonsaiWorldStateUpdateAccumulator if (slotValue == null) { if ((expectedValue == null || expectedValue.isZero()) && replacementValue != null) { maybeCreateStorageMap(storageMap, address) - .put(storageSlotKey, new BonsaiValue<>(null, replacementValue)); + .put(storageSlotKey, new DiffBasedValue<>(null, replacementValue)); } else { throw new IllegalStateException( String.format( @@ -730,7 +731,7 @@ public class BonsaiWorldStateUpdateAccumulator existingSlotValue == null ? "null" : existingSlotValue.toShortHexString())); } if (replacementValue == null && slotValue.getPrior() == null) { - final Map> thisStorageUpdate = + final Map> thisStorageUpdate = maybeCreateStorageMap(storageMap, address); thisStorageUpdate.remove(storageSlotKey); if (thisStorageUpdate.isEmpty()) { @@ -769,67 +770,6 @@ public class BonsaiWorldStateUpdateAccumulator storageKeyHashLookup.clear(); } - public static class AccountConsumingMap extends ForwardingMap { - - private final ConcurrentMap accounts; - private final Consumer consumer; - - public AccountConsumingMap( - final ConcurrentMap accounts, final Consumer consumer) { - this.accounts = accounts; - this.consumer = consumer; - } - - @Override - public T put(@Nonnull final Address address, @Nonnull final T value) { - consumer.process(address, value); - return accounts.put(address, value); - } - - public Consumer getConsumer() { - return consumer; - } - - @Override - protected Map delegate() { - return accounts; - } - } - - public static class StorageConsumingMap extends ForwardingMap { - - private final Address address; - - private final ConcurrentMap storages; - private final Consumer consumer; - - public StorageConsumingMap( - final Address address, final ConcurrentMap storages, final Consumer consumer) { - this.address = address; - this.storages = storages; - this.consumer = consumer; - } - - @Override - public T put(@Nonnull final K slotKey, @Nonnull final T value) { - consumer.process(address, slotKey); - return storages.put(slotKey, value); - } - - public Consumer getConsumer() { - return consumer; - } - - @Override - protected Map delegate() { - return storages; - } - } - - public interface Consumer { - void process(final Address address, T value); - } - protected Hash hashAndSaveAccountPreImage(final Address address) { // no need to save account preimage by default return Hash.hash(address); @@ -843,4 +783,33 @@ public class BonsaiWorldStateUpdateAccumulator } return hash; } + + public abstract DiffBasedWorldStateUpdateAccumulator copy(); + + protected abstract ACCOUNT copyAccount(final ACCOUNT account); + + protected abstract ACCOUNT copyAccount( + final ACCOUNT toCopy, final DiffBasedWorldView context, final boolean mutable); + + protected abstract ACCOUNT createAccount( + final DiffBasedWorldView context, + final Address address, + final AccountValue stateTrieAccount, + final boolean mutable); + + protected abstract ACCOUNT createAccount( + final DiffBasedWorldView context, + final Address address, + final Hash addressHash, + final long nonce, + final Wei balance, + final Hash storageRoot, + final Hash codeHash, + final boolean mutable); + + protected abstract ACCOUNT createAccount( + final DiffBasedWorldView context, final UpdateTrackingAccount tracked); + + protected abstract void assertCloseEnoughForDiffing( + final ACCOUNT source, final AccountValue account, final String context); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/AccountConsumingMap.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/AccountConsumingMap.java new file mode 100644 index 000000000..63c5b9233 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/AccountConsumingMap.java @@ -0,0 +1,49 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload; + +import org.hyperledger.besu.datatypes.Address; + +import java.util.Map; +import java.util.concurrent.ConcurrentMap; +import javax.annotation.Nonnull; + +import com.google.common.collect.ForwardingMap; + +public class AccountConsumingMap extends ForwardingMap { + + private final ConcurrentMap accounts; + private final Consumer consumer; + + public AccountConsumingMap(final ConcurrentMap accounts, final Consumer consumer) { + this.accounts = accounts; + this.consumer = consumer; + } + + @Override + public T put(@Nonnull final Address address, @Nonnull final T value) { + consumer.process(address, value); + return accounts.put(address, value); + } + + public Consumer getConsumer() { + return consumer; + } + + @Override + protected Map delegate() { + return accounts; + } +} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectlyNegativeCases.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/Consumer.java similarity index 64% rename from errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectlyNegativeCases.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/Consumer.java index 3f84b2fb1..577b0d8c5 100644 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectlyNegativeCases.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/Consumer.java @@ -1,5 +1,5 @@ /* - * Copyright ConsenSys AG. + * Copyright Hyperledger Besu Contributors. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -12,15 +12,10 @@ * * SPDX-License-Identifier: Apache-2.0 */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload; -package org.hyperledger.errorpronechecks; +import org.hyperledger.besu.datatypes.Address; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; - -public class DoNotInvokeMessageDigestDirectlyNegativeCases { - - public void callsMessageDigestGetInstance() throws NoSuchAlgorithmException { - MessageDigest dig = null; - } +public interface Consumer { + void process(final Address address, T value); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/StorageConsumingMap.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/StorageConsumingMap.java new file mode 100644 index 000000000..13b42d7b4 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/StorageConsumingMap.java @@ -0,0 +1,53 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload; + +import org.hyperledger.besu.datatypes.Address; + +import java.util.Map; +import java.util.concurrent.ConcurrentMap; +import javax.annotation.Nonnull; + +import com.google.common.collect.ForwardingMap; + +public class StorageConsumingMap extends ForwardingMap { + + private final Address address; + + private final ConcurrentMap storages; + private final Consumer consumer; + + public StorageConsumingMap( + final Address address, final ConcurrentMap storages, final Consumer consumer) { + this.address = address; + this.storages = storages; + this.consumer = consumer; + } + + @Override + public T put(@Nonnull final K slotKey, @Nonnull final T value) { + consumer.process(address, slotKey); + return storages.put(slotKey, value); + } + + public Consumer getConsumer() { + return consumer; + } + + @Override + protected Map delegate() { + return storages; + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java index 064f5a961..7ad5362ae 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java @@ -25,6 +25,7 @@ import org.immutables.value.Value; public interface DataStorageConfiguration { long DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD = 512; + boolean DEFAULT_RECEIPT_COMPACTION_ENABLED = false; DataStorageConfiguration DEFAULT_CONFIG = ImmutableDataStorageConfiguration.builder() @@ -50,6 +51,11 @@ public interface DataStorageConfiguration { Long getBonsaiMaxLayersToLoad(); + @Value.Default + default boolean getReceiptCompactionEnabled() { + return DEFAULT_RECEIPT_COMPACTION_ENABLED; + } + @Value.Default default Unstable getUnstable() { return Unstable.DEFAULT; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java index f689410a9..c35fe423a 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java @@ -15,7 +15,7 @@ package org.hyperledger.besu.ethereum.worldstate; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ExecutionContextTestFixture.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ExecutionContextTestFixture.java index 40f4c8c93..3aa483e1a 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ExecutionContextTestFixture.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ExecutionContextTestFixture.java @@ -64,7 +64,8 @@ public class ExecutionContextTestFixture { new KeyValueStoragePrefixedKeyBlockchainStorage( blockchainKeyValueStorage, new VariablesKeyValueStorage(variablesKeyValueStorage), - new MainnetBlockHeaderFunctions()), + new MainnetBlockHeaderFunctions(), + false), new NoOpMetricsSystem(), 0); this.stateArchive = createInMemoryWorldStateArchive(); diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java index 6626f4aad..9bac92549 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java @@ -25,9 +25,9 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStoragePrefixedKey import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.ForestWorldStateArchive; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState; @@ -73,7 +73,7 @@ public class InMemoryKeyValueStorageProvider extends KeyValueStorageProvider { return DefaultBlockchain.createMutable( genesisBlock, new KeyValueStoragePrefixedKeyBlockchainStorage( - keyValueStorage, variablesStorage, blockHeaderFunctions), + keyValueStorage, variablesStorage, blockHeaderFunctions, false), new NoOpMetricsSystem(), 0); } @@ -95,15 +95,15 @@ public class InMemoryKeyValueStorageProvider extends KeyValueStorageProvider { final Blockchain blockchain, final EvmConfiguration evmConfiguration) { final InMemoryKeyValueStorageProvider inMemoryKeyValueStorageProvider = new InMemoryKeyValueStorageProvider(); - final CachedMerkleTrieLoader cachedMerkleTrieLoader = - new CachedMerkleTrieLoader(new NoOpMetricsSystem()); + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader = + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()); return new BonsaiWorldStateProvider( (BonsaiWorldStateKeyValueStorage) inMemoryKeyValueStorageProvider.createWorldStateStorage( DataStorageConfiguration.DEFAULT_BONSAI_CONFIG), blockchain, Optional.empty(), - cachedMerkleTrieLoader, + bonsaiCachedMerkleTrieLoader, null, evmConfiguration); } diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java index c549c66eb..a04592d30 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java @@ -20,7 +20,7 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.MerkleTrie; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java index f8fe79458..d451a6fb7 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java @@ -41,9 +41,9 @@ import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionProcessor; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/ChainDataPrunerTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/ChainDataPrunerTest.java index 6b249e741..7082d8ae1 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/ChainDataPrunerTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/ChainDataPrunerTest.java @@ -41,7 +41,8 @@ public class ChainDataPrunerTest { new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions()); + new MainnetBlockHeaderFunctions(), + false); final ChainDataPruner chainDataPruner = new ChainDataPruner( blockchainStorage, @@ -79,7 +80,8 @@ public class ChainDataPrunerTest { new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions()); + new MainnetBlockHeaderFunctions(), + false); final ChainDataPruner chainDataPruner = new ChainDataPruner( blockchainStorage, diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchainTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchainTest.java index 66c9ac559..69fbfcdbd 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchainTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchainTest.java @@ -1055,7 +1055,8 @@ public class DefaultBlockchainTest { return new KeyValueStoragePrefixedKeyBlockchainStorage( kvStoreChain, new VariablesKeyValueStorage(kvStorageVariables), - new MainnetBlockHeaderFunctions()); + new MainnetBlockHeaderFunctions(), + false); } private DefaultBlockchain createMutableBlockchain( diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/LogTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/LogTest.java index 18fb8f332..d47efa4b0 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/LogTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/LogTest.java @@ -18,16 +18,43 @@ import static org.assertj.core.api.Assertions.assertThat; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.evm.log.Log; +import org.hyperledger.besu.evm.log.LogTopic; +import java.util.List; + +import org.apache.tuweni.bytes.Bytes; import org.junit.jupiter.api.Test; public class LogTest { + final BlockDataGenerator gen = new BlockDataGenerator(); @Test public void toFromRlp() { - final BlockDataGenerator gen = new BlockDataGenerator(); - final Log log = gen.log(); + final Log log = gen.log(2); final Log copy = Log.readFrom(RLP.input(RLP.encode(log::writeTo))); assertThat(copy).isEqualTo(log); } + + @Test + public void toFromRlpCompacted() { + final Log log = gen.log(2); + final Log copy = Log.readFrom(RLP.input(RLP.encode(rlpOut -> log.writeTo(rlpOut, true))), true); + assertThat(copy).isEqualTo(log); + } + + @Test + public void toFromRlpCompactedWithLeadingZeros() { + final Bytes logData = bytesWithLeadingZeros(10, 100); + final List logTopics = + List.of( + LogTopic.of(bytesWithLeadingZeros(20, 32)), LogTopic.of(bytesWithLeadingZeros(30, 32))); + final Log log = new Log(gen.address(), logData, logTopics); + final Log copy = Log.readFrom(RLP.input(RLP.encode(rlpOut -> log.writeTo(rlpOut, true))), true); + assertThat(copy).isEqualTo(log); + } + + private Bytes bytesWithLeadingZeros(final int noLeadingZeros, final int totalSize) { + return Bytes.concatenate( + Bytes.repeat((byte) 0, noLeadingZeros), gen.bytesValue(totalSize - noLeadingZeros)); + } } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/TransactionReceiptTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/TransactionReceiptTest.java index 00a04354c..06b9367ba 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/TransactionReceiptTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/TransactionReceiptTest.java @@ -28,7 +28,7 @@ public class TransactionReceiptTest { final BlockDataGenerator gen = new BlockDataGenerator(); final TransactionReceipt receipt = gen.receipt(); final TransactionReceipt copy = - TransactionReceipt.readFrom(RLP.input(RLP.encode(receipt::writeToWithRevertReason)), false); + TransactionReceipt.readFrom(RLP.input(RLP.encode(receipt::writeToForNetwork)), false); assertThat(copy).isEqualTo(receipt); } @@ -37,7 +37,40 @@ public class TransactionReceiptTest { final BlockDataGenerator gen = new BlockDataGenerator(); final TransactionReceipt receipt = gen.receipt(Bytes.fromHexString("0x1122334455667788")); final TransactionReceipt copy = - TransactionReceipt.readFrom(RLP.input(RLP.encode(receipt::writeToWithRevertReason))); + TransactionReceipt.readFrom( + RLP.input(RLP.encode(rlpOut -> receipt.writeToForReceiptTrie(rlpOut, true, false)))); assertThat(copy).isEqualTo(receipt); } + + @Test + public void toFromRlpCompacted() { + final BlockDataGenerator gen = new BlockDataGenerator(); + final TransactionReceipt receipt = gen.receipt(Bytes.fromHexString("0x1122334455667788")); + final TransactionReceipt copy = + TransactionReceipt.readFrom( + RLP.input(RLP.encode(rlpOut -> receipt.writeToForReceiptTrie(rlpOut, false, true)))); + assertThat(copy).isEqualTo(receipt); + } + + @Test + public void toFromRlpCompactedWithReason() { + final BlockDataGenerator gen = new BlockDataGenerator(); + final TransactionReceipt receipt = gen.receipt(Bytes.fromHexString("0x1122334455667788")); + final TransactionReceipt copy = + TransactionReceipt.readFrom( + RLP.input(RLP.encode(rlpOut -> receipt.writeToForReceiptTrie(rlpOut, true, true)))); + assertThat(copy).isEqualTo(receipt); + } + + @Test + public void uncompactedAndCompactedDecodeToSameReceipt() { + final BlockDataGenerator gen = new BlockDataGenerator(); + final TransactionReceipt receipt = gen.receipt(Bytes.fromHexString("0x1122334455667788")); + final Bytes compactedReceipt = + RLP.encode(rlpOut -> receipt.writeToForReceiptTrie(rlpOut, false, true)); + final Bytes unCompactedReceipt = + RLP.encode(rlpOut -> receipt.writeToForReceiptTrie(rlpOut, false, false)); + assertThat(TransactionReceipt.readFrom(RLP.input(compactedReceipt))).isEqualTo(receipt); + assertThat(TransactionReceipt.readFrom(RLP.input(unCompactedReceipt))).isEqualTo(receipt); + } } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/EthHashTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/EthHashTest.java index 458bc7413..10a3b15b4 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/EthHashTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/EthHashTest.java @@ -82,39 +82,21 @@ public final class EthHashTest { } assertThat(Hex.toHexString(buffer.array())) .isEqualTo( - new StringBuilder() - .append( - "7ce2991c951f7bf4c4c1bb119887ee07871eb5339d7b97b8588e85c742de90e5bafd5bbe6ce93a134fb6be9ad3e30db99d9528a2ea7846833f52e9ca119b6b54") - .append( - "8979480c46e19972bd0738779c932c1b43e665a2fd3122fc3ddb2691f353ceb0ed3e38b8f51fd55b6940290743563c9f8fa8822e611924657501a12aafab8a8d") - .append( - "88fb5fbae3a99d14792406672e783a06940a42799b1c38bc28715db6d37cb11f9f6b24e386dc52dd8c286bd8c36fa813dffe4448a9f56ebcbeea866b42f68d22") - .append( - "6c32aae4d695a23cab28fd74af53b0c2efcc180ceaaccc0b2e280103d097a03c1d1b0f0f26ce5f32a90238f9bc49f645db001ef9cd3d13d44743f841fad11a37") - .append( - "fa290c62c16042f703578921f30b9951465aae2af4a5dad43a7341d7b4a62750954965a47a1c3af638dc3495c4d62a9bab843168c9fc0114e79cffd1b2827b01") - .append( - "75d30ba054658f214e946cf24c43b40d3383fbb0493408e5c5392434ca21bbcf43200dfb876c713d201813934fa485f48767c5915745cf0986b1dc0f33e57748") - .append( - "bf483ee2aff4248dfe461ec0504a13628401020fc22638584a8f2f5206a13b2f233898c78359b21c8226024d0a7a93df5eb6c282bdbf005a4aab497e096f2847") - .append( - "76c71cee57932a8fb89f6d6b8743b60a4ea374899a94a2e0f218d5c55818cefb1790c8529a76dba31ebb0f4592d709b49587d2317970d39c086f18dd244291d9") - .append( - "eedb16705e53e3350591bd4ff4566a3595ac0f0ce24b5e112a3d033bc51b6fea0a92296dea7f5e20bf6ee6bc347d868fda193c395b9bb147e55e5a9f67cfe741") - .append( - "7eea7d699b155bd13804204df7ea91fa9249e4474dddf35188f77019c67d201e4c10d7079c5ad492a71afff9a23ca7e900ba7d1bdeaf3270514d8eb35eab8a0a") - .append( - "718bb7273aeb37768fa589ed8ab01fbf4027f4ebdbbae128d21e485f061c20183a9bc2e31edbda0727442e9d58eb0fe198440fe199e02e77c0f7b99973f1f74c") - .append( - "c9089a51ab96c94a84d66e6aa48b2d0a4543adb5a789039a2aa7b335ca85c91026c7d3c894da53ae364188c3fd92f78e01d080399884a47385aa792e38150cda") - .append( - "a8620b2ebeca41fbc773bb837b5e724d6eb2de570d99858df0d7d97067fb8103b21757873b735097b35d3bea8fd1c359a9e8a63c1540c76c9784cf8d975e995c") - .append( - "778401b94a2e66e6993ad67ad3ecdc2acb17779f1ea8606827ec92b11c728f8c3b6d3f04a3e6ed05ff81dd76d5dc5695a50377bc135aaf1671cf68b750315493") - .append( - "6c64510164d53312bf3c41740c7a237b05faf4a191bd8a95dafa068dbcf370255c725900ce5c934f36feadcfe55b687c440574c1f06f39d207a8553d39156a24") - .append( - "845f64fd8324bb85312979dead74f764c9677aab89801ad4f927f1c00f12e28f22422bb44200d1969d9ab377dd6b099dc6dbc3222e9321b2c1e84f8e2f07731c") - .toString()); + "7ce2991c951f7bf4c4c1bb119887ee07871eb5339d7b97b8588e85c742de90e5bafd5bbe6ce93a134fb6be9ad3e30db99d9528a2ea7846833f52e9ca119b6b54" + + "8979480c46e19972bd0738779c932c1b43e665a2fd3122fc3ddb2691f353ceb0ed3e38b8f51fd55b6940290743563c9f8fa8822e611924657501a12aafab8a8d" + + "88fb5fbae3a99d14792406672e783a06940a42799b1c38bc28715db6d37cb11f9f6b24e386dc52dd8c286bd8c36fa813dffe4448a9f56ebcbeea866b42f68d22" + + "6c32aae4d695a23cab28fd74af53b0c2efcc180ceaaccc0b2e280103d097a03c1d1b0f0f26ce5f32a90238f9bc49f645db001ef9cd3d13d44743f841fad11a37" + + "fa290c62c16042f703578921f30b9951465aae2af4a5dad43a7341d7b4a62750954965a47a1c3af638dc3495c4d62a9bab843168c9fc0114e79cffd1b2827b01" + + "75d30ba054658f214e946cf24c43b40d3383fbb0493408e5c5392434ca21bbcf43200dfb876c713d201813934fa485f48767c5915745cf0986b1dc0f33e57748" + + "bf483ee2aff4248dfe461ec0504a13628401020fc22638584a8f2f5206a13b2f233898c78359b21c8226024d0a7a93df5eb6c282bdbf005a4aab497e096f2847" + + "76c71cee57932a8fb89f6d6b8743b60a4ea374899a94a2e0f218d5c55818cefb1790c8529a76dba31ebb0f4592d709b49587d2317970d39c086f18dd244291d9" + + "eedb16705e53e3350591bd4ff4566a3595ac0f0ce24b5e112a3d033bc51b6fea0a92296dea7f5e20bf6ee6bc347d868fda193c395b9bb147e55e5a9f67cfe741" + + "7eea7d699b155bd13804204df7ea91fa9249e4474dddf35188f77019c67d201e4c10d7079c5ad492a71afff9a23ca7e900ba7d1bdeaf3270514d8eb35eab8a0a" + + "718bb7273aeb37768fa589ed8ab01fbf4027f4ebdbbae128d21e485f061c20183a9bc2e31edbda0727442e9d58eb0fe198440fe199e02e77c0f7b99973f1f74c" + + "c9089a51ab96c94a84d66e6aa48b2d0a4543adb5a789039a2aa7b335ca85c91026c7d3c894da53ae364188c3fd92f78e01d080399884a47385aa792e38150cda" + + "a8620b2ebeca41fbc773bb837b5e724d6eb2de570d99858df0d7d97067fb8103b21757873b735097b35d3bea8fd1c359a9e8a63c1540c76c9784cf8d975e995c" + + "778401b94a2e66e6993ad67ad3ecdc2acb17779f1ea8606827ec92b11c728f8c3b6d3f04a3e6ed05ff81dd76d5dc5695a50377bc135aaf1671cf68b750315493" + + "6c64510164d53312bf3c41740c7a237b05faf4a191bd8a95dafa068dbcf370255c725900ce5c934f36feadcfe55b687c440574c1f06f39d207a8553d39156a24" + + "845f64fd8324bb85312979dead74f764c9677aab89801ad4f927f1c00f12e28f22422bb44200d1969d9ab377dd6b099dc6dbc3222e9321b2c1e84f8e2f07731c"); } } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStoragePrefixedKeyBlockchainStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStoragePrefixedKeyBlockchainStorageTest.java index 1e89581a0..526c839b6 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStoragePrefixedKeyBlockchainStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStoragePrefixedKeyBlockchainStorageTest.java @@ -62,7 +62,7 @@ public class KeyValueStoragePrefixedKeyBlockchainStorageTest { final var blockchainStorage = new KeyValueStoragePrefixedKeyBlockchainStorage( - kvBlockchain, variablesStorage, blockHeaderFunctions); + kvBlockchain, variablesStorage, blockHeaderFunctions, false); assertNoVariablesInStorage(kvBlockchain); assertVariablesPresentInVariablesStorage(kvVariables, variableValues); @@ -80,7 +80,7 @@ public class KeyValueStoragePrefixedKeyBlockchainStorageTest { final var blockchainStorage = new KeyValueStoragePrefixedKeyBlockchainStorage( - kvBlockchain, variablesStorage, blockHeaderFunctions); + kvBlockchain, variablesStorage, blockHeaderFunctions, false); assertNoVariablesInStorage(kvBlockchain); assertVariablesPresentInVariablesStorage(kvVariables, variableValues); @@ -96,7 +96,7 @@ public class KeyValueStoragePrefixedKeyBlockchainStorageTest { final var blockchainStorage = new KeyValueStoragePrefixedKeyBlockchainStorage( - kvBlockchain, variablesStorage, blockHeaderFunctions); + kvBlockchain, variablesStorage, blockHeaderFunctions, false); assertNoVariablesInStorage(kvBlockchain); assertVariablesPresentInVariablesStorage(kvVariables, variableValues); @@ -114,6 +114,6 @@ public class KeyValueStoragePrefixedKeyBlockchainStorageTest { IllegalStateException.class, () -> new KeyValueStoragePrefixedKeyBlockchainStorage( - kvBlockchain, variablesStorage, blockHeaderFunctions)); + kvBlockchain, variablesStorage, blockHeaderFunctions, false)); } } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java similarity index 93% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java index 2d18bee32..b2fbc7138 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider.createInMemoryBlockchain; import static org.mockito.ArgumentMatchers.any; @@ -66,8 +66,8 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.evm.internal.EvmConfiguration; @@ -159,7 +159,7 @@ public abstract class AbstractIsolationTests { (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage, blockchain, Optional.of(16L), - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), null, EvmConfiguration.DEFAULT); var ws = archive.getMutable(); @@ -214,6 +214,22 @@ public abstract class AbstractIsolationTests { public Wei getMinGasPrice() { return MiningParameters.newDefault().getMinTransactionGasPrice(); } + + @Override + public org.hyperledger.besu.plugin.services.storage.DataStorageConfiguration + getDataStorageConfiguration() { + return new org.hyperledger.besu.plugin.services.storage.DataStorageConfiguration() { + @Override + public DataStorageFormat getDatabaseFormat() { + return DataStorageFormat.BONSAI; + } + + @Override + public boolean getReceiptCompactionEnabled() { + return false; + } + }; + } }) .withMetricsSystem(new NoOpMetricsSystem()) .build(); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/CachedMerkleTrieLoaderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiCachedMerkleTrieLoaderTest.java similarity index 94% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/CachedMerkleTrieLoaderTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiCachedMerkleTrieLoaderTest.java index d96c6990c..d4e42cd51 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/CachedMerkleTrieLoaderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiCachedMerkleTrieLoaderTest.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static org.assertj.core.api.Assertions.assertThat; @@ -26,8 +26,8 @@ import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; @@ -45,9 +45,9 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; -class CachedMerkleTrieLoaderTest { +class BonsaiCachedMerkleTrieLoaderTest { - private CachedMerkleTrieLoader merkleTrieLoader; + private BonsaiCachedMerkleTrieLoader merkleTrieLoader; private final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); private final BonsaiWorldStateKeyValueStorage inMemoryWorldState = Mockito.spy( @@ -69,7 +69,7 @@ class CachedMerkleTrieLoaderTest { TrieGenerator.generateTrie( worldStateStorageCoordinator, accounts.stream().map(Address::addressHash).collect(Collectors.toList())); - merkleTrieLoader = new CachedMerkleTrieLoader(new NoOpMetricsSystem()); + merkleTrieLoader = new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()); } @Test diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiSnapshotIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiSnapshotIsolationTests.java similarity index 99% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiSnapshotIsolationTests.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiSnapshotIsolationTests.java index ce983e10a..507047604 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiSnapshotIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiSnapshotIsolationTests.java @@ -14,7 +14,7 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProviderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProviderTest.java similarity index 89% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProviderTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProviderTest.java index 10d3381c7..879183ecc 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProviderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProviderTest.java @@ -13,13 +13,13 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static org.assertj.core.api.Assertions.assertThat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.BLOCKCHAIN; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; -import static org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; -import static org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; +import static org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; +import static org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.eq; @@ -36,13 +36,13 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -77,7 +77,7 @@ class BonsaiWorldStateProviderTest { @Mock SegmentedKeyValueStorageTransaction segmentedKeyValueStorageTransaction; BonsaiWorldStateProvider bonsaiWorldStateArchive; - @Mock CachedWorldStorageManager cachedWorldStorageManager; + @Mock BonsaiCachedWorldStorageManager cachedWorldStorageManager; @Mock TrieLogManager trieLogManager; @BeforeEach @@ -111,7 +111,7 @@ class BonsaiWorldStateProviderTest { new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_BONSAI_CONFIG), blockchain, - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT); assertThat(bonsaiWorldStateArchive.getMutable(chainHead, true)) @@ -128,7 +128,7 @@ class BonsaiWorldStateProviderTest { DataStorageConfiguration.DEFAULT_BONSAI_CONFIG), blockchain, Optional.of(512L), - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), null, EvmConfiguration.DEFAULT); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); @@ -150,7 +150,7 @@ class BonsaiWorldStateProviderTest { new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_BONSAI_CONFIG), blockchain, - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); final BlockHeader chainHead = blockBuilder.number(511).buildHeader(); @@ -185,7 +185,7 @@ class BonsaiWorldStateProviderTest { trieLogManager, worldStateKeyValueStorage, blockchain, - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT)); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); @@ -214,7 +214,7 @@ class BonsaiWorldStateProviderTest { trieLogManager, worldStateKeyValueStorage, blockchain, - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT)); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); @@ -254,7 +254,7 @@ class BonsaiWorldStateProviderTest { trieLogManager, worldStateKeyValueStorage, blockchain, - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT)); // initial persisted state hash key @@ -297,7 +297,7 @@ class BonsaiWorldStateProviderTest { new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_BONSAI_CONFIG), blockchain, - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT)); // initial persisted state hash key diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/LogRollingTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/LogRollingTests.java similarity index 96% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/LogRollingTests.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/LogRollingTests.java index ee63fab96..331c6b1af 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/LogRollingTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/LogRollingTests.java @@ -14,7 +14,7 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; @@ -29,11 +29,11 @@ import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.internal.EvmConfiguration; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/RollingImport.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/RollingImport.java similarity index 90% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/RollingImport.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/RollingImport.java index a43fce8f4..134d094a4 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/RollingImport.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/RollingImport.java @@ -14,7 +14,7 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static com.google.common.base.Preconditions.checkArgument; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; @@ -25,11 +25,11 @@ import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIden import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java similarity index 99% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java index 911ff1000..c4769248e 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java @@ -12,11 +12,11 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage; import static org.assertj.core.api.Assertions.assertThat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; -import static org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; +import static org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryTests.java similarity index 94% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryTests.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryTests.java index 64f96d8b7..59c208792 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryTests.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; @@ -24,6 +24,7 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManagerTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogManagerTests.java similarity index 84% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManagerTests.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogManagerTests.java index e5c716e41..a6ff8d3a7 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManagerTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogManagerTests.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.spy; @@ -22,9 +22,10 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; import org.hyperledger.besu.evm.internal.EvmConfiguration; import java.util.concurrent.atomic.AtomicBoolean; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPrunerTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogPrunerTest.java similarity index 96% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPrunerTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogPrunerTest.java index ac84d35be..19e63f6f9 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPrunerTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogPrunerTest.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; @@ -26,7 +26,10 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockDataGenerator; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogAddedEvent; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; import java.util.Optional; import java.util.function.Consumer; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateTest.java similarity index 81% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateTest.java index f875f098b..8abae1f2b 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateTest.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.worldview; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; @@ -23,8 +23,8 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiValue; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; import org.hyperledger.besu.evm.internal.EvmConfiguration; import java.util.HashMap; @@ -69,8 +69,8 @@ class BonsaiWorldStateTest { @MethodSource("priorAndUpdatedEmptyAndNullBytes") void codeUpdateDoesNothingWhenMarkedAsDeletedButAlreadyDeleted( final Bytes prior, final Bytes updated) { - final Map> codeToUpdate = - Map.of(Address.ZERO, new BonsaiValue<>(prior, updated)); + final Map> codeToUpdate = + Map.of(Address.ZERO, new DiffBasedValue<>(prior, updated)); when(bonsaiWorldStateUpdateAccumulator.getCodeToUpdate()).thenReturn(codeToUpdate); worldState.updateCode(Optional.of(bonsaiUpdater), bonsaiWorldStateUpdateAccumulator); @@ -79,8 +79,8 @@ class BonsaiWorldStateTest { @Test void codeUpdateDoesNothingWhenAddingSameAsExistingValue() { - final Map> codeToUpdate = - Map.of(Address.ZERO, new BonsaiValue<>(CODE, CODE)); + final Map> codeToUpdate = + Map.of(Address.ZERO, new DiffBasedValue<>(CODE, CODE)); when(bonsaiWorldStateUpdateAccumulator.getCodeToUpdate()).thenReturn(codeToUpdate); worldState.updateCode(Optional.of(bonsaiUpdater), bonsaiWorldStateUpdateAccumulator); @@ -90,8 +90,8 @@ class BonsaiWorldStateTest { @ParameterizedTest @MethodSource("emptyAndNullBytes") void removesCodeWhenMarkedAsDeleted(final Bytes updated) { - final Map> codeToUpdate = - Map.of(Address.ZERO, new BonsaiValue<>(CODE, updated)); + final Map> codeToUpdate = + Map.of(Address.ZERO, new DiffBasedValue<>(CODE, updated)); when(bonsaiWorldStateUpdateAccumulator.getCodeToUpdate()).thenReturn(codeToUpdate); worldState.updateCode(Optional.of(bonsaiUpdater), bonsaiWorldStateUpdateAccumulator); @@ -101,8 +101,8 @@ class BonsaiWorldStateTest { @ParameterizedTest @MethodSource("codeValueAndEmptyAndNullBytes") void addsCodeForNewCodeValue(final Bytes prior) { - final Map> codeToUpdate = - Map.of(ACCOUNT, new BonsaiValue<>(prior, CODE)); + final Map> codeToUpdate = + Map.of(ACCOUNT, new DiffBasedValue<>(prior, CODE)); when(bonsaiWorldStateUpdateAccumulator.getCodeToUpdate()).thenReturn(codeToUpdate); worldState.updateCode(Optional.of(bonsaiUpdater), bonsaiWorldStateUpdateAccumulator); @@ -112,10 +112,10 @@ class BonsaiWorldStateTest { @Test void updateCodeForMultipleValues() { - final Map> codeToUpdate = new HashMap<>(); - codeToUpdate.put(Address.fromHexString("0x1"), new BonsaiValue<>(null, CODE)); - codeToUpdate.put(Address.fromHexString("0x2"), new BonsaiValue<>(CODE, null)); - codeToUpdate.put(Address.fromHexString("0x3"), new BonsaiValue<>(Bytes.of(9), CODE)); + final Map> codeToUpdate = new HashMap<>(); + codeToUpdate.put(Address.fromHexString("0x1"), new DiffBasedValue<>(null, CODE)); + codeToUpdate.put(Address.fromHexString("0x2"), new DiffBasedValue<>(CODE, null)); + codeToUpdate.put(Address.fromHexString("0x3"), new DiffBasedValue<>(Bytes.of(9), CODE)); when(bonsaiWorldStateUpdateAccumulator.getCodeToUpdate()).thenReturn(codeToUpdate); worldState.updateCode(Optional.of(bonsaiUpdater), bonsaiWorldStateUpdateAccumulator); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProviderTest.java similarity index 97% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProviderTest.java index 2340a6e8b..5199bc19d 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProviderTest.java @@ -13,13 +13,15 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat; import static org.assertj.core.api.Assertions.assertThat; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.FullFlatDbStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.PartialFlatDbStrategy; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayerTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogLayerTests.java similarity index 98% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayerTests.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogLayerTests.java index d7d49bace..d967ca1a1 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayerTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogLayerTests.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import org.hyperledger.besu.datatypes.AccountValue; import org.hyperledger.besu.datatypes.Address; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/pruner/PrunerTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/pruner/PrunerTest.java index 804c80562..308834a6a 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/pruner/PrunerTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/pruner/PrunerTest.java @@ -63,7 +63,8 @@ public class PrunerTest { new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions()); + new MainnetBlockHeaderFunctions(), + false); final MutableBlockchain blockchain = DefaultBlockchain.createMutable(genesisBlock, blockchainStorage, metricsSystem, 0); @@ -86,7 +87,8 @@ public class PrunerTest { new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions()); + new MainnetBlockHeaderFunctions(), + false); final MutableBlockchain blockchain = DefaultBlockchain.createMutable(genesisBlock, blockchainStorage, metricsSystem, 0); @@ -114,7 +116,8 @@ public class PrunerTest { new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions()); + new MainnetBlockHeaderFunctions(), + false); final MutableBlockchain blockchain = DefaultBlockchain.createMutable(genesisBlock, blockchainStorage, metricsSystem, 0); @@ -186,7 +189,8 @@ public class PrunerTest { new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions()); + new MainnetBlockHeaderFunctions(), + false); final MutableBlockchain blockchain = DefaultBlockchain.createMutable(genesisBlock, blockchainStorage, metricsSystem, 0); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestKeyValueStorageWorldStateStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestKeyValueStorageWorldStateStorageTest.java index 08dfc86d5..d60f92448 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestKeyValueStorageWorldStateStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestKeyValueStorageWorldStateStorageTest.java @@ -66,6 +66,7 @@ public class ForestKeyValueStorageWorldStateStorageTest { storage.updater().putCode(MerkleTrie.EMPTY_TRIE_NODE).putCode(Bytes.EMPTY).commit(); assertThat(storage.getCode(Hash.EMPTY_TRIE_HASH)).contains(MerkleTrie.EMPTY_TRIE_NODE); + assertThat(storage.getCode(Hash.EMPTY)).contains(Bytes.EMPTY); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthServer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthServer.java index 76346f415..d70623d7b 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthServer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthServer.java @@ -234,7 +234,7 @@ class EthServer { } final BytesValueRLPOutput encodedReceipts = new BytesValueRLPOutput(); encodedReceipts.startList(); - maybeReceipts.get().forEach(r -> r.writeTo(encodedReceipts)); + maybeReceipts.get().forEach(r -> r.writeToForNetwork(encodedReceipts)); encodedReceipts.endList(); final int encodedSize = encodedReceipts.encodedSize(); if (responseSizeEstimate + encodedSize > maxMessageSize) { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/ReceiptsMessage.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/ReceiptsMessage.java index 9d9ed2cc7..7ca823cd8 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/ReceiptsMessage.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/ReceiptsMessage.java @@ -46,7 +46,7 @@ public final class ReceiptsMessage extends AbstractMessageData { receipts.forEach( (receiptSet) -> { tmp.startList(); - receiptSet.forEach(r -> r.writeTo(tmp)); + receiptSet.forEach(r -> r.writeToForNetwork(tmp)); tmp.endList(); }); tmp.endList(); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/AccountRangeMessage.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/AccountRangeMessage.java index 417bbd27f..c9b93734a 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/AccountRangeMessage.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/AccountRangeMessage.java @@ -24,6 +24,7 @@ import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import java.math.BigInteger; import java.util.List; import java.util.Map; +import java.util.NavigableMap; import java.util.Optional; import java.util.TreeMap; @@ -133,7 +134,7 @@ public final class AccountRangeMessage extends AbstractSnapMessageData { @Value.Immutable public interface AccountRangeData { - TreeMap accounts(); + NavigableMap accounts(); ArrayDeque proofs(); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/StorageRangeMessage.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/StorageRangeMessage.java index ab4a353fe..d89718f0f 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/StorageRangeMessage.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/StorageRangeMessage.java @@ -22,6 +22,7 @@ import org.hyperledger.besu.ethereum.rlp.RLPInput; import java.math.BigInteger; import java.util.List; +import java.util.NavigableMap; import java.util.Optional; import java.util.TreeMap; @@ -49,13 +50,13 @@ public final class StorageRangeMessage extends AbstractSnapMessageData { } public static StorageRangeMessage create( - final ArrayDeque> slots, final List proof) { + final ArrayDeque> slots, final List proof) { return create(Optional.empty(), slots, proof); } public static StorageRangeMessage create( final Optional requestId, - final ArrayDeque> slots, + final ArrayDeque> slots, final List proof) { final BytesValueRLPOutput tmp = new BytesValueRLPOutput(); tmp.startList(); @@ -88,7 +89,7 @@ public final class StorageRangeMessage extends AbstractSnapMessageData { } public SlotRangeData slotsData(final boolean withRequestId) { - final ArrayDeque> slots = new ArrayDeque<>(); + final ArrayDeque> slots = new ArrayDeque<>(); final ArrayDeque proofs = new ArrayDeque<>(); final RLPInput input = new BytesValueRLPInput(data, false); input.enterList(); @@ -120,7 +121,7 @@ public final class StorageRangeMessage extends AbstractSnapMessageData { @Value.Immutable public interface SlotRangeData { - ArrayDeque> slots(); + ArrayDeque> slots(); ArrayDeque proofs(); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java index 34c906bd9..1a82d5635 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java @@ -34,7 +34,7 @@ import org.hyperledger.besu.ethereum.eth.sync.state.PendingBlocksManager; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.trie.forest.pruner.Pruner; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.BesuMetricCategory; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/SyncMode.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/SyncMode.java index c2c1fc107..b5d0c2570 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/SyncMode.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/SyncMode.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.ethereum.eth.sync; import java.util.EnumSet; +import java.util.Locale; import org.apache.commons.lang3.StringUtils; @@ -34,10 +35,10 @@ public enum SyncMode { public String normalize() { if (this.toString().startsWith("X_")) { // removes X_ at the beginning - return StringUtils.capitalize(this.toString().substring(2).toLowerCase()); + return StringUtils.capitalize(this.toString().substring(2).toLowerCase(Locale.ROOT)); } - return StringUtils.capitalize(this.toString().toLowerCase()); + return StringUtils.capitalize(this.toString().toLowerCase(Locale.ROOT)); } public static boolean isFullSync(final SyncMode syncMode) { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java index 87a021136..7b64f885c 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java @@ -21,7 +21,7 @@ import org.hyperledger.besu.ethereum.eth.sync.ChainDownloader; import org.hyperledger.besu.ethereum.eth.sync.TrailingPeerRequirements; import org.hyperledger.besu.ethereum.eth.sync.worldstate.StalledDownloadException; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.services.tasks.TaskCollection; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStep.java index 264b29017..e9fb43ffe 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStep.java @@ -20,7 +20,7 @@ import static org.hyperledger.besu.ethereum.eth.sync.StorageExceptionManager.get import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.TrieNodeHealingRequest; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.plugin.services.exception.StorageException; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RequestDataStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RequestDataStep.java index fdde0a5ee..adbb8b736 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RequestDataStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RequestDataStep.java @@ -31,6 +31,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.AccountFlatD import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.StorageFlatDatabaseHealingRangeRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.TrieNodeHealingRequest; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; +import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.plugin.services.MetricsSystem; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncMetricsManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncMetricsManager.java index da7607de7..b8e907b2e 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncMetricsManager.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncMetricsManager.java @@ -18,6 +18,7 @@ import static io.netty.util.internal.ObjectUtil.checkNonEmpty; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncMetricsManager.Step.HEAL_TRIE; import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java index cb6ed53ce..3d93b8522 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java @@ -29,7 +29,8 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.StorageRangeDataR import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.AccountFlatDatabaseHealingRangeRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.StorageFlatDatabaseHealingRangeRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldDownloadState; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.RangeManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; @@ -46,6 +47,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.OptionalLong; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import java.util.function.Predicate; @@ -76,7 +78,7 @@ public class SnapWorldDownloadState extends WorldDownloadState protected final InMemoryTasksPriorityQueues pendingStorageFlatDatabaseHealingRequests = new InMemoryTasksPriorityQueues<>(); - private HashSet accountsHealingList = new HashSet<>(); + private Set accountsHealingList = new HashSet<>(); private DynamicPivotBlockSelector pivotBlockSelector; private final SnapSyncStatePersistenceManager snapContext; @@ -303,7 +305,7 @@ public class SnapWorldDownloadState extends WorldDownloadState } } - public synchronized void setAccountsHealingList(final HashSet addAccountToHealingList) { + public synchronized void setAccountsHealingList(final Set addAccountToHealingList) { this.accountsHealingList = addAccountToHealingList; } @@ -321,7 +323,7 @@ public class SnapWorldDownloadState extends WorldDownloadState } } - public HashSet getAccountsHealingList() { + public Set getAccountsHealingList() { return accountsHealingList; } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java index b3dae73d9..471c84a63 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java @@ -27,7 +27,8 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.context.SnapSyncStatePers import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.AccountRangeDataRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.RangeManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; @@ -35,10 +36,10 @@ import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.services.tasks.InMemoryTasksPriorityQueues; import java.time.Clock; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; @@ -153,7 +154,7 @@ public class SnapWorldStateDownloader implements WorldStateDownloader { final List currentAccountRange = snapContext.getCurrentAccountRange(); - final HashSet inconsistentAccounts = snapContext.getAccountsHealingList(); + final Set inconsistentAccounts = snapContext.getAccountsHealingList(); if (!currentAccountRange.isEmpty()) { // continue to download worldstate ranges newDownloadState.setAccountsHealingList(inconsistentAccounts); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrie.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrie.java index 01f17eb79..1cf593665 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrie.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrie.java @@ -15,12 +15,12 @@ package org.hyperledger.besu.ethereum.eth.sync.snapsync; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.trie.CommitVisitor; import org.hyperledger.besu.ethereum.trie.InnerNodeDiscoveryManager; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.Node; import org.hyperledger.besu.ethereum.trie.NodeUpdater; -import org.hyperledger.besu.ethereum.trie.SnapPutVisitor; +import org.hyperledger.besu.ethereum.trie.RangeManager; +import org.hyperledger.besu.ethereum.trie.SnapCommitVisitor; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import java.util.ArrayList; @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.NavigableMap; import java.util.Optional; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicInteger; @@ -72,7 +73,9 @@ public class StackTrie { } public void addElement( - final Bytes32 taskIdentifier, final List proofs, final TreeMap keys) { + final Bytes32 taskIdentifier, + final List proofs, + final NavigableMap keys) { this.elementsCount.addAndGet(keys.size()); this.elements.put( taskIdentifier, ImmutableTaskElement.builder().proofs(proofs).keys(keys).build()); @@ -122,7 +125,7 @@ public class StackTrie { Function.identity(), Function.identity(), startKeyHash, - keys.lastKey(), + proofs.isEmpty() ? RangeManager.MAX_RANGE : keys.lastKey(), true); final MerkleTrie trie = @@ -130,14 +133,17 @@ public class StackTrie { snapStoredNodeFactory, proofs.isEmpty() ? MerkleTrie.EMPTY_TRIE_NODE_HASH : rootHash); for (Map.Entry entry : keys.entrySet()) { - trie.put(entry.getKey(), new SnapPutVisitor<>(snapStoredNodeFactory, entry.getValue())); + trie.put(entry.getKey(), entry.getValue()); } keys.forEach(flatDatabaseUpdater::update); trie.commit( nodeUpdater, - (new CommitVisitor<>(nodeUpdater) { + (new SnapCommitVisitor<>( + nodeUpdater, + startKeyHash, + proofs.isEmpty() ? RangeManager.MAX_RANGE : keys.lastKey()) { @Override public void maybeStoreNode(final Bytes location, final Node node) { if (!node.isHealNeeded()) { @@ -180,7 +186,7 @@ public class StackTrie { } @Value.Default - public TreeMap keys() { + public NavigableMap keys() { return new TreeMap<>(); } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/context/SnapSyncStatePersistenceManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/context/SnapSyncStatePersistenceManager.java index 75ee91352..6a54dbcdf 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/context/SnapSyncStatePersistenceManager.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/context/SnapSyncStatePersistenceManager.java @@ -28,6 +28,7 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -127,7 +128,7 @@ public class SnapSyncStatePersistenceManager { .collect(Collectors.toList()); } - public HashSet getAccountsHealingList() { + public Set getAccountsHealingList() { return healContext .streamValuesFromKeysThat(notEqualsTo(SNAP_ACCOUNT_HEALING_LIST_INDEX)) .collect(Collectors.toCollection(HashSet::new)); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java index 7829ad75e..bb98534fd 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java @@ -14,12 +14,12 @@ */ package org.hyperledger.besu.ethereum.eth.sync.snapsync.request; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.MAX_RANGE; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.MIN_RANGE; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.findNewBeginElementInRange; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RequestType.ACCOUNT_RANGE; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncMetricsManager.Step.DOWNLOAD; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.StackTrie.FlatDatabaseUpdater.noop; +import static org.hyperledger.besu.ethereum.trie.RangeManager.MAX_RANGE; +import static org.hyperledger.besu.ethereum.trie.RangeManager.MIN_RANGE; +import static org.hyperledger.besu.ethereum.trie.RangeManager.findNewBeginElementInRange; import static org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator.applyForStrategy; import org.hyperledger.besu.datatypes.Hash; @@ -31,7 +31,7 @@ import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.trie.NodeUpdater; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; @@ -40,8 +40,8 @@ import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.NavigableMap; import java.util.Optional; -import java.util.TreeMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; @@ -155,7 +155,7 @@ public class AccountRangeDataRequest extends SnapDataRequest { public void addResponse( final WorldStateProofProvider worldStateProofProvider, - final TreeMap accounts, + final NavigableMap accounts, final ArrayDeque proofs) { if (!accounts.isEmpty() || !proofs.isEmpty()) { if (!worldStateProofProvider.isValidRangeProof( @@ -226,7 +226,7 @@ public class AccountRangeDataRequest extends SnapDataRequest { } @VisibleForTesting - public TreeMap getAccounts() { + public NavigableMap getAccounts() { return stackTrie.getElement(startKeyHash).keys(); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/SnapDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/SnapDataRequest.java index f3fc1f933..d9442c94c 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/SnapDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/SnapDataRequest.java @@ -31,8 +31,8 @@ import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.services.tasks.TasksPriorityProvider; -import java.util.HashSet; import java.util.Optional; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; @@ -93,7 +93,7 @@ public abstract class SnapDataRequest implements TasksPriorityProvider { } public static AccountTrieNodeHealingRequest createAccountTrieNodeDataRequest( - final Hash hash, final Bytes location, final HashSet inconsistentAccounts) { + final Hash hash, final Bytes location, final Set inconsistentAccounts) { return new AccountTrieNodeHealingRequest(hash, hash, location, inconsistentAccounts); } @@ -101,7 +101,7 @@ public abstract class SnapDataRequest implements TasksPriorityProvider { final Hash hash, final Hash rootHash, final Bytes location, - final HashSet inconsistentAccounts) { + final Set inconsistentAccounts) { return new AccountTrieNodeHealingRequest(hash, rootHash, location, inconsistentAccounts); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java index 0cb70b546..81b4897a8 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java @@ -14,16 +14,15 @@ */ package org.hyperledger.besu.ethereum.eth.sync.snapsync.request; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.MAX_RANGE; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.MIN_RANGE; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.findNewBeginElementInRange; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.getRangeCount; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RequestType.STORAGE_RANGE; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.StackTrie.FlatDatabaseUpdater.noop; +import static org.hyperledger.besu.ethereum.trie.RangeManager.MAX_RANGE; +import static org.hyperledger.besu.ethereum.trie.RangeManager.MIN_RANGE; +import static org.hyperledger.besu.ethereum.trie.RangeManager.findNewBeginElementInRange; +import static org.hyperledger.besu.ethereum.trie.RangeManager.getRangeCount; import static org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator.applyForStrategy; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncProcessState; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapWorldDownloadState; @@ -31,15 +30,16 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.StackTrie; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.trie.CompactEncoding; import org.hyperledger.besu.ethereum.trie.NodeUpdater; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.RangeManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import java.util.ArrayList; import java.util.List; +import java.util.NavigableMap; import java.util.Optional; -import java.util.TreeMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; @@ -62,7 +62,7 @@ public class StorageRangeDataRequest extends SnapDataRequest { private final Bytes32 startKeyHash; private final Bytes32 endKeyHash; - private StackTrie stackTrie; + private final StackTrie stackTrie; private Optional isProofValid; protected StorageRangeDataRequest( @@ -77,7 +77,7 @@ public class StorageRangeDataRequest extends SnapDataRequest { this.startKeyHash = startKeyHash; this.endKeyHash = endKeyHash; this.isProofValid = Optional.empty(); - addStackTrie(Optional.empty()); + this.stackTrie = new StackTrie(Hash.wrap(getStorageRoot()), startKeyHash); LOG.trace( "create get storage range data request for account {} with root hash={} from {} to {}", accountHash, @@ -129,7 +129,7 @@ public class StorageRangeDataRequest extends SnapDataRequest { public void addResponse( final SnapWorldDownloadState downloadState, final WorldStateProofProvider worldStateProofProvider, - final TreeMap slots, + final NavigableMap slots, final ArrayDeque proofs) { if (!slots.isEmpty() || !proofs.isEmpty()) { if (!worldStateProofProvider.isValidRangeProof( @@ -183,7 +183,6 @@ public class StorageRangeDataRequest extends SnapDataRequest { final StorageRangeDataRequest storageRangeDataRequest = createStorageRangeDataRequest( getRootHash(), accountHash, storageRoot, key, value); - storageRangeDataRequest.addStackTrie(Optional.of(stackTrie)); childRequests.add(storageRangeDataRequest); }); if (startKeyHash.equals(MIN_RANGE) && endKeyHash.equals(MAX_RANGE)) { @@ -203,7 +202,7 @@ public class StorageRangeDataRequest extends SnapDataRequest { return storageRoot; } - public TreeMap getSlots() { + public NavigableMap getSlots() { return stackTrie.getElement(startKeyHash).keys(); } @@ -225,11 +224,4 @@ public class StorageRangeDataRequest extends SnapDataRequest { public void setProofValid(final boolean isProofValid) { this.isProofValid = Optional.of(isProofValid); } - - public void addStackTrie(final Optional maybeStackTrie) { - stackTrie = - maybeStackTrie - .filter(StackTrie::addSegment) - .orElse(new StackTrie(Hash.wrap(getStorageRoot()), 1, 3, startKeyHash)); - } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java index 05bbeed3d..f2514b340 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java @@ -14,12 +14,11 @@ */ package org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.MAX_RANGE; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.MIN_RANGE; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncMetricsManager.Step.HEAL_FLAT; +import static org.hyperledger.besu.ethereum.trie.RangeManager.MAX_RANGE; +import static org.hyperledger.besu.ethereum.trie.RangeManager.MIN_RANGE; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; import org.hyperledger.besu.ethereum.eth.sync.snapsync.RequestType; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncProcessState; @@ -29,9 +28,10 @@ import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.CompactEncoding; import org.hyperledger.besu.ethereum.trie.MerkleTrie; +import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; @@ -41,6 +41,7 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.NavigableMap; import java.util.TreeMap; import java.util.function.Function; import java.util.stream.Stream; @@ -58,9 +59,9 @@ public class AccountFlatDatabaseHealingRangeRequest extends SnapDataRequest { private final Bytes32 startKeyHash; private final Bytes32 endKeyHash; - private TreeMap existingAccounts; + private NavigableMap existingAccounts; - private TreeMap flatDbAccounts; + private NavigableMap flatDbAccounts; private boolean isProofValid; public AccountFlatDatabaseHealingRangeRequest( @@ -132,7 +133,7 @@ public class AccountFlatDatabaseHealingRangeRequest extends SnapDataRequest { public void addLocalData( final WorldStateProofProvider worldStateProofProvider, - final TreeMap accounts, + final NavigableMap accounts, final ArrayDeque proofs) { if (!accounts.isEmpty() && !proofs.isEmpty()) { // very proof in order to check if the local flat database is valid or not diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountTrieNodeHealingRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountTrieNodeHealingRequest.java index e2f3353f2..30350f1db 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountTrieNodeHealingRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountTrieNodeHealingRequest.java @@ -34,6 +34,7 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.function.Function; import java.util.stream.Stream; @@ -43,13 +44,13 @@ import org.apache.tuweni.bytes.Bytes32; /** Represents a healing request for an account trie node. */ public class AccountTrieNodeHealingRequest extends TrieNodeHealingRequest { - private final HashSet inconsistentAccounts; + private final Set inconsistentAccounts; public AccountTrieNodeHealingRequest( final Hash hash, final Hash originalRootHash, final Bytes location, - final HashSet inconsistentAccounts) { + final Set inconsistentAccounts) { super(hash, originalRootHash, location); this.inconsistentAccounts = inconsistentAccounts; } @@ -89,7 +90,7 @@ public class AccountTrieNodeHealingRequest extends TrieNodeHealingRequest { childHash, getRootHash(), location, getSubLocation(location)); } - private HashSet getSubLocation(final Bytes location) { + private Set getSubLocation(final Bytes location) { final HashSet foundAccountsToHeal = new HashSet<>(); for (Bytes account : inconsistentAccounts) { if (account.commonPrefixLength(location) == location.size()) { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java index ac554439e..ed81fe65d 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java @@ -14,20 +14,20 @@ */ package org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.getRangeCount; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RequestType.STORAGE_RANGE; +import static org.hyperledger.besu.ethereum.trie.RangeManager.getRangeCount; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncProcessState; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapWorldDownloadState; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.trie.MerkleTrie; +import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; @@ -36,6 +36,7 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.NavigableMap; import java.util.TreeMap; import java.util.function.Function; import java.util.stream.Stream; @@ -56,7 +57,7 @@ public class StorageFlatDatabaseHealingRangeRequest extends SnapDataRequest { private final Bytes32 storageRoot; private final Bytes32 startKeyHash; private final Bytes32 endKeyHash; - private TreeMap slots; + private NavigableMap slots; private boolean isProofValid; public StorageFlatDatabaseHealingRangeRequest( @@ -120,7 +121,7 @@ public class StorageFlatDatabaseHealingRangeRequest extends SnapDataRequest { public void addLocalData( final WorldStateProofProvider worldStateProofProvider, - final TreeMap slots, + final NavigableMap slots, final ArrayDeque proofs) { if (!slots.isEmpty() && !proofs.isEmpty()) { // very proof in order to check if the local flat database is valid or not diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactions.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactions.java index b796eefac..8929e2210 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactions.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactions.java @@ -25,7 +25,6 @@ import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; import java.util.List; import java.util.Map; import java.util.NavigableMap; -import java.util.TreeMap; import java.util.TreeSet; import java.util.function.BiFunction; import java.util.function.Predicate; @@ -140,7 +139,7 @@ public abstract class AbstractPrioritizedTransactions extends AbstractSequential @Override protected void internalConsistencyCheck( - final Map> prevLayerTxsBySender) { + final Map> prevLayerTxsBySender) { super.internalConsistencyCheck(prevLayerTxsBySender); final var controlOrderByFee = new TreeSet<>(this::compareByFee); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractSequentialTransactionsLayer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractSequentialTransactionsLayer.java index 9596f6fef..245f89164 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractSequentialTransactionsLayer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractSequentialTransactionsLayer.java @@ -27,7 +27,6 @@ import java.util.Map; import java.util.NavigableMap; import java.util.OptionalLong; import java.util.Set; -import java.util.TreeMap; import java.util.function.BiFunction; public abstract class AbstractSequentialTransactionsLayer extends AbstractTransactionsLayer { @@ -142,7 +141,7 @@ public abstract class AbstractSequentialTransactionsLayer extends AbstractTransa @Override protected void internalConsistencyCheck( - final Map> prevLayerTxsBySender) { + final Map> prevLayerTxsBySender) { txsBySender.values().stream() .filter(senderTxs -> senderTxs.size() > 1) .map(NavigableMap::entrySet) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractTransactionsLayer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractTransactionsLayer.java index a323b6833..e32eb0e22 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractTransactionsLayer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractTransactionsLayer.java @@ -569,7 +569,7 @@ public abstract class AbstractTransactionsLayer implements TransactionsLayer { protected abstract String internalLogStats(); boolean consistencyCheck( - final Map> prevLayerTxsBySender) { + final Map> prevLayerTxsBySender) { final BinaryOperator noMergeExpected = (a, b) -> { throw new IllegalArgumentException(); @@ -606,7 +606,7 @@ public abstract class AbstractTransactionsLayer implements TransactionsLayer { } protected abstract void internalConsistencyCheck( - final Map> prevLayerTxsBySender); + final Map> prevLayerTxsBySender); public BlobCache getBlobCache() { return blobCache; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReadyTransactions.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReadyTransactions.java index 2820fe64b..a3dc195d1 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReadyTransactions.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReadyTransactions.java @@ -33,7 +33,6 @@ import java.util.Map; import java.util.NavigableMap; import java.util.NavigableSet; import java.util.Optional; -import java.util.TreeMap; import java.util.TreeSet; import java.util.function.BiFunction; import java.util.function.Predicate; @@ -220,7 +219,7 @@ public class ReadyTransactions extends AbstractSequentialTransactionsLayer { @Override protected void internalConsistencyCheck( - final Map> prevLayerTxsBySender) { + final Map> prevLayerTxsBySender) { super.internalConsistencyCheck(prevLayerTxsBySender); final var minNonceBySender = @@ -241,7 +240,6 @@ public class ReadyTransactions extends AbstractSequentialTransactionsLayer { : "orderByMaxFee does not match pendingTransactions"; } - assert itCurrent.hasNext() == false - : "orderByMaxFee has more elements than pendingTransactions"; + assert !itCurrent.hasNext() : "orderByMaxFee has more elements than pendingTransactions"; } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/SparseTransactions.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/SparseTransactions.java index 41e2f6d9c..e036f2fd2 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/SparseTransactions.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/SparseTransactions.java @@ -39,7 +39,6 @@ import java.util.NavigableSet; import java.util.Objects; import java.util.OptionalLong; import java.util.Set; -import java.util.TreeMap; import java.util.TreeSet; import java.util.function.BiFunction; import java.util.function.Predicate; @@ -404,7 +403,7 @@ public class SparseTransactions extends AbstractTransactionsLayer { @Override protected void internalConsistencyCheck( - final Map> prevLayerTxsBySender) { + final Map> prevLayerTxsBySender) { txsBySender.values().stream() .filter(senderTxs -> senderTxs.size() > 1) .map(NavigableMap::entrySet) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/TransactionsLayer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/TransactionsLayer.java index 85227766b..688eb6721 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/TransactionsLayer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/TransactionsLayer.java @@ -25,6 +25,7 @@ import org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult; import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.OptionalLong; @@ -106,7 +107,7 @@ public interface TransactionsLayer { private final String label; RemovalReason() { - this.label = name().toLowerCase(); + this.label = name().toLowerCase(Locale.ROOT); } public String label() { diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthServerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthServerTest.java index 7c75c6dda..7cd7bdcda 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthServerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthServerTest.java @@ -399,7 +399,7 @@ public class EthServerTest { private int calculateRlpEncodedSize(final List receipts) { final BytesValueRLPOutput rlp = new BytesValueRLPOutput(); rlp.startList(); - receipts.forEach(r -> r.writeTo(rlp)); + receipts.forEach(r -> r.writeToForNetwork(rlp)); rlp.endList(); return rlp.encodedSize(); } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetAccountRangeMessageTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetAccountRangeMessageTest.java index c8ef62a6d..a486608ef 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetAccountRangeMessageTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetAccountRangeMessageTest.java @@ -15,10 +15,10 @@ package org.hyperledger.besu.ethereum.eth.messages.snap; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.AbstractSnapMessageData; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.RawMessage; +import org.hyperledger.besu.ethereum.trie.RangeManager; import org.apache.tuweni.bytes.Bytes32; import org.assertj.core.api.Assertions; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetStorageRangeMessageTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetStorageRangeMessageTest.java index 4ff1b09b9..2492528a8 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetStorageRangeMessageTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetStorageRangeMessageTest.java @@ -15,10 +15,10 @@ package org.hyperledger.besu.ethereum.eth.messages.snap; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.AbstractSnapMessageData; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.RawMessage; +import org.hyperledger.besu.ethereum.trie.RangeManager; import java.util.List; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/StorageRangeMessageTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/StorageRangeMessageTest.java index c9179329b..da14098d0 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/StorageRangeMessageTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/StorageRangeMessageTest.java @@ -20,6 +20,7 @@ import org.hyperledger.besu.ethereum.p2p.rlpx.wire.RawMessage; import java.util.ArrayList; import java.util.List; +import java.util.NavigableMap; import java.util.TreeMap; import kotlin.collections.ArrayDeque; @@ -33,7 +34,7 @@ public final class StorageRangeMessageTest { @Test public void roundTripTest() { - final ArrayDeque> keys = new ArrayDeque<>(); + final ArrayDeque> keys = new ArrayDeque<>(); final TreeMap storage = new TreeMap<>(); storage.put(Hash.wrap(Bytes32.leftPad(Bytes.of(1))), Bytes32.random()); keys.add(storage); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointBlockImportStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointBlockImportStepTest.java index 13b471879..4813cf84d 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointBlockImportStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointBlockImportStepTest.java @@ -52,7 +52,8 @@ public class CheckPointBlockImportStepTest { new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions()); + new MainnetBlockHeaderFunctions(), + false); blockchain = DefaultBlockchain.createMutable( generateBlock(0), blockchainStorage, mock(MetricsSystem.class), 0); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java index 822ac5095..e5c8ce555 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java @@ -36,7 +36,7 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.checkpoint.Checkpoint; import org.hyperledger.besu.ethereum.eth.sync.fastsync.checkpoint.ImmutableCheckpoint; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java index 765eac4c5..e100c0369 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java @@ -31,7 +31,7 @@ import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate.FastDownloaderFactory; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java index 0c625c613..ae252a39f 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java @@ -34,7 +34,7 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate.FastWorldState import org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate.NodeDataRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.StalledDownloadException; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java index 3555a9b71..44847ffb9 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java @@ -26,7 +26,7 @@ import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.eth.manager.task.EthTask; import org.hyperledger.besu.ethereum.eth.sync.worldstate.StalledDownloadException; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloadProcess; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java index e4a827b7f..6707ed970 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java @@ -22,7 +22,7 @@ import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java index 28001a2a3..03aa292fa 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java @@ -14,7 +14,7 @@ */ package org.hyperledger.besu.ethereum.eth.sync.snapsync; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.MAX_RANGE; +import static org.hyperledger.besu.ethereum.trie.RangeManager.MAX_RANGE; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -31,7 +31,7 @@ import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.trie.patricia.StoredNodeFactory; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java index 9a53a4418..0de00f5ab 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java @@ -25,7 +25,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.AccountRangeDataR import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.BytecodeRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.StorageRangeDataRequest; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManagerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManagerTest.java index 40b967096..8ee3784d2 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManagerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManagerTest.java @@ -20,6 +20,7 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.TrieGenerator; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.trie.MerkleTrie; +import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java index 0dcc20442..35c5de5da 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java @@ -38,7 +38,8 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.context.SnapSyncStatePers import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.BytecodeRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloadProcess; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.RangeManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrieTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrieTest.java index b1b3800a1..4fcb21d92 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrieTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrieTest.java @@ -18,12 +18,15 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.TrieGenerator; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.trie.MerkleTrie; +import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; +import java.util.ArrayList; import java.util.List; import java.util.TreeMap; @@ -143,4 +146,127 @@ public class StackTrieTest { worldStateKeyValueStorage.getAccountStateTrieNode(accountStateTrie.getRootHash())) .isPresent(); } + + @Test + public void shouldNotSaveNodeWithChildNotInTheRange() { + final ForestWorldStateKeyValueStorage worldStateStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final WorldStateStorageCoordinator worldStateStorageCoordinator = + new WorldStateStorageCoordinator(worldStateStorage); + + final MerkleTrie trie = + new StoredMerklePatriciaTrie<>( + (location, hash) -> worldStateStorage.getAccountStateTrieNode(hash).map(Bytes::wrap), + b -> b, + b -> b); + + trie.put(Bytes32.rightPad(Bytes.of(0x10)), Bytes.of(0x01)); + trie.put(Bytes32.rightPad(Bytes.of(0x11)), Bytes.of(0x01)); + trie.put(Bytes32.rightPad(Bytes.of(0x20)), Bytes.of(0x01)); + trie.put(Bytes32.rightPad(Bytes.of(0x21)), Bytes.of(0x01)); + trie.put(Bytes32.rightPad(Bytes.of(0x01)), Bytes.of(0x02)); + trie.put(Bytes32.rightPad(Bytes.of(0x02)), Bytes.of(0x03)); + trie.put(Bytes32.rightPad(Bytes.of(0x03)), Bytes.of(0x04)); + + final ForestWorldStateKeyValueStorage.Updater updater = worldStateStorage.updater(); + trie.commit((location, hash, value) -> updater.putAccountStateTrieNode(hash, value)); + updater.commit(); + + final Bytes32 startRange = Bytes32.rightPad(Bytes.of(0x02)); + + final RangeStorageEntriesCollector collector = + RangeStorageEntriesCollector.createCollector( + startRange, RangeManager.MAX_RANGE, 15, Integer.MAX_VALUE); + final TrieIterator visitor = RangeStorageEntriesCollector.createVisitor(collector); + final TreeMap entries = + (TreeMap) + trie.entriesFrom( + root -> + RangeStorageEntriesCollector.collectEntries( + collector, visitor, root, startRange)); + + final WorldStateProofProvider worldStateProofProvider = + new WorldStateProofProvider(worldStateStorageCoordinator); + + // generate the proof + final List proofs = + worldStateProofProvider.getAccountProofRelatedNodes( + Hash.wrap(trie.getRootHash()), startRange); + proofs.addAll( + worldStateProofProvider.getAccountProofRelatedNodes( + Hash.wrap(trie.getRootHash()), entries.lastKey())); + + // try to commit with stack trie + final ForestWorldStateKeyValueStorage recreatedWorldStateStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final StackTrie stackTrie = new StackTrie(Hash.wrap(trie.getRootHash()), 0, 256, startRange); + stackTrie.addSegment(); + stackTrie.addElement(Bytes32.random(), proofs, entries); + final ForestWorldStateKeyValueStorage.Updater updaterStackTrie = + recreatedWorldStateStorage.updater(); + stackTrie.commit( + (location, hash, value) -> updaterStackTrie.putAccountStateTrieNode(hash, value)); + updaterStackTrie.commit(); + + // verify the state of the db + Assertions.assertThat(worldStateStorage.getAccountStateTrieNode(trie.getRootHash())) + .isPresent(); + Assertions.assertThat(recreatedWorldStateStorage.getAccountStateTrieNode(trie.getRootHash())) + .isNotPresent(); + } + + @Test + public void shouldSaveNodeWithAllChildsInTheRange() { + final ForestWorldStateKeyValueStorage worldStateStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + + final MerkleTrie trie = + new StoredMerklePatriciaTrie<>( + (location, hash) -> worldStateStorage.getAccountStateTrieNode(hash).map(Bytes::wrap), + b -> b, + b -> b); + + trie.put(Bytes32.rightPad(Bytes.of(0x10)), Bytes.of(0x01)); + trie.put(Bytes32.rightPad(Bytes.of(0x11)), Bytes.of(0x01)); + trie.put(Bytes32.rightPad(Bytes.of(0x20)), Bytes.of(0x01)); + trie.put(Bytes32.rightPad(Bytes.of(0x21)), Bytes.of(0x01)); + trie.put(Bytes32.rightPad(Bytes.of(0x01)), Bytes.of(0x02)); + trie.put(Bytes32.rightPad(Bytes.of(0x02)), Bytes.of(0x03)); + trie.put(Bytes32.rightPad(Bytes.of(0x03)), Bytes.of(0x04)); + + final ForestWorldStateKeyValueStorage.Updater updater = worldStateStorage.updater(); + trie.commit((location, hash, value) -> updater.putAccountStateTrieNode(hash, value)); + updater.commit(); + + final Bytes32 startRange = Bytes32.rightPad(Bytes.of(0x00)); + + final RangeStorageEntriesCollector collector = + RangeStorageEntriesCollector.createCollector( + startRange, RangeManager.MAX_RANGE, 15, Integer.MAX_VALUE); + final TrieIterator visitor = RangeStorageEntriesCollector.createVisitor(collector); + final TreeMap entries = + (TreeMap) + trie.entriesFrom( + root -> + RangeStorageEntriesCollector.collectEntries( + collector, visitor, root, startRange)); + + // try to commit with stack trie + final ForestWorldStateKeyValueStorage recreatedWorldStateStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final StackTrie stackTrie = new StackTrie(Hash.wrap(trie.getRootHash()), 0, 256, startRange); + stackTrie.addSegment(); + stackTrie.addElement(Bytes32.random(), new ArrayList<>(), entries); + final ForestWorldStateKeyValueStorage.Updater updaterStackTrie = + recreatedWorldStateStorage.updater(); + stackTrie.commit( + (location, hash, value) -> updaterStackTrie.putAccountStateTrieNode(hash, value)); + updaterStackTrie.commit(); + + // verify the state of the db + Assertions.assertThat(worldStateStorage.getAccountStateTrieNode(trie.getRootHash())) + .isPresent(); + Assertions.assertThat(recreatedWorldStateStorage.getAccountStateTrieNode(trie.getRootHash())) + .isPresent(); + } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java index 63895f4e6..81b9d3857 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java @@ -24,9 +24,10 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.StorageRangeDataR import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.MerkleTrie; +import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; @@ -50,6 +51,7 @@ public class TaskGenerator { new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_BONSAI_CONFIG); + final WorldStateStorageCoordinator worldStateStorageCoordinator = new WorldStateStorageCoordinator(worldStateKeyValueStorage); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java index 2799d6319..dba37b5af 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java @@ -17,7 +17,6 @@ package org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.core.TrieGenerator; -import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncMetricsManager; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncProcessState; @@ -27,9 +26,10 @@ import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.CompactEncoding; import org.hyperledger.besu.ethereum.trie.MerkleTrie; +import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java index 41d026513..da5354f71 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java @@ -20,7 +20,6 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.core.TrieGenerator; -import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncProcessState; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapWorldDownloadState; @@ -29,9 +28,10 @@ import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.MerkleTrie; +import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; @@ -89,7 +89,6 @@ class StorageFlatDatabaseHealingRangeRequestTest { DataStorageConfiguration.DEFAULT_BONSAI_CONFIG); worldStateStorageCoordinator = new WorldStateStorageCoordinator(worldStateKeyValueStorage); proofProvider = new WorldStateProofProvider(worldStateStorageCoordinator); - trie = TrieGenerator.generateTrie( worldStateStorageCoordinator, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java index 1f87a2d78..5cec07c15 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java @@ -21,7 +21,7 @@ import org.hyperledger.besu.ethereum.core.TrieGenerator; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.MerkleTrie; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/DataStoreModule.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/DataStoreModule.java index 65585b644..05dff0941 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/DataStoreModule.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/DataStoreModule.java @@ -142,6 +142,9 @@ public class DataStoreModule { @Named("variables") final KeyValueStorage variablesKeyValueStorage, final BlockHeaderFunctions blockHashFunction) { return new KeyValueStoragePrefixedKeyBlockchainStorage( - keyValueStorage, new VariablesKeyValueStorage(variablesKeyValueStorage), blockHashFunction); + keyValueStorage, + new VariablesKeyValueStorage(variablesKeyValueStorage), + blockHashFunction, + false); } } diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/benchmarks/BenchmarkExecutor.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/benchmarks/BenchmarkExecutor.java index 730dd833f..ccc9152f7 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/benchmarks/BenchmarkExecutor.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/benchmarks/BenchmarkExecutor.java @@ -36,6 +36,7 @@ import org.hyperledger.besu.evm.gascalculator.ShanghaiGasCalculator; import org.hyperledger.besu.evm.precompile.PrecompiledContract; import java.io.PrintStream; +import java.util.Locale; import java.util.concurrent.TimeUnit; import com.google.common.base.Stopwatch; @@ -127,7 +128,7 @@ public abstract class BenchmarkExecutor { * @return a gas calculator */ public static GasCalculator gasCalculatorForFork(final String fork) { - return switch (EvmSpecVersion.valueOf(fork.toUpperCase())) { + return switch (EvmSpecVersion.valueOf(fork.toUpperCase(Locale.ROOT))) { case HOMESTEAD -> new HomesteadGasCalculator(); case FRONTIER -> new FrontierGasCalculator(); case BYZANTIUM -> new ByzantiumGasCalculator(); diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryStatus.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryStatus.java index dc7049156..5311a2212 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryStatus.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryStatus.java @@ -14,6 +14,8 @@ */ package org.hyperledger.besu.ethereum.p2p.discovery; +import java.util.Locale; + /** The status of a {@link DiscoveryPeer}, in relation to the peer discovery state machine. */ public enum PeerDiscoveryStatus { @@ -40,6 +42,6 @@ public enum PeerDiscoveryStatus { @Override public String toString() { - return name().toLowerCase(); + return name().toLowerCase(Locale.ROOT); } } diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/peers/EnodeURLImpl.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/peers/EnodeURLImpl.java index 55ca94a1a..e1d158f3f 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/peers/EnodeURLImpl.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/peers/EnodeURLImpl.java @@ -23,6 +23,7 @@ import org.hyperledger.besu.util.NetworkUtility; import java.net.InetAddress; import java.net.URI; import java.net.UnknownHostException; +import java.util.Locale; import java.util.Objects; import java.util.Optional; import java.util.OptionalInt; @@ -157,7 +158,7 @@ public class EnodeURLImpl implements EnodeURL { public static Bytes parseNodeId(final String nodeId) { int expectedSize = EnodeURLImpl.NODE_ID_SIZE * 2; - if (nodeId.toLowerCase().startsWith("0x")) { + if (nodeId.toLowerCase(Locale.ROOT).startsWith("0x")) { expectedSize += 2; } checkArgument( diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/wire/Capability.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/wire/Capability.java index 5a3a57df4..f530337ee 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/wire/Capability.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/wire/Capability.java @@ -84,7 +84,7 @@ public class Capability { } @Override - /** Returned string is sanitized since it contains user input */ + // Returned string is sanitized since it contains user input public String toString() { return Encode.forJava(name) + "/" + version; } diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/wire/PeerInfo.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/wire/PeerInfo.java index bd98270e0..492280e85 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/wire/PeerInfo.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/wire/PeerInfo.java @@ -118,7 +118,7 @@ public class PeerInfo implements Comparable { } @Override - /** Returned string is sanitized since it contains user input */ + // Returned string is sanitized since it contains user input public String toString() { final StringBuilder sb = new StringBuilder("PeerInfo{"); sb.append("version=").append(version); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTimestampsTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTimestampsTest.java index 90ee8b422..d75847050 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTimestampsTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTimestampsTest.java @@ -21,7 +21,6 @@ import org.hyperledger.besu.ethereum.p2p.discovery.internal.MockPeerDiscoveryAge import org.hyperledger.besu.ethereum.p2p.discovery.internal.Packet; import java.util.Collections; -import java.util.concurrent.atomic.AtomicLong; import org.junit.jupiter.api.Test; @@ -41,8 +40,8 @@ public class PeerDiscoveryTimestampsTest { final Packet pong = helper.createPongPacket(agent, Hash.hash(agentPing.getHash())); helper.sendMessageBetweenAgents(testAgent, agent, pong); - final AtomicLong lastSeen = new AtomicLong(); - final AtomicLong firstDiscovered = new AtomicLong(); + long lastSeen; + long firstDiscovered; assertThat(agent.streamDiscoveredPeers()).hasSize(1); @@ -50,16 +49,16 @@ public class PeerDiscoveryTimestampsTest { assertThat(p.getLastSeen()).isGreaterThan(0); assertThat(p.getFirstDiscovered()).isGreaterThan(0); - lastSeen.set(p.getLastSeen()); - firstDiscovered.set(p.getFirstDiscovered()); + lastSeen = p.getLastSeen(); + firstDiscovered = p.getFirstDiscovered(); helper.sendMessageBetweenAgents(testAgent, agent, testAgentPing); assertThat(agent.streamDiscoveredPeers()).hasSize(1); p = agent.streamDiscoveredPeers().iterator().next(); - assertThat(p.getLastSeen()).isGreaterThan(lastSeen.get()); - assertThat(p.getFirstDiscovered()).isEqualTo(firstDiscovered.get()); + assertThat(p.getLastSeen()).isGreaterThan(lastSeen); + assertThat(p.getFirstDiscovered()).isEqualTo(firstDiscovered); } @Test @@ -74,9 +73,9 @@ public class PeerDiscoveryTimestampsTest { assertThat(agent.streamDiscoveredPeers()).hasSize(1); - final AtomicLong lastContacted = new AtomicLong(); - final AtomicLong lastSeen = new AtomicLong(); - final AtomicLong firstDiscovered = new AtomicLong(); + final long lastContacted; + final long lastSeen; + final long firstDiscovered; DiscoveryPeer peer = agent.streamDiscoveredPeers().iterator().next(); final long lc = peer.getLastContacted(); @@ -87,9 +86,9 @@ public class PeerDiscoveryTimestampsTest { assertThat(ls).isGreaterThan(0); assertThat(fd).isGreaterThan(0); - lastContacted.set(lc); - lastSeen.set(ls); - firstDiscovered.set(fd); + lastContacted = lc; + lastSeen = ls; + firstDiscovered = fd; // Send another packet and ensure that timestamps are updated accordingly. // Sleep beforehand to make sure timestamps will be different. @@ -102,8 +101,8 @@ public class PeerDiscoveryTimestampsTest { peer = agent.streamDiscoveredPeers().iterator().next(); - assertThat(peer.getLastContacted()).isGreaterThan(lastContacted.get()); - assertThat(peer.getLastSeen()).isGreaterThan(lastSeen.get()); - assertThat(peer.getFirstDiscovered()).isEqualTo(firstDiscovered.get()); + assertThat(peer.getLastContacted()).isGreaterThan(lastContacted); + assertThat(peer.getLastSeen()).isGreaterThan(lastSeen); + assertThat(peer.getFirstDiscovered()).isEqualTo(firstDiscovered); } } diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/MockConnectionInitializer.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/MockConnectionInitializer.java index 814e9a797..64b7335cd 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/MockConnectionInitializer.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/MockConnectionInitializer.java @@ -19,6 +19,7 @@ import org.hyperledger.besu.ethereum.p2p.rlpx.ConnectCallback; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.messages.DisconnectMessage; import org.hyperledger.besu.util.Subscribers; +import java.net.InetAddress; import java.net.InetSocketAddress; import java.util.HashMap; import java.util.Map; @@ -58,7 +59,7 @@ public class MockConnectionInitializer implements ConnectionInitializer { @Override public CompletableFuture start() { final InetSocketAddress socketAddress = - new InetSocketAddress("127.0.0.1", NEXT_PORT.incrementAndGet()); + new InetSocketAddress(InetAddress.getLoopbackAddress(), NEXT_PORT.incrementAndGet()); return CompletableFuture.completedFuture(socketAddress); } diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java index 952ebf7c2..72a939f7f 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java @@ -55,6 +55,7 @@ import org.hyperledger.besu.ethereum.rlp.RLPException; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.data.EnodeURL; +import java.net.InetAddress; import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Arrays; @@ -92,7 +93,8 @@ public class DeFramerTest { private final PeerConnection peerConnection = mock(PeerConnection.class); private final CompletableFuture connectFuture = new CompletableFuture<>(); private final int remotePort = 12345; - private final InetSocketAddress remoteAddress = new InetSocketAddress("127.0.0.1", remotePort); + private final InetSocketAddress remoteAddress = + new InetSocketAddress(InetAddress.getLoopbackAddress(), remotePort); private final int p2pVersion = 5; private final String clientId = "abc"; diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java index c64bbec90..ee4730ffd 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java @@ -17,11 +17,13 @@ package org.hyperledger.besu.ethereum.referencetests; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiAccount; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiValue; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiPreImageProxy; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiPreImageProxy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.Consumer; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.StorageConsumingMap; import org.hyperledger.besu.evm.internal.EvmConfiguration; import java.util.concurrent.ConcurrentHashMap; @@ -32,8 +34,8 @@ public class BonsaiReferenceTestUpdateAccumulator extends BonsaiWorldStateUpdate private final BonsaiPreImageProxy preImageProxy; public BonsaiReferenceTestUpdateAccumulator( - final BonsaiWorldView world, - final Consumer> accountPreloader, + final DiffBasedWorldView world, + final Consumer> accountPreloader, final Consumer storagePreloader, final BonsaiPreImageProxy preImageProxy, final EvmConfiguration evmConfiguration) { @@ -65,7 +67,7 @@ public class BonsaiReferenceTestUpdateAccumulator extends BonsaiWorldStateUpdate getStorageToUpdate() .forEach( (k, v) -> { - StorageConsumingMap> newMap = + StorageConsumingMap> newMap = new StorageConsumingMap<>(k, new ConcurrentHashMap<>(), v.getConsumer()); v.forEach((key, value) -> newMap.put(key, value.copy())); copy.getStorageToUpdate().put(k, newMap); diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java index 1c536e96a..f16452f93 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java @@ -18,16 +18,18 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.NoOpCachedWorldStorageManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiPreImageProxy; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateLayerStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogAddedEvent; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.NoOpBonsaiCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiPreImageProxy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateLayerStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogAddedEvent; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.worldstate.WorldUpdater; @@ -54,14 +56,14 @@ public class BonsaiReferenceTestWorldState extends BonsaiWorldState protected BonsaiReferenceTestWorldState( final BonsaiReferenceTestWorldStateStorage worldStateKeyValueStorage, - final CachedMerkleTrieLoader cachedMerkleTrieLoader, - final CachedWorldStorageManager cachedWorldStorageManager, + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader, + final DiffBasedCachedWorldStorageManager cachedWorldStorageManager, final TrieLogManager trieLogManager, final BonsaiPreImageProxy preImageProxy, final EvmConfiguration evmConfiguration) { super( worldStateKeyValueStorage, - cachedMerkleTrieLoader, + bonsaiCachedMerkleTrieLoader, cachedWorldStorageManager, trieLogManager, evmConfiguration); @@ -72,21 +74,21 @@ public class BonsaiReferenceTestWorldState extends BonsaiWorldState new BonsaiReferenceTestUpdateAccumulator( this, (addr, value) -> - cachedMerkleTrieLoader.preLoadAccount( + bonsaiCachedMerkleTrieLoader.preLoadAccount( getWorldStateStorage(), worldStateRootHash, addr), (addr, value) -> - cachedMerkleTrieLoader.preLoadStorageSlot(getWorldStateStorage(), addr, value), + bonsaiCachedMerkleTrieLoader.preLoadStorageSlot( + getWorldStateStorage(), addr, value), preImageProxy, evmConfiguration)); } @Override public ReferenceTestWorldState copy() { - var layerCopy = - new BonsaiReferenceTestWorldStateStorage(worldStateKeyValueStorage, preImageProxy); + var layerCopy = new BonsaiReferenceTestWorldStateStorage(getWorldStateStorage(), preImageProxy); return new BonsaiReferenceTestWorldState( layerCopy, - cachedMerkleTrieLoader, + bonsaiCachedMerkleTrieLoader, cachedWorldStorageManager, trieLogManager, preImageProxy, @@ -187,8 +189,9 @@ public class BonsaiReferenceTestWorldState extends BonsaiWorldState private BonsaiWorldState createBonsaiWorldState(final boolean isFrozen) { BonsaiWorldState bonsaiWorldState = new BonsaiWorldState( - new BonsaiWorldStateLayerStorage(worldStateKeyValueStorage), - cachedMerkleTrieLoader, + new BonsaiWorldStateLayerStorage( + (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage), + bonsaiCachedMerkleTrieLoader, cachedWorldStorageManager, trieLogManager, evmConfiguration); @@ -209,8 +212,11 @@ public class BonsaiReferenceTestWorldState extends BonsaiWorldState final Map accounts, final EvmConfiguration evmConfiguration) { final ObservableMetricsSystem metricsSystem = new NoOpMetricsSystem(); - final CachedMerkleTrieLoader cachedMerkleTrieLoader = new CachedMerkleTrieLoader(metricsSystem); + + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader = + new BonsaiCachedMerkleTrieLoader(metricsSystem); final TrieLogManager trieLogManager = new ReferenceTestsInMemoryTrieLogManager(); + final BonsaiPreImageProxy preImageProxy = new BonsaiPreImageProxy.BonsaiReferenceTestPreImageProxy(); @@ -223,13 +229,13 @@ public class BonsaiReferenceTestWorldState extends BonsaiWorldState final BonsaiReferenceTestWorldStateStorage worldStateKeyValueStorage = new BonsaiReferenceTestWorldStateStorage(bonsaiWorldStateKeyValueStorage, preImageProxy); - final NoOpCachedWorldStorageManager noOpCachedWorldStorageManager = - new NoOpCachedWorldStorageManager(bonsaiWorldStateKeyValueStorage); + final NoOpBonsaiCachedWorldStorageManager noOpCachedWorldStorageManager = + new NoOpBonsaiCachedWorldStorageManager(bonsaiWorldStateKeyValueStorage); final BonsaiReferenceTestWorldState worldState = new BonsaiReferenceTestWorldState( worldStateKeyValueStorage, - cachedMerkleTrieLoader, + bonsaiCachedMerkleTrieLoader, noOpCachedWorldStorageManager, trieLogManager, preImageProxy, @@ -260,10 +266,10 @@ public class BonsaiReferenceTestWorldState extends BonsaiWorldState @Override public synchronized void saveTrieLog( - final BonsaiWorldStateUpdateAccumulator localUpdater, + final DiffBasedWorldStateUpdateAccumulator localUpdater, final Hash forWorldStateRootHash, final BlockHeader forBlockHeader, - final BonsaiWorldState forWorldState) { + final DiffBasedWorldState forWorldState) { // notify trie log added observers, synchronously TrieLog trieLog = trieLogFactory.create(localUpdater, forBlockHeader); trieLogCache.put(forBlockHeader.getHash(), trieLogFactory.serialize(trieLog)); diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java index 68057523a..0993d4425 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java @@ -16,11 +16,11 @@ package org.hyperledger.besu.ethereum.referencetests; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiAccount; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiPreImageProxy; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateLayerStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiPreImageProxy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateLayerStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; import org.hyperledger.besu.evm.account.AccountStorageEntry; import org.hyperledger.besu.evm.worldstate.WorldState; @@ -64,7 +64,7 @@ public class BonsaiReferenceTestWorldStateStorage extends BonsaiWorldStateLayerS } public Stream streamAccounts( - final BonsaiWorldView context, final Bytes32 startKeyHash, final int limit) { + final DiffBasedWorldView context, final Bytes32 startKeyHash, final int limit) { return streamFlatAccounts(startKeyHash, UInt256.MAX_VALUE, limit) .entrySet() // map back to addresses using preImage provider: diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestBlockchain.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestBlockchain.java index cfe577bf7..f73e27139 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestBlockchain.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestBlockchain.java @@ -188,6 +188,7 @@ public class ReferenceTestBlockchain implements Blockchain { } @Override + @SuppressWarnings("unused") public Comparator getBlockChoiceRule() { return (a, b) -> { throw new NonDeterministicOperationException( diff --git a/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/RetestethContext.java b/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/RetestethContext.java index e34d9f447..cad559ae1 100644 --- a/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/RetestethContext.java +++ b/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/RetestethContext.java @@ -279,7 +279,7 @@ public class RetestethContext { return DefaultBlockchain.createMutable( genesisBlock, new KeyValueStoragePrefixedKeyBlockchainStorage( - keyValueStorage, variablesStorage, blockHeaderFunctions), + keyValueStorage, variablesStorage, blockHeaderFunctions, false), new NoOpMetricsSystem(), 100); } diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/InnerNodeDiscoveryManager.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/InnerNodeDiscoveryManager.java index 5190ada7e..d20b47e9e 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/InnerNodeDiscoveryManager.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/InnerNodeDiscoveryManager.java @@ -14,6 +14,9 @@ */ package org.hyperledger.besu.ethereum.trie; +import static org.hyperledger.besu.ethereum.trie.RangeManager.createPath; +import static org.hyperledger.besu.ethereum.trie.RangeManager.isInRange; + import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.trie.patricia.BranchNode; import org.hyperledger.besu.ethereum.trie.patricia.ExtensionNode; @@ -21,7 +24,6 @@ import org.hyperledger.besu.ethereum.trie.patricia.LeafNode; import org.hyperledger.besu.ethereum.trie.patricia.StoredNodeFactory; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.function.Function; @@ -37,7 +39,7 @@ public class InnerNodeDiscoveryManager extends StoredNodeFactory { private final List innerNodes = new ArrayList<>(); - private final Bytes startKeyHash, endKeyHash; + private final Bytes startKeyPath, endKeyPath; private final boolean allowMissingElementInRange; @@ -49,8 +51,8 @@ public class InnerNodeDiscoveryManager extends StoredNodeFactory { final Bytes32 endKeyHash, final boolean allowMissingElementInRange) { super(nodeLoader, valueSerializer, valueDeserializer); - this.startKeyHash = createPath(startKeyHash); - this.endKeyHash = createPath(endKeyHash); + this.startKeyPath = createPath(startKeyHash); + this.endKeyPath = createPath(endKeyHash); this.allowMissingElementInRange = allowMissingElementInRange; } @@ -62,7 +64,7 @@ public class InnerNodeDiscoveryManager extends StoredNodeFactory { final Supplier errMessage) { final ExtensionNode vNode = (ExtensionNode) super.decodeExtension(location, path, valueRlp, errMessage); - if (isInRange(Bytes.concatenate(location, Bytes.of(0)))) { + if (isInRange(Bytes.concatenate(location, Bytes.of(0)), startKeyPath, endKeyPath)) { innerNodes.add( ImmutableInnerNode.builder() .location(location) @@ -78,7 +80,7 @@ public class InnerNodeDiscoveryManager extends StoredNodeFactory { final BranchNode vBranchNode = super.decodeBranch(location, nodeRLPs, errMessage); final List> children = vBranchNode.getChildren(); for (int i = 0; i < children.size(); i++) { - if (isInRange(Bytes.concatenate(location, Bytes.of(i)))) { + if (isInRange(Bytes.concatenate(location, Bytes.of(i)), startKeyPath, endKeyPath)) { innerNodes.add( ImmutableInnerNode.builder() .location(location) @@ -97,7 +99,7 @@ public class InnerNodeDiscoveryManager extends StoredNodeFactory { final Supplier errMessage) { final LeafNode vLeafNode = super.decodeLeaf(location, path, valueRlp, errMessage); final Bytes concatenatePath = Bytes.concatenate(location, path); - if (isInRange(concatenatePath.slice(0, concatenatePath.size() - 1))) { + if (isInRange(concatenatePath.slice(0, concatenatePath.size() - 1), startKeyPath, endKeyPath)) { innerNodes.add(ImmutableInnerNode.builder().location(location).path(path).build()); } return vLeafNode; @@ -106,10 +108,16 @@ public class InnerNodeDiscoveryManager extends StoredNodeFactory { @Override public Optional> retrieve(final Bytes location, final Bytes32 hash) throws MerkleTrieException { + return super.retrieve(location, hash) + .map( + vNode -> { + vNode.markDirty(); + return vNode; + }) .or( () -> { - if (!allowMissingElementInRange && isInRange(location)) { + if (!allowMissingElementInRange && isInRange(location, startKeyPath, endKeyPath)) { return Optional.empty(); } return Optional.of(new MissingNode<>(hash, location)); @@ -120,23 +128,6 @@ public class InnerNodeDiscoveryManager extends StoredNodeFactory { return List.copyOf(innerNodes); } - private boolean isInRange(final Bytes location) { - return !location.isEmpty() - && Arrays.compare(location.toArrayUnsafe(), startKeyHash.toArrayUnsafe()) >= 0 - && Arrays.compare(location.toArrayUnsafe(), endKeyHash.toArrayUnsafe()) <= 0; - } - - private Bytes createPath(final Bytes bytes) { - final MutableBytes path = MutableBytes.create(bytes.size() * 2); - int j = 0; - for (int i = 0; i < bytes.size(); i += 1, j += 2) { - final byte b = bytes.get(i); - path.set(j, (byte) ((b >>> 4) & 0x0f)); - path.set(j + 1, (byte) (b & 0x0f)); - } - return path; - } - public static Bytes32 decodePath(final Bytes bytes) { final MutableBytes32 decoded = MutableBytes32.create(); final MutableBytes path = MutableBytes.create(Bytes32.SIZE * 2); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManager.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/RangeManager.java similarity index 72% rename from ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManager.java rename to ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/RangeManager.java index 83f58c10f..f0699831d 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManager.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/RangeManager.java @@ -12,23 +12,24 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.eth.sync.snapsync; +package org.hyperledger.besu.ethereum.trie; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.trie.InnerNodeDiscoveryManager; -import org.hyperledger.besu.ethereum.trie.MerkleTrieException; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import java.math.BigInteger; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.NavigableMap; import java.util.Optional; import java.util.TreeMap; import java.util.function.Function; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.bytes.MutableBytes; /** * This class helps to generate ranges according to several parameters (the start and the end of the @@ -43,7 +44,7 @@ public class RangeManager { private RangeManager() {} public static int getRangeCount( - final Bytes32 min, final Bytes32 max, final TreeMap items) { + final Bytes32 min, final Bytes32 max, final NavigableMap items) { if (min.equals(MIN_RANGE) && max.equals(MAX_RANGE)) { return MAX_RANGE .toUnsignedBigInteger() @@ -120,7 +121,7 @@ public class RangeManager { public static Optional findNewBeginElementInRange( final Bytes32 worldstateRootHash, final List proofs, - final TreeMap receivedKeys, + final NavigableMap receivedKeys, final Bytes32 endKeyHash) { if (receivedKeys.isEmpty() || receivedKeys.lastKey().compareTo(endKeyHash) >= 0) { return Optional.empty(); @@ -152,4 +153,42 @@ public class RangeManager { private static Bytes32 format(final BigInteger data) { return Bytes32.leftPad(Bytes.of(data.toByteArray()).trimLeadingZeros()); } + + /** + * Checks if a given location is within a specified range. This method determines whether a given + * location (represented as {@link Bytes}) falls within the range defined by a start key path and + * an end key path. + * + * @param location The location to check, represented as {@link Bytes}. + * @param startKeyPath The start of the range as path, represented as {@link Bytes}. + * @param endKeyPath The end of the range as path, represented as {@link Bytes}. + * @return {@code true} if the location is within the range (inclusive); {@code false} otherwise. + */ + public static boolean isInRange( + final Bytes location, final Bytes startKeyPath, final Bytes endKeyPath) { + final MutableBytes path = MutableBytes.create(Bytes32.SIZE * 2); + path.set(0, location); + return !location.isEmpty() + && Arrays.compare(path.toArrayUnsafe(), startKeyPath.toArrayUnsafe()) >= 0 + && Arrays.compare(path.toArrayUnsafe(), endKeyPath.toArrayUnsafe()) <= 0; + } + + /** + * Transforms a list of bytes into a path. This method processes a sequence of bytes, expanding + * each byte into two separate bytes to form a new path. The resulting path will have twice the + * length of the input byte sequence. + * + * @param bytes The byte sequence to be transformed into a path. + * @return A {@link Bytes} object representing the newly formed path. + */ + public static Bytes createPath(final Bytes bytes) { + final MutableBytes path = MutableBytes.create(bytes.size() * 2); + int j = 0; + for (int i = 0; i < bytes.size(); i += 1, j += 2) { + final byte b = bytes.get(i); + path.set(j, (byte) ((b >>> 4) & 0x0f)); + path.set(j + 1, (byte) (b & 0x0f)); + } + return path; + } } diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/SnapCommitVisitor.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/SnapCommitVisitor.java new file mode 100644 index 000000000..099674e22 --- /dev/null +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/SnapCommitVisitor.java @@ -0,0 +1,138 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie; + +import static org.hyperledger.besu.ethereum.trie.RangeManager.createPath; + +import org.hyperledger.besu.ethereum.trie.patricia.BranchNode; +import org.hyperledger.besu.ethereum.trie.patricia.ExtensionNode; + +import java.util.Arrays; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.bytes.MutableBytes; + +/** + * Implements a visitor for persisting changes to nodes during a snap synchronization process, + * focusing specifically on nodes that are marked as "dirty" but not as "heal needed". + * + *

This visitor plays a crucial role in the snap synchronization by identifying nodes that have + * been modified and require persistence ("dirty" nodes). The key functionality of this visitor is + * its selective persistence approach: it only persists changes to dirty nodes that are not marked + * as "heal needed". This strategy is designed to prevent future inconsistencies within the tree by + * ensuring that only nodes that are both modified and currently consistent with the rest of the + * structure are persisted. Nodes marked as "heal needed" are excluded from immediate persistence to + * allow for their proper healing in a controlled manner, ensuring the integrity and consistency of + * the data structure. + */ +public class SnapCommitVisitor extends CommitVisitor implements LocationNodeVisitor { + + private final Bytes startKeyPath; + private final Bytes endKeyPath; + + public SnapCommitVisitor( + final NodeUpdater nodeUpdater, final Bytes32 startKeyHash, final Bytes32 endKeyHash) { + super(nodeUpdater); + this.startKeyPath = createPath(startKeyHash); + this.endKeyPath = createPath(endKeyHash); + } + + /** + * Visits an extension node during a traversal operation. + * + *

This method is called when visiting an extension node. It checks if the node is marked as + * "dirty" (indicating changes that have not been persisted). If the node is clean, the method + * returns immediately. For dirty nodes, it recursively visits any dirty child nodes, + * concatenating the current location with the extension node's path to form the full path to the + * child. + * + *

Additionally, it checks if the child node requires healing (e.g., if it's falls outside the + * specified range defined by {@code startKeyPath} and {@code endKeyPath}). If healing is needed, + * the extension node is marked accordingly. + * + *

Finally, it attempts to persist the extension node if applicable. + * + * @param location The current location represented as {@link Bytes}. + * @param extensionNode The extension node being visited. + */ + @Override + public void visit(final Bytes location, final ExtensionNode extensionNode) { + if (!extensionNode.isDirty()) { + return; + } + + final Node child = extensionNode.getChild(); + if (child.isDirty()) { + child.accept(Bytes.concatenate(location, extensionNode.getPath()), this); + } + if (child.isHealNeeded() + || !isInRange( + Bytes.concatenate(location, extensionNode.getPath()), startKeyPath, endKeyPath)) { + extensionNode.markHealNeeded(); // not save an incomplete node + } + + maybeStoreNode(location, extensionNode); + } + + /** + * Visits a branch node during a traversal operation. + * + *

This method is invoked when visiting a branch node. It first checks if the branch node is + * marked as "dirty" (indicating changes that have not been persisted). If the node is clean, the + * method returns immediately. + * + *

For dirty branch nodes, it iterates through each child node. For each child, if the child is + * dirty, it recursively visits the child, passing along the concatenated path (current location + * plus the child's index) to the child's accept method. + * + *

Additionally, it checks if the child node requires healing (e.g., if it's falls outside the + * specified range of interest defined by {@code startKeyPath} and {@code endKeyPath}). If healing + * is needed, the branch node is marked accordingly. + * + *

Finally, it attempts to persist the branch node if applicable. + * + * @param location The current location represented as {@link Bytes}. + * @param branchNode The branch node being visited. + */ + @Override + public void visit(final Bytes location, final BranchNode branchNode) { + if (!branchNode.isDirty()) { + return; + } + + for (int i = 0; i < branchNode.maxChild(); ++i) { + Bytes index = Bytes.of(i); + final Node child = branchNode.child((byte) i); + if (child.isDirty()) { + child.accept(Bytes.concatenate(location, index), this); + } + if (child.isHealNeeded() + || !isInRange(Bytes.concatenate(location, index), startKeyPath, endKeyPath)) { + branchNode.markHealNeeded(); // not save an incomplete node + } + } + + maybeStoreNode(location, branchNode); + } + + private boolean isInRange( + final Bytes location, final Bytes startKeyPath, final Bytes endKeyPath) { + final MutableBytes path = MutableBytes.create(Bytes32.SIZE * 2); + path.set(0, location); + return Arrays.compare(path.toArrayUnsafe(), startKeyPath.toArrayUnsafe()) >= 0 + && Arrays.compare(path.toArrayUnsafe(), endKeyPath.toArrayUnsafe()) <= 0; + } +} diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/SnapPutVisitor.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/SnapPutVisitor.java deleted file mode 100644 index 0cce55dcf..000000000 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/SnapPutVisitor.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright Hyperledger Besu Contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.trie; - -import org.hyperledger.besu.ethereum.trie.patricia.BranchNode; -import org.hyperledger.besu.ethereum.trie.patricia.ExtensionNode; -import org.hyperledger.besu.ethereum.trie.patricia.PutVisitor; - -import org.apache.tuweni.bytes.Bytes; - -public class SnapPutVisitor extends PutVisitor { - - public SnapPutVisitor(final NodeFactory nodeFactory, final V value) { - super(nodeFactory, value); - } - - @Override - public Node visit(final BranchNode branchNode, final Bytes path) { - final Node visit = super.visit(branchNode, path); - for (Node child : visit.getChildren()) { - if (child.isHealNeeded() || (child instanceof StoredNode && child.getValue().isEmpty())) { - visit.markHealNeeded(); // not save an incomplete node - return visit; - } - } - return visit; - } - - @Override - public Node visit(final ExtensionNode extensionNode, final Bytes path) { - final Node visit = super.visit(extensionNode, path); - for (Node child : visit.getChildren()) { - if (child.isHealNeeded() || (child instanceof StoredNode && child.getValue().isEmpty())) { - visit.markHealNeeded(); // not save an incomplete node - return visit; - } - } - return visit; - } -} diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/patricia/BranchNode.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/patricia/BranchNode.java index ac5eec524..b5d845136 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/patricia/BranchNode.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/patricia/BranchNode.java @@ -55,7 +55,7 @@ public class BranchNode implements Node { public BranchNode( final Bytes location, - final ArrayList> children, + final List> children, final Optional value, final NodeFactory nodeFactory, final Function valueSerializer) { diff --git a/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/SnapCommitVisitorTest.java b/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/SnapCommitVisitorTest.java new file mode 100644 index 000000000..5b2815298 --- /dev/null +++ b/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/SnapCommitVisitorTest.java @@ -0,0 +1,127 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie; + +import static org.mockito.Mockito.mock; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.trie.patricia.BranchNode; +import org.hyperledger.besu.ethereum.trie.patricia.ExtensionNode; +import org.hyperledger.besu.ethereum.trie.patricia.StoredNodeFactory; + +import java.util.ArrayList; +import java.util.Optional; +import java.util.function.Function; + +import org.apache.tuweni.bytes.Bytes; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +@SuppressWarnings("unchecked") +public class SnapCommitVisitorTest { + + @Test + public void shouldDetectValidBranch() { + final StoredNodeFactory storedNodeFactory = mock(StoredNodeFactory.class); + final ArrayList> children = new ArrayList<>(); + for (int i = 0; i < 16; i++) { + children.add( + new StoredNode<>( + storedNodeFactory, Bytes.concatenate(Bytes.of(0x00), Bytes.of(i)), Hash.ZERO)); + } + final BranchNode validBranchNode = + new BranchNode<>( + Bytes.of(0x00), + children, + Optional.of(Bytes.of(0x00)), + storedNodeFactory, + Function.identity()); + validBranchNode.markDirty(); + final SnapCommitVisitor snapCommitVisitor = + new SnapCommitVisitor<>( + (location, hash, value) -> {}, RangeManager.MIN_RANGE, RangeManager.MAX_RANGE); + Assertions.assertThat(validBranchNode.isHealNeeded()).isFalse(); + snapCommitVisitor.visit(validBranchNode.getLocation().get(), validBranchNode); + Assertions.assertThat(validBranchNode.isHealNeeded()).isFalse(); + } + + @Test + public void shouldDetectBranchWithChildrenNotInTheRange() { + final StoredNodeFactory storedNodeFactory = mock(StoredNodeFactory.class); + final ArrayList> children = new ArrayList<>(); + for (int i = 0; i < 16; i++) { + children.add( + new StoredNode<>( + storedNodeFactory, Bytes.concatenate(Bytes.of(0x01), Bytes.of(i)), Hash.ZERO)); + } + + final BranchNode invalidBranchNode = + new BranchNode<>( + Bytes.of(0x01), + children, + Optional.of(Bytes.of(0x00)), + storedNodeFactory, + Function.identity()); + invalidBranchNode.markDirty(); + final SnapCommitVisitor snapCommitVisitor = + new SnapCommitVisitor<>( + (location, hash, value) -> {}, + RangeManager.MIN_RANGE, + Hash.fromHexString( + "0x1effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")); + Assertions.assertThat(invalidBranchNode.isHealNeeded()).isFalse(); + snapCommitVisitor.visit(invalidBranchNode.getLocation().get(), invalidBranchNode); + Assertions.assertThat(invalidBranchNode.isHealNeeded()).isTrue(); + } + + @Test + public void shouldDetectValidExtension() { + final StoredNodeFactory storedNodeFactory = mock(StoredNodeFactory.class); + final ExtensionNode validExtensionNode = + new ExtensionNode<>( + Bytes.of(0x00), + Bytes.of(0x01), + new StoredNode<>(storedNodeFactory, Bytes.of((byte) 0x00, (byte) 0x01), Hash.ZERO), + storedNodeFactory); + validExtensionNode.markDirty(); + final SnapCommitVisitor snapCommitVisitor = + new SnapCommitVisitor<>( + (location, hash, value) -> {}, RangeManager.MIN_RANGE, RangeManager.MAX_RANGE); + Assertions.assertThat(validExtensionNode.isHealNeeded()).isFalse(); + snapCommitVisitor.visit(validExtensionNode.getLocation().get(), validExtensionNode); + Assertions.assertThat(validExtensionNode.isHealNeeded()).isFalse(); + } + + @Test + public void shouldDetectExtensionWithChildNotInRange() { + final StoredNodeFactory storedNodeFactory = mock(StoredNodeFactory.class); + final ExtensionNode inValidExtensionNode = + new ExtensionNode<>( + Bytes.of(0x00), + Bytes.of(0x03), + new StoredNode<>(storedNodeFactory, Bytes.of((byte) 0x00, (byte) 0x03), Hash.ZERO), + storedNodeFactory); + inValidExtensionNode.markDirty(); + final SnapCommitVisitor snapCommitVisitor = + new SnapCommitVisitor<>( + (location, hash, value) -> {}, + RangeManager.MIN_RANGE, + Hash.fromHexString( + "0x02ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")); + Assertions.assertThat(inValidExtensionNode.isHealNeeded()).isFalse(); + snapCommitVisitor.visit(inValidExtensionNode.getLocation().get(), inValidExtensionNode); + Assertions.assertThat(inValidExtensionNode.isHealNeeded()).isTrue(); + } +} diff --git a/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/SnapPutVisitorTest.java b/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/SnapPutVisitorTest.java deleted file mode 100644 index cbedbe9a2..000000000 --- a/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/SnapPutVisitorTest.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright Hyperledger Besu Contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.trie; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyByte; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.trie.patricia.BranchNode; -import org.hyperledger.besu.ethereum.trie.patricia.ExtensionNode; -import org.hyperledger.besu.ethereum.trie.patricia.LeafNode; -import org.hyperledger.besu.ethereum.trie.patricia.StoredNodeFactory; - -import java.util.ArrayList; -import java.util.Optional; -import java.util.function.Function; - -import org.apache.tuweni.bytes.Bytes; -import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.Test; - -@SuppressWarnings("unchecked") -public class SnapPutVisitorTest { - - @Test - public void shouldDetectValidBranch() { - final StoredNodeFactory storedNodeFactory = mock(StoredNodeFactory.class); - when(storedNodeFactory.createBranch(any(), any())) - .thenReturn( - new LeafNode( - Bytes.EMPTY, Bytes.of(0x00), storedNodeFactory, Function.identity())); - final ArrayList> children = new ArrayList<>(); - for (int i = 0; i < 16; i++) { - children.add(new StoredNode<>(storedNodeFactory, Bytes.EMPTY, Hash.ZERO)); - } - final BranchNode invalidBranchNode = - new BranchNode<>( - Bytes.EMPTY, - children, - Optional.of(Bytes.of(0x00)), - storedNodeFactory, - Function.identity()); - final SnapPutVisitor snapPutVisitor = - new SnapPutVisitor<>(storedNodeFactory, Bytes.EMPTY); - Node visit = - snapPutVisitor.visit(invalidBranchNode, Bytes.of(CompactEncoding.LEAF_TERMINATOR)); - Assertions.assertThat(visit.isHealNeeded()).isFalse(); - } - - @Test - public void shouldDetectBranchWithMissingChildren() { - final StoredNodeFactory storedNodeFactory = mock(StoredNodeFactory.class); - when(storedNodeFactory.createBranch(any(), any())) - .thenReturn(new MissingNode<>(Hash.ZERO, Bytes.EMPTY)); - final ArrayList> children = new ArrayList<>(); - for (int i = 0; i < 16; i++) { - children.add(new StoredNode<>(storedNodeFactory, Bytes.EMPTY, Hash.ZERO)); - } - final BranchNode invalidBranchNode = - new BranchNode<>( - Bytes.EMPTY, - children, - Optional.of(Bytes.of(0x00)), - storedNodeFactory, - Function.identity()); - final SnapPutVisitor snapPutVisitor = - new SnapPutVisitor<>(storedNodeFactory, Bytes.EMPTY); - Node visit = - snapPutVisitor.visit(invalidBranchNode, Bytes.of(CompactEncoding.LEAF_TERMINATOR)); - Assertions.assertThat(visit.isHealNeeded()).isTrue(); - } - - @Test - public void shouldDetectValidExtension() { - final StoredNodeFactory storedNodeFactory = mock(StoredNodeFactory.class); - when(storedNodeFactory.createBranch(any(), any())) - .thenReturn( - new LeafNode<>(Bytes.EMPTY, Bytes.of(0x00), storedNodeFactory, Function.identity())); - final ArrayList> children = new ArrayList<>(); - for (int i = 0; i < 16; i++) { - children.add(new StoredNode<>(storedNodeFactory, Bytes.EMPTY, Hash.ZERO)); - } - final BranchNode invalidBranchNode = - new BranchNode<>( - Bytes.EMPTY, - children, - Optional.of(Bytes.of(0x00)), - storedNodeFactory, - Function.identity()); - final SnapPutVisitor snapPutVisitor = - new SnapPutVisitor<>(storedNodeFactory, Bytes.EMPTY); - Node visit = - snapPutVisitor.visit(invalidBranchNode, Bytes.of(CompactEncoding.LEAF_TERMINATOR)); - Assertions.assertThat(visit.isHealNeeded()).isFalse(); - } - - @Test - public void shouldDetectExtensionWithMissingChildren() { - final StoredNodeFactory storedNodeFactory = mock(StoredNodeFactory.class); - when(storedNodeFactory.createBranch(anyByte(), any(), anyByte(), any())) - .thenReturn(new MissingNode<>(Hash.ZERO, Bytes.EMPTY)); - when(storedNodeFactory.createLeaf(any(), any())) - .thenReturn(new MissingNode<>(Hash.ZERO, Bytes.EMPTY)); - final ExtensionNode invalidBranchNode = - new ExtensionNode<>( - Bytes.of(0x00), - new StoredNode<>(storedNodeFactory, Bytes.EMPTY, Hash.ZERO), - storedNodeFactory); - final SnapPutVisitor snapPutVisitor = - new SnapPutVisitor<>(storedNodeFactory, Bytes.EMPTY); - Node visit = - snapPutVisitor.visit(invalidBranchNode, Bytes.of(CompactEncoding.LEAF_TERMINATOR)); - Assertions.assertThat(visit.isHealNeeded()).isTrue(); - } -} diff --git a/evm/src/main/java/org/hyperledger/besu/evm/log/Log.java b/evm/src/main/java/org/hyperledger/besu/evm/log/Log.java index 02b523445..0c578dfd2 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/log/Log.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/log/Log.java @@ -26,6 +26,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.bytes.MutableBytes; /** * A log entry is a tuple of a logger’s address (the address of the contract that added the logs), a @@ -60,13 +62,38 @@ public class Log { * @param out the output in which to encode the log entry. */ public void writeTo(final RLPOutput out) { + writeTo(out, false); + } + + /** + * Writes the log entry to the provided RLP output. + * + * @param out the output in which to encode the log entry. + * @param compacted whether to compact the rlp log entry by trimming leading zeros on topics and + * data. + */ + public void writeTo(final RLPOutput out, final boolean compacted) { out.startList(); out.writeBytes(logger); - out.writeList(topics, (topic, listOut) -> listOut.writeBytes(topic)); - out.writeBytes(data); + if (compacted) { + out.writeList(topics, (topic, listOut) -> encodeTrimmedData(listOut, topic)); + encodeTrimmedData(out, data); + } else { + out.writeList(topics, (topic, listOut) -> listOut.writeBytes(topic)); + out.writeBytes(data); + } out.endList(); } + private void encodeTrimmedData(final RLPOutput rlpOutput, final Bytes data) { + rlpOutput.startList(); + final Bytes shortData = data.trimLeadingZeros(); + final int zeroLeadDataSize = data.size() - shortData.size(); + rlpOutput.writeIntScalar(zeroLeadDataSize); + rlpOutput.writeBytes(shortData); + rlpOutput.endList(); + } + /** * Reads the log entry from the provided RLP input. * @@ -74,14 +101,45 @@ public class Log { * @return the read log entry. */ public static Log readFrom(final RLPInput in) { + return readFrom(in, false); + } + + /** + * Reads the log entry from the provided RLP input. + * + * @param in the input from which to decode the log entry. + * @param compacted whether to compact the rlp log entry by trimming leading zeros on topics and + * data. + * @return the read log entry. + */ + public static Log readFrom(final RLPInput in, final boolean compacted) { in.enterList(); final Address logger = Address.wrap(in.readBytes()); - final List topics = in.readList(listIn -> LogTopic.wrap(listIn.readBytes32())); - final Bytes data = in.readBytes(); + + final List topics; + final Bytes data; + if (compacted) { + topics = in.readList(listIn -> LogTopic.wrap(Bytes32.wrap(readTrimmedData(in)))); + data = Bytes.wrap(readTrimmedData(in)); + } else { + topics = in.readList(listIn -> LogTopic.wrap(listIn.readBytes32())); + data = in.readBytes(); + } + in.leaveList(); return new Log(logger, data, topics); } + private static Bytes readTrimmedData(final RLPInput in) { + in.enterList(); + final int zeroLeadDataSize = in.readIntScalar(); + final Bytes shortData = in.readBytes(); + final MutableBytes data = MutableBytes.create(zeroLeadDataSize + shortData.size()); + data.set(zeroLeadDataSize, shortData); + in.leaveList(); + return data; + } + /** * Gets logger address. * diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index bef38a5b3..02b465977 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -9,6 +9,14 @@ + + + + + + + + @@ -558,6 +566,11 @@ + + + + + @@ -603,6 +616,14 @@ + + + + + + + + @@ -622,9 +643,6 @@ - - - @@ -637,14 +655,6 @@ - - - - - - - - @@ -666,11 +676,6 @@ - - - - - @@ -739,12 +744,12 @@ - - - + + + - - + + @@ -776,20 +781,28 @@ - - - + + + - - + + - - - + + + - - + + + + + + + + + + @@ -812,20 +825,17 @@ - - - - - - + + + - - - + + + - - + + @@ -844,14 +854,6 @@ - - - - - - - - @@ -876,11 +878,6 @@ - - - - - @@ -993,6 +990,19 @@ + + + + + + + + + + + + + @@ -1001,22 +1011,9 @@ - - - - - - - - - - - - - - - - + + + @@ -1034,12 +1031,12 @@ - - - + + + - - + + @@ -1071,9 +1068,9 @@ - - - + + + @@ -1099,27 +1096,6 @@ - - - - - - - - - - - - - - - - - - - - - @@ -1443,12 +1419,25 @@ - - - + + + - - + + + + + + + + + + + + + + + @@ -2916,17 +2905,17 @@ - - - + + + - - - + + + - - + + @@ -3527,14 +3516,6 @@ - - - - - - - - @@ -3843,14 +3824,6 @@ - - - - - - - - @@ -3859,11 +3832,6 @@ - - - - - @@ -4668,14 +4636,6 @@ - - - - - - - - @@ -4692,6 +4652,14 @@ + + + + + + + + @@ -5492,14 +5460,6 @@ - - - - - - - - @@ -5566,12 +5526,12 @@ - - - + + + - - + + diff --git a/gradle/versions.gradle b/gradle/versions.gradle index dbe70c1d6..629d15982 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -39,12 +39,7 @@ dependencyManagement { entry'dagger' } - dependencySet(group: 'com.google.errorprone', version: '2.18.0') { - entry 'error_prone_annotation' - entry 'error_prone_check_api' - entry 'error_prone_core' - entry 'error_prone_test_helpers' - } + dependency 'org.hyperledger.besu:besu-errorprone-checks:1.0.0' dependency 'com.google.guava:guava:33.0.0-jre' diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/MetricsHttpService.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/MetricsHttpService.java index 2c2023c19..c547b3d4c 100644 --- a/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/MetricsHttpService.java +++ b/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/MetricsHttpService.java @@ -26,6 +26,7 @@ import java.io.OutputStreamWriter; import java.net.InetSocketAddress; import java.net.SocketException; import java.nio.charset.StandardCharsets; +import java.util.Locale; import java.util.Optional; import java.util.Set; import java.util.TreeSet; @@ -175,7 +176,10 @@ public class MetricsHttpService implements MetricsService { private boolean hostIsInAllowlist(final String hostHeader) { if (config.getHostsAllowlist().stream() .anyMatch( - allowlistEntry -> allowlistEntry.toLowerCase().equals(hostHeader.toLowerCase()))) { + allowlistEntry -> + allowlistEntry + .toLowerCase(Locale.ROOT) + .equals(hostHeader.toLowerCase(Locale.ROOT)))) { return true; } else { LOG.trace("Host not in allowlist: '{}'", hostHeader); diff --git a/metrics/rocksdb/src/main/java/org/hyperledger/besu/metrics/rocksdb/RocksDBStats.java b/metrics/rocksdb/src/main/java/org/hyperledger/besu/metrics/rocksdb/RocksDBStats.java index 01e7aca7a..c8a0cb89b 100644 --- a/metrics/rocksdb/src/main/java/org/hyperledger/besu/metrics/rocksdb/RocksDBStats.java +++ b/metrics/rocksdb/src/main/java/org/hyperledger/besu/metrics/rocksdb/RocksDBStats.java @@ -22,6 +22,7 @@ import org.hyperledger.besu.plugin.services.metrics.MetricCategory; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Locale; import io.prometheus.client.Collector; import org.rocksdb.HistogramData; @@ -175,7 +176,7 @@ public class RocksDBStats { return; } for (final TickerType ticker : TICKERS) { - final String promCounterName = ticker.name().toLowerCase(); + final String promCounterName = ticker.name().toLowerCase(Locale.ROOT); metricsSystem.createLongGauge( category, promCounterName, @@ -192,7 +193,7 @@ public class RocksDBStats { final Statistics stats, final HistogramType histogram) { return new Collector() { final String metricName = - KVSTORE_ROCKSDB_STATS.getName() + "_" + histogram.name().toLowerCase(); + KVSTORE_ROCKSDB_STATS.getName() + "_" + histogram.name().toLowerCase(Locale.ROOT); @Override public List collect() { diff --git a/nat/src/main/java/org/hyperledger/besu/nat/docker/HostBasedIpDetector.java b/nat/src/main/java/org/hyperledger/besu/nat/docker/HostBasedIpDetector.java index 34922624c..a7cd1c656 100644 --- a/nat/src/main/java/org/hyperledger/besu/nat/docker/HostBasedIpDetector.java +++ b/nat/src/main/java/org/hyperledger/besu/nat/docker/HostBasedIpDetector.java @@ -27,6 +27,7 @@ public class HostBasedIpDetector implements IpDetector { private static final String HOSTNAME = "HOST_IP"; @Override + @SuppressWarnings("AddressSelection") public Optional detectAdvertisedIp() { try { return Optional.of(InetAddress.getByName(HOSTNAME).getHostAddress()); diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index 33bbf533c..3f68e61ef 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -69,7 +69,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = '/FHIztl2tLW5Gzc0qnfEeuVQa6ljVfUce7YE6JLDdZU=' + knownHash = 'YH+8rbilrhatRAh8rK8/36qxwrqkybBaaNeg+AkZ0c4=' } check.dependsOn('checkAPIChanges') diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/BesuConfiguration.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/BesuConfiguration.java index c28995cc6..ca92a1243 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/BesuConfiguration.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/BesuConfiguration.java @@ -16,6 +16,7 @@ package org.hyperledger.besu.plugin.services; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.plugin.Unstable; +import org.hyperledger.besu.plugin.services.storage.DataStorageConfiguration; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.nio.file.Path; @@ -43,6 +44,7 @@ public interface BesuConfiguration extends BesuService { * @return Database format. */ @Unstable + @Deprecated DataStorageFormat getDatabaseFormat(); /** @@ -52,4 +54,12 @@ public interface BesuConfiguration extends BesuService { */ @Unstable Wei getMinGasPrice(); + + /** + * Database storage configuration. + * + * @return Database storage configuration. + */ + @Unstable + DataStorageConfiguration getDataStorageConfiguration(); } diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/DataStorageConfiguration.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/DataStorageConfiguration.java new file mode 100644 index 000000000..80abc9369 --- /dev/null +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/DataStorageConfiguration.java @@ -0,0 +1,40 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.hyperledger.besu.plugin.services.storage; + +import org.hyperledger.besu.plugin.Unstable; + +/** Data storage configuration */ +@Unstable +public interface DataStorageConfiguration { + + /** + * Database format. This sets the list of segmentIdentifiers that should be initialized. + * + * @return Database format. + */ + @Unstable + DataStorageFormat getDatabaseFormat(); + + /** + * Whether receipt compaction is enabled. When enabled this reduces the storage needed for + * receipts. + * + * @return Whether receipt compaction is enabled + */ + @Unstable + boolean getReceiptCompactionEnabled(); +} diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactory.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactory.java index bf5f92165..909a2c5e9 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactory.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactory.java @@ -46,7 +46,9 @@ public class RocksDBKeyValuePrivacyStorageFactory implements PrivacyKeyValueStor private static final Set SUPPORTED_VERSIONS = EnumSet.of( PrivacyVersionedStorageFormat.FOREST_WITH_VARIABLES, - PrivacyVersionedStorageFormat.BONSAI_WITH_VARIABLES); + PrivacyVersionedStorageFormat.FOREST_WITH_RECEIPT_COMPACTION, + PrivacyVersionedStorageFormat.BONSAI_WITH_VARIABLES, + PrivacyVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION); private static final String PRIVATE_DATABASE_PATH = "private"; private final RocksDBKeyValueStorageFactory publicFactory; private DatabaseMetadata databaseMetadata; @@ -145,7 +147,8 @@ public class RocksDBKeyValuePrivacyStorageFactory implements PrivacyKeyValueStor privacyMetadata = existingPrivacyMetadata; final int existingPrivacyVersion = maybeExistingPrivacyVersion.getAsInt(); final var runtimeVersion = - PrivacyVersionedStorageFormat.defaultForNewDB(commonConfiguration.getDatabaseFormat()); + PrivacyVersionedStorageFormat.defaultForNewDB( + commonConfiguration.getDataStorageConfiguration()); if (existingPrivacyVersion > runtimeVersion.getPrivacyVersion().getAsInt()) { final var maybeDowngradedMetadata = @@ -194,6 +197,16 @@ public class RocksDBKeyValuePrivacyStorageFactory implements PrivacyKeyValueStor // In case we do an automated downgrade, then we also need to update the metadata on disk to // reflect the change to the runtime version, and return it. + // Besu supports both formats of receipts so no downgrade is needed + if (runtimeVersion == PrivacyVersionedStorageFormat.BONSAI_WITH_VARIABLES + || runtimeVersion == PrivacyVersionedStorageFormat.FOREST_WITH_VARIABLES) { + LOG.warn( + "Database contains compacted receipts but receipt compaction is not enabled, new receipts will " + + "be not stored in the compacted format. If you want to remove compacted receipts from the " + + "database it is necessary to resync Besu. Besu can support both compacted and non-compacted receipts."); + return Optional.empty(); + } + // for the moment there are supported automated downgrades, so we just fail. String error = String.format( @@ -216,6 +229,18 @@ public class RocksDBKeyValuePrivacyStorageFactory implements PrivacyKeyValueStor // In case we do an automated upgrade, then we also need to update the metadata on disk to // reflect the change to the runtime version, and return it. + // Besu supports both formats of receipts so no upgrade is needed other than updating metadata + if (runtimeVersion == PrivacyVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION + || runtimeVersion == PrivacyVersionedStorageFormat.FOREST_WITH_RECEIPT_COMPACTION) { + final DatabaseMetadata metadata = new DatabaseMetadata(runtimeVersion); + try { + metadata.writeToDirectory(dataDir); + return Optional.of(metadata); + } catch (IOException e) { + throw new StorageException("Database upgrade to use receipt compaction failed", e); + } + } + // for the moment there are no planned automated upgrades, so we just fail. String error = String.format( diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java index eaf95e82a..bcb86ea13 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java @@ -14,7 +14,9 @@ */ package org.hyperledger.besu.plugin.services.storage.rocksdb; +import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION; import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat.BONSAI_WITH_VARIABLES; +import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat.FOREST_WITH_RECEIPT_COMPACTION; import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat.FOREST_WITH_VARIABLES; import org.hyperledger.besu.plugin.services.BesuConfiguration; @@ -55,7 +57,11 @@ public class RocksDBKeyValueStorageFactory implements KeyValueStorageFactory { private static final Logger LOG = LoggerFactory.getLogger(RocksDBKeyValueStorageFactory.class); private static final EnumSet SUPPORTED_VERSIONED_FORMATS = - EnumSet.of(FOREST_WITH_VARIABLES, BONSAI_WITH_VARIABLES); + EnumSet.of( + FOREST_WITH_VARIABLES, + FOREST_WITH_RECEIPT_COMPACTION, + BONSAI_WITH_VARIABLES, + BONSAI_WITH_RECEIPT_COMPACTION); private static final String NAME = "rocksdb"; private final RocksDBMetricsFactory rocksDBMetricsFactory; private DatabaseMetadata databaseMetadata; @@ -220,12 +226,13 @@ public class RocksDBKeyValueStorageFactory implements KeyValueStorageFactory { if (!metadata .getVersionedStorageFormat() .getFormat() - .equals(commonConfiguration.getDatabaseFormat())) { + .equals(commonConfiguration.getDataStorageConfiguration().getDatabaseFormat())) { handleFormatMismatch(commonConfiguration, dataDir, metadata); } final var runtimeVersion = - BaseVersionedStorageFormat.defaultForNewDB(commonConfiguration.getDatabaseFormat()); + BaseVersionedStorageFormat.defaultForNewDB( + commonConfiguration.getDataStorageConfiguration()); if (metadata.getVersionedStorageFormat().getVersion() > runtimeVersion.getVersion()) { final var maybeDowngradedMetadata = @@ -247,7 +254,7 @@ public class RocksDBKeyValueStorageFactory implements KeyValueStorageFactory { LOG.info("Existing database at {}. Metadata {}. Processing WAL...", dataDir, metadata); } else { - metadata = DatabaseMetadata.defaultForNewDb(commonConfiguration.getDatabaseFormat()); + metadata = DatabaseMetadata.defaultForNewDb(commonConfiguration); LOG.info( "No existing database at {}. Using default metadata for new db {}", dataDir, metadata); if (!dataDirExists) { @@ -275,7 +282,7 @@ public class RocksDBKeyValueStorageFactory implements KeyValueStorageFactory { + "Please check your config.", dataDir, existingMetadata.getVersionedStorageFormat().getFormat().name(), - commonConfiguration.getDatabaseFormat()); + commonConfiguration.getDataStorageConfiguration().getDatabaseFormat()); throw new StorageException(error); } @@ -290,6 +297,15 @@ public class RocksDBKeyValueStorageFactory implements KeyValueStorageFactory { // In case we do an automated downgrade, then we also need to update the metadata on disk to // reflect the change to the runtime version, and return it. + // Besu supports both formats of receipts so no downgrade is needed + if (runtimeVersion == BONSAI_WITH_VARIABLES || runtimeVersion == FOREST_WITH_VARIABLES) { + LOG.warn( + "Database contains compacted receipts but receipt compaction is not enabled, new receipts will " + + "be not stored in the compacted format. If you want to remove compacted receipts from the " + + "database it is necessary to resync Besu. Besu can support both compacted and non-compacted receipts."); + return Optional.empty(); + } + // for the moment there are supported automated downgrades, so we just fail. String error = String.format( @@ -312,6 +328,18 @@ public class RocksDBKeyValueStorageFactory implements KeyValueStorageFactory { // In case we do an automated upgrade, then we also need to update the metadata on disk to // reflect the change to the runtime version, and return it. + // Besu supports both formats of receipts so no upgrade is needed other than updating metadata + if (runtimeVersion == BONSAI_WITH_RECEIPT_COMPACTION + || runtimeVersion == FOREST_WITH_RECEIPT_COMPACTION) { + final DatabaseMetadata metadata = new DatabaseMetadata(runtimeVersion); + try { + metadata.writeToDirectory(dataDir); + return Optional.of(metadata); + } catch (IOException e) { + throw new StorageException("Database upgrade to use receipt compaction failed", e); + } + } + // for the moment there are no planned automated upgrades, so we just fail. String error = String.format( diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/BaseVersionedStorageFormat.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/BaseVersionedStorageFormat.java index a18b907a6..474d5e992 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/BaseVersionedStorageFormat.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/BaseVersionedStorageFormat.java @@ -14,6 +14,7 @@ */ package org.hyperledger.besu.plugin.services.storage.rocksdb.configuration; +import org.hyperledger.besu.plugin.services.storage.DataStorageConfiguration; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.util.OptionalInt; @@ -27,13 +28,23 @@ public enum BaseVersionedStorageFormat implements VersionedStorageFormat { * make BlobDB more effective */ FOREST_WITH_VARIABLES(DataStorageFormat.FOREST, 2), + /** + * Current Forest version, with receipts using compaction, in order to make Receipts use less disk + * space + */ + FOREST_WITH_RECEIPT_COMPACTION(DataStorageFormat.FOREST, 3), /** Original Bonsai version, not used since replace by BONSAI_WITH_VARIABLES */ BONSAI_ORIGINAL(DataStorageFormat.BONSAI, 1), /** * Current Bonsai version, with blockchain variables in a dedicated column family, in order to * make BlobDB more effective */ - BONSAI_WITH_VARIABLES(DataStorageFormat.BONSAI, 2); + BONSAI_WITH_VARIABLES(DataStorageFormat.BONSAI, 2), + /** + * Current Bonsai version, with receipts using compaction, in order to make Receipts use less disk + * space + */ + BONSAI_WITH_RECEIPT_COMPACTION(DataStorageFormat.BONSAI, 3); private final DataStorageFormat format; private final int version; @@ -46,13 +57,18 @@ public enum BaseVersionedStorageFormat implements VersionedStorageFormat { /** * Return the default version for new db for a specific format * - * @param format data storage format + * @param configuration data storage configuration * @return the version to use for new db */ - public static BaseVersionedStorageFormat defaultForNewDB(final DataStorageFormat format) { - return switch (format) { - case FOREST -> FOREST_WITH_VARIABLES; - case BONSAI -> BONSAI_WITH_VARIABLES; + public static BaseVersionedStorageFormat defaultForNewDB( + final DataStorageConfiguration configuration) { + return switch (configuration.getDatabaseFormat()) { + case FOREST -> configuration.getReceiptCompactionEnabled() + ? FOREST_WITH_RECEIPT_COMPACTION + : FOREST_WITH_VARIABLES; + case BONSAI -> configuration.getReceiptCompactionEnabled() + ? BONSAI_WITH_RECEIPT_COMPACTION + : BONSAI_WITH_VARIABLES; }; } diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadata.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadata.java index 4e18e3502..c9cdd4b8b 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadata.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadata.java @@ -14,6 +14,7 @@ */ package org.hyperledger.besu.plugin.services.storage.rocksdb.configuration; +import org.hyperledger.besu.plugin.services.BesuConfiguration; import org.hyperledger.besu.plugin.services.exception.StorageException; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; @@ -46,18 +47,25 @@ public class DatabaseMetadata { .enable(SerializationFeature.INDENT_OUTPUT); private final VersionedStorageFormat versionedStorageFormat; - private DatabaseMetadata(final VersionedStorageFormat versionedStorageFormat) { + /** + * Instantiates a new Database metadata. + * + * @param versionedStorageFormat the version storage format + */ + public DatabaseMetadata(final VersionedStorageFormat versionedStorageFormat) { this.versionedStorageFormat = versionedStorageFormat; } /** * Return the default metadata for new db for a specific format * - * @param dataStorageFormat data storage format + * @param besuConfiguration besu configuration * @return the metadata to use for new db */ - public static DatabaseMetadata defaultForNewDb(final DataStorageFormat dataStorageFormat) { - return new DatabaseMetadata(BaseVersionedStorageFormat.defaultForNewDB(dataStorageFormat)); + public static DatabaseMetadata defaultForNewDb(final BesuConfiguration besuConfiguration) { + return new DatabaseMetadata( + BaseVersionedStorageFormat.defaultForNewDB( + besuConfiguration.getDataStorageConfiguration())); } /** @@ -222,6 +230,7 @@ public class DatabaseMetadata { case FOREST -> switch (versionedStorageFormat.getVersion()) { case 1 -> PrivacyVersionedStorageFormat.FOREST_ORIGINAL; case 2 -> PrivacyVersionedStorageFormat.FOREST_WITH_VARIABLES; + case 3 -> PrivacyVersionedStorageFormat.FOREST_WITH_RECEIPT_COMPACTION; default -> throw new StorageException( "Unsupported database with format FOREST and version " + versionedStorageFormat.getVersion()); @@ -229,6 +238,7 @@ public class DatabaseMetadata { case BONSAI -> switch (versionedStorageFormat.getVersion()) { case 1 -> PrivacyVersionedStorageFormat.BONSAI_ORIGINAL; case 2 -> PrivacyVersionedStorageFormat.BONSAI_WITH_VARIABLES; + case 3 -> PrivacyVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION; default -> throw new StorageException( "Unsupported database with format BONSAI and version " + versionedStorageFormat.getVersion()); diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/PrivacyVersionedStorageFormat.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/PrivacyVersionedStorageFormat.java index 9a86d6ec1..a883516f6 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/PrivacyVersionedStorageFormat.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/PrivacyVersionedStorageFormat.java @@ -14,6 +14,7 @@ */ package org.hyperledger.besu.plugin.services.storage.rocksdb.configuration; +import org.hyperledger.besu.plugin.services.storage.DataStorageConfiguration; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.util.OptionalInt; @@ -27,13 +28,23 @@ public enum PrivacyVersionedStorageFormat implements VersionedStorageFormat { * make BlobDB more effective */ FOREST_WITH_VARIABLES(BaseVersionedStorageFormat.FOREST_WITH_VARIABLES, 1), + /** + * Current Forest version, with receipts using compaction, in order to make Receipts use less disk + * space + */ + FOREST_WITH_RECEIPT_COMPACTION(BaseVersionedStorageFormat.FOREST_WITH_VARIABLES, 2), /** Original Bonsai version, not used since replace by BONSAI_WITH_VARIABLES */ BONSAI_ORIGINAL(BaseVersionedStorageFormat.BONSAI_ORIGINAL, 1), /** * Current Bonsai version, with blockchain variables in a dedicated column family, in order to * make BlobDB more effective */ - BONSAI_WITH_VARIABLES(BaseVersionedStorageFormat.BONSAI_WITH_VARIABLES, 1); + BONSAI_WITH_VARIABLES(BaseVersionedStorageFormat.BONSAI_WITH_VARIABLES, 1), + /** + * Current Bonsai version, with receipts using compaction, in order to make Receipts use less disk + * space + */ + BONSAI_WITH_RECEIPT_COMPACTION(BaseVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION, 2); private final VersionedStorageFormat baseVersionedStorageFormat; private final OptionalInt privacyVersion; @@ -47,13 +58,18 @@ public enum PrivacyVersionedStorageFormat implements VersionedStorageFormat { /** * Return the default version for new db for a specific format * - * @param format data storage format + * @param configuration data storage configuration * @return the version to use for new db */ - public static VersionedStorageFormat defaultForNewDB(final DataStorageFormat format) { - return switch (format) { - case FOREST -> FOREST_WITH_VARIABLES; - case BONSAI -> BONSAI_WITH_VARIABLES; + public static VersionedStorageFormat defaultForNewDB( + final DataStorageConfiguration configuration) { + return switch (configuration.getDatabaseFormat()) { + case FOREST -> configuration.getReceiptCompactionEnabled() + ? FOREST_WITH_RECEIPT_COMPACTION + : FOREST_WITH_VARIABLES; + case BONSAI -> configuration.getReceiptCompactionEnabled() + ? BONSAI_WITH_RECEIPT_COMPACTION + : BONSAI_WITH_VARIABLES; }; } diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java index 003d640be..5dd485cb7 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java @@ -15,13 +15,17 @@ package org.hyperledger.besu.plugin.services.storage.rocksdb; import static org.assertj.core.api.Assertions.assertThat; +import static org.hyperledger.besu.plugin.services.storage.DataStorageFormat.BONSAI; import static org.hyperledger.besu.plugin.services.storage.DataStorageFormat.FOREST; import static org.hyperledger.besu.plugin.services.storage.rocksdb.segmented.RocksDBColumnarKeyValueStorageTest.TestSegment; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.when; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.BesuConfiguration; +import org.hyperledger.besu.plugin.services.storage.DataStorageConfiguration; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.DatabaseMetadata; @@ -35,6 +39,8 @@ import java.util.List; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -42,6 +48,7 @@ import org.mockito.junit.jupiter.MockitoExtension; public class RocksDBKeyValuePrivacyStorageFactoryTest { @Mock private RocksDBFactoryConfiguration rocksDbConfiguration; @Mock private BesuConfiguration commonConfiguration; + @Mock private DataStorageConfiguration dataStorageConfiguration; @TempDir private Path temporaryFolder; private final ObservableMetricsSystem metricsSystem = new NoOpMetricsSystem(); private final SegmentIdentifier segment = TestSegment.BAR; @@ -94,20 +101,25 @@ public class RocksDBKeyValuePrivacyStorageFactoryTest { } } - @Test - public void shouldUpdateCorrectMetadataFileForLatestVersion() throws Exception { + @ParameterizedTest + @EnumSource(DataStorageFormat.class) + public void shouldUpdateCorrectMetadataFileForLatestVersion( + final DataStorageFormat dataStorageFormat) throws Exception { final Path tempDataDir = temporaryFolder.resolve("data"); final Path tempDatabaseDir = temporaryFolder.resolve("db"); - mockCommonConfiguration(tempDataDir, tempDatabaseDir); + mockCommonConfiguration(tempDataDir, tempDatabaseDir, dataStorageFormat); final RocksDBKeyValueStorageFactory storageFactory = new RocksDBKeyValueStorageFactory( () -> rocksDbConfiguration, segments, RocksDBMetricsFactory.PRIVATE_ROCKS_DB_METRICS); try (final var storage = storageFactory.create(segment, commonConfiguration, metricsSystem)) { - + final BaseVersionedStorageFormat expectedBaseVersion = + dataStorageFormat == BONSAI + ? BaseVersionedStorageFormat.BONSAI_WITH_VARIABLES + : BaseVersionedStorageFormat.FOREST_WITH_VARIABLES; assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat()) - .isEqualTo(BaseVersionedStorageFormat.FOREST_WITH_VARIABLES); + .isEqualTo(expectedBaseVersion); } storageFactory.close(); @@ -116,16 +128,67 @@ public class RocksDBKeyValuePrivacyStorageFactoryTest { try (final var storage = privacyStorageFactory.create(segment, commonConfiguration, metricsSystem)) { - + final PrivacyVersionedStorageFormat expectedPrivacyVersion = + dataStorageFormat == BONSAI + ? PrivacyVersionedStorageFormat.BONSAI_WITH_VARIABLES + : PrivacyVersionedStorageFormat.FOREST_WITH_VARIABLES; assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat()) - .isEqualTo(PrivacyVersionedStorageFormat.FOREST_WITH_VARIABLES); + .isEqualTo(expectedPrivacyVersion); + } + privacyStorageFactory.close(); + } + + @ParameterizedTest + @EnumSource(DataStorageFormat.class) + public void shouldUpdateCorrectMetadataFileForLatestVersionWithReceiptCompaction( + final DataStorageFormat dataStorageFormat) throws Exception { + final Path tempDataDir = temporaryFolder.resolve("data"); + final Path tempDatabaseDir = temporaryFolder.resolve("db"); + mockCommonConfiguration(tempDataDir, tempDatabaseDir, dataStorageFormat); + when(dataStorageConfiguration.getReceiptCompactionEnabled()).thenReturn(true); + + final RocksDBKeyValueStorageFactory storageFactory = + new RocksDBKeyValueStorageFactory( + () -> rocksDbConfiguration, segments, RocksDBMetricsFactory.PRIVATE_ROCKS_DB_METRICS); + + try (final var storage = storageFactory.create(segment, commonConfiguration, metricsSystem)) { + final BaseVersionedStorageFormat expectedBaseVersion = + dataStorageFormat == BONSAI + ? BaseVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION + : BaseVersionedStorageFormat.FOREST_WITH_RECEIPT_COMPACTION; + assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat()) + .isEqualTo(expectedBaseVersion); + } + storageFactory.close(); + + final RocksDBKeyValuePrivacyStorageFactory privacyStorageFactory = + new RocksDBKeyValuePrivacyStorageFactory(storageFactory); + + try (final var storage = + privacyStorageFactory.create(segment, commonConfiguration, metricsSystem)) { + final PrivacyVersionedStorageFormat expectedPrivacyVersion = + dataStorageFormat == BONSAI + ? PrivacyVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION + : PrivacyVersionedStorageFormat.FOREST_WITH_RECEIPT_COMPACTION; + assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat()) + .isEqualTo(expectedPrivacyVersion); } privacyStorageFactory.close(); } private void mockCommonConfiguration(final Path tempDataDir, final Path tempDatabaseDir) { + mockCommonConfiguration(tempDataDir, tempDatabaseDir, FOREST); + } + + private void mockCommonConfiguration( + final Path tempDataDir, + final Path tempDatabaseDir, + final DataStorageFormat dataStorageFormat) { when(commonConfiguration.getStoragePath()).thenReturn(tempDatabaseDir); when(commonConfiguration.getDataPath()).thenReturn(tempDataDir); - when(commonConfiguration.getDatabaseFormat()).thenReturn(FOREST); + when(dataStorageConfiguration.getDatabaseFormat()).thenReturn(dataStorageFormat); + lenient() + .when(commonConfiguration.getDataStorageConfiguration()) + .thenReturn(dataStorageConfiguration); } } diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java index e2c56f0ac..1748f280c 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java @@ -27,6 +27,7 @@ import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.BesuConfiguration; import org.hyperledger.besu.plugin.services.exception.StorageException; +import org.hyperledger.besu.plugin.services.storage.DataStorageConfiguration; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat; @@ -43,6 +44,8 @@ import org.junit.jupiter.api.condition.DisabledOnOs; import org.junit.jupiter.api.condition.OS; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -51,16 +54,19 @@ public class RocksDBKeyValueStorageFactoryTest { @Mock private RocksDBFactoryConfiguration rocksDbConfiguration; @Mock private BesuConfiguration commonConfiguration; + @Mock private DataStorageConfiguration dataStorageConfiguration; @TempDir public Path temporaryFolder; private final ObservableMetricsSystem metricsSystem = new NoOpMetricsSystem(); private final SegmentIdentifier segment = TestSegment.FOO; private final List segments = List.of(TestSegment.DEFAULT, segment); - @Test - public void shouldCreateCorrectMetadataFileForLatestVersionForNewDb() throws Exception { + @ParameterizedTest + @EnumSource(DataStorageFormat.class) + public void shouldCreateCorrectMetadataFileForLatestVersionForNewDb( + final DataStorageFormat dataStorageFormat) throws Exception { final Path tempDataDir = temporaryFolder.resolve("data"); final Path tempDatabaseDir = temporaryFolder.resolve("db"); - mockCommonConfiguration(tempDataDir, tempDatabaseDir, FOREST); + mockCommonConfiguration(tempDataDir, tempDatabaseDir, dataStorageFormat); final RocksDBKeyValueStorageFactory storageFactory = new RocksDBKeyValueStorageFactory( @@ -68,8 +74,36 @@ public class RocksDBKeyValueStorageFactoryTest { try (final var storage = storageFactory.create(segment, commonConfiguration, metricsSystem)) { // Side effect is creation of the Metadata version file + final BaseVersionedStorageFormat expectedVersion = + dataStorageFormat == BONSAI + ? BaseVersionedStorageFormat.BONSAI_WITH_VARIABLES + : BaseVersionedStorageFormat.FOREST_WITH_VARIABLES; assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat()) - .isEqualTo(BaseVersionedStorageFormat.FOREST_WITH_VARIABLES); + .isEqualTo(expectedVersion); + } + } + + @ParameterizedTest + @EnumSource(DataStorageFormat.class) + public void shouldCreateCorrectMetadataFileForLatestVersionForNewDbWithReceiptCompaction( + final DataStorageFormat dataStorageFormat) throws Exception { + final Path tempDataDir = temporaryFolder.resolve("data"); + final Path tempDatabaseDir = temporaryFolder.resolve("db"); + mockCommonConfiguration(tempDataDir, tempDatabaseDir, dataStorageFormat); + when(dataStorageConfiguration.getReceiptCompactionEnabled()).thenReturn(true); + + final RocksDBKeyValueStorageFactory storageFactory = + new RocksDBKeyValueStorageFactory( + () -> rocksDbConfiguration, segments, RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); + + try (final var storage = storageFactory.create(segment, commonConfiguration, metricsSystem)) { + // Side effect is creation of the Metadata version file + final BaseVersionedStorageFormat expectedVersion = + dataStorageFormat == BONSAI + ? BaseVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION + : BaseVersionedStorageFormat.FOREST_WITH_RECEIPT_COMPACTION; + assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat()) + .isEqualTo(expectedVersion); } } @@ -273,6 +307,9 @@ public class RocksDBKeyValueStorageFactoryTest { final Path tempDataDir, final Path tempDatabaseDir, final DataStorageFormat format) { when(commonConfiguration.getStoragePath()).thenReturn(tempDatabaseDir); when(commonConfiguration.getDataPath()).thenReturn(tempDataDir); - lenient().when(commonConfiguration.getDatabaseFormat()).thenReturn(format); + lenient().when(dataStorageConfiguration.getDatabaseFormat()).thenReturn(format); + lenient() + .when(commonConfiguration.getDataStorageConfiguration()) + .thenReturn(dataStorageConfiguration); } } diff --git a/settings.gradle b/settings.gradle index 15a0f8617..3ca01a260 100644 --- a/settings.gradle +++ b/settings.gradle @@ -37,7 +37,6 @@ include 'crypto:services' include 'datatypes' include 'enclave' include 'evm' -include 'errorprone-checks' include 'ethereum:api' include 'ethereum:blockcreation' include 'ethereum:core' diff --git a/util/src/main/java/org/hyperledger/besu/util/Subscribers.java b/util/src/main/java/org/hyperledger/besu/util/Subscribers.java index 6704ed957..7ca1e7127 100644 --- a/util/src/main/java/org/hyperledger/besu/util/Subscribers.java +++ b/util/src/main/java/org/hyperledger/besu/util/Subscribers.java @@ -165,9 +165,7 @@ public class Subscribers { } @Override - public void forEach(final Consumer action) { - return; - } + public void forEach(final Consumer action) {} @Override public int getSubscriberCount() { diff --git a/util/src/test/java/org/hyperledger/besu/util/log/LogUtilTest.java b/util/src/test/java/org/hyperledger/besu/util/log/LogUtilTest.java index 10fef9662..25ed58c33 100644 --- a/util/src/test/java/org/hyperledger/besu/util/log/LogUtilTest.java +++ b/util/src/test/java/org/hyperledger/besu/util/log/LogUtilTest.java @@ -114,6 +114,7 @@ public class LogUtilTest { return lines; } + @SuppressWarnings("InfiniteRecursion") private void recurseTimesAndThrow(final int times) { if (times < 1) { throw new RuntimeException("FakeStackOverflowError"); diff --git a/wslsh.bat b/wslsh.bat new file mode 100644 index 000000000..57e1d6230 --- /dev/null +++ b/wslsh.bat @@ -0,0 +1,2 @@ +set WSLENV=architecture/u +wsl /bin/bash %* \ No newline at end of file