vacp2p/status-linea-besu
1
0
Fork 0
mirror of https://github.com/vacp2p/status-linea-besu.git synced 2026-01-09 22:07:59 -05:00
Code Issues Packages Projects Releases Wiki Activity

Default bonsai to fully flat db and code storage by codehash (#6894)

Browse Source 
* change to full flat db and code stored by code hash by default
* deprecate --Xsnapsync-synchronizer-flat-db-healing-enabled, add DataStorageOption for --Xbonsai-full-flat-db-enabled

Signed-off-by: garyschulte <garyschulte@gmail.com>
This commit is contained in:
garyschulte
2024-05-03 16:52:55 -07:00
committed by GitHub
parent d0a32bcf9c
commit 0a24acccc2
15 changed files with 135 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@@ -45,6 +45,7 @@
- Allow users to specify which plugins are registered [#6700](https://github.com/hyperledger/besu/pull/6700)
- Layered txpool tuning for blob transactions [#6940](https://github.com/hyperledger/besu/pull/6940)
- Update Gradle to 7.6.4 [#7030](https://github.com/hyperledger/besu/pull/7030)
- Default bonsai to use full-flat db and code-storage-by-code-hash [#6984](https://github.com/hyperledger/besu/pull/6894)
### Bug fixes
- Fix txpool dump/restore race condition [#6665](https://github.com/hyperledger/besu/pull/6665)

4
besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java
View File

@@ -1706,8 +1706,8 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
CommandLineUtils.failIfOptionDoesntMeetRequirement(
commandLine,
"--Xsnapsync-synchronizer-flat option can only be used when -Xsnapsync-synchronizer-flat-db-healing-enabled is true",
unstableSynchronizerOptions.isSnapsyncFlatDbHealingEnabled(),
"--Xsnapsync-synchronizer-flat option can only be used when --Xbonsai-full-flat-db-enabled is true",
dataStorageOptions.toDomainObject().getUnstable().getBonsaiFullFlatDbEnabled(),
asList(
"--Xsnapsync-synchronizer-flat-account-healed-count-per-request",
"--Xsnapsync-synchronizer-flat-slot-healed-count-per-request"));

16
besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java
View File

@@ -17,6 +17,7 @@ package org.hyperledger.besu.cli.options.stable;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_RECEIPT_COMPACTION_ENABLED;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT;
@@ -93,6 +94,18 @@ public class DataStorageOptions implements CLIOptions<DataStorageConfiguration>
"The max number of blocks to load and prune trie logs for at startup. (default: ${DEFAULT-VALUE})")
private int bonsaiTrieLogPruningWindowSize = DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE;
// TODO: --Xsnapsync-synchronizer-flat-db-healing-enabled is deprecated, remove it in a future
// release
@CommandLine.Option(
hidden = true,
names = {
"--Xbonsai-full-flat-db-enabled",
"--Xsnapsync-synchronizer-flat-db-healing-enabled"
},
arity = "1",
description = "Enables bonsai full flat database strategy. (default: ${DEFAULT-VALUE})")
private Boolean bonsaiFullFlatDbEnabled = DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED;
@CommandLine.Option(
hidden = true,
names = {"--Xbonsai-code-using-code-hash-enabled"},
@@ -161,6 +174,8 @@ public class DataStorageOptions implements CLIOptions<DataStorageConfiguration>
domainObject.getUnstable().getBonsaiLimitTrieLogsEnabled();
dataStorageOptions.unstableOptions.bonsaiTrieLogPruningWindowSize =
domainObject.getUnstable().getBonsaiTrieLogPruningWindowSize();
dataStorageOptions.unstableOptions.bonsaiFullFlatDbEnabled =
domainObject.getUnstable().getBonsaiFullFlatDbEnabled();
dataStorageOptions.unstableOptions.bonsaiCodeUsingCodeHashEnabled =
domainObject.getUnstable().getBonsaiCodeStoredByCodeHashEnabled();
@@ -177,6 +192,7 @@ public class DataStorageOptions implements CLIOptions<DataStorageConfiguration>
ImmutableDataStorageConfiguration.Unstable.builder()
.bonsaiLimitTrieLogsEnabled(unstableOptions.bonsaiLimitTrieLogsEnabled)
.bonsaiTrieLogPruningWindowSize(unstableOptions.bonsaiTrieLogPruningWindowSize)
.bonsaiFullFlatDbEnabled(unstableOptions.bonsaiFullFlatDbEnabled)
.bonsaiCodeStoredByCodeHashEnabled(unstableOptions.bonsaiCodeUsingCodeHashEnabled)
.build())
.build();

43
besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java
View File

@@ -81,9 +81,6 @@ public class SynchronizerOptions implements CLIOptions<SynchronizerConfiguration
private static final String SNAP_FLAT_STORAGE_HEALED_COUNT_PER_REQUEST_FLAG =
"--Xsnapsync-synchronizer-flat-slot-healed-count-per-request";
private static final String SNAP_FLAT_DB_HEALING_ENABLED_FLAG =
"--Xsnapsync-synchronizer-flat-db-healing-enabled";
private static final String SNAP_SERVER_ENABLED_FLAG = "--Xsnapsync-server-enabled";
private static final String CHECKPOINT_POST_MERGE_FLAG = "--Xcheckpoint-post-merge-enabled";
@@ -292,18 +289,11 @@ public class SynchronizerOptions implements CLIOptions<SynchronizerConfiguration
private int snapsyncFlatStorageHealedCountPerRequest =
SnapSyncConfiguration.DEFAULT_LOCAL_FLAT_STORAGE_COUNT_TO_HEAL_PER_REQUEST;
@CommandLine.Option(
names = SNAP_FLAT_DB_HEALING_ENABLED_FLAG,
hidden = true,
paramLabel = "<Boolean>",
description = "Snap sync flat db healing enabled (default: ${DEFAULT-VALUE})")
private Boolean snapsyncFlatDbHealingEnabled =
SnapSyncConfiguration.DEFAULT_IS_FLAT_DB_HEALING_ENABLED;
@CommandLine.Option(
names = SNAP_SERVER_ENABLED_FLAG,
hidden = true,
paramLabel = "<Boolean>",
arity = "0..1",
description = "Snap sync server enabled (default: ${DEFAULT-VALUE})")
private Boolean snapsyncServerEnabled = SnapSyncConfiguration.DEFAULT_SNAP_SERVER_ENABLED;
@@ -316,15 +306,6 @@ public class SynchronizerOptions implements CLIOptions<SynchronizerConfiguration
private SynchronizerOptions() {}
/**
* Flag to know whether the flat db healing feature is enabled or disabled.
*
* @return true is the flat db healing is enabled
*/
public boolean isSnapsyncFlatDbHealingEnabled() {
return snapsyncFlatDbHealingEnabled;
}
/**
* Flag to know whether the Snap sync server feature is enabled or disabled.
*
@@ -382,9 +363,8 @@ public class SynchronizerOptions implements CLIOptions<SynchronizerConfiguration
config.getSnapSyncConfiguration().getLocalFlatAccountCountToHealPerRequest();
options.snapsyncFlatStorageHealedCountPerRequest =
config.getSnapSyncConfiguration().getLocalFlatStorageCountToHealPerRequest();
options.snapsyncFlatDbHealingEnabled =
config.getSnapSyncConfiguration().isFlatDbHealingEnabled();
options.checkpointPostMergeSyncEnabled = config.isCheckpointPostMergeEnabled();
options.snapsyncServerEnabled = config.getSnapSyncConfiguration().isSnapServerEnabled();
return options;
}
@@ -416,7 +396,6 @@ public class SynchronizerOptions implements CLIOptions<SynchronizerConfiguration
.trienodeCountPerRequest(snapsyncTrieNodeCountPerRequest)
.localFlatAccountCountToHealPerRequest(snapsyncFlatAccountHealedCountPerRequest)
.localFlatStorageCountToHealPerRequest(snapsyncFlatStorageHealedCountPerRequest)
.isFlatDbHealingEnabled(snapsyncFlatDbHealingEnabled)
.isSnapServerEnabled(snapsyncServerEnabled)
.build());
builder.checkpointPostMergeEnabled(checkpointPostMergeSyncEnabled);
@@ -469,17 +448,13 @@ public class SynchronizerOptions implements CLIOptions<SynchronizerConfiguration
SNAP_BYTECODE_COUNT_PER_REQUEST_FLAG,
OptionParser.format(snapsyncBytecodeCountPerRequest),
SNAP_TRIENODE_COUNT_PER_REQUEST_FLAG,
OptionParser.format(snapsyncTrieNodeCountPerRequest));
if (isSnapsyncFlatDbHealingEnabled()) {
value.addAll(
Arrays.asList(
SNAP_FLAT_ACCOUNT_HEALED_COUNT_PER_REQUEST_FLAG,
OptionParser.format(snapsyncFlatAccountHealedCountPerRequest),
SNAP_FLAT_STORAGE_HEALED_COUNT_PER_REQUEST_FLAG,
OptionParser.format(snapsyncFlatStorageHealedCountPerRequest),
SNAP_SERVER_ENABLED_FLAG,
OptionParser.format(snapsyncServerEnabled)));
}
OptionParser.format(snapsyncTrieNodeCountPerRequest),
SNAP_FLAT_ACCOUNT_HEALED_COUNT_PER_REQUEST_FLAG,
OptionParser.format(snapsyncFlatAccountHealedCountPerRequest),
SNAP_FLAT_STORAGE_HEALED_COUNT_PER_REQUEST_FLAG,
OptionParser.format(snapsyncFlatStorageHealedCountPerRequest),
SNAP_SERVER_ENABLED_FLAG,
OptionParser.format(snapsyncServerEnabled));
return value;
}
}

37
besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java
View File

@@ -2338,29 +2338,48 @@ public class BesuCommandTest extends CommandTestAbstract {
}
@Test
public void snapsyncHealingOptionShouldBeDisabledByDefault() {
public void bonsaiFlatDbShouldBeEnabledByDefault() {
final TestBesuCommand besuCommand = parseCommand();
assertThat(besuCommand.unstableSynchronizerOptions.isSnapsyncFlatDbHealingEnabled()).isFalse();
assertThat(
besuCommand
.getDataStorageOptions()
.toDomainObject()
.getUnstable()
.getBonsaiFullFlatDbEnabled())
.isTrue();
}
@Test
public void snapsyncHealingOptionShouldWork() {
final TestBesuCommand besuCommand =
parseCommand("--Xsnapsync-synchronizer-flat-db-healing-enabled", "true");
assertThat(besuCommand.unstableSynchronizerOptions.isSnapsyncFlatDbHealingEnabled()).isTrue();
final TestBesuCommand besuCommand = parseCommand("--Xbonsai-full-flat-db-enabled", "false");
assertThat(
besuCommand
.dataStorageOptions
.toDomainObject()
.getUnstable()
.getBonsaiFullFlatDbEnabled())
.isFalse();
}
@Test
public void snapsyncForHealingFeaturesShouldFailWhenHealingIsNotEnabled() {
parseCommand("--Xsnapsync-synchronizer-flat-account-healed-count-per-request", "100");
parseCommand(
"--Xbonsai-full-flat-db-enabled",
"false",
"--Xsnapsync-synchronizer-flat-account-healed-count-per-request",
"100");
assertThat(commandErrorOutput.toString(UTF_8))
.contains(
"--Xsnapsync-synchronizer-flat option can only be used when -Xsnapsync-synchronizer-flat-db-healing-enabled is true");
"--Xsnapsync-synchronizer-flat option can only be used when --Xbonsai-full-flat-db-enabled is true");
parseCommand("--Xsnapsync-synchronizer-flat-slot-healed-count-per-request", "100");
parseCommand(
"--Xbonsai-full-flat-db-enabled",
"false",
"--Xsnapsync-synchronizer-flat-slot-healed-count-per-request",
"100");
assertThat(commandErrorOutput.toString(UTF_8))
.contains(
"--Xsnapsync-synchronizer-flat option can only be used when -Xsnapsync-synchronizer-flat-db-healing-enabled is true");
"--Xsnapsync-synchronizer-flat option can only be used when --Xbonsai-full-flat-db-enabled is true");
}
@Test

1
besu/src/test/java/org/hyperledger/besu/cli/options/SynchronizerOptionsTest.java
View File

@@ -78,6 +78,7 @@ public class SynchronizerOptionsTest
.storageCountPerRequest(SnapSyncConfiguration.DEFAULT_STORAGE_COUNT_PER_REQUEST + 2)
.bytecodeCountPerRequest(
SnapSyncConfiguration.DEFAULT_BYTECODE_COUNT_PER_REQUEST + 2)
.isSnapServerEnabled(Boolean.TRUE)
.build());
}

2
besu/src/test/resources/everything_config.toml
View File

@@ -218,6 +218,8 @@ receipt-compaction-enabled=true
# feature flags
Xsecp256k1-native-enabled=false
Xaltbn128-native-enabled=false
Xsnapsync-server-enabled=true
Xbonsai-full-flat-db-enabled=true
# compatibility flags
compatibility-eth64-forkid-enabled=false

30
ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProvider.java
View File

@@ -16,6 +16,7 @@ package org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat;
import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE;
import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE;
import static org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.FullFlatDbStrategy;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.PartialFlatDbStrategy;
@@ -70,12 +71,36 @@ public class FlatDbStrategyProvider {
@VisibleForTesting
FlatDbMode deriveFlatDbStrategy(final SegmentedKeyValueStorage composedWorldStateStorage) {
final FlatDbMode requestedFlatDbMode =
dataStorageConfiguration.getUnstable().getBonsaiFullFlatDbEnabled()
? FlatDbMode.FULL
: FlatDbMode.PARTIAL;
final var existingTrieData =
composedWorldStateStorage.get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY).isPresent();
var flatDbMode =
FlatDbMode.fromVersion(
composedWorldStateStorage
.get(TRIE_BRANCH_STORAGE, FLAT_DB_MODE)
.map(Bytes::wrap)
.orElse(FlatDbMode.PARTIAL.getVersion()));
.orElseGet(
() -> {
// if we do not have a db-supplied config for flatdb, derive it:
// default to partial if trie data exists, but the flat config does not,
// and default to the storage config otherwise
var flatDbModeVal =
existingTrieData
? FlatDbMode.PARTIAL.getVersion()
: requestedFlatDbMode.getVersion();
// persist this config in the db
var setDbModeTx = composedWorldStateStorage.startTransaction();
setDbModeTx.put(
TRIE_BRANCH_STORAGE, FLAT_DB_MODE, flatDbModeVal.toArrayUnsafe());
setDbModeTx.commit();
return flatDbModeVal;
}));
LOG.info("Bonsai flat db mode found {}", flatDbMode);
return flatDbMode;
@@ -123,7 +148,7 @@ public class FlatDbStrategyProvider {
public void upgradeToFullFlatDbMode(final SegmentedKeyValueStorage composedWorldStateStorage) {
final SegmentedKeyValueStorageTransaction transaction =
composedWorldStateStorage.startTransaction();
// TODO: consider ARCHIVE mode
LOG.info("setting FlatDbStrategy to FULL");
transaction.put(
TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.FULL.getVersion().toArrayUnsafe());
transaction.commit();
@@ -134,6 +159,7 @@ public class FlatDbStrategyProvider {
final SegmentedKeyValueStorage composedWorldStateStorage) {
final SegmentedKeyValueStorageTransaction transaction =
composedWorldStateStorage.startTransaction();
LOG.info("setting FlatDbStrategy to PARTIAL");
transaction.put(
TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.PARTIAL.getVersion().toArrayUnsafe());
transaction.commit();

18
ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java
View File

@@ -38,6 +38,13 @@ public interface DataStorageConfiguration {
.bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD)
.build();
DataStorageConfiguration DEFAULT_BONSAI_PARTIAL_DB_CONFIG =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(DataStorageFormat.BONSAI)
.bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD)
.unstable(Unstable.DEFAULT_PARTIAL)
.build();
DataStorageConfiguration DEFAULT_FOREST_CONFIG =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(DataStorageFormat.FOREST)
@@ -65,11 +72,15 @@ public interface DataStorageConfiguration {
boolean DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED = false;
long MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
int DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE = 30_000;
boolean DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED = false;
boolean DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED = true;
boolean DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED = true;
DataStorageConfiguration.Unstable DEFAULT =
ImmutableDataStorageConfiguration.Unstable.builder().build();
DataStorageConfiguration.Unstable DEFAULT_PARTIAL =
ImmutableDataStorageConfiguration.Unstable.builder().bonsaiFullFlatDbEnabled(false).build();
@Value.Default
default boolean getBonsaiLimitTrieLogsEnabled() {
return DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED;
@@ -80,6 +91,11 @@ public interface DataStorageConfiguration {
return DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE;
}
@Value.Default
default boolean getBonsaiFullFlatDbEnabled() {
return DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED;
}
@Value.Default
default boolean getBonsaiCodeStoredByCodeHashEnabled() {
return DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED;

5
ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java
View File

@@ -152,8 +152,11 @@ public abstract class AbstractIsolationTests {
@BeforeEach
public void createStorage() {
worldStateKeyValueStorage =
// FYI: BonsaiSnapshoIsolationTests work with frozen/cached worldstates, using PARTIAL
// flat db strategy allows the tests to make account assertions based on trie
// (whereas a full db strategy will not, since the worldstates are frozen/cached)
createKeyValueStorageProvider()
.createWorldStateStorage(DataStorageConfiguration.DEFAULT_BONSAI_CONFIG);
.createWorldStateStorage(DataStorageConfiguration.DEFAULT_BONSAI_PARTIAL_DB_CONFIG);
archive =
new BonsaiWorldStateProvider(
(BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage,

48
ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java
View File

@@ -45,7 +45,6 @@ import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator;
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage;
import java.util.Arrays;
import java.util.Collection;
@@ -79,18 +78,19 @@ public class BonsaiWorldStateKeyValueStorageTest {
BonsaiWorldStateKeyValueStorage storage;
public void setUp(final FlatDbMode flatDbMode) {
storage = emptyStorage();
if (flatDbMode.equals(FlatDbMode.FULL)) {
storage.upgradeToFullFlatDbMode();
}
public BonsaiWorldStateKeyValueStorage setUp(final FlatDbMode flatDbMode) {
return setUp(flatDbMode, false);
}
public void setUp(final FlatDbMode flatDbMode, final boolean useCodeHashStorage) {
public BonsaiWorldStateKeyValueStorage setUp(
final FlatDbMode flatDbMode, final boolean useCodeHashStorage) {
storage = emptyStorage(useCodeHashStorage);
if (flatDbMode.equals(FlatDbMode.FULL)) {
storage.upgradeToFullFlatDbMode();
} else if (flatDbMode.equals(FlatDbMode.PARTIAL)) {
storage.downgradeToPartialFlatDbMode();
}
return storage;
}
@ParameterizedTest
@@ -215,9 +215,8 @@ public class BonsaiWorldStateKeyValueStorageTest {
@ParameterizedTest
@MethodSource("flatDbMode")
void getAccount_notLoadFromTrieWhenEmptyAndFlatDbFullMode(final FlatDbMode flatDbMode) {
setUp(flatDbMode);
Assumptions.assumeTrue(flatDbMode == FlatDbMode.FULL);
final BonsaiWorldStateKeyValueStorage storage = spy(emptyStorage());
final BonsaiWorldStateKeyValueStorage storage = spy(setUp(flatDbMode));
final WorldStateStorageCoordinator coordinator = new WorldStateStorageCoordinator(storage);
final MerkleTrie<Bytes, Bytes> trie = TrieGenerator.generateTrie(coordinator, 1);
@@ -233,8 +232,8 @@ public class BonsaiWorldStateKeyValueStorageTest {
updater.commit();
// remove flat database
storage.downgradeToPartialFlatDbMode();
storage.clearFlatDatabase();
storage.upgradeToFullFlatDbMode();
Mockito.reset(storage);
@@ -247,9 +246,8 @@ public class BonsaiWorldStateKeyValueStorageTest {
@ParameterizedTest
@MethodSource("flatDbMode")
void getAccount_loadFromTrieWhenEmptyAndFlatDbPartialMode(final FlatDbMode flatDbMode) {
setUp(flatDbMode);
Assumptions.assumeTrue(flatDbMode == FlatDbMode.PARTIAL);
final BonsaiWorldStateKeyValueStorage storage = spy(emptyStorage());
final BonsaiWorldStateKeyValueStorage storage = spy(setUp(flatDbMode));
final WorldStateStorageCoordinator coordinator = new WorldStateStorageCoordinator(storage);
final MerkleTrie<Bytes, Bytes> trie = TrieGenerator.generateTrie(coordinator, 1);
final TreeMap<Bytes32, Bytes> accounts =
@@ -277,9 +275,8 @@ public class BonsaiWorldStateKeyValueStorageTest {
@ParameterizedTest
@MethodSource("flatDbMode")
void shouldUsePartialDBStrategyAfterDowngradingMode(final FlatDbMode flatDbMode) {
setUp(flatDbMode);
Assumptions.assumeTrue(flatDbMode == FlatDbMode.PARTIAL);
final BonsaiWorldStateKeyValueStorage storage = spy(emptyStorage());
final BonsaiWorldStateKeyValueStorage storage = spy(setUp(flatDbMode));
final WorldStateStorageCoordinator coordinator = new WorldStateStorageCoordinator(storage);
final MerkleTrie<Bytes, Bytes> trie = TrieGenerator.generateTrie(coordinator, 1);
final TreeMap<Bytes32, Bytes> accounts =
@@ -309,9 +306,9 @@ public class BonsaiWorldStateKeyValueStorageTest {
@ParameterizedTest
@MethodSource("flatDbMode")
void getStorage_loadFromTrieWhenEmptyWithPartialMode(final FlatDbMode flatDbMode) {
setUp(flatDbMode);
final BonsaiWorldStateKeyValueStorage storage = spy(setUp(flatDbMode));
Assumptions.assumeTrue(flatDbMode == FlatDbMode.PARTIAL);
final BonsaiWorldStateKeyValueStorage storage = spy(emptyStorage());
final WorldStateStorageCoordinator coordinator = new WorldStateStorageCoordinator(storage);
final MerkleTrie<Bytes, Bytes> trie = TrieGenerator.generateTrie(coordinator, 1);
final TreeMap<Bytes32, Bytes> accounts =
@@ -359,9 +356,8 @@ public class BonsaiWorldStateKeyValueStorageTest {
@ParameterizedTest
@MethodSource("flatDbMode")
void getStorage_loadFromTrieWhenEmptyWithFullMode(final FlatDbMode flatDbMode) {
setUp(flatDbMode);
Assumptions.assumeTrue(flatDbMode == FlatDbMode.FULL);
final BonsaiWorldStateKeyValueStorage storage = spy(emptyStorage());
final BonsaiWorldStateKeyValueStorage storage = spy(setUp(flatDbMode));
storage.upgradeToFullFlatDbMode();
final WorldStateStorageCoordinator coordinator = new WorldStateStorageCoordinator(storage);
final MerkleTrie<Bytes, Bytes> trie = TrieGenerator.generateTrie(coordinator, 1);
@@ -380,8 +376,7 @@ public class BonsaiWorldStateKeyValueStorageTest {
@ParameterizedTest
@MethodSource("flatDbMode")
void clear_reloadFlatDbStrategy(final FlatDbMode flatDbMode) {
setUp(flatDbMode);
final BonsaiWorldStateKeyValueStorage storage = spy(emptyStorage());
final BonsaiWorldStateKeyValueStorage storage = spy(setUp(flatDbMode));
// save world state root hash
final BonsaiWorldStateKeyValueStorage.Updater updater = storage.updater();
@@ -489,7 +484,7 @@ public class BonsaiWorldStateKeyValueStorageTest {
void successfulPruneReturnsTrue() {
final KeyValueStorage mockTrieLogStorage = mock(KeyValueStorage.class);
when(mockTrieLogStorage.tryDelete(any())).thenReturn(true);
final BonsaiWorldStateKeyValueStorage storage = setupMockStorage(mockTrieLogStorage);
final BonsaiWorldStateKeyValueStorage storage = setupSpyStorage(mockTrieLogStorage);
assertThat(storage.pruneTrieLog(Hash.ZERO)).isTrue();
}
@@ -497,7 +492,7 @@ public class BonsaiWorldStateKeyValueStorageTest {
void failedPruneReturnsFalse() {
final KeyValueStorage mockTrieLogStorage = mock(KeyValueStorage.class);
when(mockTrieLogStorage.tryDelete(any())).thenReturn(false);
final BonsaiWorldStateKeyValueStorage storage = setupMockStorage(mockTrieLogStorage);
final BonsaiWorldStateKeyValueStorage storage = setupSpyStorage(mockTrieLogStorage);
assertThat(storage.pruneTrieLog(Hash.ZERO)).isFalse();
}
@@ -505,18 +500,17 @@ public class BonsaiWorldStateKeyValueStorageTest {
void exceptionalPruneReturnsFalse() {
final KeyValueStorage mockTrieLogStorage = mock(KeyValueStorage.class);
when(mockTrieLogStorage.tryDelete(any())).thenThrow(new RuntimeException("test exception"));
final BonsaiWorldStateKeyValueStorage storage = setupMockStorage(mockTrieLogStorage);
final BonsaiWorldStateKeyValueStorage storage = setupSpyStorage(mockTrieLogStorage);
assertThat(storage.pruneTrieLog(Hash.ZERO)).isFalse();
}
private BonsaiWorldStateKeyValueStorage setupMockStorage(
private BonsaiWorldStateKeyValueStorage setupSpyStorage(
final KeyValueStorage mockTrieLogStorage) {
final StorageProvider mockStorageProvider = mock(StorageProvider.class);
final StorageProvider mockStorageProvider = spy(new InMemoryKeyValueStorageProvider());
when(mockStorageProvider.getStorageBySegmentIdentifier(
KeyValueSegmentIdentifier.TRIE_LOG_STORAGE))
.thenReturn(mockTrieLogStorage);
when(mockStorageProvider.getStorageBySegmentIdentifiers(any()))
.thenReturn(mock(SegmentedKeyValueStorage.class));
return new BonsaiWorldStateKeyValueStorage(
mockStorageProvider,
new NoOpMetricsSystem(),

10
ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProviderTest.java
View File

@@ -63,7 +63,7 @@ class FlatDbStrategyProviderTest {
@Test
void loadsPartialFlatDbStrategyWhenNoFlatDbModeStored() {
flatDbStrategyProvider.loadFlatDbStrategy(composedWorldStateStorage);
assertThat(flatDbStrategyProvider.getFlatDbMode()).isEqualTo(FlatDbMode.PARTIAL);
assertThat(flatDbStrategyProvider.getFlatDbMode()).isEqualTo(FlatDbMode.FULL);
}
@Test
@@ -76,7 +76,7 @@ class FlatDbStrategyProviderTest {
assertThat(flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage))
.isInstanceOf(FullFlatDbStrategy.class);
assertThat(flatDbStrategyProvider.flatDbStrategy.codeStorageStrategy)
.isInstanceOf(AccountHashCodeStorageStrategy.class);
.isInstanceOf(CodeHashCodeStorageStrategy.class);
}
@ParameterizedTest
@@ -99,7 +99,7 @@ class FlatDbStrategyProviderTest {
codeByHashEnabled
? CodeHashCodeStorageStrategy.class
: AccountHashCodeStorageStrategy.class;
assertThat(flatDbStrategyProvider.flatDbMode).isEqualTo(FlatDbMode.PARTIAL);
assertThat(flatDbStrategyProvider.flatDbMode).isEqualTo(FlatDbMode.FULL);
assertThat(flatDbStrategyProvider.flatDbStrategy.codeStorageStrategy)
.isInstanceOf(expectedCodeStorageClass);
}
@@ -129,7 +129,7 @@ class FlatDbStrategyProviderTest {
transaction.commit();
flatDbStrategyProvider.loadFlatDbStrategy(composedWorldStateStorage);
assertThat(flatDbStrategyProvider.flatDbMode).isEqualTo(FlatDbMode.PARTIAL);
assertThat(flatDbStrategyProvider.flatDbMode).isEqualTo(FlatDbMode.FULL);
assertThat(flatDbStrategyProvider.flatDbStrategy.codeStorageStrategy)
.isInstanceOf(AccountHashCodeStorageStrategy.class);
}
@@ -158,7 +158,7 @@ class FlatDbStrategyProviderTest {
transaction.commit();
flatDbStrategyProvider.loadFlatDbStrategy(composedWorldStateStorage);
assertThat(flatDbStrategyProvider.flatDbMode).isEqualTo(FlatDbMode.PARTIAL);
assertThat(flatDbStrategyProvider.flatDbMode).isEqualTo(FlatDbMode.FULL);
assertThat(flatDbStrategyProvider.flatDbStrategy.codeStorageStrategy)
.isInstanceOf(CodeHashCodeStorageStrategy.class);
}

7
ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncConfiguration.java
View File

@@ -36,8 +36,6 @@ public class SnapSyncConfiguration {
public static final int DEFAULT_LOCAL_FLAT_STORAGE_COUNT_TO_HEAL_PER_REQUEST =
1024; // The default number of flat slots entries to verify and heal per request.
public static final Boolean DEFAULT_IS_FLAT_DB_HEALING_ENABLED = Boolean.FALSE;
public static final Boolean DEFAULT_SNAP_SERVER_ENABLED = Boolean.FALSE;
public static SnapSyncConfiguration getDefault() {
@@ -79,11 +77,6 @@ public class SnapSyncConfiguration {
return DEFAULT_LOCAL_FLAT_STORAGE_COUNT_TO_HEAL_PER_REQUEST;
}
@Value.Default
public Boolean isFlatDbHealingEnabled() {
return DEFAULT_IS_FLAT_DB_HEALING_ENABLED;
}
@Value.Default
public Boolean isSnapServerEnabled() {
return DEFAULT_SNAP_SERVER_ENABLED;

10
ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java
View File

@@ -184,16 +184,6 @@ public class SnapWorldStateDownloader implements WorldStateDownloader {
} else {
// start from scratch
worldStateStorageCoordinator.clear();
// we have to upgrade to full flat db mode if we are in bonsai mode
if (snapSyncConfiguration.isFlatDbHealingEnabled()) {
worldStateStorageCoordinator.applyOnMatchingStrategy(
DataStorageFormat.BONSAI,
strategy -> {
BonsaiWorldStateKeyValueStorage onBonsai =
(BonsaiWorldStateKeyValueStorage) strategy;
onBonsai.upgradeToFullFlatDbMode();
});
}
ranges.forEach(
(key, value) ->
newDownloadState.enqueueRequest(

2
ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java
View File

@@ -118,7 +118,7 @@ public class SnapWorldDownloadStateTest {
new BonsaiWorldStateKeyValueStorage(
new InMemoryKeyValueStorageProvider(),
new NoOpMetricsSystem(),
DataStorageConfiguration.DEFAULT_BONSAI_CONFIG);
DataStorageConfiguration.DEFAULT_BONSAI_PARTIAL_DB_CONFIG);
} else {
worldStateKeyValueStorage =
new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage());