vacp2p/linea-besu
1
0
Fork 0
mirror of https://github.com/vacp2p/linea-besu.git synced 2026-01-09 21:17:54 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add FlatDbStrategy (#5901)

Browse Source 
* move FlatDbReader to FlatDbStrategy (including writes), add getNearestTo

Signed-off-by: garyschulte <garyschulte@gmail.com>
This commit is contained in:
garyschulte
2023-09-22 10:51:39 -07:00
committed by GitHub
parent 9d7ee2b52b
commit d81e1f3042
19 changed files with 389 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java
View File

@@ -47,7 +47,7 @@ public class BonsaiSnapshotWorldStateKeyValueStorage extends BonsaiWorldStateKey
final ObservableMetricsSystem metricsSystem) {
super(
parentWorldStateStorage.flatDbMode,
parentWorldStateStorage.flatDbReaderStrategy,
parentWorldStateStorage.flatDbStrategy,
segmentedWorldStateStorage,
trieLogStorage,
metricsSystem);
@@ -77,7 +77,8 @@ public class BonsaiSnapshotWorldStateKeyValueStorage extends BonsaiWorldStateKey
public BonsaiUpdater updater() {
return new Updater(
((SnappedKeyValueStorage) composedWorldStateStorage).getSnapshotTransaction(),
trieLogStorage.startTransaction());
trieLogStorage.startTransaction(),
flatDbStrategy);
}
@Override

100
ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java
View File

@@ -21,9 +21,9 @@ import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIden
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.bonsai.storage.flat.FlatDbReaderStrategy;
import org.hyperledger.besu.ethereum.bonsai.storage.flat.FullFlatDbReaderStrategy;
import org.hyperledger.besu.ethereum.bonsai.storage.flat.PartialFlatDbReaderStrategy;
import org.hyperledger.besu.ethereum.bonsai.storage.flat.FlatDbStrategy;
import org.hyperledger.besu.ethereum.bonsai.storage.flat.FullFlatDbStrategy;
import org.hyperledger.besu.ethereum.bonsai.storage.flat.PartialFlatDbStrategy;
import org.hyperledger.besu.ethereum.storage.StorageProvider;
import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier;
import org.hyperledger.besu.ethereum.trie.MerkleTrie;
@@ -67,7 +67,7 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
public static final byte[] FLAT_DB_MODE = "flatDbStatus".getBytes(StandardCharsets.UTF_8);
protected FlatDbMode flatDbMode;
protected FlatDbReaderStrategy flatDbReaderStrategy;
protected FlatDbStrategy flatDbStrategy;
protected final SegmentedKeyValueStorage composedWorldStateStorage;
protected final KeyValueStorage trieLogStorage;
@@ -94,33 +94,49 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
public BonsaiWorldStateKeyValueStorage(
final FlatDbMode flatDbMode,
final FlatDbReaderStrategy flatDbReaderStrategy,
final FlatDbStrategy flatDbStrategy,
final SegmentedKeyValueStorage composedWorldStateStorage,
final KeyValueStorage trieLogStorage,
final ObservableMetricsSystem metricsSystem) {
this.flatDbMode = flatDbMode;
this.flatDbReaderStrategy = flatDbReaderStrategy;
this.flatDbStrategy = flatDbStrategy;
this.composedWorldStateStorage = composedWorldStateStorage;
this.trieLogStorage = trieLogStorage;
this.metricsSystem = metricsSystem;
}
public void loadFlatDbStrategy() {
this.flatDbMode =
private void loadFlatDbStrategy() {
// derive our flatdb strategy from db or default:
var newFlatDbMode = deriveFlatDbStrategy();
// if flatDbMode is not loaded or has changed, reload flatDbStrategy
if (this.flatDbMode == null || !this.flatDbMode.equals(newFlatDbMode)) {
this.flatDbMode = newFlatDbMode;
if (flatDbMode == FlatDbMode.FULL) {
this.flatDbStrategy = new FullFlatDbStrategy(metricsSystem);
} else {
this.flatDbStrategy = new PartialFlatDbStrategy(metricsSystem);
}
}
}
public FlatDbMode deriveFlatDbStrategy() {
var flatDbMode =
FlatDbMode.fromVersion(
composedWorldStateStorage
.get(TRIE_BRANCH_STORAGE, FLAT_DB_MODE)
.map(Bytes::wrap)
.orElse(
FlatDbMode.PARTIAL
.getVersion())); // for backward compatibility we use partial as
// default
.orElse(FlatDbMode.PARTIAL.getVersion()));
LOG.info("Bonsai flat db mode found {}", flatDbMode);
if (flatDbMode == FlatDbMode.FULL) {
this.flatDbReaderStrategy = new FullFlatDbReaderStrategy(metricsSystem);
} else {
this.flatDbReaderStrategy = new PartialFlatDbReaderStrategy(metricsSystem);
return flatDbMode;
}
public FlatDbStrategy getFlatDbStrategy() {
if (flatDbStrategy == null) {
loadFlatDbStrategy();
}
return flatDbStrategy;
}
@Override
@@ -133,22 +149,18 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
return flatDbMode;
}
public FlatDbReaderStrategy getFlatDbReaderStrategy() {
return flatDbReaderStrategy;
}
@Override
public Optional<Bytes> getCode(final Bytes32 codeHash, final Hash accountHash) {
if (codeHash.equals(Hash.EMPTY)) {
return Optional.of(Bytes.EMPTY);
} else {
return getFlatDbReaderStrategy().getCode(codeHash, accountHash, composedWorldStateStorage);
return getFlatDbStrategy().getFlatCode(codeHash, accountHash, composedWorldStateStorage);
}
}
public Optional<Bytes> getAccount(final Hash accountHash) {
return getFlatDbReaderStrategy()
.getAccount(
return getFlatDbStrategy()
.getFlatAccount(
this::getWorldStateRootHash,
this::getAccountStateTrieNode,
accountHash,
@@ -226,8 +238,8 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
final Supplier<Optional<Hash>> storageRootSupplier,
final Hash accountHash,
final StorageSlotKey storageSlotKey) {
return getFlatDbReaderStrategy()
.getStorageValueByStorageSlotKey(
return getFlatDbStrategy()
.getFlatStorageValueByStorageSlotKey(
this::getWorldStateRootHash,
storageRootSupplier,
(location, hash) -> getAccountStorageTrieNode(accountHash, location, hash),
@@ -239,14 +251,14 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
@Override
public Map<Bytes32, Bytes> streamFlatAccounts(
final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) {
return getFlatDbReaderStrategy()
return getFlatDbStrategy()
.streamAccountFlatDatabase(composedWorldStateStorage, startKeyHash, endKeyHash, max);
}
@Override
public Map<Bytes32, Bytes> streamFlatStorages(
final Hash accountHash, final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) {
return getFlatDbReaderStrategy()
return getFlatDbStrategy()
.streamStorageFlatDatabase(
composedWorldStateStorage, accountHash, startKeyHash, endKeyHash, max);
}
@@ -273,6 +285,7 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
public void upgradeToFullFlatDbMode() {
final SegmentedKeyValueStorageTransaction transaction =
composedWorldStateStorage.startTransaction();
// TODO: consider ARCHIVE mode
transaction.put(
TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.FULL.getVersion().toArrayUnsafe());
transaction.commit();
@@ -291,7 +304,7 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
@Override
public void clear() {
subscribers.forEach(BonsaiStorageSubscriber::onClearStorage);
getFlatDbReaderStrategy().clearAll(composedWorldStateStorage);
getFlatDbStrategy().clearAll(composedWorldStateStorage);
composedWorldStateStorage.clear(TRIE_BRANCH_STORAGE);
trieLogStorage.clear();
loadFlatDbStrategy(); // force reload of flat db reader strategy
@@ -306,13 +319,15 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
@Override
public void clearFlatDatabase() {
subscribers.forEach(BonsaiStorageSubscriber::onClearFlatDatabaseStorage);
getFlatDbReaderStrategy().resetOnResync(composedWorldStateStorage);
getFlatDbStrategy().resetOnResync(composedWorldStateStorage);
}
@Override
public BonsaiUpdater updater() {
return new Updater(
composedWorldStateStorage.startTransaction(), trieLogStorage.startTransaction());
composedWorldStateStorage.startTransaction(),
trieLogStorage.startTransaction(),
flatDbStrategy);
}
@Override
@@ -351,18 +366,21 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
private final SegmentedKeyValueStorageTransaction composedWorldStateTransaction;
private final KeyValueStorageTransaction trieLogStorageTransaction;
private final FlatDbStrategy flatDbStrategy;
public Updater(
final SegmentedKeyValueStorageTransaction composedWorldStateTransaction,
final KeyValueStorageTransaction trieLogStorageTransaction) {
final KeyValueStorageTransaction trieLogStorageTransaction,
final FlatDbStrategy flatDbStrategy) {
this.composedWorldStateTransaction = composedWorldStateTransaction;
this.trieLogStorageTransaction = trieLogStorageTransaction;
this.flatDbStrategy = flatDbStrategy;
}
@Override
public BonsaiUpdater removeCode(final Hash accountHash) {
composedWorldStateTransaction.remove(CODE_STORAGE, accountHash.toArrayUnsafe());
flatDbStrategy.removeFlatCode(composedWorldStateTransaction, accountHash);
return this;
}
@@ -372,14 +390,13 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
// Don't save empty values
return this;
}
composedWorldStateTransaction.put(
CODE_STORAGE, accountHash.toArrayUnsafe(), code.toArrayUnsafe());
flatDbStrategy.putFlatCode(composedWorldStateTransaction, accountHash, codeHash, code);
return this;
}
@Override
public BonsaiUpdater removeAccountInfoState(final Hash accountHash) {
composedWorldStateTransaction.remove(ACCOUNT_INFO_STATE, accountHash.toArrayUnsafe());
flatDbStrategy.removeFlatAccount(composedWorldStateTransaction, accountHash);
return this;
}
@@ -389,8 +406,7 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
// Don't save empty values
return this;
}
composedWorldStateTransaction.put(
ACCOUNT_INFO_STATE, accountHash.toArrayUnsafe(), accountValue.toArrayUnsafe());
flatDbStrategy.putFlatAccount(composedWorldStateTransaction, accountHash, accountValue);
return this;
}
@@ -441,18 +457,16 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
@Override
public synchronized BonsaiUpdater putStorageValueBySlotHash(
final Hash accountHash, final Hash slotHash, final Bytes storage) {
composedWorldStateTransaction.put(
ACCOUNT_STORAGE_STORAGE,
Bytes.concatenate(accountHash, slotHash).toArrayUnsafe(),
storage.toArrayUnsafe());
flatDbStrategy.putFlatAccountStorageValueByStorageSlotHash(
composedWorldStateTransaction, accountHash, slotHash, storage);
return this;
}
@Override
public synchronized void removeStorageValueBySlotHash(
final Hash accountHash, final Hash slotHash) {
composedWorldStateTransaction.remove(
ACCOUNT_STORAGE_STORAGE, Bytes.concatenate(accountHash, slotHash).toArrayUnsafe());
flatDbStrategy.removeFlatAccountStorageValueByStorageSlotHash(
composedWorldStateTransaction, accountHash, slotHash);
}
@Override

76
ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java → ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbStrategy.java
View File

@@ -26,6 +26,7 @@ import org.hyperledger.besu.metrics.BesuMetricCategory;
import org.hyperledger.besu.plugin.services.MetricsSystem;
import org.hyperledger.besu.plugin.services.metrics.Counter;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction;
import java.util.Map;
import java.util.Optional;
@@ -40,11 +41,11 @@ import org.apache.tuweni.bytes.Bytes32;
import org.apache.tuweni.rlp.RLP;
/**
* This class represents a FlatDbReaderStrategy, which is responsible for reading data from flat
* databases. It implements various methods for retrieving account data, code data, and storage data
* from the corresponding KeyValueStorage.
* This class represents a FlatDbReaderStrategy, which is responsible for reading and writing data
* from flat databases. It implements various methods for storing and retrieving account data, code
* data, and storage data from the corresponding KeyValueStorage.
*/
public abstract class FlatDbReaderStrategy {
public abstract class FlatDbStrategy {
protected final MetricsSystem metricsSystem;
protected final Counter getAccountCounter;
@@ -53,7 +54,7 @@ public abstract class FlatDbReaderStrategy {
protected final Counter getStorageValueCounter;
protected final Counter getStorageValueFlatDatabaseCounter;
public FlatDbReaderStrategy(final MetricsSystem metricsSystem) {
public FlatDbStrategy(final MetricsSystem metricsSystem) {
this.metricsSystem = metricsSystem;
getAccountCounter =
@@ -84,7 +85,7 @@ public abstract class FlatDbReaderStrategy {
/*
* Retrieves the account data for the given account hash, using the world state root hash supplier and node loader.
*/
public abstract Optional<Bytes> getAccount(
public abstract Optional<Bytes> getFlatAccount(
Supplier<Optional<Bytes>> worldStateRootHashSupplier,
NodeLoader nodeLoader,
Hash accountHash,
@@ -94,7 +95,7 @@ public abstract class FlatDbReaderStrategy {
* Retrieves the storage value for the given account hash and storage slot key, using the world state root hash supplier, storage root supplier, and node loader.
*/
public abstract Optional<Bytes> getStorageValueByStorageSlotKey(
public abstract Optional<Bytes> getFlatStorageValueByStorageSlotKey(
Supplier<Optional<Bytes>> worldStateRootHashSupplier,
Supplier<Optional<Hash>> storageRootSupplier,
NodeLoader nodeLoader,
@@ -105,7 +106,7 @@ public abstract class FlatDbReaderStrategy {
/*
* Retrieves the code data for the given code hash and account hash.
*/
public Optional<Bytes> getCode(
public Optional<Bytes> getFlatCode(
final Bytes32 codeHash, final Hash accountHash, final SegmentedKeyValueStorage storage) {
if (codeHash.equals(Hash.EMPTY)) {
return Optional.of(Bytes.EMPTY);
@@ -117,6 +118,65 @@ public abstract class FlatDbReaderStrategy {
}
}
/*
* Puts the account data for the given account hash, using the world state root hash supplier and node loader.
*/
public void putFlatAccount(
final SegmentedKeyValueStorageTransaction transaction,
final Hash accountHash,
final Bytes accountValue) {
transaction.put(ACCOUNT_INFO_STATE, accountHash.toArrayUnsafe(), accountValue.toArrayUnsafe());
}
public void removeFlatAccount(
final SegmentedKeyValueStorageTransaction transaction, final Hash accountHash) {
transaction.remove(ACCOUNT_INFO_STATE, accountHash.toArrayUnsafe());
}
/*
* Puts the storage value for the given account hash and storage slot key, using the world state root hash supplier, storage root supplier, and node loader.
*/
public void putFlatAccountStorageValueByStorageSlotHash(
final SegmentedKeyValueStorageTransaction transaction,
final Hash accountHash,
final Hash slotHash,
final Bytes storage) {
transaction.put(
ACCOUNT_STORAGE_STORAGE,
Bytes.concatenate(accountHash, slotHash).toArrayUnsafe(),
storage.toArrayUnsafe());
}
/*
* Removes the storage value for the given account hash and storage slot key, using the world state root hash supplier, storage root supplier, and node loader.
*/
public void removeFlatAccountStorageValueByStorageSlotHash(
final SegmentedKeyValueStorageTransaction transaction,
final Hash accountHash,
final Hash slotHash) {
transaction.remove(
ACCOUNT_STORAGE_STORAGE, Bytes.concatenate(accountHash, slotHash).toArrayUnsafe());
}
/*
* Removes code for the given account hash.
*/
public void removeFlatCode(
final SegmentedKeyValueStorageTransaction transaction, final Hash accountHash) {
transaction.remove(CODE_STORAGE, accountHash.toArrayUnsafe());
}
/*
* Puts the code data for the given code hash and account hash.
*/
public void putFlatCode(
final SegmentedKeyValueStorageTransaction transaction,
final Hash accountHash,
final Bytes32 codeHash,
final Bytes code) {
transaction.put(CODE_STORAGE, accountHash.toArrayUnsafe(), code.toArrayUnsafe());
}
public void clearAll(final SegmentedKeyValueStorage storage) {
storage.clear(ACCOUNT_INFO_STATE);
storage.clear(ACCOUNT_STORAGE_STORAGE);

8
ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FullFlatDbReaderStrategy.java → ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FullFlatDbStrategy.java
View File

@@ -31,13 +31,13 @@ import java.util.function.Supplier;
import org.apache.tuweni.bytes.Bytes;
public class FullFlatDbReaderStrategy extends FlatDbReaderStrategy {
public class FullFlatDbStrategy extends FlatDbStrategy {
protected final Counter getAccountNotFoundInFlatDatabaseCounter;
protected final Counter getStorageValueNotFoundInFlatDatabaseCounter;
public FullFlatDbReaderStrategy(final MetricsSystem metricsSystem) {
public FullFlatDbStrategy(final MetricsSystem metricsSystem) {
super(metricsSystem);
getAccountNotFoundInFlatDatabaseCounter =
@@ -54,7 +54,7 @@ public class FullFlatDbReaderStrategy extends FlatDbReaderStrategy {
}
@Override
public Optional<Bytes> getAccount(
public Optional<Bytes> getFlatAccount(
final Supplier<Optional<Bytes>> worldStateRootHashSupplier,
final NodeLoader nodeLoader,
final Hash accountHash,
@@ -71,7 +71,7 @@ public class FullFlatDbReaderStrategy extends FlatDbReaderStrategy {
}
@Override
public Optional<Bytes> getStorageValueByStorageSlotKey(
public Optional<Bytes> getFlatStorageValueByStorageSlotKey(
final Supplier<Optional<Bytes>> worldStateRootHashSupplier,
final Supplier<Optional<Hash>> storageRootSupplier,
final NodeLoader nodeLoader,

8
ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/PartialFlatDbReaderStrategy.java → ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/PartialFlatDbStrategy.java
View File

@@ -44,7 +44,7 @@ import org.apache.tuweni.rlp.RLP;
* methods, which checks if the data is present in the flat database, and if not, queries the merkle
* trie
*/
public class PartialFlatDbReaderStrategy extends FlatDbReaderStrategy {
public class PartialFlatDbStrategy extends FlatDbStrategy {
protected final Counter getAccountMerkleTrieCounter;
protected final Counter getAccountMissingMerkleTrieCounter;
@@ -52,7 +52,7 @@ public class PartialFlatDbReaderStrategy extends FlatDbReaderStrategy {
protected final Counter getStorageValueMerkleTrieCounter;
protected final Counter getStorageValueMissingMerkleTrieCounter;
public PartialFlatDbReaderStrategy(final MetricsSystem metricsSystem) {
public PartialFlatDbStrategy(final MetricsSystem metricsSystem) {
super(metricsSystem);
getAccountMerkleTrieCounter =
metricsSystem.createCounter(
@@ -80,7 +80,7 @@ public class PartialFlatDbReaderStrategy extends FlatDbReaderStrategy {
}
@Override
public Optional<Bytes> getAccount(
public Optional<Bytes> getFlatAccount(
final Supplier<Optional<Bytes>> worldStateRootHashSupplier,
final NodeLoader nodeLoader,
final Hash accountHash,
@@ -111,7 +111,7 @@ public class PartialFlatDbReaderStrategy extends FlatDbReaderStrategy {
}
@Override
public Optional<Bytes> getStorageValueByStorageSlotKey(
public Optional<Bytes> getFlatStorageValueByStorageSlotKey(
final Supplier<Optional<Bytes>> worldStateRootHashSupplier,
final Supplier<Optional<Hash>> storageRootSupplier,
final NodeLoader nodeLoader,

4
ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java
View File

@@ -498,7 +498,9 @@ public class BonsaiWorldState
@Override
public Hash frontierRootHash() {
return calculateRootHash(
Optional.of(new BonsaiWorldStateKeyValueStorage.Updater(noOpSegmentedTx, noOpTx)),
Optional.of(
new BonsaiWorldStateKeyValueStorage.Updater(
noOpSegmentedTx, noOpTx, worldStateStorage.getFlatDbStrategy())),
accumulator.copy());
}

5
ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorageTest.java → ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java
View File

@@ -12,7 +12,7 @@
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.ethereum.bonsai;
package org.hyperledger.besu.ethereum.bonsai.storage;
import static org.assertj.core.api.Assertions.assertThat;
import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY;
@@ -26,7 +26,6 @@ import static org.mockito.Mockito.verify;
import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider;
import org.hyperledger.besu.ethereum.core.TrieGenerator;
import org.hyperledger.besu.ethereum.rlp.RLP;
@@ -372,7 +371,7 @@ public class BonsaiWorldStateKeyValueStorageTest {
// clear
storage.clear();
assertThat(storage.getFlatDbReaderStrategy()).isNotNull();
assertThat(storage.getFlatDbStrategy()).isNotNull();
assertThat(storage.getAccount(Hash.ZERO)).isEmpty();
}

2
plugin-api/build.gradle
View File

@@ -69,7 +69,7 @@ Calculated : ${currentHash}
tasks.register('checkAPIChanges', FileStateChecker) {
description = "Checks that the API for the Plugin-API project does not change without deliberate thought"
files = sourceSets.main.allJava.files
knownHash = 'tpSnjt4HgqSiOTJhBbYdB0r1nFX4QZbicjfloI71Wf0='
knownHash = 'BhqPyj1fT50NWuHTgzgCmW1ynAPj/2QiGWraq5OwgOQ='
}
check.dependsOn('checkAPIChanges')

31
plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentedKeyValueStorage.java
View File

@@ -23,6 +23,7 @@ import java.util.function.Predicate;
import java.util.stream.Stream;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.tuweni.bytes.Bytes;
/** Service provided by Besu to facilitate persistent data storage. */
public interface SegmentedKeyValueStorage extends Closeable {
@@ -37,6 +38,18 @@ public interface SegmentedKeyValueStorage extends Closeable {
*/
Optional<byte[]> get(SegmentIdentifier segment, byte[] key) throws StorageException;
/**
* Find the key and corresponding value "nearest to" the specified key. Nearest is defined as
* either matching the supplied key or the key lexicographically prior to it.
*
* @param segmentIdentifier segment to scan
* @param key key for which we are searching for the nearest match.
* @return Optional of NearestKeyValue-wrapped matched key and corresponding value.
* @throws StorageException the storage exception
*/
Optional<NearestKeyValue> getNearestTo(final SegmentIdentifier segmentIdentifier, Bytes key)
throws StorageException;
/**
* Contains key.
*
@@ -144,4 +157,22 @@ public interface SegmentedKeyValueStorage extends Closeable {
* @return boolean indicating whether the underlying storage is closed.
*/
boolean isClosed();
/**
* record type used to wrap responses from getNearestTo, includes the matched key and the value.
*
* @param key the matched (nearest) key
* @param value the corresponding value
*/
record NearestKeyValue(Bytes key, Optional<byte[]> value) {
/**
* Convenience method to map the Optional value to Bytes.
*
* @return Optional of Bytes.
*/
public Optional<Bytes> wrapBytes() {
return value.map(Bytes::wrap);
}
}
}

15
plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueSnapshot.java
View File

@@ -33,8 +33,11 @@ import java.util.function.Predicate;
import java.util.stream.Stream;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.tuweni.bytes.Bytes;
import org.rocksdb.AbstractRocksIterator;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.OptimisticTransactionDB;
import org.rocksdb.RocksIterator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -73,6 +76,18 @@ public class RocksDBColumnarKeyValueSnapshot
return snapTx.get(segment, key);
}
@Override
public Optional<NearestKeyValue> getNearestTo(
final SegmentIdentifier segmentIdentifier, final Bytes key) throws StorageException {
try (final RocksIterator rocksIterator = snapTx.getIterator(segmentIdentifier)) {
rocksIterator.seekForPrev(key.toArrayUnsafe());
return Optional.of(rocksIterator)
.filter(AbstractRocksIterator::isValid)
.map(it -> new NearestKeyValue(Bytes.of(it.key()), Optional.of(it.value())));
}
}
@Override
public Stream<Pair<byte[], byte[]>> stream(final SegmentIdentifier segment) {
throwIfClosed();

14
plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java
View File

@@ -44,6 +44,7 @@ import com.google.common.base.Splitter;
import com.google.common.collect.Streams;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.tuweni.bytes.Bytes;
import org.rocksdb.AbstractRocksIterator;
import org.rocksdb.BlockBasedTableConfig;
import org.rocksdb.BloomFilter;
import org.rocksdb.ColumnFamilyDescriptor;
@@ -318,6 +319,19 @@ public abstract class RocksDBColumnarKeyValueStorage implements SegmentedKeyValu
}
}
@Override
public Optional<NearestKeyValue> getNearestTo(
final SegmentIdentifier segmentIdentifier, final Bytes key) throws StorageException {
try (final RocksIterator rocksIterator =
getDB().newIterator(safeColumnHandle(segmentIdentifier))) {
rocksIterator.seekForPrev(key.toArrayUnsafe());
return Optional.of(rocksIterator)
.filter(AbstractRocksIterator::isValid)
.map(it -> new NearestKeyValue(Bytes.of(it.key()), Optional.of(it.value())));
}
}
@Override
public Stream<Pair<byte[], byte[]>> stream(final SegmentIdentifier segmentIdentifier) {
final RocksIterator rocksIterator = getDB().newIterator(safeColumnHandle(segmentIdentifier));

13
plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBSnapshotTransaction.java
View File

@@ -137,6 +137,19 @@ public class RocksDBSnapshotTransaction
}
}
/**
* get a RocksIterator that reads through the transaction to represent the current state.
*
* <p>be sure to close this iterator, like in a try-with-resources block, otherwise a native
* memory leak might occur.
*
* @param segmentId id for the segment to iterate over.
* @return RocksIterator
*/
public RocksIterator getIterator(final SegmentIdentifier segmentId) {
return snapTx.getIterator(readOptions, columnFamilyMapper.apply(segmentId));
}
/**
* Stream.
*

2
services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java
View File

@@ -32,7 +32,7 @@ import org.apache.tuweni.bytes.Bytes;
*/
public class InMemoryKeyValueStorage extends SegmentedKeyValueStorageAdapter {
private static final SegmentIdentifier SEGMENT_IDENTIFIER =
static final SegmentIdentifier SEGMENT_IDENTIFIER =
new SegmentIdentifier() {
private static final String NAME = "SEGMENT_IDENTIFIER";

20
services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java
View File

@@ -93,6 +93,26 @@ public class LayeredKeyValueStorage extends SegmentedInMemoryKeyValueStorage
}
}
@Override
public Optional<NearestKeyValue> getNearestTo(
final SegmentIdentifier segmentIdentifier, final Bytes key) throws StorageException {
Optional<NearestKeyValue> ourNearest = super.getNearestTo(segmentIdentifier, key);
Optional<NearestKeyValue> parentNearest = parent.getNearestTo(segmentIdentifier, key);
if (ourNearest.isPresent() && parentNearest.isPresent()) {
// Both are present, return the one closer to the key
int ourDistance = ourNearest.get().key().commonPrefixLength(key);
int parentDistance = parentNearest.get().key().commonPrefixLength(key);
return (ourDistance <= parentDistance) ? ourNearest : parentNearest;
} else if (ourNearest.isPresent()) {
// Only ourNearest is present
return ourNearest;
} else {
// return parentNearest, which may be an empty Optional
return parentNearest;
}
}
@Override
public Stream<Pair<byte[], byte[]>> stream(final SegmentIdentifier segmentId) {
throwIfClosed();

27
services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedInMemoryKeyValueStorage.java
View File

@@ -24,6 +24,7 @@ import org.hyperledger.besu.plugin.services.storage.SnappableKeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.SnappedKeyValueStorage;
import java.io.PrintStream;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -113,6 +114,32 @@ public class SegmentedInMemoryKeyValueStorage
}
}
@Override
public Optional<NearestKeyValue> getNearestTo(
final SegmentIdentifier segmentIdentifier, final Bytes key) throws StorageException {
final Lock lock = rwLock.readLock();
lock.lock();
try {
// TODO: revisit this for sort performance
Comparator<Map.Entry<Bytes, Optional<byte[]>>> comparing =
Comparator.comparing(
(Map.Entry<Bytes, Optional<byte[]>> a) -> a.getKey().commonPrefixLength(key))
.thenComparing(Map.Entry.comparingByKey());
return this.hashValueStore
.computeIfAbsent(segmentIdentifier, s -> new HashMap<>())
.entrySet()
.stream()
// only return keys equal to or less than
.filter(e -> e.getKey().compareTo(key) <= 0)
.sorted(comparing.reversed())
.findFirst()
.map(z -> new NearestKeyValue(z.getKey(), z.getValue()));
} finally {
lock.unlock();
}
}
@Override
public Set<byte[]> getAllKeysThat(
final SegmentIdentifier segmentIdentifier, final Predicate<byte[]> returnCondition) {

4
services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedKeyValueStorageAdapter.java
View File

@@ -51,6 +51,10 @@ public class SegmentedKeyValueStorageAdapter implements KeyValueStorage {
this.storage = storage;
}
SegmentedKeyValueStorage getSegmentedStore() {
return this.storage;
}
@Override
public void clear() {
throwIfClosed();

82
services/kvstore/src/test/java/org/hyperledger/besu/services/kvstore/AbstractSegmentedKeyValueStorageTest.java Normal file
View File

@@ -0,0 +1,82 @@
/*
* Copyright Hyperledger Besu Contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*
*/
package org.hyperledger.besu.services.kvstore;
import static org.assertj.core.api.Assertions.assertThat;
import static org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage.SEGMENT_IDENTIFIER;
import org.hyperledger.besu.kvstore.AbstractKeyValueStorageTest;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction;
import java.util.stream.IntStream;
import org.apache.tuweni.bytes.Bytes;
import org.junit.jupiter.api.Test;
public abstract class AbstractSegmentedKeyValueStorageTest extends AbstractKeyValueStorageTest {
public abstract SegmentedKeyValueStorage createSegmentedStore();
@Test
public void assertSegmentedIsNearestTo() throws Exception {
try (final var store = this.createSegmentedStore()) {
// create 10 entries
final SegmentedKeyValueStorageTransaction tx = store.startTransaction();
IntStream.range(1, 10)
.forEach(
i -> {
final byte[] key = bytesFromHexString("000" + i);
final byte[] value = bytesFromHexString("0FFF");
tx.put(SEGMENT_IDENTIFIER, key, value);
// different common prefix, and reversed order of bytes:
final byte[] key2 = bytesFromHexString("010" + (10 - i));
final byte[] value2 = bytesFromHexString("0FFF");
tx.put(SEGMENT_IDENTIFIER, key2, value2);
});
tx.commit();
// assert 0009 is closest to 000F
var val = store.getNearestTo(SEGMENT_IDENTIFIER, Bytes.fromHexString("000F"));
assertThat(val).isPresent();
assertThat(val.get().key()).isEqualTo(Bytes.fromHexString("0009"));
// assert 0109 is closest to 010D
var val2 = store.getNearestTo(SEGMENT_IDENTIFIER, Bytes.fromHexString("010D"));
assertThat(val2).isPresent();
assertThat(val2.get().key()).isEqualTo(Bytes.fromHexString("0109"));
// assert 0103 is closest to 0103
var val3 = store.getNearestTo(SEGMENT_IDENTIFIER, Bytes.fromHexString("0103"));
assertThat(val3).isPresent();
assertThat(val3.get().key()).isEqualTo(Bytes.fromHexString("0103"));
// assert 0003 is closest to 0003
var val4 = store.getNearestTo(SEGMENT_IDENTIFIER, Bytes.fromHexString("0003"));
assertThat(val4).isPresent();
assertThat(val4.get().key()).isEqualTo(Bytes.fromHexString("0003"));
// assert 0001 is closest to 0001
var val5 = store.getNearestTo(SEGMENT_IDENTIFIER, Bytes.fromHexString("0001"));
assertThat(val5).isPresent();
assertThat(val5.get().key()).isEqualTo(Bytes.fromHexString("0001"));
// assert 0000 is not present
var val6 = store.getNearestTo(SEGMENT_IDENTIFIER, Bytes.fromHexString("0000"));
assertThat(val6).isNotPresent();
}
}
}

9
services/kvstore/src/test/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorageTest.java
View File

@@ -14,13 +14,18 @@
*/
package org.hyperledger.besu.services.kvstore;
import org.hyperledger.besu.kvstore.AbstractKeyValueStorageTest;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage;
public class InMemoryKeyValueStorageTest extends AbstractKeyValueStorageTest {
public class InMemoryKeyValueStorageTest extends AbstractSegmentedKeyValueStorageTest {
@Override
protected KeyValueStorage createStore() {
return new InMemoryKeyValueStorage();
}
@Override
public SegmentedKeyValueStorage createSegmentedStore() {
return new SegmentedInMemoryKeyValueStorage();
}
}

33
services/kvstore/src/test/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorageTest.java Normal file
View File

@@ -0,0 +1,33 @@
/*
* Copyright Hyperledger Besu Contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*
*/
package org.hyperledger.besu.services.kvstore;
import static org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage.SEGMENT_IDENTIFIER;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage;
public class LayeredKeyValueStorageTest extends AbstractSegmentedKeyValueStorageTest {
@Override
protected KeyValueStorage createStore() {
return new SegmentedKeyValueStorageAdapter(SEGMENT_IDENTIFIER, createSegmentedStore());
}
@Override
public SegmentedKeyValueStorage createSegmentedStore() {
return new LayeredKeyValueStorage(new SegmentedInMemoryKeyValueStorage());
}
}