vacp2p/linea-besu
1
0
Fork 0
mirror of https://github.com/vacp2p/linea-besu.git synced 2026-01-09 21:17:54 -05:00
Code Issues Packages Projects Releases Wiki Activity

Implement getNearest methods (#7258)

Browse Source 
Signed-off-by: Karim Taam <karim.t2am@gmail.com>
This commit is contained in:
Karim Taam
2024-08-16 13:58:24 +01:00
committed by GitHub
parent 8523ed0e43
commit dff99f7c17
12 changed files with 634 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
plugin-api/build.gradle
View File

@@ -70,7 +70,7 @@ Calculated : ${currentHash}
tasks.register('checkAPIChanges', FileStateChecker) {
description = "Checks that the API for the Plugin-API project does not change without deliberate thought"
files = sourceSets.main.allJava.files
knownHash = 'W1gv5UjqU+RJZJN6xPNjVfjuz7nKIcBgmh1j2XON4EU='
knownHash = '6Hy3eaCpnxehyDO3smSAr1i2DsB2q/V37/m8POycikI='
}
check.dependsOn('checkAPIChanges')

34
plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentedKeyValueStorage.java
View File

@@ -39,15 +39,35 @@ public interface SegmentedKeyValueStorage extends Closeable {
Optional<byte[]> get(SegmentIdentifier segment, byte[] key) throws StorageException;
/**
* Find the key and corresponding value "nearest to" the specified key. Nearest is defined as
* either matching the supplied key or the key lexicographically prior to it.
* Finds the key and corresponding value that is "nearest before" the specified key. "Nearest
* before" is defined as the closest key that is either exactly matching the supplied key or
* lexicographically before it.
*
* @param segmentIdentifier segment to scan
* @param key key for which we are searching for the nearest match.
* @return Optional of NearestKeyValue-wrapped matched key and corresponding value.
* @throws StorageException the storage exception
* @param segmentIdentifier The segment to scan for the nearest key.
* @param key The key for which we are searching for the nearest match before it.
* @return An Optional of NearestKeyValue, wrapping the matched key and its corresponding value,
* if found.
* @throws StorageException If an error occurs during the retrieval process.
*/
Optional<NearestKeyValue> getNearestTo(final SegmentIdentifier segmentIdentifier, Bytes key)
Optional<NearestKeyValue> getNearestBefore(final SegmentIdentifier segmentIdentifier, Bytes key)
throws StorageException;
/**
* Finds the key and corresponding value that is "nearest after" the specified key. "Nearest
* after" is defined as the closest key that is either exactly matching the supplied key or
* lexicographically after it.
*
* <p>This method aims to find the next key in sequence after the provided key, considering the
* order of keys within the specified segment. It is particularly useful for iterating over keys
* in a sorted manner starting from a given key.
*
* @param segmentIdentifier The segment to scan for the nearest key.
* @param key The key for which we are searching for the nearest match after it.
* @return An Optional of NearestKeyValue, wrapping the matched key and its corresponding value,
* if found.
* @throws StorageException If an error occurs during the retrieval process.
*/
Optional<NearestKeyValue> getNearestAfter(final SegmentIdentifier segmentIdentifier, Bytes key)
throws StorageException;
/**

13
plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueSnapshot.java
View File

@@ -76,7 +76,7 @@ public class RocksDBColumnarKeyValueSnapshot
}
@Override
public Optional<NearestKeyValue> getNearestTo(
public Optional<NearestKeyValue> getNearestBefore(
final SegmentIdentifier segmentIdentifier, final Bytes key) throws StorageException {
try (final RocksIterator rocksIterator = snapTx.getIterator(segmentIdentifier)) {
@@ -87,6 +87,17 @@ public class RocksDBColumnarKeyValueSnapshot
}
}
@Override
public Optional<NearestKeyValue> getNearestAfter(
final SegmentIdentifier segmentIdentifier, final Bytes key) throws StorageException {
try (final RocksIterator rocksIterator = snapTx.getIterator(segmentIdentifier)) {
rocksIterator.seek(key.toArrayUnsafe());
return Optional.of(rocksIterator)
.filter(AbstractRocksIterator::isValid)
.map(it -> new NearestKeyValue(Bytes.of(it.key()), Optional.of(it.value())));
}
}
@Override
public Stream<Pair<byte[], byte[]>> stream(final SegmentIdentifier segment) {
throwIfClosed();

15
plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java
View File

@@ -354,7 +354,7 @@ public abstract class RocksDBColumnarKeyValueStorage implements SegmentedKeyValu
}
@Override
public Optional<NearestKeyValue> getNearestTo(
public Optional<NearestKeyValue> getNearestBefore(
final SegmentIdentifier segmentIdentifier, final Bytes key) throws StorageException {
try (final RocksIterator rocksIterator =
@@ -366,6 +366,19 @@ public abstract class RocksDBColumnarKeyValueStorage implements SegmentedKeyValu
}
}
@Override
public Optional<NearestKeyValue> getNearestAfter(
final SegmentIdentifier segmentIdentifier, final Bytes key) throws StorageException {
try (final RocksIterator rocksIterator =
getDB().newIterator(safeColumnHandle(segmentIdentifier))) {
rocksIterator.seek(key.toArrayUnsafe());
return Optional.of(rocksIterator)
.filter(AbstractRocksIterator::isValid)
.map(it -> new NearestKeyValue(Bytes.of(it.key()), Optional.of(it.value())));
}
}
@Override
public Stream<Pair<byte[], byte[]>> stream(final SegmentIdentifier segmentIdentifier) {
final RocksIterator rocksIterator = getDB().newIterator(safeColumnHandle(segmentIdentifier));

336
plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/NearestKeyValueStorageTest.java Normal file
View File

@@ -0,0 +1,336 @@
/*
* Copyright contributors to Hyperledger Besu.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.plugin.services.storage.rocksdb;
import static org.assertj.core.api.Assertions.assertThat;
import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_BACKGROUND_THREAD_COUNT;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_CACHE_CAPACITY;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_IS_HIGH_SPEC;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_MAX_OPEN_FILES;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier;
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
import org.hyperledger.besu.plugin.services.BesuConfiguration;
import org.hyperledger.besu.plugin.services.storage.DataStorageConfiguration;
import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage.NearestKeyValue;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction;
import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBFactoryConfiguration;
import org.hyperledger.besu.services.kvstore.LayeredKeyValueStorage;
import org.hyperledger.besu.services.kvstore.SegmentedInMemoryKeyValueStorage;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.stream.IntStream;
import org.apache.tuweni.bytes.Bytes;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.junit.jupiter.MockitoExtension;
@ExtendWith(MockitoExtension.class)
public class NearestKeyValueStorageTest {
@TempDir private static Path tempDir;
private static RocksDBKeyValueStorageFactory rocksdbStorageFactory;
private static BesuConfiguration commonConfiguration;
@BeforeAll
public static void setup() throws IOException {
rocksdbStorageFactory =
new RocksDBKeyValueStorageFactory(
() ->
new RocksDBFactoryConfiguration(
DEFAULT_MAX_OPEN_FILES,
DEFAULT_BACKGROUND_THREAD_COUNT,
DEFAULT_CACHE_CAPACITY,
DEFAULT_IS_HIGH_SPEC),
Arrays.asList(KeyValueSegmentIdentifier.values()),
RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS);
Utils.createDatabaseMetadataV2(tempDir, DataStorageFormat.BONSAI, 2);
mockCommonConfiguration(tempDir);
}
@Test
public void testNearestRocksdbWithInMemoryKeyValueStorage() {
final SegmentedKeyValueStorage rockdDBKeyValueStorage =
getRocksDBKeyValueStorage(TRIE_BRANCH_STORAGE);
final SegmentedKeyValueStorageTransaction rocksDbTransaction =
rockdDBKeyValueStorage.startTransaction();
final SegmentedKeyValueStorage inMemoryDBKeyValueStorage = getInMemoryDBKeyValueStorage();
final SegmentedKeyValueStorageTransaction inMemoryDBTransaction =
inMemoryDBKeyValueStorage.startTransaction();
IntStream.range(1, 10)
.forEach(
i -> {
final byte[] key = Bytes.fromHexString("0x000" + i).toArrayUnsafe();
final byte[] value = Bytes.fromHexString("0FFF").toArrayUnsafe();
rocksDbTransaction.put(TRIE_BRANCH_STORAGE, key, value);
inMemoryDBTransaction.put(TRIE_BRANCH_STORAGE, key, value);
// different common prefix, and reversed order of bytes:
final byte[] key2 = Bytes.fromHexString("0x010" + (10 - i)).toArrayUnsafe();
final byte[] value2 = Bytes.fromHexString("0FFF").toArrayUnsafe();
rocksDbTransaction.put(TRIE_BRANCH_STORAGE, key2, value2);
inMemoryDBTransaction.put(TRIE_BRANCH_STORAGE, key2, value2);
// different size:
final byte[] key3 = Bytes.fromHexString("0x01011" + (10 - i)).toArrayUnsafe();
final byte[] value3 = Bytes.fromHexString("0FFF").toArrayUnsafe();
rocksDbTransaction.put(TRIE_BRANCH_STORAGE, key3, value3);
inMemoryDBTransaction.put(TRIE_BRANCH_STORAGE, key3, value3);
final byte[] key4 = Bytes.fromHexString("0x0" + (10 - i)).toArrayUnsafe();
final byte[] value4 = Bytes.fromHexString("0FFF").toArrayUnsafe();
rocksDbTransaction.put(TRIE_BRANCH_STORAGE, key4, value4);
inMemoryDBTransaction.put(TRIE_BRANCH_STORAGE, key4, value4);
});
rocksDbTransaction.commit();
inMemoryDBTransaction.commit();
// compare rocksdb implementation with inmemory implementation
rockdDBKeyValueStorage.stream(TRIE_BRANCH_STORAGE)
.forEach(
pair -> {
final Bytes key = Bytes.of(pair.getKey());
assertThat(
isNearestKeyValueTheSame(
inMemoryDBKeyValueStorage.getNearestBefore(TRIE_BRANCH_STORAGE, key),
rockdDBKeyValueStorage.getNearestBefore(TRIE_BRANCH_STORAGE, key)))
.isTrue();
assertThat(
isNearestKeyValueTheSame(
inMemoryDBKeyValueStorage.getNearestAfter(TRIE_BRANCH_STORAGE, key),
rockdDBKeyValueStorage.getNearestAfter(TRIE_BRANCH_STORAGE, key)))
.isTrue();
final Bytes biggerKey = Bytes.concatenate(key, Bytes.of(0x01));
assertThat(
isNearestKeyValueTheSame(
inMemoryDBKeyValueStorage.getNearestBefore(
TRIE_BRANCH_STORAGE, biggerKey),
rockdDBKeyValueStorage.getNearestBefore(TRIE_BRANCH_STORAGE, biggerKey)))
.isTrue();
assertThat(
isNearestKeyValueTheSame(
inMemoryDBKeyValueStorage.getNearestAfter(TRIE_BRANCH_STORAGE, biggerKey),
rockdDBKeyValueStorage.getNearestAfter(TRIE_BRANCH_STORAGE, biggerKey)))
.isTrue();
final Bytes smallerKey = key.slice(0, key.size() - 1);
assertThat(
isNearestKeyValueTheSame(
inMemoryDBKeyValueStorage.getNearestBefore(
TRIE_BRANCH_STORAGE, smallerKey),
rockdDBKeyValueStorage.getNearestBefore(TRIE_BRANCH_STORAGE, smallerKey)))
.isTrue();
assertThat(
isNearestKeyValueTheSame(
inMemoryDBKeyValueStorage.getNearestAfter(
TRIE_BRANCH_STORAGE, smallerKey),
rockdDBKeyValueStorage.getNearestAfter(TRIE_BRANCH_STORAGE, smallerKey)))
.isTrue();
final Bytes reversedKey = key.reverse();
assertThat(
isNearestKeyValueTheSame(
inMemoryDBKeyValueStorage.getNearestBefore(
TRIE_BRANCH_STORAGE, reversedKey),
rockdDBKeyValueStorage.getNearestBefore(
TRIE_BRANCH_STORAGE, reversedKey)))
.isTrue();
assertThat(
isNearestKeyValueTheSame(
inMemoryDBKeyValueStorage.getNearestAfter(
TRIE_BRANCH_STORAGE, reversedKey),
rockdDBKeyValueStorage.getNearestAfter(TRIE_BRANCH_STORAGE, reversedKey)))
.isTrue();
});
}
@Test
public void testNearestRocksdbWithLayeredKeyValueStorage() {
final SegmentedKeyValueStorage rockdDBKeyValueStorage =
getRocksDBKeyValueStorage(TRIE_BRANCH_STORAGE);
final SegmentedKeyValueStorageTransaction rocksDbTransaction =
rockdDBKeyValueStorage.startTransaction();
final SegmentedKeyValueStorage inMemoryDBKeyValueStorage = getInMemoryDBKeyValueStorage();
final SegmentedKeyValueStorageTransaction inMemoryDBTransaction =
inMemoryDBKeyValueStorage.startTransaction();
final LayeredKeyValueStorage layeredDBKeyValueStorage =
new LayeredKeyValueStorage(inMemoryDBKeyValueStorage);
final SegmentedKeyValueStorageTransaction layeredDBTransaction =
layeredDBKeyValueStorage.startTransaction();
IntStream.range(1, 10)
.forEach(
i -> {
final byte[] key = Bytes.fromHexString("0x000" + i).toArrayUnsafe();
final byte[] value = Bytes.fromHexString("0FFF").toArrayUnsafe();
rocksDbTransaction.put(TRIE_BRANCH_STORAGE, key, value);
// as we have several layers I store sometimes in the child layer and sometimes in the
// parent
if (i % 2 == 0) {
inMemoryDBTransaction.put(TRIE_BRANCH_STORAGE, key, value);
} else {
layeredDBTransaction.put(TRIE_BRANCH_STORAGE, key, value);
}
// different common prefix, and reversed order of bytes:
final byte[] key2 = Bytes.fromHexString("0x010" + (10 - i)).toArrayUnsafe();
final byte[] value2 = Bytes.fromHexString("0FFF").toArrayUnsafe();
rocksDbTransaction.put(TRIE_BRANCH_STORAGE, key2, value2);
// as we have several layers I store sometimes in the child layer and sometimes in the
// parent
if (i % 2 == 0) {
inMemoryDBTransaction.put(TRIE_BRANCH_STORAGE, key2, value2);
} else {
layeredDBTransaction.put(TRIE_BRANCH_STORAGE, key2, value2);
}
// different size:
final byte[] key3 = Bytes.fromHexString("0x01011" + (10 - i)).toArrayUnsafe();
final byte[] value3 = Bytes.fromHexString("0FFF").toArrayUnsafe();
rocksDbTransaction.put(TRIE_BRANCH_STORAGE, key3, value3);
// as we have several layers I store sometimes in the child layer and sometimes in the
// parent
if (i % 2 == 0) {
inMemoryDBTransaction.put(TRIE_BRANCH_STORAGE, key3, value3);
} else {
layeredDBTransaction.put(TRIE_BRANCH_STORAGE, key3, value3);
}
final byte[] key4 = Bytes.fromHexString("0x0" + (10 - i)).toArrayUnsafe();
final byte[] value4 = Bytes.fromHexString("0FFF").toArrayUnsafe();
rocksDbTransaction.put(TRIE_BRANCH_STORAGE, key4, value4);
// as we have several layers I store sometimes in the child layer and sometimes in the
// parent
if (i % 2 == 0) {
inMemoryDBTransaction.put(TRIE_BRANCH_STORAGE, key4, value4);
} else {
layeredDBTransaction.put(TRIE_BRANCH_STORAGE, key4, value4);
}
});
rocksDbTransaction.commit();
inMemoryDBTransaction.commit();
layeredDBTransaction.commit();
// compare rocksdb implementation with inmemory implementation
rockdDBKeyValueStorage.stream(TRIE_BRANCH_STORAGE)
.forEach(
pair -> {
final Bytes key = Bytes.of(pair.getKey());
assertThat(
isNearestKeyValueTheSame(
layeredDBKeyValueStorage.getNearestBefore(TRIE_BRANCH_STORAGE, key),
rockdDBKeyValueStorage.getNearestBefore(TRIE_BRANCH_STORAGE, key)))
.isTrue();
assertThat(
isNearestKeyValueTheSame(
layeredDBKeyValueStorage.getNearestAfter(TRIE_BRANCH_STORAGE, key),
rockdDBKeyValueStorage.getNearestAfter(TRIE_BRANCH_STORAGE, key)))
.isTrue();
final Bytes biggerKey = Bytes.concatenate(key, Bytes.of(0x01));
assertThat(
isNearestKeyValueTheSame(
layeredDBKeyValueStorage.getNearestBefore(TRIE_BRANCH_STORAGE, biggerKey),
rockdDBKeyValueStorage.getNearestBefore(TRIE_BRANCH_STORAGE, biggerKey)))
.isTrue();
assertThat(
isNearestKeyValueTheSame(
layeredDBKeyValueStorage.getNearestAfter(TRIE_BRANCH_STORAGE, biggerKey),
rockdDBKeyValueStorage.getNearestAfter(TRIE_BRANCH_STORAGE, biggerKey)))
.isTrue();
final Bytes smallerKey = key.slice(0, key.size() - 1);
assertThat(
isNearestKeyValueTheSame(
layeredDBKeyValueStorage.getNearestBefore(
TRIE_BRANCH_STORAGE, smallerKey),
rockdDBKeyValueStorage.getNearestBefore(TRIE_BRANCH_STORAGE, smallerKey)))
.isTrue();
assertThat(
isNearestKeyValueTheSame(
layeredDBKeyValueStorage.getNearestAfter(TRIE_BRANCH_STORAGE, smallerKey),
rockdDBKeyValueStorage.getNearestAfter(TRIE_BRANCH_STORAGE, smallerKey)))
.isTrue();
final Bytes reversedKey = key.reverse();
assertThat(
isNearestKeyValueTheSame(
layeredDBKeyValueStorage.getNearestBefore(
TRIE_BRANCH_STORAGE, reversedKey),
rockdDBKeyValueStorage.getNearestBefore(
TRIE_BRANCH_STORAGE, reversedKey)))
.isTrue();
assertThat(
isNearestKeyValueTheSame(
layeredDBKeyValueStorage.getNearestAfter(
TRIE_BRANCH_STORAGE, reversedKey),
rockdDBKeyValueStorage.getNearestAfter(TRIE_BRANCH_STORAGE, reversedKey)))
.isTrue();
});
}
private SegmentedKeyValueStorage getRocksDBKeyValueStorage(final SegmentIdentifier segment) {
return rocksdbStorageFactory.create(
List.of(segment), commonConfiguration, new NoOpMetricsSystem());
}
private SegmentedKeyValueStorage getInMemoryDBKeyValueStorage() {
return new SegmentedInMemoryKeyValueStorage();
}
private static void mockCommonConfiguration(final Path tempDataDir) {
commonConfiguration = mock(BesuConfiguration.class);
when(commonConfiguration.getStoragePath()).thenReturn(tempDataDir);
when(commonConfiguration.getDataPath()).thenReturn(tempDataDir);
DataStorageConfiguration dataStorageConfiguration = mock(DataStorageConfiguration.class);
when(dataStorageConfiguration.getDatabaseFormat()).thenReturn(DataStorageFormat.BONSAI);
lenient()
.when(commonConfiguration.getDataStorageConfiguration())
.thenReturn(dataStorageConfiguration);
}
private boolean isNearestKeyValueTheSame(
final Optional<NearestKeyValue> v1, final Optional<NearestKeyValue> v2) {
if (v1.isPresent() && v2.isPresent()) {
final NearestKeyValue nearestKeyValue1 = v1.get();
final NearestKeyValue nearestKeyValue2 = v2.get();
if (nearestKeyValue1.key().equals(nearestKeyValue2.key())) {
if (nearestKeyValue1.value().isPresent() && nearestKeyValue2.value().isPresent()) {
return Arrays.equals(nearestKeyValue1.value().get(), nearestKeyValue2.value().get());
}
} else if (nearestKeyValue1.value().isEmpty() && nearestKeyValue2.value().isEmpty()) {
return true;
}
} else if (v1.isEmpty() && v2.isEmpty()) {
return true;
}
return false;
}
}

61
services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/KeyComparator.java Normal file
View File

@@ -0,0 +1,61 @@
/*
* Copyright contributors to Hyperledger Besu.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.services.kvstore;
import org.apache.tuweni.bytes.Bytes;
/**
* This class is a comparator that allows comparing two byte arrays from left to right.
*
* <p>For example:
*
* <p>>0x01 is smaller than 0x0101 or 0x01 is smaller than 0x02.
*/
public class KeyComparator {
/** Instantiates a new KeyComparator */
public KeyComparator() {}
/**
* Compares two keys from left to right.
*
* <p>This method performs a byte-by-byte comparison between two keys, starting from the left
* (most significant byte). It is designed to compare keys in a way that reflects their
* hierarchical or sequential order.
*
* <p>The method returns: - A negative integer if {@code key1} is lexicographically less than
* key2. - Zero if key1 and key2 are equal. - A positive integer if key1 is lexicographically
* greater than key2.
*
* <p>If the keys are of unequal length but identical for the length of the shorter key (prefix),
* the shorter key is considered to be lexicographically less than the longer key. This is
* consistent with the lexicographic ordering used by rocksdb.
*
* @param key1 the first key compare.
* @param key2 the second key to compare with.
* @return the value 0 if key1 is equal to key2; a value less than 0 if key1 is lexicographically
* less than key2; and a value greater than 0 if key1 is lexicographically greater than key2.
*/
public static int compareKeyLeftToRight(final Bytes key1, final Bytes key2) {
int minLength = Math.min(key1.size(), key2.size());
for (int i = 0; i < minLength; i++) {
int compare = Byte.compareUnsigned(key1.get(i), key2.get(i));
if (compare != 0) {
return compare;
}
}
return Integer.compare(key1.size(), key2.size());
}
}

71
services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java
View File

@@ -33,6 +33,7 @@ import java.util.Spliterators;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.Lock;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
@@ -101,25 +102,77 @@ public class LayeredKeyValueStorage extends SegmentedInMemoryKeyValueStorage
}
@Override
public Optional<NearestKeyValue> getNearestTo(
public Optional<NearestKeyValue> getNearestBefore(
final SegmentIdentifier segmentIdentifier, final Bytes key) throws StorageException {
Optional<NearestKeyValue> ourNearest = super.getNearestTo(segmentIdentifier, key);
Optional<NearestKeyValue> parentNearest = parent.getNearestTo(segmentIdentifier, key);
return getNearest(
key,
k -> super.getNearestBefore(segmentIdentifier, k),
k -> parent.getNearestBefore(segmentIdentifier, k),
false);
}
@Override
public Optional<NearestKeyValue> getNearestAfter(
final SegmentIdentifier segmentIdentifier, final Bytes key) throws StorageException {
return getNearest(
key,
k -> super.getNearestAfter(segmentIdentifier, k),
k -> parent.getNearestAfter(segmentIdentifier, k),
true);
}
private Optional<NearestKeyValue> getNearest(
final Bytes key,
final Function<Bytes, Optional<NearestKeyValue>> ourNearestFunction,
final Function<Bytes, Optional<NearestKeyValue>> parentNearestFunction,
final boolean isAfter)
throws StorageException {
final Optional<NearestKeyValue> ourNearest = ourNearestFunction.apply(key);
final Optional<NearestKeyValue> parentNearest = parentNearestFunction.apply(key);
if (ourNearest.isPresent() && parentNearest.isPresent()) {
// Both are present, return the one closer to the key
int ourDistance = ourNearest.get().key().commonPrefixLength(key);
int parentDistance = parentNearest.get().key().commonPrefixLength(key);
return (ourDistance <= parentDistance) ? ourNearest : parentNearest;
return compareNearest(ourNearest, parentNearest, key, isAfter);
} else if (ourNearest.isPresent()) {
// Only ourNearest is present
return ourNearest;
} else {
// return parentNearest, which may be an empty Optional
return parentNearest;
}
}
private Optional<NearestKeyValue> compareNearest(
final Optional<NearestKeyValue> ourNearest,
final Optional<NearestKeyValue> parentNearest,
final Bytes key,
final boolean isAfter) {
final int ourDistance = ourNearest.get().key().compareTo(key);
final int parentDistance = parentNearest.get().key().compareTo(key);
if (ourDistance == 0) {
return ourNearest;
} else if (parentDistance == 0) {
return parentNearest;
} else {
final int ourCommonPrefixLength = ourNearest.get().key().commonPrefixLength(key);
final int parentCommonPrefixLength = parentNearest.get().key().commonPrefixLength(key);
if (ourCommonPrefixLength != parentCommonPrefixLength) {
return ourCommonPrefixLength > parentCommonPrefixLength ? ourNearest : parentNearest;
} else {
// When searching for a key, if isAfter is true, we choose the next smallest key after our
// target because both found keys are after it.
// If isAfter is false, meaning we're doing a seekForPrev, we select the largest key that
// comes before our target, as it's the nearest one.
// For example : if the searched key is 0x0101 and we found 0x0001 and 0x0100 when isAfter
// == false we will take 0x0100
if (ourNearest.get().key().compareTo(parentNearest.get().key()) > 0) {
return isAfter ? parentNearest : ourNearest;
} else {
return isAfter ? ourNearest : parentNearest;
}
}
}
}
@Override
public Stream<Pair<byte[], byte[]>> stream(final SegmentIdentifier segmentId) {
throwIfClosed();

73
services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedInMemoryKeyValueStorage.java
View File

@@ -15,6 +15,7 @@
package org.hyperledger.besu.services.kvstore;
import static java.util.stream.Collectors.toUnmodifiableSet;
import static org.hyperledger.besu.services.kvstore.KeyComparator.compareKeyLeftToRight;
import org.hyperledger.besu.plugin.services.exception.StorageException;
import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier;
@@ -39,6 +40,7 @@ import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -143,26 +145,67 @@ public class SegmentedInMemoryKeyValueStorage
}
@Override
public Optional<NearestKeyValue> getNearestTo(
public Optional<NearestKeyValue> getNearestBefore(
final SegmentIdentifier segmentIdentifier, final Bytes key) throws StorageException {
return getNearest(
segmentIdentifier,
e ->
compareKeyLeftToRight(e.getKey(), key) <= 0
&& e.getKey().commonPrefixLength(key) >= e.getKey().size(),
e -> compareKeyLeftToRight(e.getKey(), key) < 0,
false);
}
@Override
public Optional<NearestKeyValue> getNearestAfter(
final SegmentIdentifier segmentIdentifier, final Bytes key) throws StorageException {
return getNearest(
segmentIdentifier,
e ->
compareKeyLeftToRight(e.getKey(), key) >= 0
&& e.getKey().commonPrefixLength(key) >= e.getKey().size(),
e -> compareKeyLeftToRight(e.getKey(), key) >= 0,
true);
}
private Optional<NearestKeyValue> getNearest(
final SegmentIdentifier segmentIdentifier,
final Predicate<Map.Entry<Bytes, Optional<byte[]>>> samePrefixPredicate,
final Predicate<Map.Entry<Bytes, Optional<byte[]>>> fallbackPredicate,
final boolean useMin)
throws StorageException {
final Lock lock = rwLock.readLock();
lock.lock();
try {
// TODO: revisit this for sort performance
Comparator<Map.Entry<Bytes, Optional<byte[]>>> comparing =
Comparator.comparing(
(Map.Entry<Bytes, Optional<byte[]>> a) -> a.getKey().commonPrefixLength(key))
.thenComparing(Map.Entry.comparingByKey());
return this.hashValueStore
.computeIfAbsent(segmentIdentifier, s -> newSegmentMap())
.entrySet()
.stream()
// only return keys equal to or less than
.filter(e -> e.getKey().compareTo(key) <= 0)
.sorted(comparing.reversed())
.findFirst()
.map(z -> new NearestKeyValue(z.getKey(), z.getValue()));
final Map<Bytes, Optional<byte[]>> segmentMap =
this.hashValueStore.computeIfAbsent(segmentIdentifier, s -> newSegmentMap());
final Function<Predicate<Map.Entry<Bytes, Optional<byte[]>>>, Optional<NearestKeyValue>>
findNearest =
(predicate) -> {
final Stream<Map.Entry<Bytes, Optional<byte[]>>> filteredStream =
segmentMap.entrySet().stream().filter(predicate);
// Depending on the useMin flag, find either the minimum or maximum entry according
// to key order
final Optional<Map.Entry<Bytes, Optional<byte[]>>> sortedStream =
useMin
? filteredStream.min(
(t1, t2) -> compareKeyLeftToRight(t1.getKey(), t2.getKey()))
: filteredStream.max(
(t1, t2) -> compareKeyLeftToRight(t1.getKey(), t2.getKey()));
return sortedStream.map(
entry -> new NearestKeyValue(entry.getKey(), entry.getValue()));
};
// First, attempt to find a key-value pair that matches the same prefix
final Optional<NearestKeyValue> withSamePrefix = findNearest.apply(samePrefixPredicate);
if (withSamePrefix.isPresent()) {
return withSamePrefix;
}
// If a matching entry with a common prefix is not found, the next step is to search for the
// nearest key that comes after or before the requested one.
return findNearest.apply(fallbackPredicate);
} finally {
lock.unlock();
}

81
services/kvstore/src/test/java/org/hyperledger/besu/services/kvstore/AbstractSegmentedKeyValueStorageTest.java
View File

@@ -1,81 +0,0 @@
/*
* Copyright contributors to Hyperledger Besu.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.services.kvstore;
import static org.assertj.core.api.Assertions.assertThat;
import static org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage.SEGMENT_IDENTIFIER;
import org.hyperledger.besu.kvstore.AbstractKeyValueStorageTest;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction;
import java.util.stream.IntStream;
import org.apache.tuweni.bytes.Bytes;
import org.junit.jupiter.api.Test;
public abstract class AbstractSegmentedKeyValueStorageTest extends AbstractKeyValueStorageTest {
public abstract SegmentedKeyValueStorage createSegmentedStore();
@Test
public void assertSegmentedIsNearestTo() throws Exception {
try (final var store = this.createSegmentedStore()) {
// create 10 entries
final SegmentedKeyValueStorageTransaction tx = store.startTransaction();
IntStream.range(1, 10)
.forEach(
i -> {
final byte[] key = bytesFromHexString("000" + i);
final byte[] value = bytesFromHexString("0FFF");
tx.put(SEGMENT_IDENTIFIER, key, value);
// different common prefix, and reversed order of bytes:
final byte[] key2 = bytesFromHexString("010" + (10 - i));
final byte[] value2 = bytesFromHexString("0FFF");
tx.put(SEGMENT_IDENTIFIER, key2, value2);
});
tx.commit();
// assert 0009 is closest to 000F
var val = store.getNearestTo(SEGMENT_IDENTIFIER, Bytes.fromHexString("000F"));
assertThat(val).isPresent();
assertThat(val.get().key()).isEqualTo(Bytes.fromHexString("0009"));
// assert 0109 is closest to 010D
var val2 = store.getNearestTo(SEGMENT_IDENTIFIER, Bytes.fromHexString("010D"));
assertThat(val2).isPresent();
assertThat(val2.get().key()).isEqualTo(Bytes.fromHexString("0109"));
// assert 0103 is closest to 0103
var val3 = store.getNearestTo(SEGMENT_IDENTIFIER, Bytes.fromHexString("0103"));
assertThat(val3).isPresent();
assertThat(val3.get().key()).isEqualTo(Bytes.fromHexString("0103"));
// assert 0003 is closest to 0003
var val4 = store.getNearestTo(SEGMENT_IDENTIFIER, Bytes.fromHexString("0003"));
assertThat(val4).isPresent();
assertThat(val4.get().key()).isEqualTo(Bytes.fromHexString("0003"));
// assert 0001 is closest to 0001
var val5 = store.getNearestTo(SEGMENT_IDENTIFIER, Bytes.fromHexString("0001"));
assertThat(val5).isPresent();
assertThat(val5.get().key()).isEqualTo(Bytes.fromHexString("0001"));
// assert 0000 is not present
var val6 = store.getNearestTo(SEGMENT_IDENTIFIER, Bytes.fromHexString("0000"));
assertThat(val6).isNotPresent();
}
}
}

31
services/kvstore/src/test/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorageTest.java
View File

@@ -1,31 +0,0 @@
/*
* Copyright ConsenSys AG.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.services.kvstore;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage;
public class InMemoryKeyValueStorageTest extends AbstractSegmentedKeyValueStorageTest {
@Override
protected KeyValueStorage createStore() {
return new InMemoryKeyValueStorage();
}
@Override
public SegmentedKeyValueStorage createSegmentedStore() {
return new SegmentedInMemoryKeyValueStorage();
}
}

63
services/kvstore/src/test/java/org/hyperledger/besu/services/kvstore/KeyComparatorTest.java Normal file
View File

@@ -0,0 +1,63 @@
/*
* Copyright contributors to Hyperledger Besu.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.services.kvstore;
import static org.junit.jupiter.api.Assertions.assertEquals;
import org.apache.tuweni.bytes.Bytes;
import org.junit.jupiter.api.Test;
public class KeyComparatorTest {
@Test
public void testEmptyVs01() {
Bytes key1 = Bytes.EMPTY;
Bytes key2 = Bytes.fromHexString("0x01");
int result = KeyComparator.compareKeyLeftToRight(key1, key2);
assertEquals(-1, result, "Empty key should be considered smaller than 0x01");
}
@Test
public void test01Vs02() {
Bytes key1 = Bytes.fromHexString("0x01");
Bytes key2 = Bytes.fromHexString("0x02");
int result = KeyComparator.compareKeyLeftToRight(key1, key2);
assertEquals(-1, result, "0x01 should be considered smaller than 0x02");
}
@Test
public void test01Vs0100() {
Bytes key1 = Bytes.fromHexString("0x01");
Bytes key2 = Bytes.fromHexString("0x0100");
int result = KeyComparator.compareKeyLeftToRight(key1, key2);
assertEquals(-1, result, "0x01 should be considered smaller than 0x0100");
}
@Test
public void test01Vs0201() {
Bytes key1 = Bytes.fromHexString("0x01");
Bytes key2 = Bytes.fromHexString("0x0201");
int result = KeyComparator.compareKeyLeftToRight(key1, key2);
assertEquals(-1, result, "0x01 should be considered smaller than 0x0201");
}
@Test
public void test0101Vs02() {
Bytes key1 = Bytes.fromHexString("0x0101");
Bytes key2 = Bytes.fromHexString("0x02");
int result = KeyComparator.compareKeyLeftToRight(key1, key2);
assertEquals(-1, result, "0x0101 should be considered smaller than 0x02");
}
}

32
services/kvstore/src/test/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorageTest.java
View File

@@ -1,32 +0,0 @@
/*
* Copyright contributors to Hyperledger Besu.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.services.kvstore;
import static org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage.SEGMENT_IDENTIFIER;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage;
public class LayeredKeyValueStorageTest extends AbstractSegmentedKeyValueStorageTest {
@Override
protected KeyValueStorage createStore() {
return new SegmentedKeyValueStorageAdapter(SEGMENT_IDENTIFIER, createSegmentedStore());
}
@Override
public SegmentedKeyValueStorage createSegmentedStore() {
return new LayeredKeyValueStorage(new SegmentedInMemoryKeyValueStorage());
}
}