vacp2p/linea-besu
1
0
Fork 0
mirror of https://github.com/vacp2p/linea-besu.git synced 2026-01-08 20:47:59 -05:00
Code Issues Packages Projects Releases Wiki Activity

Reduce receipt size (#6602)

Browse Source 
Signed-off-by: Jason Frame <jason.frame@consensys.net>
This commit is contained in:
Jason Frame
2024-03-26 11:00:05 +10:00
committed by GitHub
parent bb9ba13cf8
commit 15d54afdf9
48 changed files with 602 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactory.java
View File

@@ -46,7 +46,9 @@ public class RocksDBKeyValuePrivacyStorageFactory implements PrivacyKeyValueStor
private static final Set<PrivacyVersionedStorageFormat> SUPPORTED_VERSIONS =
EnumSet.of(
PrivacyVersionedStorageFormat.FOREST_WITH_VARIABLES,
PrivacyVersionedStorageFormat.BONSAI_WITH_VARIABLES);
PrivacyVersionedStorageFormat.FOREST_WITH_RECEIPT_COMPACTION,
PrivacyVersionedStorageFormat.BONSAI_WITH_VARIABLES,
PrivacyVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION);
private static final String PRIVATE_DATABASE_PATH = "private";
private final RocksDBKeyValueStorageFactory publicFactory;
private DatabaseMetadata databaseMetadata;
@@ -145,7 +147,8 @@ public class RocksDBKeyValuePrivacyStorageFactory implements PrivacyKeyValueStor
privacyMetadata = existingPrivacyMetadata;
final int existingPrivacyVersion = maybeExistingPrivacyVersion.getAsInt();
final var runtimeVersion =
PrivacyVersionedStorageFormat.defaultForNewDB(commonConfiguration.getDatabaseFormat());
PrivacyVersionedStorageFormat.defaultForNewDB(
commonConfiguration.getDataStorageConfiguration());
if (existingPrivacyVersion > runtimeVersion.getPrivacyVersion().getAsInt()) {
final var maybeDowngradedMetadata =
@@ -194,6 +197,16 @@ public class RocksDBKeyValuePrivacyStorageFactory implements PrivacyKeyValueStor
// In case we do an automated downgrade, then we also need to update the metadata on disk to
// reflect the change to the runtime version, and return it.
// Besu supports both formats of receipts so no downgrade is needed
if (runtimeVersion == PrivacyVersionedStorageFormat.BONSAI_WITH_VARIABLES
|| runtimeVersion == PrivacyVersionedStorageFormat.FOREST_WITH_VARIABLES) {
LOG.warn(
"Database contains compacted receipts but receipt compaction is not enabled, new receipts will "
+ "be not stored in the compacted format. If you want to remove compacted receipts from the "
+ "database it is necessary to resync Besu. Besu can support both compacted and non-compacted receipts.");
return Optional.empty();
}
// for the moment there are supported automated downgrades, so we just fail.
String error =
String.format(
@@ -216,6 +229,18 @@ public class RocksDBKeyValuePrivacyStorageFactory implements PrivacyKeyValueStor
// In case we do an automated upgrade, then we also need to update the metadata on disk to
// reflect the change to the runtime version, and return it.
// Besu supports both formats of receipts so no upgrade is needed other than updating metadata
if (runtimeVersion == PrivacyVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION
|| runtimeVersion == PrivacyVersionedStorageFormat.FOREST_WITH_RECEIPT_COMPACTION) {
final DatabaseMetadata metadata = new DatabaseMetadata(runtimeVersion);
try {
metadata.writeToDirectory(dataDir);
return Optional.of(metadata);
} catch (IOException e) {
throw new StorageException("Database upgrade to use receipt compaction failed", e);
}
}
// for the moment there are no planned automated upgrades, so we just fail.
String error =
String.format(

38
plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java
View File

@@ -14,7 +14,9 @@
*/
package org.hyperledger.besu.plugin.services.storage.rocksdb;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat.BONSAI_WITH_VARIABLES;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat.FOREST_WITH_RECEIPT_COMPACTION;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat.FOREST_WITH_VARIABLES;
import org.hyperledger.besu.plugin.services.BesuConfiguration;
@@ -55,7 +57,11 @@ public class RocksDBKeyValueStorageFactory implements KeyValueStorageFactory {
private static final Logger LOG = LoggerFactory.getLogger(RocksDBKeyValueStorageFactory.class);
private static final EnumSet<BaseVersionedStorageFormat> SUPPORTED_VERSIONED_FORMATS =
EnumSet.of(FOREST_WITH_VARIABLES, BONSAI_WITH_VARIABLES);
EnumSet.of(
FOREST_WITH_VARIABLES,
FOREST_WITH_RECEIPT_COMPACTION,
BONSAI_WITH_VARIABLES,
BONSAI_WITH_RECEIPT_COMPACTION);
private static final String NAME = "rocksdb";
private final RocksDBMetricsFactory rocksDBMetricsFactory;
private DatabaseMetadata databaseMetadata;
@@ -220,12 +226,13 @@ public class RocksDBKeyValueStorageFactory implements KeyValueStorageFactory {
if (!metadata
.getVersionedStorageFormat()
.getFormat()
.equals(commonConfiguration.getDatabaseFormat())) {
.equals(commonConfiguration.getDataStorageConfiguration().getDatabaseFormat())) {
handleFormatMismatch(commonConfiguration, dataDir, metadata);
}
final var runtimeVersion =
BaseVersionedStorageFormat.defaultForNewDB(commonConfiguration.getDatabaseFormat());
BaseVersionedStorageFormat.defaultForNewDB(
commonConfiguration.getDataStorageConfiguration());
if (metadata.getVersionedStorageFormat().getVersion() > runtimeVersion.getVersion()) {
final var maybeDowngradedMetadata =
@@ -247,7 +254,7 @@ public class RocksDBKeyValueStorageFactory implements KeyValueStorageFactory {
LOG.info("Existing database at {}. Metadata {}. Processing WAL...", dataDir, metadata);
} else {
metadata = DatabaseMetadata.defaultForNewDb(commonConfiguration.getDatabaseFormat());
metadata = DatabaseMetadata.defaultForNewDb(commonConfiguration);
LOG.info(
"No existing database at {}. Using default metadata for new db {}", dataDir, metadata);
if (!dataDirExists) {
@@ -275,7 +282,7 @@ public class RocksDBKeyValueStorageFactory implements KeyValueStorageFactory {
+ "Please check your config.",
dataDir,
existingMetadata.getVersionedStorageFormat().getFormat().name(),
commonConfiguration.getDatabaseFormat());
commonConfiguration.getDataStorageConfiguration().getDatabaseFormat());
throw new StorageException(error);
}
@@ -290,6 +297,15 @@ public class RocksDBKeyValueStorageFactory implements KeyValueStorageFactory {
// In case we do an automated downgrade, then we also need to update the metadata on disk to
// reflect the change to the runtime version, and return it.
// Besu supports both formats of receipts so no downgrade is needed
if (runtimeVersion == BONSAI_WITH_VARIABLES || runtimeVersion == FOREST_WITH_VARIABLES) {
LOG.warn(
"Database contains compacted receipts but receipt compaction is not enabled, new receipts will "
+ "be not stored in the compacted format. If you want to remove compacted receipts from the "
+ "database it is necessary to resync Besu. Besu can support both compacted and non-compacted receipts.");
return Optional.empty();
}
// for the moment there are supported automated downgrades, so we just fail.
String error =
String.format(
@@ -312,6 +328,18 @@ public class RocksDBKeyValueStorageFactory implements KeyValueStorageFactory {
// In case we do an automated upgrade, then we also need to update the metadata on disk to
// reflect the change to the runtime version, and return it.
// Besu supports both formats of receipts so no upgrade is needed other than updating metadata
if (runtimeVersion == BONSAI_WITH_RECEIPT_COMPACTION
|| runtimeVersion == FOREST_WITH_RECEIPT_COMPACTION) {
final DatabaseMetadata metadata = new DatabaseMetadata(runtimeVersion);
try {
metadata.writeToDirectory(dataDir);
return Optional.of(metadata);
} catch (IOException e) {
throw new StorageException("Database upgrade to use receipt compaction failed", e);
}
}
// for the moment there are no planned automated upgrades, so we just fail.
String error =
String.format(

28
plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/BaseVersionedStorageFormat.java
View File

@@ -14,6 +14,7 @@
*/
package org.hyperledger.besu.plugin.services.storage.rocksdb.configuration;
import org.hyperledger.besu.plugin.services.storage.DataStorageConfiguration;
import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
import java.util.OptionalInt;
@@ -27,13 +28,23 @@ public enum BaseVersionedStorageFormat implements VersionedStorageFormat {
* make BlobDB more effective
*/
FOREST_WITH_VARIABLES(DataStorageFormat.FOREST, 2),
/**
* Current Forest version, with receipts using compaction, in order to make Receipts use less disk
* space
*/
FOREST_WITH_RECEIPT_COMPACTION(DataStorageFormat.FOREST, 3),
/** Original Bonsai version, not used since replace by BONSAI_WITH_VARIABLES */
BONSAI_ORIGINAL(DataStorageFormat.BONSAI, 1),
/**
* Current Bonsai version, with blockchain variables in a dedicated column family, in order to
* make BlobDB more effective
*/
BONSAI_WITH_VARIABLES(DataStorageFormat.BONSAI, 2);
BONSAI_WITH_VARIABLES(DataStorageFormat.BONSAI, 2),
/**
* Current Bonsai version, with receipts using compaction, in order to make Receipts use less disk
* space
*/
BONSAI_WITH_RECEIPT_COMPACTION(DataStorageFormat.BONSAI, 3);
private final DataStorageFormat format;
private final int version;
@@ -46,13 +57,18 @@ public enum BaseVersionedStorageFormat implements VersionedStorageFormat {
/**
* Return the default version for new db for a specific format
*
* @param format data storage format
* @param configuration data storage configuration
* @return the version to use for new db
*/
public static BaseVersionedStorageFormat defaultForNewDB(final DataStorageFormat format) {
return switch (format) {
case FOREST -> FOREST_WITH_VARIABLES;
case BONSAI -> BONSAI_WITH_VARIABLES;
public static BaseVersionedStorageFormat defaultForNewDB(
final DataStorageConfiguration configuration) {
return switch (configuration.getDatabaseFormat()) {
case FOREST -> configuration.getReceiptCompactionEnabled()
? FOREST_WITH_RECEIPT_COMPACTION
: FOREST_WITH_VARIABLES;
case BONSAI -> configuration.getReceiptCompactionEnabled()
? BONSAI_WITH_RECEIPT_COMPACTION
: BONSAI_WITH_VARIABLES;
};
}

18
plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadata.java
View File

@@ -14,6 +14,7 @@
*/
package org.hyperledger.besu.plugin.services.storage.rocksdb.configuration;
import org.hyperledger.besu.plugin.services.BesuConfiguration;
import org.hyperledger.besu.plugin.services.exception.StorageException;
import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
@@ -46,18 +47,25 @@ public class DatabaseMetadata {
.enable(SerializationFeature.INDENT_OUTPUT);
private final VersionedStorageFormat versionedStorageFormat;
private DatabaseMetadata(final VersionedStorageFormat versionedStorageFormat) {
/**
* Instantiates a new Database metadata.
*
* @param versionedStorageFormat the version storage format
*/
public DatabaseMetadata(final VersionedStorageFormat versionedStorageFormat) {
this.versionedStorageFormat = versionedStorageFormat;
}
/**
* Return the default metadata for new db for a specific format
*
* @param dataStorageFormat data storage format
* @param besuConfiguration besu configuration
* @return the metadata to use for new db
*/
public static DatabaseMetadata defaultForNewDb(final DataStorageFormat dataStorageFormat) {
return new DatabaseMetadata(BaseVersionedStorageFormat.defaultForNewDB(dataStorageFormat));
public static DatabaseMetadata defaultForNewDb(final BesuConfiguration besuConfiguration) {
return new DatabaseMetadata(
BaseVersionedStorageFormat.defaultForNewDB(
besuConfiguration.getDataStorageConfiguration()));
}
/**
@@ -222,6 +230,7 @@ public class DatabaseMetadata {
case FOREST -> switch (versionedStorageFormat.getVersion()) {
case 1 -> PrivacyVersionedStorageFormat.FOREST_ORIGINAL;
case 2 -> PrivacyVersionedStorageFormat.FOREST_WITH_VARIABLES;
case 3 -> PrivacyVersionedStorageFormat.FOREST_WITH_RECEIPT_COMPACTION;
default -> throw new StorageException(
"Unsupported database with format FOREST and version "
+ versionedStorageFormat.getVersion());
@@ -229,6 +238,7 @@ public class DatabaseMetadata {
case BONSAI -> switch (versionedStorageFormat.getVersion()) {
case 1 -> PrivacyVersionedStorageFormat.BONSAI_ORIGINAL;
case 2 -> PrivacyVersionedStorageFormat.BONSAI_WITH_VARIABLES;
case 3 -> PrivacyVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION;
default -> throw new StorageException(
"Unsupported database with format BONSAI and version "
+ versionedStorageFormat.getVersion());

28
plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/PrivacyVersionedStorageFormat.java
View File

@@ -14,6 +14,7 @@
*/
package org.hyperledger.besu.plugin.services.storage.rocksdb.configuration;
import org.hyperledger.besu.plugin.services.storage.DataStorageConfiguration;
import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
import java.util.OptionalInt;
@@ -27,13 +28,23 @@ public enum PrivacyVersionedStorageFormat implements VersionedStorageFormat {
* make BlobDB more effective
*/
FOREST_WITH_VARIABLES(BaseVersionedStorageFormat.FOREST_WITH_VARIABLES, 1),
/**
* Current Forest version, with receipts using compaction, in order to make Receipts use less disk
* space
*/
FOREST_WITH_RECEIPT_COMPACTION(BaseVersionedStorageFormat.FOREST_WITH_VARIABLES, 2),
/** Original Bonsai version, not used since replace by BONSAI_WITH_VARIABLES */
BONSAI_ORIGINAL(BaseVersionedStorageFormat.BONSAI_ORIGINAL, 1),
/**
* Current Bonsai version, with blockchain variables in a dedicated column family, in order to
* make BlobDB more effective
*/
BONSAI_WITH_VARIABLES(BaseVersionedStorageFormat.BONSAI_WITH_VARIABLES, 1);
BONSAI_WITH_VARIABLES(BaseVersionedStorageFormat.BONSAI_WITH_VARIABLES, 1),
/**
* Current Bonsai version, with receipts using compaction, in order to make Receipts use less disk
* space
*/
BONSAI_WITH_RECEIPT_COMPACTION(BaseVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION, 2);
private final VersionedStorageFormat baseVersionedStorageFormat;
private final OptionalInt privacyVersion;
@@ -47,13 +58,18 @@ public enum PrivacyVersionedStorageFormat implements VersionedStorageFormat {
/**
* Return the default version for new db for a specific format
*
* @param format data storage format
* @param configuration data storage configuration
* @return the version to use for new db
*/
public static VersionedStorageFormat defaultForNewDB(final DataStorageFormat format) {
return switch (format) {
case FOREST -> FOREST_WITH_VARIABLES;
case BONSAI -> BONSAI_WITH_VARIABLES;
public static VersionedStorageFormat defaultForNewDB(
final DataStorageConfiguration configuration) {
return switch (configuration.getDatabaseFormat()) {
case FOREST -> configuration.getReceiptCompactionEnabled()
? FOREST_WITH_RECEIPT_COMPACTION
: FOREST_WITH_VARIABLES;
case BONSAI -> configuration.getReceiptCompactionEnabled()
? BONSAI_WITH_RECEIPT_COMPACTION
: BONSAI_WITH_VARIABLES;
};
}

79
plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java
View File

@@ -15,13 +15,17 @@
package org.hyperledger.besu.plugin.services.storage.rocksdb;
import static org.assertj.core.api.Assertions.assertThat;
import static org.hyperledger.besu.plugin.services.storage.DataStorageFormat.BONSAI;
import static org.hyperledger.besu.plugin.services.storage.DataStorageFormat.FOREST;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.segmented.RocksDBColumnarKeyValueStorageTest.TestSegment;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.when;
import org.hyperledger.besu.metrics.ObservableMetricsSystem;
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
import org.hyperledger.besu.plugin.services.BesuConfiguration;
import org.hyperledger.besu.plugin.services.storage.DataStorageConfiguration;
import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier;
import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat;
import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.DatabaseMetadata;
@@ -35,6 +39,8 @@ import java.util.List;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
@@ -42,6 +48,7 @@ import org.mockito.junit.jupiter.MockitoExtension;
public class RocksDBKeyValuePrivacyStorageFactoryTest {
@Mock private RocksDBFactoryConfiguration rocksDbConfiguration;
@Mock private BesuConfiguration commonConfiguration;
@Mock private DataStorageConfiguration dataStorageConfiguration;
@TempDir private Path temporaryFolder;
private final ObservableMetricsSystem metricsSystem = new NoOpMetricsSystem();
private final SegmentIdentifier segment = TestSegment.BAR;
@@ -94,20 +101,25 @@ public class RocksDBKeyValuePrivacyStorageFactoryTest {
}
}
@Test
public void shouldUpdateCorrectMetadataFileForLatestVersion() throws Exception {
@ParameterizedTest
@EnumSource(DataStorageFormat.class)
public void shouldUpdateCorrectMetadataFileForLatestVersion(
final DataStorageFormat dataStorageFormat) throws Exception {
final Path tempDataDir = temporaryFolder.resolve("data");
final Path tempDatabaseDir = temporaryFolder.resolve("db");
mockCommonConfiguration(tempDataDir, tempDatabaseDir);
mockCommonConfiguration(tempDataDir, tempDatabaseDir, dataStorageFormat);
final RocksDBKeyValueStorageFactory storageFactory =
new RocksDBKeyValueStorageFactory(
() -> rocksDbConfiguration, segments, RocksDBMetricsFactory.PRIVATE_ROCKS_DB_METRICS);
try (final var storage = storageFactory.create(segment, commonConfiguration, metricsSystem)) {
final BaseVersionedStorageFormat expectedBaseVersion =
dataStorageFormat == BONSAI
? BaseVersionedStorageFormat.BONSAI_WITH_VARIABLES
: BaseVersionedStorageFormat.FOREST_WITH_VARIABLES;
assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat())
.isEqualTo(BaseVersionedStorageFormat.FOREST_WITH_VARIABLES);
.isEqualTo(expectedBaseVersion);
}
storageFactory.close();
@@ -116,16 +128,67 @@ public class RocksDBKeyValuePrivacyStorageFactoryTest {
try (final var storage =
privacyStorageFactory.create(segment, commonConfiguration, metricsSystem)) {
final PrivacyVersionedStorageFormat expectedPrivacyVersion =
dataStorageFormat == BONSAI
? PrivacyVersionedStorageFormat.BONSAI_WITH_VARIABLES
: PrivacyVersionedStorageFormat.FOREST_WITH_VARIABLES;
assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat())
.isEqualTo(PrivacyVersionedStorageFormat.FOREST_WITH_VARIABLES);
.isEqualTo(expectedPrivacyVersion);
}
privacyStorageFactory.close();
}
@ParameterizedTest
@EnumSource(DataStorageFormat.class)
public void shouldUpdateCorrectMetadataFileForLatestVersionWithReceiptCompaction(
final DataStorageFormat dataStorageFormat) throws Exception {
final Path tempDataDir = temporaryFolder.resolve("data");
final Path tempDatabaseDir = temporaryFolder.resolve("db");
mockCommonConfiguration(tempDataDir, tempDatabaseDir, dataStorageFormat);
when(dataStorageConfiguration.getReceiptCompactionEnabled()).thenReturn(true);
final RocksDBKeyValueStorageFactory storageFactory =
new RocksDBKeyValueStorageFactory(
() -> rocksDbConfiguration, segments, RocksDBMetricsFactory.PRIVATE_ROCKS_DB_METRICS);
try (final var storage = storageFactory.create(segment, commonConfiguration, metricsSystem)) {
final BaseVersionedStorageFormat expectedBaseVersion =
dataStorageFormat == BONSAI
? BaseVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION
: BaseVersionedStorageFormat.FOREST_WITH_RECEIPT_COMPACTION;
assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat())
.isEqualTo(expectedBaseVersion);
}
storageFactory.close();
final RocksDBKeyValuePrivacyStorageFactory privacyStorageFactory =
new RocksDBKeyValuePrivacyStorageFactory(storageFactory);
try (final var storage =
privacyStorageFactory.create(segment, commonConfiguration, metricsSystem)) {
final PrivacyVersionedStorageFormat expectedPrivacyVersion =
dataStorageFormat == BONSAI
? PrivacyVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION
: PrivacyVersionedStorageFormat.FOREST_WITH_RECEIPT_COMPACTION;
assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat())
.isEqualTo(expectedPrivacyVersion);
}
privacyStorageFactory.close();
}
private void mockCommonConfiguration(final Path tempDataDir, final Path tempDatabaseDir) {
mockCommonConfiguration(tempDataDir, tempDatabaseDir, FOREST);
}
private void mockCommonConfiguration(
final Path tempDataDir,
final Path tempDatabaseDir,
final DataStorageFormat dataStorageFormat) {
when(commonConfiguration.getStoragePath()).thenReturn(tempDatabaseDir);
when(commonConfiguration.getDataPath()).thenReturn(tempDataDir);
when(commonConfiguration.getDatabaseFormat()).thenReturn(FOREST);
when(dataStorageConfiguration.getDatabaseFormat()).thenReturn(dataStorageFormat);
lenient()
.when(commonConfiguration.getDataStorageConfiguration())
.thenReturn(dataStorageConfiguration);
}
}

47
plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java
View File

@@ -27,6 +27,7 @@ import org.hyperledger.besu.metrics.ObservableMetricsSystem;
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
import org.hyperledger.besu.plugin.services.BesuConfiguration;
import org.hyperledger.besu.plugin.services.exception.StorageException;
import org.hyperledger.besu.plugin.services.storage.DataStorageConfiguration;
import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier;
import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat;
@@ -43,6 +44,8 @@ import org.junit.jupiter.api.condition.DisabledOnOs;
import org.junit.jupiter.api.condition.OS;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
@@ -51,16 +54,19 @@ public class RocksDBKeyValueStorageFactoryTest {
@Mock private RocksDBFactoryConfiguration rocksDbConfiguration;
@Mock private BesuConfiguration commonConfiguration;
@Mock private DataStorageConfiguration dataStorageConfiguration;
@TempDir public Path temporaryFolder;
private final ObservableMetricsSystem metricsSystem = new NoOpMetricsSystem();
private final SegmentIdentifier segment = TestSegment.FOO;
private final List<SegmentIdentifier> segments = List.of(TestSegment.DEFAULT, segment);
@Test
public void shouldCreateCorrectMetadataFileForLatestVersionForNewDb() throws Exception {
@ParameterizedTest
@EnumSource(DataStorageFormat.class)
public void shouldCreateCorrectMetadataFileForLatestVersionForNewDb(
final DataStorageFormat dataStorageFormat) throws Exception {
final Path tempDataDir = temporaryFolder.resolve("data");
final Path tempDatabaseDir = temporaryFolder.resolve("db");
mockCommonConfiguration(tempDataDir, tempDatabaseDir, FOREST);
mockCommonConfiguration(tempDataDir, tempDatabaseDir, dataStorageFormat);
final RocksDBKeyValueStorageFactory storageFactory =
new RocksDBKeyValueStorageFactory(
@@ -68,8 +74,36 @@ public class RocksDBKeyValueStorageFactoryTest {
try (final var storage = storageFactory.create(segment, commonConfiguration, metricsSystem)) {
// Side effect is creation of the Metadata version file
final BaseVersionedStorageFormat expectedVersion =
dataStorageFormat == BONSAI
? BaseVersionedStorageFormat.BONSAI_WITH_VARIABLES
: BaseVersionedStorageFormat.FOREST_WITH_VARIABLES;
assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat())
.isEqualTo(BaseVersionedStorageFormat.FOREST_WITH_VARIABLES);
.isEqualTo(expectedVersion);
}
}
@ParameterizedTest
@EnumSource(DataStorageFormat.class)
public void shouldCreateCorrectMetadataFileForLatestVersionForNewDbWithReceiptCompaction(
final DataStorageFormat dataStorageFormat) throws Exception {
final Path tempDataDir = temporaryFolder.resolve("data");
final Path tempDatabaseDir = temporaryFolder.resolve("db");
mockCommonConfiguration(tempDataDir, tempDatabaseDir, dataStorageFormat);
when(dataStorageConfiguration.getReceiptCompactionEnabled()).thenReturn(true);
final RocksDBKeyValueStorageFactory storageFactory =
new RocksDBKeyValueStorageFactory(
() -> rocksDbConfiguration, segments, RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS);
try (final var storage = storageFactory.create(segment, commonConfiguration, metricsSystem)) {
// Side effect is creation of the Metadata version file
final BaseVersionedStorageFormat expectedVersion =
dataStorageFormat == BONSAI
? BaseVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION
: BaseVersionedStorageFormat.FOREST_WITH_RECEIPT_COMPACTION;
assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat())
.isEqualTo(expectedVersion);
}
}
@@ -273,6 +307,9 @@ public class RocksDBKeyValueStorageFactoryTest {
final Path tempDataDir, final Path tempDatabaseDir, final DataStorageFormat format) {
when(commonConfiguration.getStoragePath()).thenReturn(tempDatabaseDir);
when(commonConfiguration.getDataPath()).thenReturn(tempDataDir);
lenient().when(commonConfiguration.getDatabaseFormat()).thenReturn(format);
lenient().when(dataStorageConfiguration.getDatabaseFormat()).thenReturn(format);
lenient()
.when(commonConfiguration.getDataStorageConfiguration())
.thenReturn(dataStorageConfiguration);
}
}