[Pruning Bugfix] Prevent race condition in key deletion. (#760)

* add doomed key check (busy-waiting for now) Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * optional and logging Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * remove logging Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * sleeping and hardening Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * rename segments Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * move away from atomic references to regular vars Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * remove hardened segment parameter Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * increase sleep Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * spotless Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * remove unnecessary interface Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * rename Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * move commit waiting outside of timer Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * set default lock timeout to 1ms Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * add default lock timeout to tests Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * Revert "rename segments" This reverts commit 184eefaaa0ccc857b0caff2b382f8338ff225d5d. Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * fix jmh compilation error Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * add documentation Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * bump up sleep to 1ms Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * (POC) Add lock to ensure that we don't prune while comitting Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * remove unnecessary persist (#569) Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * flesh out @mbaxter's idea and remove my code Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * iterator changes Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * hybridize with doomed key Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * comment Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * move doomed key unset to after node added listener Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * update instead of getting and setting doomedKeyRef in commit Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * comment Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * invert condition Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * remove locks Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * remove `removeAllKeysUnless` Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * more remove removeAllKeysUnless Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * reuse streamKeys Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * remove test Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * set default lock timeout to 1ms Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * add default lock timeout to tests Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * fix jmh compilation error Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * revert back to locks instead of doomedkey Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * change delete to not guarantee deletion Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * plugin hash Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * javadoc Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * Revert "change delete to not guarantee deletion" This reverts commit 2289bb34cfe73bb34990db3b5ef3d614222b8c5b. Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * skip key deletion on timeout Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * clear in rollback Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * Revert "fix jmh compilation error" This reverts commit b64ecf86568789583966e534685258d484987deb. Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * Revert "add default lock timeout to tests" This reverts commit aff6aa6065a02bbc7eaa313d58034de48dd1d9ce. Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * Revert "set default lock timeout to 1ms" This reverts commit 267fe0a642bf9cca89d53e09830454b7137324e6. Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * use noSlowDown write option instead of global timeout Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * add back tests Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * close tryDeleteOptions Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * remove unnecessary lock Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * move increment inside try Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * use StorageException subclass instead of field Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * revert accidental deletion in javadoc Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * tryDelete javadoc Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * add trace for skipping key deletion Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * merge catch and finally try blocks Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * switch from exception to boolean return value Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * tweak Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * changelog changes Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * add api back Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * add back throws javadoc Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> Co-authored-by: Meredith Baxter <meredith.baxter@consensys.net> Co-authored-by: MadelineMurray <43356962+MadelineMurray@users.noreply.github.com>
2026-01-09 15:37:54 -05:00 · 2020-05-06 18:14:20 -04:00
parent 69f6493f91
commit 43eccbbb67
14 changed files with 360 additions and 79 deletions
--- a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java
+++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java
@@ -29,7 +29,9 @@ import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.function.Predicate;
+import java.util.stream.Stream;

+import com.google.common.collect.ImmutableSet;
 import org.apache.tuweni.bytes.Bytes;

 public class InMemoryKeyValueStorage implements KeyValueStorage {
@@ -72,6 +74,22 @@ public class InMemoryKeyValueStorage implements KeyValueStorage {
    }
  }

+  @Override
+  public Set<byte[]> getAllKeysThat(final Predicate<byte[]> returnCondition) {
+    return streamKeys().filter(returnCondition).collect(toUnmodifiableSet());
+  }
+
+  @Override
+  public Stream<byte[]> streamKeys() {
+    final Lock lock = rwLock.readLock();
+    lock.lock();
+    try {
+      return ImmutableSet.copyOf(hashValueStore.keySet()).stream().map(Bytes::toArrayUnsafe);
+    } finally {
+      lock.unlock();
+    }
+  }
+
  @Override
  public long removeAllKeysUnless(final Predicate<byte[]> retainCondition) throws StorageException {
    final Lock lock = rwLock.writeLock();
@@ -86,17 +104,17 @@ public class InMemoryKeyValueStorage implements KeyValueStorage {
  }

  @Override
-  public Set<byte[]> getAllKeysThat(final Predicate<byte[]> returnCondition) {
-    final Lock lock = rwLock.readLock();
-    lock.lock();
-    try {
-      return hashValueStore.keySet().stream()
-          .map(Bytes::toArrayUnsafe)
-          .filter(returnCondition)
-          .collect(toUnmodifiableSet());
-    } finally {
-      lock.unlock();
+  public boolean tryDelete(final byte[] key) {
+    final Lock lock = rwLock.writeLock();
+    if (lock.tryLock()) {
+      try {
+        hashValueStore.remove(Bytes.wrap(key));
+      } finally {
+        lock.unlock();
+      }
+      return true;
    }
+    return false;
  }

  @Override
--- a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LimitedInMemoryKeyValueStorage.java
+++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LimitedInMemoryKeyValueStorage.java
@@ -14,6 +14,8 @@
 */
 package org.hyperledger.besu.services.kvstore;

+import static java.util.stream.Collectors.toUnmodifiableSet;
+
 import org.hyperledger.besu.plugin.services.exception.StorageException;
 import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
 import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction;
@@ -27,10 +29,11 @@ import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.function.Predicate;
-import java.util.stream.Collectors;
+import java.util.stream.Stream;

 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
+import com.google.common.collect.ImmutableSet;
 import org.apache.tuweni.bytes.Bytes;

 /**
@@ -77,18 +80,46 @@ public class LimitedInMemoryKeyValueStorage implements KeyValueStorage {
  }

  @Override
-  public long removeAllKeysUnless(final Predicate<byte[]> retainCondition) throws StorageException {
-    final long initialSize = storage.size();
-    storage.asMap().keySet().removeIf(key -> !retainCondition.test(key.toArrayUnsafe()));
-    return initialSize - storage.size();
+  public Set<byte[]> getAllKeysThat(final Predicate<byte[]> returnCondition) {
+    return streamKeys().filter(returnCondition).collect(toUnmodifiableSet());
  }

  @Override
-  public Set<byte[]> getAllKeysThat(final Predicate<byte[]> returnCondition) {
-    return storage.asMap().keySet().stream()
-        .map(Bytes::toArrayUnsafe)
-        .filter(returnCondition)
-        .collect(Collectors.toSet());
+  public Stream<byte[]> streamKeys() {
+    final Lock lock = rwLock.readLock();
+    lock.lock();
+    try {
+      return ImmutableSet.copyOf(storage.asMap().keySet()).stream().map(Bytes::toArrayUnsafe);
+    } finally {
+      lock.unlock();
+    }
+  }
+
+  @Override
+  public long removeAllKeysUnless(final Predicate<byte[]> retainCondition) throws StorageException {
+    final Lock lock = rwLock.writeLock();
+    lock.lock();
+    try {
+      final long initialSize = storage.size();
+      storage.asMap().keySet().removeIf(key -> !retainCondition.test(key.toArrayUnsafe()));
+      return initialSize - storage.size();
+    } finally {
+      lock.unlock();
+    }
+  }
+
+  @Override
+  public boolean tryDelete(final byte[] key) {
+    final Lock lock = rwLock.writeLock();
+    if (lock.tryLock()) {
+      try {
+        storage.invalidate(Bytes.wrap(key));
+      } finally {
+        lock.unlock();
+      }
+      return true;
+    }
+    return false;
  }

  @Override
--- a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedKeyValueStorage.java
+++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedKeyValueStorage.java
@@ -21,6 +21,7 @@ import java.io.Closeable;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Predicate;
+import java.util.stream.Stream;

 /**
 * Service provided by besu to facilitate persistent data storage.
@@ -49,7 +50,37 @@ public interface SegmentedKeyValueStorage<S> extends Closeable {
   */
  Transaction<S> startTransaction() throws StorageException;

-  long removeAllKeysUnless(S segmentHandle, Predicate<byte[]> inUseCheck);
+  /**
+   * Returns a stream of all keys for the segment.
+   *
+   * @param segmentHandle The segment handle whose keys we want to stream.
+   * @return A stream of all keys in the specified segment.
+   */
+  Stream<byte[]> streamKeys(final S segmentHandle);
+
+  /**
+   * Performs an evaluation against each key in the store, keeping the entries that pass, removing
+   * those that fail.
+   *
+   * @param segmentHandle The segment handle whose keys we want to stream.
+   * @param retainCondition predicate to evaluate each key against, unless the result is {@code
+   *     null}, both the key and associated value must be removed.
+   * @return the number of keys removed.
+   */
+  long removeAllKeysUnless(final S segmentHandle, Predicate<byte[]> retainCondition)
+      throws StorageException;
+
+  /**
+   * Delete the value corresponding to the given key in the given segment if a write lock can be
+   * instantly acquired on the underlying storage. Do nothing otherwise.
+   *
+   * @param segmentHandle The segment handle whose keys we want to stream.
+   * @param key The key to delete.
+   * @throws StorageException any problem encountered during the deletion attempt.
+   * @return false if the lock on the underlying storage could not be instantly acquired, true
+   *     otherwise
+   */
+  boolean tryDelete(S segmentHandle, byte[] key) throws StorageException;

  Set<byte[]> getAllKeysThat(S segmentHandle, Predicate<byte[]> returnCondition);

--- a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedKeyValueStorageAdapter.java
+++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedKeyValueStorageAdapter.java
@@ -23,9 +23,9 @@ import java.io.IOException;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Predicate;
+import java.util.stream.Stream;

 public class SegmentedKeyValueStorageAdapter<S> implements KeyValueStorage {
-
  private final S segmentHandle;
  private final SegmentedKeyValueStorage<S> storage;

@@ -50,14 +50,24 @@ public class SegmentedKeyValueStorageAdapter<S> implements KeyValueStorage {
    return storage.get(segmentHandle, key);
  }

+  @Override
+  public Set<byte[]> getAllKeysThat(final Predicate<byte[]> returnCondition) {
+    return storage.getAllKeysThat(segmentHandle, returnCondition);
+  }
+
+  @Override
+  public Stream<byte[]> streamKeys() {
+    return storage.streamKeys(segmentHandle);
+  }
+
  @Override
  public long removeAllKeysUnless(final Predicate<byte[]> retainCondition) throws StorageException {
    return storage.removeAllKeysUnless(segmentHandle, retainCondition);
  }

  @Override
-  public Set<byte[]> getAllKeysThat(final Predicate<byte[]> returnCondition) {
-    return storage.getAllKeysThat(segmentHandle, returnCondition);
+  public boolean tryDelete(final byte[] key) {
+    return storage.tryDelete(segmentHandle, key);
  }

  @Override