Start external services before we start plugins (#3118)

* Start external services before we start mining Reasons: - you may wish to query the blockchain using rpc before you start processing - you may wish to register a websocket e.g to listen for blocks currently you will probably miss one because we start processing before we start the ws service Signed-off-by: Antony Denyer <git@antonydenyer.co.uk> * Add beforeExternalServices hook for plugins Signed-off-by: Antony Denyer <git@antonydenyer.co.uk> * move the acceptance test plugins move the test plugins that are used in the acceptance tests into the same package namespace as the acceptance tests so that they are ignored by sonar for code coverage Signed-off-by: Antony Denyer <git@antonydenyer.co.uk> * Fix typo in log Signed-off-by: Antony Denyer <git@antonydenyer.co.uk> Co-authored-by: Sally MacFarlane <sally.macfarlane@consensys.net>
2026-01-09 15:37:54 -05:00 · 2021-12-20 05:59:33 +00:00
parent 06419d43c6
commit 424de5a55b
20 changed files with 142 additions and 68 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,6 +19,7 @@
 - The invalid value is now shown when `--bootnodes` cannot parse an item to make it easier to identify which option is invalid.
 - Adding two new options to be able to specify desired TLS protocol version and Java cipher suites [#3105](https://github.com/hyperledger/besu/pull/3105)
 - Implements [EIP-4399](https://eips.ethereum.org/EIPS/eip-4399) to repurpose DIFFICULTY opcode after the merge as a source of entropy from the Beacon chain. [#3081](https://github.com/hyperledger/besu/issues/3081)
+- Re-order external services (e.g JsonRpcHttpService) to start before blocks start processing [#3118](https://github.com/hyperledger/besu/pull/3118)

 ### Bug Fixes
 - Change the base docker image from Debian Buster to Ubuntu 20.04 [#3171](https://github.com/hyperledger/besu/issues/3171) fixes [#3045](https://github.com/hyperledger/besu/issues/3045)
--- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java
+++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java
@@ -176,15 +176,6 @@ public class ThreadBesuNodeRunner implements BesuNodeRunner {
    final RunnerBuilder runnerBuilder = new RunnerBuilder();
    runnerBuilder.permissioningConfiguration(node.getPermissioningConfiguration());

-    besuPluginContext.addService(
-        BesuEvents.class,
-        new BesuEventsImpl(
-            besuController.getProtocolContext().getBlockchain(),
-            besuController.getProtocolManager().getBlockBroadcaster(),
-            besuController.getTransactionPool(),
-            besuController.getSyncState()));
-    besuPluginContext.startPlugins();
-
    final Runner runner =
        runnerBuilder
            .vertx(Vertx.vertx())
@@ -215,7 +206,20 @@ public class ThreadBesuNodeRunner implements BesuNodeRunner {
            .rpcEndpointService(new RpcEndpointServiceImpl())
            .build();

-    runner.start();
+    besuPluginContext.beforeExternalServices();
+
+    runner.startExternalServices();
+
+    besuPluginContext.addService(
+        BesuEvents.class,
+        new BesuEventsImpl(
+            besuController.getProtocolContext().getBlockchain(),
+            besuController.getProtocolManager().getBlockBroadcaster(),
+            besuController.getTransactionPool(),
+            besuController.getSyncState()));
+    besuPluginContext.startPlugins();
+
+    runner.startEthereumMainLoop();

    besuRunners.put(node.getName(), runner);
    ThreadContext.remove("node");
--- a/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/BadCLIOptionsPlugin.java
+++ b/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/BadCLIOptionsPlugin.java
@@ -13,7 +13,7 @@
 * SPDX-License-Identifier: Apache-2.0
 *
 */
-package org.hyperledger.besu.plugins;
+package org.hyperledger.besu.tests.acceptance.plugins;

 import static java.nio.charset.StandardCharsets.UTF_8;

--- a/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/TestBesuEventsPlugin.java
+++ b/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/TestBesuEventsPlugin.java
@@ -12,7 +12,7 @@
 *
 * SPDX-License-Identifier: Apache-2.0
 */
-package org.hyperledger.besu.plugins;
+package org.hyperledger.besu.tests.acceptance.plugins;

 import org.hyperledger.besu.plugin.BesuContext;
 import org.hyperledger.besu.plugin.BesuPlugin;
--- a/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/TestPermissioningPlugin.java
+++ b/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/TestPermissioningPlugin.java
@@ -13,7 +13,7 @@
 * SPDX-License-Identifier: Apache-2.0
 */

-package org.hyperledger.besu.plugins;
+package org.hyperledger.besu.tests.acceptance.plugins;

 import org.hyperledger.besu.plugin.BesuContext;
 import org.hyperledger.besu.plugin.BesuPlugin;
@@ -29,25 +29,25 @@ import picocli.CommandLine.Option;
 public class TestPermissioningPlugin implements BesuPlugin {
  private static final Logger LOG = LogManager.getLogger();

-  private final String aliceNode =
+  private static final String aliceNode =
      "09b02f8a5fddd222ade4ea4528faefc399623af3f736be3c44f03e2df22fb792f3931a4d9573d333ca74343305762a753388c3422a86d98b713fc91c1ea04842";

-  private final String bobNode =
+  private static final String bobNode =
      "af80b90d25145da28c583359beb47b21796b2fe1a23c1511e443e7a64dfdb27d7434c380f0aa4c500e220aa1a9d068514b1ff4d5019e624e7ba1efe82b340a59";

-  private final String charlieNode =
+  private static final String charlieNode =
      "ce7edc292d7b747fab2f23584bbafaffde5c8ff17cf689969614441e0527b90015ea9fee96aed6d9c0fc2fbe0bd1883dee223b3200246ff1e21976bdbc9a0fc8";

  PermissioningService service;

  @Override
  public void register(final BesuContext context) {
-    context.getService(PicoCLIOptions.class).get().addPicoCLIOptions("permissioning", this);
-    service = context.getService(PermissioningService.class).get();
+    context.getService(PicoCLIOptions.class).orElseThrow().addPicoCLIOptions("permissioning", this);
+    service = context.getService(PermissioningService.class).orElseThrow();
  }

  @Override
-  public void start() {
+  public void beforeExternalServices() {
    if (enabled) {
      service.registerNodePermissioningProvider(
          (sourceEnode, destinationEnode) -> {
@@ -79,6 +79,9 @@ public class TestPermissioningPlugin implements BesuPlugin {
    }
  }

+  @Override
+  public void start() {}
+
  private boolean transactionMessage(final int code) {
    return code == 0x02 || code == 0x08 || code == 0x09 || code == 0x0a;
  }
--- a/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/TestPicoCLIPlugin.java
+++ b/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/TestPicoCLIPlugin.java
@@ -12,7 +12,7 @@
 *
 * SPDX-License-Identifier: Apache-2.0
 */
-package org.hyperledger.besu.plugins;
+package org.hyperledger.besu.tests.acceptance.plugins;

 import org.hyperledger.besu.plugin.BesuContext;
 import org.hyperledger.besu.plugin.BesuPlugin;
--- a/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/TestPrivacyServicePlugin.java
+++ b/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/TestPrivacyServicePlugin.java
@@ -12,15 +12,15 @@
 *
 * SPDX-License-Identifier: Apache-2.0
 */
-package org.hyperledger.besu.plugins;
+package org.hyperledger.besu.tests.acceptance.plugins;

 import org.hyperledger.besu.plugin.BesuContext;
 import org.hyperledger.besu.plugin.BesuPlugin;
 import org.hyperledger.besu.plugin.services.PicoCLIOptions;
 import org.hyperledger.besu.plugin.services.PrivacyPluginService;
-import org.hyperledger.besu.plugins.privacy.TestPrivacyGroupGenesisProvider;
-import org.hyperledger.besu.plugins.privacy.TestPrivacyPluginPayloadProvider;
-import org.hyperledger.besu.plugins.privacy.TestSigningPrivateMarkerTransactionFactory;
+import org.hyperledger.besu.tests.acceptance.plugins.privacy.TestPrivacyGroupGenesisProvider;
+import org.hyperledger.besu.tests.acceptance.plugins.privacy.TestPrivacyPluginPayloadProvider;
+import org.hyperledger.besu.tests.acceptance.plugins.privacy.TestSigningPrivateMarkerTransactionFactory;

 import com.google.auto.service.AutoService;
 import org.apache.logging.log4j.LogManager;
@@ -43,16 +43,19 @@ public class TestPrivacyServicePlugin implements BesuPlugin {
  public void register(final BesuContext context) {
    this.context = context;

-    context.getService(PicoCLIOptions.class).get().addPicoCLIOptions("privacy-service", this);
-    pluginService = context.getService(PrivacyPluginService.class).get();
+    context
+        .getService(PicoCLIOptions.class)
+        .orElseThrow()
+        .addPicoCLIOptions("privacy-service", this);
+    pluginService = context.getService(PrivacyPluginService.class).orElseThrow();
    pluginService.setPrivacyGroupGenesisProvider(privacyGroupGenesisProvider);

    LOG.info("Registering Plugins with options " + this);
  }

  @Override
-  public void start() {
-    LOG.info("Start Plugins with options " + this);
+  public void beforeExternalServices() {
+    LOG.info("Start Plugins with options {}", this);

    TestPrivacyPluginPayloadProvider payloadProvider = new TestPrivacyPluginPayloadProvider();

@@ -69,6 +72,9 @@ public class TestPrivacyServicePlugin implements BesuPlugin {
    }
  }

+  @Override
+  public void start() {}
+
  @Override
  public void stop() {}

--- a/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/TestRpcEndpointServicePlugin.java
+++ b/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/TestRpcEndpointServicePlugin.java
@@ -12,7 +12,7 @@
 *
 * SPDX-License-Identifier: Apache-2.0
 */
-package org.hyperledger.besu.plugins;
+package org.hyperledger.besu.tests.acceptance.plugins;

 import static com.google.common.base.Preconditions.checkArgument;

--- a/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/privacy/TestPrivacyGroupGenesisProvider.java
+++ b/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/privacy/TestPrivacyGroupGenesisProvider.java
@@ -13,7 +13,7 @@
 * SPDX-License-Identifier: Apache-2.0
 *
 */
-package org.hyperledger.besu.plugins.privacy;
+package org.hyperledger.besu.tests.acceptance.plugins.privacy;

 import org.hyperledger.besu.datatypes.Wei;
 import org.hyperledger.besu.plugin.data.Address;
--- a/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/privacy/TestPrivacyPluginPayloadProvider.java
+++ b/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/privacy/TestPrivacyPluginPayloadProvider.java
@@ -13,7 +13,7 @@
 * SPDX-License-Identifier: Apache-2.0
 *
 */
-package org.hyperledger.besu.plugins.privacy;
+package org.hyperledger.besu.tests.acceptance.plugins.privacy;

 import static org.hyperledger.besu.ethereum.privacy.PrivateTransaction.readFrom;
 import static org.hyperledger.besu.ethereum.privacy.PrivateTransaction.serialize;
--- a/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/privacy/TestSigningPrivateMarkerTransactionFactory.java
+++ b/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/privacy/TestSigningPrivateMarkerTransactionFactory.java
@@ -12,7 +12,7 @@
 *
 * SPDX-License-Identifier: Apache-2.0
 */
-package org.hyperledger.besu.plugins.privacy;
+package org.hyperledger.besu.tests.acceptance.plugins.privacy;

 import static org.apache.logging.log4j.LogManager.getLogger;
 import static org.hyperledger.besu.datatypes.Address.extract;
@@ -66,12 +66,12 @@ public class TestSigningPrivateMarkerTransactionFactory implements PrivateMarker
            .gasLimit(unsignedPrivateMarkerTransaction.getGasLimit())
            .to(
                org.hyperledger.besu.datatypes.Address.fromPlugin(
-                    unsignedPrivateMarkerTransaction.getTo().get()))
+                    unsignedPrivateMarkerTransaction.getTo().orElseThrow()))
            .value(Wei.fromQuantity(unsignedPrivateMarkerTransaction.getValue()))
            .payload(unsignedPrivateMarkerTransaction.getPayload())
            .signAndBuild(aliceFixedSigningKey);

-    LOG.info("Signing PMT from " + sender);
+    LOG.info("Signing PMT from {}", sender);

    final BytesValueRLPOutput out = new BytesValueRLPOutput();
    transaction.writeTo(out);
--- a/acceptance-tests/test-plugins/src/test/java/org/hyperledger/besu/services/BesuPluginContextImplTest.java
+++ b/acceptance-tests/test-plugins/src/test/java/org/hyperledger/besu/services/BesuPluginContextImplTest.java
@@ -18,7 +18,7 @@ import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatExceptionOfType;

 import org.hyperledger.besu.plugin.BesuPlugin;
-import org.hyperledger.besu.plugins.TestPicoCLIPlugin;
+import org.hyperledger.besu.tests.acceptance.plugins.TestPicoCLIPlugin;

 import java.io.File;
 import java.io.IOException;
@@ -27,6 +27,7 @@ import java.nio.file.Path;
 import java.util.List;
 import java.util.Optional;

+import org.assertj.core.api.Assertions;
 import org.assertj.core.api.ThrowableAssert;
 import org.junit.After;
 import org.junit.BeforeClass;
@@ -57,10 +58,11 @@ public class BesuPluginContextImplTest {
    assertThat(contextImpl.getPlugins()).isNotEmpty();

    final Optional<TestPicoCLIPlugin> testPluginOptional = findTestPlugin(contextImpl.getPlugins());
-    assertThat(testPluginOptional).isPresent();
+    Assertions.assertThat(testPluginOptional).isPresent();
    final TestPicoCLIPlugin testPicoCLIPlugin = testPluginOptional.get();
    assertThat(testPicoCLIPlugin.getState()).isEqualTo("registered");

+    contextImpl.beforeExternalServices();
    contextImpl.startPlugins();
    assertThat(testPicoCLIPlugin.getState()).isEqualTo("started");

@@ -77,6 +79,9 @@ public class BesuPluginContextImplTest {
    contextImpl.registerPlugins(new File(".").toPath());
    assertThat(contextImpl.getPlugins()).isNotInstanceOfAny(TestPicoCLIPlugin.class);

+    contextImpl.beforeExternalServices();
+    assertThat(contextImpl.getPlugins()).isNotInstanceOfAny(TestPicoCLIPlugin.class);
+
    contextImpl.startPlugins();
    assertThat(contextImpl.getPlugins()).isNotInstanceOfAny(TestPicoCLIPlugin.class);

@@ -98,6 +103,7 @@ public class BesuPluginContextImplTest {
    final TestPicoCLIPlugin testPicoCLIPlugin = testPluginOptional.get();
    assertThat(testPicoCLIPlugin.getState()).isEqualTo("registered");

+    contextImpl.beforeExternalServices();
    contextImpl.startPlugins();
    assertThat(testPicoCLIPlugin.getState()).isEqualTo("failstart");
    assertThat(contextImpl.getPlugins()).isNotInstanceOfAny(TestPicoCLIPlugin.class);
@@ -120,6 +126,7 @@ public class BesuPluginContextImplTest {
    final TestPicoCLIPlugin testPicoCLIPlugin = testPluginOptional.get();
    assertThat(testPicoCLIPlugin.getState()).isEqualTo("registered");

+    contextImpl.beforeExternalServices();
    contextImpl.startPlugins();
    assertThat(testPicoCLIPlugin.getState()).isEqualTo("started");

@@ -140,6 +147,7 @@ public class BesuPluginContextImplTest {
    assertThatExceptionOfType(IllegalStateException.class).isThrownBy(registerPlugins);
    assertThatExceptionOfType(IllegalStateException.class).isThrownBy(contextImpl::stopPlugins);

+    contextImpl.beforeExternalServices();
    contextImpl.startPlugins();
    assertThatExceptionOfType(IllegalStateException.class).isThrownBy(registerPlugins);
    assertThatExceptionOfType(IllegalStateException.class).isThrownBy(contextImpl::startPlugins);
--- a/besu/src/main/java/org/hyperledger/besu/Runner.java
+++ b/besu/src/main/java/org/hyperledger/besu/Runner.java
@@ -103,31 +103,35 @@ public class Runner implements AutoCloseable {
            cacher -> new AutoTransactionLogBloomCachingService(blockchain, cacher));
  }

-  public void start() {
+  public void startExternalServices() {
+    LOG.info("Starting external services ... ");
+    metrics.ifPresent(service -> waitForServiceToStart("metrics", service.start()));
+
+    jsonRpc.ifPresent(service -> waitForServiceToStart("jsonRpc", service.start()));
+    graphQLHttp.ifPresent(service -> waitForServiceToStart("graphQLHttp", service.start()));
+    websocketRpc.ifPresent(service -> waitForServiceToStart("websocketRpc", service.start()));
+    stratumServer.ifPresent(server -> waitForServiceToStart("stratum", server.start()));
+    autoTransactionLogBloomCachingService.ifPresent(AutoTransactionLogBloomCachingService::start);
+    ethStatsService.ifPresent(EthStatsService::start);
+  }
+
+  public void startEthereumMainLoop() {
    try {
      LOG.info("Starting Ethereum main loop ... ");
-      metrics.ifPresent(service -> waitForServiceToStart("metrics", service.start()));
      natService.start();
      networkRunner.start();
      if (networkRunner.getNetwork().isP2pEnabled()) {
        besuController.getSynchronizer().start();
      }
      besuController.getMiningCoordinator().start();
-      stratumServer.ifPresent(server -> waitForServiceToStart("stratum", server.start()));
      vertx.setPeriodic(
          TimeUnit.MINUTES.toMillis(1),
          time ->
              besuController.getTransactionPool().getPendingTransactions().evictOldTransactions());
-      jsonRpc.ifPresent(service -> waitForServiceToStart("jsonRpc", service.start()));
-      graphQLHttp.ifPresent(service -> waitForServiceToStart("graphQLHttp", service.start()));
-      websocketRpc.ifPresent(service -> waitForServiceToStart("websocketRpc", service.start()));
-      ethStatsService.ifPresent(EthStatsService::start);
      LOG.info("Ethereum main loop is up.");
      writeBesuPortsToFile();
      writeBesuNetworksToFile();
-      autoTransactionLogBloomCachingService.ifPresent(AutoTransactionLogBloomCachingService::start);
      writePidFile();
-
    } catch (final Exception ex) {
      LOG.error("Startup failed", ex);
      throw new IllegalStateException(ex);
--- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java
+++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java
@@ -1268,10 +1268,18 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
      validateOptions();
      configure();
      initController();
+
+      besuPluginContext.beforeExternalServices();
+
+      var runner = buildRunner();
+      runner.startExternalServices();
+
      startPlugins();
      validatePluginOptions();
      preSynchronization();
-      startSynchronization();
+
+      runner.startEthereumMainLoop();
+      runner.awaitStop();

    } catch (final Exception e) {
      throw new ParameterException(this.commandLine, e.getMessage(), e);
@@ -1432,8 +1440,8 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
    preSynchronizationTaskRunner.runTasks(besuController);
  }

-  private void startSynchronization() {
-    synchronize(
+  private Runner buildRunner() {
+    return synchronize(
        besuController,
        p2pEnabled,
        p2pTLSConfiguration,
@@ -2453,7 +2461,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
  }

  // Blockchain synchronization from peers.
-  private void synchronize(
+  private Runner synchronize(
      final BesuController controller,
      final boolean p2pEnabled,
      final Optional<TLSConfiguration> p2pTLSConfiguration,
@@ -2519,8 +2527,8 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
            .build();

    addShutdownHook(runner);
-    runner.start();
-    runner.awaitStop();
+
+    return runner;
  }

  protected Vertx createVertx(final VertxOptions vertxOptions) {
--- a/besu/src/main/java/org/hyperledger/besu/cli/presynctasks/PreSynchronizationTask.java
+++ b/besu/src/main/java/org/hyperledger/besu/cli/presynctasks/PreSynchronizationTask.java
@@ -14,12 +14,13 @@
 */
 package org.hyperledger.besu.cli.presynctasks;

+import org.hyperledger.besu.Runner;
 import org.hyperledger.besu.cli.BesuCommand;
 import org.hyperledger.besu.controller.BesuController;

 /**
 * All PreSynchronizationTask instances execute after the {@link BesuController} instance in {@link
- * BesuCommand} is ready and before {@link BesuCommand#startSynchronization()} is called
+ * BesuCommand} is ready and before {@link Runner#startEthereumMainLoop()} is called
 */
 public interface PreSynchronizationTask {

--- a/besu/src/main/java/org/hyperledger/besu/services/BesuPluginContextImpl.java
+++ b/besu/src/main/java/org/hyperledger/besu/services/BesuPluginContextImpl.java
@@ -53,8 +53,10 @@ public class BesuPluginContextImpl implements BesuContext, PluginVersionsProvide
    UNINITIALIZED,
    REGISTERING,
    REGISTERED,
-    STARTING,
-    STARTED,
+    BEFORE_EXTERNAL_SERVICES_STARTED,
+    BEFORE_EXTERNAL_SERVICES_FINISHED,
+    BEFORE_MAIN_LOOP_STARTED,
+    BEFORE_MAIN_LOOP_FINISHED,
    STOPPING,
    STOPPED
  }
@@ -126,13 +128,43 @@ public class BesuPluginContextImpl implements BesuContext, PluginVersionsProvide
    pluginVersions.add(pluginVersion);
  }

-  public void startPlugins() {
+  public void beforeExternalServices() {
    checkState(
        state == Lifecycle.REGISTERED,
        "BesuContext should be in state %s but it was in %s",
        Lifecycle.REGISTERED,
        state);
-    state = Lifecycle.STARTING;
+    state = Lifecycle.BEFORE_EXTERNAL_SERVICES_STARTED;
+    final Iterator<BesuPlugin> pluginsIterator = plugins.iterator();
+
+    while (pluginsIterator.hasNext()) {
+      final BesuPlugin plugin = pluginsIterator.next();
+
+      try {
+        plugin.beforeExternalServices();
+        LOG.debug(
+            "beforeExternalServices called on plugin of type {}.", plugin.getClass().getName());
+      } catch (final Exception e) {
+        LOG.error(
+            "Error calling `beforeExternalServices` on plugin of type "
+                + plugin.getClass().getName()
+                + ", stop will not be called.",
+            e);
+        pluginsIterator.remove();
+      }
+    }
+
+    LOG.debug("Plugin startup complete.");
+    state = Lifecycle.BEFORE_EXTERNAL_SERVICES_FINISHED;
+  }
+
+  public void startPlugins() {
+    checkState(
+        state == Lifecycle.BEFORE_EXTERNAL_SERVICES_FINISHED,
+        "BesuContext should be in state %s but it was in %s",
+        Lifecycle.BEFORE_EXTERNAL_SERVICES_FINISHED,
+        state);
+    state = Lifecycle.BEFORE_MAIN_LOOP_STARTED;
    final Iterator<BesuPlugin> pluginsIterator = plugins.iterator();

    while (pluginsIterator.hasNext()) {
@@ -152,14 +184,14 @@ public class BesuPluginContextImpl implements BesuContext, PluginVersionsProvide
    }

    LOG.debug("Plugin startup complete.");
-    state = Lifecycle.STARTED;
+    state = Lifecycle.BEFORE_MAIN_LOOP_FINISHED;
  }

  public void stopPlugins() {
    checkState(
-        state == Lifecycle.STARTED,
+        state == Lifecycle.BEFORE_MAIN_LOOP_FINISHED,
        "BesuContext should be in state %s but it was in %s",
-        Lifecycle.STARTED,
+        Lifecycle.BEFORE_MAIN_LOOP_FINISHED,
        state);
    state = Lifecycle.STOPPING;

--- a/besu/src/test/java/org/hyperledger/besu/RunnerBuilderTest.java
+++ b/besu/src/test/java/org/hyperledger/besu/RunnerBuilderTest.java
@@ -145,7 +145,7 @@ public final class RunnerBuilderTest {
            .forkIdSupplier(() -> Collections.singletonList(Bytes.EMPTY))
            .rpcEndpointService(new RpcEndpointServiceImpl())
            .build();
-    runner.start();
+    runner.startEthereumMainLoop();

    final EnodeURL expectedEodeURL =
        EnodeURLImpl.builder()
@@ -189,7 +189,7 @@ public final class RunnerBuilderTest {
            .forkIdSupplier(() -> Collections.singletonList(Bytes.EMPTY))
            .rpcEndpointService(new RpcEndpointServiceImpl())
            .build();
-    runner.start();
+    runner.startEthereumMainLoop();
    when(besuController.getProtocolSchedule().streamMilestoneBlocks())
        .thenAnswer(__ -> Stream.of(1L, 2L));
    for (int i = 0; i < 2; ++i) {
--- a/besu/src/test/java/org/hyperledger/besu/RunnerTest.java
+++ b/besu/src/test/java/org/hyperledger/besu/RunnerTest.java
@@ -230,8 +230,8 @@ public final class RunnerTest {
            .rpcEndpointService(new RpcEndpointServiceImpl())
            .build();
    try {
-
-      runnerAhead.start();
+      runnerAhead.startExternalServices();
+      runnerAhead.startEthereumMainLoop();
      assertThat(pidPath.toFile().exists()).isTrue();

      final SynchronizerConfiguration syncConfigBehind =
@@ -284,7 +284,8 @@ public final class RunnerTest {
              .forkIdSupplier(() -> controllerBehind.getProtocolManager().getForkIdAsBytesList())
              .build();

-      runnerBehind.start();
+      runnerBehind.startExternalServices();
+      runnerBehind.startEthereumMainLoop();

      final int behindJsonRpcPort = runnerBehind.getJsonRpcPort().get();
      final Call.Factory client = new OkHttpClient();
--- a/plugin-api/build.gradle
+++ b/plugin-api/build.gradle
@@ -64,7 +64,7 @@ Calculated : ${currentHash}
 tasks.register('checkAPIChanges', FileStateChecker) {
  description = "Checks that the API for the Plugin-API project does not change without deliberate thought"
  files = sourceSets.main.allJava.files
-  knownHash = 'QPxUj/HJM7DZkbUq538/S2f3MmwPOqaPX/BBTkKznQk='
+  knownHash = 'DhISJBKoARZyoiT387UIA7QfcEwyOcIYm17f/fLsJRM='
 }
 check.dependsOn('checkAPIChanges')

--- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/BesuPlugin.java
+++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/BesuPlugin.java
@@ -51,9 +51,15 @@ public interface BesuPlugin {
  void register(BesuContext context);

  /**
-   * Called once Besu has loaded configuration and is starting up. The plugin should begin
-   * operation, including registering any event listener with Besu services and starting any
-   * background threads the plugin requires.
+   * Called once when besu has loaded configuration but before external services have been started
+   * e.g metrics and http
+   */
+  default void beforeExternalServices() {}
+
+  /**
+   * Called once Besu has loaded configuration and has started external services but before the main
+   * loop is up. The plugin should begin operation, including registering any event listener with
+   * Besu services and starting any background threads the plugin requires.
   */
  void start();