github/reth
1
1
Fork 0
mirror of https://github.com/paradigmxyz/reth.git synced 2026-01-26 15:48:13 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: local engine (#10803)

Browse Source
This commit is contained in:
greged93
2024-09-19 15:07:15 +02:00
committed by GitHub
parent 89ca7a9ef1
commit 668807802b
11 changed files with 478 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.github/assets/check_wasm.sh vendored
View File

@@ -66,6 +66,7 @@ exclude_crates=(
reth-rpc-types
reth-stages
reth-storage-errors
reth-engine-local
# The following are not supposed to be working
reth # all of the crates below
reth-invalid-block-hooks # reth-provider

70
Cargo.lock generated
View File

@@ -1529,9 +1529,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "bytes"
version = "1.7.1"
version = "1.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50"
checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3"
dependencies = [
"serde",
]
@@ -1606,9 +1606,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.1.19"
version = "1.1.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d74707dde2ba56f86ae90effb3b43ddd369504387e718014de010cec7959800"
checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0"
dependencies = [
"jobserver",
"libc",
@@ -4032,9 +4032,9 @@ checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4"
[[package]]
name = "iri-string"
version = "0.7.4"
version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e0f755bd3806e06ad4f366f92639415d99a339a2c7ecf8c26ccea2097c11cb6"
checksum = "9c25163201be6ded9e686703e85532f8f852ea1f92ba625cb3c51f7fe6d07a4a"
dependencies = [
"memchr",
"serde",
@@ -4410,7 +4410,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4"
dependencies = [
"cfg-if",
"windows-targets 0.48.5",
"windows-targets 0.52.6",
]
[[package]]
@@ -5086,9 +5086,9 @@ dependencies = [
[[package]]
name = "once_cell"
version = "1.20.0"
version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ea5043e58958ee56f3e15a90aee535795cd7dfd319846288d93c5b57d85cbe"
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
[[package]]
name = "oorandom"
@@ -6946,6 +6946,35 @@ dependencies = [
"typenum",
]
[[package]]
name = "reth-engine-local"
version = "1.0.7"
dependencies = [
"eyre",
"futures-util",
"reth-beacon-consensus",
"reth-chain-state",
"reth-chainspec",
"reth-config",
"reth-db",
"reth-engine-tree",
"reth-ethereum-engine-primitives",
"reth-exex-test-utils",
"reth-node-types",
"reth-payload-builder",
"reth-payload-primitives",
"reth-primitives",
"reth-provider",
"reth-prune",
"reth-rpc-types",
"reth-stages-api",
"reth-tracing",
"reth-transaction-pool",
"tokio",
"tokio-stream",
"tracing",
]
[[package]]
name = "reth-engine-primitives"
version = "1.0.7"
@@ -8081,6 +8110,7 @@ dependencies = [
"futures-util",
"metrics",
"pin-project",
"reth-chain-state",
"reth-errors",
"reth-ethereum-engine-primitives",
"reth-metrics",
@@ -9990,9 +10020,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "symbolic-common"
version = "12.11.0"
version = "12.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c1db5ac243c7d7f8439eb3b8f0357888b37cf3732957e91383b0ad61756374e"
checksum = "9fdf97c441f18a4f92425b896a4ec7a27e03631a0b1047ec4e34e9916a9a167e"
dependencies = [
"debugid",
"memmap2",
@@ -10002,9 +10032,9 @@ dependencies = [
[[package]]
name = "symbolic-demangle"
version = "12.11.0"
version = "12.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea26e430c27d4a8a5dea4c4b81440606c7c1a415bd611451ef6af8c81416afc3"
checksum = "bc8ece6b129e97e53d1fbb3f61d33a6a9e5369b11d01228c068094d6d134eaea"
dependencies = [
"cpp_demangle",
"rustc-demangle",
@@ -10450,9 +10480,9 @@ dependencies = [
[[package]]
name = "toml_edit"
version = "0.22.20"
version = "0.22.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d"
checksum = "3b072cee73c449a636ffd6f32bd8de3a9f7119139aff882f44943ce2986dc5cf"
dependencies = [
"indexmap 2.5.0",
"serde",
@@ -10829,9 +10859,9 @@ checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe"
[[package]]
name = "unicode-normalization"
version = "0.1.23"
version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5"
checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956"
dependencies = [
"tinyvec",
]
@@ -11096,9 +11126,9 @@ dependencies = [
[[package]]
name = "webpki-roots"
version = "0.26.5"
version = "0.26.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bd24728e5af82c6c4ec1b66ac4844bdf8156257fccda846ec58b42cd0cdbe6a"
checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958"
dependencies = [
"rustls-pki-types",
]
@@ -11131,7 +11161,7 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
dependencies = [
"windows-sys 0.48.0",
"windows-sys 0.59.0",
]
[[package]]

2
Cargo.toml
View File

@@ -27,6 +27,7 @@ members = [
"crates/consensus/debug-client/",
"crates/e2e-test-utils/",
"crates/engine/invalid-block-hooks/",
"crates/engine/local",
"crates/engine/primitives/",
"crates/engine/service",
"crates/engine/tree/",
@@ -314,6 +315,7 @@ reth-dns-discovery = { path = "crates/net/dns" }
reth-downloaders = { path = "crates/net/downloaders" }
reth-e2e-test-utils = { path = "crates/e2e-test-utils" }
reth-ecies = { path = "crates/net/ecies" }
reth-engine-local = { path = "crates/engine/local" }
reth-engine-primitives = { path = "crates/engine/primitives" }
reth-engine-tree = { path = "crates/engine/tree" }
reth-engine-service = { path = "crates/engine/service" }

46
crates/engine/local/Cargo.toml Normal file
View File

@@ -0,0 +1,46 @@
[package]
name = "reth-engine-local"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
exclude.workspace = true
[dependencies]
# reth
reth-beacon-consensus.workspace = true
reth-engine-tree.workspace = true
reth-node-types.workspace = true
reth-payload-builder.workspace = true
reth-payload-primitives.workspace = true
reth-primitives.workspace = true
reth-provider.workspace = true
reth-prune.workspace = true
reth-transaction-pool.workspace = true
reth-stages-api.workspace = true
# async
tokio.workspace = true
tokio-stream.workspace = true
futures-util.workspace = true
# misc
eyre.workspace = true
tracing.workspace = true
[dev-dependencies]
reth-chainspec.workspace = true
reth-chain-state.workspace = true
reth-config.workspace = true
reth-db = { workspace = true, features = ["test-utils"] }
reth-ethereum-engine-primitives.workspace = true
reth-exex-test-utils.workspace = true
reth-payload-builder = { workspace = true, features = ["test-utils"] }
reth-provider = { workspace = true, features = ["test-utils"] }
reth-rpc-types.workspace = true
reth-tracing.workspace = true
[lints]
workspace = true

3
crates/engine/local/src/lib.rs Normal file
View File

@@ -0,0 +1,3 @@
//! A local engine service that can be used to drive a dev chain.
pub mod miner;
pub mod service;

60
crates/engine/local/src/miner.rs Normal file
View File

@@ -0,0 +1,60 @@
//! Contains the implementation of the mining mode for the local engine.
use futures_util::{stream::Fuse, StreamExt};
use reth_primitives::TxHash;
use reth_transaction_pool::TransactionPool;
use std::{
future::Future,
pin::Pin,
task::{Context, Poll},
time::Duration,
};
use tokio::time::Interval;
use tokio_stream::wrappers::ReceiverStream;
/// A mining mode for the local dev engine.
#[derive(Debug)]
pub enum MiningMode {
/// In this mode a block is built as soon as
/// a valid transaction reaches the pool.
Instant(Fuse<ReceiverStream<TxHash>>),
/// In this mode a block is built at a fixed interval.
Interval(Interval),
}
impl MiningMode {
/// Constructor for a [`MiningMode::Instant`]
pub fn instant<Pool: TransactionPool>(pool: Pool) -> Self {
let rx = pool.pending_transactions_listener();
Self::Instant(ReceiverStream::new(rx).fuse())
}
/// Constructor for a [`MiningMode::Interval`]
pub fn interval(duration: Duration) -> Self {
let start = tokio::time::Instant::now() + duration;
Self::Interval(tokio::time::interval_at(start, duration))
}
}
impl Future for MiningMode {
type Output = ();
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
let this = self.get_mut();
match this {
Self::Instant(rx) => {
// drain all transactions notifications
if let Poll::Ready(Some(_)) = rx.poll_next_unpin(cx) {
return Poll::Ready(())
}
Poll::Pending
}
Self::Interval(interval) => {
if interval.poll_tick(cx).is_ready() {
return Poll::Ready(())
}
Poll::Pending
}
}
}
}

298
crates/engine/local/src/service.rs Normal file
View File

@@ -0,0 +1,298 @@
//! Provides a local dev service engine that can be used to run a dev chain.
//!
//! [`LocalEngineService`] polls the payload builder based on a mining mode
//! which can be set to `Instant` or `Interval`. The `Instant` mode will
//! constantly poll the payload builder and initiate block building
//! with a single transaction. The `Interval` mode will initiate block
//! building at a fixed interval.
use crate::miner::MiningMode;
use reth_beacon_consensus::EngineNodeTypes;
use reth_engine_tree::persistence::PersistenceHandle;
use reth_payload_builder::PayloadBuilderHandle;
use reth_payload_primitives::{
BuiltPayload, PayloadAttributesBuilder, PayloadBuilderAttributes, PayloadTypes,
};
use reth_primitives::B256;
use reth_provider::ProviderFactory;
use reth_prune::PrunerWithFactory;
use reth_stages_api::MetricEventsSender;
use std::fmt::Formatter;
use tokio::sync::oneshot;
use tracing::debug;
/// Provides a local dev service engine that can be used to drive the
/// chain forward.
pub struct LocalEngineService<N, B>
where
N: EngineNodeTypes,
B: PayloadAttributesBuilder<PayloadAttributes = <N::Engine as PayloadTypes>::PayloadAttributes>,
{
/// The payload builder for the engine
payload_builder: PayloadBuilderHandle<N::Engine>,
/// The payload attribute builder for the engine
payload_attributes_builder: B,
/// A handle to the persistence layer
persistence_handle: PersistenceHandle,
/// The hash of the current head
head: B256,
/// The mining mode for the engine
mode: MiningMode,
}
impl<N, B> std::fmt::Debug for LocalEngineService<N, B>
where
N: EngineNodeTypes,
B: PayloadAttributesBuilder<PayloadAttributes = <N::Engine as PayloadTypes>::PayloadAttributes>,
{
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
f.debug_struct("LocalEngineService")
.field("payload_builder", &self.payload_builder)
.field("payload_attributes_builder", &self.payload_attributes_builder)
.field("persistence_handle", &self.persistence_handle)
.field("head", &self.head)
.field("mode", &self.mode)
.finish()
}
}
impl<N, B> LocalEngineService<N, B>
where
N: EngineNodeTypes,
B: PayloadAttributesBuilder<PayloadAttributes = <N::Engine as PayloadTypes>::PayloadAttributes>,
{
/// Constructor for [`LocalEngineService`].
pub fn new(
payload_builder: PayloadBuilderHandle<N::Engine>,
payload_attributes_builder: B,
provider: ProviderFactory<N>,
pruner: PrunerWithFactory<ProviderFactory<N>>,
sync_metrics_tx: MetricEventsSender,
head: B256,
mode: MiningMode,
) -> Self {
let persistence_handle =
PersistenceHandle::spawn_service(provider, pruner, sync_metrics_tx);
Self { payload_builder, payload_attributes_builder, persistence_handle, head, mode }
}
/// Spawn the [`LocalEngineService`] on a tokio green thread. The service will poll the payload
/// builder with two varying modes, [`MiningMode::Instant`] or [`MiningMode::Interval`]
/// which will respectively either execute the block as soon as it finds a
/// transaction in the pool or build the block based on an interval.
pub fn spawn_new(
payload_builder: PayloadBuilderHandle<N::Engine>,
payload_attributes_builder: B,
provider: ProviderFactory<N>,
pruner: PrunerWithFactory<ProviderFactory<N>>,
sync_metrics_tx: MetricEventsSender,
head: B256,
mode: MiningMode,
) {
let engine = Self::new(
payload_builder,
payload_attributes_builder,
provider,
pruner,
sync_metrics_tx,
head,
mode,
);
// Spawn the engine
tokio::spawn(engine.run());
}
/// Runs the [`LocalEngineService`] in a loop, polling the miner and building
/// payloads.
async fn run(mut self) {
loop {
// Wait for the interval or the pool to receive a transaction
(&mut self.mode).await;
// Start a new payload building job
let new_head = self.build_and_save_payload().await;
if new_head.is_err() {
debug!(target: "local_engine", err = ?new_head.unwrap_err(), "failed payload building");
continue
}
// Update the head
self.head = new_head.expect("not error");
}
}
/// Builds a payload by initiating a new payload job via the [`PayloadBuilderHandle`],
/// saving the execution outcome to persistence and returning the current head of the
/// chain.
async fn build_and_save_payload(&self) -> eyre::Result<B256> {
let payload_attributes = self.payload_attributes_builder.build()?;
let payload_builder_attributes =
<N::Engine as PayloadTypes>::PayloadBuilderAttributes::try_new(
self.head,
payload_attributes,
)
.map_err(|_| eyre::eyre!("failed to fetch payload attributes"))?;
let payload = self
.payload_builder
.send_and_resolve_payload(payload_builder_attributes)
.await?
.await?;
let block = payload.executed_block().map(|block| vec![block]).unwrap_or_default();
let (tx, rx) = oneshot::channel();
let _ = self.persistence_handle.save_blocks(block, tx);
// Wait for the persistence_handle to complete
let new_head = rx.await?.ok_or_else(|| eyre::eyre!("missing new head"))?;
Ok(new_head.hash)
}
}
#[cfg(test)]
mod tests {
use super::*;
use reth_chainspec::MAINNET;
use reth_config::PruneConfig;
use reth_db::test_utils::{create_test_rw_db, create_test_static_files_dir};
use reth_ethereum_engine_primitives::EthEngineTypes;
use reth_exex_test_utils::TestNode;
use reth_node_types::NodeTypesWithDBAdapter;
use reth_payload_builder::test_utils::spawn_test_payload_service;
use reth_primitives::B256;
use reth_provider::{providers::StaticFileProvider, BlockReader, ProviderFactory};
use reth_prune::PrunerBuilder;
use reth_transaction_pool::{
test_utils::{testing_pool, MockTransaction},
TransactionPool,
};
use std::{convert::Infallible, time::Duration};
use tokio::sync::mpsc::unbounded_channel;
#[derive(Debug)]
struct TestPayloadAttributesBuilder;
impl PayloadAttributesBuilder for TestPayloadAttributesBuilder {
type PayloadAttributes = reth_rpc_types::engine::PayloadAttributes;
type Error = Infallible;
fn build(&self) -> Result<Self::PayloadAttributes, Self::Error> {
Ok(reth_rpc_types::engine::PayloadAttributes {
timestamp: 0,
prev_randao: Default::default(),
suggested_fee_recipient: Default::default(),
withdrawals: None,
parent_beacon_block_root: None,
})
}
}
#[tokio::test]
async fn test_local_engine_service_interval() -> eyre::Result<()> {
reth_tracing::init_test_tracing();
// Start the provider and the pruner
let (_, static_dir_path) = create_test_static_files_dir();
let provider = ProviderFactory::<NodeTypesWithDBAdapter<TestNode, _>>::new(
create_test_rw_db(),
MAINNET.clone(),
StaticFileProvider::read_write(static_dir_path).unwrap(),
);
let pruner = PrunerBuilder::new(PruneConfig::default())
.build_with_provider_factory(provider.clone());
// Start the payload builder service
let payload_handle = spawn_test_payload_service::<EthEngineTypes>();
// Sync metric channel
let (sync_metrics_tx, _) = unbounded_channel();
// Get the attributes for start of block building
let genesis_hash = B256::random();
// Launch the LocalEngineService in interval mode
let period = Duration::from_secs(1);
LocalEngineService::spawn_new(
payload_handle,
TestPayloadAttributesBuilder,
provider.clone(),
pruner,
sync_metrics_tx,
genesis_hash,
MiningMode::interval(period),
);
// Wait 4 intervals
tokio::time::sleep(4 * period).await;
// Assert a block has been build
let block = provider.block_by_number(0)?;
assert!(block.is_some());
Ok(())
}
#[tokio::test]
async fn test_local_engine_service_instant() -> eyre::Result<()> {
reth_tracing::init_test_tracing();
// Start the provider and the pruner
let (_, static_dir_path) = create_test_static_files_dir();
let provider = ProviderFactory::<NodeTypesWithDBAdapter<TestNode, _>>::new(
create_test_rw_db(),
MAINNET.clone(),
StaticFileProvider::read_write(static_dir_path).unwrap(),
);
let pruner = PrunerBuilder::new(PruneConfig::default())
.build_with_provider_factory(provider.clone());
// Start the payload builder service
let payload_handle = spawn_test_payload_service::<EthEngineTypes>();
// Start a transaction pool
let pool = testing_pool();
// Sync metric channel
let (sync_metrics_tx, _) = unbounded_channel();
// Get the attributes for start of block building
let genesis_hash = B256::random();
// Launch the LocalEngineService in instant mode
LocalEngineService::spawn_new(
payload_handle,
TestPayloadAttributesBuilder,
provider.clone(),
pruner,
sync_metrics_tx,
genesis_hash,
MiningMode::instant(pool.clone()),
);
// Wait for a small period to assert block building is
// triggered by adding a transaction to the pool
let period = Duration::from_millis(500);
tokio::time::sleep(period).await;
let block = provider.block_by_number(0)?;
assert!(block.is_none());
// Add a transaction to the pool
let transaction = MockTransaction::legacy().with_gas_price(10);
pool.add_transaction(Default::default(), transaction).await?;
// Wait for block building
let period = Duration::from_secs(2);
tokio::time::sleep(period).await;
// Assert a block has been build
let block = provider.block_by_number(0)?;
assert!(block.is_some());
Ok(())
}
}

3
crates/payload/builder/Cargo.toml
View File

@@ -20,6 +20,7 @@ reth-errors.workspace = true
reth-provider.workspace = true
reth-payload-primitives.workspace = true
reth-ethereum-engine-primitives.workspace = true
reth-chain-state = { workspace = true, optional = true }
# alloy
alloy-primitives.workspace = true
@@ -42,4 +43,4 @@ tracing.workspace = true
revm.workspace = true
[features]
test-utils = []
test-utils = ["reth-chain-state"]

3
crates/payload/builder/src/test_utils.rs
View File

@@ -6,6 +6,7 @@ use crate::{
PayloadJobGenerator,
};
use alloy_primitives::U256;
use reth_chain_state::ExecutedBlock;
use reth_payload_primitives::PayloadTypes;
use reth_primitives::Block;
use reth_provider::CanonStateNotification;
@@ -87,7 +88,7 @@ impl PayloadJob for TestPayloadJob {
self.attr.payload_id(),
Block::default().seal_slow(),
U256::ZERO,
None,
Some(ExecutedBlock::default()),
))
}

4
crates/payload/primitives/src/lib.rs
View File

@@ -15,7 +15,9 @@ pub use error::{EngineObjectValidationError, PayloadBuilderError, VersionSpecifi
/// Contains traits to abstract over payload attributes types and default implementations of the
/// [`PayloadAttributes`] trait for ethereum mainnet and optimism types.
mod traits;
pub use traits::{BuiltPayload, PayloadAttributes, PayloadBuilderAttributes};
pub use traits::{
BuiltPayload, PayloadAttributes, PayloadAttributesBuilder, PayloadBuilderAttributes,
};
mod payload;
pub use payload::PayloadOrAttributes;

11
crates/payload/primitives/src/traits.rs
View File

@@ -145,3 +145,14 @@ impl PayloadAttributes for OptimismPayloadAttributes {
Ok(())
}
}
/// A builder that can return the current payload attribute.
pub trait PayloadAttributesBuilder: std::fmt::Debug + Send + Sync + 'static {
/// The payload attributes type returned by the builder.
type PayloadAttributes: PayloadAttributes;
/// The error type returned by [`PayloadAttributesBuilder::build`].
type Error: std::error::Error + Send + Sync;
/// Return a new payload attribute from the builder.
fn build(&self) -> Result<Self::PayloadAttributes, Self::Error>;
}