diff --git a/.gitignore b/.gitignore index 7d1cd95..8df3f4b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ /target +# local run +/storage + # heaptrack *.zst perf.data diff --git a/Cargo.lock b/Cargo.lock index 92702c9..b7af866 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1188,6 +1188,29 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "aws-lc-rs" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08b5d4e069cbc868041a64bd68dc8cb39a0d79585cd6c5a24caa8c2d622121be" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbfd150b5dbdb988bcc8fb1fe787eb6b7ee6180ca24da683b61ea5405f3d43ff" +dependencies = [ + "bindgen", + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "axum" version = "0.8.3" @@ -1284,12 +1307,15 @@ dependencies = [ "itertools 0.10.5", "lazy_static", "lazycell", + "log", + "prettyplease", "proc-macro2", "quote", "regex", "rustc-hash 1.1.0", "shlex", "syn 2.0.100", + "which", ] [[package]] @@ -1581,6 +1607,15 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" +[[package]] +name = "cmake" +version = "0.1.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.3" @@ -1645,6 +1680,16 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -1972,6 +2017,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + [[package]] name = "enum-ordinalize" version = "4.3.0" @@ -2141,6 +2192,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "funty" version = "2.0.0" @@ -2400,6 +2457,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "home" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "http" version = "1.3.1" @@ -2467,6 +2533,23 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + [[package]] name = "hyper-timeout" version = "0.5.2" @@ -2895,6 +2978,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.9.4" @@ -2986,6 +3075,57 @@ dependencies = [ "zeroize", ] +[[package]] +name = "metrics" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25dea7ac8057892855ec285c440160265225438c3c45072613c25a4b26e98ef5" +dependencies = [ + "ahash", + "portable-atomic", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b166dea96003ee2531cf14833efedced545751d800f03535801d833313f8c15" +dependencies = [ + "base64", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "indexmap 2.9.0", + "ipnet", + "metrics", + "metrics-util", + "quanta", + "thiserror 2.0.12", + "tokio", + "tracing", +] + +[[package]] +name = "metrics-util" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe8db7a05415d0f919ffb905afa37784f71901c9a773188876984b4f769ab986" +dependencies = [ + "aho-corasick", + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.15.2", + "indexmap 2.9.0", + "metrics", + "ordered-float", + "quanta", + "radix_trie", + "rand 0.9.1", + "rand_xoshiro", + "sketches-ddsketch", +] + [[package]] name = "mime" version = "0.3.17" @@ -3045,11 +3185,20 @@ dependencies = [ "openssl-probe", "openssl-sys", "schannel", - "security-framework", + "security-framework 2.11.1", "security-framework-sys", "tempfile", ] +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec", +] + [[package]] name = "nom" version = "7.1.3" @@ -3222,6 +3371,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + [[package]] name = "overload" version = "0.1.1" @@ -3429,6 +3587,12 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "portable-atomic" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" + [[package]] name = "powerfmt" version = "0.2.0" @@ -3597,6 +3761,9 @@ dependencies = [ "derive_more", "futures", "http", + "metrics", + "metrics-exporter-prometheus", + "metrics-util", "nom 8.0.0", "num-bigint", "parking_lot 0.12.4", @@ -3637,6 +3804,32 @@ dependencies = [ "tracing-test", ] +[[package]] +name = "prover_client" +version = "0.1.0" +dependencies = [ + "alloy", + "prost", + "tokio", + "tonic", + "tonic-build", +] + +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi 0.11.0+wasi-snapshot-preview1", + "web-sys", + "winapi", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -3664,6 +3857,16 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + [[package]] name = "rand" version = "0.8.5" @@ -3735,6 +3938,24 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.3", +] + +[[package]] +name = "raw-cpuid" +version = "11.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6df7ab838ed27997ba19a4664507e6f82b41fe6e20be42929332156e5e85146" +dependencies = [ + "bitflags 2.9.0", +] + [[package]] name = "rayon" version = "1.7.0" @@ -4018,6 +4239,19 @@ dependencies = [ "semver 1.0.26", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.9.0", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + [[package]] name = "rustix" version = "1.0.5" @@ -4027,7 +4261,7 @@ dependencies = [ "bitflags 2.9.0", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.9.4", "windows-sys 0.59.0", ] @@ -4037,6 +4271,7 @@ version = "0.23.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321" dependencies = [ + "aws-lc-rs", "once_cell", "ring", "rustls-pki-types", @@ -4045,6 +4280,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework 3.2.0", +] + [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -4066,6 +4313,7 @@ version = "0.103.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -4162,7 +4410,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags 2.9.0", - "core-foundation", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" +dependencies = [ + "bitflags 2.9.0", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -4368,6 +4629,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "sketches-ddsketch" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1e9a774a6c28142ac54bb25d25562e6bcf957493a184f15ad4eebccb23e410a" + [[package]] name = "slab" version = "0.4.9" @@ -4548,7 +4815,7 @@ dependencies = [ "fastrand", "getrandom 0.3.2", "once_cell", - "rustix", + "rustix 1.0.5", "windows-sys 0.59.0", ] @@ -4558,7 +4825,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45c6481c4829e4cc63825e62c49186a34538b7b2750b73b266581ffb612fb5ed" dependencies = [ - "rustix", + "rustix 1.0.5", "windows-sys 0.59.0", ] @@ -5320,6 +5587,18 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index d7cff06..4569a8b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "smart_contract", "prover", "prover_cli", + "prover_client", ] resolver = "2" diff --git a/Readme.md b/Readme.md index cdd6e81..a654097 100644 --- a/Readme.md +++ b/Readme.md @@ -5,9 +5,17 @@ * docker build --progress=plain --no-cache -t prover . * sudo docker run -p 50051:50051 prover --mock-sc true --mock-user mock/mock_user_1.json -## Run +## Run prover -RUST_LOG=debug cargo run -- -i 127.0.0.1 -r "wss://eth-mainnet.g.alchemy.com/v2/__MY_TOKEN__" +RUST_LOG=debug cargo run -p prover_cli -- -i 127.0.0.1 -r "wss://eth-mainnet.g.alchemy.com/v2/__MY_TOKEN__" + +## Run prover + Mock + +RUST_LOG=debug cargo run -p prover_cli -- -i 127.0.0.1 --metrics-ip 127.0.0.1 --mock-sc true --mock-user mock/mock_user_1.json + +## Run prover client (for tests) + +RUST_LOG=debug cargo run -p prover_client ## Debug diff --git a/prover/Cargo.toml b/prover/Cargo.toml index 554cc80..845ddeb 100644 --- a/prover/Cargo.toml +++ b/prover/Cargo.toml @@ -37,6 +37,9 @@ nom = "8.0" claims = "0.8" clap_config = "0.1" toml = "0.8" +metrics = "0.24" +metrics-exporter-prometheus = "0.17" +metrics-util = "0.20" rln = { git = "https://github.com/vacp2p/zerokit", features = ["pmtree-ft"] } zerokit_utils = { git = "https://github.com/vacp2p/zerokit", package = "zerokit_utils", features = ["default"] } rln_proof = { path = "../rln_proof" } diff --git a/prover/src/args.rs b/prover/src/args.rs index 58b5b0a..ab09530 100644 --- a/prover/src/args.rs +++ b/prover/src/args.rs @@ -91,6 +91,14 @@ pub struct AppArgs { help_heading = "config" )] pub no_config: Option, + #[arg( + long = "metrics-ip", + default_value = "::1", + help = "Prometheus Metrics ip" + )] + pub metrics_ip: IpAddr, + #[arg(long = "metrics-port", default_value = "30031", help = "Metrics port")] + pub metrics_port: u16, // Hidden option - expect user set it via a config file #[arg( long = "broadcast-channel-size", diff --git a/prover/src/epoch_service.rs b/prover/src/epoch_service.rs index 9155602..8469305 100644 --- a/prover/src/epoch_service.rs +++ b/prover/src/epoch_service.rs @@ -4,11 +4,13 @@ use std::time::Duration; // third-party use chrono::{DateTime, NaiveDate, NaiveDateTime, OutOfRangeError, TimeDelta, Utc}; use derive_more::{Deref, From, Into}; +use metrics::{gauge, histogram}; use parking_lot::RwLock; use tokio::sync::Notify; use tracing::{debug, error}; // internal use crate::error::AppError; +use crate::metrics::{EPOCH_SERVICE_CURRENT_EPOCH, EPOCH_SERVICE_CURRENT_EPOCH_SLICE, EPOCH_SERVICE_DRIFT_MILLIS}; /// Duration of an epoch (1 day) const EPOCH_DURATION: Duration = Duration::from_secs(TimeDelta::days(1).num_seconds() as u64); @@ -89,7 +91,9 @@ impl EpochService { { let now_ = tokio::time::Instant::now(); debug!("awake at: {:?}, drift by: {:?}", now_, now_ - wait_until); + histogram!(EPOCH_SERVICE_DRIFT_MILLIS.name, "prover" => "epoch service").record(now_ - wait_until); } + // Note: could use checked_add() here, but it's quite impossible to have an overflow here // it would mean that the epoch_slice_duration is insanely large and wait_until // overflows as a timestamp @@ -106,6 +110,11 @@ impl EpochService { current_epoch, current_epoch_slice ); + // Note: based on this link https://doc.rust-lang.org/reference/expressions/operator-expr.html#type-cast-expressions + // "Casting from an integer to float will produce the closest possible float *" + gauge!(EPOCH_SERVICE_CURRENT_EPOCH.name, "prover" => "epoch service").set(i64::from(current_epoch) as f64); + gauge!(EPOCH_SERVICE_CURRENT_EPOCH_SLICE.name, "prover" => "epoch service").set(i64::from(current_epoch_slice) as f64); + // println!("Epoch changed: {}", current_epoch); self.epoch_changes.notify_one(); } diff --git a/prover/src/grpc_service.rs b/prover/src/grpc_service.rs index 7c42e4d..b31fa54 100644 --- a/prover/src/grpc_service.rs +++ b/prover/src/grpc_service.rs @@ -8,6 +8,7 @@ use async_channel::Sender; use bytesize::ByteSize; use futures::TryFutureExt; use http::Method; +use metrics::counter; use num_bigint::BigUint; use tokio::sync::{broadcast, mpsc}; use tonic::{ @@ -21,6 +22,8 @@ use url::Url; use crate::error::{AppError, ProofGenerationStringError}; use crate::proof_generation::{ProofGenerationData, ProofSendingData}; use crate::user_db::{UserDb, UserTierInfo}; +use crate::metrics::{USER_REGISTERED, USER_REGISTERED_REQUESTS, SEND_TRANSACTION_REQUESTS, GET_USER_TIER_INFO_REQUESTS, GET_PROOFS_LISTENERS, GaugeWrapper}; +use crate::user_db_error::RegisterError; use rln_proof::RlnIdentifier; use smart_contract::{ KarmaAmountExt, @@ -39,7 +42,6 @@ pub mod prover_proto { pub(crate) const FILE_DESCRIPTOR_SET: &[u8] = tonic::include_file_descriptor_set!("prover_descriptor"); } -use crate::user_db_error::RegisterError; use prover_proto::{ GetUserTierInfoReply, GetUserTierInfoRequest, @@ -100,6 +102,8 @@ where &self, request: Request, ) -> Result, Status> { + + counter!(SEND_TRANSACTION_REQUESTS.name, "service" => "grpc").increment(1); debug!("send_transaction request: {:?}", request); let req = request.into_inner(); @@ -154,6 +158,7 @@ where request: Request, ) -> Result, Status> { debug!("register_user request: {:?}", request); + counter!(USER_REGISTERED_REQUESTS.name, "service" => "grpc").increment(1); let req = request.into_inner(); let user = if let Some(user) = req.user { @@ -192,6 +197,8 @@ where let reply = RegisterUserReply { status: status.into(), }; + + counter!(USER_REGISTERED.name, "service" => "grpc").increment(1); Ok(Response::new(reply)) } @@ -201,13 +208,20 @@ where &self, request: Request, ) -> Result, Status> { + debug!("get_proofs request: {:?}", request); + let gauge = GaugeWrapper::new(GET_PROOFS_LISTENERS.name, "service", "grpc"); + // Channel to send proof to the connected grpc client (aka the Verifier) let (tx, rx) = mpsc::channel(self.proof_sender_channel_size); // Channel to receive a RLN proof (from one proof service) let mut rx2 = self.broadcast_channel.0.subscribe(); tokio::spawn(async move { + // FIXME: Should we send the error here? + + let gauge_ = gauge; + while let Ok(Ok(data)) = rx2.recv().await { let rln_proof = RlnProof { sender: data.tx_sender.to_vec(), @@ -224,6 +238,9 @@ where break; }; } + + // Note: will be dropped anyway but better be explicit here :) + drop(gauge_); }); Ok(Response::new(ReceiverStream::new(rx))) @@ -233,7 +250,9 @@ where &self, request: Request, ) -> Result, Status> { + debug!("request: {:?}", request); + counter!(GET_USER_TIER_INFO_REQUESTS.name, "service" => "grpc").increment(1); let req = request.into_inner(); diff --git a/prover/src/lib.rs b/prover/src/lib.rs index 78076a1..faeb0e3 100644 --- a/prover/src/lib.rs +++ b/prover/src/lib.rs @@ -3,6 +3,7 @@ mod args; mod epoch_service; mod error; mod grpc_service; +pub mod metrics; mod mock; mod proof_generation; mod proof_service; @@ -336,6 +337,8 @@ mod tests { mock_user: None, config_path: Default::default(), no_config: Some(true), + metrics_ip: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), + metrics_port: 30031, broadcast_channel_size: 100, proof_service_count: 8, transaction_channel_size: 100, @@ -382,6 +385,8 @@ mod tests { mock_user: None, config_path: Default::default(), no_config: Some(true), + metrics_ip: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), + metrics_port: 30031, broadcast_channel_size: 100, proof_service_count: 8, transaction_channel_size: 100, diff --git a/prover/src/metrics.rs b/prover/src/metrics.rs new file mode 100644 index 0000000..d0365c4 --- /dev/null +++ b/prover/src/metrics.rs @@ -0,0 +1,146 @@ +use std::net::{IpAddr, SocketAddr}; +use metrics::gauge; +// third-party +use metrics_exporter_prometheus::PrometheusBuilder; +// use metrics_util::MetricKindMask; +use tracing::{ + // debug, + // error, + info, +}; + +pub struct Metric { + pub name: &'static str, + description: &'static str, +} + +pub const USER_REGISTERED_REQUESTS: Metric = Metric { + name: "user_registered_requests", + description: "Number of RegisterUser grpc requests", +}; +pub const USER_REGISTERED: Metric = Metric { + name: "user_registered", + description: "Number of registered users in the prover", +}; +pub const SEND_TRANSACTION_REQUESTS: Metric = Metric { + name: "send_transaction_requests", + description: "Number of SendTransaction grpc requests", +}; + +pub const GET_USER_TIER_INFO_REQUESTS: Metric = Metric { + name: "get_user_tier_info_requests", + description: "Number of GetUserTierInfo grpc requests", +}; + +pub const EPOCH_SERVICE_CURRENT_EPOCH: Metric = Metric { + name: "epoch_service_current_epoch", + description: "Current epoch in the epoch service", +}; + +pub const EPOCH_SERVICE_CURRENT_EPOCH_SLICE: Metric = Metric { + name: "epoch_service_current_epoch_slice", + description: "Current epoch slice in the epoch service", +}; + +pub const EPOCH_SERVICE_DRIFT_MILLIS: Metric = Metric { + name: "epoch_service_drift_millis", + description: "Drift in milliseconds (when epoch service is waiting for the next epoch slice)", +}; + +pub const PROOF_SERVICE_GEN_PROOF_TIME: Metric = Metric { + name: "proof_service_gen_proof_time", + description: "Generation time of a proof in milliseconds", +}; + +pub const GET_PROOFS_LISTENERS: Metric = Metric { + name: "get_proof_listeners", + description: "Current number of active subscription to grpc get_proofs server streaming endpoint", +}; + +pub const COUNTERS: [Metric; 4] = [ + USER_REGISTERED, + USER_REGISTERED_REQUESTS, + SEND_TRANSACTION_REQUESTS, + GET_USER_TIER_INFO_REQUESTS +]; +pub const GAUGES: [Metric; 3] = [ + EPOCH_SERVICE_CURRENT_EPOCH, + EPOCH_SERVICE_CURRENT_EPOCH_SLICE, + GET_PROOFS_LISTENERS, +]; +pub const HISTOGRAMS: [Metric; 2] = [EPOCH_SERVICE_DRIFT_MILLIS, PROOF_SERVICE_GEN_PROOF_TIME]; + +pub fn init_metrics(ip: IpAddr, port: &u16) { + info!("Initializing metrics exporter (port: {})", port); + + // Install in the current tokio runtime + PrometheusBuilder::new() + // .idle_timeout( + // MetricKindMask::COUNTER | MetricKindMask::HISTOGRAM, + // Some(Duration::from_secs(10)), + // ) + .with_http_listener(SocketAddr::new( + // IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), + ip, + port.to_owned(), + )) + .install() + .expect("failed to install Prometheus recorder"); + + for name in COUNTERS { + register_counter(name) + } + + for name in GAUGES { + register_gauge(name) + } + + for name in HISTOGRAMS { + register_histogram(name) + } +} + +/// Registers a counter with the given name. +fn register_counter(metric: Metric) { + metrics::describe_counter!(metric.name, metric.description); + let _counter = metrics::counter!(metric.name); +} + +/// Registers a gauge with the given name. +fn register_gauge(metric: Metric) { + metrics::describe_gauge!(metric.name, metric.description); + let _gauge = ::metrics::gauge!(metric.name); +} + +/// Registers a histogram with the given name. +fn register_histogram(metric: Metric) { + metrics::describe_histogram!(metric.name, metric.description); + let _histogram = ::metrics::histogram!(metric.name); +} + +/// A Wrapper around a metric gauge +/// +/// Increment the given metric gauge on a new and decrement on drop +/// Useful in a closure (or an async closure) +pub struct GaugeWrapper { + gauge_name: &'static str, + gauge_app: &'static str, + gauge_label: &'static str, +} + +impl GaugeWrapper { + pub fn new(gauge_name: &'static str, gauge_app: &'static str, gauge_label: &'static str) -> Self { + gauge!(gauge_name, gauge_app => gauge_label).increment(1.0); + Self { + gauge_name, + gauge_app, + gauge_label, + } + } +} + +impl Drop for GaugeWrapper { + fn drop(&mut self) { + gauge!(self.gauge_name, self.gauge_app => self.gauge_label).decrement(1.0); + } +} \ No newline at end of file diff --git a/prover/src/proof_service.rs b/prover/src/proof_service.rs index 4efce44..f37dea4 100644 --- a/prover/src/proof_service.rs +++ b/prover/src/proof_service.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use ark_bn254::Fr; use ark_serialize::CanonicalSerialize; use async_channel::Receiver; +use metrics::histogram; use parking_lot::RwLock; use rln::hashers::hash_to_field; use rln::protocol::serialize_proof_values; @@ -15,6 +16,7 @@ use crate::proof_generation::{ProofGenerationData, ProofSendingData}; use crate::user_db::UserDb; use crate::user_db_types::RateLimit; use rln_proof::{RlnData, compute_rln_proof_and_values}; +use crate::metrics::PROOF_SERVICE_GEN_PROOF_TIME; const PROOF_SIZE: usize = 512; @@ -49,6 +51,7 @@ impl ProofService { } pub(crate) async fn serve(&self) -> Result<(), AppError> { + loop { let received = self.receiver.recv().await; @@ -66,6 +69,9 @@ impl ProofService { // Move to a task (as generating the proof can take quite some time) let blocking_task = tokio::task::spawn_blocking(move || { + + let proof_generation_start = std::time::Instant::now(); + let message_id = { let mut m_id = proof_generation_data.tx_counter; // Note: Zerokit can only recover user secret hash with 2 messages with the @@ -112,6 +118,9 @@ impl ProofService { .write_all(&serialize_proof_values(&proof_values)) .map_err(ProofGenerationError::SerializationWrite)?; + histogram!(PROOF_SERVICE_GEN_PROOF_TIME.name, "prover" => "proof service") + .record(proof_generation_start.elapsed().as_secs_f64()); + Ok::, ProofGenerationError>(output_buffer.into_inner()) }); diff --git a/prover_cli/src/main.rs b/prover_cli/src/main.rs index d98a125..2f49410 100644 --- a/prover_cli/src/main.rs +++ b/prover_cli/src/main.rs @@ -10,7 +10,7 @@ use tracing::{ }; use tracing_subscriber::{EnvFilter, fmt, layer::SubscriberExt, util::SubscriberInitExt}; // internal -use prover::{AppArgs, AppArgsConfig, run_prover}; +use prover::{AppArgs, AppArgsConfig, metrics::init_metrics, run_prover}; #[tokio::main] async fn main() -> Result<(), Box> { @@ -56,5 +56,7 @@ async fn main() -> Result<(), Box return Err("Please provide rpc url (--ws-rpc-url) or mock (--mock-sc)".into()); } + init_metrics(app_args.metrics_ip, &app_args.metrics_port); + run_prover(app_args).await } diff --git a/prover_client/Cargo.toml b/prover_client/Cargo.toml new file mode 100644 index 0000000..81f4a4f --- /dev/null +++ b/prover_client/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "prover_client" +version = "0.1.0" +edition = "2024" + +[dependencies] +# TODO: workspace? +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } +# TODO: workspace? +tonic = "0.13.1" +prost = "0.13" +alloy.workspace = true + +[build-dependencies] +tonic-build = "*" diff --git a/prover_client/build.rs b/prover_client/build.rs new file mode 100644 index 0000000..0271ab9 --- /dev/null +++ b/prover_client/build.rs @@ -0,0 +1,12 @@ +use std::env; +use std::path::PathBuf; + +fn main() -> Result<(), Box> { + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + tonic_build::configure() + .file_descriptor_set_path(out_dir.join("prover_descriptor.bin")) + .compile_protos(&["../proto/net/vac/prover/prover.proto"], &["../proto"]) + .unwrap(); + + Ok(()) +} diff --git a/prover_client/src/main.rs b/prover_client/src/main.rs new file mode 100644 index 0000000..2d0c4d0 --- /dev/null +++ b/prover_client/src/main.rs @@ -0,0 +1,40 @@ +use alloy::primitives::Address; +use std::str::FromStr; +use tonic::Response; + +pub mod prover_proto { + + // Include generated code (see build.rs) + tonic::include_proto!("prover"); +} + +use prover_proto::{ + Address as GrpcAddress, RegisterUserReply, RegisterUserRequest, + // RegistrationStatus, + rln_prover_client::RlnProverClient, +}; + +#[tokio::main] +async fn main() { + // FIXME: clap + let url = "http://127.0.0.1:42942"; + let addr = "0xb20a608c624Ca5003905aA834De7156C68b2E1d0"; + + let addr = Address::from_str(addr).unwrap(); + + let grpc_addr = GrpcAddress { + value: addr.to_vec(), + }; + + let mut client = RlnProverClient::connect(url).await.unwrap(); + let request_0 = RegisterUserRequest { + user: Some(grpc_addr), + }; + let request = tonic::Request::new(request_0); + let response: Response = client.register_user(request).await.unwrap(); + + println!( + "RegisterUSerReply status: {:?}", + response.into_inner().status + ); +}