From 2d399d5e245f9375fe31fb69d46bb0a446d2a093 Mon Sep 17 00:00:00 2001 From: Hendrik Eeckhaut Date: Wed, 23 Jul 2025 10:58:46 +0200 Subject: [PATCH] chore: Update latency/bandwidth plots for new harness (#923) * Updated latency/bandwidth plots for new harness * Fix harness Docker build --- .github/workflows/bench.yml | 12 +- Cargo.lock | 432 ++++++++++++++++++++++++++++++ Cargo.toml | 1 + crates/harness/.gitignore | 2 +- crates/harness/build.sh | 3 +- crates/harness/docker.md | 5 +- crates/harness/harness.Dockerfile | 2 +- crates/harness/plot/Cargo.toml | 19 ++ crates/harness/plot/bin/plot.rs | 277 +++++++++++++++++++ 9 files changed, 744 insertions(+), 9 deletions(-) create mode 100644 crates/harness/plot/Cargo.toml create mode 100644 crates/harness/plot/bin/plot.rs diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 0793872b3..b2e44d22c 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -25,13 +25,17 @@ jobs: - name: Run Benchmarks run: | - docker run -it --privileged -v ./crates/harness/:/benches tlsn-bench bash -c "runner setup; runner --target ${{ github.event.inputs.bench_type }} test" + docker run --privileged -v ./crates/harness/:/benches tlsn-bench bash -c "runner setup; runner --target ${{ github.event.inputs.bench_type }} bench" + - name: Plot Benchmarks + run: | + docker run -v ./crates/harness/:/benches tlsn-bench bash -c "tlsn-harness-plot /benches/bench.toml /benches/metrics.csv --min-max-band --prover-kind ${{ github.event.inputs.bench_type }}" - name: Upload graphs uses: actions/upload-artifact@v4 with: name: benchmark_graphs path: | - ./crates/benches/binary/runtime_vs_latency.html - ./crates/benches/binary/runtime_vs_bandwidth.html - ./crates/benches/binary/download_size_vs_memory.html \ No newline at end of file + ./crates/harness/metrics.csv + ./crates/harness/bench.toml + ./crates/harness/runtime_vs_latency.html + ./crates/harness/runtime_vs_bandwidth.html \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index b99ed5085..29d0a1337 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1198,6 +1198,12 @@ dependencies = [ "url", ] +[[package]] +name = "az" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b7e4c2464d97fe331d41de9d5db0def0a96f4d823b8b32a2efd503578988973" + [[package]] name = "backtrace" version = "0.3.75" @@ -1231,6 +1237,16 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.8.0" @@ -1311,6 +1327,26 @@ dependencies = [ "which 4.4.2", ] +[[package]] +name = "bindgen" +version = "0.71.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" +dependencies = [ + "bitflags 2.9.1", + "cexpr", + "clang-sys", + "itertools 0.12.1", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 2.1.1", + "shlex", + "syn 2.0.104", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -1496,6 +1532,26 @@ dependencies = [ "serde", ] +[[package]] +name = "capacity_builder" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f2d24a6dcf0cd402a21b65d35340f3a49ff3475dc5fdac91d22d2733e6641c6" +dependencies = [ + "capacity_builder_macros", + "itoa", +] + +[[package]] +name = "capacity_builder_macros" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b4a6cae9efc04cc6cbb8faf338d2c497c165c83e74509cf4dbedea948bbf6e5" +dependencies = [ + "quote", + "syn 2.0.104", +] + [[package]] name = "cargo-emit" version = "0.2.1" @@ -1534,6 +1590,20 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" +[[package]] +name = "charming" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ffae2e616ae7d66b2e9ea369f1c7650042bdcdc1dc08b04b027107007b4f09" +dependencies = [ + "deno_core", + "handlebars", + "serde", + "serde_json", + "serde_v8", + "serde_with", +] + [[package]] name = "chromiumoxide" version = "0.7.0" @@ -1875,6 +1945,12 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "cooked-waker" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147be55d677052dabc6b22252d5dd0fd4c29c8c27aa4f2fbef0f94aa003b406f" + [[package]] name = "cookie" version = "0.18.1" @@ -1925,6 +2001,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + [[package]] name = "criterion" version = "0.5.1" @@ -2177,6 +2262,126 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "serde", + "uuid", +] + +[[package]] +name = "deno_core" +version = "0.348.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7902cd1dde9e8ffc953428933f37f9b943dc123265709220cded6277946cadf7" +dependencies = [ + "anyhow", + "az", + "bincode", + "bit-set", + "bit-vec", + "bytes", + "capacity_builder", + "cooked-waker", + "deno_core_icudata", + "deno_error", + "deno_ops", + "deno_path_util", + "deno_unsync", + "futures", + "indexmap 2.10.0", + "libc", + "parking_lot", + "percent-encoding", + "pin-project 1.1.10", + "serde", + "serde_json", + "serde_v8", + "smallvec", + "sourcemap", + "static_assertions", + "thiserror 2.0.12", + "tokio", + "url", + "v8", + "wasm_dep_analyzer", +] + +[[package]] +name = "deno_core_icudata" +version = "0.74.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe4dccb6147bb3f3ba0c7a48e993bfeb999d2c2e47a81badee80e2b370c8d695" + +[[package]] +name = "deno_error" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e983933fb4958fbe1e0a63c1e89a2af72b12c409e86404e547955564e6e217b8" +dependencies = [ + "deno_error_macro", + "libc", + "serde", + "serde_json", + "tokio", + "url", +] + +[[package]] +name = "deno_error_macro" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ad5ae3ef15db33e917d6ed54b53d0a98d068c4d1217eb35a4997423203c3ef" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + +[[package]] +name = "deno_ops" +version = "0.224.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55db7994eccbdad457ca5091c545af712234c8f6e4389cadb2a24d2ecbbe7e89" +dependencies = [ + "indexmap 2.10.0", + "proc-macro-rules", + "proc-macro2", + "quote", + "stringcase", + "strum", + "strum_macros", + "syn 2.0.104", + "thiserror 2.0.12", +] + +[[package]] +name = "deno_path_util" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8850326ea9cb786aafd938f3de9866432904c0bae3aa0139a7a4e570b0174f6" +dependencies = [ + "deno_error", + "percent-encoding", + "sys_traits", + "thiserror 2.0.12", + "url", +] + +[[package]] +name = "deno_unsync" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6742a724e8becb372a74c650a1aefb8924a5b8107f7d75b3848763ea24b27a87" +dependencies = [ + "futures-util", + "parking_lot", + "tokio", +] + [[package]] name = "der" version = "0.7.10" @@ -2227,6 +2432,15 @@ dependencies = [ "derive_builder_macro 0.12.0", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro 0.20.2", +] + [[package]] name = "derive_builder_core" version = "0.11.2" @@ -2251,6 +2465,18 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "derive_builder_macro" version = "0.11.2" @@ -2271,6 +2497,16 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core 0.20.2", + "syn 2.0.104", +] + [[package]] name = "derive_more" version = "1.0.0" @@ -2632,6 +2868,16 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "fslock" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04412b8935272e3a9bae6f48c7bfff74c2911f60525404edfdd28e49884c3bfb" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "funty" version = "2.0.0" @@ -2884,6 +3130,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "gzip-header" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95cc527b92e6029a62960ad99aa8a6660faa4555fe5f731aab13aa6a921795a2" +dependencies = [ + "crc32fast", +] + [[package]] name = "h2" version = "0.4.11" @@ -2913,6 +3168,22 @@ dependencies = [ "crunchy", ] +[[package]] +name = "handlebars" +version = "6.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759e2d5aea3287cb1190c8ec394f42866cb5bf74fcbf213f354e3c856ea26098" +dependencies = [ + "derive_builder 0.20.2", + "log", + "num-order", + "pest", + "pest_derive", + "serde", + "serde_json", + "thiserror 2.0.12", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -3306,6 +3577,12 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "if_chain" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb56e1aa765b4b4f3aadfab769793b7087bb03a4ea4920644a6d238e2df5b9ed" + [[package]] name = "impl-codec" version = "0.6.0" @@ -4451,6 +4728,7 @@ checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", + "rand 0.8.5", ] [[package]] @@ -4468,6 +4746,21 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-modular" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17bb261bf36fa7d83f4c294f834e91256769097b3cb505d44831e0a179ac647f" + +[[package]] +name = "num-order" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "537b596b97c40fcf8056d153049eb22f481c17ebce72a513ec9286e4986d1bb6" +dependencies = [ + "num-modular", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -4608,6 +4901,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "overload" version = "0.1.1" @@ -5053,6 +5352,29 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "proc-macro-rules" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07c277e4e643ef00c1233393c673f655e3672cf7eb3ba08a00bdd0ea59139b5f" +dependencies = [ + "proc-macro-rules-macros", + "proc-macro2", + "syn 2.0.104", +] + +[[package]] +name = "proc-macro-rules-macros" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "207fffb0fe655d1d47f6af98cc2793405e85929bdbc420d685554ff07be27ac7" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "proc-macro2" version = "1.0.95" @@ -5843,6 +6165,7 @@ version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" dependencies = [ + "indexmap 2.10.0", "itoa", "memchr", "ryu", @@ -5880,6 +6203,20 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_v8" +version = "0.257.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c57c63e9203f1d1383df0943e080955999e984e96ecc9557c3bae48e483edfa" +dependencies = [ + "deno_error", + "num-bigint", + "serde", + "smallvec", + "thiserror 2.0.12", + "v8", +] + [[package]] name = "serde_with" version = "3.14.0" @@ -6104,6 +6441,24 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "sourcemap" +version = "9.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22afbcb92ce02d23815b9795523c005cb9d3c214f8b7a66318541c240ea7935" +dependencies = [ + "base64-simd", + "bitvec", + "data-encoding", + "debugid", + "if_chain", + "rustc-hash 2.1.1", + "serde", + "serde_json", + "unicode-id-start", + "url", +] + [[package]] name = "spansy" version = "0.1.0" @@ -6146,6 +6501,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "stringcase" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72abeda133c49d7bddece6c154728f83eec8172380c80ab7096da9487e20d27c" + [[package]] name = "strsim" version = "0.8.0" @@ -6270,6 +6631,26 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "sys_traits" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc4707edf3196e8037ee45018d1bb1bfb233b0e4fc440fa3d3f25bc69bfdaf26" +dependencies = [ + "sys_traits_macros", +] + +[[package]] +name = "sys_traits_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "181f22127402abcf8ee5c83ccd5b408933fec36a6095cf82cda545634692657e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "tap" version = "1.0.1" @@ -6669,6 +7050,18 @@ dependencies = [ "ws_stream_wasm", ] +[[package]] +name = "tlsn-harness-plot" +version = "0.0.0" +dependencies = [ + "charming", + "clap 4.5.40", + "csv", + "itertools 0.14.0", + "tlsn-harness-core", + "toml", +] + [[package]] name = "tlsn-harness-runner" version = "0.1.0" @@ -7371,6 +7764,12 @@ version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" +[[package]] +name = "unicode-id-start" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f322b60f6b9736017344fa0635d64be2f458fbc04eef65f6be22976dd1ffd5b" + [[package]] name = "unicode-ident" version = "1.0.18" @@ -7442,6 +7841,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -7474,6 +7874,22 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "v8" +version = "137.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b387c1c5731284e756c03280032068e68e5b52f6c4714492403c30f650ad52" +dependencies = [ + "bindgen 0.71.1", + "bitflags 2.9.1", + "fslock", + "gzip-header", + "home", + "miniz_oxide", + "paste", + "which 6.0.3", +] + [[package]] name = "valuable" version = "0.1.1" @@ -7504,6 +7920,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "wait-timeout" version = "0.2.1" @@ -7618,6 +8040,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm_dep_analyzer" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eeee3bdea6257cc36d756fa745a70f9d393571e47d69e0ed97581676a5369ca" +dependencies = [ + "deno_error", + "thiserror 2.0.12", +] + [[package]] name = "web-spawn" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 221befbe1..15618c900 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ members = [ "crates/harness/core", "crates/harness/executor", "crates/harness/runner", + "crates/harness/plot", "crates/tlsn", ] resolver = "2" diff --git a/crates/harness/.gitignore b/crates/harness/.gitignore index 13b1b6709..375a00c13 100644 --- a/crates/harness/.gitignore +++ b/crates/harness/.gitignore @@ -1,4 +1,4 @@ *.svg *.html -bin/ +/bin/ diff --git a/crates/harness/build.sh b/crates/harness/build.sh index 590993b54..7cbd48c6c 100755 --- a/crates/harness/build.sh +++ b/crates/harness/build.sh @@ -3,7 +3,7 @@ # Ensure the script runs in the folder that contains this script cd "$(dirname "$0")" -cargo build --release --package tlsn-harness-runner --package tlsn-harness-executor --package tlsn-server-fixture +cargo build --release --package tlsn-harness-runner --package tlsn-harness-executor --package tlsn-server-fixture --package tlsn-harness-plot mkdir -p bin @@ -11,5 +11,6 @@ cp ../../target/release/tlsn-harness-runner bin/runner cp ../../target/release/tlsn-harness-executor-native bin/executor-native cp ../../target/release/tlsn-server-fixture bin/server-fixture cp ../../target/release/tlsn-harness-wasm-server bin/wasm-server +cp ../../target/release/tlsn-harness-plot bin/tlsn-harness-plot ./build.wasm.sh diff --git a/crates/harness/docker.md b/crates/harness/docker.md index 8f32c44bb..c4d5bbbe6 100644 --- a/crates/harness/docker.md +++ b/crates/harness/docker.md @@ -7,11 +7,12 @@ docker build --pull -t tlsn-bench . -f ./crates/harness/harness.Dockerfile Next run the benches with: ``` -docker run -it --privileged -v ./crates/harness/:/benches tlsn-bench bash -c "runner setup; runner test" +docker run -it --privileged -v ./crates/harness/:/benches tlsn-bench bash -c "runner setup; runner bench" ``` The `--privileged` parameter is required because this test bench needs permission to create networks with certain parameters To run the benches in a browser run: ``` -+docker run -it --privileged -v ./crates/harness/:/benches tlsn-bench bash -c "cd /; runner setup; runner --target browser test" +docker run -it --privileged -v ./crates/harness/:/benches tlsn-bench bash -c "cd /; runner setup; runner --target browser bench" ``` + diff --git a/crates/harness/harness.Dockerfile b/crates/harness/harness.Dockerfile index 897a43d9b..ac0af606b 100644 --- a/crates/harness/harness.Dockerfile +++ b/crates/harness/harness.Dockerfile @@ -6,7 +6,7 @@ RUN \ apt update && apt install -y clang; \ rustup install nightly; \ rustup component add rust-src --toolchain nightly; \ - cargo install wasm-pack; + cargo install --git https://github.com/rustwasm/wasm-pack.git --rev 32e52ca; COPY . . RUN \ cd crates/harness; \ diff --git a/crates/harness/plot/Cargo.toml b/crates/harness/plot/Cargo.toml new file mode 100644 index 000000000..fd8de3a82 --- /dev/null +++ b/crates/harness/plot/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "tlsn-harness-plot" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] +tlsn-harness-core = { workspace = true } +# tlsn-server-fixture = { workspace = true } +charming = { version = "0.5.1", features = ["ssr"] } +csv = "1.3.0" +clap = { workspace = true, features = ["derive", "env"] } +itertools = "0.14.0" +toml = { workspace = true } + + +[[bin]] +name = "tlsn-harness-plot" +path = "bin/plot.rs" diff --git a/crates/harness/plot/bin/plot.rs b/crates/harness/plot/bin/plot.rs new file mode 100644 index 000000000..0a107aba1 --- /dev/null +++ b/crates/harness/plot/bin/plot.rs @@ -0,0 +1,277 @@ +use std::f32; + +use charming::{ + Chart, HtmlRenderer, + component::{Axis, Legend, Title}, + element::{AreaStyle, LineStyle, NameLocation, Orient, TextStyle, Tooltip, Trigger}, + series::Line, + theme::Theme, +}; +use clap::Parser; +use harness_core::bench::{BenchItems, Measurement}; +use itertools::Itertools; + +const THEME: Theme = Theme::Default; + +#[derive(Parser, Debug)] +#[command(author, version, about)] +struct Cli { + /// Path to the Bench.toml file with benchmark spec + toml: String, + + /// Path to the CSV file with benchmark results + csv: String, + + /// Prover kind: native or browser + #[arg(short, long, value_enum, default_value = "native")] + prover_kind: ProverKind, + + /// Add min/max bands to plots + #[arg(long, default_value_t = false)] + min_max_band: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] +enum ProverKind { + Native, + Browser, +} + +impl std::fmt::Display for ProverKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ProverKind::Native => write!(f, "Native"), + ProverKind::Browser => write!(f, "Browser"), + } + } +} + +fn main() -> Result<(), Box> { + let cli = Cli::parse(); + + let mut rdr = csv::Reader::from_path(&cli.csv)?; + + let items: BenchItems = toml::from_str(&std::fs::read_to_string(&cli.toml)?)?; + let groups = items.group; + + // Prepare data for plotting. + let all_data: Vec = rdr + .deserialize::() + .collect::, _>>()?; + + for group in groups { + if group.protocol_latency.is_some() { + let latency = group.protocol_latency.unwrap(); + plot_runtime_vs( + &all_data, + cli.min_max_band, + &group.name, + |r| r.bandwidth as f32 / 1000.0, // Kbps to Mbps + "Runtime vs Bandwidth", + format!("{} ms Latency, {} mode", latency, cli.prover_kind), + "runtime_vs_bandwidth.html", + "Bandwidth (Mbps)", + )?; + } + + if group.bandwidth.is_some() { + let bandwidth = group.bandwidth.unwrap(); + plot_runtime_vs( + &all_data, + cli.min_max_band, + &group.name, + |r| r.latency as f32, + "Runtime vs Latency", + format!("{} bps bandwidth, {} mode", bandwidth, cli.prover_kind), + "runtime_vs_latency.html", + "Latency (ms)", + )?; + } + } + + Ok(()) +} + +struct DataPoint { + min: f32, + mean: f32, + max: f32, +} + +struct Points { + preprocess: DataPoint, + online: DataPoint, + total: DataPoint, +} + +#[allow(clippy::too_many_arguments)] +fn plot_runtime_vs( + all_data: &[Measurement], + show_min_max: bool, + group: &str, + x_value: Fx, + title: &str, + subtitle: String, + output_file: &str, + x_axis_label: &str, +) -> Result> +where + Fx: Fn(&Measurement) -> f32, +{ + fn data_point(values: &[f32]) -> DataPoint { + let mean = values.iter().copied().sum::() / values.len() as f32; + let max = values.iter().copied().reduce(f32::max).unwrap_or_default(); + let min = values.iter().copied().reduce(f32::min).unwrap_or_default(); + DataPoint { min, mean, max } + } + + let stats: Vec<(f32, Points)> = all_data + .iter() + .filter(|r| r.group.as_deref() == Some(group)) + .map(|r| { + ( + x_value(r), + r.time_preprocess as f32 / 1000.0, // ms to s + r.time_online as f32 / 1000.0, + r.time_total as f32 / 1000.0, + ) + }) + .sorted_by(|a, b| a.0.partial_cmp(&b.0).unwrap()) + .chunk_by(|entry| entry.0) + .into_iter() + .map(|(x, group)| { + let group_vec: Vec<_> = group.collect(); + let preprocess = data_point( + &group_vec + .iter() + .map(|(_, t, _, _)| *t) + .collect::>(), + ); + let online = data_point( + &group_vec + .iter() + .map(|(_, _, t, _)| *t) + .collect::>(), + ); + let total = data_point( + &group_vec + .iter() + .map(|(_, _, _, t)| *t) + .collect::>(), + ); + ( + x, + Points { + preprocess, + online, + total, + }, + ) + }) + .collect(); + + let mut chart = Chart::new() + .title( + Title::new() + .text(title) + .left("center") + .subtext(subtitle) + .subtext_style(TextStyle::new().font_size(16)), + ) + .tooltip(Tooltip::new().trigger(Trigger::Axis)) + .legend( + Legend::new() + .data(vec!["Preprocess Mean", "Online Mean", "Total Mean"]) + .top("80") + .right("110") + .orient(Orient::Vertical) + .item_gap(10), + ) + .x_axis( + Axis::new() + .name(x_axis_label) + .scale(true) + .name_location(NameLocation::Middle) + .name_gap(30) + .name_text_style(TextStyle::new().font_size(21)), + ) + .y_axis( + Axis::new() + .name("Time (seconds)") + .scale(true) + .name_location(NameLocation::Middle) + .name_rotation(90) + .name_gap(30) + .name_text_style(TextStyle::new().font_size(21)), + ); + + chart = add_mean_series(chart, &stats, "Preprocess Mean", |p| p.preprocess.mean); + chart = add_mean_series(chart, &stats, "Online Mean", |p| p.online.mean); + chart = add_mean_series(chart, &stats, "Total Mean", |p| p.total.mean); + + if show_min_max { + chart = add_min_max_band( + chart, + &stats, + "Preprocess Min/Max", + |p| &p.preprocess, + "#ccc", + ); + chart = add_min_max_band(chart, &stats, "Online Min/Max", |p| &p.online, "#ccc"); + chart = add_min_max_band(chart, &stats, "Total Min/Max", |p| &p.total, "#ccc"); + } + // Save the chart as HTML file. + HtmlRenderer::new(title, 1000, 800) + .theme(THEME) + .save(&chart, output_file) + .unwrap(); + + Ok(chart) +} + +fn add_mean_series( + chart: Chart, + stats: &[(f32, Points)], + name: &str, + extract: impl Fn(&Points) -> f32, +) -> Chart { + chart.series( + Line::new() + .name(name) + .data( + stats + .iter() + .map(|(x, points)| vec![*x, extract(points)]) + .collect(), + ) + .symbol_size(6), + ) +} + +fn add_min_max_band( + chart: Chart, + stats: &[(f32, Points)], + name: &str, + extract: impl Fn(&Points) -> &DataPoint, + color: &str, +) -> Chart { + chart.series( + Line::new() + .name(name) + .data( + stats + .iter() + .map(|(x, points)| vec![*x, extract(points).max]) + .chain( + stats + .iter() + .rev() + .map(|(x, points)| vec![*x, extract(points).min]), + ) + .collect(), + ) + .show_symbol(false) + .line_style(LineStyle::new().opacity(0.0)) + .area_style(AreaStyle::new().opacity(0.3).color(color)), + ) +}