Run cargo bench in PR CI tests and report result to PR and Github Pages (#2198)

Run benchmarks in PRs, fail and warn on the PR if we got more than 20%
slower, add benchmark results to https://docs.powdr.org/dev/bench/

.github/workflows/deploy-book.yml

@@ -36,7 +36,8 @@ jobs:
         cd gh-pages
         # Delete the ref to avoid keeping history.
         git update-ref -d refs/heads/gh-pages
-        rm -rf *
+        # Delete everything except the `dev` folder, as it contains benchmarks we should keep.
+        find . -mindepth 1 -maxdepth 1 ! -name "dev" -exec rm -rf {} +
         mv ../book/* .
         git add .
         git commit -m "Deploy $GITHUB_SHA to gh-pages"

.github/workflows/nightly-tests.yml

@@ -86,5 +86,3 @@ jobs:
     - name: Run tests
       # Number threads is set to 1 because the runner does not have enough memeory for more.
       run: PILCOM=$(pwd)/pilcom/ cargo test --all --release --verbose --all-features -- --include-ignored --nocapture --test-threads=1
-    - name: Run benchmarks
-      run: cargo bench --workspace --all-features

.github/workflows/pr-tests.yml

@@ -237,3 +237,55 @@ jobs:
       env:
         PILCOM: ${{ github.workspace }}/pilcom/
         POWDR_STD: ${{ github.workspace }}/std/
+
+  bench:
+    needs: build
+    runs-on: warp-ubuntu-2404-x64-4x
+    permissions:
+      contents: write
+      deployments: write
+      pull-requests: write
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+    - name: ⚡ Restore rust cache
+      id: cache
+      uses: WarpBuilds/cache/restore@v1
+      with:
+        path: |
+          ~/.cargo/registry/index/
+          ~/.cargo/registry/cache/
+          ~/.cargo/git/db/
+          target/
+          Cargo.lock
+        key: ${{ runner.os }}-cargo-pr-tests
+    - name: Install Rust toolchain 1.81
+      run: rustup toolchain install 1.81-x86_64-unknown-linux-gnu
+    - name: Install nightly
+      run: rustup toolchain install nightly-2024-08-01-x86_64-unknown-linux-gnu
+    - name: Install std source
+      run: rustup component add rust-src --toolchain nightly-2024-08-01-x86_64-unknown-linux-gnu
+    - name: Install riscv target
+      run: rustup target add riscv32imac-unknown-none-elf --toolchain nightly-2024-08-01-x86_64-unknown-linux-gnu
+    - name: Install test dependencies
+      run: sudo apt-get update && sudo apt-get install -y binutils-riscv64-unknown-elf lld
+    - name: Install EStarkPolygon prover dependencies
+      run: sudo apt-get update && sudo apt-get install -y nlohmann-json3-dev libpqxx-dev nasm libgrpc++-dev libprotobuf-dev protobuf-compiler-grpc uuid-dev build-essential cmake pkg-config git
+    - name: Install pilcom
+      run: git clone https://github.com/0xPolygonHermez/pilcom.git  && cd pilcom && npm install
+    - name: Run benchmarks
+      # we add `|| exit 1` to make sure the step fails if `cargo bench` fails
+      run: cargo bench --workspace --all-features -- --output-format bencher | tee output.txt || exit 1
+    - name: Store benchmark result
+      uses: benchmark-action/github-action-benchmark@v1
+      with:
+        name: Benchmarks
+        tool: 'cargo'
+        output-file-path: output.txt
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        auto-push: true
+        alert-threshold: '120%'
+        comment-on-alert: true
+        summary-always: true

airgen/Cargo.toml

@@ -18,3 +18,6 @@ itertools = "0.13"
 
 [lints]
 workspace = true
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

analysis/Cargo.toml

@@ -28,3 +28,6 @@ development = ["env_logger"]
 
 [lints]
 workspace = true
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

asm-to-pil/Cargo.toml

@@ -22,4 +22,7 @@ powdr-analysis = { path = "../analysis" }
 powdr-importer = { path = "../importer" }
 
 [lints]
-workspace = true
+workspace = true
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

ast/Cargo.toml

@@ -28,3 +28,6 @@ powdr-parser.workspace = true
 
 [lints]
 workspace = true
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

backend-utils/Cargo.toml

@@ -14,3 +14,6 @@ powdr-number.workspace = true
 powdr-executor-utils.workspace = true
 log = "0.4.22"
 itertools = "0.13.0"
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

backend/Cargo.toml

@@ -89,3 +89,6 @@ development = ["env_logger"]
 
 [lints]
 workspace = true
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

cargo-powdr/Cargo.toml

@@ -13,3 +13,4 @@ clap = { version = "^4.3", features = ["derive"] }
 [[bin]]
 name = "cargo-powdr"
 path = "src/main.rs"
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

cli-rs/Cargo.toml

@@ -27,3 +27,4 @@ clap-markdown = "0.1.3"
 [[bin]]
 name = "powdr-rs"
 path = "src/main.rs"
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

cli/Cargo.toml

@@ -43,6 +43,7 @@ env_logger = "0.10.0"
 [[bin]]
 name = "powdr"
 path = "src/main.rs"
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458
 
 [lints.clippy]
 uninlined_format_args = "deny"

executor-utils/Cargo.toml

@@ -11,3 +11,6 @@ powdr-number.workspace = true
 powdr-ast.workspace = true
 
 serde = { version = "1.0", default-features = false, features = ["alloc", "derive", "rc"] }
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

executor/Cargo.toml

@@ -35,3 +35,6 @@ development = ["env_logger"]
 
 [lints]
 workspace = true
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

importer/Cargo.toml

@@ -17,3 +17,6 @@ pretty_assertions = "1.4.0"
 
 [lints]
 workspace = true
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

isa-utils/Cargo.toml

@@ -6,3 +6,6 @@ edition = { workspace = true }
 license = { workspace = true }
 homepage = { workspace = true }
 repository = { workspace = true }
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

jit-compiler/Cargo.toml

@@ -25,3 +25,6 @@ test-log = "0.2.12"
 
 [lints]
 workspace = true
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

linker/Cargo.toml

@@ -26,3 +26,6 @@ powdr-parser.workspace = true
 
 [lints]
 workspace = true
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

number/Cargo.toml

@@ -40,3 +40,6 @@ development = ["env_logger"]
 
 [lints]
 workspace = true
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

parser-util/Cargo.toml

@@ -25,3 +25,6 @@ development = ["env_logger"]
 
 [lints]
 workspace = true
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

parser/Cargo.toml

@@ -33,3 +33,6 @@ development = ["env_logger"]
 
 [lints]
 workspace = true
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

pil-analyzer/Cargo.toml

@@ -27,3 +27,6 @@ development = ["env_logger"]
 
 [lints]
 workspace = true
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

pilopt/Cargo.toml

@@ -20,3 +20,6 @@ powdr-pil-analyzer.workspace = true
 
 [lints]
 workspace = true
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

pipeline/Cargo.toml

@@ -64,3 +64,6 @@ harness = false
 
 [lints]
 workspace = true
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

pipeline/benches/evaluator_benchmark.rs

@@ -167,6 +167,7 @@ fn jit_benchmark(c: &mut Criterion) {
             });
         });
     }
+
     group.finish();
 }
 

plonky3/Cargo.toml

@@ -6,12 +6,10 @@ license.workspace = true
 homepage.workspace = true
 repository.workspace = true
 
-[lib]
-name = "powdr_plonky3"
-
 [[bin]]
 name = "gen_poseidon_consts"
 path = "src/bin/gen_poseidon_consts.rs"
+bench = false                           # See https://github.com/bheisler/criterion.rs/issues/458
 
 [features]
 default = []
@@ -74,6 +72,7 @@ powdr-riscv-runtime = { path = "../riscv-runtime", features = [
   "getrandom",
   "allow_fake_rand",
 ] }
-indexmap = { version = "1.9.3", features = [
-  "std",
-] }
+indexmap = { version = "1.9.3", features = ["std"] }
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

powdr-test/Cargo.toml

@@ -15,3 +15,6 @@ env_logger = "0.10.2"
 
 [lints]
 workspace = true
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

powdr/Cargo.toml

@@ -43,3 +43,6 @@ estark-starky-simd = ["powdr-backend/estark-starky-simd", "powdr-pipeline/estark
 
 [lints.clippy]
 uninlined_format_args = "deny"
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

riscv-executor/Cargo.toml

@@ -28,3 +28,6 @@ inferno = "0.11.19"
 
 [lints.clippy]
 uninlined_format_args = "deny"
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

riscv-syscalls/Cargo.toml

@@ -11,3 +11,6 @@ repository = { workspace = true }
 
 [lints]
 workspace = true
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

riscv/Cargo.toml

@@ -67,3 +67,6 @@ harness = false
 
 [lints]
 workspace = true
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458

schemas/Cargo.toml

@@ -12,13 +12,21 @@ repository.workspace = true
 powdr-ast.workspace = true
 powdr-number.workspace = true
 
-serde = { version = "1.0", default-features = false, features = ["alloc", "derive", "rc"] }
-schemars = { version = "0.8.16", features = ["preserve_order"]}
+serde = { version = "1.0", default-features = false, features = [
+    "alloc",
+    "derive",
+    "rc",
+] }
+schemars = { version = "0.8.16", features = ["preserve_order"] }
 serde_cbor = "0.11.2"
 
 [[bin]]
 name = "powdr-schemas"
 path = "bin/main.rs"
+bench = false          # See https://github.com/bheisler/criterion.rs/issues/458
 
 [lints.clippy]
 uninlined_format_args = "deny"
+
+[lib]
+bench = false # See https://github.com/bheisler/criterion.rs/issues/458