mirror of
https://github.com/powdr-labs/powdr.git
synced 2026-01-07 01:04:01 -05:00
Nightly: Run more benchmarks (#3286)
Changes: - For Keccak and Matmul, we only ran the benchmark with 0 and 100 APCs. I changed it to run with 0, 3, 10, 30. I think more does not make sense, because the effectiveness plots ([matmul](https://github.com/powdr-labs/bench-results/blob/gh-pages/results/2025-09-15-0630/matmul/100apc/effectiveness.png), [keccak](https://github.com/powdr-labs/bench-results/blob/gh-pages/results/2025-09-15-0630/keccak/100apc/effectiveness.png)) do not suggest that there are more basic blocks worth accelerating. - For reth, I also added more steps. - Named experiments more consistently (`apc000`, `apc003`, ...) - Added existing benchmarks for SHA, Pairing, and U256 - Increased the size of the U256 benchmark to get more meaningful results Manual nightly run: https://github.com/powdr-labs/powdr/actions/runs/17772555964 Which leads to the following bench results: https://github.com/powdr-labs/bench-results/tree/gh-pages/results/2025-09-16-2213 Note that the `test_apc` job duration goes from from 2:48h -> 4:34h.
This commit is contained in:
Georg Wiese
31
.github/workflows/nightly-tests.yml
vendored
31
.github/workflows/nightly-tests.yml
vendored
@@ -172,17 +172,37 @@ jobs:
|
||||
# prove with no APCs
|
||||
MODE="prove-stark" APC=0 ./run.sh || exit 1
|
||||
echo "Finished proving with no APCs"
|
||||
mv metrics.json $RES_DIR/noapc.json
|
||||
mv metrics.json $RES_DIR/apc000.json
|
||||
python ../openvm/scripts/plot_trace_cells.py -o $RES_DIR/trace_cells_apc000.png $RES_DIR/apc000.json > $RES_DIR/trace_cells_apc000.txt
|
||||
|
||||
# prove with 3 APCs
|
||||
MODE="prove-stark" APC=3 ./run.sh || exit 1
|
||||
echo "Finished proving with 3 APCs"
|
||||
mv metrics.json $RES_DIR/apc003.json
|
||||
python ../openvm/scripts/plot_trace_cells.py -o $RES_DIR/trace_cells_apc003.png $RES_DIR/apc003.json > $RES_DIR/trace_cells_apc003.txt
|
||||
|
||||
# prove with 10 APCs
|
||||
MODE="prove-stark" APC=10 ./run.sh || exit 1
|
||||
echo "Finished proving with 10 APCs"
|
||||
mv metrics.json $RES_DIR/apc010.json
|
||||
python ../openvm/scripts/plot_trace_cells.py -o $RES_DIR/trace_cells_apc010.png $RES_DIR/apc010.json > $RES_DIR/trace_cells_apc010.txt
|
||||
|
||||
# prove with 30 APCs
|
||||
MODE="prove-stark" APC=30 ./run.sh || exit 1
|
||||
echo "Finished proving with 30 APCs"
|
||||
mv metrics.json $RES_DIR/apc030.json
|
||||
python ../openvm/scripts/plot_trace_cells.py -o $RES_DIR/trace_cells_apc030.png $RES_DIR/apc030.json > $RES_DIR/trace_cells_apc030.txt
|
||||
|
||||
# prove with 100 APCs, recording mem usage
|
||||
MODE="prove-stark" APC=100 psrecord --include-children --interval 1 --log $RES_DIR/psrecord.csv --log-format csv --plot $RES_DIR/psrecord.png "./run.sh" || exit 1
|
||||
echo "Finished proving with 100 APCs"
|
||||
mv metrics.json $RES_DIR/100apc.json
|
||||
mv metrics.json $RES_DIR/apc100.json
|
||||
python ../openvm/scripts/plot_trace_cells.py -o $RES_DIR/trace_cells_apc100.png $RES_DIR/apc100.json > $RES_DIR/trace_cells_apc100.txt
|
||||
|
||||
# The APC candidates would be the same for all runs, so just keep the last one
|
||||
mv apcs/apc_candidates.json $RES_DIR/apc_candidates.json
|
||||
|
||||
python ../openvm/scripts/basic_metrics.py --csv $RES_DIR/noapc.json $RES_DIR/100apc.json > $RES_DIR/basic_metrics.csv
|
||||
python ../openvm/scripts/plot_trace_cells.py -o $RES_DIR/trace_cells.png $RES_DIR/100apc.json > $RES_DIR/trace_cells.txt
|
||||
python ../openvm/scripts/basic_metrics.py --csv $RES_DIR/apc000.json $RES_DIR/apc003.json $RES_DIR/apc010.json $RES_DIR/apc030.json $RES_DIR/apc100.json > $RES_DIR/basic_metrics.csv
|
||||
python ../autoprecompiles/scripts/plot_effectiveness.py $RES_DIR/apc_candidates.json --output $RES_DIR/effectiveness.png
|
||||
|
||||
mv $RES_DIR ../results/
|
||||
@@ -217,6 +237,9 @@ jobs:
|
||||
- name: Run tests (excluding larger ones)
|
||||
run: cargo test --release --verbose -p powdr-openvm -- --include-ignored --nocapture --test-threads=1 --skip _prove_large
|
||||
|
||||
- name: Run u256_prove_large
|
||||
run: cargo test --release --verbose -p powdr-openvm -- --include-ignored --nocapture --test-threads=1 u256_prove_large
|
||||
|
||||
- name: Run keccak_prove_large
|
||||
run: cargo test --release --verbose -p powdr-openvm -- --include-ignored --nocapture --test-threads=1 keccak_prove_large
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ use openvm_ruint::aliases::U256;
|
||||
|
||||
openvm::entry!(main);
|
||||
|
||||
const N: usize = 32;
|
||||
const N: usize = 70;
|
||||
type Matrix = [[U256; N]; N];
|
||||
|
||||
pub fn get_matrix(val: u32) -> Matrix {
|
||||
|
||||
@@ -5,7 +5,7 @@ use ruint::aliases::U256;
|
||||
|
||||
openvm::entry!(main);
|
||||
|
||||
const N: usize = 32;
|
||||
const N: usize = 70;
|
||||
type Matrix = [[U256; N]; N];
|
||||
|
||||
pub fn get_matrix(val: u32) -> Matrix {
|
||||
|
||||
@@ -50,8 +50,59 @@ mkdir -p "$dir"
|
||||
pushd "$dir"
|
||||
|
||||
run_bench guest-keccak-manual-precompile "$input" 0 manual
|
||||
run_bench guest-keccak "$input" 0 noapc
|
||||
run_bench guest-keccak "$input" 100 100apc
|
||||
run_bench guest-keccak "$input" 0 apc000
|
||||
run_bench guest-keccak "$input" 3 apc003
|
||||
run_bench guest-keccak "$input" 10 apc010
|
||||
run_bench guest-keccak "$input" 30 apc030
|
||||
|
||||
python3 $SCRIPTS_DIR/basic_metrics.py --csv **/metrics.json > basic_metrics.csv
|
||||
popd
|
||||
|
||||
### SHA256
|
||||
dir="results/sha256"
|
||||
input="30000"
|
||||
|
||||
mkdir -p "$dir"
|
||||
pushd "$dir"
|
||||
|
||||
run_bench guest-sha256-manual-precompile "$input" 0 manual
|
||||
run_bench guest-sha256 "$input" 0 apc000
|
||||
run_bench guest-sha256 "$input" 3 apc003
|
||||
run_bench guest-sha256 "$input" 10 apc010
|
||||
run_bench guest-sha256 "$input" 30 apc030
|
||||
|
||||
python3 $SCRIPTS_DIR/basic_metrics.py --csv **/metrics.json > basic_metrics.csv
|
||||
popd
|
||||
|
||||
### Pairing
|
||||
dir="results/pairing"
|
||||
input="0" # No input
|
||||
|
||||
mkdir -p "$dir"
|
||||
pushd "$dir"
|
||||
|
||||
run_bench guest-pairing-manual-precompile "$input" 0 manual
|
||||
run_bench guest-pairing "$input" 0 apc000
|
||||
run_bench guest-pairing "$input" 3 apc003
|
||||
run_bench guest-pairing "$input" 10 apc010
|
||||
run_bench guest-pairing "$input" 30 apc030
|
||||
run_bench guest-pairing "$input" 100 apc100
|
||||
|
||||
python3 $SCRIPTS_DIR/basic_metrics.py --csv **/metrics.json > basic_metrics.csv
|
||||
popd
|
||||
|
||||
### U256
|
||||
dir="results/u256"
|
||||
input="0" # No input
|
||||
|
||||
mkdir -p "$dir"
|
||||
pushd "$dir"
|
||||
|
||||
run_bench guest-u256-manual-precompile "$input" 0 manual
|
||||
run_bench guest-u256 "$input" 0 apc000
|
||||
run_bench guest-u256 "$input" 3 apc003
|
||||
run_bench guest-u256 "$input" 10 apc010
|
||||
run_bench guest-u256 "$input" 30 apc030
|
||||
|
||||
python3 $SCRIPTS_DIR/basic_metrics.py --csv **/metrics.json > basic_metrics.csv
|
||||
popd
|
||||
@@ -62,8 +113,10 @@ dir="results/matmul"
|
||||
mkdir -p "$dir"
|
||||
pushd "$dir"
|
||||
|
||||
run_bench guest-matmul 0 0 noapc
|
||||
run_bench guest-matmul 0 100 100apc
|
||||
run_bench guest-matmul 0 0 apc000
|
||||
run_bench guest-matmul 0 3 apc003
|
||||
run_bench guest-matmul 0 10 apc010
|
||||
run_bench guest-matmul 0 30 apc030
|
||||
|
||||
python3 "$SCRIPTS_DIR"/basic_metrics.py --csv **/metrics.json > basic_metrics.csv
|
||||
popd
|
||||
|
||||
@@ -1300,7 +1300,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
#[ignore = "Too much RAM"]
|
||||
fn u256_prove() {
|
||||
fn u256_prove_large() {
|
||||
use std::time::Instant;
|
||||
|
||||
let stdin = StdIn::default();
|
||||
|
||||
Reference in New Issue
Block a user