From 990e8b97eea57599d2f93c5689141e5f5bdc6690 Mon Sep 17 00:00:00 2001 From: wozeparrot Date: Mon, 20 Oct 2025 18:30:34 -0700 Subject: [PATCH] feat: log openpilot 0.10.1 times (#12816) --- .github/workflows/benchmark.yml | 6 +++--- examples/openpilot/compile3.py | 8 ++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c563f79c62..39893f4388 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -626,11 +626,11 @@ jobs: - name: benchmark openpilot 0.9.9 dmonitoring run: BENCHMARK_LOG=openpilot_0_9_9_dmonitoring PYTHONPATH=. NOLOCALS=1 FLOAT16=1 IMAGE=2 QCOM=1 taskset -c 4-7 python3 test/external/external_benchmark_openpilot.py https://github.com/commaai/openpilot/raw/v0.9.9/selfdrive/modeld/models/dmonitoring_model.onnx - name: openpilot compile3 0.10.1 driving_vision - run: PYTHONPATH="." ASSERT_MIN_STEP_TIME=25 DEV=QCOM FLOAT16=1 IMAGE=2 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/cf6376aa9a090f0da26c280ef69eabf9bbdd51d1faac9ed392919c3db69be916 + run: BENCHMARK_LOG=openpilot_0_10_1_vision PYTHONPATH="." ASSERT_MIN_STEP_TIME=25 DEV=QCOM FLOAT16=1 IMAGE=2 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/cf6376aa9a090f0da26c280ef69eabf9bbdd51d1faac9ed392919c3db69be916 - name: openpilot compile3 0.10.1 driving_policy - run: PYTHONPATH="." ASSERT_MIN_STEP_TIME=7 DEV=QCOM FLOAT16=1 IMAGE=2 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/refs/heads/master/selfdrive/modeld/models/driving_policy.onnx + run: BENCHMARK_LOG=openpilot_0_10_1_policy PYTHONPATH="." ASSERT_MIN_STEP_TIME=7 DEV=QCOM FLOAT16=1 IMAGE=2 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/refs/heads/master/selfdrive/modeld/models/driving_policy.onnx - name: openpilot compile3 0.10.1 dmonitoring - run: PYTHONPATH="." ASSERT_MIN_STEP_TIME=12 DEV=QCOM FLOAT16=1 IMAGE=2 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/refs/heads/master/selfdrive/modeld/models/dmonitoring_model.onnx + run: BENCHMARK_LOG=openpilot_0_10_1_dmonitoring PYTHONPATH="." ASSERT_MIN_STEP_TIME=12 DEV=QCOM FLOAT16=1 IMAGE=2 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/refs/heads/master/selfdrive/modeld/models/dmonitoring_model.onnx - name: benchmark MobileNetV2 on DSP run: | # generate quantized weights diff --git a/examples/openpilot/compile3.py b/examples/openpilot/compile3.py index 02b8496b26..1c831aa48d 100644 --- a/examples/openpilot/compile3.py +++ b/examples/openpilot/compile3.py @@ -121,6 +121,12 @@ def test_vs_onnx(new_inputs, test_val, onnx_file, tol): print("test vs onnx passed") return timings +def bench(run, inputs): + from extra.bench_log import WallTimeEvent, BenchEvent + for _ in range(10): + with WallTimeEvent(BenchEvent.STEP): + run(**inputs).numpy() + if __name__ == "__main__": onnx_file = fetch(OPENPILOT_MODEL) inputs, outputs = compile(onnx_file) @@ -131,3 +137,5 @@ if __name__ == "__main__": if not getenv("FLOAT16"): test_vs_onnx(inputs, outputs, onnx_file, 1e-4) + if getenv("BENCHMARK_LOG", ""): + bench(pickle_loaded, inputs)