refactor: split benchmarks into ml benchmarks and concrete benchmarks

.github/workflows/daily-benchmarks.yaml

@@ -47,18 +47,18 @@ jobs:
           host: ${{ steps.amd-public-ip.outputs.value }}
           username: ${{ secrets.BENCHMARKS_EC2_USERNAME }}
           key: ${{ secrets.BENCHMARKS_EC2_SSH_KEY }}
-          command_timeout: 240m
+          command_timeout: 720m
           script: |
-            cd ~/concrete-framework-internal
+            cd ~/project
             git pull
             make docker_publish_measurements
             docker system prune -f
 
-      - name: Copy AMD EC2 Instance Logs
-        run: scp -o StrictHostKeyChecking=no -i ~/ssh-key ${{ secrets.BENCHMARKS_EC2_USERNAME }}@${{ steps.amd-public-ip.outputs.value }}:~/concrete-framework-internal/logs/latest.log ~/latest.log
+      - name: Copy AMD EC2 Instance Concrete Logs
+        run: scp -o StrictHostKeyChecking=no -i ~/ssh-key ${{ secrets.BENCHMARKS_EC2_USERNAME }}@${{ steps.amd-public-ip.outputs.value }}:~/project/logs/latest.concrete.log ~/latest.concrete.log
 
-      - name: Copy AMD EC2 Instance Logs
-        run: scp -o StrictHostKeyChecking=no -i ~/ssh-key ${{ secrets.BENCHMARKS_EC2_USERNAME }}@${{ steps.amd-public-ip.outputs.value }}:~/concrete-framework-internal/.benchmarks/findings.json ~/findings.json
+      - name: Copy AMD EC2 Instance ML Logs
+        run: scp -o StrictHostKeyChecking=no -i ~/ssh-key ${{ secrets.BENCHMARKS_EC2_USERNAME }}@${{ steps.amd-public-ip.outputs.value }}:~/project/logs/latest.ml.log ~/latest.ml.log
 
       - name: Stop AMD EC2 Instance
         if: ${{ always() }}
@@ -68,14 +68,8 @@ jobs:
       - name: Upload Logs of AMD EC2 Instance
         uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2
         with:
-          name: amd-logs
-          path: ~/latest.log
-
-      - name: Upload Findings of AMD EC2 Instance
-        uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2
-        with:
-          name: amd-findings
-          path: ~/findings.json
+          name: amd
+          path: ~/latest.*.log
 
       - name: Start Intel EC2 Instance
         run: |
@@ -102,18 +96,18 @@ jobs:
           host: ${{ steps.intel-public-ip.outputs.value }}
           username: ${{ secrets.BENCHMARKS_EC2_USERNAME }}
           key: ${{ secrets.BENCHMARKS_EC2_SSH_KEY }}
-          command_timeout: 240m
+          command_timeout: 720m
           script: |
-            cd ~/concrete-framework-internal
+            cd ~/project
             git pull
             make docker_publish_measurements
             docker system prune -f
 
-      - name: Copy Intel EC2 Instance Logs
-        run: scp -o StrictHostKeyChecking=no -i ~/ssh-key ${{ secrets.BENCHMARKS_EC2_USERNAME }}@${{ steps.intel-public-ip.outputs.value }}:~/concrete-framework-internal/logs/latest.log ~/latest.log
+      - name: Copy Intel EC2 Instance Concrete Logs
+        run: scp -o StrictHostKeyChecking=no -i ~/ssh-key ${{ secrets.BENCHMARKS_EC2_USERNAME }}@${{ steps.intel-public-ip.outputs.value }}:~/project/logs/latest.concrete.log ~/latest.concrete.log
 
-      - name: Copy Intel EC2 Instance Findings
-        run: scp -o StrictHostKeyChecking=no -i ~/ssh-key ${{ secrets.BENCHMARKS_EC2_USERNAME }}@${{ steps.intel-public-ip.outputs.value }}:~/concrete-framework-internal/.benchmarks/findings.json ~/findings.json
+      - name: Copy Intel EC2 Instance ML Logs
+        run: scp -o StrictHostKeyChecking=no -i ~/ssh-key ${{ secrets.BENCHMARKS_EC2_USERNAME }}@${{ steps.intel-public-ip.outputs.value }}:~/project/logs/latest.ml.log ~/latest.ml.log
 
       - name: Stop Intel EC2 Instance
         if: ${{ always() }}
@@ -123,14 +117,8 @@ jobs:
       - name: Upload Logs of Intel EC2 Instance
         uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2
         with:
-          name: intel-logs
-          path: ~/latest.log
-
-      - name: Upload Findings of Intel EC2 Instance
-        uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2
-        with:
-          name: intel-findings
-          path: ~/findings.json
+          name: intel
+          path: ~/latest.*.log
 
       - name: Send Slack Notification
         if: ${{ always() }}

Makefile

@@ -60,7 +60,7 @@ pylint_tests:
 pylint_benchmarks:
 	@# Disable duplicate code detection, docstring requirement, too many locals/statements
 	find ./benchmarks/ -type f -name "*.py" | xargs poetry run pylint \
-	--disable=R0801,R0914,R0915,C0103,C0114,C0115,C0116 --rcfile=pylintrc
+	--disable=R0801,R0914,R0915,C0103,C0114,C0115,C0116,W0108 --rcfile=pylintrc
 
 .PHONY: pylint_script # Run pylint on scripts
 pylint_script:
@@ -115,9 +115,16 @@ mypy_ns:
 mypy_test:
 	find ./tests/ -name "*.py" | xargs poetry run mypy --ignore-missing-imports
 
+.PHONY: mypy_concrete_benchmark # Run mypy on concrete benchmark files
+mypy_concrete_benchmark:
+	find ./benchmarks/concrete/ -name "*.py" | xargs poetry run mypy --ignore-missing-imports
+
+.PHONY: mypy_ml_benchmark # Run mypy on ml benchmark files
+mypy_ml_benchmark:
+	find ./benchmarks/ml/ -name "*.py" | xargs poetry run mypy --ignore-missing-imports
+
 .PHONY: mypy_benchmark # Run mypy on benchmark files
-mypy_benchmark:
-	find ./benchmarks/ -name "*.py" | xargs poetry run mypy --ignore-missing-imports
+mypy_benchmark: mypy_concrete_benchmark mypy_ml_benchmark
 
 .PHONY: mypy_script # Run mypy on scripts
 mypy_script:
@@ -212,10 +219,17 @@ finalize_nb:
 pytest_nb:
 	find docs -name "*.ipynb" | grep -v _build | grep -v .ipynb_checkpoints | xargs poetry run pytest -Wignore --nbmake
 
-.PHONY: benchmark # Launch benchmark
-benchmark:
+.PHONY: concrete_benchmark # Launch concrete benchmarks
+concrete_benchmark:
 	rm -rf progress.json && \
-	for script in benchmarks/*.py; do \
+	for script in benchmarks/concrete/*.py; do \
+	  poetry run python $$script; \
+	done
+
+.PHONY: ml_benchmark # Launch ml benchmarks
+ml_benchmark:
+	rm -rf progress.json && \
+	for script in benchmarks/ml/*.py; do \
 	  poetry run python $$script; \
 	done
 

benchmarks/ml/common.py

@@ -0,0 +1,16 @@
+import concrete.numpy as hnp
+from concrete.numpy import compile as compile_
+
+# This is only for benchmarks to speed up compilation times
+# pylint: disable=protected-access
+compile_._COMPILE_FHE_INSECURE_KEY_CACHE_DIR = "/tmp/keycache"
+# pylint: enable=protected-access
+
+BENCHMARK_CONFIGURATION = hnp.CompilationConfiguration(
+    check_every_input_in_inputset=True,
+    dump_artifacts_on_unexpected_failures=True,
+    enable_topological_optimizations=True,
+    enable_unsafe_features=True,
+    treat_warnings_as_errors=True,
+    use_insecure_key_cache=True,
+)

script/progress_tracker_utils/benchmark_and_publish_findings_in_docker.sh

@@ -5,6 +5,13 @@
 
 set -e
 
+# shellcheck disable=SC1091
+if [ -f .env ]
+then
+    # shellcheck disable=SC1091
+    set -a; source .env; set +a
+fi
+
 DEV_VENV_PATH="/home/dev_user/dev_venv"
 
 # shellcheck disable=SC1090,SC1091
@@ -18,27 +25,40 @@ cd /src/ && make sync_env
 
 mkdir -p logs
 
-initial_log=logs/$(date -u --iso-8601=seconds).log
+initial_concrete_log=logs/$(date -u --iso-8601=seconds).concrete.log
+make -s concrete_benchmark 2>&1 | tee -a "$initial_concrete_log"
 
-make benchmark > "$initial_log"
+final_concrete_log=logs/$(date -u --iso-8601=seconds).concrete.log
+cat -s "$initial_concrete_log" | sed '1d; $d' > "$final_concrete_log"
 
-final_log=logs/$(date -u --iso-8601=seconds).log
+# sed above removes the first and the last lines of the log
+# which are empty to provide a nice console output
+# but empty lines are useless for logs so we get rid of them
 
-cat -s "$initial_log" | sed '1d; $d' > "$final_log"
-rm "$initial_log"
-
-cp "$final_log" logs/latest.log
-
-if [ -f .env ]
-then
-  # Set the last two environment variables in `.env` for the curl command below
-  # (https://gist.github.com/mihow/9c7f559807069a03e302605691f85572)
-  # shellcheck disable=SC2002,SC2046
-  export $(cat .env | tail -n 2 | sed 's/#.*//g' | xargs  -d '\n')
-fi
+rm "$initial_concrete_log"
+cp "$final_concrete_log" logs/latest.concrete.log
 
 curl \
-     -H 'Authorization: Bearer '"$PROGRESS_TRACKER_TOKEN"'' \
+     -H 'Authorization: Bearer '"$CONCRETE_PROGRESS_TRACKER_TOKEN"'' \
      -H 'Content-Type: application/json' \
      -d @progress.json \
-     -X POST "$PROGRESS_TRACKER_URL"/measurement
+     -X POST "$CONCRETE_PROGRESS_TRACKER_URL"/measurement
+
+initial_ml_log=logs/$(date -u --iso-8601=seconds).ml.log
+make -s ml_benchmark 2>&1 | tee -a "$initial_ml_log"
+
+final_ml_log=logs/$(date -u --iso-8601=seconds).ml.log
+cat -s "$initial_ml_log" | sed '1d; $d' > "$final_ml_log"
+
+# sed above removes the first and the last lines of the log
+# which are empty to provide a nice console output
+# but empty lines are useless for logs so we get rid of them
+
+rm "$initial_ml_log"
+cp "$final_ml_log" logs/latest.ml.log
+
+curl \
+     -H 'Authorization: Bearer '"$ML_PROGRESS_TRACKER_TOKEN"'' \
+     -H 'Content-Type: application/json' \
+     -d @progress.json \
+     -X POST "$ML_PROGRESS_TRACKER_URL"/measurement