feat: implement performing and publishing benchmarks with a single make target

2026-04-17 03:00:54 -04:00 · 2021-09-10 10:16:04 +03:00
parent bd8dca11d5
commit c253219277
10 changed files with 273 additions and 39 deletions
--- a/.github/workflows/daily-benchmarks.yaml
+++ b/.github/workflows/daily-benchmarks.yaml
@@ -0,0 +1,67 @@
+name: Daily Benchmarks
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 22 * * *' # Everyday @ 22:00
+
+jobs:
+  perform:
+    name: Run Benchmarks on EC2 and Publish Results to Progress Tracker
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-3 # Europe (Paris)
+
+      - name: Start EC2 Instance
+        run: |
+          aws ec2 start-instances --instance-ids ${{ secrets.BENCHMARKS_EC2_INSTANCE_ID }}
+
+      - name: Get Public IP Address of EC2 Instance
+        id: public-ip
+        run: echo "::set-output name=value::$(aws ec2 describe-instances --region eu-west-3 --instance-ids ${{ secrets.BENCHMARKS_EC2_INSTANCE_ID }} --query 'Reservations[].Instances[].PublicIpAddress' --output text)"
+
+      - name: Connect To EC2 Instance, Perform Benchmarks, Publish Results
+        uses: appleboy/ssh-action@1d1b21ca96111b1eb4c03c21c14ebb971d2200f6
+        with:
+          host: ${{ steps.public-ip.outputs.value }}
+          username: ${{ secrets.BENCHMARKS_EC2_USERNAME }}
+          key: ${{ secrets.BENCHMARKS_EC2_SSH_KEY }}
+          script: |
+            cd ~/concretefhe-internal
+            make docker_publish_measurements
+
+      - name: Copy Logs
+        uses: appleboy/scp-action@edc8ec9139a2687bcebf0249d0352ff2a988df00
+        with:
+          host: ${{ steps.public-ip.outputs.value }}
+          username: ${{ secrets.BENCHMARKS_EC2_USERNAME }}
+          key: ${{ secrets.BENCHMARKS_EC2_SSH_KEY }}
+          source: "~/concretefhe-internal/logs/latest.log"
+          target: "~/latest.log"
+
+      - name: Stop EC2 Instance
+        if: ${{ always() }}
+        run: |
+          aws ec2 stop-instances --instance-ids ${{ secrets.BENCHMARKS_EC2_INSTANCE_ID }}
+
+      - name: Upload Logs
+        uses: actions/upload-artifact@v2
+        with:
+          name: logs
+          path: ~/latest.log
+
+      - name: Send Slack Notification
+        if: ${{ always() }}
+        continue-on-error: true
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
+          SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
+          SLACK_COLOR: ${{ job.status }}
+          SLACK_MESSAGE: 'Publishing benchmarks finished with status ${{ job.status }} (${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})'
+          SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
--- a/.gitignore
+++ b/.gitignore
@@ -132,8 +132,8 @@ dmypy.json
 # Pyre type checker
 .pyre/

-# Benchmark results
-.benchmarks.json
+# Benchmark Artifacts
+.benchmarks

 # concrete compilation artifacts
 .artifacts
--- a/13
+++ b/13
@@ -131,6 +131,16 @@ docker_build_and_start: docker_build docker_start
 docker_bas: docker_build_and_start
 .PHONY: docker_bas

+docker_publish_measurements: docker_build
+	git pull
+	mkdir -p .benchmarks
+	python script/progress_tracker_utils/extract_machine_info.py
+	docker run --rm -it \
+	--volume /"$$(pwd)":/src \
+	$(DEV_DOCKER_IMG) \
+	 /bin/bash -i ./script/progress_tracker_utils/benchmark_and_publish_findings_in_docker.sh
+.PHONY: docker_publish_measurements
+
 docs: clean_docs
 	@# Generate the auto summary of documentations
 	poetry run sphinx-apidoc -o docs/_apidoc $(SRC_DIR)
@@ -165,8 +175,7 @@ pytest_nb:
 .PHONY: pytest_nb

 benchmark:
-	poetry run python script/progress_tracker_utils/measure.py benchmarks \
-	--output .benchmarks.json
+	poetry run python script/progress_tracker_utils/measure.py benchmarks
 .PHONY: benchmark

 jupyter:
--- a/benchmarks/linear_regression.py
+++ b/benchmarks/linear_regression.py
@@ -151,9 +151,12 @@ def main():

    loss = 0
    for x_i, y_i in zip(x_q, y):
+        x_i = [int(value) for value in x_i]
+
        # Measure: Evaluation Time (ms)
        prediction = QuantizedArray(engine.run(*x_i), y_parameters).dequantize()
        # Measure: End
+
        loss += (prediction - y_i) ** 2

    # Measure: Loss = loss / len(y)
--- a/benchmarks/logistic_regression.py
+++ b/benchmarks/logistic_regression.py
@@ -220,6 +220,8 @@ def main():

    correct = 0
    for x_i, y_i in zip(x_q, y):
+        x_i = [int(value) for value in x_i]
+
        # Measure: Evaluation Time (ms)
        prediction = round(QuantizedArray(engine.run(*x_i), y_parameters).dequantize())
        # Measure: End
--- a/poetry.lock
+++ b/poetry.lock
@@ -207,7 +207,7 @@ six = "*"

 [[package]]
 name = "decorator"
-version = "5.0.9"
+version = "5.1.0"
 description = "Decorators for Humans"
 category = "dev"
 optional = false
@@ -223,21 +223,18 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"

 [[package]]
 name = "diff-cover"
-version = "6.3.5"
-description = "Run coverage and linting reports on diffs"
+version = "6.2.1"
+description = "Automatically find diff lines that need test coverage."
 category = "dev"
 optional = false
-python-versions = ">=3.6.2,<4.0.0"
+python-versions = ">= 3.6"

 [package.dependencies]
 chardet = ">=3.0.0"
 Jinja2 = ">=2.7.1"
-jinja2_pluralize = ">=0.3.0,<0.4.0"
-pluggy = ">=0.13.1,<0.14.0"
-Pygments = ">=2.9.0,<3.0.0"
-
-[package.extras]
-toml = ["tomli (>=1.2.1,<2.0.0)"]
+jinja2-pluralize = "*"
+pluggy = "*"
+pygments = "*"

 [[package]]
 name = "docutils"
@@ -270,7 +267,7 @@ pyflakes = ">=2.3.0,<2.4.0"

 [[package]]
 name = "flake8-bugbear"
-version = "21.4.3"
+version = "21.9.1"
 description = "A plugin for flake8 finding likely bugs and design problems in your program. Contains warnings that don't belong in pyflakes and pycodestyle."
 category = "dev"
 optional = false
@@ -813,7 +810,7 @@ python-versions = ">=3.5"

 [[package]]
 name = "networkx"
-version = "2.6.2"
+version = "2.6.3"
 description = "Python package for creating and manipulating graphs and networks"
 category = "main"
 optional = false
@@ -951,14 +948,15 @@ test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock

 [[package]]
 name = "pluggy"
-version = "0.13.1"
+version = "1.0.0"
 description = "plugin and hook calling mechanisms for python"
 category = "dev"
 optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+python-versions = ">=3.6"

 [package.extras]
 dev = ["pre-commit", "tox"]
+testing = ["pytest", "pytest-benchmark"]

 [[package]]
 name = "prometheus-client"
@@ -982,6 +980,17 @@ python-versions = ">=3.6.2"
 [package.dependencies]
 wcwidth = "*"

+[[package]]
+name = "psutil"
+version = "5.8.0"
+description = "Cross-platform lib for process and system monitoring in Python."
+category = "dev"
+optional = false
+python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+
+[package.extras]
+test = ["ipaddress", "mock", "unittest2", "enum34", "pywin32", "wmi"]
+
 [[package]]
 name = "ptyprocess"
 version = "0.7.0"
@@ -998,6 +1007,14 @@ category = "dev"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"

+[[package]]
+name = "py-cpuinfo"
+version = "8.0.0"
+description = "Get CPU info with pure Python 2 & 3"
+category = "dev"
+optional = false
+python-versions = "*"
+
 [[package]]
 name = "pycodestyle"
 version = "2.7.0"
@@ -1147,6 +1164,17 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
 [package.dependencies]
 six = ">=1.5"

+[[package]]
+name = "python-dotenv"
+version = "0.19.0"
+description = "Read key-value pairs from a .env file and set them as environment variables"
+category = "dev"
+optional = false
+python-versions = ">=3.5"
+
+[package.extras]
+cli = ["click (>=5.0)"]
+
 [[package]]
 name = "pytz"
 version = "2021.1"
@@ -1278,7 +1306,7 @@ python-versions = "*"

 [[package]]
 name = "sphinx"
-version = "4.1.2"
+version = "4.2.0"
 description = "Python documentation generator"
 category = "dev"
 optional = false
@@ -1541,7 +1569,7 @@ python-versions = "*"
 [metadata]
 lock-version = "1.1"
 python-versions = ">=3.8,<3.9"
-content-hash = "8a3be3fe122eddfb9a28a4f789c4581311ada706406277b5e84beb431369163b"
+content-hash = "b2650d98938713e30a9141050ab90466bd9ba8a1eb38eedffcaa5bbd0f150cc5"

 [metadata.files]
 alabaster = [
@@ -1723,16 +1751,16 @@ cycler = [
    {file = "cycler-0.10.0.tar.gz", hash = "sha256:cd7b2d1018258d7247a71425e9f26463dfb444d411c39569972f4ce586b0c9d8"},
 ]
 decorator = [
-    {file = "decorator-5.0.9-py3-none-any.whl", hash = "sha256:6e5c199c16f7a9f0e3a61a4a54b3d27e7dad0dbdde92b944426cb20914376323"},
-    {file = "decorator-5.0.9.tar.gz", hash = "sha256:72ecfba4320a893c53f9706bebb2d55c270c1e51a28789361aa93e4a21319ed5"},
+    {file = "decorator-5.1.0-py3-none-any.whl", hash = "sha256:7b12e7c3c6ab203a29e157335e9122cb03de9ab7264b137594103fd4a683b374"},
+    {file = "decorator-5.1.0.tar.gz", hash = "sha256:e59913af105b9860aa2c8d3272d9de5a56a4e608db9a2f167a8480b323d529a7"},
 ]
 defusedxml = [
    {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"},
    {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"},
 ]
 diff-cover = [
-    {file = "diff_cover-6.3.5-py3-none-any.whl", hash = "sha256:d8f0949c8a57f7ed4b93d0012fbe9e77898571070cd35bd33e29d0c4b2045b0d"},
-    {file = "diff_cover-6.3.5.tar.gz", hash = "sha256:53b90abb1d33ef0361d15c8761d260f572b981b82a104dfb70de67a461d8cd42"},
+    {file = "diff_cover-6.2.1-py3-none-any.whl", hash = "sha256:cfe6333c9b83a28f91214e06e3a86108aeeb6a9a31233944ce8ef236121ba899"},
+    {file = "diff_cover-6.2.1.tar.gz", hash = "sha256:b22fe97d118fadb2528bbc0a1f9fc370491b29b1b3fe2e2f1cdb94bf028814c7"},
 ]
 docutils = [
    {file = "docutils-0.16-py2.py3-none-any.whl", hash = "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af"},
@@ -1747,8 +1775,8 @@ flake8 = [
    {file = "flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"},
 ]
 flake8-bugbear = [
-    {file = "flake8-bugbear-21.4.3.tar.gz", hash = "sha256:2346c81f889955b39e4a368eb7d508de723d9de05716c287dc860a4073dc57e7"},
-    {file = "flake8_bugbear-21.4.3-py36.py37.py38-none-any.whl", hash = "sha256:4f305dca96be62bf732a218fe6f1825472a621d3452c5b994d8f89dae21dbafa"},
+    {file = "flake8-bugbear-21.9.1.tar.gz", hash = "sha256:2f60c8ce0dc53d51da119faab2d67dea978227f0f92ed3c44eb7d65fb2e06a96"},
+    {file = "flake8_bugbear-21.9.1-py36.py37.py38-none-any.whl", hash = "sha256:45bfdccfb9f2d8aa140e33cac8f46f1e38215c13d5aa8650e7e188d84e2f94c6"},
 ]
 idna = [
    {file = "idna-3.2-py3-none-any.whl", hash = "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a"},
@@ -2054,8 +2082,8 @@ nest-asyncio = [
    {file = "nest_asyncio-1.5.1.tar.gz", hash = "sha256:afc5a1c515210a23c461932765691ad39e8eba6551c055ac8d5546e69250d0aa"},
 ]
 networkx = [
-    {file = "networkx-2.6.2-py3-none-any.whl", hash = "sha256:5fcb7004be69e8fbdf07dcb502efa5c77cadcaad6982164134eeb9721f826c2e"},
-    {file = "networkx-2.6.2.tar.gz", hash = "sha256:2306f1950ce772c5a59a57f5486d59bb9cab98497c45fc49cbc45ac0dec119bb"},
+    {file = "networkx-2.6.3-py3-none-any.whl", hash = "sha256:80b6b89c77d1dfb64a4c7854981b60aeea6360ac02c6d4e4913319e0a313abef"},
+    {file = "networkx-2.6.3.tar.gz", hash = "sha256:c0946ed31d71f1b732b5aaa6da5a0388a345019af232ce2f49c766e2d6795c51"},
 ]
 notebook = [
    {file = "notebook-6.4.3-py3-none-any.whl", hash = "sha256:b50eafa8208d5db966efd1caa4076b4dfc51815e02a805b32ecd717e9e6cc071"},
@@ -2179,8 +2207,8 @@ platformdirs = [
    {file = "platformdirs-2.3.0.tar.gz", hash = "sha256:15b056538719b1c94bdaccb29e5f81879c7f7f0f4a153f46086d155dffcd4f0f"},
 ]
 pluggy = [
-    {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"},
-    {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"},
+    {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"},
+    {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"},
 ]
 prometheus-client = [
    {file = "prometheus_client-0.11.0-py2.py3-none-any.whl", hash = "sha256:b014bc76815eb1399da8ce5fc84b7717a3e63652b0c0f8804092c9363acab1b2"},
@@ -2190,6 +2218,36 @@ prompt-toolkit = [
    {file = "prompt_toolkit-3.0.20-py3-none-any.whl", hash = "sha256:6076e46efae19b1e0ca1ec003ed37a933dc94b4d20f486235d436e64771dcd5c"},
    {file = "prompt_toolkit-3.0.20.tar.gz", hash = "sha256:eb71d5a6b72ce6db177af4a7d4d7085b99756bf656d98ffcc4fecd36850eea6c"},
 ]
+psutil = [
+    {file = "psutil-5.8.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:0066a82f7b1b37d334e68697faba68e5ad5e858279fd6351c8ca6024e8d6ba64"},
+    {file = "psutil-5.8.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:0ae6f386d8d297177fd288be6e8d1afc05966878704dad9847719650e44fc49c"},
+    {file = "psutil-5.8.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:12d844996d6c2b1d3881cfa6fa201fd635971869a9da945cf6756105af73d2df"},
+    {file = "psutil-5.8.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:02b8292609b1f7fcb34173b25e48d0da8667bc85f81d7476584d889c6e0f2131"},
+    {file = "psutil-5.8.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:6ffe81843131ee0ffa02c317186ed1e759a145267d54fdef1bc4ea5f5931ab60"},
+    {file = "psutil-5.8.0-cp27-none-win32.whl", hash = "sha256:ea313bb02e5e25224e518e4352af4bf5e062755160f77e4b1767dd5ccb65f876"},
+    {file = "psutil-5.8.0-cp27-none-win_amd64.whl", hash = "sha256:5da29e394bdedd9144c7331192e20c1f79283fb03b06e6abd3a8ae45ffecee65"},
+    {file = "psutil-5.8.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:74fb2557d1430fff18ff0d72613c5ca30c45cdbfcddd6a5773e9fc1fe9364be8"},
+    {file = "psutil-5.8.0-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:74f2d0be88db96ada78756cb3a3e1b107ce8ab79f65aa885f76d7664e56928f6"},
+    {file = "psutil-5.8.0-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:99de3e8739258b3c3e8669cb9757c9a861b2a25ad0955f8e53ac662d66de61ac"},
+    {file = "psutil-5.8.0-cp36-cp36m-win32.whl", hash = "sha256:36b3b6c9e2a34b7d7fbae330a85bf72c30b1c827a4366a07443fc4b6270449e2"},
+    {file = "psutil-5.8.0-cp36-cp36m-win_amd64.whl", hash = "sha256:52de075468cd394ac98c66f9ca33b2f54ae1d9bff1ef6b67a212ee8f639ec06d"},
+    {file = "psutil-5.8.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c6a5fd10ce6b6344e616cf01cc5b849fa8103fbb5ba507b6b2dee4c11e84c935"},
+    {file = "psutil-5.8.0-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:61f05864b42fedc0771d6d8e49c35f07efd209ade09a5afe6a5059e7bb7bf83d"},
+    {file = "psutil-5.8.0-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:0dd4465a039d343925cdc29023bb6960ccf4e74a65ad53e768403746a9207023"},
+    {file = "psutil-5.8.0-cp37-cp37m-win32.whl", hash = "sha256:1bff0d07e76114ec24ee32e7f7f8d0c4b0514b3fae93e3d2aaafd65d22502394"},
+    {file = "psutil-5.8.0-cp37-cp37m-win_amd64.whl", hash = "sha256:fcc01e900c1d7bee2a37e5d6e4f9194760a93597c97fee89c4ae51701de03563"},
+    {file = "psutil-5.8.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6223d07a1ae93f86451d0198a0c361032c4c93ebd4bf6d25e2fb3edfad9571ef"},
+    {file = "psutil-5.8.0-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:d225cd8319aa1d3c85bf195c4e07d17d3cd68636b8fc97e6cf198f782f99af28"},
+    {file = "psutil-5.8.0-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:28ff7c95293ae74bf1ca1a79e8805fcde005c18a122ca983abf676ea3466362b"},
+    {file = "psutil-5.8.0-cp38-cp38-win32.whl", hash = "sha256:ce8b867423291cb65cfc6d9c4955ee9bfc1e21fe03bb50e177f2b957f1c2469d"},
+    {file = "psutil-5.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:90f31c34d25b1b3ed6c40cdd34ff122b1887a825297c017e4cbd6796dd8b672d"},
+    {file = "psutil-5.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6323d5d845c2785efb20aded4726636546b26d3b577aded22492908f7c1bdda7"},
+    {file = "psutil-5.8.0-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:245b5509968ac0bd179287d91210cd3f37add77dad385ef238b275bad35fa1c4"},
+    {file = "psutil-5.8.0-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:90d4091c2d30ddd0a03e0b97e6a33a48628469b99585e2ad6bf21f17423b112b"},
+    {file = "psutil-5.8.0-cp39-cp39-win32.whl", hash = "sha256:ea372bcc129394485824ae3e3ddabe67dc0b118d262c568b4d2602a7070afdb0"},
+    {file = "psutil-5.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:f4634b033faf0d968bb9220dd1c793b897ab7f1189956e1aa9eae752527127d3"},
+    {file = "psutil-5.8.0.tar.gz", hash = "sha256:0c9ccb99ab76025f2f0bbecf341d4656e9c1351db8cc8a03ccd62e318ab4b5c6"},
+]
 ptyprocess = [
    {file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"},
    {file = "ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"},
@@ -2198,6 +2256,9 @@ py = [
    {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"},
    {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"},
 ]
+py-cpuinfo = [
+    {file = "py-cpuinfo-8.0.0.tar.gz", hash = "sha256:5f269be0e08e33fd959de96b34cd4aeeeacac014dd8305f70eb28d06de2345c5"},
+]
 pycodestyle = [
    {file = "pycodestyle-2.7.0-py2.py3-none-any.whl", hash = "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068"},
    {file = "pycodestyle-2.7.0.tar.gz", hash = "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef"},
@@ -2288,6 +2349,10 @@ python-dateutil = [
    {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"},
    {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"},
 ]
+python-dotenv = [
+    {file = "python-dotenv-0.19.0.tar.gz", hash = "sha256:f521bc2ac9a8e03c736f62911605c5d83970021e3fa95b37d769e2bbbe9b6172"},
+    {file = "python_dotenv-0.19.0-py2.py3-none-any.whl", hash = "sha256:aae25dc1ebe97c420f50b81fb0e5c949659af713f31fdb63c749ca68748f34b1"},
+]
 pytz = [
    {file = "pytz-2021.1-py2.py3-none-any.whl", hash = "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798"},
    {file = "pytz-2021.1.tar.gz", hash = "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da"},
@@ -2449,8 +2514,8 @@ snowballstemmer = [
    {file = "snowballstemmer-2.1.0.tar.gz", hash = "sha256:e997baa4f2e9139951b6f4c631bad912dfd3c792467e2f03d7239464af90e914"},
 ]
 sphinx = [
-    {file = "Sphinx-4.1.2-py3-none-any.whl", hash = "sha256:46d52c6cee13fec44744b8c01ed692c18a640f6910a725cbb938bc36e8d64544"},
-    {file = "Sphinx-4.1.2.tar.gz", hash = "sha256:3092d929cd807926d846018f2ace47ba2f3b671b309c7a89cd3306e80c826b13"},
+    {file = "Sphinx-4.2.0-py3-none-any.whl", hash = "sha256:98a535c62a4fcfcc362528592f69b26f7caec587d32cd55688db580be0287ae0"},
+    {file = "Sphinx-4.2.0.tar.gz", hash = "sha256:94078db9184491e15bce0a56d9186e0aec95f16ac20b12d00e06d4e36f1058a6"},
 ]
 sphinx-rtd-theme = [
    {file = "sphinx_rtd_theme-0.5.2-py2.py3-none-any.whl", hash = "sha256:4a05bdbe8b1446d77a01e20a23ebc6777c74f43237035e76be89699308987d6f"},
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -33,6 +33,9 @@ Sphinx = "^4.1.1"
 sphinx-rtd-theme = "^0.5.2"
 myst-parser = "^0.15.1"
 tqdm = "^4.62.2"
+psutil = "^5.8.0"
+py-cpuinfo = "^8.0.0"
+python-dotenv = "^0.19.0"

 [build-system]
 requires = ["poetry-core>=1.0.0"]
--- a/script/progress_tracker_utils/benchmark_and_publish_findings_in_docker.sh
+++ b/script/progress_tracker_utils/benchmark_and_publish_findings_in_docker.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Run benchmarks while logging the intermediate results
+# Publish findings in the progress tracker
+
+initial_log=logs/$(date -u --iso-8601=seconds).log
+
+mkdir -p logs
+make -s benchmark > "$initial_log"
+
+final_log=logs/$(date -u --iso-8601=seconds).log
+
+cat -s "$initial_log" | sed '1d; $d' > "$final_log"
+rm "$initial_log"
+
+cp "$final_log" logs/latest.log
+
+if [ -f .env ]
+then
+  # Set the last two environment variables in `.env` for the curl command below
+  # (https://gist.github.com/mihow/9c7f559807069a03e302605691f85572)
+  export $(cat .env | tail -n 2 | sed 's/#.*//g' | xargs  -d '\n')
+fi
+
+curl \
+     -H 'Authorization: Bearer '"$PROGRESS_TRACKER_TOKEN"'' \
+     -H 'Content-Type: application/json' \
+     -d @.benchmarks/findings.json \
+     -X POST "$PROGRESS_TRACKER_URL"/measurement
--- a/script/progress_tracker_utils/extract_machine_info.py
+++ b/script/progress_tracker_utils/extract_machine_info.py
@@ -0,0 +1,50 @@
+import cpuinfo
+import dotenv
+import json
+import os
+import platform
+import psutil
+import urllib.parse
+
+
+def main():
+    dotenv.load_dotenv()
+
+    properties = []
+
+    cpu_value = cpuinfo.get_cpu_info()["brand_raw"].replace("(R)", "®").replace("(TM)", "™")
+    properties.append(["CPU", cpu_value])
+
+    vcpu_value = os.getenv("VCPU")
+    if vcpu_value is not None:
+        properties.append(["vCPU", vcpu_value])
+
+    ram_value = f"{psutil.virtual_memory().total / (1024 ** 3):.2f} GB"
+    properties.append(["RAM", ram_value])
+
+    os_value = os.getenv("OS_NAME")
+    if os_value is None:
+        os_value = f"{platform.system()} {platform.release()}"
+    properties.append(["OS", os_value])
+
+    name = os.getenv("MACHINE_NAME").strip()
+    if name is None:
+        name = platform.node().strip()
+
+    id_ = name.lower()
+    id_ = id_.replace(" ", "-")
+    id_ = id_.replace("_", "-")
+    id_ = id_.replace(".", "-")
+    id_ = id_.replace("(", "")
+    id_ = id_.replace(")", "")
+    id_ = id_.replace("$/h", "-dollars-per-hour")
+    id_ = id_.strip()
+    id_ = urllib.parse.quote_plus(id_)
+
+    machine = {"id": id_, "name": name, "properties": properties}
+    with open(".benchmarks/machine.json", "w") as f:
+        json.dump(machine, f, indent=2, ensure_ascii=False)
+
+
+if __name__ == "__main__":
+    main()
--- a/script/progress_tracker_utils/measure.py
+++ b/script/progress_tracker_utils/measure.py
@@ -10,9 +10,9 @@ import tqdm
 def name_to_id(name):
    """Convert a human readable name to a url friendly id (e.g., `x + y` to `x-plus-y`)"""

+    name = name.replace("-", "minus")
    name = name.replace("**", "-to-the-power-of-")
    name = name.replace("+", "plus")
-    name = name.replace("-", "minus")
    name = name.replace("*", "times")
    name = name.replace("/", "over")
    name = name.replace("%", "percent")
@@ -191,7 +191,12 @@ def perform_measurements(script, script_without_extension, target_id, metrics, s
                working = False

                pbar.write(f"        Failed (exited with {process.returncode})")
-                pbar.write(f"")
+                pbar.write(f"        --------------------{'-' * len(str(process.returncode))}-")
+
+                stderr = process.stderr.decode("utf-8")
+                for line in stderr.split("\n"):
+                    if line.strip() != "":
+                        pbar.write(f"            {line}")

                pbar.update(samples)
                break
@@ -235,17 +240,18 @@ def main():
    parser = argparse.ArgumentParser(description="Measurement script for the progress tracker")

    parser.add_argument("base", type=str, help="directory which contains the benchmarks")
-    parser.add_argument("--output", type=str, help="file which the results will be saved to")
    parser.add_argument("--samples", type=int, default=30, help="number of samples to take")
    parser.add_argument("--keep", action='store_true', help="flag to keep measurement scripts")

    args = parser.parse_args()

    base = pathlib.Path(args.base)
-    output = pathlib.Path(args.output)
    samples = args.samples

-    result = {"metrics": {}, "targets": {}}
+    with open(".benchmarks/machine.json", "r") as f:
+        machine = json.load(f)
+
+    result = {"machine": machine, "metrics": {}, "targets": {}}
    scripts = list(base.glob("*.py"))

    # Process each script under the base directory
@@ -303,8 +309,8 @@ def main():
        perform_measurements(script, script_without_extension, target_id, metrics, samples, result)

        # Dump the latest results to the output file
-        with open(output, "w") as f:
-            json.dump(result, f, indent=2)
+        with open(".benchmarks/findings.json", "w") as f:
+            json.dump(result, f, indent=2, ensure_ascii=False)

        # Delete the modified script if the user doesn't care
        if not args.keep: