diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml
index a89a089b4..978e43ad2 100644
--- a/.github/workflows/continuous-integration.yml
+++ b/.github/workflows/continuous-integration.yml
@@ -85,19 +85,13 @@ jobs:
           submodules: recursive
           token: ${{ secrets.GH_TOKEN }}
 
-      - name: Install rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-
-      - name: Concrete-Optimizer
-        run: |
-          cd compiler
-          make concrete-optimizer-lib
-
       - name: Create build dir
         run: mkdir build
 
+      - name: Set up ssh auth in docker
+        run: |
+          echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}"
+
       - name: Build compiler
         uses: addnab/docker-run-action@v3
         id: build-compiler
@@ -110,9 +104,13 @@ jobs:
             -v ${{ github.workspace }}/llvm-project:/llvm-project
             -v ${{ github.workspace }}/compiler:/compiler
             -v ${{ github.workspace }}/build:/build
+            -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket
+            -e SSH_AUTH_SOCK=/ssh.socket
           shell: bash
           run: |
             set -e
+            cd /compiler/concrete-optimizer
+            cargo build --release -p concrete-optimizer-cpp
             cd /compiler
             rm -rf /build/*
             pip install pytest
@@ -152,6 +150,8 @@ jobs:
             -v ${{ github.workspace }}/compiler:/compiler
             -v ${{ github.workspace }}/llvm-project:/llvm-project
             -v ${{ github.workspace }}/docs:/docs
+            -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket
+            -e SSH_AUTH_SOCK=/ssh.socket
           shell: bash
           run: |
             set -e
@@ -348,22 +348,16 @@ jobs:
           submodules: recursive
           token: ${{ secrets.GH_TOKEN }}
 
-      - name: Install Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-
-      - name: Concrete-Optimizer
-        run: |
-          cd compiler
-          make concrete-optimizer-lib
-
       - name: Update Python Version
         run: cd compiler && make update-python-version
 
       - name: Login to Github Container Registry
         run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
 
+      - name: Set up ssh auth in docker
+        run: |
+          echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}"
+
       - name: Build Wheel
         uses: addnab/docker-run-action@v3
         with:
@@ -375,6 +369,8 @@ jobs:
             -v ${{ github.workspace }}/llvm-project:/llvm-project
             -v ${{ github.workspace }}/compiler:/compiler
             -v ${{ github.workspace }}/wheels:/wheels
+            -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket
+            -e SSH_AUTH_SOCK=/ssh.socket
           shell: bash
           run: |
             set -e
@@ -424,16 +420,6 @@ jobs:
           submodules: recursive
           token: ${{ secrets.GH_TOKEN }}
 
-      - name: Install Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-
-      - name: Concrete-Optimizer
-        run: |
-          cd compiler
-          make concrete-optimizer-lib
-
       - name: Login to Github Container Registry
         run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
 
@@ -510,11 +496,6 @@ jobs:
           cd compiler
           make concrete-optimizer-lib
 
-      - name: Install rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
@@ -704,22 +685,20 @@ jobs:
           submodules: recursive
           token: ${{ secrets.GH_TOKEN }}
 
-      - name: Install rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-
-      - name: Concrete-Optimizer
-        run: |
-          cd compiler
-          make concrete-optimizer-lib
-
       - name: Login to Registry
         run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
 
+      - name: Set up ssh auth in docker
+        run: |
+          echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}"
+
       # label was initially a need from the frontend CI
       - name: Build Image
-        run: docker image build --no-cache --label "commit-sha=${{ github.sha }}" -t ${{ matrix.image }} -f ${{ matrix.dockerfile }} .
+        run: |
+          docker image build --no-cache \
+            -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket \
+            -e SSH_AUTH_SOCK=/ssh.socket \
+            --label "commit-sha=${{ github.sha }}" -t ${{ matrix.image }} -f ${{ matrix.dockerfile }} .
 
       - name: Tag and Publish Image
         run: |
@@ -755,10 +734,17 @@ jobs:
         if: contains(steps.changed-files.outputs.modified_files, 'builders/Dockerfile.hpx-env')
         run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
 
+      - name: Set up ssh auth in docker
+        run: |
+          echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}"
+
       - name: Build Tag and Publish
         if: ${{ steps.login.conclusion != 'skipped' }}
-        run: |
-          docker build -t "${IMAGE}" -f builders/Dockerfile.hpx-env .
+        run:
+          docker build \
+            -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket  \
+            -e SSH_AUTH_SOCK=/ssh.socket \
+            -t "${IMAGE}" -f builders/Dockerfile.hpx-env .
           docker push "${IMAGE}:latest"
 
       - name: Is Image Built
@@ -791,7 +777,10 @@ jobs:
       - name: Build Tag and Publish
         if: ${{ steps.login.conclusion != 'skipped' }}
         run: |
-          docker build -t "${IMAGE}" -f builders/Dockerfile.cuda-env .
+          docker build \
+            -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket \
+            -e SSH_AUTH_SOCK=/ssh.socket \
+            -t "${IMAGE}" -f builders/Dockerfile.cuda-env .
           docker image tag "${IMAGE}" "${IMAGE}:${TAG}"
           docker push "${IMAGE}:latest"
           docker push "${IMAGE}:${TAG}"