Optimize CI workflows and add core SDK checks (#885)

* Parallelize mobile SDK CI and add core workflow

* fix tests for now

* test build improvements

* fix: add explicit Corepack setup to fix Yarn version issues in CI

* security: add minimal permissions, pin actions to SHAs, and improve checksum verification

* refactor: remove redundant Corepack setup from npm-publish and core-sdk-ci workflows

* simplify action versions

.github/actions/yarn-install/action.yml

@@ -16,7 +16,8 @@ runs:
       run: |
         corepack enable
         corepack prepare yarn@4.6.0 --activate
-        yarn set version 4.6.0
+        # Ensure we're using the correct version
+        yarn --version
 
     - name: Setup Node.js
       uses: actions/setup-node@v4

.github/workflows/circuits-build.yml

@@ -17,6 +17,11 @@ on:
 jobs:
   build:
     runs-on: ["self-hosted", "selfxyz-org", "ubuntu-22-04", "128ram"]
+    permissions:
+      contents: read
+    env:
+      CIRCOM_VERSION: "2.1.9"
+      CIRCOM_SHA256: "e5575829252d763b7818049df9de2ef9304df834697de77fa63ce7babc23c967"
 
     steps:
       - name: Checkout Repository
@@ -33,30 +38,55 @@ jobs:
             nlohmann-json3-dev \
             wget
 
-      # TODO(): Use caching
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 22
-
-      - name: Install dependencies
-        run: |
-          corepack enable
-          yarn set version 4.6.0
-          cd circuits && yarn
-
       - name: Setup Rust
         uses: dtolnay/rust-toolchain@stable
 
+      - name: Restore Circom binary
+        id: circom-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: ~/.cache/circom
+          key: circom-v2.1.9
+
       - name: Download Circom Binary v2.1.9
+        if: steps.circom-cache.outputs.cache-hit != 'true'
         run: |
-          mkdir -p /home/runner/work
-          wget -qO /home/runner/work/circom https://github.com/iden3/circom/releases/download/v2.1.9/circom-linux-amd64
-          chmod +x /home/runner/work/circom
-          sudo mv /home/runner/work/circom /bin/circom
+          mkdir -p ~/.cache/circom
+          wget -qO ~/.cache/circom/circom https://github.com/iden3/circom/releases/download/v${{ env.CIRCOM_VERSION }}/circom-linux-amd64
+          chmod +x ~/.cache/circom/circom
+          # Verify checksum - SHA256 for circom v2.1.9 linux-amd64
+          echo "${{ env.CIRCOM_SHA256 }}  ~/.cache/circom/circom" | sha256sum -c -
+
+      - name: Save Circom cache
+        if: steps.circom-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: ~/.cache/circom
+          key: circom-v2.1.9
+
+      - name: Verify Circom checksum (cache hit)
+        if: steps.circom-cache.outputs.cache-hit == 'true'
+        run: |
+          echo "${{ env.CIRCOM_SHA256 }}  ~/.cache/circom/circom" | sha256sum -c -
+
+      - name: Add Circom to PATH
+        run: echo "$HOME/.cache/circom" >> "$GITHUB_PATH"
+
+      - name: Install dependencies
+        uses: ./.github/actions/yarn-install
+        with:
+          working_directory: circuits
 
       - name: Print Circom version
         run: circom --version
 
+      - name: Restore circuits cache
+        id: circuits-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: output
+          key: circuits-output-v1-${{ hashFiles('circuits/circuits/**', 'circuits/scripts/**', 'circuits/*.{json,yml,yaml}') }}
+
       - name: Build cpp circuits
         run: |
           chmod +x circuits/scripts/build/build_cpp.sh && \
@@ -65,8 +95,15 @@ jobs:
           ./circuits/scripts/build/build_cpp.sh disclose &&
           ./circuits/scripts/build/build_cpp.sh dsc
 
+      - name: Save circuits cache
+        if: steps.circuits-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: output
+          key: circuits-output-v1-${{ hashFiles('circuits/circuits/**', 'circuits/scripts/**', 'circuits/*.{json,yml,yaml}') }}
+
       - name: Upload Artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4
         with:
           name: circuits
           path: output/

.github/workflows/circuits.yml

@@ -21,6 +21,11 @@ jobs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     environment: development
+    permissions:
+      contents: read
+    env:
+      CIRCOM_VERSION: "2.1.9"
+      CIRCOM_SHA256: "e5575829252d763b7818049df9de2ef9304df834697de77fa63ce7babc23c967"
     steps:
       - uses: actions/checkout@v4
 
@@ -35,25 +40,47 @@ jobs:
             nasm \
             nlohmann-json3-dev
 
-      - name: Set Node.js 22.x
-        uses: actions/setup-node@v4
-        with:
-          node-version: 22
       - name: Setup Rust
         uses: dtolnay/rust-toolchain@stable
 
+      - name: Restore Circom binary
+        id: circom-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: ~/.cache/circom
+          key: circom-v2.1.9
+
       - name: Download Circom Binary v2.1.9
+        if: steps.circom-cache.outputs.cache-hit != 'true'
         run: |
-          wget -qO /home/runner/work/circom https://github.com/iden3/circom/releases/download/v2.1.9/circom-linux-amd64
-          chmod +x /home/runner/work/circom
-          sudo mv /home/runner/work/circom /bin/circom
+          mkdir -p ~/.cache/circom
+          wget -qO ~/.cache/circom/circom https://github.com/iden3/circom/releases/download/v${{ env.CIRCOM_VERSION }}/circom-linux-amd64
+          chmod +x ~/.cache/circom/circom
+          # Verify checksum - SHA256 for circom v2.1.9 linux-amd64
+          echo "${{ env.CIRCOM_SHA256 }}  ~/.cache/circom/circom" | sha256sum -c -
+
+      - name: Save Circom cache
+        if: steps.circom-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: ~/.cache/circom
+          key: circom-v2.1.9
+
+      - name: Verify Circom checksum (cache hit)
+        if: steps.circom-cache.outputs.cache-hit == 'true'
+        run: |
+          echo "${{ env.CIRCOM_SHA256 }}  ~/.cache/circom/circom" | sha256sum -c -
+
+      - name: Add Circom to PATH
+        run: echo "$HOME/.cache/circom" >> "$GITHUB_PATH"
 
       - name: Print Circom version
         run: circom --version
-      - name: "enable yarn"
-        run: corepack enable yarn
+
       - name: Install Yarn dependencies
-        run: yarn workspaces focus @selfxyz/circuits
+        uses: ./.github/actions/yarn-install
+        with:
+          working_directory: circuits
 
       - name: Run lint
         run: yarn workspace @selfxyz/circuits lint

.github/workflows/common-ci.yml

@@ -3,39 +3,73 @@ name: Common CI
 on:
   pull_request:
 jobs:
-  lint:
+  build:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
     steps:
       - uses: actions/checkout@v4
       - name: Install Dependencies
         uses: ./.github/actions/yarn-install
-      - name: Build dependencies
-        shell: bash
+      - name: Build @selfxyz/common
         run: yarn workspace @selfxyz/common build
+      - name: Cache build artifacts
+        uses: actions/cache/save@v4
+        with:
+          path: common/dist
+          key: common-build-${{ runner.os }}-${{ github.sha }}
+
+  lint:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Dependencies
+        uses: ./.github/actions/yarn-install
       - name: Run linter
-        run: yarn lint
+        run: yarn workspace @selfxyz/common lint
 
   type-check:
     runs-on: ubuntu-latest
+    needs: build
+    permissions:
+      contents: read
     steps:
       - uses: actions/checkout@v4
       - name: Install Dependencies
         uses: ./.github/actions/yarn-install
-      - name: Build dependencies
-        shell: bash
+      - name: Restore build artifacts
+        id: restore-common-dist
+        uses: actions/cache/restore@v4
+        with:
+          path: common/dist
+          key: common-build-${{ runner.os }}-${{ github.sha }}
+          fail-on-cache-miss: false
+      - name: Build @selfxyz/common (fallback on cache miss)
+        if: steps.restore-common-dist.outputs.cache-hit != 'true'
         run: yarn workspace @selfxyz/common build
       - name: Yarn types
-        shell: bash
         run: yarn types
 
   test-common:
     runs-on: ubuntu-latest
+    needs: build
+    permissions:
+      contents: read
     steps:
       - uses: actions/checkout@v4
       - name: Install Dependencies
         uses: ./.github/actions/yarn-install
-      - name: Build dependencies
-        shell: bash
+      - name: Restore build artifacts
+        id: restore-common-dist
+        uses: actions/cache/restore@v4
+        with:
+          path: common/dist
+          key: common-build-${{ runner.os }}-${{ github.sha }}
+          fail-on-cache-miss: false
+      - name: Build @selfxyz/common (fallback on cache miss)
+        if: steps.restore-common-dist.outputs.cache-hit != 'true'
         run: yarn workspace @selfxyz/common build
       - name: Run @selfxyz/common tests
         run: yarn workspace @selfxyz/common test

.github/workflows/core-sdk-ci.yml

@@ -0,0 +1,120 @@
+name: Core SDK CI
+
+on:
+  pull_request:
+    paths:
+      - "sdk/core/**"
+      - "common/**"
+      - ".github/workflows/core-sdk-ci.yml"
+      - ".github/actions/**"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Dependencies
+        uses: ./.github/actions/yarn-install
+      - name: Build dependencies
+        run: |
+          yarn workspace @selfxyz/common build
+          yarn workspace @selfxyz/core build
+      - name: Cache build artifacts
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            common/dist
+            sdk/core/dist
+            node_modules
+            sdk/core/node_modules
+            common/node_modules
+          key: core-sdk-build-${{ github.sha }}
+
+  lint:
+    runs-on: ubuntu-latest
+    needs: build
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Corepack
+        run: |
+          corepack enable
+          corepack prepare yarn@4.6.0 --activate
+      - name: Restore build artifacts
+        id: build-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            common/dist
+            sdk/core/dist
+            node_modules
+            sdk/core/node_modules
+            common/node_modules
+          key: core-sdk-build-${{ github.sha }}
+          fail-on-cache-miss: true
+      - name: Install Dependencies
+        if: steps.build-cache.outputs.cache-hit != 'true'
+        uses: ./.github/actions/yarn-install
+      - name: Run linter
+        run: yarn workspace @selfxyz/core lint
+
+  types:
+    runs-on: ubuntu-latest
+    needs: build
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Corepack
+        run: |
+          corepack enable
+          corepack prepare yarn@4.6.0 --activate
+      - name: Restore build artifacts
+        id: build-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            common/dist
+            sdk/core/dist
+            node_modules
+            sdk/core/node_modules
+            common/node_modules
+          key: core-sdk-build-${{ github.sha }}
+          fail-on-cache-miss: true
+      - name: Install Dependencies
+        if: steps.build-cache.outputs.cache-hit != 'true'
+        uses: ./.github/actions/yarn-install
+      - name: Type checking
+        run: yarn workspace @selfxyz/core types
+
+  test:
+    runs-on: ubuntu-latest
+    needs: build
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Corepack
+        run: |
+          corepack enable
+          corepack prepare yarn@4.6.0 --activate
+      - name: Restore build artifacts
+        id: build-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            common/dist
+            sdk/core/dist
+            node_modules
+            sdk/core/node_modules
+            common/node_modules
+          key: core-sdk-build-${{ github.sha }}
+          fail-on-cache-miss: true
+      - name: Install Dependencies
+        if: steps.build-cache.outputs.cache-hit != 'true'
+        uses: ./.github/actions/yarn-install
+      - name: Run tests
+        run: yarn workspace @selfxyz/core test

.github/workflows/mobile-sdk-ci.yml

@@ -31,8 +31,7 @@ jobs:
             common/node_modules
           key: mobile-sdk-alpha-build-${{ github.sha }}
 
-  # Consolidated quality checks job
-  quality-checks:
+  lint:
     runs-on: ubuntu-latest
     needs: build
     steps:
@@ -52,9 +51,66 @@ jobs:
           fail-on-cache-miss: true
       - name: Run linter
         run: yarn workspace @selfxyz/mobile-sdk-alpha lint
+
+  format:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Dependencies
+        uses: ./.github/actions/yarn-install
+      - name: Restore build artifacts
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            common/dist
+            packages/mobile-sdk-alpha/dist
+            node_modules
+            packages/mobile-sdk-alpha/node_modules
+            common/node_modules
+          key: mobile-sdk-alpha-build-${{ github.sha }}
+          fail-on-cache-miss: true
       - name: Check Prettier formatting
         run: yarn workspace @selfxyz/mobile-sdk-alpha prettier --check .
+
+  types:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Dependencies
+        uses: ./.github/actions/yarn-install
+      - name: Restore build artifacts
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            common/dist
+            packages/mobile-sdk-alpha/dist
+            node_modules
+            packages/mobile-sdk-alpha/node_modules
+            common/node_modules
+          key: mobile-sdk-alpha-build-${{ github.sha }}
+          fail-on-cache-miss: true
       - name: Type checking
         run: yarn workspace @selfxyz/mobile-sdk-alpha types
+
+  test:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Dependencies
+        uses: ./.github/actions/yarn-install
+      - name: Restore build artifacts
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            common/dist
+            packages/mobile-sdk-alpha/dist
+            node_modules
+            packages/mobile-sdk-alpha/node_modules
+            common/node_modules
+          key: mobile-sdk-alpha-build-${{ github.sha }}
+          fail-on-cache-miss: true
       - name: Run tests
         run: yarn workspace @selfxyz/mobile-sdk-alpha test

sdk/core/package.json

@@ -42,6 +42,7 @@
     "lint": "prettier --check .",
     "prepublishOnly": "npm run build",
     "publish": "yarn npm publish --access public",
+    "test": "node -e \"require('./dist/index.cjs'); console.log('Core SDK smoke test: import OK')\"",
     "types": "yarn build"
   },
   "dependencies": {