diff --git a/.github/workflows/pypi-gpu.yml b/.github/workflows/pypi-gpu.yml index db7f7ba5..16726899 100644 --- a/.github/workflows/pypi-gpu.yml +++ b/.github/workflows/pypi-gpu.yml @@ -27,6 +27,8 @@ jobs: target: [x86_64] env: RELEASE_TAG: ${{ github.ref_name }} + RUSTFLAGS: "-C linker=gcc" + OPENSSL_NO_VENDOR: 1 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 with: @@ -36,6 +38,16 @@ jobs: python-version: 3.12 architecture: x64 + + + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential g++ gcc cmake libclang-dev llvm-dev libstdc++-12-dev libc6 libc6-dev libssl-dev pkg-config + + - name: Force rebuild icicle dependencies + run: cargo clean -p icicle-runtime -p icicle-core -p icicle-hash -p icicle-bn254 + - name: Set pyproject.toml version to match github tag and rename ezkl to ezkl-gpu shell: bash run: | @@ -71,7 +83,7 @@ jobs: target: ${{ matrix.target }} manylinux: auto container: off - args: --release --out dist --features python-bindings,icicle + args: --release --out dist --features python-bindings,gpu-accelerated - name: Install built wheel if: matrix.target == 'x86_64' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3e73644d..33717b24 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -47,6 +47,8 @@ jobs: TARGET_DIR: ./target RUST_BACKTRACE: 1 PCRE2_SYS_STATIC: 1 + RUSTFLAGS: "-C linker=gcc" + OPENSSL_NO_VENDOR: 1 steps: - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 #v1.0.6 with: @@ -59,6 +61,13 @@ jobs: with: persist-credentials: false + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential g++ gcc cmake libclang-dev llvm-dev libstdc++-12-dev libc6 libc6-dev libssl-dev pkg-config + + - name: Force rebuild icicle dependencies + run: cargo clean -p icicle-runtime -p icicle-core -p icicle-hash -p icicle-bn254 - name: Get release version from tag shell: bash @@ -79,7 +88,7 @@ jobs: sudo apt-get update - name: Build release binary - run: cargo build --release -Z sparse-registry --features icicle + run: cargo build --release -Z sparse-registry --features gpu-accelerated - name: Build archive shell: bash diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 792f89f3..a2e84fda 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -46,9 +46,11 @@ jobs: build: permissions: contents: read - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 env: EVM_VERIFIER_EZKL_TOKEN: ${{ secrets.EVM_VERIFIER_EZKL_TOKEN }} + RUSTFLAGS: "-C linker=gcc" + OPENSSL_NO_VENDOR: 1 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 with: @@ -60,13 +62,19 @@ jobs: toolchain: nightly-2025-05-01 override: true components: rustfmt, clippy + - name: install libc6 + run: sudo apt-get install -y libc6 + - name: Install cmake and build dependencies + run: sudo apt-get update && sudo apt-get install -y cmake build-essential g++ gcc libclang-dev llvm-dev libstdc++-12-dev libc6-dev libssl-dev pkg-config + - name: Force rebuild icicle dependencies + run: cargo clean -p icicle-runtime -p icicle-core -p icicle-hash -p icicle-bn254 - name: Build run: cargo build --verbose docs: permissions: contents: read - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 env: EVM_VERIFIER_EZKL_TOKEN: ${{ secrets.EVM_VERIFIER_EZKL_TOKEN }} steps: @@ -78,15 +86,21 @@ jobs: toolchain: nightly-2025-05-01 override: true components: rustfmt, clippy + - name: install libc6 + run: sudo apt-get install -y libc6 + - name: Install cmake and build dependencies + run: sudo apt-get update && sudo apt-get install -y cmake build-essential g++ gcc libclang-dev llvm-dev libstdc++-12-dev libc6-dev libssl-dev pkg-config - name: Docs run: cargo doc --verbose library-tests: permissions: contents: read - runs-on: ubuntu-latest-32-cores + runs-on: ubuntu-22.04 env: EVM_VERIFIER_EZKL_TOKEN: ${{ secrets.EVM_VERIFIER_EZKL_TOKEN }} + RUSTFLAGS: "-C linker=gcc" + OPENSSL_NO_VENDOR: 1 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 with: @@ -96,6 +110,12 @@ jobs: toolchain: nightly-2025-05-01 override: true components: rustfmt, clippy + - name: install libc6 + run: sudo apt-get install -y libc6 + - name: Install cmake and build dependencies + run: sudo apt-get update && sudo apt-get install -y cmake build-essential g++ gcc libclang-dev llvm-dev libstdc++-12-dev libc6-dev libssl-dev pkg-config + - name: Force rebuild icicle dependencies + run: cargo clean -p icicle-runtime -p icicle-core -p icicle-hash -p icicle-bn254 - uses: baptiste0928/cargo-install@91c5da15570085bcde6f4d7aed98cb82d6769fd3 #v3.3.0 with: crate: cargo-nextest @@ -105,53 +125,21 @@ jobs: run: cargo test --doc --verbose - name: Library tests run: cargo nextest run --lib --verbose - - name: Library tests (original lookup) - run: cargo nextest run --lib --verbose --no-default-features --features ezkl,eth-original-lookup - # ultra-overflow-tests-gpu: - # runs-on: GPU - # env: - # ENABLE_ICICLE_GPU: true - # steps: - # - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 - # with: - # persist-credentials: false - # - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 #v1.0.6 - # with: - # toolchain: nightly-2025-05-01 - # override: true - # components: rustfmt, clippy - # - uses: baptiste0928/cargo-install@91c5da15570085bcde6f4d7aed98cb82d6769fd3 #v3.3.0 - # with: - # crate: cargo-nextest - # locked: true - # - uses: mwilliamson/setup-wasmtime-action@bf814d7d8fc3c3a77dfe114bd9fb8a2c575f6ad6 #v2.0.0 - # with: - # wasmtime-version: "3.0.1" - # # - name: Matmul overflow (wasi) - # # run: cargo wasi test matmul_col_ultra_overflow -- --include-ignored --nocapture - # # - name: Conv overflow (wasi) - # # run: cargo wasi test conv_col_ultra_overflow -- --include-ignored --nocapture - # - name: lookup overflow - # run: cargo nextest run lookup_ultra_overflow --no-capture --features icicle -- --include-ignored - # - name: Matmul overflow - # run: RUST_LOG=debug cargo nextest run matmul_col_ultra_overflow --no-capture --features icicle -- --include-ignored - # - name: Conv overflow - # run: RUST_LOG=debug cargo nextest run conv_col_ultra_overflow --no-capture --features icicle -- --include-ignored - # - name: Conv + relu overflow - # run: cargo nextest run conv_relu_col_ultra_overflow --no-capture --features icicle -- --include-ignored - ultra-overflow-tests_og-lookup: + ultra-overflow-tests-gpu: permissions: contents: read - runs-on: [ non-gpu, non-sgx ] + runs-on: gpu env: + ENABLE_ICICLE_GPU: true EVM_VERIFIER_EZKL_TOKEN: ${{ secrets.EVM_VERIFIER_EZKL_TOKEN }} + RUSTFLAGS: "-C linker=gcc" + OPENSSL_NO_VENDOR: 1 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 with: persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 #v1.0.6 with: toolchain: nightly-2025-05-01 @@ -164,25 +152,36 @@ jobs: - uses: mwilliamson/setup-wasmtime-action@bf814d7d8fc3c3a77dfe114bd9fb8a2c575f6ad6 #v2.0.0 with: wasmtime-version: "3.0.1" + - name: Setup GPU dependencies + run: sudo ./setup-gpu.sh --yes + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential g++ gcc cmake libclang-dev llvm-dev libstdc++-12-dev libc6 libc6-dev libssl-dev pkg-config + - name: Force rebuild icicle dependencies + run: cargo clean -p icicle-runtime -p icicle-core -p icicle-hash -p icicle-bn254 # - name: Matmul overflow (wasi) # run: cargo wasi test matmul_col_ultra_overflow -- --include-ignored --nocapture # - name: Conv overflow (wasi) # run: cargo wasi test conv_col_ultra_overflow -- --include-ignored --nocapture - name: lookup overflow - run: cargo nextest run --release lookup_ultra_overflow --no-capture --no-default-features --features ezkl,eth-original-lookup -- --include-ignored + run: cargo nextest run lookup_ultra_overflow --no-capture --features gpu-accelerated -- --include-ignored - name: Matmul overflow - run: RUST_LOG=debug cargo nextest run --release matmul_col_ultra_overflow --no-capture --no-default-features --features ezkl,eth-original-lookup -- --include-ignored + run: RUST_LOG=debug cargo nextest run matmul_col_ultra_overflow --no-capture --features gpu-accelerated -- --include-ignored - name: Conv overflow - run: RUST_LOG=debug cargo nextest run --release conv_col_ultra_overflow --no-capture --no-default-features --features ezkl,eth-original-lookup -- --include-ignored + run: RUST_LOG=debug cargo nextest run conv_col_ultra_overflow --no-capture --features gpu-accelerated -- --include-ignored - name: Conv + relu overflow - run: cargo nextest run --release conv_relu_col_ultra_overflow --no-capture --no-default-features --features ezkl,eth-original-lookup -- --include-ignored + run: cargo nextest run conv_relu_col_ultra_overflow --no-capture --features gpu-accelerated -- --include-ignored ultra-overflow-tests: permissions: contents: read runs-on: [ non-gpu, non-sgx ] + needs: [build, library-tests, docs] env: EVM_VERIFIER_EZKL_TOKEN: ${{ secrets.EVM_VERIFIER_EZKL_TOKEN }} + RUSTFLAGS: "-C linker=gcc" + OPENSSL_NO_VENDOR: 1 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 @@ -195,7 +194,12 @@ jobs: toolchain: nightly-2025-05-01 override: true components: rustfmt, clippy - + - name: install libc6 + run: sudo apt-get install -y libc6 + - name: Install cmake and build dependencies + run: sudo apt-get update && sudo apt-get install -y cmake build-essential g++ gcc libclang-dev llvm-dev libstdc++-12-dev libc6-dev libssl-dev pkg-config + - name: Force rebuild icicle dependencies + run: cargo clean -p icicle-runtime -p icicle-core -p icicle-hash -p icicle-bn254 - uses: baptiste0928/cargo-install@91c5da15570085bcde6f4d7aed98cb82d6769fd3 #v3.3.0 with: crate: cargo-nextest @@ -219,9 +223,11 @@ jobs: model-serialization: permissions: contents: read - runs-on: ubuntu-latest-16-cores + runs-on: ubuntu-22.04 env: EVM_VERIFIER_EZKL_TOKEN: ${{ secrets.EVM_VERIFIER_EZKL_TOKEN }} + RUSTFLAGS: "-C linker=gcc" + OPENSSL_NO_VENDOR: 1 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 @@ -234,6 +240,12 @@ jobs: toolchain: nightly-2025-05-01 override: true components: rustfmt, clippy + - name: install libc6 + run: sudo apt-get install -y libc6 + - name: Install cmake and build dependencies + run: sudo apt-get update && sudo apt-get install -y cmake build-essential g++ gcc libclang-dev llvm-dev libstdc++-12-dev libc6-dev libssl-dev pkg-config + - name: Force rebuild icicle dependencies + run: cargo clean -p icicle-runtime -p icicle-core -p icicle-hash -p icicle-bn254 - uses: baptiste0928/cargo-install@91c5da15570085bcde6f4d7aed98cb82d6769fd3 #v3.3.0 with: crate: cargo-nextest @@ -244,11 +256,12 @@ jobs: wasm32-tests: permissions: contents: read - runs-on: ubuntu-latest-64-cores + runs-on: ubuntu-22.04 env: EVM_VERIFIER_EZKL_TOKEN: ${{ secrets.EVM_VERIFIER_EZKL_TOKEN }} # add `atomics` and `bulk-memory` to RUSTFLAGS to enable wasm-bindgen tests RUSTFLAGS: "-C target-feature=+atomics,+bulk-memory" + OPENSSL_NO_VENDOR: 1 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 @@ -261,10 +274,16 @@ jobs: toolchain: nightly-2025-05-01 override: true components: rustfmt, clippy + - name: install libc6 + run: sudo apt-get install -y libc6 + - name: Install cmake and build dependencies + run: sudo apt-get update && sudo apt-get install -y cmake build-essential g++ gcc libclang-dev llvm-dev libstdc++-12-dev libc6-dev libssl-dev pkg-config + - name: Force rebuild icicle dependencies + run: cargo clean -p icicle-runtime -p icicle-core -p icicle-hash -p icicle-bn254 - uses: jetli/wasm-pack-action@0d096b08b4e5a7de8c28de67e11e945404e9eefa #v0.4.0 with: - # Pin to version 0.12.1 - version: "v0.12.1" + # Pin to version 0.13.1 + version: "v0.13.1" - uses: nanasess/setup-chromedriver@affb1ea8848cbb080be372c1e8d7a5c173e9298f #v2.3.0 # with: # chromedriver-version: "115.0.5790.102" @@ -287,8 +306,11 @@ jobs: permissions: contents: read runs-on: [ non-gpu, non-sgx ] + needs: [build, library-tests, docs] env: EVM_VERIFIER_EZKL_TOKEN: ${{ secrets.EVM_VERIFIER_EZKL_TOKEN }} + RUSTFLAGS: "-C linker=gcc" + OPENSSL_NO_VENDOR: 1 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 @@ -304,6 +326,8 @@ jobs: with: crate: cargo-nextest locked: true + - name: Force rebuild icicle dependencies + run: cargo clean -p icicle-runtime -p icicle-core -p icicle-hash -p icicle-bn254 # - name: The Worm Mock # run: cargo nextest run --verbose tests::large_mock_::large_tests_5_expects -- --include-ignored - name: Large 1D Conv Mock @@ -357,10 +381,11 @@ jobs: permissions: contents: read runs-on: [ non-gpu, non-sgx ] - # needs: [build, library-tests, docs, python-tests, python-integration-tests] + needs: [build, library-tests, docs, python-tests, python-integration-tests] env: EVM_VERIFIER_EZKL_TOKEN: ${{ secrets.EVM_VERIFIER_EZKL_TOKEN }} RUSTFLAGS: "-C target-feature=+atomics,+bulk-memory" + OPENSSL_NO_VENDOR: 1 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 @@ -375,12 +400,14 @@ jobs: components: rustfmt, clippy - uses: jetli/wasm-pack-action@0d096b08b4e5a7de8c28de67e11e945404e9eefa #v0.4.0 with: - # Pin to version 0.12.1 - version: "v0.12.1" + # Pin to version 0.13.1 + version: "v0.13.1" - uses: baptiste0928/cargo-install@91c5da15570085bcde6f4d7aed98cb82d6769fd3 #v3.3.0 with: crate: cargo-nextest locked: true + - name: Force rebuild icicle dependencies + run: cargo clean -p icicle-runtime -p icicle-core -p icicle-hash -p icicle-bn254 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 with: persist-credentials: false @@ -388,10 +415,10 @@ jobs: uses: pnpm/action-setup@eae0cfeb286e66ffb5155f1a79b90583a127a68b #v2.4.1 with: version: 8 - - name: Use Node.js 18.12.1 + - name: Use Node.js 22.17.1 uses: actions/setup-node@1a4442cacd436585916779262731d5b162bc6ec7 #v3.8.2 with: - node-version: "18.12.1" + node-version: "22.17.1" cache: "pnpm" - name: "Add rust-src" run: rustup component add rust-src --toolchain nightly-2025-05-01-x86_64-unknown-linux-gnu @@ -438,8 +465,8 @@ jobs: # components: rustfmt, clippy # - uses: jetli/wasm-pack-action@0d096b08b4e5a7de8c28de67e11e945404e9eefa #v0.4.0 # with: - # # Pin to version 0.12.1 - # version: 'v0.12.1' + # # Pin to version 0.13.1 + # version: 'v0.13.1' # - name: Add rust-src # run: rustup component add rust-src --toolchain nightly-2025-05-01 # - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 @@ -460,10 +487,8 @@ jobs: permissions: contents: read runs-on: [ non-gpu, non-sgx ] - needs: [build, library-tests, docs] env: EVM_VERIFIER_EZKL_TOKEN: ${{ secrets.EVM_VERIFIER_EZKL_TOKEN }} - RUSTFLAGS: "-C target-feature=+atomics,+bulk-memory" steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 @@ -478,13 +503,15 @@ jobs: components: rustfmt, clippy - uses: jetli/wasm-pack-action@0d096b08b4e5a7de8c28de67e11e945404e9eefa #v0.4.0 with: - # Pin to version 0.12.1 - version: "v0.12.1" + # Pin to version 0.13.1 + version: "v0.13.1" - name: Add wasm32-unknown-unknown target run: rustup target add wasm32-unknown-unknown - name: Add rust-src run: rustup component add rust-src --toolchain nightly-2025-05-01-x86_64-unknown-linux-gnu + - name: Force rebuild icicle dependencies + run: cargo clean -p icicle-runtime -p icicle-core -p icicle-hash -p icicle-bn254 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 with: persist-credentials: false @@ -492,10 +519,10 @@ jobs: uses: pnpm/action-setup@eae0cfeb286e66ffb5155f1a79b90583a127a68b #v2.4.1 with: version: 8 - - name: Use Node.js 18.12.1 + - name: Use Node.js 22.17.1 uses: actions/setup-node@1a4442cacd436585916779262731d5b162bc6ec7 #v3.8.2 with: - node-version: "18.12.1" + node-version: "22.17.1" cache: "pnpm" - name: Install dependencies for js tests run: | @@ -507,9 +534,9 @@ jobs: with: crate: cargo-nextest locked: true - - name: Build wasm package for nodejs target. - run: | - wasm-pack build --target nodejs --out-dir ./tests/wasm/nodejs . -- -Z build-std="panic_abort,std" + # - name: Build wasm package for nodejs target. + # run: | + # wasm-pack build --target nodejs --out-dir ./tests/wasm/nodejs . -- -Z build-std="panic_abort,std" - name: KZG prove and verify tests (public outputs + column overflow) run: cargo nextest run --verbose tests::kzg_prove_and_verify_with_overflow_::w - name: KZG prove and verify tests (public outputs + fixed params + column overflow) @@ -518,10 +545,6 @@ jobs: run: cargo nextest run --verbose tests::kzg_prove_and_verify_with_overflow_hashed_inputs_ - name: KZG prove and verify tests (public outputs) run: cargo nextest run --verbose tests::kzg_prove_and_verify_tight_lookup_::t - - name: IPA prove and verify tests - run: cargo nextest run --verbose tests::ipa_prove_and_verify_::t --test-threads 1 - - name: IPA prove and verify tests (ipa outputs) - run: cargo nextest run --verbose tests::ipa_prove_and_verify_ipa_output - name: KZG prove and verify tests single inner col run: cargo nextest run --verbose tests::kzg_prove_and_verify_single_col - name: KZG prove and verify tests triple inner col @@ -541,42 +564,53 @@ jobs: - name: KZG prove and verify tests (hashed outputs) run: cargo nextest run --verbose tests::kzg_prove_and_verify_hashed - # prove-and-verify-tests-gpu: - # runs-on: GPU - # env: - # ENABLE_ICICLE_GPU: true - # steps: - # - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 - # with: - # persist-credentials: false - # - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 #v1.0.6 - # with: - # toolchain: nightly-2025-05-01 - # override: true - # components: rustfmt, clippy - # - name: Add rust-src - # run: rustup component add rust-src --toolchain nightly-2025-05-01-x86_64-unknown-linux-gnu - # - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 - # - uses: baptiste0928/cargo-install@91c5da15570085bcde6f4d7aed98cb82d6769fd3 #v3.3.0 - # with: - # crate: cargo-nextest - # locked: true - # - name: KZG prove and verify tests (kzg outputs) - # run: cargo nextest run --verbose tests::kzg_prove_and_verify_kzg_output --features icicle --test-threads 1 - # - name: KZG prove and verify tests (public outputs + column overflow) - # run: cargo nextest run --verbose tests::kzg_prove_and_verify_with_overflow_::w --features icicle --test-threads 1 - # - name: KZG prove and verify tests (public outputs + fixed params + column overflow) - # run: cargo nextest run --verbose tests::kzg_prove_and_verify_with_overflow_fixed_params_ --features icicle --test-threads 1 - # - name: KZG prove and verify tests (public outputs) - # run: cargo nextest run --verbose tests::kzg_prove_and_verify_::t --features icicle --test-threads 1 - # - name: KZG prove and verify tests (public outputs + column overflow) - # run: cargo nextest run --verbose tests::kzg_prove_and_verify_::t --features icicle --test-threads 1 - # - name: KZG prove and verify tests (public inputs) - # run: cargo nextest run --verbose tests::kzg_prove_and_verify_public_input --features icicle --test-threads 1 - # - name: KZG prove and verify tests (fixed params) - # run: cargo nextest run --verbose tests::kzg_prove_and_verify_fixed_params --features icicle --test-threads 1 - # - name: KZG prove and verify tests (hashed outputs) - # run: cargo nextest run --verbose tests::kzg_prove_and_verify_hashed --features icicle --test-threads 1 + prove-and-verify-tests-gpu: + permissions: + contents: read + runs-on: gpu + needs: [build, library-tests, docs] + env: + ENABLE_ICICLE_GPU: true + EVM_VERIFIER_EZKL_TOKEN: ${{ secrets.EVM_VERIFIER_EZKL_TOKEN }} + RUSTFLAGS: "-C linker=gcc" + OPENSSL_NO_VENDOR: 1 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 + with: + persist-credentials: false + - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 #v1.0.6 + with: + toolchain: nightly-2025-05-01 + override: true + components: rustfmt, clippy + - name: Add rust-src + run: rustup component add rust-src --toolchain nightly-2025-05-01-x86_64-unknown-linux-gnu + - uses: baptiste0928/cargo-install@91c5da15570085bcde6f4d7aed98cb82d6769fd3 #v3.3.0 + with: + crate: cargo-nextest + locked: true + - name: Setup GPU dependencies + run: sudo ./setup-gpu.sh --yes + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential g++ gcc cmake libclang-dev llvm-dev libstdc++-12-dev libc6 libc6-dev libssl-dev pkg-config + - name: Force rebuild icicle dependencies + run: cargo clean -p icicle-runtime -p icicle-core -p icicle-hash -p icicle-bn254 + - name: KZG prove and verify tests (kzg outputs) + run: cargo nextest run --verbose tests::kzg_prove_and_verify_kzg_output --features gpu-accelerated --test-threads 1 + - name: KZG prove and verify tests (public outputs + column overflow) + run: cargo nextest run --verbose tests::kzg_prove_and_verify_with_overflow_::w --features gpu-accelerated --test-threads 1 + - name: KZG prove and verify tests (public outputs + fixed params + column overflow) + run: cargo nextest run --verbose tests::kzg_prove_and_verify_with_overflow_fixed_params_ --features gpu-accelerated --test-threads 1 + - name: KZG prove and verify tests (public outputs) + run: cargo nextest run --verbose tests::kzg_prove_and_verify_::t --features gpu-accelerated --test-threads 1 + - name: KZG prove and verify tests (public inputs) + run: cargo nextest run --verbose tests::kzg_prove_and_verify_public_input --features gpu-accelerated --test-threads 1 + - name: KZG prove and verify tests (fixed params) + run: cargo nextest run --verbose tests::kzg_prove_and_verify_fixed_params --features gpu-accelerated --test-threads 1 + - name: KZG prove and verify tests (hashed outputs) + run: cargo nextest run --verbose tests::kzg_prove_and_verify_hashed --features gpu-accelerated --test-threads 1 prove-and-verify-mock-aggr-tests: permissions: @@ -604,25 +638,39 @@ jobs: - name: Mock aggr tests (KZG) run: cargo nextest run --verbose tests_aggr::kzg_aggr_mock_prove_and_verify_ --test-threads 8 - # prove-and-verify-aggr-tests-gpu: - # runs-on: GPU - # env: - # ENABLE_ICICLE_GPU: true - # steps: - # - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 - # with: - # persist-credentials: false - # - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 #v1.0.6 - # with: - # toolchain: nightly-2025-05-01 - # override: true - # components: rustfmt, clippy - # - uses: baptiste0928/cargo-install@91c5da15570085bcde6f4d7aed98cb82d6769fd3 #v3.3.0 - # with: - # crate: cargo-nextest - # locked: true - # - name: KZG tests - # run: cargo nextest run --verbose tests_aggr::kzg_aggr_prove_and_verify_ --features icicle --test-threads 1 -- --include-ignored + prove-and-verify-aggr-tests-gpu: + permissions: + contents: read + runs-on: gpu + needs: [build, library-tests, docs, python-tests, python-integration-tests] + env: + ENABLE_ICICLE_GPU: true + EVM_VERIFIER_EZKL_TOKEN: ${{ secrets.EVM_VERIFIER_EZKL_TOKEN }} + RUSTFLAGS: "-C linker=gcc" + OPENSSL_NO_VENDOR: 1 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 + with: + persist-credentials: false + - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 #v1.0.6 + with: + toolchain: nightly-2025-05-01 + override: true + components: rustfmt, clippy + - uses: baptiste0928/cargo-install@91c5da15570085bcde6f4d7aed98cb82d6769fd3 #v3.3.0 + with: + crate: cargo-nextest + locked: true + - name: Setup GPU dependencies + run: sudo ./setup-gpu.sh --yes + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential g++ gcc cmake libclang-dev llvm-dev libstdc++-12-dev libc6 libc6-dev libssl-dev pkg-config + - name: Force rebuild icicle dependencies + run: cargo clean -p icicle-runtime -p icicle-core -p icicle-hash -p icicle-bn254 + - name: KZG tests + run: cargo nextest run --verbose tests_aggr::kzg_aggr_prove_and_verify_ --features gpu-accelerated --test-threads 1 -- --include-ignored prove-and-verify-aggr-tests: permissions: @@ -683,10 +731,12 @@ jobs: examples: permissions: contents: read - runs-on: ubuntu-latest-32-cores + runs-on: ubuntu-22.04 needs: [build, library-tests, docs] env: EVM_VERIFIER_EZKL_TOKEN: ${{ secrets.EVM_VERIFIER_EZKL_TOKEN }} + RUSTFLAGS: "-C linker=gcc" + OPENSSL_NO_VENDOR: 1 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 @@ -699,6 +749,12 @@ jobs: toolchain: nightly-2025-05-01 override: true components: rustfmt, clippy + - name: install libc6 + run: sudo apt-get install -y libc6 + - name: Install cmake and build dependencies + run: sudo apt-get update && sudo apt-get install -y cmake build-essential g++ gcc libclang-dev llvm-dev libstdc++-12-dev libc6-dev libssl-dev pkg-config + - name: Force rebuild icicle dependencies + run: cargo clean -p icicle-runtime -p icicle-core -p icicle-hash -p icicle-bn254 - uses: baptiste0928/cargo-install@91c5da15570085bcde6f4d7aed98cb82d6769fd3 #v3.3.0 with: crate: cargo-nextest @@ -784,6 +840,7 @@ jobs: permissions: contents: read runs-on: large-self-hosted + needs: [build, library-tests, docs, python-tests] env: EVM_VERIFIER_EZKL_TOKEN: ${{ secrets.EVM_VERIFIER_EZKL_TOKEN }} @@ -849,6 +906,8 @@ jobs: runs-on: macos-latest env: EVM_VERIFIER_EZKL_TOKEN: ${{ secrets.EVM_VERIFIER_EZKL_TOKEN }} + RUSTFLAGS: "-C linker=gcc" + OPENSSL_NO_VENDOR: 1 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 @@ -865,6 +924,8 @@ jobs: with: crate: cargo-nextest locked: true + - name: Force rebuild icicle dependencies + run: cargo clean -p icicle-runtime -p icicle-core -p icicle-hash -p icicle-bn254 - name: Run ios tests run: CARGO_BUILD_TARGET=aarch64-apple-darwin RUSTUP_TOOLCHAIN=nightly-2025-05-01-aarch64-apple-darwin cargo test --test ios_integration_tests --features ios-bindings-test --no-default-features @@ -876,6 +937,8 @@ jobs: env: EVM_VERIFIER_EZKL_TOKEN: ${{ secrets.EVM_VERIFIER_EZKL_TOKEN }} + RUSTFLAGS: "-C linker=gcc" + OPENSSL_NO_VENDOR: 1 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 @@ -887,6 +950,8 @@ jobs: toolchain: nightly-2025-05-01 override: true components: rustfmt, clippy + - name: Force rebuild icicle dependencies + run: cargo clean -p icicle-runtime -p icicle-core -p icicle-hash -p icicle-bn254 - name: Build EzklCoreBindings run: CONFIGURATION=debug cargo run --bin ios_gen_bindings --features "ios-bindings uuid camino uniffi_bindgen" --no-default-features @@ -918,7 +983,7 @@ jobs: cd ezkl-swift-package xcodebuild test \ -scheme EzklPackage \ - -destination 'platform=iOS Simulator,name=iPhone 15 Pro,OS=17.5' \ + -destination 'platform=iOS Simulator,name=iPhone 16 Pro,OS=18.2' \ -resultBundlePath ../testResults - name: Run Example App Tests @@ -927,7 +992,7 @@ jobs: xcodebuild test \ -project Example.xcodeproj \ -scheme EzklApp \ - -destination 'platform=iOS Simulator,name=iPhone 15 Pro,OS=17.5' \ + -destination 'platform=iOS Simulator,name=iPhone 16 Pro,OS=18.2' \ -parallel-testing-enabled NO \ -resultBundlePath ../../exampleTestResults \ -skip-testing:EzklAppUITests/EzklAppUITests/testButtonClicksInOrder \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index b09d348d..5a156d0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -881,17 +881,6 @@ dependencies = [ "syn 2.0.101", ] -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] - [[package]] name = "aurora-engine-modexp" version = "1.2.0" @@ -970,29 +959,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bindgen" -version = "0.69.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" -dependencies = [ - "bitflags 2.9.0", - "cexpr", - "clang-sys", - "itertools 0.12.1", - "lazy_static", - "lazycell", - "log", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "rustc-hash 1.1.0", - "shlex", - "syn 2.0.101", - "which", -] - [[package]] name = "bit-set" version = "0.5.3" @@ -1211,15 +1177,6 @@ dependencies = [ "shlex", ] -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom", -] - [[package]] name = "cfg-if" version = "1.0.0" @@ -1270,29 +1227,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" dependencies = [ "ciborium-io", - "half 2.6.0", -] - -[[package]] -name = "clang-sys" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" -dependencies = [ - "glob", - "libc", - "libloading", -] - -[[package]] -name = "clap" -version = "2.34.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" -dependencies = [ - "bitflags 1.3.2", - "textwrap 0.11.0", - "unicode-width 0.1.14", + "half", ] [[package]] @@ -1323,7 +1258,7 @@ version = "4.5.47" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c06f5378ea264ad4f82bbc826628b5aad714a75abf6ece087e923010eb937fb6" dependencies = [ - "clap 4.5.37", + "clap", ] [[package]] @@ -1510,32 +1445,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "criterion" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f" -dependencies = [ - "atty", - "cast", - "clap 2.34.0", - "criterion-plot 0.4.5", - "csv", - "itertools 0.10.5", - "lazy_static", - "num-traits", - "oorandom", - "plotters", - "rayon", - "regex", - "serde", - "serde_cbor", - "serde_derive", - "serde_json", - "tinytemplate", - "walkdir", -] - [[package]] name = "criterion" version = "0.5.1" @@ -1545,8 +1454,8 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.5.37", - "criterion-plot 0.5.0", + "clap", + "criterion-plot", "is-terminal", "itertools 0.10.5", "num-traits", @@ -1562,16 +1471,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "criterion-plot" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876" -dependencies = [ - "cast", - "itertools 0.10.5", -] - [[package]] name = "criterion-plot" version = "0.5.0" @@ -1644,27 +1543,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "csv" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" -dependencies = [ - "csv-core", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "csv-core" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" -dependencies = [ - "memchr", -] - [[package]] name = "dashmap" version = "5.5.3" @@ -2065,12 +1943,12 @@ dependencies = [ "bincode", "camino", "chrono", - "clap 4.5.37", + "clap", "clap_complete", "colored", "colored_json", "console_error_panic_hook", - "criterion 0.5.1", + "criterion", "ecc", "env_logger 0.10.2", "ethabi", @@ -2082,6 +1960,7 @@ dependencies = [ "halo2_solidity_verifier", "halo2curves 0.7.0 (git+https://github.com/privacy-scaling-explorations/halo2curves?rev=b753a832e92d5c86c5c997327a9cf9de86a18851)", "hex", + "icicle-runtime", "indicatif", "instant", "itertools 0.10.5", @@ -2328,7 +2207,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" dependencies = [ - "rustix 1.0.5", + "rustix", "windows-sys 0.59.0", ] @@ -2521,12 +2400,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "half" -version = "1.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403" - [[package]] name = "half" version = "2.6.0" @@ -2541,7 +2414,7 @@ dependencies = [ [[package]] name = "halo2_gadgets" version = "0.2.0" -source = "git+https://github.com/zkonduit/halo2#f441c920be45f8f05d2c06a173d82e8885a5ed4d" +source = "git+https://github.com/zkonduit/halo2?branch=ac%2Fconditional-compilation-icicle2#1dd2090741f006fd031a07da7f3c9dfce5e0015e" dependencies = [ "arrayvec 0.7.6", "bitvec", @@ -2558,7 +2431,7 @@ dependencies = [ [[package]] name = "halo2_proofs" version = "0.3.0" -source = "git+https://github.com/zkonduit/halo2#f441c920be45f8f05d2c06a173d82e8885a5ed4d" +source = "git+https://github.com/zkonduit/halo2#1dd2090741f006fd031a07da7f3c9dfce5e0015e?branch=ac%2Fconditional-compilation-icicle2#1dd2090741f006fd031a07da7f3c9dfce5e0015e" dependencies = [ "bincode", "blake2b_simd", @@ -2568,7 +2441,7 @@ dependencies = [ "halo2curves 0.7.0 (git+https://github.com/privacy-scaling-explorations/halo2curves?rev=b753a832e92d5c86c5c997327a9cf9de86a18851)", "icicle-bn254", "icicle-core", - "icicle-cuda-runtime", + "icicle-runtime", "instant", "lazy_static", "log", @@ -2576,7 +2449,7 @@ dependencies = [ "mopro-msm", "rand_chacha 0.3.1", "rand_core 0.6.4", - "rustc-hash 2.1.1", + "rustc-hash", "serde", "sha3 0.9.1", "tracing", @@ -2791,15 +2664,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.3.9" @@ -2991,33 +2855,45 @@ dependencies = [ [[package]] name = "icicle-bn254" -version = "2.8.0" -source = "git+https://github.com/ingonyama-zk/icicle?branch=ezkl-icicle2#5dfe006a0f1bc62ea82ca297709bbf3d22a2ca25" +version = "3.7.0" +source = "git+https://github.com/ingonyama-zk/icicle?branch=emir%2Fgate_eval_2#012e00694f4cf399fe7a42d9cfbfa6cd7a60f876" dependencies = [ "cmake", - "criterion 0.3.6", "icicle-core", - "icicle-cuda-runtime", + "icicle-hash", + "icicle-runtime", ] [[package]] name = "icicle-core" -version = "2.8.0" -source = "git+https://github.com/ingonyama-zk/icicle?branch=ezkl-icicle2#5dfe006a0f1bc62ea82ca297709bbf3d22a2ca25" +version = "3.7.0" +source = "git+https://github.com/ingonyama-zk/icicle?branch=emir%2Fgate_eval_2#012e00694f4cf399fe7a42d9cfbfa6cd7a60f876" dependencies = [ - "criterion 0.3.6", "hex", - "icicle-cuda-runtime", + "icicle-runtime", + "once_cell", + "rand 0.8.5", "rayon", ] [[package]] -name = "icicle-cuda-runtime" -version = "2.8.0" -source = "git+https://github.com/ingonyama-zk/icicle?branch=ezkl-icicle2#5dfe006a0f1bc62ea82ca297709bbf3d22a2ca25" +name = "icicle-hash" +version = "3.7.0" +source = "git+https://github.com/ingonyama-zk/icicle?branch=emir%2Fgate_eval_2#012e00694f4cf399fe7a42d9cfbfa6cd7a60f876" dependencies = [ - "bindgen", - "bitflags 1.3.2", + "cmake", + "icicle-core", + "icicle-runtime", + "rand 0.8.5", +] + +[[package]] +name = "icicle-runtime" +version = "3.7.0" +source = "git+https://github.com/ingonyama-zk/icicle?branch=emir%2Fgate_eval_2#012e00694f4cf399fe7a42d9cfbfa6cd7a60f876" +dependencies = [ + "cmake", + "once_cell", ] [[package]] @@ -3470,28 +3346,12 @@ dependencies = [ "spin", ] -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "libc" version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" -[[package]] -name = "libloading" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" -dependencies = [ - "cfg-if", - "windows-targets 0.52.6", -] - [[package]] name = "libm" version = "0.2.13" @@ -3519,12 +3379,6 @@ dependencies = [ "redox_syscall", ] -[[package]] -name = "linux-raw-sys" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" - [[package]] name = "linux-raw-sys" version = "0.9.4" @@ -3963,7 +3817,7 @@ dependencies = [ "num-traits", "pyo3", "pyo3-build-config", - "rustc-hash 2.1.1", + "rustc-hash", ] [[package]] @@ -4394,16 +4248,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" -[[package]] -name = "prettyplease" -version = "0.2.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" -dependencies = [ - "proc-macro2", - "syn 2.0.101", -] - [[package]] name = "primal-check" version = "0.3.4" @@ -4656,7 +4500,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.1.1", + "rustc-hash", "rustls", "socket2", "thiserror 2.0.12", @@ -4675,7 +4519,7 @@ dependencies = [ "getrandom 0.3.2", "rand 0.9.1", "ring", - "rustc-hash 2.1.1", + "rustc-hash", "rustls", "rustls-pki-types", "slab", @@ -5173,12 +5017,6 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.1.1" @@ -5224,19 +5062,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "rustix" -version = "0.38.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" -dependencies = [ - "bitflags 2.9.0", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - [[package]] name = "rustix" version = "1.0.5" @@ -5246,7 +5071,7 @@ dependencies = [ "bitflags 2.9.0", "errno", "libc", - "linux-raw-sys 0.9.4", + "linux-raw-sys", "windows-sys 0.59.0", ] @@ -5494,16 +5319,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_cbor" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" -dependencies = [ - "half 1.8.3", - "serde", -] - [[package]] name = "serde_derive" version = "1.0.219" @@ -5950,7 +5765,7 @@ dependencies = [ "fastrand", "getrandom 0.3.2", "once_cell", - "rustix 1.0.5", + "rustix", "windows-sys 0.59.0", ] @@ -5996,15 +5811,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "textwrap" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -dependencies = [ - "unicode-width 0.1.14", -] - [[package]] name = "textwrap" version = "0.16.2" @@ -6398,7 +6204,7 @@ dependencies = [ "downcast-rs", "dyn-clone", "dyn-hash", - "half 2.6.0", + "half", "itertools 0.12.1", "lazy_static", "maplit", @@ -6433,7 +6239,7 @@ dependencies = [ "downcast-rs", "dyn-clone", "dyn-hash", - "half 2.6.0", + "half", "lazy_static", "liquid", "liquid-core", @@ -6614,7 +6420,7 @@ dependencies = [ "once_cell", "paste", "serde", - "textwrap 0.16.2", + "textwrap", "toml 0.5.11", "uniffi_meta", "uniffi_testing", @@ -6706,7 +6512,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cef408229a3a407fafa4c36dc4f6ece78a6fb258ab28d2b64bddd49c8cb680f6" dependencies = [ "anyhow", - "textwrap 0.16.2", + "textwrap", "uniffi_meta", "uniffi_testing", "weedle2", @@ -7046,18 +6852,6 @@ dependencies = [ "nom", ] -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix 0.38.44", -] - [[package]] name = "winapi" version = "0.3.9" @@ -7435,7 +7229,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d65cbf2f12c15564212d48f4e3dfb87923d25d611f2aed18f4cb23f0413d89e" dependencies = [ "libc", - "rustix 1.0.5", + "rustix", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 4a62a90a..a28f4b25 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,12 +16,12 @@ crate-type = ["cdylib", "rlib", "staticlib"] [dependencies] -halo2_gadgets = { git = "https://github.com/zkonduit/halo2" } +halo2_gadgets = { git = "https://github.com/zkonduit/halo2", branch= "ac/conditional-compilation-icicle2" } halo2curves = { git = "https://github.com/privacy-scaling-explorations/halo2curves", rev = "b753a832e92d5c86c5c997327a9cf9de86a18851", features = [ "derive_serde", ] } -halo2_proofs = { git = "https://github.com/zkonduit/halo2", package = "halo2_proofs", features = [ - "circuit-params", +halo2_proofs = { git = "https://github.com/zkonduit/halo2", package = "halo2_proofs", branch= "ac/conditional-compilation-icicle2", features = [ + "circuit-params", "mv-lookup" ] } rand = { version = "0.8", default-features = false } itertools = { version = "0.10.3", default-features = false } @@ -33,10 +33,10 @@ thiserror = { version = "1.0.38", default-features = false } hex = { version = "0.4.3", default-features = false } halo2_wrong_ecc = { git = "https://github.com/zkonduit/halo2wrong", branch = "ac/chunked-mv-lookup", package = "ecc" } snark-verifier = { git = "https://github.com/zkonduit/snark-verifier", branch = "ac/chunked-mv-lookup", features = [ - "derive_serde", + "derive_serde", "mv-lookup" ] } halo2_solidity_verifier = { git = "https://github.com/zkonduit/ezkl-verifier", branch = "main", optional = true, features = [ - "evm", + "evm", "mv-lookup", ] } maybe-rayon = { version = "0.1.1", default-features = false } bincode = { version = "1.3.3", default-features = false } @@ -103,6 +103,10 @@ uniffi_bindgen = { version = "=0.28.0", optional = true } camino = { version = "^1.1", optional = true } uuid = { version = "1.10.0", features = ["v4"], optional = true } +# GPU / device related things (optional - only enabled with gpu-accelerated feature) +icicle-runtime = { git = "https://github.com/ingonyama-zk/icicle", branch="emir/gate_eval_2", package="icicle-runtime", optional = true } + + [target.'cfg(not(all(target_arch = "wasm32", target_os = "unknown")))'.dependencies] colored = { version = "2.0.0", default-features = false, optional = true } env_logger = { version = "0.10.0", default-features = false, optional = true } @@ -222,7 +226,8 @@ required-features = ["python-bindings"] [features] web = ["wasm-bindgen-rayon"] default = [ - "eth-mv-lookup", + "eth", + "dep:halo2_solidity_verifier", "ezkl", "precompute-coset", "no-banner", @@ -236,7 +241,7 @@ universal-bindings = [ "mv-lookup", "precompute-coset", "parallel-poly-read", - "solidity-verifier-mv-lookup", + "dep:halo2_solidity_verifier" ] logging = ["dep:colored", "dep:env_logger", "dep:chrono"] ios-bindings = ["universal-bindings"] @@ -262,10 +267,6 @@ ezkl = [ "logging", ] eth = ["dep:alloy", "dep:foundry-compilers", "dep:ethabi"] -solidity-verifier = ["dep:halo2_solidity_verifier"] -solidity-verifier-mv-lookup = ["halo2_solidity_verifier/mv-lookup"] -eth-mv-lookup = ["solidity-verifier-mv-lookup", "mv-lookup", "eth"] -eth-original-lookup = ["eth", "solidity-verifier"] parallel-poly-read = [ "halo2_proofs/circuit-params", "halo2_proofs/parallel-poly-read", @@ -274,7 +275,7 @@ mv-lookup = ["halo2_proofs/mv-lookup", "snark-verifier/mv-lookup"] asm = ["halo2curves/asm", "halo2_proofs/asm"] precompute-coset = ["halo2_proofs/precompute-coset"] det-prove = [] -icicle = ["halo2_proofs/icicle_gpu"] +gpu-accelerated = ["halo2_proofs/gpu-accelerated", "dep:icicle-runtime"] empty-cmd = [] no-banner = [] no-update = [] @@ -285,9 +286,21 @@ mimalloc = ["dep:mimalloc"] reusable-verifier = [] +[patch.'https://github.com/zkonduit/halo2'] +halo2_proofs = { git = "https://github.com/zkonduit/halo2#1dd2090741f006fd031a07da7f3c9dfce5e0015e", package = "halo2_proofs", branch= "ac/conditional-compilation-icicle2", features = [ + "circuit-params", "mv-lookup" +] } + +[patch.'https://github.com/zkonduit/halo2#f441c920be45f8f05d2c06a173d82e8885a5ed4d'] +halo2_proofs = { git = "https://github.com/zkonduit/halo2#1dd2090741f006fd031a07da7f3c9dfce5e0015e", package = "halo2_proofs", branch= "ac/conditional-compilation-icicle2", features = [ + "circuit-params", "mv-lookup" +] } + + [patch.crates-io] uniffi_testing = { git = "https://github.com/ElusAegis/uniffi-rs", branch = "feat/testing-feature-build-fix" } + [profile.release] # debug = true rustflags = ["-C", "relocation-model=pic"] @@ -302,3 +315,5 @@ opt-level = 3 [package.metadata.wasm-pack.profile.release] wasm-opt = ["-O4", "--flexible-inline-max-function-size", "4294967295"] + + diff --git a/setup-gpu.sh b/setup-gpu.sh new file mode 100755 index 00000000..2716c8c4 --- /dev/null +++ b/setup-gpu.sh @@ -0,0 +1,229 @@ +#!/bin/bash + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Default installation directory +DEFAULT_INSTALL_DIR="/opt/icicle/lib/backend/halo2" + +# Halo2 repository details +HALO2_REPO="https://github.com/zkonduit/halo2" +HALO2_BRANCH="ac/conditional-compilation-icicle2" + +# Parse command line arguments +AUTO_YES=false +for arg in "$@"; do + case $arg in + -y|--yes) + AUTO_YES=true + shift + ;; + -h|--help) + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " -y, --yes Automatically answer 'yes' to all prompts" + echo " -h, --help Show this help message" + exit 0 + ;; + *) + echo "Unknown option: $arg" + echo "Use -h or --help for usage information" + exit 1 + ;; + esac +done + +echo -e "${GREEN}EZKL GPU Setup Script${NC}" +echo -e "${GREEN}=====================${NC}" +echo "" + +# Parse commit hash from Cargo.lock +echo "Parsing halo2 commit hash from Cargo.lock..." +if [ ! -f "Cargo.lock" ]; then + echo -e "${RED}Error: Cargo.lock not found. Please run this script from the project root.${NC}" + exit 1 +fi + +HALO2_COMMIT=$(grep "github\.com/zkonduit/halo2?" Cargo.lock | grep -v "halo2wrong" | head -1 | grep -o "#[a-f0-9]\{40\}" | cut -c2-) + +if [ -z "$HALO2_COMMIT" ]; then + echo -e "${RED}Error: Could not parse halo2 commit hash from Cargo.lock${NC}" + exit 1 +fi + +echo -e "${GREEN}Found halo2 commit: $HALO2_COMMIT${NC}" +echo "" +echo "This script will:" +echo "1. Sparse checkout the halo2 repository at commit $HALO2_COMMIT" +echo "2. Extract only the icicle/backend/cuda/ directory" +echo "3. Set the ICICLE_BACKEND_INSTALL_DIR environment variable" +echo "" + +# Check if user wants to override the default directory +if [ "$AUTO_YES" = true ]; then + INSTALL_DIR="$DEFAULT_INSTALL_DIR" + echo -e "${GREEN}Using default installation directory: ${INSTALL_DIR}${NC}" +else + echo -e "${YELLOW}Default installation directory: ${DEFAULT_INSTALL_DIR}${NC}" + read -p "Do you want to use a different directory? [y/N]: " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + read -p "Enter the installation directory: " INSTALL_DIR + INSTALL_DIR="${INSTALL_DIR/#\~/$HOME}" # Expand ~ to $HOME + else + INSTALL_DIR="$DEFAULT_INSTALL_DIR" + fi + + # Confirm the installation directory + echo "" + echo -e "${YELLOW}Installation directory: ${INSTALL_DIR}${NC}" + read -p "Continue with this directory? [y/N]: " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo -e "${RED}Setup cancelled by user.${NC}" + exit 1 + fi +fi + +# Check if ICICLE_BACKEND_INSTALL_DIR is already set +if [ ! -z "$ICICLE_BACKEND_INSTALL_DIR" ] && [ "$AUTO_YES" = false ]; then + echo "" + echo -e "${YELLOW}Warning: ICICLE_BACKEND_INSTALL_DIR is already set to: $ICICLE_BACKEND_INSTALL_DIR${NC}" + read -p "Do you want to override it? [y/N]: " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo -e "${RED}Setup cancelled by user.${NC}" + exit 1 + fi +elif [ ! -z "$ICICLE_BACKEND_INSTALL_DIR" ] && [ "$AUTO_YES" = true ]; then + echo -e "${GREEN}Overriding existing ICICLE_BACKEND_INSTALL_DIR (was: $ICICLE_BACKEND_INSTALL_DIR)${NC}" +fi + +echo "" +echo -e "${GREEN}Starting GPU setup...${NC}" + +# Create installation directory +echo "Creating installation directory..." +mkdir -p "$INSTALL_DIR" + +# Create temporary directory for sparse checkout +TEMP_DIR=$(mktemp -d) +echo "Using temporary directory: $TEMP_DIR" + +# Clone with sparse checkout +echo "Cloning halo2 repository with sparse checkout..." +cd "$TEMP_DIR" +git clone --filter=blob:none --sparse "$HALO2_REPO" halo2 +cd halo2 + +# Checkout the specific branch and commit +echo "Checking out branch $HALO2_BRANCH at commit $HALO2_COMMIT..." +git checkout "$HALO2_BRANCH" +git checkout "$HALO2_COMMIT" + +# Configure sparse checkout +echo "Configuring sparse checkout for icicle/backend/cuda/..." +git sparse-checkout init --cone +git sparse-checkout set icicle/backend/cuda/ + +# Copy the icicle directory to the installation location +if [ -d "icicle/backend/cuda" ]; then + echo "Copying icicle/backend/cuda/ to $INSTALL_DIR..." + cp -r icicle/backend/cuda/* "$INSTALL_DIR/" + echo -e "${GREEN}Files copied successfully!${NC}" +else + echo -e "${RED}Error: icicle/backend/cuda directory not found in the repository${NC}" + exit 1 +fi + +# Clean up temporary directory +echo "Cleaning up temporary files..." +rm -rf "$TEMP_DIR" + +# Ask user about setting environment variable permanently +SETUP_PERMANENT_ENV=false +if [ "$AUTO_YES" = true ]; then + SETUP_PERMANENT_ENV=true + echo "" + echo -e "${GREEN}Setting ICICLE_BACKEND_INSTALL_DIR environment variable permanently...${NC}" +else + echo "" + echo -e "${YELLOW}Do you want to set ICICLE_BACKEND_INSTALL_DIR environment variable permanently?${NC}" + echo "This will add 'export ICICLE_BACKEND_INSTALL_DIR=\"$INSTALL_DIR\"' to your shell configuration file." + read -p "Set environment variable permanently? [y/N]: " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + SETUP_PERMANENT_ENV=true + fi +fi + +if [ "$SETUP_PERMANENT_ENV" = true ]; then + echo "Setting ICICLE_BACKEND_INSTALL_DIR environment variable..." + + # Detect shell and set environment variable accordingly + if [ -n "$ZSH_VERSION" ]; then + SHELL_RC="$HOME/.zshrc" + elif [ -n "$BASH_VERSION" ]; then + SHELL_RC="$HOME/.bashrc" + else + # Try to detect based on $SHELL + case "$SHELL" in + */zsh) + SHELL_RC="$HOME/.zshrc" + ;; + */bash) + SHELL_RC="$HOME/.bashrc" + ;; + *) + SHELL_RC="$HOME/.profile" + ;; + esac + fi + + # Add environment variable to shell configuration + ENV_EXPORT="export ICICLE_BACKEND_INSTALL_DIR=\"$INSTALL_DIR\"" + + # Check if the variable is already set in the file + if [ -f "$SHELL_RC" ] && grep -q "ICICLE_BACKEND_INSTALL_DIR" "$SHELL_RC"; then + # Replace existing line + if [[ "$OSTYPE" == "darwin"* ]]; then + # macOS + sed -i '' "s|export ICICLE_BACKEND_INSTALL_DIR=.*|$ENV_EXPORT|" "$SHELL_RC" + else + # Linux + sed -i "s|export ICICLE_BACKEND_INSTALL_DIR=.*|$ENV_EXPORT|" "$SHELL_RC" + fi + echo "Updated existing ICICLE_BACKEND_INSTALL_DIR in $SHELL_RC" + else + # Add new line + echo "$ENV_EXPORT" >> "$SHELL_RC" + echo "Added ICICLE_BACKEND_INSTALL_DIR to $SHELL_RC" + fi + + echo -e "${GREEN}Environment variable set permanently.${NC}" +else + echo "Skipping permanent environment variable setup." +fi + +# Export for current session regardless +export ICICLE_BACKEND_INSTALL_DIR="$INSTALL_DIR" +echo "Environment variable set for current session." + +echo "" +echo -e "${GREEN}GPU setup completed successfully!${NC}" +echo "" +echo -e "${YELLOW}Important:${NC}" +echo "1. The ICICLE_BACKEND_INSTALL_DIR environment variable has been set to: $INSTALL_DIR" +if [ "$SETUP_PERMANENT_ENV" = true ]; then + echo "2. Please restart your terminal or run: source $SHELL_RC" +else + echo "2. To use GPU features, set: export ICICLE_BACKEND_INSTALL_DIR=\"$INSTALL_DIR\"" +fi +echo "3. You can now build with GPU support using: cargo build --features gpu-accelerated" +echo "" +echo -e "${GREEN}Setup complete!${NC}" \ No newline at end of file diff --git a/src/bin/ezkl.rs b/src/bin/ezkl.rs index d48d26ff..2ad5b479 100644 --- a/src/bin/ezkl.rs +++ b/src/bin/ezkl.rs @@ -15,9 +15,6 @@ use log::{error, info}; #[cfg(not(any(target_arch = "wasm32", feature = "no-banner")))] use rand::prelude::SliceRandom; #[cfg(all(feature = "ezkl", not(target_arch = "wasm32")))] -#[cfg(feature = "icicle")] -use std::env; - #[tokio::main(flavor = "current_thread")] #[cfg(all(feature = "ezkl", not(target_arch = "wasm32")))] pub async fn main() { @@ -31,12 +28,7 @@ pub async fn main() { init_logger(); #[cfg(not(any(target_arch = "wasm32", feature = "no-banner")))] banner(); - #[cfg(feature = "icicle")] - if env::var("ENABLE_ICICLE_GPU").is_ok() { - info!("Running with ICICLE GPU"); - } else { - info!("Running with CPU"); - } + debug!( "command: \n {}", &command.as_json().to_colored_json_auto().unwrap() diff --git a/src/bindings/python.rs b/src/bindings/python.rs index 8d19bf73..c7a5234e 100644 --- a/src/bindings/python.rs +++ b/src/bindings/python.rs @@ -93,7 +93,6 @@ impl From for G1 { } } - /// pyclass containing the struct used for G1 #[pyclass] #[derive(Debug, Clone)] @@ -125,7 +124,6 @@ impl From for G1Affine { } } - /// Python class containing the struct used for run_args /// /// Returns diff --git a/src/circuit/ops/chip.rs b/src/circuit/ops/chip.rs index 7c5ebd29..1bc73bfa 100644 --- a/src/circuit/ops/chip.rs +++ b/src/circuit/ops/chip.rs @@ -7,12 +7,7 @@ use halo2_proofs::{ }; use log::debug; #[cfg(feature = "python-bindings")] -use pyo3::{ - conversion::FromPyObject, - exceptions::PyValueError, - IntoPyObject, - prelude::*, -}; +use pyo3::{conversion::FromPyObject, exceptions::PyValueError, prelude::*, IntoPyObject}; use serde::{Deserialize, Serialize}; #[cfg(all(feature = "ezkl", not(target_arch = "wasm32")))] use tosubcommand::ToFlags; diff --git a/src/circuit/ops/mod.rs b/src/circuit/ops/mod.rs index 65ea2e33..005b0da5 100644 --- a/src/circuit/ops/mod.rs +++ b/src/circuit/ops/mod.rs @@ -364,7 +364,15 @@ impl< }; Ok(Some(if self.decomp { log::debug!("constraining constant to be decomp"); - super::layouts::decompose(config, region, &[&value], ®ion.base(), ®ion.legs(), false)?.1 + super::layouts::decompose( + config, + region, + &[&value], + ®ion.base(), + ®ion.legs(), + false, + )? + .1 } else { log::debug!("constraining constant to be identity"); super::layouts::identity(config, region, &[&value])? diff --git a/src/commands.rs b/src/commands.rs index c6f68215..d5fc5fb2 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -269,9 +269,7 @@ impl<'py> IntoPyObject<'py> for CalibrationTarget { fn into_pyobject(self, py: Python<'py>) -> Result { let result = match self { - CalibrationTarget::Resources { col_overflow: true } => { - "resources/col-overflow" - } + CalibrationTarget::Resources { col_overflow: true } => "resources/col-overflow", CalibrationTarget::Resources { col_overflow: false, } => "resources", diff --git a/src/execute.rs b/src/execute.rs index 8357d443..3bb6a9b4 100644 --- a/src/execute.rs +++ b/src/execute.rs @@ -26,6 +26,8 @@ use colored::Colorize; #[cfg(unix)] use gag::Gag; use halo2_proofs::dev::VerifyFailure; +#[cfg(feature = "gpu-accelerated")] +use halo2_proofs::icicle::try_load_and_set_backend_device; use halo2_proofs::plonk::{self, Circuit}; use halo2_proofs::poly::commitment::{CommitmentScheme, Params}; use halo2_proofs::poly::commitment::{ParamsProver, Verifier}; @@ -46,6 +48,8 @@ use halo2_solidity_verifier; use halo2curves::bn256::{Bn256, Fr, G1Affine}; use halo2curves::ff::{FromUniformBytes, WithSmallOrderMulGroup}; use halo2curves::serde::SerdeObject; +#[cfg(feature = "gpu-accelerated")] +use icicle_runtime::{stream::IcicleStream, warmup}; use indicatif::{ProgressBar, ProgressStyle}; use instant::Instant; use itertools::Itertools; @@ -87,6 +91,22 @@ lazy_static! { } +/// Set the device used for computation. +#[cfg(feature = "gpu-accelerated")] +pub fn set_device() { + if std::env::var("ICICLE_BACKEND_INSTALL_DIR").is_ok() { + info!("Running with ICICLE GPU"); + try_load_and_set_backend_device("CUDA"); + match warmup(&IcicleStream::default()) { + Ok(_) => info!("GPU warmed :)"), + Err(e) => log::error!("GPU warmup failed: {:?}", e), + } + } else { + info!("Running with CPU: 'ICICLE_BACKEND_INSTALL_DIR' not set"); + try_load_and_set_backend_device("CPU"); + } +} + /// A wrapper for execution errors #[derive(Debug, Error)] pub enum ExecutionError { @@ -108,6 +128,8 @@ lazy_static::lazy_static! { /// Run an ezkl command with given args pub async fn run(command: Commands) -> Result { + #[cfg(feature = "gpu-accelerated")] + set_device(); // set working dir std::env::set_current_dir(WORKING_DIR.as_path())?; diff --git a/src/graph/input.rs b/src/graph/input.rs index 324f4f18..be98a36f 100644 --- a/src/graph/input.rs +++ b/src/graph/input.rs @@ -427,8 +427,6 @@ mod tests { } } - - #[cfg(feature = "python-bindings")] use crate::pfsys::field_to_string; @@ -443,13 +441,9 @@ impl<'py> IntoPyObject<'py> for FileSourceInner { FileSourceInner::Field(data) => { let s = field_to_string(&data); Ok(pyo3::types::PyString::new(py, &s).into_any()) - }, - FileSourceInner::Bool(data) => { - Ok(pyo3::types::PyBool::new(py, data).as_any().clone()) - }, - FileSourceInner::Float(data) => { - Ok(pyo3::types::PyFloat::new(py, data).into_any()) - }, + } + FileSourceInner::Bool(data) => Ok(pyo3::types::PyBool::new(py, data).as_any().clone()), + FileSourceInner::Float(data) => Ok(pyo3::types::PyFloat::new(py, data).into_any()), } } } diff --git a/src/graph/mod.rs b/src/graph/mod.rs index 64297f68..231af48d 100644 --- a/src/graph/mod.rs +++ b/src/graph/mod.rs @@ -844,24 +844,60 @@ impl<'de> Deserialize<'de> for GraphSettings { use serde::de::Error; // For bincode compatibility, deserialize in the same order as tuple serialization - let run_args = seq.next_element()?.ok_or_else(|| Error::invalid_length(0, &self))?; - let num_rows = seq.next_element()?.ok_or_else(|| Error::invalid_length(1, &self))?; - let total_assignments = seq.next_element()?.ok_or_else(|| Error::invalid_length(2, &self))?; - let total_const_size = seq.next_element()?.ok_or_else(|| Error::invalid_length(3, &self))?; - let dynamic_lookup_params = seq.next_element()?.ok_or_else(|| Error::invalid_length(4, &self))?; - let shuffle_params = seq.next_element()?.ok_or_else(|| Error::invalid_length(5, &self))?; - let model_instance_shapes = seq.next_element()?.ok_or_else(|| Error::invalid_length(6, &self))?; - let model_output_scales = seq.next_element()?.ok_or_else(|| Error::invalid_length(7, &self))?; - let model_input_scales = seq.next_element()?.ok_or_else(|| Error::invalid_length(8, &self))?; - let module_sizes = seq.next_element()?.ok_or_else(|| Error::invalid_length(9, &self))?; - let required_lookups = seq.next_element()?.ok_or_else(|| Error::invalid_length(10, &self))?; - let required_range_checks = seq.next_element()?.ok_or_else(|| Error::invalid_length(11, &self))?; - let check_mode = seq.next_element()?.ok_or_else(|| Error::invalid_length(12, &self))?; - let version = seq.next_element()?.ok_or_else(|| Error::invalid_length(13, &self))?; - let num_blinding_factors = seq.next_element()?.ok_or_else(|| Error::invalid_length(14, &self))?; - let timestamp = seq.next_element()?.ok_or_else(|| Error::invalid_length(15, &self))?; - let input_types = seq.next_element()?.ok_or_else(|| Error::invalid_length(16, &self))?; - let output_types = seq.next_element()?.ok_or_else(|| Error::invalid_length(17, &self))?; + let run_args = seq + .next_element()? + .ok_or_else(|| Error::invalid_length(0, &self))?; + let num_rows = seq + .next_element()? + .ok_or_else(|| Error::invalid_length(1, &self))?; + let total_assignments = seq + .next_element()? + .ok_or_else(|| Error::invalid_length(2, &self))?; + let total_const_size = seq + .next_element()? + .ok_or_else(|| Error::invalid_length(3, &self))?; + let dynamic_lookup_params = seq + .next_element()? + .ok_or_else(|| Error::invalid_length(4, &self))?; + let shuffle_params = seq + .next_element()? + .ok_or_else(|| Error::invalid_length(5, &self))?; + let model_instance_shapes = seq + .next_element()? + .ok_or_else(|| Error::invalid_length(6, &self))?; + let model_output_scales = seq + .next_element()? + .ok_or_else(|| Error::invalid_length(7, &self))?; + let model_input_scales = seq + .next_element()? + .ok_or_else(|| Error::invalid_length(8, &self))?; + let module_sizes = seq + .next_element()? + .ok_or_else(|| Error::invalid_length(9, &self))?; + let required_lookups = seq + .next_element()? + .ok_or_else(|| Error::invalid_length(10, &self))?; + let required_range_checks = seq + .next_element()? + .ok_or_else(|| Error::invalid_length(11, &self))?; + let check_mode = seq + .next_element()? + .ok_or_else(|| Error::invalid_length(12, &self))?; + let version = seq + .next_element()? + .ok_or_else(|| Error::invalid_length(13, &self))?; + let num_blinding_factors = seq + .next_element()? + .ok_or_else(|| Error::invalid_length(14, &self))?; + let timestamp = seq + .next_element()? + .ok_or_else(|| Error::invalid_length(15, &self))?; + let input_types = seq + .next_element()? + .ok_or_else(|| Error::invalid_length(16, &self))?; + let output_types = seq + .next_element()? + .ok_or_else(|| Error::invalid_length(17, &self))?; Ok(GraphSettings { run_args, @@ -884,20 +920,35 @@ impl<'de> Deserialize<'de> for GraphSettings { output_types, }) } - } // Universal deserializer that works with both JSON (map) and bincode (tuple) if deserializer.is_human_readable() { // JSON format - use struct/map deserialization with flattened fields const FIELDS: &'static [&'static str] = &[ - "run_args", "num_rows", "total_assignments", "total_const_size", - "total_dynamic_col_size", "max_dynamic_input_len", "num_dynamic_lookups", - "num_shuffles", "total_shuffle_col_size", "model_instance_shapes", - "model_output_scales", "model_input_scales", "module_sizes", - "required_lookups", "required_range_checks", "check_mode", "version", - "num_blinding_factors", "timestamp", "input_types", "output_types", - "dynamic_lookup_params", "shuffle_params", + "run_args", + "num_rows", + "total_assignments", + "total_const_size", + "total_dynamic_col_size", + "max_dynamic_input_len", + "num_dynamic_lookups", + "num_shuffles", + "total_shuffle_col_size", + "model_instance_shapes", + "model_output_scales", + "model_input_scales", + "module_sizes", + "required_lookups", + "required_range_checks", + "check_mode", + "version", + "num_blinding_factors", + "timestamp", + "input_types", + "output_types", + "dynamic_lookup_params", + "shuffle_params", ]; deserializer.deserialize_struct("GraphSettings", FIELDS, GraphSettingsVisitor) } else { @@ -2162,7 +2213,6 @@ pub mod tests { let deserialized: GraphSettings = serde_json::from_str(&json_str).unwrap(); assert_eq!(original, deserialized); - // now do JSON bytes let json_bytes = serde_json::to_vec(&original).unwrap(); let deserialized_from_bytes: GraphSettings = serde_json::from_slice(&json_bytes).unwrap(); @@ -2253,6 +2303,5 @@ pub mod tests { }"#; let _backwards_compatible: GraphSettings = serde_json::from_str(old_format_json).unwrap(); - } } diff --git a/src/graph/vars.rs b/src/graph/vars.rs index f865420f..7c6a8bb0 100644 --- a/src/graph/vars.rs +++ b/src/graph/vars.rs @@ -8,9 +8,7 @@ use halo2curves::ff::PrimeField; use itertools::Itertools; use log::debug; #[cfg(feature = "python-bindings")] -use pyo3::{ - exceptions::PyValueError, FromPyObject, IntoPyObject, PyResult, Python, -}; +use pyo3::{exceptions::PyValueError, FromPyObject, IntoPyObject, PyResult, Python}; use serde::{Deserialize, Serialize}; #[cfg(all(feature = "ezkl", not(target_arch = "wasm32")))] use tosubcommand::ToFlags; @@ -131,7 +129,9 @@ impl<'py> IntoPyObject<'py> for Visibility { .map(|o| o.to_string()) .collect_vec() .join(","); - Ok(format!("hashed/private/{}", outlets).into_pyobject(py)?.into_any()) + Ok(format!("hashed/private/{}", outlets) + .into_pyobject(py)? + .into_any()) } } } diff --git a/src/pfsys/mod.rs b/src/pfsys/mod.rs index 710b4b75..de6b97ce 100644 --- a/src/pfsys/mod.rs +++ b/src/pfsys/mod.rs @@ -314,7 +314,6 @@ impl ToFlags for TranscriptType { } } - #[cfg(feature = "python-bindings")] /// pub fn g1affine_to_pydict(g1affine_dict: &pyo3::Bound<'_, PyDict>, g1affine: &G1Affine) { @@ -404,7 +403,8 @@ where #[cfg(feature = "python-bindings")] use pyo3::{types::PyDict, IntoPyObject, Python}; #[cfg(feature = "python-bindings")] -impl<'py, F: PrimeField + SerdeObject + Serialize, C: CurveAffine + Serialize> IntoPyObject<'py> for Snark +impl<'py, F: PrimeField + SerdeObject + Serialize, C: CurveAffine + Serialize> IntoPyObject<'py> + for Snark where C::Scalar: Serialize + DeserializeOwned, C::ScalarExt: Serialize + DeserializeOwned, diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 39a02358..b3cc1a1d 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -163,10 +163,16 @@ mod native_tests { let data = GraphData::from_path(format!("{}/{}/input.json", test_dir, test).into()) .expect("failed to load input data"); - let duplicated_input_data = data.input_data.into_iter().map(|input| { - (0..num_batches) - .map(move |_| input.clone()).flatten().collect::>() - }).collect::>(); + let duplicated_input_data = data + .input_data + .into_iter() + .map(|input| { + (0..num_batches) + .map(move |_| input.clone()) + .flatten() + .collect::>() + }) + .collect::>(); let duplicated_data = GraphData::new(duplicated_input_data); @@ -307,63 +313,63 @@ mod native_tests { "integer_div", // 98 ]; - const WASM_TESTS: [&str; 46] = [ - "1l_mlp", - "1l_slice", - "1l_concat", - "1l_flatten", + const WASM_TESTS: [&str; 44] = [ + "1l_mlp", // 0 + "1l_slice", // 1 + "1l_concat", // 2 + "1l_flatten", // 3 // "1l_average", - "1l_div", - "1l_pad", - "1l_reshape", - "1l_eltwise_div", - "1l_sigmoid", - "1l_sqrt", - "1l_softmax", + "1l_div", // 4 + "1l_pad", // 5 + "1l_reshape", // 6 + "1l_eltwise_div", // 7 + "1l_sigmoid", // 8 + "1l_sqrt", // 9 + "1l_softmax", // 10 // "1l_instance_norm", - "1l_batch_norm", - "1l_prelu", - "1l_leakyrelu", - "1l_gelu_noappx", + "1l_batch_norm", // 11 + "1l_prelu", // 12 + "1l_leakyrelu", // 13 + "1l_gelu_noappx", // 14 // "1l_gelu_tanh_appx", - "1l_relu", - "1l_downsample", - "1l_tanh", - "2l_relu_sigmoid_small", - "2l_relu_fc", - "2l_relu_small", - "2l_relu_sigmoid", - "1l_conv", - "2l_sigmoid_small", - "2l_relu_sigmoid_conv", - "3l_relu_conv_fc", - "4l_relu_conv_fc", - "1l_erf", - "1l_var", - "1l_elu", - "min", - "max", - "1l_max_pool", - "1l_conv_transpose", - "1l_upsample", - "1l_identity", + "1l_relu", // 15 + "1l_downsample", // 16 + "1l_tanh", // 17 + "2l_relu_sigmoid_small", // 18 + "2l_relu_fc", // 19 + "2l_relu_small", // 20 + "2l_relu_sigmoid", // 21 + "1l_conv", // 22 + "2l_sigmoid_small", // 23 + "2l_relu_sigmoid_conv", // 24 + // "3l_relu_conv_fc", + // "4l_relu_conv_fc", + "1l_erf", // 25 + "1l_var", // 26 + "1l_elu", // 27 + "min", // 28 + "max", // 29 + "1l_max_pool", // 30 + "1l_conv_transpose", // 31 + "1l_upsample", // 32 + "1l_identity", // 33 // "idolmodel", - "trig", - "prelu_gmm", - "lstm", - "rnn", - "quantize_dequantize", - "1l_where", - "boolean", - "boolean_identity", - "gradient_boosted_trees", - "1l_topk", - // "xgboost", - // "lightgbm", - // "hummingbird_decision_tree", + "trig", // 34 + "prelu_gmm", // 35 + "lstm", // 36 + "rnn", // 37 + "quantize_dequantize", // 38 + "1l_where", // 39 + "boolean", // 40 + "boolean_identity", // 41 + "gradient_boosted_trees", // 42 + "1l_topk", // 43 + // "xgboost", + // "lightgbm", + // "hummingbird_decision_tree", ]; - #[cfg(not(feature = "icicle"))] + #[cfg(not(feature = "gpu-accelerated"))] const TESTS_AGGR: [&str; 21] = [ "1l_mlp", "1l_flatten", @@ -388,7 +394,7 @@ mod native_tests { "1l_max_pool", ]; - #[cfg(feature = "icicle")] + #[cfg(feature = "gpu-accelerated")] const TESTS_AGGR: [&str; 3] = ["1l_mlp", "1l_flatten", "1l_average"]; const TESTS_EVM: [&str; 23] = [ @@ -448,11 +454,12 @@ mod native_tests { use crate::native_tests::TESTS_AGGR; use test_case::test_case; use crate::native_tests::aggr_prove_and_verify; + #[cfg(not(feature = "gpu-accelerated"))] use crate::native_tests::kzg_aggr_mock_prove_and_verify; use tempdir::TempDir; use ezkl::Commitments; - #[cfg(not(feature="icicle"))] + #[cfg(not(feature="gpu-accelerated"))] seq!(N in 0..=20 { #(#[test_case(TESTS_AGGR[N])])* @@ -486,7 +493,7 @@ mod native_tests { }); - #[cfg(feature="icicle")] + #[cfg(feature="gpu-accelerated")] seq!(N in 0..=2 { #(#[test_case(TESTS_AGGR[N])])* fn kzg_aggr_prove_and_verify_(test: &str) { @@ -514,8 +521,8 @@ mod native_tests { use crate::native_tests::mock; use crate::native_tests::accuracy_measurement; use crate::native_tests::prove_and_verify; - use crate::native_tests::run_js_tests; - use crate::native_tests::render_circuit; + // use crate::native_tests::run_js_tests; + // use crate::native_tests::render_circuit; use crate::native_tests::model_serialization_different_binaries; use tempdir::TempDir; @@ -545,15 +552,15 @@ mod native_tests { seq!(N in 0..=98 { - #(#[test_case(TESTS[N])])* - #[ignore] - fn render_circuit_(test: &str) { - crate::native_tests::init_binary(); - let test_dir = TempDir::new(test).unwrap(); - let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); - render_circuit(path, test.to_string()); - test_dir.close().unwrap(); - } + // #(#[test_case(TESTS[N])])* + // #[ignore] + // fn render_circuit_(test: &str) { + // crate::native_tests::init_binary(); + // let test_dir = TempDir::new(test).unwrap(); + // let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); + // render_circuit(path, test.to_string()); + // test_dir.close().unwrap(); + // } @@ -905,7 +912,7 @@ mod native_tests { }); - seq!(N in 0..=45 { + seq!(N in 0..=43 { #(#[test_case(WASM_TESTS[N])])* fn kzg_prove_and_verify_with_overflow_(test: &str) { @@ -915,8 +922,8 @@ mod native_tests { env_logger::init(); let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 1, None, true, "single", Commitments::KZG, 2); - #[cfg(not(feature = "icicle"))] - run_js_tests(path, test.to_string(), "testWasm", false); + // #[cfg(not(feature = "gpu-accelerated"))] + // run_js_tests(path, test.to_string(), "testWasm", false); test_dir.close().unwrap(); } @@ -928,8 +935,8 @@ mod native_tests { env_logger::init(); let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); prove_and_verify(path, test.to_string(), "safe", "hashed", "private", "public", 1, None, true, "single", Commitments::KZG, 2); - #[cfg(not(feature = "icicle"))] - run_js_tests(path, test.to_string(), "testWasm", false); + // #[cfg(not(feature = "gpu-accelerated"))] + // run_js_tests(path, test.to_string(), "testWasm", false); test_dir.close().unwrap(); } @@ -941,8 +948,8 @@ mod native_tests { env_logger::init(); let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); prove_and_verify(path, test.to_string(), "safe", "private", "fixed", "public", 1, None, true, "single", Commitments::KZG, 2); - #[cfg(not(feature = "icicle"))] - run_js_tests(path, test.to_string(), "testWasm", false); + // #[cfg(not(feature = "gpu-accelerated"))] + // run_js_tests(path, test.to_string(), "testWasm", false); test_dir.close().unwrap(); } @@ -989,6 +996,7 @@ mod native_tests { use crate::native_tests::kzg_evm_aggr_prove_and_verify; use tempdir::TempDir; use crate::native_tests::Hardfork; + #[cfg(not(feature = "gpu-accelerated"))] use crate::native_tests::run_js_tests; use ezkl::logger::init_logger; use crate::native_tests::lazy_static; @@ -1089,7 +1097,7 @@ mod native_tests { let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); let _anvil_child = crate::native_tests::start_anvil(false, Hardfork::Latest); kzg_evm_prove_and_verify(2, path, test.to_string(), "private", "private", "public"); - #[cfg(not(feature = "icicle"))] + #[cfg(not(feature = "gpu-accelerated"))] run_js_tests(path, test.to_string(), "testBrowserEvmVerify", false); test_dir.close().unwrap(); @@ -1103,7 +1111,7 @@ mod native_tests { let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); let mut _anvil_child = crate::native_tests::start_anvil(false, Hardfork::Latest); kzg_evm_prove_and_verify(2, path, test.to_string(), "hashed", "private", "private"); - #[cfg(not(feature = "icicle"))] + #[cfg(not(feature = "gpu-accelerated"))] run_js_tests(path, test.to_string(), "testBrowserEvmVerify", false); test_dir.close().unwrap(); } @@ -1120,7 +1128,7 @@ mod native_tests { let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); let mut _anvil_child = crate::native_tests::start_anvil(false, hardfork); kzg_evm_prove_and_verify(2, path, test.to_string(), "polycommit", "private", "public"); - #[cfg(not(feature = "icicle"))] + #[cfg(not(feature = "gpu-accelerated"))] run_js_tests(path, test.to_string(), "testBrowserEvmVerify", false); test_dir.close().unwrap(); } @@ -1133,7 +1141,7 @@ mod native_tests { let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); let _anvil_child = crate::native_tests::start_anvil(false, Hardfork::Latest); kzg_evm_prove_and_verify(2, path, test.to_string(), "private", "hashed", "public"); - #[cfg(not(feature = "icicle"))] + #[cfg(not(feature = "gpu-accelerated"))] run_js_tests(path, test.to_string(), "testBrowserEvmVerify", false); test_dir.close().unwrap(); @@ -1146,7 +1154,7 @@ mod native_tests { let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); let _anvil_child = crate::native_tests::start_anvil(false, Hardfork::Latest); kzg_evm_prove_and_verify(2, path, test.to_string(), "private", "private", "hashed"); - #[cfg(not(feature = "icicle"))] + #[cfg(not(feature = "gpu-accelerated"))] run_js_tests(path, test.to_string(), "testBrowserEvmVerify", false); test_dir.close().unwrap(); } @@ -1159,7 +1167,7 @@ mod native_tests { let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); let _anvil_child = crate::native_tests::start_anvil(false, Hardfork::Latest); kzg_evm_prove_and_verify(2, path, test.to_string(), "private", "polycommit", "public"); - #[cfg(not(feature = "icicle"))] + #[cfg(not(feature = "gpu-accelerated"))] run_js_tests(path, test.to_string(), "testBrowserEvmVerify", false); test_dir.close().unwrap(); } @@ -1172,7 +1180,7 @@ mod native_tests { let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); let _anvil_child = crate::native_tests::start_anvil(false, Hardfork::Latest); kzg_evm_prove_and_verify(2, path, test.to_string(), "private", "private", "polycommit"); - #[cfg(not(feature = "icicle"))] + #[cfg(not(feature = "gpu-accelerated"))] run_js_tests(path, test.to_string(), "testBrowserEvmVerify", false); test_dir.close().unwrap(); } @@ -1184,7 +1192,7 @@ mod native_tests { let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); let _anvil_child = crate::native_tests::start_anvil(false, Hardfork::Latest); kzg_evm_prove_and_verify(2, path, test.to_string(), "polycommit", "polycommit", "polycommit"); - #[cfg(not(feature = "icicle"))] + #[cfg(not(feature = "gpu-accelerated"))] run_js_tests(path, test.to_string(), "testBrowserEvmVerify", false); test_dir.close().unwrap(); } @@ -1551,24 +1559,25 @@ mod native_tests { assert!(status.success()); } - // Mock prove (fast, but does not cover some potential issues) - fn render_circuit(test_dir: &str, example_name: String) { - let status = Command::new(format!("{}/{}", *CARGO_TARGET_DIR, TEST_BINARY)) - .args([ - "render-circuit", - "-M", - format!("{}/{}/network.onnx", test_dir, example_name).as_str(), - "-O", - format!("{}/{}/render.png", test_dir, example_name).as_str(), - "--lookup-range=-32768->32768", - "-K=17", - ]) - .status() - .expect("failed to execute process"); - assert!(status.success()); - } + // // Mock prove (fast, but does not cover some potential issues) + // fn render_circuit(test_dir: &str, example_name: String) { + // let status = Command::new(format!("{}/{}", *CARGO_TARGET_DIR, TEST_BINARY)) + // .args([ + // "render-circuit", + // "-M", + // format!("{}/{}/network.onnx", test_dir, example_name).as_str(), + // "-O", + // format!("{}/{}/render.png", test_dir, example_name).as_str(), + // "--lookup-range=-32768->32768", + // "-K=17", + // ]) + // .status() + // .expect("failed to execute process"); + // assert!(status.success()); + // } // prove-serialize-verify, the usual full path + #[cfg(not(feature = "gpu-accelerated"))] fn kzg_aggr_mock_prove_and_verify(test_dir: &str, example_name: String) { prove_and_verify( test_dir, @@ -2225,6 +2234,7 @@ mod native_tests { } // run js browser evm verify tests for a given example + #[cfg(not(feature = "gpu-accelerated"))] fn run_js_tests(test_dir: &str, example_name: String, js_test: &str, vk: bool) { let example = format!("--example={}", example_name); let dir = format!("--dir={}", test_dir); @@ -2243,7 +2253,7 @@ mod native_tests { #[allow(unused_variables)] fn build_ezkl() { - #[cfg(feature = "icicle")] + #[cfg(feature = "gpu-accelerated")] let args = [ "build", "--profile=test-runs", @@ -2262,18 +2272,8 @@ mod native_tests { "macos-metal", ]; // not macos-metal and not icicle - #[cfg(all(not(feature = "icicle"), not(feature = "macos-metal")))] + #[cfg(all(not(feature = "gpu-accelerated"), not(feature = "macos-metal")))] let args = ["build", "--profile=test-runs", "--bin", "ezkl"]; - #[cfg(feature = "eth-original-lookup")] - let args = [ - "build", - "--profile=test-runs", - "--bin", - "ezkl", - "--no-default-features", - "--features", - "ezkl,solidity-verifier,eth", - ]; #[cfg(feature = "reusable-verifier")] let args = [ "build", diff --git a/tests/python/binding_tests.py b/tests/python/binding_tests.py index 817bea97..85ab05d6 100644 --- a/tests/python/binding_tests.py +++ b/tests/python/binding_tests.py @@ -352,7 +352,7 @@ def test_prove_and_verify(): "for-aggr", srs_path=srs_path, ) - assert res['transcript_type'] == 'Poseidon' + assert res['transcript_type'] == 'poseidon' assert os.path.isfile(proof_path) settings_path = os.path.join(folder_path, 'settings.json') @@ -388,7 +388,7 @@ def test_prove_evm(): "single", srs_path=srs_path, ) - assert res['transcript_type'] == 'EVM' + assert res['transcript_type'] == 'evm' assert os.path.isfile(proof_path) diff --git a/tests/wasm.rs b/tests/wasm.rs index b8f11ea7..aa5a0b1f 100644 --- a/tests/wasm.rs +++ b/tests/wasm.rs @@ -42,21 +42,21 @@ mod wasm32 { pub const SRS1: &[u8] = include_bytes!("assets/kzg1.srs"); pub const VERIFIER_BYTECODE: &[u8] = include_bytes!("assets/wasm.code"); - #[wasm_bindgen_test] - async fn can_verify_aggr() { - let value = verifyAggr( - wasm_bindgen::Clamped(PROOF_AGGR.to_vec()), - wasm_bindgen::Clamped(VK_AGGR.to_vec()), - 21, - wasm_bindgen::Clamped(SRS1.to_vec()), - "kzg", - ) - .map_err(|_| "failed") - .unwrap(); + // #[wasm_bindgen_test] + // async fn can_verify_aggr() { + // let value = verifyAggr( + // wasm_bindgen::Clamped(PROOF_AGGR.to_vec()), + // wasm_bindgen::Clamped(VK_AGGR.to_vec()), + // 21, + // wasm_bindgen::Clamped(SRS1.to_vec()), + // "kzg", + // ) + // .map_err(|_| "failed") + // .unwrap(); - // should not fail - assert!(value); - } + // // should not fail + // assert!(value); + // } #[wasm_bindgen_test] async fn verify_encode_verifier_calldata() {