From 578e0507b3eba373c8bb01eba081ba6da4329293 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vinh=20Tr=E1=BB=8Bnh?= <108657096+vinhtc27@users.noreply.github.com> Date: Wed, 30 Jul 2025 19:18:30 +0700 Subject: [PATCH] feat: add wasm parallel testcase and simplify the witness_calculator.js (#328) - Tested the parallel feature for rln-wasm on this branch: https://github.com/vacp2p/zerokit/tree/benchmark-v0.9.0 - Simplified the test case by using the default generated witness_calculator.js file for both Node and browser tests - Added a WASM parallel test case using the latest wasm-bindgen-rayon version 1.3.0 - [Successful CI run](https://github.com/vacp2p/zerokit/actions/runs/16570298449) with Cargo.lock is included, but it fails if ignored from the codebase. - Requires publishing new pmtree version [on this PR](https://github.com/vacp2p/pmtree/pull/4) before merging this branch. --- .github/workflows/ci.yml | 83 ++--- .github/workflows/nightly-release.yml | 10 +- .gitignore | 11 +- rln-wasm/.gitignore | 21 +- rln-wasm/Cargo.toml | 4 +- rln-wasm/Makefile.toml | 8 +- ...lculator_node.js => witness_calculator.js} | 9 +- .../resources/witness_calculator_browser.js | 335 ------------------ rln-wasm/tests/browser.rs | 14 +- rln-wasm/tests/node.rs | 28 +- rln/Cargo.toml | 3 +- utils/Cargo.toml | 4 +- 12 files changed, 105 insertions(+), 425 deletions(-) rename rln-wasm/resources/{witness_calculator_node.js => witness_calculator.js} (97%) delete mode 100644 rln-wasm/resources/witness_calculator_browser.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec5310e..acb64bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,66 +88,41 @@ jobs: - name: Install dependencies run: make installdeps - name: Build rln-wasm - run: | - if [ ${{ matrix.feature }} == default ]; then - cargo make build - else - cargo make build_${{ matrix.feature }} - fi + run: cargo make build working-directory: ${{ matrix.crate }} - - name: Test rln-wasm - run: | - if [ ${{ matrix.feature }} == default ]; then - cargo make test --release - else - cargo make test_${{ matrix.feature }} --release - fi + - name: Test rln-wasm on node + run: cargo make test --release working-directory: ${{ matrix.crate }} - name: Test rln-wasm on browser - run: | - if [ ${{ matrix.feature }} == default ]; then - cargo make test_browser --release - else - cargo make test_browser_${{ matrix.feature }} --release - fi + run: cargo make test_browser --release working-directory: ${{ matrix.crate }} - # rln-wasm-parallel-test: - # strategy: - # matrix: - # platform: [ubuntu-latest, macos-latest] - # crate: [rln-wasm] - # feature: ["parallel"] - # runs-on: ${{ matrix.platform }} - # timeout-minutes: 60 + rln-wasm-parallel-test: + strategy: + matrix: + platform: [ubuntu-latest, macos-latest] + crate: [rln-wasm] + feature: ["parallel"] + runs-on: ${{ matrix.platform }} + timeout-minutes: 60 - # name: Test - ${{ matrix.crate }} - ${{ matrix.platform }} - ${{ matrix.feature }} - # steps: - # - uses: actions/checkout@v4 - # - name: Install nightly toolchain - # uses: dtolnay/rust-toolchain@nightly - # with: - # components: rust-src - # targets: wasm32-unknown-unknown - # - uses: Swatinem/rust-cache@v2 - # - name: Install dependencies - # run: make installdeps - # - name: Build rln-wasm in parallel mode - # run: | - # if [ ${{ matrix.feature }} == default ]; then - # cargo make build - # else - # cargo make build_${{ matrix.feature }} - # fi - # working-directory: ${{ matrix.crate }} - # - name: Test rln-wasm in parallel mode - # run: | - # if [ ${{ matrix.feature }} == default ]; then - # cargo make test --release - # else - # cargo make test_${{ matrix.feature }} --release - # fi - # working-directory: ${{ matrix.crate }} + name: Test - ${{ matrix.crate }} - ${{ matrix.platform }} - ${{ matrix.feature }} + steps: + - uses: actions/checkout@v4 + - name: Install nightly toolchain + uses: dtolnay/rust-toolchain@nightly + with: + components: rust-src + targets: wasm32-unknown-unknown + - uses: Swatinem/rust-cache@v2 + - name: Install dependencies + run: make installdeps + - name: Build rln-wasm in parallel mode + run: cargo make build_parallel + working-directory: ${{ matrix.crate }} + - name: Test rln-wasm in parallel mode on browser + run: cargo make test_parallel --release + working-directory: ${{ matrix.crate }} lint: strategy: diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml index 0395475..b6534d2 100644 --- a/.github/workflows/nightly-release.yml +++ b/.github/workflows/nightly-release.yml @@ -125,16 +125,8 @@ jobs: wasm-bindgen --target web --split-linked-modules --out-dir ./pkg \ ./target/wasm32-unknown-unknown/release/rln_wasm.wasm - - find ./pkg/snippets -name "workerHelpers.worker.js" \ - -exec sed -i.bak 's|from '\''\.\.\/\.\.\/\.\.\/'\'';|from "../../../rln_wasm.js";|g' {} \; \ - -exec rm -f {}.bak \; - - find ./pkg/snippets -name "workerHelpers.worker.js" \ - -exec sed -i.bak 's|await initWbg(module, memory);|await initWbg({ module, memory });|g' {} \; \ - -exec rm -f {}.bak \; else - wasm-pack build --release --target web --scope waku --features ${{ matrix.feature }} + wasm-pack build --release --target web --scope waku fi sed -i.bak 's/rln-wasm/zerokit-rln-wasm/g' pkg/package.json && rm pkg/package.json.bak diff --git a/.gitignore b/.gitignore index f81eff8..2b43e5e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,15 @@ +# Common files to ignore in Rust projects .DS_Store .idea *.log tmp/ -rln/pmtree_db -rln-cli/database -# Generated by Cargo -# will have compiled files and executables +# Generated by Cargo will have compiled files and executables /target -/Cargo.lock +Cargo.lock + +# Generated by rln-cli +rln-cli/database # Generated by Nix result diff --git a/rln-wasm/.gitignore b/rln-wasm/.gitignore index 4e30131..3068f54 100644 --- a/rln-wasm/.gitignore +++ b/rln-wasm/.gitignore @@ -1,6 +1,21 @@ +# Common files to ignore in Rust projects +.DS_Store +.idea +*.log +tmp/ + +# Generated by Cargo will have compiled files and executables /target -**/*.rs.bk Cargo.lock -bin/ + +# Generated by rln-wasm pkg/ -wasm-pack.log + +# Generated by Nix +result + +# These are backup files generated by rustfmt +**/*.rs.bk + +# MSVC Windows builds of rustc generate these, which store debugging information +*.pdb diff --git a/rln-wasm/Cargo.toml b/rln-wasm/Cargo.toml index 320bffb..7c1d57f 100644 --- a/rln-wasm/Cargo.toml +++ b/rln-wasm/Cargo.toml @@ -16,7 +16,9 @@ num-bigint = { version = "0.4.6", default-features = false } js-sys = "0.3.77" wasm-bindgen = "0.2.100" serde-wasm-bindgen = "0.6.5" -wasm-bindgen-rayon = { version = "1.2.0", optional = true } +wasm-bindgen-rayon = { version = "1.3.0", features = [ + "no-bundler", +], optional = true } # The `console_error_panic_xhook` crate provides better debugging of panics by # logging them with `console.error`. This is great for development, but requires diff --git a/rln-wasm/Makefile.toml b/rln-wasm/Makefile.toml index 10b9467..6b9a9d3 100644 --- a/rln-wasm/Makefile.toml +++ b/rln-wasm/Makefile.toml @@ -37,9 +37,7 @@ args = [ [tasks.post_build_parallel] script = ''' -wasm-bindgen --target web --split-linked-modules --out-dir ./pkg ./target/wasm32-unknown-unknown/release/rln_wasm.wasm && \ -find ./pkg/snippets -name "workerHelpers.worker.js" -exec sed -i.bak 's|from '\''\.\.\/\.\.\/\.\.\/'\'';|from "../../../rln_wasm.js";|g' {} \; -exec rm -f {}.bak \; && \ -find ./pkg/snippets -name "workerHelpers.worker.js" -exec sed -i.bak 's|await initWbg(module, memory);|await initWbg({ module, memory });|g' {} \; -exec rm -f {}.bak \; +wasm-bindgen --target web --split-linked-modules --out-dir ./pkg ./target/wasm32-unknown-unknown/release/rln_wasm.wasm ''' [tasks.pack_rename] @@ -77,8 +75,6 @@ args = [ "test", "--release", "--chrome", - # "--firefox", - # "--safari", "--headless", "--target", "wasm32-unknown-unknown", @@ -98,8 +94,6 @@ args = [ "test", "--release", "--chrome", - # "--firefox", - # "--safari", "--headless", "--target", "wasm32-unknown-unknown", diff --git a/rln-wasm/resources/witness_calculator_node.js b/rln-wasm/resources/witness_calculator.js similarity index 97% rename from rln-wasm/resources/witness_calculator_node.js rename to rln-wasm/resources/witness_calculator.js index c43da74..dce0821 100644 --- a/rln-wasm/resources/witness_calculator_node.js +++ b/rln-wasm/resources/witness_calculator.js @@ -1,5 +1,8 @@ -// Node.js module compatible witness calculator -module.exports = async function builder(code, options) { +// File generated with https://github.com/iden3/circom +// following the instructions from: +// https://github.com/vacp2p/zerokit/tree/master/rln#advanced-custom-circuit-compilation + +export async function builder(code, options) { options = options || {}; let wasmModule; @@ -102,7 +105,7 @@ module.exports = async function builder(code, options) { // Then append the value to the message we are creating msgStr += fromArray32(arr).toString(); } -}; +} class WitnessCalculator { constructor(instance, sanityCheck) { diff --git a/rln-wasm/resources/witness_calculator_browser.js b/rln-wasm/resources/witness_calculator_browser.js deleted file mode 100644 index 39b8280..0000000 --- a/rln-wasm/resources/witness_calculator_browser.js +++ /dev/null @@ -1,335 +0,0 @@ -// Browser compatible witness calculator -(function (global) { - async function builder(code, options) { - options = options || {}; - - let wasmModule; - try { - wasmModule = await WebAssembly.compile(code); - } catch (err) { - console.log(err); - console.log( - "\nTry to run circom --c in order to generate c++ code instead\n" - ); - throw new Error(err); - } - - let wc; - - let errStr = ""; - let msgStr = ""; - - const instance = await WebAssembly.instantiate(wasmModule, { - runtime: { - exceptionHandler: function (code) { - let err; - if (code == 1) { - err = "Signal not found.\n"; - } else if (code == 2) { - err = "Too many signals set.\n"; - } else if (code == 3) { - err = "Signal already set.\n"; - } else if (code == 4) { - err = "Assert Failed.\n"; - } else if (code == 5) { - err = "Not enough memory.\n"; - } else if (code == 6) { - err = "Input signal array access exceeds the size.\n"; - } else { - err = "Unknown error.\n"; - } - throw new Error(err + errStr); - }, - printErrorMessage: function () { - errStr += getMessage() + "\n"; - // console.error(getMessage()); - }, - writeBufferMessage: function () { - const msg = getMessage(); - // Any calls to `log()` will always end with a `\n`, so that's when we print and reset - if (msg === "\n") { - console.log(msgStr); - msgStr = ""; - } else { - // If we've buffered other content, put a space in between the items - if (msgStr !== "") { - msgStr += " "; - } - // Then append the message to the message we are creating - msgStr += msg; - } - }, - showSharedRWMemory: function () { - printSharedRWMemory(); - }, - }, - }); - - const sanityCheck = options; - // options && - // ( - // options.sanityCheck || - // options.logGetSignal || - // options.logSetSignal || - // options.logStartComponent || - // options.logFinishComponent - // ); - - wc = new WitnessCalculator(instance, sanityCheck); - return wc; - - function getMessage() { - var message = ""; - var c = instance.exports.getMessageChar(); - while (c != 0) { - message += String.fromCharCode(c); - c = instance.exports.getMessageChar(); - } - return message; - } - - function printSharedRWMemory() { - const shared_rw_memory_size = instance.exports.getFieldNumLen32(); - const arr = new Uint32Array(shared_rw_memory_size); - for (let j = 0; j < shared_rw_memory_size; j++) { - arr[shared_rw_memory_size - 1 - j] = - instance.exports.readSharedRWMemory(j); - } - - // If we've buffered other content, put a space in between the items - if (msgStr !== "") { - msgStr += " "; - } - // Then append the value to the message we are creating - msgStr += fromArray32(arr).toString(); - } - } - - class WitnessCalculator { - constructor(instance, sanityCheck) { - this.instance = instance; - - this.version = this.instance.exports.getVersion(); - this.n32 = this.instance.exports.getFieldNumLen32(); - - this.instance.exports.getRawPrime(); - const arr = new Uint32Array(this.n32); - for (let i = 0; i < this.n32; i++) { - arr[this.n32 - 1 - i] = this.instance.exports.readSharedRWMemory(i); - } - this.prime = fromArray32(arr); - - this.witnessSize = this.instance.exports.getWitnessSize(); - - this.sanityCheck = sanityCheck; - } - - circom_version() { - return this.instance.exports.getVersion(); - } - - async _doCalculateWitness(input, sanityCheck) { - //input is assumed to be a map from signals to arrays of bigints - this.instance.exports.init(this.sanityCheck || sanityCheck ? 1 : 0); - const keys = Object.keys(input); - var input_counter = 0; - keys.forEach((k) => { - const h = fnvHash(k); - const hMSB = parseInt(h.slice(0, 8), 16); - const hLSB = parseInt(h.slice(8, 16), 16); - const fArr = flatArray(input[k]); - let signalSize = this.instance.exports.getInputSignalSize(hMSB, hLSB); - if (signalSize < 0) { - throw new Error(`Signal ${k} not found\n`); - } - if (fArr.length < signalSize) { - throw new Error(`Not enough values for input signal ${k}\n`); - } - if (fArr.length > signalSize) { - throw new Error(`Too many values for input signal ${k}\n`); - } - for (let i = 0; i < fArr.length; i++) { - const arrFr = toArray32(BigInt(fArr[i]) % this.prime, this.n32); - for (let j = 0; j < this.n32; j++) { - this.instance.exports.writeSharedRWMemory( - j, - arrFr[this.n32 - 1 - j] - ); - } - try { - this.instance.exports.setInputSignal(hMSB, hLSB, i); - input_counter++; - } catch (err) { - // console.log(`After adding signal ${i} of ${k}`) - throw new Error(err); - } - } - }); - if (input_counter < this.instance.exports.getInputSize()) { - throw new Error( - `Not all inputs have been set. Only ${input_counter} out of ${this.instance.exports.getInputSize()}` - ); - } - } - - async calculateWitness(input, sanityCheck) { - const w = []; - - await this._doCalculateWitness(input, sanityCheck); - - for (let i = 0; i < this.witnessSize; i++) { - this.instance.exports.getWitness(i); - const arr = new Uint32Array(this.n32); - for (let j = 0; j < this.n32; j++) { - arr[this.n32 - 1 - j] = this.instance.exports.readSharedRWMemory(j); - } - w.push(fromArray32(arr)); - } - - return w; - } - - async calculateBinWitness(input, sanityCheck) { - const buff32 = new Uint32Array(this.witnessSize * this.n32); - const buff = new Uint8Array(buff32.buffer); - await this._doCalculateWitness(input, sanityCheck); - - for (let i = 0; i < this.witnessSize; i++) { - this.instance.exports.getWitness(i); - const pos = i * this.n32; - for (let j = 0; j < this.n32; j++) { - buff32[pos + j] = this.instance.exports.readSharedRWMemory(j); - } - } - - return buff; - } - - async calculateWTNSBin(input, sanityCheck) { - const buff32 = new Uint32Array( - this.witnessSize * this.n32 + this.n32 + 11 - ); - const buff = new Uint8Array(buff32.buffer); - await this._doCalculateWitness(input, sanityCheck); - - //"wtns" - buff[0] = "w".charCodeAt(0); - buff[1] = "t".charCodeAt(0); - buff[2] = "n".charCodeAt(0); - buff[3] = "s".charCodeAt(0); - - //version 2 - buff32[1] = 2; - - //number of sections: 2 - buff32[2] = 2; - - //id section 1 - buff32[3] = 1; - - const n8 = this.n32 * 4; - //id section 1 length in 64bytes - const idSection1length = 8 + n8; - const idSection1lengthHex = idSection1length.toString(16); - buff32[4] = parseInt(idSection1lengthHex.slice(0, 8), 16); - buff32[5] = parseInt(idSection1lengthHex.slice(8, 16), 16); - - //this.n32 - buff32[6] = n8; - - //prime number - this.instance.exports.getRawPrime(); - - var pos = 7; - for (let j = 0; j < this.n32; j++) { - buff32[pos + j] = this.instance.exports.readSharedRWMemory(j); - } - pos += this.n32; - - // witness size - buff32[pos] = this.witnessSize; - pos++; - - //id section 2 - buff32[pos] = 2; - pos++; - - // section 2 length - const idSection2length = n8 * this.witnessSize; - const idSection2lengthHex = idSection2length.toString(16); - buff32[pos] = parseInt(idSection2lengthHex.slice(0, 8), 16); - buff32[pos + 1] = parseInt(idSection2lengthHex.slice(8, 16), 16); - - pos += 2; - for (let i = 0; i < this.witnessSize; i++) { - this.instance.exports.getWitness(i); - for (let j = 0; j < this.n32; j++) { - buff32[pos + j] = this.instance.exports.readSharedRWMemory(j); - } - pos += this.n32; - } - - return buff; - } - } - - function toArray32(rem, size) { - const res = []; //new Uint32Array(size); //has no unshift - const radix = BigInt(0x100000000); - while (rem) { - res.unshift(Number(rem % radix)); - rem = rem / radix; - } - if (size) { - var i = size - res.length; - while (i > 0) { - res.unshift(0); - i--; - } - } - return res; - } - - function fromArray32(arr) { - //returns a BigInt - var res = BigInt(0); - const radix = BigInt(0x100000000); - for (let i = 0; i < arr.length; i++) { - res = res * radix + BigInt(arr[i]); - } - return res; - } - - function flatArray(a) { - var res = []; - fillArray(res, a); - return res; - - function fillArray(res, a) { - if (Array.isArray(a)) { - for (let i = 0; i < a.length; i++) { - fillArray(res, a[i]); - } - } else { - res.push(a); - } - } - } - - function fnvHash(str) { - const uint64_max = BigInt(2) ** BigInt(64); - let hash = BigInt("0xCBF29CE484222325"); - for (var i = 0; i < str.length; i++) { - hash ^= BigInt(str[i].charCodeAt()); - hash *= BigInt(0x100000001b3); - hash %= uint64_max; - } - let shash = hash.toString(16); - let n = 16 - shash.length; - shash = "0".repeat(n).concat(shash); - return shash; - } - - // Make it globally available - global.witnessCalculatorBuilder = builder; -})(typeof self !== "undefined" ? self : window); diff --git a/rln-wasm/tests/browser.rs b/rln-wasm/tests/browser.rs index 8e2786e..1d4c19e 100644 --- a/rln-wasm/tests/browser.rs +++ b/rln-wasm/tests/browser.rs @@ -29,7 +29,15 @@ mod tests { } export function initWitnessCalculator(jsCode) { - eval(jsCode); + const processedCode = jsCode + .replace(/export\s+async\s+function\s+builder/, 'async function builder') + .replace(/export\s*\{\s*builder\s*\};?/g, ''); + + const moduleFunc = new Function(processedCode + '\nreturn { builder };'); + const witnessCalculatorModule = moduleFunc(); + + window.witnessCalculatorBuilder = witnessCalculatorModule.builder; + if (typeof window.witnessCalculatorBuilder !== 'function') { return false; } @@ -63,7 +71,7 @@ mod tests { async fn calculateWitness(circom_data: &[u8], inputs: Object) -> Result; } - const WITNESS_CALCULATOR_JS: &str = include_str!("../resources/witness_calculator_browser.js"); + const WITNESS_CALCULATOR_JS: &str = include_str!("../resources/witness_calculator.js"); const ARKZKEY_BYTES: &[u8] = include_bytes!("../../rln/resources/tree_height_20/rln_final.arkzkey"); @@ -89,7 +97,7 @@ mod tests { .expect("Failed to initialize thread pool"); } - // Initialize the witness calculator + // Initialize witness calculator initWitnessCalculator(WITNESS_CALCULATOR_JS) .expect("Failed to initialize witness calculator"); diff --git a/rln-wasm/tests/node.rs b/rln-wasm/tests/node.rs index e03b51a..2bf5073 100644 --- a/rln-wasm/tests/node.rs +++ b/rln-wasm/tests/node.rs @@ -18,22 +18,39 @@ mod tests { OptimalMerkleProof, OptimalMerkleTree, ZerokitMerkleProof, ZerokitMerkleTree, }; + const WITNESS_CALCULATOR_JS: &str = include_str!("../resources/witness_calculator.js"); + #[wasm_bindgen(inline_js = r#" const fs = require("fs"); + let witnessCalculatorModule = null; + module.exports = { + initWitnessCalculator: function(code) { + const processedCode = code + .replace(/export\s+async\s+function\s+builder/, 'async function builder') + .replace(/export\s*\{\s*builder\s*\};?/g, ''); + + const moduleFunc = new Function(processedCode + '\nreturn { builder };'); + witnessCalculatorModule = moduleFunc(); + + if (typeof witnessCalculatorModule.builder !== 'function') { + return false; + } + return true; + }, + readFile: function (path) { return fs.readFileSync(path); }, calculateWitness: async function (circom_path, inputs) { - const wc = require("resources/witness_calculator_node.js"); const wasmFile = fs.readFileSync(circom_path); const wasmFileBuffer = wasmFile.slice( wasmFile.byteOffset, wasmFile.byteOffset + wasmFile.byteLength ); - const witnessCalculator = await wc(wasmFileBuffer); + const witnessCalculator = await witnessCalculatorModule.builder(wasmFileBuffer); const calculatedWitness = await witnessCalculator.calculateWitness( inputs, false @@ -45,6 +62,9 @@ mod tests { }; "#)] extern "C" { + #[wasm_bindgen(catch)] + fn initWitnessCalculator(code: &str) -> Result; + #[wasm_bindgen(catch)] fn readFile(path: &str) -> Result; @@ -58,6 +78,10 @@ mod tests { #[wasm_bindgen_test] pub async fn rln_wasm_benchmark() { + // Initialize witness calculator + initWitnessCalculator(WITNESS_CALCULATOR_JS) + .expect("Failed to initialize witness calculator"); + let mut results = String::from("\nbenchmarks:\n"); let iterations = 10; diff --git a/rln/Cargo.toml b/rln/Cargo.toml index 8d57cc0..a69f766 100644 --- a/rln/Cargo.toml +++ b/rln/Cargo.toml @@ -30,7 +30,7 @@ ark-serialize = { version = "0.5.0", default-features = false } thiserror = "2.0.12" # utilities -rayon = { version = "1.7.0" } +rayon = { version = "1.10.0", optional = true } byteorder = "1.5.0" cfg-if = "1.0" num-bigint = { version = "0.4.6", default-features = false, features = ["std"] } @@ -58,6 +58,7 @@ criterion = { version = "0.7.0", features = ["html_reports"] } default = ["parallel", "pmtree-ft"] stateless = [] parallel = [ + "rayon", "utils/parallel", "ark-ff/parallel", "ark-ec/parallel", diff --git a/utils/Cargo.toml b/utils/Cargo.toml index ef6f3f3..ec1d20d 100644 --- a/utils/Cargo.toml +++ b/utils/Cargo.toml @@ -14,12 +14,12 @@ bench = false [dependencies] ark-ff = { version = "0.5.0", default-features = false } num-bigint = { version = "0.4.6", default-features = false } -pmtree = { package = "vacp2p_pmtree", version = "2.0.2", optional = true } +pmtree = { package = "vacp2p_pmtree", version = "2.0.3", optional = true } sled = "0.34.7" serde_json = "1.0.141" lazy_static = "1.5.0" hex = "0.4.3" -rayon = "1.7.0" +rayon = "1.10.0" thiserror = "2.0" [dev-dependencies]