github/zerokit
1
1
Fork 0
mirror of https://github.com/vacp2p/zerokit.git synced 2026-01-07 21:53:59 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: add wasm parallel testcase and simplify the witness_calculator.js (#328)

Browse Source 
- Tested the parallel feature for rln-wasm on this branch:
https://github.com/vacp2p/zerokit/tree/benchmark-v0.9.0
- Simplified the test case by using the default generated
witness_calculator.js file for both Node and browser tests
- Added a WASM parallel test case using the latest wasm-bindgen-rayon
version 1.3.0
- [Successful CI
run](https://github.com/vacp2p/zerokit/actions/runs/16570298449) with
Cargo.lock is included, but it fails if ignored from the codebase.
- Requires publishing new pmtree version [on this
PR](https://github.com/vacp2p/pmtree/pull/4) before merging this branch.
This commit is contained in:
Vinh Trịnh
2025-07-30 19:18:30 +07:00
committed by GitHub
parent bf1e184da9
commit 578e0507b3
12 changed files with 105 additions and 425 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
.github/workflows/ci.yml vendored
View File

@@ -88,66 +88,41 @@ jobs:
- name: Install dependencies
run: make installdeps
- name: Build rln-wasm
run: |
if [ ${{ matrix.feature }} == default ]; then
cargo make build
else
cargo make build_${{ matrix.feature }}
fi
run: cargo make build
working-directory: ${{ matrix.crate }}
- name: Test rln-wasm
run: |
if [ ${{ matrix.feature }} == default ]; then
cargo make test --release
else
cargo make test_${{ matrix.feature }} --release
fi
- name: Test rln-wasm on node
run: cargo make test --release
working-directory: ${{ matrix.crate }}
- name: Test rln-wasm on browser
run: |
if [ ${{ matrix.feature }} == default ]; then
cargo make test_browser --release
else
cargo make test_browser_${{ matrix.feature }} --release
fi
run: cargo make test_browser --release
working-directory: ${{ matrix.crate }}
# rln-wasm-parallel-test:
# strategy:
# matrix:
# platform: [ubuntu-latest, macos-latest]
# crate: [rln-wasm]
# feature: ["parallel"]
# runs-on: ${{ matrix.platform }}
# timeout-minutes: 60
rln-wasm-parallel-test:
strategy:
matrix:
platform: [ubuntu-latest, macos-latest]
crate: [rln-wasm]
feature: ["parallel"]
runs-on: ${{ matrix.platform }}
timeout-minutes: 60
# name: Test - ${{ matrix.crate }} - ${{ matrix.platform }} - ${{ matrix.feature }}
# steps:
# - uses: actions/checkout@v4
# - name: Install nightly toolchain
# uses: dtolnay/rust-toolchain@nightly
# with:
# components: rust-src
# targets: wasm32-unknown-unknown
# - uses: Swatinem/rust-cache@v2
# - name: Install dependencies
# run: make installdeps
# - name: Build rln-wasm in parallel mode
# run: |
# if [ ${{ matrix.feature }} == default ]; then
# cargo make build
# else
# cargo make build_${{ matrix.feature }}
# fi
# working-directory: ${{ matrix.crate }}
# - name: Test rln-wasm in parallel mode
# run: |
# if [ ${{ matrix.feature }} == default ]; then
# cargo make test --release
# else
# cargo make test_${{ matrix.feature }} --release
# fi
# working-directory: ${{ matrix.crate }}
name: Test - ${{ matrix.crate }} - ${{ matrix.platform }} - ${{ matrix.feature }}
steps:
- uses: actions/checkout@v4
- name: Install nightly toolchain
uses: dtolnay/rust-toolchain@nightly
with:
components: rust-src
targets: wasm32-unknown-unknown
- uses: Swatinem/rust-cache@v2
- name: Install dependencies
run: make installdeps
- name: Build rln-wasm in parallel mode
run: cargo make build_parallel
working-directory: ${{ matrix.crate }}
- name: Test rln-wasm in parallel mode on browser
run: cargo make test_parallel --release
working-directory: ${{ matrix.crate }}
lint:
strategy:

10
.github/workflows/nightly-release.yml vendored
View File

@@ -125,16 +125,8 @@ jobs:
wasm-bindgen --target web --split-linked-modules --out-dir ./pkg \
./target/wasm32-unknown-unknown/release/rln_wasm.wasm
find ./pkg/snippets -name "workerHelpers.worker.js" \
-exec sed -i.bak 's|from '\''\.\.\/\.\.\/\.\.\/'\'';|from "../../../rln_wasm.js";|g' {} \; \
-exec rm -f {}.bak \;
find ./pkg/snippets -name "workerHelpers.worker.js" \
-exec sed -i.bak 's|await initWbg(module, memory);|await initWbg({ module, memory });|g' {} \; \
-exec rm -f {}.bak \;
else
wasm-pack build --release --target web --scope waku --features ${{ matrix.feature }}
wasm-pack build --release --target web --scope waku
fi
sed -i.bak 's/rln-wasm/zerokit-rln-wasm/g' pkg/package.json && rm pkg/package.json.bak

11
.gitignore vendored
View File

@@ -1,14 +1,15 @@
# Common files to ignore in Rust projects
.DS_Store
.idea
*.log
tmp/
rln/pmtree_db
rln-cli/database
# Generated by Cargo
# will have compiled files and executables
# Generated by Cargo will have compiled files and executables
/target
/Cargo.lock
Cargo.lock
# Generated by rln-cli
rln-cli/database
# Generated by Nix
result

21
rln-wasm/.gitignore vendored
View File

@@ -1,6 +1,21 @@
# Common files to ignore in Rust projects
.DS_Store
.idea
*.log
tmp/
# Generated by Cargo will have compiled files and executables
/target
**/*.rs.bk
Cargo.lock
bin/
# Generated by rln-wasm
pkg/
wasm-pack.log
# Generated by Nix
result
# These are backup files generated by rustfmt
**/*.rs.bk
# MSVC Windows builds of rustc generate these, which store debugging information
*.pdb

4
rln-wasm/Cargo.toml
View File

@@ -16,7 +16,9 @@ num-bigint = { version = "0.4.6", default-features = false }
js-sys = "0.3.77"
wasm-bindgen = "0.2.100"
serde-wasm-bindgen = "0.6.5"
wasm-bindgen-rayon = { version = "1.2.0", optional = true }
wasm-bindgen-rayon = { version = "1.3.0", features = [
"no-bundler",
], optional = true }
# The `console_error_panic_xhook` crate provides better debugging of panics by
# logging them with `console.error`. This is great for development, but requires

8
rln-wasm/Makefile.toml
View File

@@ -37,9 +37,7 @@ args = [
[tasks.post_build_parallel]
script = '''
wasm-bindgen --target web --split-linked-modules --out-dir ./pkg ./target/wasm32-unknown-unknown/release/rln_wasm.wasm && \
find ./pkg/snippets -name "workerHelpers.worker.js" -exec sed -i.bak 's|from '\''\.\.\/\.\.\/\.\.\/'\'';|from "../../../rln_wasm.js";|g' {} \; -exec rm -f {}.bak \; && \
find ./pkg/snippets -name "workerHelpers.worker.js" -exec sed -i.bak 's|await initWbg(module, memory);|await initWbg({ module, memory });|g' {} \; -exec rm -f {}.bak \;
wasm-bindgen --target web --split-linked-modules --out-dir ./pkg ./target/wasm32-unknown-unknown/release/rln_wasm.wasm
'''
[tasks.pack_rename]
@@ -77,8 +75,6 @@ args = [
"test",
"--release",
"--chrome",
# "--firefox",
# "--safari",
"--headless",
"--target",
"wasm32-unknown-unknown",
@@ -98,8 +94,6 @@ args = [
"test",
"--release",
"--chrome",
# "--firefox",
# "--safari",
"--headless",
"--target",
"wasm32-unknown-unknown",

9
rln-wasm/resources/witness_calculator_node.js → rln-wasm/resources/witness_calculator.js
View File

@@ -1,5 +1,8 @@
// Node.js module compatible witness calculator
module.exports = async function builder(code, options) {
// File generated with https://github.com/iden3/circom
// following the instructions from:
// https://github.com/vacp2p/zerokit/tree/master/rln#advanced-custom-circuit-compilation
export async function builder(code, options) {
options = options || {};
let wasmModule;
@@ -102,7 +105,7 @@ module.exports = async function builder(code, options) {
// Then append the value to the message we are creating
msgStr += fromArray32(arr).toString();
}
};
}
class WitnessCalculator {
constructor(instance, sanityCheck) {

335
rln-wasm/resources/witness_calculator_browser.js
View File

@@ -1,335 +0,0 @@
// Browser compatible witness calculator
(function (global) {
async function builder(code, options) {
options = options || {};
let wasmModule;
try {
wasmModule = await WebAssembly.compile(code);
} catch (err) {
console.log(err);
console.log(
"\nTry to run circom --c in order to generate c++ code instead\n"
);
throw new Error(err);
}
let wc;
let errStr = "";
let msgStr = "";
const instance = await WebAssembly.instantiate(wasmModule, {
runtime: {
exceptionHandler: function (code) {
let err;
if (code == 1) {
err = "Signal not found.\n";
} else if (code == 2) {
err = "Too many signals set.\n";
} else if (code == 3) {
err = "Signal already set.\n";
} else if (code == 4) {
err = "Assert Failed.\n";
} else if (code == 5) {
err = "Not enough memory.\n";
} else if (code == 6) {
err = "Input signal array access exceeds the size.\n";
} else {
err = "Unknown error.\n";
}
throw new Error(err + errStr);
},
printErrorMessage: function () {
errStr += getMessage() + "\n";
// console.error(getMessage());
},
writeBufferMessage: function () {
const msg = getMessage();
// Any calls to `log()` will always end with a `\n`, so that's when we print and reset
if (msg === "\n") {
console.log(msgStr);
msgStr = "";
} else {
// If we've buffered other content, put a space in between the items
if (msgStr !== "") {
msgStr += " ";
}
// Then append the message to the message we are creating
msgStr += msg;
}
},
showSharedRWMemory: function () {
printSharedRWMemory();
},
},
});
const sanityCheck = options;
// options &&
// (
// options.sanityCheck ||
// options.logGetSignal ||
// options.logSetSignal ||
// options.logStartComponent ||
// options.logFinishComponent
// );
wc = new WitnessCalculator(instance, sanityCheck);
return wc;
function getMessage() {
var message = "";
var c = instance.exports.getMessageChar();
while (c != 0) {
message += String.fromCharCode(c);
c = instance.exports.getMessageChar();
}
return message;
}
function printSharedRWMemory() {
const shared_rw_memory_size = instance.exports.getFieldNumLen32();
const arr = new Uint32Array(shared_rw_memory_size);
for (let j = 0; j < shared_rw_memory_size; j++) {
arr[shared_rw_memory_size - 1 - j] =
instance.exports.readSharedRWMemory(j);
}
// If we've buffered other content, put a space in between the items
if (msgStr !== "") {
msgStr += " ";
}
// Then append the value to the message we are creating
msgStr += fromArray32(arr).toString();
}
}
class WitnessCalculator {
constructor(instance, sanityCheck) {
this.instance = instance;
this.version = this.instance.exports.getVersion();
this.n32 = this.instance.exports.getFieldNumLen32();
this.instance.exports.getRawPrime();
const arr = new Uint32Array(this.n32);
for (let i = 0; i < this.n32; i++) {
arr[this.n32 - 1 - i] = this.instance.exports.readSharedRWMemory(i);
}
this.prime = fromArray32(arr);
this.witnessSize = this.instance.exports.getWitnessSize();
this.sanityCheck = sanityCheck;
}
circom_version() {
return this.instance.exports.getVersion();
}
async _doCalculateWitness(input, sanityCheck) {
//input is assumed to be a map from signals to arrays of bigints
this.instance.exports.init(this.sanityCheck || sanityCheck ? 1 : 0);
const keys = Object.keys(input);
var input_counter = 0;
keys.forEach((k) => {
const h = fnvHash(k);
const hMSB = parseInt(h.slice(0, 8), 16);
const hLSB = parseInt(h.slice(8, 16), 16);
const fArr = flatArray(input[k]);
let signalSize = this.instance.exports.getInputSignalSize(hMSB, hLSB);
if (signalSize < 0) {
throw new Error(`Signal ${k} not found\n`);
}
if (fArr.length < signalSize) {
throw new Error(`Not enough values for input signal ${k}\n`);
}
if (fArr.length > signalSize) {
throw new Error(`Too many values for input signal ${k}\n`);
}
for (let i = 0; i < fArr.length; i++) {
const arrFr = toArray32(BigInt(fArr[i]) % this.prime, this.n32);
for (let j = 0; j < this.n32; j++) {
this.instance.exports.writeSharedRWMemory(
j,
arrFr[this.n32 - 1 - j]
);
}
try {
this.instance.exports.setInputSignal(hMSB, hLSB, i);
input_counter++;
} catch (err) {
// console.log(`After adding signal ${i} of ${k}`)
throw new Error(err);
}
}
});
if (input_counter < this.instance.exports.getInputSize()) {
throw new Error(
`Not all inputs have been set. Only ${input_counter} out of ${this.instance.exports.getInputSize()}`
);
}
}
async calculateWitness(input, sanityCheck) {
const w = [];
await this._doCalculateWitness(input, sanityCheck);
for (let i = 0; i < this.witnessSize; i++) {
this.instance.exports.getWitness(i);
const arr = new Uint32Array(this.n32);
for (let j = 0; j < this.n32; j++) {
arr[this.n32 - 1 - j] = this.instance.exports.readSharedRWMemory(j);
}
w.push(fromArray32(arr));
}
return w;
}
async calculateBinWitness(input, sanityCheck) {
const buff32 = new Uint32Array(this.witnessSize * this.n32);
const buff = new Uint8Array(buff32.buffer);
await this._doCalculateWitness(input, sanityCheck);
for (let i = 0; i < this.witnessSize; i++) {
this.instance.exports.getWitness(i);
const pos = i * this.n32;
for (let j = 0; j < this.n32; j++) {
buff32[pos + j] = this.instance.exports.readSharedRWMemory(j);
}
}
return buff;
}
async calculateWTNSBin(input, sanityCheck) {
const buff32 = new Uint32Array(
this.witnessSize * this.n32 + this.n32 + 11
);
const buff = new Uint8Array(buff32.buffer);
await this._doCalculateWitness(input, sanityCheck);
//"wtns"
buff[0] = "w".charCodeAt(0);
buff[1] = "t".charCodeAt(0);
buff[2] = "n".charCodeAt(0);
buff[3] = "s".charCodeAt(0);
//version 2
buff32[1] = 2;
//number of sections: 2
buff32[2] = 2;
//id section 1
buff32[3] = 1;
const n8 = this.n32 * 4;
//id section 1 length in 64bytes
const idSection1length = 8 + n8;
const idSection1lengthHex = idSection1length.toString(16);
buff32[4] = parseInt(idSection1lengthHex.slice(0, 8), 16);
buff32[5] = parseInt(idSection1lengthHex.slice(8, 16), 16);
//this.n32
buff32[6] = n8;
//prime number
this.instance.exports.getRawPrime();
var pos = 7;
for (let j = 0; j < this.n32; j++) {
buff32[pos + j] = this.instance.exports.readSharedRWMemory(j);
}
pos += this.n32;
// witness size
buff32[pos] = this.witnessSize;
pos++;
//id section 2
buff32[pos] = 2;
pos++;
// section 2 length
const idSection2length = n8 * this.witnessSize;
const idSection2lengthHex = idSection2length.toString(16);
buff32[pos] = parseInt(idSection2lengthHex.slice(0, 8), 16);
buff32[pos + 1] = parseInt(idSection2lengthHex.slice(8, 16), 16);
pos += 2;
for (let i = 0; i < this.witnessSize; i++) {
this.instance.exports.getWitness(i);
for (let j = 0; j < this.n32; j++) {
buff32[pos + j] = this.instance.exports.readSharedRWMemory(j);
}
pos += this.n32;
}
return buff;
}
}
function toArray32(rem, size) {
const res = []; //new Uint32Array(size); //has no unshift
const radix = BigInt(0x100000000);
while (rem) {
res.unshift(Number(rem % radix));
rem = rem / radix;
}
if (size) {
var i = size - res.length;
while (i > 0) {
res.unshift(0);
i--;
}
}
return res;
}
function fromArray32(arr) {
//returns a BigInt
var res = BigInt(0);
const radix = BigInt(0x100000000);
for (let i = 0; i < arr.length; i++) {
res = res * radix + BigInt(arr[i]);
}
return res;
}
function flatArray(a) {
var res = [];
fillArray(res, a);
return res;
function fillArray(res, a) {
if (Array.isArray(a)) {
for (let i = 0; i < a.length; i++) {
fillArray(res, a[i]);
}
} else {
res.push(a);
}
}
}
function fnvHash(str) {
const uint64_max = BigInt(2) ** BigInt(64);
let hash = BigInt("0xCBF29CE484222325");
for (var i = 0; i < str.length; i++) {
hash ^= BigInt(str[i].charCodeAt());
hash *= BigInt(0x100000001b3);
hash %= uint64_max;
}
let shash = hash.toString(16);
let n = 16 - shash.length;
shash = "0".repeat(n).concat(shash);
return shash;
}
// Make it globally available
global.witnessCalculatorBuilder = builder;
})(typeof self !== "undefined" ? self : window);

14
rln-wasm/tests/browser.rs
View File

@@ -29,7 +29,15 @@ mod tests {
}
export function initWitnessCalculator(jsCode) {
eval(jsCode);
const processedCode = jsCode
.replace(/export\s+async\s+function\s+builder/, 'async function builder')
.replace(/export\s*\{\s*builder\s*\};?/g, '');
const moduleFunc = new Function(processedCode + '\nreturn { builder };');
const witnessCalculatorModule = moduleFunc();
window.witnessCalculatorBuilder = witnessCalculatorModule.builder;
if (typeof window.witnessCalculatorBuilder !== 'function') {
return false;
}
@@ -63,7 +71,7 @@ mod tests {
async fn calculateWitness(circom_data: &[u8], inputs: Object) -> Result<JsValue, JsValue>;
}
const WITNESS_CALCULATOR_JS: &str = include_str!("../resources/witness_calculator_browser.js");
const WITNESS_CALCULATOR_JS: &str = include_str!("../resources/witness_calculator.js");
const ARKZKEY_BYTES: &[u8] =
include_bytes!("../../rln/resources/tree_height_20/rln_final.arkzkey");
@@ -89,7 +97,7 @@ mod tests {
.expect("Failed to initialize thread pool");
}
// Initialize the witness calculator
// Initialize witness calculator
initWitnessCalculator(WITNESS_CALCULATOR_JS)
.expect("Failed to initialize witness calculator");

28
rln-wasm/tests/node.rs
View File

@@ -18,22 +18,39 @@ mod tests {
OptimalMerkleProof, OptimalMerkleTree, ZerokitMerkleProof, ZerokitMerkleTree,
};
const WITNESS_CALCULATOR_JS: &str = include_str!("../resources/witness_calculator.js");
#[wasm_bindgen(inline_js = r#"
const fs = require("fs");
let witnessCalculatorModule = null;
module.exports = {
initWitnessCalculator: function(code) {
const processedCode = code
.replace(/export\s+async\s+function\s+builder/, 'async function builder')
.replace(/export\s*\{\s*builder\s*\};?/g, '');
const moduleFunc = new Function(processedCode + '\nreturn { builder };');
witnessCalculatorModule = moduleFunc();
if (typeof witnessCalculatorModule.builder !== 'function') {
return false;
}
return true;
},
readFile: function (path) {
return fs.readFileSync(path);
},
calculateWitness: async function (circom_path, inputs) {
const wc = require("resources/witness_calculator_node.js");
const wasmFile = fs.readFileSync(circom_path);
const wasmFileBuffer = wasmFile.slice(
wasmFile.byteOffset,
wasmFile.byteOffset + wasmFile.byteLength
);
const witnessCalculator = await wc(wasmFileBuffer);
const witnessCalculator = await witnessCalculatorModule.builder(wasmFileBuffer);
const calculatedWitness = await witnessCalculator.calculateWitness(
inputs,
false
@@ -45,6 +62,9 @@ mod tests {
};
"#)]
extern "C" {
#[wasm_bindgen(catch)]
fn initWitnessCalculator(code: &str) -> Result<bool, JsValue>;
#[wasm_bindgen(catch)]
fn readFile(path: &str) -> Result<Uint8Array, JsValue>;
@@ -58,6 +78,10 @@ mod tests {
#[wasm_bindgen_test]
pub async fn rln_wasm_benchmark() {
// Initialize witness calculator
initWitnessCalculator(WITNESS_CALCULATOR_JS)
.expect("Failed to initialize witness calculator");
let mut results = String::from("\nbenchmarks:\n");
let iterations = 10;

3
rln/Cargo.toml
View File

@@ -30,7 +30,7 @@ ark-serialize = { version = "0.5.0", default-features = false }
thiserror = "2.0.12"
# utilities
rayon = { version = "1.7.0" }
rayon = { version = "1.10.0", optional = true }
byteorder = "1.5.0"
cfg-if = "1.0"
num-bigint = { version = "0.4.6", default-features = false, features = ["std"] }
@@ -58,6 +58,7 @@ criterion = { version = "0.7.0", features = ["html_reports"] }
default = ["parallel", "pmtree-ft"]
stateless = []
parallel = [
"rayon",
"utils/parallel",
"ark-ff/parallel",
"ark-ec/parallel",

4
utils/Cargo.toml
View File

@@ -14,12 +14,12 @@ bench = false
[dependencies]
ark-ff = { version = "0.5.0", default-features = false }
num-bigint = { version = "0.4.6", default-features = false }
pmtree = { package = "vacp2p_pmtree", version = "2.0.2", optional = true }
pmtree = { package = "vacp2p_pmtree", version = "2.0.3", optional = true }
sled = "0.34.7"
serde_json = "1.0.141"
lazy_static = "1.5.0"
hex = "0.4.3"
rayon = "1.7.0"
rayon = "1.10.0"
thiserror = "2.0"
[dev-dependencies]