diff --git a/examples/mean/mean.ipynb b/examples/mean/mean.ipynb index 5f29efa..88985a4 100644 --- a/examples/mean/mean.ipynb +++ b/examples/mean/mean.ipynb @@ -25,7 +25,7 @@ "metadata": {}, "outputs": [], "source": [ - "from zkstats.core import create_dummy, verifier_define_calculation, prover_gen_witness_array,prover_gen_settings, setup, prover_gen_proof, verifier_verify, generate_data_commitment" + "from zkstats.core import create_dummy, verifier_define_calculation,prover_gen_settings, setup, prover_gen_proof, verifier_verify, generate_data_commitment" ] }, { @@ -58,7 +58,8 @@ "# this is just dummy random value\n", "sel_dummy_data_path = os.path.join('shared/sel_dummy_data.json')\n", "data_commitment_path = os.path.join('shared/data_commitment.json')\n", - "witness_array_path = os.path.join('shared/witness_array.json')" + "precal_witness_path = os.path.join('shared/precal_witness_arr.json')\n", + "# aggregate_witness_path = os.path.join('shared/aggregate_witness.json')" ] }, { @@ -106,18 +107,47 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 6, "metadata": {}, "outputs": [ { - "ename": "AttributeError", - "evalue": "type object 'Model' has no attribute 'clone'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[8], line 17\u001b[0m\n\u001b[1;32m 14\u001b[0m _, prover_model \u001b[39m=\u001b[39m computation_to_model(computation, error)\n\u001b[1;32m 16\u001b[0m \u001b[39m# prover gen witness array file\u001b[39;00m\n\u001b[0;32m---> 17\u001b[0m prover_gen_witness_array(data_path,selected_columns,sel_data_path,prover_model\u001b[39m.\u001b[39;49mclone(), witness_array_path)\n\u001b[1;32m 19\u001b[0m \u001b[39m# prover gen_settings a\u001b[39;00m\n\u001b[1;32m 20\u001b[0m prover_gen_settings(data_path, selected_columns, sel_data_path, prover_model, prover_model_path, scales, \u001b[39m\"\u001b[39m\u001b[39mresources\u001b[39m\u001b[39m\"\u001b[39m, settings_path)\n", - "\u001b[0;31mAttributeError\u001b[0m: type object 'Model' has no attribute 'clone'" + "name": "stdout", + "output_type": "stream", + "text": [ + "Prover side\n", + "final op: \n", + "==== Generate & Calibrate Setting ====\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/jernkun/Desktop/zk-stats-lib/zkstats/computation.py:208: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect.\n", + " is_precise_aggregated = torch.tensor(1.0)\n", + "/Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages/torch/onnx/symbolic_opset9.py:2174: FutureWarning: 'torch.onnx.symbolic_opset9._cast_Bool' is deprecated in version 2.0 and will be removed in the future. Please Avoid using this function and create a Cast node instead.\n", + " return fn(g, to_cast_func(g, input, False), to_cast_func(g, other, False))\n", + "/Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages/torch/onnx/utils.py:1703: UserWarning: The exported ONNX model failed ONNX shape inference. The model will not be executable by the ONNX Runtime. If this is unintended and you believe there is a bug, please report an issue at https://github.com/pytorch/pytorch/issues. Error reported by strict ONNX shape inference: [ShapeInferenceError] (op_type:Where, node name: /Where): Y has inconsistent type tensor(float) (Triggered internally at /Users/runner/work/pytorch/pytorch/pytorch/torch/csrc/jit/serialization/export.cpp:1490.)\n", + " _C._check_onnx_proto(proto)\n", + "\n", + "\n", + " <------------- Numerical Fidelity Report (input_scale: 3, param_scale: 3, scale_input_multiplier: 10) ------------->\n", + "\n", + "+--------------+--------------+-------------+-----------+----------------+------------------+---------------+---------------+--------------------+--------------------+------------------------+\n", + "| mean_error | median_error | max_error | min_error | mean_abs_error | median_abs_error | max_abs_error | min_abs_error | mean_squared_error | mean_percent_error | mean_abs_percent_error |\n", + "+--------------+--------------+-------------+-----------+----------------+------------------+---------------+---------------+--------------------+--------------------+------------------------+\n", + "| 0.0044994354 | 0.008998871 | 0.008998871 | 0 | 0.0044994354 | 0.008998871 | 0.008998871 | 0 | 0.00004048984 | 0.00010678871 | 0.00010678871 |\n", + "+--------------+--------------+-------------+-----------+----------------+------------------+---------------+---------------+--------------------+--------------------+------------------------+\n", + "\n", + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "scale: [3]\n", + "setting: {\"run_args\":{\"tolerance\":{\"val\":0.0,\"scale\":1.0},\"input_scale\":3,\"param_scale\":3,\"scale_rebase_multiplier\":10,\"lookup_range\":[-288,300],\"logrows\":12,\"num_inner_cols\":2,\"variables\":[[\"batch_size\",1]],\"input_visibility\":{\"Hashed\":{\"hash_is_public\":true,\"outlets\":[]}},\"output_visibility\":\"Public\",\"param_visibility\":\"Fixed\",\"div_rebasing\":false,\"rebase_frac_zero_constants\":false,\"check_mode\":\"UNSAFE\"},\"num_rows\":3936,\"total_assignments\":1017,\"total_const_size\":361,\"model_instance_shapes\":[[1],[1]],\"model_output_scales\":[0,3],\"model_input_scales\":[3],\"module_sizes\":{\"kzg\":[],\"poseidon\":[3936,[1]]},\"required_lookups\":[{\"GreaterThan\":{\"a\":0.0}},\"Abs\"],\"required_range_checks\":[],\"check_mode\":\"UNSAFE\",\"version\":\"9.1.0\",\"num_blinding_factors\":null,\"timestamp\":1714794824466}\n" ] } ], @@ -135,12 +165,7 @@ "\n", "\n", "# Prover/ data owner side\n", - "_, prover_model = computation_to_model(computation, error)\n", - "\n", - "# prover gen witness array file\n", - "prover_gen_witness_array(data_path,selected_columns,sel_data_path,prover_model, witness_array_path)\n", - "\n", - "# prover gen_settings a\n", + "_, prover_model = computation_to_model(computation, precal_witness_path, True, error)\n", "prover_gen_settings(data_path, selected_columns, sel_data_path, prover_model, prover_model_path, scales, \"resources\", settings_path)\n", "\n" ] @@ -158,37 +183,24 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "witness array: [tensor(42.1340)]\n", - "verrrr\n", - "x sy: \n", - "final op: \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/jernkun/Desktop/zk-stats-lib/zkstats/computation.py:191: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect.\n", - " is_precise_aggregated = torch.tensor(1.0)\n", - "/Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages/torch/onnx/symbolic_opset9.py:2174: FutureWarning: 'torch.onnx.symbolic_opset9._cast_Bool' is deprecated in version 2.0 and will be removed in the future. Please Avoid using this function and create a Cast node instead.\n", - " return fn(g, to_cast_func(g, input, False), to_cast_func(g, other, False))\n", - "/Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages/torch/onnx/utils.py:1703: UserWarning: The exported ONNX model failed ONNX shape inference. The model will not be executable by the ONNX Runtime. If this is unintended and you believe there is a bug, please report an issue at https://github.com/pytorch/pytorch/issues. Error reported by strict ONNX shape inference: [ShapeInferenceError] (op_type:Where, node name: /Where): Y has inconsistent type tensor(float) (Triggered internally at /Users/runner/work/pytorch/pytorch/pytorch/torch/csrc/jit/serialization/export.cpp:1490.)\n", - " _C._check_onnx_proto(proto)\n" + "Verifier side\n", + "mean tensor arr: [tensor(42.1340)]\n", + "final op: \n" ] } ], "source": [ - "witness_array_data = json.loads(open(witness_array_path, \"r\").read())['value']\n", - "witness_array = [torch.tensor(witness_array_data[0])]\n", - "print('witness array: ', witness_array)\n", - "_, verifier_model = computation_to_model(computation, error, witness_array)\n", + "# witness_array_data = json.loads(open(witness_array_path, \"r\").read())['value']\n", + "# witness_array = [torch.tensor(witness_array_data[0])]\n", + "# print('witness array: ', witness_array)\n", + "_, verifier_model = computation_to_model(computation, precal_witness_path, False,error)\n", "\n", "verifier_define_calculation(dummy_data_path, selected_columns, sel_dummy_data_path,verifier_model, verifier_model_path)" ] @@ -209,28 +221,22 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ + "==== setting up ezkl ====\n", + "Time setup: 0.5510029792785645 seconds\n", "=======================================\n", "==== Generating Witness ====\n", - "witness boolean: 42.125\n" - ] - }, - { - "ename": "IndexError", - "evalue": "list index out of range", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[12], line 7\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39m=======================================\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 6\u001b[0m \u001b[39m# Prover generates proof\u001b[39;00m\n\u001b[0;32m----> 7\u001b[0m prover_gen_proof(prover_model_path, sel_data_path, witness_path, prover_compiled_model_path, settings_path, proof_path, pk_path)\n", - "File \u001b[0;32m~/Desktop/zk-stats-lib/zkstats/core.py:180\u001b[0m, in \u001b[0;36mprover_gen_proof\u001b[0;34m(prover_model_path, sel_data_path, witness_path, prover_compiled_model_path, settings_path, proof_path, pk_path)\u001b[0m\n\u001b[1;32m 178\u001b[0m \u001b[39m# print(\"witness boolean: \", ezkl.vecu64_to_float(witness['outputs'][0][0], output_scale[0]))\u001b[39;00m\n\u001b[1;32m 179\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mwitness boolean: \u001b[39m\u001b[39m\"\u001b[39m, ezkl\u001b[39m.\u001b[39mfelt_to_float(witness[\u001b[39m'\u001b[39m\u001b[39moutputs\u001b[39m\u001b[39m'\u001b[39m][\u001b[39m0\u001b[39m][\u001b[39m0\u001b[39m], output_scale[\u001b[39m0\u001b[39m]))\n\u001b[0;32m--> 180\u001b[0m \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(\u001b[39mlen\u001b[39m(witness[\u001b[39m'\u001b[39;49m\u001b[39moutputs\u001b[39;49m\u001b[39m'\u001b[39;49m][\u001b[39m1\u001b[39;49m])):\n\u001b[1;32m 181\u001b[0m \u001b[39m# print(\"witness result\", i+1,\":\", ezkl.vecu64_to_float(witness['outputs'][1][i], output_scale[1]))\u001b[39;00m\n\u001b[1;32m 182\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mwitness result\u001b[39m\u001b[39m\"\u001b[39m, i\u001b[39m+\u001b[39m\u001b[39m1\u001b[39m,\u001b[39m\"\u001b[39m\u001b[39m:\u001b[39m\u001b[39m\"\u001b[39m, ezkl\u001b[39m.\u001b[39mfelt_to_float(witness[\u001b[39m'\u001b[39m\u001b[39moutputs\u001b[39m\u001b[39m'\u001b[39m][\u001b[39m1\u001b[39m][i], output_scale[\u001b[39m1\u001b[39m]))\n\u001b[1;32m 184\u001b[0m \u001b[39m# GENERATE A PROOF\u001b[39;00m\n", - "\u001b[0;31mIndexError\u001b[0m: list index out of range" + "witness boolean: 1.0\n", + "witness result 1 : 42.125\n", + "==== Generating Proof ====\n", + "proof: {'instances': [['11e5950d0c875140b38d8b4bc0997697b7b183cfdbc19e767d87caf0020da12a', '0100000000000000000000000000000000000000000000000000000000000000', '5101000000000000000000000000000000000000000000000000000000000000']], 'proof': '0x06ad4caccfb7e3c66c1efec358439068d1ee0ace048d2fedfa9bea5abfbff31f2ab38f76eaba54e3ec6ce079567b40cd805060f74ab84a116f7e1854a2540b390e2f962b6a50502bebdec5b5c4ecd09ddbe05c811685b233670e69187e66d9d41a41ee6cab93a1cc137d02816530bf5c270505777a2cdc810f4516e0ac2d805a1f6fe755373deaf0dcbd69bf864d3a015d8ab35a85c953c76d9c8265235b75a207f2914474f8596fb47fbe7b007302f7c673253eb3ef9c23fdf66d0b387f059628b9c9684c58ea6ce84fa4e9e6823a3f8410e702b5ae27addd03579656c1f7770aa5f5aa3629d08ccb7f07e11a1c1d91ac453e40e00eff453c8f1552e8b62a75136bfd94af9c2bf9c91409acfc29f7fb22cae93928c92779fef52f8fd77dc937118bfefe3c0b47c51d1f4686f718d91e80a8cc3de307dd4901cb0dd56de8767f2047b3ba48cf44e05bbd32c8a29f3d7eb2ab1ed46b847afdcf5a3a66518768902eaf93fc912ca5214f582b572dc244859079c491989f57272ccfb54c7323a21911ac0696e4e705521a72a88d8d11c3da0f65c84a57c4622fb497e31c1e57fd602775004b8a94b85bad1e34113e8b9cb45a07f78740e653e8b5c25698f7df4b48011695477126406f8f03ae3ac154f459dc1aee5abc42c871677aaeea3a131905010345b1fdecbc2b1ba2618f50fc9e46e52d84edf475ac828774fe52af673a1405908f9a6bb377ba3e19d95be73d1f77d80a2a8df05c3f8ed3f648cbce832f70044abd7c60c5bacd701820f5ae14362d66a17f6ee8cd4e39d39d06fc5e2eab7a196403b8f091e092508911daa32fd847e3b83ab431fed0baa60106a26e3d7f730e23a3cb4386422e41e24b1eeae4ea86419b937beb321d0a9dafcf9570afb341161377416d8cbb451075147c682ef3ceb7dd50cb874f4d3cecd973e4434b60d512106651494e9a0f64c81b849c2dba84b44cd19fb74299d861c8ff68c2fbbcd503613a24a193edd56e5bca0c3437a2d5f87ae479cc2ac3427212f2e29698ab892d441b5dd7a532e8b36eddffad661530aaaa65f3a7dcc8f164d88e53be1605ad161377416d8cbb451075147c682ef3ceb7dd50cb874f4d3cecd973e4434b60d512106651494e9a0f64c81b849c2dba84b44cd19fb74299d861c8ff68c2fbbcd5177ad48ae8d4b90e17163725948afe868b44a678e653cdb10528b16968cbdb9516620dcf4ecb62923d6cefc1b8de5f3133191ea63cbcd256ef923699b9f07d8d25d21d33ae01a849b72ed09ae07f46f56a45a2ba1242d2620618b0e3a725b018035be97fc2507db53cfdd39dc34027e1a0f06ddddff408cb86745f60198eb5cd027715e08b7c4157acbb0f1194368eaeda33ff981ae6f805d9d188e8bd132d7215c290ffe6de7e5051a7f07bec1686fe7ef39874bab0cd9c532f5b662287fc19168313b74460ec9a267e465ea4a7e71efac47054f1664dd4390968d67a09e981074f74eb5197d21442cb054c9d9c89f5212f6043840c17cde6eba0f9b3a780ff18ab592f48dcb90ef350335c2bda94ff236efd33c9d259bfd071a9c9ab7e062a091bb2198b311fa46477d85925d386d17b2492484c3e9d34387d065d7df78c702accca3a09db6d518405e94c4d376fcbf014f6e2c961f4993961e8cfc8fa3cbf0f49a83d080a22aca85d40b2063f0bc3005fd443f4b7f8609e5c0fe8c138f5800754c5aa18175fd6ffc79ddc95e9497c197899924ffb89bcf1ea2799e394eecb30641c2bd169223274aa786c24f7e28839ac51bc1373fe0226e3ea168a9ea1b70052cbdbdf0652df3c39c481280bfdc5df3b2a5bc45fd7a55a3066836c77e7210e3b114424e9177e7c8846915408683efed6f5ba2ae94bf180520f761c61edb2052db594927abb7c3a935dea16c0a7f265d6ca5d006a67e877fceb10e953685c117450fe599731b75af0ef72291af9877dc779c6478c41198516cb4a434790bc21eb9b35841be19ae7ec6575b6bc38bda8c03548fa06b5f8f09342430915de201cfa5daf56282775960f45f8d7e49eef3e8f94ccdab8e0c54562637828b826132e22243c00e86a16e94063062293228bdd15fceba81ac4f5e8dd71287cf104b809b3046b44b2661cdf05fdbfa94ba8f2a28f00e7fcbe89989541fc075cdb0d8a2682b9b86cd58fc69ee1e07c1184ec5f9621ff3cc41f1f4e0f66ceed15bd26021e676d31c969d5ba3184ff6d2a083e47ed9d0d6b24c22c9e9cde9fba176254bc06bf1da0bc6cce4655c6b27edbf680f5d604f1ee30a97b7587f92f740672b6a526eb527b6714b1770d5c4578afef1ded9594f0a93675c39c36f1d616aa84670b1e71befcf7f288900924d0f3cd9eab5679eaa04e3cd79b50a7a42b8afcde2ee12639f181acb8c4f109a0dbb74bd017d4730b4d2e0ef5f2c2d88c05d85ca180651752982ee4c102c9f1571ad99fee5615175984a4c1450c78086f453b6ac140722bcdc34455a9eaca540132cd8cd381dc46907773411c7846a065256575a524af21d6ba3875b40213e7c4890439e0c71c0155cc35710bde0c9a88eecd136954f1194b9f584a221ecc6febc35dcd6287e94d19e1bdfa4081f93083b6ef02dcd26502e8d902a2904a72338ed544771227eb07630ab32da2644f4447082ae38164bb2996a0b9fcd85b91368f1ade91d3b21120d35288215d4ea518469c3491fa5d012548a5cda092fde3d93b8e309a3832b2e67839bbd8922f86561c0bcb11a8eaf71dbfcb713558fa5137edabc2e77f81ce7f8f0477ad4dcd20f9d2c19f1f14b3272a31aa119ac137e6e0bd14d87248ec76e49a273e4d1d45c495809ee7f9322b03304be4673bfe25402513ee7272bf53bdd000224d2bd568e5347e36d83b536fad2875158822282617a65933397bb47876bb663a27dd5a4a8f3b685721925f46520a509c2eaa0a15375660436e079dd25e97e39af0b2b32b0e33479abfb06f61601b2c0473b591f172c83639ef0990d566f292e13c7eb6bf2fb11794e24c8c34a118fa5feec3d7d1af3ed4571051fd06c92206cedafdbdd11dbfe6bc8e46fdce582e6da3f94703c7eae2c44551077acb069ecbc721ad5a53c1a9cf3f8866d708b12fa9c30312bc2859c8dcc27099f22cbd17a2e89094a0195eba02b0f62e4f81b71aa630a29125cb70e4b626873ac87a05d39532b50feac4fa548f08bd95b5d3a210842ba7abab58ab02f50deada3179cc5ad8f1d5df118f1d8353585b93d02e8a1fb81ee1bb0e415463d9ec28bd3f49eea383a4487597582b751d8526f0b4fea60e8c77b170f28535f1f1dbe88434566700fd05d80f22f72765c71fffff3a7d8f2c694d1fb56b8622847c6bbeb0064d05fdb742f105c02981f1f73bc545bf5eb20a497dc01b7146473af1db40a2efbafd8bcd80fdae86ecdc3a4487f0b42fd08426a932927b1705860d3ed28c42113ada70d703000e8ef4308034788f4fdcacca04d893931361abf93d3939fed7be765b3091a08bc3900c75f277d785ea21b9ad20338f75da25bd115e51656501cede3f883bf6d123ebf0de1971568b6aea5d8000000000000000000000000000000000000000000000000000000000000000002d67d40d2ea164e715971eb261df9e091ef3983b9b25ce625c7283070c0a5352061ba9bfd0de4af1b07bdf30ef9a503d5d6c0aa66a4c9f53182544036da9d31a00000000000000000000000000000000000000000000000000000000000000002e4dc99331b14793d7d96ff32ed492db22c539930bd8d1c2c1eae7400add1e3c0000000000000000000000000000000000000000000000000000000000000000010bf21cc5841619425afdb0b4cdc838ded30c673a9ed400f2b40a876a8a149f0000000000000000000000000000000000000000000000000000000000000000137591ee71554dce32282ed14bd4cc79bf86147be8afc95e649b65369840a08a061ba9bfd0de4af1b07bdf30ef9a503d5d6c0aa66a4c9f53182544036da9d31a00000000000000000000000000000000000000000000000000000000000000001358b6adf07e3048764f96bf46ca20342533c4036ab5d140af9e330b3a05229f00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a8dc1559f2585c044d13053321e41233fe73a0c08c4590cfe1d2445283145572ffde846be4e93e7d9b06fc6deda68be3a7d1ffe832bd311bd15d4b7028238090f4985eec88854a5c284c4d50b0e6b2a08be1cb487d2c3d0076685df24314a390000000000000000000000000000000000000000000000000000000000000000138f9ec9677e8a9749e788b9e673e79709e0e4743d0c3dc4602e906cd30761e8000000000000000000000000000000000000000000000000000000000000000006c612578ceff4e67ccaddd0609884d2753f1c88f19e7c47f605496a759f40bb096f6d2a1ae30d91c62325ef86af7930ba520548510cf17562e183ca6aff0e600272eb2aace08cf1e7328ba23e71e3af6919fb1f02fb662df7cc3292dbe9d5ad1c3893f6e1bceb1a4ec6b963fb24cbb15e0f15322478e1b55b5a524dcb8ffd250ef2ae8b48a93d25a39ff204406227bae395f7daf31d00b0db25d6b240f4d1b00b4639a963fa6bc66b51189630de04a48fbfcd0f2a46cebddc6e6ccbe00f0d7d079582ed17d2ef81e00a4fc54e0f8a7c31e8e12dffdc8f1ca27d3b39889ee08f1208d8c20adbef24d738296ef366de7514cdd02327b56f9b030b6c909f176e870d177dcb1683fc4f74dbc5160a99908499018801af955b2b3467caed04d0522810b07ea0d9f052e42a2c28e5057f11769aa826ad041d8e2bd187555d03c0b83c1208c738735117399212c615fccc6bbb065dce03f8eaf1f0991b83cc9b2285e01492a07eab8e790c359d642c29e6013c155a80ad7993f81462b1101e37c81fdd034034a9ed3f2e66d9f6f4d21671e3179c17a34ae73d06969baff1d5c059886821522639e3e0cb4cf6a89c5d70ec6501146cfac3d963e20b1a6adac09424638f2bfec1eac372fdec8117548dc6d5d7c5325f37c0e78ccb676d30c742034e52c61659ecb11933fedef7b52af0e74f0ce9daeb92d1c08acebb98f770f4c617913b30280deba7d62d8b566afc517540edc5daf94beedf0c6f11c7ed91031877d5ce0daf05ceb600b5d215306b137da40b4fced855c6f4d727ba348ffd696030fa73052910348680078279eee6750c3092454cff9f53eb388655938721cf884fb6961876d3dfb5e271d93c64c51a314f779d6fe7dced747cf5b3fba09905d33fef0422b5aebf77670200578bf5a640b6756f71f8d82cea3f75cca9f42da13f4208e52115d3ff7fd89cad31eb2da263e2a45ed813f875f205a36400e87bd287e4d42405f0e4304bc38c99f3ff9d21f85f746f1081ac14738c892d83ccc39e78e94f022af4e4b31be86770b928326b3465cee8363bd21a60ff6d2080abe289074511bd230bc186c1a26c9e76a4694fd5efb5be2dd45e1bd8c9924ed76974428fb3abca1f6254ee17b5e4f3a16a92649ae00b65596f587db822cd8a928fe0e038f17e460f6186800b419e28d94be226d741b4a6cf15e18ed92a2b64fc18b6aa81aa26a23059aa16f89d2ad9dc1dc992ecd91a1e6aedbea57b53e0c47f3a3020bf826d552cd0543efeaf917d3b21627881194634a120c11d260c0ad210ec0f3df498f54b093351b265c6a4cda71b2fcb83cd30e194628908b283df490c643a39813301330ba6b8efece58c967d7e6dc12f7b0c3be47d9a7c8f9120fea0d6caaf4b57c3841f57f2417d76f4b0680fd75cb125ed55fdb0d256d9a0bef99d2f9a3de9ab44d50f6186800b419e28d94be226d741b4a6cf15e18ed92a2b64fc18b6aa81aa26a21791da9d20127d06876a9e7fe4d78859e1dfbe28196de327f125cce1d332f9e7243104f57f5ca5f68b2b027d9405d1e324d04ccfd6476fa24ceaaf4ea754606601d05f30997c436ff7e28f392dd8f4b43508aaa5136c23247b1b276e8b04695f234b5b7649ba28d4eabf8ca609c06bf57a84fa3d5f17da8a48c9caaeb4ad2d0e', 'transcript_type': 'EVM'}\n", + "Time gen prf: 0.6677489280700684 seconds\n" ] } ], @@ -246,19 +252,14 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "metadata": {}, "outputs": [ { - "ename": "RuntimeError", - "evalue": "Failed to run verify: The constraint system is not satisfied", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[10], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[39m# Verifier verifies\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m res \u001b[39m=\u001b[39m verifier_verify(proof_path, settings_path, vk_path, selected_columns, data_commitment_path)\n\u001b[1;32m 3\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mVerifier gets result:\u001b[39m\u001b[39m\"\u001b[39m, res)\n", - "File \u001b[0;32m~/Desktop/zk-stats-lib/zkstats/core.py:216\u001b[0m, in \u001b[0;36mverifier_verify\u001b[0;34m(proof_path, settings_path, vk_path, selected_columns, data_commitment_path)\u001b[0m\n\u001b[1;32m 205\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 206\u001b[0m \u001b[39mVerify the proof and return the result.\u001b[39;00m\n\u001b[1;32m 207\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 212\u001b[0m \u001b[39m be stored in `expected_data_commitments[i]`.\u001b[39;00m\n\u001b[1;32m 213\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 215\u001b[0m \u001b[39m# 1. First check the zk proof is valid\u001b[39;00m\n\u001b[0;32m--> 216\u001b[0m res \u001b[39m=\u001b[39m ezkl\u001b[39m.\u001b[39;49mverify(\n\u001b[1;32m 217\u001b[0m proof_path,\n\u001b[1;32m 218\u001b[0m settings_path,\n\u001b[1;32m 219\u001b[0m vk_path,\n\u001b[1;32m 220\u001b[0m )\n\u001b[1;32m 221\u001b[0m \u001b[39m# TODO: change asserts to return boolean\u001b[39;00m\n\u001b[1;32m 222\u001b[0m \u001b[39massert\u001b[39;00m res \u001b[39m==\u001b[39m \u001b[39mTrue\u001b[39;00m\n", - "\u001b[0;31mRuntimeError\u001b[0m: Failed to run verify: The constraint system is not satisfied" + "name": "stdout", + "output_type": "stream", + "text": [ + "Verifier gets result: [42.125]\n" ] } ], diff --git a/zkstats/computation.py b/zkstats/computation.py index 1ee1734..a7c3ee2 100644 --- a/zkstats/computation.py +++ b/zkstats/computation.py @@ -3,6 +3,7 @@ from typing import Callable, Type, Optional, Union import torch from torch import nn +import json from .ops import ( Operation, @@ -43,20 +44,23 @@ class State: self.error: float = error # Pointer to the current operation index. If None, it's in stage 1. If not None, it's in stage 3. self.current_op_index: Optional[int] = None - self.witness_array: Optional[list[torch.Tensor]] = None + self.precal_witness_path: str = None + self.precal_witness:dict = {} + self.isProver:bool = None + def set_ready_for_exporting_onnx(self) -> None: self.current_op_index = 0 - def set_witness(self,witness_array) -> None: - self.witness_array = witness_array + # def set_witness(self,witness_array) -> None: + # self.witness_array = witness_array + # def set_aggregate_witness_path(self,aggregate_witness_path) -> None: + # self.aggregate_witness_path = aggregate_witness_path + # def get_aggregate_witness(self) -> list[torch.Tensor]: + # return self.aggregate_witness def mean(self, x: torch.Tensor) -> torch.Tensor: """ Calculate the mean of the input tensor. The behavior should conform to [statistics.mean](https://docs.python.org/3/library/statistics.html#statistics.mean) in Python standard library. """ - # if self.witness_array is not None: - # print('self.wtiness ', self.witness_array) - # return self._call_op([x], Mean, self.witness_array) - # else: return self._call_op([x], Mean) def median(self, x: torch.Tensor) -> torch.Tensor: @@ -150,10 +154,23 @@ class State: def _call_op(self, x: list[torch.Tensor], op_type: Type[Operation]) -> Union[torch.Tensor, tuple[IsResultPrecise, torch.Tensor]]: if self.current_op_index is None: - if self.witness_array is not None: - op = op_type.create(x, self.error, self.witness_array) - else: + # for prover + if self.isProver: + print('Prover side') op = op_type.create(x, self.error) + # print('oppy : ', op) + # print('is check pri 1: ', isinstance(op,Mean)) + if isinstance(op,Mean): + self.precal_witness['Mean'] = [op.result.data.item()] + # for verifier + else: + print('Verifier side') + # if isinstance(op,Mean): + precal_witness = json.loads(open(self.precal_witness_path, "r").read()) + # tensor_arr = [] + # for ele in data: + # tensor_arr.append(torch.tensor(ele)) + op = op_type.create(x, self.error, precal_witness) self.ops.append(op) return op.result else: @@ -199,8 +216,8 @@ class State: # return as where result return is_precise_aggregated, op.result+x[1]-x[1] else: - # return as a single number - # return is_precise_aggregated, torch.tensor(40.0)+(x[0]-x[0])[0][0][0] + if self.isProver: + json.dump(self.precal_witness, open(self.precal_witness_path, 'w')) return is_precise_aggregated, op.result+(x[0]-x[0])[0][0][0] elif current_op_index > len_ops - 1: @@ -213,6 +230,9 @@ class State: else: # return single float number # return torch.where(x[0], x[1], 9999999) + # print('oppy else: ', op) + # print('is check else: ', isinstance(op,Mean)) + # self.aggregate_witness.append(op.result) return op.result+(x[0]-x[0])[0][0][0] @@ -234,7 +254,7 @@ class IModel(nn.Module): TComputation = Callable[[State, list[torch.Tensor]], torch.Tensor] -def computation_to_model(computation: TComputation, error: float = DEFAULT_ERROR, witness_array: Optional[list[torch.Tensor]] = None ) -> tuple[State, Type[IModel]]: +def computation_to_model(computation: TComputation, precal_witness_path:str, isProver:bool ,error: float = DEFAULT_ERROR ) -> tuple[State, Type[IModel]]: """ Create a torch model from a `computation` function defined by user :param computation: A function that takes a State and a list of torch.Tensor, and returns a torch.Tensor @@ -244,16 +264,18 @@ def computation_to_model(computation: TComputation, error: float = DEFAULT_ERROR """ state = State(error) # if it's verifier - if witness_array is not None: - state.set_witness(witness_array) + state.precal_witness_path= precal_witness_path + state.isProver = isProver + class Model(IModel): def preprocess(self, x: list[torch.Tensor]) -> None: computation(state, x) state.set_ready_for_exporting_onnx() def forward(self, *x: list[torch.Tensor]) -> tuple[IsResultPrecise, torch.Tensor]: - print('x sy: ') + # print('x sy: ') return computation(state, x) + # print('state:: ', state.aggregate_witness_path) return state, Model diff --git a/zkstats/core.py b/zkstats/core.py index f61a6d4..e4b4ad1 100644 --- a/zkstats/core.py +++ b/zkstats/core.py @@ -54,24 +54,24 @@ def create_dummy(data_path: str, dummy_data_path: str) -> None: # =================================================================================================== # =================================================================================================== -def prover_gen_witness_array( - data_path:str, - selected_columns:list[str], - sel_data_path:list[str], - prover_model: Type[IModel], - witness_array_path:str -): - data_tensor_array = _process_data(data_path, selected_columns, sel_data_path) +# def prover_gen_witness_array( +# data_path:str, +# selected_columns:list[str], +# sel_data_path:list[str], +# prover_model: Type[IModel], +# witness_array_path:str +# ): +# data_tensor_array = _process_data(data_path, selected_columns, sel_data_path) - circuit = prover_model() - # cloned_circuit = circuit.clone() - circuit.eval() - # be careful of tuple here --> array --> tuple need something like in export_onnx - one_witness = circuit.forward(data_tensor_array[0]).data.item() - print('one witness: ', one_witness) +# circuit = prover_model() +# # cloned_circuit = circuit.clone() +# circuit.eval() +# # be careful of tuple here --> array --> tuple need something like in export_onnx +# one_witness = circuit.forward(data_tensor_array[0]).data.item() +# print('one witness: ', one_witness) - data ={'value':[one_witness]} - json.dump(data, open(witness_array_path, 'w')) +# data ={'value':[one_witness]} +# json.dump(data, open(witness_array_path, 'w')) def prover_gen_settings( diff --git a/zkstats/ops.py b/zkstats/ops.py index 73c50be..7b78b82 100644 --- a/zkstats/ops.py +++ b/zkstats/ops.py @@ -36,16 +36,20 @@ class Where(Operation): class Mean(Operation): @classmethod - def create(cls, x: list[torch.Tensor], error: float, witness_array:Optional[list[torch.Tensor]] = None ) -> 'Mean': + def create(cls, x: list[torch.Tensor], error: float, precal_witness:dict = None ) -> 'Mean': # support where statement, hopefully we can use 'nan' once onnx.isnan() is supported - if witness_array is None: + if precal_witness is None: # this is prover - print('provvv') + # print('provvv') return cls(torch.mean(x[0][x[0]!=MagicNumber]), error) else: # this is verifier - print('verrrr') - return cls(witness_array[0], error) + # print('verrrr') + tensor_arr = [] + for ele in precal_witness['Mean']: + tensor_arr.append(torch.tensor(ele)) + print("mean tensor arr: ", tensor_arr) + return cls(tensor_arr[0], error) def ezkl(self, x: list[torch.Tensor]) -> IsResultPrecise: