mirror of
https://github.com/MPCStats/zk-stats-demo.git
synced 2026-01-08 21:28:00 -05:00
use 0.1.7 and add verifier notebook
This commit is contained in:
mhchia
File diff suppressed because one or more lines are too long
308
public/assets/verifier.ipynb
Normal file
308
public/assets/verifier.ipynb
Normal file
@@ -0,0 +1,308 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Requirement already satisfied: torch in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (2.2.2)\n",
|
||||
"Requirement already satisfied: zkstats==0.1.7 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (0.1.7)\n",
|
||||
"Requirement already satisfied: click<9.0.0,>=8.1.7 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from zkstats==0.1.7) (8.1.7)\n",
|
||||
"Requirement already satisfied: ezkl==9.1.0 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from zkstats==0.1.7) (9.1.0)\n",
|
||||
"Requirement already satisfied: matplotlib<4.0.0,>=3.8.2 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from zkstats==0.1.7) (3.8.4)\n",
|
||||
"Requirement already satisfied: numpy<2.0.0,>=1.26.2 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from zkstats==0.1.7) (1.26.4)\n",
|
||||
"Requirement already satisfied: onnx<2.0.0,>=1.15.0 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from zkstats==0.1.7) (1.16.0)\n",
|
||||
"Requirement already satisfied: requests<3.0.0,>=2.31.0 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from zkstats==0.1.7) (2.31.0)\n",
|
||||
"Requirement already satisfied: scipy<2.0.0,>=1.11.4 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from zkstats==0.1.7) (1.13.0)\n",
|
||||
"Requirement already satisfied: statistics<2.0.0,>=1.0.3 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from zkstats==0.1.7) (1.0.3.5)\n",
|
||||
"Requirement already satisfied: filelock in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from torch) (3.13.4)\n",
|
||||
"Requirement already satisfied: typing-extensions>=4.8.0 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from torch) (4.11.0)\n",
|
||||
"Requirement already satisfied: sympy in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from torch) (1.12)\n",
|
||||
"Requirement already satisfied: networkx in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from torch) (3.2.1)\n",
|
||||
"Requirement already satisfied: jinja2 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from torch) (3.1.3)\n",
|
||||
"Requirement already satisfied: fsspec in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from torch) (2024.3.1)\n",
|
||||
"Requirement already satisfied: contourpy>=1.0.1 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from matplotlib<4.0.0,>=3.8.2->zkstats==0.1.7) (1.2.1)\n",
|
||||
"Requirement already satisfied: cycler>=0.10 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from matplotlib<4.0.0,>=3.8.2->zkstats==0.1.7) (0.12.1)\n",
|
||||
"Requirement already satisfied: fonttools>=4.22.0 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from matplotlib<4.0.0,>=3.8.2->zkstats==0.1.7) (4.51.0)\n",
|
||||
"Requirement already satisfied: kiwisolver>=1.3.1 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from matplotlib<4.0.0,>=3.8.2->zkstats==0.1.7) (1.4.5)\n",
|
||||
"Requirement already satisfied: packaging>=20.0 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from matplotlib<4.0.0,>=3.8.2->zkstats==0.1.7) (24.0)\n",
|
||||
"Requirement already satisfied: pillow>=8 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from matplotlib<4.0.0,>=3.8.2->zkstats==0.1.7) (10.3.0)\n",
|
||||
"Requirement already satisfied: pyparsing>=2.3.1 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from matplotlib<4.0.0,>=3.8.2->zkstats==0.1.7) (3.1.2)\n",
|
||||
"Requirement already satisfied: python-dateutil>=2.7 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from matplotlib<4.0.0,>=3.8.2->zkstats==0.1.7) (2.9.0.post0)\n",
|
||||
"Requirement already satisfied: protobuf>=3.20.2 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from onnx<2.0.0,>=1.15.0->zkstats==0.1.7) (4.25.3)\n",
|
||||
"Requirement already satisfied: charset-normalizer<4,>=2 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from requests<3.0.0,>=2.31.0->zkstats==0.1.7) (3.3.2)\n",
|
||||
"Requirement already satisfied: idna<4,>=2.5 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from requests<3.0.0,>=2.31.0->zkstats==0.1.7) (3.7)\n",
|
||||
"Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from requests<3.0.0,>=2.31.0->zkstats==0.1.7) (2.2.1)\n",
|
||||
"Requirement already satisfied: certifi>=2017.4.17 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from requests<3.0.0,>=2.31.0->zkstats==0.1.7) (2024.2.2)\n",
|
||||
"Requirement already satisfied: docutils>=0.3 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from statistics<2.0.0,>=1.0.3->zkstats==0.1.7) (0.21.1)\n",
|
||||
"Requirement already satisfied: MarkupSafe>=2.0 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from jinja2->torch) (2.1.5)\n",
|
||||
"Requirement already satisfied: mpmath>=0.19 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from sympy->torch) (1.3.0)\n",
|
||||
"Requirement already satisfied: six>=1.5 in /Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages (from python-dateutil>=2.7->matplotlib<4.0.0,>=3.8.2->zkstats==0.1.7) (1.16.0)\n",
|
||||
"\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.1\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
|
||||
"Note: you may need to restart the kernel to use updated packages.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%pip install torch zkstats==0.1.7"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"import os\n",
|
||||
"from zkstats.core import (\n",
|
||||
" setup,\n",
|
||||
")\n",
|
||||
"from zkstats.computation import computation_to_model\n",
|
||||
"\n",
|
||||
"cwd = os.getcwd()\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Paths to the output files\n",
|
||||
"output_dir = f\"{cwd}/out\"\n",
|
||||
"os.makedirs(output_dir, exist_ok=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Shared by the data provider beforehand\n",
|
||||
"data_shape = {'x': 7, 'y': 7}\n",
|
||||
"data_commitment_path = f\"{output_dir}/data_commitment.json\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"User select the columns"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# FIXME: this should be provided by users\n",
|
||||
"# selected_columns = [\"x\", \"y\"]\n",
|
||||
"selected_columns = [\"x\"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## User-defined Computation\n",
|
||||
"\n",
|
||||
"A computation should be of type `TComputation`. For example, the following code snippet defines a computation that computes the sum of the private data.\n",
|
||||
"\n",
|
||||
"```python\n",
|
||||
"def computation(state: State, x: list[torch.Tensor]):\n",
|
||||
" out_0 = state.median(x[0])\n",
|
||||
" out_1 = state.median(x[1])\n",
|
||||
" return state.mean(torch.cat([out_0.unsqueeze(0), out_1.unsqueeze(0)]).reshape(-1,1))\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
"FIXME: The following code snippet is entirely from the user. You MUST check\n",
|
||||
"1. the code only performs zkstats-related operations.\n",
|
||||
"2. the computation must not leak any information about the private data."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# This is just a dummy computation. Replace it with user's computation\n",
|
||||
"import torch\n",
|
||||
"from zkstats.computation import State\n",
|
||||
"\n",
|
||||
"def computation(state: State, x: list[torch.Tensor]):\n",
|
||||
" # out_0 = state.median(x[0])\n",
|
||||
" # out_1 = state.median(x[1])\n",
|
||||
" # # return state.mean(torch.cat([out_0.unsqueeze(0), out_1.unsqueeze(0)]).reshape(-1,1)), out_0\n",
|
||||
" # return out_0, out_1\n",
|
||||
" return state.mean(x[0]), state.median(x[0])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Get proof, settings, and precal_witness from the data provider"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"proof_path = f\"{output_dir}/model.pf\"\n",
|
||||
"settings_path = f\"{output_dir}/settings.json\"\n",
|
||||
"precal_witness_path = f\"{output_dir}/precal_witness.json\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Verify the proof to ensure it is correct\n",
|
||||
"NOTE: The following section is to illustrate what should be done on the user (data consumer) side. This step is not required by the data provider."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages/torch/onnx/symbolic_opset9.py:2174: FutureWarning: 'torch.onnx.symbolic_opset9._cast_Bool' is deprecated in version 2.0 and will be removed in the future. Please Avoid using this function and create a Cast node instead.\n",
|
||||
" return fn(g, to_cast_func(g, input, False), to_cast_func(g, other, False))\n",
|
||||
"/Users/mhchia/Library/Caches/pypoetry/virtualenvs/zkstats-brXmXluj-py3.12/lib/python3.12/site-packages/torch/onnx/utils.py:1703: UserWarning: The exported ONNX model failed ONNX shape inference. The model will not be executable by the ONNX Runtime. If this is unintended and you believe there is a bug, please report an issue at https://github.com/pytorch/pytorch/issues. Error reported by strict ONNX shape inference: [ShapeInferenceError] (op_type:Where, node name: /Where_1): Y has inconsistent type tensor(float) (Triggered internally at /Users/runner/work/pytorch/pytorch/pytorch/torch/csrc/jit/serialization/export.cpp:1490.)\n",
|
||||
" _C._check_onnx_proto(proto)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"==== setting up ezkl ====\n",
|
||||
"Time setup: 0.5691819190979004 seconds\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[51.5, 46.25]"
|
||||
]
|
||||
},
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"import numpy as np\n",
|
||||
"from typing import Dict\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def create_dummy(shape_info: Dict[str, int], dummy_data_path: str) -> None:\n",
|
||||
" \"\"\"\n",
|
||||
" Create a dummy data file with randomized data based on the provided shape information.\n",
|
||||
"\n",
|
||||
" Parameters:\n",
|
||||
" - shape_info (dict): A dictionary where keys are column names and values are the number of elements (shape).\n",
|
||||
" - dummy_data_path (str): The path to save the dummy data file.\n",
|
||||
" \"\"\"\n",
|
||||
" dummy_data = {}\n",
|
||||
" for col, length in shape_info.items():\n",
|
||||
" # Generate random data for each column\n",
|
||||
" dummy_data[col] = np.round(np.random.uniform(0, 100, length), 1).tolist()\n",
|
||||
"\n",
|
||||
" with open(dummy_data_path, 'w') as f:\n",
|
||||
" json.dump(dummy_data, f)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"from zkstats.core import verifier_define_calculation, verifier_verify\n",
|
||||
"\n",
|
||||
"verifier_model_path = f\"{output_dir}/verifier_model.onnx\"\n",
|
||||
"verifier_compiled_model_path = f\"{output_dir}/verifier_model.compiled\"\n",
|
||||
"verifier_vk_path = f\"{output_dir}/verifier_model.vk\"\n",
|
||||
"verifier_pk_path = f\"{output_dir}/verifier_model.pk\"\n",
|
||||
"dummy_data_path = f\"{output_dir}/dummy_data.json\"\n",
|
||||
"sel_dummy_data_path = f\"{output_dir}/sel_dummy_data.json\"\n",
|
||||
"\n",
|
||||
"# NOTE: generate the verifier model with the `precal_witness_path` provided by the prover\n",
|
||||
"_, verifier_model = computation_to_model(computation, precal_witness_path, isProver=False)\n",
|
||||
"# Determine which srs to use with the logrows in the settings.json\n",
|
||||
"with open(settings_path, \"r\") as f:\n",
|
||||
" settings = json.load(f)\n",
|
||||
"logrows = int(settings[\"run_args\"][\"logrows\"])\n",
|
||||
"srs_path = f'~/.ezkl/srs/kzg{logrows}.srs'\n",
|
||||
"\n",
|
||||
"# create dummy data with the same shape as the original data\n",
|
||||
"create_dummy(data_shape, dummy_data_path)\n",
|
||||
"# generate the verifier model given the dummy data and the selected columns\n",
|
||||
"verifier_define_calculation(dummy_data_path, selected_columns, sel_dummy_data_path, verifier_model, verifier_model_path)\n",
|
||||
"# generate the verification key\n",
|
||||
"setup(verifier_model_path, verifier_compiled_model_path, settings_path, verifier_vk_path, verifier_pk_path)\n",
|
||||
"# verify the proof\n",
|
||||
"verifier_verify(proof_path, settings_path, verifier_vk_path, selected_columns, data_commitment_path)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Print the file paths. You should share the following files back to the user for them to verify the proof. You **SHOULD NOT** share more files otherwise data might be leaked."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Model onnx:\t\t /Users/mhchia/projects/work/pse/demo-next/public/assets/out/verifier_model.onnx\n",
|
||||
"Settings:\t\t /Users/mhchia/projects/work/pse/demo-next/public/assets/out/settings.json\n",
|
||||
"Proof:\t\t\t /Users/mhchia/projects/work/pse/demo-next/public/assets/out/model.pf\n",
|
||||
"Verification key:\t /Users/mhchia/projects/work/pse/demo-next/public/assets/out/verifier_model.vk\n",
|
||||
"Srs path:\t\t ~/.ezkl/srs/kzg11.srs\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(\"Model onnx:\\t\\t\", verifier_model_path)\n",
|
||||
"print(\"Settings:\\t\\t\", settings_path)\n",
|
||||
"print(\"Proof:\\t\\t\\t\", proof_path)\n",
|
||||
"print(\"Verification key:\\t\", verifier_vk_path)\n",
|
||||
"print(\"Srs path:\\t\\t\", srs_path)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": ".venv",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.12.1"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
Reference in New Issue
Block a user