github/zk-email-verify
1
1
Fork 0
mirror of https://github.com/zkemail/zk-email-verify.git synced 2026-01-09 13:38:03 -05:00
Code Issues Packages Projects Releases Wiki Activity

helpers: add eslint and lint files

Browse Source
This commit is contained in:
Saleel
2024-04-15 12:53:25 +05:30
parent 737871c6db
commit f2fb77c6ab
17 changed files with 16186 additions and 2167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12488
package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

32
packages/helpers/.eslintrc.json Normal file
View File

@@ -0,0 +1,32 @@
{
"env": {
"browser": true,
"es2021": true,
"node": true
},
"extends": [
"airbnb-base",
"airbnb-typescript/base"
],
"parser": "@typescript-eslint/parser",
"parserOptions": {
"ecmaVersion": 12,
"sourceType": "module",
"project": "./tsconfig.json"
},
"plugins": [
"@typescript-eslint"
],
"rules": {
"max-len": ["error", { "code": 150 }],
"import/prefer-default-export": "off",
"no-await-in-loop": "off",
"no-restricted-syntax": "off",
"no-plusplus": "off",
"no-bitwise": "off",
"no-console": "off",
"no-continue": "off",
"@typescript-eslint/no-use-before-define": "off",
"prefer-destructuring": "off"
}
}

7
packages/helpers/package.json
View File

@@ -5,6 +5,7 @@
"scripts": {
"build": "tsc",
"test": "jest tests/**/*.test.ts",
"lint": "eslint src/**/*.ts tests/**/*.ts|js",
"prepublish": "yarn build",
"publish": "yarn npm publish --access=public"
},
@@ -35,8 +36,14 @@
"@types/node": "^18.0.6",
"@types/node-forge": "^1.3.2",
"@types/psl": "^1.1.2",
"@typescript-eslint/eslint-plugin": "^7.0.0",
"@typescript-eslint/parser": "^7.0.0",
"babel-jest": "^29.5.0",
"babel-preset-jest": "^29.5.0",
"eslint": "^7.32.0",
"eslint-config-airbnb-base": "^15.0.0",
"eslint-config-airbnb-typescript": "^18.0.0",
"eslint-plugin-import": "^2.29.1",
"jest": "^29.5.0",
"msw": "^1.2.2",
"typescript": "^5.2.2"

46
packages/helpers/src/binary-format.ts
View File

@@ -1,4 +1,4 @@
import { CIRCOM_BIGINT_N, CIRCOM_BIGINT_K } from "./constants";
import { CIRCOM_BIGINT_N, CIRCOM_BIGINT_K } from './constants';
export function bytesToString(bytes: Uint8Array): string {
return new TextDecoder().decode(bytes);
@@ -6,7 +6,7 @@ export function bytesToString(bytes: Uint8Array): string {
// stringToUint8Array
export function stringToBytes(str: string) {
const encodedText = new TextEncoder().encode(str);
// const encodedText = new TextEncoder().encode(str);
const toReturn = Uint8Array.from(str, (x) => x.charCodeAt(0));
// const buf = Buffer.from(str, "utf8");
return toReturn;
@@ -38,7 +38,7 @@ export function bufferToUint8Array(buf: Buffer): Uint8Array {
}
export function bufferToHex(buf: Buffer): String {
return buf.toString("hex");
return buf.toString('hex');
}
export function Uint8ArrayToCharArray(a: Uint8Array): string[] {
@@ -48,15 +48,15 @@ export function Uint8ArrayToCharArray(a: Uint8Array): string[] {
export async function Uint8ArrayToString(a: Uint8Array): Promise<string> {
return Array.from(a)
.map((x) => x.toString())
.join(";");
.join(';');
}
export async function Uint8ArrayToHex(a: Uint8Array): Promise<string> {
return Buffer.from(a).toString("hex");
return Buffer.from(a).toString('hex');
}
export function bufferToString(buf: Buffer): String {
let intermediate = bufferToUint8Array(buf);
const intermediate = bufferToUint8Array(buf);
return bytesToString(intermediate);
}
@@ -70,7 +70,7 @@ export function bytesToBigInt(bytes: Uint8Array) {
export function bigIntToChunkedBytes(num: BigInt | bigint, bytesPerChunk: number, numChunks: number) {
const res = [];
const bigintNum: bigint = typeof num == "bigint" ? num : num.valueOf();
const bigintNum: bigint = typeof num === 'bigint' ? num : num.valueOf();
const msk = (1n << BigInt(bytesPerChunk)) - 1n;
for (let i = 0; i < numChunks; ++i) {
res.push(((bigintNum >> BigInt(i * bytesPerChunk)) & msk).toString());
@@ -83,7 +83,7 @@ export function toCircomBigIntBytes(num: BigInt | bigint) {
}
// https://stackoverflow.com/a/69585881
const HEX_STRINGS = "0123456789abcdef";
const HEX_STRINGS = '0123456789abcdef';
const MAP_HEX = {
0: 0,
1: 1,
@@ -113,7 +113,7 @@ const MAP_HEX = {
export function toHex(bytes: Uint8Array): string {
return Array.from(bytes || [])
.map((b) => HEX_STRINGS[b >> 4] + HEX_STRINGS[b & 15])
.join("");
.join('');
}
// Mimics Buffer.from(x, 'hex') logic
@@ -121,10 +121,10 @@ export function toHex(bytes: Uint8Array): string {
// https://github.com/nodejs/node/blob/v14.18.1/src/string_bytes.cc#L246-L261
export function fromHex(hexString: string): Uint8Array {
let hexStringTrimmed: string = hexString;
if (hexString[0] === "0" && hexString[1] === "x") {
if (hexString[0] === '0' && hexString[1] === 'x') {
hexStringTrimmed = hexString.slice(2);
}
const bytes = new Uint8Array(Math.floor((hexStringTrimmed || "").length / 2));
const bytes = new Uint8Array(Math.floor((hexStringTrimmed || '').length / 2));
let i;
for (i = 0; i < bytes.length; i++) {
const a = MAP_HEX[hexStringTrimmed[i * 2] as keyof typeof MAP_HEX];
@@ -139,22 +139,22 @@ export function fromHex(hexString: string): Uint8Array {
// Works only on 32 bit sha text lengths
export function int64toBytes(num: number): Uint8Array {
let arr = new ArrayBuffer(8); // an Int32 takes 4 bytes
let view = new DataView(arr);
const arr = new ArrayBuffer(8); // an Int32 takes 4 bytes
const view = new DataView(arr);
view.setInt32(4, num, false); // byteOffset = 0; litteEndian = false
return new Uint8Array(arr);
}
// Works only on 32 bit sha text lengths
export function int8toBytes(num: number): Uint8Array {
let arr = new ArrayBuffer(1); // an Int8 takes 4 bytes
let view = new DataView(arr);
const arr = new ArrayBuffer(1); // an Int8 takes 4 bytes
const view = new DataView(arr);
view.setUint8(0, num); // byteOffset = 0; litteEndian = false
return new Uint8Array(arr);
}
export function bitsToUint8(bits: string[]): Uint8Array {
let bytes = new Uint8Array(bits.length);
const bytes = new Uint8Array(bits.length);
for (let i = 0; i < bits.length; i += 1) {
bytes[i] = parseInt(bits[i], 2);
}
@@ -162,12 +162,12 @@ export function bitsToUint8(bits: string[]): Uint8Array {
}
export function uint8ToBits(uint8: Uint8Array): string {
return uint8.reduce((acc, byte) => acc + byte.toString(2).padStart(8, "0"), "");
return uint8.reduce((acc, byte) => acc + byte.toString(2).padStart(8, '0'), '');
}
export function mergeUInt8Arrays(a1: Uint8Array, a2: Uint8Array): Uint8Array {
// sum of individual array lengths
var mergedArray = new Uint8Array(a1.length + a2.length);
const mergedArray = new Uint8Array(a1.length + a2.length);
mergedArray.set(a1);
mergedArray.set(a2, a1.length);
return mergedArray;
@@ -180,9 +180,9 @@ export function assert(cond: boolean, errorMessage: string) {
}
export function packedNBytesToString(packedBytes: bigint[], n: number = 7): string {
let chars: number[] = [];
const chars: number[] = [];
for (let i = 0; i < packedBytes.length; i++) {
for (var k = 0n; k < n; k++) {
for (let k = 0n; k < n; k++) {
chars.push(Number((packedBytes[i] >> (k * 8n)) % 256n));
}
}
@@ -190,14 +190,14 @@ export function packedNBytesToString(packedBytes: bigint[], n: number = 7): stri
}
export function packBytesIntoNBytes(messagePaddedRaw: Uint8Array | string, n = 7): Array<bigint> {
const messagePadded: Uint8Array = typeof messagePaddedRaw === "string" ? stringToBytes(messagePaddedRaw) : messagePaddedRaw;
let output: Array<bigint> = [];
const messagePadded: Uint8Array = typeof messagePaddedRaw === 'string' ? stringToBytes(messagePaddedRaw) : messagePaddedRaw;
const output: Array<bigint> = [];
for (let i = 0; i < messagePadded.length; i++) {
if (i % n === 0) {
output.push(0n);
}
const j = (i / n) | 0;
console.assert(j === output.length - 1, "Not editing the index of the last element -- packing loop invariants bug!");
console.assert(j === output.length - 1, 'Not editing the index of the last element -- packing loop invariants bug!');
output[j] += BigInt(messagePadded[i]) << BigInt((i % n) * 8);
}
return output;

100
packages/helpers/src/chunked-zkey.ts
View File

@@ -1,19 +1,17 @@
import localforage from "localforage";
import localforage from 'localforage';
// @ts-ignore
import pako from "pako";
import pako from 'pako';
// @ts-ignore
import * as snarkjs from "snarkjs";
import * as snarkjs from 'snarkjs';
const compressed = true;
const zkeyExtension = ".gz";
const zkeyExtensionRegEx = new RegExp(`\\b${zkeyExtension}$\\b`, "i"); // = /.gz$/i
const zkeySuffix = ["b", "c", "d", "e", "f", "g", "h", "i", "j", "k"];
const zkeyExtension = '.gz';
const zkeyExtensionRegEx = new RegExp(`\\b${zkeyExtension}$\\b`, 'i'); // = /.gz$/i
const zkeySuffix = ['b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k'];
// uncompresses single .gz file.
// returns the contents as an ArrayBuffer
export const uncompressGz = async (
arrayBuffer: ArrayBuffer
arrayBuffer: ArrayBuffer,
): Promise<ArrayBuffer> => {
const output = pako.ungzip(arrayBuffer);
const buff = output.buffer;
@@ -22,19 +20,19 @@ export const uncompressGz = async (
// We can use this function to ensure the type stored in localforage is correct.
async function storeArrayBuffer(keyname: string, buffer: ArrayBuffer) {
return await localforage.setItem(keyname, buffer);
return localforage.setItem(keyname, buffer);
}
async function downloadWithRetries(link: string, downloadAttempts: number) {
for (let i = 1; i <= downloadAttempts; i++) {
console.log(`download attempt ${i} for ${link}`);
let response = await fetch(link, { method: "GET" });
if (response.status == 200) {
const response = await fetch(link, { method: 'GET' });
if (response.status === 200) {
return response;
}
}
throw new Error(
`Error downloading ${link} after ${downloadAttempts} retries`
`Error downloading ${link} after ${downloadAttempts} retries`,
);
}
@@ -44,7 +42,7 @@ async function downloadWithRetries(link: string, downloadAttempts: number) {
export async function downloadFromFilename(
baseUrl: string,
filename: string,
compressed = false
compressed = false,
) {
const link = baseUrl + filename;
@@ -56,53 +54,51 @@ export async function downloadFromFilename(
} else {
// uncompress the data
const zkeyUncompressed = await uncompressGz(zkeyBuff);
const rawFilename = filename.replace(zkeyExtensionRegEx, ""); // replace .gz with ""
const rawFilename = filename.replace(zkeyExtensionRegEx, ''); // replace .gz with ""
// store the uncompressed data
console.log("storing file in localforage", rawFilename);
console.log('storing file in localforage', rawFilename);
await storeArrayBuffer(rawFilename, zkeyUncompressed);
console.log("stored file in localforage", rawFilename);
console.log('stored file in localforage', rawFilename);
// await localforage.setItem(filename, zkeyBuff);
}
console.log(`Storage of ${filename} successful!`);
}
export const downloadProofFiles = async function (
export async function downloadProofFiles(
baseUrl: string,
circuitName: string,
onFileDownloaded: () => void
onFileDownloaded: () => void,
) {
const filePromises = [];
for (const c of zkeySuffix) {
const targzFilename = `${circuitName}.zkey${c}${zkeyExtension}`;
const itemCompressed = await localforage.getItem(targzFilename);
// const itemCompressed = await localforage.getItem(targzFilename);
const item = await localforage.getItem(`${circuitName}.zkey${c}`);
if (item) {
console.log(
`${circuitName}.zkey${c}${
item ? "" : zkeyExtension
} already found in localforage!`
item ? '' : zkeyExtension
} already found in localforage!`,
);
onFileDownloaded();
continue;
}
filePromises.push(
downloadFromFilename(baseUrl, targzFilename, compressed).then(() =>
onFileDownloaded()
)
downloadFromFilename(baseUrl, targzFilename, true).then(() => onFileDownloaded()),
);
}
console.log(filePromises);
await Promise.all(filePromises);
};
}
export async function generateProof(input: any, baseUrl: string, circuitName: string) {
// TODO: figure out how to generate this s.t. it passes build
console.log("generating proof for input");
console.log('generating proof for input');
console.log(input);
const { proof, publicSignals } = await snarkjs.groth16.fullProve(
input,
`${baseUrl}${circuitName}.wasm`,
`${circuitName}.zkey`
`${circuitName}.zkey`,
);
console.log(`Generated proof ${JSON.stringify(proof)}`);
@@ -113,45 +109,45 @@ export async function generateProof(input: any, baseUrl: string, circuitName: st
}
export async function verifyProof(proof: any, publicSignals: any, baseUrl: string, circuitName: string) {
console.log("PROOF", proof);
console.log("PUBLIC SIGNALS", publicSignals);
console.log('PROOF', proof);
console.log('PUBLIC SIGNALS', publicSignals);
const response = await downloadWithRetries(`${baseUrl}${circuitName}.vkey.json`, 3);
const vkey = await response.json();
console.log("vkey", vkey);
console.log('vkey', vkey);
const proofVerified = await snarkjs.groth16.verify(
vkey,
publicSignals,
proof
proof,
);
console.log("proofV", proofVerified);
console.log('proofV', proofVerified);
return proofVerified;
}
function bigIntToArray(n: number, k: number, x: bigint) {
let divisor = 1n;
for (var idx = 0; idx < n; idx++) {
divisor = divisor * 2n;
for (let idx = 0; idx < n; idx++) {
divisor *= 2n;
}
let ret = [];
var x_temp = BigInt(x);
for (var idx = 0; idx < k; idx++) {
ret.push(x_temp % divisor);
x_temp = x_temp / divisor;
const ret = [];
let temp = BigInt(x);
for (let idx = 0; idx < k; idx++) {
ret.push(temp % divisor);
temp /= divisor;
}
return ret;
}
// taken from generation code in dizkus-circuits tests
function pubkeyToXYArrays(pk: string) {
const XArr = bigIntToArray(64, 4, BigInt("0x" + pk.slice(4, 4 + 64))).map(
(el) => el.toString()
const XArr = bigIntToArray(64, 4, BigInt(`0x${pk.slice(4, 4 + 64)}`)).map(
(el) => el.toString(),
);
const YArr = bigIntToArray(64, 4, BigInt("0x" + pk.slice(68, 68 + 64))).map(
(el) => el.toString()
const YArr = bigIntToArray(64, 4, BigInt(`0x${pk.slice(68, 68 + 64)}`)).map(
(el) => el.toString(),
);
return [XArr, YArr];
@@ -159,11 +155,11 @@ function pubkeyToXYArrays(pk: string) {
// taken from generation code in dizkus-circuits tests
function sigToRSArrays(sig: string) {
const rArr = bigIntToArray(64, 4, BigInt("0x" + sig.slice(2, 2 + 64))).map(
(el) => el.toString()
const rArr = bigIntToArray(64, 4, BigInt(`0x${sig.slice(2, 2 + 64)}`)).map(
(el) => el.toString(),
);
const sArr = bigIntToArray(64, 4, BigInt("0x" + sig.slice(66, 66 + 64))).map(
(el) => el.toString()
const sArr = bigIntToArray(64, 4, BigInt(`0x${sig.slice(66, 66 + 64)}`)).map(
(el) => el.toString(),
);
return [rArr, sArr];
@@ -173,8 +169,8 @@ export function buildInput(pubkey: string, msghash: string, sig: string) {
const [r, s] = sigToRSArrays(sig);
return {
r: r,
s: s,
r,
s,
msghash: bigIntToArray(64, 4, BigInt(msghash)),
pubkey: pubkeyToXYArrays(pubkey),
};

5
packages/helpers/src/constants.ts
View File

@@ -1,7 +1,6 @@
export const CIRCOM_FIELD_MODULUS = 21888242871839275222246405745257275088548364400416034343698204186575808495617n;
export const MAX_HEADER_PADDED_BYTES = 1024; // Default value for max size to be used in circuit
export const MAX_BODY_PADDED_BYTES = 1536; // Default value for max size to be used in circuit
export const MAX_HEADER_PADDED_BYTES = 1024; // Default value for max size to be used in circuit
export const MAX_BODY_PADDED_BYTES = 1536; // Default value for max size to be used in circuit
export const CIRCOM_BIGINT_N = 121;
export const CIRCOM_BIGINT_K = 17;

60
packages/helpers/src/dkim/index.ts
View File

@@ -1,10 +1,10 @@
import { pki } from "node-forge";
import { DkimVerifier } from "../../lib/mailauth/dkim-verifier";
import { writeToStream } from "../../lib/mailauth/tools";
import sanitizers from "./sanitizers";
import { pki } from 'node-forge';
import { DkimVerifier } from '../../lib/mailauth/dkim-verifier';
import { writeToStream } from '../../lib/mailauth/tools';
import sanitizers from './sanitizers';
// `./mailauth` is modified version of https://github.com/postalsys/mailauth
// Main modification are including emailHeaders in the DKIM result, making it work in the browser, add types
// Main modification are including emailHeaders in the DKIM result, making it work in the browser, add types
// TODO: Fork the repo and make the changes; consider upstream to original repo
export interface DKIMVerificationResult {
@@ -22,45 +22,43 @@ export interface DKIMVerificationResult {
}
/**
*
*
* @param email Entire email data as a string or buffer
* @param domain Domain to verify DKIM signature for. If not provided, the domain is extracted from the `From` header
* @param enableSanitization If true, email will be applied with various sanitization to try and pass DKIM verification
* @returns
* @returns
*/
export async function verifyDKIMSignature(
email: Buffer | string,
domain: string = "",
enableSanitization: boolean = true
domain: string = '',
enableSanitization: boolean = true,
): Promise<DKIMVerificationResult> {
const emailStr = email.toString();
const pgpMarkers = ["BEGIN PGP MESSAGE", "BEGIN PGP SIGNED MESSAGE"];
const pgpMarkers = ['BEGIN PGP MESSAGE', 'BEGIN PGP SIGNED MESSAGE'];
const isPGPEncoded = pgpMarkers.some((marker) => emailStr.includes(marker));
if (isPGPEncoded) {
throw new Error("PGP encoded emails are not supported.");
throw new Error('PGP encoded emails are not supported.');
}
let dkimResult = await tryVerifyDKIM(email, domain);
// If DKIM verification fails, try again after sanitizing email
let appliedSanitization;
if (dkimResult.status.comment === "bad signature" && enableSanitization) {
if (dkimResult.status.comment === 'bad signature' && enableSanitization) {
const results = await Promise.all(
sanitizers.map((sanitize) =>
tryVerifyDKIM(sanitize(emailStr), domain).then((result) => ({
result,
sanitizer: sanitize.name,
}))
)
sanitizers.map((sanitize) => tryVerifyDKIM(sanitize(emailStr), domain).then((result) => ({
result,
sanitizer: sanitize.name,
}))),
);
const passed = results.find((r) => r.result.status.result === "pass");
const passed = results.find((r) => r.result.status.result === 'pass');
if (passed) {
console.log(
`DKIM: Verification passed after applying sanitization "${passed.sanitizer}"`
`DKIM: Verification passed after applying sanitization "${passed.sanitizer}"`,
);
dkimResult = passed.result;
appliedSanitization = passed.sanitizer;
@@ -77,19 +75,19 @@ export async function verifyDKIMSignature(
bodyHash,
} = dkimResult;
if (result !== "pass") {
if (result !== 'pass') {
throw new Error(
`DKIM signature verification failed for domain ${signingDomain}. Reason: ${comment}`
`DKIM signature verification failed for domain ${signingDomain}. Reason: ${comment}`,
);
}
const pubKeyData = pki.publicKeyFromPem(publicKey.toString());
return {
signature: BigInt("0x" + Buffer.from(signature, "base64").toString("hex")),
signature: BigInt(`0x${Buffer.from(signature, 'base64').toString('hex')}`),
headers: status.signedHeaders,
body: body,
bodyHash: bodyHash,
body,
bodyHash,
signingDomain: dkimResult.signingDomain,
publicKey: BigInt(pubKeyData.n.toString()),
selector: dkimResult.selector,
@@ -100,28 +98,28 @@ export async function verifyDKIMSignature(
};
}
async function tryVerifyDKIM(email: Buffer | string, domain: string = "") {
let dkimVerifier = new DkimVerifier({});
async function tryVerifyDKIM(email: Buffer | string, domain: string = '') {
const dkimVerifier = new DkimVerifier({});
await writeToStream(dkimVerifier, email as any);
let domainToVerifyDKIM = domain;
if (!domainToVerifyDKIM) {
if (dkimVerifier.headerFrom.length > 1) {
throw new Error(
"Multiple From header in email and domain for verification not specified"
'Multiple From header in email and domain for verification not specified',
);
}
domainToVerifyDKIM = dkimVerifier.headerFrom[0].split("@")[1];
domainToVerifyDKIM = dkimVerifier.headerFrom[0].split('@')[1];
}
const dkimResult = dkimVerifier.results.find(
(d: any) => d.signingDomain === domainToVerifyDKIM
(d: any) => d.signingDomain === domainToVerifyDKIM,
);
if (!dkimResult) {
throw new Error(
`DKIM signature not found for domain ${domainToVerifyDKIM}`
`DKIM signature not found for domain ${domainToVerifyDKIM}`,
);
}

13
packages/helpers/src/dkim/sanitizers.ts
View File

@@ -1,6 +1,6 @@
function getHeaderValue(email: string, header: string) {
const headerStartIndex = email.indexOf(`${header}: `) + header.length + 2;
const headerEndIndex = email.indexOf("\n", headerStartIndex);
const headerEndIndex = email.indexOf('\n', headerStartIndex);
const headerValue = email.substring(headerStartIndex, headerEndIndex);
return headerValue;
@@ -15,17 +15,17 @@ function setHeaderValue(email: string, header: string, value: string) {
// TODO: Add test for this
function revertGoogleMessageId(email: string): string {
// (Optional check) This only happens when google does ARC
if (!email.includes("ARC-Authentication-Results")) {
if (!email.includes('ARC-Authentication-Results')) {
return email;
}
const googleReplacedMessageId = getHeaderValue(
email,
"X-Google-Original-Message-ID"
'X-Google-Original-Message-ID',
);
if (googleReplacedMessageId) {
return setHeaderValue(email, "Message-ID", googleReplacedMessageId);
return setHeaderValue(email, 'Message-ID', googleReplacedMessageId);
}
return email;
@@ -34,7 +34,7 @@ function revertGoogleMessageId(email: string): string {
// Remove labels inserted to Subject - `[ListName] Newsletter 2024` to `Newsletter 2024`
function removeLabels(email: string): string {
// Replace Subject: [label] with Subject:
const sanitized = email.replace(/Subject: \[.*\]/, "Subject:");
const sanitized = email.replace(/Subject: \[.*\]/, 'Subject:');
return sanitized;
}
@@ -62,10 +62,9 @@ function insert13Before10(email: string): string {
// Replace `=09` with `\t` in email
// TODO: Add test for this
function sanitizeTabs(email: string): string {
return email.replace(`=09`, `\t`);
return email.replace('=09', '\t');
}
const sanitizers = [
revertGoogleMessageId,
removeLabels,

10
packages/helpers/src/index.ts
View File

@@ -1,5 +1,5 @@
export * from "./binary-format";
export * from "./constants";
export * from "../lib/fast-sha256";
export * from "./input-generators";
export * from "./sha-utils";
export * from './binary-format';
export * from './constants';
export * from '../lib/fast-sha256';
export * from './input-generators';
export * from './sha-utils';

43
packages/helpers/src/input-generators.ts
View File

@@ -1,7 +1,7 @@
import { Uint8ArrayToCharArray, toCircomBigIntBytes } from "./binary-format";
import { MAX_BODY_PADDED_BYTES, MAX_HEADER_PADDED_BYTES } from "./constants";
import { DKIMVerificationResult, verifyDKIMSignature } from "./dkim";
import { generatePartialSHA, sha256Pad } from "./sha-utils";
import { Uint8ArrayToCharArray, toCircomBigIntBytes } from './binary-format';
import { MAX_BODY_PADDED_BYTES, MAX_HEADER_PADDED_BYTES } from './constants';
import { DKIMVerificationResult, verifyDKIMSignature } from './dkim';
import { generatePartialSHA, sha256Pad } from './sha-utils';
type CircuitInput = {
emailHeader: string[];
@@ -21,9 +21,8 @@ type InputGenerationArgs = {
maxBodyLength?: number; // Max length of the email body after shaPrecomputeSelector including padding
};
/**
*
*
* @description Generate circuit inputs for the EmailVerifier circuit from raw email content
* @param rawEmail Full email content as a buffer or string
* @param params Arguments to control the input generation
@@ -31,19 +30,18 @@ type InputGenerationArgs = {
*/
export async function generateEmailVerifierInputs(
rawEmail: Buffer | string,
params: InputGenerationArgs = {}
params: InputGenerationArgs = {},
) {
const dkimResult = await verifyDKIMSignature(rawEmail);
return generateEmailVerifierInputsFromDKIMResult(
dkimResult,
params
params,
);
}
/**
*
*
* @description Generate circuit inputs for the EmailVerifier circuit from DKIMVerification result
* @param dkimResult DKIMVerificationResult containing email data and verification result
* @param params Arguments to control the input generation
@@ -51,14 +49,16 @@ export async function generateEmailVerifierInputs(
*/
export function generateEmailVerifierInputsFromDKIMResult(
dkimResult: DKIMVerificationResult,
params: InputGenerationArgs = {}
params: InputGenerationArgs = {},
): CircuitInput {
const { headers, body, bodyHash, publicKey, signature } = dkimResult;
const {
headers, body, bodyHash, publicKey, signature,
} = dkimResult;
// SHA add padding
const [messagePadded, messagePaddedLen] = sha256Pad(
headers,
params.maxHeadersLength || MAX_HEADER_PADDED_BYTES
params.maxHeadersLength || MAX_HEADER_PADDED_BYTES,
);
const circuitInputs: CircuitInput = {
@@ -71,7 +71,7 @@ export function generateEmailVerifierInputsFromDKIMResult(
if (!params.ignoreBodyHashCheck) {
if (!body || !bodyHash) {
throw new Error(
`body and bodyHash are required when ignoreBodyHashCheck is false`
'body and bodyHash are required when ignoreBodyHashCheck is false',
);
}
@@ -83,16 +83,15 @@ export function generateEmailVerifierInputsFromDKIMResult(
const bodySHALength = Math.floor((body.length + 63 + 65) / 64) * 64;
const [bodyPadded, bodyPaddedLen] = sha256Pad(
body,
Math.max(maxBodyLength, bodySHALength)
Math.max(maxBodyLength, bodySHALength),
);
const { precomputedSha, bodyRemaining, bodyRemainingLength } =
generatePartialSHA({
body: bodyPadded,
bodyLength: bodyPaddedLen,
selectorString: params.shaPrecomputeSelector,
maxRemainingBodyLength: maxBodyLength,
});
const { precomputedSha, bodyRemaining, bodyRemainingLength } = generatePartialSHA({
body: bodyPadded,
bodyLength: bodyPaddedLen,
selectorString: params.shaPrecomputeSelector,
maxRemainingBodyLength: maxBodyLength,
});
circuitInputs.emailBodyLength = bodyRemainingLength.toString();
circuitInputs.precomputedSHA = Uint8ArrayToCharArray(precomputedSha);

53
packages/helpers/src/sha-utils.ts
View File

@@ -1,10 +1,15 @@
import * as CryptoJS from 'crypto';
import { assert, int64toBytes, int8toBytes, mergeUInt8Arrays } from "./binary-format";
import { Hash } from "../lib/fast-sha256";
import {
assert,
int64toBytes,
int8toBytes,
mergeUInt8Arrays,
} from './binary-format';
import { Hash } from '../lib/fast-sha256';
export function findIndexInUint8Array(
array: Uint8Array,
selector: Uint8Array
selector: Uint8Array,
): number {
let i = 0;
let j = 0;
@@ -24,6 +29,7 @@ export function findIndexInUint8Array(
export function padUint8ArrayWithZeros(array: Uint8Array, length: number) {
while (array.length < length) {
// eslint-disable-next-line no-param-reassign
array = mergeUInt8Arrays(array, int8toBytes(0));
}
return array;
@@ -48,7 +54,7 @@ export function generatePartialSHA({
selectorIndex = findIndexInUint8Array(body, selector);
if (selectorIndex === -1) {
throw new Error(`Provider SHA precompute selector not found in the body`);
throw new Error('Provider SHA precompute selector not found in the body');
}
}
@@ -60,12 +66,12 @@ export function generatePartialSHA({
if (bodyRemainingLength > maxRemainingBodyLength) {
throw new Error(
`Remaining body ${bodyRemainingLength} after the selector is longer than max (${maxRemainingBodyLength})`
`Remaining body ${bodyRemainingLength} after the selector is longer than max (${maxRemainingBodyLength})`,
);
}
if (bodyRemaining.length % 64 !== 0) {
throw new Error(`Remaining body was not padded correctly with int64s`);
throw new Error('Remaining body was not padded correctly with int64s');
}
bodyRemaining = padUint8ArrayWithZeros(bodyRemaining, maxRemainingBodyLength);
@@ -88,23 +94,30 @@ export function partialSha(msg: Uint8Array, msgLen: number): Uint8Array {
}
// Puts an end selector, a bunch of 0s, then the length, then fill the rest with 0s.
export function sha256Pad(prehash_prepad_m: Uint8Array, maxShaBytes: number): [Uint8Array, number] {
let length_bits = prehash_prepad_m.length * 8; // bytes to bits
let length_in_bytes = int64toBytes(length_bits);
prehash_prepad_m = mergeUInt8Arrays(prehash_prepad_m, int8toBytes(2 ** 7)); // Add the 1 on the end, length 505
export function sha256Pad(
message: Uint8Array,
maxShaBytes: number,
): [Uint8Array, number] {
const msgLen = message.length * 8; // bytes to bits
const msgLenBytes = int64toBytes(msgLen);
let res = mergeUInt8Arrays(message, int8toBytes(2 ** 7)); // Add the 1 on the end, length 505
// while ((prehash_prepad_m.length * 8 + length_in_bytes.length * 8) % 512 !== 0) {
while ((prehash_prepad_m.length * 8 + length_in_bytes.length * 8) % 512 !== 0) {
prehash_prepad_m = mergeUInt8Arrays(prehash_prepad_m, int8toBytes(0));
while ((res.length * 8 + msgLenBytes.length * 8) % 512 !== 0) {
res = mergeUInt8Arrays(res, int8toBytes(0));
}
prehash_prepad_m = mergeUInt8Arrays(prehash_prepad_m, length_in_bytes);
assert((prehash_prepad_m.length * 8) % 512 === 0, "Padding did not complete properly!");
let messageLen = prehash_prepad_m.length;
while (prehash_prepad_m.length < maxShaBytes) {
prehash_prepad_m = mergeUInt8Arrays(prehash_prepad_m, int64toBytes(0));
res = mergeUInt8Arrays(res, msgLenBytes);
assert((res.length * 8) % 512 === 0, 'Padding did not complete properly!');
const messageLen = res.length;
while (res.length < maxShaBytes) {
res = mergeUInt8Arrays(res, int64toBytes(0));
}
assert(
prehash_prepad_m.length === maxShaBytes,
`Padding to max length did not complete properly! Your padded message is ${prehash_prepad_m.length} long but max is ${maxShaBytes}!`
res.length === maxShaBytes,
`Padding to max length did not complete properly! Your padded message is ${res.length} long but max is ${maxShaBytes}!`,
);
return [prehash_prepad_m, messageLen];
return [res, messageLen];
}

40
packages/helpers/tests/__mocks__/localforage.ts
View File

@@ -2,32 +2,30 @@ import fs from 'fs';
import path from 'path';
const getUncompressedTestFile = (): ArrayBuffer => {
const buffer = fs.readFileSync(path.join(__dirname, `../test-data/compressed-files/uncompressed-value.txt`));
const buffer = fs.readFileSync(path.join(__dirname, '../test-data/compressed-files/uncompressed-value.txt'));
return buffer;
}
const tempStorage: Record<string, ArrayBuffer> = {
"email.zkeyb": getUncompressedTestFile(),
"email.zkeyc": getUncompressedTestFile(),
"email.zkeyd": getUncompressedTestFile(),
"email.zkeye": getUncompressedTestFile(),
"email.zkeyf": getUncompressedTestFile(),
"email.zkeyg": getUncompressedTestFile(),
"email.zkeyh": getUncompressedTestFile(),
"email.zkeyi": getUncompressedTestFile(),
"email.zkeyj": getUncompressedTestFile(),
"email.zkeyk": getUncompressedTestFile(),
};
const getItem = jest.fn((key)=>{
return tempStorage[key];
});
const tempStorage: Record<string, ArrayBuffer> = {
'email.zkeyb': getUncompressedTestFile(),
'email.zkeyc': getUncompressedTestFile(),
'email.zkeyd': getUncompressedTestFile(),
'email.zkeye': getUncompressedTestFile(),
'email.zkeyf': getUncompressedTestFile(),
'email.zkeyg': getUncompressedTestFile(),
'email.zkeyh': getUncompressedTestFile(),
'email.zkeyi': getUncompressedTestFile(),
'email.zkeyj': getUncompressedTestFile(),
'email.zkeyk': getUncompressedTestFile(),
};
const getItem = jest.fn((key) => tempStorage[key]);
const setItem = jest.fn();
const locaforage = {
getItem,
setItem
};
getItem,
setItem,
};
export default locaforage;
export default locaforage;

51
packages/helpers/tests/chunked-zkey.test.ts
View File

@@ -1,43 +1,43 @@
import fs from 'fs';
import path from 'path';
import { StringDecoder } from "string_decoder";
import _localforage from "localforage";
import { downloadFromFilename, downloadProofFiles, uncompressGz as uncompress } from "../src/chunked-zkey";
import { server } from './mocks/server.js'
import { MOCK_BASE_URL } from "./mocks/handlers.js";
import { StringDecoder } from 'string_decoder';
import _localforage from 'localforage';
import { downloadFromFilename, downloadProofFiles, uncompressGz as uncompress } from '../src/chunked-zkey';
import { server } from './mocks/server';
import { MOCK_BASE_URL } from './mocks/handlers';
// this is mocked in __mocks__/localforage.ts
jest.mock("localforage");
jest.mock('localforage');
const localforage = _localforage as jest.Mocked<typeof _localforage>;
// Establish API mocking before all tests.
beforeAll(() => server.listen())
beforeAll(() => server.listen());
// Reset any request handlers that we may add during the tests,
// so they don't affect other tests.
afterEach(() => server.resetHandlers())
afterEach(() => server.resetHandlers());
// Clean up after the tests are finished.
afterAll(() => server.close())
afterAll(() => server.close());
// localforage should be storing ArrayBuffers.
// We can use this function to simplify checking the mocked value of the ArrayBuffer.
const decodeArrayBufferToString = (buffer: ArrayBuffer): string => {
const decodeArrayBufferToString = (buffer: ArrayBuffer): string => {
const decoder = new StringDecoder('utf8');
const str = decoder.write(Buffer.from(buffer));
return str;
}
};
const getCompressedTestFile = (): ArrayBuffer => {
const buffer = fs.readFileSync(path.join(__dirname, `test-data/compressed-files/compressed.txt.gz`));
const buffer = fs.readFileSync(path.join(__dirname, 'test-data/compressed-files/compressed.txt.gz'));
return buffer;
}
};
const getUncompressedTestFile = (): ArrayBuffer => {
const buffer = fs.readFileSync(path.join(__dirname, `test-data/compressed-files/uncompressed-value.txt`));
const buffer = fs.readFileSync(path.join(__dirname, 'test-data/compressed-files/uncompressed-value.txt'));
return buffer;
}
};
describe('Uncompress GZ file', () => {
test('Uncompresss a GZ file', async () => {
@@ -52,15 +52,14 @@ describe('Uncompress GZ file', () => {
});
describe('Test zkp fetch and store', () => {
afterEach(()=>{
afterEach(() => {
jest.resetAllMocks();
});
test('should fetch a gz file, uncompress it, and store it in indexeddb', async () => {
const filename = "email.zkeyb.gz";
const filename = 'email.zkeyb.gz';
// downloadFileFromFilename requests the file from the server, which we mocked with msw.
// The server returns a gz file of a file containing "not compressed 👍",
// The server returns a gz file of a file containing "not compressed 👍",
// which is defined in __fixtures__/compressed-files/compressed.txt.gz
await downloadFromFilename(MOCK_BASE_URL, filename, true);
// check that localforage.setItem was called once to save the zkey file.
@@ -70,28 +69,28 @@ describe('Test zkp fetch and store', () => {
// expect to be called with...
const str = decodeArrayBufferToString(decompressedBuffer);
expect(filenameRaw).toBe("email.zkeyb");
expect(filenameRaw).toBe('email.zkeyb');
// check that it decompressed the file correctly.
expect(str).toBe("not compressed 👍");
expect(str).toBe('not compressed 👍');
});
test('should should download all the zkeys and save them in local storage for snarkjs to access.', async () => {
// downloadProofFiles calls downloadFromFilename 10 times, one for each zkey, b-k.
const onDownloaded = jest.fn();
await downloadProofFiles(MOCK_BASE_URL, "email", onDownloaded);
await downloadProofFiles(MOCK_BASE_URL, 'email', onDownloaded);
expect(localforage.setItem).toBeCalledTimes(10);
// check the first one
const filenameRawB = localforage.setItem.mock.calls[0][0];
const decompressedBufferB = localforage.setItem.mock.calls[0][1] as ArrayBuffer;
expect(filenameRawB).toBe("email.zkeyb");
expect(decodeArrayBufferToString(decompressedBufferB)).toBe("not compressed 👍");
expect(filenameRawB).toBe('email.zkeyb');
expect(decodeArrayBufferToString(decompressedBufferB)).toBe('not compressed 👍');
// ... c d e f g h i j ... assume these are fine too.
// check the last one
const filenameRawK = localforage.setItem.mock.calls[9][0];
const decompressedBufferK = localforage.setItem.mock.calls[9][1] as ArrayBuffer;
expect(filenameRawK).toBe("email.zkeyk");
expect(decodeArrayBufferToString(decompressedBufferK)).toBe("not compressed 👍");
expect(filenameRawK).toBe('email.zkeyk');
expect(decodeArrayBufferToString(decompressedBufferK)).toBe('not compressed 👍');
expect(onDownloaded).toBeCalledTimes(10);
});
});

52
packages/helpers/tests/dkim.test.ts
View File

@@ -1,24 +1,24 @@
import { verifyDKIMSignature } from "../src/dkim";
import fs from "fs";
import path from "path";
import fs from 'fs';
import path from 'path';
import { verifyDKIMSignature } from '../src/dkim';
jest.setTimeout(10000);
describe("DKIM signature verification", () => {
it("should pass for valid email", async () => {
describe('DKIM signature verification', () => {
it('should pass for valid email', async () => {
const email = fs.readFileSync(
path.join(__dirname, `test-data/email-good.eml`)
path.join(__dirname, 'test-data/email-good.eml'),
);
const result = await verifyDKIMSignature(email);
expect(result.signingDomain).toBe("icloud.com");
expect(result.signingDomain).toBe('icloud.com');
expect(result.appliedSanitization).toBeFalsy();
});
it("should fail for invalid selector", async () => {
it('should fail for invalid selector', async () => {
const email = fs.readFileSync(
path.join(__dirname, `test-data/email-invalid-selector.eml`)
path.join(__dirname, 'test-data/email-invalid-selector.eml'),
);
expect.assertions(1);
@@ -27,14 +27,14 @@ describe("DKIM signature verification", () => {
await verifyDKIMSignature(email);
} catch (e) {
expect(e.message).toBe(
"DKIM signature verification failed for domain icloud.com. Reason: no key"
'DKIM signature verification failed for domain icloud.com. Reason: no key',
);
}
});
it("should fail for tampered body", async () => {
it('should fail for tampered body', async () => {
const email = fs.readFileSync(
path.join(__dirname, `test-data/email-body-tampered.eml`)
path.join(__dirname, 'test-data/email-body-tampered.eml'),
);
expect.assertions(1);
@@ -43,15 +43,15 @@ describe("DKIM signature verification", () => {
await verifyDKIMSignature(email);
} catch (e) {
expect(e.message).toBe(
"DKIM signature verification failed for domain icloud.com. Reason: body hash did not verify"
'DKIM signature verification failed for domain icloud.com. Reason: body hash did not verify',
);
}
});
it("should fail for when DKIM signature is not present for domain", async () => {
it('should fail for when DKIM signature is not present for domain', async () => {
// In this email From address is user@gmail.com, but the DKIM signature is only for icloud.com
const email = fs.readFileSync(
path.join(__dirname, `test-data/email-invalid-domain.eml`)
path.join(__dirname, 'test-data/email-invalid-domain.eml'),
);
expect.assertions(1);
@@ -60,45 +60,45 @@ describe("DKIM signature verification", () => {
await verifyDKIMSignature(email);
} catch (e) {
expect(e.message).toBe(
"DKIM signature not found for domain gmail.com"
'DKIM signature not found for domain gmail.com',
);
}
});
it("should be able to override domain", async () => {
it('should be able to override domain', async () => {
// From address domain is icloud.com
const email = fs.readFileSync(
path.join(__dirname, `test-data/email-different-domain.eml`)
path.join(__dirname, 'test-data/email-different-domain.eml'),
);
// Should pass with default domain
await verifyDKIMSignature(email);
// Should fail because the email wont have a DKIM signature with the overridden domain
// Can be replaced with a better test email where signer is actually
// Can be replaced with a better test email where signer is actually
// different from From domain and the below check pass.
expect.assertions(1);
try {
await verifyDKIMSignature(email, "domain.com");
await verifyDKIMSignature(email, 'domain.com');
} catch (e) {
expect(e.message).toBe(
"DKIM signature not found for domain domain.com"
'DKIM signature not found for domain domain.com',
);
}
});
});
describe("DKIM with sanitization", () => {
it("should pass after removing label from Subject", async () => {
describe('DKIM with sanitization', () => {
it('should pass after removing label from Subject', async () => {
const email = fs.readFileSync(
path.join(__dirname, `test-data/email-good.eml`)
path.join(__dirname, 'test-data/email-good.eml'),
);
// Add a label to the subject
const tamperedEmail = email.toString().replace("Subject: ", "Subject: [EmailListABC]");
const tamperedEmail = email.toString().replace('Subject: ', 'Subject: [EmailListABC]');
const result = await verifyDKIMSignature(tamperedEmail);
expect(result.appliedSanitization).toBe("removeLabels");
expect(result.appliedSanitization).toBe('removeLabels');
});
});

36
packages/helpers/tests/input-generators.test.ts
View File

@@ -1,14 +1,14 @@
import { generateEmailVerifierInputs } from "../src/input-generators";
import fs from "fs";
import path from "path";
import { bytesToString } from "../src/binary-format";
import fs from 'fs';
import path from 'path';
import { generateEmailVerifierInputs } from '../src/input-generators';
import { bytesToString } from '../src/binary-format';
jest.setTimeout(10000);
describe("Input generators", () => {
it("should generate input from raw email", async () => {
describe('Input generators', () => {
it('should generate input from raw email', async () => {
const email = fs.readFileSync(
path.join(__dirname, `test-data/email-good.eml`)
path.join(__dirname, 'test-data/email-good.eml'),
);
const inputs = await generateEmailVerifierInputs(email);
@@ -22,9 +22,9 @@ describe("Input generators", () => {
expect(inputs.bodyHashIndex).toBeDefined();
});
it("should generate input without body params when ignoreBodyHash is true", async () => {
it('should generate input without body params when ignoreBodyHash is true', async () => {
const email = fs.readFileSync(
path.join(__dirname, `test-data/email-good.eml`)
path.join(__dirname, 'test-data/email-good.eml'),
);
const inputs = await generateEmailVerifierInputs(email, {
@@ -40,37 +40,37 @@ describe("Input generators", () => {
expect(inputs.bodyHashIndex).toBeFalsy();
});
it("should generate input with SHA precompute selector", async () => {
it('should generate input with SHA precompute selector', async () => {
const email = fs.readFileSync(
path.join(__dirname, `test-data/email-good-large.eml`)
path.join(__dirname, 'test-data/email-good-large.eml'),
);
const inputs = await generateEmailVerifierInputs(email, {
shaPrecomputeSelector: "thousands",
shaPrecomputeSelector: 'thousands',
});
expect(inputs.emailBody).toBeDefined();
const strBody = bytesToString(
Uint8Array.from(inputs.emailBody!.map((b) => Number(b)))
Uint8Array.from(inputs.emailBody!.map((b) => Number(b))),
);
const expected = `h hundreds of thousands of blocks.`; // will round till previous 64x th byte
const expected = 'h hundreds of thousands of blocks.'; // will round till previous 64x th byte
expect(strBody.startsWith(expected)).toBeTruthy();
});
it("should throw if SHA precompute selector is invalid", async () => {
it('should throw if SHA precompute selector is invalid', async () => {
const email = fs.readFileSync(
path.join(__dirname, `test-data/email-good.eml`)
path.join(__dirname, 'test-data/email-good.eml'),
);
expect(async () => {
await generateEmailVerifierInputs(email, {
shaPrecomputeSelector: "Bla Bla",
shaPrecomputeSelector: 'Bla Bla',
});
}).rejects.toThrow(
"Provider SHA precompute selector not found in the body"
'Provider SHA precompute selector not found in the body',
);
});
});

2
packages/helpers/tsconfig.json
View File

@@ -1,5 +1,5 @@
{
"include": ["src/**/*"],
"include": ["src/**/*", "tests/**/*", "lib/mailauth", "lib/fast-sha256.ts"],
"compilerOptions": {
"target": "es2020",
"module": "commonjs",

5315
yarn.lock
View File

File diff suppressed because it is too large Load Diff