From 88fbac9fcf8d5b9eabb1219b7b2b2b781bbf7861 Mon Sep 17 00:00:00 2001 From: Cayman Date: Thu, 23 Oct 2025 14:28:25 -0400 Subject: [PATCH] feat: use bytes from lodestar-bun (#8562) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Motivation** - #7280 **Description** - Build upon the isomorphic bytes code in the utils package - refactor browser/nodejs selection to use conditional imports (like how we've been handling bun / nodejs selection - Use Uint8Array.fromHex and toHex (mentioned in https://github.com/ChainSafe/lodestar/pull/8275#issuecomment-3228184163) - Refactor the bytes perf tests to include bun - Add lodestar-bun dependency (also add missing dependency in beacon-node package) Results from my machine ``` bytes utils ✔ nodejs block root to RootHex using toHex 5500338 ops/s 181.8070 ns/op - 1048 runs 0.444 s ✔ nodejs block root to RootHex using toRootHex 7466866 ops/s 133.9250 ns/op - 2189 runs 0.477 s ✔ nodejs fromHex(blob) 7001.930 ops/s 142.8178 us/op - 10 runs 1.94 s ✔ nodejs fromHexInto(blob) 1744.298 ops/s 573.2965 us/op - 10 runs 6.33 s ✔ nodejs block root to RootHex using the deprecated toHexString 1609510 ops/s 621.3070 ns/op - 309 runs 0.704 s ✔ browser block root to RootHex using toHex 1854390 ops/s 539.2610 ns/op - 522 runs 0.807 s ✔ browser block root to RootHex using toRootHex 2060543 ops/s 485.3090 ns/op - 597 runs 0.805 s ✔ browser fromHex(blob) 1632.601 ops/s 612.5196 us/op - 10 runs 6.77 s ✔ browser fromHexInto(blob) 1751.718 ops/s 570.8683 us/op - 10 runs 6.36 s ✔ browser block root to RootHex using the deprecated toHexString 1596024 ops/s 626.5570 ns/op - 457 runs 0.805 s ✔ bun block root to RootHex using toHex 1.249563e+7 ops/s 80.02800 ns/op - 4506 runs 0.518 s ✔ bun block root to RootHex using toRootHex 1.262626e+7 ops/s 79.20000 ns/op - 3716 runs 0.409 s ✔ bun fromHex(blob) 26995.09 ops/s 37.04377 us/op - 10 runs 0.899 s ✔ bun fromHexInto(blob) 31539.09 ops/s 31.70668 us/op - 13 runs 0.914 s ✔ bun block root to RootHex using the deprecated toHexString 1.252944e+7 ops/s 79.81200 ns/op - 3616 runs 0.414 s ``` --- packages/beacon-node/package.json | 1 + packages/utils/package.json | 8 ++ packages/utils/src/bytes.ts | 84 ------------ packages/utils/src/bytes/browser.ts | 65 +++++++++ packages/utils/src/bytes/bun.ts | 54 ++++++++ packages/utils/src/bytes/index.ts | 29 ---- packages/utils/src/bytes/nodejs.ts | 2 + packages/utils/src/format.ts | 22 ++- packages/utils/src/index.ts | 3 +- packages/utils/test/perf/bytes.test.ts | 177 ++++++++++--------------- packages/utils/tsconfig.build.json | 1 + packages/utils/tsconfig.json | 1 + 12 files changed, 222 insertions(+), 225 deletions(-) delete mode 100644 packages/utils/src/bytes.ts create mode 100644 packages/utils/src/bytes/bun.ts delete mode 100644 packages/utils/src/bytes/index.ts diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index d791fa2707..f712b62e10 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -145,6 +145,7 @@ "@libp2p/prometheus-metrics": "^4.3.15", "@libp2p/tcp": "^10.1.8", "@lodestar/api": "^1.35.0", + "@lodestar/bun": "git+https://github.com/ChainSafe/lodestar-bun.git", "@lodestar/config": "^1.35.0", "@lodestar/db": "^1.35.0", "@lodestar/fork-choice": "^1.35.0", diff --git a/packages/utils/package.json b/packages/utils/package.json index ca600a1c70..a512a776b3 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -20,6 +20,13 @@ "import": "./lib/index.js" } }, + "imports": { + "#bytes": { + "bun": "./src/bytes/bun.ts", + "browser": "./lib/bytes/browser.js", + "default": "./lib/bytes/nodejs.js" + } + }, "files": [ "src", "lib", @@ -41,6 +48,7 @@ "types": "lib/index.d.ts", "dependencies": { "@chainsafe/as-sha256": "^1.2.0", + "@lodestar/bun": "git+https://github.com/ChainSafe/lodestar-bun.git", "any-signal": "^4.1.1", "bigint-buffer": "^1.1.5", "case": "^1.6.3", diff --git a/packages/utils/src/bytes.ts b/packages/utils/src/bytes.ts deleted file mode 100644 index c992be441d..0000000000 --- a/packages/utils/src/bytes.ts +++ /dev/null @@ -1,84 +0,0 @@ -import {toBigIntBE, toBigIntLE, toBufferBE, toBufferLE} from "bigint-buffer"; - -type Endianness = "le" | "be"; - -const hexByByte: string[] = []; -/** - * @deprecated Use toHex() instead. - */ -export function toHexString(bytes: Uint8Array): string { - let hex = "0x"; - for (const byte of bytes) { - if (!hexByByte[byte]) { - hexByByte[byte] = byte < 16 ? "0" + byte.toString(16) : byte.toString(16); - } - hex += hexByByte[byte]; - } - return hex; -} - -/** - * Return a byte array from a number or BigInt - */ -export function intToBytes(value: bigint | number, length: number, endianness: Endianness = "le"): Buffer { - return bigIntToBytes(BigInt(value), length, endianness); -} - -/** - * Convert byte array in LE to integer. - */ -export function bytesToInt(value: Uint8Array, endianness: Endianness = "le"): number { - return Number(bytesToBigInt(value, endianness)); -} - -export function bigIntToBytes(value: bigint, length: number, endianness: Endianness = "le"): Buffer { - if (endianness === "le") { - return toBufferLE(value, length); - } - if (endianness === "be") { - return toBufferBE(value, length); - } - throw new Error("endianness must be either 'le' or 'be'"); -} - -export function bytesToBigInt(value: Uint8Array, endianness: Endianness = "le"): bigint { - if (!(value instanceof Uint8Array)) { - throw new TypeError("expected a Uint8Array"); - } - - if (endianness === "le") { - return toBigIntLE(value as Buffer); - } - if (endianness === "be") { - return toBigIntBE(value as Buffer); - } - throw new Error("endianness must be either 'le' or 'be'"); -} - -export function formatBytes(bytes: number): string { - if (bytes < 0) { - throw new Error("bytes must be a positive number, got " + bytes); - } - - if (bytes === 0) { - return "0 Bytes"; - } - - // size of a kb - const k = 1024; - - // only support up to GB - const units = ["Bytes", "KB", "MB", "GB"]; - const i = Math.min(Math.floor(Math.log(bytes) / Math.log(k)), units.length - 1); - const formattedSize = (bytes / Math.pow(k, i)).toFixed(2); - - return `${formattedSize} ${units[i]}`; -} - -export function xor(a: Uint8Array, b: Uint8Array): Uint8Array { - const length = Math.min(a.length, b.length); - for (let i = 0; i < length; i++) { - a[i] = a[i] ^ b[i]; - } - return a; -} diff --git a/packages/utils/src/bytes/browser.ts b/packages/utils/src/bytes/browser.ts index 27abb185e4..50c92b22e1 100644 --- a/packages/utils/src/bytes/browser.ts +++ b/packages/utils/src/bytes/browser.ts @@ -121,3 +121,68 @@ function charCodeToByte(charCode: number): number { throw new Error(`Invalid hex character code: ${charCode}`); } + +import {toBigIntBE, toBigIntLE, toBufferBE, toBufferLE} from "bigint-buffer"; + +type Endianness = "le" | "be"; + +const hexByByte: string[] = []; +/** + * @deprecated Use toHex() instead. + */ +export function toHexString(bytes: Uint8Array): string { + let hex = "0x"; + for (const byte of bytes) { + if (!hexByByte[byte]) { + hexByByte[byte] = byte < 16 ? "0" + byte.toString(16) : byte.toString(16); + } + hex += hexByByte[byte]; + } + return hex; +} + +/** + * Return a byte array from a number or BigInt + */ +export function intToBytes(value: bigint | number, length: number, endianness: Endianness = "le"): Buffer { + return bigIntToBytes(BigInt(value), length, endianness); +} + +/** + * Convert byte array in LE to integer. + */ +export function bytesToInt(value: Uint8Array, endianness: Endianness = "le"): number { + return Number(bytesToBigInt(value, endianness)); +} + +export function bigIntToBytes(value: bigint, length: number, endianness: Endianness = "le"): Buffer { + if (endianness === "le") { + return toBufferLE(value, length); + } + if (endianness === "be") { + return toBufferBE(value, length); + } + throw new Error("endianness must be either 'le' or 'be'"); +} + +export function bytesToBigInt(value: Uint8Array, endianness: Endianness = "le"): bigint { + if (!(value instanceof Uint8Array)) { + throw new TypeError("expected a Uint8Array"); + } + + if (endianness === "le") { + return toBigIntLE(value as Buffer); + } + if (endianness === "be") { + return toBigIntBE(value as Buffer); + } + throw new Error("endianness must be either 'le' or 'be'"); +} + +export function xor(a: Uint8Array, b: Uint8Array): Uint8Array { + const length = Math.min(a.length, b.length); + for (let i = 0; i < length; i++) { + a[i] = a[i] ^ b[i]; + } + return a; +} diff --git a/packages/utils/src/bytes/bun.ts b/packages/utils/src/bytes/bun.ts new file mode 100644 index 0000000000..44cf0662db --- /dev/null +++ b/packages/utils/src/bytes/bun.ts @@ -0,0 +1,54 @@ +import {bytes} from "@lodestar/bun"; + +export function toHex(data: Uint8Array): string { + return `0x${data.toHex()}`; +} + +export function toRootHex(root: Uint8Array): string { + if (root.length !== 32) { + throw Error(`Expect root to be 32 bytes, got ${root.length}`); + } + return `0x${root.toHex()}`; +} + +export function toPubkeyHex(pubkey: Uint8Array): string { + if (pubkey.length !== 48) { + throw Error(`Expect pubkey to be 48 bytes, got ${pubkey.length}`); + } + return `0x${pubkey.toHex()}`; +} + +export function fromHex(hex: string): Uint8Array { + if (hex.startsWith("0x")) { + hex = hex.slice(2); + } + + if (hex.length % 2 !== 0) { + throw new Error(`hex string length ${hex.length} must be multiple of 2`); + } + + return Uint8Array.fromHex(hex); +} + +export function fromHexInto(hex: string, buffer: Uint8Array): void { + if (hex.startsWith("0x")) { + hex = hex.slice(2); + } + + if (hex.length !== buffer.length * 2) { + throw new Error(`hex string length ${hex.length} must be exactly double the buffer length ${buffer.length}`); + } + + buffer.setFromHex(hex); +} + +export const toHexString = toHex; + +export const { + intToBytes, + bytesToInt, + // naming differences from upstream + intToBytes: bigIntToBytes, + bytesToBigint: bytesToBigInt, +} = bytes; +export {xor} from "./browser.ts"; diff --git a/packages/utils/src/bytes/index.ts b/packages/utils/src/bytes/index.ts deleted file mode 100644 index 51823ae03b..0000000000 --- a/packages/utils/src/bytes/index.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { - fromHex as browserFromHex, - fromHexInto as browserFromHexInto, - toHex as browserToHex, - toPubkeyHex as browserToPubkeyHex, - toRootHex as browserToRootHex, -} from "./browser.js"; -import { - fromHex as nodeFromHex, - toHex as nodeToHex, - toPubkeyHex as nodeToPubkeyHex, - toRootHex as nodeToRootHex, -} from "./nodejs.js"; - -let toHex = browserToHex; -let toRootHex = browserToRootHex; -let toPubkeyHex = browserToPubkeyHex; -let fromHex = browserFromHex; -// there is no fromHexInto for NodeJs as the performance of browserFromHexInto is >100x faster -const fromHexInto = browserFromHexInto; - -if (typeof Buffer !== "undefined") { - toHex = nodeToHex; - toRootHex = nodeToRootHex; - toPubkeyHex = nodeToPubkeyHex; - fromHex = nodeFromHex; -} - -export {toHex, toRootHex, toPubkeyHex, fromHex, fromHexInto}; diff --git a/packages/utils/src/bytes/nodejs.ts b/packages/utils/src/bytes/nodejs.ts index b1172bb60b..7f544d44fc 100644 --- a/packages/utils/src/bytes/nodejs.ts +++ b/packages/utils/src/bytes/nodejs.ts @@ -61,3 +61,5 @@ export function fromHex(hex: string): Uint8Array { } /// the performance of fromHexInto using a preallocated buffer is very bad compared to browser so I moved it to the benchmark + +export {bigIntToBytes, bytesToBigInt, bytesToInt, fromHexInto, intToBytes, toHexString, xor} from "./browser.ts"; diff --git a/packages/utils/src/format.ts b/packages/utils/src/format.ts index 88c08a127e..6bcacb717b 100644 --- a/packages/utils/src/format.ts +++ b/packages/utils/src/format.ts @@ -1,4 +1,4 @@ -import {toRootHex} from "./bytes/index.js"; +import {toRootHex} from "#bytes"; import {ETH_TO_WEI} from "./ethConversion.js"; /** @@ -117,3 +117,23 @@ export function prettyPrintIndices(indices: number[]): string { const increments = groupSequentialIndices(indices); return `[${increments.join(", ")}]`; } + +export function formatBytes(bytes: number): string { + if (bytes < 0) { + throw new Error("bytes must be a positive number, got " + bytes); + } + + if (bytes === 0) { + return "0 Bytes"; + } + + // size of a kb + const k = 1024; + + // only support up to GB + const units = ["Bytes", "KB", "MB", "GB"]; + const i = Math.min(Math.floor(Math.log(bytes) / Math.log(k)), units.length - 1); + const formattedSize = (bytes / Math.pow(k, i)).toFixed(2); + + return `${formattedSize} ${units[i]}`; +} diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index e360b3374a..c98435c1c1 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -1,7 +1,6 @@ +export * from "#bytes"; export * from "./assert.js"; export * from "./base64.js"; -export * from "./bytes/index.js"; -export * from "./bytes.js"; export * from "./command.js"; export * from "./diff.js"; export * from "./err.js"; diff --git a/packages/utils/test/perf/bytes.test.ts b/packages/utils/test/perf/bytes.test.ts index 6a87111c9d..32ab1233a5 100644 --- a/packages/utils/test/perf/bytes.test.ts +++ b/packages/utils/test/perf/bytes.test.ts @@ -1,14 +1,8 @@ import {bench, describe} from "@chainsafe/benchmark"; -import { - fromHex as browserFromHex, - fromHexInto as browserFromHexInto, - toHex as browserToHex, - toRootHex as browserToRootHex, -} from "../../src/bytes/browser.js"; -import {fromHex, toHex, toRootHex} from "../../src/bytes/nodejs.js"; -import {toHexString} from "../../src/bytes.js"; +import * as browser from "../../src/bytes/browser.ts"; +import * as nodejs from "../../src/bytes/nodejs.ts"; -describe("bytes utils", () => { +describe("bytes utils", async () => { const runsFactor = 1000; const blockRoot = new Uint8Array(Array.from({length: 32}, (_, i) => i)); // FIELD_ELEMENTS_PER_BLOB * BYTES_PER_FIELD_ELEMENT = 4096 * 32 = 131072 @@ -17,111 +11,76 @@ describe("bytes utils", () => { for (let i = 0; i < blob.length; i++) { blob[i] = i % 256; } - const blobHex = toHex(blob); + const blobHex = nodejs.toHex(blob); - bench({ - id: "nodejs block root to RootHex using toHex", - fn: () => { - for (let i = 0; i < runsFactor; i++) { - toHex(blockRoot); - } + const implementations = [ + { + name: "nodejs", + impl: nodejs, }, - runsFactor, - }); + { + name: "browser", + impl: browser, + }, + Boolean(globalThis.Bun) && { + name: "bun", + impl: await import("../../src/bytes/bun.ts"), + }, + ].filter(Boolean) as { + name: string; + impl: typeof nodejs; + }[]; - bench({ - id: "nodejs block root to RootHex using toRootHex", - fn: () => { - for (let i = 0; i < runsFactor; i++) { - toRootHex(blockRoot); - } - }, - runsFactor, - }); + for (const {name, impl} of implementations) { + bench({ + id: `${name} block root to RootHex using toHex`, + fn: () => { + for (let i = 0; i < runsFactor; i++) { + impl.toHex(blockRoot); + } + }, + runsFactor, + }); - bench({ - id: "nodejs fromhex(blob)", - fn: () => { - for (let i = 0; i < runsFactor; i++) { - fromHex(blobHex); - } - }, - }); + bench({ + id: `${name} block root to RootHex using toRootHex`, + fn: () => { + for (let i = 0; i < runsFactor; i++) { + impl.toRootHex(blockRoot); + } + }, + runsFactor, + }); - const buffer = Buffer.alloc(BLOB_LEN); - bench({ - id: "nodejs fromHexInto(blob)", - fn: () => { - for (let i = 0; i < runsFactor; i++) { - nodeJsFromHexInto(blobHex, buffer); - } - }, - }); + bench({ + id: `${name} fromHex(blob)`, + fn: () => { + for (let i = 0; i < runsFactor; i++) { + impl.fromHex(blobHex); + } + }, + runsFactor, + }); - bench({ - id: "browser block root to RootHex using the deprecated toHexString", - fn: () => { - for (let i = 0; i < runsFactor; i++) { - toHexString(blockRoot); - } - }, - runsFactor, - }); + const buffer = new Uint8Array(BLOB_LEN); + bench({ + id: `${name} fromHexInto(blob)`, + fn: () => { + for (let i = 0; i < runsFactor; i++) { + impl.fromHexInto(blobHex, buffer); + } + }, + runsFactor, + }); - bench({ - id: "browser block root to RootHex using toHex", - fn: () => { - for (let i = 0; i < runsFactor; i++) { - browserToHex(blockRoot); - } - }, - runsFactor, - }); - - bench({ - id: "browser block root to RootHex using toRootHex", - fn: () => { - for (let i = 0; i < runsFactor; i++) { - browserToRootHex(blockRoot); - } - }, - runsFactor, - }); - - const buf = new Uint8Array(BLOB_LEN); - bench({ - id: "browser fromHexInto(blob)", - fn: () => { - for (let i = 0; i < runsFactor; i++) { - browserFromHexInto(blobHex, buf); - } - }, - runsFactor, - }); - - bench({ - id: "browser fromHex(blob)", - fn: () => { - for (let i = 0; i < runsFactor; i++) { - browserFromHex(blobHex); - } - }, - }); + bench({ + id: `${name} block root to RootHex using the deprecated toHexString`, + fn: () => { + for (let i = 0; i < runsFactor; i++) { + impl.toHexString(blockRoot); + } + }, + runsFactor, + }); + } }); - -/** - * this function is so slow compared to browser's implementation we only maintain it here to compare performance - * - nodejs fromHexInto(blob) 3.562495 ops/s 280.7022 ms/op - 10 runs 3.50 s - * - browser fromHexInto(blob) 535.0952 ops/s 1.868826 ms/op - 10 runs 20.8 s - */ -function nodeJsFromHexInto(hex: string, buffer: Buffer): void { - if (hex.startsWith("0x")) { - hex = hex.slice(2); - } - - if (hex.length !== buffer.length * 2) { - throw new Error(`hex string length ${hex.length} must be exactly double the buffer length ${buffer.length}`); - } - - buffer.write(hex, "hex"); -} diff --git a/packages/utils/tsconfig.build.json b/packages/utils/tsconfig.build.json index 92235557ba..e67663675a 100644 --- a/packages/utils/tsconfig.build.json +++ b/packages/utils/tsconfig.build.json @@ -2,6 +2,7 @@ "extends": "../../tsconfig.build.json", "include": ["src"], "compilerOptions": { + "rootDir": "src", "outDir": "lib" } } diff --git a/packages/utils/tsconfig.json b/packages/utils/tsconfig.json index a0f4f2a31e..4d40b7e8d2 100644 --- a/packages/utils/tsconfig.json +++ b/packages/utils/tsconfig.json @@ -2,6 +2,7 @@ "extends": "../../tsconfig.json", "include": ["src", "test"], "compilerOptions": { + "rootDir": "../..", "outDir": "lib" } }