github/lodestar
1
1
Fork 0
mirror of https://github.com/ChainSafe/lodestar.git synced 2026-01-07 23:04:06 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: use bytes from lodestar-bun (#8562)

Browse Source 
**Motivation**

- #7280 

**Description**

- Build upon the isomorphic bytes code in the utils package
- refactor browser/nodejs selection to use conditional imports (like how
we've been handling bun / nodejs selection
- Use Uint8Array.fromHex and toHex (mentioned in
https://github.com/ChainSafe/lodestar/pull/8275#issuecomment-3228184163)
- Refactor the bytes perf tests to include bun
- Add lodestar-bun dependency (also add missing dependency in
beacon-node package)

Results from my machine
```
  bytes utils
    ✔ nodejs block root to RootHex using toHex                             5500338 ops/s    181.8070 ns/op        -       1048 runs  0.444 s
    ✔ nodejs block root to RootHex using toRootHex                         7466866 ops/s    133.9250 ns/op        -       2189 runs  0.477 s
    ✔ nodejs fromHex(blob)                                                7001.930 ops/s    142.8178 us/op        -         10 runs   1.94 s
    ✔ nodejs fromHexInto(blob)                                            1744.298 ops/s    573.2965 us/op        -         10 runs   6.33 s
    ✔ nodejs block root to RootHex using the deprecated toHexString        1609510 ops/s    621.3070 ns/op        -        309 runs  0.704 s
    ✔ browser block root to RootHex using toHex                            1854390 ops/s    539.2610 ns/op        -        522 runs  0.807 s
    ✔ browser block root to RootHex using toRootHex                        2060543 ops/s    485.3090 ns/op        -        597 runs  0.805 s
    ✔ browser fromHex(blob)                                               1632.601 ops/s    612.5196 us/op        -         10 runs   6.77 s
    ✔ browser fromHexInto(blob)                                           1751.718 ops/s    570.8683 us/op        -         10 runs   6.36 s
    ✔ browser block root to RootHex using the deprecated toHexString       1596024 ops/s    626.5570 ns/op        -        457 runs  0.805 s
    ✔ bun block root to RootHex using toHex                            1.249563e+7 ops/s    80.02800 ns/op        -       4506 runs  0.518 s
    ✔ bun block root to RootHex using toRootHex                        1.262626e+7 ops/s    79.20000 ns/op        -       3716 runs  0.409 s
    ✔ bun fromHex(blob)                                                   26995.09 ops/s    37.04377 us/op        -         10 runs  0.899 s
    ✔ bun fromHexInto(blob)                                               31539.09 ops/s    31.70668 us/op        -         13 runs  0.914 s
    ✔ bun block root to RootHex using the deprecated toHexString       1.252944e+7 ops/s    79.81200 ns/op        -       3616 runs  0.414 s
```
This commit is contained in:
Cayman
2025-10-23 14:28:25 -04:00
committed by GitHub
parent 57b1f6e666
commit 88fbac9fcf
12 changed files with 222 additions and 225 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
packages/beacon-node/package.json
View File

@@ -145,6 +145,7 @@
"@libp2p/prometheus-metrics": "^4.3.15",
"@libp2p/tcp": "^10.1.8",
"@lodestar/api": "^1.35.0",
"@lodestar/bun": "git+https://github.com/ChainSafe/lodestar-bun.git",
"@lodestar/config": "^1.35.0",
"@lodestar/db": "^1.35.0",
"@lodestar/fork-choice": "^1.35.0",

8
packages/utils/package.json
View File

@@ -20,6 +20,13 @@
"import": "./lib/index.js"
}
},
"imports": {
"#bytes": {
"bun": "./src/bytes/bun.ts",
"browser": "./lib/bytes/browser.js",
"default": "./lib/bytes/nodejs.js"
}
},
"files": [
"src",
"lib",
@@ -41,6 +48,7 @@
"types": "lib/index.d.ts",
"dependencies": {
"@chainsafe/as-sha256": "^1.2.0",
"@lodestar/bun": "git+https://github.com/ChainSafe/lodestar-bun.git",
"any-signal": "^4.1.1",
"bigint-buffer": "^1.1.5",
"case": "^1.6.3",

84
packages/utils/src/bytes.ts
View File

@@ -1,84 +0,0 @@
import {toBigIntBE, toBigIntLE, toBufferBE, toBufferLE} from "bigint-buffer";
type Endianness = "le" | "be";
const hexByByte: string[] = [];
/**
* @deprecated Use toHex() instead.
*/
export function toHexString(bytes: Uint8Array): string {
let hex = "0x";
for (const byte of bytes) {
if (!hexByByte[byte]) {
hexByByte[byte] = byte < 16 ? "0" + byte.toString(16) : byte.toString(16);
}
hex += hexByByte[byte];
}
return hex;
}
/**
* Return a byte array from a number or BigInt
*/
export function intToBytes(value: bigint | number, length: number, endianness: Endianness = "le"): Buffer {
return bigIntToBytes(BigInt(value), length, endianness);
}
/**
* Convert byte array in LE to integer.
*/
export function bytesToInt(value: Uint8Array, endianness: Endianness = "le"): number {
return Number(bytesToBigInt(value, endianness));
}
export function bigIntToBytes(value: bigint, length: number, endianness: Endianness = "le"): Buffer {
if (endianness === "le") {
return toBufferLE(value, length);
}
if (endianness === "be") {
return toBufferBE(value, length);
}
throw new Error("endianness must be either 'le' or 'be'");
}
export function bytesToBigInt(value: Uint8Array, endianness: Endianness = "le"): bigint {
if (!(value instanceof Uint8Array)) {
throw new TypeError("expected a Uint8Array");
}
if (endianness === "le") {
return toBigIntLE(value as Buffer);
}
if (endianness === "be") {
return toBigIntBE(value as Buffer);
}
throw new Error("endianness must be either 'le' or 'be'");
}
export function formatBytes(bytes: number): string {
if (bytes < 0) {
throw new Error("bytes must be a positive number, got " + bytes);
}
if (bytes === 0) {
return "0 Bytes";
}
// size of a kb
const k = 1024;
// only support up to GB
const units = ["Bytes", "KB", "MB", "GB"];
const i = Math.min(Math.floor(Math.log(bytes) / Math.log(k)), units.length - 1);
const formattedSize = (bytes / Math.pow(k, i)).toFixed(2);
return `${formattedSize} ${units[i]}`;
}
export function xor(a: Uint8Array, b: Uint8Array): Uint8Array {
const length = Math.min(a.length, b.length);
for (let i = 0; i < length; i++) {
a[i] = a[i] ^ b[i];
}
return a;
}

65
packages/utils/src/bytes/browser.ts
View File

@@ -121,3 +121,68 @@ function charCodeToByte(charCode: number): number {
throw new Error(`Invalid hex character code: ${charCode}`);
}
import {toBigIntBE, toBigIntLE, toBufferBE, toBufferLE} from "bigint-buffer";
type Endianness = "le" | "be";
const hexByByte: string[] = [];
/**
* @deprecated Use toHex() instead.
*/
export function toHexString(bytes: Uint8Array): string {
let hex = "0x";
for (const byte of bytes) {
if (!hexByByte[byte]) {
hexByByte[byte] = byte < 16 ? "0" + byte.toString(16) : byte.toString(16);
}
hex += hexByByte[byte];
}
return hex;
}
/**
* Return a byte array from a number or BigInt
*/
export function intToBytes(value: bigint | number, length: number, endianness: Endianness = "le"): Buffer {
return bigIntToBytes(BigInt(value), length, endianness);
}
/**
* Convert byte array in LE to integer.
*/
export function bytesToInt(value: Uint8Array, endianness: Endianness = "le"): number {
return Number(bytesToBigInt(value, endianness));
}
export function bigIntToBytes(value: bigint, length: number, endianness: Endianness = "le"): Buffer {
if (endianness === "le") {
return toBufferLE(value, length);
}
if (endianness === "be") {
return toBufferBE(value, length);
}
throw new Error("endianness must be either 'le' or 'be'");
}
export function bytesToBigInt(value: Uint8Array, endianness: Endianness = "le"): bigint {
if (!(value instanceof Uint8Array)) {
throw new TypeError("expected a Uint8Array");
}
if (endianness === "le") {
return toBigIntLE(value as Buffer);
}
if (endianness === "be") {
return toBigIntBE(value as Buffer);
}
throw new Error("endianness must be either 'le' or 'be'");
}
export function xor(a: Uint8Array, b: Uint8Array): Uint8Array {
const length = Math.min(a.length, b.length);
for (let i = 0; i < length; i++) {
a[i] = a[i] ^ b[i];
}
return a;
}

54
packages/utils/src/bytes/bun.ts Normal file
View File

@@ -0,0 +1,54 @@
import {bytes} from "@lodestar/bun";
export function toHex(data: Uint8Array): string {
return `0x${data.toHex()}`;
}
export function toRootHex(root: Uint8Array): string {
if (root.length !== 32) {
throw Error(`Expect root to be 32 bytes, got ${root.length}`);
}
return `0x${root.toHex()}`;
}
export function toPubkeyHex(pubkey: Uint8Array): string {
if (pubkey.length !== 48) {
throw Error(`Expect pubkey to be 48 bytes, got ${pubkey.length}`);
}
return `0x${pubkey.toHex()}`;
}
export function fromHex(hex: string): Uint8Array {
if (hex.startsWith("0x")) {
hex = hex.slice(2);
}
if (hex.length % 2 !== 0) {
throw new Error(`hex string length ${hex.length} must be multiple of 2`);
}
return Uint8Array.fromHex(hex);
}
export function fromHexInto(hex: string, buffer: Uint8Array): void {
if (hex.startsWith("0x")) {
hex = hex.slice(2);
}
if (hex.length !== buffer.length * 2) {
throw new Error(`hex string length ${hex.length} must be exactly double the buffer length ${buffer.length}`);
}
buffer.setFromHex(hex);
}
export const toHexString = toHex;
export const {
intToBytes,
bytesToInt,
// naming differences from upstream
intToBytes: bigIntToBytes,
bytesToBigint: bytesToBigInt,
} = bytes;
export {xor} from "./browser.ts";

29
packages/utils/src/bytes/index.ts
View File

@@ -1,29 +0,0 @@
import {
fromHex as browserFromHex,
fromHexInto as browserFromHexInto,
toHex as browserToHex,
toPubkeyHex as browserToPubkeyHex,
toRootHex as browserToRootHex,
} from "./browser.js";
import {
fromHex as nodeFromHex,
toHex as nodeToHex,
toPubkeyHex as nodeToPubkeyHex,
toRootHex as nodeToRootHex,
} from "./nodejs.js";
let toHex = browserToHex;
let toRootHex = browserToRootHex;
let toPubkeyHex = browserToPubkeyHex;
let fromHex = browserFromHex;
// there is no fromHexInto for NodeJs as the performance of browserFromHexInto is >100x faster
const fromHexInto = browserFromHexInto;
if (typeof Buffer !== "undefined") {
toHex = nodeToHex;
toRootHex = nodeToRootHex;
toPubkeyHex = nodeToPubkeyHex;
fromHex = nodeFromHex;
}
export {toHex, toRootHex, toPubkeyHex, fromHex, fromHexInto};

2
packages/utils/src/bytes/nodejs.ts
View File

@@ -61,3 +61,5 @@ export function fromHex(hex: string): Uint8Array {
}
/// the performance of fromHexInto using a preallocated buffer is very bad compared to browser so I moved it to the benchmark
export {bigIntToBytes, bytesToBigInt, bytesToInt, fromHexInto, intToBytes, toHexString, xor} from "./browser.ts";

22
packages/utils/src/format.ts
View File

@@ -1,4 +1,4 @@
import {toRootHex} from "./bytes/index.js";
import {toRootHex} from "#bytes";
import {ETH_TO_WEI} from "./ethConversion.js";
/**
@@ -117,3 +117,23 @@ export function prettyPrintIndices(indices: number[]): string {
const increments = groupSequentialIndices(indices);
return `[${increments.join(", ")}]`;
}
export function formatBytes(bytes: number): string {
if (bytes < 0) {
throw new Error("bytes must be a positive number, got " + bytes);
}
if (bytes === 0) {
return "0 Bytes";
}
// size of a kb
const k = 1024;
// only support up to GB
const units = ["Bytes", "KB", "MB", "GB"];
const i = Math.min(Math.floor(Math.log(bytes) / Math.log(k)), units.length - 1);
const formattedSize = (bytes / Math.pow(k, i)).toFixed(2);
return `${formattedSize} ${units[i]}`;
}

3
packages/utils/src/index.ts
View File

@@ -1,7 +1,6 @@
export * from "#bytes";
export * from "./assert.js";
export * from "./base64.js";
export * from "./bytes/index.js";
export * from "./bytes.js";
export * from "./command.js";
export * from "./diff.js";
export * from "./err.js";

177
packages/utils/test/perf/bytes.test.ts
View File

@@ -1,14 +1,8 @@
import {bench, describe} from "@chainsafe/benchmark";
import {
fromHex as browserFromHex,
fromHexInto as browserFromHexInto,
toHex as browserToHex,
toRootHex as browserToRootHex,
} from "../../src/bytes/browser.js";
import {fromHex, toHex, toRootHex} from "../../src/bytes/nodejs.js";
import {toHexString} from "../../src/bytes.js";
import * as browser from "../../src/bytes/browser.ts";
import * as nodejs from "../../src/bytes/nodejs.ts";
describe("bytes utils", () => {
describe("bytes utils", async () => {
const runsFactor = 1000;
const blockRoot = new Uint8Array(Array.from({length: 32}, (_, i) => i));
// FIELD_ELEMENTS_PER_BLOB * BYTES_PER_FIELD_ELEMENT = 4096 * 32 = 131072
@@ -17,111 +11,76 @@ describe("bytes utils", () => {
for (let i = 0; i < blob.length; i++) {
blob[i] = i % 256;
}
const blobHex = toHex(blob);
const blobHex = nodejs.toHex(blob);
bench({
id: "nodejs block root to RootHex using toHex",
fn: () => {
for (let i = 0; i < runsFactor; i++) {
toHex(blockRoot);
}
const implementations = [
{
name: "nodejs",
impl: nodejs,
},
runsFactor,
});
{
name: "browser",
impl: browser,
},
Boolean(globalThis.Bun) && {
name: "bun",
impl: await import("../../src/bytes/bun.ts"),
},
].filter(Boolean) as {
name: string;
impl: typeof nodejs;
}[];
bench({
id: "nodejs block root to RootHex using toRootHex",
fn: () => {
for (let i = 0; i < runsFactor; i++) {
toRootHex(blockRoot);
}
},
runsFactor,
});
for (const {name, impl} of implementations) {
bench({
id: `${name} block root to RootHex using toHex`,
fn: () => {
for (let i = 0; i < runsFactor; i++) {
impl.toHex(blockRoot);
}
},
runsFactor,
});
bench({
id: "nodejs fromhex(blob)",
fn: () => {
for (let i = 0; i < runsFactor; i++) {
fromHex(blobHex);
}
},
});
bench({
id: `${name} block root to RootHex using toRootHex`,
fn: () => {
for (let i = 0; i < runsFactor; i++) {
impl.toRootHex(blockRoot);
}
},
runsFactor,
});
const buffer = Buffer.alloc(BLOB_LEN);
bench({
id: "nodejs fromHexInto(blob)",
fn: () => {
for (let i = 0; i < runsFactor; i++) {
nodeJsFromHexInto(blobHex, buffer);
}
},
});
bench({
id: `${name} fromHex(blob)`,
fn: () => {
for (let i = 0; i < runsFactor; i++) {
impl.fromHex(blobHex);
}
},
runsFactor,
});
bench({
id: "browser block root to RootHex using the deprecated toHexString",
fn: () => {
for (let i = 0; i < runsFactor; i++) {
toHexString(blockRoot);
}
},
runsFactor,
});
const buffer = new Uint8Array(BLOB_LEN);
bench({
id: `${name} fromHexInto(blob)`,
fn: () => {
for (let i = 0; i < runsFactor; i++) {
impl.fromHexInto(blobHex, buffer);
}
},
runsFactor,
});
bench({
id: "browser block root to RootHex using toHex",
fn: () => {
for (let i = 0; i < runsFactor; i++) {
browserToHex(blockRoot);
}
},
runsFactor,
});
bench({
id: "browser block root to RootHex using toRootHex",
fn: () => {
for (let i = 0; i < runsFactor; i++) {
browserToRootHex(blockRoot);
}
},
runsFactor,
});
const buf = new Uint8Array(BLOB_LEN);
bench({
id: "browser fromHexInto(blob)",
fn: () => {
for (let i = 0; i < runsFactor; i++) {
browserFromHexInto(blobHex, buf);
}
},
runsFactor,
});
bench({
id: "browser fromHex(blob)",
fn: () => {
for (let i = 0; i < runsFactor; i++) {
browserFromHex(blobHex);
}
},
});
bench({
id: `${name} block root to RootHex using the deprecated toHexString`,
fn: () => {
for (let i = 0; i < runsFactor; i++) {
impl.toHexString(blockRoot);
}
},
runsFactor,
});
}
});
/**
* this function is so slow compared to browser's implementation we only maintain it here to compare performance
* - nodejs fromHexInto(blob) 3.562495 ops/s 280.7022 ms/op - 10 runs 3.50 s
* - browser fromHexInto(blob) 535.0952 ops/s 1.868826 ms/op - 10 runs 20.8 s
*/
function nodeJsFromHexInto(hex: string, buffer: Buffer): void {
if (hex.startsWith("0x")) {
hex = hex.slice(2);
}
if (hex.length !== buffer.length * 2) {
throw new Error(`hex string length ${hex.length} must be exactly double the buffer length ${buffer.length}`);
}
buffer.write(hex, "hex");
}

1
packages/utils/tsconfig.build.json
View File

@@ -2,6 +2,7 @@
"extends": "../../tsconfig.build.json",
"include": ["src"],
"compilerOptions": {
"rootDir": "src",
"outDir": "lib"
}
}

1
packages/utils/tsconfig.json
View File

@@ -2,6 +2,7 @@
"extends": "../../tsconfig.json",
"include": ["src", "test"],
"compilerOptions": {
"rootDir": "../..",
"outDir": "lib"
}
}