github/lodestar
1
1
Fork 0
mirror of https://github.com/ChainSafe/lodestar.git synced 2026-01-09 15:48:08 -05:00
Code Issues Packages Projects Releases Wiki Activity

chore: various bun tweaks (#8449)

Browse Source 
**Motivation**

- #7280 

**Description**

- combined with #8448, `surpressTranspileTS: true` is needed for workers
to use typescript source directly without transpiling it (and
incorrectly loading it via a commonjs loader)
- avoid worker `resourceLimits` - Bun doesn't implement it, it throws if
you use it
- use `datastore-fs` instead of `datastore-level` for backing our libp2p
database -- Note this is changed unilaterally (affecting current nodejs
usage)
- use assemblyscript chacha20-poly1305 - bun doesn't support the native
crypto implementation
This commit is contained in:
Cayman
2025-09-23 07:49:36 -04:00
committed by GitHub
parent 6494939cd0
commit cec9bef1db
9 changed files with 90 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/beacon-node/package.json
View File

@@ -115,7 +115,7 @@
"@chainsafe/discv5": "^11.0.4",
"@chainsafe/enr": "^5.0.1",
"@chainsafe/libp2p-gossipsub": "^14.1.2",
"@chainsafe/libp2p-noise": "^16.1.0",
"@chainsafe/libp2p-noise": "^16.1.5",
"@chainsafe/persistent-merkle-tree": "^1.2.1",
"@chainsafe/prometheus-gc-stats": "^1.0.0",
"@chainsafe/pubkey-index-map": "^3.0.0",
@@ -150,7 +150,7 @@
"@lodestar/validator": "^1.34.1",
"@multiformats/multiaddr": "^12.1.3",
"datastore-core": "^10.0.2",
"datastore-level": "^11.0.3",
"datastore-fs": "^10.0.6",
"deepmerge": "^4.3.1",
"fastify": "^5.2.1",
"interface-datastore": "^8.3.0",

1
packages/beacon-node/src/chain/archiveStore/historicalState/historicalStateRegen.ts
View File

@@ -38,6 +38,7 @@ export class HistoricalStateRegen implements HistoricalStateWorkerApi {
};
const worker = new Worker(path.join(WORKER_DIR, "worker.js"), {
suppressTranspileTS: Boolean(globalThis.Bun),
workerData,
} as ConstructorParameters<typeof Worker>[1]);

1
packages/beacon-node/src/chain/bls/multithread/index.ts
View File

@@ -264,6 +264,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
for (let i = 0; i < poolSize; i++) {
const workerData: WorkerData = {workerId: i};
const worker = new Worker(path.join(workerDir, "worker.js"), {
suppressTranspileTS: Boolean(globalThis.Bun),
workerData,
} as ConstructorParameters<typeof Worker>[1]);

12
packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts
View File

@@ -130,8 +130,12 @@ export class WorkerNetworkCore implements INetworkCore {
loggerOpts: modules.logger.toOpts(),
};
const worker = new Worker(path.join(workerDir, "networkCoreWorker.js"), {
const workerOpts: ConstructorParameters<typeof Worker>[1] = {
workerData,
};
if (globalThis.Bun) {
workerOpts.suppressTranspileTS = true;
} else {
/**
* maxYoungGenerationSizeMb defaults to 152mb through the cli option defaults.
* That default value was determined via https://github.com/ChainSafe/lodestar/issues/2115 and
@@ -143,8 +147,10 @@ export class WorkerNetworkCore implements INetworkCore {
* showed that there is a pretty big window of "correct" values but we can always tune as
* necessary
*/
resourceLimits: {maxYoungGenerationSizeMb: opts.maxYoungGenerationSizeMb},
} as ConstructorParameters<typeof Worker>[1]);
workerOpts.resourceLimits = {maxYoungGenerationSizeMb: opts.maxYoungGenerationSizeMb};
}
const worker = new Worker(path.join(workerDir, "networkCoreWorker.js"), workerOpts);
// biome-ignore lint/suspicious/noExplicitAny: Don't know any specific interface for the spawn
const networkThreadApi = (await spawn<any>(worker, {

5
packages/beacon-node/src/network/discv5/index.ts
View File

@@ -51,7 +51,10 @@ export class Discv5Worker extends (EventEmitter as {new (): StrictEventEmitter<E
loggerOpts: opts.logger.toOpts(),
genesisTime: opts.genesisTime,
};
const worker = new Worker("./worker.js", {workerData} as ConstructorParameters<typeof Worker>[1]);
const worker = new Worker("./worker.js", {
suppressTranspileTS: Boolean(globalThis.Bun),
workerData,
} as ConstructorParameters<typeof Worker>[1]);
const workerApi = await spawn<Discv5WorkerApi>(worker, {
// A Lodestar Node may do very expensive task at start blocking the event loop and causing

11
packages/beacon-node/src/network/libp2p/index.ts
View File

@@ -9,6 +9,7 @@ import {createLibp2p} from "libp2p";
import {Registry} from "prom-client";
import {ENR} from "@chainsafe/enr";
import {noise} from "@chainsafe/libp2p-noise";
import {asCrypto, defaultCrypto} from "@chainsafe/libp2p-noise/crypto";
import {Libp2p, LodestarComponents} from "../interface.js";
import {NetworkOptions, defaultNetworkOptions} from "../options.js";
import {Eth2PeerDataStore} from "../peers/datastore.js";
@@ -63,13 +64,21 @@ export async function createNodeJsLibp2p(
}
}
const noiseCrypto = {
...defaultCrypto,
};
if (globalThis.Bun) {
noiseCrypto.chaCha20Poly1305Decrypt = asCrypto.chaCha20Poly1305Decrypt;
noiseCrypto.chaCha20Poly1305Encrypt = asCrypto.chaCha20Poly1305Encrypt;
}
return createLibp2p({
privateKey,
addresses: {
listen: localMultiaddrs,
announce: [],
},
connectionEncrypters: [noise()],
connectionEncrypters: [noise({crypto: noiseCrypto})],
// Reject connections when the server's connection count gets high
transports: [
tcp({

8
packages/beacon-node/src/network/peers/datastore.ts
View File

@@ -1,6 +1,6 @@
import {AbortOptions} from "@libp2p/interface";
import {BaseDatastore} from "datastore-core";
import {LevelDatastore} from "datastore-level";
import {FsDatastore} from "datastore-fs";
import {Key, KeyQuery, Pair, Query} from "interface-datastore";
type MemoryItem = {
@@ -22,7 +22,7 @@ type MemoryItem = {
* - Update lastAccessedMs
*/
export class Eth2PeerDataStore extends BaseDatastore {
private _dbDatastore: LevelDatastore;
private _dbDatastore: FsDatastore;
private _memoryDatastore: Map<string, MemoryItem>;
/** Same to PersistentPeerStore of the old libp2p implementation */
private _dirtyItems = new Set<string>();
@@ -32,7 +32,7 @@ export class Eth2PeerDataStore extends BaseDatastore {
private _maxMemoryItems: number;
constructor(
dbDatastore: LevelDatastore | string,
dbDatastore: FsDatastore | string,
{threshold = 5, maxMemoryItems = 50}: {threshold?: number | undefined; maxMemoryItems?: number | undefined} = {}
) {
super();
@@ -44,7 +44,7 @@ export class Eth2PeerDataStore extends BaseDatastore {
throw Error(`Threshold ${threshold} should be at most maxMemoryItems ${maxMemoryItems}`);
}
this._dbDatastore = typeof dbDatastore === "string" ? new LevelDatastore(dbDatastore) : dbDatastore;
this._dbDatastore = typeof dbDatastore === "string" ? new FsDatastore(dbDatastore) : dbDatastore;
this._memoryDatastore = new Map();
this._threshold = threshold;
this._maxMemoryItems = maxMemoryItems;

8
packages/beacon-node/test/unit/network/peers/datastore.test.ts
View File

@@ -1,18 +1,18 @@
import {LevelDatastore} from "datastore-level";
import {FsDatastore} from "datastore-fs";
import {Key} from "interface-datastore";
import {MockedObject, afterEach, beforeEach, describe, expect, it, vi} from "vitest";
import {Eth2PeerDataStore} from "../../../../src/network/peers/datastore.js";
vi.mock("datastore-level");
vi.mock("datastore-fs");
describe("Eth2PeerDataStore", () => {
let eth2Datastore: Eth2PeerDataStore;
let dbDatastoreStub: MockedObject<LevelDatastore>;
let dbDatastoreStub: MockedObject<FsDatastore>;
beforeEach(() => {
vi.useFakeTimers({now: Date.now()});
dbDatastoreStub = vi.mocked(new LevelDatastore({} as any));
dbDatastoreStub = vi.mocked(new FsDatastore({} as any));
eth2Datastore = new Eth2PeerDataStore(dbDatastoreStub, {threshold: 2, maxMemoryItems: 3});
vi.spyOn(dbDatastoreStub, "put").mockResolvedValue({} as any);

112
yarn.lock
View File

@@ -659,10 +659,10 @@
uint8arraylist "^2.4.8"
uint8arrays "^5.0.1"
"@chainsafe/libp2p-noise@^16.1.0":
version "16.1.4"
resolved "https://registry.yarnpkg.com/@chainsafe/libp2p-noise/-/libp2p-noise-16.1.4.tgz#6788c1ad3e2567f37189e552fc77e67c57f88505"
integrity sha512-f4FlyRVndcs4PoioOIZWrFc6wfO/mrAj7H63o0+eA0O2xhcoRkxHh6zna4W+WtScaF/Ua/UULgiNGuKNpLvLlQ==
"@chainsafe/libp2p-noise@^16.1.5":
version "16.1.5"
resolved "https://registry.yarnpkg.com/@chainsafe/libp2p-noise/-/libp2p-noise-16.1.5.tgz#f846fcd9ca94d75450fbaba4d0b9ccbbb3473b68"
integrity sha512-yac0bknwfuYdXXOAFGVL4fYR0de0p1Uk7W0gIkuxlj8JSqhkHr+kEt3958PfnfmX880QJAIogsh/ZupFjg0uJA==
dependencies:
"@chainsafe/as-chacha20poly1305" "^0.1.0"
"@chainsafe/as-sha256" "^1.0.0"
@@ -3973,19 +3973,6 @@ abstract-level@^1.0.0, abstract-level@^1.0.2, abstract-level@^1.0.3:
module-error "^1.0.1"
queue-microtask "^1.2.3"
abstract-level@^1.0.4:
version "1.0.4"
resolved "https://registry.yarnpkg.com/abstract-level/-/abstract-level-1.0.4.tgz#3ad8d684c51cc9cbc9cf9612a7100b716c414b57"
integrity sha512-eUP/6pbXBkMbXFdx4IH2fVgvB7M0JvR7/lIL33zcs0IBcwjdzSSl31TOJsaCzmKSSDF9h8QYSOJux4Nd4YJqFg==
dependencies:
buffer "^6.0.3"
catering "^2.1.0"
is-buffer "^2.0.5"
level-supports "^4.0.0"
level-transcoder "^1.0.1"
module-error "^1.0.1"
queue-microtask "^1.2.3"
abstract-logging@^2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/abstract-logging/-/abstract-logging-2.0.1.tgz#6b0c371df212db7129b57d2e7fcf282b8bf1c839"
@@ -5590,20 +5577,19 @@ datastore-core@^10.0.0, datastore-core@^10.0.2:
it-sort "^3.0.6"
it-take "^3.0.6"
datastore-level@^11.0.3:
version "11.0.3"
resolved "https://registry.yarnpkg.com/datastore-level/-/datastore-level-11.0.3.tgz#b8dff3d0bede396629bb0e6505787b5952673e3f"
integrity sha512-MU0DhzRsJqlNUf1Z1zap+4AGY+fWhKjvFJU1W50+jJ12x+g1e/yOkdS3aGYHal9LBvYEJF+/TSPvMxnC8Gxnkw==
datastore-fs@^10.0.6:
version "10.0.6"
resolved "https://registry.yarnpkg.com/datastore-fs/-/datastore-fs-10.0.6.tgz#70cca661673c4be2f11cfaa947498d4bd457d9ec"
integrity sha512-809BIs3wxJMg8/BYDwm734c06oKfrP5svD5CahPPHOIOOk9fSJc3j0OyP1yrb+oUk1OU/V5m1kmeO+eU0tHNCQ==
dependencies:
datastore-core "^10.0.0"
interface-datastore "^8.0.0"
interface-store "^6.0.0"
it-filter "^3.1.3"
it-glob "^3.0.3"
it-map "^3.1.3"
it-sort "^3.0.8"
it-take "^3.0.8"
level "^8.0.1"
race-signal "^1.1.3"
it-parallel-batch "^3.0.8"
race-signal "^2.0.0"
steno "^4.0.2"
dateformat@^3.0.3:
version "3.0.3"
@@ -6438,6 +6424,17 @@ fast-glob@^3.2.9:
merge2 "^1.3.0"
micromatch "^4.0.4"
fast-glob@^3.3.3:
version "3.3.3"
resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.3.3.tgz#d06d585ce8dba90a16b0505c543c3ccfb3aeb818"
integrity sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==
dependencies:
"@nodelib/fs.stat" "^2.0.2"
"@nodelib/fs.walk" "^1.2.3"
glob-parent "^5.1.2"
merge2 "^1.3.0"
micromatch "^4.0.8"
fast-json-stringify@^6.0.0:
version "6.0.0"
resolved "https://registry.yarnpkg.com/fast-json-stringify/-/fast-json-stringify-6.0.0.tgz#15c5e85b567ead695773bf55938b56aaaa57d805"
@@ -7988,6 +7985,11 @@ it-all@^3.0.8:
resolved "https://registry.yarnpkg.com/it-all/-/it-all-3.0.9.tgz#9b9b54ddb42786260c3d9e25feeaa02e667be1cc"
integrity sha512-fz1oJJ36ciGnu2LntAlE6SA97bFZpW7Rnt0uEc1yazzR2nKokZLr8lIRtgnpex4NsmaBcvHF+Z9krljWFy/mmg==
it-batch@^3.0.0:
version "3.0.9"
resolved "https://registry.yarnpkg.com/it-batch/-/it-batch-3.0.9.tgz#d0e3b57403908d3ea14a44ec2b40026b0c44e709"
integrity sha512-z6p89Q8gm2urBtF3JcpnbJogacijWk3m1uc3xZYI3x0eJUoYLUbgF8IxJ2fnuVObV7yRv3SixfwGCufaZY1NCg==
it-byte-stream@^2.0.0, it-byte-stream@^2.0.2:
version "2.0.3"
resolved "https://registry.yarnpkg.com/it-byte-stream/-/it-byte-stream-2.0.3.tgz#42bfac48d918d865113aa5838d7463b250752405"
@@ -8011,13 +8013,6 @@ it-filter@^3.1.1:
dependencies:
it-peekable "^3.0.0"
it-filter@^3.1.3:
version "3.1.4"
resolved "https://registry.yarnpkg.com/it-filter/-/it-filter-3.1.4.tgz#bcbeb74edd45c6b8d522e6581edf8a4c0bbb02af"
integrity sha512-80kWEKgiFEa4fEYD3mwf2uygo1dTQ5Y5midKtL89iXyjinruA/sNXl6iFkTcdNedydjvIsFhWLiqRPQP4fAwWQ==
dependencies:
it-peekable "^3.0.0"
it-foreach@^2.1.3:
version "2.1.4"
resolved "https://registry.yarnpkg.com/it-foreach/-/it-foreach-2.1.4.tgz#f7295feefe40b47569863b34271efc3682f62708"
@@ -8025,6 +8020,13 @@ it-foreach@^2.1.3:
dependencies:
it-peekable "^3.0.0"
it-glob@^3.0.3:
version "3.0.4"
resolved "https://registry.yarnpkg.com/it-glob/-/it-glob-3.0.4.tgz#6d008a45d54573541bcd8ead0b2dac7cef907ddd"
integrity sha512-73PbGBTK/dHp5PX4l8pkQH1ozCONP0U+PB3qMqltxPonRJQNomINE3Hn9p02m2GOu95VoeVvSZdHI2N+qub0pw==
dependencies:
fast-glob "^3.3.3"
it-length-prefixed-stream@^2.0.0, it-length-prefixed-stream@^2.0.1, it-length-prefixed-stream@^2.0.2:
version "2.0.3"
resolved "https://registry.yarnpkg.com/it-length-prefixed-stream/-/it-length-prefixed-stream-2.0.3.tgz#8020d64e2dc7fea0999f9f4eb95455e43749f49b"
@@ -8087,6 +8089,13 @@ it-pair@^2.0.6:
it-stream-types "^2.0.1"
p-defer "^4.0.0"
it-parallel-batch@^3.0.8:
version "3.0.9"
resolved "https://registry.yarnpkg.com/it-parallel-batch/-/it-parallel-batch-3.0.9.tgz#4fcaa8328e2b534471d8dceec907fb6325673b01"
integrity sha512-TszXWqqLG8IG5DUEnC4cgH9aZI6CsGS7sdkXTiiacMIj913bFy7+ohU3IqsFURCcZkpnXtNLNzrYnXISsKBhbQ==
dependencies:
it-batch "^3.0.0"
it-parallel@^3.0.11:
version "3.0.13"
resolved "https://registry.yarnpkg.com/it-parallel/-/it-parallel-3.0.13.tgz#bc08878b76d384e7099e3ab7058127261449def5"
@@ -8160,13 +8169,6 @@ it-sort@^3.0.6:
dependencies:
it-all "^3.0.0"
it-sort@^3.0.8:
version "3.0.9"
resolved "https://registry.yarnpkg.com/it-sort/-/it-sort-3.0.9.tgz#2a1ffca46c1a947b1f230f6249a2e5484bd9c2cf"
integrity sha512-jsM6alGaPiQbcAJdzMsuMh00uJcI+kD9TBoScB8TR75zUFOmHvhSsPi+Dmh2zfVkcoca+14EbfeIZZXTUGH63w==
dependencies:
it-all "^3.0.0"
it-stream-types@^2.0.1, it-stream-types@^2.0.2:
version "2.0.2"
resolved "https://registry.yarnpkg.com/it-stream-types/-/it-stream-types-2.0.2.tgz#60bbace90096796b4e6cc3bfab99cf9f2b86c152"
@@ -8177,11 +8179,6 @@ it-take@^3.0.6:
resolved "https://registry.yarnpkg.com/it-take/-/it-take-3.0.6.tgz#509283b69b88f823350b256392525267609f1925"
integrity sha512-uqw3MRzf9to1SOLxaureGa73lK8k8ZB/asOApTAkvrzUqCznGtKNgPFH7uYIWlt4UuWq/hU6I+U4Fm5xpjN8Vg==
it-take@^3.0.8:
version "3.0.9"
resolved "https://registry.yarnpkg.com/it-take/-/it-take-3.0.9.tgz#50c99ef24991f87bc4351a9a537db292e7facfe2"
integrity sha512-XMeUbnjOcgrhFXPUqa7H0VIjYSV/BvyxxjCp76QHVAFDJw2LmR1SHxUFiqyGeobgzJr7P2ZwSRRJQGn4D2BVlA==
jackspeak@^2.0.3:
version "2.3.3"
resolved "https://registry.yarnpkg.com/jackspeak/-/jackspeak-2.3.3.tgz#95e4cbcc03b3eb357bf6bcce14a903fb3d1151e1"
@@ -8534,15 +8531,6 @@ level@^8.0.0:
browser-level "^1.0.1"
classic-level "^1.2.0"
level@^8.0.1:
version "8.0.1"
resolved "https://registry.yarnpkg.com/level/-/level-8.0.1.tgz#737161db1bc317193aca4e7b6f436e7e1df64379"
integrity sha512-oPBGkheysuw7DmzFQYyFe8NAia5jFLAgEnkgWnK3OXAuJr8qFT+xBQIwokAZPME2bhPFzS8hlYcL16m8UZrtwQ==
dependencies:
abstract-level "^1.0.4"
browser-level "^1.0.1"
classic-level "^1.2.0"
libnpmaccess@7.0.2:
version "7.0.2"
resolved "https://registry.yarnpkg.com/libnpmaccess/-/libnpmaccess-7.0.2.tgz#7f056c8c933dd9c8ba771fa6493556b53c5aac52"
@@ -9438,9 +9426,9 @@ nan@^2.16.0, nan@^2.17.0:
integrity sha512-bk3gXBZDGILuuo/6sKtr0DQmSThYHLtNCdSdXk9YkxD/jK6X2vmCyyXBBxyqZ4XcnzTyYEAThfX3DCEnLf6igw==
nan@^2.19.0:
version "2.22.2"
resolved "https://registry.yarnpkg.com/nan/-/nan-2.22.2.tgz#6b504fd029fb8f38c0990e52ad5c26772fdacfbb"
integrity sha512-DANghxFkS1plDdRsX0X9pm0Z6SJNN6gBdtXfanwoZ8hooC5gosGFSBGRYHUVPz1asKA/kMRqDRdHrluZ61SpBQ==
version "2.23.0"
resolved "https://registry.yarnpkg.com/nan/-/nan-2.23.0.tgz#24aa4ddffcc37613a2d2935b97683c1ec96093c6"
integrity sha512-1UxuyYGdoQHcGg87Lkqm3FzefucTa0NAiOcuRsDmysep3c1LVCRK2krrUDafMWtjSG04htvAmvg96+SDknOmgQ==
nanoid@^3.3.8:
version "3.3.8"
@@ -10747,6 +10735,11 @@ race-signal@^1.1.3:
resolved "https://registry.yarnpkg.com/race-signal/-/race-signal-1.1.3.tgz#688c117d626161abfd5ee6d9b5d84bd59df54ee5"
integrity sha512-Mt2NznMgepLfORijhQMncE26IhkmjEphig+/1fKC0OtaKwys/gpvpmswSjoN01SS+VO951mj0L4VIDXdXsjnfA==
race-signal@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/race-signal/-/race-signal-2.0.0.tgz#979d58493098ad6a6b556ef5b07b897d253d6b0f"
integrity sha512-P31bLhE4ByBX/70QDXMutxnqgwrF1WUXea1O8DXuviAgkdbQ1iQMQotNgzJIBC9yUSn08u/acZrMUhgw7w6GpA==
randombytes@^2.0.0, randombytes@^2.0.1, randombytes@^2.0.5:
version "2.1.0"
resolved "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz"
@@ -11669,6 +11662,11 @@ stdin-discarder@^0.1.0:
dependencies:
bl "^5.0.0"
steno@^4.0.2:
version "4.0.2"
resolved "https://registry.yarnpkg.com/steno/-/steno-4.0.2.tgz#9bd9b0ffc226a1f9436f29132c8b8e7199d22c50"
integrity sha512-yhPIQXjrlt1xv7dyPQg2P17URmXbuM5pdGkpiMB3RenprfiBlvK415Lctfe0eshk90oA7/tNq7WEiMK8RSP39A==
stream-browserify@^2.0.1:
version "2.0.2"
resolved "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.2.tgz"