github/lodestar
1
1
Fork 0
mirror of https://github.com/ChainSafe/lodestar.git synced 2026-01-10 08:08:16 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add assertions to sim tests (#4510)

Browse Source 
* Add support for run simulation env in child process

* Update the simulation node runner

* Add assertions placeholders

* Update assertions for single node

* Update simulation env for the multiple nodes

* Updaate the assertions for single node

* Updat asssertions for multiple nodes

* Add a simulation test for four nodes

* Add --dev option for testing environment

* Fix linting errors

* Update code as per feedback

* Update the execution.engineMock to be optional

* Add fork detection test

* Update forks tests

* Update the assertions to split in multiple files

* Add sim tests to the pipeline

* Fix lint error

* Update the simulation log format

* Add simulation tracker

* Skip the multi node test for now

* Updte the assertions style

* Update tests structure to run in sequence

* Fix the issues after rebase

* Update the export

* Improve the stats printing

* Update the logging for the sim tests

* Update the logs to hide rc config

* Update the usage of far future epoch value

* Update the fork tests to check the finality

* Add external signer support

* Skip the multi-node test

* Update the assertions style

* Update the node tests structure

* Update the nodes numbers

* Update the logging for childprocess

* Remove existing sim tests for same scenarios

* Update the logging for the process loading

* Remove duplicate sim tests

* Increase genesis delay to not miss slots on CI server

* Update simulation id

* Update acceptableParticipationRate to 70%

* Update the calculation for calcultating participation

* Update the event wait logic for multiple nodes

* Remove the extra log

* Fix forkchoice bug for low pivot slot

* Update the simulation env trackers

* Fix multi node key index

* Update simulations to use longer epochs

* Revert unused changes

* Update assertion messages

* Revert change for minimal param preset

* Update test description

* Remove unused code
This commit is contained in:
Nazar Hussain
2022-09-19 23:47:40 +02:00
committed by GitHub
parent 31c6f57c2e
commit 31f220e321
20 changed files with 1255 additions and 289 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.dockerignore
View File

@@ -35,6 +35,7 @@ packages/**/lib
packages/*/spec-tests
packages/*/benchmark_data
packages/beacon-node/test-logs/
packages/cli/test-logs/
packages/state-transition/test-cache
benchmark_data
invalidSszObjects/

19
.github/workflows/test-sim.yml vendored
View File

@@ -36,12 +36,9 @@ jobs:
if: steps.cache-deps.outputs.cache-hit == 'true'
# </common-build>
- name: Simulation single thread single node test
run: yarn test:sim:singleThread
working-directory: packages/beacon-node
- name: Simulation single thread multi node test
run: yarn test:sim:singleThreadMultiNode
working-directory: packages/beacon-node
- name: Simulation tests for CLI
run: yarn test:sim
working-directory: packages/cli
# - name: Simulation multi thread multi node test phase0
# run: yarn test:sim:multiThread
# working-directory: packages/beacon-node
@@ -54,9 +51,15 @@ jobs:
# run: yarn test:sim:multiThread
# working-directory: packages/beacon-node
# env: {RUN_ONLY_SIM_TEST: altair-epoch2}
- name: Upload debug log test files
- name: Upload debug log test files for "packages/beacon-node"
if: ${{ always() }}
uses: actions/upload-artifact@v2
with:
name: debug-test-logs
name: debug-test-logs-beacon-node
path: packages/beacon-node/test-logs
- name: Upload debug log test files for "packages/cli"
if: ${{ always() }}
uses: actions/upload-artifact@v2
with:
name: debug-test-logs-cli
path: packages/cli/test-logs

1
.gitignore vendored
View File

@@ -29,6 +29,7 @@ validators
packages/*/spec-tests
packages/*/benchmark_data
packages/beacon-node/test-logs/
packages/cli/test-logs/
packages/state-transition/test-cache
benchmark_data
invalidSszObjects/

2
packages/beacon-node/package.json
View File

@@ -86,8 +86,6 @@
"test:unit": "yarn test:unit:minimal && yarn test:unit:mainnet",
"test:e2e": "mocha 'test/e2e/**/*.test.ts'",
"test:sim": "mocha 'test/sim/**/*.test.ts'",
"test:sim:singleThread": "mocha 'test/sim/singleNodeSingleThread.test.ts'",
"test:sim:singleThreadMultiNode": "mocha 'test/sim/multiNodeSingleThread.test.ts'",
"test:sim:multiThread": "mocha 'test/sim/multiNodeMultiThread.test.ts'",
"test:sim:merge-interop": "mocha 'test/sim/merge-interop.test.ts'",
"download-spec-tests": "node --loader=ts-node/esm test/spec/downloadTests.ts",

131
packages/beacon-node/test/sim/multiNodeSingleThread.test.ts
View File

@@ -1,131 +0,0 @@
import {IChainConfig} from "@lodestar/config";
import {phase0} from "@lodestar/types";
import {Validator} from "@lodestar/validator/lib";
import {ILogger, sleep, TimestampFormatCode} from "@lodestar/utils";
import {SLOTS_PER_EPOCH} from "@lodestar/params";
import {BeaconNode} from "../../src/index.js";
import {getDevBeaconNode} from "../utils/node/beacon.js";
import {waitForEvent} from "../utils/events/resolver.js";
import {getAndInitDevValidators} from "../utils/node/validator.js";
import {ChainEvent} from "../../src/chain/index.js";
import {testLogger, LogLevel, TestLoggerOpts} from "../utils/logger.js";
import {connect} from "../utils/network.js";
import {simTestInfoTracker} from "../utils/node/simTest.js";
import {logFilesDir} from "./params.js";
/* eslint-disable no-console, @typescript-eslint/naming-convention */
describe("Run multi node single thread interop validators (no eth1) until checkpoint", function () {
const testParams: Pick<IChainConfig, "SECONDS_PER_SLOT"> = {
SECONDS_PER_SLOT: 3,
};
const testCases: {
nodeCount: number;
validatorsPerNode: number;
event: ChainEvent.justified | ChainEvent.finalized;
altairForkEpoch: number;
}[] = [
// Test phase0 to justification
{nodeCount: 4, validatorsPerNode: 32, event: ChainEvent.justified, altairForkEpoch: Infinity},
// Test altair only
{nodeCount: 4, validatorsPerNode: 32, event: ChainEvent.justified, altairForkEpoch: 0},
// Test phase -> altair fork transition
{nodeCount: 4, validatorsPerNode: 32, event: ChainEvent.justified, altairForkEpoch: 2},
];
const afterEachCallbacks: (() => Promise<unknown> | void)[] = [];
afterEach("Stop nodes and validators", async function () {
this.timeout("10 min");
while (afterEachCallbacks.length > 0) {
const callback = afterEachCallbacks.pop();
if (callback) await callback();
}
});
// TODO test multiNode with remote;
for (const {nodeCount, validatorsPerNode, event, altairForkEpoch} of testCases) {
it(`singleThread ${nodeCount} nodes / ${validatorsPerNode} vc / 1 validator > until ${event}, altairForkEpoch ${altairForkEpoch}`, async function () {
this.timeout("10 min");
const nodes: BeaconNode[] = [];
const validators: Validator[] = [];
const loggers: ILogger[] = [];
// delay a bit so regular sync sees it's up to date and sync is completed from the beginning
const genesisSlotsDelay = 30;
const genesisTime = Math.floor(Date.now() / 1000) + genesisSlotsDelay * testParams.SECONDS_PER_SLOT;
for (let i = 0; i < nodeCount; i++) {
const testLoggerOpts: TestLoggerOpts = {
logLevel: LogLevel.info,
logFile: `${logFilesDir}/singlethread_multinode_altair-${altairForkEpoch}.log`,
timestampFormat: {
format: TimestampFormatCode.EpochSlot,
genesisTime,
slotsPerEpoch: SLOTS_PER_EPOCH,
secondsPerSlot: testParams.SECONDS_PER_SLOT,
},
};
const logger = testLogger(`Node ${i}`, testLoggerOpts);
const bn = await getDevBeaconNode({
params: {...testParams, ALTAIR_FORK_EPOCH: altairForkEpoch},
options: {api: {rest: {port: 10000 + i}}},
validatorCount: nodeCount * validatorsPerNode,
genesisTime,
logger,
});
const {validators: nodeValidators} = await getAndInitDevValidators({
node: bn,
validatorsPerClient: validatorsPerNode,
validatorClientCount: 1,
startIndex: i * validatorsPerNode,
testLoggerOpts,
});
afterEachCallbacks.push(async () => await Promise.all(validators.map((validator) => validator.close())));
afterEachCallbacks.push(async () => {
await Promise.all(validators.map((validator) => validator.close()));
console.log("--- Stopped all validators ---");
// wait for 1 slot
await sleep(1 * testParams.SECONDS_PER_SLOT * 1000);
stopInfoTracker();
await Promise.all(nodes.map((node) => node.close()));
console.log("--- Stopped all nodes ---");
// Wait a bit for nodes to shutdown
await sleep(3000);
});
loggers.push(logger);
nodes.push(bn);
validators.push(...nodeValidators);
}
const stopInfoTracker = simTestInfoTracker(nodes[0], loggers[0]);
afterEachCallbacks.push(async () => {
stopInfoTracker();
await Promise.all(nodes.map((node) => node.close()));
console.log("--- Stopped all nodes ---");
// Wait a bit for nodes to shutdown
await sleep(3000);
});
// Connect all nodes with each other
for (let i = 0; i < nodeCount; i++) {
for (let j = 0; j < nodeCount; j++) {
if (i !== j) {
await connect(nodes[i].network, nodes[j].network.peerId, nodes[j].network.localMultiaddrs);
}
}
}
// Wait for justified checkpoint on all nodes
await Promise.all(nodes.map((node) => waitForEvent<phase0.Checkpoint>(node.chain.emitter, event, 240000)));
console.log("--- All nodes reached justified checkpoint ---");
});
}
});

148
packages/beacon-node/test/sim/singleNodeSingleThread.test.ts
View File

@@ -1,148 +0,0 @@
import {SLOTS_PER_EPOCH} from "@lodestar/params";
import {phase0} from "@lodestar/types";
import {toHexString} from "@chainsafe/ssz";
import {sleep, TimestampFormatCode} from "@lodestar/utils";
import {IChainConfig} from "@lodestar/config";
import {getDevBeaconNode} from "../utils/node/beacon.js";
import {waitForEvent} from "../utils/events/resolver.js";
import {getAndInitDevValidators} from "../utils/node/validator.js";
import {ChainEvent} from "../../src/chain/index.js";
import {BeaconRestApiServerOpts} from "../../src/api/rest/index.js";
import {testLogger, TestLoggerOpts, LogLevel} from "../utils/logger.js";
import {simTestInfoTracker} from "../utils/node/simTest.js";
import {INTEROP_BLOCK_HASH} from "../../src/node/utils/interop/state.js";
import {createExternalSignerServer} from "../../../validator/test/utils/createExternalSignerServer.js";
import {logFilesDir} from "./params.js";
/* eslint-disable no-console, @typescript-eslint/naming-convention */
describe("Run single node single thread interop validators (no eth1) until checkpoint", function () {
const testParams: Pick<IChainConfig, "SECONDS_PER_SLOT"> = {
SECONDS_PER_SLOT: 2,
};
const validatorClientCount = 1;
const validatorsPerClient = 32 * 4;
const testCases: {
event: ChainEvent.justified | ChainEvent.finalized;
altairEpoch: number;
bellatrixEpoch: number;
withExternalSigner?: boolean;
}[] = [
// phase0 fork only
{event: ChainEvent.finalized, altairEpoch: Infinity, bellatrixEpoch: Infinity},
// altair fork only
{event: ChainEvent.finalized, altairEpoch: 0, bellatrixEpoch: Infinity},
// altair fork at epoch 2
{event: ChainEvent.finalized, altairEpoch: 2, bellatrixEpoch: Infinity},
// bellatrix fork at epoch 0
{event: ChainEvent.finalized, altairEpoch: 0, bellatrixEpoch: 0},
// Remote signer with altair
{event: ChainEvent.justified, altairEpoch: 0, bellatrixEpoch: Infinity, withExternalSigner: true},
];
const afterEachCallbacks: (() => Promise<unknown> | unknown)[] = [];
afterEach(async () => {
// Run the afterEachCallbacks in a specific order decided latter in the test
for (let i = 0; i < afterEachCallbacks.length; i++) {
await afterEachCallbacks[i]?.();
}
afterEachCallbacks.length = 0;
});
for (const {event, altairEpoch, bellatrixEpoch, withExternalSigner} of testCases) {
const testIdStr = [
`altair-${altairEpoch}`,
`bellatrix-${bellatrixEpoch}`,
`vc-${validatorClientCount}`,
`vPc-${validatorsPerClient}`,
withExternalSigner ? "external_signer" : "local_signer",
`event-${event}`,
].join("_");
it(`singleNode ${testIdStr}`, async function () {
// Should reach justification in 3 epochs max, and finalization in 4 epochs max
const expectedEpochsToFinish = event === ChainEvent.justified ? 3 : 4;
// 1 epoch of margin of error
const epochsOfMargin = 1;
const timeoutSetupMargin = 5 * 1000; // Give extra 5 seconds of margin
// delay a bit so regular sync sees it's up to date and sync is completed from the beginning
// allow time for bls worker threads to warm up
const genesisSlotsDelay = 20;
const timeout =
((epochsOfMargin + expectedEpochsToFinish) * SLOTS_PER_EPOCH + genesisSlotsDelay) *
testParams.SECONDS_PER_SLOT *
1000;
this.timeout(timeout + 2 * timeoutSetupMargin);
const genesisTime = Math.floor(Date.now() / 1000) + genesisSlotsDelay * testParams.SECONDS_PER_SLOT;
const testLoggerOpts: TestLoggerOpts = {
logLevel: LogLevel.info,
logFile: `${logFilesDir}/singlethread_singlenode_${testIdStr}.log`,
timestampFormat: {
format: TimestampFormatCode.EpochSlot,
genesisTime,
slotsPerEpoch: SLOTS_PER_EPOCH,
secondsPerSlot: testParams.SECONDS_PER_SLOT,
},
};
const loggerNodeA = testLogger("Node-A", testLoggerOpts);
const bn = await getDevBeaconNode({
params: {...testParams, ALTAIR_FORK_EPOCH: altairEpoch, BELLATRIX_FORK_EPOCH: bellatrixEpoch},
options: {
api: {rest: {enabled: true} as BeaconRestApiServerOpts},
sync: {isSingleNode: true},
network: {allowPublishToZeroPeers: true},
executionEngine: {mode: "mock", genesisBlockHash: toHexString(INTEROP_BLOCK_HASH)},
},
validatorCount: validatorClientCount * validatorsPerClient,
logger: loggerNodeA,
genesisTime,
});
afterEachCallbacks[3] = () => bn.close();
const stopInfoTracker = simTestInfoTracker(bn, loggerNodeA);
afterEachCallbacks[2] = () => stopInfoTracker();
const externalSignerPort = 38000;
const externalSignerUrl = `http://localhost:${externalSignerPort}`;
const {validators, secretKeys} = await getAndInitDevValidators({
node: bn,
validatorsPerClient,
validatorClientCount,
startIndex: 0,
// At least one sim test must use the REST API for beacon <-> validator comms
useRestApi: true,
testLoggerOpts,
externalSignerUrl: withExternalSigner ? externalSignerUrl : undefined,
});
if (withExternalSigner) {
const server = createExternalSignerServer(secretKeys);
afterEachCallbacks[0] = () => server.close();
await server.listen(externalSignerPort);
}
// TODO: Previous code waited for 1 slot between stopping the validators and stopInfoTracker()
afterEachCallbacks[1] = () => Promise.all(validators.map((v) => v.close()));
// Wait for test to complete
await waitForEvent<phase0.Checkpoint>(bn.chain.emitter, event, timeout);
console.log(`\nGot event ${event}, stopping validators and nodes\n`);
// wait for 1 slot
await sleep(1 * bn.config.SECONDS_PER_SLOT * 1000);
console.log("\n\nDone\n\n");
await sleep(1000);
});
}
});

1
packages/cli/package.json
View File

@@ -34,6 +34,7 @@
"pretest": "yarn run check-types",
"test:unit": "nyc --cache-dir .nyc_output/.cache -e .ts mocha 'test/unit/**/*.test.ts'",
"test:e2e": "mocha --timeout 30000 'test/e2e/**/*.test.ts'",
"test:sim": "LODESTAR_PRESET=minimal mocha 'test/simulation/**/*.test.ts'",
"test": "yarn test:unit && yarn test:e2e",
"coverage": "codecov -F lodestar",
"check-readme": "typescript-docs-verifier"

15
packages/cli/src/options/beaconNodeOptions/execution.ts
View File

@@ -7,10 +7,18 @@ export type ExecutionEngineArgs = {
"execution.timeout": number;
"execution.retryAttempts": number;
"execution.retryDelay": number;
"execution.engineMock"?: boolean;
"jwt-secret"?: string;
};
export function parseArgs(args: ExecutionEngineArgs): IBeaconNodeOptions["executionEngine"] {
if (args["execution.engineMock"]) {
return {
mode: "mock",
genesisBlockHash: "",
};
}
return {
urls: args["execution.urls"],
timeout: args["execution.timeout"],
@@ -59,6 +67,13 @@ export const options: ICliCommandOptions<ExecutionEngineArgs> = {
group: "execution",
},
"execution.engineMock": {
description: "Set the execution engine to mock mode",
type: "boolean",
hidden: true,
group: "execution",
},
"jwt-secret": {
description:
"File path to a shared hex-encoded jwt secret which will be used to generate and bundle HS256 encoded jwt tokens for authentication with the EL client's rpc server hosting engine apis. Secret to be exactly same as the one used by the corresponding EL client.",

109
packages/cli/test/simulation/simulation.test.ts Normal file
View File

@@ -0,0 +1,109 @@
import {join} from "node:path";
import chai from "chai";
import chaiAsPromised from "chai-as-promised";
import {Epoch} from "@lodestar/types";
import {logFilesDir, SimulationEnvironment} from "../utils/simulation/index.js";
import {
missedBlocksAssertions,
attestationParticipationAssertions,
nodeAssertions,
} from "../utils/simulation/assertions.js";
chai.use(chaiAsPromised);
const nodeCases: {beaconNodes: number; validatorClients: number; validatorsPerClient: number}[] = [
{beaconNodes: 4, validatorClients: 1, validatorsPerClient: 32},
];
const forksCases: {
title: string;
params: {
altairEpoch: number;
bellatrixEpoch: number;
withExternalSigner?: boolean;
runTill: Epoch;
};
}[] = [
{
title: "mixed forks",
params: {altairEpoch: 2, bellatrixEpoch: 4, runTill: 6},
},
// {
// title: "mixed forks with remote signer",
// params: {altairEpoch: 2, bellatrixEpoch: 4, withExternalSigner: true, runTill: 6},
// },
];
for (const {beaconNodes, validatorClients, validatorsPerClient} of nodeCases) {
for (const {
title,
params: {altairEpoch, bellatrixEpoch, withExternalSigner, runTill},
} of forksCases) {
const testIdStr = [
`beaconNodes-${beaconNodes}`,
`validatorClients-${validatorClients}`,
`validatorsPerClient-${validatorsPerClient}`,
`altair-${altairEpoch}`,
`bellatrix-${bellatrixEpoch}`,
`externalSigner-${withExternalSigner ? "yes" : "no"}`,
].join("_");
console.log(
JSON.stringify({
beaconNodes,
validatorClients,
validatorsPerClient,
altairEpoch,
bellatrixEpoch,
withExternalSigner,
})
);
const env = new SimulationEnvironment({
beaconNodes,
validatorClients,
validatorsPerClient,
altairEpoch,
bellatrixEpoch,
logFilesDir: join(logFilesDir, testIdStr),
externalSigner: withExternalSigner,
});
describe(`simulation test - ${testIdStr}`, function () {
this.timeout("5m");
describe(title, async () => {
before("start env", async () => {
await env.start();
await env.network.connectAllNodes();
});
after("stop env", async () => {
env.resetCounter();
await env.stop();
});
describe("nodes env", () => {
nodeAssertions(env);
});
for (let epoch = 0; epoch <= runTill; epoch += 1) {
describe(`epoch - ${epoch}`, () => {
before("wait for epoch", async () => {
// Wait for one extra slot to make sure epoch transition is complete on the state
await env.waitForEndOfSlot(env.clock.getLastSlotOfEpoch(epoch) + 1);
env.tracker.printNoesInfo();
});
describe("missed blocks", () => {
missedBlocksAssertions(env);
});
describe("attestation participation", () => {
attestationParticipationAssertions(env, epoch);
});
});
}
});
});
}
}

71
packages/cli/test/utils/simulation/EpochClock.ts Normal file
View File

@@ -0,0 +1,71 @@
export const MS_IN_SEC = 1000;
export class EpochClock {
private readonly genesisTime: number;
private readonly secondsPerSlot: number;
readonly slotsPerEpoch: number;
constructor({
genesisTime,
secondsPerSlot,
slotsPerEpoch,
}: {
genesisTime: number;
secondsPerSlot: number;
slotsPerEpoch: number;
}) {
this.genesisTime = genesisTime;
this.secondsPerSlot = secondsPerSlot;
this.slotsPerEpoch = slotsPerEpoch;
}
timeSinceGenesis(): number {
return Math.floor(Date.now() / MS_IN_SEC - this.genesisTime);
}
get currentSlot(): number {
return this.getSlotFor();
}
get currentEpoch(): number {
return Math.floor(this.currentSlot / this.slotsPerEpoch);
}
getLastSlotOfEpoch(epoch: number): number {
return (epoch + 1) * this.slotsPerEpoch - 1;
}
getFirstSlotOfEpoch(epoch: number): number {
return epoch * this.slotsPerEpoch;
}
getEpochForSlot(slot: number): number {
return Math.floor(slot / this.slotsPerEpoch);
}
getSlotIndexInEpoch(slot: number): number {
return slot % this.slotsPerEpoch;
}
getSlotFor(timeStamp?: number): number {
const time = timeStamp ?? Math.floor(Date.now() / MS_IN_SEC);
const elapsedTime = time - this.genesisTime;
return Math.floor(elapsedTime / this.secondsPerSlot);
}
getSlotTime(slot: number): number {
const slotGenesisTimeOffset = slot * this.secondsPerSlot;
return this.genesisTime + slotGenesisTimeOffset;
}
isFirstSlotOfEpoch(slot: number): boolean {
return slot % this.slotsPerEpoch === 0;
}
isLastSlotOfEpoch(slot: number): boolean {
return slot % this.slotsPerEpoch === this.slotsPerEpoch - 1;
}
}

71
packages/cli/test/utils/simulation/ExternalSignerServer.ts Normal file
View File

@@ -0,0 +1,71 @@
import fastify from "fastify";
import {fromHexString, toHexString} from "@chainsafe/ssz";
import type {SecretKey} from "@chainsafe/bls/types";
import {EXTERNAL_SIGNER_BASE_PORT} from "./utils.js";
export class ExternalSignerServer {
static totalProcessCount = 0;
readonly address: string = "127.0.0.1";
readonly port: number;
private server: ReturnType<typeof fastify>;
constructor(secretKeys: SecretKey[]) {
const secretKeyMap = new Map<string, SecretKey>();
for (const secretKey of secretKeys) {
const pubkeyHex = toHexString(secretKey.toPublicKey().toBytes());
secretKeyMap.set(pubkeyHex, secretKey);
}
ExternalSignerServer.totalProcessCount++;
this.port = EXTERNAL_SIGNER_BASE_PORT + ExternalSignerServer.totalProcessCount;
this.server = fastify();
this.server.get("/upcheck", async () => {
return {status: "OK"};
});
this.server.get("/keys", async () => {
return {keys: Array.from(secretKeyMap.keys())};
});
/* eslint-disable @typescript-eslint/naming-convention */
this.server.post<{
Params: {
/** BLS public key as a hex string. */
identifier: string;
};
Body: {
/** Data to sign as a hex string */
signingRoot: string;
};
}>("/sign/:identifier", async (req) => {
const pubkeyHex: string = req.params.identifier;
const signingRootHex: string = req.body.signingRoot;
const secretKey = secretKeyMap.get(pubkeyHex);
if (!secretKey) {
throw Error(`pubkey not known ${pubkeyHex}`);
}
return {signature: secretKey.sign(fromHexString(signingRootHex)).toHex()};
});
}
get url(): string {
return `http://${this.address}:${this.port}`;
}
async start(): Promise<void> {
console.log(`Starting external signer server at ${this.url}.`);
await this.server.listen(this.port, this.address);
console.log(`Started external signer server at ${this.url}.`);
}
async stop(): Promise<void> {
console.log(`Stopping external signer server at ${this.url}.`);
await this.server.close();
console.log(`Stopped external signer server at ${this.url}.`);
}
}

139
packages/cli/test/utils/simulation/LodestarBeaconNodeProcess.ts Normal file
View File

@@ -0,0 +1,139 @@
import {join} from "node:path";
import {ChildProcess} from "node:child_process";
import {mkdir, writeFile} from "node:fs/promises";
import type {SecretKey} from "@chainsafe/bls/types";
import {Api, getClient} from "@lodestar/api";
import {nodeUtils} from "@lodestar/beacon-node/node";
import {IChainForkConfig} from "@lodestar/config";
import {IBeaconArgs} from "../../../src/cmds/beacon/options.js";
import {getBeaconConfigFromArgs} from "../../../src/config/beaconParams.js";
import {IGlobalArgs} from "../../../src/options/globalOptions.js";
import {LodestarValidatorProcess} from "./LodestarValidatorProcess.js";
import {BeaconNodeConstructor, BeaconNodeProcess, SimulationParams, ValidatorProcess} from "./types.js";
import {BN_P2P_BASE_PORT, BN_P2P_REST_PORT, closeChildProcess, spawnProcessAndWait, __dirname} from "./utils.js";
// eslint-disable-next-line @typescript-eslint/naming-convention
export const LodestarBeaconNodeProcess: BeaconNodeConstructor = class LodestarBeaconNodeProcess
implements BeaconNodeProcess {
static totalProcessCount = 0;
readonly params: SimulationParams;
readonly secretKeys: Record<number, SecretKey[]> = {};
readonly address: string;
readonly port: number;
readonly restPort: number;
readonly id: string;
readonly api: Api;
peerId?: string;
readonly multiaddrs: string[];
readonly validatorClients: ValidatorProcess[] = [];
private rootDir: string;
private beaconProcess!: ChildProcess;
private rcConfig: IBeaconArgs & IGlobalArgs;
private config!: IChainForkConfig;
constructor(params: SimulationParams, rootDir: string) {
this.params = params;
this.rootDir = rootDir;
LodestarBeaconNodeProcess.totalProcessCount += 1;
this.address = "127.0.0.1";
this.port = BN_P2P_BASE_PORT + LodestarBeaconNodeProcess.totalProcessCount;
this.restPort = BN_P2P_REST_PORT + LodestarBeaconNodeProcess.totalProcessCount;
this.id = `NODE-${LodestarBeaconNodeProcess.totalProcessCount}`;
this.rcConfig = ({
network: "dev",
preset: "minimal",
dataDir: this.rootDir,
genesisStateFile: `${this.rootDir}/genesis.ssz`,
rest: true,
"rest.address": this.address,
"rest.port": this.restPort,
"rest.namespace": "*",
"sync.isSingleNode": this.params.beaconNodes === 1,
"network.allowPublishToZeroPeers": this.params.beaconNodes === 1,
eth1: false,
discv5: this.params.beaconNodes > 1,
"network.connectToDiscv5Bootnodes": this.params.beaconNodes > 1,
"execution.engineMock": true,
listenAddress: this.address,
port: this.port,
metrics: false,
bootnodes: [],
"params.SECONDS_PER_SLOT": String(this.params.secondsPerSlot),
"params.GENESIS_DELAY": String(this.params.genesisSlotsDelay),
"params.ALTAIR_FORK_EPOCH": String(this.params.altairEpoch),
"params.BELLATRIX_FORK_EPOCH": String(this.params.bellatrixEpoch),
logPrefix: this.id,
logFormatGenesisTime: `${this.params.genesisTime}`,
logFile: `${this.params.logFilesDir}/${this.id}.log`,
logFileLevel: "debug",
logLevel: "info",
} as unknown) as IBeaconArgs & IGlobalArgs;
this.multiaddrs = [`/ip4/${this.address}/tcp/${this.port}`];
this.config = getBeaconConfigFromArgs(this.rcConfig).config;
this.config = getBeaconConfigFromArgs(this.rcConfig).config;
this.api = getClient({baseUrl: `http://${this.address}:${this.restPort}`}, {config: this.config});
for (let clientIndex = 0; clientIndex < this.params.validatorClients; clientIndex += 1) {
this.validatorClients.push(
new LodestarValidatorProcess(this.params, {
rootDir: join(this.rootDir, `validator-${clientIndex}`),
config: getBeaconConfigFromArgs(this.rcConfig).config,
server: `http://${this.address}:${this.restPort}/`,
clientIndex: clientIndex + LodestarBeaconNodeProcess.totalProcessCount - 1,
})
);
}
}
async start(): Promise<void> {
const {state} = nodeUtils.initDevState(
this.config,
this.params.validatorClients * this.params.validatorsPerClient,
{
genesisTime: this.params.genesisTime,
}
);
await mkdir(this.rootDir);
await writeFile(join(this.rootDir, "genesis.ssz"), state.serialize());
await writeFile(join(this.rootDir, "rc_config.json"), JSON.stringify(this.rcConfig, null, 2));
console.log(`Starting lodestar beacon node "${this.id}".`, {dataDir: this.rootDir});
this.beaconProcess = await spawnProcessAndWait(
`${__dirname}/../../../bin/lodestar.js`,
["beacon", "--rcConfig", `${this.rootDir}/rc_config.json`, "--network", "dev"],
async () => this.ready(),
`Waiting for "${this.id}" to start.`
);
console.log(`Beacon node "${this.id}" started.`);
this.peerId = (await this.api.node.getNetworkIdentity()).data.peerId;
for (let clientIndex = 0; clientIndex < this.params.validatorClients; clientIndex++) {
await this.validatorClients[clientIndex].start();
}
}
async stop(): Promise<void> {
console.log(`Stopping node "${this.id}".`);
await Promise.all(this.validatorClients.map((p) => p.stop()));
if (this.beaconProcess !== undefined) {
await closeChildProcess(this.beaconProcess);
}
}
async ready(): Promise<boolean> {
const health = await this.api.node.getHealth();
return health === 200 || health === 206;
}
};

154
packages/cli/test/utils/simulation/LodestarValidatorProcess.ts Normal file
View File

@@ -0,0 +1,154 @@
import {join} from "node:path";
import {ChildProcess} from "node:child_process";
import {mkdir, writeFile} from "node:fs/promises";
import type {SecretKey} from "@chainsafe/bls/types";
import {Api, getClient} from "@lodestar/api/keymanager";
import {Keystore} from "@chainsafe/bls-keystore";
import {IChainForkConfig} from "@lodestar/config";
import {interopSecretKey} from "@lodestar/state-transition";
import {IGlobalArgs} from "../../../src/options/globalOptions.js";
import {IValidatorCliArgs} from "../../../lib/cmds/validator/options.js";
import {SimulationParams, ValidatorConstructor, ValidatorProcess} from "./types.js";
import {closeChildProcess, KEY_MANAGER_BASE_PORT, spawnProcessAndWait, __dirname} from "./utils.js";
import {ExternalSignerServer} from "./ExternalSignerServer.js";
// eslint-disable-next-line @typescript-eslint/naming-convention
export const LodestarValidatorProcess: ValidatorConstructor = class LodestarValidatorProcess
implements ValidatorProcess {
static totalProcessCount = 0;
readonly params: SimulationParams;
readonly address = "127.0.0.1";
readonly keyManagerPort: number;
readonly id: string;
readonly keyManagerApi: Api;
readonly secretKeys: SecretKey[] = [];
readonly externalSigner?: ExternalSignerServer;
private rootDir: string;
private clientIndex: number;
private validatorProcess!: ChildProcess;
private rcConfig: IValidatorCliArgs & IGlobalArgs;
private forkConfig: IChainForkConfig;
constructor(
params: SimulationParams,
{
rootDir,
clientIndex,
server,
config,
}: {
rootDir: string;
clientIndex: number;
server: string;
config: IChainForkConfig;
}
) {
this.params = params;
this.rootDir = rootDir;
this.clientIndex = clientIndex;
LodestarValidatorProcess.totalProcessCount += 1;
this.keyManagerPort = KEY_MANAGER_BASE_PORT + LodestarValidatorProcess.totalProcessCount;
this.id = `VAL-${LodestarValidatorProcess.totalProcessCount}`;
this.forkConfig = config;
const validatorSecretKeys = Array.from({length: this.params.validatorsPerClient}, (_, i) => {
return interopSecretKey(this.clientIndex * this.params.validatorsPerClient + i);
});
this.secretKeys = validatorSecretKeys;
this.rcConfig = ({
network: "dev",
preset: "minimal",
dataDir: join(this.rootDir, this.id),
server,
keymanager: true,
"keymanager.authEnabled": false,
"keymanager.address": this.address,
"keymanager.port": this.keyManagerPort,
"params.SECONDS_PER_SLOT": String(this.params.secondsPerSlot),
"params.GENESIS_DELAY": String(this.params.genesisSlotsDelay),
"params.ALTAIR_FORK_EPOCH": String(this.params.altairEpoch),
"params.BELLATRIX_FORK_EPOCH": String(this.params.bellatrixEpoch),
logPrefix: this.id,
logFormatGenesisTime: this.params.genesisTime,
logFile: join(this.params.logFilesDir, `${this.id}.log`),
logFileLevel: "debug",
logLevel: "info",
} as unknown) as IValidatorCliArgs & IGlobalArgs;
if (this.params.externalSigner) {
this.externalSigner = new ExternalSignerServer(this.secretKeys);
this.rcConfig["externalSigner.url"] = this.externalSigner.url;
}
this.keyManagerApi = getClient(
{baseUrl: `http://${this.address}:${this.keyManagerPort}`},
{config: this.forkConfig}
);
}
async start(): Promise<void> {
await mkdir(this.rootDir);
await mkdir(`${this.rootDir}/keystores`);
await writeFile(join(this.rootDir, "password.txt"), "password");
await writeFile(join(this.rootDir, "rc_config.json"), JSON.stringify(this.rcConfig, null, 2));
for (const key of this.secretKeys) {
const keystore = await Keystore.create("password", key.toBytes(), key.toPublicKey().toBytes(), "");
await writeFile(
join(this.rootDir, "keystores", `${key.toPublicKey().toHex()}.json`),
JSON.stringify(keystore.toObject(), null, 2)
);
}
if (this.externalSigner) {
await this.externalSigner.start();
}
console.log(`Starting lodestar validator "${this.id}".`, {dataDir: this.rootDir});
this.validatorProcess = await spawnProcessAndWait(
`${__dirname}/../../../bin/lodestar.js`,
[
"validator",
"--network",
"dev",
"--rcConfig",
`${this.rootDir}/rc_config.json`,
"--importKeystores",
`${this.rootDir}/keystores`,
"--importKeystoresPassword",
`${this.rootDir}/password.txt`,
],
async () => this.ready(),
`Waiting for "${this.id}" to start.`
);
console.log(`Validator "${this.id}" started.`);
}
async stop(): Promise<void> {
console.log(`Stopping validator "${this.id}".`);
if (this.externalSigner) {
await this.externalSigner.stop();
}
if (this.validatorProcess !== undefined) {
await closeChildProcess(this.validatorProcess);
}
}
async ready(): Promise<boolean> {
try {
await this.keyManagerApi.listKeys();
return true;
} catch {
return false;
}
}
};

154
packages/cli/test/utils/simulation/SimulationEnvironment.ts Normal file
View File

@@ -0,0 +1,154 @@
import {mkdir, rm} from "node:fs/promises";
import {join} from "node:path";
import {EventEmitter} from "node:events";
import tmp from "tmp";
import {activePreset} from "@lodestar/params";
import {routes} from "@lodestar/api/beacon";
import {BeaconNodeProcess, SimulationOptionalParams, SimulationParams, SimulationRequiredParams} from "./types.js";
import {EpochClock, MS_IN_SEC} from "./EpochClock.js";
import {SimulationTracker} from "./SimulationTracker.js";
import {
LodestarBeaconNodeProcess,
defaultSimulationParams,
LodestarValidatorProcess,
getSimulationId,
} from "./index.js";
export class SimulationEnvironment {
readonly params: SimulationParams;
readonly id: string;
readonly rootDir: string;
readonly nodes: BeaconNodeProcess[] = [];
readonly clock: EpochClock;
readonly acceptableParticipationRate = 1;
readonly tracker: SimulationTracker;
readonly emitter: EventEmitter;
readonly controller: AbortController;
readonly network = {
connectAllNodes: async (): Promise<void> => {
for (let i = 0; i < this.params.beaconNodes; i += 1) {
for (let j = 0; j < this.params.beaconNodes; j += 1) {
const peerId = this.nodes[j].peerId;
if (i === j || !peerId) continue;
await this.nodes[i].api.lodestar.connectPeer(peerId, this.nodes[j].multiaddrs);
}
}
},
connectNodesToFirstNode: async (): Promise<void> => {
const firstNode = this.nodes[0];
if (!firstNode.peerId) throw Error("First node must be started");
for (let i = 1; i < this.params.beaconNodes; i += 1) {
const node = this.nodes[i];
await node.api.lodestar.connectPeer(firstNode.peerId, firstNode.multiaddrs);
}
},
};
constructor(params: SimulationRequiredParams & Partial<SimulationOptionalParams>) {
const paramsWithDefaults = {...defaultSimulationParams, ...params} as SimulationRequiredParams &
SimulationOptionalParams;
const genesisTime =
Math.floor(Date.now() / 1000) + paramsWithDefaults.genesisSlotsDelay * paramsWithDefaults.secondsPerSlot;
this.params = {
...paramsWithDefaults,
genesisTime,
slotsPerEpoch: activePreset.SLOTS_PER_EPOCH,
} as SimulationParams;
this.controller = new AbortController();
this.id = getSimulationId(this.params);
this.rootDir = join(tmp.dirSync({unsafeCleanup: true}).name, this.id);
this.clock = new EpochClock({
genesisTime,
secondsPerSlot: this.params.secondsPerSlot,
slotsPerEpoch: this.params.slotsPerEpoch,
});
this.emitter = new EventEmitter();
for (let i = 1; i <= this.params.beaconNodes; i += 1) {
const nodeRootDir = `${this.rootDir}/node-${i}`;
this.nodes.push(new LodestarBeaconNodeProcess(this.params, nodeRootDir));
}
this.tracker = new SimulationTracker(this.nodes, this.clock, this.params, this.controller.signal);
}
async start(): Promise<this> {
await mkdir(this.rootDir);
await Promise.all(this.nodes.map((p) => p.start()));
await this.tracker.start();
return this;
}
async stop(): Promise<void> {
this.controller.abort();
await this.tracker.stop();
await Promise.all(this.nodes.map((p) => p.stop()));
await rm(this.rootDir, {recursive: true});
}
// TODO: Add timeout support
waitForEvent(event: routes.events.EventType, node?: BeaconNodeProcess): Promise<routes.events.BeaconEvent> {
console.log(`Waiting for event "${event}" on "${node?.id ?? "any node"}"`);
return new Promise((resolve) => {
const handler = (beaconEvent: routes.events.BeaconEvent, eventNode: BeaconNodeProcess): void => {
if (!node) {
this.emitter.removeListener(event, handler);
resolve(beaconEvent);
}
if (node && eventNode === node) {
this.emitter.removeListener(event, handler);
resolve(beaconEvent);
}
};
this.tracker.emitter.addListener(event, handler);
});
}
waitForStartOfSlot(slot: number): Promise<this> {
console.log("Waiting for start of slot", {target: slot, current: this.clock.currentSlot});
return new Promise((resolve) => {
const slotTime = this.clock.getSlotTime(slot) * MS_IN_SEC - Date.now();
const timeout = setTimeout(() => {
resolve(this);
}, slotTime);
this.controller.signal.addEventListener(
"abort",
() => {
clearTimeout(timeout);
},
{once: true}
);
});
}
waitForEndOfSlot(slot: number): Promise<this> {
return this.waitForStartOfSlot(slot + 1);
}
waitForStartOfEpoch(epoch: number): Promise<this> {
return this.waitForStartOfSlot(this.clock.getFirstSlotOfEpoch(epoch));
}
waitForEndOfEpoch(epoch: number): Promise<this> {
return this.waitForEndOfSlot(this.clock.getLastSlotOfEpoch(epoch));
}
resetCounter(): void {
LodestarBeaconNodeProcess.totalProcessCount = 0;
LodestarValidatorProcess.totalProcessCount = 0;
}
}

188
packages/cli/test/utils/simulation/SimulationTracker.ts Normal file
View File

@@ -0,0 +1,188 @@
import EventEmitter from "node:events";
import {routes} from "@lodestar/api/beacon";
import {altair, Epoch, Slot} from "@lodestar/types";
import {EpochClock} from "./EpochClock.js";
import {BeaconNodeProcess, SimulationParams} from "./types.js";
import {computeAttestation, computeAttestationParticipation, computeInclusionDelay, getForkName} from "./utils.js";
const participationHeading = (id: string): string => `${id}-P-H/S/T`;
const missedBlocksHeading = (id: string): string => `${id}-M`;
export class SimulationTracker {
readonly producedBlocks: Map<string, Map<Slot, boolean>>;
readonly attestationsPerBlock: Map<string, Map<Slot, number>>;
readonly inclusionDelayPerBlock: Map<string, Map<Slot, number>>;
readonly attestationParticipation: Map<string, Map<Epoch, {head: number; source: number; target: number}>>;
private lastSeenSlot: Map<string, Slot>;
readonly emitter = new EventEmitter();
private signal: AbortSignal;
private nodes: BeaconNodeProcess[];
private clock: EpochClock;
private params: SimulationParams;
constructor(nodes: BeaconNodeProcess[], clock: EpochClock, params: SimulationParams, signal: AbortSignal) {
this.signal = signal;
this.nodes = nodes;
this.clock = clock;
this.params = params;
this.producedBlocks = new Map();
this.attestationsPerBlock = new Map();
this.inclusionDelayPerBlock = new Map();
this.attestationParticipation = new Map();
this.lastSeenSlot = new Map();
for (let i = 0; i < nodes.length; i += 1) {
this.producedBlocks.set(nodes[i].id, new Map());
this.attestationsPerBlock.set(nodes[i].id, new Map());
this.inclusionDelayPerBlock.set(nodes[i].id, new Map());
this.attestationParticipation.set(nodes[i].id, new Map());
this.lastSeenSlot.set(nodes[i].id, 0);
}
}
get missedBlocks(): Map<string, Slot[]> {
const missedBlocks: Map<string, Slot[]> = new Map();
const minSlot = Math.min(...this.lastSeenSlot.values());
for (let i = 0; i < this.nodes.length; i++) {
const missedBlocksForNode: Slot[] = [];
for (let s = 0; s < minSlot; s++) {
if (!this.producedBlocks.get(this.nodes[i].id)?.get(s)) {
missedBlocksForNode.push(s);
}
}
missedBlocks.set(this.nodes[i].id, missedBlocksForNode);
}
return missedBlocks;
}
async start(): Promise<void> {
for (let i = 0; i < this.nodes.length; i += 1) {
this.nodes[i].api.events.eventstream(
[routes.events.EventType.block, routes.events.EventType.head, routes.events.EventType.finalizedCheckpoint],
this.signal,
async (event) => {
this.emitter.emit(event.type, event, this.nodes[i]);
switch (event.type) {
case routes.events.EventType.block:
await this.onBlock(event.message, this.nodes[i]);
return;
case routes.events.EventType.finalizedCheckpoint:
this.onFinalizedCheckpoint(event.message, this.nodes[i]);
return;
}
}
);
}
}
async stop(): Promise<void> {
// Do nothing;
}
private async onBlock(
event: routes.events.EventData[routes.events.EventType.block],
node: BeaconNodeProcess
): Promise<void> {
const slot = event.slot;
const lastSeenSlot = this.lastSeenSlot.get(node.id);
const blockAttestations = await node.api.beacon.getBlockAttestations(slot);
if (lastSeenSlot !== undefined && slot > lastSeenSlot) {
this.lastSeenSlot.set(node.id, slot);
}
this.producedBlocks.get(node.id)?.set(slot, true);
this.attestationsPerBlock.get(node.id)?.set(slot, computeAttestation(blockAttestations.data));
this.inclusionDelayPerBlock.get(node.id)?.set(slot, computeInclusionDelay(blockAttestations.data, slot));
if (this.clock.isFirstSlotOfEpoch(slot)) {
const state = await node.api.debug.getStateV2("head");
const participation = computeAttestationParticipation(state.data as altair.BeaconState);
this.attestationParticipation
.get(node.id)
// As the `computeAttestationParticipation` using previousEpochParticipation for calculations
?.set(participation.epoch, {
head: participation.head,
source: participation.source,
target: participation.target,
});
}
}
private onHead(_event: routes.events.EventData[routes.events.EventType.head], _node: BeaconNodeProcess): void {
// TODO: Add head tracking
}
private onFinalizedCheckpoint(
_event: routes.events.EventData[routes.events.EventType.finalizedCheckpoint],
_node: BeaconNodeProcess
): void {
// TODO: Add checkpoint tracking
}
printNoesInfo(): void {
/* eslint-disable @typescript-eslint/naming-convention */
const maxSlot = Math.max(...this.lastSeenSlot.values());
const records: Record<string, unknown>[] = [];
for (let slot = 0; slot <= maxSlot; slot++) {
const epoch = this.clock.getEpochForSlot(slot);
const record: Record<string, unknown> = {
F: getForkName(epoch, this.params),
Eph: `${this.clock.getEpochForSlot(slot)}/${this.clock.getSlotIndexInEpoch(slot)}`,
slot,
};
for (const node of this.nodes) {
record[missedBlocksHeading(node.id)] = this.producedBlocks.get(node.id)?.get(slot) ? "" : "x";
}
for (const node of this.nodes) {
record[participationHeading(node.id)] = "";
}
records.push(record);
if (this.clock.isLastSlotOfEpoch(slot)) {
const epoch = this.clock.getEpochForSlot(slot);
const record: Record<string, unknown> = {
F: getForkName(epoch, this.params),
Eph: epoch,
slot: "---",
};
for (const node of this.nodes) {
record[missedBlocksHeading(node.id)] = this.missedBlocks.get(node.id)?.filter((s) => s <= slot).length;
}
for (const node of this.nodes) {
const participation = this.attestationParticipation.get(node.id)?.get(epoch);
const participationStr =
participation?.head != null
? `${participation?.head.toFixed(2)}/${participation?.source.toFixed(2)}/${participation?.target.toFixed(
2
)}`
: "";
record[participationHeading(node.id)] = participationStr;
}
records.push(record);
}
}
console.table(records);
console.log(
["M - Missed Blocks", "P - Attestation Participation", "H - Head", "S - Source", "T - Target"].join(" | ")
);
/* eslint-enable @typescript-eslint/naming-convention */
}
}

111
packages/cli/test/utils/simulation/assertions.ts Normal file
View File

@@ -0,0 +1,111 @@
import chai, {expect} from "chai";
import chaiAsPromised from "chai-as-promised";
import {routes} from "@lodestar/api/beacon";
import {Epoch} from "@lodestar/types";
import {SimulationEnvironment} from "./SimulationEnvironment.js";
chai.use(chaiAsPromised);
export function nodeAssertions(env: SimulationEnvironment): void {
it("test env should have correct number of nodes", () => {
expect(env.nodes.length).to.equal(
env.params.beaconNodes,
`should have "${env.params.beaconNodes}" number of nodes. Found: ${env.nodes.length}`
);
});
for (const node of env.nodes) {
describe(node.id, () => {
it("should have correct sync status", async () => {
const health = await node.api.node.getHealth();
expect(health === routes.node.NodeHealth.SYNCING || health === routes.node.NodeHealth.READY).to.equal(
true,
`Node "${node.id}" health is neither READY or SYNCING`
);
});
it("should have correct number of validator clients", async () => {
expect(node.validatorClients).to.have.lengthOf(
env.params.validatorClients,
`Node "${node.id}" have correct "${env.params.validatorClients}" of validator clients. Found: ${node.validatorClients.length}`
);
});
for (const validator of node.validatorClients) {
describe(validator.id, () => {
it("should have correct keys loaded", async () => {
const keys = (await validator.keyManagerApi.listKeys()).data.map((k) => k.validatingPubkey).sort();
const existingKeys = validator.secretKeys.map((k) => k.toPublicKey().toHex()).sort();
expect(keys).to.eql(
existingKeys,
`Validator "${validator.id}" should have correct number of keys loaded. Generated Keys: ${existingKeys}, Loaded Keys: ${keys}`
);
});
});
}
});
}
}
export function attestationParticipationAssertions(env: SimulationEnvironment, epoch: Epoch): void {
if (epoch < env.params.altairEpoch) return;
for (const node of env.nodes) {
describe(`${node.id}`, () => {
it("should have correct attestation on head", () => {
const participation = env.tracker.attestationParticipation.get(node.id)?.get(epoch);
expect(participation?.head).to.be.gte(
env.acceptableParticipationRate,
`node "${node.id}" has low participation rate on head for epoch ${epoch}. participationRate: ${participation?.head}, acceptableParticipationRate: ${env.acceptableParticipationRate}`
);
});
it("should have correct attestation on target", () => {
const participation = env.tracker.attestationParticipation.get(node.id)?.get(epoch);
expect(participation?.target).to.be.gte(
env.acceptableParticipationRate,
`node "${node.id}" has low participation rate on target for epoch ${epoch}. participationRate: ${participation?.target}, acceptableParticipationRate: ${env.acceptableParticipationRate}`
);
});
it("should have correct attestation on source", () => {
const participation = env.tracker.attestationParticipation.get(node.id)?.get(epoch);
expect(participation?.source).to.be.gte(
env.acceptableParticipationRate,
`node "${node.id}" has low participation rate on source for epoch ${epoch}. participationRate: ${participation?.source}, acceptableParticipationRate: ${env.acceptableParticipationRate}`
);
});
});
}
}
export function missedBlocksAssertions(env: SimulationEnvironment): void {
if (env.params.beaconNodes === 1) {
it("should not have any missed blocks than genesis", () => {
expect(env.tracker.missedBlocks.get(env.nodes[0].id)).to.be.eql(
[0],
"single node should not miss any blocks other than genesis"
);
});
return;
}
for (const node of env.nodes) {
describe(node.id, () => {
it("should have same missed blocks as first node", () => {
const missedBlocksOnFirstNode = env.tracker.missedBlocks.get(env.nodes[0].id);
const missedBlocksOnNodeN = env.tracker.missedBlocks.get(node.id);
expect(missedBlocksOnNodeN).to.eql(
missedBlocksOnFirstNode,
`node "${node.id}" has different missed blocks than node 0. missedBlocksOnNodeN: ${missedBlocksOnNodeN}, missedBlocksOnFirstNode: ${missedBlocksOnFirstNode}`
);
});
});
}
}

5
packages/cli/test/utils/simulation/index.ts Normal file
View File

@@ -0,0 +1,5 @@
export * from "./LodestarBeaconNodeProcess.js";
export * from "./LodestarValidatorProcess.js";
export * from "./SimulationEnvironment.js";
export * from "./utils.js";
export * from "./types.js";

70
packages/cli/test/utils/simulation/types.ts Normal file
View File

@@ -0,0 +1,70 @@
import type {SecretKey} from "@chainsafe/bls/types";
import {Api} from "@lodestar/api";
import {Api as KeyManagerApi} from "@lodestar/api/keymanager";
import {IChainForkConfig} from "@lodestar/config";
import {BeaconStateAllForks} from "@lodestar/state-transition/";
export type SimulationRequiredParams = {
beaconNodes: number;
validatorClients: number;
altairEpoch: number;
bellatrixEpoch: number;
logFilesDir: string;
};
export type SimulationOptionalParams = {
validatorsPerClient: number;
withExternalSigner: boolean;
secondsPerSlot: number;
genesisSlotsDelay: number;
anchorState?: BeaconStateAllForks;
externalSigner: boolean;
};
export type RunTimeSimulationParams = {
genesisTime: number;
slotsPerEpoch: number;
};
export interface BeaconNodeProcess {
ready(): Promise<boolean>;
start(): Promise<void>;
stop(): Promise<void>;
id: string;
peerId?: string;
multiaddrs: string[];
api: Api;
address: string;
port: number;
restPort: number;
validatorClients: ValidatorProcess[];
}
export interface BeaconNodeConstructor {
new (params: SimulationParams, rootDir: string): BeaconNodeProcess;
totalProcessCount: number;
}
export interface ValidatorProcess {
ready(): Promise<boolean>;
start(): Promise<void>;
stop(): Promise<void>;
id: string;
secretKeys: SecretKey[];
keyManagerApi: KeyManagerApi;
}
export interface ValidatorConstructor {
new (
params: SimulationParams,
options: {
rootDir: string;
clientIndex: number;
server: string;
config: IChainForkConfig;
}
): ValidatorProcess;
totalProcessCount: number;
}
export type SimulationParams = SimulationRequiredParams & Required<SimulationOptionalParams> & RunTimeSimulationParams;

147
packages/cli/test/utils/simulation/utils.ts Normal file
View File

@@ -0,0 +1,147 @@
import {dirname} from "node:path";
import {fileURLToPath} from "node:url";
import {ChildProcess, spawn} from "node:child_process";
import {altair, Epoch, phase0, Slot} from "@lodestar/types";
import {
TIMELY_HEAD_FLAG_INDEX,
TIMELY_TARGET_FLAG_INDEX,
TIMELY_SOURCE_FLAG_INDEX,
SLOTS_PER_EPOCH,
ForkName,
} from "@lodestar/params";
import {SimulationOptionalParams, SimulationParams} from "./types.js";
const TIMELY_HEAD = 1 << TIMELY_HEAD_FLAG_INDEX;
const TIMELY_SOURCE = 1 << TIMELY_SOURCE_FLAG_INDEX;
const TIMELY_TARGET = 1 << TIMELY_TARGET_FLAG_INDEX;
// Global variable __dirname no longer available in ES6 modules.
// Solutions: https://stackoverflow.com/questions/46745014/alternative-for-dirname-in-node-js-when-using-es6-modules
// eslint-disable-next-line @typescript-eslint/naming-convention
export const __dirname = dirname(fileURLToPath(import.meta.url));
export const logFilesDir = "test-logs";
export const defaultSimulationParams: SimulationOptionalParams = {
validatorsPerClient: 32,
withExternalSigner: false,
secondsPerSlot: 2,
// delay a bit so regular sync sees it's up to date and sync is completed from the beginning
// allow time for bls worker threads to warm up
genesisSlotsDelay: 30,
externalSigner: false,
};
export const getSimulationId = ({
beaconNodes,
validatorClients,
validatorsPerClient,
withExternalSigner,
altairEpoch,
bellatrixEpoch,
}: SimulationParams): string =>
[
`beaconNodes-${beaconNodes}`,
`validatorClients-${validatorClients}`,
`validatorsPerClient-${validatorsPerClient}`,
`altair-${altairEpoch}`,
`bellatrix-${bellatrixEpoch}`,
`externalSigner-${withExternalSigner ? "yes" : "no"}`,
].join("_");
export const spawnProcessAndWait = async (
module: string,
args: string[],
ready: (childProcess: ChildProcess) => Promise<boolean>,
message: string
): Promise<ChildProcess> => {
return new Promise((resolve, reject) => {
void (async () => {
const childProcess = spawn(module, args, {
detached: false,
stdio: process.env.SHOW_LOGS ? "inherit" : "ignore",
// eslint-disable-next-line @typescript-eslint/naming-convention
env: {...process.env, NODE_ENV: "test"},
});
childProcess.on("error", reject);
childProcess.on("exit", (code: number) => {
reject(new Error(`lodestar exited with code ${code}`));
});
// TODO: Add support for timeout
// To safe the space in logs log only for once.
console.log(message);
const intervalId = setInterval(async () => {
if (await ready(childProcess)) {
clearInterval(intervalId);
resolve(childProcess);
}
}, 1000);
})();
});
};
export const closeChildProcess = async (childProcess: ChildProcess, signal?: "SIGTERM"): Promise<void> => {
return new Promise((resolve) => {
childProcess.on("close", resolve);
childProcess.kill(signal);
});
};
export const computeAttestationParticipation = (
state: altair.BeaconState
): {epoch: number; head: number; source: number; target: number} => {
// Attestation to be computed at the end of epoch. At that time the "currentEpochParticipation" is all set to zero
// and we have to use "previousEpochParticipation" instead.
const previousEpochParticipation = state.previousEpochParticipation;
// As we calculated the participation from the previous epoch
const epoch = Math.floor(state.slot / SLOTS_PER_EPOCH) - 1;
const totalAttestingBalance = {head: 0, source: 0, target: 0};
let totalEffectiveBalance = 0;
for (let i = 0; i < previousEpochParticipation.length; i++) {
totalAttestingBalance.head +=
previousEpochParticipation[i] & TIMELY_HEAD ? state.validators[i].effectiveBalance : 0;
totalAttestingBalance.source +=
previousEpochParticipation[i] & TIMELY_SOURCE ? state.validators[i].effectiveBalance : 0;
totalAttestingBalance.target +=
previousEpochParticipation[i] & TIMELY_TARGET ? state.validators[i].effectiveBalance : 0;
totalEffectiveBalance += state.validators[i].effectiveBalance;
}
totalAttestingBalance.head = totalAttestingBalance.head / totalEffectiveBalance;
totalAttestingBalance.source = totalAttestingBalance.source / totalEffectiveBalance;
totalAttestingBalance.target = totalAttestingBalance.target / totalEffectiveBalance;
return {...totalAttestingBalance, epoch};
};
export const computeAttestation = (attestations: phase0.Attestation[]): number => {
return Array.from(attestations).reduce((total, att) => total + att.aggregationBits.getTrueBitIndexes().length, 0);
};
export const computeInclusionDelay = (attestations: phase0.Attestation[], slot: Slot): number => {
return avg(Array.from(attestations).map((att) => slot - att.data.slot));
};
export const avg = (arr: number[]): number => {
return arr.length === 0 ? 0 : arr.reduce((p, c) => p + c, 0) / arr.length;
};
export const getForkName = (epoch: Epoch, params: SimulationParams): ForkName => {
if (epoch < params.altairEpoch) {
return ForkName.phase0;
} else if (epoch < params.bellatrixEpoch) {
return ForkName.altair;
} else {
return ForkName.bellatrix;
}
};
export const FAR_FUTURE_EPOCH = 10 ** 12;
export const BN_P2P_BASE_PORT = 4000;
export const BN_P2P_REST_PORT = 5000;
export const KEY_MANAGER_BASE_PORT = 6000;
export const EXTERNAL_SIGNER_BASE_PORT = 7000;

7
packages/state-transition/src/util/epoch.ts
View File

@@ -15,6 +15,13 @@ export function computeStartSlotAtEpoch(epoch: Epoch): Slot {
return epoch * SLOTS_PER_EPOCH;
}
/**
* Return the end slot of the given epoch.
*/
export function computeEndSlotAtEpoch(epoch: Epoch): Slot {
return computeStartSlotAtEpoch(epoch + 1) - 1;
}
/**
* Return the epoch at which an activation or exit triggered in ``epoch`` takes effect.
*/