diff --git a/.benchrc.yaml b/.benchrc.yaml index df1b6f8d48..bb9ec45c62 100644 --- a/.benchrc.yaml +++ b/.benchrc.yaml @@ -1,4 +1,10 @@ +# Mocha opts colors: true require: - ts-node/register - packages/lodestar/test/setupBLS.ts + +# benchmark opts +threshold: 3 +maxMs: 60_000 +minRuns: 10 diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index dfe0a7b656..35b60d3cef 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -47,7 +47,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # benchmark options BENCHMARK_S3: true - BENCHMARK_threshold: 3 # S3 credentials S3_ACCESS_KEY: ${{ secrets.S3_BENCH_LODESTAR_ACCESS_KEY }} S3_SECRET_KEY: ${{ secrets.S3_BENCH_LODESTAR_SECRET_KEY }} diff --git a/package.json b/package.json index d88c312cda..0a6b20ad9b 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "check-readme": "lerna run check-readme" }, "devDependencies": { - "@dapplion/benchmark": "^0.2.0", + "@dapplion/benchmark": "^0.2.2", "@types/chai": "4.2.0", "@types/chai-as-promised": "^7.1.2", "@types/mocha": "^8.0.3", diff --git a/packages/beacon-state-transition/test/perf/allForks/hashing.test.ts b/packages/beacon-state-transition/test/perf/allForks/hashing.test.ts index baab133648..e791d30085 100644 --- a/packages/beacon-state-transition/test/perf/allForks/hashing.test.ts +++ b/packages/beacon-state-transition/test/perf/allForks/hashing.test.ts @@ -1,4 +1,4 @@ -import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {itBench} from "@dapplion/benchmark"; import {phase0, ssz} from "@chainsafe/lodestar-types"; import {TreeBacked} from "@chainsafe/ssz"; import {generatePerformanceStatePhase0, getPubkeys, numValidators} from "../util"; @@ -7,8 +7,6 @@ import {unshuffleList} from "../../../src"; // Test cost of hashing state after some modifications describe("state hashTreeRoot", () => { - setBenchOpts({maxMs: 60 * 1000}); - const vc = numValidators; const indicesShuffled: number[] = []; let stateOg: TreeBacked; @@ -26,7 +24,7 @@ describe("state hashTreeRoot", () => { const validator: phase0.Validator = ssz.phase0.Validator.defaultValue(); const balance = BigInt(31e9); - const testCases: {id: string; track?: boolean; fn: (state: TreeBacked) => void}[] = [ + const testCases: {id: string; noTrack?: boolean; fn: (state: TreeBacked) => void}[] = [ { id: "No change", fn: () => { @@ -40,7 +38,7 @@ describe("state hashTreeRoot", () => { const idxs = indicesShuffled.slice(0, count); testCases.push({ id: `${count} full validator`, - track: count >= 512, + noTrack: count < 512, fn: (state) => { for (const i of idxs) state.validators[i] = validator; }, @@ -51,7 +49,7 @@ describe("state hashTreeRoot", () => { const idxs = indicesShuffled.slice(0, count); testCases.push({ id: `${count} validator.effectiveBalance`, - track: count >= 512, + noTrack: count < 512, fn: (state) => { for (const i of idxs) state.validators[i].effectiveBalance = balance; }, @@ -63,17 +61,17 @@ describe("state hashTreeRoot", () => { const idxs = indicesShuffled.slice(0, count); testCases.push({ id: `${count} balances`, - track: count >= 512, + noTrack: count < 512, fn: (state) => { for (const i of idxs) state.balances[i] = balance; }, }); } - for (const {id, track, fn} of testCases) { + for (const {id, noTrack, fn} of testCases) { itBench, TreeBacked>({ id: `state hashTreeRoot - ${id}`, - threshold: !track ? Infinity : undefined, + noThreshold: noTrack, beforeEach: () => { fn(stateOg); return stateOg; diff --git a/packages/beacon-state-transition/test/perf/allForks/slot/slots.test.ts b/packages/beacon-state-transition/test/perf/allForks/slot/slots.test.ts index f5072d51ba..18e408c24b 100644 --- a/packages/beacon-state-transition/test/perf/allForks/slot/slots.test.ts +++ b/packages/beacon-state-transition/test/perf/allForks/slot/slots.test.ts @@ -1,4 +1,4 @@ -import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {itBench} from "@dapplion/benchmark"; import {generatePerfTestCachedStatePhase0} from "../../util"; import {processSlot} from "../../../../src/allForks/slot"; import {State} from "../../types"; @@ -6,8 +6,6 @@ import {State} from "../../types"; // Test advancing through an empty slot, without any epoch transition describe("processSlot", () => { - setBenchOpts({maxMs: 60 * 1000}); - for (const slotCount of [1, 32]) { itBench({ id: `processSlot - ${slotCount} slots`, diff --git a/packages/beacon-state-transition/test/perf/allForks/util/epochContext.test.ts b/packages/beacon-state-transition/test/perf/allForks/util/epochContext.test.ts index 30038a0373..342f5572b8 100644 --- a/packages/beacon-state-transition/test/perf/allForks/util/epochContext.test.ts +++ b/packages/beacon-state-transition/test/perf/allForks/util/epochContext.test.ts @@ -1,5 +1,5 @@ import {Epoch} from "@chainsafe/lodestar-types"; -import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {itBench} from "@dapplion/benchmark"; import {allForks, computeEpochAtSlot} from "../../../../src"; import {generatePerfTestCachedStatePhase0, numValidators} from "../../util"; @@ -22,8 +22,6 @@ describe("epochCtx.getCommitteeAssignments", () => { if (state.validators.length !== numValidators) throw Error("constant numValidators is wrong"); }); - setBenchOpts({maxMs: 10 * 1000}); - // the new way of getting attester duties for (const reqCount of [1, 100, 1000]) { const validatorCount = numValidators; @@ -34,7 +32,7 @@ describe("epochCtx.getCommitteeAssignments", () => { itBench({ id: `getCommitteeAssignments - req ${reqCount} vs - ${validatorCount} vc`, // Only run for 1000 in CI to ensure performance does not degrade - threshold: reqCount < 1000 ? Infinity : undefined, + noThreshold: reqCount < 1000, fn: () => { state.epochCtx.getCommitteeAssignments(epoch, indices); }, diff --git a/packages/beacon-state-transition/test/perf/altair/block/processBlock.test.ts b/packages/beacon-state-transition/test/perf/altair/block/processBlock.test.ts index 00e79afb6c..3adc05966e 100644 --- a/packages/beacon-state-transition/test/perf/altair/block/processBlock.test.ts +++ b/packages/beacon-state-transition/test/perf/altair/block/processBlock.test.ts @@ -1,4 +1,4 @@ -import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {itBench} from "@dapplion/benchmark"; import { ACTIVE_PRESET, MAX_ATTESTATIONS, @@ -66,8 +66,6 @@ import {StateBlock} from "../../types"; // describe("altair processBlock", () => { - setBenchOpts({maxMs: 60 * 1000, minRuns: 10}); - if (ACTIVE_PRESET !== PresetName.mainnet) { throw Error(`ACTIVE_PRESET ${ACTIVE_PRESET} must be mainnet`); } diff --git a/packages/beacon-state-transition/test/perf/altair/epoch/epoch.test.ts b/packages/beacon-state-transition/test/perf/altair/epoch/epoch.test.ts index 764f4bef8f..8106b39f8c 100644 --- a/packages/beacon-state-transition/test/perf/altair/epoch/epoch.test.ts +++ b/packages/beacon-state-transition/test/perf/altair/epoch/epoch.test.ts @@ -9,7 +9,9 @@ const slot = computeStartSlotAtEpoch(epoch) - 1; const stateId = `${network}_e${epoch}`; describe(`altair processEpoch - ${stateId}`, () => { - setBenchOpts({maxMs: 60 * 1000, minRuns: 10}); + setBenchOpts({ + yieldEventLoopAfterEach: true, // So SubTree(s)'s WeakRef can be garbage collected https://github.com/nodejs/node/issues/39902 + }); const stateOg = beforeValue(async () => { const state = await getNetworkCachedState(network, slot, 300_000); @@ -19,6 +21,7 @@ describe(`altair processEpoch - ${stateId}`, () => { itBench({ id: `altair processEpoch - ${stateId}`, + yieldEventLoopAfterEach: true, // So SubTree(s)'s WeakRef can be garbage collected https://github.com/nodejs/node/issues/39902 beforeEach: () => stateOg.value.clone() as CachedBeaconState, fn: (state) => { const epochProcess = allForks.beforeProcessEpoch(state); @@ -31,7 +34,7 @@ describe(`altair processEpoch - ${stateId}`, () => { // Only in local environment compute a full breakdown of the cost of each step describe(`altair processEpoch steps - ${stateId}`, () => { - setBenchOpts({threshold: Infinity, minRuns: 10}); + setBenchOpts({noThreshold: true}); benchmarkAltairEpochSteps(stateOg, stateId); }); diff --git a/packages/beacon-state-transition/test/perf/altair/epoch/processInactivityUpdates.test.ts b/packages/beacon-state-transition/test/perf/altair/epoch/processInactivityUpdates.test.ts index ab323f4f57..a1ea4fe472 100644 --- a/packages/beacon-state-transition/test/perf/altair/epoch/processInactivityUpdates.test.ts +++ b/packages/beacon-state-transition/test/perf/altair/epoch/processInactivityUpdates.test.ts @@ -1,4 +1,4 @@ -import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {itBench} from "@dapplion/benchmark"; import {altair} from "../../../../src"; import {FlagFactors, generateBalanceDeltasEpochProcess} from "../../phase0/epoch/util"; import {StateAltairEpoch} from "../../types"; @@ -20,8 +20,6 @@ import {mutateInactivityScores} from "./util"; // - all inactivityScores > 0 describe("altair processInactivityUpdates", () => { - setBenchOpts({maxMs: 60 * 1000, minRuns: 10}); - const vc = numValidators; const testCases: {id: string; isInInactivityLeak: boolean; flagFactors: FlagFactors; factorWithPositive: number}[] = [ @@ -42,6 +40,7 @@ describe("altair processInactivityUpdates", () => { for (const {id, isInInactivityLeak, flagFactors, factorWithPositive} of testCases) { itBench({ id: `altair processInactivityUpdates - ${vc} ${id}`, + yieldEventLoopAfterEach: true, // So SubTree(s)'s WeakRef can be garbage collected https://github.com/nodejs/node/issues/39902 before: () => { const state = generatePerfTestCachedStateAltair({goBackOneSlot: true}); const epochProcess = generateBalanceDeltasEpochProcess(state, isInInactivityLeak, flagFactors); diff --git a/packages/beacon-state-transition/test/perf/altair/epoch/processParticipationFlagUpdates.test.ts b/packages/beacon-state-transition/test/perf/altair/epoch/processParticipationFlagUpdates.test.ts index f16c99dbeb..bdcded583d 100644 --- a/packages/beacon-state-transition/test/perf/altair/epoch/processParticipationFlagUpdates.test.ts +++ b/packages/beacon-state-transition/test/perf/altair/epoch/processParticipationFlagUpdates.test.ts @@ -1,4 +1,4 @@ -import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {itBench} from "@dapplion/benchmark"; import {altair} from "../../../../src"; import {StateAltair} from "../../types"; import {generatePerfTestCachedStateAltair, numValidators} from "../../util"; @@ -6,12 +6,11 @@ import {generatePerfTestCachedStateAltair, numValidators} from "../../util"; // PERF: Cost = 'proportional' to $VALIDATOR_COUNT. Just copies a tree and recreates another describe("altair processParticipationFlagUpdates", () => { - setBenchOpts({maxMs: 60 * 1000, minRuns: 10}); - const vc = numValidators; itBench({ id: `altair processParticipationFlagUpdates - ${vc} anycase`, + yieldEventLoopAfterEach: true, // So SubTree(s)'s WeakRef can be garbage collected https://github.com/nodejs/node/issues/39902 before: () => generatePerfTestCachedStateAltair({goBackOneSlot: true}), beforeEach: (state) => state.clone(), fn: (state) => altair.processParticipationFlagUpdates(state), diff --git a/packages/beacon-state-transition/test/perf/altair/epoch/processRewardsAndPenalties.test.ts b/packages/beacon-state-transition/test/perf/altair/epoch/processRewardsAndPenalties.test.ts index 63ab089f6b..397a7d41f2 100644 --- a/packages/beacon-state-transition/test/perf/altair/epoch/processRewardsAndPenalties.test.ts +++ b/packages/beacon-state-transition/test/perf/altair/epoch/processRewardsAndPenalties.test.ts @@ -1,4 +1,4 @@ -import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {itBench} from "@dapplion/benchmark"; import {altair} from "../../../../src"; import {generatePerfTestCachedStateAltair, numValidators} from "../../util"; import {FlagFactors, generateBalanceDeltasEpochProcess} from "../../phase0/epoch/util"; @@ -13,8 +13,6 @@ import {mutateInactivityScores} from "./util"; // inactivityScore > 0 describe("altair processRewardsAndPenalties", () => { - setBenchOpts({maxMs: 60 * 1000, minRuns: 10}); - const vc = numValidators; const testCases: {id: string; isInInactivityLeak: boolean; flagFactors: FlagFactors; factorWithPositive: number}[] = [ { @@ -43,6 +41,7 @@ describe("altair processRewardsAndPenalties", () => { for (const {id, isInInactivityLeak, flagFactors, factorWithPositive} of testCases) { itBench({ id: `altair processRewardsAndPenalties - ${vc} ${id}`, + yieldEventLoopAfterEach: true, // So SubTree(s)'s WeakRef can be garbage collected https://github.com/nodejs/node/issues/39902 before: () => { const state = generatePerfTestCachedStateAltair({goBackOneSlot: true}); const epochProcess = generateBalanceDeltasEpochProcess(state, isInInactivityLeak, flagFactors); diff --git a/packages/beacon-state-transition/test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts b/packages/beacon-state-transition/test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts index 2f3a5b59ea..a8b0ef3585 100644 --- a/packages/beacon-state-transition/test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts +++ b/packages/beacon-state-transition/test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts @@ -1,16 +1,15 @@ -import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {itBench} from "@dapplion/benchmark"; import {StateAltair} from "../../types"; import {generatePerfTestCachedStateAltair, numValidators} from "../../util"; // PERF: Cost = once per epoch compute committee, proportional to $VALIDATOR_COUNT describe("altair processSyncCommitteeUpdates", () => { - setBenchOpts({maxMs: 60 * 1000, minRuns: 10}); - const vc = numValidators; itBench({ id: `altair processSyncCommitteeUpdates - ${vc}`, + yieldEventLoopAfterEach: true, // So SubTree(s)'s WeakRef can be garbage collected https://github.com/nodejs/node/issues/39902 before: () => generatePerfTestCachedStateAltair({goBackOneSlot: true}), beforeEach: (state) => state.clone(), fn: (state) => state.rotateSyncCommittee(), diff --git a/packages/beacon-state-transition/test/perf/dataStructures/arrayish.test.ts b/packages/beacon-state-transition/test/perf/dataStructures/arrayish.test.ts index a441f14ca0..fc2078ad43 100644 --- a/packages/beacon-state-transition/test/perf/dataStructures/arrayish.test.ts +++ b/packages/beacon-state-transition/test/perf/dataStructures/arrayish.test.ts @@ -46,7 +46,7 @@ const runsFactor = 1000; describe("Tree (persistent-merkle-tree)", () => { // Don't track regressions in CI - setBenchOpts({maxMs: 10 * 1000, threshold: Infinity}); + setBenchOpts({noThreshold: true}); const d = 40; const gih = toGindex(d, BigInt(ih)); @@ -58,7 +58,7 @@ describe("Tree (persistent-merkle-tree)", () => { tree = getTree(d, n); }); - itBench({id: `Tree ${d} ${n} create`, timeout: 60_000}, () => { + itBench({id: `Tree ${d} ${n} create`, timeoutBench: 60_000}, () => { getTree(d, n); }); @@ -101,7 +101,7 @@ describe("Tree (persistent-merkle-tree)", () => { describe("MutableVector", () => { // Don't track regressions in CI - setBenchOpts({maxMs: 10 * 1000, threshold: Infinity}); + setBenchOpts({noThreshold: true}); let items: number[]; let mutableVector: MutableVector; @@ -143,7 +143,7 @@ describe("MutableVector", () => { describe("Array", () => { // Don't track regressions in CI - setBenchOpts({maxMs: 10 * 1000, threshold: Infinity}); + setBenchOpts({noThreshold: true}); let arr: number[]; diff --git a/packages/beacon-state-transition/test/perf/misc/aggregationBits.test.ts b/packages/beacon-state-transition/test/perf/misc/aggregationBits.test.ts index 652dfac555..85bdec06da 100644 --- a/packages/beacon-state-transition/test/perf/misc/aggregationBits.test.ts +++ b/packages/beacon-state-transition/test/perf/misc/aggregationBits.test.ts @@ -5,7 +5,7 @@ import {BitList, List, readonlyValues, TreeBacked} from "@chainsafe/ssz"; import {zipIndexesCommitteeBits} from "../../../src"; describe("aggregationBits", () => { - setBenchOpts({maxMs: 60 * 1000, threshold: Infinity}); + setBenchOpts({noThreshold: true}); const len = MAX_VALIDATORS_PER_COMMITTEE; const idPrefix = `aggregationBits - ${len} els`; diff --git a/packages/beacon-state-transition/test/perf/misc/rootEquals.test.ts b/packages/beacon-state-transition/test/perf/misc/rootEquals.test.ts index 94b6293833..0c9ba3d069 100644 --- a/packages/beacon-state-transition/test/perf/misc/rootEquals.test.ts +++ b/packages/beacon-state-transition/test/perf/misc/rootEquals.test.ts @@ -10,7 +10,7 @@ import {byteArrayEquals, fromHexString} from "@chainsafe/ssz"; // byteArrayEquals with valueOf() 853971.0 ops/s 1.171000 us/op 9963051 runs 16.07 s describe("root equals", () => { - setBenchOpts({maxMs: 60 * 1000, threshold: Infinity}); + setBenchOpts({noThreshold: true}); const stateRoot = fromHexString("0x6c86ca3c4c6688cf189421b8a68bf2dbc91521609965e6f4e207d44347061fee"); const rootTree = ssz.Root.createTreeBackedFromStruct(stateRoot); diff --git a/packages/beacon-state-transition/test/perf/phase0/block/processBlock.test.ts b/packages/beacon-state-transition/test/perf/phase0/block/processBlock.test.ts index d4d55a0ec4..f20ead289f 100644 --- a/packages/beacon-state-transition/test/perf/phase0/block/processBlock.test.ts +++ b/packages/beacon-state-transition/test/perf/phase0/block/processBlock.test.ts @@ -1,4 +1,4 @@ -import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {itBench} from "@dapplion/benchmark"; import { ACTIVE_PRESET, MAX_ATTESTATIONS, @@ -68,8 +68,6 @@ import {StateBlock} from "../../types"; // describe("phase0 processBlock", () => { - setBenchOpts({maxMs: 60 * 1000, minRuns: 10}); - if (ACTIVE_PRESET !== PresetName.mainnet) { throw Error(`ACTIVE_PRESET ${ACTIVE_PRESET} must be mainnet`); } diff --git a/packages/beacon-state-transition/test/perf/phase0/epoch/afterProcessEpoch.test.ts b/packages/beacon-state-transition/test/perf/phase0/epoch/afterProcessEpoch.test.ts index bff8f2f5bf..ef0c931e59 100644 --- a/packages/beacon-state-transition/test/perf/phase0/epoch/afterProcessEpoch.test.ts +++ b/packages/beacon-state-transition/test/perf/phase0/epoch/afterProcessEpoch.test.ts @@ -1,4 +1,4 @@ -import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {itBench} from "@dapplion/benchmark"; import {allForks} from "../../../../src"; import {StateEpoch} from "../../types"; import {generatePerfTestCachedStatePhase0, perfStateId} from "../../util"; @@ -7,10 +7,9 @@ import {generatePerfTestCachedStatePhase0, perfStateId} from "../../util"; // network conditions. See also individual benchmarks for shuffling computations. describe("phase0 afterProcessEpoch", () => { - setBenchOpts({maxMs: 60 * 1000, minRuns: 10}); - itBench({ id: `phase0 afterProcessEpoch - ${perfStateId}`, + yieldEventLoopAfterEach: true, // So SubTree(s)'s WeakRef can be garbage collected https://github.com/nodejs/node/issues/39902 before: () => { const state = generatePerfTestCachedStatePhase0({goBackOneSlot: true}); const epochProcess = allForks.beforeProcessEpoch(state); diff --git a/packages/beacon-state-transition/test/perf/phase0/epoch/beforeProcessEpoch.test.ts b/packages/beacon-state-transition/test/perf/phase0/epoch/beforeProcessEpoch.test.ts index 0c898e5646..570ec87206 100644 --- a/packages/beacon-state-transition/test/perf/phase0/epoch/beforeProcessEpoch.test.ts +++ b/packages/beacon-state-transition/test/perf/phase0/epoch/beforeProcessEpoch.test.ts @@ -1,4 +1,4 @@ -import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {itBench} from "@dapplion/benchmark"; import {allForks} from "../../../../src"; import {State} from "../../types"; import {generatePerfTestCachedStatePhase0, perfStateId} from "../../util"; @@ -9,10 +9,9 @@ import {generatePerfTestCachedStatePhase0, perfStateId} from "../../util"; // 3. Iterate over status to compute total balances. Cost = 'proportional' to $VALIDATOR_COUNT not network conditions describe("phase0 beforeProcessEpoch", () => { - setBenchOpts({maxMs: 60 * 1000, minRuns: 10}); - itBench({ id: `phase0 beforeProcessEpoch - ${perfStateId}`, + yieldEventLoopAfterEach: true, // So SubTree(s)'s WeakRef can be garbage collected https://github.com/nodejs/node/issues/39902 before: () => generatePerfTestCachedStatePhase0({goBackOneSlot: true}) as allForks.CachedBeaconState, beforeEach: (state) => state.clone(), diff --git a/packages/beacon-state-transition/test/perf/phase0/epoch/epoch.test.ts b/packages/beacon-state-transition/test/perf/phase0/epoch/epoch.test.ts index d4f9c3642d..ccc4a1d61c 100644 --- a/packages/beacon-state-transition/test/perf/phase0/epoch/epoch.test.ts +++ b/packages/beacon-state-transition/test/perf/phase0/epoch/epoch.test.ts @@ -10,7 +10,9 @@ const slot = computeStartSlotAtEpoch(epoch) - 1; const stateId = `${network}_e${epoch}`; describe(`phase0 processEpoch - ${stateId}`, () => { - setBenchOpts({maxMs: 60 * 1000, minRuns: 10}); + setBenchOpts({ + yieldEventLoopAfterEach: true, // So SubTree(s)'s WeakRef can be garbage collected https://github.com/nodejs/node/issues/39902 + }); const stateOg = beforeValue(async () => { const state = await getNetworkCachedState(network, slot, 300_000); @@ -32,7 +34,7 @@ describe(`phase0 processEpoch - ${stateId}`, () => { // Only in local environment compute a full breakdown of the cost of each step describe(`phase0 processEpoch steps - ${stateId}`, () => { - setBenchOpts({threshold: Infinity, minRuns: 10}); + setBenchOpts({noThreshold: true}); benchmarkPhase0EpochSteps(stateOg, stateId); }); diff --git a/packages/beacon-state-transition/test/perf/phase0/epoch/processEffectiveBalanceUpdates.test.ts b/packages/beacon-state-transition/test/perf/phase0/epoch/processEffectiveBalanceUpdates.test.ts index 0adecf3a3f..8ef64b6eb3 100644 --- a/packages/beacon-state-transition/test/perf/phase0/epoch/processEffectiveBalanceUpdates.test.ts +++ b/packages/beacon-state-transition/test/perf/phase0/epoch/processEffectiveBalanceUpdates.test.ts @@ -1,4 +1,4 @@ -import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {itBench} from "@dapplion/benchmark"; import {ssz} from "@chainsafe/lodestar-types"; import {config} from "@chainsafe/lodestar-config/default"; import {allForks} from "../../../../src"; @@ -14,8 +14,6 @@ import {StateEpoch} from "../../types"; // statuses: All balances are low enough to trigger an effective balance change describe("phase0 processEffectiveBalanceUpdates", () => { - setBenchOpts({maxMs: 60 * 1000, minRuns: 5}); - const vc = numValidators; const testCases: {id: string; changeRatio: number}[] = [ // Normal (optimal) mainnet network conditions: No effectiveBalance is udpated @@ -32,6 +30,8 @@ describe("phase0 processEffectiveBalanceUpdates", () => { for (const {id, changeRatio} of testCases) { itBench({ id: `phase0 processEffectiveBalanceUpdates - ${vc} ${id}`, + yieldEventLoopAfterEach: true, // So SubTree(s)'s WeakRef can be garbage collected https://github.com/nodejs/node/issues/39902 + minRuns: 5, // Worst case is very slow before: () => getEffectiveBalanceTestData(vc, changeRatio), beforeEach: ({state, epochProcess}) => ({state: state.clone(), epochProcess}), fn: ({state, epochProcess}) => allForks.processEffectiveBalanceUpdates(state, epochProcess), diff --git a/packages/beacon-state-transition/test/perf/phase0/epoch/processRegistryUpdates.test.ts b/packages/beacon-state-transition/test/perf/phase0/epoch/processRegistryUpdates.test.ts index c0d24dc060..49b5f52de1 100644 --- a/packages/beacon-state-transition/test/perf/phase0/epoch/processRegistryUpdates.test.ts +++ b/packages/beacon-state-transition/test/perf/phase0/epoch/processRegistryUpdates.test.ts @@ -1,5 +1,4 @@ -import {itBench, setBenchOpts} from "@dapplion/benchmark"; -import {sleep} from "@chainsafe/lodestar-utils"; +import {itBench} from "@dapplion/benchmark"; import {allForks} from "../../../../src"; import {beforeProcessEpoch} from "../../../../src/allForks"; import {generatePerfTestCachedStatePhase0, numValidators} from "../../util"; @@ -17,8 +16,6 @@ import {StateEpoch} from "../../types"; // - indicesToEject: 0 describe("phase0 processRegistryUpdates", () => { - setBenchOpts({maxMs: 60 * 1000, minRuns: 5}); - const vc = numValidators; const testCases: {id: string; notTrack?: boolean; lengths: IndicesLengths}[] = [ // Normal (optimal) mainnet network conditions: No effectiveBalance is udpated @@ -57,15 +54,14 @@ describe("phase0 processRegistryUpdates", () => { for (const {id, notTrack, lengths} of testCases) { itBench({ id: `phase0 processRegistryUpdates - ${vc} ${id}`, - threshold: notTrack ? Infinity : undefined, + // WeakRef keeps a strong reference to its constructor value until the event loop ticks. + // Without this `sleep(0)` all the SubTree(s) created updating the validators registry + // won't be garabage collected causing an OOM crash. Tracking issue https://github.com/nodejs/node/issues/39902 + yieldEventLoopAfterEach: true, + minRuns: 5, // Worst case is very slow + noThreshold: notTrack, before: () => getRegistryUpdatesTestData(vc, lengths), - beforeEach: async ({state, epochProcess}) => { - // WeakRef keeps a strong reference to its constructor value until the event loop ticks. - // Without this `sleep(0)` all the SubTree(s) created updating the validators registry - // won't be garabage collected causing an OOM crash. Tracking issue https://github.com/nodejs/node/issues/39902 - await sleep(0); - return {state: state.clone(), epochProcess}; - }, + beforeEach: async ({state, epochProcess}) => ({state: state.clone(), epochProcess}), fn: ({state, epochProcess}) => allForks.processRegistryUpdates(state, epochProcess), }); } diff --git a/packages/beacon-state-transition/test/perf/phase0/epoch/processRewardsAndPenalties.test.ts b/packages/beacon-state-transition/test/perf/phase0/epoch/processRewardsAndPenalties.test.ts index 5b7cfb9a79..80c6049eb1 100644 --- a/packages/beacon-state-transition/test/perf/phase0/epoch/processRewardsAndPenalties.test.ts +++ b/packages/beacon-state-transition/test/perf/phase0/epoch/processRewardsAndPenalties.test.ts @@ -1,4 +1,4 @@ -import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {itBench} from "@dapplion/benchmark"; import {phase0} from "../../../../src"; import {generatePerfTestCachedStatePhase0, numValidators} from "../../util"; import {StatePhase0Epoch} from "../../types"; @@ -15,8 +15,6 @@ import {FlagFactors, generateBalanceDeltasEpochProcess} from "./util"; // - eligibleAttester: 98% describe("phase0 getAttestationDeltas", () => { - setBenchOpts({maxMs: 60 * 1000, minRuns: 10}); - const vc = numValidators; const testCases: {id: string; isInInactivityLeak: boolean; flagFactors: FlagFactors}[] = [ { @@ -43,6 +41,7 @@ describe("phase0 getAttestationDeltas", () => { for (const {id, isInInactivityLeak, flagFactors} of testCases) { itBench({ id: `phase0 getAttestationDeltas - ${vc} ${id}`, + yieldEventLoopAfterEach: true, // So SubTree(s)'s WeakRef can be garbage collected https://github.com/nodejs/node/issues/39902 before: () => { const state = generatePerfTestCachedStatePhase0({goBackOneSlot: true}); const epochProcess = generateBalanceDeltasEpochProcess(state, isInInactivityLeak, flagFactors); diff --git a/packages/beacon-state-transition/test/perf/phase0/epoch/processSlashingsAllForks.test.ts b/packages/beacon-state-transition/test/perf/phase0/epoch/processSlashingsAllForks.test.ts index e78fbdf1cf..be45e15900 100644 --- a/packages/beacon-state-transition/test/perf/phase0/epoch/processSlashingsAllForks.test.ts +++ b/packages/beacon-state-transition/test/perf/phase0/epoch/processSlashingsAllForks.test.ts @@ -1,4 +1,4 @@ -import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {itBench} from "@dapplion/benchmark"; import {allForks} from "../../../../src"; import {beforeProcessEpoch} from "../../../../src/allForks"; import {generatePerfTestCachedStatePhase0, numValidators} from "../../util"; @@ -11,8 +11,6 @@ import {StateEpoch} from "../../types"; // - On normal mainnet conditions indicesToSlash = 0 describe("phase0 processSlashings", () => { - setBenchOpts({maxMs: 60 * 1000, minRuns: 5}); - const vc = numValidators; const testCases: {id: string; indicesToSlashLen: number}[] = [ // Normal (optimal) mainnet network conditions: No slashings. Ignore this case since it does nothing @@ -27,6 +25,8 @@ describe("phase0 processSlashings", () => { for (const {id, indicesToSlashLen} of testCases) { itBench({ id: `phase0 processSlashings - ${vc} ${id}`, + yieldEventLoopAfterEach: true, // So SubTree(s)'s WeakRef can be garbage collected https://github.com/nodejs/node/issues/39902 + minRuns: 5, // Worst case is very slow before: () => getProcessSlashingsTestData(indicesToSlashLen), beforeEach: ({state, epochProcess}) => ({state: state.clone(), epochProcess}), fn: ({state, epochProcess}) => allForks.processRegistryUpdates(state, epochProcess), diff --git a/packages/beacon-state-transition/test/perf/shuffle/numberMath.test.ts b/packages/beacon-state-transition/test/perf/shuffle/numberMath.test.ts index abe7e1ec11..08e57b090a 100644 --- a/packages/beacon-state-transition/test/perf/shuffle/numberMath.test.ts +++ b/packages/beacon-state-transition/test/perf/shuffle/numberMath.test.ts @@ -1,8 +1,6 @@ -import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {itBench} from "@dapplion/benchmark"; describe.skip("shuffle number math ops", () => { - setBenchOpts({maxMs: 10 * 1000}); - const forRuns = 100e5; const j = forRuns / 2; diff --git a/packages/beacon-state-transition/test/perf/shuffle/shuffle.test.ts b/packages/beacon-state-transition/test/perf/shuffle/shuffle.test.ts index 4bf0a93ce9..1b723af1e0 100644 --- a/packages/beacon-state-transition/test/perf/shuffle/shuffle.test.ts +++ b/packages/beacon-state-transition/test/perf/shuffle/shuffle.test.ts @@ -1,4 +1,4 @@ -import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {itBench} from "@dapplion/benchmark"; import {unshuffleList} from "../../../src"; // Lightouse Lodestar @@ -7,8 +7,6 @@ import {unshuffleList} from "../../../src"; // 4000000 1.5617 s 4.9690 s (x3) describe("shuffle list", () => { - setBenchOpts({maxMs: 30 * 1000}); - // eslint-disable-next-line @typescript-eslint/naming-convention const seed = new Uint8Array([42, 32]); diff --git a/packages/lodestar/test/perf/api/impl/validator/attester.test.ts b/packages/lodestar/test/perf/api/impl/validator/attester.test.ts index 9e6cd502ba..da0d07b681 100644 --- a/packages/lodestar/test/perf/api/impl/validator/attester.test.ts +++ b/packages/lodestar/test/perf/api/impl/validator/attester.test.ts @@ -1,4 +1,4 @@ -import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {itBench} from "@dapplion/benchmark"; import {PointFormat} from "@chainsafe/bls"; import { generatePerfTestCachedStatePhase0, @@ -19,8 +19,6 @@ import { // ✓ getPubkeys - persistent - req 1000 vs - 200000 vc 56593.10 ops/s 17.67000 us/op - 111477 runs 2.00 s describe("api / impl / validator", () => { - setBenchOpts({maxMs: 10 * 1000}); - let state: ReturnType; before(function () { @@ -33,7 +31,7 @@ describe("api / impl / validator", () => { for (const reqCount of reqCounts) { itBench({ id: `getPubkeys - index2pubkey - req ${reqCount} vs - ${numValidators} vc`, - threshold: Infinity, + noThreshold: true, fn: () => { for (let i = 0; i < reqCount; i++) { const pubkey = state.index2pubkey[i]; @@ -46,7 +44,7 @@ describe("api / impl / validator", () => { for (const reqCount of reqCounts) { itBench({ id: `getPubkeys - validatorsArr - req ${reqCount} vs - ${numValidators} vc`, - threshold: Infinity, + noThreshold: true, fn: () => { for (let i = 0; i < reqCount; i++) { const validator = state.validators[i]; @@ -60,7 +58,7 @@ describe("api / impl / validator", () => { itBench({ id: `getPubkeys - persistent - req ${reqCount} vs - ${numValidators} vc`, // Only track regressions for 1000 in CI to ensure performance does not degrade - threshold: reqCount < 1000 ? Infinity : undefined, + noThreshold: reqCount < 1000, fn: () => { const validators = state.validators.persistent; for (let i = 0; i < reqCount; i++) { diff --git a/packages/lodestar/test/perf/bls/bls.test.ts b/packages/lodestar/test/perf/bls/bls.test.ts index b416002f01..a70a835503 100644 --- a/packages/lodestar/test/perf/bls/bls.test.ts +++ b/packages/lodestar/test/perf/bls/bls.test.ts @@ -1,10 +1,8 @@ -import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {itBench} from "@dapplion/benchmark"; import {bls, PublicKey, SecretKey, Signature} from "@chainsafe/bls"; import {linspace} from "../../../src/util/numpy"; describe("BLS ops", function () { - setBenchOpts({maxMs: 60 * 1000}); - type Keypair = {publicKey: PublicKey; secretKey: SecretKey}; type BlsSet = {publicKey: PublicKey; message: Uint8Array; signature: Signature}; diff --git a/packages/lodestar/test/perf/chain/opPools/aggregatedAttestationPool.test.ts b/packages/lodestar/test/perf/chain/opPools/aggregatedAttestationPool.test.ts index e837b51e65..ebae0b9427 100644 --- a/packages/lodestar/test/perf/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/lodestar/test/perf/chain/opPools/aggregatedAttestationPool.test.ts @@ -1,4 +1,4 @@ -import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {itBench} from "@dapplion/benchmark"; import {expect} from "chai"; import { allForks, @@ -16,8 +16,6 @@ import {ssz} from "@chainsafe/lodestar-types"; // getAttestationsForBlock // ✓ getAttestationsForBlock 4.410948 ops/s 226.7086 ms/op - 64 runs 51.8 s describe("getAttestationsForBlock", () => { - setBenchOpts({maxMs: 60 * 1000}); - let originalState: allForks.CachedBeaconState; before(function () { diff --git a/packages/lodestar/test/perf/chain/stateCache/stateContextCheckpointsCache.test.ts b/packages/lodestar/test/perf/chain/stateCache/stateContextCheckpointsCache.test.ts index 308ed45de3..80c664e777 100644 --- a/packages/lodestar/test/perf/chain/stateCache/stateContextCheckpointsCache.test.ts +++ b/packages/lodestar/test/perf/chain/stateCache/stateContextCheckpointsCache.test.ts @@ -5,7 +5,7 @@ import {generateCachedState} from "../../../utils/state"; import {CheckpointStateCache} from "../../../../src/chain/stateCache"; describe("CheckpointStateCache perf tests", function () { - setBenchOpts({maxMs: 10 * 1000, threshold: Infinity}); + setBenchOpts({noThreshold: true}); let state: CachedBeaconState; let checkpoint: phase0.Checkpoint; diff --git a/packages/lodestar/test/perf/chain/validation/aggregateAndProof.test.ts b/packages/lodestar/test/perf/chain/validation/aggregateAndProof.test.ts index 745ea68171..629d3714d3 100644 --- a/packages/lodestar/test/perf/chain/validation/aggregateAndProof.test.ts +++ b/packages/lodestar/test/perf/chain/validation/aggregateAndProof.test.ts @@ -1,12 +1,10 @@ -import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {itBench} from "@dapplion/benchmark"; import {ssz} from "@chainsafe/lodestar-types"; import {validateGossipAggregateAndProof} from "../../../../src/chain/validation"; import {generateTestCachedBeaconStateOnlyValidators} from "@chainsafe/lodestar-beacon-state-transition/test/perf/util"; import {getAggregateAndProofValidData} from "../../../utils/validationData/aggregateAndProof"; describe("validate gossip signedAggregateAndProof", () => { - setBenchOpts({maxMs: 60 * 1000}); - const vc = 64; const stateSlot = 100; diff --git a/packages/lodestar/test/perf/chain/validation/attestation.test.ts b/packages/lodestar/test/perf/chain/validation/attestation.test.ts index 8e1e988f40..29f3e3dc61 100644 --- a/packages/lodestar/test/perf/chain/validation/attestation.test.ts +++ b/packages/lodestar/test/perf/chain/validation/attestation.test.ts @@ -1,12 +1,10 @@ -import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {itBench} from "@dapplion/benchmark"; import {ssz} from "@chainsafe/lodestar-types"; import {validateGossipAttestation} from "../../../../src/chain/validation"; import {generateTestCachedBeaconStateOnlyValidators} from "@chainsafe/lodestar-beacon-state-transition/test/perf/util"; import {getAttestationValidData} from "../../../utils/validationData/attestation"; describe("validate gossip attestation", () => { - setBenchOpts({maxMs: 60 * 1000}); - const vc = 64; const stateSlot = 100; diff --git a/yarn.lock b/yarn.lock index 2ef90a3829..8a52835fbf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -521,10 +521,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@dapplion/benchmark@^0.2.0": - version "0.2.0" - resolved "https://registry.yarnpkg.com/@dapplion/benchmark/-/benchmark-0.2.0.tgz#0595ac6903e62be918bfc8fb6a557ecb0ded10d5" - integrity sha512-fPXqgCdV82uxqg1hxIYoXOSJHYaCOSDdf3AUMM33T21qGdBSjdGUbd4uXkw59Ec7rYNAuoahOTG9Albn9pLkRQ== +"@dapplion/benchmark@^0.2.2": + version "0.2.2" + resolved "https://registry.yarnpkg.com/@dapplion/benchmark/-/benchmark-0.2.2.tgz#cc9c78e5b8d2d04311bcf30cbc4c67783e2efc6f" + integrity sha512-1wkplmCOVo5U3vv6Vej9MkQ1KtWtTT7WIpwVKyZdNIPnLHG82X4JwySuaaXeAKwiMs0kQzE/+ax5gNwsfqqYpQ== dependencies: "@actions/cache" "^1.0.7" "@actions/github" "^5.0.0"