test: support .env.test to reuse env variables across the tests (#6408)

* Variables using .env file

* Add env variable support to normal tests

* Fix the order of variables

* Update the order of jobs

* Export the varible using actions toolkit

* Update the workflow tasks

* Fix the sim workflow

* Add .env.test file support to tests

* Fix the lint

* Update the task description

* Move lodestar preset to env file

* Fix the directory path

* newline

* Update debugging spec tests section

* Update the env variable for preset

* Fix the path for setup files

* Update code as per feedback

* Fix the e2e variables

* Update doc

* Fix the bash script

* Fix sim geth runne

* Update the env file

* Fix e2e tests

* Update the script tasks

* Update the script tasks

* Add minimal for e2e tests

* Add minimal for e2e tests

* Update comments in preset tests

* Downgrade nethermind version

* Load env file in e2e env

* Add the issue link in env variable

* Update bash script for failsafe current dir

* Fix the mistaken genesis extension for el nodes

* Add missing capella wait for one test

* Downgrade the geth version

* Update .env.test

---------

Co-authored-by: Nico Flaig <nflaig@protonmail.com>
This commit is contained in:
Nazar Hussain
2024-02-12 14:36:33 +01:00
committed by GitHub
parent 1e67579442
commit da0bcc8f19
26 changed files with 115 additions and 47 deletions

13
.env.test Normal file
View File

@@ -0,0 +1,13 @@
# We use these images during sim and e2e tests
# TODO: Upgrade Geth once the Nethermind issue is resolved else it's causing following error
# Rejected peer id=134e2c1a76745626 addr=192.168.0.3:9052 conn=staticdial err="useless peer"
GETH_DOCKER_IMAGE=ethereum/client-go:v1.11.6
# Use either image or local binary for the testing
GETH_BINARY_DIR=
LIGHTHOUSE_DOCKER_IMAGE=sigp/lighthouse:v4.6.0-amd64-modern-dev
# We can't upgrade nethermind further due to genesis hash mismatch with the geth
# https://github.com/NethermindEth/nethermind/issues/6683
NETHERMIND_DOCKER_IMAGE=nethermind/nethermind:1.18.2
# We mostly use mainnet for unit testing
# Changing this value may impact the tests which are written with mainnet in mind
LODESTAR_PRESET=mainnet

5
.github/actions/dotenv/action.yml vendored Normal file
View File

@@ -0,0 +1,5 @@
name: "Setup env variables using .env file"
description: "Load .env file from root of repo and setup for CI runner"
runs:
using: "node20"
main: index.js

29
.github/actions/dotenv/index.js vendored Normal file
View File

@@ -0,0 +1,29 @@
const fs = require("fs");
const core = require("@actions/core");
const dotEnv = require("dotenv");
const envFile = ".env.test";
if (!fs.existsSync(envFile)) {
core.setFailed("File .env not found");
}
const result = dotEnv.config({path: envFile});
if (result.error) {
core.setFailed(result.error.message);
} else {
core.setOutput("env", result.parsed);
core.info("Env file loaded");
core.info("Populating env variables...");
for (const key in result.parsed) {
const value = result.parsed[key];
core.info(`${key}=${value}`);
// Export variable
core.exportVariable(key, value);
// Set to output so it can be used in as the input for the next job/step
core.setOutput(key, value);
}
}

View File

@@ -22,11 +22,6 @@ on:
type: number
default: 40
env:
GETH_DOCKER_IMAGE: ethereum/client-go:v1.13.11
LIGHTHOUSE_DOCKER_IMAGE: sigp/lighthouse:latest-amd64-modern-dev
NETHERMIND_DOCKER_IMAGE: nethermind/nethermind:1.25.3
jobs:
tests-sim:
name: Sim tests
@@ -58,6 +53,9 @@ jobs:
if: steps.cache-deps.outputs.cache-hit == 'true'
# </common-build>
- name: Load env variables
uses: ./.github/actions/dotenv
- name: Download required docker images before running tests
run: |
docker pull ${{env.GETH_DOCKER_IMAGE}}

View File

@@ -12,10 +12,6 @@ on:
pull_request:
workflow_dispatch:
env:
GETH_DOCKER_IMAGE: ethereum/client-go:v1.11.6
NETHERMIND_DOCKER_IMAGE: nethermind/nethermind:1.18.0
jobs:
build:
name: Build
@@ -230,6 +226,9 @@ jobs:
key: ${{ runner.os }}-node-${{ matrix.node }}-${{ github.sha }}
fail-on-cache-miss: true
- name: Load env variables
uses: ./.github/actions/dotenv
- name: Run the e2e test environment
run: scripts/run_e2e_env.sh start

View File

@@ -35,7 +35,7 @@ To run tests:
Note that to run `test:e2e`, first ensure that the environment is correctly setup by running the `run_e2e_env.sh` script.
```sh
GETH_DOCKER_IMAGE=ethereum/client-go:v1.11.6 NETHERMIND_DOCKER_IMAGE=nethermind/nethermind:1.18.0 ./scripts/run_e2e_env.sh start
./scripts/run_e2e_env.sh start
```
Similarly, run `yarn download-spec-tests` before running `yarn test:spec`.
@@ -63,9 +63,9 @@ If you observe following error running any of the test files that means you are
- Spec tests often compare full expected vs actual states in JSON format.
- A single logical error can cause many spec tests to fail. To focus on a single test at a time you can use vitest's option `--bail 1` to stop at the first failed test
- To then run only that failed test you can run against a specific file as use vitest's filters option `-t <pattern>` to run only one case
- Before running the tests, make sure to switch to the package directory (e.g. `packages/beacon-node`) to speed up test execution
```sh
cd packages/beacon-node
LODESTAR_PRESET=minimal yarn vitest --run --bail 1 --config vitest.spec.config.ts test/spec/presets/sanity.test.ts -t attester_slashing
```

View File

@@ -12,7 +12,7 @@ There are a number of sim tests that are available and each has a slightly diffe
### Environment Variables
To see what typical values for these are check out the `test-sim.yaml` workflow file in the `.github/workflows` directory.
To see what typical values for these are check out the `.env.test` file in the root directory.
- `GETH_DOCKER_IMAGE`: The geth docker image that will be used
- `NETHERMIND_IMAGE`: The nethermind docker image that will be used
@@ -23,10 +23,7 @@ To see what typical values for these are check out the `test-sim.yaml` workflow
The multi-fork sim test checks most of the functionality Lodestar provides. Is verifies that Lodestar is capable of peering, moving through all of the forks and using various sync methods in a testnet environment. Lodestar is tested with both Geth and Nethermind as the execution client. It also checks a Lighthouse/Geth node for cross client compatibility.
```sh
GETH_DOCKER_IMAGE=ethereum/client-go:v1.11.6 \
LIGHTHOUSE_DOCKER_IMAGE=sigp/lighthouse:latest-amd64-modern-dev \
NETHERMIND_DOCKER_IMAGE=nethermind/nethermind:1.18.0 \
yarn workspace @chainsafe/lodestar test:sim:multifork
yarn workspace @chainsafe/lodestar test:sim:multifork
```
### `test:sim:endpoints`
@@ -34,8 +31,7 @@ GETH_DOCKER_IMAGE=ethereum/client-go:v1.11.6 \
This tests that various endpoints of the beacon node and validator client are working as expected.
```sh
GETH_DOCKER_IMAGE=ethereum/client-go:v1.11.6 \
yarn workspace @chainsafe/lodestar test:sim:endpoints
yarn workspace @chainsafe/lodestar test:sim:endpoints
```
### `test:sim:deneb`
@@ -47,9 +43,7 @@ This test is still included in our CI but is no longer as important as it once w
Checks that Lodestar is compatible with other consensus validators and vice-versa. All tests use Geth as the EL.
```sh
GETH_DOCKER_IMAGE=ethereum/client-go:v1.11.6 \
LIGHTHOUSE_DOCKER_IMAGE=sigp/lighthouse:latest-amd64-modern-dev \
yarn workspace @chainsafe/lodestar test:sim:mixedclient
yarn workspace @chainsafe/lodestar test:sim:mixedclient
```
## Sim Test Infrastructure

View File

@@ -38,13 +38,14 @@
"test-coverage:e2e-sim": "c8 --config .c8rc.json --report-dir coverage/e2e-sim/ --all npm run test:e2e:sim",
"test-coverage:spec": "c8 --config .c8rc.json --report-dir coverage/spec/ --all npm run test:spec",
"benchmark": "yarn benchmark:files 'packages/*/test/perf/**/*.test.ts'",
"benchmark:files": "LODESTAR_PRESET=mainnet NODE_OPTIONS='--max-old-space-size=4096 --loader=ts-node/esm' benchmark --config .benchrc.yaml --defaultBranch unstable",
"benchmark:files": "NODE_OPTIONS='--max-old-space-size=4096 --loader=ts-node/esm' benchmark --config .benchrc.yaml --defaultBranch unstable",
"release:create-rc": "node scripts/release/create_rc.mjs",
"release:tag-rc": "node scripts/release/tag_rc.mjs",
"release:tag-stable": "node scripts/release/tag_stable.mjs",
"release:publish": "lerna publish from-package --yes --no-verify-access"
},
"devDependencies": {
"@actions/core": "^1.10.1",
"@chainsafe/eslint-plugin-node": "^11.2.3",
"@dapplion/benchmark": "^0.2.4",
"@types/mocha": "^10.0.6",
@@ -54,6 +55,7 @@
"@vitest/coverage-v8": "^1.2.1",
"@vitest/browser": "^1.2.1",
"crypto-browserify": "^3.12.0",
"dotenv": "^16.4.1",
"electron": "^26.2.2",
"eslint": "^8.50.0",
"eslint-import-resolver-typescript": "^3.6.1",

View File

@@ -75,8 +75,8 @@
"lint": "eslint --color --ext .ts src/ test/",
"lint:fix": "yarn run lint --fix",
"test": "yarn test:unit && yarn test:e2e",
"test:unit:minimal": "vitest --run --segfaultRetry 3 --dir test/unit/",
"test:unit:mainnet": "LODESTAR_PRESET=mainnet vitest --run --dir test/unit-mainnet",
"test:unit:minimal": "LODESTAR_PRESET=minimal vitest --run --segfaultRetry 3 --dir test/unit/",
"test:unit:mainnet": "LODESTAR_PRESET=mainnet vitest --run --segfaultRetry 3 --dir test/unit-mainnet",
"test:unit": "wrapper() { yarn test:unit:minimal $@ && yarn test:unit:mainnet $@; }; wrapper",
"test:e2e": "LODESTAR_PRESET=minimal vitest --run --segfaultRetry 3 --config vitest.e2e.config.ts --dir test/e2e",
"test:sim": "vitest --run test/sim/**/*.test.ts",

View File

@@ -32,11 +32,11 @@
"lint:fix": "yarn run lint --fix",
"test:unit": "vitest --run --dir test/unit/",
"test:e2e": "vitest --run --config vitest.e2e.config.ts --dir test/e2e/",
"test:sim:multifork": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/multi_fork.test.ts",
"test:sim:mixedclient": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/mixed_client.test.ts",
"test:sim:endpoints": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/endpoints.test.ts",
"test:sim:deneb": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/deneb.test.ts",
"test:sim:backup_eth_provider": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/backup_eth_provider.test.ts",
"test:sim:multifork": "LODESTAR_PRESET=minimal DOTENV_CONFIG_PATH=../../.env.test node -r dotenv/config --loader ts-node/esm test/sim/multi_fork.test.ts",
"test:sim:mixedclient": "LODESTAR_PRESET=minimal DOTENV_CONFIG_PATH=../../.env.test node -r dotenv/config --loader ts-node/esm test/sim/mixed_client.test.ts",
"test:sim:endpoints": "LODESTAR_PRESET=minimal DOTENV_CONFIG_PATH=../../.env.test node -r dotenv/config --loader ts-node/esm test/sim/endpoints.test.ts",
"test:sim:deneb": "LODESTAR_PRESET=minimal DOTENV_CONFIG_PATH=../../.env.test node -r dotenv/config --loader ts-node/esm test/sim/deneb.test.ts",
"test:sim:backup_eth_provider": "LODESTAR_PRESET=minimal DOTENV_CONFIG_PATH=../../.env.test node -r dotenv/config --loader ts-node/esm test/sim/backup_eth_provider.test.ts",
"test": "yarn test:unit && yarn test:e2e",
"check-readme": "typescript-docs-verifier"
},

View File

@@ -14,6 +14,7 @@ const {forkConfig} = defineSimTestConfig({
BELLATRIX_FORK_EPOCH: bellatrixForkEpoch,
CAPELLA_FORK_EPOCH: capellaForkEpoch,
runTillEpoch: Infinity,
initialNodes: 2,
});
const env = await SimulationEnvironment.initWithDefaults(

View File

@@ -21,7 +21,7 @@ export const generateGethNode: ExecutionNodeGenerator<ExecutionClient.Geth> = (o
const {id, mode, ttd, address, mining, clientOptions, nodeIndex} = opts;
const ports = getNodePorts(nodeIndex);
const isDocker = process.env.GETH_DOCKER_IMAGE !== undefined;
const isDocker = !!process.env.GETH_DOCKER_IMAGE;
const binaryPath = isDocker ? "" : `${process.env.GETH_BINARY_DIR}/geth`;
const {rootDir, rootDirMounted, genesisFilePathMounted, logFilePath, jwtsecretFilePathMounted} = getNodeMountedPaths(
opts.paths,

View File

@@ -23,7 +23,7 @@ export function getNodePaths<
return {
rootDir: executionRootDir,
dataDir: path.join(executionRootDir, "data"),
genesisFilePath: path.join(executionRootDir, "genesis.ssz"),
genesisFilePath: path.join(executionRootDir, "genesis.json"),
jwtsecretFilePath: path.join(executionRootDir, "jwtsecret.txt"),
logFilePath: path.join(logsDir, `${id}-${client}.log`),
} as R;

View File

@@ -61,7 +61,7 @@
"test:browsers:chrome": "vitest --run --browser chrome --config ./vitest.browser.config.ts --dir test/unit",
"test:browsers:firefox": "vitest --run --browser firefox --config ./vitest.browser.config.ts --dir test/unit",
"test:browsers:electron": "echo 'Electron tests will be introduced back in the future as soon vitest supports electron.'",
"test:e2e": "LODESTAR_PRESET=minimal vitest --run --config vitest.e2e.config.ts --dir test/e2e",
"test:e2e": "vitest --run --config vitest.e2e.config.ts --dir test/e2e",
"check-readme": "typescript-docs-verifier"
},
"types": "lib/index.d.ts",

View File

@@ -58,7 +58,7 @@
"test:browsers:chrome": "vitest --run --browser chrome --config ./vitest.browser.config.ts --dir test/unit",
"test:browsers:firefox": "vitest --run --browser firefox --config ./vitest.browser.config.ts --dir test/unit",
"test:browsers:electron": "echo 'Electron tests will be introduced back in the future as soon vitest supports electron.'",
"test:e2e": "LODESTAR_PRESET=minimal vitest --run --config vitest.e2e.config.ts --dir test/e2e/",
"test:e2e": "vitest --run --config vitest.e2e.config.ts --dir test/e2e/",
"check-readme": "typescript-docs-verifier"
},
"repository": {

View File

@@ -21,8 +21,8 @@ describe("Override preset", function () {
vi.setConfig({testTimeout: 30_000});
it("Should correctly override preset", async () => {
// These commands can not run with minimal preset
if (process.env.LODESTAR_PRESET === "minimal") delete process.env.LODESTAR_PRESET;
// `LODESTAR_PRESET` must not be set to properly test preset override
if (process.env.LODESTAR_PRESET) delete process.env.LODESTAR_PRESET;
await exec(`node --loader ts-node/esm ${path.join(__dirname, scriptNames.ok)}`);
});

View File

@@ -21,8 +21,8 @@ describe("setPreset", function () {
vi.setConfig({testTimeout: 30_000});
it("Should correctly set preset", async () => {
// These commands can not run with minimal preset
if (process.env.LODESTAR_PRESET === "minimal") delete process.env.LODESTAR_PRESET;
// `LODESTAR_PRESET` must not be set to properly test setting preset
if (process.env.LODESTAR_PRESET) delete process.env.LODESTAR_PRESET;
await exec(`node --loader ts-node/esm ${path.join(__dirname, scriptNames.ok)}`);
});

View File

@@ -3,14 +3,16 @@ import {describe, it, expect, beforeAll} from "vitest";
import Web3 from "web3";
import {LCTransport} from "../../src/interfaces.js";
import {createVerifiedExecutionProvider} from "../../src/web3_provider.js";
import {rpcUrl, beaconUrl, config} from "../utils/e2e_env.js";
import {rpcUrl, beaconUrl, config, waitForCapellaFork} from "../utils/e2e_env.js";
import {getVerificationFailedMessage} from "../../src/utils/json_rpc.js";
/* prettier-ignore */
describe("web3_batch_requests", function () {
let web3: Web3;
beforeAll(() => {
beforeAll(async () => {
await waitForCapellaFork();
const {provider} = createVerifiedExecutionProvider(new Web3.providers.HttpProvider(rpcUrl), {
transport: LCTransport.Rest,
urls: [beaconUrl],

View File

@@ -64,7 +64,7 @@
"test": "yarn test:unit",
"test:constants:minimal": "LODESTAR_PRESET=minimal vitest --run --dir test/constants/",
"test:constants:mainnet": "LODESTAR_PRESET=mainnet vitest --run --dir test/constants/",
"test:unit": "wrapper() { yarn test:constants:minimal $@ && yarn test:constants:mainnet $@ && vitest --run --dir test/unit/ $@; }; wrapper",
"test:unit": "wrapper() { yarn test:constants:minimal $@ && yarn test:constants:mainnet $@ && LODESTAR_PRESET=mainnet vitest --run --dir test/unit/ $@; }; wrapper",
"test:browsers": "yarn test:browsers:chrome && yarn test:browsers:firefox && yarn test:browsers:electron",
"test:browsers:chrome": "vitest --run --browser chrome --config ./vitest.browser.config.ts --dir test/unit",
"test:browsers:firefox": "vitest --run --browser firefox --config ./vitest.browser.config.ts --dir test/unit",

View File

@@ -30,7 +30,7 @@
"test:unit": "vitest --run --dir test/unit/",
"test": "yarn test:unit && yarn test:e2e",
"test:spec": "vitest --run --config vitest.spec.config.ts --dir test/spec/",
"test:e2e": "LODESTAR_PRESET=mainnet vitest --run --config vitest.e2e.config.ts --dir test/e2e",
"test:e2e": "vitest --run --config vitest.e2e.config.ts --dir test/e2e",
"download-spec-tests": "node --loader=ts-node/esm test/spec/downloadTests.ts",
"check-readme": "typescript-docs-verifier"
},

View File

@@ -1,18 +1,19 @@
#!/bin/bash
DIR="$(CDPATH= cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
function start_app() {
mkdir -p test-logs/e2e-test-env
export LODESTAR_PRESET=minimal
nohup node --loader ts-node/esm packages/cli/test/scripts/e2e_test_env.ts > test-logs/e2e-test-env/simulation.out 2>&1 &
export DOTENV_CONFIG_PATH="$DIR/../.env.test"
nohup node -r dotenv/config --loader ts-node/esm packages/cli/test/scripts/e2e_test_env.ts > test-logs/e2e-test-env/simulation.out 2>&1 &
echo $! > test-logs/e2e-test-env/simulation.pid
echo "Wait for the node to be ready"
npx wait-port -t 120000 0.0.0.0:5001
}
function stop_app() {
kill -9 $(cat test-logs/e2e-test-env/simulation.pid)
# Incase the process pid file is not present
kill -9 $(lsof -t -i:5001)
kill -s TERM $(cat test-logs/e2e-test-env/simulation.pid)
}

View File

@@ -0,0 +1,8 @@
import path from "node:path";
// It's a dev dependency
// eslint-disable-next-line import/no-extraneous-dependencies
import {config} from "dotenv";
// eslint-disable-next-line @typescript-eslint/naming-convention
const __dirname = new URL(".", import.meta.url).pathname;
config({path: path.join(__dirname, "../../../.env.test")});

View File

@@ -24,7 +24,7 @@ export default defineConfig({
"**/.{idea,git,cache,output,temp}/**",
"**/{karma,rollup,webpack,vite,vitest,jest,ava,babel,nyc,cypress,tsup,build}.config.*",
],
setupFiles: [path.join(__dirname, "./scripts/vitest/customMatchers.ts")],
setupFiles: [path.join(__dirname, "./scripts/vitest/setupFiles/customMatchers.ts")],
reporters: ["default", "hanging-process"],
coverage: {
enabled: false,

View File

@@ -14,7 +14,10 @@ export default defineConfig({
"**/.{idea,git,cache,output,temp}/**",
"**/{karma,rollup,webpack,vite,vitest,jest,ava,babel,nyc,cypress,tsup,build}.config.*",
],
setupFiles: [path.join(__dirname, "./scripts/vitest/customMatchers.ts")],
setupFiles: [
path.join(__dirname, "./scripts/vitest/setupFiles/customMatchers.ts"),
path.join(__dirname, "./scripts/vitest/setupFiles/dotenv.ts"),
],
reporters: ["default", "hanging-process"],
coverage: {
enabled: process.env.CI === "true",

View File

@@ -22,6 +22,14 @@
semver "^6.1.0"
uuid "^3.3.3"
"@actions/core@^1.10.1":
version "1.10.1"
resolved "https://registry.yarnpkg.com/@actions/core/-/core-1.10.1.tgz#61108e7ac40acae95ee36da074fa5850ca4ced8a"
integrity sha512-3lBR9EDAY+iYIpTnTIXmWcNbX3T2kCkAEQGIQx4NVQ0575nk2k3GRZDTPQG+vVtS2izSLmINlxXf0uLtnrTP+g==
dependencies:
"@actions/http-client" "^2.0.1"
uuid "^8.3.2"
"@actions/core@^1.2.6":
version "1.10.0"
resolved "https://registry.yarnpkg.com/@actions/core/-/core-1.10.0.tgz#44551c3c71163949a2f06e94d9ca2157a0cfac4f"
@@ -5600,6 +5608,11 @@ dotenv-expand@~10.0.0:
resolved "https://registry.yarnpkg.com/dotenv-expand/-/dotenv-expand-10.0.0.tgz#12605d00fb0af6d0a592e6558585784032e4ef37"
integrity sha512-GopVGCpVS1UKH75VKHGuQFqS1Gusej0z4FyQkPdwjil2gNIv+LNsqBlboOzpJFZKVT95GkCyWJbBSdFEFUWI2A==
dotenv@^16.4.1:
version "16.4.1"
resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.4.1.tgz#1d9931f1d3e5d2959350d1250efab299561f7f11"
integrity sha512-CjA3y+Dr3FyFDOAMnxZEGtnW9KBR2M0JvvUtXNW+dYJL5ROWxP9DUHCwgFqpMk0OXCc0ljhaNTr2w/kutYIcHQ==
dotenv@~16.3.1:
version "16.3.1"
resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.3.1.tgz#369034de7d7e5b120972693352a3bf112172cc3e"