AtHeartEngineer/lodestar
1
0
Fork 0
mirror of https://github.com/AtHeartEngineering/lodestar.git synced 2026-01-09 01:28:11 -05:00
Code Issues Packages Projects Releases Wiki Activity

Simplify gitData and version guessing (#3992)

Browse Source 
Don't print double slash in version string

Dont add git-data.json to NPM releases

Write git-data.json only in from source docker build

Remove numCommits

Test git-data.json generation from within the test

Move comment

Revert "Dont add git-data.json to NPM releases"

This reverts commit 5fe2d388825f3e3a834058478071e8364b0d761c.

Simplify gitData and version guessing

Run cmd
This commit is contained in:
Lion - dapplion
2022-05-10 12:07:27 +02:00
committed by GitHub
parent 7fd5a4f416
commit 15d8ae2c2c
13 changed files with 142 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
docker/from_source.Dockerfile
View File

@@ -30,6 +30,12 @@ RUN yarn install --non-interactive --frozen-lockfile --ignore-scripts
COPY . .
RUN yarn install --non-interactive --frozen-lockfile && yarn build
# To have access to the specific branch and commit used to build this source,
# a git-data.json file is created by persisting git data at build time. Then,
# a version string like `v0.35.0-beta.0/HEAD/82219149 (git)` can be shown in
# the terminal and in the logs; which is very useful to track tests better.
RUN cd packages/cli && yarn write-git-data
# Copy built src + node_modules to a new layer to prune unnecessary fs
# Previous layer weights 7.25GB, while this final 488MB (as of Oct 2020)
FROM node:16-alpine

4
packages/cli/src/cli.ts
View File

@@ -3,9 +3,9 @@ import yargs from "yargs";
import {cmds} from "./cmds";
import {globalOptions, rcConfigOption} from "./options";
import {registerCommandToYargs} from "./util";
import {getVersion} from "./util/version";
import {getVersionData} from "./util/version";
const version = getVersion();
const {version} = getVersionData();
const topBanner = `🌟 Lodestar: TypeScript Implementation of the Ethereum Consensus Beacon Chain.
* Version: ${version}
* by ChainSafe Systems, 2018-2022`;

11
packages/cli/src/cmds/beacon/handler.ts
View File

@@ -14,7 +14,7 @@ import {initializeOptionsAndConfig, persistOptionsAndConfig} from "../init/handl
import {IBeaconArgs} from "./options";
import {getBeaconPaths} from "./paths";
import {initBeaconState} from "./initBeaconState";
import {getVersion, getVersionGitData} from "../../util/version";
import {getVersionData} from "../../util/version";
import {deleteOldPeerstorePreV036} from "../../migrations";
/**
@@ -26,16 +26,15 @@ export async function beaconHandler(args: IBeaconArgs & IGlobalArgs): Promise<vo
const {beaconNodeOptions, config} = await initializeOptionsAndConfig(args);
await persistOptionsAndConfig(args);
const version = getVersion();
const gitData = getVersionGitData();
const {version, commit} = getVersionData();
const beaconPaths = getBeaconPaths(args);
// TODO: Rename db.name to db.path or db.location
beaconNodeOptions.set({db: {name: beaconPaths.dbDir}});
beaconNodeOptions.set({chain: {persistInvalidSszObjectsDir: beaconPaths.persistInvalidSszObjectsDir}});
// Add metrics metadata to show versioning + network info in Prometheus + Grafana
beaconNodeOptions.set({metrics: {metadata: {...gitData, version, network: args.network}}});
beaconNodeOptions.set({metrics: {metadata: {version, commit, network: args.network}}});
// Add detailed version string for API node/version endpoint
beaconNodeOptions.set({api: {version: version}});
beaconNodeOptions.set({api: {version}});
// ENR setup
const peerId = await readPeerId(beaconPaths.peerIdFile);
@@ -53,7 +52,7 @@ export async function beaconHandler(args: IBeaconArgs & IGlobalArgs): Promise<vo
abortController.abort();
}, logger.info.bind(logger));
logger.info("Lodestar", {version: version, network: args.network});
logger.info("Lodestar", {network: args.network, version, commit});
if (ACTIVE_PRESET === PresetName.minimal) logger.info("ACTIVE_PRESET == minimal preset");
// peerstore migration

4
packages/cli/src/cmds/dev/handler.ts
View File

@@ -20,7 +20,7 @@ import {initializeOptionsAndConfig} from "../init/handler";
import {mkdir, initBLS, getCliLogger} from "../../util";
import {getBeaconPaths} from "../beacon/paths";
import {getValidatorPaths} from "../validator/paths";
import {getVersion} from "../../util/version";
import {getVersionData} from "../../util/version";
/**
* Run a beacon node with validator
@@ -68,7 +68,7 @@ export async function devHandler(args: IDevArgs & IGlobalArgs): Promise<void> {
// BeaconNode setup
const libp2p = await createNodeJsLibp2p(peerId, options.network, {peerStoreDir: beaconPaths.peerStoreDir});
const logger = getCliLogger(args, beaconPaths, config);
logger.info("Lodestar", {version: getVersion(), network: args.network});
logger.info("Lodestar", {network: args.network, ...getVersionData()});
if (ACTIVE_PRESET === PresetName.minimal) logger.info("ACTIVE_PRESET == minimal preset");
const db = new BeaconDb({config, controller: new LevelDbController(options.db, {logger})});

16
packages/cli/src/cmds/validator/handler.ts
View File

@@ -8,7 +8,7 @@ import {getBeaconConfigFromArgs} from "../../config";
import {IGlobalArgs} from "../../options";
import {YargsError, getDefaultGraffiti, initBLS, mkdir, getCliLogger} from "../../util";
import {onGracefulShutdown} from "../../util";
import {getVersion, getVersionGitData} from "../../util/version";
import {getVersionData} from "../../util/version";
import {getBeaconPaths} from "../beacon/paths";
import {getValidatorPaths} from "./paths";
import {IValidatorCliArgs, validatorMetricsDefaultOptions} from "./options";
@@ -28,9 +28,8 @@ export async function validatorHandler(args: IValidatorCliArgs & IGlobalArgs): P
const logger = getCliLogger(args, beaconPaths, config);
const version = getVersion();
const gitData = getVersionGitData();
logger.info("Lodestar", {version: version, network: args.network});
const {version, commit} = getVersionData();
logger.info("Lodestar", {network: args.network, version, commit});
const dbPath = validatorPaths.validatorsDbDir;
mkdir(dbPath);
@@ -102,14 +101,7 @@ export async function validatorHandler(args: IValidatorCliArgs & IGlobalArgs): P
const register = args["metrics.enabled"] ? new RegistryMetricCreator() : null;
const metrics =
register &&
getMetrics((register as unknown) as MetricsRegister, {
semver: gitData.semver ?? "-",
branch: gitData.branch ?? "-",
commit: gitData.commit ?? "-",
version,
network: args.network,
});
register && getMetrics((register as unknown) as MetricsRegister, {version, commit, network: args.network});
// Start metrics server if metrics are enabled.
// Collect NodeJS metrics defined in the Lodestar repo

30
packages/cli/src/util/gitData/gitDataPath.ts
View File

@@ -1,11 +1,23 @@
/**
* Persist git data and distribute through NPM so CLI consumers can know exactly
* at what commit was this src build. This is used in the metrics and to log initially.
*/
import path from "node:path";
import fs from "node:fs";
// Persist git data and distribute through NPM so CLI consumers can know exactly
// at what commit was this src build. This is used in the metrics and to log initially.
//
// - For NPM release (stable): Only the version is persisted. Once must then track the version's tag
// in Github to resolve that version to a specific commit. While this is okay, git-data.json gives
// a gurantee of the exact commit at build time.
//
// - For NPM release (nightly): canary commits include the commit, so this feature is not really
// necessary. However, it's more cumbersome to have conditional logic on stable / nightly.
//
// - For build from source: .git folder is available in the context of the built code, so it can extract
// branch and commit directly without the need for .git-data.json.
//
// - For build from source dockerized: This feature is required to know the branch and commit, since
// git data is not persisted past the build. However, .dockerignore prevents .git folder from being
// copied into the container's context, so .git-data.json can't be generated.
/**
* WARNING!! If you change this path make sure to update:
* - 'packages/cli/package.json' -> .files -> `".git-data.json"`
@@ -14,14 +26,10 @@ export const gitDataPath = path.resolve(__dirname, "../../../.git-data.json");
/** Git data type used to construct version information string and persistence. */
export type GitData = {
/** v0.28.2 */
semver?: string;
/** "developer-feature" */
branch?: string;
branch: string;
/** "80c248bb392f512cc115d95059e22239a17bbd7d" */
commit?: string;
/** +7 (commits since last tag) */
numCommits?: string;
commit: string;
};
/** Writes a persistent git data file. */

135
packages/cli/src/util/gitData/index.ts
View File

@@ -1,108 +1,65 @@
import {execSync} from "node:child_process";
/**
* This file is created in the build step and is distributed through NPM
* MUST be in sync with `-/gitDataPath.ts` and `package.json` files.
*/
// This file is created in the build step and is distributed through NPM
// MUST be in sync with `-/gitDataPath.ts` and `package.json` files.
import {readGitDataFile, GitData} from "./gitDataPath";
/** Silent shell that won't pollute stdout, or stderr */
function shell(cmd: string): string {
return execSync(cmd, {stdio: ["ignore", "pipe", "ignore"]})
.toString()
.trim();
/** Reads git data from a persisted file or local git data at build time. */
export function readAndGetGitData(): GitData {
try {
// Gets git data containing current branch and commit info from persistent file.
let persistedGitData: Partial<GitData>;
try {
persistedGitData = readGitDataFile();
} catch (e) {
persistedGitData = {};
}
const currentGitData = getGitData();
return {
// If the CLI is run from source, prioritze current git data
// over `.git-data.json` file, which might be stale here.
branch: currentGitData.branch ?? persistedGitData.branch ?? "",
commit: currentGitData.commit ?? persistedGitData.commit ?? "",
};
} catch (e) {
return {
branch: "",
commit: "",
};
}
}
/** Gets git data containing current branch and commit info from CLI. */
export function getGitData(): GitData {
return {
branch: getBranch(),
commit: getCommit(),
};
}
/** Tries to get branch from git CLI. */
function getBranch(): string | undefined {
function getBranch(): string {
try {
return shell("git rev-parse --abbrev-ref HEAD");
return shellSilent("git rev-parse --abbrev-ref HEAD");
} catch (e) {
return undefined;
return "";
}
}
/** Tries to get commit from git from git CLI. */
function getCommit(): string | undefined {
function getCommit(): string {
try {
return shell("git rev-parse --verify HEAD");
return shellSilent("git rev-parse --verify HEAD");
} catch (e) {
return undefined;
return "";
}
}
/** Tries to get the latest tag from git CLI. */
function getLatestTag(): string | undefined {
try {
return shell("git describe --abbrev=0");
} catch (e) {
return undefined;
}
}
/** Gets number of commits since latest tag/release. */
function getCommitsSinceRelease(): number | undefined {
let numCommits = 0;
const latestTag: string | undefined = getLatestTag();
try {
numCommits = +shell(`git rev-list ${latestTag}..HEAD --count`);
} catch (e) {
return undefined;
}
return numCommits;
}
/** Reads git data from a persisted file or local git data at build time. */
export function readLodestarGitData(): GitData {
try {
const currentGitData = getGitData();
const persistedGitData = getPersistedGitData();
// If the CLI is run from source, prioritze current git data
// over `.git-data.json` file, which might be stale here.
let gitData = {...persistedGitData, ...currentGitData};
// If the CLI is not run from the git repository, fall back to persistent
if (!gitData.semver || !gitData.branch || !gitData.commit) {
gitData = persistedGitData;
}
return {
semver: gitData?.semver,
branch: gitData?.branch || "N/A",
commit: gitData?.commit || "N/A",
numCommits: gitData?.numCommits || "",
};
} catch (e) {
return {semver: "", branch: "", commit: "", numCommits: ""};
}
}
/** Wrapper for updating git data. ONLY to be used with build scripts! */
export function forceUpdateGitData(): Partial<GitData> {
return getGitData();
}
/** Gets git data containing current branch and commit info from CLI. */
function getGitData(): Partial<GitData> {
const numCommits: number | undefined = getCommitsSinceRelease();
let strCommits = "";
if (numCommits !== undefined && numCommits > 0) {
strCommits = `+${numCommits}`;
}
return {
branch: getBranch(),
commit: getCommit(),
semver: getLatestTag(),
numCommits: strCommits,
};
}
/** Gets git data containing current branch and commit info from persistent file. */
function getPersistedGitData(): Partial<GitData> {
try {
return readGitDataFile();
} catch (e) {
return {};
}
/** Silent shell that won't pollute stdout, or stderr */
function shellSilent(cmd: string): string {
return execSync(cmd, {stdio: ["ignore", "pipe", "ignore"]})
.toString()
.trim();
}

12
packages/cli/src/util/gitData/writeGitData.ts
View File

@@ -1,12 +1,10 @@
#!/usr/bin/env node
/**
* Persist git data and distribute through NPM so CLI consumers can know exactly
* at what commit was this source build. This is also used in the metrics and to log initially.
*/
// For RATIONALE of this file, check packages/cli/src/util/gitData/gitDataPath.ts
// Persist exact commit in NPM distributions for easier tracking of the build
import {getGitData} from "./index";
import {writeGitDataFile} from "./gitDataPath";
import {forceUpdateGitData} from "./index";
/** Script to write the git data file (json) used by the build procedures to persist git data. */
writeGitDataFile(forceUpdateGitData());
// Script to write the git data file (json) used by the build procedures to persist git data.
writeGitDataFile(getGitData());

4
packages/cli/src/util/graffiti.ts
View File

@@ -1,4 +1,4 @@
import {getVersion} from "./version";
import {getVersionData} from "./version";
const lodestarPackageName = "Lodestar";
@@ -8,7 +8,7 @@ const lodestarPackageName = "Lodestar";
*/
export function getDefaultGraffiti(): string {
try {
const version = getVersion();
const {version} = getVersionData();
return `${lodestarPackageName}-${version}`;
} catch (e) {
// eslint-disable-next-line no-console

69
packages/cli/src/util/version.ts
View File

@@ -1,57 +1,52 @@
import fs from "node:fs";
import findUp from "find-up";
import {readLodestarGitData} from "./gitData";
import {GitData} from "./gitData/gitDataPath";
import {readAndGetGitData} from "./gitData";
type VersionJson = {
/** "0.28.2" */
version: string;
};
enum ReleaseTrack {
git = "git",
npm = "npm",
nightly = "nightly",
alpha = "alpha",
beta = "beta",
rc = "release candidate",
stable = "stable",
lts = "long term support",
}
/** Defines default release track, i.e., the "stability" of tag releases */
const defaultReleaseTrack = ReleaseTrack.alpha;
const BRANCH_IGNORE = /^(HEAD|master|main)$/;
/**
* Gathers all information on package version including Git data.
* @returns a version string, e.g., `v0.28.2/developer-feature/+7/80c248bb (nightly)`
* @returns a version string, e.g.
* - Stable release: `v0.36.0/80c248bb`
* - Nightly release: `v0.36.0-dev.80c248bb/80c248bb`
* - Test branch: `v0.36.0/developer-feature/80c248bb`
*/
export function getVersion(): string {
const gitData: GitData = readLodestarGitData();
let semver: string | undefined = gitData.semver;
const numCommits: string | undefined = gitData.numCommits;
const commitSlice: string | undefined = gitData.commit?.slice(0, 8);
export function getVersionData(): {
version: string;
commit: string;
} {
const parts: string[] = [];
// ansible github branch deployment returns no semver
semver = semver ?? `v${getLocalVersion()}`;
// Tag release has numCommits as "0"
if (!commitSlice || numCommits === "0") {
return `${semver} (${defaultReleaseTrack})`;
/** Returns local version from `lerna.json` or `package.json` as `"0.28.2"` */
const localVersion = readCliPackageJson() || readVersionFromLernaJson();
if (localVersion) {
parts.push(`v${localVersion}`);
}
// Otherwise get branch and commit information
return `${semver}/${gitData.branch}/${numCommits}/${commitSlice} (${ReleaseTrack.git})`;
}
const {branch, commit} = readAndGetGitData();
/** Exposes raw version data wherever needed for reporting (metrics, grafana). */
export function getVersionGitData(): GitData {
return readLodestarGitData();
}
// Add branch only if not present and not an ignore value
if (branch && !BRANCH_IGNORE.test(branch)) parts.push(branch);
/** Returns local version from `lerna.json` or `package.json` as `"0.28.2"` */
function getLocalVersion(): string | undefined {
return readVersionFromLernaJson() || readCliPackageJson();
// Add commit only if present. 7 characters to be consistent with Github
if (commit) {
const commitShort = commit.slice(0, 7);
// Don't add commit if it's already in the version string (nightly versions)
if (!localVersion || !localVersion.includes(commitShort)) {
parts.push(commitShort);
}
}
return {
// Guard against empty parts array
version: parts.length > 0 ? parts.join("/") : "unknown",
commit,
};
}
/** Read version information from lerna.json */

17
packages/cli/test/unit/util/gitData.test.ts
View File

@@ -1,15 +1,28 @@
import {expect} from "chai";
import child_process from "node:child_process";
import fs from "node:fs";
import path from "node:path";
import findUp from "find-up";
import {gitDataPath, readGitDataFile} from "../../../src/util/gitData/gitDataPath";
import {getGitData} from "../../../src/util";
const WRITE_GIT_DATA_CMD = "npm run write-git-data";
describe("util / gitData", () => {
before(() => {
const pkgJsonPath = findUp.sync("package.json", {cwd: __dirname});
if (!pkgJsonPath) {
throw Error("No package.json found");
}
const pkgJsonDir = path.resolve(path.dirname(pkgJsonPath));
child_process.execSync(WRITE_GIT_DATA_CMD, {cwd: pkgJsonDir});
});
it("gitData file must exist", () => {
const gitData = readGitDataFile();
if (!gitData.branch) throw Error("No gitData.branch");
if (!gitData.commit) throw Error("No gitData.commit");
expect(gitData).to.deep.equal(getGitData(), "Wrong git-data.json contents");
});
it("gitData path must be included in the package.json", () => {

8
packages/lodestar/src/metrics/options.ts
View File

@@ -5,14 +5,10 @@
import {HttpMetricsServerOpts} from "./server";
export type LodestarMetadata = {
/** "0.16.0" */
semver: string;
/** "developer/feature-1" */
branch: string;
/** "v0.16.0/developer/feature-1/ac99f2b5" */
version: string;
/** "4f816b16dfde718e2d74f95f2c8292596138c248" */
commit: string;
/** "0.16.0 developer/feature-1 ac99f2b5" */
version: string;
/** "prater" */
network: string;
};

8
packages/validator/src/metrics.ts
View File

@@ -60,14 +60,10 @@ export interface MetricsRegister {
export type Metrics = ReturnType<typeof getMetrics>;
export type LodestarGitData = {
/** "0.16.0" */
semver: string;
/** "developer/feature-1" */
branch: string;
/** "4f816b16dfde718e2d74f95f2c8292596138c248" */
commit: string;
/** "0.16.0 developer/feature-1 ac99f2b5" */
version: string;
/** "4f816b16dfde718e2d74f95f2c8292596138c248" */
commit: string;
/** "prater" */
network: string;
};