Simplify gitData and version guessing (#3992)

Don't print double slash in version string

Dont add git-data.json to NPM releases

Write git-data.json only in from source docker build

Remove numCommits

Test git-data.json generation from within the test

Move comment

Revert "Dont add git-data.json to NPM releases"

This reverts commit 5fe2d388825f3e3a834058478071e8364b0d761c.

Simplify gitData and version guessing

Run cmd

docker/from_source.Dockerfile

@@ -30,6 +30,12 @@ RUN yarn install --non-interactive --frozen-lockfile --ignore-scripts
 COPY . .
 RUN yarn install --non-interactive --frozen-lockfile && yarn build
 
+# To have access to the specific branch and commit used to build this source,
+# a git-data.json file is created by persisting git data at build time. Then,
+# a version string like `v0.35.0-beta.0/HEAD/82219149 (git)` can be shown in
+# the terminal and in the logs; which is very useful to track tests better.
+RUN cd packages/cli && yarn write-git-data
+
 # Copy built src + node_modules to a new layer to prune unnecessary fs
 # Previous layer weights 7.25GB, while this final 488MB (as of Oct 2020)
 FROM node:16-alpine

packages/cli/src/cli.ts

@@ -3,9 +3,9 @@ import yargs from "yargs";
 import {cmds} from "./cmds";
 import {globalOptions, rcConfigOption} from "./options";
 import {registerCommandToYargs} from "./util";
-import {getVersion} from "./util/version";
+import {getVersionData} from "./util/version";
 
-const version = getVersion();
+const {version} = getVersionData();
 const topBanner = `🌟 Lodestar: TypeScript Implementation of the Ethereum Consensus Beacon Chain.
   * Version: ${version}
   * by ChainSafe Systems, 2018-2022`;

packages/cli/src/cmds/beacon/handler.ts

@@ -14,7 +14,7 @@ import {initializeOptionsAndConfig, persistOptionsAndConfig} from "../init/handl
 import {IBeaconArgs} from "./options";
 import {getBeaconPaths} from "./paths";
 import {initBeaconState} from "./initBeaconState";
-import {getVersion, getVersionGitData} from "../../util/version";
+import {getVersionData} from "../../util/version";
 import {deleteOldPeerstorePreV036} from "../../migrations";
 
 /**
@@ -26,16 +26,15 @@ export async function beaconHandler(args: IBeaconArgs & IGlobalArgs): Promise<vo
   const {beaconNodeOptions, config} = await initializeOptionsAndConfig(args);
   await persistOptionsAndConfig(args);
 
-  const version = getVersion();
+  const {version, commit} = getVersionData();
-  const gitData = getVersionGitData();
   const beaconPaths = getBeaconPaths(args);
   // TODO: Rename db.name to db.path or db.location
   beaconNodeOptions.set({db: {name: beaconPaths.dbDir}});
   beaconNodeOptions.set({chain: {persistInvalidSszObjectsDir: beaconPaths.persistInvalidSszObjectsDir}});
   // Add metrics metadata to show versioning + network info in Prometheus + Grafana
-  beaconNodeOptions.set({metrics: {metadata: {...gitData, version, network: args.network}}});
+  beaconNodeOptions.set({metrics: {metadata: {version, commit, network: args.network}}});
   // Add detailed version string for API node/version endpoint
-  beaconNodeOptions.set({api: {version: version}});
+  beaconNodeOptions.set({api: {version}});
 
   // ENR setup
   const peerId = await readPeerId(beaconPaths.peerIdFile);
@@ -53,7 +52,7 @@ export async function beaconHandler(args: IBeaconArgs & IGlobalArgs): Promise<vo
     abortController.abort();
   }, logger.info.bind(logger));
 
-  logger.info("Lodestar", {version: version, network: args.network});
+  logger.info("Lodestar", {network: args.network, version, commit});
   if (ACTIVE_PRESET === PresetName.minimal) logger.info("ACTIVE_PRESET == minimal preset");
 
   // peerstore migration

packages/cli/src/cmds/dev/handler.ts

@@ -20,7 +20,7 @@ import {initializeOptionsAndConfig} from "../init/handler";
 import {mkdir, initBLS, getCliLogger} from "../../util";
 import {getBeaconPaths} from "../beacon/paths";
 import {getValidatorPaths} from "../validator/paths";
-import {getVersion} from "../../util/version";
+import {getVersionData} from "../../util/version";
 
 /**
  * Run a beacon node with validator
@@ -68,7 +68,7 @@ export async function devHandler(args: IDevArgs & IGlobalArgs): Promise<void> {
   // BeaconNode setup
   const libp2p = await createNodeJsLibp2p(peerId, options.network, {peerStoreDir: beaconPaths.peerStoreDir});
   const logger = getCliLogger(args, beaconPaths, config);
-  logger.info("Lodestar", {version: getVersion(), network: args.network});
+  logger.info("Lodestar", {network: args.network, ...getVersionData()});
   if (ACTIVE_PRESET === PresetName.minimal) logger.info("ACTIVE_PRESET == minimal preset");
 
   const db = new BeaconDb({config, controller: new LevelDbController(options.db, {logger})});

packages/cli/src/cmds/validator/handler.ts

@@ -8,7 +8,7 @@ import {getBeaconConfigFromArgs} from "../../config";
 import {IGlobalArgs} from "../../options";
 import {YargsError, getDefaultGraffiti, initBLS, mkdir, getCliLogger} from "../../util";
 import {onGracefulShutdown} from "../../util";
-import {getVersion, getVersionGitData} from "../../util/version";
+import {getVersionData} from "../../util/version";
 import {getBeaconPaths} from "../beacon/paths";
 import {getValidatorPaths} from "./paths";
 import {IValidatorCliArgs, validatorMetricsDefaultOptions} from "./options";
@@ -28,9 +28,8 @@ export async function validatorHandler(args: IValidatorCliArgs & IGlobalArgs): P
 
   const logger = getCliLogger(args, beaconPaths, config);
 
-  const version = getVersion();
+  const {version, commit} = getVersionData();
-  const gitData = getVersionGitData();
+  logger.info("Lodestar", {network: args.network, version, commit});
-  logger.info("Lodestar", {version: version, network: args.network});
 
   const dbPath = validatorPaths.validatorsDbDir;
   mkdir(dbPath);
@@ -102,14 +101,7 @@ export async function validatorHandler(args: IValidatorCliArgs & IGlobalArgs): P
 
   const register = args["metrics.enabled"] ? new RegistryMetricCreator() : null;
   const metrics =
-    register &&
+    register && getMetrics((register as unknown) as MetricsRegister, {version, commit, network: args.network});
-    getMetrics((register as unknown) as MetricsRegister, {
-      semver: gitData.semver ?? "-",
-      branch: gitData.branch ?? "-",
-      commit: gitData.commit ?? "-",
-      version,
-      network: args.network,
-    });
 
   // Start metrics server if metrics are enabled.
   // Collect NodeJS metrics defined in the Lodestar repo

packages/cli/src/util/gitData/gitDataPath.ts

@@ -1,11 +1,23 @@
-/**
- * Persist git data and distribute through NPM so CLI consumers can know exactly
- * at what commit was this src build. This is used in the metrics and to log initially.
- */
-
 import path from "node:path";
 import fs from "node:fs";
 
+// Persist git data and distribute through NPM so CLI consumers can know exactly
+// at what commit was this src build. This is used in the metrics and to log initially.
+//
+// - For NPM release (stable): Only the version is persisted. Once must then track the version's tag
+//   in Github to resolve that version to a specific commit. While this is okay, git-data.json gives
+//   a gurantee of the exact commit at build time.
+//
+// - For NPM release (nightly): canary commits include the commit, so this feature is not really
+//   necessary. However, it's more cumbersome to have conditional logic on stable / nightly.
+//
+// - For build from source: .git folder is available in the context of the built code, so it can extract
+//   branch and commit directly without the need for .git-data.json.
+//
+// - For build from source dockerized: This feature is required to know the branch and commit, since
+//   git data is not persisted past the build. However, .dockerignore prevents .git folder from being
+//   copied into the container's context, so .git-data.json can't be generated.
+
 /**
  * WARNING!! If you change this path make sure to update:
  * - 'packages/cli/package.json' -> .files -> `".git-data.json"`
@@ -14,14 +26,10 @@ export const gitDataPath = path.resolve(__dirname, "../../../.git-data.json");
 
 /** Git data type used to construct version information string and persistence. */
 export type GitData = {
-  /** v0.28.2 */
-  semver?: string;
   /** "developer-feature" */
-  branch?: string;
+  branch: string;
   /** "80c248bb392f512cc115d95059e22239a17bbd7d" */
-  commit?: string;
+  commit: string;
-  /** +7 (commits since last tag) */
-  numCommits?: string;
 };
 
 /** Writes a persistent git data file. */

packages/cli/src/util/gitData/index.ts

@@ -1,108 +1,65 @@
 import {execSync} from "node:child_process";
 
-/**
+// This file is created in the build step and is distributed through NPM
- * This file is created in the build step and is distributed through NPM
+// MUST be in sync with `-/gitDataPath.ts` and `package.json` files.
- * MUST be in sync with `-/gitDataPath.ts` and `package.json` files.
- */
 import {readGitDataFile, GitData} from "./gitDataPath";
 
-/** Silent shell that won't pollute stdout, or stderr */
+/** Reads git data from a persisted file or local git data at build time. */
-function shell(cmd: string): string {
+export function readAndGetGitData(): GitData {
-  return execSync(cmd, {stdio: ["ignore", "pipe", "ignore"]})
+  try {
-    .toString()
+    // Gets git data containing current branch and commit info from persistent file.
-    .trim();
+    let persistedGitData: Partial<GitData>;
+    try {
+      persistedGitData = readGitDataFile();
+    } catch (e) {
+      persistedGitData = {};
+    }
+
+    const currentGitData = getGitData();
+
+    return {
+      // If the CLI is run from source, prioritze current git data
+      // over `.git-data.json` file, which might be stale here.
+      branch: currentGitData.branch ?? persistedGitData.branch ?? "",
+      commit: currentGitData.commit ?? persistedGitData.commit ?? "",
+    };
+  } catch (e) {
+    return {
+      branch: "",
+      commit: "",
+    };
+  }
+}
+
+/** Gets git data containing current branch and commit info from CLI. */
+export function getGitData(): GitData {
+  return {
+    branch: getBranch(),
+    commit: getCommit(),
+  };
 }
 
 /** Tries to get branch from git CLI. */
-function getBranch(): string | undefined {
+function getBranch(): string {
   try {
-    return shell("git rev-parse --abbrev-ref HEAD");
+    return shellSilent("git rev-parse --abbrev-ref HEAD");
   } catch (e) {
-    return undefined;
+    return "";
   }
 }
 
 /** Tries to get commit from git from git CLI. */
-function getCommit(): string | undefined {
+function getCommit(): string {
   try {
-    return shell("git rev-parse --verify HEAD");
+    return shellSilent("git rev-parse --verify HEAD");
   } catch (e) {
-    return undefined;
+    return "";
   }
 }
 
-/** Tries to get the latest tag from git CLI. */
+/** Silent shell that won't pollute stdout, or stderr */
-function getLatestTag(): string | undefined {
+function shellSilent(cmd: string): string {
-  try {
+  return execSync(cmd, {stdio: ["ignore", "pipe", "ignore"]})
-    return shell("git describe --abbrev=0");
+    .toString()
-  } catch (e) {
+    .trim();
-    return undefined;
-  }
-}
-
-/** Gets number of commits since latest tag/release. */
-function getCommitsSinceRelease(): number | undefined {
-  let numCommits = 0;
-  const latestTag: string | undefined = getLatestTag();
-  try {
-    numCommits = +shell(`git rev-list ${latestTag}..HEAD --count`);
-  } catch (e) {
-    return undefined;
-  }
-  return numCommits;
-}
-
-/** Reads git data from a persisted file or local git data at build time. */
-export function readLodestarGitData(): GitData {
-  try {
-    const currentGitData = getGitData();
-    const persistedGitData = getPersistedGitData();
-
-    // If the CLI is run from source, prioritze current git data
-    // over `.git-data.json` file, which might be stale here.
-    let gitData = {...persistedGitData, ...currentGitData};
-
-    // If the CLI is not run from the git repository, fall back to persistent
-    if (!gitData.semver || !gitData.branch || !gitData.commit) {
-      gitData = persistedGitData;
-    }
-
-    return {
-      semver: gitData?.semver,
-      branch: gitData?.branch || "N/A",
-      commit: gitData?.commit || "N/A",
-      numCommits: gitData?.numCommits || "",
-    };
-  } catch (e) {
-    return {semver: "", branch: "", commit: "", numCommits: ""};
-  }
-}
-
-/** Wrapper for updating git data. ONLY to be used with build scripts! */
-export function forceUpdateGitData(): Partial<GitData> {
-  return getGitData();
-}
-
-/** Gets git data containing current branch and commit info from CLI. */
-function getGitData(): Partial<GitData> {
-  const numCommits: number | undefined = getCommitsSinceRelease();
-  let strCommits = "";
-  if (numCommits !== undefined && numCommits > 0) {
-    strCommits = `+${numCommits}`;
-  }
-  return {
-    branch: getBranch(),
-    commit: getCommit(),
-    semver: getLatestTag(),
-    numCommits: strCommits,
-  };
-}
-
-/** Gets git data containing current branch and commit info from persistent file. */
-function getPersistedGitData(): Partial<GitData> {
-  try {
-    return readGitDataFile();
-  } catch (e) {
-    return {};
-  }
 }

packages/cli/src/util/gitData/writeGitData.ts

@@ -1,12 +1,10 @@
 #!/usr/bin/env node
 
-/**
+// For RATIONALE of this file, check packages/cli/src/util/gitData/gitDataPath.ts
- * Persist git data and distribute through NPM so CLI consumers can know exactly
+// Persist exact commit in NPM distributions for easier tracking of the build
- * at what commit was this source build. This is also used in the metrics and to log initially.
- */
 
+import {getGitData} from "./index";
 import {writeGitDataFile} from "./gitDataPath";
-import {forceUpdateGitData} from "./index";
 
-/** Script to write the git data file (json) used by the build procedures to persist git data. */
+// Script to write the git data file (json) used by the build procedures to persist git data.
-writeGitDataFile(forceUpdateGitData());
+writeGitDataFile(getGitData());

packages/cli/src/util/graffiti.ts

@@ -1,4 +1,4 @@
-import {getVersion} from "./version";
+import {getVersionData} from "./version";
 
 const lodestarPackageName = "Lodestar";
 
@@ -8,7 +8,7 @@ const lodestarPackageName = "Lodestar";
  */
 export function getDefaultGraffiti(): string {
   try {
-    const version = getVersion();
+    const {version} = getVersionData();
     return `${lodestarPackageName}-${version}`;
   } catch (e) {
     // eslint-disable-next-line no-console

packages/cli/src/util/version.ts

@@ -1,57 +1,52 @@
 import fs from "node:fs";
 import findUp from "find-up";
-import {readLodestarGitData} from "./gitData";
+import {readAndGetGitData} from "./gitData";
-import {GitData} from "./gitData/gitDataPath";
 
 type VersionJson = {
   /** "0.28.2" */
   version: string;
 };
 
-enum ReleaseTrack {
+const BRANCH_IGNORE = /^(HEAD|master|main)$/;
-  git = "git",
-  npm = "npm",
-  nightly = "nightly",
-  alpha = "alpha",
-  beta = "beta",
-  rc = "release candidate",
-  stable = "stable",
-  lts = "long term support",
-}
-
-/** Defines default release track, i.e., the "stability" of tag releases */
-const defaultReleaseTrack = ReleaseTrack.alpha;
 
 /**
  * Gathers all information on package version including Git data.
- * @returns a version string, e.g., `v0.28.2/developer-feature/+7/80c248bb (nightly)`
+ * @returns a version string, e.g.
+ * - Stable release: `v0.36.0/80c248bb`
+ * - Nightly release: `v0.36.0-dev.80c248bb/80c248bb`
+ * - Test branch: `v0.36.0/developer-feature/80c248bb`
  */
-export function getVersion(): string {
+export function getVersionData(): {
-  const gitData: GitData = readLodestarGitData();
+  version: string;
-  let semver: string | undefined = gitData.semver;
+  commit: string;
-  const numCommits: string | undefined = gitData.numCommits;
+} {
-  const commitSlice: string | undefined = gitData.commit?.slice(0, 8);
+  const parts: string[] = [];
 
-  // ansible github branch deployment returns no semver
+  /** Returns local version from `lerna.json` or `package.json` as `"0.28.2"` */
-  semver = semver ?? `v${getLocalVersion()}`;
+  const localVersion = readCliPackageJson() || readVersionFromLernaJson();
-
+  if (localVersion) {
-  // Tag release has numCommits as "0"
+    parts.push(`v${localVersion}`);
-  if (!commitSlice || numCommits === "0") {
-    return `${semver} (${defaultReleaseTrack})`;
   }
 
-  // Otherwise get branch and commit information
+  const {branch, commit} = readAndGetGitData();
-  return `${semver}/${gitData.branch}/${numCommits}/${commitSlice} (${ReleaseTrack.git})`;
-}
 
-/** Exposes raw version data wherever needed for reporting (metrics, grafana). */
+  // Add branch only if not present and not an ignore value
-export function getVersionGitData(): GitData {
+  if (branch && !BRANCH_IGNORE.test(branch)) parts.push(branch);
-  return readLodestarGitData();
-}
 
-/** Returns local version from `lerna.json` or `package.json` as `"0.28.2"` */
+  // Add commit only if present. 7 characters to be consistent with Github
-function getLocalVersion(): string | undefined {
+  if (commit) {
-  return readVersionFromLernaJson() || readCliPackageJson();
+    const commitShort = commit.slice(0, 7);
+    // Don't add commit if it's already in the version string (nightly versions)
+    if (!localVersion || !localVersion.includes(commitShort)) {
+      parts.push(commitShort);
+    }
+  }
+
+  return {
+    // Guard against empty parts array
+    version: parts.length > 0 ? parts.join("/") : "unknown",
+    commit,
+  };
 }
 
 /** Read version information from lerna.json */

packages/cli/test/unit/util/gitData.test.ts

@@ -1,15 +1,28 @@
 import {expect} from "chai";
+import child_process from "node:child_process";
 import fs from "node:fs";
 import path from "node:path";
 import findUp from "find-up";
 import {gitDataPath, readGitDataFile} from "../../../src/util/gitData/gitDataPath";
+import {getGitData} from "../../../src/util";
+
+const WRITE_GIT_DATA_CMD = "npm run write-git-data";
 
 describe("util / gitData", () => {
+  before(() => {
+    const pkgJsonPath = findUp.sync("package.json", {cwd: __dirname});
+    if (!pkgJsonPath) {
+      throw Error("No package.json found");
+    }
+
+    const pkgJsonDir = path.resolve(path.dirname(pkgJsonPath));
+    child_process.execSync(WRITE_GIT_DATA_CMD, {cwd: pkgJsonDir});
+  });
+
   it("gitData file must exist", () => {
     const gitData = readGitDataFile();
 
-    if (!gitData.branch) throw Error("No gitData.branch");
+    expect(gitData).to.deep.equal(getGitData(), "Wrong git-data.json contents");
-    if (!gitData.commit) throw Error("No gitData.commit");
   });
 
   it("gitData path must be included in the package.json", () => {

packages/lodestar/src/metrics/options.ts

@@ -5,14 +5,10 @@
 import {HttpMetricsServerOpts} from "./server";
 
 export type LodestarMetadata = {
-  /** "0.16.0" */
+  /** "v0.16.0/developer/feature-1/ac99f2b5" */
-  semver: string;
+  version: string;
-  /** "developer/feature-1" */
-  branch: string;
   /** "4f816b16dfde718e2d74f95f2c8292596138c248" */
   commit: string;
-  /** "0.16.0 developer/feature-1 ac99f2b5" */
-  version: string;
   /** "prater" */
   network: string;
 };

packages/validator/src/metrics.ts

@@ -60,14 +60,10 @@ export interface MetricsRegister {
 export type Metrics = ReturnType<typeof getMetrics>;
 
 export type LodestarGitData = {
-  /** "0.16.0" */
-  semver: string;
-  /** "developer/feature-1" */
-  branch: string;
-  /** "4f816b16dfde718e2d74f95f2c8292596138c248" */
-  commit: string;
   /** "0.16.0 developer/feature-1 ac99f2b5" */
   version: string;
+  /** "4f816b16dfde718e2d74f95f2c8292596138c248" */
+  commit: string;
   /** "prater" */
   network: string;
 };