feat: add script for assemble linea besu packages and makefile (#1050)

* feat: add script for assemble linea besu packages and makefile * feat: update README.md for make commands to build linea-besu-package locally * feat: revise the make commands for build and run e2e tests * feat: add timeout for shomei frontend e2e test * feat: revise README * feat: update e2e test timeout and reduce log * feat: update e2e test timeout * feat: revise Makefile for compatible sed -i command on MacOS and Linux * feat: update e2e test timeout * feat: added log info for e2e test * feat: revise the e2e test to avoid timeout * feat: revise the e2e test to avoid timeout
2026-01-08 03:43:56 -05:00 · 2025-06-06 02:22:27 +08:00
parent c845a2548d
commit 62b422b634
11 changed files with 150 additions and 69 deletions
--- a/.github/actions/linea-besu-package/assemble/action.yml
+++ b/.github/actions/linea-besu-package/assemble/action.yml
@@ -100,48 +100,13 @@ runs:
      shell: bash
      run: |
        echo "dockerimage=consensys/linea-besu-package:${{ steps.dockertag.outputs.dockertag }}" >> "$GITHUB_OUTPUT"
-        
-    - name: download and untar the linea-besu archive
-      shell: bash
-      run: |
-        cd /tmp/
-        echo "downloading besu from linea-besu-upstream: ${{ steps.dotenv.outputs.LINEA_BESU_TAR_GZ }}"
-        wget -nv "${{ steps.dotenv.outputs.LINEA_BESU_BASE_URL }}${{ steps.dotenv.outputs.LINEA_BESU_TAR_GZ }}/${{ steps.dotenv.outputs.LINEA_BESU_FILENAME_PREFIX }}-${{ steps.dotenv.outputs.LINEA_BESU_TAR_GZ }}.tar.gz"
-        tar -xvf ${{ steps.dotenv.outputs.LINEA_BESU_FILENAME_PREFIX }}-${{ steps.dotenv.outputs.LINEA_BESU_TAR_GZ }}.tar.gz
-        mv /tmp/${{ steps.dotenv.outputs.LINEA_BESU_FILENAME_PREFIX }}-${{ steps.dotenv.outputs.LINEA_BESU_TAR_GZ }} /tmp/besu

-    - name: copy the list of versions to the container
-      shell: bash      
-      run: |
-        cp ./linea-besu-package/versions.env /tmp/besu/versions.txt
-        mkdir -p /tmp/besu/plugins
-
-    - name: get the plugins
-      shell: bash      
-      run: |
-        cd /tmp/besu/plugins
-
-        echo "getting linea_sequencer_plugin_version: ${{ steps.dotenv.outputs.LINEA_SEQUENCER_PLUGIN_VERSION }}" 
-        wget -nv "https://github.com/Consensys/linea-sequencer/releases/download/v${{ steps.dotenv.outputs.LINEA_SEQUENCER_PLUGIN_VERSION }}/linea-sequencer-v${{ steps.dotenv.outputs.LINEA_SEQUENCER_PLUGIN_VERSION }}.jar" -P /tmp/besu/plugins
-
-        echo "getting linea_finalized_tag_updater_plugin_version: ${{ steps.dotenv.outputs.LINEA_FINALIZED_TAG_UPDATER_PLUGIN_VERSION }}" 
-        wget -nv "https://github.com/Consensys/linea-monorepo/releases/download/linea-finalized-tag-updater-v${{ steps.dotenv.outputs.LINEA_FINALIZED_TAG_UPDATER_PLUGIN_VERSION }}/linea-finalized-tag-updater-v${{ steps.dotenv.outputs.LINEA_FINALIZED_TAG_UPDATER_PLUGIN_VERSION }}.jar" -P /tmp/besu/plugins
-
-        echo "getting linea_staterecovery_plugin_version: ${{ steps.dotenv.outputs.LINEA_STATERECOVERY_PLUGIN_VERSION }}"
-        wget -nv "https://github.com/Consensys/linea-monorepo/releases/download/linea-staterecovery-v${{ steps.dotenv.outputs.LINEA_STATERECOVERY_PLUGIN_VERSION }}/linea-staterecovery-besu-plugin-v${{ steps.dotenv.outputs.LINEA_STATERECOVERY_PLUGIN_VERSION }}.jar" -P /tmp/besu/plugins
-
-        echo "getting linea_tracer_plugin_version: ${{ steps.dotenv.outputs.LINEA_TRACER_PLUGIN_VERSION }}" 
-        wget -nv "https://github.com/Consensys/linea-tracer/releases/download/${{ steps.dotenv.outputs.LINEA_TRACER_PLUGIN_VERSION }}/linea-tracer-${{ steps.dotenv.outputs.LINEA_TRACER_PLUGIN_VERSION }}.jar" -P /tmp/besu/plugins
-
-        echo "getting shomei_plugin_version: ${{ steps.dotenv.outputs.SHOMEI_PLUGIN_VERSION }}" 
-        wget -nv "https://github.com/Consensys/besu-shomei-plugin/releases/download/v${{ steps.dotenv.outputs.SHOMEI_PLUGIN_VERSION }}/besu-shomei-plugin-v${{ steps.dotenv.outputs.SHOMEI_PLUGIN_VERSION }}.jar" -P /tmp/besu/plugins
-
-    - name: place the packages together for preparing docker image build
+    - name: assemble the packages to linea-besu and show folder structure
      shell: bash    
      run: |
-        cd linea-besu-package/linea-besu
-        mv /tmp/besu ./
-        mv config/ genesis/ profiles/ besu/
+        cd linea-besu-package
+        make assemble
+        mv ./tmp/besu ./linea-besu
        sudo apt update
        sudo apt-get install --no-install-recommends --assume-yes tree
        tree .
--- a/.github/workflows/linea-besu-package-release.yml
+++ b/.github/workflows/linea-besu-package-release.yml
@@ -48,7 +48,7 @@ jobs:
    outputs:
      versions_env: ${{ steps.filter.outputs.versions-env }}
      linea_configs: ${{ steps.filter.outputs.linea-configs }}
-      dockerfile: ${{ steps.filter.outputs.dockerfile }}
+      docker-files: ${{ steps.filter.outputs.docker-files }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
@@ -61,8 +61,9 @@ jobs:
          filters: |
            versions-env:
              - 'linea-besu-package/versions.env'
-            dockerfile:
+            docker-files:
              - 'linea-besu-package/linea-besu/Dockerfile'
+              - 'linea-besu-package/scripts/assemble-packages.sh'
            linea-configs:
              - 'linea-besu-package/linea-besu/**'

@@ -73,7 +74,7 @@ jobs:
    with:
      release_tag_prefix: ${{ inputs.release_tag_prefix }}
      run_test: true
-      run_e2e_test: ${{ github.event_name == 'workflow_dispatch' || needs.filter-commit-changes.outputs.versions_env == 'true' || needs.filter-commit-changes.outputs.dockerfile == 'true' }}
+      run_e2e_test: ${{ github.event_name == 'workflow_dispatch' || needs.filter-commit-changes.outputs.versions_env == 'true' || needs.filter-commit-changes.outputs.docker-files == 'true' }}
      push_image: ${{ github.event_name == 'workflow_dispatch' }}
      skip_e2e_test: ${{ inputs.skip_e2e_test == 'true' }}
      expected_traces_api_ver: ${{ inputs.expected_traces_api_ver }}
--- a/.github/workflows/reuse-linea-besu-package-run-test.yml
+++ b/.github/workflows/reuse-linea-besu-package-run-test.yml
@@ -59,7 +59,7 @@ jobs:
          eval $COMMAND

      - name: Verify besu container
-        run: bash linea-besu-package/docker/scripts/BesuContainerVerify.sh
+        run: bash linea-besu-package/scripts/besu-container-verify.sh
        env:
          CONTAINER_NAME: ${{ env.CONTAINER_NAME }}

--- a/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/app/L1DependentApp.kt
+++ b/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/app/L1DependentApp.kt
@@ -564,7 +564,7 @@ class L1DependentApp(
      measurementSupplier = maxBlobEndBlockNumberTracker,
    )

-    val highestAggregationTracker = HighestULongTracker(lastProcessedBlockNumber)
+    val highestAggregationTracker = HighestULongTracker(lastConsecutiveAggregatedBlockNumber)
    metricsFacade.createGauge(
      category = LineaMetricsCategory.AGGREGATION,
      name = "proven.highest.block.number",
--- a/docker/compose-spec-l2-services.yml
+++ b/docker/compose-spec-l2-services.yml
@@ -201,7 +201,7 @@ services:
  coordinator:
    hostname: coordinator
    container_name: coordinator
-    image: consensys/linea-coordinator:${COORDINATOR_TAG:-c3eb71d}
+    image: consensys/linea-coordinator:${COORDINATOR_TAG:-7e306e2}
    platform: linux/amd64
    profiles: [ "l2", "debug" ]
    depends_on:
--- a/e2e/src/common/utils.ts
+++ b/e2e/src/common/utils.ts
@@ -521,7 +521,7 @@ export async function execDockerCommand(command: string, containerName: string):
        logger.error(`Error executing (${dockerCommand}). error=${stderr}`);
        reject(error);
      }
-      logger.info(`Execution success (${dockerCommand}). output=${stdout}`);
+      logger.debug(`Execution success (${dockerCommand}). output=${stdout}`);
      resolve(stdout);
    });
  });
--- a/e2e/src/shomei-get-proof.spec.ts
+++ b/e2e/src/shomei-get-proof.spec.ts
@@ -13,14 +13,12 @@ describe("Shomei Linea get proof test suite", () => {
    "Call linea_getProof to Shomei frontend node and get a valid proof",
    async () => {
      const shomeiImageTag = await getDockerImageTag("shomei-frontend", "consensys/linea-shomei");
-      logger.debug(`shomeiImageTag=${shomeiImageTag}`);
+      logger.info(`shomeiImageTag=${shomeiImageTag}`);

-      const currentL2BlockNumber = await awaitUntil(
+      let targetL2BlockNumber = await awaitUntil(
        async () => {
          try {
-            return await lineaRollupV6.currentL2BlockNumber({
-              blockTag: "finalized",
-            });
+            return await lineaRollupV6.currentL2BlockNumber({ blockTag: "finalized" });
          } catch (err) {
            if (!(err as Error).message.includes("could not decode result data")) {
              throw err;
@@ -30,30 +28,55 @@ describe("Shomei Linea get proof test suite", () => {
        },
        (currentL2BlockNumber: bigint) => currentL2BlockNumber > 1n,
        2000,
-        100000,
+        150000,
      );

-      expect(currentL2BlockNumber).toBeGreaterThan(1n);
-
-      logger.debug(`currentL2BlockNumber=${currentL2BlockNumber}`);
+      expect(targetL2BlockNumber).toBeGreaterThan(1n);

+      const finalizedL2BlockNumbers = [targetL2BlockNumber!];
      const provingAddress = "0xfe3b557e8fb62b89f4916b721be55ceb828dbd73"; // from genesis file
      const getProofResponse = await awaitUntil(
-        async () =>
-          lineaShomeiFrontenedClient.lineaGetProof(provingAddress, [], "0x" + currentL2BlockNumber!.toString(16)),
+        async () => {
+          let getProofResponse;
+          // Need to put all the latest currentL2BlockNumber in a list and traverse to get the proof
+          // from one of them as we don't know on which finalized L2 block number the shomei frontend
+          // was being notified
+          for (const finalizedL2BlockNumber of finalizedL2BlockNumbers) {
+            getProofResponse = await lineaShomeiFrontenedClient.lineaGetProof(
+              provingAddress,
+              [],
+              "0x" + finalizedL2BlockNumber.toString(16),
+            );
+            if (getProofResponse?.result) {
+              targetL2BlockNumber = finalizedL2BlockNumber;
+              break;
+            }
+          }
+          if (!getProofResponse?.result) {
+            const latestFinalizedL2BlockNumber = await lineaRollupV6.currentL2BlockNumber({ blockTag: "finalized" });
+            if (!finalizedL2BlockNumbers.includes(latestFinalizedL2BlockNumber)) {
+              finalizedL2BlockNumbers.push(latestFinalizedL2BlockNumber);
+              logger.info(`finalizedL2BlockNumbers=${JSON.stringify(finalizedL2BlockNumbers.map((it) => Number(it)))}`);
+            }
+          }
+          return getProofResponse;
+        },
        (getProofResponse) => getProofResponse?.result,
        2000,
-        100000,
+        150000,
      );

+      logger.info(`targetL2BlockNumber=${targetL2BlockNumber}`);
+
      const {
        result: { zkEndStateRootHash },
      } = await lineaShomeiClient.rollupGetZkEVMStateMerkleProofV0(
-        Number(currentL2BlockNumber),
-        Number(currentL2BlockNumber),
+        Number(targetL2BlockNumber),
+        Number(targetL2BlockNumber),
        shomeiImageTag,
      );

+      logger.info(`zkEndStateRootHash=${zkEndStateRootHash}`);
      expect(zkEndStateRootHash).toBeDefined();

      const l2SparseMerkleProofContract = config.getL2SparseMerkleProofContract();
@@ -80,6 +103,6 @@ describe("Shomei Linea get proof test suite", () => {

      expect(isInvalid).toBeTruthy();
    },
-    100_000,
+    150_000,
  );
 });
--- a/linea-besu-package/Makefile
+++ b/linea-besu-package/Makefile
@@ -0,0 +1,36 @@
+include versions.env
+
+TAG ?= local
+PLATFORM ?=
+
+assemble:
+	./scripts/assemble-packages.sh
+
+build:
+	$(MAKE) assemble
+	echo "IMAGE TAG=$(TAG)"
+	echo "PLATFORM=$(PLATFORM)"
+	if [ "$(PLATFORM)" = "" ]; then \
+		echo "Building image for default platform"; \
+		docker buildx build --file linea-besu/Dockerfile --tag consensys/linea-besu-package:$(TAG) ./tmp --progress=plain --no-cache --load; \
+	else \
+		echo "Building image for $(PLATFORM)"; \
+		docker buildx build --file linea-besu/Dockerfile --platform $(PLATFORM) --tag consensys/linea-besu-package:$(TAG) ./tmp --progress=plain --no-cache --load; \
+	fi
+
+run-e2e-test:
+	echo "EXPECTED_TRACES_API_VERSION=${LINEA_TRACER_PLUGIN_VERSION}"
+	if [ "$(shell uname)" = "Linux" ]; then \
+		sed -i'' 's/^\(expected-traces-api-version-v2=\).*/\1"${LINEA_TRACER_PLUGIN_VERSION}"/' ../config/coordinator/coordinator-docker.config.toml; \
+		sed -i'' 's/^\(expected-traces-api-version-v2=\).*/\1"${LINEA_TRACER_PLUGIN_VERSION}"/' ../config/coordinator/coordinator-docker-traces-v2-override.config.toml; \
+	elif [ "$(shell uname)" = "Darwin" ]; then \
+		sed -i '' 's/^\(expected-traces-api-version-v2=\).*/\1"${LINEA_TRACER_PLUGIN_VERSION}"/' ../config/coordinator/coordinator-docker.config.toml; \
+		sed -i '' 's/^\(expected-traces-api-version-v2=\).*/\1"${LINEA_TRACER_PLUGIN_VERSION}"/' ../config/coordinator/coordinator-docker-traces-v2-override.config.toml; \
+	fi
+	cd .. && BESU_PACKAGE_TAG=$(TAG) make start-env-with-tracing-v2-ci && pnpm run -F e2e test:e2e:local
+
+run-e2e-test-cleanup:
+	cd .. && make clean-environment
+
+clean:
+	rm -fr tmp || true
--- a/linea-besu-package/README.md
+++ b/linea-besu-package/README.md
@@ -61,20 +61,37 @@ Linea Finalization status is based on L1 RPC endpoint's response
 bin/besu --profile=advanced-mainnet --plugin-linea-l1-rpc-endpoint=YOUR_L1_RPC_ENDPOINT
 ```

-## Build from source
+## Build from source locally

-1. Make a branch with changes to `linea-besu-package/versions.env` as needed
-2. Create a PR for the branch
-3. Go to the [actions tab](https://github.com/Consensys/linea-monorepo/actions) to check if the workflow completed successfully
-4. Go to the [releases page](https://github.com/Consensys/linea-monorepo/releases?q=linea-besu-package&expanded=true) and you should find the corresponding release info along with the docker image tag
+1. Make changes to `linea-besu-package/versions.env` as needed
+
+2. Cd into `linea-besu-package`
+
+3. Run `make clean && make build` (check the Makefile for build options)
+
+4. The docker image (i.e. default as `consensys/linea-besu-package:local`) should be created locally 
+
+### Note: 
+
+To build with specific platform (e.g. linux/amd64) and image tag (e.g. xxx), do the following:
+```
+make clean && PLATFORM=linux/amd64 TAG=xxx make build
+```
+
+To run the e2e test locally with the locally-built `linea-besu-package` image (e.g. tagged as xxx), do the following:
+```
+TAG=xxx make run-e2e-test
+```

 ## How-To Release

-1. Go to the [actions tab](https://github.com/Consensys/linea-monorepo/actions) and click on the workflow `linea-besu-package-release` for release with besu and plugin versions based on `linea-besu-package/versions.env`
+1. Make a branch with changes to `linea-besu-package/versions.env` as needed

-2. If release prefix is not given, `LINEA_TRACER_PLUGIN_VERSION` in the target `versions.env` file will be used, and the resultant release tag would be `linea-besu-package-[releasePrefix]-[YYYYMMDDHHMMSS]-[shortenCommitHash]` and the docker image tag would be `[releasePrefix]-[YYYYMMDDHHMMSS]-[shortenCommitHash]`
+2. Go to the [actions tab](https://github.com/Consensys/linea-monorepo/actions) and click on the workflow `linea-besu-package-release` and select the target branch for making a release with besu and plugin versions based on `linea-besu-package/versions.env`

-3. Go to the [releases page](https://github.com/Consensys/linea-monorepo/releases?q=linea-besu-package&expanded=true) and you should find the corresponding release info along with the docker image tag
+3. If release prefix is not given, `LINEA_TRACER_PLUGIN_VERSION` in the target `versions.env` file will be used, and the resultant release tag would be `linea-besu-package-[releasePrefix]-[YYYYMMDDHHMMSS]-[shortenCommitHash]` and the docker image tag would be `[releasePrefix]-[YYYYMMDDHHMMSS]-[shortenCommitHash]`
+
+4. Once the workflow is done successfully, go to the [releases page](https://github.com/Consensys/linea-monorepo/releases?q=linea-besu-package&expanded=true) and you should find the corresponding release info along with the docker image tag

 Additionally, the `latest` tag will be updated to match this release

--- a/linea-besu-package/scripts/assemble-packages.sh
+++ b/linea-besu-package/scripts/assemble-packages.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+source ./versions.env
+
+mkdir -p ./tmp
+pushd ./tmp
+
+echo "downloading besu from linea-besu-upstream: $LINEA_BESU_TAR_GZ"
+wget -nv $LINEA_BESU_BASE_URL$LINEA_BESU_TAR_GZ/$LINEA_BESU_FILENAME_PREFIX-$LINEA_BESU_TAR_GZ.tar.gz 
+tar -xvf $LINEA_BESU_FILENAME_PREFIX-$LINEA_BESU_TAR_GZ.tar.gz
+mv $LINEA_BESU_FILENAME_PREFIX-$LINEA_BESU_TAR_GZ ./besu
+
+echo "copying the versions.env to the container as versions.txt"
+cp ../versions.env ./besu/versions.txt
+
+mkdir -p ./besu/plugins
+cd ./besu/plugins
+
+echo "downloading the plugins"
+echo "getting linea_sequencer_plugin_version: $LINEA_SEQUENCER_PLUGIN_VERSION" 
+wget -nv https://github.com/Consensys/linea-sequencer/releases/download/v$LINEA_SEQUENCER_PLUGIN_VERSION/linea-sequencer-v$LINEA_SEQUENCER_PLUGIN_VERSION.jar
+
+echo "getting linea_finalized_tag_updater_plugin_version: $LINEA_FINALIZED_TAG_UPDATER_PLUGIN_VERSION"
+wget -nv https://github.com/Consensys/linea-monorepo/releases/download/linea-finalized-tag-updater-v$LINEA_FINALIZED_TAG_UPDATER_PLUGIN_VERSION/linea-finalized-tag-updater-v$LINEA_FINALIZED_TAG_UPDATER_PLUGIN_VERSION.jar
+
+echo "getting linea_staterecovery_plugin_version: $LINEA_STATERECOVERY_PLUGIN_VERSION"
+wget -nv https://github.com/Consensys/linea-monorepo/releases/download/linea-staterecovery-v$LINEA_STATERECOVERY_PLUGIN_VERSION/linea-staterecovery-besu-plugin-v$LINEA_STATERECOVERY_PLUGIN_VERSION.jar
+
+echo "getting linea_tracer_plugin_version: $LINEA_TRACER_PLUGIN_VERSION" 
+wget -nv https://github.com/Consensys/linea-tracer/releases/download/$LINEA_TRACER_PLUGIN_VERSION/linea-tracer-$LINEA_TRACER_PLUGIN_VERSION.jar
+
+echo "getting shomei_plugin_version: $SHOMEI_PLUGIN_VERSION" 
+wget -nv https://github.com/Consensys/besu-shomei-plugin/releases/download/v$SHOMEI_PLUGIN_VERSION/besu-shomei-plugin-v$SHOMEI_PLUGIN_VERSION.jar
+
+popd
+
+echo "placing the packages, config, profiles together for preparing docker image build"
+cd ./linea-besu
+cp -r config genesis profiles ../tmp/besu/
--- a/linea-besu-package/docker/scripts/BesuContainerVerify.sh
+++ b/linea-besu-package/docker/scripts/BesuContainerVerify.sh