feat: add script for assemble linea besu packages and makefile (#1050)

* feat: add script for assemble linea besu packages and makefile

* feat: update README.md for make commands to build linea-besu-package locally

* feat: revise the make commands for build and run e2e tests

* feat: add timeout for shomei frontend e2e test

* feat: revise README

* feat: update e2e test timeout and reduce log

* feat: update e2e test timeout

* feat: revise Makefile for compatible sed -i command on MacOS and Linux

* feat: update e2e test timeout

* feat: added log info for e2e test

* feat: revise the e2e test to avoid timeout

* feat: revise the e2e test to avoid timeout

.github/actions/linea-besu-package/assemble/action.yml

@@ -100,48 +100,13 @@ runs:
       shell: bash
       run: |
         echo "dockerimage=consensys/linea-besu-package:${{ steps.dockertag.outputs.dockertag }}" >> "$GITHUB_OUTPUT"
-        
-    - name: download and untar the linea-besu archive
-      shell: bash
-      run: |
-        cd /tmp/
-        echo "downloading besu from linea-besu-upstream: ${{ steps.dotenv.outputs.LINEA_BESU_TAR_GZ }}"
-        wget -nv "${{ steps.dotenv.outputs.LINEA_BESU_BASE_URL }}${{ steps.dotenv.outputs.LINEA_BESU_TAR_GZ }}/${{ steps.dotenv.outputs.LINEA_BESU_FILENAME_PREFIX }}-${{ steps.dotenv.outputs.LINEA_BESU_TAR_GZ }}.tar.gz"
-        tar -xvf ${{ steps.dotenv.outputs.LINEA_BESU_FILENAME_PREFIX }}-${{ steps.dotenv.outputs.LINEA_BESU_TAR_GZ }}.tar.gz
-        mv /tmp/${{ steps.dotenv.outputs.LINEA_BESU_FILENAME_PREFIX }}-${{ steps.dotenv.outputs.LINEA_BESU_TAR_GZ }} /tmp/besu
 
-    - name: copy the list of versions to the container
+    - name: assemble the packages to linea-besu and show folder structure
-      shell: bash      
-      run: |
-        cp ./linea-besu-package/versions.env /tmp/besu/versions.txt
-        mkdir -p /tmp/besu/plugins
-
-    - name: get the plugins
-      shell: bash      
-      run: |
-        cd /tmp/besu/plugins
-
-        echo "getting linea_sequencer_plugin_version: ${{ steps.dotenv.outputs.LINEA_SEQUENCER_PLUGIN_VERSION }}" 
-        wget -nv "https://github.com/Consensys/linea-sequencer/releases/download/v${{ steps.dotenv.outputs.LINEA_SEQUENCER_PLUGIN_VERSION }}/linea-sequencer-v${{ steps.dotenv.outputs.LINEA_SEQUENCER_PLUGIN_VERSION }}.jar" -P /tmp/besu/plugins
-
-        echo "getting linea_finalized_tag_updater_plugin_version: ${{ steps.dotenv.outputs.LINEA_FINALIZED_TAG_UPDATER_PLUGIN_VERSION }}" 
-        wget -nv "https://github.com/Consensys/linea-monorepo/releases/download/linea-finalized-tag-updater-v${{ steps.dotenv.outputs.LINEA_FINALIZED_TAG_UPDATER_PLUGIN_VERSION }}/linea-finalized-tag-updater-v${{ steps.dotenv.outputs.LINEA_FINALIZED_TAG_UPDATER_PLUGIN_VERSION }}.jar" -P /tmp/besu/plugins
-
-        echo "getting linea_staterecovery_plugin_version: ${{ steps.dotenv.outputs.LINEA_STATERECOVERY_PLUGIN_VERSION }}"
-        wget -nv "https://github.com/Consensys/linea-monorepo/releases/download/linea-staterecovery-v${{ steps.dotenv.outputs.LINEA_STATERECOVERY_PLUGIN_VERSION }}/linea-staterecovery-besu-plugin-v${{ steps.dotenv.outputs.LINEA_STATERECOVERY_PLUGIN_VERSION }}.jar" -P /tmp/besu/plugins
-
-        echo "getting linea_tracer_plugin_version: ${{ steps.dotenv.outputs.LINEA_TRACER_PLUGIN_VERSION }}" 
-        wget -nv "https://github.com/Consensys/linea-tracer/releases/download/${{ steps.dotenv.outputs.LINEA_TRACER_PLUGIN_VERSION }}/linea-tracer-${{ steps.dotenv.outputs.LINEA_TRACER_PLUGIN_VERSION }}.jar" -P /tmp/besu/plugins
-
-        echo "getting shomei_plugin_version: ${{ steps.dotenv.outputs.SHOMEI_PLUGIN_VERSION }}" 
-        wget -nv "https://github.com/Consensys/besu-shomei-plugin/releases/download/v${{ steps.dotenv.outputs.SHOMEI_PLUGIN_VERSION }}/besu-shomei-plugin-v${{ steps.dotenv.outputs.SHOMEI_PLUGIN_VERSION }}.jar" -P /tmp/besu/plugins
-
-    - name: place the packages together for preparing docker image build
       shell: bash    
       run: |
-        cd linea-besu-package/linea-besu
+        cd linea-besu-package
-        mv /tmp/besu ./
+        make assemble
-        mv config/ genesis/ profiles/ besu/
+        mv ./tmp/besu ./linea-besu
         sudo apt update
         sudo apt-get install --no-install-recommends --assume-yes tree
         tree .

.github/workflows/linea-besu-package-release.yml

@@ -48,7 +48,7 @@ jobs:
     outputs:
       versions_env: ${{ steps.filter.outputs.versions-env }}
       linea_configs: ${{ steps.filter.outputs.linea-configs }}
-      dockerfile: ${{ steps.filter.outputs.dockerfile }}
+      docker-files: ${{ steps.filter.outputs.docker-files }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -61,8 +61,9 @@ jobs:
           filters: |
             versions-env:
               - 'linea-besu-package/versions.env'
-            dockerfile:
+            docker-files:
               - 'linea-besu-package/linea-besu/Dockerfile'
+              - 'linea-besu-package/scripts/assemble-packages.sh'
             linea-configs:
               - 'linea-besu-package/linea-besu/**'
 
@@ -73,7 +74,7 @@ jobs:
     with:
       release_tag_prefix: ${{ inputs.release_tag_prefix }}
       run_test: true
-      run_e2e_test: ${{ github.event_name == 'workflow_dispatch' || needs.filter-commit-changes.outputs.versions_env == 'true' || needs.filter-commit-changes.outputs.dockerfile == 'true' }}
+      run_e2e_test: ${{ github.event_name == 'workflow_dispatch' || needs.filter-commit-changes.outputs.versions_env == 'true' || needs.filter-commit-changes.outputs.docker-files == 'true' }}
       push_image: ${{ github.event_name == 'workflow_dispatch' }}
       skip_e2e_test: ${{ inputs.skip_e2e_test == 'true' }}
       expected_traces_api_ver: ${{ inputs.expected_traces_api_ver }}

.github/workflows/reuse-linea-besu-package-run-test.yml

@@ -59,7 +59,7 @@ jobs:
           eval $COMMAND
 
       - name: Verify besu container
-        run: bash linea-besu-package/docker/scripts/BesuContainerVerify.sh
+        run: bash linea-besu-package/scripts/besu-container-verify.sh
         env:
           CONTAINER_NAME: ${{ env.CONTAINER_NAME }}
 

coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/app/L1DependentApp.kt

@@ -564,7 +564,7 @@ class L1DependentApp(
       measurementSupplier = maxBlobEndBlockNumberTracker,
     )
 
-    val highestAggregationTracker = HighestULongTracker(lastProcessedBlockNumber)
+    val highestAggregationTracker = HighestULongTracker(lastConsecutiveAggregatedBlockNumber)
     metricsFacade.createGauge(
       category = LineaMetricsCategory.AGGREGATION,
       name = "proven.highest.block.number",

docker/compose-spec-l2-services.yml

@@ -201,7 +201,7 @@ services:
   coordinator:
     hostname: coordinator
     container_name: coordinator
-    image: consensys/linea-coordinator:${COORDINATOR_TAG:-c3eb71d}
+    image: consensys/linea-coordinator:${COORDINATOR_TAG:-7e306e2}
     platform: linux/amd64
     profiles: [ "l2", "debug" ]
     depends_on:

e2e/src/common/utils.ts

@@ -521,7 +521,7 @@ export async function execDockerCommand(command: string, containerName: string):
         logger.error(`Error executing (${dockerCommand}). error=${stderr}`);
         reject(error);
       }
-      logger.info(`Execution success (${dockerCommand}). output=${stdout}`);
+      logger.debug(`Execution success (${dockerCommand}). output=${stdout}`);
       resolve(stdout);
     });
   });

e2e/src/shomei-get-proof.spec.ts

@@ -13,14 +13,12 @@ describe("Shomei Linea get proof test suite", () => {
     "Call linea_getProof to Shomei frontend node and get a valid proof",
     async () => {
       const shomeiImageTag = await getDockerImageTag("shomei-frontend", "consensys/linea-shomei");
-      logger.debug(`shomeiImageTag=${shomeiImageTag}`);
+      logger.info(`shomeiImageTag=${shomeiImageTag}`);
 
-      const currentL2BlockNumber = await awaitUntil(
+      let targetL2BlockNumber = await awaitUntil(
         async () => {
           try {
-            return await lineaRollupV6.currentL2BlockNumber({
+            return await lineaRollupV6.currentL2BlockNumber({ blockTag: "finalized" });
-              blockTag: "finalized",
-            });
           } catch (err) {
             if (!(err as Error).message.includes("could not decode result data")) {
               throw err;
@@ -30,30 +28,55 @@ describe("Shomei Linea get proof test suite", () => {
         },
         (currentL2BlockNumber: bigint) => currentL2BlockNumber > 1n,
         2000,
-        100000,
+        150000,
       );
 
-      expect(currentL2BlockNumber).toBeGreaterThan(1n);
+      expect(targetL2BlockNumber).toBeGreaterThan(1n);
-
-      logger.debug(`currentL2BlockNumber=${currentL2BlockNumber}`);
 
+      const finalizedL2BlockNumbers = [targetL2BlockNumber!];
       const provingAddress = "0xfe3b557e8fb62b89f4916b721be55ceb828dbd73"; // from genesis file
       const getProofResponse = await awaitUntil(
-        async () =>
+        async () => {
-          lineaShomeiFrontenedClient.lineaGetProof(provingAddress, [], "0x" + currentL2BlockNumber!.toString(16)),
+          let getProofResponse;
+          // Need to put all the latest currentL2BlockNumber in a list and traverse to get the proof
+          // from one of them as we don't know on which finalized L2 block number the shomei frontend
+          // was being notified
+          for (const finalizedL2BlockNumber of finalizedL2BlockNumbers) {
+            getProofResponse = await lineaShomeiFrontenedClient.lineaGetProof(
+              provingAddress,
+              [],
+              "0x" + finalizedL2BlockNumber.toString(16),
+            );
+            if (getProofResponse?.result) {
+              targetL2BlockNumber = finalizedL2BlockNumber;
+              break;
+            }
+          }
+          if (!getProofResponse?.result) {
+            const latestFinalizedL2BlockNumber = await lineaRollupV6.currentL2BlockNumber({ blockTag: "finalized" });
+            if (!finalizedL2BlockNumbers.includes(latestFinalizedL2BlockNumber)) {
+              finalizedL2BlockNumbers.push(latestFinalizedL2BlockNumber);
+              logger.info(`finalizedL2BlockNumbers=${JSON.stringify(finalizedL2BlockNumbers.map((it) => Number(it)))}`);
+            }
+          }
+          return getProofResponse;
+        },
         (getProofResponse) => getProofResponse?.result,
         2000,
-        100000,
+        150000,
       );
 
+      logger.info(`targetL2BlockNumber=${targetL2BlockNumber}`);
+
       const {
         result: { zkEndStateRootHash },
       } = await lineaShomeiClient.rollupGetZkEVMStateMerkleProofV0(
-        Number(currentL2BlockNumber),
+        Number(targetL2BlockNumber),
-        Number(currentL2BlockNumber),
+        Number(targetL2BlockNumber),
         shomeiImageTag,
       );
 
+      logger.info(`zkEndStateRootHash=${zkEndStateRootHash}`);
       expect(zkEndStateRootHash).toBeDefined();
 
       const l2SparseMerkleProofContract = config.getL2SparseMerkleProofContract();
@@ -80,6 +103,6 @@ describe("Shomei Linea get proof test suite", () => {
 
       expect(isInvalid).toBeTruthy();
     },
-    100_000,
+    150_000,
   );
 });

linea-besu-package/Makefile

@@ -0,0 +1,36 @@
+include versions.env
+
+TAG ?= local
+PLATFORM ?=
+
+assemble:
+	./scripts/assemble-packages.sh
+
+build:
+	$(MAKE) assemble
+	echo "IMAGE TAG=$(TAG)"
+	echo "PLATFORM=$(PLATFORM)"
+	if [ "$(PLATFORM)" = "" ]; then \
+		echo "Building image for default platform"; \
+		docker buildx build --file linea-besu/Dockerfile --tag consensys/linea-besu-package:$(TAG) ./tmp --progress=plain --no-cache --load; \
+	else \
+		echo "Building image for $(PLATFORM)"; \
+		docker buildx build --file linea-besu/Dockerfile --platform $(PLATFORM) --tag consensys/linea-besu-package:$(TAG) ./tmp --progress=plain --no-cache --load; \
+	fi
+
+run-e2e-test:
+	echo "EXPECTED_TRACES_API_VERSION=${LINEA_TRACER_PLUGIN_VERSION}"
+	if [ "$(shell uname)" = "Linux" ]; then \
+		sed -i'' 's/^\(expected-traces-api-version-v2=\).*/\1"${LINEA_TRACER_PLUGIN_VERSION}"/' ../config/coordinator/coordinator-docker.config.toml; \
+		sed -i'' 's/^\(expected-traces-api-version-v2=\).*/\1"${LINEA_TRACER_PLUGIN_VERSION}"/' ../config/coordinator/coordinator-docker-traces-v2-override.config.toml; \
+	elif [ "$(shell uname)" = "Darwin" ]; then \
+		sed -i '' 's/^\(expected-traces-api-version-v2=\).*/\1"${LINEA_TRACER_PLUGIN_VERSION}"/' ../config/coordinator/coordinator-docker.config.toml; \
+		sed -i '' 's/^\(expected-traces-api-version-v2=\).*/\1"${LINEA_TRACER_PLUGIN_VERSION}"/' ../config/coordinator/coordinator-docker-traces-v2-override.config.toml; \
+	fi
+	cd .. && BESU_PACKAGE_TAG=$(TAG) make start-env-with-tracing-v2-ci && pnpm run -F e2e test:e2e:local
+
+run-e2e-test-cleanup:
+	cd .. && make clean-environment
+
+clean:
+	rm -fr tmp || true

linea-besu-package/README.md

@@ -61,20 +61,37 @@ Linea Finalization status is based on L1 RPC endpoint's response
 bin/besu --profile=advanced-mainnet --plugin-linea-l1-rpc-endpoint=YOUR_L1_RPC_ENDPOINT
 ```
 
-## Build from source
+## Build from source locally
 
-1. Make a branch with changes to `linea-besu-package/versions.env` as needed
+1. Make changes to `linea-besu-package/versions.env` as needed
-2. Create a PR for the branch
+
-3. Go to the [actions tab](https://github.com/Consensys/linea-monorepo/actions) to check if the workflow completed successfully
+2. Cd into `linea-besu-package`
-4. Go to the [releases page](https://github.com/Consensys/linea-monorepo/releases?q=linea-besu-package&expanded=true) and you should find the corresponding release info along with the docker image tag
+
+3. Run `make clean && make build` (check the Makefile for build options)
+
+4. The docker image (i.e. default as `consensys/linea-besu-package:local`) should be created locally 
+
+### Note: 
+
+To build with specific platform (e.g. linux/amd64) and image tag (e.g. xxx), do the following:
+```
+make clean && PLATFORM=linux/amd64 TAG=xxx make build
+```
+
+To run the e2e test locally with the locally-built `linea-besu-package` image (e.g. tagged as xxx), do the following:
+```
+TAG=xxx make run-e2e-test
+```
 
 ## How-To Release
 
-1. Go to the [actions tab](https://github.com/Consensys/linea-monorepo/actions) and click on the workflow `linea-besu-package-release` for release with besu and plugin versions based on `linea-besu-package/versions.env`
+1. Make a branch with changes to `linea-besu-package/versions.env` as needed
 
-2. If release prefix is not given, `LINEA_TRACER_PLUGIN_VERSION` in the target `versions.env` file will be used, and the resultant release tag would be `linea-besu-package-[releasePrefix]-[YYYYMMDDHHMMSS]-[shortenCommitHash]` and the docker image tag would be `[releasePrefix]-[YYYYMMDDHHMMSS]-[shortenCommitHash]`
+2. Go to the [actions tab](https://github.com/Consensys/linea-monorepo/actions) and click on the workflow `linea-besu-package-release` and select the target branch for making a release with besu and plugin versions based on `linea-besu-package/versions.env`
 
-3. Go to the [releases page](https://github.com/Consensys/linea-monorepo/releases?q=linea-besu-package&expanded=true) and you should find the corresponding release info along with the docker image tag
+3. If release prefix is not given, `LINEA_TRACER_PLUGIN_VERSION` in the target `versions.env` file will be used, and the resultant release tag would be `linea-besu-package-[releasePrefix]-[YYYYMMDDHHMMSS]-[shortenCommitHash]` and the docker image tag would be `[releasePrefix]-[YYYYMMDDHHMMSS]-[shortenCommitHash]`
+
+4. Once the workflow is done successfully, go to the [releases page](https://github.com/Consensys/linea-monorepo/releases?q=linea-besu-package&expanded=true) and you should find the corresponding release info along with the docker image tag
 
 Additionally, the `latest` tag will be updated to match this release
 

linea-besu-package/scripts/assemble-packages.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+source ./versions.env
+
+mkdir -p ./tmp
+pushd ./tmp
+
+echo "downloading besu from linea-besu-upstream: $LINEA_BESU_TAR_GZ"
+wget -nv $LINEA_BESU_BASE_URL$LINEA_BESU_TAR_GZ/$LINEA_BESU_FILENAME_PREFIX-$LINEA_BESU_TAR_GZ.tar.gz 
+tar -xvf $LINEA_BESU_FILENAME_PREFIX-$LINEA_BESU_TAR_GZ.tar.gz
+mv $LINEA_BESU_FILENAME_PREFIX-$LINEA_BESU_TAR_GZ ./besu
+
+echo "copying the versions.env to the container as versions.txt"
+cp ../versions.env ./besu/versions.txt
+
+mkdir -p ./besu/plugins
+cd ./besu/plugins
+
+echo "downloading the plugins"
+echo "getting linea_sequencer_plugin_version: $LINEA_SEQUENCER_PLUGIN_VERSION" 
+wget -nv https://github.com/Consensys/linea-sequencer/releases/download/v$LINEA_SEQUENCER_PLUGIN_VERSION/linea-sequencer-v$LINEA_SEQUENCER_PLUGIN_VERSION.jar
+
+echo "getting linea_finalized_tag_updater_plugin_version: $LINEA_FINALIZED_TAG_UPDATER_PLUGIN_VERSION"
+wget -nv https://github.com/Consensys/linea-monorepo/releases/download/linea-finalized-tag-updater-v$LINEA_FINALIZED_TAG_UPDATER_PLUGIN_VERSION/linea-finalized-tag-updater-v$LINEA_FINALIZED_TAG_UPDATER_PLUGIN_VERSION.jar
+
+echo "getting linea_staterecovery_plugin_version: $LINEA_STATERECOVERY_PLUGIN_VERSION"
+wget -nv https://github.com/Consensys/linea-monorepo/releases/download/linea-staterecovery-v$LINEA_STATERECOVERY_PLUGIN_VERSION/linea-staterecovery-besu-plugin-v$LINEA_STATERECOVERY_PLUGIN_VERSION.jar
+
+echo "getting linea_tracer_plugin_version: $LINEA_TRACER_PLUGIN_VERSION" 
+wget -nv https://github.com/Consensys/linea-tracer/releases/download/$LINEA_TRACER_PLUGIN_VERSION/linea-tracer-$LINEA_TRACER_PLUGIN_VERSION.jar
+
+echo "getting shomei_plugin_version: $SHOMEI_PLUGIN_VERSION" 
+wget -nv https://github.com/Consensys/besu-shomei-plugin/releases/download/v$SHOMEI_PLUGIN_VERSION/besu-shomei-plugin-v$SHOMEI_PLUGIN_VERSION.jar
+
+popd
+
+echo "placing the packages, config, profiles together for preparing docker image build"
+cd ./linea-besu
+cp -r config genesis profiles ../tmp/besu/