From 62b422b6347c95a9778939d5dc6f7a7a1f68f41a Mon Sep 17 00:00:00 2001 From: jonesho <81145364+jonesho@users.noreply.github.com> Date: Fri, 6 Jun 2025 02:22:27 +0800 Subject: [PATCH] feat: add script for assemble linea besu packages and makefile (#1050) * feat: add script for assemble linea besu packages and makefile * feat: update README.md for make commands to build linea-besu-package locally * feat: revise the make commands for build and run e2e tests * feat: add timeout for shomei frontend e2e test * feat: revise README * feat: update e2e test timeout and reduce log * feat: update e2e test timeout * feat: revise Makefile for compatible sed -i command on MacOS and Linux * feat: update e2e test timeout * feat: added log info for e2e test * feat: revise the e2e test to avoid timeout * feat: revise the e2e test to avoid timeout --- .../linea-besu-package/assemble/action.yml | 43 ++------------- .../workflows/linea-besu-package-release.yml | 7 +-- .../reuse-linea-besu-package-run-test.yml | 2 +- .../zkevm/coordinator/app/L1DependentApp.kt | 2 +- docker/compose-spec-l2-services.yml | 2 +- e2e/src/common/utils.ts | 2 +- e2e/src/shomei-get-proof.spec.ts | 53 +++++++++++++------ linea-besu-package/Makefile | 36 +++++++++++++ linea-besu-package/README.md | 33 +++++++++--- .../scripts/assemble-packages.sh | 39 ++++++++++++++ .../besu-container-verify.sh} | 0 11 files changed, 150 insertions(+), 69 deletions(-) create mode 100644 linea-besu-package/Makefile create mode 100755 linea-besu-package/scripts/assemble-packages.sh rename linea-besu-package/{docker/scripts/BesuContainerVerify.sh => scripts/besu-container-verify.sh} (100%) diff --git a/.github/actions/linea-besu-package/assemble/action.yml b/.github/actions/linea-besu-package/assemble/action.yml index cb7b1902..3eb836da 100644 --- a/.github/actions/linea-besu-package/assemble/action.yml +++ b/.github/actions/linea-besu-package/assemble/action.yml @@ -100,48 +100,13 @@ runs: shell: bash run: | echo "dockerimage=consensys/linea-besu-package:${{ steps.dockertag.outputs.dockertag }}" >> "$GITHUB_OUTPUT" - - - name: download and untar the linea-besu archive - shell: bash - run: | - cd /tmp/ - echo "downloading besu from linea-besu-upstream: ${{ steps.dotenv.outputs.LINEA_BESU_TAR_GZ }}" - wget -nv "${{ steps.dotenv.outputs.LINEA_BESU_BASE_URL }}${{ steps.dotenv.outputs.LINEA_BESU_TAR_GZ }}/${{ steps.dotenv.outputs.LINEA_BESU_FILENAME_PREFIX }}-${{ steps.dotenv.outputs.LINEA_BESU_TAR_GZ }}.tar.gz" - tar -xvf ${{ steps.dotenv.outputs.LINEA_BESU_FILENAME_PREFIX }}-${{ steps.dotenv.outputs.LINEA_BESU_TAR_GZ }}.tar.gz - mv /tmp/${{ steps.dotenv.outputs.LINEA_BESU_FILENAME_PREFIX }}-${{ steps.dotenv.outputs.LINEA_BESU_TAR_GZ }} /tmp/besu - - name: copy the list of versions to the container - shell: bash - run: | - cp ./linea-besu-package/versions.env /tmp/besu/versions.txt - mkdir -p /tmp/besu/plugins - - - name: get the plugins - shell: bash - run: | - cd /tmp/besu/plugins - - echo "getting linea_sequencer_plugin_version: ${{ steps.dotenv.outputs.LINEA_SEQUENCER_PLUGIN_VERSION }}" - wget -nv "https://github.com/Consensys/linea-sequencer/releases/download/v${{ steps.dotenv.outputs.LINEA_SEQUENCER_PLUGIN_VERSION }}/linea-sequencer-v${{ steps.dotenv.outputs.LINEA_SEQUENCER_PLUGIN_VERSION }}.jar" -P /tmp/besu/plugins - - echo "getting linea_finalized_tag_updater_plugin_version: ${{ steps.dotenv.outputs.LINEA_FINALIZED_TAG_UPDATER_PLUGIN_VERSION }}" - wget -nv "https://github.com/Consensys/linea-monorepo/releases/download/linea-finalized-tag-updater-v${{ steps.dotenv.outputs.LINEA_FINALIZED_TAG_UPDATER_PLUGIN_VERSION }}/linea-finalized-tag-updater-v${{ steps.dotenv.outputs.LINEA_FINALIZED_TAG_UPDATER_PLUGIN_VERSION }}.jar" -P /tmp/besu/plugins - - echo "getting linea_staterecovery_plugin_version: ${{ steps.dotenv.outputs.LINEA_STATERECOVERY_PLUGIN_VERSION }}" - wget -nv "https://github.com/Consensys/linea-monorepo/releases/download/linea-staterecovery-v${{ steps.dotenv.outputs.LINEA_STATERECOVERY_PLUGIN_VERSION }}/linea-staterecovery-besu-plugin-v${{ steps.dotenv.outputs.LINEA_STATERECOVERY_PLUGIN_VERSION }}.jar" -P /tmp/besu/plugins - - echo "getting linea_tracer_plugin_version: ${{ steps.dotenv.outputs.LINEA_TRACER_PLUGIN_VERSION }}" - wget -nv "https://github.com/Consensys/linea-tracer/releases/download/${{ steps.dotenv.outputs.LINEA_TRACER_PLUGIN_VERSION }}/linea-tracer-${{ steps.dotenv.outputs.LINEA_TRACER_PLUGIN_VERSION }}.jar" -P /tmp/besu/plugins - - echo "getting shomei_plugin_version: ${{ steps.dotenv.outputs.SHOMEI_PLUGIN_VERSION }}" - wget -nv "https://github.com/Consensys/besu-shomei-plugin/releases/download/v${{ steps.dotenv.outputs.SHOMEI_PLUGIN_VERSION }}/besu-shomei-plugin-v${{ steps.dotenv.outputs.SHOMEI_PLUGIN_VERSION }}.jar" -P /tmp/besu/plugins - - - name: place the packages together for preparing docker image build + - name: assemble the packages to linea-besu and show folder structure shell: bash run: | - cd linea-besu-package/linea-besu - mv /tmp/besu ./ - mv config/ genesis/ profiles/ besu/ + cd linea-besu-package + make assemble + mv ./tmp/besu ./linea-besu sudo apt update sudo apt-get install --no-install-recommends --assume-yes tree tree . diff --git a/.github/workflows/linea-besu-package-release.yml b/.github/workflows/linea-besu-package-release.yml index e4353b09..bfa60196 100644 --- a/.github/workflows/linea-besu-package-release.yml +++ b/.github/workflows/linea-besu-package-release.yml @@ -48,7 +48,7 @@ jobs: outputs: versions_env: ${{ steps.filter.outputs.versions-env }} linea_configs: ${{ steps.filter.outputs.linea-configs }} - dockerfile: ${{ steps.filter.outputs.dockerfile }} + docker-files: ${{ steps.filter.outputs.docker-files }} steps: - name: Checkout uses: actions/checkout@v4 @@ -61,8 +61,9 @@ jobs: filters: | versions-env: - 'linea-besu-package/versions.env' - dockerfile: + docker-files: - 'linea-besu-package/linea-besu/Dockerfile' + - 'linea-besu-package/scripts/assemble-packages.sh' linea-configs: - 'linea-besu-package/linea-besu/**' @@ -73,7 +74,7 @@ jobs: with: release_tag_prefix: ${{ inputs.release_tag_prefix }} run_test: true - run_e2e_test: ${{ github.event_name == 'workflow_dispatch' || needs.filter-commit-changes.outputs.versions_env == 'true' || needs.filter-commit-changes.outputs.dockerfile == 'true' }} + run_e2e_test: ${{ github.event_name == 'workflow_dispatch' || needs.filter-commit-changes.outputs.versions_env == 'true' || needs.filter-commit-changes.outputs.docker-files == 'true' }} push_image: ${{ github.event_name == 'workflow_dispatch' }} skip_e2e_test: ${{ inputs.skip_e2e_test == 'true' }} expected_traces_api_ver: ${{ inputs.expected_traces_api_ver }} diff --git a/.github/workflows/reuse-linea-besu-package-run-test.yml b/.github/workflows/reuse-linea-besu-package-run-test.yml index 9278d0de..c9cbeed6 100644 --- a/.github/workflows/reuse-linea-besu-package-run-test.yml +++ b/.github/workflows/reuse-linea-besu-package-run-test.yml @@ -59,7 +59,7 @@ jobs: eval $COMMAND - name: Verify besu container - run: bash linea-besu-package/docker/scripts/BesuContainerVerify.sh + run: bash linea-besu-package/scripts/besu-container-verify.sh env: CONTAINER_NAME: ${{ env.CONTAINER_NAME }} diff --git a/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/app/L1DependentApp.kt b/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/app/L1DependentApp.kt index bf6d893f..a7e84521 100644 --- a/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/app/L1DependentApp.kt +++ b/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/app/L1DependentApp.kt @@ -564,7 +564,7 @@ class L1DependentApp( measurementSupplier = maxBlobEndBlockNumberTracker, ) - val highestAggregationTracker = HighestULongTracker(lastProcessedBlockNumber) + val highestAggregationTracker = HighestULongTracker(lastConsecutiveAggregatedBlockNumber) metricsFacade.createGauge( category = LineaMetricsCategory.AGGREGATION, name = "proven.highest.block.number", diff --git a/docker/compose-spec-l2-services.yml b/docker/compose-spec-l2-services.yml index 97667755..ff5d31bc 100644 --- a/docker/compose-spec-l2-services.yml +++ b/docker/compose-spec-l2-services.yml @@ -201,7 +201,7 @@ services: coordinator: hostname: coordinator container_name: coordinator - image: consensys/linea-coordinator:${COORDINATOR_TAG:-c3eb71d} + image: consensys/linea-coordinator:${COORDINATOR_TAG:-7e306e2} platform: linux/amd64 profiles: [ "l2", "debug" ] depends_on: diff --git a/e2e/src/common/utils.ts b/e2e/src/common/utils.ts index 95c93db9..aeff8b59 100644 --- a/e2e/src/common/utils.ts +++ b/e2e/src/common/utils.ts @@ -521,7 +521,7 @@ export async function execDockerCommand(command: string, containerName: string): logger.error(`Error executing (${dockerCommand}). error=${stderr}`); reject(error); } - logger.info(`Execution success (${dockerCommand}). output=${stdout}`); + logger.debug(`Execution success (${dockerCommand}). output=${stdout}`); resolve(stdout); }); }); diff --git a/e2e/src/shomei-get-proof.spec.ts b/e2e/src/shomei-get-proof.spec.ts index 40aaafa9..09cf4d55 100644 --- a/e2e/src/shomei-get-proof.spec.ts +++ b/e2e/src/shomei-get-proof.spec.ts @@ -13,14 +13,12 @@ describe("Shomei Linea get proof test suite", () => { "Call linea_getProof to Shomei frontend node and get a valid proof", async () => { const shomeiImageTag = await getDockerImageTag("shomei-frontend", "consensys/linea-shomei"); - logger.debug(`shomeiImageTag=${shomeiImageTag}`); + logger.info(`shomeiImageTag=${shomeiImageTag}`); - const currentL2BlockNumber = await awaitUntil( + let targetL2BlockNumber = await awaitUntil( async () => { try { - return await lineaRollupV6.currentL2BlockNumber({ - blockTag: "finalized", - }); + return await lineaRollupV6.currentL2BlockNumber({ blockTag: "finalized" }); } catch (err) { if (!(err as Error).message.includes("could not decode result data")) { throw err; @@ -30,30 +28,55 @@ describe("Shomei Linea get proof test suite", () => { }, (currentL2BlockNumber: bigint) => currentL2BlockNumber > 1n, 2000, - 100000, + 150000, ); - expect(currentL2BlockNumber).toBeGreaterThan(1n); - - logger.debug(`currentL2BlockNumber=${currentL2BlockNumber}`); + expect(targetL2BlockNumber).toBeGreaterThan(1n); + const finalizedL2BlockNumbers = [targetL2BlockNumber!]; const provingAddress = "0xfe3b557e8fb62b89f4916b721be55ceb828dbd73"; // from genesis file const getProofResponse = await awaitUntil( - async () => - lineaShomeiFrontenedClient.lineaGetProof(provingAddress, [], "0x" + currentL2BlockNumber!.toString(16)), + async () => { + let getProofResponse; + // Need to put all the latest currentL2BlockNumber in a list and traverse to get the proof + // from one of them as we don't know on which finalized L2 block number the shomei frontend + // was being notified + for (const finalizedL2BlockNumber of finalizedL2BlockNumbers) { + getProofResponse = await lineaShomeiFrontenedClient.lineaGetProof( + provingAddress, + [], + "0x" + finalizedL2BlockNumber.toString(16), + ); + if (getProofResponse?.result) { + targetL2BlockNumber = finalizedL2BlockNumber; + break; + } + } + if (!getProofResponse?.result) { + const latestFinalizedL2BlockNumber = await lineaRollupV6.currentL2BlockNumber({ blockTag: "finalized" }); + if (!finalizedL2BlockNumbers.includes(latestFinalizedL2BlockNumber)) { + finalizedL2BlockNumbers.push(latestFinalizedL2BlockNumber); + logger.info(`finalizedL2BlockNumbers=${JSON.stringify(finalizedL2BlockNumbers.map((it) => Number(it)))}`); + } + } + return getProofResponse; + }, (getProofResponse) => getProofResponse?.result, 2000, - 100000, + 150000, ); + logger.info(`targetL2BlockNumber=${targetL2BlockNumber}`); + const { result: { zkEndStateRootHash }, } = await lineaShomeiClient.rollupGetZkEVMStateMerkleProofV0( - Number(currentL2BlockNumber), - Number(currentL2BlockNumber), + Number(targetL2BlockNumber), + Number(targetL2BlockNumber), shomeiImageTag, ); + logger.info(`zkEndStateRootHash=${zkEndStateRootHash}`); expect(zkEndStateRootHash).toBeDefined(); const l2SparseMerkleProofContract = config.getL2SparseMerkleProofContract(); @@ -80,6 +103,6 @@ describe("Shomei Linea get proof test suite", () => { expect(isInvalid).toBeTruthy(); }, - 100_000, + 150_000, ); }); diff --git a/linea-besu-package/Makefile b/linea-besu-package/Makefile new file mode 100644 index 00000000..f68643c6 --- /dev/null +++ b/linea-besu-package/Makefile @@ -0,0 +1,36 @@ +include versions.env + +TAG ?= local +PLATFORM ?= + +assemble: + ./scripts/assemble-packages.sh + +build: + $(MAKE) assemble + echo "IMAGE TAG=$(TAG)" + echo "PLATFORM=$(PLATFORM)" + if [ "$(PLATFORM)" = "" ]; then \ + echo "Building image for default platform"; \ + docker buildx build --file linea-besu/Dockerfile --tag consensys/linea-besu-package:$(TAG) ./tmp --progress=plain --no-cache --load; \ + else \ + echo "Building image for $(PLATFORM)"; \ + docker buildx build --file linea-besu/Dockerfile --platform $(PLATFORM) --tag consensys/linea-besu-package:$(TAG) ./tmp --progress=plain --no-cache --load; \ + fi + +run-e2e-test: + echo "EXPECTED_TRACES_API_VERSION=${LINEA_TRACER_PLUGIN_VERSION}" + if [ "$(shell uname)" = "Linux" ]; then \ + sed -i'' 's/^\(expected-traces-api-version-v2=\).*/\1"${LINEA_TRACER_PLUGIN_VERSION}"/' ../config/coordinator/coordinator-docker.config.toml; \ + sed -i'' 's/^\(expected-traces-api-version-v2=\).*/\1"${LINEA_TRACER_PLUGIN_VERSION}"/' ../config/coordinator/coordinator-docker-traces-v2-override.config.toml; \ + elif [ "$(shell uname)" = "Darwin" ]; then \ + sed -i '' 's/^\(expected-traces-api-version-v2=\).*/\1"${LINEA_TRACER_PLUGIN_VERSION}"/' ../config/coordinator/coordinator-docker.config.toml; \ + sed -i '' 's/^\(expected-traces-api-version-v2=\).*/\1"${LINEA_TRACER_PLUGIN_VERSION}"/' ../config/coordinator/coordinator-docker-traces-v2-override.config.toml; \ + fi + cd .. && BESU_PACKAGE_TAG=$(TAG) make start-env-with-tracing-v2-ci && pnpm run -F e2e test:e2e:local + +run-e2e-test-cleanup: + cd .. && make clean-environment + +clean: + rm -fr tmp || true diff --git a/linea-besu-package/README.md b/linea-besu-package/README.md index e827c15b..b9e77aca 100644 --- a/linea-besu-package/README.md +++ b/linea-besu-package/README.md @@ -61,20 +61,37 @@ Linea Finalization status is based on L1 RPC endpoint's response bin/besu --profile=advanced-mainnet --plugin-linea-l1-rpc-endpoint=YOUR_L1_RPC_ENDPOINT ``` -## Build from source +## Build from source locally -1. Make a branch with changes to `linea-besu-package/versions.env` as needed -2. Create a PR for the branch -3. Go to the [actions tab](https://github.com/Consensys/linea-monorepo/actions) to check if the workflow completed successfully -4. Go to the [releases page](https://github.com/Consensys/linea-monorepo/releases?q=linea-besu-package&expanded=true) and you should find the corresponding release info along with the docker image tag +1. Make changes to `linea-besu-package/versions.env` as needed + +2. Cd into `linea-besu-package` + +3. Run `make clean && make build` (check the Makefile for build options) + +4. The docker image (i.e. default as `consensys/linea-besu-package:local`) should be created locally + +### Note: + +To build with specific platform (e.g. linux/amd64) and image tag (e.g. xxx), do the following: +``` +make clean && PLATFORM=linux/amd64 TAG=xxx make build +``` + +To run the e2e test locally with the locally-built `linea-besu-package` image (e.g. tagged as xxx), do the following: +``` +TAG=xxx make run-e2e-test +``` ## How-To Release -1. Go to the [actions tab](https://github.com/Consensys/linea-monorepo/actions) and click on the workflow `linea-besu-package-release` for release with besu and plugin versions based on `linea-besu-package/versions.env` +1. Make a branch with changes to `linea-besu-package/versions.env` as needed -2. If release prefix is not given, `LINEA_TRACER_PLUGIN_VERSION` in the target `versions.env` file will be used, and the resultant release tag would be `linea-besu-package-[releasePrefix]-[YYYYMMDDHHMMSS]-[shortenCommitHash]` and the docker image tag would be `[releasePrefix]-[YYYYMMDDHHMMSS]-[shortenCommitHash]` +2. Go to the [actions tab](https://github.com/Consensys/linea-monorepo/actions) and click on the workflow `linea-besu-package-release` and select the target branch for making a release with besu and plugin versions based on `linea-besu-package/versions.env` -3. Go to the [releases page](https://github.com/Consensys/linea-monorepo/releases?q=linea-besu-package&expanded=true) and you should find the corresponding release info along with the docker image tag +3. If release prefix is not given, `LINEA_TRACER_PLUGIN_VERSION` in the target `versions.env` file will be used, and the resultant release tag would be `linea-besu-package-[releasePrefix]-[YYYYMMDDHHMMSS]-[shortenCommitHash]` and the docker image tag would be `[releasePrefix]-[YYYYMMDDHHMMSS]-[shortenCommitHash]` + +4. Once the workflow is done successfully, go to the [releases page](https://github.com/Consensys/linea-monorepo/releases?q=linea-besu-package&expanded=true) and you should find the corresponding release info along with the docker image tag Additionally, the `latest` tag will be updated to match this release diff --git a/linea-besu-package/scripts/assemble-packages.sh b/linea-besu-package/scripts/assemble-packages.sh new file mode 100755 index 00000000..c8de0705 --- /dev/null +++ b/linea-besu-package/scripts/assemble-packages.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +source ./versions.env + +mkdir -p ./tmp +pushd ./tmp + +echo "downloading besu from linea-besu-upstream: $LINEA_BESU_TAR_GZ" +wget -nv $LINEA_BESU_BASE_URL$LINEA_BESU_TAR_GZ/$LINEA_BESU_FILENAME_PREFIX-$LINEA_BESU_TAR_GZ.tar.gz +tar -xvf $LINEA_BESU_FILENAME_PREFIX-$LINEA_BESU_TAR_GZ.tar.gz +mv $LINEA_BESU_FILENAME_PREFIX-$LINEA_BESU_TAR_GZ ./besu + +echo "copying the versions.env to the container as versions.txt" +cp ../versions.env ./besu/versions.txt + +mkdir -p ./besu/plugins +cd ./besu/plugins + +echo "downloading the plugins" +echo "getting linea_sequencer_plugin_version: $LINEA_SEQUENCER_PLUGIN_VERSION" +wget -nv https://github.com/Consensys/linea-sequencer/releases/download/v$LINEA_SEQUENCER_PLUGIN_VERSION/linea-sequencer-v$LINEA_SEQUENCER_PLUGIN_VERSION.jar + +echo "getting linea_finalized_tag_updater_plugin_version: $LINEA_FINALIZED_TAG_UPDATER_PLUGIN_VERSION" +wget -nv https://github.com/Consensys/linea-monorepo/releases/download/linea-finalized-tag-updater-v$LINEA_FINALIZED_TAG_UPDATER_PLUGIN_VERSION/linea-finalized-tag-updater-v$LINEA_FINALIZED_TAG_UPDATER_PLUGIN_VERSION.jar + +echo "getting linea_staterecovery_plugin_version: $LINEA_STATERECOVERY_PLUGIN_VERSION" +wget -nv https://github.com/Consensys/linea-monorepo/releases/download/linea-staterecovery-v$LINEA_STATERECOVERY_PLUGIN_VERSION/linea-staterecovery-besu-plugin-v$LINEA_STATERECOVERY_PLUGIN_VERSION.jar + +echo "getting linea_tracer_plugin_version: $LINEA_TRACER_PLUGIN_VERSION" +wget -nv https://github.com/Consensys/linea-tracer/releases/download/$LINEA_TRACER_PLUGIN_VERSION/linea-tracer-$LINEA_TRACER_PLUGIN_VERSION.jar + +echo "getting shomei_plugin_version: $SHOMEI_PLUGIN_VERSION" +wget -nv https://github.com/Consensys/besu-shomei-plugin/releases/download/v$SHOMEI_PLUGIN_VERSION/besu-shomei-plugin-v$SHOMEI_PLUGIN_VERSION.jar + +popd + +echo "placing the packages, config, profiles together for preparing docker image build" +cd ./linea-besu +cp -r config genesis profiles ../tmp/besu/ \ No newline at end of file diff --git a/linea-besu-package/docker/scripts/BesuContainerVerify.sh b/linea-besu-package/scripts/besu-container-verify.sh similarity index 100% rename from linea-besu-package/docker/scripts/BesuContainerVerify.sh rename to linea-besu-package/scripts/besu-container-verify.sh