GitHub Actions CI/CD (#6427)

Implements a CI/CD pipeline using Github Actions, to replace the current CircleCI implementation.

https://wiki.hyperledger.org/pages/viewpage.action?pageId=80774216


---------

Signed-off-by: Justin Florentine <justin+github@florentine.us>
Signed-off-by: Gabriel Fukushima <gabrielfukushima@gmail.com>
Signed-off-by: Sally MacFarlane <macfarla.github@gmail.com>
Signed-off-by: jflo <justin+github@florentine.us>
Signed-off-by: RoboCopsGoneSock <158174948+RoboCopsGoneSock@users.noreply.github.com>
Signed-off-by: Danno Ferrin (shemnon) <danno.ferrin@shemnon.com>
Signed-off-by: Danno Ferrin <danno.ferrin@swirldslabs.com>
Signed-off-by: Karim Taam <karim.t2am@gmail.com>
Signed-off-by: Ameziane H <ameziane.hamlat@consensys.net>
Signed-off-by: ahamlat <ameziane.hamlat@consensys.net>
Signed-off-by: garyschulte <garyschulte@gmail.com>
Co-authored-by: Fabio Di Fabio <fabio.difabio@consensys.net>
Co-authored-by: Gabriel Fukushima <gabrielfukushima@gmail.com>
Co-authored-by: Sally MacFarlane <macfarla.github@gmail.com>
Co-authored-by: RoboCopsGoneSock <158174948+RoboCopsGoneSock@users.noreply.github.com>
Co-authored-by: Danno Ferrin <danno.ferrin@shemnon.com>
Co-authored-by: Karim TAAM <karim.t2am@gmail.com>
Co-authored-by: garyschulte <garyschulte@gmail.com>

.github/workflows/acceptance-tests.yml

@@ -0,0 +1,114 @@
+name: acceptance-tests
+on:
+  pull_request:
+  pull_request_review:
+    types: [submitted]
+
+env:
+  GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false"
+  total-runners: 16
+
+jobs:
+  shouldRun:
+    name: checks to ensure we should run
+    # necessary because there is no single PR approved event, need to check all comments/approvals/denials
+    runs-on: ubuntu-22.04
+    outputs:
+      shouldRun: ${{steps.shouldRun.outputs.result}}
+    steps:
+      - name: required check
+        id: shouldRun
+        uses: actions/github-script@v7.0.1
+        env:
+          # fun fact, this changes based on incoming event, it will be different when we run this on pushes to main
+          RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+        with:
+          script: |
+            const { RELEVANT_SHA } = process.env;
+            const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: RELEVANT_SHA,
+            });
+            const acceptanceTested = statuses && statuses.filter(({ context }) => context === 'acceptance-tests');
+            const alreadyRun = acceptanceTested && acceptanceTested.find(({ state }) => state === 'success') > 0;
+            const { data: reviews } = await github.rest.pulls.listReviews({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number,
+            });
+            const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED');
+            const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' &&  (approvingReviews.length > 0);
+            
+              console.log("tests should be run = %j", shouldRun);
+              console.log("alreadyRun = %j", alreadyRun);
+              console.log("approvingReviews = %j", approvingReviews.length);
+            
+            return shouldRun;
+  acceptanceTestEthereum:
+    runs-on: ubuntu-22.04
+    name: "Acceptance Runner"
+    needs: shouldRun
+    permissions:
+      statuses: write
+      checks: write
+    if: ${{ needs.shouldRun.outputs.shouldRun == 'true'}}
+    strategy:
+      fail-fast: true
+      matrix:
+        runner_index: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4.1.1
+      - name: Set up Java
+        uses: actions/setup-java@v4.0.0
+        with:
+          distribution: temurin
+          java-version: 17
+      - name: get acceptance test report
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          branch: main
+          name_is_regexp: true
+          name: 'acceptance-node-\d*\d-test-results'
+          path: tmp/junit-xml-reports-downloaded
+          if_no_artifact_found: true
+      - name: setup gradle
+        uses: gradle/gradle-build-action@v2.12.0
+      - name: Split tests
+        id: split-tests
+        uses: r7kamura/split-tests-by-timings@v0
+        with:
+          reports: tmp/junit-xml-reports-downloaded
+          glob: 'acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/**/*Test.java'
+          total: ${{env.total-runners}}
+          index: ${{ matrix.runner_index }}
+      - name: write out test list
+        run: echo "${{ steps.split-tests.outputs.paths }}" >> testList.txt
+      - name: format gradle args
+        #regex means: first truncate file paths to align with package name, then swap path delimiter with package delimiter,
+        #then drop file extension, then insert --tests option between each.
+        run: cat testList.txt | sed -e 's@acceptance-tests/tests/src/test/java/@--tests\ @g;s@/@.@g;s/\.java//g'  > gradleArgs.txt
+      - name: run acceptance tests
+        run: ./gradlew acceptanceTest `cat gradleArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
+      - name: cleanup tempfiles
+        run: rm testList.txt gradleArgs.txt
+      - name: Upload Acceptance Test Results
+        uses: actions/upload-artifact@v3.1.0
+        with:
+          name: acceptance-node-${{matrix.runner_index}}-test-results
+          path: 'acceptance-tests/tests/build/test-results/acceptanceTest/TEST-*.xml'
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v4
+        if: (success() || failure()) # always run even if the build step fails
+        with:
+          report_paths: 'acceptance-tests/tests/build/test-results/acceptanceTest/TEST-*.xml'
+  acceptance-tests:
+    runs-on: ubuntu-22.04
+    needs: [ acceptanceTestEthereum ]
+    permissions:
+      checks: write
+      statuses: write
+    steps:
+      - name: consolidation
+        run: echo "consolidating statuses"

.github/workflows/artifacts.yml

@@ -0,0 +1,76 @@
+
+name: artifacts
+
+on:
+  release:
+    types:
+      - prereleased
+
+jobs:
+  artifacts:
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: write
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4.1.1
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4.0.0
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+      - name: setup gradle
+        uses: gradle/gradle-build-action@v2.12.0
+      - name: assemble distributions
+        run:
+          ./gradlew -Prelease.releaseVersion=${{github.ref_name}} assemble -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
+      - name: hashes
+        id: hashes
+        run: |
+          cd build/distributions
+          echo "zipSha=$(shasum -a 256 besu*.zip)" >> $GITHUB_OUTPUT
+          echo "tarSha=$(shasum -a 256 besu*.tar.gz)" >> $GITHUB_OUTPUT
+      - name: upload tarball
+        uses: actions/upload-artifact@v3
+        with:
+          path: 'build/distributions/besu*.tar.gz'
+          name: besu-${{ github.ref_name }}.tar.gz
+      - name: upload zipfile
+        uses: actions/upload-artifact@v3
+        with:
+          path: 'build/distributions/besu*.zip'
+          name: besu-${{ github.ref_name }}.zip
+      - name: Upload Release assets
+        uses: softprops/action-gh-release@v1
+        with:
+          append_body: true
+          files: |
+            build/distributions/besu*.tar.gz
+            build/distributions/besu*.zip
+          body: |
+            ${{steps.hashes.outputs.tarSha}}
+            ${{steps.hashes.outputs.zipSha}}
+  testWindows:
+    runs-on: windows-2022
+    needs: artifacts
+    timeout-minutes: 10
+    if: ${{ github.actor != 'dependabot[bot]' }}
+    steps:
+      - name: Set up Java
+        uses: actions/setup-java@v4.0.0
+        with:
+          distribution: adopt
+          java-version: 17
+      - name: Download zip
+        uses: actions/download-artifact@v3
+        with:
+          name: besu-${{ github.ref_name }}.zip
+      - name: test Besu
+        run: |
+          unzip besu-*.zip -d besu-tmp
+          cd besu-tmp
+          mv besu-* ../besu
+          cd ..
+          besu\bin\besu.bat --help
+          besu\bin\besu.bat --version
+

.github/workflows/checks.yml

@@ -1,36 +0,0 @@
-name: checks
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-  workflow_dispatch:
-
-jobs:
-  spotless:
-    runs-on: [besu-research-ubuntu-16]
-    if: ${{ github.actor != 'dependabot[bot]' }}
-    steps:
-      - name: Checkout Repo
-        uses: actions/checkout@v4
-      - name: Set up Java
-        uses: actions/setup-java@v4
-        with:
-          distribution: 'temurin'
-          java-version: 17
-          cache: gradle
-      - name: spotless
-        run: ./gradlew --no-daemon --parallel clean spotlessCheck
-  javadoc_17:
-    runs-on: [besu-research-ubuntu-8]
-    if: ${{ github.actor != 'dependabot[bot]' }}
-    steps:
-      - name: Checkout Repo
-        uses: actions/checkout@v4
-      - name: Set up Java 17
-        uses: actions/setup-java@v4
-        with:
-          distribution: 'temurin'
-          java-version: 17
-          cache: gradle
-      - name: javadoc (JDK 17)
-        run: ./gradlew --no-daemon clean javadoc

.github/workflows/codeql.yml

@@ -14,40 +14,29 @@ name: "CodeQL"
 on:
   push:
     branches: [ main ]
-  pull_request:
+    pull_request:
-    branches: [ main ]
+      branches: [ main ]
-    paths-ignore:
+      paths-ignore:
-      - '**/*.json'
+        - '**/*.json'
-      - '**/*.md'
+        - '**/*.md'
-      - '**/*.properties'
+        - '**/*.properties'
-      - '**/*.txt'
+        - '**/*.txt'
 jobs:
   analyze:
     name: Analyze
-    runs-on: [besu-research-ubuntu-16]
+    runs-on: ubuntu-22.04
     permissions:
       actions: read
       contents: read
       security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'java' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
-
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v4.1.1
-
     - name: Set up Java
-      uses: actions/setup-java@v4
+      uses: actions/setup-java@v4.0.0
       with:
         distribution: 'temurin'
         java-version: 17
-        cache: gradle
-
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v2
@@ -58,10 +47,10 @@ jobs:
         # Prefix the list here with "+" to use these queries and those in the config file.
         queries: security-and-quality,security-extended
 
-    # Autobuild failed (OOM)
+    - name: setup gradle
-    # Hence, supply memory args for gradle build
+      uses: gradle/gradle-build-action@v2.12.0
-    - run: |
+    - name: compileJava noscan
-        JAVA_OPTS="-Xmx1000M" ./gradlew --no-scan compileJava
+      run: |
-
+        JAVA_OPTS="-Xmx2048M" ./gradlew --no-scan compileJava
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@v2

.github/workflows/dco-merge-group.yml

@@ -1,10 +0,0 @@
-name: dco
-on:
-  merge_group:
-
-jobs:
-  dco:
-    runs-on: [besu-research-ubuntu-8]
-    if: ${{ github.actor != 'dependabot[bot]' }}
-    steps:
-      - run: echo "This DCO job runs on merge_queue event and doesn't check PR contents"

.github/workflows/dco.yml

@@ -1,20 +0,0 @@
-name: dco
-on:
-  pull_request:
-  workflow_dispatch:
-
-jobs:
-  dco:
-    runs-on: [besu-research-ubuntu-8]
-    if: ${{ github.actor != 'dependabot[bot]' }}
-    steps:
-      - run: echo "This DCO job runs on pull_request event and workflow_dispatch"
-      - name: Get PR Commits
-        id: 'get-pr-commits'
-        uses: tim-actions/get-pr-commits@v1.2.0
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-      - name: DCO Check
-        uses: tim-actions/dco@v1.1.0
-        with:
-          commits: ${{ steps.get-pr-commits.outputs.commits }}

.github/workflows/docker.yml

@@ -0,0 +1,113 @@
+name: docker
+on:
+  release:
+    types:
+      - prereleased
+env:
+  registry: ghcr.io
+
+jobs:
+  hadolint:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4.1.1
+      - name: Set up Java
+        uses: actions/setup-java@v4.0.0
+        with:
+          distribution: temurin
+          java-version: 17
+      - name: setup gradle
+        uses: gradle/gradle-build-action@v2.12.0
+      - name: hadoLint_openj9-jdk_17
+        run: docker run --rm -i hadolint/hadolint < docker/openj9-jdk-17/Dockerfile
+      - name: hadoLint_openjdk_17
+        run: docker run --rm -i hadolint/hadolint < docker/openjdk-17/Dockerfile
+      - name: hadoLint_openjdk_17_debug
+        run: docker run --rm -i hadolint/hadolint < docker/openjdk-17-debug/Dockerfile
+      - name: hadoLint_openjdk_latest
+        run: docker run --rm -i hadolint/hadolint < docker/openjdk-latest/Dockerfile
+      - name: hadoLint_graalvm
+        run: docker run --rm -i hadolint/hadolint < docker/graalvm/Dockerfile
+  buildDocker:
+    needs: hadolint
+    permissions:
+      contents: read
+      packages: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - ubuntu-22.04
+          - [self-hosted, ARM64]
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - name: Prepare
+        id: prep
+        run: |
+            platform=${{ matrix.platform }}
+            if [ "$platform" = 'ubuntu-22.04' ]; then 
+              echo "PLATFORM_PAIR=linux-amd64" >> $GITHUB_OUTPUT
+              echo "ARCH=amd64" >> $GITHUB_OUTPUT
+            else
+              echo "PLATFORM_PAIR=linux-arm64" >> $GITHUB_OUTPUT
+              echo "ARCH=arm64" >> $GITHUB_OUTPUT
+            fi
+      - name: Checkout Repo
+        uses: actions/checkout@v4.1.1
+      - name: short sha
+        id: shortSha
+        run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Set up Java
+        uses: actions/setup-java@v4.0.0
+        with:
+          distribution: temurin
+          java-version: 17
+      - name: setup gradle
+        uses: gradle/gradle-build-action@v2.12.0
+      - name: install goss
+        run: |
+          mkdir -p docker/reports
+          curl -L https://github.com/aelsabbahy/goss/releases/download/v0.4.4/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} -o ./docker/tests/goss-${{ steps.prep.outputs.PLATFORM_PAIR }}
+      - name: build and test docker
+        uses: gradle/gradle-build-action@v2.12.0
+        env:
+          architecture: ${{ steps.prep.outputs.ARCH }}
+        with:
+          arguments: testDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Prelease.releaseVersion=${{ github.ref_name }}
+      - name: login to ghcr
+        uses: docker/login-action@v3.0.0
+        with:
+          registry: ${{ env.registry }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: publish
+        env:
+          architecture: ${{ steps.prep.outputs.ARCH }}
+        run: ./gradlew --no-daemon dockerUpload -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Prelease.releaseVersion=${{ github.ref_name }}
+  multiArch:
+    needs: buildDocker
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4.1.1
+      - name: Set up Java
+        uses: actions/setup-java@v4.0.0
+        with:
+          distribution: temurin
+          java-version: 17
+      - name: setup gradle
+        uses: gradle/gradle-build-action@v2.12.0
+      - name: login to ghcr
+        uses: docker/login-action@v3.0.0
+        with:
+          registry: ${{ env.registry }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: multi-arch docker
+        run: ./gradlew manifestDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Prelease.releaseVersion=${{ github.ref_name }}
+

.github/workflows/gradle-wrapper-validation.yml

@@ -1,11 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-name: "Validate Gradle Wrapper"
-on: [push, pull_request]
-
-jobs:
-  validation:
-    name: "Gradle Wrapper Validation"
-    runs-on: [besu-research-ubuntu-8]
-    steps:
-      - uses: actions/checkout@v4
-      - uses: gradle/wrapper-validation-action@v1

.github/workflows/integration-tests.yml

@@ -0,0 +1,73 @@
+name: integration-tests
+on:
+  pull_request:
+  pull_request_review:
+    types:
+      - submitted
+
+env:
+  GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false"
+
+jobs:
+  shouldRun:
+    name: checks to ensure we should run
+    runs-on: ubuntu-22.04
+    outputs:
+      shouldRun: ${{steps.shouldRun.outputs.result}}
+    steps:
+      - name: required check
+        id: shouldRun
+        uses: actions/github-script@v7.0.1
+        env:
+          # fun fact, this changes based on incoming event, it will be different when we run this on pushes to main
+          RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+        with:
+          script: |
+            const { RELEVANT_SHA } = process.env;
+            const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: RELEVANT_SHA,
+            });
+            
+            const intTested = statuses && statuses.filter(({ context }) => context === 'integration-tests');
+            const alreadyRun = intTested && intTested.find(({ state }) => state === 'success') > 0;
+            const { data: reviews } = await github.rest.pulls.listReviews({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number,
+            });
+            const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED');
+            const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (approvingReviews.length > 0);
+
+              console.log("tests should be run = %j", shouldRun);
+              console.log("alreadyRun = %j", alreadyRun);
+              console.log("approvingReviews = %j", approvingReviews.length);
+
+            return shouldRun;
+  integration-tests:
+    runs-on: ubuntu-22.04
+    needs: shouldRun
+    if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }}
+    permissions:
+      statuses: write
+      checks: write
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4.1.1
+      - name: Set up Java
+        uses: actions/setup-java@v4.0.0
+        with:
+          distribution: temurin
+          java-version: 17
+      - name: setup gradle
+        uses: gradle/gradle-build-action@v2.12.0
+      - name: run integration tests
+        run: ./gradlew integrationTest compileJmh -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v4
+        if: (success() || failure())
+        with:
+          report_paths: '**/build/test-results/integrationTest/TEST-*.xml'
+
+

.github/workflows/nightly.yml

@@ -0,0 +1,121 @@
+name: nightly
+
+on:
+  workflow_dispatch:
+  schedule:
+    # * is a special character in YAML so you have to quote this string
+    # expression evaluates to midnight every night
+    - cron:  '0 0 * * *'
+
+env:
+  nightly-tag: develop
+  registry: ghcr.io
+
+jobs:
+  hadolint:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4.1.1
+      - name: Set up Java
+        uses: actions/setup-java@v4.0.0
+        with:
+          distribution: temurin
+          java-version: 17
+      - name: setup gradle
+        uses: gradle/gradle-build-action@v2.12.0
+      - name: hadoLint_openj9-jdk_17
+        run: docker run --rm -i hadolint/hadolint < docker/openj9-jdk-17/Dockerfile
+      - name: hadoLint_openjdk_17
+        run: docker run --rm -i hadolint/hadolint < docker/openjdk-17/Dockerfile
+      - name: hadoLint_openjdk_17_debug
+        run: docker run --rm -i hadolint/hadolint < docker/openjdk-17-debug/Dockerfile
+      - name: hadoLint_openjdk_latest
+        run: docker run --rm -i hadolint/hadolint < docker/openjdk-latest/Dockerfile
+      - name: hadoLint_graalvm
+        run: docker run --rm -i hadolint/hadolint < docker/graalvm/Dockerfile
+  buildDocker:
+    needs: hadolint
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - ubuntu-22.04
+          - [self-hosted, ARM64]
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - name: Prepare
+        id: prep
+        run: |
+          platform=${{ matrix.platform }}
+          if [ "$platform" = 'ubuntu-22.04' ]; then 
+            echo "PLATFORM_PAIR=linux-amd64" >> $GITHUB_OUTPUT
+            echo "ARCH=amd64" >> $GITHUB_OUTPUT
+          else
+            echo "PLATFORM_PAIR=linux-arm64" >> $GITHUB_OUTPUT
+            echo "ARCH=arm64" >> $GITHUB_OUTPUT
+          fi
+      - name: Checkout Repo
+        uses: actions/checkout@v4.1.1
+      - name: short sha
+        id: shortSha
+        run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Set up Java
+        uses: actions/setup-java@v4.0.0
+        with:
+          distribution: temurin
+          java-version: 17
+      - name: setup gradle
+        uses: gradle/gradle-build-action@v2.12.0
+      - name: build image
+        uses: gradle/gradle-build-action@v2.12.0
+        with:
+          arguments: distDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Pbranch=main
+      - name: install goss
+        run: |
+          mkdir -p docker/reports
+          curl -L https://github.com/aelsabbahy/goss/releases/download/v0.4.4/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} -o ./docker/tests/goss-${{ steps.prep.outputs.PLATFORM_PAIR }}
+      - name: test docker
+        uses: gradle/gradle-build-action@v2.12.0
+        env:
+          architecture: ${{ steps.prep.outputs.ARCH }}
+        with:
+          arguments: testDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Pbranch=main
+      - name: login to ghcr
+        uses: docker/login-action@v3.0.0
+        with:
+          registry: ${{ env.registry }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: publish
+        env:
+          architecture: ${{ steps.prep.outputs.ARCH }}
+        run: ./gradlew --no-daemon dockerUpload -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Pbranch=main
+  multiArch:
+    permissions:
+      contents: read
+      packages: write
+    needs: buildDocker
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4.1.1
+      - name: Set up Java
+        uses: actions/setup-java@v4.0.0
+        with:
+          distribution: temurin
+          java-version: 17
+      - name: setup gradle
+        uses: gradle/gradle-build-action@v2.12.0
+      - name: Login to DockerHub
+        uses: docker/login-action@v3.0.0
+        with:
+          registry: ${{ env.registry }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: multi-arch docker
+        run: ./gradlew manifestDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Pbranch=main
+

.github/workflows/parallel-unit-tests.yml

@@ -0,0 +1,49 @@
+name: parallel-unit-tests
+#experimental work in progress - trying to figure out how to split tests across multi-modules by runtime
+on:
+  workflow_dispatch:
+
+env:
+  GRADLE_OPTS: "-Dorg.gradle.daemon=false"
+  total-runners: 4
+jobs:
+  junit:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        runner_index:
+          - 0
+          - 1
+          - 2
+          - 3
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4.1.1
+      - name: Split tests
+        id: split-tests
+        uses: chaosaffe/split-tests@v1-alpha.1
+        with:
+          glob: '**/src/test/java/**/*.java'
+          split-total: ${{ env.total-runners }}
+          split-index: ${{ matrix.runner_index }}
+          line-count: true
+      - name: Set up Java
+        uses: actions/setup-java@v4.0.0
+        with:
+          distribution: adopt
+          java-version: 17
+          cache: gradle
+      - name: write out test list
+        run: echo "${{ steps.split-tests.outputs.test-suite }}" >> testList.txt
+      - name: debug testfile paths
+        run: cat testList.txt
+      - name: format gradle args
+        # regex means: truncate file paths to align with package name, replacing with tests switch, then drop file extension,
+        # then swap path delimiter with package delimiter
+        run: cat testList.txt | sed -e 's/[^ ]*src\/test\/java\//--tests\ /g' -e 's/\.java//g' -e 's/\//\./g' >> gradleArgs.txt
+      - name: debug test class list
+        run: cat gradleArgs.txt
+      - name: run unit tests
+        run: ./gradlew test `cat gradleArgs.txt`
+

.github/workflows/pr-checklist-on-open.yml

@@ -6,9 +6,11 @@ on:
 jobs:
   checklist:
     name: "add checklist as a comment on newly opened PRs"
-    runs-on: [besu-research-ubuntu-8]
+    runs-on: ubuntu-22.04
+    permissions:
+      pull-requests: write
     steps:
-      - uses: actions/github-script@v5
+      - uses: actions/github-script@v7.0.1
         with:
           github-token: ${{secrets.GITHUB_TOKEN}}
           script: |
@@ -16,5 +18,5 @@ jobs:
               issue_number: context.issue.number,
               owner: context.repo.owner,
               repo: context.repo.repo,
-              body: '- [ ] I thought about documentation and added the `doc-change-required` label to this PR if [updates are required](https://wiki.hyperledger.org/display/BESU/Documentation).\n- [ ] I thought about the changelog and included a [changelog update if required](https://wiki.hyperledger.org/display/BESU/Changelog).\n- [ ] If my PR includes database changes (e.g. KeyValueSegmentIdentifier) I have thought about compatibility and performed forwards and backwards compatibility tests'
+              body: '- [ ] I thought about documentation and added the `doc-change-required` label to this PR if [updates are required](https://wiki.hyperledger.org/display/BESU/Documentation).\n- [ ] I thought about the changelog and included a [changelog update if required](https://wiki.hyperledger.org/display/BESU/Changelog).\n- [ ] If my PR includes database changes (e.g. KeyValueSegmentIdentifier) I have thought about compatibility and performed forwards and backwards compatibility tests\n- [ ] I thought about running CI.\n- [ ] If I did not run CI, I ran as much locally as possible before pushing.\n-'
             })

.github/workflows/pre-review.yml

@@ -0,0 +1,103 @@
+name: pre-review
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  statuses: write
+  checks: write
+
+jobs:
+  repolint:
+    name: "Repository Linting"
+    runs-on: ubuntu-22.04
+    container: ghcr.io/todogroup/repolinter:v0.11.2
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v4.1.1
+    - name: Lint Repo
+      run: bundle exec /app/bin/repolinter.js --rulesetUrl https://raw.githubusercontent.com/hyperledger-labs/hyperledger-community-management-tools/main/repo_structure/repolint.json --format markdown
+  gradle-wrapper:
+    name: "Gradle Wrapper Validation"
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4.1.1
+      - uses: gradle/wrapper-validation-action@v1.1.0
+  spotless:
+    runs-on: ubuntu-22.04
+    if: ${{ github.actor != 'dependabot[bot]' }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4.1.1
+      - name: Set up Java
+        uses: actions/setup-java@v4.0.0
+        with:
+          distribution: temurin
+          java-version: 17
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2.12.0
+      - name: run spotless
+        run: ./gradlew spotlessCheck -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
+  compile:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    needs: [spotless, gradle-wrapper, repolint]
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4.1.1
+      - name: Set up Java
+        uses: actions/setup-java@v4.0.0
+        with:
+          distribution: temurin
+          java-version: 17
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2.12.0
+      - name: Gradle Compile
+        run: ./gradlew build -x test -x spotlessCheck -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
+  unitTests:
+    env:
+        GRADLEW_UNIT_TEST_ARGS: ${{matrix.gradle_args}}
+    runs-on: ubuntu-22.04
+    needs: [ compile ]
+    permissions:
+      checks: write
+      statuses: write
+    strategy:
+      fail-fast: true
+      matrix:
+        gradle_args:
+          - "test -x besu:test -x consensus:test -x crypto:test -x ethereum:eth:test -x ethereum:api:test -x ethereum:core:test"
+          - "besu:test consensus:test crypto:test"
+          - "ethereum:api:testBonsai"
+          - "ethereum:api:testForest"
+          - "ethereum:api:testRemainder"
+          - "ethereum:core:test"
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4.1.1
+      - name: Set up Java
+        uses: actions/setup-java@v4.0.0
+        with:
+          distribution: temurin
+          java-version: 17
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2.12.0
+      - name: run unit tests
+        id: unitTest
+        run: ./gradlew $GRADLEW_UNIT_TEST_ARGS -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v4
+        if: success() || failure() # always run even if the build step fails
+        with:
+          report_paths: '**/test-results/**/TEST-*.xml'
+          annotate_only: true
+  pre-review:
+    runs-on: ubuntu-22.04
+    needs: [unitTests]
+    permissions:
+      checks: write
+      statuses: write
+    steps:
+      - name: consolidation
+        run: echo "consolidating statuses"

.github/workflows/reference-tests.yml

@@ -0,0 +1,147 @@
+name: reference-tests
+on:
+  pull_request:
+  pull_request_review:
+    types:
+      - submitted
+
+env:
+  GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false"
+  total-runners: 6
+
+jobs:
+  shouldRun:
+    name: checks to ensure we should run
+    # necessary because there is no single PR approved event, need to check all comments/approvals/denials
+    # might also be a job running, and additional approvals
+    runs-on: ubuntu-22.04
+    outputs:
+      shouldRun: ${{steps.shouldRun.outputs.result}}
+    steps:
+      - name: required check
+        id: shouldRun
+        uses: actions/github-script@v7.0.1
+        env:
+          # fun fact, this changes based on incoming event, it will be different when we run this on pushes to main
+          RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+        with:
+          script: |
+            const { RELEVANT_SHA } = process.env;
+            const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: RELEVANT_SHA,
+            });
+            
+            
+            const refTested = statuses && statuses.filter(({ context }) => context === 'reference-tests');
+            const alreadyRun = refTested && refTested.find(({ state }) => state === 'success') > 0;
+            const { data: reviews } = await github.rest.pulls.listReviews({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number,
+            });
+            const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED');
+            const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' &&  (approvingReviews.length > 0);
+
+              console.log("tests should be run = %j", shouldRun);
+              console.log("alreadyRun = %j", alreadyRun);
+              console.log("approvingReviews = %j", approvingReviews.length);
+
+            return shouldRun;
+
+  prepareReferenceTestEthereum:
+    runs-on: ubuntu-22.04
+    needs: shouldRun
+    if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4.1.1
+        with:
+          submodules: recursive
+          set-safe-directory: true
+      - name: Set up Java
+        uses: actions/setup-java@v4.0.0
+        with:
+          distribution: temurin
+          java-version: 17
+      - name: setup gradle
+        uses: gradle/gradle-build-action@v2.12.0
+      - name: execute generate reference tests
+        run: ./gradlew ethereum:referencetests:blockchainReferenceTests ethereum:referencetests:generalstateReferenceTests ethereum:referencetests:generalstateRegressionReferenceTests -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
+      - name: store generated tests
+        uses: actions/upload-artifact@v3
+        with:
+          name: 'reference-tests'
+          path: 'ethereum/referencetests/build/generated/sources/reference-test/**/*.java'
+
+  referenceTestEthereum:
+    runs-on: ubuntu-22.04
+    permissions:
+      statuses: write
+      checks: write
+    needs:
+      - prepareReferenceTestEthereum
+    if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }}
+    strategy:
+      fail-fast: true
+      matrix:
+        runner_index: [0,1,2,3,4,5]
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4.1.1
+        with:
+          submodules: recursive
+      - name: Set up Java
+        uses: actions/setup-java@v4.0.0
+        with:
+          distribution: adopt-openj9
+          java-version: 17
+      - name: retrieve generated tests
+        uses: actions/download-artifact@v3.0.2
+        with:
+          name: 'reference-tests'
+          path: 'ethereum/referencetests/build/generated/sources/reference-test/'
+      - name: get reference test report
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          branch: main
+          name_is_regexp: true
+          name: 'reference-test-node-\d*\d-results'
+          path: tmp/ref-xml-reports-downloaded
+          if_no_artifact_found: true
+      - name: setup gradle
+        uses: gradle/gradle-build-action@v2.12.0
+      - name: Split tests
+        id: split-tests
+        uses: r7kamura/split-tests-by-timings@v0
+        with:
+          reports: tmp/ref-xml-reports-downloaded
+          glob: 'ethereum/referencetests/build/generated/sources/reference-test/**/*.java'
+          total: ${{env.total-runners}}
+          index: ${{ matrix.runner_index }}
+      - name: compose gradle args
+        run: echo ${{ steps.split-tests.outputs.paths }} | sed -e 's/^.*java\///' -e 's@/@.@g' -e 's/\.java//' -e 's/^/--tests /' > refTestArgs.txt
+      - name: run reference tests
+        run: ./gradlew ethereum:referenceTests:referenceTests `cat refTestArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
+      - name: Upload Test Report
+        uses: actions/upload-artifact@v3
+        if: always() # always run even if the previous step fails
+        with:
+          name: reference-test-node-${{matrix.runner_index}}-results
+          path: '**/build/test-results/referenceTests/TEST-*.xml'
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v4
+        if: success() || failure() # always run even if the build step fails
+        with:
+          report_paths: '**/build/test-results/referenceTest/TEST-*.xml'
+  reference-tests:
+    runs-on: ubuntu-22.04
+    needs: [ referenceTestEthereum ]
+    permissions:
+      checks: write
+      statuses: write
+    steps:
+      - name: consolidation
+        run: echo "consolidating statuses"
+

.github/workflows/release.yml

@@ -1,13 +1,14 @@
 name: release besu
 on:
+  workflow_dispatch:
   release:
-    types: released
+    types: [released]
 jobs:
   dockerPromoteX64:
-    runs-on: [besu-research-ubuntu-16]
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v4.1.1
-      - uses: actions/setup-java@v4
+      - uses: actions/setup-java@v4.0.0
         with:
           distribution: 'temurin' # See 'Supported distributions' for available options
           java-version: '17'
@@ -15,7 +16,7 @@ jobs:
       - name: Login to DockerHub
         run: echo '${{ secrets.DOCKER_PASSWORD_RW }}' | docker login -u '${{ secrets.DOCKER_USER_RW }}' --password-stdin
       - name: Setup Gradle
-        uses: gradle/gradle-build-action@v2
+        uses: gradle/gradle-build-action@v2.12.0
       - name: Docker upload
         run: ./gradlew "-Prelease.releaseVersion=${{ github.ref_name }}" "-PdockerOrgName=${{ secrets.DOCKER_ORG }}" dockerUploadRelease
       - name: Docker manifest

.github/workflows/repolinter.yml

@@ -1,24 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# Hyperledger Repolinter Action
-name: Repolinter
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - master
-      - main
-  pull_request:
-    branches:
-      - master
-      - main
-
-jobs:
-  build:
-    runs-on: [besu-research-ubuntu-16]
-    container: ghcr.io/todogroup/repolinter:v0.10.1
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@v4
-      - name: Lint Repo
-        run: bundle exec /app/bin/repolinter.js --rulesetUrl https://raw.githubusercontent.com/hyperledger-labs/hyperledger-community-management-tools/main/repo_structure/repolint.json --format markdown

.github/workflows/sonarcloud.yml

@@ -14,24 +14,26 @@ permissions:
 jobs:
   Analysis:
     runs-on: ubuntu-latest
-    if: github.repository == 'hyperledger/besu'
     steps:
       - name: checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v4.1.1
       - name: Set up JDK 17
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@v4.0.0
         with:
           distribution: 'temurin'
           java-version: '17'
-          cache: gradle
       - name: Cache SonarCloud packages
         uses: actions/cache@v3
         with:
           path: ~/.sonar/cache
           key: ${{ runner.os }}-sonar
           restore-keys: ${{ runner.os }}-sonar
+      - name: setup gradle
+        uses: gradle/gradle-build-action@v2.12.0
       - name: Build and analyze
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-        run: ./gradlew build sonarqube --continue --info
+          SONAR_ORGANIZATION: ${{ env.SONAR_ORGANIZATION }}
+          SONAR_PROJECT_KEY: $ {{ env.SONAR_PROJECT_KEY }}
+        run: ./gradlew build sonarqube --continue --info -Dorg.gradle.parallel=true -Dorg.gradle.caching=true

CHANGELOG.md

@@ -19,6 +19,7 @@
 - Log blob count when importing a block via Engine API [#6466](https://github.com/hyperledger/besu/pull/6466)
 - Introduce `--Xbonsai-limit-trie-logs-enabled` experimental feature which by default will only retain the latest 512 trie logs, saving about 3GB per week in database growth [#5390](https://github.com/hyperledger/besu/issues/5390)
 - Introduce `besu storage x-trie-log prune` experimental offline subcommand which will prune all redundant trie logs except the latest 512 [#6303](https://github.com/hyperledger/besu/pull/6303)
+- Github Actions based build.
 - Introduce caching mechanism to optimize Keccak hash calculations for account storage slots during block processing [#6452](https://github.com/hyperledger/besu/pull/6452)
 - Added configuration options for `pragueTime` to genesis file for Prague fork development [#6473](https://github.com/hyperledger/besu/pull/6473)
 - Moving trielog storage to RocksDB's blobdb to improve write amplications [#6289](https://github.com/hyperledger/besu/pull/6289)

build.gradle

@@ -36,8 +36,9 @@ plugins {
 
 sonarqube {
   properties {
-    property "sonar.projectKey", "hyperledger_besu"
+    property "sonar.projectKey", "$System.env.SONAR_PROJECT_KEY"
-    property "sonar.organization", "hyperledger"
+    property "sonar.organization", "$System.env.SONAR_ORGANIZATION"
+    property "sonar.gradle.skipCompile", "true"
     property "sonar.host.url", "https://sonarcloud.io"
     property "sonar.coverage.jacoco.xmlReportPaths", "${buildDir}/reports/jacoco/jacocoRootReport/jacocoRootReport.xml"
     property "sonar.coverage.exclusions", "acceptance-tests/**/*"
@@ -646,6 +647,8 @@ task autocomplete(type: JavaExec) {
   }
 }
 
+def archiveBuildVersion = project.hasProperty('release.releaseVersion') ? project.property('release.releaseVersion') : "${rootProject.version}"
+
 installDist { dependsOn checkLicense, untunedStartScripts, evmToolStartScripts }
 
 distTar {
@@ -654,6 +657,7 @@ distTar {
     delete fileTree(dir: 'build/distributions', include: '*.tar.gz')
   }
   compression = Compression.GZIP
+  setVersion(archiveBuildVersion)
   archiveExtension = 'tar.gz'
 }
 
@@ -662,6 +666,7 @@ distZip {
   doFirst {
     delete fileTree(dir: 'build/distributions', include: '*.zip')
   }
+  setVersion(archiveBuildVersion)
 }
 
 publishing {
@@ -983,6 +988,12 @@ task checkSpdxHeader(type: CheckSpdxHeader) {
   ].join("|")
 }
 
+jacocoTestReport {
+  reports {
+    xml.enabled true
+  }
+}
+
 task jacocoRootReport(type: org.gradle.testing.jacoco.tasks.JacocoReport) {
   additionalSourceDirs.from files(subprojects.sourceSets.main.allSource.srcDirs)
   sourceDirectories.from files(subprojects.sourceSets.main.allSource.srcDirs)

ethereum/api/build.gradle

@@ -168,3 +168,29 @@ tasks.register('generateTestBlockchain') {
   }
 }
 test.dependsOn(generateTestBlockchain)
+/*
+ Utility tasks used to separate out long running suites of tests so they can be parallelized in CI
+ */
+tasks.register("testBonsai", Test) {
+  useJUnitPlatform()
+  filter {
+    includeTestsMatching("org.hyperledger.besu.ethereum.api.jsonrpc.bonsai.*")
+  }
+  dependsOn(generateTestBlockchain)
+}
+
+tasks.register("testForest", Test) {
+  useJUnitPlatform()
+  filter {
+    includeTestsMatching("org.hyperledger.besu.ethereum.api.jsonrpc.forest.*")
+  }
+  dependsOn(generateTestBlockchain)
+}
+
+tasks.register("testRemainder", Test) {
+  useJUnitPlatform()
+  filter {
+    excludeTestsMatching("org.hyperledger.besu.ethereum.api.jsonrpc.bonsai.*")
+    excludeTestsMatching("org.hyperledger.besu.ethereum.api.jsonrpc.forest.*")
+  }
+}