vacp2p/status-linea-besu
1
0
Fork 0
mirror of https://github.com/vacp2p/status-linea-besu.git synced 2026-01-09 13:58:02 -05:00
Code Issues Packages Projects Releases Wiki Activity

GitHub Actions CI/CD (#6427)

Browse Source 
Implements a CI/CD pipeline using Github Actions, to replace the current CircleCI implementation.

https://wiki.hyperledger.org/pages/viewpage.action?pageId=80774216


---------

Signed-off-by: Justin Florentine <justin+github@florentine.us>
Signed-off-by: Gabriel Fukushima <gabrielfukushima@gmail.com>
Signed-off-by: Sally MacFarlane <macfarla.github@gmail.com>
Signed-off-by: jflo <justin+github@florentine.us>
Signed-off-by: RoboCopsGoneSock <158174948+RoboCopsGoneSock@users.noreply.github.com>
Signed-off-by: Danno Ferrin (shemnon) <danno.ferrin@shemnon.com>
Signed-off-by: Danno Ferrin <danno.ferrin@swirldslabs.com>
Signed-off-by: Karim Taam <karim.t2am@gmail.com>
Signed-off-by: Ameziane H <ameziane.hamlat@consensys.net>
Signed-off-by: ahamlat <ameziane.hamlat@consensys.net>
Signed-off-by: garyschulte <garyschulte@gmail.com>
Co-authored-by: Fabio Di Fabio <fabio.difabio@consensys.net>
Co-authored-by: Gabriel Fukushima <gabrielfukushima@gmail.com>
Co-authored-by: Sally MacFarlane <macfarla.github@gmail.com>
Co-authored-by: RoboCopsGoneSock <158174948+RoboCopsGoneSock@users.noreply.github.com>
Co-authored-by: Danno Ferrin <danno.ferrin@shemnon.com>
Co-authored-by: Karim TAAM <karim.t2am@gmail.com>
Co-authored-by: garyschulte <garyschulte@gmail.com>
This commit is contained in:
Justin Florentine
2024-01-31 08:41:02 -05:00
committed by GitHub
parent c10cb3d3a9
commit 9cb6600456
20 changed files with 869 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
.github/workflows/acceptance-tests.yml vendored Normal file
View File

@@ -0,0 +1,114 @@
name: acceptance-tests
on:
pull_request:
pull_request_review:
types: [submitted]
env:
GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false"
total-runners: 16
jobs:
shouldRun:
name: checks to ensure we should run
# necessary because there is no single PR approved event, need to check all comments/approvals/denials
runs-on: ubuntu-22.04
outputs:
shouldRun: ${{steps.shouldRun.outputs.result}}
steps:
- name: required check
id: shouldRun
uses: actions/github-script@v7.0.1
env:
# fun fact, this changes based on incoming event, it will be different when we run this on pushes to main
RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
with:
script: |
const { RELEVANT_SHA } = process.env;
const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({
owner: context.repo.owner,
repo: context.repo.repo,
ref: RELEVANT_SHA,
});
const acceptanceTested = statuses && statuses.filter(({ context }) => context === 'acceptance-tests');
const alreadyRun = acceptanceTested && acceptanceTested.find(({ state }) => state === 'success') > 0;
const { data: reviews } = await github.rest.pulls.listReviews({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: context.issue.number,
});
const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED');
const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (approvingReviews.length > 0);
console.log("tests should be run = %j", shouldRun);
console.log("alreadyRun = %j", alreadyRun);
console.log("approvingReviews = %j", approvingReviews.length);
return shouldRun;
acceptanceTestEthereum:
runs-on: ubuntu-22.04
name: "Acceptance Runner"
needs: shouldRun
permissions:
statuses: write
checks: write
if: ${{ needs.shouldRun.outputs.shouldRun == 'true'}}
strategy:
fail-fast: true
matrix:
runner_index: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
steps:
- name: Checkout Repo
uses: actions/checkout@v4.1.1
- name: Set up Java
uses: actions/setup-java@v4.0.0
with:
distribution: temurin
java-version: 17
- name: get acceptance test report
uses: dawidd6/action-download-artifact@v2
with:
branch: main
name_is_regexp: true
name: 'acceptance-node-\d*\d-test-results'
path: tmp/junit-xml-reports-downloaded
if_no_artifact_found: true
- name: setup gradle
uses: gradle/gradle-build-action@v2.12.0
- name: Split tests
id: split-tests
uses: r7kamura/split-tests-by-timings@v0
with:
reports: tmp/junit-xml-reports-downloaded
glob: 'acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/**/*Test.java'
total: ${{env.total-runners}}
index: ${{ matrix.runner_index }}
- name: write out test list
run: echo "${{ steps.split-tests.outputs.paths }}" >> testList.txt
- name: format gradle args
#regex means: first truncate file paths to align with package name, then swap path delimiter with package delimiter,
#then drop file extension, then insert --tests option between each.
run: cat testList.txt | sed -e 's@acceptance-tests/tests/src/test/java/@--tests\ @g;s@/@.@g;s/\.java//g' > gradleArgs.txt
- name: run acceptance tests
run: ./gradlew acceptanceTest `cat gradleArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
- name: cleanup tempfiles
run: rm testList.txt gradleArgs.txt
- name: Upload Acceptance Test Results
uses: actions/upload-artifact@v3.1.0
with:
name: acceptance-node-${{matrix.runner_index}}-test-results
path: 'acceptance-tests/tests/build/test-results/acceptanceTest/TEST-*.xml'
- name: Publish Test Report
uses: mikepenz/action-junit-report@v4
if: (success() || failure()) # always run even if the build step fails
with:
report_paths: 'acceptance-tests/tests/build/test-results/acceptanceTest/TEST-*.xml'
acceptance-tests:
runs-on: ubuntu-22.04
needs: [ acceptanceTestEthereum ]
permissions:
checks: write
statuses: write
steps:
- name: consolidation
run: echo "consolidating statuses"

76
.github/workflows/artifacts.yml vendored Normal file
View File

@@ -0,0 +1,76 @@
name: artifacts
on:
release:
types:
- prereleased
jobs:
artifacts:
runs-on: ubuntu-22.04
permissions:
contents: write
steps:
- name: checkout
uses: actions/checkout@v4.1.1
- name: Set up JDK 17
uses: actions/setup-java@v4.0.0
with:
distribution: 'temurin'
java-version: '17'
- name: setup gradle
uses: gradle/gradle-build-action@v2.12.0
- name: assemble distributions
run:
./gradlew -Prelease.releaseVersion=${{github.ref_name}} assemble -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
- name: hashes
id: hashes
run: |
cd build/distributions
echo "zipSha=$(shasum -a 256 besu*.zip)" >> $GITHUB_OUTPUT
echo "tarSha=$(shasum -a 256 besu*.tar.gz)" >> $GITHUB_OUTPUT
- name: upload tarball
uses: actions/upload-artifact@v3
with:
path: 'build/distributions/besu*.tar.gz'
name: besu-${{ github.ref_name }}.tar.gz
- name: upload zipfile
uses: actions/upload-artifact@v3
with:
path: 'build/distributions/besu*.zip'
name: besu-${{ github.ref_name }}.zip
- name: Upload Release assets
uses: softprops/action-gh-release@v1
with:
append_body: true
files: |
build/distributions/besu*.tar.gz
build/distributions/besu*.zip
body: |
${{steps.hashes.outputs.tarSha}}
${{steps.hashes.outputs.zipSha}}
testWindows:
runs-on: windows-2022
needs: artifacts
timeout-minutes: 10
if: ${{ github.actor != 'dependabot[bot]' }}
steps:
- name: Set up Java
uses: actions/setup-java@v4.0.0
with:
distribution: adopt
java-version: 17
- name: Download zip
uses: actions/download-artifact@v3
with:
name: besu-${{ github.ref_name }}.zip
- name: test Besu
run: |
unzip besu-*.zip -d besu-tmp
cd besu-tmp
mv besu-* ../besu
cd ..
besu\bin\besu.bat --help
besu\bin\besu.bat --version

36
.github/workflows/checks.yml vendored
View File

@@ -1,36 +0,0 @@
name: checks
on:
push:
branches: [ main ]
pull_request:
workflow_dispatch:
jobs:
spotless:
runs-on: [besu-research-ubuntu-16]
if: ${{ github.actor != 'dependabot[bot]' }}
steps:
- name: Checkout Repo
uses: actions/checkout@v4
- name: Set up Java
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: 17
cache: gradle
- name: spotless
run: ./gradlew --no-daemon --parallel clean spotlessCheck
javadoc_17:
runs-on: [besu-research-ubuntu-8]
if: ${{ github.actor != 'dependabot[bot]' }}
steps:
- name: Checkout Repo
uses: actions/checkout@v4
- name: Set up Java 17
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: 17
cache: gradle
- name: javadoc (JDK 17)
run: ./gradlew --no-daemon clean javadoc

41
.github/workflows/codeql.yml vendored
View File

@@ -14,40 +14,29 @@ name: "CodeQL"
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
paths-ignore:
- '**/*.json'
- '**/*.md'
- '**/*.properties'
- '**/*.txt'
pull_request:
branches: [ main ]
paths-ignore:
- '**/*.json'
- '**/*.md'
- '**/*.properties'
- '**/*.txt'
jobs:
analyze:
name: Analyze
runs-on: [besu-research-ubuntu-16]
runs-on: ubuntu-22.04
permissions:
actions: read
contents: read
security-events: write
strategy:
fail-fast: false
matrix:
language: [ 'java' ]
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
# Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
steps:
- name: Checkout repository
uses: actions/checkout@v4
uses: actions/checkout@v4.1.1
- name: Set up Java
uses: actions/setup-java@v4
uses: actions/setup-java@v4.0.0
with:
distribution: 'temurin'
java-version: 17
cache: gradle
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v2
@@ -58,10 +47,10 @@ jobs:
# Prefix the list here with "+" to use these queries and those in the config file.
queries: security-and-quality,security-extended
# Autobuild failed (OOM)
# Hence, supply memory args for gradle build
- run: |
JAVA_OPTS="-Xmx1000M" ./gradlew --no-scan compileJava
- name: setup gradle
uses: gradle/gradle-build-action@v2.12.0
- name: compileJava noscan
run: |
JAVA_OPTS="-Xmx2048M" ./gradlew --no-scan compileJava
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2

10
.github/workflows/dco-merge-group.yml vendored
View File

@@ -1,10 +0,0 @@
name: dco
on:
merge_group:
jobs:
dco:
runs-on: [besu-research-ubuntu-8]
if: ${{ github.actor != 'dependabot[bot]' }}
steps:
- run: echo "This DCO job runs on merge_queue event and doesn't check PR contents"

20
.github/workflows/dco.yml vendored
View File

@@ -1,20 +0,0 @@
name: dco
on:
pull_request:
workflow_dispatch:
jobs:
dco:
runs-on: [besu-research-ubuntu-8]
if: ${{ github.actor != 'dependabot[bot]' }}
steps:
- run: echo "This DCO job runs on pull_request event and workflow_dispatch"
- name: Get PR Commits
id: 'get-pr-commits'
uses: tim-actions/get-pr-commits@v1.2.0
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: DCO Check
uses: tim-actions/dco@v1.1.0
with:
commits: ${{ steps.get-pr-commits.outputs.commits }}

113
.github/workflows/docker.yml vendored Normal file
View File

@@ -0,0 +1,113 @@
name: docker
on:
release:
types:
- prereleased
env:
registry: ghcr.io
jobs:
hadolint:
runs-on: ubuntu-22.04
steps:
- name: Checkout Repo
uses: actions/checkout@v4.1.1
- name: Set up Java
uses: actions/setup-java@v4.0.0
with:
distribution: temurin
java-version: 17
- name: setup gradle
uses: gradle/gradle-build-action@v2.12.0
- name: hadoLint_openj9-jdk_17
run: docker run --rm -i hadolint/hadolint < docker/openj9-jdk-17/Dockerfile
- name: hadoLint_openjdk_17
run: docker run --rm -i hadolint/hadolint < docker/openjdk-17/Dockerfile
- name: hadoLint_openjdk_17_debug
run: docker run --rm -i hadolint/hadolint < docker/openjdk-17-debug/Dockerfile
- name: hadoLint_openjdk_latest
run: docker run --rm -i hadolint/hadolint < docker/openjdk-latest/Dockerfile
- name: hadoLint_graalvm
run: docker run --rm -i hadolint/hadolint < docker/graalvm/Dockerfile
buildDocker:
needs: hadolint
permissions:
contents: read
packages: write
strategy:
fail-fast: false
matrix:
platform:
- ubuntu-22.04
- [self-hosted, ARM64]
runs-on: ${{ matrix.platform }}
steps:
- name: Prepare
id: prep
run: |
platform=${{ matrix.platform }}
if [ "$platform" = 'ubuntu-22.04' ]; then
echo "PLATFORM_PAIR=linux-amd64" >> $GITHUB_OUTPUT
echo "ARCH=amd64" >> $GITHUB_OUTPUT
else
echo "PLATFORM_PAIR=linux-arm64" >> $GITHUB_OUTPUT
echo "ARCH=arm64" >> $GITHUB_OUTPUT
fi
- name: Checkout Repo
uses: actions/checkout@v4.1.1
- name: short sha
id: shortSha
run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
- name: Set up Java
uses: actions/setup-java@v4.0.0
with:
distribution: temurin
java-version: 17
- name: setup gradle
uses: gradle/gradle-build-action@v2.12.0
- name: install goss
run: |
mkdir -p docker/reports
curl -L https://github.com/aelsabbahy/goss/releases/download/v0.4.4/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} -o ./docker/tests/goss-${{ steps.prep.outputs.PLATFORM_PAIR }}
- name: build and test docker
uses: gradle/gradle-build-action@v2.12.0
env:
architecture: ${{ steps.prep.outputs.ARCH }}
with:
arguments: testDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Prelease.releaseVersion=${{ github.ref_name }}
- name: login to ghcr
uses: docker/login-action@v3.0.0
with:
registry: ${{ env.registry }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: publish
env:
architecture: ${{ steps.prep.outputs.ARCH }}
run: ./gradlew --no-daemon dockerUpload -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Prelease.releaseVersion=${{ github.ref_name }}
multiArch:
needs: buildDocker
runs-on: ubuntu-22.04
permissions:
contents: read
packages: write
steps:
- name: Checkout Repo
uses: actions/checkout@v4.1.1
- name: Set up Java
uses: actions/setup-java@v4.0.0
with:
distribution: temurin
java-version: 17
- name: setup gradle
uses: gradle/gradle-build-action@v2.12.0
- name: login to ghcr
uses: docker/login-action@v3.0.0
with:
registry: ${{ env.registry }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: multi-arch docker
run: ./gradlew manifestDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Prelease.releaseVersion=${{ github.ref_name }}

11
.github/workflows/gradle-wrapper-validation.yml vendored
View File

@@ -1,11 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
name: "Validate Gradle Wrapper"
on: [push, pull_request]
jobs:
validation:
name: "Gradle Wrapper Validation"
runs-on: [besu-research-ubuntu-8]
steps:
- uses: actions/checkout@v4
- uses: gradle/wrapper-validation-action@v1

73
.github/workflows/integration-tests.yml vendored Normal file
View File

@@ -0,0 +1,73 @@
name: integration-tests
on:
pull_request:
pull_request_review:
types:
- submitted
env:
GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false"
jobs:
shouldRun:
name: checks to ensure we should run
runs-on: ubuntu-22.04
outputs:
shouldRun: ${{steps.shouldRun.outputs.result}}
steps:
- name: required check
id: shouldRun
uses: actions/github-script@v7.0.1
env:
# fun fact, this changes based on incoming event, it will be different when we run this on pushes to main
RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
with:
script: |
const { RELEVANT_SHA } = process.env;
const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({
owner: context.repo.owner,
repo: context.repo.repo,
ref: RELEVANT_SHA,
});
const intTested = statuses && statuses.filter(({ context }) => context === 'integration-tests');
const alreadyRun = intTested && intTested.find(({ state }) => state === 'success') > 0;
const { data: reviews } = await github.rest.pulls.listReviews({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: context.issue.number,
});
const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED');
const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (approvingReviews.length > 0);
console.log("tests should be run = %j", shouldRun);
console.log("alreadyRun = %j", alreadyRun);
console.log("approvingReviews = %j", approvingReviews.length);
return shouldRun;
integration-tests:
runs-on: ubuntu-22.04
needs: shouldRun
if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }}
permissions:
statuses: write
checks: write
steps:
- name: Checkout Repo
uses: actions/checkout@v4.1.1
- name: Set up Java
uses: actions/setup-java@v4.0.0
with:
distribution: temurin
java-version: 17
- name: setup gradle
uses: gradle/gradle-build-action@v2.12.0
- name: run integration tests
run: ./gradlew integrationTest compileJmh -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
- name: Publish Test Report
uses: mikepenz/action-junit-report@v4
if: (success() || failure())
with:
report_paths: '**/build/test-results/integrationTest/TEST-*.xml'

121
.github/workflows/nightly.yml vendored Normal file
View File

@@ -0,0 +1,121 @@
name: nightly
on:
workflow_dispatch:
schedule:
# * is a special character in YAML so you have to quote this string
# expression evaluates to midnight every night
- cron: '0 0 * * *'
env:
nightly-tag: develop
registry: ghcr.io
jobs:
hadolint:
runs-on: ubuntu-22.04
steps:
- name: Checkout Repo
uses: actions/checkout@v4.1.1
- name: Set up Java
uses: actions/setup-java@v4.0.0
with:
distribution: temurin
java-version: 17
- name: setup gradle
uses: gradle/gradle-build-action@v2.12.0
- name: hadoLint_openj9-jdk_17
run: docker run --rm -i hadolint/hadolint < docker/openj9-jdk-17/Dockerfile
- name: hadoLint_openjdk_17
run: docker run --rm -i hadolint/hadolint < docker/openjdk-17/Dockerfile
- name: hadoLint_openjdk_17_debug
run: docker run --rm -i hadolint/hadolint < docker/openjdk-17-debug/Dockerfile
- name: hadoLint_openjdk_latest
run: docker run --rm -i hadolint/hadolint < docker/openjdk-latest/Dockerfile
- name: hadoLint_graalvm
run: docker run --rm -i hadolint/hadolint < docker/graalvm/Dockerfile
buildDocker:
needs: hadolint
permissions:
contents: read
packages: write
strategy:
fail-fast: false
matrix:
platform:
- ubuntu-22.04
- [self-hosted, ARM64]
runs-on: ${{ matrix.platform }}
steps:
- name: Prepare
id: prep
run: |
platform=${{ matrix.platform }}
if [ "$platform" = 'ubuntu-22.04' ]; then
echo "PLATFORM_PAIR=linux-amd64" >> $GITHUB_OUTPUT
echo "ARCH=amd64" >> $GITHUB_OUTPUT
else
echo "PLATFORM_PAIR=linux-arm64" >> $GITHUB_OUTPUT
echo "ARCH=arm64" >> $GITHUB_OUTPUT
fi
- name: Checkout Repo
uses: actions/checkout@v4.1.1
- name: short sha
id: shortSha
run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
- name: Set up Java
uses: actions/setup-java@v4.0.0
with:
distribution: temurin
java-version: 17
- name: setup gradle
uses: gradle/gradle-build-action@v2.12.0
- name: build image
uses: gradle/gradle-build-action@v2.12.0
with:
arguments: distDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Pbranch=main
- name: install goss
run: |
mkdir -p docker/reports
curl -L https://github.com/aelsabbahy/goss/releases/download/v0.4.4/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} -o ./docker/tests/goss-${{ steps.prep.outputs.PLATFORM_PAIR }}
- name: test docker
uses: gradle/gradle-build-action@v2.12.0
env:
architecture: ${{ steps.prep.outputs.ARCH }}
with:
arguments: testDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Pbranch=main
- name: login to ghcr
uses: docker/login-action@v3.0.0
with:
registry: ${{ env.registry }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: publish
env:
architecture: ${{ steps.prep.outputs.ARCH }}
run: ./gradlew --no-daemon dockerUpload -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Pbranch=main
multiArch:
permissions:
contents: read
packages: write
needs: buildDocker
runs-on: ubuntu-22.04
steps:
- name: Checkout Repo
uses: actions/checkout@v4.1.1
- name: Set up Java
uses: actions/setup-java@v4.0.0
with:
distribution: temurin
java-version: 17
- name: setup gradle
uses: gradle/gradle-build-action@v2.12.0
- name: Login to DockerHub
uses: docker/login-action@v3.0.0
with:
registry: ${{ env.registry }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: multi-arch docker
run: ./gradlew manifestDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Pbranch=main

49
.github/workflows/parallel-unit-tests.yml vendored Normal file
View File

@@ -0,0 +1,49 @@
name: parallel-unit-tests
#experimental work in progress - trying to figure out how to split tests across multi-modules by runtime
on:
workflow_dispatch:
env:
GRADLE_OPTS: "-Dorg.gradle.daemon=false"
total-runners: 4
jobs:
junit:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
runner_index:
- 0
- 1
- 2
- 3
steps:
- name: Checkout Repo
uses: actions/checkout@v4.1.1
- name: Split tests
id: split-tests
uses: chaosaffe/split-tests@v1-alpha.1
with:
glob: '**/src/test/java/**/*.java'
split-total: ${{ env.total-runners }}
split-index: ${{ matrix.runner_index }}
line-count: true
- name: Set up Java
uses: actions/setup-java@v4.0.0
with:
distribution: adopt
java-version: 17
cache: gradle
- name: write out test list
run: echo "${{ steps.split-tests.outputs.test-suite }}" >> testList.txt
- name: debug testfile paths
run: cat testList.txt
- name: format gradle args
# regex means: truncate file paths to align with package name, replacing with tests switch, then drop file extension,
# then swap path delimiter with package delimiter
run: cat testList.txt | sed -e 's/[^ ]*src\/test\/java\//--tests\ /g' -e 's/\.java//g' -e 's/\//\./g' >> gradleArgs.txt
- name: debug test class list
run: cat gradleArgs.txt
- name: run unit tests
run: ./gradlew test `cat gradleArgs.txt`

8
.github/workflows/pr-checklist-on-open.yml vendored
View File

@@ -6,9 +6,11 @@ on:
jobs:
checklist:
name: "add checklist as a comment on newly opened PRs"
runs-on: [besu-research-ubuntu-8]
runs-on: ubuntu-22.04
permissions:
pull-requests: write
steps:
- uses: actions/github-script@v5
- uses: actions/github-script@v7.0.1
with:
github-token: ${{secrets.GITHUB_TOKEN}}
script: |
@@ -16,5 +18,5 @@ jobs:
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: '- [ ] I thought about documentation and added the `doc-change-required` label to this PR if [updates are required](https://wiki.hyperledger.org/display/BESU/Documentation).\n- [ ] I thought about the changelog and included a [changelog update if required](https://wiki.hyperledger.org/display/BESU/Changelog).\n- [ ] If my PR includes database changes (e.g. KeyValueSegmentIdentifier) I have thought about compatibility and performed forwards and backwards compatibility tests'
body: '- [ ] I thought about documentation and added the `doc-change-required` label to this PR if [updates are required](https://wiki.hyperledger.org/display/BESU/Documentation).\n- [ ] I thought about the changelog and included a [changelog update if required](https://wiki.hyperledger.org/display/BESU/Changelog).\n- [ ] If my PR includes database changes (e.g. KeyValueSegmentIdentifier) I have thought about compatibility and performed forwards and backwards compatibility tests\n- [ ] I thought about running CI.\n- [ ] If I did not run CI, I ran as much locally as possible before pushing.\n-'
})

103
.github/workflows/pre-review.yml vendored Normal file
View File

@@ -0,0 +1,103 @@
name: pre-review
on:
pull_request:
workflow_dispatch:
permissions:
statuses: write
checks: write
jobs:
repolint:
name: "Repository Linting"
runs-on: ubuntu-22.04
container: ghcr.io/todogroup/repolinter:v0.11.2
steps:
- name: Checkout Code
uses: actions/checkout@v4.1.1
- name: Lint Repo
run: bundle exec /app/bin/repolinter.js --rulesetUrl https://raw.githubusercontent.com/hyperledger-labs/hyperledger-community-management-tools/main/repo_structure/repolint.json --format markdown
gradle-wrapper:
name: "Gradle Wrapper Validation"
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4.1.1
- uses: gradle/wrapper-validation-action@v1.1.0
spotless:
runs-on: ubuntu-22.04
if: ${{ github.actor != 'dependabot[bot]' }}
steps:
- name: Checkout Repo
uses: actions/checkout@v4.1.1
- name: Set up Java
uses: actions/setup-java@v4.0.0
with:
distribution: temurin
java-version: 17
- name: Setup Gradle
uses: gradle/gradle-build-action@v2.12.0
- name: run spotless
run: ./gradlew spotlessCheck -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
compile:
runs-on: ubuntu-22.04
timeout-minutes: 30
needs: [spotless, gradle-wrapper, repolint]
steps:
- name: Checkout Repo
uses: actions/checkout@v4.1.1
- name: Set up Java
uses: actions/setup-java@v4.0.0
with:
distribution: temurin
java-version: 17
- name: Setup Gradle
uses: gradle/gradle-build-action@v2.12.0
- name: Gradle Compile
run: ./gradlew build -x test -x spotlessCheck -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
unitTests:
env:
GRADLEW_UNIT_TEST_ARGS: ${{matrix.gradle_args}}
runs-on: ubuntu-22.04
needs: [ compile ]
permissions:
checks: write
statuses: write
strategy:
fail-fast: true
matrix:
gradle_args:
- "test -x besu:test -x consensus:test -x crypto:test -x ethereum:eth:test -x ethereum:api:test -x ethereum:core:test"
- "besu:test consensus:test crypto:test"
- "ethereum:api:testBonsai"
- "ethereum:api:testForest"
- "ethereum:api:testRemainder"
- "ethereum:core:test"
steps:
- name: Checkout Repo
uses: actions/checkout@v4.1.1
- name: Set up Java
uses: actions/setup-java@v4.0.0
with:
distribution: temurin
java-version: 17
- name: Setup Gradle
uses: gradle/gradle-build-action@v2.12.0
- name: run unit tests
id: unitTest
run: ./gradlew $GRADLEW_UNIT_TEST_ARGS -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
- name: Publish Test Report
uses: mikepenz/action-junit-report@v4
if: success() || failure() # always run even if the build step fails
with:
report_paths: '**/test-results/**/TEST-*.xml'
annotate_only: true
pre-review:
runs-on: ubuntu-22.04
needs: [unitTests]
permissions:
checks: write
statuses: write
steps:
- name: consolidation
run: echo "consolidating statuses"

147
.github/workflows/reference-tests.yml vendored Normal file
View File

@@ -0,0 +1,147 @@
name: reference-tests
on:
pull_request:
pull_request_review:
types:
- submitted
env:
GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false"
total-runners: 6
jobs:
shouldRun:
name: checks to ensure we should run
# necessary because there is no single PR approved event, need to check all comments/approvals/denials
# might also be a job running, and additional approvals
runs-on: ubuntu-22.04
outputs:
shouldRun: ${{steps.shouldRun.outputs.result}}
steps:
- name: required check
id: shouldRun
uses: actions/github-script@v7.0.1
env:
# fun fact, this changes based on incoming event, it will be different when we run this on pushes to main
RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
with:
script: |
const { RELEVANT_SHA } = process.env;
const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({
owner: context.repo.owner,
repo: context.repo.repo,
ref: RELEVANT_SHA,
});
const refTested = statuses && statuses.filter(({ context }) => context === 'reference-tests');
const alreadyRun = refTested && refTested.find(({ state }) => state === 'success') > 0;
const { data: reviews } = await github.rest.pulls.listReviews({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: context.issue.number,
});
const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED');
const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (approvingReviews.length > 0);
console.log("tests should be run = %j", shouldRun);
console.log("alreadyRun = %j", alreadyRun);
console.log("approvingReviews = %j", approvingReviews.length);
return shouldRun;
prepareReferenceTestEthereum:
runs-on: ubuntu-22.04
needs: shouldRun
if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }}
steps:
- name: Checkout Repo
uses: actions/checkout@v4.1.1
with:
submodules: recursive
set-safe-directory: true
- name: Set up Java
uses: actions/setup-java@v4.0.0
with:
distribution: temurin
java-version: 17
- name: setup gradle
uses: gradle/gradle-build-action@v2.12.0
- name: execute generate reference tests
run: ./gradlew ethereum:referencetests:blockchainReferenceTests ethereum:referencetests:generalstateReferenceTests ethereum:referencetests:generalstateRegressionReferenceTests -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
- name: store generated tests
uses: actions/upload-artifact@v3
with:
name: 'reference-tests'
path: 'ethereum/referencetests/build/generated/sources/reference-test/**/*.java'
referenceTestEthereum:
runs-on: ubuntu-22.04
permissions:
statuses: write
checks: write
needs:
- prepareReferenceTestEthereum
if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }}
strategy:
fail-fast: true
matrix:
runner_index: [0,1,2,3,4,5]
steps:
- name: Checkout Repo
uses: actions/checkout@v4.1.1
with:
submodules: recursive
- name: Set up Java
uses: actions/setup-java@v4.0.0
with:
distribution: adopt-openj9
java-version: 17
- name: retrieve generated tests
uses: actions/download-artifact@v3.0.2
with:
name: 'reference-tests'
path: 'ethereum/referencetests/build/generated/sources/reference-test/'
- name: get reference test report
uses: dawidd6/action-download-artifact@v2
with:
branch: main
name_is_regexp: true
name: 'reference-test-node-\d*\d-results'
path: tmp/ref-xml-reports-downloaded
if_no_artifact_found: true
- name: setup gradle
uses: gradle/gradle-build-action@v2.12.0
- name: Split tests
id: split-tests
uses: r7kamura/split-tests-by-timings@v0
with:
reports: tmp/ref-xml-reports-downloaded
glob: 'ethereum/referencetests/build/generated/sources/reference-test/**/*.java'
total: ${{env.total-runners}}
index: ${{ matrix.runner_index }}
- name: compose gradle args
run: echo ${{ steps.split-tests.outputs.paths }} | sed -e 's/^.*java\///' -e 's@/@.@g' -e 's/\.java//' -e 's/^/--tests /' > refTestArgs.txt
- name: run reference tests
run: ./gradlew ethereum:referenceTests:referenceTests `cat refTestArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
- name: Upload Test Report
uses: actions/upload-artifact@v3
if: always() # always run even if the previous step fails
with:
name: reference-test-node-${{matrix.runner_index}}-results
path: '**/build/test-results/referenceTests/TEST-*.xml'
- name: Publish Test Report
uses: mikepenz/action-junit-report@v4
if: success() || failure() # always run even if the build step fails
with:
report_paths: '**/build/test-results/referenceTest/TEST-*.xml'
reference-tests:
runs-on: ubuntu-22.04
needs: [ referenceTestEthereum ]
permissions:
checks: write
statuses: write
steps:
- name: consolidation
run: echo "consolidating statuses"

11
.github/workflows/release.yml vendored
View File

@@ -1,13 +1,14 @@
name: release besu
on:
workflow_dispatch:
release:
types: released
types: [released]
jobs:
dockerPromoteX64:
runs-on: [besu-research-ubuntu-16]
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v4
- uses: actions/checkout@v4.1.1
- uses: actions/setup-java@v4.0.0
with:
distribution: 'temurin' # See 'Supported distributions' for available options
java-version: '17'
@@ -15,7 +16,7 @@ jobs:
- name: Login to DockerHub
run: echo '${{ secrets.DOCKER_PASSWORD_RW }}' | docker login -u '${{ secrets.DOCKER_USER_RW }}' --password-stdin
- name: Setup Gradle
uses: gradle/gradle-build-action@v2
uses: gradle/gradle-build-action@v2.12.0
- name: Docker upload
run: ./gradlew "-Prelease.releaseVersion=${{ github.ref_name }}" "-PdockerOrgName=${{ secrets.DOCKER_ORG }}" dockerUploadRelease
- name: Docker manifest

24
.github/workflows/repolinter.yml vendored
View File

@@ -1,24 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# Hyperledger Repolinter Action
name: Repolinter
on:
workflow_dispatch:
push:
branches:
- master
- main
pull_request:
branches:
- master
- main
jobs:
build:
runs-on: [besu-research-ubuntu-16]
container: ghcr.io/todogroup/repolinter:v0.10.1
steps:
- name: Checkout Code
uses: actions/checkout@v4
- name: Lint Repo
run: bundle exec /app/bin/repolinter.js --rulesetUrl https://raw.githubusercontent.com/hyperledger-labs/hyperledger-community-management-tools/main/repo_structure/repolint.json --format markdown

12
.github/workflows/sonarcloud.yml vendored
View File

@@ -14,24 +14,26 @@ permissions:
jobs:
Analysis:
runs-on: ubuntu-latest
if: github.repository == 'hyperledger/besu'
steps:
- name: checkout
uses: actions/checkout@v4
uses: actions/checkout@v4.1.1
- name: Set up JDK 17
uses: actions/setup-java@v4
uses: actions/setup-java@v4.0.0
with:
distribution: 'temurin'
java-version: '17'
cache: gradle
- name: Cache SonarCloud packages
uses: actions/cache@v3
with:
path: ~/.sonar/cache
key: ${{ runner.os }}-sonar
restore-keys: ${{ runner.os }}-sonar
- name: setup gradle
uses: gradle/gradle-build-action@v2.12.0
- name: Build and analyze
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
run: ./gradlew build sonarqube --continue --info
SONAR_ORGANIZATION: ${{ env.SONAR_ORGANIZATION }}
SONAR_PROJECT_KEY: $ {{ env.SONAR_PROJECT_KEY }}
run: ./gradlew build sonarqube --continue --info -Dorg.gradle.parallel=true -Dorg.gradle.caching=true

1
CHANGELOG.md
View File

@@ -19,6 +19,7 @@
- Log blob count when importing a block via Engine API [#6466](https://github.com/hyperledger/besu/pull/6466)
- Introduce `--Xbonsai-limit-trie-logs-enabled` experimental feature which by default will only retain the latest 512 trie logs, saving about 3GB per week in database growth [#5390](https://github.com/hyperledger/besu/issues/5390)
- Introduce `besu storage x-trie-log prune` experimental offline subcommand which will prune all redundant trie logs except the latest 512 [#6303](https://github.com/hyperledger/besu/pull/6303)
- Github Actions based build.
- Introduce caching mechanism to optimize Keccak hash calculations for account storage slots during block processing [#6452](https://github.com/hyperledger/besu/pull/6452)
- Added configuration options for `pragueTime` to genesis file for Prague fork development [#6473](https://github.com/hyperledger/besu/pull/6473)
- Moving trielog storage to RocksDB's blobdb to improve write amplications [#6289](https://github.com/hyperledger/besu/pull/6289)

15
build.gradle
View File

@@ -36,8 +36,9 @@ plugins {
sonarqube {
properties {
property "sonar.projectKey", "hyperledger_besu"
property "sonar.organization", "hyperledger"
property "sonar.projectKey", "$System.env.SONAR_PROJECT_KEY"
property "sonar.organization", "$System.env.SONAR_ORGANIZATION"
property "sonar.gradle.skipCompile", "true"
property "sonar.host.url", "https://sonarcloud.io"
property "sonar.coverage.jacoco.xmlReportPaths", "${buildDir}/reports/jacoco/jacocoRootReport/jacocoRootReport.xml"
property "sonar.coverage.exclusions", "acceptance-tests/**/*"
@@ -646,6 +647,8 @@ task autocomplete(type: JavaExec) {
}
}
def archiveBuildVersion = project.hasProperty('release.releaseVersion') ? project.property('release.releaseVersion') : "${rootProject.version}"
installDist { dependsOn checkLicense, untunedStartScripts, evmToolStartScripts }
distTar {
@@ -654,6 +657,7 @@ distTar {
delete fileTree(dir: 'build/distributions', include: '*.tar.gz')
}
compression = Compression.GZIP
setVersion(archiveBuildVersion)
archiveExtension = 'tar.gz'
}
@@ -662,6 +666,7 @@ distZip {
doFirst {
delete fileTree(dir: 'build/distributions', include: '*.zip')
}
setVersion(archiveBuildVersion)
}
publishing {
@@ -983,6 +988,12 @@ task checkSpdxHeader(type: CheckSpdxHeader) {
].join("|")
}
jacocoTestReport {
reports {
xml.enabled true
}
}
task jacocoRootReport(type: org.gradle.testing.jacoco.tasks.JacocoReport) {
additionalSourceDirs.from files(subprojects.sourceSets.main.allSource.srcDirs)
sourceDirectories.from files(subprojects.sourceSets.main.allSource.srcDirs)

26
ethereum/api/build.gradle
View File

@@ -168,3 +168,29 @@ tasks.register('generateTestBlockchain') {
}
}
test.dependsOn(generateTestBlockchain)
/*
Utility tasks used to separate out long running suites of tests so they can be parallelized in CI
*/
tasks.register("testBonsai", Test) {
useJUnitPlatform()
filter {
includeTestsMatching("org.hyperledger.besu.ethereum.api.jsonrpc.bonsai.*")
}
dependsOn(generateTestBlockchain)
}
tasks.register("testForest", Test) {
useJUnitPlatform()
filter {
includeTestsMatching("org.hyperledger.besu.ethereum.api.jsonrpc.forest.*")
}
dependsOn(generateTestBlockchain)
}
tasks.register("testRemainder", Test) {
useJUnitPlatform()
filter {
excludeTestsMatching("org.hyperledger.besu.ethereum.api.jsonrpc.bonsai.*")
excludeTestsMatching("org.hyperledger.besu.ethereum.api.jsonrpc.forest.*")
}
}