From ed4213fdff75700314b57d284ee25c3174e7a0f4 Mon Sep 17 00:00:00 2001 From: Bjerg Date: Tue, 20 Dec 2022 22:18:53 +0100 Subject: [PATCH] ci: speed up fuzz tests (#544) * ci: run fuzz tests in matrix * ci: fail if fuzzing fails * ci: rename fuzz step * ci: clean up env * ci: fix fuzz test target matching * ci: prebuild fuzz corpus * ci: collect coverage while running tests Instead of running tests twice, we can just run them once while collecting coverage data. * ci: temporarily disable `reth-eth-wire` fuzzing * ci: collect fuzz coverage data * ci: fix fuzz coverage collection * ci: re-enable `reth-eth-wire` --- .github/scripts/fuzz.sh | 16 +++++--- .github/workflows/ci.yml | 80 +++++++++++++++++----------------------- 2 files changed, 44 insertions(+), 52 deletions(-) diff --git a/.github/scripts/fuzz.sh b/.github/scripts/fuzz.sh index b77d4330df..ec7c01d195 100755 --- a/.github/scripts/fuzz.sh +++ b/.github/scripts/fuzz.sh @@ -1,13 +1,18 @@ #!/bin/bash +set -e # Runs fuzz tests using `cargo test-fuzz`. PACKAGE=$1 TEST_TIME=${2:-5} -echo "::group::$PACKAGE" - echo Building corpus. -cargo test -p $PACKAGE --no-run --all-features +cargo test -p $PACKAGE + +# We configure coverage after building a corpus to only include +# fuzz tests in the coverage report. +echo Configuring coverage. +source <(cargo llvm-cov show-env --export-prefix) +cargo llvm-cov clean --workspace # Gets the list of tests present in the package. TESTS=$(cargo test-fuzz --list -p $PACKAGE | head -n -3 | tail -n+9 | cat - <(echo \"--list\"]) | cat - | jq -r ".[]") @@ -16,8 +21,9 @@ for test in $TESTS do echo Running test: $test set -x - cargo test-fuzz --no-ui -p "$PACKAGE" $test -- -V $TEST_TIME + cargo test-fuzz --no-ui --exact -p "$PACKAGE" $test -- -V $TEST_TIME set +x done; -echo "::endgroup::" \ No newline at end of file +echo Building coverage report. +cargo llvm-cov report --lcov --output-path lcov.info diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e3082fb4d..39b1993fa2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,6 @@ env: RUSTFLAGS: -D warnings CARGO_TERM_COLOR: always GETH_BUILD: 1.10.26-e5eb32ac - AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES: 1 concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -25,6 +24,8 @@ jobs: uses: actions/checkout@v3 - name: Install toolchain uses: dtolnay/rust-toolchain@nightly + with: + components: llvm-tools-preview - uses: Swatinem/rust-cache@v2 with: cache-on-failure: true @@ -42,22 +43,37 @@ jobs: - name: Install latest nextest release uses: taiki-e/install-action@nextest + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov - - name: Test with latest nextest release - uses: actions-rs/cargo@v1 + - name: Run tests + run: cargo llvm-cov nextest --lcov --output-path lcov.info --locked --workspace --all-features + + - name: Upload coverage data to codecov + uses: codecov/codecov-action@v3 with: - command: nextest - args: run --locked --workspace --all-features + token: ${{ secrets.CODECOV_TOKEN }} + files: lcov.info + flags: unit-tests fuzz: # Pin to `20.04` instead of `ubuntu-latest`, until ubuntu-latest migration is complete # See also runs-on: ubuntu-20.04 + strategy: + matrix: + target: + - reth-primitives + - reth-db + - reth-eth-wire + - reth-codecs steps: - name: Checkout sources uses: actions/checkout@v3 - name: Install toolchain uses: dtolnay/rust-toolchain@nightly + with: + components: llvm-tools-preview - uses: Swatinem/rust-cache@v2 with: cache-on-failure: true @@ -67,18 +83,24 @@ jobs: with: command: install args: cargo-test-fuzz afl - - - name: check for cargo afl + - name: Force install cargo-afl run: | cargo install --force afl cargo afl --version + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov - name: Run fuzz tests run: | - ./.github/scripts/fuzz.sh reth-primitives - ./.github/scripts/fuzz.sh reth-db - ./.github/scripts/fuzz.sh reth-eth-wire - ./.github/scripts/fuzz.sh reth-codecs + ./.github/scripts/fuzz.sh ${{ matrix.target }} + env: + AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES: 1 + - name: Upload coverage data to codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: lcov.info + flags: fuzz-tests lint: # Pin to `20.04` instead of `ubuntu-latest`, until ubuntu-latest migration is complete @@ -106,39 +128,3 @@ jobs: with: args: --all --all-features token: ${{ secrets.GITHUB_TOKEN }} - - coverage: - # Pin to `20.04` instead of `ubuntu-latest`, until ubuntu-latest migration is complete - # See also - runs-on: ubuntu-20.04 - continue-on-error: true - steps: - - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@stable - with: - components: llvm-tools-preview - - uses: Swatinem/rust-cache@v2 - - - name: Install latest nextest release - uses: taiki-e/install-action@nextest - - name: Install cargo-llvm-cov - uses: taiki-e/install-action@cargo-llvm-cov - - - name: Install geth - run: | - mkdir -p "$HOME/bin" - wget -q https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-$GETH_BUILD.tar.gz - tar -xvf geth-linux-amd64-$GETH_BUILD.tar.gz - mv geth-linux-amd64-$GETH_BUILD/geth $HOME/bin/geth - chmod u+x "$HOME/bin/geth" - export PATH=$HOME/bin:$PATH - echo $HOME/bin >> $GITHUB_PATH - geth version - - - name: Collect coverage data - run: cargo llvm-cov nextest --lcov --output-path lcov.info --locked --workspace --all-features - - name: Upload coverage data to codecov - uses: codecov/codecov-action@v3 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: lcov.info