From ae1987a359dc5cb38bcd9d9eddb9ebd27f351596 Mon Sep 17 00:00:00 2001 From: Pascal Jufer Date: Mon, 25 Oct 2021 17:22:06 +0200 Subject: [PATCH] Revise GitHub workflows (#9011) * Fix check for changed files & reformat workflows * Test performance of paths-filter action * Revise workflows * Better wording * Fix naming of "Pre-Check" * Point out values * Abreviate to make it look cleaner in report --- .github/workflows/assign-author-to-pr.yml | 10 ++- .github/workflows/auto-merge-crowdin.yml | 3 +- .github/workflows/codeql-analysis.yml | 40 ++++------ .github/workflows/continuous-integration.yml | 55 +++++++++++++ .github/workflows/e2e-full.yml | 65 ---------------- .github/workflows/e2e-tests.yml | 64 +++++++++++++++ .github/workflows/lint.yml | 32 +++----- .github/workflows/release.yml | 82 +++++++++++++------- .github/workflows/sync-dockerhub-readme.yml | 11 +-- .github/workflows/tests.yml | 46 ----------- .github/workflows/unit-tests.yml | 33 ++++++++ 11 files changed, 242 insertions(+), 199 deletions(-) create mode 100644 .github/workflows/continuous-integration.yml delete mode 100644 .github/workflows/e2e-full.yml create mode 100644 .github/workflows/e2e-tests.yml delete mode 100644 .github/workflows/tests.yml create mode 100644 .github/workflows/unit-tests.yml diff --git a/.github/workflows/assign-author-to-pr.yml b/.github/workflows/assign-author-to-pr.yml index 0ddb497140..f2e3239cf4 100644 --- a/.github/workflows/assign-author-to-pr.yml +++ b/.github/workflows/assign-author-to-pr.yml @@ -1,12 +1,14 @@ -name: Assign author to PR +name: Assign Author to PR on: pull_request_target: - types: [opened] + types: + - opened jobs: assign-author: - name: Assign author to PR + name: Assign Author runs-on: ubuntu-latest steps: - - uses: technote-space/assign-author@v1 + - name: Assign author + uses: technote-space/assign-author@v1 diff --git a/.github/workflows/auto-merge-crowdin.yml b/.github/workflows/auto-merge-crowdin.yml index c12baf9c74..a70892769e 100644 --- a/.github/workflows/auto-merge-crowdin.yml +++ b/.github/workflows/auto-merge-crowdin.yml @@ -4,12 +4,14 @@ on: pull_request: branches: - main + # Only run if pull request includes changes in translation files paths: - 'app/src/lang/translations/*.yaml' jobs: auto-merge: name: Auto-Merge + # Only auto-merge if it looks like an actual pull request from Crowdin if: github.event.pull_request.user.login == 'rijkvanzanten' && github.event.pull_request.title == 'New Crowdin updates' && github.head_ref == 'translations' runs-on: ubuntu-latest steps: @@ -20,4 +22,3 @@ jobs: MERGE_LABELS: '' UPDATE_LABELS: '' MERGE_METHOD: squash - MERGE_COMMIT_MESSAGE: '{pullRequest.title} (#{pullRequest.number})' diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 15c5dea9de..268ae9387e 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,16 +1,7 @@ -name: "CodeQL" +name: CodeQL Analysis on: - push: - branches: [ main ] - # Don't run on translation updates - paths-ignore: - - 'app/src/lang/translations/*.yaml' - pull_request: - branches: [ main ] - # Don't run on translation updates - paths-ignore: - - 'app/src/lang/translations/*.yaml' + workflow_call: schedule: - cron: '42 23 * * 5' @@ -22,23 +13,22 @@ jobs: actions: read contents: read security-events: write - strategy: fail-fast: false matrix: - language: [ 'javascript' ] - + language: + - javascript steps: - - name: Checkout repository - uses: actions/checkout@v2 + - name: Checkout repository + uses: actions/checkout@v2 - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - - - name: Autobuild - uses: github/codeql-action/autobuild@v1 + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + - name: Perform CodeQL analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml new file mode 100644 index 0000000000..116cc54ddf --- /dev/null +++ b/.github/workflows/continuous-integration.yml @@ -0,0 +1,55 @@ +name: CI + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + pre_check: + name: Pre-Check + runs-on: ubuntu-latest + outputs: + # Returns 'true' if workflow is skippable for one of the following reasons: + # - Exact same files have been successfully checked in older run + # - Only files that do not require checks have been changed (paths_ignore) + should_skip: ${{ steps.skip_check.outputs.should_skip }} + steps: + - name: Check if workflow is skippable + id: skip_check + uses: fkirc/skip-duplicate-actions@master + # Run all the jobs if the pre-check should fail for any reason + # (the pre-check will be marked as 'passed', better solution pending at https://github.com/actions/toolkit/issues/399) + continue-on-error: true + with: + # Cancel outdated workflow-runs + cancel_others: 'true' + paths_ignore: '["app/src/lang/translations/*.yaml"]' + + lint: + name: Lint + needs: pre_check + if: needs.pre_check.outputs.should_skip != 'true' + uses: directus/directus/.github/workflows/lint.yml@main + + codeql_analysis: + name: CodeQL Analysis + needs: pre_check + if: needs.pre_check.outputs.should_skip != 'true' + uses: directus/directus/.github/workflows/codeql-analysis.yml@main + + unit_tests: + name: Unit Tests + needs: pre_check + if: needs.pre_check.outputs.should_skip != 'true' + uses: directus/directus/.github/workflows/unit-tests.yml@main + + e2e_tests: + name: End-to-End Tests + needs: pre_check + # Only run on push events (on main branch) for now + if: github.event_name == 'push' && needs.pre_check.outputs.should_skip != 'true' + uses: directus/directus/.github/workflows/e2e-tests.yml@main diff --git a/.github/workflows/e2e-full.yml b/.github/workflows/e2e-full.yml deleted file mode 100644 index 58b4aa0493..0000000000 --- a/.github/workflows/e2e-full.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: E2E - -on: - push: - branches: - - main - # Don't run on translation updates - paths-ignore: - - 'app/src/lang/translations/*.yaml' - -jobs: - tests: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - db: ['mssql', 'mysql', 'postgres', 'maria'] #sqlite - # node-version: ['12-alpine', '14-alpine', '16-alpine'] - node-version: ['16-alpine'] - env: - CACHED_IMAGE: ghcr.io/directus/directus-e2e-test-cache:${{ matrix.node-version }} - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.9.1 - with: - access_token: ${{ secrets.GITHUB_TOKEN }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Pull cached directus image - run: | - docker pull $CACHED_IMAGE || true - docker tag $CACHED_IMAGE directus-test-image || true - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 - with: - node-version: '16' - cache: npm - - name: restore node_modules cache - uses: actions/cache@v2 - with: - path: | - node_modules - **/node_modules - key: ${{ runner.os }}-${{ hashFiles('**/package-lock.json') }} - - name: Install dependencies - run: | - npm ci - - name: Build - run: | - npm run build - - name: Run tests - env: - TEST_NODE_VERSION: ${{ matrix.node-version }} - TEST_DB: ${{ matrix.db }} - run: npm run test:e2e - - name: Push cached image - # only push the new cache image on the main branch once per node version - if: github.ref == 'refs/heads/main' && github.repository == 'directus/directus' && matrix.db == 'sqlite3' - run: | - docker tag directus-test-image $CACHED_IMAGE - docker push $CACHED_IMAGE || true diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml new file mode 100644 index 0000000000..9c171f0e38 --- /dev/null +++ b/.github/workflows/e2e-tests.yml @@ -0,0 +1,64 @@ +name: End-to-End Tests + +on: + workflow_call: + +jobs: + test: + name: Test + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + db: + - mssql + - mysql + - postgres + - maria + # - sqlite3 + node-version: + # - 12-alpine + # - 14-alpine + - 16-alpine + env: + CACHED_IMAGE: ghcr.io/directus/directus-e2e-test-cache:${{ matrix.node-version }} + steps: + - name: Login to GitHub container registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Pull cached test image + run: | + docker pull $CACHED_IMAGE || true + docker tag $CACHED_IMAGE directus-test-image || true + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Setup Node.js + uses: actions/setup-node@v2 + with: + node-version: '16' + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Build + run: npm run build + + - name: Run tests + env: + TEST_NODE_VERSION: ${{ matrix.node-version }} + TEST_DB: ${{ matrix.db }} + run: npm run test:e2e + + - name: Push updated test image + # Only push the new cache image on the main branch once per node version + if: github.ref == 'refs/heads/main' && github.repository == 'directus/directus' && matrix.db == 'postgres' + run: | + docker tag directus-test-image $CACHED_IMAGE + docker push $CACHED_IMAGE || true diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index fc7bb83e25..b859c7e87d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,37 +1,23 @@ -name: Run linters +name: Lint on: - push: - branches: - - main - # Don't run on translation updates - paths-ignore: - - 'app/src/lang/translations/*.yaml' - pull_request: - branches: - - main - # Don't run on translation updates - paths-ignore: - - 'app/src/lang/translations/*.yaml' + workflow_call: jobs: lint: + name: Lint runs-on: ubuntu-latest - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.9.1 - with: - access_token: ${{ secrets.GITHUB_TOKEN }} + - name: Checkout repository + uses: actions/checkout@v2 - - uses: actions/checkout@v2 - - - uses: actions/setup-node@v2 + - name: Setup Node.js + uses: actions/setup-node@v2 with: - node-version: "16" + node-version: '16' cache: npm - - name: Install Dependencies + - name: Install dependencies run: npm ci - name: Run linters diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 38b1ed10ab..9588bb6240 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,7 +3,7 @@ name: Release on: push: tags: - - "v*" + - 'v*' env: GHCR_IMAGE: ghcr.io/${{ github.repository }} @@ -11,12 +11,13 @@ env: jobs: create-release: + name: Create Release runs-on: ubuntu-latest steps: - - name: Checkout code + - name: Checkout repository uses: actions/checkout@v2 - - name: Create Release + - name: Create release id: create_release uses: actions/create-release@v1 env: @@ -30,72 +31,92 @@ jobs: prerelease: false build: + name: Build Packages runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Use Node.js + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Setup Node.js uses: actions/setup-node@v2 with: - node-version: "16.x" + node-version: '16' cache: npm # See https://github.com/npm/cli/issues/3637 - - run: npm i -g npm@7.20.2 + - name: Downgrade NPM + run: npm i -g npm@7.20.2 - - uses: c-hive/gha-npm-cache@v1 - - run: npm ci - - run: npm run build - - run: node docker/pack + - name: Setup NPM cache + uses: c-hive/gha-npm-cache@v1 + + - name: Install dependencies + run: npm ci + + - name: Build + run: | + npm run build + node docker/pack - name: Cache build artifacts uses: actions/cache@v2 with: - path: "**/dist" + path: '**/dist' key: build-artifacts-${{ github.sha }} publish-npm: + name: Publish to NPM runs-on: ubuntu-latest needs: build + # Skip this step in forks if: ${{ github.repository_owner == 'directus' }} steps: - - uses: actions/checkout@v2 + - name: Checkout repository + uses: actions/checkout@v2 + - name: Restore build artifacts uses: actions/cache@v2 with: - path: "**/dist" + path: '**/dist' key: build-artifacts-${{ github.sha }} - - name: Use Node.js + + - name: Setup Node.js uses: actions/setup-node@v2 with: - node-version: "16.x" - registry-url: "https://registry.npmjs.org" + node-version: '16' + registry-url: https://registry.npmjs.org cache: npm # See https://github.com/npm/cli/issues/3637 - - run: npm i -g npm@7.20.2 + - name: Downgrade NPM + run: npm i -g npm@7.20.2 - - run: npm ci + - name: Install dependencies + run: npm ci - - run: npx lerna publish from-git --no-verify-access --yes + - name: Publish packages to NPM + run: npx lerna publish from-git --no-verify-access --yes env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} build-images: + name: Build Images runs-on: ubuntu-latest needs: build steps: - - uses: actions/checkout@v2 + - name: Checkout repository + uses: actions/checkout@v2 - name: Restore build artifacts uses: actions/cache@v2 with: - path: "**/dist" + path: '**/dist' key: build-artifacts-${{ github.sha }} - - name: Set up QEMU + - name: Setup QEMU uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx + - name: Setup Docker Buildx uses: docker/setup-buildx-action@v1 - name: Cache Docker layers @@ -106,7 +127,7 @@ jobs: restore-keys: | ${{ runner.os }}-buildx- - - name: Docker meta + - name: Extract metadata for Docker image id: meta uses: docker/metadata-action@v3 with: @@ -121,7 +142,7 @@ jobs: type=semver,pattern={{major}}.{{minor}} type=semver,pattern={{major}} - - name: Login to DockerHub + - name: Login to Docker Hub uses: docker/login-action@v1 if: ${{ env.DOCKERHUB_IMAGE }} with: @@ -140,16 +161,17 @@ jobs: uses: docker/build-push-action@v2 with: context: . - file: "./docker/Dockerfile" + file: ./docker/Dockerfile tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} platforms: linux/amd64,linux/arm64 push: true cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new - # Temp fix - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 + + # Temp fix: + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 - name: Move cache run: | rm -rf /tmp/.buildx-cache diff --git a/.github/workflows/sync-dockerhub-readme.yml b/.github/workflows/sync-dockerhub-readme.yml index 1e8fc57c5e..1df6ab44e6 100644 --- a/.github/workflows/sync-dockerhub-readme.yml +++ b/.github/workflows/sync-dockerhub-readme.yml @@ -4,16 +4,17 @@ on: push: branches: - main - paths: # ensures this workflow only runs when the readme.md or this file changes. - - 'readme.md' - - '.github/workflows/sync-dockerhub-readme.yml' - workflow_dispatch: + # Ensures this workflow only runs when the readme.md or workflow file itself changes + paths: + - readme.md + - .github/workflows/sync-dockerhub-readme.yml jobs: sync-dockerhub-readme: + name: Sync Readme to Docker Hub runs-on: ubuntu-latest steps: - - name: Checkout code + - name: Checkout repository uses: actions/checkout@v2 - name: Sync Readme to Docker Hub diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index 5b10e06fcd..0000000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Run tests - -on: - push: - branches: - - main - # Don't run on translation updates - paths-ignore: - - 'app/src/lang/translations/*.yaml' - pull_request: - branches: - - main - # Don't run on translation updates - paths-ignore: - - 'app/src/lang/translations/*.yaml' - -jobs: - test: - runs-on: ubuntu-latest - strategy: - matrix: - node: ["16"] - name: Node ${{ matrix.node }} - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.9.1 - with: - access_token: ${{ secrets.GITHUB_TOKEN }} - - - uses: actions/checkout@v2 - - - uses: actions/setup-node@v2 - with: - node-version: ${{ matrix.node }} - cache: npm - - - name: Install Dependencies - run: npm ci - - - name: Build Dependencies - run: npm run build - - - name: Run Tests - run: npm run test - env: - SECRET: TEST_SECRET diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 0000000000..40767f7411 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,33 @@ +name: Unit Tests + +on: + workflow_call: + +jobs: + test: + name: Test (Node.js ${{ matrix.node-version }}) + runs-on: ubuntu-latest + strategy: + matrix: + node-version: + - '16' + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Setup Node.js + uses: actions/setup-node@v2 + with: + node-version: ${{ matrix.node-version }} + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Build packages + run: npm run build + + - name: Run tests + run: npm run test + env: + SECRET: TEST_SECRET