diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a15b0cd7..207f5703 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -152,11 +152,18 @@ jobs:
       - name: Setup Bundler 1.x for Rails 4.x
         if: ${{ matrix.gemfile == 'gemfiles/Gemfile-rails-4-1' || matrix.gemfile == 'gemfiles/Gemfile-rails-4-2' }}
         run: echo "BUNDLER_VERSION=1.17.3" >> $GITHUB_ENV
+      - name: Setup Rubygems version as default for Ruby < 2.5
+        if: ${{ matrix.ruby < '2.5' }}
+        run: echo "RUBYGEMS_VERSION=default" >> $GITHUB_ENV
+      - name: Setup Rubygems version as 3.2.3 for Ruby 2.5
+        if: ${{ matrix.ruby == '2.5' }}
+        run: echo "RUBYGEMS_VERSION=3.2.3" >> $GITHUB_ENV
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
           bundler-cache: true # runs bundle install and caches installed gems automatically
-          bundler: ${{ env.BUNDLER_VERSION || 'latest' }}
+          bundler: ${{ env.BUNDLER_VERSION || 'default' }}
+          rubygems: ${{ env.RUBYGEMS_VERSION || 'latest' }}
       - uses: supercharge/mongodb-github-action@1.3.0
         if: ${{ matrix.env == 'DEVISE_ORM=mongoid' }}
       - run: bundle exec rake