From 37e467d5805e97242f882d58c05c36c09be33573 Mon Sep 17 00:00:00 2001 From: Maximilian Downey Twiss Date: Thu, 26 Jun 2025 22:14:20 +1000 Subject: [PATCH] Refactor tools/update_ruby_gem_packages.rb to use package objects, the rubygems.org api, and more rubification (#11671) * Trim -ruby versions in PackageUtils.get_clean_version * Refactor tools/update_ruby_gem_packages.rb to use package objects, the rubygems.org api, and more rubification --- .github/workflows/Updater.yml | 2 +- lib/package_utils.rb | 2 + tests/lib/package_utils.rb | 4 ++ tools/update_ruby_gem_packages.rb | 76 ++++++++++++++++++++----------- 4 files changed, 57 insertions(+), 27 deletions(-) diff --git a/.github/workflows/Updater.yml b/.github/workflows/Updater.yml index 02589901c..d7c270c28 100644 --- a/.github/workflows/Updater.yml +++ b/.github/workflows/Updater.yml @@ -79,7 +79,7 @@ jobs: env: UPDATE_BRANCH_NAME: ${{ steps.set-variables.outputs.UPDATE_BRANCH_NAME }} run: | - ruby tools/update_ruby_gem_packages.rb + LD_LIBRARY_PATH=/usr/local/lib ruby tools/update_ruby_gem_packages.rb # Create a new branch with the updated package files only # if there are updated packages. Otherwise exit early. if [ -n "$(git status --porcelain)" ]; then diff --git a/lib/package_utils.rb b/lib/package_utils.rb index 25289dfeb..239ad81c2 100644 --- a/lib/package_utils.rb +++ b/lib/package_utils.rb @@ -81,6 +81,8 @@ class PackageUtils pkg_version.delete_prefix!('kde-') # Delete -py3.12, futureproofed until Python 4 pkg_version.gsub!(/-py3\.\d{2}/, '') + # Delete -ruby3.4, futureproofed until Ruby 4 or Ruby 3.10 + pkg_version.gsub!(/-ruby3\.\d{1}/, '') # Delete -perl 5.40, futureproofed until Perl 5.100 pkg_version.gsub!(/-perl5\.\d{2}/, '') # Delete -llvm18, futureproofed until llvm 100 diff --git a/tests/lib/package_utils.rb b/tests/lib/package_utils.rb index ba86fda67..20794bfe0 100644 --- a/tests/lib/package_utils.rb +++ b/tests/lib/package_utils.rb @@ -148,6 +148,10 @@ class PackageUtilsTest < Minitest::Test assert_equal('1.2.3', PackageUtils.get_clean_version('1.2.3-py3.12')) end + def test_get_clean_ruby_version + assert_equal('99.95', PackageUtils.get_clean_version('99.95-ruby3.4')) + end + def test_get_clean_perl_version assert_equal('0.004.2', PackageUtils.get_clean_version('0.004.2-perl5.40')) end diff --git a/tools/update_ruby_gem_packages.rb b/tools/update_ruby_gem_packages.rb index eeefca501..35c5ea77d 100755 --- a/tools/update_ruby_gem_packages.rb +++ b/tools/update_ruby_gem_packages.rb @@ -5,45 +5,68 @@ # tools/update_ruby_gem_packages.rb # Add >LOCAL< lib to LOAD_PATH -$LOAD_PATH.unshift '../lib' +$LOAD_PATH.unshift './lib' + +require 'json' +require 'net/http' require_relative '../lib/color' require_relative '../lib/const' -require_relative '../lib/gem_compact_index_client' +require_relative '../lib/package' +require_relative '../lib/package_utils' require_relative '../lib/require_gem' require_gem 'concurrent-ruby' +require_gem 'ruby-libversion', 'ruby_libversion' def check_for_updated_ruby_packages - gems = BasicCompactIndexClient.new.gems - + # Create a thread pool for parallelization. pool = Concurrent::ThreadPoolExecutor.new( min_threads: 1, max_threads: CREW_NPROC.to_i + 1, max_queue: 0, # unbounded work queue fallback_policy: :caller_runs ) + + # Currently, the only packages that use the ruby buildsystem are ruby_* packages, so this finds all the packages we need to check. relevant_gem_packages = Dir['packages/ruby_*.rb'] + + # Get the total number of files to check, and then the length of that number, so status updates can be formatted. total_files_to_check = relevant_gem_packages.length numlength = total_files_to_check.to_s.length updateable_packages = {} + packages_without_gem_versions = [] relevant_gem_packages.each_with_index do |package, index| pool.post do - untested_package_name = package.gsub(%r{^packages/ruby_}, '').gsub(/.rb$/, '') - gem_test = gems.grep(/#{"^#{untested_package_name}\\s.*$"}/).last.blank? ? gems.grep(/#{"^\(#{passed_name.gsub(/^ruby_/, '').gsub('_', ')*.(')}\\s\).*$"}/).last : gems.grep(/#{"^#{untested_package_name}\\s.*$"}/).last - abort "Cannot find #{passed_name} gem to install.".lightred if gem_test.blank? - gem_test_name = gem_test.split.first - gem_test_versions = gem_test.split[1].split(',') - # Any version with a letter is considered a prerelease as per - # https://github.com/rubygems/rubygems/blob/b5798efd348935634d4e0e2b846d4f455582db48/lib/rubygems/version.rb#L305 - gem_test_versions.delete_if { |i| i.match?(/[a-zA-Z]/) } - gem_test_version = gem_test_versions.max - ruby_gem_name = gem_test_name.blank? ? Gem::SpecFetcher.fetcher.suggest_gems_from_name(untested_package_name).first : gem_test_name - ruby_gem_version = gem_test_name.blank? ? Gem.latest_version_for(untested_package_name).to_s : gem_test_version - next package if ruby_gem_version.blank? + pkg = Package.load_package(package) + gem_name = pkg.name.sub('ruby_', '') + # We replace all dashes with underscores in our initial package names, but some gems actually use underscores, so we need special cases. + # This list was created by looking at what packages were listed as not having updates in rubygems, and then looking up the upstream name for them. + if %w[connection_pool error_highlight mini_mime multi_xml mutex_m power_assert regexp_parser repl_type_completor ruby2_keywords syntax_suggest].include?(gem_name) + # These gems used underscores originally, so don't replace anything + elsif gem_name == 'language_server_protocol' + # These gems have an underscore then a dash, but there's only one, so we hardcode the logic for now. + gem_name = 'language_server-protocol' + elsif gem_name == 'unicode_display_width' + # These gems have a dash then an underscore, but there's only one, so we hardcode the logic for now. + gem_name = 'unicode-display_width' + else + # In the common case, the gem name used only dashes, which we all replaced with underscores. + gem_name.gsub!('_', '-') + end - relevant_gem_packages.delete(package) - puts "[#{(index + 1).to_s.rjust(numlength)}/#{total_files_to_check}] Checking rubygems for updates to #{ruby_gem_name} in #{package}...".orange - pkg_version = `sed -n -e 's/^\ \ version //p' #{package}`.chomp.delete("'").delete('"').gsub(/-\#{CREW_RUBY_VER}/, '').split('-').first - next package unless Gem::Version.new(ruby_gem_version) > Gem::Version.new(pkg_version) + puts "[#{(index + 1).to_s.rjust(numlength)}/#{total_files_to_check}] Checking rubygems for updates to #{gem_name} in #{package}...".orange + + gem_version = JSON.parse(Net::HTTP.get(URI("https://rubygems.org/api/v1/versions/#{gem_name}/latest.json")))['version'] + + if gem_version == 'unknown' + packages_without_gem_versions << gem_name + next + end + + # Any version with a letter is considered a prerelease as per + # https://docs.ruby-lang.org/en/master/Gem/Version.html#method-i-prerelease-3F + next if gem_version.match?(/[a-zA-Z]/) + + next unless Libversion.version_compare2(PackageUtils.get_clean_version(pkg.version), gem_version) == -1 updateable_packages[package] = ruby_gem_version end @@ -51,8 +74,8 @@ def check_for_updated_ruby_packages pool.shutdown pool.wait_for_termination - puts "Done checking for updates to #{total_files_to_check} ruby gems.\r".orange - puts "Updated version#{relevant_gem_packages.length > 1 ? 's were' : ' was'} could not be found for: #{relevant_gem_packages.map { |i| i.gsub('.rb', '').sub('ruby_', '').gsub('_', '-').gsub('packages/', '') }.join(' ')}".orange + puts "Done checking rubygems for updates to #{total_files_to_check} ruby packages.\r".orange + puts "Updated versions were not listed in rubygems for: #{packages_without_gem_versions.join(' ')}".orange return updateable_packages end @@ -61,10 +84,11 @@ def update_package_files(updateable_packages) return if updateable_packages.empty? updateable_packages.each_pair do |package, new_version| - package_name = package.gsub(%r{^packages/ruby_}, '').gsub(/.rb$/, '') - old_version = `sed -n -e 's/^\ \ version //p' #{package}`.chomp.delete("'").delete('"').gsub(/-\#{CREW_RUBY_VER}/, '').split('-').first - puts "Updating #{package_name} from #{old_version} to #{new_version}".lightblue - system "sed \"s,^\ \ version\ .*,\ \ version \\\"#{new_version}-\#{CREW_RUBY_VER}\\\",g;w #{package}.new\" #{package} && mv #{package}.new #{package}" + pkg = Package.load_package(package) + puts "Updating #{pkg.name.gsub('_', '-')} from #{pkg.version} to #{new_version}".lightblue + file = File.read(package) + file.sub!(PackageUtils.get_clean_version(pkg.version), new_version) + File.write(package, file) end end