From c022369cfb57e45acd4fd9c837cf7106a738c980 Mon Sep 17 00:00:00 2001 From: Maximilian Downey Twiss Date: Sat, 29 Mar 2025 05:20:59 +1100 Subject: [PATCH] Refactor tools/update_python_pip_packages.rb to use package objects and more rubification (#11623) --- .github/workflows/Updater.yml | 15 ++++++-- tools/update_python_pip_packages.rb | 59 ++++++++++++++++------------- 2 files changed, 45 insertions(+), 29 deletions(-) diff --git a/.github/workflows/Updater.yml b/.github/workflows/Updater.yml index 5378e8565..9b7a2219f 100644 --- a/.github/workflows/Updater.yml +++ b/.github/workflows/Updater.yml @@ -24,11 +24,20 @@ jobs: persist-credentials: true - uses: ruby/setup-ruby@v1 with: - ruby-version: '3.3.6' + ruby-version: '3.4.2' - name: Install Python pip run: sudo apt install -y python3-pip - - name: Install activesupport - run: sudo apt install -y ruby-activesupport + - name: Install ruby-libversion # Hopefully this will get added as an Ubuntu/Debian package so we don't have to do this manually. + working-directory: ${{ runner.temp }} + run: | + git clone --depth 1 -b 3.0.3 https://github.com/repology/libversion + cd libversion + mkdir build + cd build + cmake .. + make -j $(nproc) + sudo make install + sudo gem install ruby-libversion - name: Set workflow & branch variables id: set-variables run: | diff --git a/tools/update_python_pip_packages.rb b/tools/update_python_pip_packages.rb index fae6c3243..c7a5687c5 100755 --- a/tools/update_python_pip_packages.rb +++ b/tools/update_python_pip_packages.rb @@ -6,47 +6,57 @@ # Usage in root of cloned chromebrew repo: # tools/update_python_pip_packages.rb -# Add >LOCAL< lib to LOAD_PATH -$LOAD_PATH.unshift '../lib' +# Add >LOCAL< lib to LOAD_PATH so that packages can be loaded +$LOAD_PATH.unshift './lib' + require_relative '../lib/color' require_relative '../lib/const' +require_relative '../lib/package' +require_relative '../lib/package_utils' require_relative '../lib/require_gem' -require_gem('activesupport', 'active_support/core_ext/object/blank') require_gem 'concurrent-ruby' +require_gem 'ruby-libversion', 'ruby_libversion' def check_for_updated_python_packages - # Get a list of all the python/pip packages to check for updates. - py3_files = Dir['packages/py3_*.rb'] - pip_files = `grep -l "^require 'buildsystems/pip'" packages/*.rb`.chomp.split - relevant_pip_packages = (py3_files + pip_files).uniq! - # Sets the correct pip configuration values if they are not already set. - pip_config = `pip config list`.chomp + pip_config = `pip config list` system "pip config --user set global.index-url #{CREW_GITLAB_PKG_REPO}/pypi/simple", %i[err out] => File::NULL unless pip_config.include?("global.index-url='#{CREW_GITLAB_PKG_REPO}/pypi/simple'") system 'pip config --user set global.extra-index-url https://pypi.org/simple', %i[err out] => File::NULL unless pip_config.include?("global.extra-index-url='https://pypi.org/simple'") system 'pip config --user set global.trusted-host gitlab.com', %i[err out] => File::NULL unless pip_config.include?("global.trusted-host='gitlab.com'") + # Get a list of all the python/pip packages to check for updates. + py3_files = Dir['packages/py3_*.rb'] + pip_files = `grep -l "^require 'buildsystems/pip'" packages/*.rb`.split + relevant_pip_packages = (py3_files + pip_files).uniq! + + # Create a thread pool for parallelization. pool = Concurrent::ThreadPoolExecutor.new( min_threads: 1, max_threads: CREW_NPROC.to_i + 1, max_queue: 0, # unbounded work queue fallback_policy: :caller_runs ) + + # Get the total number of files to check, and then the length of that number, so status updates can be formatted. total_files_to_check = relevant_pip_packages.length numlength = total_files_to_check.to_s.length updateable_packages = {} + packages_without_pypi_versions = [] relevant_pip_packages.each_with_index do |package, index| pool.post do - pip_name = package.gsub('.rb', '').sub('py3_', '').gsub('_', '-').gsub('packages/', '') - prerelease = system("grep -q '^\ \ prerelease' #{package}") ? '--pre' : nil + pkg = Package.load_package(package) + pip_name = pkg.name.sub('py3_', '').gsub('_', '-') # The \e[1A\e[K[] is to ensure the concurrency doesn't mess up the order of the printed status updates. - puts "\e[1A\e[K[#{(index + 1).to_s.rjust(numlength)}/#{total_files_to_check}] Checking pypi for #{prerelease.blank? ? '' : 'prerelease '}updates to #{pip_name}...\r".orange - pip_version = `python3 -m pip index versions #{prerelease} #{pip_name} 2>/dev/null | head -n 1 | awk '{print $2}'`.chomp.delete('()') - next package if pip_version.blank? + puts "\e[1A\e[K[#{(index + 1).to_s.rjust(numlength)}/#{total_files_to_check}] Checking pypi for#{' prerelease' if pkg.prerelease?} updates to #{pip_name}...\r".orange + # Attempt to query pip for the latest version of the package, but if it fails we take note of that and move to the next package. + begin + pip_version = `pip index versions #{'--pre' if pkg.prerelease?} #{pip_name} 2>/dev/null`.match(/#{Regexp.escape(pip_name)} \(([^)]+)\)/)[1] + rescue NoMethodError + packages_without_pypi_versions << pip_name + next + end - relevant_pip_packages.delete(package) - pkg_version = `sed -n -e 's/^\ \ version //p' #{package}`.chomp.delete("'").delete('"').gsub('-#{CREW_ICU_VER}', '').gsub('-#{CREW_PY_VER}', '') # rubocop:disable Lint/InterpolationCheck - next package unless Gem::Version.new(pip_version) > Gem::Version.new(pkg_version) + next unless Libversion.version_compare2(PackageUtils.get_clean_version(pkg.version), pip_version) == -1 updateable_packages[package] = pip_version end @@ -55,7 +65,7 @@ def check_for_updated_python_packages pool.wait_for_termination puts "Done checking pypi for updates to #{total_files_to_check} python packages.".orange - puts "Updated version#{relevant_pip_packages.length > 1 ? 's were' : ' was'} not listed in pypi for: #{relevant_pip_packages.map { |i| i.gsub('.rb', '').sub('ruby_', '').gsub('_', '-').gsub('packages/', '') }.join(' ')}".orange + puts "Updated versions were not listed in pypi for: #{packages_without_pypi_versions.join(' ')}".orange return updateable_packages end @@ -64,14 +74,11 @@ def update_package_files(updateable_packages) return if updateable_packages.empty? updateable_packages.each_pair do |package, new_version| - package_name = package.gsub('.rb', '').sub('py3_', '').gsub('_', '-').gsub('packages/', '') - old_version = `sed -n -e 's/^\ \ version //p' #{package}`.chomp.delete("'").delete('"').gsub('-#{CREW_ICU_VER}', '').gsub('-#{CREW_PY_VER}', '') # rubocop:disable Lint/InterpolationCheck - puts "Updating #{package_name} from #{old_version} to #{new_version}".lightblue - if package_name == 'pyicu' - system "sed -i \"s,^\ \ version\ .*,\ \ version \\\"#{new_version}-\#{CREW_ICU_VER}-\#{CREW_PY_VER}\\\",\" #{package}" - else - system "sed -i \"s,^\ \ version\ .*,\ \ version \\\"#{new_version}-\#{CREW_PY_VER}\\\",\" #{package}" - end + pkg = Package.load_package(package) + puts "Updating #{pkg.name.gsub('_', '-')} from #{pkg.version} to #{new_version}".lightblue + file = File.read(package) + file.sub!(PackageUtils.get_clean_version(pkg.version), new_version) + File.write(package, file) end end