Refactor tools/update_python_pip_packages.rb to use package objects and more rubification (#11623)

.github/workflows/Updater.yml

@@ -24,11 +24,20 @@ jobs:
           persist-credentials: true
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: '3.3.6'
+          ruby-version: '3.4.2'
       - name: Install Python pip
         run: sudo apt install -y python3-pip
-      - name: Install activesupport
-        run: sudo apt install -y ruby-activesupport
+      - name: Install ruby-libversion  # Hopefully this will get added as an Ubuntu/Debian package so we don't have to do this manually.
+        working-directory: ${{ runner.temp }}
+        run: |
+          git clone --depth 1 -b 3.0.3 https://github.com/repology/libversion
+          cd libversion
+          mkdir build
+          cd build
+          cmake ..
+          make -j $(nproc)
+          sudo make install
+          sudo gem install ruby-libversion
       - name: Set workflow & branch variables
         id: set-variables
         run: |

tools/update_python_pip_packages.rb

@@ -6,47 +6,57 @@
 # Usage in root of cloned chromebrew repo:
 # tools/update_python_pip_packages.rb
 
-# Add >LOCAL< lib to LOAD_PATH
-$LOAD_PATH.unshift '../lib'
+# Add >LOCAL< lib to LOAD_PATH so that packages can be loaded
+$LOAD_PATH.unshift './lib'
+
 require_relative '../lib/color'
 require_relative '../lib/const'
+require_relative '../lib/package'
+require_relative '../lib/package_utils'
 require_relative '../lib/require_gem'
-require_gem('activesupport', 'active_support/core_ext/object/blank')
 require_gem 'concurrent-ruby'
+require_gem 'ruby-libversion', 'ruby_libversion'
 
 def check_for_updated_python_packages
-  # Get a list of all the python/pip packages to check for updates.
-  py3_files = Dir['packages/py3_*.rb']
-  pip_files = `grep -l "^require 'buildsystems/pip'" packages/*.rb`.chomp.split
-  relevant_pip_packages = (py3_files + pip_files).uniq!
-
   # Sets the correct pip configuration values if they are not already set.
-  pip_config = `pip config list`.chomp
+  pip_config = `pip config list`
   system "pip config --user set global.index-url #{CREW_GITLAB_PKG_REPO}/pypi/simple", %i[err out] => File::NULL unless pip_config.include?("global.index-url='#{CREW_GITLAB_PKG_REPO}/pypi/simple'")
   system 'pip config --user set global.extra-index-url https://pypi.org/simple', %i[err out] => File::NULL unless pip_config.include?("global.extra-index-url='https://pypi.org/simple'")
   system 'pip config --user set global.trusted-host gitlab.com', %i[err out] => File::NULL unless pip_config.include?("global.trusted-host='gitlab.com'")
 
+  # Get a list of all the python/pip packages to check for updates.
+  py3_files = Dir['packages/py3_*.rb']
+  pip_files = `grep -l "^require 'buildsystems/pip'" packages/*.rb`.split
+  relevant_pip_packages = (py3_files + pip_files).uniq!
+
+  # Create a thread pool for parallelization.
   pool = Concurrent::ThreadPoolExecutor.new(
     min_threads: 1,
     max_threads: CREW_NPROC.to_i + 1,
     max_queue: 0, # unbounded work queue
     fallback_policy: :caller_runs
   )
+
+  # Get the total number of files to check, and then the length of that number, so status updates can be formatted.
   total_files_to_check = relevant_pip_packages.length
   numlength = total_files_to_check.to_s.length
   updateable_packages = {}
+  packages_without_pypi_versions = []
   relevant_pip_packages.each_with_index do |package, index|
     pool.post do
-      pip_name = package.gsub('.rb', '').sub('py3_', '').gsub('_', '-').gsub('packages/', '')
-      prerelease = system("grep -q '^\ \ prerelease' #{package}") ? '--pre' : nil
+      pkg = Package.load_package(package)
+      pip_name = pkg.name.sub('py3_', '').gsub('_', '-')
       # The \e[1A\e[K[] is to ensure the concurrency doesn't mess up the order of the printed status updates.
-      puts "\e[1A\e[K[#{(index + 1).to_s.rjust(numlength)}/#{total_files_to_check}] Checking pypi for #{prerelease.blank? ? '' : 'prerelease '}updates to #{pip_name}...\r".orange
-      pip_version = `python3 -m pip index versions #{prerelease} #{pip_name} 2>/dev/null | head -n 1 | awk '{print $2}'`.chomp.delete('()')
-      next package if pip_version.blank?
+      puts "\e[1A\e[K[#{(index + 1).to_s.rjust(numlength)}/#{total_files_to_check}] Checking pypi for#{' prerelease' if pkg.prerelease?} updates to #{pip_name}...\r".orange
+      # Attempt to query pip for the latest version of the package, but if it fails we take note of that and move to the next package.
+      begin
+        pip_version = `pip index versions #{'--pre' if pkg.prerelease?} #{pip_name} 2>/dev/null`.match(/#{Regexp.escape(pip_name)} \(([^)]+)\)/)[1]
+      rescue NoMethodError
+        packages_without_pypi_versions << pip_name
+        next
+      end
 
-      relevant_pip_packages.delete(package)
-      pkg_version = `sed -n -e 's/^\ \ version //p' #{package}`.chomp.delete("'").delete('"').gsub('-#{CREW_ICU_VER}', '').gsub('-#{CREW_PY_VER}', '') # rubocop:disable Lint/InterpolationCheck
-      next package unless Gem::Version.new(pip_version) > Gem::Version.new(pkg_version)
+      next unless Libversion.version_compare2(PackageUtils.get_clean_version(pkg.version), pip_version) == -1
 
       updateable_packages[package] = pip_version
     end
@@ -55,7 +65,7 @@ def check_for_updated_python_packages
   pool.wait_for_termination
 
   puts "Done checking pypi for updates to #{total_files_to_check} python packages.".orange
-  puts "Updated version#{relevant_pip_packages.length > 1 ? 's were' : ' was'} not listed in pypi for: #{relevant_pip_packages.map { |i| i.gsub('.rb', '').sub('ruby_', '').gsub('_', '-').gsub('packages/', '') }.join(' ')}".orange
+  puts "Updated versions were not listed in pypi for: #{packages_without_pypi_versions.join(' ')}".orange
 
   return updateable_packages
 end
@@ -64,14 +74,11 @@ def update_package_files(updateable_packages)
   return if updateable_packages.empty?
 
   updateable_packages.each_pair do |package, new_version|
-    package_name = package.gsub('.rb', '').sub('py3_', '').gsub('_', '-').gsub('packages/', '')
-    old_version = `sed -n -e 's/^\ \ version //p' #{package}`.chomp.delete("'").delete('"').gsub('-#{CREW_ICU_VER}', '').gsub('-#{CREW_PY_VER}', '') # rubocop:disable Lint/InterpolationCheck
-    puts "Updating #{package_name} from #{old_version} to #{new_version}".lightblue
-    if package_name == 'pyicu'
-      system "sed -i \"s,^\ \ version\ .*,\ \ version \\\"#{new_version}-\#{CREW_ICU_VER}-\#{CREW_PY_VER}\\\",\" #{package}"
-    else
-      system "sed -i \"s,^\ \ version\ .*,\ \ version \\\"#{new_version}-\#{CREW_PY_VER}\\\",\" #{package}"
-    end
+    pkg = Package.load_package(package)
+    puts "Updating #{pkg.name.gsub('_', '-')} from #{pkg.version} to #{new_version}".lightblue
+    file = File.read(package)
+    file.sub!(PackageUtils.get_clean_version(pkg.version), new_version)
+    File.write(package, file)
   end
 end