Some pip refactoring... (#10463)

* Pip updates

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

* Add wheel upload capability to crew.

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

* reupload i686 binary of py3_twine

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

* Add :in_install.

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

* Mention uri in uri download failed message.

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

* Have gem maintenance use cleanup instead of pristine.

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

* Add binaries for py3_importlib_metadata

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

* update scripts...

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

* Suggested changes.

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

* dogfooding build_updated_packages script to make more updates...

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

* More changes to use CREW_GITLAB_PKG_REPO.

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

* Suggested changes and tools updates.

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

* Added a :prerelease property to buildsystems/pip because properties from package.rb are not seen by buildsystems/pip.rb

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

* Add more binaries

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

* update i686 hash

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

* update last hash

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

---------

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

tools/update_ruby_gem_packages.rb

@@ -1,12 +1,59 @@
 #!/usr/bin/env ruby
 # update_ruby_gem_packages version 1.2 (for Chromebrew)
 # Author: Satadru Pramanik (satmandu) satadru at gmail dot com
-# Usage in packages dir: ../tools/update_ruby_gem_packages.rb
-require 'parallel'
+# Usage in root of cloned chromebrew repo:
+# tools/update_ruby_gem_packages.rb
+require_relative '../lib/color'
+require_relative '../lib/const'
+def require_gem(gem_name_and_require = nil, require_override = nil)
+  # Allow only loading gems when needed.
+  return if gem_name_and_require.nil?
 
-Parallel.map(Dir['ruby_*.rb']) do |package|
-  gem_name = package.gsub('.rb', '').sub('ruby_', '').gsub('_', '-')
-  gem_version = Gem.latest_spec_for(gem_name).version.to_s
-  puts "Updating #{gem_name} package file #{package} to #{gem_version}"
-  system "sed -i \"s,^\ \ version\ .*,\ \ version '#{gem_version}-ruby-#{RUBY_VERSION.slice(/(?:.*(?=\.))/)}',\" #{package}"
+  gem_name = gem_name_and_require.split('/')[0]
+  begin
+    gem gem_name
+  rescue LoadError
+    puts " -> install #{gem_name} gem".orange
+    Gem.install(gem_name)
+    gem gem_name
+  end
+  requires = if require_override.nil?
+               gem_name_and_require.split('/')[1].nil? ? gem_name_and_require.split('/')[0] : gem_name_and_require
+             else
+               require_override
+             end
+  require requires
 end
+require_gem('activesupport', 'active_support/core_ext/object/blank')
+require_gem 'concurrent-ruby'
+
+ruby_ver_string = "-ruby-#{RUBY_VERSION.slice(/(?:.*(?=\.))/)}"
+pool = Concurrent::ThreadPoolExecutor.new(
+  min_threads: 1,
+  max_threads: CREW_NPROC.to_i + 1,
+  max_queue: 0, # unbounded work queue
+  fallback_policy: :caller_runs
+)
+relevant_gem_packages = Dir['packages/ruby_*.rb']
+total_files_to_check = relevant_gem_packages.length
+numlength = total_files_to_check.to_s.length
+relevant_gem_packages.each_with_index do |package, index|
+  pool.post do
+    gem_name = package.gsub('.rb', '').sub('ruby_', '').gsub('_', '-').gsub('packages/', '')
+    puts "[#{(index + 1).to_s.rjust(numlength)}/#{total_files_to_check}] Checking rubygems for updates to #{gem_name}...".orange
+    pkg_version = `sed -n -e 's/^\ \ version //p' #{package}`.chomp.delete("'").delete('"').gsub(ruby_ver_string, '')
+    gem_version = Gem.latest_spec_for(gem_name).version.to_s
+    next package if gem_version.blank?
+    if Gem::Version.new(gem_version) > Gem::Version.new(pkg_version)
+      puts "Updating #{gem_name} from #{pkg_version} to #{gem_version}".lightblue
+      system "sed -i \"s,^\ \ version\ .*,\ \ version '#{gem_version}-ruby-#{RUBY_VERSION.slice(/(?:.*(?=\.))/)}',\" #{package}"
+    end
+    relevant_gem_packages.delete(package)
+  end
+end
+pool.shutdown
+pool.wait_for_termination
+puts "Done checking for updates to #{total_files_to_check} ruby gems.\r".orange
+return if relevant_gem_packages.blank?
+
+puts "Updated version#{relevant_gem_packages.length > 1 ? 's were' : ' was'} could not be found for: #{relevant_gem_packages.map { |i| i.gsub('.rb', '').sub('ruby_', '').gsub('_', '-').gsub('packages/', '') }.join(' ')}".orange