Update package updater to handle llvm and gcc suite packages. — container_essential,libclc → 21.1.6,openmp → 21.1.6 (#13551)

* Update package updater to handle llvm and gcc suite packages.

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

* Add x86_64 binaries for openmp, libclc.

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

* Add help options.

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

* Expand help text.

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

* openmp: Package File Update Run on linux/386 container.

* Update gexiv2 anitya mapping.

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

* openmp: Package File Update Run on linux/amd64 container.

---------

Signed-off-by: Satadru Pramanik <satadru@gmail.com>
Co-authored-by: Satadru Pramanik <satadru@gmail.com>
Co-authored-by: chromebrew-actions[bot] <chromebrew-actions[bot]@users.noreply.github.com>
Co-authored-by: satmandu <satmandu@users.noreply.github.com>

lib/color.rb

@@ -12,9 +12,11 @@ class String
        purple: { normal: 35, background: 45 },
          cyan: { normal: 36, background: 46 },
     lightgray: { normal: 37, background: 47 },
+    lightgrey: { normal: 37, background: 47 },
 
            # colors with bold varient available
            gray: { normal: 90, background: 100, bold: 30 },
+           grey: { normal: 90, background: 100, bold: 30 },
        lightred: { normal: 91, background: 101, bold: 31 },
      lightgreen: { normal: 92, background: 102, bold: 32 },
          yellow: { normal: 93, background: 103, bold: 33 },

lib/const.rb

@@ -410,6 +410,7 @@ unless defined?(CREW_ANITYA_PACKAGE_NAME_MAPPINGS)
     { pkg_name: 'gcr_3', anitya_pkg: 'gcr~3', comments: '' },
     { pkg_name: 'gcr_4', anitya_pkg: 'gcr', comments: '' },
     { pkg_name: 'gemacs', anitya_pkg: 'emacs', comments: '' },
+    { pkg_name: 'gexiv2', anitya_pkg: 'gexiv2~14.x', comments: '' },
     { pkg_name: 'gnome_docking_library', anitya_pkg: 'gdl', comments: '' },
     { pkg_name: 'gnu_time', anitya_pkg: 'time', comments: '' },
     { pkg_name: 'go_tools', anitya_pkg: 'golang-x-tools', comments: '' },

packages/container_essential.rb

@@ -3,7 +3,7 @@ require 'package'
 class Container_essential < Package
   description 'A collection of extra tools and packages needed in the docker build containers.'
   homepage 'https://github.com/chromebrew/chromebrew'
-  version '1.0'
+  version '1.1'
   license 'GPL-3+'
   compatibility 'all'
 
@@ -15,6 +15,8 @@ class Container_essential < Package
 
   depends_on 'psmisc'
   depends_on 'sudo_rs'
+  # xxd is used to pass arguments into the container to the /bin/chromebrewstart docker container start CMD.
+  depends_on 'xxd_standalone'
   depends_on 'util_linux'
 
   if CREW_GLIBC_INTERPRETER.nil?

packages/libclc.rb

@@ -17,9 +17,9 @@ class Libclc < Package
   binary_compression 'tar.zst'
 
   binary_sha256({
-    aarch64: 'b654ab4da781f0bc59a6e444275480add0ca53c4139bee3a48f36c877f4e616c',
+    aarch64: '54ade18f0566de160f26e9f553ce307a3441530f29a5b638f340a757dbb2dd46',
-     armv7l: 'b654ab4da781f0bc59a6e444275480add0ca53c4139bee3a48f36c877f4e616c',
+     armv7l: '54ade18f0566de160f26e9f553ce307a3441530f29a5b638f340a757dbb2dd46',
-     x86_64: '3457d53ef12ccaf8788d9dcfc29eb4e0e1ff0d62937936f74060b19580431b37'
+     x86_64: 'c62dca7429ebc864f8cfb54f88f52078746fb42e30b151c625c4e2a11fad7733'
   })
 
   depends_on 'llvm_dev' => :build

packages/openmp.rb

@@ -17,10 +17,10 @@ class Openmp < Package
   binary_compression 'tar.zst'
 
   binary_sha256({
-    aarch64: 'fb6da8ebb677a802dcc0941b9a332c329db8a5219fe2c01cfaa2414e41f3e304',
+    aarch64: '3b19fd3adcb8fe69eec06070a0d8cf257fa14602b3a13cf3bfb668b92edfa500',
-     armv7l: 'fb6da8ebb677a802dcc0941b9a332c329db8a5219fe2c01cfaa2414e41f3e304',
+     armv7l: '3b19fd3adcb8fe69eec06070a0d8cf257fa14602b3a13cf3bfb668b92edfa500',
-       i686: '9f7be32395bf445897d0d2f7efcec7be39f7c23736692f058d61f9961826da3d',
+       i686: '5e5cd17acfb4dafd4d847656d9a67f21e30a26bd5565c07fbfacbc80e8e5a27d',
-     x86_64: 'afd29b5eae9649eaa29bfdd31fd2bb3f9950a27fad3740ba0bbc5c96e5ee5221'
+     x86_64: '8a16faced504313444a76c85940d484e599e2b93075f3fa5f917c0cf904e7618'
   })
 
   depends_on 'gcc_lib' # R

tools/build_updated_packages.rb

@@ -1,5 +1,5 @@
 #!/usr/local/bin/ruby
-# build_updated_packages version 3.9 (for Chromebrew)
+# build_updated_packages version 4.0 (for Chromebrew)
 # This updates the versions in python pip packages by calling
 # tools/update_python_pip_packages.rb, checks for updated ruby packages
 # by calling tools/update_ruby_gem_packages.rb, and then checks if any
@@ -15,11 +15,36 @@
 
 require 'English'
 require 'fileutils'
-require_relative '../lib/color'
+
-require_relative '../lib/const'
+crew_local_repo_root = `git rev-parse --show-toplevel 2> /dev/null`.chomp
-require_relative '../lib/package'
+# When invoked from crew, pwd is CREW_DEST_DIR, so crew_local_repo_root
-require_relative '../lib/require_gem'
+# is empty.
-$LOAD_PATH.unshift File.expand_path('../lib', __dir__)
+crew_local_repo_root = '../' if crew_local_repo_root.to_s.empty?
+require File.join(crew_local_repo_root, 'lib/color')
+require File.join(crew_local_repo_root, 'lib/const')
+require File.join(crew_local_repo_root, 'lib/package')
+require File.join(crew_local_repo_root, 'lib/require_gem')
+require_gem 'highline'
+require_gem 'timeout'
+
+# Add >LOCAL< lib to LOAD_PATH
+$LOAD_PATH.unshift File.join(crew_local_repo_root, 'lib')
+
+OPTIONS = %w[-h --help --check-all-python --check-all-ruby --continue-after-failed-builds --skip -v --verbose -vv]
+
+if ARGV.include?('-h') || ARGV.include?('--help')
+  abort <<~EOM
+    Usage: ./build_updated_packages.rb [<package>] #{OPTIONS.to_s.gsub('"', '')}
+    Example: ./build_updated_packages.rb abcde -v
+    If <package> is omitted, recently updated files will be checked for needed builds.
+    Passing --continue-after-failed-builds will continue if builds fail.
+    Passing --skip will skip update checks.
+    Passing --check-all-python will check py3_ pip packages for updates.
+    Passing --check-all-ruby will check ruby_ gem packages for updates.
+    Passing --verbose or -v will display verbose output.
+    Passing -vv will display very verbose output.
+  EOM
+end
 
 abort "\nGITLAB_TOKEN environment variable not set.\n".lightred if ENV['GITLAB_TOKEN'].nil?
 abort "\nGITLAB_TOKEN_USERNAME environment variable not set.\n".lightred if ENV['GITLAB_TOKEN_USERNAME'].nil?
@@ -29,8 +54,22 @@ CONTINUE_AFTER_FAILED_BUILDS = ARGV.include?('--continue-after-failed-builds')
 SKIP_UPDATE_CHECKS = ARGV.include?('--skip')
 CHECK_ALL_PYTHON = ARGV.include?('--check-all-python')
 CHECK_ALL_RUBY = ARGV.include?('--check-all-ruby')
-require_gem 'highline'
+VERBOSE = ARGV.include?('-v') || ARGV.include?('--verbose') || ARGV.include?('-vv')
-require_gem 'timeout'
+VERY_VERBOSE = ARGV.include?('-vv')
+
+updated_packages = []
+# Handle multiple packages being passed.
+argv = ARGV.map(&:split).flatten
+if argv.length.positive? && !(argv.length == 1 && OPTIONS.include?(argv[0]))
+  argv.each do |arg|
+    arg = arg.gsub('.rb', '')
+    next unless arg =~ /^[0-9a-zA-Z_*]+$/
+
+    filename = File.join(crew_local_repo_root, "packages/#{arg}.rb")
+    updated_packages.push "packages/#{arg}.rb" if File.exist?(filename)
+    puts "#{arg} has been added to the list of packages to be checked for updates.".lightblue
+  end
+end
 
 build = {}
 
@@ -39,7 +78,18 @@ excluded_packages = Set[
   { pkg_name: 'py3_unsupported_python', commwnts: 'Stub package.' },
   { pkg_name: 'terraform', comments: 'Needs manual update due to package structure.' }
 ]
-excluded_pkgs = excluded_packages.map { |h| h[:pkg_name] }
+excluded_pkgs = excluded_packages.map { it[:pkg_name] }
+
+dependent_packages_to_check = Set[
+  { pkg_name: 'gcc_build', downstream_packages: 'gcc_lib', comments: '' },
+  { pkg_name: 'gcc_lib', downstream_packages: 'gcc_dev', comments: '' },
+  { pkg_name: 'gcc_dev', downstream_packages: 'libssp', comments: '' },
+  { pkg_name: "#{CREW_LLVM_VER}_build", downstream_packages: "#{CREW_LLVM_VER}_lib", comments: '' },
+  { pkg_name: "#{CREW_LLVM_VER}_lib", downstream_packages: "#{CREW_LLVM_VER}_dev", comments: '' },
+  { pkg_name: "#{CREW_LLVM_VER}_dev", downstream_packages: 'openmp spirv_llvm_translator libclc', comments: '' },
+  { pkg_name: 'spirv_llvm_translator', downstream_packages: 'libclc', comments: '' }
+]
+dependent_pkgs = dependent_packages_to_check.to_h { [it[:pkg_name], it[:downstream_packages]] }
 
 def self.agree_default_yes(message = nil)
   Timeout.timeout(CREW_AGREE_TIMEOUT_SECONDS) do
@@ -56,16 +106,16 @@ def self.check_build_uploads(architectures_to_check = nil, name = nil)
   remote_binary = { armv7l: nil, i686: nil, x86_64: nil }
   remote_binary.keys.each do |arch|
     arch_specific_url = "#{CREW_GITLAB_PKG_REPO}/generic/#{name}/#{@pkg_obj.version}_#{arch}/#{name}-#{@pkg_obj.version}-chromeos-#{arch}.#{@pkg_obj.binary_compression}"
-    puts "Checking: curl -sI #{arch_specific_url}" if CREW_VERBOSE
+    puts "Checking: curl -sI #{arch_specific_url}" if VERBOSE
     remote_binary[arch.to_sym] = `curl -sI #{arch_specific_url}`.lines.first.split[1] == '200'
-    puts "#{arch_specific_url} found!" if remote_binary[arch.to_sym] && CREW_VERBOSE
+    puts "#{arch_specific_url} found!" if remote_binary[arch.to_sym] && VERBOSE
   end
   system "crew update_package_file #{name}" unless remote_binary.values.all?(nil)
 
   builds_needed = architectures_to_check.dup
   architectures_to_check.each do |arch|
     builds_needed.delete(arch) if remote_binary[arch.to_sym]
-    puts "builds_needed for #{name} is now #{builds_needed}" if CREW_VERBOSE
+    puts "builds_needed for #{name} is now #{builds_needed}" if VERBOSE
   end
   return builds_needed
 end
@@ -75,9 +125,9 @@ def update_hashes_and_manifests(name = nil)
     remote_binary = { armv7l: nil, i686: nil, x86_64: nil }
     remote_binary.keys.each do |arch|
       arch_specific_url = "#{CREW_GITLAB_PKG_REPO}/generic/#{name}/#{@pkg_obj.version}_#{arch}/#{name}-#{@pkg_obj.version}-chromeos-#{arch}.#{@pkg_obj.binary_compression}"
-      puts "Checking: curl -sI #{arch_specific_url}" if CREW_VERBOSE
+      puts "Checking: curl -sI #{arch_specific_url}" if VERBOSE
       remote_binary[arch.to_sym] = `curl -sI #{arch_specific_url}`.lines.first.split[1] == '200'
-      puts "#{arch_specific_url} found!" if remote_binary[arch.to_sym] && CREW_VERBOSE
+      puts "#{arch_specific_url} found!" if remote_binary[arch.to_sym] && VERBOSE
     end
     # Add build hashes.
     system "crew update_package_file #{name}" unless remote_binary.values.all?(nil)
@@ -117,17 +167,9 @@ else
 end
 changed_files = `git diff HEAD --name-only`.chomp.split
 changed_files_previous_commit = `git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/master)..$(git rev-parse --verify HEAD)`.chomp.split
-updated_packages = changed_files.select { |c| c.include?('packages/') }
+updated_packages.push(*changed_files.select { |c| c.include?('packages/') })
 updated_packages.push(*changed_files_previous_commit.select { |c| c.include?('packages/') })
 
-if updated_packages.empty?
-  puts 'No packages need to be updated.'.orange
-else
-  puts 'These changed packages will be checked to see if they need updated binaries:'.orange
-  updated_packages.uniq!
-  updated_packages.each { |p| puts p.sub('packages/', '').sub('.rb', '').to_s.lightblue }
-end
-
 crew_update_packages = `CREW_NO_GIT=1 CREW_UNATTENDED=1 crew update | grep "\\[\\""  | jq -r '.[]'`.chomp.split.map(&'packages/'.method(:+)).map { |i| i.concat('.rb') }
 if CHECK_ALL_PYTHON
   py_packages = `grep -l CREW_PY_VER packages/*`.chomp.split
@@ -139,17 +181,38 @@ if CHECK_ALL_RUBY
 end
 updated_packages.push(*crew_update_packages)
 updated_packages.uniq!
+
 # Remove packages that don't need to be checked for updates from the
 # check list.
 exclusion_regex = "(#{excluded_pkgs.join('|')})"
 updated_packages.delete_if { |d| /#{exclusion_regex}/.match(d) }
 
+# Add additional packages that need to be checked for updates.
+inclusion_regex = "(#{dependent_pkgs.keys.join('|')})"
+updated_packages.each do |p|
+  pkg = p.sub('packages/', '').sub('.rb', '')
+  next unless /#{inclusion_regex}/.match(pkg)
+  dependent_pkgs[pkg].split.each do |dependent_pkg|
+    if File.exist?(File.join(crew_local_repo_root, "packages/#{dependent_pkg}.rb"))
+      puts "Will check #{dependent_pkg} since dependent package of #{pkg}...".orange if VERY_VERBOSE
+      updated_packages.push "packages/#{dependent_pkg}.rb"
+    end
+  end
+end
+
+if updated_packages.empty?
+  puts 'No packages need to be updated.'.orange
+else
+  puts 'These packages will be checked to see if they need updated binaries:'.orange
+  updated_packages.uniq!
+  updated_packages.each { |p| puts p.sub('packages/', '').sub('.rb', '').to_s.lightblue }
+end
+
 updated_packages.each do |pkg|
-  puts "pkg is #{pkg}" if CREW_VERBOSE
   name = pkg.sub('packages/', '').sub('.rb', '')
   next unless File.file?(pkg)
 
-  puts "Evaluating #{name} package...".orange
+  puts "Evaluating #{name} package...".lightpurple
   system "rubocop -c .rubocop.yml -A #{pkg}"
   @pkg_obj = Package.load_package(pkg)