From 0beafd117f6c16c65438569f08f19c93d2c9fb8c Mon Sep 17 00:00:00 2001 From: "chromebrew-actions[bot]" <220035932+chromebrew-actions[bot]@users.noreply.github.com> Date: Fri, 21 Nov 2025 11:48:08 -0600 Subject: [PATCH] =?UTF-8?q?Update=20package=20updater=20to=20handle=20llvm?= =?UTF-8?q?=20and=20gcc=20suite=20packages.=20=E2=80=94=20container=5Fesse?= =?UTF-8?q?ntial,libclc=20=E2=86=92=2021.1.6,openmp=20=E2=86=92=2021.1.6?= =?UTF-8?q?=20(#13551)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update package updater to handle llvm and gcc suite packages. Signed-off-by: Satadru Pramanik * Add x86_64 binaries for openmp, libclc. Signed-off-by: Satadru Pramanik * Add help options. Signed-off-by: Satadru Pramanik * Expand help text. Signed-off-by: Satadru Pramanik * openmp: Package File Update Run on linux/386 container. * Update gexiv2 anitya mapping. Signed-off-by: Satadru Pramanik * openmp: Package File Update Run on linux/amd64 container. --------- Signed-off-by: Satadru Pramanik Co-authored-by: Satadru Pramanik Co-authored-by: chromebrew-actions[bot] Co-authored-by: satmandu --- lib/color.rb | 2 + lib/const.rb | 1 + packages/container_essential.rb | 4 +- packages/libclc.rb | 6 +- packages/openmp.rb | 8 +-- tools/build_updated_packages.rb | 113 +++++++++++++++++++++++++------- 6 files changed, 101 insertions(+), 33 deletions(-) diff --git a/lib/color.rb b/lib/color.rb index c5d725f1c..66836567e 100644 --- a/lib/color.rb +++ b/lib/color.rb @@ -12,9 +12,11 @@ class String purple: { normal: 35, background: 45 }, cyan: { normal: 36, background: 46 }, lightgray: { normal: 37, background: 47 }, + lightgrey: { normal: 37, background: 47 }, # colors with bold varient available gray: { normal: 90, background: 100, bold: 30 }, + grey: { normal: 90, background: 100, bold: 30 }, lightred: { normal: 91, background: 101, bold: 31 }, lightgreen: { normal: 92, background: 102, bold: 32 }, yellow: { normal: 93, background: 103, bold: 33 }, diff --git a/lib/const.rb b/lib/const.rb index 07690e7fd..d7a3ac44e 100644 --- a/lib/const.rb +++ b/lib/const.rb @@ -410,6 +410,7 @@ unless defined?(CREW_ANITYA_PACKAGE_NAME_MAPPINGS) { pkg_name: 'gcr_3', anitya_pkg: 'gcr~3', comments: '' }, { pkg_name: 'gcr_4', anitya_pkg: 'gcr', comments: '' }, { pkg_name: 'gemacs', anitya_pkg: 'emacs', comments: '' }, + { pkg_name: 'gexiv2', anitya_pkg: 'gexiv2~14.x', comments: '' }, { pkg_name: 'gnome_docking_library', anitya_pkg: 'gdl', comments: '' }, { pkg_name: 'gnu_time', anitya_pkg: 'time', comments: '' }, { pkg_name: 'go_tools', anitya_pkg: 'golang-x-tools', comments: '' }, diff --git a/packages/container_essential.rb b/packages/container_essential.rb index 625c6cb92..bbe3760f6 100644 --- a/packages/container_essential.rb +++ b/packages/container_essential.rb @@ -3,7 +3,7 @@ require 'package' class Container_essential < Package description 'A collection of extra tools and packages needed in the docker build containers.' homepage 'https://github.com/chromebrew/chromebrew' - version '1.0' + version '1.1' license 'GPL-3+' compatibility 'all' @@ -15,6 +15,8 @@ class Container_essential < Package depends_on 'psmisc' depends_on 'sudo_rs' + # xxd is used to pass arguments into the container to the /bin/chromebrewstart docker container start CMD. + depends_on 'xxd_standalone' depends_on 'util_linux' if CREW_GLIBC_INTERPRETER.nil? diff --git a/packages/libclc.rb b/packages/libclc.rb index 87225a086..aa04d6898 100644 --- a/packages/libclc.rb +++ b/packages/libclc.rb @@ -17,9 +17,9 @@ class Libclc < Package binary_compression 'tar.zst' binary_sha256({ - aarch64: 'b654ab4da781f0bc59a6e444275480add0ca53c4139bee3a48f36c877f4e616c', - armv7l: 'b654ab4da781f0bc59a6e444275480add0ca53c4139bee3a48f36c877f4e616c', - x86_64: '3457d53ef12ccaf8788d9dcfc29eb4e0e1ff0d62937936f74060b19580431b37' + aarch64: '54ade18f0566de160f26e9f553ce307a3441530f29a5b638f340a757dbb2dd46', + armv7l: '54ade18f0566de160f26e9f553ce307a3441530f29a5b638f340a757dbb2dd46', + x86_64: 'c62dca7429ebc864f8cfb54f88f52078746fb42e30b151c625c4e2a11fad7733' }) depends_on 'llvm_dev' => :build diff --git a/packages/openmp.rb b/packages/openmp.rb index 1f3240443..6e7bac553 100644 --- a/packages/openmp.rb +++ b/packages/openmp.rb @@ -17,10 +17,10 @@ class Openmp < Package binary_compression 'tar.zst' binary_sha256({ - aarch64: 'fb6da8ebb677a802dcc0941b9a332c329db8a5219fe2c01cfaa2414e41f3e304', - armv7l: 'fb6da8ebb677a802dcc0941b9a332c329db8a5219fe2c01cfaa2414e41f3e304', - i686: '9f7be32395bf445897d0d2f7efcec7be39f7c23736692f058d61f9961826da3d', - x86_64: 'afd29b5eae9649eaa29bfdd31fd2bb3f9950a27fad3740ba0bbc5c96e5ee5221' + aarch64: '3b19fd3adcb8fe69eec06070a0d8cf257fa14602b3a13cf3bfb668b92edfa500', + armv7l: '3b19fd3adcb8fe69eec06070a0d8cf257fa14602b3a13cf3bfb668b92edfa500', + i686: '5e5cd17acfb4dafd4d847656d9a67f21e30a26bd5565c07fbfacbc80e8e5a27d', + x86_64: '8a16faced504313444a76c85940d484e599e2b93075f3fa5f917c0cf904e7618' }) depends_on 'gcc_lib' # R diff --git a/tools/build_updated_packages.rb b/tools/build_updated_packages.rb index 6c15c82cc..6430205a6 100755 --- a/tools/build_updated_packages.rb +++ b/tools/build_updated_packages.rb @@ -1,5 +1,5 @@ #!/usr/local/bin/ruby -# build_updated_packages version 3.9 (for Chromebrew) +# build_updated_packages version 4.0 (for Chromebrew) # This updates the versions in python pip packages by calling # tools/update_python_pip_packages.rb, checks for updated ruby packages # by calling tools/update_ruby_gem_packages.rb, and then checks if any @@ -15,11 +15,36 @@ require 'English' require 'fileutils' -require_relative '../lib/color' -require_relative '../lib/const' -require_relative '../lib/package' -require_relative '../lib/require_gem' -$LOAD_PATH.unshift File.expand_path('../lib', __dir__) + +crew_local_repo_root = `git rev-parse --show-toplevel 2> /dev/null`.chomp +# When invoked from crew, pwd is CREW_DEST_DIR, so crew_local_repo_root +# is empty. +crew_local_repo_root = '../' if crew_local_repo_root.to_s.empty? +require File.join(crew_local_repo_root, 'lib/color') +require File.join(crew_local_repo_root, 'lib/const') +require File.join(crew_local_repo_root, 'lib/package') +require File.join(crew_local_repo_root, 'lib/require_gem') +require_gem 'highline' +require_gem 'timeout' + +# Add >LOCAL< lib to LOAD_PATH +$LOAD_PATH.unshift File.join(crew_local_repo_root, 'lib') + +OPTIONS = %w[-h --help --check-all-python --check-all-ruby --continue-after-failed-builds --skip -v --verbose -vv] + +if ARGV.include?('-h') || ARGV.include?('--help') + abort <<~EOM + Usage: ./build_updated_packages.rb [] #{OPTIONS.to_s.gsub('"', '')} + Example: ./build_updated_packages.rb abcde -v + If is omitted, recently updated files will be checked for needed builds. + Passing --continue-after-failed-builds will continue if builds fail. + Passing --skip will skip update checks. + Passing --check-all-python will check py3_ pip packages for updates. + Passing --check-all-ruby will check ruby_ gem packages for updates. + Passing --verbose or -v will display verbose output. + Passing -vv will display very verbose output. + EOM +end abort "\nGITLAB_TOKEN environment variable not set.\n".lightred if ENV['GITLAB_TOKEN'].nil? abort "\nGITLAB_TOKEN_USERNAME environment variable not set.\n".lightred if ENV['GITLAB_TOKEN_USERNAME'].nil? @@ -29,8 +54,22 @@ CONTINUE_AFTER_FAILED_BUILDS = ARGV.include?('--continue-after-failed-builds') SKIP_UPDATE_CHECKS = ARGV.include?('--skip') CHECK_ALL_PYTHON = ARGV.include?('--check-all-python') CHECK_ALL_RUBY = ARGV.include?('--check-all-ruby') -require_gem 'highline' -require_gem 'timeout' +VERBOSE = ARGV.include?('-v') || ARGV.include?('--verbose') || ARGV.include?('-vv') +VERY_VERBOSE = ARGV.include?('-vv') + +updated_packages = [] +# Handle multiple packages being passed. +argv = ARGV.map(&:split).flatten +if argv.length.positive? && !(argv.length == 1 && OPTIONS.include?(argv[0])) + argv.each do |arg| + arg = arg.gsub('.rb', '') + next unless arg =~ /^[0-9a-zA-Z_*]+$/ + + filename = File.join(crew_local_repo_root, "packages/#{arg}.rb") + updated_packages.push "packages/#{arg}.rb" if File.exist?(filename) + puts "#{arg} has been added to the list of packages to be checked for updates.".lightblue + end +end build = {} @@ -39,7 +78,18 @@ excluded_packages = Set[ { pkg_name: 'py3_unsupported_python', commwnts: 'Stub package.' }, { pkg_name: 'terraform', comments: 'Needs manual update due to package structure.' } ] -excluded_pkgs = excluded_packages.map { |h| h[:pkg_name] } +excluded_pkgs = excluded_packages.map { it[:pkg_name] } + +dependent_packages_to_check = Set[ + { pkg_name: 'gcc_build', downstream_packages: 'gcc_lib', comments: '' }, + { pkg_name: 'gcc_lib', downstream_packages: 'gcc_dev', comments: '' }, + { pkg_name: 'gcc_dev', downstream_packages: 'libssp', comments: '' }, + { pkg_name: "#{CREW_LLVM_VER}_build", downstream_packages: "#{CREW_LLVM_VER}_lib", comments: '' }, + { pkg_name: "#{CREW_LLVM_VER}_lib", downstream_packages: "#{CREW_LLVM_VER}_dev", comments: '' }, + { pkg_name: "#{CREW_LLVM_VER}_dev", downstream_packages: 'openmp spirv_llvm_translator libclc', comments: '' }, + { pkg_name: 'spirv_llvm_translator', downstream_packages: 'libclc', comments: '' } +] +dependent_pkgs = dependent_packages_to_check.to_h { [it[:pkg_name], it[:downstream_packages]] } def self.agree_default_yes(message = nil) Timeout.timeout(CREW_AGREE_TIMEOUT_SECONDS) do @@ -56,16 +106,16 @@ def self.check_build_uploads(architectures_to_check = nil, name = nil) remote_binary = { armv7l: nil, i686: nil, x86_64: nil } remote_binary.keys.each do |arch| arch_specific_url = "#{CREW_GITLAB_PKG_REPO}/generic/#{name}/#{@pkg_obj.version}_#{arch}/#{name}-#{@pkg_obj.version}-chromeos-#{arch}.#{@pkg_obj.binary_compression}" - puts "Checking: curl -sI #{arch_specific_url}" if CREW_VERBOSE + puts "Checking: curl -sI #{arch_specific_url}" if VERBOSE remote_binary[arch.to_sym] = `curl -sI #{arch_specific_url}`.lines.first.split[1] == '200' - puts "#{arch_specific_url} found!" if remote_binary[arch.to_sym] && CREW_VERBOSE + puts "#{arch_specific_url} found!" if remote_binary[arch.to_sym] && VERBOSE end system "crew update_package_file #{name}" unless remote_binary.values.all?(nil) builds_needed = architectures_to_check.dup architectures_to_check.each do |arch| builds_needed.delete(arch) if remote_binary[arch.to_sym] - puts "builds_needed for #{name} is now #{builds_needed}" if CREW_VERBOSE + puts "builds_needed for #{name} is now #{builds_needed}" if VERBOSE end return builds_needed end @@ -75,9 +125,9 @@ def update_hashes_and_manifests(name = nil) remote_binary = { armv7l: nil, i686: nil, x86_64: nil } remote_binary.keys.each do |arch| arch_specific_url = "#{CREW_GITLAB_PKG_REPO}/generic/#{name}/#{@pkg_obj.version}_#{arch}/#{name}-#{@pkg_obj.version}-chromeos-#{arch}.#{@pkg_obj.binary_compression}" - puts "Checking: curl -sI #{arch_specific_url}" if CREW_VERBOSE + puts "Checking: curl -sI #{arch_specific_url}" if VERBOSE remote_binary[arch.to_sym] = `curl -sI #{arch_specific_url}`.lines.first.split[1] == '200' - puts "#{arch_specific_url} found!" if remote_binary[arch.to_sym] && CREW_VERBOSE + puts "#{arch_specific_url} found!" if remote_binary[arch.to_sym] && VERBOSE end # Add build hashes. system "crew update_package_file #{name}" unless remote_binary.values.all?(nil) @@ -117,17 +167,9 @@ else end changed_files = `git diff HEAD --name-only`.chomp.split changed_files_previous_commit = `git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/master)..$(git rev-parse --verify HEAD)`.chomp.split -updated_packages = changed_files.select { |c| c.include?('packages/') } +updated_packages.push(*changed_files.select { |c| c.include?('packages/') }) updated_packages.push(*changed_files_previous_commit.select { |c| c.include?('packages/') }) -if updated_packages.empty? - puts 'No packages need to be updated.'.orange -else - puts 'These changed packages will be checked to see if they need updated binaries:'.orange - updated_packages.uniq! - updated_packages.each { |p| puts p.sub('packages/', '').sub('.rb', '').to_s.lightblue } -end - crew_update_packages = `CREW_NO_GIT=1 CREW_UNATTENDED=1 crew update | grep "\\[\\"" | jq -r '.[]'`.chomp.split.map(&'packages/'.method(:+)).map { |i| i.concat('.rb') } if CHECK_ALL_PYTHON py_packages = `grep -l CREW_PY_VER packages/*`.chomp.split @@ -139,17 +181,38 @@ if CHECK_ALL_RUBY end updated_packages.push(*crew_update_packages) updated_packages.uniq! + # Remove packages that don't need to be checked for updates from the # check list. exclusion_regex = "(#{excluded_pkgs.join('|')})" updated_packages.delete_if { |d| /#{exclusion_regex}/.match(d) } +# Add additional packages that need to be checked for updates. +inclusion_regex = "(#{dependent_pkgs.keys.join('|')})" +updated_packages.each do |p| + pkg = p.sub('packages/', '').sub('.rb', '') + next unless /#{inclusion_regex}/.match(pkg) + dependent_pkgs[pkg].split.each do |dependent_pkg| + if File.exist?(File.join(crew_local_repo_root, "packages/#{dependent_pkg}.rb")) + puts "Will check #{dependent_pkg} since dependent package of #{pkg}...".orange if VERY_VERBOSE + updated_packages.push "packages/#{dependent_pkg}.rb" + end + end +end + +if updated_packages.empty? + puts 'No packages need to be updated.'.orange +else + puts 'These packages will be checked to see if they need updated binaries:'.orange + updated_packages.uniq! + updated_packages.each { |p| puts p.sub('packages/', '').sub('.rb', '').to_s.lightblue } +end + updated_packages.each do |pkg| - puts "pkg is #{pkg}" if CREW_VERBOSE name = pkg.sub('packages/', '').sub('.rb', '') next unless File.file?(pkg) - puts "Evaluating #{name} package...".orange + puts "Evaluating #{name} package...".lightpurple system "rubocop -c .rubocop.yml -A #{pkg}" @pkg_obj = Package.load_package(pkg)