From 723cfbf1befbae20ae9c5d9251a0b3ac5d8209a9 Mon Sep 17 00:00:00 2001 From: "Satadru Pramanik, DO, MPH, MEng" Date: Sat, 28 Jun 2025 12:42:19 -0400 Subject: [PATCH] Adjust crew update command's fixup usage. (#12118) * Adjust crew update command. Signed-off-by: Satadru Pramanik * fixup Signed-off-by: Satadru Pramanik --------- Signed-off-by: Satadru Pramanik --- bin/crew | 12 +++++------ lib/const.rb | 2 +- lib/fixup.rb | 60 ++++++++++++---------------------------------------- 3 files changed, 20 insertions(+), 54 deletions(-) diff --git a/bin/crew b/bin/crew index e6ae83aec..138d9377e 100755 --- a/bin/crew +++ b/bin/crew @@ -246,13 +246,11 @@ def update GIT_REPAIR_COMMANDS end - system(<<~GIT_UPDATE_COMMANDS, chdir: CREW_LIB_PATH, exception: true) - ## Update crew from git. - # Set sparse-checkout folders. - git sparse-checkout set packages manifest/#{ARCH} lib commands bin crew tests tools - git sparse-checkout reapply - git fetch #{CREW_REPO} #{CREW_BRANCH} - GIT_UPDATE_COMMANDS + ## Update crew from git. + # Set sparse-checkout folders. + system "git sparse-checkout set packages manifest/#{ARCH} lib commands bin crew tests tools", chdir: CREW_LIB_PATH, exception: true + system 'git sparse-checkout reapply', chdir: CREW_LIB_PATH, exception: true + system "git fetch #{CREW_REPO} #{CREW_BRANCH}", chdir: CREW_LIB_PATH, exception: true # Now that we've fetched all the new changes, see if lib/const.rb was changed. # We do this before resetting to FETCH_HEAD because we lose the original HEAD when doing so. to_update = `cd #{CREW_LIB_PATH} && git show --name-only HEAD..FETCH_HEAD`.include?('lib/const.rb') diff --git a/lib/const.rb b/lib/const.rb index ea03ecf66..8a0947384 100644 --- a/lib/const.rb +++ b/lib/const.rb @@ -4,7 +4,7 @@ require 'etc' require 'open3' OLD_CREW_VERSION ||= defined?(CREW_VERSION) ? CREW_VERSION : '1.0' -CREW_VERSION ||= '1.62.2' unless defined?(CREW_VERSION) && CREW_VERSION == OLD_CREW_VERSION +CREW_VERSION ||= '1.62.3' unless defined?(CREW_VERSION) && CREW_VERSION == OLD_CREW_VERSION # Kernel architecture. KERN_ARCH ||= Etc.uname[:machine] diff --git a/lib/fixup.rb b/lib/fixup.rb index f31204708..45d611101 100644 --- a/lib/fixup.rb +++ b/lib/fixup.rb @@ -1,8 +1,8 @@ # lib/fixup.rb # Add fixups to be run during crew update here. +require 'English' require 'etc' require 'json' -require 'open3' require_relative 'color' require_relative 'convenience_functions' require_relative 'package' @@ -290,53 +290,21 @@ end if File.exist?("#{CREW_PREFIX}/bin/upx") && File.exist?("#{CREW_PREFIX}/bin/patchelf") abort("No Upx found! Please run 'crew install upx'").lightred unless File.file?("#{CREW_PREFIX}/bin/upx") abort("No Patchelf found! Please run 'crew install patchelf'").lightred unless File.file?("#{CREW_PREFIX}/bin/patchelf") - puts "Running upx to uncompress binaries #{'and patchelf to patch binary interpreter paths ' unless CREW_PRE_GLIBC_STANDALONE}if needed.".lightblue - # Look for installed binaries and libraries in /usr/local and the lib - # prefix directories. - execfiles = Find.find("#{CREW_PREFIX}/bin", CREW_LIB_PREFIX).select { |p| File.executable?(p) } - return if execfiles.empty? + # Using xargs to spawn parallel processes since spawning parallel + # processes using ruby gems like concurrent ruby leads to out of memory + # errors on arm. + # Running find twice because it involves less ruby overhead than saving + # the output in memory, and also doing that in ruby is VERY SLOW. + puts 'Please wait while upx is run to uncompress binaries...'.lightblue + Kernel.system "#{CREW_PREFIX}/bin/find #{CREW_PREFIX}/bin -type f -print0 | xargs -0 -P#{CREW_NPROC} -n1 -r bash -c 'header=$(head -c4 ${0}); elfheader='$(printf '\\\177ELF')' ; arheader=\\!\\ File::NULL, exception: false - @localdir = File.expand_path('../../.') - require_gem 'concurrent-ruby' - pool = Concurrent::ThreadPoolExecutor.new( - min_threads: 1, - max_threads: CREW_NPROC, - max_queue: 0, # unbounded work queue - fallback_policy: :caller_runs - ) - execfiles.each do |execfiletopatch| - next unless File.file?(execfiletopatch) - - pool.post do - # Decompress the binary if compressed. - system "upx -qq -d #{execfiletopatch}", %i[err] => File::NULL, exception: false - - # Check for existing interpreter. - next if CREW_GLIBC_INTERPRETER.blank? - - @interpreter, _read_interpreter_stderr_s, @read_interpreter_status = Open3.capture3("patchelf --print-interpreter #{execfiletopatch}") - # Set interpreter unless the interpreter read failed or is already - # set appropriately. - unless @read_interpreter_status && @interpreter == CREW_GLIBC_INTERPRETER - puts "Running patchelf on #{execfiletopatch} to set interpreter".orange if CREW_VERBOSE - _set_interpreter_stdout, @set_interpreter_stderr = Open3.capture3("patchelf --set-interpreter #{CREW_GLIBC_INTERPRETER} #{execfiletopatch}") - puts "#{execfiletopatch}: @set_interpreter_stderr: #{@set_interpreter_stderr.chomp}".lightpurple if !@set_interpreter_stderr.blank? && CREW_VERBOSE - end - # Try to read any existing rpath. - @read_rpath_stdout_s, @read_rpath_stderr_s, @read_rpath_status = Open3.capture3("patchelf --print-rpath #{execfiletopatch}") - @exec_rpath = @read_rpath_stdout_s.chomp - @rpath_status = @read_rpath_status - puts "#{execfiletopatch}: @read_rpath_stderr_s: #{@read_rpath_stderr_s}".lightpurple if !@read_rpath_stderr_s.blank? && CREW_VERBOSE - # Set rpath if rpath read didn't fail, an rpath exists, and does not - # already contain CREW_GLIBC_PREFIX. - next if !@read_rpath_rpath_status || @exec_rpath.blank? || @exec_rpath.include?(CREW_GLIBC_PREFIX) - puts "#{execfiletopatch.gsub(@localdir, '')} has an existing rpath of #{@exec_rpath}".lightpurple if CREW_VERBOSE - puts "Prefixing #{CREW_GLIBC_PREFIX} to #{@exec_rpath} rpath for #{execfiletopatch.gsub(@localdir, '')}.".lightblue - @set_rpath_stdout_s, @set_rpath_stderr_s, @set_rpath_status = Open3.capture3("patchelf --set-rpath #{CREW_GLIBC_PREFIX}:#{@exec_rpath} #{execfiletopatch}") - puts "#{execfiletopatch}: @set_rpath_stderr_s: #{@set_rpath_stderr_s}".lightpurple if !@set_rpath_stderr_s.blank? && CREW_VERBOSE + unless CREW_GLIBC_INTERPRETER.blank? + puts 'Please wait while patchelf is run to patch binary interpreter and rpath if needed...'.lightblue + if CREW_VERBOSE + Kernel.system "#{CREW_PREFIX}/bin/find #{CREW_PREFIX}/bin #{CREW_LIB_PREFIX} -type f ! -name '*.a' ! -name '*.o' ! -name '*.gox' -print0 | xargs -0 -P#{CREW_NPROC} -n1 -r bash -c 'header=$(head -c4 ${0}); elfheader='$(printf '\\\177ELF')' ; arheader=\\!\\ File::NULL, exception: false + else + Kernel.system "#{CREW_PREFIX}/bin/find #{CREW_PREFIX}/bin #{CREW_LIB_PREFIX} -type f ! -name '*.a' ! -name '*.o' ! -name '*.gox' -print0 | xargs -0 -P#{CREW_NPROC} -n1 -r bash -c 'header=$(head -c4 ${0}); elfheader='$(printf '\\\177ELF')' ; arheader=\\!\\ File::NULL, exception: false end - pool.shutdown - pool.wait_for_termination end else abort 'Please install upx and patchelf first by running \'crew install upx patchelf\'.'.lightred