diff --git a/.github/workflows/Unit-Test.yml b/.github/workflows/Unit-Test.yml index 490079258..017e472f4 100644 --- a/.github/workflows/Unit-Test.yml +++ b/.github/workflows/Unit-Test.yml @@ -25,9 +25,9 @@ jobs: ALL_CHANGED_FILES: ${{ steps.changed-ruby-files.outputs.all_changed_files }} run: | export PR_NUMBER="${{ github.event.pull_request.number || github.event.issue.number }}" && \ - echo "ALL_CHANGED FILES is/are ${ALL_CHANGED_FILES}." && \ + echo "ALL_CHANGED FILES: ${ALL_CHANGED_FILES}." && \ export CHANGED_PACKAGES="$(echo ${ALL_CHANGED_FILES} | sed -e 's,packages/,,g' -e 's,\.rb,,g' | sort)" && \ - echo "CHANGED PACKAGES is/are ${CHANGED_PACKAGES}." && \ + echo "CHANGED PACKAGES: ${CHANGED_PACKAGES}." && \ export GLIBC_227_COMPATIBLE_PACKAGES="$(for i in ${CHANGED_PACKAGES} ; do if ! grep -q min_glibc packages/$i.rb; then echo $i ; else grep min_glibc packages/$i.rb | tr -d \' | awk '{exit $2 < 2.27}' || echo $i ; fi ; done | xargs)" && \ echo "PR #${PR_NUMBER} has these Glibc 2.27 compatible packages: ${GLIBC_227_COMPATIBLE_PACKAGES}" && \ export GLIBC_237_COMPATIBLE_PACKAGES="$(for i in ${CHANGED_PACKAGES} ; do grep min_glibc packages/$i.rb | tr -d \' | awk '{exit $2 < 2.37}' || echo $i ; done | xargs)" && \ @@ -39,8 +39,8 @@ jobs: if [[ $GLIBC_237_COMPATIBLE_PACKAGES ]] ; then x86_64_CONTAINER="satmandu/crewbuild:hatch-x86_64.m126"; else x86_64_CONTAINER="satmandu/crewbuild:nocturne-x86_64.m90"; fi &&\ export container_cmdline="/usr/bin/sudo -u chronos LD_LIBRARY_PATH=/usr/local/lib64:/lib64 /usr/local/bin/bash -c \"export ALL_CHANGED_FILES='${ALL_CHANGED_FILES}' && \ export CHANGED_PACKAGES='${CHANGED_PACKAGES}' && \ - echo 'ALL_CHANGED FILES is/are ${ALL_CHANGED_FILES}.' && \ - echo 'CHANGED PACKAGES is/are ${CHANGED_PACKAGES}.' && \ + echo 'ALL_CHANGED FILES: ${ALL_CHANGED_FILES}.' && \ + echo 'CHANGED PACKAGES: ${CHANGED_PACKAGES}.' && \ echo 'CREW_REPO is ${{ github.event.pull_request.head.repo.clone_url }}' && \ echo 'CREW_BRANCH is ${{ github.head_ref }}' && \ CREW_REPO=${{ github.event.pull_request.head.repo.clone_url }} CREW_BRANCH=${{ github.head_ref }} crew update && \ @@ -58,9 +58,9 @@ jobs: ALL_CHANGED_FILES: ${{ steps.changed-ruby-files.outputs.all_changed_files }} run: | export PR_NUMBER="${{ github.event.pull_request.number || github.event.issue.number }}" && \ - echo "ALL_CHANGED FILES is/are ${ALL_CHANGED_FILES}." && \ + echo "ALL_CHANGED FILES: ${ALL_CHANGED_FILES}." && \ export CHANGED_PACKAGES="$(echo ${ALL_CHANGED_FILES} | sed -e 's,packages/,,g' -e 's,\.rb,,g' | sort)" && \ - echo "CHANGED PACKAGES is/are ${CHANGED_PACKAGES}." && \ + echo "CHANGED PACKAGES: ${CHANGED_PACKAGES}." && \ export GLIBC_227_COMPATIBLE_PACKAGES="$(for i in ${CHANGED_PACKAGES} ; do if ! grep -q min_glibc packages/$i.rb; then echo $i ; else grep min_glibc packages/$i.rb | tr -d \' | awk '{exit $2 < 2.27}' || echo $i ; fi ; done | xargs)" && \ echo "PR #${PR_NUMBER} has these Glibc 2.27 compatible packages: ${GLIBC_227_COMPATIBLE_PACKAGES}" && \ export GLIBC_237_COMPATIBLE_PACKAGES="$(for i in ${CHANGED_PACKAGES} ; do grep min_glibc packages/$i.rb | tr -d \' | awk '{exit $2 < 2.37}' || echo $i ; done | xargs)" && \ @@ -73,8 +73,8 @@ jobs: if [[ $GLIBC_237_COMPATIBLE_PACKAGES ]] ; then armv7l_CONTAINER="satmandu/crewbuild:strongbad-armv7l.m126"; else armv7l_CONTAINER="satmandu/crewbuild:fievel-armv7l.m91"; fi &&\ export container_cmdline="/usr/bin/sudo -u chronos LD_LIBRARY_PATH=/usr/lib:/lib /usr/local/bin/bash -c \"export ALL_CHANGED_FILES='${ALL_CHANGED_FILES}' && \ export CHANGED_PACKAGES='${CHANGED_PACKAGES}' && \ - echo 'ALL_CHANGED FILES is/are ${ALL_CHANGED_FILES}.' && \ - echo 'CHANGED PACKAGES is/are ${CHANGED_PACKAGES}.' && \ + echo 'ALL_CHANGED FILES: ${ALL_CHANGED_FILES}.' && \ + echo 'CHANGED PACKAGES: ${CHANGED_PACKAGES}.' && \ echo 'CREW_REPO is ${{ github.event.pull_request.head.repo.clone_url }}' && \ echo 'CREW_BRANCH is ${{ github.head_ref }}' && \ CREW_REPO=${{ github.event.pull_request.head.repo.clone_url }} CREW_BRANCH=${{ github.head_ref }} crew update && \ diff --git a/bin/crew b/bin/crew index 9d7a62231..f5ce957d3 100755 --- a/bin/crew +++ b/bin/crew @@ -143,29 +143,6 @@ at_exit do ExitMessage.print end -def load_json - # load_json(): (re)load device.json - json_path = File.join(CREW_CONFIG_PATH, 'device.json') - @device = JSON.load_file(json_path, symbolize_names: true) - - # symbolize also values - @device.transform_values! { |val| val.is_a?(String) ? val.to_sym : val } -end - -def save_json(json_object) - crewlog 'Saving device.json...' - begin - File.write File.join(CREW_CONFIG_PATH, 'device.json.tmp'), JSON.pretty_generate(JSON.parse(json_object.to_json)) - rescue StandardError - puts 'Error writing updated packages json file!'.lightred - abort - end - - # Copy over original if the write to the tmp file succeeds. - FileUtils.cp("#{CREW_CONFIG_PATH}/device.json.tmp", File.join(CREW_CONFIG_PATH, 'device.json')) && FileUtils.rm("#{CREW_CONFIG_PATH}/device.json.tmp") - load_json -end - def print_current_package(extra = false) status = if PackageUtils.installed?(@pkg.name) :installed @@ -205,7 +182,7 @@ def set_package(pkg_path) end def generate_compatible - puts 'Generating compatible packages...'.orange if CREW_VERBOSE + puts 'Determining package compatibility...'.orange if CREW_VERBOSE @device[:compatible_packages] = [] Dir["#{CREW_PACKAGES_PATH}/*.rb"].each do |filename| pkg_name = File.basename filename, '.rb' @@ -223,11 +200,12 @@ def generate_compatible puts "#{pkg_name} is not a compatible package.".lightred end end - File.open(File.join(CREW_CONFIG_PATH, 'device.json'), 'w') do |file| - output = JSON.parse @device.to_json - file.write JSON.pretty_generate(output) - end - puts 'Generating compatible packages done.'.orange if CREW_VERBOSE + puts 'Determining essential dependencies from CREW_ESSENTIAL_PACKAGES...'.orange if CREW_VERBOSE + @device[:essential_deps] = [] + @device[:essential_deps].concat(CREW_ESSENTIAL_PACKAGES.flat_map { |i| Package.load_package("#{i}.rb").get_deps_list }.push(*CREW_ESSENTIAL_PACKAGES).uniq.sort) + crewlog "Essential packages: #{@device[:essential_deps]}" + PackageUtils.save_json(@device) + puts 'Determined compatibility & which packages are essential.'.orange if CREW_VERBOSE end def search(pkg_name, pkg_path: File.join(CREW_PACKAGES_PATH, "#{pkg_name}.rb"), silent: false) @@ -715,7 +693,7 @@ def pre_install(dest_dir) @pkg.preinstall # Reload device.json in case preinstall modified it via # running 'crew remove packages...' - load_json + @device = PackageUtils.load_json end end @@ -1303,7 +1281,7 @@ def install # remove it just before the file copy if @pkg.in_upgrade - puts 'Removing since upgrade or reinstall...' + puts 'Attempting removal since this is an upgrade or reinstall...' Command.remove(@pkg, CREW_VERBOSE) end @@ -1335,7 +1313,7 @@ def install # Add to installed packages list in devices.json, but remove first if it is already there. crewlog "Adding package #{@pkg.name} to device.json." @device[:installed_packages].delete_if { |entry| entry[:name] == @pkg.name } and @device[:installed_packages].push(name: @pkg.name, version: @pkg.version, sha256: PackageUtils.get_sha256(@pkg, build_from_source: @opt_source)) - save_json(@device) + PackageUtils.save_json(@device) crewlog "#{@pkg.name} in device.json after install: #{`jq --arg key '#{@pkg.name}' -e '.installed_packages[] | select(.name == $key )' #{File.join(CREW_CONFIG_PATH, 'device.json')}`}" if Kernel.system 'which jq', %i[out err] => File::NULL end @@ -1430,81 +1408,6 @@ def archive_package(crew_archive_dest) end end -def print_deps_tree(args) - warn 'Walking through dependencies recursively, this may take a while...', '' - - # dep_hash: Hash object returned by @pkg.get_deps_list - dep_hash = @pkg.get_deps_list(hash: true, include_build_deps: args['--include-build-deps'] || 'auto', exclude_buildessential: args['--exclude-buildessential']) - - # convert returned hash to json and format it - json_view = JSON.pretty_generate(dep_hash) - - # convert formatted json string to tree structure - tree_view = json_view.gsub(/\{\s*/m, '└─────').gsub(/[\[\]{},":]/, '').gsub(/^\s*$\n/, '').gsub(/\s*$/, '') - - # add pipe char to connect endpoints and starting points, improve readability - # find the horizontal location of all arrow symbols - index_with_pipe_char = tree_view.lines.map { |line| line.index('└') }.compact.uniq - - # determine whatever a pipe char should be added according to the horizontal location of arrow symbols - tree_view = tree_view.lines.each_with_index.map do |line, line_i| - index_with_pipe_char.each do |char_i| - # check if there have any non-space char (pkg_names) between starting point ([line_i][char_i]) and endpoint vertically ([next_arrow_line_offset][char_i]) - # (used to determine if the starting point and endpoint are in same branch, use pipe char to connect them if true) - next_arrow_line_offset = tree_view.lines[line_i..].index { |l| l[char_i] == '└' } - have_line_with_non_empty_char = tree_view.lines[line_i + 1..line_i + next_arrow_line_offset.to_i - 1].any? { |l| l[char_i].nil? or l[char_i] =~ /\S/ } - - line[char_i] = '│' if next_arrow_line_offset && (line[char_i] == ' ') && !have_line_with_non_empty_char - end - next line - end.join - - # replace arrow symbols with a tee symbol on branch intersection - tree_view = tree_view.lines.each_with_index.map do |line, line_i| - # orig_arrow_index_connecter: the horizontal location of the arrow symbol used to connect parent branch - # - # example: - # └───┬─chrome - # └─────buildessential - # ^ - orig_arrow_index_connecter = line.index('└') - # orig_arrow_index_newbranch: the horizontal location of the "box drawing char" symbol MIGHT be - # required to convert to tee char in order to connect child branch, - # located at 3 chars later of orig_arrow_index_connecter - # - # example: - # v - # └─────chrome - # └─────buildessential - # - # which might need to be convert to: - # └───┬─chrome - # └─────buildessential - orig_arrow_index_newbranch = orig_arrow_index_connecter + 4 - - # if the char under the processing arrow symbol (orig_arrow_index_connecter) is also arrow or pipe, change the processing char to tee symbol - line[orig_arrow_index_connecter] = '├' if orig_arrow_index_connecter && tree_view.lines[line_i + 1].to_s[orig_arrow_index_connecter] =~ (/[└│]/) - # if the char under the processing arrow symbol (orig_arrow_index_newbranch) is also arrow or pipe, change the processing char to tee symbol - line[orig_arrow_index_newbranch] = '┬' if orig_arrow_index_newbranch && tree_view.lines[line_i + 1].to_s[orig_arrow_index_newbranch] =~ (/[└├]/) - next line # return modified line - end.join - - if String.use_color - puts <<~EOT, '' - \e[45m \e[0m: satisfied dependency - \e[46m \e[0m: build dependency - \e[47m \e[0m: runtime dependency - EOT - # (the first string in each #{} is used for commenting only, will not be included in output) - - # replace special symbols returned by @pkg.get_deps_list to actual color code - tree_view.gsub!(/\*(.+)\*/, '\1'.lightcyan) - tree_view.gsub!(/\+(.+)\+/, "\e[45m\\1\e[0m") - end - - puts tree_view -end - def upload(pkg_name = nil, pkg_version = nil, gitlab_token = nil, binary_compression = nil) abort "\nPackage to be uploaded was not specified.\n".lightred if pkg_name.nil? abort "\nGITLAB_TOKEN environment variable not set.\n".lightred if gitlab_token.nil? @@ -1751,7 +1654,7 @@ def deps_command(args) if args['--tree'] # call `print_deps_tree` (print dependency tree) if --tree is specified - print_deps_tree(args) + @pkg.print_deps_tree(args) elsif args['--deep'] system "#{CREW_LIB_PATH}/tools/getrealdeps.rb #{name}" else @@ -1806,7 +1709,7 @@ def install_command(args) end def list_command(args) - Command.list(args['available'], args['installed'], args['compatible'], args['incompatible'], CREW_VERBOSE) + Command.list(args['available'], args['compatible'], args['incompatible'], args['essential'], args['installed'], CREW_VERBOSE) end def postinstall_command(args) @@ -1922,9 +1825,9 @@ Signal.trap('INT') do exit 1 end -load_json +@device = PackageUtils.load_json -@last_update_check = File.file?(File.join(CREW_LIB_PATH, '.git/FETCH_HEAD')) ? `stat -c %Y #{File.join(CREW_LIB_PATH, '.git/FETCH_HEAD')}`.chomp.to_i : `stat -c %Y #{File.join(CREW_LIB_PATH, 'lib/const.rb')}`.chomp.to_i +@last_update_check = Dir["#{CREW_LIB_PATH}/{.git/FETCH_HEAD,lib/const.rb}"].compact.map { |i| File.mtime(i).utc.to_i }.max crewlog("The last update was #{time_difference(@last_update_check, Time.now.to_i)} ago.") puts "It has been more than #{CREW_UPDATE_CHECK_INTERVAL} day#{CREW_UPDATE_CHECK_INTERVAL < 2 ? '' : 's'} since crew was last updated. Please run 'crew update'".lightpurple if Time.now.to_i - @last_update_check > (CREW_UPDATE_CHECK_INTERVAL * 3600 * 24) command_name = args.select { |k, v| v && command?(k) }.keys[0] diff --git a/commands/list.rb b/commands/list.rb index 74151b031..7342ef31d 100644 --- a/commands/list.rb +++ b/commands/list.rb @@ -6,7 +6,7 @@ require_relative '../lib/package' require_relative '../lib/package_utils' class Command - def self.list(available, installed, compatible, incompatible, verbose) + def self.list(available, compatible, incompatible, essential, installed, verbose) device_json = JSON.load_file(File.join(CREW_CONFIG_PATH, 'device.json'), symbolize_names: true) installed_packages = {} device_json[:installed_packages].each do |package| @@ -20,19 +20,6 @@ class Command pkg = Package.load_package(filename) puts pkg_name if PackageUtils.compatible?(pkg) end - elsif installed - if verbose - installed_packages['======='] = '=======' - installed_packages['Package'] = 'Version' - first_col_width = installed_packages.keys.max { |a, b| a.size <=> b.size }.size - installed_packages.sort.to_h.each do |package, version| - puts "#{package.ljust(first_col_width)} #{version}".lightgreen - end - else - installed_packages.each_key do |package| - puts package.lightgreen - end - end elsif compatible Dir["#{CREW_PACKAGES_PATH}/*.rb"].each do |filename| pkg_name = File.basename(filename, '.rb') @@ -46,6 +33,21 @@ class Command pkg = Package.load_package(filename) puts pkg_name.lightred unless PackageUtils.compatible?(pkg) end + elsif essential + puts device_json[:essential_deps].join("\n") + elsif installed + if verbose + installed_packages['======='] = '=======' + installed_packages['Package'] = 'Version' + first_col_width = installed_packages.keys.max { |a, b| a.size <=> b.size }.size + installed_packages.sort.to_h.each do |package, version| + puts "#{package.ljust(first_col_width)} #{version}".lightgreen + end + else + installed_packages.each_key do |package| + puts package.lightgreen + end + end end end end diff --git a/commands/remove.rb b/commands/remove.rb index 5434208f5..70cefb985 100644 --- a/commands/remove.rb +++ b/commands/remove.rb @@ -1,11 +1,11 @@ require 'fileutils' -require 'json' require_relative '../lib/const' +require_relative '../lib/package' require_relative '../lib/package_utils' class Command def self.remove(pkg, verbose) - device_json = JSON.load_file(File.join(CREW_CONFIG_PATH, 'device.json')) + device_json = PackageUtils.load_json # Make sure the package is actually installed before we attempt to remove it. unless PackageUtils.installed?(pkg.name) @@ -13,10 +13,23 @@ class Command return end - # Don't remove any of the packages ruby (and thus crew) needs to run. - if CREW_ESSENTIAL_PACKAGES.include?(pkg.name) - puts "Refusing to remove essential package #{pkg.name}.".lightred - return + # Determine dependencies of packages in CREW_ESSENTIAL_PACKAGES and + # their dependencies, as those are needed for ruby and crew to run, + # and thus should not be removed. + # essential_deps = recursive_deps(CREW_ESSENTIAL_PACKAGES) + essential_deps = device_json[:essential_deps] + crewlog "Essential Deps are #{essential_deps}." + if essential_deps.include?(pkg.name) + return if pkg.in_upgrade + + puts <<~ESSENTIAL_PACKAGE_WARNING_EOF.gsub(/^(?=\w)/, ' ').lightred + #{pkg.name.capitalize} is considered an essential package needed for + Chromebrew to function and thus cannot be removed. + ESSENTIAL_PACKAGE_WARNING_EOF + + # Exit with failure if attempt to remove an essential package + # is made. + exit 1 end # Perform any operations required prior to package removal. @@ -34,17 +47,40 @@ class Command # Remove the files and directories installed by the package. unless pkg.is_fake? Dir.chdir CREW_CONFIG_PATH do - # Remove all files installed by the package. + # Remove all files installed by the package in CREW_PREFIX and + # HOME. + # Exceptions: + # 1. The file exists in another installed package. + # 2. The file is in one of the filelists for packages in + # CREW_ESSENTIAL_FILES, or a dependendent package of + # CREW_ESSENTIAL_PACKAGES. flist = File.join(CREW_META_PATH, "#{pkg.name}.filelist") if File.file?(flist) - File.foreach(flist, chomp: true) do |line| - next unless line.start_with?(CREW_PREFIX) - if system("grep --exclude #{pkg.name}.filelist -Fxq '#{line}' ./meta/*.filelist") - puts "#{line} is in another package. It will not be removed during the removal of #{pkg.name}.".orange - else - puts "Removing file #{line}".yellow if verbose - FileUtils.remove_file line, exception: false - end + # When searching for files to delete we exclude the files from + # all packages and dependent packages of CREW_ESSENTIAL_PACKAGES. + essential_deps_exclude_froms = essential_deps.map { |i| File.file?("#{File.join(CREW_META_PATH, i.to_s)}.filelist") ? "--exclude-from=#{File.join(CREW_META_PATH, i.to_s)}.filelist" : '' }.join(' ') + + # When making a list of all files from crew filelists we again + # ignore all files from packages and dependent packages of + # CREW_ESSENTIAL_PACKAGES. + essential_deps_excludes = essential_deps.map { |i| File.file?("#{File.join(CREW_META_PATH, i.to_s)}.filelist") ? "--exclude=#{File.join(CREW_META_PATH, i.to_s)}.filelist" : '' }.join(' ') + + package_files = `grep -h #{essential_deps_exclude_froms} \"^#{CREW_PREFIX}\\|^#{HOME}\" #{flist}`.split("\n").uniq.sort + all_other_files = `grep -h --exclude #{pkg.name}.filelist #{essential_deps_excludes} \"^#{CREW_PREFIX}\\|^#{HOME}\" #{CREW_META_PATH}/*.filelist`.split("\n").uniq.sort + + # We want the difference of these arrays. + unique_to_package_files = package_files - all_other_files + + # We want the intersection of these arrays. + package_files_that_overlap = all_other_files & package_files + + unless package_files_that_overlap.empty? + puts "The following file(s) in other packages will not be deleted during the removal of #{pkg.name}:".orange + puts package_files_that_overlap.join("\n").orange + end + unique_to_package_files.each do |file| + puts "Removing file #{file}".yellow if CREW_VERBOSE + FileUtils.remove_file file, exception: false end FileUtils.remove_file flist end @@ -64,10 +100,10 @@ class Command # Remove the package from the list of installed packages in device.json. puts "Removing package #{pkg.name} from device.json".yellow if verbose - device_json['installed_packages'].delete_if { |entry| entry['name'] == pkg.name } + device_json[:installed_packages].delete_if { |entry| entry[:name] == pkg.name } # Update device.json with our changes. - save_json(device_json) + PackageUtils.save_json(device_json) # Perform any operations required after package removal. pkg.postremove diff --git a/install.sh b/install.sh index 6c5f5b2dd..bbc9b7e4a 100755 --- a/install.sh +++ b/install.sh @@ -83,6 +83,7 @@ fi # Chromebrew directories. CREW_LIB_PATH="${CREW_PREFIX}/lib/crew" CREW_CONFIG_PATH="${CREW_PREFIX}/etc/crew" +CREW_META_PATH="${CREW_CONFIG_PATH}/meta" CREW_BREW_DIR="${CREW_PREFIX}/tmp/crew" CREW_DEST_DIR="${CREW_BREW_DIR}/dest" : "${CREW_CACHE_DIR:=$CREW_PREFIX/tmp/packages}" @@ -261,8 +262,8 @@ function extract_install () { echo_intra "Installing ${1} ..." tar cpf - ./*/* | (cd /; tar xp --keep-directory-symlink -m -f -) - mv ./dlist "${CREW_CONFIG_PATH}/meta/${1}.directorylist" - mv ./filelist "${CREW_CONFIG_PATH}/meta/${1}.filelist" + mv ./dlist "${CREW_META_PATH}/${1}.directorylist" + mv ./filelist "${CREW_META_PATH}/${1}.filelist" } function update_device_json () { @@ -278,12 +279,15 @@ function install_ruby_gem () { gem update -N --system for gem in "$@"; do ruby_gem="${gem}" - echo_intra "Installing ${ruby_gem} gem..." + echo_intra "Installing ${ruby_gem^} gem..." gem install -N "${ruby_gem}" --conservative gem_version="$(ruby -e "gem('${ruby_gem}')" -e "puts Gem.loaded_specs['${ruby_gem}'].version.to_s")" json_gem_version="${gem_version}-ruby-${rubymajorversion}" crew_gem_package="ruby_${ruby_gem//-/_}" update_device_json "${crew_gem_package}" "${json_gem_version}" "" + gem_filelist_path="${CREW_META_PATH}/${crew_gem_package}.filelist" + echo_intra "Saving ${ruby_gem^} filelist..." + gem contents "${ruby_gem}" > "${gem_filelist_path}" echo_success "${ruby_gem^} gem installed." BOOTSTRAP_PACKAGES+=" ${crew_gem_package}" done @@ -358,14 +362,15 @@ else # Make the git default branch error messages go away. git config --global init.defaultBranch main - # Help handle situations where GitHub is down. - git config --local http.lowSpeedLimit 1000 - git config --local http.lowSpeedTime 5 # Setup the folder with git information. git init --ref-format=reftable git remote add origin "https://github.com/${OWNER}/${REPO}" + # Help handle situations where GitHub is down. + git config --local http.lowSpeedLimit 1000 + git config --local http.lowSpeedTime 5 + # Checkout, overwriting local files. git fetch --all git checkout -f "${BRANCH}" diff --git a/lib/const.rb b/lib/const.rb index 6deba99e5..c2d1995fe 100644 --- a/lib/const.rb +++ b/lib/const.rb @@ -2,7 +2,7 @@ # Defines common constants used in different parts of crew require 'etc' -CREW_VERSION = '1.50.3' +CREW_VERSION = '1.50.4' # Kernel architecture. KERN_ARCH = Etc.uname[:machine] @@ -307,25 +307,25 @@ CREW_DOCOPT = <<~DOCOPT Usage: crew build [options] [-k|--keep] [-v|--verbose] ... - crew check [-V|--version] [-v|--verbose] ... - crew const [-v|--verbose] [ ...] + crew check [options] [-V|--version] [-v|--verbose] ... + crew const [options] [-v|--verbose] [ ...] crew deps [options] [--deep] [-t|--tree] [-b|--include-build-deps] [--exclude-buildessential] [-v|--verbose] ... crew download [options] [-s|--source] [-v|--verbose] ... - crew files ... - crew help [] [-v|--verbose] [] + crew files [options] ... + crew help [options] [] [-v|--verbose] [] crew install [options] [-k|--keep] [-s|--source] [-S|--recursive-build] [-v|--verbose] ... - crew list [options] [-v|--verbose] (available|installed|compatible|incompatible) + crew list [options] [-v|--verbose] (available|compatible|incompatible|essential|installed) crew postinstall [options] [-v|--verbose] ... - crew prop [] + crew prop [options] [] crew reinstall [options] [-k|--keep] [-s|--source] [-S|--recursive-build] [-v|--verbose] ... - crew remove [-v|--verbose] ... - crew search [-v|--verbose] ... - crew sysinfo [-v|--verbose] - crew test [-v|--verbose] [ ...] + crew remove [options] [-v|--verbose] ... + crew search [options] [-v|--verbose] ... + crew sysinfo [options] [-v|--verbose] + crew test [options] [-v|--verbose] [ ...] crew update [options] [-v|--verbose] [] crew upgrade [options] [-k|--keep] [-s|--source] [-v|--verbose] [ ...] crew upload [options] [-v|--verbose] [ ...] - crew whatprovides ... + crew whatprovides [options] ... -b --include-build-deps Include build dependencies in output. -t --tree Print dependencies in a tree-structure format. @@ -340,8 +340,6 @@ CREW_DOCOPT = <<~DOCOPT -V --version Display the crew version. -h --help Show this screen. -D --debug Enable debugging. - - version #{CREW_VERSION} DOCOPT # All available crew commands. diff --git a/lib/package.rb b/lib/package.rb index 4e63fb55e..595080d28 100644 --- a/lib/package.rb +++ b/lib/package.rb @@ -1,4 +1,5 @@ require 'English' +require 'json' require_relative 'const' require_relative 'color' require_relative 'package_helpers' @@ -36,7 +37,10 @@ class Package pkg_name = File.basename(pkg_file, '.rb') class_name = pkg_name.capitalize - # read and eval package script under 'Package' class + # Read and eval package script under 'Package' class, using the + # newest file available. + pkg_file = Dir["{#{CREW_LOCAL_REPO_ROOT}/packages,#{CREW_PACKAGES_PATH}}/#{pkg_name}.rb"].max { |a, b| File.mtime(a) <=> File.mtime(b) } + class_eval(File.read(pkg_file, encoding: Encoding::UTF_8), pkg_file) unless const_defined?("Package::#{class_name}") pkg_obj = const_get(class_name) @@ -48,9 +52,9 @@ class Package end def self.dependencies - # We need instance variable in derived class, so not define it here, - # base class. Instead of define it, we initialize it in a function - # called from derived classees. + # We need instance variable in derived class, so do not define it here, + # base class. Instead of defining it, we initialize it in a function + # called from derived classes. @dependencies ||= {} end @@ -76,14 +80,14 @@ class Package # @checked_list ||= {} # create @checked_list placeholder if not exist - # add current package to @checked_list for preventing extra checks + # Add current package to @checked_list for preventing extra checks. @checked_list.merge!({ pkg_name => pkg_tags }) pkg_obj = load_package(File.join(CREW_PACKAGES_PATH, "#{pkg_name}.rb")) is_source = pkg_obj.source?(ARCH.to_sym) or pkg_obj.build_from_source deps = pkg_obj.dependencies - # append buildessential to deps if building from source is needed/specified + # Append buildessential to deps if building from source is needed/specified. if ((include_build_deps == true) || ((include_build_deps == 'auto') && is_source)) && \ !pkg_obj.no_compile_needed? && \ !exclude_buildessential && \ @@ -92,30 +96,30 @@ class Package deps = { 'buildessential' => [[:build]] }.merge(deps) end - # parse dependencies recursively + # Parse dependencies recursively. expanded_deps = deps.uniq.map do |dep, (dep_tags, ver_check)| - # check build dependencies only if building from source is needed/specified + # Check build dependencies only if building from source is needed/specified. # Do not recursively find :build based build dependencies. next unless (include_build_deps == true && @crew_current_package == pkg_obj.name) || \ ((include_build_deps == 'auto') && is_source && @crew_current_package == pkg_obj.name) || \ !dep_tags.include?(:build) - # overwrite tags if parent dependency is a build dependency + # Overwrite tags if parent dependency is a build dependency. # (for build dependencies highlighting) tags = pkg_tags.include?(:build) ? pkg_tags : dep_tags if @checked_list.keys.none?(dep) - # check dependency by calling this function recursively + # Check dependency by calling this function recursively. next \ send( __method__, dep, pkg_tags: tags, ver_check:, include_self: true, top_level: false, hash:, return_attr:, include_build_deps:, highlight_build_deps:, exclude_buildessential: ) elsif hash && top_level - # will be dropped here if current dependency is already checked and #{top_level} is set to true + # Will be dropped here if current dependency is already checked and #{top_level} is set to true. # - # the '+' symbol tell `print_deps_tree` (`bin/crew`) to color this package as "satisfied dependency" - # the '*' symbol tell `print_deps_tree` (`bin/crew`) to color this package as "build dependency" + # The '+' symbol tell `print_deps_tree` (`bin/crew`) to color this package as a "satisfied dependency". + # The '*' symbol tell `print_deps_tree` (`bin/crew`) to color this package as a "build dependency". if highlight_build_deps && tags.include?(:build) next { "+*#{dep}*+" => [] } elsif highlight_build_deps @@ -127,54 +131,129 @@ class Package end.compact if hash - # the '*' symbol tell `print_deps_tree` (`bin/crew`) to color this package as "build dependency" + # The '*' symbol tell `print_deps_tree` (`bin/crew`) to color this package as a "build dependency". if highlight_build_deps && pkg_tags.include?(:build) return { "*#{pkg_name}*" => expanded_deps } else return { pkg_name => expanded_deps } end elsif include_self - # return pkg_name itself if this function is called as a recursive loop (see `expanded_deps`) + # Return pkg_name itself if this function is called as a recursive loop (see `expanded_deps`). if return_attr return [expanded_deps, { pkg_name => [pkg_tags, ver_check] }].flatten else return [expanded_deps, pkg_name].flatten end else - # if this function is called outside of this function, return parsed dependencies only + # If this function is called outside of this function, return parsed dependencies only. return expanded_deps.flatten end end + def self.print_deps_tree(args) + warn 'Walking through dependencies recursively, this may take a while...', '' + + # dep_hash: Hash object returned by @pkg.get_deps_list + dep_hash = get_deps_list(hash: true, include_build_deps: args['--include-build-deps'] || 'auto', exclude_buildessential: args['--exclude-buildessential']) + + # convert returned hash to json and format it + json_view = JSON.pretty_generate(dep_hash) + + # convert formatted json string to tree structure + tree_view = json_view.gsub(/\{\s*/m, '└─────').gsub(/[\[\]{},":]/, '').gsub(/^\s*$\n/, '').gsub(/\s*$/, '') + + # add pipe char to connect endpoints and starting points, improve readability + # find the horizontal location of all arrow symbols + index_with_pipe_char = tree_view.lines.map { |line| line.index('└') }.compact.uniq + + # determine whatever a pipe char should be added according to the horizontal location of arrow symbols + tree_view = tree_view.lines.each_with_index.map do |line, line_i| + index_with_pipe_char.each do |char_i| + # check if there have any non-space char (pkg_names) between starting point ([line_i][char_i]) and endpoint vertically ([next_arrow_line_offset][char_i]) + # (used to determine if the starting point and endpoint are in same branch, use pipe char to connect them if true) + next_arrow_line_offset = tree_view.lines[line_i..].index { |l| l[char_i] == '└' } + have_line_with_non_empty_char = tree_view.lines[line_i + 1..line_i + next_arrow_line_offset.to_i - 1].any? { |l| l[char_i].nil? or l[char_i] =~ /\S/ } + + line[char_i] = '│' if next_arrow_line_offset && (line[char_i] == ' ') && !have_line_with_non_empty_char + end + next line + end.join + + # replace arrow symbols with a tee symbol on branch intersection + tree_view = tree_view.lines.each_with_index.map do |line, line_i| + # orig_arrow_index_connecter: the horizontal location of the arrow symbol used to connect parent branch + # + # example: + # └───┬─chrome + # └─────buildessential + # ^ + orig_arrow_index_connecter = line.index('└') + # orig_arrow_index_newbranch: the horizontal location of the "box drawing char" symbol MIGHT be + # required to convert to tee char in order to connect child branch, + # located at 3 chars later of orig_arrow_index_connecter + # + # example: + # v + # └─────chrome + # └─────buildessential + # + # which might need to be convert to: + # └───┬─chrome + # └─────buildessential + orig_arrow_index_newbranch = orig_arrow_index_connecter + 4 + + # if the char under the processing arrow symbol (orig_arrow_index_connecter) is also arrow or pipe, change the processing char to tee symbol + line[orig_arrow_index_connecter] = '├' if orig_arrow_index_connecter && tree_view.lines[line_i + 1].to_s[orig_arrow_index_connecter] =~ (/[└│]/) + # if the char under the processing arrow symbol (orig_arrow_index_newbranch) is also arrow or pipe, change the processing char to tee symbol + line[orig_arrow_index_newbranch] = '┬' if orig_arrow_index_newbranch && tree_view.lines[line_i + 1].to_s[orig_arrow_index_newbranch] =~ (/[└├]/) + next line # return modified line + end.join + + if String.use_color + puts <<~EOT, '' + \e[45m \e[0m: satisfied dependency + \e[46m \e[0m: build dependency + \e[47m \e[0m: runtime dependency + EOT + # (the first string in each #{} is used for commenting only, will not be included in output) + + # replace special symbols returned by @pkg.get_deps_list to actual color code + tree_view.gsub!(/\*(.+)\*/, '\1'.lightcyan) + tree_view.gsub!(/\+(.+)\+/, "\e[45m\\1\e[0m") + end + + puts tree_view + end + def self.depends_on(dependency, ver_range = nil) @dependencies ||= {} ver_check = nil dep_tags = [] - # add element in "[ name, [ tag1, tag2, ... ] ]" format + # Add element in "[ name, [ tag1, tag2, ... ] ]" format. if dependency.is_a?(Hash) - # parse "depends_on name => " + # Parse "depends_on name => ". dep_name, tags = dependency.first - # convert `tags` to array in case `tags` is a symbol + # Convert `tags` to array in case `tags` is a symbol. dep_tags += [tags].flatten else - # parse "depends_on name" + # Parse "depends_on name". dep_name = dependency end - # process dependency version range if specified + # Process dependency version range if specified. # example: # depends_on name, '>= 1.0' # - # operator can be '>=', '==', '<=', '<', '>' + # Operators can be: '>=', '==', '<=', '<', or '>' if ver_range operator, target_ver = ver_range.split(' ', 2) # lambda for comparing the given range with installed version ver_check = lambda do |installed_ver| unless Gem::Version.new(installed_ver).send(operator.to_sym, Gem::Version.new(target_ver)) - # print error if the range is not fulfilled + # Print error if the range is not fulfilled. warn <<~EOT.lightred Package #{name} depends on '#{dep_name}' (#{operator} #{target_ver}), however version '#{installed_ver}' is currently installed :/ @@ -203,33 +282,33 @@ class Package # Replace CREW_ARCH_FLAGS if @arch_flags_override is true. @crew_env_options_hash = @arch_flags_override ? @crew_env_options_hash.each { |k, v| @crew_env_options_hash[k] = v.gsub(CREW_ARCH_FLAGS, CREW_ARCH_FLAGS_OVERRIDE) } : @crew_env_options_hash - # add "-j#" argument to "make" at compile-time, if necessary + # Add "-j#" argument to "make" at compile-time, if necessary. # Order of precedence to assign the number of threads: # 1. The value of '-j#' from the package make argument # 2. The value of ENV["CREW_NPROC"] # 3. The value of `nproc`.strip - # See lib/const.rb for more details + # See lib/const.rb for more details. - # add exception option to opt_args + # Add exception option to opt_args. opt_args.merge!(exception: true) unless opt_args.key?(:exception) - # extract env hash + # Extract env hash. if args[0].is_a?(Hash) env = @crew_env_options_hash.merge(args[0]) - args.delete_at(0) # remove env hash from args array + args.delete_at(0) # Remove env hash from args array. else env = @crew_env_options_hash end - cmd_args = args # after removing the env hash, all remaining args must be command args + cmd_args = args # After removing the env hash, all remaining args must be command args. make_threads = CREW_NPROC modded_make_cmd = false - # append -j placeholder to `make` commands only if '-j#' does not exist + # Append -j placeholder to `make` commands only if '-j#' does not exist. unless cmd_args.grep(/-j[[:space:]]?[0-9]+/).any? if cmd_args.size == 1 - # involve a shell if the command is passed in one single string + # Involve a shell if the command is passed in one single string. cmd_args = ['bash', '-c', cmd_args[0].sub(/^(make)\b/, '\\1 <<>>')] modded_make_cmd = true elsif cmd_args[0] == 'make' @@ -240,23 +319,23 @@ class Package begin if modded_make_cmd - # replace placeholder with '-j#' arg and execute the actual command + # Replace placeholder with '-j#' arg and execute the actual command. Kernel.system(env, *cmd_args.map { |arg| arg.sub('<<>>', "-j#{make_threads}") }, **opt_args) else Kernel.system(env, *cmd_args, **opt_args) end rescue RuntimeError => e if modded_make_cmd && make_threads != 1 - # retry with single thread if command is `make` and is modified by crew + # Retry with single thread if command is `make` and is modified by crew. warn "Command \"#{cmd_args.map { |arg| arg.sub('<<>>', "-j#{make_threads}") }.join(' ')}\" failed, retrying with \"-j1\"...".yellow make_threads = 1 retry else - # exit with error + # Exit with error. raise e end rescue StandardError => e - # print failed line number and error message + # Print failed line number and error message. puts "#{e.backtrace[1]}: #{e.message}".orange raise InstallError, "`#{env.map { |k, v| "#{k}=\"#{v}\"" }.join(' ')} #{cmd_args.join(' ')}` exited with #{$CHILD_STATUS.exitstatus}".lightred end diff --git a/lib/package_utils.rb b/lib/package_utils.rb index 184f0e26b..1acc25a23 100644 --- a/lib/package_utils.rb +++ b/lib/package_utils.rb @@ -1,10 +1,27 @@ require 'json' require_relative 'const' +require_relative 'crewlog' class PackageUtils + def self.load_json + return JSON.load_file(File.join(CREW_CONFIG_PATH, 'device.json'), symbolize_names: true).transform_values! { |val| val.is_a?(String) ? val.to_sym : val } + end + + def self.save_json(json_object) + crewlog 'Saving device.json...' + begin + File.write File.join(CREW_CONFIG_PATH, 'device.json.tmp'), JSON.pretty_generate(JSON.parse(json_object.to_json)) + rescue StandardError + puts 'Error writing updated packages json file!'.lightred + abort + end + + # Copy over original if the write to the tmp file succeeds. + FileUtils.cp("#{CREW_CONFIG_PATH}/device.json.tmp", File.join(CREW_CONFIG_PATH, 'device.json')) && FileUtils.rm("#{CREW_CONFIG_PATH}/device.json.tmp") + end + def self.installed?(pkg_name) - device_json = JSON.load_file(File.join(CREW_CONFIG_PATH, 'device.json')) - return device_json['installed_packages'].any? { |elem| elem['name'] == pkg_name } + return load_json[:installed_packages].any? { |elem| elem[:name] == pkg_name } end def self.compatible?(pkg) diff --git a/tests/commands/list.rb b/tests/commands/list.rb new file mode 100644 index 000000000..b0de513c1 --- /dev/null +++ b/tests/commands/list.rb @@ -0,0 +1,22 @@ +require 'minitest/autorun' +require_relative '../../commands/list' +require_relative '../../lib/package_utils' + +# Add lib to LOAD_PATH +$LOAD_PATH << File.join(CREW_LIB_PATH, 'lib') + +String.use_color = false + +class ListCommandTest < Minitest::Test + def setup + @essential_deps = PackageUtils.load_json[:essential_deps].join("\n") + "\n".to_s + end + + def test_list_essential_deps + expected_output = @essential_deps + assert_output(expected_output, nil) do + # Command.list(args['available'], args['compatible'], args['incompatible'], args['essential'], args['installed'], CREW_VERBOSE) + Command.list(false, false, false, true, false, false) + end + end +end diff --git a/tests/commands/remove.rb b/tests/commands/remove.rb new file mode 100644 index 000000000..a255cc9d6 --- /dev/null +++ b/tests/commands/remove.rb @@ -0,0 +1,56 @@ +require 'minitest/autorun' +require_relative '../../commands/remove' +require_relative '../../lib/package_utils' + +# Add lib to LOAD_PATH +$LOAD_PATH << File.join(CREW_LIB_PATH, 'lib') + +# This is needed to force --no-color mode. +String.use_color = false + +class RemoveCommandTest < Minitest::Test + def setup + essential_deps = PackageUtils.load_json[:essential_deps] + @random_essential_package_name = essential_deps[rand(0...(essential_deps.length - 1))] + puts <<~ESSENTIAL_PACKAGE_REMOVAL_TEST_EOF + + Testing the removal of essential package #{@random_essential_package_name}, which was picked at random from one the essential packages: #{essential_deps.join(', ')} + (This should fail.) + + ESSENTIAL_PACKAGE_REMOVAL_TEST_EOF + @random_essential_pkg = Package.load_package("#{@random_essential_package_name}.rb") + + @normal_package_name = 'xxd_standalone' + puts <<~NORMAL_PACKAGE_REMOVAL_TEST_EOF + + Testing the removal of normal package #{@normal_package_name}. + (This should succeed.) + + NORMAL_PACKAGE_REMOVAL_TEST_EOF + @normal_pkg = Package.load_package("#{@normal_package_name}.rb") + end + + def test_remove_essential_package + # expected_output = %( #{@random_essential_package_name.capitalize} is considered an essential package needed for + # Chromebrew to function and thus cannot be removed. + # ) + # assert_output(expected_output, nil) do + # Command.remove(@random_essential_pkg, true) + # end + + assert_raises(SystemExit) { Command.remove(@random_essential_pkg, true) } + end + + def test_remove_normal_package + expected_output = <<~EOT + #{@normal_package_name} removed + EOT + assert_output(/^#{Regexp.escape(expected_output.chomp)}!/, nil) do + until PackageUtils.installed?(@normal_package_name) + system "crew install -d #{@normal_package_name} &>/dev/null", out: File::NULL + sleep 2 + end + Command.remove(@normal_pkg, true) + end + end +end diff --git a/tests/lib/docopt.rb b/tests/lib/docopt.rb new file mode 100644 index 000000000..39ef9d4fb --- /dev/null +++ b/tests/lib/docopt.rb @@ -0,0 +1,12 @@ +require 'minitest/autorun' +require_relative '../../lib/const' +require_relative '../../lib/docopt' + +class DocoptParseTest < Minitest::Test + def test_crew_with_docopt_flags + expected_output = 'true' + assert_output(expected_output, nil) do + print Docopt.docopt(CREW_DOCOPT, argv: '-d const ARCH').is_a?(Hash) + end + end +end diff --git a/tests/unit_test.sh b/tests/unit_test.sh index 07da86ab2..0c7aa41e6 100755 --- a/tests/unit_test.sh +++ b/tests/unit_test.sh @@ -1,14 +1,17 @@ #!/bin/bash -e # This is for use as a Github CI Pull Request Unit Test. -echo "ALL_CHANGED FILES is/are ${ALL_CHANGED_FILES}." -echo "CHANGED_PACKAGES is/are ${CHANGED_PACKAGES}." +echo "ALL_CHANGED FILES: ${ALL_CHANGED_FILES}." +echo "CHANGED_PACKAGES: ${CHANGED_PACKAGES}." cd /usr/local/lib/crew/packages/ yes | crew upgrade yes | crew install vim yes | crew remove vim ruby ../tests/commands/const.rb ruby ../tests/commands/help.rb +ruby ../tests/commands/list.rb ruby ../tests/commands/prop.rb +ruby ../tests/commands/remove.rb +ruby ../tests/lib/docopt.rb if [[ -n ${ALL_CHANGED_FILES-} ]]; then # for file in ${ALL_CHANGED_FILES}; do # ruby ../tests/prop_test "$file" @@ -25,7 +28,12 @@ if [[ -n ${ALL_CHANGED_FILES-} ]]; then ruby ../tests/buildsystem_test "${pkg}" echo "Testing install/removal of compatible package ${pkg}." yes | time crew install "${pkg}" - yes | time crew remove "${pkg}" + # Removal of essential packages is expected to fail. + if [[ $(crew list -d essential) == *"${pkg}"* ]]; then + yes | time crew remove "${pkg}" || true + else + yes | time crew remove "${pkg}" + fi fi done fi