From e44d98aa8c4daa12c6e09f9b4dd503d95649a3bc Mon Sep 17 00:00:00 2001 From: supechicken Date: Fri, 11 Mar 2022 02:07:16 +0800 Subject: [PATCH] crew: Rewrite dependency resolver algorithm (#6840) * Rewrite dependency resolver algorithm * Add color for build dependencies, show satisfied dependencies * Update help message * Fix typo --- bin/crew | 177 +++++++++++++++++++++++++++++-------------------- lib/const.rb | 2 +- lib/package.rb | 93 +++++++++++++++++++++++++- 3 files changed, 199 insertions(+), 73 deletions(-) diff --git a/bin/crew b/bin/crew index cad38b361..8770da290 100755 --- a/bin/crew +++ b/bin/crew @@ -25,7 +25,7 @@ Usage: crew autoremove [options] crew build [options] [-k|--keep] ... crew const [options] [ ...] - crew deps [options] ... + crew deps [options] [-t|--tree] [-b|--include-build-deps] [--exclude-buildessential] ... crew download [options] ... crew files [options] ... crew help [] @@ -39,15 +39,17 @@ Usage: crew upgrade [options] [-k|--keep] [-s|--build-from-source] [ ...] crew whatprovides [options] ... - -c --color Use colors even if standard out is not a tty. - -d --no-color Disable colors even if standard out is a tty. - -k --keep Keep the `CREW_BREW_DIR` (#{CREW_BREW_DIR}) directory. - -L --license Display the crew license. - -s --build-from-source Build from source even if pre-compiled binary exists. - -S --recursive-build Build from source, including all dependencies, even if pre-compiled binaries exist. - -v --verbose Show extra information. - -V --version Display the crew version. - -h --help Show this screen. + -b --include-build-deps Include build dependencies in output. + -t --tree Print dependencies in a tree-structure format. + -c --color Use colors even if standard out is not a tty. + -d --no-color Disable colors even if standard out is a tty. + -k --keep Keep the `CREW_BREW_DIR` (#{CREW_BREW_DIR}) directory. + -L --license Display the crew license. + -s --build-from-source Build from source even if pre-compiled binary exists. + -S --recursive-build Build from source, including all dependencies, even if pre-compiled binaries exist. + -v --verbose Show extra information. + -V --version Display the crew version. + -h --help Show this screen. version #{CREW_VERSION} DOCOPT @@ -358,7 +360,11 @@ def help(pkgName) when "deps" puts <<~EOT Display dependencies of package(s). - Usage: crew deps [ ...] + Usage: crew deps [-t|--tree] [-b|--include-build-deps] [--exclude-buildessential] [ ...] + + If `-t` or `--tree` specified, dependencies will be printed in a tree-structure format + If `-b` or `--include-build-deps` specified, build dependencies will be included in output + It `--exclude-buildessential` specified, `buildessential` and its dependencies will not be inserted automatically EOT when "download" puts <<~EOT @@ -1300,40 +1306,13 @@ def resolve_dependencies_and_install end def expand_dependencies - def push_dependencies - return unless @pkg - if @pkg.is_binary?(@device[:architecture]) || - (!@pkg.in_upgrade && !@pkg.build_from_source && @device[:installed_packages].any? { |pkg| pkg[:name] == @pkg.name }) - # retrieve name of dependencies that doesn't contain :build tag - check_deps = @pkg.dependencies.select {|k, v| !v.include?(:build)}.map {|k, v| k} - else - # retrieve name of all dependencies - check_deps = @pkg.dependencies.map {|k, v| k} - end - # check all dependencies recursively - check_deps.each do |dep| - # build unique dependencies list - unless @dependencies&.include?(dep) || dep == @pkgName - @dependencies << dep - search dep, true - push_dependencies - end - end - end - push_dependencies + @dependencies = @pkg.get_deps_list.reject {|depName| @device[:installed_packages].any? { |pkg| pkg[:name] == depName } } end def resolve_dependencies - abort "Package #{@pkg.name} is not compatible with your device architecture (#{ARCH}) :/".lightred unless @device[:compatible_packages].any? do |elem| elem[:name] == @pkg.name end + abort "Package #{@pkg.name} is not compatible with your device architecture (#{ARCH}) :/".lightred unless @device[:compatible_packages].any? {|elem| elem[:name] == @pkg.name } @dependencies = [] - if @pkg.build_from_source - # make sure all buildessential packages are installed - pkgname = @pkg.name - search 'buildessential', true - expand_dependencies - search pkgname, true - end expand_dependencies # leave only not installed packages in dependencies @@ -1343,39 +1322,14 @@ def resolve_dependencies puts 'The following packages also need to be installed: ' - deps = @dependencies - # populate arrays with common elements - begin_packages = deps & CREW_FIRST_PACKAGES - end_packages = deps & CREW_LAST_PACKAGES - @dependencies.each do |dep| - depends = nil unless File.exist?("#{CREW_PACKAGES_PATH}#{dep}.rb") abort "Dependency #{dep} was not found.".lightred end - File.open("#{CREW_PACKAGES_PATH}#{dep}.rb") do |f| - f.each_line do |line| - found = line[/depends_on/] if line.ascii_only? - if found - depends = true - break - end - end - end - # if a dependency package has no other dependencies, push to the front - begin_packages.push dep unless depends - end - # Remove elements in another array - deps -= begin_packages - deps -= end_packages - - @dependencies = (begin_packages + deps + end_packages).uniq - - @dependencies.each do |dep| - print dep + ' ' end - puts + puts @dependencies.join(' ') + print 'Do you agree? [Y/n] ' response = STDIN.getc case response @@ -1633,6 +1587,83 @@ def remove(pkgName) puts "#{pkgName.capitalize} removed!".lightgreen end +def print_deps_tree (args) + STDERR.puts 'Walking through dependencies recursively, this may take a while...', '' + + # depHash: Hash object returned by @pkg.get_deps_list + depHash = @pkg.get_deps_list( hash: true, include_build_deps: (args['--include-build-deps'] || 'auto'), exclude_buildessential: args['--exclude-buildessential'] ) + + # convert returned hash to json and format it + jsonView = JSON.pretty_generate(depHash) + + # convert formatted json string to tree structure + treeView = jsonView.gsub(/\{\s*/m, '└─────').gsub(/[\[\]\{\}\,\"\:]/, '').gsub(/^\s*$\n/, '').gsub(/\s*$/, '') + + # add pipe char to connect endpoints and starting points, improve readability + # find the horizontal location of all arrow symbols + index_with_pipe_char = treeView.lines.map {|line| line.index('└') } .reject(&:nil?).uniq + + # determine whatever a pipe char should be added according to the horizontal location of arrow symbols + treeView = treeView.lines.each_with_index.map do |line, line_i| + index_with_pipe_char.each do |char_i| + # check if there have any non-space char (pkgNames) between starting point ([line_i][char_i]) and endpoint vertically ([next_arrow_line_offset][char_i]) + # (used to determine if the starting point and endpoint are in same branch, use pipe char to connect them if true) + next_arrow_line_offset = treeView.lines[line_i..-1].index {|l| l[char_i] == '└' } + have_line_with_non_empty_char = treeView.lines[line_i+1..line_i+next_arrow_line_offset.to_i-1].any? {|l| l[char_i].nil? or l[char_i] =~ /\S/ } + + if next_arrow_line_offset and line[char_i] == ' ' and !have_line_with_non_empty_char + line[char_i] = '│' + end + end + next line + end.join + + # replace arrow symbols with a tee symbol on branch intersection + treeView = treeView.lines.each_with_index.map do |line, line_i| + # orig_arrow_index_connecter: the horizontal location of the arrow symbol used to connect parent branch + # + # example: + # └───┬─chrome + # └─────buildessential + # ^ + orig_arrow_index_connecter = line.index('└') + # orig_arrow_index_newbranch: the horizontal location of the "box drawing char" symbol MIGHT be + # required to convert to tee char in order to connect child branch, + # located at 3 chars later of orig_arrow_index_connecter + # + # example: + # v + # └─────chrome + # └─────buildessential + # + # which might need to be convert to: + # └───┬─chrome + # └─────buildessential + orig_arrow_index_newbranch = orig_arrow_index_connecter + 4 + + # if the char under the processing arrow symbol (orig_arrow_index_connecter) is also arrow or pipe, change the processing char to tee symbol + line[orig_arrow_index_connecter] = '├' if orig_arrow_index_connecter and treeView.lines[line_i+1].to_s[orig_arrow_index_connecter] =~ /[└│]/ + # if the char under the processing arrow symbol (orig_arrow_index_newbranch) is also arrow or pipe, change the processing char to tee symbol + line[orig_arrow_index_newbranch] = '┬' if orig_arrow_index_newbranch and treeView.lines[line_i+1].to_s[orig_arrow_index_newbranch] =~ /[└├]/ + next line # return modified line + end.join + + if String.use_color + puts <<~EOT, '' + #{'purple -->'; "\e[45m \e[0m"}: satisfied dependency + #{'lightcyan -->'; "\e[46m \e[0m"}: build dependency + #{'white -->'; "\e[47m \e[0m"}: runtime dependency + EOT + # (the first string in each #{} is used for commenting only, will not be included in output) + + # replace special symbols returned by @pkg.get_deps_list to actual color code + treeView.gsub!(/\*(.+)\*/, '\1'.lightcyan) + treeView.gsub!(/\+(.+)\+/, "\e[45m\\1\e[0m") + end + + puts treeView +end + def autoremove_command(args) deps_of_installed_pkgs = @device[:installed_packages].map do |pkg| # ignore deleted/non-exist package recipes @@ -1688,12 +1719,16 @@ end def deps_command(args) args[""].each do |name| - @dependencies = [] @pkgName = name search @pkgName - print_current_package - expand_dependencies - puts @dependencies + + if args['--tree'] + # call `print_deps_tree` (print dependency tree) if --tree is specified + print_deps_tree(args) + else + # print dependencies according to the install order if --tree is not specified + puts @pkg.get_deps_list( include_build_deps: (args['--include-build-deps'] || 'auto'), exclude_buildessential: args['--exclude-buildessential'] ) + end end end diff --git a/lib/const.rb b/lib/const.rb index adefb7aaf..a507e30ac 100644 --- a/lib/const.rb +++ b/lib/const.rb @@ -1,6 +1,6 @@ # Defines common constants used in different parts of crew -CREW_VERSION = '1.22.10' +CREW_VERSION = '1.23.0' ARCH_ACTUAL = `uname -m`.chomp # This helps with virtualized builds on aarch64 machines diff --git a/lib/package.rb b/lib/package.rb index 3e6a64974..84ff62708 100644 --- a/lib/package.rb +++ b/lib/package.rb @@ -29,7 +29,98 @@ class Package # base class. Instead of define it, we initialize it in a function # called from derived classees. @dependencies ||= Hash.new - @dependencies + end + + def self.get_deps_list (pkgName = self.name, hash: false, include_build_deps: 'auto', include_self: false, + pkgTags: [], highlight_build_deps: true, exclude_buildessential: false, top_level: true) + # get_deps_list: get dependencies list of pkgName (current package by default) + # + # pkgName: package to check dependencies, current package by default + # hash: return result in nested hash, used by `print_deps_tree` (`bin/crew`) + # + # include_build_deps: if set to true, force all build dependencies to be returned. + # if set to false, all build dependencies will not be returned + # if set to "auto" (default), return build dependencies if pre-built binaries not available + # + # include_self: include #{pkgName} itself in returned result, only used in recursive calls (see `expandedDeps` below) + # highlight_build_deps: include corresponding symbols in return value, you can convert it to actual ascii color codes later + # exclude_buildessential: do not insert `buildessential` dependency automatically + # + # top_level: if set to true, return satisfied dependencies + # (dependencies that might be a sub-dependency of a dependency that checked before), + # always set to false if this function is called in recursive loop (see `expandedDeps` below) + # + @checked_list ||= Hash.new # create @checked_list placeholder if not exist + + # add current package to @checked_list for preventing extra checks + @checked_list.merge!({ pkgName => pkgTags }) + + pkgObj = Object.const_get(pkgName.capitalize) + is_source = pkgObj.is_source?(ARCH.to_sym) or pkgObj.build_from_source + deps = pkgObj.dependencies + + # append buildessential to deps if building from source is needed/specified + if ( include_build_deps == true or (include_build_deps == 'auto' and is_source) ) and \ + !exclude_buildessential and \ + !@checked_list.keys.include?('buildessential') + + deps = ({ 'buildessential' => [ :build ] }).merge(deps) + end + + # parse dependencies recursively + expandedDeps = deps.uniq.map do |dep, depTags| + # check build dependencies only if building from source is needed/specified + if include_build_deps == true or \ + (include_build_deps == 'auto' and is_source) or \ + !depTags.include?(:build) + + # overwrite tags if parent dependency is a build dependency + # (for build dependencies highlighting) + tags = (pkgTags.include?(:build)) ? pkgTags : depTags + + if @checked_list.keys.none?(dep) + require_relative "#{CREW_PACKAGES_PATH}/#{dep}.rb" + # check dependency by calling this function recursively + next send(__method__, dep, + hash: hash, + pkgTags: tags, + include_build_deps: include_build_deps, + highlight_build_deps: highlight_build_deps, + exclude_buildessential: exclude_buildessential, + include_self: true, + top_level: false + ) + + elsif hash and top_level + # will be dropped here if current dependency is already checked and #{top_level} is set to true + # + # the '+' symbol tell `print_deps_tree` (`bin/crew`) to color this package as "satisfied dependency" + # the '*' symbol tell `print_deps_tree` (`bin/crew`) to color this package as "build dependency" + if highlight_build_deps and tags.include?(:build) + next { "+*#{dep}*+" => [] } + elsif highlight_build_deps + next { "+#{dep}+" => [] } + else + next { dep => [] } + end + end + end + end.reject(&:nil?) + + if hash + # the '*' symbol tell `print_deps_tree` (`bin/crew`) to color this package as "build dependency" + if highlight_build_deps and pkgTags.include?(:build) + return { "*#{pkgName}*" => expandedDeps } + else + return { pkgName => expandedDeps } + end + elsif include_self + # return pkgName itself if this function is called as a recursive loop (see `expandedDeps`) + return [ expandedDeps, pkgName ].flatten + else + # if this function is called outside of this function, return parsed dependencies only + return expandedDeps.flatten + end end boolean_property.each do |prop|