crew: Rewrite dependency resolver algorithm (#6840)

* Rewrite dependency resolver algorithm

* Add color for build dependencies, show satisfied dependencies

* Update help message

* Fix typo

bin/crew

@@ -25,7 +25,7 @@ Usage:
   crew autoremove [options]
   crew build [options] [-k|--keep] <name> ...
   crew const [options] [<name> ...]
-  crew deps [options] <name> ...
+  crew deps [options] [-t|--tree] [-b|--include-build-deps] [--exclude-buildessential] <name> ...
   crew download [options] <name> ...
   crew files [options] <name> ...
   crew help [<command>]
@@ -39,15 +39,17 @@ Usage:
   crew upgrade [options] [-k|--keep] [-s|--build-from-source] [<name> ...]
   crew whatprovides [options] <pattern> ...
 
-  -c --color              Use colors even if standard out is not a tty.
-  -d --no-color           Disable colors even if standard out is a tty.
-  -k --keep               Keep the `CREW_BREW_DIR` (#{CREW_BREW_DIR}) directory.
-  -L --license            Display the crew license.
-  -s --build-from-source  Build from source even if pre-compiled binary exists.
-  -S --recursive-build    Build from source, including all dependencies, even if pre-compiled binaries exist.
-  -v --verbose            Show extra information.
-  -V --version            Display the crew version.
-  -h --help               Show this screen.
+  -b --include-build-deps  Include build dependencies in output.
+  -t --tree                Print dependencies in a tree-structure format.
+  -c --color               Use colors even if standard out is not a tty.
+  -d --no-color            Disable colors even if standard out is a tty.
+  -k --keep                Keep the `CREW_BREW_DIR` (#{CREW_BREW_DIR}) directory.
+  -L --license             Display the crew license.
+  -s --build-from-source   Build from source even if pre-compiled binary exists.
+  -S --recursive-build     Build from source, including all dependencies, even if pre-compiled binaries exist.
+  -v --verbose             Show extra information.
+  -V --version             Display the crew version.
+  -h --help                Show this screen.
 
 version #{CREW_VERSION}
 DOCOPT
@@ -358,7 +360,11 @@ def help(pkgName)
   when "deps"
     puts <<~EOT
       Display dependencies of package(s).
-      Usage: crew deps <package1> [<package2> ...]
+      Usage: crew deps [-t|--tree] [-b|--include-build-deps] [--exclude-buildessential] <package1> [<package2> ...]
+
+      If `-t` or `--tree` specified, dependencies will be printed in a tree-structure format
+      If `-b` or `--include-build-deps` specified, build dependencies will be included in output
+      It `--exclude-buildessential` specified, `buildessential` and its dependencies will not be inserted automatically
     EOT
   when "download"
     puts <<~EOT
@@ -1300,40 +1306,13 @@ def resolve_dependencies_and_install
 end
 
 def expand_dependencies
-  def push_dependencies
-    return unless @pkg
-    if @pkg.is_binary?(@device[:architecture]) ||
-      (!@pkg.in_upgrade && !@pkg.build_from_source && @device[:installed_packages].any? { |pkg| pkg[:name] == @pkg.name })
-      # retrieve name of dependencies that doesn't contain :build tag
-      check_deps = @pkg.dependencies.select {|k, v| !v.include?(:build)}.map {|k, v| k}
-    else
-      # retrieve name of all dependencies
-      check_deps = @pkg.dependencies.map {|k, v| k}
-    end
-    # check all dependencies recursively
-    check_deps.each do |dep|
-      # build unique dependencies list
-      unless @dependencies&.include?(dep) || dep == @pkgName
-        @dependencies << dep
-        search dep, true
-        push_dependencies
-      end
-    end
-  end
-  push_dependencies
+  @dependencies = @pkg.get_deps_list.reject {|depName| @device[:installed_packages].any? { |pkg| pkg[:name] == depName } }
 end
 
 def resolve_dependencies
-  abort "Package #{@pkg.name} is not compatible with your device architecture (#{ARCH}) :/".lightred unless @device[:compatible_packages].any? do |elem| elem[:name] == @pkg.name end
+  abort "Package #{@pkg.name} is not compatible with your device architecture (#{ARCH}) :/".lightred unless @device[:compatible_packages].any? {|elem| elem[:name] == @pkg.name }
 
   @dependencies = []
-  if @pkg.build_from_source
-    # make sure all buildessential packages are installed
-    pkgname = @pkg.name
-    search 'buildessential', true
-    expand_dependencies
-    search pkgname, true
-  end
   expand_dependencies
 
   # leave only not installed packages in dependencies
@@ -1343,39 +1322,14 @@ def resolve_dependencies
 
   puts 'The following packages also need to be installed: '
 
-  deps = @dependencies
-  # populate arrays with common elements
-  begin_packages = deps & CREW_FIRST_PACKAGES
-  end_packages = deps & CREW_LAST_PACKAGES
-
   @dependencies.each do |dep|
-    depends = nil
     unless File.exist?("#{CREW_PACKAGES_PATH}#{dep}.rb")
       abort "Dependency #{dep} was not found.".lightred
     end
-    File.open("#{CREW_PACKAGES_PATH}#{dep}.rb") do |f|
-      f.each_line do |line|
-        found = line[/depends_on/] if line.ascii_only?
-        if found
-          depends = true
-          break
-        end
-      end
-    end
-    # if a dependency package has no other dependencies, push to the front
-    begin_packages.push dep unless depends
-  end
-  # Remove elements in another array
-  deps -= begin_packages
-  deps -= end_packages
-
-  @dependencies = (begin_packages + deps + end_packages).uniq
-
-  @dependencies.each do |dep|
-    print dep + ' '
   end
 
-  puts
+  puts @dependencies.join(' ')
+
   print 'Do you agree? [Y/n] '
   response = STDIN.getc
   case response
@@ -1633,6 +1587,83 @@ def remove(pkgName)
   puts "#{pkgName.capitalize} removed!".lightgreen
 end
 
+def print_deps_tree (args)
+  STDERR.puts 'Walking through dependencies recursively, this may take a while...', ''
+
+  # depHash: Hash object returned by @pkg.get_deps_list
+  depHash = @pkg.get_deps_list( hash: true, include_build_deps: (args['--include-build-deps'] || 'auto'), exclude_buildessential: args['--exclude-buildessential'] )
+
+  # convert returned hash to json and format it
+  jsonView = JSON.pretty_generate(depHash)
+
+  # convert formatted json string to tree structure
+  treeView = jsonView.gsub(/\{\s*/m, '└─────').gsub(/[\[\]\{\}\,\"\:]/, '').gsub(/^\s*$\n/, '').gsub(/\s*$/, '')
+
+  # add pipe char to connect endpoints and starting points, improve readability
+  # find the horizontal location of all arrow symbols
+  index_with_pipe_char = treeView.lines.map {|line| line.index('└') } .reject(&:nil?).uniq
+
+  # determine whatever a pipe char should be added according to the horizontal location of arrow symbols
+  treeView = treeView.lines.each_with_index.map do |line, line_i|
+    index_with_pipe_char.each do |char_i|
+      # check if there have any non-space char (pkgNames) between starting point ([line_i][char_i]) and endpoint vertically ([next_arrow_line_offset][char_i])
+      # (used to determine if the starting point and endpoint are in same branch, use pipe char to connect them if true)
+      next_arrow_line_offset = treeView.lines[line_i..-1].index {|l| l[char_i] == '└' }
+      have_line_with_non_empty_char = treeView.lines[line_i+1..line_i+next_arrow_line_offset.to_i-1].any? {|l| l[char_i].nil? or l[char_i] =~ /\S/ }
+
+      if next_arrow_line_offset and line[char_i] == ' ' and !have_line_with_non_empty_char
+        line[char_i] = '│'
+      end
+    end
+    next line
+  end.join
+
+  # replace arrow symbols with a tee symbol on branch intersection
+  treeView = treeView.lines.each_with_index.map do |line, line_i|
+    # orig_arrow_index_connecter: the horizontal location of the arrow symbol used to connect parent branch
+    #
+    # example:
+    # └───┬─chrome
+    #     └─────buildessential
+    #     ^
+    orig_arrow_index_connecter = line.index('└')
+    # orig_arrow_index_newbranch: the horizontal location of the "box drawing char" symbol MIGHT be
+    #                             required to convert to tee char in order to connect child branch,
+    #                             located at 3 chars later of orig_arrow_index_connecter
+    #
+    # example:
+    #     v
+    # └─────chrome
+    #     └─────buildessential
+    #
+    # which might need to be convert to:
+    # └───┬─chrome
+    #     └─────buildessential
+    orig_arrow_index_newbranch = orig_arrow_index_connecter + 4
+
+    # if the char under the processing arrow symbol (orig_arrow_index_connecter) is also arrow or pipe, change the processing char to tee symbol
+    line[orig_arrow_index_connecter] = '├' if orig_arrow_index_connecter and treeView.lines[line_i+1].to_s[orig_arrow_index_connecter] =~ /[└│]/
+    # if the char under the processing arrow symbol (orig_arrow_index_newbranch) is also arrow or pipe, change the processing char to tee symbol
+    line[orig_arrow_index_newbranch] = '┬' if orig_arrow_index_newbranch and treeView.lines[line_i+1].to_s[orig_arrow_index_newbranch] =~ /[└├]/
+    next line # return modified line
+  end.join
+
+  if String.use_color
+    puts <<~EOT, ''
+      #{'purple    -->'; "\e[45m \e[0m"}: satisfied dependency
+      #{'lightcyan -->'; "\e[46m \e[0m"}: build dependency
+      #{'white     -->'; "\e[47m \e[0m"}: runtime dependency
+    EOT
+    # (the first string in each #{} is used for commenting only, will not be included in output)
+
+    # replace special symbols returned by @pkg.get_deps_list to actual color code
+    treeView.gsub!(/\*(.+)\*/, '\1'.lightcyan)
+    treeView.gsub!(/\+(.+)\+/, "\e[45m\\1\e[0m")
+  end
+
+  puts treeView
+end
+
 def autoremove_command(args)
   deps_of_installed_pkgs = @device[:installed_packages].map do |pkg|
     # ignore deleted/non-exist package recipes
@@ -1688,12 +1719,16 @@ end
 
 def deps_command(args)
   args["<name>"].each do |name|
-    @dependencies = []
     @pkgName = name
     search @pkgName
-    print_current_package
-    expand_dependencies
-    puts @dependencies
+
+    if args['--tree']
+      # call `print_deps_tree` (print dependency tree) if --tree is specified
+      print_deps_tree(args)
+    else
+      # print dependencies according to the install order if --tree is not specified
+      puts @pkg.get_deps_list( include_build_deps: (args['--include-build-deps'] || 'auto'), exclude_buildessential: args['--exclude-buildessential'] )
+    end
   end
 end
 

lib/const.rb

@@ -1,6 +1,6 @@
 # Defines common constants used in different parts of crew
 
-CREW_VERSION = '1.22.10'
+CREW_VERSION = '1.23.0'
 
 ARCH_ACTUAL = `uname -m`.chomp
 # This helps with virtualized builds on aarch64 machines

lib/package.rb

@@ -29,7 +29,98 @@ class Package
     # base class.  Instead of define it, we initialize it in a function
     # called from derived classees.
     @dependencies ||= Hash.new
-    @dependencies
+  end
+
+  def self.get_deps_list (pkgName = self.name, hash: false, include_build_deps: 'auto', include_self: false,
+                          pkgTags: [], highlight_build_deps: true, exclude_buildessential: false, top_level: true)
+    # get_deps_list: get dependencies list of pkgName (current package by default)
+    #
+    #                pkgName: package to check dependencies, current package by default
+    #                   hash: return result in nested hash, used by `print_deps_tree` (`bin/crew`)
+    #
+    #     include_build_deps: if set to true, force all build dependencies to be returned.
+    #                         if set to false, all build dependencies will not be returned
+    #                         if set to "auto" (default), return build dependencies if pre-built binaries not available
+    #
+    #           include_self: include #{pkgName} itself in returned result, only used in recursive calls (see `expandedDeps` below)
+    #   highlight_build_deps: include corresponding symbols in return value, you can convert it to actual ascii color codes later
+    # exclude_buildessential: do not insert `buildessential` dependency automatically
+    #
+    #              top_level: if set to true, return satisfied dependencies 
+    #                         (dependencies that might be a sub-dependency of a dependency that checked before),
+    #                         always set to false if this function is called in recursive loop (see `expandedDeps` below)
+    #
+    @checked_list ||= Hash.new # create @checked_list placeholder if not exist
+
+    # add current package to @checked_list for preventing extra checks
+    @checked_list.merge!({ pkgName => pkgTags })
+
+    pkgObj = Object.const_get(pkgName.capitalize)
+    is_source = pkgObj.is_source?(ARCH.to_sym) or pkgObj.build_from_source
+    deps = pkgObj.dependencies
+
+    # append buildessential to deps if building from source is needed/specified
+    if ( include_build_deps == true or (include_build_deps == 'auto' and is_source) ) and \
+       !exclude_buildessential and \
+       !@checked_list.keys.include?('buildessential')
+
+      deps = ({ 'buildessential' => [ :build ] }).merge(deps)
+    end
+
+    # parse dependencies recursively
+    expandedDeps = deps.uniq.map do |dep, depTags|
+                     # check build dependencies only if building from source is needed/specified
+                     if include_build_deps == true or \
+                        (include_build_deps == 'auto' and is_source) or \
+                        !depTags.include?(:build)
+
+                       # overwrite tags if parent dependency is a build dependency
+                       # (for build dependencies highlighting)
+                       tags = (pkgTags.include?(:build)) ? pkgTags : depTags
+
+                       if @checked_list.keys.none?(dep)
+                         require_relative "#{CREW_PACKAGES_PATH}/#{dep}.rb"
+                         # check dependency by calling this function recursively
+                         next send(__method__, dep,
+                                                     hash: hash,
+                                                  pkgTags: tags,
+                                       include_build_deps: include_build_deps,
+                                     highlight_build_deps: highlight_build_deps,
+                                   exclude_buildessential: exclude_buildessential,
+                                             include_self: true,
+                                                top_level: false
+                                  )
+
+                       elsif hash and top_level
+                         # will be dropped here if current dependency is already checked and #{top_level} is set to true
+                         #
+                         # the '+' symbol tell `print_deps_tree` (`bin/crew`) to color this package as "satisfied dependency"
+                         # the '*' symbol tell `print_deps_tree` (`bin/crew`) to color this package as "build dependency"
+                         if highlight_build_deps and tags.include?(:build)
+                           next { "+*#{dep}*+" => [] }
+                         elsif highlight_build_deps
+                           next { "+#{dep}+" => [] }
+                         else
+                           next { dep => [] }
+                         end
+                       end
+                     end
+                   end.reject(&:nil?)
+
+    if hash
+      # the '*' symbol tell `print_deps_tree` (`bin/crew`) to color this package as "build dependency"
+      if highlight_build_deps and pkgTags.include?(:build)
+        return { "*#{pkgName}*" => expandedDeps }
+      else
+        return { pkgName => expandedDeps }
+      end
+    elsif include_self
+      # return pkgName itself if this function is called as a recursive loop (see `expandedDeps`)
+      return [ expandedDeps, pkgName ].flatten
+    else
+      # if this function is called outside of this function, return parsed dependencies only
+      return expandedDeps.flatten
+    end
   end
 
   boolean_property.each do |prop|