Add gitlab and sourceforge fallbacks to version.rb (#13226)

* Add gitlab and sourceforge fallbacks to version.rb

Add Compile? to version line output

Update options and help to include -vv

Fix candidate selection logic

Add more very verbose debugging output

* Rubocop lint

* More rubocop linting

* Handle package not found searches more elegantly

* Set VERBOSE to true if VERY_VERBOSE

bin/crew

@@ -91,6 +91,7 @@ String.use_color = args['--color'] || !args['--no-color']
 @update        = @opt_update ? '-u' : ''
 @verbose       = CREW_VERBOSE ? 'v' : ''
 @short_verbose = CREW_VERBOSE ? '-v' : ''
+@very_verbose  = CREW_VERY_VERBOSE ? '-vv' : ''
 
 # Make sure crew work directories exist.
 FileUtils.mkdir_p CREW_BREW_DIR
@@ -2150,7 +2151,7 @@ def upstream_command(args)
   # Pass the whole argument input to tools/version.rb, stripping out any file extensions that occur as a result of using a wildcard argument in a directory where it matches files.
   args = { '<name>' => args.split } if args.is_a? String
   Dir.chdir CREW_PACKAGES_PATH do
-    system "../tools/version.rb #{args['<name>'].join(' ').gsub('.rb', '')} #{@json} #{@update} #{@short_verbose}"
+    system "../tools/version.rb #{args['<name>'].join(' ').gsub('.rb', '')} #{@json} #{@update} #{@short_verbose} #{@very_verbose}"
   end
 end
 

commands/help.rb

@@ -162,10 +162,11 @@ class Command
     when 'upstream'
       puts <<~EOT
         Check if an upstream version is available for package(s).
-        Usage: crew upstream [-j|--json|-u|--update-package-files|-v|--verbose] [<package1> <package2> ...]
+        Usage: crew upstream [-j|--json|-u|--update-package-files|-v|--verbose|-vv] [<package1> <package2> ...]
         If `-j` or `--json` is present, output will be in json format.
         If `-u` or `--update-package-files` is present, it will attempt to update the package version.
         If `-v` or `--verbose` is present, extra information will be displayed.
+        If `-vv` is present, very verbose information will be displayed.
       EOT
     when 'version'
       puts <<~EOT

lib/const.rb

@@ -4,7 +4,7 @@ require 'etc'
 require 'open3'
 
 OLD_CREW_VERSION ||= defined?(CREW_VERSION) ? CREW_VERSION : '1.0'
-CREW_VERSION ||= '1.67.9' unless defined?(CREW_VERSION) && CREW_VERSION == OLD_CREW_VERSION
+CREW_VERSION ||= '1.67.10' unless defined?(CREW_VERSION) && CREW_VERSION == OLD_CREW_VERSION
 
 # Kernel architecture.
 KERN_ARCH ||= Etc.uname[:machine]
@@ -133,15 +133,16 @@ CREW_MUSL_PREFIX      ||= File.join(CREW_PREFIX, '/share/musl/')
 CREW_DEST_MUSL_PREFIX ||= File.join(CREW_DEST_DIR, CREW_MUSL_PREFIX)
 MUSL_LIBC_VERSION     ||= File.executable?("#{CREW_MUSL_PREFIX}/lib/libc.so") ? `#{CREW_MUSL_PREFIX}/lib/libc.so 2>&1`[/\bVersion\s+\K\S+/] : nil unless defined?(MUSL_LIBC_VERSION)
 
-CREW_DEST_HOME          ||= File.join(CREW_DEST_DIR, HOME)
+CREW_DEST_HOME  ||= File.join(CREW_DEST_DIR, HOME)
-CREW_NO_GIT             ||= ENV.fetch('CREW_NO_GIT', false) unless defined?(CREW_NO_GIT)
+CREW_NO_GIT     ||= ENV.fetch('CREW_NO_GIT', false) unless defined?(CREW_NO_GIT)
-CREW_UNATTENDED         ||= ENV.fetch('CREW_UNATTENDED', false) unless defined?(CREW_UNATTENDED)
+CREW_UNATTENDED ||= ENV.fetch('CREW_UNATTENDED', false) unless defined?(CREW_UNATTENDED)
 
 CREW_STANDALONE_UPGRADE_ORDER = %w[libxcrypt crew_preload glibc openssl ruby python3 perl icu4c sommelier] unless defined?(CREW_STANDALONE_UPGRADE_ORDER)
 
-CREW_DEBUG   ||= ARGV.intersect?(%w[-D --debug]) unless defined?(CREW_DEBUG)
+CREW_DEBUG        ||= ARGV.include?('-D') || ARGV.include?('--debug') unless defined?(CREW_DEBUG)
-CREW_FORCE   ||= ARGV.intersect?(%w[-f --force]) unless defined?(CREW_FORCE)
+CREW_FORCE        ||= ARGV.include?('-f') || ARGV.include?('--force') unless defined?(CREW_FORCE)
-CREW_VERBOSE ||= ARGV.intersect?(%w[-v --verbose]) unless defined?(CREW_VERBOSE)
+CREW_VERBOSE      ||= ARGV.include?('-v') || ARGV.include?('--verbose') || ARGV.include?('-vv') unless defined?(CREW_VERBOSE)
+CREW_VERY_VERBOSE ||= ARGV.include?('-vv') unless defined?(CREW_VERY_VERBOSE)
 
 # Set CREW_NPROC from environment variable, `distcc -j`, or `nproc`.
 CREW_NPROC ||=
@@ -421,6 +422,7 @@ unless defined?(CREW_ANITYA_PACKAGE_NAME_MAPPINGS)
     { pkg_name: 'signal_desktop', anitya_pkg: 'signal', comments: '' },
     { pkg_name: 'tepl_6', anitya_pkg: 'libgedit-tepl', comments: '' },
     { pkg_name: 'upx', anitya_pkg: 'upx', comments: 'Prefer to GitHub' },
+    { pkg_name: 'vidstab', anitya_pkg: 'vid.stab', comments: 'Prefer to GitHub' },
     { pkg_name: 'vim_runtime', anitya_pkg: 'vim', comments: '' },
     { pkg_name: 'webkitgtk_6', anitya_pkg: 'webkitgtk~stable', comments: '' },
     { pkg_name: 'xauth', anitya_pkg: 'xorg-x11-xauth', comments: '' },
@@ -476,7 +478,7 @@ CREW_DOCOPT ||= <<~DOCOPT
     crew update_package_file [options] [-v|--verbose] [<name> ...]
     crew upgrade [options] [-f|--force] [-k|--keep] [--regenerate-filelist] [-s|--source] [-v|--verbose] [<name> ...]
     crew upload [options] [-f|--force] [-v|--verbose] [<name> ...]
-    crew upstream [options] [-j|--json|-u|--update-package-files|-v|--verbose] <name> ...
+    crew upstream [options] [-j|--json|-u|--update-package-files|-v|--verbose|-vv] <name> ...
     crew version [options] [<name>]
     crew whatprovides [options] <pattern> ...
 

tools/version.rb

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
-# version.rb version 3.6 (for Chromebrew)
+# version.rb version 3.7 (for Chromebrew)
 
-OPTIONS = %w[-h --help -j --json -u --update-package-files -v --verbose]
+OPTIONS = %w[-h --help -j --json -u --update-package-files -v --verbose -vv]
 
 if ARGV.include?('-h') || ARGV.include?('--help')
   abort <<~EOM
@@ -13,6 +13,7 @@ if ARGV.include?('-h') || ARGV.include?('--help')
     field in the package file.
     Passing --json or -j will only give json output.
     Passing --verbose or -v will display verbose output.
+    Passing -vv will display very verbose output.
   EOM
 end
 
@@ -40,7 +41,7 @@ $LOAD_PATH.unshift File.join(crew_local_repo_root, 'lib')
 
 OUTPUT_JSON = ARGV.include?('-j') || ARGV.include?('--json')
 UPDATE_PACKAGE_FILES = ARGV.include?('-u') || ARGV.include?('--update-package-files')
-VERBOSE = ARGV.include?('-v') || ARGV.include?('--verbose')
+VERBOSE = ARGV.include?('-v') || ARGV.include?('--verbose') || ARGV.include?('-vv')
 VERY_VERBOSE = ARGV.include?('-vv')
 bc_updated = {}
 @pkg_names = {}
@@ -49,11 +50,12 @@ version_line_string = {}
 versions_updated = {}
 versions = []
 
-def get_version(name, homepage, source)
+def get_version(name, homepage, source, version)
   anitya_name_mapping_idx = CREW_ANITYA_PACKAGE_NAME_MAPPINGS.keys.find_index { |i| i == name }
-  anitya_name = name.gsub(/\Apy\d_|\Aperl_|\Aruby_/, '').tr('_', '-')
+  anitya_name = name.gsub(/\Apy\d_|\Aperl_|\Aruby_/, '')
   anitya_name = CREW_ANITYA_PACKAGE_NAME_MAPPINGS.values[anitya_name_mapping_idx] unless anitya_name_mapping_idx.nil?
   anitya_id = get_anitya_id(anitya_name, homepage)
+  anitya_name = @new_anitya_name unless @new_anitya_name.nil?
   # If anitya_id cannot be determined, a Range can be returned, and
   # .nonzero? does not work with Ranges.
   anitya_id = nil if anitya_id.is_a? Range
@@ -61,6 +63,7 @@ def get_version(name, homepage, source)
   if anitya_id&.nonzero?
     # Get the latest stable version of the package from anitya.
     json = JSON.parse(Net::HTTP.get(URI("https://release-monitoring.org/api/v2/versions/?project_id=#{anitya_id}")))
+    puts json if VERY_VERBOSE
     return if json['stable_versions'].nil?
     return json['stable_versions'][0]
   elsif source.nil? || %w[Pip].include?(@pkg.superclass.to_s)
@@ -83,7 +86,9 @@ def get_version(name, homepage, source)
     end
     source.sub!('www.', '')
     url = URI.parse(source)
-    if url.host == 'github.com'
+    puts "source_url host: #{url.host}" if VERY_VERBOSE
+    case url.host
+    when 'github.com'
       url_parts = url.path.split('/')
       unless url_parts.count < 3
         repo = "#{url_parts[1]}/#{url_parts[2].gsub(/.git\z/, '')}"
@@ -91,12 +96,40 @@ def get_version(name, homepage, source)
         if File.which('gh')
           # This allows us to only get non-pre-release versions from
           # GitHub if such releases exist.
+          puts "gh release ls --exclude-pre-releases --exclude-drafts -L 1 -R #{repo} --json tagName -q '.[] | .tagName'" if VERY_VERBOSE
           github_ver = `gh release ls --exclude-pre-releases --exclude-drafts -L 1 -R #{repo} --json tagName -q '.[] | .tagName'`.chomp if system 'gh auth status >/dev/null', exception: false
         else
-          github_ver = `curl https://api.github.com/repos/#{repo}/releases/latest -s | jq .tag -r`.chomp
+          puts "curl https://api.github.com/repos/#{repo}/releases/latest -s | jq .tag_name -r" if VERY_VERBOSE
+          github_ver = `curl https://api.github.com/repos/#{repo}/releases/latest -s | jq .tag_name -r`.chomp
+          puts "curl https://api.github.com/repos/#{repo}/tags -s | jq '.[0].name' -r" if VERY_VERBOSE && (github_ver.blank? || github_ver == 'null')
+          github_ver = `curl https://api.github.com/repos/#{repo}/tags -s | jq '.[0].name' -r`.chomp if github_ver.blank? || github_ver == 'null'
         end
         return github_ver unless github_ver.blank? || github_ver == 'null'
       end
+    when 'gitlab.com'
+      url_parts = url.path.split('/')
+      unless url_parts.count < 3
+        repo = "#{url_parts[1]}/#{url_parts[2].gsub(/.git\z/, '')}"
+        puts "GitLab Repo is #{repo}" if VERBOSE
+        puts "curl https://#{url.host}/#{repo}/-/releases/permalink/latest -s | jq .tag_name -r" if VERY_VERBOSE
+        gitlab_ver = `curl https://#{url.host}/#{repo}/-/releases/permalink/latest -s | jq .tag_name -r`.chomp
+        return gitlab_ver unless gitlab_ver.blank? || gitlab_ver == 'null'
+      end
+    when 'downloads.sourceforge.net'
+      url_parts = url.path.split('/')
+      unless url_parts.count < 3
+        repo = url_parts[2]
+        filename = url_parts.last
+        puts "Sourceforge Repo is #{repo}" if VERBOSE
+        puts "curl -L -s https://sourceforge.net/projects/#{repo}/best_release.json | jq .release.filename -r" if VERY_VERBOSE
+        sourceforge_file = `curl -L -s https://sourceforge.net/projects/#{repo}/best_release.json | jq .release.filename -r`.chomp
+        best_release = sourceforge_file.split('/').last
+        if filename == best_release
+          return version
+        else
+          return best_release
+        end
+      end
     end
   end
 end
@@ -109,22 +142,40 @@ def get_anitya_id(name, homepage)
   # Find out how many packages Anitya has with the provided name.
   puts "url is https://release-monitoring.org/api/v2/projects/?name=#{CGI.escape(name)}" if VERY_VERBOSE
   json = JSON.parse(Net::HTTP.get(URI("https://release-monitoring.org/api/v2/projects/?name=#{CGI.escape(name)}")))
+  puts json if VERY_VERBOSE
   number_of_packages = json['total_items']
 
   puts "number_of_packages = #{number_of_packages}" if VERY_VERBOSE
   if number_of_packages == 1 # We assume we have the right package, take the ID and move on.
     return json['items'][0]['id']
   elsif number_of_packages.zero? # Anitya either doesn't have this package, or has it under a different name.
-    # If it has it under a different name, check if it has the name used by Chromebrew.
+    @new_anitya_name = nil
-    json2 = JSON.parse(Net::HTTP.get(URI("https://release-monitoring.org/api/v2/packages/?name=#{name.tr('-', '_')}")))
+    name_candidate = name.tr('-', '_') if name.include?('-')
-    return if json2['total_items'].zero?
+    name_candidate = name.tr('_', '-') if name.include?('_')
+    if name_candidate && name_candidate != name
+      if VERY_VERBOSE
+        puts "No Anitya package found with #{name}. Attempting a new search with #{name_candidate}."
+        puts "url is https://release-monitoring.org/api/v2/projects/?name=#{name_candidate}"
+      end
+      json = JSON.parse(Net::HTTP.get(URI("https://release-monitoring.org/api/v2/projects/?name=#{name_candidate}")))
+      number_of_packages = json['total_items']
+      if number_of_packages.zero?
+        puts "No Anitya package found with #{name_candidate}." if VERY_VERBOSE
+        return
+      end
 
-    (0..(json2['total_items'] - 1)).each do |i|
+      puts "number_of_packages = #{number_of_packages}" if VERY_VERBOSE
-      # If it has it under a different name, make sure that is the Chromebrew mapping, and not some other distribution,
+      (0..(number_of_packages - 1)).each do |i|
-      # because that could lead to overmatching.
+        next if json['items'][i].nil?
-      next unless json2['items'][i]['distribution'] == 'Chromebrew'
+        homepage_domain = homepage.gsub(%r{http(s)?://(www\.)?}, '').chomp('/')
-
+        puts "homepage_domain = #{homepage.gsub(%r{http(s)?://(www\.)?}, '').chomp('/')}" if VERY_VERBOSE
-      return get_anitya_id(json2['items'][i]['project'], homepage) if json2['items'][i]['name'] == name.tr('-', '_')
+        candidate_homepage_domain = json['items'][i]['homepage'].gsub(%r{http(s)?://(www\.)?}, '').chomp('/')
+        puts "candidate_homepage_domain = #{json['items'][i]['homepage'].gsub(%r{http(s)?://(www\.)?}, '').chomp('/')}" if VERY_VERBOSE
+        if homepage_domain == candidate_homepage_domain
+          @new_anitya_name = name_candidate
+          return json['items'][i]['id']
+        end
+      end
     end
   else # Anitya has more than one package with this exact name.
     candidates = []
@@ -142,6 +193,7 @@ def get_anitya_id(name, homepage)
         candidates.append(i)
       end
     end
+    puts "candidates = #{candidates}" if VERY_VERBOSE
 
     if candidates.length == 1 # If there's only one candidate left, we're done.
       return json['items'][candidates[0]]['id']
@@ -157,8 +209,13 @@ def get_anitya_id(name, homepage)
         # We assume there is only one candidate with the same name and homepage as their crew counterpart.
         # Even if there are multiple candidates with the same name and homepage, its probably fine to treat them as identical.
         # If it isn't fine to treat them as identical, something has gone horribly wrong.
-        return json['items'][candidate]['id'] if homepage.chomp('/') == json['items'][candidate]['homepage'].chomp('/')
+        homepage_domain = homepage.gsub(%r{http(s)?://(www\.)?}, '').chomp('/')
+        puts "homepage_domain = #{homepage.gsub(%r{http(s)?://(www\.)?}, '').chomp('/')}" if VERY_VERBOSE
+        candidate_homepage_domain = json['items'][candidate]['homepage'].gsub(%r{http(s)?://(www\.)?}, '').chomp('/')
+        puts "candidate_homepage_domain = #{json['items'][candidate]['homepage'].gsub(%r{http(s)?://(www\.)?}, '').chomp('/')}" if VERY_VERBOSE
+        return json['items'][candidate]['id'] if homepage_domain == candidate_homepage_domain
       end
+      puts 'no candidates found.' if VERY_VERBOSE
 
       # If we're still here, that means none of the candidates had the same homepage as their crew counterpart.
       # Not much we can do at this point to find the version, and its better to be cautious to avoid getting the wrong candidate.
@@ -198,8 +255,8 @@ if filelist.length.positive?
   status_field_length = 17
   version_field_length = 13
 
-  puts "#{'Package'.ljust(package_field_length)}#{'Status'.ljust(status_field_length)}#{'Current'.ljust(version_field_length)}#{'Upstream'.ljust(version_field_length)}Updatable?" unless OUTPUT_JSON
+  puts "#{'Package'.ljust(package_field_length)}#{'Status'.ljust(status_field_length)}#{'Current'.ljust(version_field_length)}#{'Upstream'.ljust(version_field_length)}#{'Updatable?'.ljust(version_field_length)}Compile?" unless OUTPUT_JSON
-  puts "#{'-------'.ljust(package_field_length)}#{'------'.ljust(status_field_length)}#{'-------'.ljust(version_field_length)}#{'--------'.ljust(version_field_length)}----------" unless OUTPUT_JSON
+  puts "#{'-------'.ljust(package_field_length)}#{'------'.ljust(status_field_length)}#{'-------'.ljust(version_field_length)}#{'--------'.ljust(version_field_length)}#{'----------'.ljust(version_field_length)}--------" unless OUTPUT_JSON
   filelist.each do |filename|
     @pkg = Package.load_package(filename)
     cleaned_pkg_version = PackageUtils.get_clean_version(@pkg.version)
@@ -281,7 +338,7 @@ if filelist.length.positive?
       end
     else
       # Get the upstream version.
-      upstream_version = get_version(@pkg.name, @pkg.homepage, @pkg.source_url)
+      upstream_version = get_version(@pkg.name, @pkg.homepage, @pkg.source_url, @pkg.version)
     end
     # Some packages don't work with this yet, so gracefully exit now rather than throwing false positives.
     versions_updated[@pkg.name.to_sym] = 'Not Found.' if upstream_version.nil? || upstream_version.to_s.chomp == 'null'
@@ -403,11 +460,12 @@ if filelist.length.positive?
     when 'Up to date.'
       version_status_string = 'Up to date.'.ljust(status_field_length).lightgreen
     end
-    updatable_string = (updatable_pkg[@pkg.name.to_sym] == 'Yes' ? 'Yes'.lightgreen : 'No'.lightred) if updatable_string.nil?
+    updatable_string = (updatable_pkg[@pkg.name.to_sym] == 'Yes' ? 'Yes'.ljust(version_field_length).lightgreen : 'No'.ljust(version_field_length).lightred) if updatable_string.nil?
+    compile_string = @pkg.no_compile_needed? ? 'No'.lightred : 'Yes'.lightgreen
     versions.push(package: @pkg.name, update_status: versions_updated[@pkg.name.to_sym], version: cleaned_pkg_version, upstream_version: upstream_version.chomp)
 
     addendum_string = "#{@pkg.name} cannot be automatically updated: ".red + "#{updatable_pkg[@pkg.name.to_sym]}\n".purple unless updatable_pkg[@pkg.name.to_sym] == 'Yes'
-    version_line_string[@pkg.name.to_sym] = "#{@pkg.name.ljust(package_field_length)}#{version_status_string}#{cleaned_pkg_version.ljust(version_field_length)}#{upstream_version.chomp.ljust(version_field_length)}#{updatable_string}\n"
+    version_line_string[@pkg.name.to_sym] = "#{@pkg.name.ljust(package_field_length)}#{version_status_string}#{cleaned_pkg_version.ljust(version_field_length)}#{upstream_version.chomp.ljust(version_field_length)}#{updatable_string}#{compile_string}\n"
     print version_line_string[@pkg.name.to_sym] if !OUTPUT_JSON && ((versions_updated[@pkg.name.to_sym] == 'Outdated.' && updatable_pkg[@pkg.name.to_sym] == 'Yes') || VERBOSE)
     print addendum_string unless addendum_string.blank? || OUTPUT_JSON || !VERBOSE