diff --git a/bin/crew b/bin/crew index 0be52128a..0e67833d9 100755 --- a/bin/crew +++ b/bin/crew @@ -91,6 +91,7 @@ String.use_color = args['--color'] || !args['--no-color'] @update = @opt_update ? '-u' : '' @verbose = CREW_VERBOSE ? 'v' : '' @short_verbose = CREW_VERBOSE ? '-v' : '' +@very_verbose = CREW_VERY_VERBOSE ? '-vv' : '' # Make sure crew work directories exist. FileUtils.mkdir_p CREW_BREW_DIR @@ -2150,7 +2151,7 @@ def upstream_command(args) # Pass the whole argument input to tools/version.rb, stripping out any file extensions that occur as a result of using a wildcard argument in a directory where it matches files. args = { '' => args.split } if args.is_a? String Dir.chdir CREW_PACKAGES_PATH do - system "../tools/version.rb #{args[''].join(' ').gsub('.rb', '')} #{@json} #{@update} #{@short_verbose}" + system "../tools/version.rb #{args[''].join(' ').gsub('.rb', '')} #{@json} #{@update} #{@short_verbose} #{@very_verbose}" end end diff --git a/commands/help.rb b/commands/help.rb index 2c55b1f16..3b66f7b29 100644 --- a/commands/help.rb +++ b/commands/help.rb @@ -162,10 +162,11 @@ class Command when 'upstream' puts <<~EOT Check if an upstream version is available for package(s). - Usage: crew upstream [-j|--json|-u|--update-package-files|-v|--verbose] [ ...] + Usage: crew upstream [-j|--json|-u|--update-package-files|-v|--verbose|-vv] [ ...] If `-j` or `--json` is present, output will be in json format. If `-u` or `--update-package-files` is present, it will attempt to update the package version. If `-v` or `--verbose` is present, extra information will be displayed. + If `-vv` is present, very verbose information will be displayed. EOT when 'version' puts <<~EOT diff --git a/lib/const.rb b/lib/const.rb index 410743f50..6c8763efc 100644 --- a/lib/const.rb +++ b/lib/const.rb @@ -4,7 +4,7 @@ require 'etc' require 'open3' OLD_CREW_VERSION ||= defined?(CREW_VERSION) ? CREW_VERSION : '1.0' -CREW_VERSION ||= '1.67.9' unless defined?(CREW_VERSION) && CREW_VERSION == OLD_CREW_VERSION +CREW_VERSION ||= '1.67.10' unless defined?(CREW_VERSION) && CREW_VERSION == OLD_CREW_VERSION # Kernel architecture. KERN_ARCH ||= Etc.uname[:machine] @@ -133,15 +133,16 @@ CREW_MUSL_PREFIX ||= File.join(CREW_PREFIX, '/share/musl/') CREW_DEST_MUSL_PREFIX ||= File.join(CREW_DEST_DIR, CREW_MUSL_PREFIX) MUSL_LIBC_VERSION ||= File.executable?("#{CREW_MUSL_PREFIX}/lib/libc.so") ? `#{CREW_MUSL_PREFIX}/lib/libc.so 2>&1`[/\bVersion\s+\K\S+/] : nil unless defined?(MUSL_LIBC_VERSION) -CREW_DEST_HOME ||= File.join(CREW_DEST_DIR, HOME) -CREW_NO_GIT ||= ENV.fetch('CREW_NO_GIT', false) unless defined?(CREW_NO_GIT) -CREW_UNATTENDED ||= ENV.fetch('CREW_UNATTENDED', false) unless defined?(CREW_UNATTENDED) +CREW_DEST_HOME ||= File.join(CREW_DEST_DIR, HOME) +CREW_NO_GIT ||= ENV.fetch('CREW_NO_GIT', false) unless defined?(CREW_NO_GIT) +CREW_UNATTENDED ||= ENV.fetch('CREW_UNATTENDED', false) unless defined?(CREW_UNATTENDED) CREW_STANDALONE_UPGRADE_ORDER = %w[libxcrypt crew_preload glibc openssl ruby python3 perl icu4c sommelier] unless defined?(CREW_STANDALONE_UPGRADE_ORDER) -CREW_DEBUG ||= ARGV.intersect?(%w[-D --debug]) unless defined?(CREW_DEBUG) -CREW_FORCE ||= ARGV.intersect?(%w[-f --force]) unless defined?(CREW_FORCE) -CREW_VERBOSE ||= ARGV.intersect?(%w[-v --verbose]) unless defined?(CREW_VERBOSE) +CREW_DEBUG ||= ARGV.include?('-D') || ARGV.include?('--debug') unless defined?(CREW_DEBUG) +CREW_FORCE ||= ARGV.include?('-f') || ARGV.include?('--force') unless defined?(CREW_FORCE) +CREW_VERBOSE ||= ARGV.include?('-v') || ARGV.include?('--verbose') || ARGV.include?('-vv') unless defined?(CREW_VERBOSE) +CREW_VERY_VERBOSE ||= ARGV.include?('-vv') unless defined?(CREW_VERY_VERBOSE) # Set CREW_NPROC from environment variable, `distcc -j`, or `nproc`. CREW_NPROC ||= @@ -421,6 +422,7 @@ unless defined?(CREW_ANITYA_PACKAGE_NAME_MAPPINGS) { pkg_name: 'signal_desktop', anitya_pkg: 'signal', comments: '' }, { pkg_name: 'tepl_6', anitya_pkg: 'libgedit-tepl', comments: '' }, { pkg_name: 'upx', anitya_pkg: 'upx', comments: 'Prefer to GitHub' }, + { pkg_name: 'vidstab', anitya_pkg: 'vid.stab', comments: 'Prefer to GitHub' }, { pkg_name: 'vim_runtime', anitya_pkg: 'vim', comments: '' }, { pkg_name: 'webkitgtk_6', anitya_pkg: 'webkitgtk~stable', comments: '' }, { pkg_name: 'xauth', anitya_pkg: 'xorg-x11-xauth', comments: '' }, @@ -476,7 +478,7 @@ CREW_DOCOPT ||= <<~DOCOPT crew update_package_file [options] [-v|--verbose] [ ...] crew upgrade [options] [-f|--force] [-k|--keep] [--regenerate-filelist] [-s|--source] [-v|--verbose] [ ...] crew upload [options] [-f|--force] [-v|--verbose] [ ...] - crew upstream [options] [-j|--json|-u|--update-package-files|-v|--verbose] ... + crew upstream [options] [-j|--json|-u|--update-package-files|-v|--verbose|-vv] ... crew version [options] [] crew whatprovides [options] ... diff --git a/tools/version.rb b/tools/version.rb index 81b09d43e..4aab10db3 100755 --- a/tools/version.rb +++ b/tools/version.rb @@ -1,7 +1,7 @@ #!/usr/bin/env ruby -# version.rb version 3.6 (for Chromebrew) +# version.rb version 3.7 (for Chromebrew) -OPTIONS = %w[-h --help -j --json -u --update-package-files -v --verbose] +OPTIONS = %w[-h --help -j --json -u --update-package-files -v --verbose -vv] if ARGV.include?('-h') || ARGV.include?('--help') abort <<~EOM @@ -13,6 +13,7 @@ if ARGV.include?('-h') || ARGV.include?('--help') field in the package file. Passing --json or -j will only give json output. Passing --verbose or -v will display verbose output. + Passing -vv will display very verbose output. EOM end @@ -40,7 +41,7 @@ $LOAD_PATH.unshift File.join(crew_local_repo_root, 'lib') OUTPUT_JSON = ARGV.include?('-j') || ARGV.include?('--json') UPDATE_PACKAGE_FILES = ARGV.include?('-u') || ARGV.include?('--update-package-files') -VERBOSE = ARGV.include?('-v') || ARGV.include?('--verbose') +VERBOSE = ARGV.include?('-v') || ARGV.include?('--verbose') || ARGV.include?('-vv') VERY_VERBOSE = ARGV.include?('-vv') bc_updated = {} @pkg_names = {} @@ -49,11 +50,12 @@ version_line_string = {} versions_updated = {} versions = [] -def get_version(name, homepage, source) +def get_version(name, homepage, source, version) anitya_name_mapping_idx = CREW_ANITYA_PACKAGE_NAME_MAPPINGS.keys.find_index { |i| i == name } - anitya_name = name.gsub(/\Apy\d_|\Aperl_|\Aruby_/, '').tr('_', '-') + anitya_name = name.gsub(/\Apy\d_|\Aperl_|\Aruby_/, '') anitya_name = CREW_ANITYA_PACKAGE_NAME_MAPPINGS.values[anitya_name_mapping_idx] unless anitya_name_mapping_idx.nil? anitya_id = get_anitya_id(anitya_name, homepage) + anitya_name = @new_anitya_name unless @new_anitya_name.nil? # If anitya_id cannot be determined, a Range can be returned, and # .nonzero? does not work with Ranges. anitya_id = nil if anitya_id.is_a? Range @@ -61,6 +63,7 @@ def get_version(name, homepage, source) if anitya_id&.nonzero? # Get the latest stable version of the package from anitya. json = JSON.parse(Net::HTTP.get(URI("https://release-monitoring.org/api/v2/versions/?project_id=#{anitya_id}"))) + puts json if VERY_VERBOSE return if json['stable_versions'].nil? return json['stable_versions'][0] elsif source.nil? || %w[Pip].include?(@pkg.superclass.to_s) @@ -83,7 +86,9 @@ def get_version(name, homepage, source) end source.sub!('www.', '') url = URI.parse(source) - if url.host == 'github.com' + puts "source_url host: #{url.host}" if VERY_VERBOSE + case url.host + when 'github.com' url_parts = url.path.split('/') unless url_parts.count < 3 repo = "#{url_parts[1]}/#{url_parts[2].gsub(/.git\z/, '')}" @@ -91,12 +96,40 @@ def get_version(name, homepage, source) if File.which('gh') # This allows us to only get non-pre-release versions from # GitHub if such releases exist. + puts "gh release ls --exclude-pre-releases --exclude-drafts -L 1 -R #{repo} --json tagName -q '.[] | .tagName'" if VERY_VERBOSE github_ver = `gh release ls --exclude-pre-releases --exclude-drafts -L 1 -R #{repo} --json tagName -q '.[] | .tagName'`.chomp if system 'gh auth status >/dev/null', exception: false else - github_ver = `curl https://api.github.com/repos/#{repo}/releases/latest -s | jq .tag -r`.chomp + puts "curl https://api.github.com/repos/#{repo}/releases/latest -s | jq .tag_name -r" if VERY_VERBOSE + github_ver = `curl https://api.github.com/repos/#{repo}/releases/latest -s | jq .tag_name -r`.chomp + puts "curl https://api.github.com/repos/#{repo}/tags -s | jq '.[0].name' -r" if VERY_VERBOSE && (github_ver.blank? || github_ver == 'null') + github_ver = `curl https://api.github.com/repos/#{repo}/tags -s | jq '.[0].name' -r`.chomp if github_ver.blank? || github_ver == 'null' end return github_ver unless github_ver.blank? || github_ver == 'null' end + when 'gitlab.com' + url_parts = url.path.split('/') + unless url_parts.count < 3 + repo = "#{url_parts[1]}/#{url_parts[2].gsub(/.git\z/, '')}" + puts "GitLab Repo is #{repo}" if VERBOSE + puts "curl https://#{url.host}/#{repo}/-/releases/permalink/latest -s | jq .tag_name -r" if VERY_VERBOSE + gitlab_ver = `curl https://#{url.host}/#{repo}/-/releases/permalink/latest -s | jq .tag_name -r`.chomp + return gitlab_ver unless gitlab_ver.blank? || gitlab_ver == 'null' + end + when 'downloads.sourceforge.net' + url_parts = url.path.split('/') + unless url_parts.count < 3 + repo = url_parts[2] + filename = url_parts.last + puts "Sourceforge Repo is #{repo}" if VERBOSE + puts "curl -L -s https://sourceforge.net/projects/#{repo}/best_release.json | jq .release.filename -r" if VERY_VERBOSE + sourceforge_file = `curl -L -s https://sourceforge.net/projects/#{repo}/best_release.json | jq .release.filename -r`.chomp + best_release = sourceforge_file.split('/').last + if filename == best_release + return version + else + return best_release + end + end end end end @@ -109,22 +142,40 @@ def get_anitya_id(name, homepage) # Find out how many packages Anitya has with the provided name. puts "url is https://release-monitoring.org/api/v2/projects/?name=#{CGI.escape(name)}" if VERY_VERBOSE json = JSON.parse(Net::HTTP.get(URI("https://release-monitoring.org/api/v2/projects/?name=#{CGI.escape(name)}"))) + puts json if VERY_VERBOSE number_of_packages = json['total_items'] puts "number_of_packages = #{number_of_packages}" if VERY_VERBOSE if number_of_packages == 1 # We assume we have the right package, take the ID and move on. return json['items'][0]['id'] elsif number_of_packages.zero? # Anitya either doesn't have this package, or has it under a different name. - # If it has it under a different name, check if it has the name used by Chromebrew. - json2 = JSON.parse(Net::HTTP.get(URI("https://release-monitoring.org/api/v2/packages/?name=#{name.tr('-', '_')}"))) - return if json2['total_items'].zero? + @new_anitya_name = nil + name_candidate = name.tr('-', '_') if name.include?('-') + name_candidate = name.tr('_', '-') if name.include?('_') + if name_candidate && name_candidate != name + if VERY_VERBOSE + puts "No Anitya package found with #{name}. Attempting a new search with #{name_candidate}." + puts "url is https://release-monitoring.org/api/v2/projects/?name=#{name_candidate}" + end + json = JSON.parse(Net::HTTP.get(URI("https://release-monitoring.org/api/v2/projects/?name=#{name_candidate}"))) + number_of_packages = json['total_items'] + if number_of_packages.zero? + puts "No Anitya package found with #{name_candidate}." if VERY_VERBOSE + return + end - (0..(json2['total_items'] - 1)).each do |i| - # If it has it under a different name, make sure that is the Chromebrew mapping, and not some other distribution, - # because that could lead to overmatching. - next unless json2['items'][i]['distribution'] == 'Chromebrew' - - return get_anitya_id(json2['items'][i]['project'], homepage) if json2['items'][i]['name'] == name.tr('-', '_') + puts "number_of_packages = #{number_of_packages}" if VERY_VERBOSE + (0..(number_of_packages - 1)).each do |i| + next if json['items'][i].nil? + homepage_domain = homepage.gsub(%r{http(s)?://(www\.)?}, '').chomp('/') + puts "homepage_domain = #{homepage.gsub(%r{http(s)?://(www\.)?}, '').chomp('/')}" if VERY_VERBOSE + candidate_homepage_domain = json['items'][i]['homepage'].gsub(%r{http(s)?://(www\.)?}, '').chomp('/') + puts "candidate_homepage_domain = #{json['items'][i]['homepage'].gsub(%r{http(s)?://(www\.)?}, '').chomp('/')}" if VERY_VERBOSE + if homepage_domain == candidate_homepage_domain + @new_anitya_name = name_candidate + return json['items'][i]['id'] + end + end end else # Anitya has more than one package with this exact name. candidates = [] @@ -142,6 +193,7 @@ def get_anitya_id(name, homepage) candidates.append(i) end end + puts "candidates = #{candidates}" if VERY_VERBOSE if candidates.length == 1 # If there's only one candidate left, we're done. return json['items'][candidates[0]]['id'] @@ -157,8 +209,13 @@ def get_anitya_id(name, homepage) # We assume there is only one candidate with the same name and homepage as their crew counterpart. # Even if there are multiple candidates with the same name and homepage, its probably fine to treat them as identical. # If it isn't fine to treat them as identical, something has gone horribly wrong. - return json['items'][candidate]['id'] if homepage.chomp('/') == json['items'][candidate]['homepage'].chomp('/') + homepage_domain = homepage.gsub(%r{http(s)?://(www\.)?}, '').chomp('/') + puts "homepage_domain = #{homepage.gsub(%r{http(s)?://(www\.)?}, '').chomp('/')}" if VERY_VERBOSE + candidate_homepage_domain = json['items'][candidate]['homepage'].gsub(%r{http(s)?://(www\.)?}, '').chomp('/') + puts "candidate_homepage_domain = #{json['items'][candidate]['homepage'].gsub(%r{http(s)?://(www\.)?}, '').chomp('/')}" if VERY_VERBOSE + return json['items'][candidate]['id'] if homepage_domain == candidate_homepage_domain end + puts 'no candidates found.' if VERY_VERBOSE # If we're still here, that means none of the candidates had the same homepage as their crew counterpart. # Not much we can do at this point to find the version, and its better to be cautious to avoid getting the wrong candidate. @@ -198,8 +255,8 @@ if filelist.length.positive? status_field_length = 17 version_field_length = 13 - puts "#{'Package'.ljust(package_field_length)}#{'Status'.ljust(status_field_length)}#{'Current'.ljust(version_field_length)}#{'Upstream'.ljust(version_field_length)}Updatable?" unless OUTPUT_JSON - puts "#{'-------'.ljust(package_field_length)}#{'------'.ljust(status_field_length)}#{'-------'.ljust(version_field_length)}#{'--------'.ljust(version_field_length)}----------" unless OUTPUT_JSON + puts "#{'Package'.ljust(package_field_length)}#{'Status'.ljust(status_field_length)}#{'Current'.ljust(version_field_length)}#{'Upstream'.ljust(version_field_length)}#{'Updatable?'.ljust(version_field_length)}Compile?" unless OUTPUT_JSON + puts "#{'-------'.ljust(package_field_length)}#{'------'.ljust(status_field_length)}#{'-------'.ljust(version_field_length)}#{'--------'.ljust(version_field_length)}#{'----------'.ljust(version_field_length)}--------" unless OUTPUT_JSON filelist.each do |filename| @pkg = Package.load_package(filename) cleaned_pkg_version = PackageUtils.get_clean_version(@pkg.version) @@ -281,7 +338,7 @@ if filelist.length.positive? end else # Get the upstream version. - upstream_version = get_version(@pkg.name, @pkg.homepage, @pkg.source_url) + upstream_version = get_version(@pkg.name, @pkg.homepage, @pkg.source_url, @pkg.version) end # Some packages don't work with this yet, so gracefully exit now rather than throwing false positives. versions_updated[@pkg.name.to_sym] = 'Not Found.' if upstream_version.nil? || upstream_version.to_s.chomp == 'null' @@ -403,11 +460,12 @@ if filelist.length.positive? when 'Up to date.' version_status_string = 'Up to date.'.ljust(status_field_length).lightgreen end - updatable_string = (updatable_pkg[@pkg.name.to_sym] == 'Yes' ? 'Yes'.lightgreen : 'No'.lightred) if updatable_string.nil? + updatable_string = (updatable_pkg[@pkg.name.to_sym] == 'Yes' ? 'Yes'.ljust(version_field_length).lightgreen : 'No'.ljust(version_field_length).lightred) if updatable_string.nil? + compile_string = @pkg.no_compile_needed? ? 'No'.lightred : 'Yes'.lightgreen versions.push(package: @pkg.name, update_status: versions_updated[@pkg.name.to_sym], version: cleaned_pkg_version, upstream_version: upstream_version.chomp) addendum_string = "#{@pkg.name} cannot be automatically updated: ".red + "#{updatable_pkg[@pkg.name.to_sym]}\n".purple unless updatable_pkg[@pkg.name.to_sym] == 'Yes' - version_line_string[@pkg.name.to_sym] = "#{@pkg.name.ljust(package_field_length)}#{version_status_string}#{cleaned_pkg_version.ljust(version_field_length)}#{upstream_version.chomp.ljust(version_field_length)}#{updatable_string}\n" + version_line_string[@pkg.name.to_sym] = "#{@pkg.name.ljust(package_field_length)}#{version_status_string}#{cleaned_pkg_version.ljust(version_field_length)}#{upstream_version.chomp.ljust(version_field_length)}#{updatable_string}#{compile_string}\n" print version_line_string[@pkg.name.to_sym] if !OUTPUT_JSON && ((versions_updated[@pkg.name.to_sym] == 'Outdated.' && updatable_pkg[@pkg.name.to_sym] == 'Yes') || VERBOSE) print addendum_string unless addendum_string.blank? || OUTPUT_JSON || !VERBOSE