diff --git a/tests/tools/version.rb b/tests/tools/version.rb index eb4835eb83..4229dea749 100644 --- a/tests/tools/version.rb +++ b/tests/tools/version.rb @@ -127,7 +127,7 @@ class VersionMonitorTest < Minitest::Test update_package_test_wrapper(input_file, '1.0', input_file.sub('1.16.1', '1.0')) end - def test_update_package_file_binary_compression + def test_update_package_file_xz_binary_compression input_file = <<~EOF class Bar < Package version '3.65.0' @@ -138,6 +138,30 @@ class VersionMonitorTest < Minitest::Test update_package_test_wrapper(input_file, '3.65.0', input_file.sub("binary_compression 'tar.xz'", "binary_compression 'tar.zst'"), bc_updated: true) end + def test_update_package_file_xz_binary_compression_no_zstd + input_file = <<~EOF + class Foobarr < Package + version '1.0.0' + binary_compression 'tar.xz' + + no_zstd + end + EOF + + update_package_test_wrapper(input_file, '1.0.0', input_file) + end + + def test_update_package_file_tpxz_binary_compression + input_file = <<~EOF + class Frob < Package + version '1ca' + binary_compression 'tpxz' + end + EOF + + update_package_test_wrapper(input_file, '1ca', input_file.sub("binary_compression 'tpxz'", "binary_compression 'tar.zst'"), bc_updated: true) + end + def test_update_package_file_version_source_sha256 input_file = <<~'EOF' class Qux < Package @@ -171,7 +195,6 @@ class VersionMonitorTest < Minitest::Test update_package_test_wrapper(input_file, '0.7.2', input_file.sub('0.5.0', '0.7.2').sub('49a0a2c75a464f35b17c2254f979e48b460350ad3eccffdec53f9ee746950950', '29c7291985ad391fc8af930ba89c7441d5764aa3415ef1d77171aea0b34d35b9').sub('7d4c073a0342cf39bdb99c32b4749f1c022cf2cffdfb080c12c106aa9d341708', '70423609f27b504d6c0c47e340f33652aea975e45f312324f2dbf91c95a3b188')) end - # TODO: Why is this the only one that returns a false bc_updated value? def test_update_package_file_version_bad_source_sha256 input_file = <<~'EOF' class Quuux < Package @@ -182,18 +205,17 @@ class VersionMonitorTest < Minitest::Test end EOF - update_package_test_wrapper(input_file, '9.1.8', input_file, version_updated: 'Bad Source', bc_updated: false) + update_package_test_wrapper(input_file, '9.1.8', input_file, version_updated: 'Bad Source', bc_updated: nil) end - # TODO: Why doesn't this have the same behavior as the single source_url case? def test_update_package_file_version_bad_multi_source_sha256 input_file = <<~'EOF' class Quuuux < Package version '1.4.0' binary_compression 'tar.zst' source_url({ - armv7l: "ftp://example.hats/ia32-#{version}.tar.bz3", - i686: "ftp://example.hats/strongarmv7-#{version}.tar.bz3" + armv7l: "http://example.net/ia32-#{version}.tar.bz3", + i686: "http://example.net/strongarmv7-#{version}.tar.bz3" }) source_sha256({ armv7l: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', @@ -202,6 +224,6 @@ class VersionMonitorTest < Minitest::Test end EOF - update_package_test_wrapper(input_file, '2.7.0', input_file.sub('1.4.0', '2.7.0')) + update_package_test_wrapper(input_file, '2.7.0', input_file, version_updated: 'Bad Source', bc_updated: nil) end end diff --git a/tools/version.rb b/tools/version.rb index d7944351fe..875d730694 100755 --- a/tools/version.rb +++ b/tools/version.rb @@ -257,95 +257,69 @@ def get_anitya_id(name, homepage, buildsystem) end def update_package_file(filename, upstream_version) - @pkg = Package.load_package(filename, true) - - versions_updated = {} - bc_updated = {} - + pkg = Package.load_package(filename) file = File.read(filename) - FileUtils.cp filename, "#{filename}.bak" - if file.sub!(PackageUtils.get_clean_version(@pkg.version), upstream_version).nil? - versions_updated[@pkg.name.to_sym] = false - else - # Version update succeeded. Now check for a sha256 update. - old_hash = {} - new_hash = {} - # Handle source_url whether hash or not. - if !@pkg.source_sha256.nil? && @pkg.source_sha256.is_a?(Hash) && @pkg.source_sha256&.key?(ARCH.to_sym) - # Get old hashes - (@pkg.source_url.keys.map &:to_s).each do |arch| - puts "old source_url: #{@pkg.source_url[arch.to_sym]}" if CREW_VERBOSE && !CREW_OUTPUT_JSON - old_hash[arch] = @pkg.source_sha256[arch.to_sym] - puts "old hash: #{old_hash[arch]}" if CREW_VERBOSE && !CREW_OUTPUT_JSON - end - File.write(filename, file) - @pkg = Package.load_package(filename, true) - (@pkg.source_url.keys.map &:to_s).each do |arch| - puts "new source_url: #{@pkg.source_url[arch.to_sym]}" if CREW_VERBOSE && !CREW_OUTPUT_JSON - status = `curl -fsI #{@pkg.source_url[arch.to_sym]}`.lines.first.split[1] - puts "new source_url response status: #{status}" if CREW_VERBOSE && !CREW_OUTPUT_JSON - unless %w[200 302].include?(status) - versions_updated[@pkg.name.to_sym] = 'Bad Source' - puts "#{@pkg.source_url[arch.to_sym]} is a bad source".lightred if CREW_VERBOSE && !CREW_OUTPUT_JSON - if File.file?("#{filename}.bak") - FileUtils.cp "#{filename}.bak", filename - FileUtils.rm "#{filename}.bak" - end - return versions_updated[@pkg.name.to_sym], false - end - new_hash[arch] = `curl -Ls #{@pkg.source_url[arch.to_sym]} | sha256sum - | awk '{print $1}'`.chomp - puts "new hash: #{new_hash[arch]}" if CREW_VERBOSE && !CREW_OUTPUT_JSON - file.sub!(old_hash[arch], new_hash[arch]) - end - elsif !@pkg.source_sha256.nil? && !@pkg.source_sha256.is_a?(Hash) - arch = :all - # Get old hashes - old_hash[arch] = @pkg.source_sha256 - puts "old source_url: #{@pkg.source_url}" if CREW_VERBOSE && !CREW_OUTPUT_JSON - puts "old hash: #{old_hash[arch]}" if CREW_VERBOSE && !CREW_OUTPUT_JSON - File.write(filename, file) - # Now get new hashes - @pkg = Package.load_package(filename, true) - puts "new source_url: #{@pkg.source_url}" if CREW_VERBOSE && !CREW_OUTPUT_JSON - status = `curl -fsI #{@pkg.source_url}`.lines.first.split[1] - unless %w[200 302].include?(status) - versions_updated[@pkg.name.to_sym] = 'Bad Source' - puts "#{@pkg.source_url} is a bad source.".lightred if CREW_VERBOSE && !CREW_OUTPUT_JSON - if File.file?("#{filename}.bak") - FileUtils.cp "#{filename}.bak", filename - FileUtils.rm "#{filename}.bak" - end - return versions_updated[@pkg.name.to_sym], false - end - new_hash[arch] = `curl -Ls #{@pkg.source_url} | sha256sum - | awk '{print $1}'`.chomp - puts "new hash: #{new_hash[arch]}" if CREW_VERBOSE && !CREW_OUTPUT_JSON - file.sub!(old_hash[arch], new_hash[arch]) - end - File.write(filename, file) - puts "Successfully updated #{filename} to version #{upstream_version}.".lightgreen - local_repo_root = '' - Dir.chdir(ENV.fetch('PWD', nil)) do - local_repo_root = `git rev-parse --show-toplevel 2> /dev/null`.chomp - end - if local_repo_root && File.file?("#{local_repo_root}/packages/#{@pkg.name}.rb") && (Pathname.new(filename).realpath.to_s != "#{local_repo_root}/packages/#{@pkg.name}.rb") - FileUtils.cp filename, "#{local_repo_root}/packages/#{@pkg.name}.rb" - puts "Successfully updated #{local_repo_root}/packages/#{@pkg.name}.rb to version #{upstream_version}.".lightgreen - end - versions_updated[@pkg.name.to_sym] = true - FileUtils.rm "#{filename}.bak" if File.file?("#{filename}.bak") - end - if @pkg.binary_compression == 'tar.xz' && !@pkg.no_zstd? - file = File.read(filename) - if file.sub!("binary_compression 'tar.xz'", "binary_compression 'tar.zst'").nil? - bc_updated[@pkg.name.to_sym] = false + # This doesn't handle the case where a package has a tpxz binary compression and is no_zstd and so we would want to change it to tar.xz, because we only have 61 packages with a tpxz binary compression and none of them are no_zstd. + if (pkg.binary_compression == 'tar.xz' && !pkg.no_zstd?) || pkg.binary_compression == 'tpxz' + if file.sub!(/binary_compression '.*'/, "binary_compression 'tar.zst'").nil? + bc_updated = false else File.write(filename, file) - bc_updated[@pkg.name.to_sym] = true + bc_updated = true end end - return versions_updated[@pkg.name.to_sym], bc_updated[@pkg.name.to_sym] + # Update the version in the loaded file, exiting early if it failed. + return false, bc_updated if file.sub!(PackageUtils.get_clean_version(pkg.version), upstream_version).nil? + + # Version update succeeded. Now check for a sha256 update. + unless pkg.source_sha256.nil? + # Get old hashes + old_source_url = pkg.source_url.is_a?(Hash) ? pkg.source_url.values : [pkg.source_url] + old_source_sha256 = pkg.source_sha256.is_a?(Hash) ? pkg.source_sha256.values : [pkg.source_sha256] + old_source_url.zip(old_source_sha256).each do |url, sha256| + puts "old source_url: #{url}" if CREW_VERBOSE && !CREW_OUTPUT_JSON + puts "old source_sha256: #{sha256}" if CREW_VERBOSE && !CREW_OUTPUT_JSON + end + + # Now get new hashes + # This essentially replicates Package.load_package, but this way we don't have to make an early write to the actual package file before we confirm all the modifications are correct. + Package.class_eval(file) + pkg = Package.const_get(File.basename(filename, '.rb').capitalize) + + new_source_url = pkg.source_url.is_a?(Hash) ? pkg.source_url.values : [pkg.source_url] + new_source_url.zip(old_source_sha256).each do |url, old_sha256| + puts "new source_url: #{url}" if CREW_VERBOSE && !CREW_OUTPUT_JSON + # Download the URL, following redirects until we reach the final destination. + response = Net::HTTP.get_response(URI.parse(url)) + loop do + break unless response.is_a?(Net::HTTPRedirection) + url = response['location'] + response = Net::HTTP.get_response(URI.parse(url)) + end + puts "new source_url response status: #{response.code}" if CREW_VERBOSE && !CREW_OUTPUT_JSON + unless response.code == '200' + puts "#{url} is a bad source".lightred if CREW_VERBOSE && !CREW_OUTPUT_JSON + return 'Bad Source', bc_updated + end + new_sha256 = Digest::SHA256.hexdigest(response.body) + puts "new source_sha256: #{new_source_sha256}" if CREW_VERBOSE && !CREW_OUTPUT_JSON + file.sub!(old_sha256, new_sha256) + end + end + File.write(filename, file) + puts "Successfully updated #{filename} to version #{upstream_version}.".lightgreen + local_repo_root = '' + Dir.chdir(ENV.fetch('PWD', nil)) do + local_repo_root = `git rev-parse --show-toplevel 2> /dev/null`.chomp + end + if local_repo_root && File.file?("#{local_repo_root}/packages/#{pkg.name}.rb") && (Pathname.new(filename).realpath.to_s != "#{local_repo_root}/packages/#{pkg.name}.rb") + FileUtils.cp filename, "#{local_repo_root}/packages/#{pkg.name}.rb" + puts "Successfully updated #{local_repo_root}/packages/#{pkg.name}.rb to version #{upstream_version}.".lightgreen + end + + return true, bc_updated end # If we have been required from another file (i.e. for testing) don't run any of this, as we're only interested in the functions up above. @@ -466,16 +440,13 @@ if filelist.length.positive? crewlog "Libversion.version_compare2(PackageUtils.get_clean_version(@pkg.version), upstream_version): #{Libversion.version_compare2(PackageUtils.get_clean_version(@pkg.version), upstream_version)}" crewlog "Libversion.version_compare2(PackageUtils.get_clean_version(@pkg.version), upstream_version) >= 0: #{Libversion.version_compare2(PackageUtils.get_clean_version(@pkg.version), upstream_version) >= 0}" end + versions_updated[@pkg.name.to_sym], bc_updated[@pkg.name.to_sym] = update_package_file(filename, upstream_version) if (Libversion.version_compare2(PackageUtils.get_clean_version(@pkg.version), upstream_version) == -1) && UPDATE_PACKAGE_FILES && !CREW_UPDATER_EXCLUDED_PKGS.key?(@pkg.name) && updatable_pkg[@pkg.name.to_sym] == 'Yes' + versions_updated[@pkg.name.to_sym] = if UPDATE_PACKAGE_FILES && !CREW_UPDATER_EXCLUDED_PKGS.key?(@pkg.name) && versions_updated[@pkg.name.to_sym] + 'Updated.' + else + CREW_UPDATER_EXCLUDED_PKGS.key?(@pkg.name) ? 'Update manually.' : 'Outdated.' + end versions_updated[@pkg.name.to_sym] = 'Up to date.' if Libversion.version_compare2(PackageUtils.get_clean_version(@pkg.version), upstream_version) >= 0 - if (Libversion.version_compare2(PackageUtils.get_clean_version(@pkg.version), upstream_version) == -1) && UPDATE_PACKAGE_FILES && !CREW_UPDATER_EXCLUDED_PKGS.key?(@pkg.name) && updatable_pkg[@pkg.name.to_sym] == 'Yes' - versions_updated[@pkg.name.to_sym], bc_updated[@pkg.name.to_sym] = update_package_file(filename, upstream_version) - - versions_updated[@pkg.name.to_sym] = if UPDATE_PACKAGE_FILES && !CREW_UPDATER_EXCLUDED_PKGS.key?(@pkg.name) && versions_updated[@pkg.name.to_sym] - 'Updated.' - else - CREW_UPDATER_EXCLUDED_PKGS.key?(@pkg.name) ? 'Update manually.' : 'Outdated.' - end - end end version_status_string = ''.ljust(status_field_length) updatable_string = nil