diff --git a/bin/crew b/bin/crew index c52f02dd9..9fb60f9e2 100755 --- a/bin/crew +++ b/bin/crew @@ -435,6 +435,8 @@ def download uri = URI.parse url filename = File.basename(uri.path) + # # If we're downloading a binary, reset the filename to what it would have been if we didn't download from the API. + filename = "#{@pkg.name}-#{@pkg.version}-chromeos-#{ARCH}.#{@pkg.binary_compression}" if filename.eql?('download') sha256sum = PackageUtils.get_sha256(@pkg, build_from_source: @opt_source || @pkg.build_from_source) @extract_dir = "#{@pkg.name}.#{Time.now.utc.strftime('%Y%m%d%H%M%S')}.dir" @@ -1448,13 +1450,6 @@ def upload(pkg_name = nil, pkg_version = nil, gitlab_token = nil, binary_compres new_url = "#{base_url}/#{package}/#{new_version}_#{arch}/#{new_tarfile}".gsub("#{release_dir}/", '') token_label = gitlab_token.split('-').first == 'glpat' ? 'PRIVATE-TOKEN' : 'DEPLOY-TOKEN' - if `curl -sI #{new_url}`.lines.first.split[1] == '200' - puts "\n#{new_tarfile} has already been uploaded.\nPlease change the #{package} package version from #{new_version} and try again.\n".lightred - unless Package.agree_default_no('Do you want to overwrite the existing upload instead') - puts 'Will NOT overwite the existing upload.'.orange - next - end - end puts "curl -# --header \"#{token_label}: #{gitlab_token}\" --upload-file \"#{new_tarfile}\" \"#{new_url}\" | cat" if CREW_VERBOSE output = `curl -# --header "#{token_label}: #{gitlab_token}" --upload-file "#{new_tarfile}" "#{new_url}" | cat`.chomp if output.include?('201 Created') diff --git a/lib/const.rb b/lib/const.rb index f937ff812..3e33fed8f 100644 --- a/lib/const.rb +++ b/lib/const.rb @@ -3,7 +3,7 @@ require 'etc' OLD_CREW_VERSION ||= defined?(CREW_VERSION) ? CREW_VERSION : '1.0' -CREW_VERSION ||= '1.51.7' unless defined?(CREW_VERSION) && CREW_VERSION == OLD_CREW_VERSION +CREW_VERSION ||= '1.51.8' unless defined?(CREW_VERSION) && CREW_VERSION == OLD_CREW_VERSION # Kernel architecture. KERN_ARCH ||= Etc.uname[:machine] diff --git a/lib/package_utils.rb b/lib/package_utils.rb index 11269d1d9..986699e4e 100644 --- a/lib/package_utils.rb +++ b/lib/package_utils.rb @@ -1,6 +1,7 @@ # lib/package_utils.rb # Utility functions that take either a package object or a component of a package object as primary input. require 'json' +require 'net/http' require_relative 'const' class PackageUtils @@ -23,7 +24,7 @@ class PackageUtils def self.get_url(pkg, build_from_source: false) if !build_from_source && pkg.binary_sha256&.key?(ARCH.to_sym) - return "https://gitlab.com/api/v4/projects/26210301/packages/generic/#{pkg.name}/#{pkg.version}_#{ARCH}/#{pkg.name}-#{pkg.version}-chromeos-#{ARCH}.#{pkg.binary_compression}" + return get_binary_url(pkg) elsif pkg.source_url.is_a?(Hash) && pkg.source_url&.key?(ARCH.to_sym) return pkg.source_url[ARCH.to_sym] else @@ -41,6 +42,34 @@ class PackageUtils end end + def self.get_binary_url(pkg) + # Fall back to the old method if querying the gitlab API doesn't work for whatever reason. + fallback_url = "https://gitlab.com/api/v4/projects/26210301/packages/generic/#{pkg.name}/#{pkg.version}_#{ARCH}/#{pkg.name}-#{pkg.version}-chromeos-#{ARCH}.#{pkg.binary_compression}" + # List all the packages with the name and version of the package file. + # The name search is fuzzy, so we need to refine it further (otherwise packages like vim, gvim and vim_runtime would break). + packages = JSON.parse(Net::HTTP.get(URI("https://gitlab.com/api/v4/projects/26210301/packages?package_name=#{pkg.name}&package_version=#{pkg.version}_#{ARCH}"))) + # Loop over each result until we get an exact name match, then return the package ID for that match. + package_id = 0 + (0..packages.count - 1).each do |i| + next unless packages[i]['name'] == pkg.name + package_id = packages[i]['id'] + end + # Return early if we weren't able to find the package ID, so that the CREW_CACHE_ENABLED hack to test packages without uploading them still works. + # When we're doing that, we're calling download knowing that there isn't an actual file to download, but relying on CREW_CACHE_ENABLED to save us before we get there. + return fallback_url if package_id.zero? + # List all the package files for that package ID. + package_files = JSON.parse(Net::HTTP.get(URI("https://gitlab.com/api/v4/projects/26210301/packages/#{package_id}/package_files"))) + # Bail out if we weren't actually able to find a package. + return fallback_url if package_files.is_a?(Hash) && package_files['message'] == '404 Not found' + # Loop over each result until we find a matching file_sha256 to our binary_sha256. + (0..package_files.count - 1).each do |i| + next unless package_files[i]['file_sha256'] == pkg.binary_sha256[ARCH.to_sym] + return "https://gitlab.com/chromebrew/binaries/-/package_files/#{package_files[i]['id']}/download" + end + # If we're still here, the likely cause is that the file sha256s are mismatched. + return fallback_url + end + def self.get_clean_version(pkg_version) # Trim kde- suffixes in qt5 packages so nothing else gets confused. pkg_version.delete_prefix!('kde-') diff --git a/packages/hello_world_chromebrew.rb b/packages/hello_world_chromebrew.rb index a43a1aa47..c261435b4 100755 --- a/packages/hello_world_chromebrew.rb +++ b/packages/hello_world_chromebrew.rb @@ -12,10 +12,10 @@ class Hello_world_chromebrew < Package # These are needed to successfully build and check for dependencies. binary_sha256({ - aarch64: '000', - armv7l: '000', - i686: '111', - x86_64: '222' + aarch64: '827f9794864aa76f4c99fd31f989077f1fa65771386f73db30a7681842f8736d', + armv7l: '827f9794864aa76f4c99fd31f989077f1fa65771386f73db30a7681842f8736d', + i686: 'ff0942c505b04982fed187bcda123adead37b3ac2dcfd7e2f0543ca179e81df6', + x86_64: '3081f1f25950c91f75db41095b644a2783987a3a7ef2832fc2b85bf138bb006f' }) # Register dependencies (use the following line as a basis) diff --git a/tests/lib/package_utils.rb b/tests/lib/package_utils.rb index 0eaafe729..eb76b0460 100644 --- a/tests/lib/package_utils.rb +++ b/tests/lib/package_utils.rb @@ -50,15 +50,38 @@ class PackageUtilsTest < Minitest::Test refute(PackageUtils.compatible?(pkg)) end - def test_get_binary_url + def test_get_binary_url_old_hash pkg = Class.new(Package) - pkg.name = 'test_package' + pkg.name = 'hello_world_chromebrew' pkg.instance_eval do - version '1.0' + version '1.1' binary_compression 'tar.zst' - binary_sha256({ ARCH.to_sym => '0000000000000000000000000000000000000000000000000000000000000000' }) + binary_sha256({ + aarch64: '29185a6c4a8ecc3532606649f5831e37cd977c24bfd4fb1c12328ca5ba966ff1', + armv7l: '29185a6c4a8ecc3532606649f5831e37cd977c24bfd4fb1c12328ca5ba966ff1', + i686: '6e46c31245e9e17eeecf03d61454bb7312b6e769bfaf352ced527ee93cc62518', + x86_64: '83f674b3e8fe29c7e761ce6adc27fd7df927d985140ff6d1ae64a0046339322b' + }) end - assert_equal("https://gitlab.com/api/v4/projects/26210301/packages/generic/test_package/1.0_#{ARCH}/test_package-1.0-chromeos-#{ARCH}.tar.zst", PackageUtils.get_url(pkg)) + package_file_id = { aarch64: '137956464', armv7l: '137956464', i686: '137956442', x86_64: '137956370' } + assert_equal("https://gitlab.com/chromebrew/binaries/-/package_files/#{package_file_id[ARCH.to_sym]}/download", PackageUtils.get_url(pkg)) + end + + def test_get_binary_url_new_hash + pkg = Class.new(Package) + pkg.name = 'hello_world_chromebrew' + pkg.instance_eval do + version '1.1' + binary_compression 'tar.zst' + binary_sha256({ + aarch64: '827f9794864aa76f4c99fd31f989077f1fa65771386f73db30a7681842f8736d', + armv7l: '827f9794864aa76f4c99fd31f989077f1fa65771386f73db30a7681842f8736d', + i686: 'ff0942c505b04982fed187bcda123adead37b3ac2dcfd7e2f0543ca179e81df6', + x86_64: '3081f1f25950c91f75db41095b644a2783987a3a7ef2832fc2b85bf138bb006f' + }) + end + package_file_id = { aarch64: '137956527', armv7l: '137956527', i686: '137956521', x86_64: '137956520' } + assert_equal("https://gitlab.com/chromebrew/binaries/-/package_files/#{package_file_id[ARCH.to_sym]}/download", PackageUtils.get_url(pkg)) end def test_get_source_url_hash