Reland: Download the correct file when there are multiple versions of a package binary (#10336)

bin/crew

@@ -435,6 +435,8 @@ def download
 
   uri = URI.parse url
   filename = File.basename(uri.path)
+  # # If we're downloading a binary, reset the filename to what it would have been if we didn't download from the API.
+  filename = "#{@pkg.name}-#{@pkg.version}-chromeos-#{ARCH}.#{@pkg.binary_compression}" if filename.eql?('download')
   sha256sum = PackageUtils.get_sha256(@pkg, build_from_source: @opt_source || @pkg.build_from_source)
   @extract_dir = "#{@pkg.name}.#{Time.now.utc.strftime('%Y%m%d%H%M%S')}.dir"
 
@@ -1448,13 +1450,6 @@ def upload(pkg_name = nil, pkg_version = nil, gitlab_token = nil, binary_compres
       new_url = "#{base_url}/#{package}/#{new_version}_#{arch}/#{new_tarfile}".gsub("#{release_dir}/", '')
       token_label = gitlab_token.split('-').first == 'glpat' ? 'PRIVATE-TOKEN' : 'DEPLOY-TOKEN'
 
-      if `curl -sI #{new_url}`.lines.first.split[1] == '200'
-        puts "\n#{new_tarfile} has already been uploaded.\nPlease change the #{package} package version from #{new_version} and try again.\n".lightred
-        unless Package.agree_default_no('Do you want to overwrite the existing upload instead')
-          puts 'Will NOT overwite the existing upload.'.orange
-          next
-        end
-      end
       puts "curl -# --header \"#{token_label}: #{gitlab_token}\" --upload-file \"#{new_tarfile}\" \"#{new_url}\" | cat" if CREW_VERBOSE
       output = `curl -# --header "#{token_label}: #{gitlab_token}" --upload-file "#{new_tarfile}" "#{new_url}" | cat`.chomp
       if output.include?('201 Created')

lib/const.rb

@@ -3,7 +3,7 @@
 require 'etc'
 
 OLD_CREW_VERSION ||= defined?(CREW_VERSION) ? CREW_VERSION : '1.0'
-CREW_VERSION ||= '1.51.7' unless defined?(CREW_VERSION) && CREW_VERSION == OLD_CREW_VERSION
+CREW_VERSION ||= '1.51.8' unless defined?(CREW_VERSION) && CREW_VERSION == OLD_CREW_VERSION
 
 # Kernel architecture.
 KERN_ARCH ||= Etc.uname[:machine]

lib/package_utils.rb

@@ -1,6 +1,7 @@
 # lib/package_utils.rb
 # Utility functions that take either a package object or a component of a package object as primary input.
 require 'json'
+require 'net/http'
 require_relative 'const'
 
 class PackageUtils
@@ -23,7 +24,7 @@ class PackageUtils
 
   def self.get_url(pkg, build_from_source: false)
     if !build_from_source && pkg.binary_sha256&.key?(ARCH.to_sym)
-      return "https://gitlab.com/api/v4/projects/26210301/packages/generic/#{pkg.name}/#{pkg.version}_#{ARCH}/#{pkg.name}-#{pkg.version}-chromeos-#{ARCH}.#{pkg.binary_compression}"
+      return get_binary_url(pkg)
     elsif pkg.source_url.is_a?(Hash) && pkg.source_url&.key?(ARCH.to_sym)
       return pkg.source_url[ARCH.to_sym]
     else
@@ -41,6 +42,34 @@ class PackageUtils
     end
   end
 
+  def self.get_binary_url(pkg)
+    # Fall back to the old method if querying the gitlab API doesn't work for whatever reason.
+    fallback_url = "https://gitlab.com/api/v4/projects/26210301/packages/generic/#{pkg.name}/#{pkg.version}_#{ARCH}/#{pkg.name}-#{pkg.version}-chromeos-#{ARCH}.#{pkg.binary_compression}"
+    # List all the packages with the name and version of the package file.
+    # The name search is fuzzy, so we need to refine it further (otherwise packages like vim, gvim and vim_runtime would break).
+    packages = JSON.parse(Net::HTTP.get(URI("https://gitlab.com/api/v4/projects/26210301/packages?package_name=#{pkg.name}&package_version=#{pkg.version}_#{ARCH}")))
+    # Loop over each result until we get an exact name match, then return the package ID for that match.
+    package_id = 0
+    (0..packages.count - 1).each do |i|
+      next unless packages[i]['name'] == pkg.name
+      package_id = packages[i]['id']
+    end
+    # Return early if we weren't able to find the package ID, so that the CREW_CACHE_ENABLED hack to test packages without uploading them still works.
+    # When we're doing that, we're calling download knowing that there isn't an actual file to download, but relying on CREW_CACHE_ENABLED to save us before we get there.
+    return fallback_url if package_id.zero?
+    # List all the package files for that package ID.
+    package_files = JSON.parse(Net::HTTP.get(URI("https://gitlab.com/api/v4/projects/26210301/packages/#{package_id}/package_files")))
+    # Bail out if we weren't actually able to find a package.
+    return fallback_url if package_files.is_a?(Hash) && package_files['message'] == '404 Not found'
+    # Loop over each result until we find a matching file_sha256 to our binary_sha256.
+    (0..package_files.count - 1).each do |i|
+      next unless package_files[i]['file_sha256'] == pkg.binary_sha256[ARCH.to_sym]
+      return "https://gitlab.com/chromebrew/binaries/-/package_files/#{package_files[i]['id']}/download"
+    end
+    # If we're still here, the likely cause is that the file sha256s are mismatched.
+    return fallback_url
+  end
+
   def self.get_clean_version(pkg_version)
     # Trim kde- suffixes in qt5 packages so nothing else gets confused.
     pkg_version.delete_prefix!('kde-')

packages/hello_world_chromebrew.rb

@@ -12,10 +12,10 @@ class Hello_world_chromebrew < Package
 
   # These are needed to successfully build and check for dependencies.
   binary_sha256({
-    aarch64: '000',
-     armv7l: '000',
-       i686: '111',
-     x86_64: '222'
+    aarch64: '827f9794864aa76f4c99fd31f989077f1fa65771386f73db30a7681842f8736d',
+     armv7l: '827f9794864aa76f4c99fd31f989077f1fa65771386f73db30a7681842f8736d',
+       i686: 'ff0942c505b04982fed187bcda123adead37b3ac2dcfd7e2f0543ca179e81df6',
+     x86_64: '3081f1f25950c91f75db41095b644a2783987a3a7ef2832fc2b85bf138bb006f'
   })
 
   # Register dependencies (use the following line as a basis)

tests/lib/package_utils.rb

@@ -50,15 +50,38 @@ class PackageUtilsTest < Minitest::Test
     refute(PackageUtils.compatible?(pkg))
   end
 
-  def test_get_binary_url
+  def test_get_binary_url_old_hash
     pkg = Class.new(Package)
-    pkg.name = 'test_package'
+    pkg.name = 'hello_world_chromebrew'
     pkg.instance_eval do
-      version '1.0'
+      version '1.1'
       binary_compression 'tar.zst'
-      binary_sha256({ ARCH.to_sym => '0000000000000000000000000000000000000000000000000000000000000000' })
+      binary_sha256({
+        aarch64: '29185a6c4a8ecc3532606649f5831e37cd977c24bfd4fb1c12328ca5ba966ff1',
+         armv7l: '29185a6c4a8ecc3532606649f5831e37cd977c24bfd4fb1c12328ca5ba966ff1',
+           i686: '6e46c31245e9e17eeecf03d61454bb7312b6e769bfaf352ced527ee93cc62518',
+         x86_64: '83f674b3e8fe29c7e761ce6adc27fd7df927d985140ff6d1ae64a0046339322b'
+      })
     end
-    assert_equal("https://gitlab.com/api/v4/projects/26210301/packages/generic/test_package/1.0_#{ARCH}/test_package-1.0-chromeos-#{ARCH}.tar.zst", PackageUtils.get_url(pkg))
+    package_file_id = { aarch64: '137956464', armv7l: '137956464', i686: '137956442', x86_64: '137956370' }
+    assert_equal("https://gitlab.com/chromebrew/binaries/-/package_files/#{package_file_id[ARCH.to_sym]}/download", PackageUtils.get_url(pkg))
+  end
+
+  def test_get_binary_url_new_hash
+    pkg = Class.new(Package)
+    pkg.name = 'hello_world_chromebrew'
+    pkg.instance_eval do
+      version '1.1'
+      binary_compression 'tar.zst'
+      binary_sha256({
+        aarch64: '827f9794864aa76f4c99fd31f989077f1fa65771386f73db30a7681842f8736d',
+         armv7l: '827f9794864aa76f4c99fd31f989077f1fa65771386f73db30a7681842f8736d',
+           i686: 'ff0942c505b04982fed187bcda123adead37b3ac2dcfd7e2f0543ca179e81df6',
+         x86_64: '3081f1f25950c91f75db41095b644a2783987a3a7ef2832fc2b85bf138bb006f'
+      })
+    end
+    package_file_id = { aarch64: '137956527', armv7l: '137956527', i686: '137956521', x86_64: '137956520' }
+    assert_equal("https://gitlab.com/chromebrew/binaries/-/package_files/#{package_file_id[ARCH.to_sym]}/download", PackageUtils.get_url(pkg))
   end
 
   def test_get_source_url_hash