diff --git a/lib/downloader.rb b/lib/downloader.rb index fb928f1ea..c3d890e50 100644 --- a/lib/downloader.rb +++ b/lib/downloader.rb @@ -91,6 +91,26 @@ rescue StandardError => e external_downloader(uri, filename, verbose: verbose) end +def get_http_response(uri) + uri = URI(uri) if uri.is_a?(String) + ssl_error_retry = 0 + + # open http connection + Net::HTTP.start(uri.host, uri.port, { + max_retries: CREW_DOWNLOADER_RETRY, + use_ssl: uri.scheme.eql?('https'), + min_version: :TLS1_2, + ca_file: SSL_CERT_FILE, + ca_path: SSL_CERT_DIR + }) do |http| + return http.get(uri) + end +rescue OpenSSL::SSL::SSLError + # handle SSL errors + ssl_error_retry += 1 + ssl_error_retry <= 3 ? retry : raise +end + def http_downloader(uri, filename = File.basename(url), verbose: false) # http_downloader: Downloader based on net/http library ssl_error_retry = 0 @@ -98,10 +118,10 @@ def http_downloader(uri, filename = File.basename(url), verbose: false) # open http connection Net::HTTP.start(uri.host, uri.port, { max_retries: CREW_DOWNLOADER_RETRY, - use_ssl: uri.scheme.eql?('https'), + use_ssl: uri.scheme.eql?('https'), min_version: :TLS1_2, - ca_file: SSL_CERT_FILE, - ca_path: SSL_CERT_DIR + ca_file: SSL_CERT_FILE, + ca_path: SSL_CERT_DIR }) do |http| http.request(Net::HTTP::Get.new(uri)) do |response| case diff --git a/lib/gem_compact_index_client.rb b/lib/gem_compact_index_client.rb index 51afedea7..16836907c 100644 --- a/lib/gem_compact_index_client.rb +++ b/lib/gem_compact_index_client.rb @@ -1,5 +1,5 @@ require 'uri' -require 'net/http' +require_relative 'downloader' # Adapted from https://github.com/rubygems/rubygems/issues/8012#issuecomment-2375499571 # by @duckinator (Ellen Marie Dash) @@ -19,21 +19,12 @@ class BasicCompactIndexClient private def request(endpoint) - uri = URI.join(@gem_server, endpoint) + uri = URI.join(@gem_server, endpoint) + response = get_http_response(uri) - Net::HTTP.start(uri.host, uri.port, { - max_retries: CREW_DOWNLOADER_RETRY, - use_ssl: uri.scheme.eql?('https'), - min_version: :TLS1_2, - ca_file: SSL_CERT_FILE, - ca_path: SSL_CERT_DIR - }) do |http| - response = http.get(uri) + raise "got HTTP code #{response.code}, expected 200" unless response.is_a?(Net::HTTPOK) - raise "got HTTP code #{response.code}, expected 200" unless response.is_a?(Net::HTTPOK) - - return response.body - end + return response.body end def lines(data) diff --git a/lib/package_utils.rb b/lib/package_utils.rb index 8d7bb45f5..25289dfeb 100644 --- a/lib/package_utils.rb +++ b/lib/package_utils.rb @@ -1,8 +1,8 @@ # lib/package_utils.rb # Utility functions that take either a package object or a component of a package object as primary input. require 'json' -require 'net/http' require_relative 'const' +require_relative 'downloader' class PackageUtils def self.installed?(pkg_name) @@ -45,27 +45,31 @@ class PackageUtils def self.get_binary_url(pkg) # Fall back to the old method of querying if the gitlab API doesn't work for whatever reason. fallback_url = "#{CREW_GITLAB_PKG_REPO}/generic/#{pkg.name}/#{pkg.version}_#{ARCH}/#{pkg.name}-#{pkg.version}-chromeos-#{ARCH}.#{pkg.binary_compression}" + # List all the packages with the name and version of the package file. # The name search is fuzzy, so we need to refine it further (otherwise packages like vim, gvim and vim_runtime would break). - packages = JSON.parse(Net::HTTP.get(URI("#{CREW_GITLAB_PKG_REPO}?package_name=#{pkg.name}&package_version=#{pkg.version}_#{ARCH}"))) + response = get_http_response("#{CREW_GITLAB_PKG_REPO}?package_name=#{pkg.name}&package_version=#{pkg.version}_#{ARCH}") + packages = JSON.parse(response.body) + # Loop over each result until we get an exact name match, then return the package ID for that match. - package_id = 0 - (0..packages.count - 1).each do |i| - next unless packages[i]['name'] == pkg.name - package_id = packages[i]['id'] - end + package_id = packages.select(&->(p) { p['name'] == pkg.name }).dig(0, 'id') + # Return early if we weren't able to find the package ID, so that the CREW_CACHE_ENABLED hack to test packages without uploading them still works. # When we're doing that, we're calling download knowing that there isn't an actual file to download, but relying on CREW_CACHE_ENABLED to save us before we get there. - return fallback_url if package_id.zero? + return fallback_url unless package_id + # List all the package files for that package ID. - package_files = JSON.parse(Net::HTTP.get(URI("#{CREW_GITLAB_PKG_REPO}/#{package_id}/package_files"))) + response = get_http_response("#{CREW_GITLAB_PKG_REPO}/#{package_id}/package_files") + package_files = JSON.parse(response.body) + # Bail out if we weren't actually able to find a package. return fallback_url if package_files.is_a?(Hash) && package_files['message'] == '404 Not found' + # Loop over each result until we find a matching file_sha256 to our binary_sha256. - (0..package_files.count - 1).each do |i| - next unless package_files[i]['file_sha256'] == pkg.binary_sha256[ARCH.to_sym] - return "https://gitlab.com/chromebrew/binaries/-/package_files/#{package_files[i]['id']}/download" - end + file_id = package_files.select(&->(p) { p['file_sha256'] == pkg.binary_sha256[ARCH.to_sym] }).dig(0, 'id') + + return "https://gitlab.com/chromebrew/binaries/-/package_files/#{file_id}/download" if file_id + # If we're still here, the likely cause is that the file sha256s are mismatched. return fallback_url end