From 8fc7b84aaca6a07ae94b672990c07de5b13b43e7 Mon Sep 17 00:00:00 2001 From: supechicken Date: Fri, 24 Dec 2021 11:53:50 +0800 Subject: [PATCH] Add a downloader based on the `net/http` library (#6491) * Use net http (#3) * Add user agent header * Update crew * Update downloader.rb * Update const.rb * Fix const.rb crash. Co-authored-by: Satadru Pramanik Co-authored-by: Satadru Pramanik --- bin/crew | 33 ++++++------------- crew | 1 - lib/const.rb | 22 ++++++++++++- lib/convert_size.rb | 22 +++++++++++++ lib/downloader.rb | 78 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 130 insertions(+), 26 deletions(-) delete mode 120000 crew create mode 100644 lib/convert_size.rb create mode 100644 lib/downloader.rb diff --git a/bin/crew b/bin/crew index b81c82191..3cdc80961 100755 --- a/bin/crew +++ b/bin/crew @@ -11,6 +11,8 @@ require 'fileutils' require 'tmpdir' require_relative '../lib/const' require_relative '../lib/util' +require_relative '../lib/convert_size' +require_relative '../lib/downloader' # Add lib to LOAD_PATH $LOAD_PATH.unshift "#{CREW_LIB_PATH}lib" @@ -473,29 +475,6 @@ def const(var) end end -def human_size (bytes) - kilobyte = 1024.0 - megabyte = kilobyte * kilobyte - gigabyte = megabyte * kilobyte - if bytes < kilobyte - units = 'B' - size = bytes - end - if bytes >= kilobyte and bytes < megabyte - units = 'KB' - size = bytes / kilobyte - end - if bytes >= megabyte and bytes < gigabyte - units = 'MB' - size = bytes / megabyte - end - if bytes >= gigabyte - units = 'GB' - size = bytes / gigabyte - end - return sprintf('%.2f', size.to_s) + units -end - def files (pkgName) filelist = "#{CREW_META_PATH}#{pkgName}.filelist" if File.exist? filelist @@ -707,7 +686,13 @@ def download end end # Download file if not cached. - system "#{CURL} --retry 3 -#{@verbose}#LC - --insecure \'#{url}\' --output #{filename}" + # use curl if CURL environment variable set + if CURL == 'curl' + downloader url, filename, @opt_verbose + else + system "#{CURL} --retry 3 -#{@verbose}#LC - --insecure \'#{url}\' --output #{filename}" + end + abort 'Checksum mismatch. 😔 Try again.'.lightred unless Digest::SHA256.hexdigest( File.read(filename) ) == sha256sum || sha256sum =~ /^SKIP$/i puts "#{@pkg.name.capitalize} archive downloaded.".lightgreen diff --git a/crew b/crew deleted file mode 120000 index 0205cf051..000000000 --- a/crew +++ /dev/null @@ -1 +0,0 @@ -bin/crew \ No newline at end of file diff --git a/lib/const.rb b/lib/const.rb index a27f1a254..81d5852fc 100644 --- a/lib/const.rb +++ b/lib/const.rb @@ -1,6 +1,6 @@ # Defines common constants used in different parts of crew -CREW_VERSION = '1.19.5' +CREW_VERSION = '1.20.0' ARCH_ACTUAL = `uname -m`.chomp # This helps with virtualized builds on aarch64 machines @@ -88,6 +88,26 @@ end # If CURL environment variable exists use it in lieu of curl. CURL = ENV['CURL'] || 'curl' +# set certificate file location for lib/downloader.rb +SSL_CERT_FILE = if ENV['SSL_CERT_FILE'].to_s.empty? || !File.exist?(ENV['SSL_CERT_FILE']) + if File.exist?("#{CREW_PREFIX}/etc/ssl/certs/ca-certificates.crt") + "#{CREW_PREFIX}/etc/ssl/certs/ca-certificates.crt" + else + '/etc/ssl/certs/ca-certificates.crt' + end + else + ENV['SSL_CERT_FILE'] + end +SSL_CERT_DIR = if ENV['SSL_CERT_DIR'].to_s.empty? || !Dir.exist?(ENV['SSL_CERT_DIR']) + if Dir.exist?("#{CREW_PREFIX}/etc/ssl/certs") + "#{CREW_PREFIX}/etc/ssl/certs" + else + '/etc/ssl/certs' + end + else + ENV['SSL_CERT_DIR'] + end + case ARCH when 'aarch64', 'armv7l' CREW_TGT = 'armv7l-cros-linux-gnueabihf' diff --git a/lib/convert_size.rb b/lib/convert_size.rb new file mode 100644 index 000000000..e0b56f26d --- /dev/null +++ b/lib/convert_size.rb @@ -0,0 +1,22 @@ +def human_size (bytes) + kilobyte = 1024.0 + megabyte = kilobyte * kilobyte + gigabyte = megabyte * kilobyte + if bytes < kilobyte + units = 'B' + size = bytes + end + if bytes >= kilobyte and bytes < megabyte + units = 'KB' + size = bytes / kilobyte + end + if bytes >= megabyte and bytes < gigabyte + units = 'MB' + size = bytes / megabyte + end + if bytes >= gigabyte + units = 'GB' + size = bytes / gigabyte + end + return sprintf('%.2f %s', size, units) +end diff --git a/lib/downloader.rb b/lib/downloader.rb new file mode 100644 index 000000000..92f083d8e --- /dev/null +++ b/lib/downloader.rb @@ -0,0 +1,78 @@ +require 'io/console' +require 'net/http' +require 'uri' +require_relative 'const' +require_relative 'color' +require_relative 'convert_size' + +def setTermSize + # setTermSize: set progress bar size based on terminal width + # get terminal window size + @termH, @termW = IO.console.winsize + # space for progress bar after minus the reserved space for showing + # the file size and progress percentage + @progBarW = @termW - 17 + return true +end + +def downloader (url, filename = File.basename(url), retry_count = 0, verbose = false) + setTermSize + # reset width settings after terminal resized + trap('WINCH') { setTermSize } + uri = URI(url) + + Net::HTTP.start(uri.host, uri.port, { + max_retries: 3, + use_ssl: uri.scheme.eql?('https'), + ca_file: SSL_CERT_FILE, + ca_path: SSL_CERT_DIR + }) do |http| + http.request( Net::HTTP::Get.new(uri) ) do |response| + case + when response.is_a?(Net::HTTPOK) + when response.is_a?(Net::HTTPRedirection) # follow HTTP redirection + puts <<~EOT if verbose + * Follow HTTP redirection: #{response['Location']} + * + EOT + return downloader(response['Location'], filename, retry_count, verbose) + else + abort "Download failed with error #{response.code}: #{response.msg}".lightred + end + + # get target file size (should be returned by the server) + file_size = response['Content-Length'].to_i + downloaded_size = 0 + + if verbose + puts <<~EOT + * Connected to #{uri.host} port #{uri.port} + * HTTPS: #{uri.scheme.eql?('https')} + * + EOT + + response.to_hash.each_pair do |k, v| + puts "> #{k}: #{v}" + end + puts + end + + File.open(filename, 'wb') do |io| + response.read_body do |chunk| + downloaded_size += chunk.size + if file_size.positive? + # calculate downloading progress percentage with the given file size + percentage = (downloaded_size.to_f / file_size.to_f) * 100 + # show progress bar, file size and progress percentage + printf "\r[%-#{@progBarW}s] %9.9s %3d%%", + '#' * ( @progBarW * (percentage / 100) ).to_i, + human_size(file_size), + percentage + end + io.write(chunk) + end + end + puts + end + end +end