Add a downloader based on the net/http library (#6491)

* Use net http (#3)

* Add user agent header

* Update crew

* Update downloader.rb

* Update const.rb

* Fix const.rb crash.

Co-authored-by: Satadru Pramanik <satadru@umich.edu>
Co-authored-by: Satadru Pramanik <satadru@gmail.com>

bin/crew

@@ -11,6 +11,8 @@ require 'fileutils'
 require 'tmpdir'
 require_relative '../lib/const'
 require_relative '../lib/util'
+require_relative '../lib/convert_size'
+require_relative '../lib/downloader'
 
 # Add lib to LOAD_PATH
 $LOAD_PATH.unshift "#{CREW_LIB_PATH}lib"
@@ -473,29 +475,6 @@ def const(var)
   end
 end
 
-def human_size (bytes)
-  kilobyte = 1024.0
-  megabyte = kilobyte * kilobyte
-  gigabyte = megabyte * kilobyte
-  if bytes < kilobyte
-    units = 'B'
-    size = bytes
-  end
-  if bytes >= kilobyte and bytes < megabyte
-    units = 'KB'
-    size = bytes / kilobyte
-  end
-  if bytes >= megabyte and bytes < gigabyte
-    units = 'MB'
-    size = bytes / megabyte
-  end
-  if bytes >= gigabyte
-    units = 'GB'
-    size = bytes / gigabyte
-  end
-  return sprintf('%.2f', size.to_s) + units
-end
-
 def files (pkgName)
   filelist = "#{CREW_META_PATH}#{pkgName}.filelist"
   if File.exist? filelist
@@ -707,7 +686,13 @@ def download
         end
       end
       # Download file if not cached.
-      system "#{CURL} --retry 3 -#{@verbose}#LC - --insecure \'#{url}\' --output #{filename}"
+      # use curl if CURL environment variable set
+      if CURL == 'curl'
+        downloader url, filename, @opt_verbose
+      else
+        system "#{CURL} --retry 3 -#{@verbose}#LC - --insecure \'#{url}\' --output #{filename}"
+      end
+
       abort 'Checksum mismatch. 😔 Try again.'.lightred unless
         Digest::SHA256.hexdigest( File.read(filename) ) == sha256sum || sha256sum =~ /^SKIP$/i
       puts "#{@pkg.name.capitalize} archive downloaded.".lightgreen

crew

@@ -1 +0,0 @@
-bin/crew

lib/const.rb

@@ -1,6 +1,6 @@
 # Defines common constants used in different parts of crew
 
-CREW_VERSION = '1.19.5'
+CREW_VERSION = '1.20.0'
 
 ARCH_ACTUAL = `uname -m`.chomp
 # This helps with virtualized builds on aarch64 machines
@@ -88,6 +88,26 @@ end
 # If CURL environment variable exists use it in lieu of curl.
 CURL = ENV['CURL'] || 'curl'
 
+# set certificate file location for lib/downloader.rb
+SSL_CERT_FILE = if ENV['SSL_CERT_FILE'].to_s.empty? || !File.exist?(ENV['SSL_CERT_FILE'])
+                  if File.exist?("#{CREW_PREFIX}/etc/ssl/certs/ca-certificates.crt")
+                    "#{CREW_PREFIX}/etc/ssl/certs/ca-certificates.crt"
+                  else
+                    '/etc/ssl/certs/ca-certificates.crt'
+                  end
+                else
+                  ENV['SSL_CERT_FILE']
+                end
+SSL_CERT_DIR = if ENV['SSL_CERT_DIR'].to_s.empty? || !Dir.exist?(ENV['SSL_CERT_DIR'])
+                 if Dir.exist?("#{CREW_PREFIX}/etc/ssl/certs")
+                   "#{CREW_PREFIX}/etc/ssl/certs"
+                 else
+                   '/etc/ssl/certs'
+                 end
+               else
+                 ENV['SSL_CERT_DIR']
+               end
+
 case ARCH
 when 'aarch64', 'armv7l'
   CREW_TGT = 'armv7l-cros-linux-gnueabihf'

lib/convert_size.rb

@@ -0,0 +1,22 @@
+def human_size (bytes)
+  kilobyte = 1024.0
+  megabyte = kilobyte * kilobyte
+  gigabyte = megabyte * kilobyte
+  if bytes < kilobyte
+    units = 'B'
+    size = bytes
+  end
+  if bytes >= kilobyte and bytes < megabyte
+    units = 'KB'
+    size = bytes / kilobyte
+  end
+  if bytes >= megabyte and bytes < gigabyte
+    units = 'MB'
+    size = bytes / megabyte
+  end
+  if bytes >= gigabyte
+    units = 'GB'
+    size = bytes / gigabyte
+  end
+  return sprintf('%.2f %s', size, units)
+end

lib/downloader.rb

@@ -0,0 +1,78 @@
+require 'io/console'
+require 'net/http'
+require 'uri'
+require_relative 'const'
+require_relative 'color'
+require_relative 'convert_size'
+
+def setTermSize
+  # setTermSize: set progress bar size based on terminal width
+  # get terminal window size
+  @termH, @termW = IO.console.winsize
+  # space for progress bar after minus the reserved space for showing
+  # the file size and progress percentage
+  @progBarW = @termW - 17
+  return true
+end
+
+def downloader (url, filename = File.basename(url), retry_count = 0, verbose = false)
+  setTermSize
+  # reset width settings after terminal resized
+  trap('WINCH') { setTermSize }
+  uri = URI(url)
+
+  Net::HTTP.start(uri.host, uri.port, {
+      max_retries: 3,
+      use_ssl: uri.scheme.eql?('https'),
+      ca_file: SSL_CERT_FILE,
+      ca_path: SSL_CERT_DIR
+  }) do |http|
+    http.request( Net::HTTP::Get.new(uri) ) do |response|
+      case
+      when response.is_a?(Net::HTTPOK)
+      when response.is_a?(Net::HTTPRedirection) # follow HTTP redirection
+        puts <<~EOT if verbose
+          * Follow HTTP redirection: #{response['Location']}
+          *
+        EOT
+        return downloader(response['Location'], filename, retry_count, verbose)
+      else
+        abort "Download failed with error #{response.code}: #{response.msg}".lightred
+      end
+
+      # get target file size (should be returned by the server)
+      file_size = response['Content-Length'].to_i
+      downloaded_size = 0
+
+      if verbose
+        puts <<~EOT
+          * Connected to #{uri.host} port #{uri.port}
+          * HTTPS: #{uri.scheme.eql?('https')}
+          *
+        EOT
+
+        response.to_hash.each_pair do |k, v|
+          puts "> #{k}: #{v}"
+        end
+        puts
+      end
+
+      File.open(filename, 'wb') do |io|
+        response.read_body do |chunk|
+          downloaded_size += chunk.size
+          if file_size.positive?
+            # calculate downloading progress percentage with the given file size
+            percentage = (downloaded_size.to_f / file_size.to_f) * 100
+            # show progress bar, file size and progress percentage
+            printf "\r[%-#{@progBarW}s] %9.9s %3d%%",
+                   '#' * ( @progBarW * (percentage / 100) ).to_i,
+                   human_size(file_size),
+                   percentage
+          end
+          io.write(chunk)
+        end
+      end
+      puts
+    end
+  end
+end