diff --git a/bin/crew b/bin/crew index 25609bde03..50928942e4 100755 --- a/bin/crew +++ b/bin/crew @@ -704,12 +704,7 @@ def download end end # Download file if not cached. - # use curl if CREW_USE_CURL environment variable is set to 1 - unless CREW_USE_CURL - downloader url, filename, @opt_verbose - else - system "curl --retry 3 -#{@verbose}#LC - --insecure \'#{url}\' --output #{filename}" - end + downloader url, filename, @opt_verbose abort 'Checksum mismatch. 😔 Try again.'.lightred unless Digest::SHA256.hexdigest( File.read(filename) ) == sha256sum || sha256sum =~ /^SKIP$/i @@ -735,12 +730,7 @@ def download @git = true else Dir.mkdir @extract_dir - # use curl if CREW_USE_CURL environment variable is set to 1 - unless CREW_USE_CURL - downloader url, filename, @opt_verbose - else - system "curl --retry 3 -#{@verbose}#LC - --insecure \'#{url}\' --output #{filename}" - end + downloader url, filename, @opt_verbose abort 'Checksum mismatch. 😔 Try again.'.lightred unless Digest::SHA256.hexdigest( File.read(filename) ) == sha256sum || sha256sum =~ /^SKIP$/i diff --git a/lib/const.rb b/lib/const.rb index 307cf567ee..0e10b7abbe 100644 --- a/lib/const.rb +++ b/lib/const.rb @@ -83,6 +83,10 @@ end # If CREW_USE_CURL environment variable exists use curl in lieu of net/http. CREW_USE_CURL = ENV['CREW_USE_CURL'] == '1' +# Use an external downloader instead of net/http if CREW_DOWNLOADER is set, see lib/downloader.rb for more info +# About the format of the CREW_DOWNLOADER variable, see line 130-133 in lib/downloader.rb +CREW_DOWNLOADER = ( ENV['CREW_DOWNLOADER'].to_s.empty? ) ? nil : ENV['CREW_DOWNLOADER'] + # set certificate file location for lib/downloader.rb SSL_CERT_FILE = if ENV['SSL_CERT_FILE'].to_s.empty? || !File.exist?(ENV['SSL_CERT_FILE']) if File.exist?("#{CREW_PREFIX}/etc/ssl/certs/ca-certificates.crt") diff --git a/lib/downloader.rb b/lib/downloader.rb index 480c4d9705..1485e0f165 100644 --- a/lib/downloader.rb +++ b/lib/downloader.rb @@ -11,12 +11,13 @@ def setTermSize # get terminal window size begin @termH, @termW = IO.console.winsize - rescue => e - puts "Non-interactive terminals may not be able to be queried for size." - # @termW = %x[tput cols].chomp.to_i - # @termH = %x[tput lines].chomp.to_i - @termW = 80 - @termH = 25 + rescue NoMethodError => e + unless @warned + STDERR.puts 'Non-interactive terminals may not be able to be queried for size.' + @warned = true + end + + @termH, @termW = [ 25, 80 ] end # space for progress bar after minus the reserved space for showing # the file size and progress percentage @@ -24,34 +25,41 @@ def setTermSize return true end -def downloader (url, filename = File.basename(url), retry_count = 0, verbose = false) +def downloader (*args) setTermSize # reset width settings after terminal resized trap('WINCH') { setTermSize } - uri = URI(url) - case uri.scheme - when /http?/ - when 'file' - # use FileUtils to copy if it is a local file (the url protocol is file://) - if File.exist?(uri.path) - return FileUtils.cp uri.path, filename + uri = URI(args[0]) # read url from given params + + unless CREW_USE_CURL or !ENV['CREW_DOWNLOADER'].to_s.empty? + case uri.scheme + when 'http', 'https' + # use net/http if the url protocol is http(s):// + http_downloader(*args) + when 'file' + # use FileUtils to copy if it is a local file (the url protocol is file://) + if File.exist?(uri.path) + return FileUtils.cp uri.path, filename + else + abort "#{uri.path}: File not found :/".lightred + end else - abort "#{uri.path}: File not found :/".lightred + # use external downloader (curl by default) if the url protocol is not http(s):// or file:// + external_downloader(*args) end else - # fallback to curl if the url protocol is not http(s):// or file:// - 0.step(3,1) do |i| - unless system "#{CURL} --ssl -#L -C - '#{uri.to_s}' -o '#{filename}'" - puts "Retrying, #{3-i} retries left.".yellow - else - return true - end - end - # the download failed if we're still here - abort 'Download failed :/ Please check your network settings.'.lightred + # force using external downloader if either CREW_USE_CURL or ENV['CREW_DOWNLOADER'] is set + external_downloader(*args) end +end +def http_downloader (url, filename = File.basename(url), retry_count = 0, verbose = false) + # http_downloader: Downloader based on net/http library + + uri = URI(url) + + # open http connection Net::HTTP.start(uri.host, uri.port, { max_retries: 3, use_ssl: uri.scheme.eql?('https'), @@ -60,7 +68,7 @@ def downloader (url, filename = File.basename(url), retry_count = 0, verbose = f }) do |http| http.request( Net::HTTP::Get.new(uri) ) do |response| case - when response.is_a?(Net::HTTPOK) + when response.is_a?(Net::HTTPSuccess) when response.is_a?(Net::HTTPRedirection) # follow HTTP redirection puts <<~EOT if verbose * Follow HTTP redirection: #{response['Location']} @@ -68,6 +76,8 @@ def downloader (url, filename = File.basename(url), retry_count = 0, verbose = f EOT redirect_uri = URI(response['Location']) + + # add url scheme/host for redirected url based on original url if missing redirect_uri.scheme ||= uri.scheme redirect_uri.host ||= uri.host @@ -77,8 +87,8 @@ def downloader (url, filename = File.basename(url), retry_count = 0, verbose = f end # get target file size (should be returned by the server) - file_size = response['Content-Length'].to_i - downloaded_size = 0 + file_size = response['Content-Length'].to_f + downloaded_size = 0.0 if verbose puts <<~EOT @@ -87,28 +97,44 @@ def downloader (url, filename = File.basename(url), retry_count = 0, verbose = f * EOT - response.to_hash.each_pair do |k, v| - puts "> #{k}: #{v}" - end + # parse response's header to readable format + response.to_hash.each_pair {|k, v| puts "> #{k}: #{v}" } + puts end + # read file chunks from server, write it to filesystem File.open(filename, 'wb') do |io| response.read_body do |chunk| - downloaded_size += chunk.size + downloaded_size += chunk.size # record downloaded size, used for showing progress bar if file_size.positive? # calculate downloading progress percentage with the given file size - percentage = (downloaded_size.to_f / file_size.to_f) * 100 + percentage = (downloaded_size / file_size) * 100 # show progress bar, file size and progress percentage - printf "\r[%-#{@progBarW}s] %9.9s %3d%%", - '#' * ( @progBarW * (percentage / 100) ).to_i, + printf "\r""[%-#{@progBarW}.#{@progBarW}s] %9.9s %3d%%", + '#' * ( @progBarW * (percentage / 100) ), human_size(file_size), percentage end - io.write(chunk) + io.write(chunk) # write to file end end puts end end end + +def external_downloader (url, filename = File.basename(url), retry_count = 0, verbose = false) + # external_downloader: wrapper for external downloaders in CREW_DOWNLOADER (curl by default) + + # default curl cmdline, CREW_DOWNLOADER should be in this format also + # %s: Will be substitute to "--verbose" if #{verbose} set to true, otherwise will be substitute to "" + # %s: Will be substitute to #{url} + # %s: Will be substitute to #{filename} + curl_cmdline = 'curl %s -L -# --ssl --retry 3 %s -o %s' + + # use CREW_DOWNLOADER if specified, use curl by default + downloader_cmdline = CREW_DOWNLOADER || curl_cmdline + + return system (downloader_cmdline % { verbose: verbose ? '--verbose' : '', url: url, output: filename}), exception: true +end