crew: Move checksum checking part to lib/downloader.rb (#6891)

* lib/downloader: Check checksum if provided * crew: Move checksum part to downloader * Update `Checksum mismatch` message * Move the `filename` or statement to a variable * Bump version * Make `sha256sum` a required param * Add SKIP option, update packages that use `downloader` * Remove extra round bracket * Fix option order * Update usage * libspatialaudio: Check sha256sum with `downloader`, rubyize * Make retry times configurable * Correct usage
2026-01-08 23:18:10 -05:00 · 2022-03-23 15:33:11 +08:00
parent 944607d5ac
commit 7d7e491190
12 changed files with 69 additions and 61 deletions
--- a/bin/crew
+++ b/bin/crew
@@ -712,10 +712,8 @@ def download
        end
      end
      # Download file if not cached.
-      downloader url, filename, @opt_verbose
+      downloader url, sha256sum, filename, @opt_verbose

-      abort 'Checksum mismatch. 😔 Try again.'.lightred unless
-        Digest::SHA256.hexdigest( File.read(filename) ) == sha256sum || sha256sum =~ /^SKIP$/i
      puts "#{@pkg.name.capitalize} archive downloaded.".lightgreen
      # Stow file in cache if requested, if file is not from cache,
      # and cache is writable.
@@ -738,10 +736,8 @@ def download
      @git = true
    else
      Dir.mkdir @extract_dir
-      downloader url, filename, @opt_verbose
+      downloader url, sha256sum, filename, @opt_verbose

-      abort 'Checksum mismatch. 😔 Try again.'.lightred unless
-        Digest::SHA256.hexdigest( File.read(filename) ) == sha256sum || sha256sum =~ /^SKIP$/i
      puts "#{filename}: File downloaded.".lightgreen

      FileUtils.mv filename, "#{@extract_dir}/#{filename}"
--- a/lib/const.rb
+++ b/lib/const.rb
@@ -1,6 +1,6 @@
 # Defines common constants used in different parts of crew

-CREW_VERSION = '1.23.4'
+CREW_VERSION = '1.23.5'

 ARCH_ACTUAL = `uname -m`.chomp
 # This helps with virtualized builds on aarch64 machines
@@ -87,6 +87,9 @@ CREW_USE_CURL = ENV['CREW_USE_CURL'] == '1'
 # About the format of the CREW_DOWNLOADER variable, see line 130-133 in lib/downloader.rb
 CREW_DOWNLOADER = ( ENV['CREW_DOWNLOADER'].to_s.empty? ) ? nil : ENV['CREW_DOWNLOADER']

+# Downloader maximum retry count
+CREW_DOWNLOADER_RETRY = ( ENV['CREW_DOWNLOADER_RETRY'].to_s.empty? ) ? 3 : ENV['CREW_DOWNLOADER_RETRY'].to_i
+
 # set certificate file location for lib/downloader.rb
 SSL_CERT_FILE = if ENV['SSL_CERT_FILE'].to_s.empty? || !File.exist?(ENV['SSL_CERT_FILE'])
                  if File.exist?("#{CREW_PREFIX}/etc/ssl/certs/ca-certificates.crt")
--- a/lib/downloader.rb
+++ b/lib/downloader.rb
@@ -1,4 +1,5 @@
 require 'io/console'
+require 'digest/sha2'
 require 'net/http'
 require 'uri'
 require 'resolv-replace'
@@ -25,43 +26,56 @@ def setTermSize
  return true
 end

-def downloader (*args)
+def downloader (url, sha256sum, filename = File.basename(url), verbose = false)
+  # downloader: wrapper for all Chromebrew downloaders (`net/http`,`curl`...)
+  # Usage: downloader <url>, <sha256sum>, <filename::optional>, <verbose::optional>
+  #
+  #           <url>: URL that points to the target file
+  #     <sha256sum>: SHA256 checksum, verify downloaded file with given checksum
+  #      <filename>: (Optional) Output path/filename
+  #       <verbose>: (Optional) Verbose output
+  #
  setTermSize
  # reset width settings after terminal resized
  trap('WINCH') { setTermSize }

-  uri = URI(args[0]) # read url from given params
+  uri = URI(url)

  unless CREW_USE_CURL or !ENV['CREW_DOWNLOADER'].to_s.empty?
    case uri.scheme
    when 'http', 'https'
      # use net/http if the url protocol is http(s)://
-      http_downloader(*args)
+      http_downloader(url, filename, verbose)
    when 'file'
      # use FileUtils to copy if it is a local file (the url protocol is file://)
      if File.exist?(uri.path)
-        return FileUtils.cp uri.path, args[1] || File.basename(uri.path)
+        return FileUtils.cp uri.path, filename
      else
        abort "#{uri.path}: File not found :/".lightred
      end
    else
      # use external downloader (curl by default) if the url protocol is not http(s):// or file://
-      external_downloader(*args)
+      external_downloader(url, filename, verbose)
    end
  else
    # force using external downloader if either CREW_USE_CURL or ENV['CREW_DOWNLOADER'] is set
-    external_downloader(*args)
+    external_downloader(url, filename, verbose)
+  end
+
+  # verify with given checksum
+  unless sha256sum =~ /^SKIP$/i or Digest::SHA256.hexdigest( File.read(filename) ) == sha256sum
+    abort 'Checksum mismatch :/ Try again?'.lightred
  end
 end

-def http_downloader (url, filename = File.basename(url), retry_count = 0, verbose = false)
+def http_downloader (url, filename = File.basename(url), verbose = false)
  # http_downloader: Downloader based on net/http library

  uri = URI(url)

  # open http connection
  Net::HTTP.start(uri.host, uri.port, {
-      max_retries: 3,
+      max_retries: CREW_DOWNLOADER_RETRY,
      use_ssl: uri.scheme.eql?('https'),
      ca_file: SSL_CERT_FILE,
      ca_path: SSL_CERT_DIR
@@ -81,7 +95,7 @@ def http_downloader (url, filename = File.basename(url), retry_count = 0, verbos
        redirect_uri.scheme ||= uri.scheme
        redirect_uri.host ||= uri.host

-        return downloader(redirect_uri.to_s, filename, retry_count, verbose)
+        return send(__method__, redirect_uri.to_s, filename, verbose)
      else
        abort "Download failed with error #{response.code}: #{response.msg}".lightred
      end
@@ -124,17 +138,18 @@ def http_downloader (url, filename = File.basename(url), retry_count = 0, verbos
  end
 end

-def external_downloader (url, filename = File.basename(url), retry_count = 0, verbose = false)
+def external_downloader (url, filename = File.basename(url), verbose = false)
  # external_downloader: wrapper for external downloaders in CREW_DOWNLOADER (curl by default)

  # default curl cmdline, CREW_DOWNLOADER should be in this format also
  #   %<verbose>s: Will be substitute to "--verbose" if #{verbose} set to true, otherwise will be substitute to ""
+  #      %<retry>: Will be substitute to #{CREW_DOWNLOADER_RETRY}
  #       %<url>s: Will be substitute to #{url}
  #    %<output>s: Will be substitute to #{filename}
-  curl_cmdline = 'curl %<verbose>s -L -# --ssl --retry 3 %<url>s -o %<output>s'
+  curl_cmdline = 'curl %<verbose>s -L -# --ssl --retry %<retry>s %<url>s -o %<output>s'

  # use CREW_DOWNLOADER if specified, use curl by default
  downloader_cmdline = CREW_DOWNLOADER || curl_cmdline

-  return system (downloader_cmdline % { verbose: verbose ? '--verbose' : '', url: url, output: filename}), exception: true
+  return system (downloader_cmdline % { verbose: verbose ? '--verbose' : '', retry: CREW_DOWNLOADER_RETRY, url: url, output: filename}), exception: true
 end
--- a/packages/box.rb
+++ b/packages/box.rb
@@ -11,10 +11,7 @@ class Box < Package
  depends_on 'php74' unless File.exist? "#{CREW_PREFIX}/bin/php"

  def self.install
-    downloader 'https://github.com/box-project/box/releases/download/3.16.0/box.phar'
-    unless Digest::SHA256.hexdigest( File.read('box.phar') ) == 'f508e28f309d7e95a319bdcd5f13dcfbb18eb91cb7a6cac9b69bc7799d78bdf9'
-      abort 'Checksum mismatch. :/ Try again.'.lightred
-    end
+    downloader 'https://github.com/box-project/box/releases/download/3.16.0/box.phar', 'f508e28f309d7e95a319bdcd5f13dcfbb18eb91cb7a6cac9b69bc7799d78bdf9'
    FileUtils.mkdir_p "#{CREW_DEST_PREFIX}/bin"
    FileUtils.install 'box.phar', "#{CREW_DEST_PREFIX}/bin/box", mode: 0o755
  end
--- a/packages/cunit.rb
+++ b/packages/cunit.rb
@@ -24,12 +24,14 @@ class Cunit < Package
  })

  def self.patch
-    downloader 'https://httpredir.debian.org/debian/pool/main/c/cunit/cunit_2.1-3-dfsg-2.4.debian.tar.xz'
-    unless Digest::SHA256.hexdigest(File.read('cunit_2.1-3-dfsg-2.4.debian.tar.xz')) == 'e7a09a24c7db0e2aa9feb444fe38957286ebfc63b355c308957794f064b5881d'
-      abort 'Checksum mismatch. :/ Try again.'.lightred
-    end
+    downloader 'https://httpredir.debian.org/debian/pool/main/c/cunit/cunit_2.1-3-dfsg-2.4.debian.tar.xz', 'e7a09a24c7db0e2aa9feb444fe38957286ebfc63b355c308957794f064b5881d'
+
    system 'tar xf cunit_2.1-3-dfsg-2.4.debian.tar.xz'
-    system "for i in \$(cat debian/patches/series); do patch -Np1 -i debian/patches/\${i}; done"
+
+    File.foreach 'debian/patches/series' do |patch|
+      system "patch -Np1 -i debian/patches/#{patch}"
+    end
+
    system "sed -i 's:<curses.h>:<ncursesw/curses.h>:' CUnit/Sources/Curses/Curses.c"
    system "sed -i 's:ncurses:ncursesw:g' configure.in"
  end
--- a/packages/docbook_xsl.rb
+++ b/packages/docbook_xsl.rb
@@ -28,10 +28,8 @@ class Docbook_xsl < Package
  depends_on 'xmlcatmgr'

  def self.patch
-    downloader 'https://github.com/archlinux/svntogit-packages/raw/packages/docbook-xsl/trunk/765567_non-recursive_string_subst.patch'
-    unless Digest::SHA256.hexdigest(File.read('765567_non-recursive_string_subst.patch')) == '193ec26dcb37bdf12037ed4ea98d68bd550500c8e96b719685d76d7096c3f9b3'
-      abort 'Checksum mismatch. :/ Try again.'.lightred
-    end
+    downloader 'https://github.com/archlinux/svntogit-packages/raw/packages/docbook-xsl/trunk/765567_non-recursive_string_subst.patch',
+               '193ec26dcb37bdf12037ed4ea98d68bd550500c8e96b719685d76d7096c3f9b3'
    system 'patch -Np2 -i 765567_non-recursive_string_subst.patch'
  end

--- a/packages/docbook_xsl_nons.rb
+++ b/packages/docbook_xsl_nons.rb
@@ -28,10 +28,9 @@ class Docbook_xsl_nons < Package
  depends_on 'xmlcatmgr'

  def self.patch
-    downloader 'https://github.com/archlinux/svntogit-packages/raw/packages/docbook-xsl/trunk/765567_non-recursive_string_subst.patch'
-    unless Digest::SHA256.hexdigest(File.read('765567_non-recursive_string_subst.patch')) == '193ec26dcb37bdf12037ed4ea98d68bd550500c8e96b719685d76d7096c3f9b3'
-      abort 'Checksum mismatch. :/ Try again.'.lightred
-    end
+    downloader 'https://github.com/archlinux/svntogit-packages/raw/packages/docbook-xsl/trunk/765567_non-recursive_string_subst.patch',
+               '193ec26dcb37bdf12037ed4ea98d68bd550500c8e96b719685d76d7096c3f9b3'
+
    system 'patch -Np2 -i 765567_non-recursive_string_subst.patch'
  end

--- a/packages/libebml.rb
+++ b/packages/libebml.rb
@@ -24,10 +24,9 @@ class Libebml < Package
  })

  def self.patch
-    downloader 'https://salsa.debian.org/multimedia-team/libebml/-/raw/master/debian/patches/0001-include-appropriate-header-files-for-std-numeric_lim.patch'
-    unless Digest::SHA256.hexdigest(File.read('0001-include-appropriate-header-files-for-std-numeric_lim.patch')) == 'e0662dddd31ffcc0581bbba25de707fe881e556eef540522eb9b7956cb3cd32c'
-      abort 'Checksum mismatch :/ try again'.lightred
-    end
+    downloader 'https://salsa.debian.org/multimedia-team/libebml/-/raw/master/debian/patches/0001-include-appropriate-header-files-for-std-numeric_lim.patch',
+               'e0662dddd31ffcc0581bbba25de707fe881e556eef540522eb9b7956cb3cd32c'
+
    system 'patch -Np1 -i 0001-include-appropriate-header-files-for-std-numeric_lim.patch'
  end

--- a/packages/libmad.rb
+++ b/packages/libmad.rb
@@ -23,10 +23,8 @@ class Libmad < Package
  })

  def self.patch
-    downloader 'https://httpredir.debian.org/debian/pool/main/libm/libmad/libmad_0.15.1b-10.diff.gz'
-    unless Digest::SHA256.hexdigest(File.read('libmad_0.15.1b-10.diff.gz')) == 'dfeabd5d2398bf902660edc31f87ad40600f0aa732b946f864d8ee6bbf56a99c'
-      abort 'Checksum mismatch. :/ Try again.'.lightred
-    end
+    downloader 'https://httpredir.debian.org/debian/pool/main/libm/libmad/libmad_0.15.1b-10.diff.gz', 'dfeabd5d2398bf902660edc31f87ad40600f0aa732b946f864d8ee6bbf56a99c'
+
    system 'zcat libmad_0.15.1b-10.diff.gz | patch -Np1'
    system "for i in \$(cat debian/patches/series); do patch -Np1 -i debian/patches/\${i}; done"
  end
--- a/packages/libspatialaudio.rb
+++ b/packages/libspatialaudio.rb
@@ -28,20 +28,22 @@ class Libspatialaudio < Package
  def self.patch
    FileUtils.mkdir_p 'source/normal'
    Dir.chdir 'source/normal' do
-      downloader 'https://github.com/greekgoddj/mit-hrtf-lib/raw/658657aebc58480d5511bd726b8dfa73b4f49ac7/source/normal/mit_hrtf_normal_44100.h'
-      downloader 'https://github.com/greekgoddj/mit-hrtf-lib/raw/658657aebc58480d5511bd726b8dfa73b4f49ac7/source/normal/mit_hrtf_normal_48000.h'
-      downloader 'https://github.com/greekgoddj/mit-hrtf-lib/raw/658657aebc58480d5511bd726b8dfa73b4f49ac7/source/normal/mit_hrtf_normal_88200.h'
-      downloader 'https://github.com/greekgoddj/mit-hrtf-lib/raw/658657aebc58480d5511bd726b8dfa73b4f49ac7/source/normal/mit_hrtf_normal_96000.h'
-      @sha256sums = <<~EOF
-        ffee6a5f1f1e771c863c1c02be3d01e7a23359d8553b1f5aa4dba014f1eb05dd  mit_hrtf_normal_44100.h
-        807de23e95e299692516ccaebfb907e1d8caf554f38e87dd8fd07caee41aae7f  mit_hrtf_normal_48000.h
-        f106a37cd87689cde5d958e3201e6f1d214998063caa60486d2dcf672deaa70b  mit_hrtf_normal_88200.h
-        37880838efe36fea8c1eff1c9a4b4b4fb0a44e61aaec6cfda716062ccbc75fe2  mit_hrtf_normal_96000.h
-      EOF
-      File.write('sha256sums', @sha256sums)
-      system 'sha256sum -c sha256sums'
+      downloader 'https://github.com/greekgoddj/mit-hrtf-lib/raw/658657aebc58480d5511bd726b8dfa73b4f49ac7/source/normal/mit_hrtf_normal_44100.h',
+                 'ffee6a5f1f1e771c863c1c02be3d01e7a23359d8553b1f5aa4dba014f1eb05dd'
+
+      downloader 'https://github.com/greekgoddj/mit-hrtf-lib/raw/658657aebc58480d5511bd726b8dfa73b4f49ac7/source/normal/mit_hrtf_normal_48000.h',
+                 '807de23e95e299692516ccaebfb907e1d8caf554f38e87dd8fd07caee41aae7f'
+
+      downloader 'https://github.com/greekgoddj/mit-hrtf-lib/raw/658657aebc58480d5511bd726b8dfa73b4f49ac7/source/normal/mit_hrtf_normal_88200.h',
+                 'f106a37cd87689cde5d958e3201e6f1d214998063caa60486d2dcf672deaa70b'
+
+      downloader 'https://github.com/greekgoddj/mit-hrtf-lib/raw/658657aebc58480d5511bd726b8dfa73b4f49ac7/source/normal/mit_hrtf_normal_96000.h',
+                 '37880838efe36fea8c1eff1c9a4b4b4fb0a44e61aaec6cfda716062ccbc75fe2'
+    end
+
+    File.foreach 'debian/patches/series' do |patch|
+      system "patch -Np1 -i debian/patches/#{patch}"
    end
-    system "for i in \$(cat debian/patches/series); do patch -Np1 -i debian/patches/\${i}; done"
  end

  def self.build
--- a/packages/p7zip_gui.rb
+++ b/packages/p7zip_gui.rb
@@ -41,8 +41,7 @@ class P7zip_gui < Package
    patches.each do |url, sha256sum|
      patch_filename = File.basename(url)

-      downloader url, patch_filename
-      abort 'Checksum mismatch :/'.lightred unless Digest::SHA256.hexdigest( File.read(patch_filename) ) == sha256sum
+      downloader url, sha256sum, patch_filename

      system 'patch', '-p1', '-i', patch_filename
    end
--- a/packages/symfony.rb
+++ b/packages/symfony.rb
@@ -23,8 +23,8 @@ class Symfony < Package
      sha256 = '66c2daf21e3acbdda8d826a0484b02e59255401d54027cdbe2605406f77933a8'
    end
    symfony_file = "symfony_linux_#{arch}"
-    downloader "https://github.com/symfony/cli/releases/download/v#{version}/#{symfony_file}"
-    abort 'Checksum mismatch. :/ Try again.'.lightred unless Digest::SHA256.hexdigest( File.read(symfony_file) ) == sha256
+    downloader "https://github.com/symfony/cli/releases/download/v#{version}/#{symfony_file}", sha256
+
    FileUtils.mkdir_p "#{CREW_DEST_PREFIX}/bin"
    FileUtils.install symfony_file, "#{CREW_DEST_PREFIX}/bin/symfony", mode: 0o755
  end