Support source_urls ending with .git (#5598)

* support source_urls ending with .git * crew: clean up * make sha256sum more readable * change git_branch to git_tag and allow git hashes (thanks @satmandu) * support git hashes * support caching with git sources * crew: fix compressdoc and fix creating tarball * lib/const.rb: change envvar to CREW_IS_CACHING * packages/dash.rb: fix homepage and binaries * crew: remove (currently) extraneous variable long_verbose * bin/crew: fix compressdoc * packages/dash.rb: fix homepages (again) * bin/crew: change $extract_dir to @extract_dir * CREW_IS_CACHING => CREW_CACHE_ENABLED * condense tag + hash checkout logic part 1 * condense tag + hash checkout logic part 2 * Bump version Co-authored-by: satmandu <satadru@gmail.com> Co-authored-by: satmandu <satadru@umich.edu>
2026-01-11 00:18:06 -05:00 · 2021-04-15 13:32:58 -04:00
parent e5d4c39711
commit 3a53fc1c8e
4 changed files with 107 additions and 54 deletions
--- a/bin/crew
+++ b/bin/crew
@@ -111,11 +111,11 @@ String.use_color = args["--color"] || !args["--no-color"]
 if @opt_verbose then
  @fileutils_verbose = true
  @verbose = 'v'
-  @mv_verbose = '-v'
+  @short_verbose = '-v'
 else
  @fileutils_verbose = false
  @verbose = ''
-  @mv_verbose = ''
+  @short_verbose = ''
 end

@opt_src = args["--build-from-source"]
@@ -543,6 +543,11 @@ def download
  url = @pkg.get_url(@device[:architecture])
  source = @pkg.is_source?(@device[:architecture])

+  uri = URI.parse url
+  filename = File.basename(uri.path)
+  sha256sum = @pkg.get_sha256(@device[:architecture])
+  @extract_dir = @pkg.get_extract_dir
+
  if !url
    abort "No precompiled binary or source is available for #{@device[:architecture]}.".lightred
  elsif !source
@@ -553,65 +558,98 @@ def download
    puts "No precompiled binary available for your platform, downloading source..."
  end

-  uri = URI.parse url
-  filename = File.basename(uri.path)
-  if source
-    sha256sum = @pkg.source_sha256
-  else
-    sha256sum = @pkg.binary_sha256[@device[:architecture]]
-  end
  Dir.chdir CREW_BREW_DIR do
-    if ENV['CREW_CACHE_OPT']
-      FileUtils.mkdir_p CREW_CACHE_DIR, verbose: @fileutils_verbose
-      cachefile = CREW_CACHE_DIR + filename
-      if File.file?(cachefile)
-        if Digest::SHA256.hexdigest( File.read(cachefile) ) == sha256sum then
-          FileUtils.cp cachefile, CREW_BREW_DIR, verbose: @fileutils_verbose
-          puts "Archive found in cache".lightgreen
-          return {source: source, filename: filename}
+
+    case File.basename(filename)
+    # Sources that download with curl
+    when /\.zip$/i, /\.(tar(\.(gz|bz2|xz|lz))?|tgz|tbz|txz)$/i, /\.deb$/i
+      # Recall file from cache if requeseted
+      if CREW_CACHE_ENABLED
+        cachefile = CREW_CACHE_DIR + filename
+        if File.file?(cachefile)
+          if Digest::SHA256.hexdigest( File.read(cachefile) ) == sha256sum then
+            FileUtils.cp cachefile, CREW_BREW_DIR, verbose: @fileutils_verbose
+            puts "Archive found in cache".lightgreen
+            return {source: source, filename: filename}
          else
-          puts 'Cached archive checksum mismatch. :/ Will download.'.lightred
+            puts 'Cached archive checksum mismatch. :/ Will download.'.lightred
+          end
+        puts 'Cannot find cached archive. :/ Will download.'.lightred
        end
      end
-    end
+      # Download file if not cached
+      system "#{CURL} --retry 3 -#{@verbose}#LC - --insecure \'#{url}\' --output #{filename}"
+      abort 'Checksum mismatch. :/ Try again.'.lightred unless
+        Digest::SHA256.hexdigest( File.read(filename) ) == sha256sum
+      puts "Archive downloaded".lightgreen
+      # Stow file in cache if requested
+      if CREW_CACHE_ENABLED
+        puts 'Caching downloaded archive'.lightgreen
+        FileUtils.cp filename, CREW_CACHE_DIR, verbose: @fileutils_verbose
+      end

-    system "#{CURL} --retry 3 -#{@verbose}#LC - --insecure \'#{url}\' --output #{filename}"
-
-    abort 'Checksum mismatch. :/ Try again.'.lightred unless
-      Digest::SHA256.hexdigest( File.read(filename) ) == sha256sum
-    puts "Archive downloaded".lightgreen
-
-    if ENV['CREW_CACHE_OPT']
-      puts 'Caching downloaded archive'.lightgreen
-      FileUtils.cp filename, CREW_CACHE_DIR, verbose: @fileutils_verbose
+    # Sources that download with git
+    when /\.git$/i
+      # Recall repository from cache if requested
+      if CREW_CACHE_ENABLED
+        cachefile = CREW_CACHE_DIR + '/' + filename + '.tar.xz'
+        if File.file?(cachefile)
+          if system "sha256sum -c #{cachefile}.sha256"
+            FileUtils.mkdir @extract_dir
+            system "tar x#{@verbose}f #{cachefile} -C #{@extract_dir}"
+            return {source: source, filename: filename}
+          else
+            puts 'Cached repository checksum mismatch. :/ Will download.'.lightred
+          end
+        else
+          puts 'Cannot find cached repository. :/ Will download.'.lightred
+        end
+      end
+      # Download via git
+      Dir.mkdir @extract_dir
+      Dir.chdir @extract_dir do
+        system 'git init'
+        system 'git config advice.detachedHead false'
+        system 'git config init.defaultBranch master'
+        system "git remote add origin #{@pkg.source_url}", exception: true
+        system "git fetch --depth 1 origin #{@pkg.git_hashtag}", exception: true
+        system 'git checkout FETCH_HEAD'
+        system 'git submodule update --init --recursive'
+        puts "Repository downloaded".lightgreen
+      end
+      # Stow file in cache if requested
+      if CREW_CACHE_ENABLED
+        puts 'Caching downloaded archive.'.lightgreen
+          Dir.chdir "#{@extract_dir}" do system "tar c#{@verbose}Jf #{CREW_CACHE_DIR}/#{filename}.tar.xz $(find -mindepth 1 -maxdepth 1 -printf '%P\n')" end
+        system "sha256sum #{CREW_CACHE_DIR}/#{filename}.tar.xz > #{CREW_CACHE_DIR}/#{filename}.tar.xz.sha256"
+      end
    end
  end
  return {source: source, filename: filename}
 end

 def unpack (meta)
-  extract_dir = meta[:filename] + '.' + SecureRandom.alphanumeric(8) + '.dir'
  target_dir = nil
  Dir.chdir CREW_BREW_DIR do
-    FileUtils.mkdir_p extract_dir, verbose: @fileutils_verbose
+    FileUtils.mkdir_p @extract_dir, verbose: @fileutils_verbose
    case File.basename meta[:filename]
    when /\.zip$/i
-      puts "Unpacking archive using 'unzip', this may take a while..."
+      puts "Unpacking archive using 'unzip', this may take a while...".lightgreen
      _verbopt = @opt_verbose ? '-v' : '-qq'
-      system 'unzip', _verbopt, '-d', extract_dir, meta[:filename]
+      system 'unzip', _verbopt, '-d', @extract_dir, meta[:filename], exception: true
    when /\.(tar(\.(gz|bz2|xz|lz))?|tgz|tbz|txz)$/i
-      puts "Unpacking archive using 'tar', this may take a while..."
-      FileUtils.mkdir extract_dir unless Dir.exist?(extract_dir)
-      system "tar x#{@verbose}f #{meta[:filename]} -C #{extract_dir}"
+      puts "Unpacking archive using 'tar', this may take a while...".lightgreen
+      FileUtils.mkdir @extract_dir unless Dir.exist?(@extract_dir)
+      system "tar x#{@verbose}f #{meta[:filename]} -C #{@extract_dir}", exception: true
    when /\.deb$/i
-      puts "Unpacking archive using 'ar', this may take a while..."
-      FileUtils.mkdir extract_dir unless Dir.exist?(extract_dir)
-      system "ar -p #{meta[:filename]} data.tar.xz | xz -dc#{@verbose} | tar x#{@verbose} -C #{extract_dir}"
+      puts "Unpacking archive using 'ar', this may take a while...".lightgreen
+      FileUtils.mkdir @extract_dir unless Dir.exist?(@extract_dir)
+      system "ar -p #{meta[:filename]} data.tar.xz | xz -dc#{@verbose} | tar x#{@verbose} -C #{@extract_dir}", exception: true
    end
    if meta[:source] == true
      # Check the number of directories in the archive
-      entries = Dir["#{extract_dir}/*"]
-      entries = Dir[extract_dir] if entries.empty?
+      entries = Dir["#{@extract_dir}/*"]
+      entries = Dir[@extract_dir] if entries.empty?
      if entries.length == 0
        abort "Empty archive: #{meta[:filename]}".lightred
      elsif entries.length == 1 && File.directory?(entries.first)
@@ -619,11 +657,11 @@ def unpack (meta)
        target_dir = entries.first
      else
        # Use `extract_dir` otherwise
-        target_dir = extract_dir
+        target_dir = @extract_dir
      end
    else
      # Use `extract_dir` for binary distribution
-      target_dir = extract_dir
+      target_dir = @extract_dir
    end
  end
  return CREW_BREW_DIR + target_dir
@@ -639,7 +677,7 @@ def build_and_preconfigure (target_dir)
    # https://stackoverflow.com/questions/42963653/libquadmath-la-is-not-a-valid-libtool-archive-when-configuring-openmpi-with-g
    puts 'Rename all *.la files to *.la_tmp'.lightblue

-    system "find #{CREW_LIB_PREFIX} -type f -name *.la -print0 | xargs --null -I{} mv #{@mv_verbose} {} {}_tmp"
+    system "find #{CREW_LIB_PREFIX} -type f -name *.la -print0 | xargs --null -I{} mv #{@short_verbose} {} {}_tmp"

    @pkg.in_build = true
    @pkg.patch
@@ -654,7 +692,7 @@ def build_and_preconfigure (target_dir)
    # Rename all *.la_tmp back to *.la to avoid
    # cannot access '*.la': No such file or directory
    puts 'Rename all *.la_tmp files back to *.la'.lightblue
-    system "find #{CREW_LIB_PREFIX} -type f -name '*.la_tmp' -exec sh -c 'mv #{@mv_verbose} \"$1\" \"${1%.la_tmp}.la\"' _  {} \\;"
+    system "find #{CREW_LIB_PREFIX} -type f -name '*.la_tmp' -exec sh -c 'mv #{@short_verbose} \"$1\" \"${1%.la_tmp}.la\"' _  {} \\;"
  end
 end

@@ -678,7 +716,7 @@ def compress_doc (dir)

  if Dir.exist? dir
    system "find #{dir} -type f ! -perm -200 | xargs -r chmod u+w"
-    system "compressdoc --gzip -9#{@verbose} #{dir}"
+    system "compressdoc --gzip -9 #{@short_verbose} #{dir}"
  end
 end

--- a/lib/const.rb
+++ b/lib/const.rb
@@ -1,6 +1,6 @@
 # Defines common constants used in different parts of crew

-CREW_VERSION = '1.8.5'
+CREW_VERSION = '1.8.6'

 ARCH_ACTUAL = `uname -m`.strip
 # This helps with virtualized builds on aarch64 machines
@@ -36,10 +36,13 @@ else
 end

 if ENV['CREW_CACHE_DIR'].to_s == ''
-  CREW_CACHE_DIR = HOME + '/.cache/crewcache/'
+  CREW_CACHE_DIR = HOME + '/.cache/crewcache'
 else
  CREW_CACHE_DIR = ENV['CREW_CACHE_DIR']
 end
+FileUtils.mkdir_p CREW_CACHE_DIR unless Dir.exist? CREW_CACHE_DIR
+
+CREW_CACHE_ENABLED = ENV['CREW_CACHE_ENABLED']

 CREW_DEST_HOME = CREW_DEST_DIR + HOME

--- a/lib/package.rb
+++ b/lib/package.rb
@@ -1,7 +1,7 @@
 require 'package_helpers'

 class Package
-  property :description, :homepage, :version, :license, :compatibility, :binary_url, :binary_sha256, :source_url, :source_sha256, :is_fake
+  property :description, :homepage, :version, :license, :compatibility, :binary_url, :binary_sha256, :source_url, :source_sha256, :git_hashtag, :is_fake

  class << self
    attr_reader :is_fake
@@ -38,15 +38,27 @@ class Package
  end

  def self.get_url (architecture)
-    if !@build_from_source && @binary_url && @binary_url.has_key?(architecture)
+    if !@build_from_source and @binary_url and @binary_url.has_key?(architecture)
      return @binary_url[architecture]
    else
      return @source_url
    end
  end

+  def self.get_sha256 (architecture)
+    if !@build_from_source and @binary_sha256 and @binary_sha256.has_key?(architecture)
+      return @binary_sha256[architecture]
+    else
+      return @source_sha256
+    end
+  end
+
+  def self.get_extract_dir
+    name + '.' + Time.now.utc.strftime("%Y%m%d%H%M%S") + '.dir'
+  end
+
  def self.is_binary? (architecture)
-    if !@build_from_source && @binary_url && @binary_url.has_key?(architecture)
+    if !@build_from_source and @binary_url and @binary_url.has_key?(architecture)
      return true
    else
      return false
@@ -54,7 +66,7 @@ class Package
  end

  def self.is_source? (architecture)
-    if is_binary?(architecture) || is_fake?
+    if is_binary?(architecture) or is_fake?
      return false
    else
      return true
--- a/packages/dash.rb
+++ b/packages/dash.rb
@@ -2,12 +2,12 @@ require 'package'

 class Dash < Package
  description 'The Debian Almquist Shell (dash) is a POSIX-compliant shell derived from ash that executes scripts faster than bash and has fewer library dependencies.'
-  homepage 'https://salsa.debian.org/debian/dash'
+  homepage 'https://salsa.debian.org/debian/dash/'
  version '0.5.11.1'
  license 'BSD'
  compatibility 'all'
-  source_url 'https://salsa.debian.org/debian/dash/-/archive/upstream/0.5.11.1/dash-upstream-0.5.11.1.tar.bz2'
-  source_sha256 '4bb75944bb47fa6d1cf1e8a3ff941f6f1fb23497b553446e9f615d52d07ef1e7'
+  source_url 'https://salsa.debian.org/debian/dash.git'
+  git_tag 'upstream/0.5.11.2'

  binary_url ({
    aarch64: 'https://downloads.sourceforge.net/project/chromebrew/armv7l/dash-0.5.11.1-chromeos-armv7l.tar.xz',