From 3a53fc1c8e4e5eb383287d89d2b4487be6eafe46 Mon Sep 17 00:00:00 2001 From: Cassandra Watergate <55339220+saltedcoffii@users.noreply.github.com> Date: Thu, 15 Apr 2021 13:32:58 -0400 Subject: [PATCH] Support source_urls ending with .git (#5598) * support source_urls ending with .git * crew: clean up * make sha256sum more readable * change git_branch to git_tag and allow git hashes (thanks @satmandu) * support git hashes * support caching with git sources * crew: fix compressdoc and fix creating tarball * lib/const.rb: change envvar to CREW_IS_CACHING * packages/dash.rb: fix homepage and binaries * crew: remove (currently) extraneous variable long_verbose * bin/crew: fix compressdoc * packages/dash.rb: fix homepages (again) * bin/crew: change $extract_dir to @extract_dir * CREW_IS_CACHING => CREW_CACHE_ENABLED * condense tag + hash checkout logic part 1 * condense tag + hash checkout logic part 2 * Bump version Co-authored-by: satmandu Co-authored-by: satmandu --- bin/crew | 128 ++++++++++++++++++++++++++++++----------------- lib/const.rb | 7 ++- lib/package.rb | 20 ++++++-- packages/dash.rb | 6 +-- 4 files changed, 107 insertions(+), 54 deletions(-) diff --git a/bin/crew b/bin/crew index 8732ecbfbf..cb744e7150 100755 --- a/bin/crew +++ b/bin/crew @@ -111,11 +111,11 @@ String.use_color = args["--color"] || !args["--no-color"] if @opt_verbose then @fileutils_verbose = true @verbose = 'v' - @mv_verbose = '-v' + @short_verbose = '-v' else @fileutils_verbose = false @verbose = '' - @mv_verbose = '' + @short_verbose = '' end @opt_src = args["--build-from-source"] @@ -543,6 +543,11 @@ def download url = @pkg.get_url(@device[:architecture]) source = @pkg.is_source?(@device[:architecture]) + uri = URI.parse url + filename = File.basename(uri.path) + sha256sum = @pkg.get_sha256(@device[:architecture]) + @extract_dir = @pkg.get_extract_dir + if !url abort "No precompiled binary or source is available for #{@device[:architecture]}.".lightred elsif !source @@ -553,65 +558,98 @@ def download puts "No precompiled binary available for your platform, downloading source..." end - uri = URI.parse url - filename = File.basename(uri.path) - if source - sha256sum = @pkg.source_sha256 - else - sha256sum = @pkg.binary_sha256[@device[:architecture]] - end Dir.chdir CREW_BREW_DIR do - if ENV['CREW_CACHE_OPT'] - FileUtils.mkdir_p CREW_CACHE_DIR, verbose: @fileutils_verbose - cachefile = CREW_CACHE_DIR + filename - if File.file?(cachefile) - if Digest::SHA256.hexdigest( File.read(cachefile) ) == sha256sum then - FileUtils.cp cachefile, CREW_BREW_DIR, verbose: @fileutils_verbose - puts "Archive found in cache".lightgreen - return {source: source, filename: filename} + + case File.basename(filename) + # Sources that download with curl + when /\.zip$/i, /\.(tar(\.(gz|bz2|xz|lz))?|tgz|tbz|txz)$/i, /\.deb$/i + # Recall file from cache if requeseted + if CREW_CACHE_ENABLED + cachefile = CREW_CACHE_DIR + filename + if File.file?(cachefile) + if Digest::SHA256.hexdigest( File.read(cachefile) ) == sha256sum then + FileUtils.cp cachefile, CREW_BREW_DIR, verbose: @fileutils_verbose + puts "Archive found in cache".lightgreen + return {source: source, filename: filename} else - puts 'Cached archive checksum mismatch. :/ Will download.'.lightred + puts 'Cached archive checksum mismatch. :/ Will download.'.lightred + end + puts 'Cannot find cached archive. :/ Will download.'.lightred end end - end + # Download file if not cached + system "#{CURL} --retry 3 -#{@verbose}#LC - --insecure \'#{url}\' --output #{filename}" + abort 'Checksum mismatch. :/ Try again.'.lightred unless + Digest::SHA256.hexdigest( File.read(filename) ) == sha256sum + puts "Archive downloaded".lightgreen + # Stow file in cache if requested + if CREW_CACHE_ENABLED + puts 'Caching downloaded archive'.lightgreen + FileUtils.cp filename, CREW_CACHE_DIR, verbose: @fileutils_verbose + end - system "#{CURL} --retry 3 -#{@verbose}#LC - --insecure \'#{url}\' --output #{filename}" - - abort 'Checksum mismatch. :/ Try again.'.lightred unless - Digest::SHA256.hexdigest( File.read(filename) ) == sha256sum - puts "Archive downloaded".lightgreen - - if ENV['CREW_CACHE_OPT'] - puts 'Caching downloaded archive'.lightgreen - FileUtils.cp filename, CREW_CACHE_DIR, verbose: @fileutils_verbose + # Sources that download with git + when /\.git$/i + # Recall repository from cache if requested + if CREW_CACHE_ENABLED + cachefile = CREW_CACHE_DIR + '/' + filename + '.tar.xz' + if File.file?(cachefile) + if system "sha256sum -c #{cachefile}.sha256" + FileUtils.mkdir @extract_dir + system "tar x#{@verbose}f #{cachefile} -C #{@extract_dir}" + return {source: source, filename: filename} + else + puts 'Cached repository checksum mismatch. :/ Will download.'.lightred + end + else + puts 'Cannot find cached repository. :/ Will download.'.lightred + end + end + # Download via git + Dir.mkdir @extract_dir + Dir.chdir @extract_dir do + system 'git init' + system 'git config advice.detachedHead false' + system 'git config init.defaultBranch master' + system "git remote add origin #{@pkg.source_url}", exception: true + system "git fetch --depth 1 origin #{@pkg.git_hashtag}", exception: true + system 'git checkout FETCH_HEAD' + system 'git submodule update --init --recursive' + puts "Repository downloaded".lightgreen + end + # Stow file in cache if requested + if CREW_CACHE_ENABLED + puts 'Caching downloaded archive.'.lightgreen + Dir.chdir "#{@extract_dir}" do system "tar c#{@verbose}Jf #{CREW_CACHE_DIR}/#{filename}.tar.xz $(find -mindepth 1 -maxdepth 1 -printf '%P\n')" end + system "sha256sum #{CREW_CACHE_DIR}/#{filename}.tar.xz > #{CREW_CACHE_DIR}/#{filename}.tar.xz.sha256" + end end end return {source: source, filename: filename} end def unpack (meta) - extract_dir = meta[:filename] + '.' + SecureRandom.alphanumeric(8) + '.dir' target_dir = nil Dir.chdir CREW_BREW_DIR do - FileUtils.mkdir_p extract_dir, verbose: @fileutils_verbose + FileUtils.mkdir_p @extract_dir, verbose: @fileutils_verbose case File.basename meta[:filename] when /\.zip$/i - puts "Unpacking archive using 'unzip', this may take a while..." + puts "Unpacking archive using 'unzip', this may take a while...".lightgreen _verbopt = @opt_verbose ? '-v' : '-qq' - system 'unzip', _verbopt, '-d', extract_dir, meta[:filename] + system 'unzip', _verbopt, '-d', @extract_dir, meta[:filename], exception: true when /\.(tar(\.(gz|bz2|xz|lz))?|tgz|tbz|txz)$/i - puts "Unpacking archive using 'tar', this may take a while..." - FileUtils.mkdir extract_dir unless Dir.exist?(extract_dir) - system "tar x#{@verbose}f #{meta[:filename]} -C #{extract_dir}" + puts "Unpacking archive using 'tar', this may take a while...".lightgreen + FileUtils.mkdir @extract_dir unless Dir.exist?(@extract_dir) + system "tar x#{@verbose}f #{meta[:filename]} -C #{@extract_dir}", exception: true when /\.deb$/i - puts "Unpacking archive using 'ar', this may take a while..." - FileUtils.mkdir extract_dir unless Dir.exist?(extract_dir) - system "ar -p #{meta[:filename]} data.tar.xz | xz -dc#{@verbose} | tar x#{@verbose} -C #{extract_dir}" + puts "Unpacking archive using 'ar', this may take a while...".lightgreen + FileUtils.mkdir @extract_dir unless Dir.exist?(@extract_dir) + system "ar -p #{meta[:filename]} data.tar.xz | xz -dc#{@verbose} | tar x#{@verbose} -C #{@extract_dir}", exception: true end if meta[:source] == true # Check the number of directories in the archive - entries = Dir["#{extract_dir}/*"] - entries = Dir[extract_dir] if entries.empty? + entries = Dir["#{@extract_dir}/*"] + entries = Dir[@extract_dir] if entries.empty? if entries.length == 0 abort "Empty archive: #{meta[:filename]}".lightred elsif entries.length == 1 && File.directory?(entries.first) @@ -619,11 +657,11 @@ def unpack (meta) target_dir = entries.first else # Use `extract_dir` otherwise - target_dir = extract_dir + target_dir = @extract_dir end else # Use `extract_dir` for binary distribution - target_dir = extract_dir + target_dir = @extract_dir end end return CREW_BREW_DIR + target_dir @@ -639,7 +677,7 @@ def build_and_preconfigure (target_dir) # https://stackoverflow.com/questions/42963653/libquadmath-la-is-not-a-valid-libtool-archive-when-configuring-openmpi-with-g puts 'Rename all *.la files to *.la_tmp'.lightblue - system "find #{CREW_LIB_PREFIX} -type f -name *.la -print0 | xargs --null -I{} mv #{@mv_verbose} {} {}_tmp" + system "find #{CREW_LIB_PREFIX} -type f -name *.la -print0 | xargs --null -I{} mv #{@short_verbose} {} {}_tmp" @pkg.in_build = true @pkg.patch @@ -654,7 +692,7 @@ def build_and_preconfigure (target_dir) # Rename all *.la_tmp back to *.la to avoid # cannot access '*.la': No such file or directory puts 'Rename all *.la_tmp files back to *.la'.lightblue - system "find #{CREW_LIB_PREFIX} -type f -name '*.la_tmp' -exec sh -c 'mv #{@mv_verbose} \"$1\" \"${1%.la_tmp}.la\"' _ {} \\;" + system "find #{CREW_LIB_PREFIX} -type f -name '*.la_tmp' -exec sh -c 'mv #{@short_verbose} \"$1\" \"${1%.la_tmp}.la\"' _ {} \\;" end end @@ -678,7 +716,7 @@ def compress_doc (dir) if Dir.exist? dir system "find #{dir} -type f ! -perm -200 | xargs -r chmod u+w" - system "compressdoc --gzip -9#{@verbose} #{dir}" + system "compressdoc --gzip -9 #{@short_verbose} #{dir}" end end diff --git a/lib/const.rb b/lib/const.rb index e32e90965a..da8bad9058 100644 --- a/lib/const.rb +++ b/lib/const.rb @@ -1,6 +1,6 @@ # Defines common constants used in different parts of crew -CREW_VERSION = '1.8.5' +CREW_VERSION = '1.8.6' ARCH_ACTUAL = `uname -m`.strip # This helps with virtualized builds on aarch64 machines @@ -36,10 +36,13 @@ else end if ENV['CREW_CACHE_DIR'].to_s == '' - CREW_CACHE_DIR = HOME + '/.cache/crewcache/' + CREW_CACHE_DIR = HOME + '/.cache/crewcache' else CREW_CACHE_DIR = ENV['CREW_CACHE_DIR'] end +FileUtils.mkdir_p CREW_CACHE_DIR unless Dir.exist? CREW_CACHE_DIR + +CREW_CACHE_ENABLED = ENV['CREW_CACHE_ENABLED'] CREW_DEST_HOME = CREW_DEST_DIR + HOME diff --git a/lib/package.rb b/lib/package.rb index aa41fb543a..55956da552 100644 --- a/lib/package.rb +++ b/lib/package.rb @@ -1,7 +1,7 @@ require 'package_helpers' class Package - property :description, :homepage, :version, :license, :compatibility, :binary_url, :binary_sha256, :source_url, :source_sha256, :is_fake + property :description, :homepage, :version, :license, :compatibility, :binary_url, :binary_sha256, :source_url, :source_sha256, :git_hashtag, :is_fake class << self attr_reader :is_fake @@ -38,15 +38,27 @@ class Package end def self.get_url (architecture) - if !@build_from_source && @binary_url && @binary_url.has_key?(architecture) + if !@build_from_source and @binary_url and @binary_url.has_key?(architecture) return @binary_url[architecture] else return @source_url end end + def self.get_sha256 (architecture) + if !@build_from_source and @binary_sha256 and @binary_sha256.has_key?(architecture) + return @binary_sha256[architecture] + else + return @source_sha256 + end + end + + def self.get_extract_dir + name + '.' + Time.now.utc.strftime("%Y%m%d%H%M%S") + '.dir' + end + def self.is_binary? (architecture) - if !@build_from_source && @binary_url && @binary_url.has_key?(architecture) + if !@build_from_source and @binary_url and @binary_url.has_key?(architecture) return true else return false @@ -54,7 +66,7 @@ class Package end def self.is_source? (architecture) - if is_binary?(architecture) || is_fake? + if is_binary?(architecture) or is_fake? return false else return true diff --git a/packages/dash.rb b/packages/dash.rb index 8803916f46..05739c5eb3 100644 --- a/packages/dash.rb +++ b/packages/dash.rb @@ -2,12 +2,12 @@ require 'package' class Dash < Package description 'The Debian Almquist Shell (dash) is a POSIX-compliant shell derived from ash that executes scripts faster than bash and has fewer library dependencies.' - homepage 'https://salsa.debian.org/debian/dash' + homepage 'https://salsa.debian.org/debian/dash/' version '0.5.11.1' license 'BSD' compatibility 'all' - source_url 'https://salsa.debian.org/debian/dash/-/archive/upstream/0.5.11.1/dash-upstream-0.5.11.1.tar.bz2' - source_sha256 '4bb75944bb47fa6d1cf1e8a3ff941f6f1fb23497b553446e9f615d52d07ef1e7' + source_url 'https://salsa.debian.org/debian/dash.git' + git_tag 'upstream/0.5.11.2' binary_url ({ aarch64: 'https://downloads.sourceforge.net/project/chromebrew/armv7l/dash-0.5.11.1-chromeos-armv7l.tar.xz',