Miscellaneous crew changes (#9311)

* Make greater use of File.join when loading device.json

* Remove redundant check that the compatibility property exists (already checked by prop_test as part of CI)

* Don't pass architecture to lib/package.rb

* Replace @device[:architecture] with ARCH, do not create architecture section in device.json

* Refactor getting urls and sha256s in package.rb

* Use inplace sort on installed_packages instead of creating sorted_installed_packages

* Rework load_package to always only take one argument

* Only rescue package loading errors in set_package

* Remove generate_compatible

* Replace all calls to load_package with set_package

* Use _args instead of _ in prop_command

* Merge @short_verbose and @verbose

* Simplify filename detection in download

* Bump crew version
This commit is contained in:
Maximilian Downey Twiss
2024-02-13 04:38:18 +11:00
committed by GitHub
parent 59370bd08b
commit 7a5eff8945
6 changed files with 78 additions and 151 deletions

170
bin/crew
View File

@@ -124,8 +124,7 @@ String.use_color = args['--color'] || !args['--no-color']
# Verbose options
@fileutils_verbose = @opt_verbose
@verbose = @opt_verbose ? 'v' : ''
@short_verbose = @opt_verbose ? '-v' : ''
@verbose = @opt_verbose ? '-v' : ''
class ExitMessage
@messages = []
@@ -161,20 +160,14 @@ end
def load_json
# load_json(): (re)load device.json
json_path = File.join(CREW_CONFIG_PATH, 'device.json')
@device = JSON.load_file(json_path, symbolize_names: true)
@device = JSON.load_file(File.join(CREW_CONFIG_PATH, 'device.json'), symbolize_names: true)
# symbolize also values
@device.transform_values! {|val| val.is_a?(String) ? val.to_sym : val }
end
def print_package(pkgPath, extra = false)
pkgName = File.basename pkgPath, '.rb'
begin
set_package pkgName, pkgPath
rescue StandardError => e
warn "Error with #{pkgName}.rb: #{e}".red unless e.to_s.include?('uninitialized constant')
end
set_package pkgPath
print_current_package extra
end
@@ -206,11 +199,11 @@ def print_current_package(extra = false)
puts ''
end
def set_package(pkgName, pkgPath)
def set_package(pkgPath)
begin
@pkg = Package.load_package(pkgPath, pkgName)
rescue SyntaxError => e
warn "#{e.class}: #{e.message}".red
@pkg = Package.load_package(pkgPath)
rescue StandardError => e
puts "Error with #{pkgName}.rb: #{e}".lightred
end
@pkg.build_from_source = true if @opt_recursive
@@ -230,12 +223,7 @@ def list_available
next unless notInstalled
begin
set_package pkgName, filename
rescue StandardError => e
puts "Error with #{pkgName}.rb: #{e}".red unless e.to_s.include?('uninitialized constant')
end
puts pkgName if @pkg.compatible?
set_package filename
end
end
@@ -246,11 +234,11 @@ def list_installed
search package[:name], true
installed_packages.append("#{package[:name]} #{package[:version]}")
end
sorted_installed_packages = installed_packages.sort
sorted_installed_packages.unshift('======= =======')
sorted_installed_packages.unshift('Package Version')
first_col_width = sorted_installed_packages.map(&:split).map(&:first).max_by(&:size).size + 2
sorted_installed_packages.map(&:strip).each do |line|
installed_packages.sort!
installed_packages.unshift('======= =======')
installed_packages.unshift('Package Version')
first_col_width = installed_packages.map(&:split).map(&:first).max_by(&:size).size + 2
installed_packages.map(&:strip).each do |line|
puts "%-#{first_col_width}s%s".lightgreen % line.split
end
print "\n"
@@ -263,8 +251,8 @@ end
def list_compatible(compat = true)
Dir["#{CREW_PACKAGES_PATH}/*.rb"].each do |filename|
pkgName = File.basename filename, '.rb'
if @device[:compatible_packages].any? { |elem| elem[:name] == pkgName }
set_package filename
if @pkg.compatible?
if compat
if File.file? "#{CREW_META_PATH}/#{pkgName}.filelist"
puts pkgName.lightgreen
@@ -278,39 +266,9 @@ def list_compatible(compat = true)
end
end
def generate_compatible
puts 'Generating compatible packages...'.orange if @opt_verbose
@device[:compatible_packages] = []
Dir["#{CREW_PACKAGES_PATH}/*.rb"].each do |filename|
pkgName = File.basename filename, '.rb'
begin
set_package pkgName, filename
rescue StandardError => e
puts "Error with #{pkgName}.rb: #{e}".red unless e.to_s.include?('uninitialized constant')
end
puts "Checking #{pkgName} for compatibility.".orange if @opt_verbose
if @pkg.compatible?
# add to compatible packages
puts "Adding #{pkgName} #{@pkg.version} to compatible packages.".lightgreen if @opt_verbose
@device[:compatible_packages].push(name: @pkg.name)
elsif @opt_verbose
puts "#{pkgName} is not a compatible package.".lightred
end
end
File.open(File.join(CREW_CONFIG_PATH, 'device.json'), 'w') do |file|
output = JSON.parse @device.to_json
file.write JSON.pretty_generate(output)
end
puts 'Generating compatible packages done.'.orange if @opt_verbose
end
def search(pkgName, silent = false)
pkgPath = File.join(CREW_PACKAGES_PATH, "#{pkgName}.rb")
begin
return set_package(pkgName, pkgPath) if File.file?(pkgPath)
rescue StandardError => e
puts "Error with #{pkgName}.rb: #{e}".lightred unless e.to_s.include?('uninitialized constant')
end
return set_package(pkgPath) if File.file?(pkgPath)
unless File.file?(pkgPath) && silent
@pkg = nil
abort "Package #{pkgName} not found. 😞".lightred unless silent
@@ -325,12 +283,7 @@ def regexp_search(pkgPat)
.each { |f| print_package(f, @opt_verbose) }
if results.empty?
Dir["#{CREW_PACKAGES_PATH}/*.rb"].each do |packagePath|
packageName = File.basename packagePath, '.rb'
begin
set_package packageName, packagePath
rescue StandardError => e
puts "Error with #{pkgName}.rb: #{e}".red unless e.to_s.include?('uninitialized constant')
end
set_package packagePath
if @pkg.description =~ /#{pkgPat}/i
print_current_package @opt_verbose
results.push(packageName)
@@ -509,8 +462,6 @@ def help(pkgName = nil)
Update crew.
Usage: crew update
This only updates crew itself. Use `crew upgrade` to update packages.
Usage: crew update compatible
This updates the crew package compatibility list.
EOT
when 'upgrade'
puts <<~EOT
@@ -542,7 +493,7 @@ def help(pkgName = nil)
end
def cache_build
build_cachefile = File.join(CREW_CACHE_DIR, "#{@pkg.name}-#{@pkg.version}-build-#{@device[:architecture]}.tar.zst")
build_cachefile = File.join(CREW_CACHE_DIR, "#{@pkg.name}-#{@pkg.version}-build-#{ARCH}.tar.zst")
if CREW_CACHE_ENABLED && File.writable?(CREW_CACHE_DIR)
puts 'Caching build dir...'
pkg_build_dirname_absolute = File.join(CREW_BREW_DIR, @extract_dir)
@@ -637,7 +588,8 @@ def whatprovides(regexPat)
filelist, matchedFile = result.split(':', 2)
pkgName = File.basename(filelist, '.filelist')
pkgNameStatus = pkgName
if @device[:compatible_packages].any? { |elem| elem[:name] == pkgName }
set_package File.join(CREW_PACKAGES_PATH, "#{pkgName}.rb")
if @pkg.compatible?
pkgNameStatus = pkgName.lightgreen if File.file? "#{CREW_META_PATH}/#{pkgName}.filelist"
else
pkgNameStatus = pkgName.lightred
@@ -667,8 +619,6 @@ def update
# Do any fixups necessary after crew has updated from git.
load "#{CREW_LIB_PATH}/lib/fixup.rb"
# update compatible packages
generate_compatible
# check for outdated installed packages
puts 'Checking for package updates...'
@@ -680,8 +630,8 @@ def update
next
end
differentVersion = (package[:version] != @pkg.version)
hasSha = !(@pkg.get_binary_sha256(@device[:architecture]).to_s.empty? || package[:binary_sha256].to_s.empty?)
differentSha = hasSha && package[:binary_sha256] != @pkg.get_binary_sha256(@device[:architecture])
hasSha = !(@pkg.binary_sha256&.dig(ARCH.to_sym).to_s.empty? || package[:binary_sha256].to_s.empty?)
differentSha = hasSha && package[:binary_sha256] != @pkg.binary_sha256&.dig(ARCH.to_sym)
canBeUpdated += 1 if differentVersion || differentSha
@@ -718,13 +668,12 @@ def upgrade(*pkgs, build_from_source: false)
return false
end
end
pkgVer_latest = Package.load_package(pkgFile, pkgName).version
set_package File.join(CREW_PACKAGES_PATH, "#{pkgName}.rb")
pkgVer_installed = @device[:installed_packages].select { |pkg| pkg[:name] == pkgName } [0][:version]
pkgHash_latest = Package.load_package(pkgFile, pkgName).get_binary_sha256(@device[:architecture])
pkgHash_installed = @device[:installed_packages].select { |pkg| pkg[:name] == pkgName } [0][:binary_sha256]
return pkgHash_latest != pkgHash_installed unless !pkgHash_installed || pkgHash_latest == ''
return pkgVer_latest != pkgVer_installed
return @pkg.binary_sha256&.dig(ARCH.to_sym) != pkgHash_installed unless !pkgHash_installed || @pkg.binary_sha256&.dig(ARCH.to_sym) == 'null'
return @pkg.version != pkgVer_installed
end
to_be_upgraded = []
@@ -790,19 +739,18 @@ def upgrade(*pkgs, build_from_source: false)
end
def download
url = @pkg.get_url(@device[:architecture])
source = @pkg.is_source?(@device[:architecture])
url = @pkg.get_url
source = @pkg.is_source?
uri = URI.parse url
filename = File.basename(uri.path)
sha256sum = @pkg.get_sha256(@device[:architecture])
filename = File.basename(url)
sha256sum = @pkg.get_sha256
@extract_dir = @pkg.get_extract_dir
build_cachefile = File.join(CREW_CACHE_DIR, "#{@pkg.name}-#{@pkg.version}-build-#{@device[:architecture]}.tar.zst")
build_cachefile = File.join(CREW_CACHE_DIR, "#{@pkg.name}-#{@pkg.version}-build-#{ARCH}.tar.zst")
return { source:, filename: } if CREW_CACHE_BUILD && File.file?(build_cachefile)
if !url
abort "No precompiled binary or source is available for #{@device[:architecture]}.".lightred
abort "No precompiled binary or source is available for #{ARCH}.".lightred
elsif @pkg.build_from_source
puts 'Downloading source...'
elsif !source
@@ -813,12 +761,12 @@ def download
puts 'No precompiled binary available for your platform, downloading source...'
end
git = uri.scheme.eql?('git')
git = URI.parse(url).scheme.eql?('git')
Dir.chdir CREW_BREW_DIR do
case File.basename(filename)
case filename
# Sources that download with our internal downloader
when /\.zip$/i, /\.(tar(\.(gz|bz2|xz|lzma|lz|zst))?|tgz|tbz|tpxz|txz)$/i, /\.deb$/i, /\.AppImage$/i
when /\.zip$/, /\.tar.*$/, /\.deb$/, /\.AppImage$/
# Recall file from cache if requested
if CREW_CACHE_ENABLED
puts "Looking for #{@pkg.name} archive in cache".orange if @opt_verbose
@@ -870,9 +818,9 @@ def download
end
return { source:, filename: }
when /^SKIP$/i
when 'SKIP'
Dir.mkdir @extract_dir
when /\.git$/i # Source URLs which end with .git are git sources.
when /\.git$/ # Source URLs which end with .git are git sources.
git = true
else
Dir.mkdir @extract_dir
@@ -960,7 +908,7 @@ def unpack(meta)
Dir.chdir CREW_BREW_DIR do
FileUtils.mkdir_p @extract_dir, verbose: @fileutils_verbose
build_cachefile = File.join(CREW_CACHE_DIR, "#{@pkg.name}-#{@pkg.version}-build-#{@device[:architecture]}.tar.zst")
build_cachefile = File.join(CREW_CACHE_DIR, "#{@pkg.name}-#{@pkg.version}-build-#{ARCH}.tar.zst")
if CREW_CACHE_BUILD && File.file?(build_cachefile) && File.file?("#{build_cachefile}.sha256") && ( system "cd #{CREW_CACHE_DIR} && sha256sum -c #{build_cachefile}.sha256" )
@pkg.cached_build = true
puts "Extracting cached build directory from #{build_cachefile}".lightgreen
@@ -1403,12 +1351,12 @@ def install_files(src, dst = File.join( CREW_PREFIX, src.delete_prefix('./usr/lo
# Check for ACLs support.
rsync_version = `rsync --version`.chomp
if rsync_version.include?('ACLs') && !rsync_version.include?('no ACLs')
system 'rsync', "-ah#{@verbose}HAXW", '--remove-source-files', src, dst, exception: true
system 'rsync', "#{@verbose} -ahHAXW", '--remove-source-files', src, dst, exception: true
else
system 'rsync', "-ah#{@verbose}HXW", '--remove-source-files', src, dst, exception: true
system 'rsync', "#{@verbose} -ah#HXW", '--remove-source-files', src, dst, exception: true
end
else
system "cd #{src}; tar -cf - ./* | (cd #{dst}; tar -x#{@verbose}p --keep-directory-symlink -f -)", exception: true
system "cd #{src}; tar -cf - ./* | (cd #{dst}; tar #{@verbose} -xp --keep-directory-symlink -f -)", exception: true
end
end
else
@@ -1531,8 +1479,8 @@ def resolve_dependencies
# run preflight check for dependencies
@dependencies.each do |depName|
dep_pkgPath = File.join(CREW_PACKAGES_PATH, "#{depName}.rb")
Package.load_package(dep_pkgPath, depName).preflight
set_package File.join(CREW_PACKAGES_PATH, "#{depName}.rb")
@pkg.preflight
end
return if @dependencies.empty?
@@ -1604,7 +1552,7 @@ def install
# If this fails, the install should fail before we create any
# damage, and we should roughly be at maximal disk space usage at this
# point anyways.
FileUtils.cp File.join(CREW_CONFIG_PATH, 'device.json'), "#{CREW_CONFIG_PATH}/device.json.tmp"
FileUtils.cp File.join(CREW_CONFIG_PATH, 'device.json'), File.join(CREW_CONFIG_PATH, 'device.json.tmp')
# remove it just before the file copy
if @pkg.in_upgrade
@@ -1626,14 +1574,14 @@ def install
end
# add to installed packages
@device[:installed_packages].push(name: @pkg.name, version: @pkg.version, binary_sha256: @pkg.get_binary_sha256(@device[:architecture]))
File.open("#{CREW_CONFIG_PATH}/device.json.tmp", 'w') do |file|
@device[:installed_packages].push(name: @pkg.name, version: @pkg.version, binary_sha256: @pkg.binary_sha256&.dig(ARCH.to_sym))
File.open(File.join(CREW_CONFIG_PATH, 'device.json.tmp'), 'w') do |file|
output = JSON.parse @device.to_json
file.write JSON.pretty_generate(output)
end
# Copy over original if the write to the tmp file succeeds.
FileUtils.cp "#{CREW_CONFIG_PATH}/device.json.tmp", File.join(CREW_CONFIG_PATH, 'device.json')
FileUtils.rm "#{CREW_CONFIG_PATH}/device.json.tmp"
FileUtils.cp File.join(CREW_CONFIG_PATH, 'device.json.tmp'), File.join(CREW_CONFIG_PATH, 'device.json')
FileUtils.rm File.join(CREW_CONFIG_PATH, 'device.json.tmp')
end
def resolve_dependencies_and_build
@@ -1695,13 +1643,13 @@ def archive_package(crew_archive_dest)
end
if @pkg.no_zstd? || !crew_prefix_zstd_available
puts 'Using xz to compress package. This may take some time.'.lightblue
pkg_name = "#{@pkg.name}-#{@pkg.version}-chromeos-#{@device[:architecture]}.tar.xz"
pkg_name = "#{@pkg.name}-#{@pkg.version}-chromeos-#{ARCH}.tar.xz"
Dir.chdir CREW_DEST_DIR do
system "tar c#{@verbose}Jf #{crew_archive_dest}/#{pkg_name} *"
system "tar cJf #{crew_archive_dest}/#{pkg_name} * #{@verbose}"
end
else
puts 'Using zstd to compress package. This may take some time.'.lightblue
pkg_name = "#{@pkg.name}-#{@pkg.version}-chromeos-#{@device[:architecture]}.tar.zst"
pkg_name = "#{@pkg.name}-#{@pkg.version}-chromeos-#{ARCH}.tar.zst"
Dir.chdir CREW_DEST_DIR do
# Using same zstd compression options as Arch, which privilege
# decompression speed over compression speed.
@@ -1709,7 +1657,7 @@ def archive_package(crew_archive_dest)
# Use nice so that user can (possibly) do other things during compression.
if crew_prefix_zstd_available
puts 'Using standard zstd'.lightblue if @opt_verbose
system "tar c#{@verbose} * | nice -n 20 #{CREW_PREFIX}/bin/zstd -c -T0 --ultra -20 - > #{crew_archive_dest}/#{pkg_name}"
system "tar c * #{@verbose} | nice -n 20 #{CREW_PREFIX}/bin/zstd -c -T0 --ultra -20 - > #{crew_archive_dest}/#{pkg_name}"
end
end
end
@@ -1792,7 +1740,7 @@ def remove(pkgName)
@device[:installed_packages].delete_if { |elem| elem[:name] == pkgName }
# update the device manifest
File.write "#{CREW_CONFIG_PATH}/device.json", JSON.pretty_generate(JSON.parse(@device.to_json))
File.write File.join(CREW_CONFIG_PATH, 'device.json'), JSON.pretty_generate(JSON.parse(@device.to_json))
search pkgName, true
@pkg.remove unless @in_fixup
@@ -2018,8 +1966,8 @@ def build_command(args)
puts 'Compile not needed. Skipping build.'.lightred if @pkg.no_compile_needed?
puts "Package #{@pkg.name} is not compatible with your device architecture (#{ARCH}). Skipping build.".lightred unless @pkg.compatible?
puts 'Unable to build a fake package. Skipping build.'.lightred if @pkg.is_fake?
puts 'Unable to build without source. Skipping build.'.lightred unless @pkg.is_source?(ARCH) && @pkg.source_url.to_s.upcase != 'SKIP'
resolve_dependencies_and_build if !@pkg.is_fake? && @pkg.is_source?(ARCH) && @pkg.source_url.to_s.upcase != 'SKIP' && !@pkg.no_compile_needed?
puts 'Unable to build without source. Skipping build.'.lightred unless @pkg.is_source? && @pkg.source_url.to_s.upcase != 'SKIP'
resolve_dependencies_and_build if !@pkg.is_fake? && @pkg.is_source? && @pkg.source_url.to_s.upcase != 'SKIP' && !@pkg.no_compile_needed?
end
puts "Builds are located in #{CREW_LOCAL_BUILD_DIR}.".yellow
end
@@ -2116,7 +2064,7 @@ def postinstall_command(args)
end
end
def prop_command(_)
def prop_command(_args)
prop
end
@@ -2189,12 +2137,8 @@ def sysinfo_command(_args)
end
end
def update_command(args)
if args['<compatible>']
generate_compatible
else
update
end
def update_command(_args)
update
end
def upgrade_command(args) = upgrade(*args['<name>'], build_from_source: @opt_source)

View File

@@ -176,9 +176,7 @@ fi
# Create the device.json file.
cd "${CREW_CONFIG_PATH}"
echo_info "\nCreating device.json."
jq --arg key0 'architecture' --arg value0 "${ARCH}" \
--arg key1 'installed_packages' \
'. | .[$key0]=$value0 | .[$key1]=[]' <<<'{}' > device.json
jq --arg key0 'installed_packages' '. | .[$key0]=[]' <<<'{}' > device.json
# Functions to maintain packages.

View File

@@ -1,7 +1,7 @@
# lib/const.rb
# Defines common constants used in different parts of crew
CREW_VERSION = '1.43.2'
CREW_VERSION = '1.43.3'
# kernel architecture
KERN_ARCH = `uname -m`.chomp

View File

@@ -30,9 +30,10 @@ class Package
attr_accessor :name, :cached_build, :in_build, :build_from_source, :in_upgrade
end
def self.load_package(pkgFile, pkgName = File.basename(pkgFile, '.rb'))
def self.load_package(pkgFile)
# self.load_package: load a package under 'Package' class scope
#
pkgName = File.basename(pkgFile, '.rb')
className = pkgName.capitalize
# read and eval package script under 'Package' class
@@ -79,7 +80,7 @@ class Package
@checked_list.merge!({ pkgName => pkgTags })
pkgObj = load_package("#{CREW_PACKAGES_PATH}/#{pkgName}.rb")
is_source = pkgObj.is_source?(ARCH.to_sym) or pkgObj.build_from_source
is_source = pkgObj.is_source? or pkgObj.build_from_source
deps = pkgObj.dependencies
# append buildessential to deps if building from source is needed/specified
@@ -145,14 +146,7 @@ class Package
end
end
def self.compatible?
if @compatibility
return @compatibility.casecmp?('all') || @compatibility.include?(ARCH)
else
warn "#{name}: Missing `compatibility` field.".lightred
return false
end
end
def self.compatible? = @compatibility.casecmp?('all') || @compatibility.include?(ARCH)
def self.depends_on(dependency, ver_range = nil)
@dependencies ||= {}
@@ -197,38 +191,30 @@ class Package
@dependencies.store(depName, [dep_tags, ver_check])
end
def self.get_url(architecture)
if !@build_from_source && @binary_sha256 && @binary_sha256.key?(architecture)
return get_binary_url(architecture)
elsif @source_url.respond_to?(:has_key?)
return @source_url.key?(architecture) ? @source_url[architecture] : nil
def self.get_url
if is_binary?
return "https://gitlab.com/api/v4/projects/26210301/packages/generic/#{name}/#{version}_#{ARCH}/#{name}-#{version}-chromeos-#{ARCH}.#{binary_compression}"
elsif @source_url.is_a?(Hash)
return @source_url[ARCH.to_sym]
else
return @source_url
end
end
def self.get_binary_url(architecture)
architecture = 'armv7l' if architecture == 'aarch64'
return "https://gitlab.com/api/v4/projects/26210301/packages/generic/#{name}/#{version}_#{architecture}/#{name}-#{version}-chromeos-#{architecture}.#{binary_compression}"
end
def self.get_source_url(architecture) = @source_url.key?(architecture) ? @source_url[architecture] : nil
def self.get_sha256(architecture)
if !@build_from_source && @binary_sha256 && @binary_sha256.key?(architecture)
return @binary_sha256[architecture]
elsif @source_sha256.respond_to?(:has_key?)
return @source_sha256.key?(architecture) ? @source_sha256[architecture] : nil
def self.get_sha256
if is_binary?
return @binary_sha256[ARCH.to_sym]
elsif @source_sha256.is_a?(Hash)
return @source_sha256[ARCH.to_sym]
else
return @source_sha256
end
end
def self.get_binary_sha256(architecture) = @binary_sha256&.key?(architecture) ? @binary_sha256[architecture] : ''
def self.get_extract_dir = "#{name}.#{Time.now.utc.strftime('%Y%m%d%H%M%S')}.dir"
def self.is_binary?(architecture) = !@build_from_source && @binary_sha256 && @binary_sha256.key?(architecture)
def self.is_source?(architecture) = !(is_binary?(architecture) || is_fake?)
def self.is_binary? = !@build_from_source && @binary_sha256&.key?(ARCH.to_sym)
def self.is_source? = !(is_binary? || is_fake?)
def self.system(*args, **opt_args)
@crew_env_options_hash = if no_env_options?

View File

@@ -58,7 +58,7 @@ class Jdk8 < Package
EOT
end
return if File.exist?( URI( get_source_url(ARCH.to_sym) ).path )
return if File.exist?( URI( get_url ).path )
# check if we should prompt user to the archive page or download page based on #{version}
# download page only contains latest version while archive page only contains older versions
@@ -98,6 +98,6 @@ class Jdk8 < Package
def self.postinstall
# remove jdk archive after installed
FileUtils.rm_f URI( get_source_url(ARCH.to_sym) ).path
FileUtils.rm_f URI( get_url ).path
end
end

View File

@@ -16,8 +16,7 @@ puts "Running dependency cycle tests...\n".yellow
# Loads all packages
Dir['../packages/*.rb'].each do |filename|
name = File.basename(filename, '.rb')
pkg = Package.load_package(filename, name)
pkg = Package.load_package(filename)
@all_pkgs[name] = pkg
end