diff --git a/bin/crew b/bin/crew index 6be318cb1..2724f3d11 100755 --- a/bin/crew +++ b/bin/crew @@ -124,7 +124,8 @@ String.use_color = args['--color'] || !args['--no-color'] # Verbose options @fileutils_verbose = @opt_verbose -@verbose = @opt_verbose ? '-v' : '' +@verbose = @opt_verbose ? 'v' : '' +@short_verbose = @opt_verbose ? '-v' : '' class ExitMessage @messages = [] @@ -160,14 +161,20 @@ end def load_json # load_json(): (re)load device.json - @device = JSON.load_file(File.join(CREW_CONFIG_PATH, 'device.json'), symbolize_names: true) + json_path = File.join(CREW_CONFIG_PATH, 'device.json') + @device = JSON.load_file(json_path, symbolize_names: true) # symbolize also values @device.transform_values! {|val| val.is_a?(String) ? val.to_sym : val } end def print_package(pkgPath, extra = false) - set_package pkgPath + pkgName = File.basename pkgPath, '.rb' + begin + set_package pkgName, pkgPath + rescue StandardError => e + warn "Error with #{pkgName}.rb: #{e}".red unless e.to_s.include?('uninitialized constant') + end print_current_package extra end @@ -199,11 +206,11 @@ def print_current_package(extra = false) puts '' end -def set_package(pkgPath) +def set_package(pkgName, pkgPath) begin - @pkg = Package.load_package(pkgPath) - rescue StandardError => e - puts "Error with #{pkgName}.rb: #{e}".lightred + @pkg = Package.load_package(pkgPath, pkgName) + rescue SyntaxError => e + warn "#{e.class}: #{e.message}".red end @pkg.build_from_source = true if @opt_recursive @@ -223,7 +230,12 @@ def list_available next unless notInstalled - set_package filename + begin + set_package pkgName, filename + rescue StandardError => e + puts "Error with #{pkgName}.rb: #{e}".red unless e.to_s.include?('uninitialized constant') + end + puts pkgName if @pkg.compatible? end end @@ -234,11 +246,11 @@ def list_installed search package[:name], true installed_packages.append("#{package[:name]} #{package[:version]}") end - installed_packages.sort! - installed_packages.unshift('======= =======') - installed_packages.unshift('Package Version') - first_col_width = installed_packages.map(&:split).map(&:first).max_by(&:size).size + 2 - installed_packages.map(&:strip).each do |line| + sorted_installed_packages = installed_packages.sort + sorted_installed_packages.unshift('======= =======') + sorted_installed_packages.unshift('Package Version') + first_col_width = sorted_installed_packages.map(&:split).map(&:first).max_by(&:size).size + 2 + sorted_installed_packages.map(&:strip).each do |line| puts "%-#{first_col_width}s%s".lightgreen % line.split end print "\n" @@ -251,8 +263,8 @@ end def list_compatible(compat = true) Dir["#{CREW_PACKAGES_PATH}/*.rb"].each do |filename| - set_package filename - if @pkg.compatible? + pkgName = File.basename filename, '.rb' + if @device[:compatible_packages].any? { |elem| elem[:name] == pkgName } if compat if File.file? "#{CREW_META_PATH}/#{pkgName}.filelist" puts pkgName.lightgreen @@ -266,9 +278,39 @@ def list_compatible(compat = true) end end +def generate_compatible + puts 'Generating compatible packages...'.orange if @opt_verbose + @device[:compatible_packages] = [] + Dir["#{CREW_PACKAGES_PATH}/*.rb"].each do |filename| + pkgName = File.basename filename, '.rb' + begin + set_package pkgName, filename + rescue StandardError => e + puts "Error with #{pkgName}.rb: #{e}".red unless e.to_s.include?('uninitialized constant') + end + puts "Checking #{pkgName} for compatibility.".orange if @opt_verbose + if @pkg.compatible? + # add to compatible packages + puts "Adding #{pkgName} #{@pkg.version} to compatible packages.".lightgreen if @opt_verbose + @device[:compatible_packages].push(name: @pkg.name) + elsif @opt_verbose + puts "#{pkgName} is not a compatible package.".lightred + end + end + File.open(File.join(CREW_CONFIG_PATH, 'device.json'), 'w') do |file| + output = JSON.parse @device.to_json + file.write JSON.pretty_generate(output) + end + puts 'Generating compatible packages done.'.orange if @opt_verbose +end + def search(pkgName, silent = false) pkgPath = File.join(CREW_PACKAGES_PATH, "#{pkgName}.rb") - return set_package(pkgPath) if File.file?(pkgPath) + begin + return set_package(pkgName, pkgPath) if File.file?(pkgPath) + rescue StandardError => e + puts "Error with #{pkgName}.rb: #{e}".lightred unless e.to_s.include?('uninitialized constant') + end unless File.file?(pkgPath) && silent @pkg = nil abort "Package #{pkgName} not found. 😞".lightred unless silent @@ -283,7 +325,12 @@ def regexp_search(pkgPat) .each { |f| print_package(f, @opt_verbose) } if results.empty? Dir["#{CREW_PACKAGES_PATH}/*.rb"].each do |packagePath| - set_package packagePath + packageName = File.basename packagePath, '.rb' + begin + set_package packageName, packagePath + rescue StandardError => e + puts "Error with #{pkgName}.rb: #{e}".red unless e.to_s.include?('uninitialized constant') + end if @pkg.description =~ /#{pkgPat}/i print_current_package @opt_verbose results.push(packageName) @@ -462,6 +509,8 @@ def help(pkgName = nil) Update crew. Usage: crew update This only updates crew itself. Use `crew upgrade` to update packages. + Usage: crew update compatible + This updates the crew package compatibility list. EOT when 'upgrade' puts <<~EOT @@ -493,7 +542,7 @@ def help(pkgName = nil) end def cache_build - build_cachefile = File.join(CREW_CACHE_DIR, "#{@pkg.name}-#{@pkg.version}-build-#{ARCH}.tar.zst") + build_cachefile = File.join(CREW_CACHE_DIR, "#{@pkg.name}-#{@pkg.version}-build-#{@device[:architecture]}.tar.zst") if CREW_CACHE_ENABLED && File.writable?(CREW_CACHE_DIR) puts 'Caching build dir...' pkg_build_dirname_absolute = File.join(CREW_BREW_DIR, @extract_dir) @@ -588,8 +637,7 @@ def whatprovides(regexPat) filelist, matchedFile = result.split(':', 2) pkgName = File.basename(filelist, '.filelist') pkgNameStatus = pkgName - set_package File.join(CREW_PACKAGES_PATH, "#{pkgName}.rb") - if @pkg.compatible? + if @device[:compatible_packages].any? { |elem| elem[:name] == pkgName } pkgNameStatus = pkgName.lightgreen if File.file? "#{CREW_META_PATH}/#{pkgName}.filelist" else pkgNameStatus = pkgName.lightred @@ -619,6 +667,8 @@ def update # Do any fixups necessary after crew has updated from git. load "#{CREW_LIB_PATH}/lib/fixup.rb" + # update compatible packages + generate_compatible # check for outdated installed packages puts 'Checking for package updates...' @@ -630,8 +680,8 @@ def update next end differentVersion = (package[:version] != @pkg.version) - hasSha = !(@pkg.binary_sha256&.dig(ARCH.to_sym).to_s.empty? || package[:binary_sha256].to_s.empty?) - differentSha = hasSha && package[:binary_sha256] != @pkg.binary_sha256&.dig(ARCH.to_sym) + hasSha = !(@pkg.get_binary_sha256(@device[:architecture]).to_s.empty? || package[:binary_sha256].to_s.empty?) + differentSha = hasSha && package[:binary_sha256] != @pkg.get_binary_sha256(@device[:architecture]) canBeUpdated += 1 if differentVersion || differentSha @@ -668,12 +718,13 @@ def upgrade(*pkgs, build_from_source: false) return false end end - set_package File.join(CREW_PACKAGES_PATH, "#{pkgName}.rb") + pkgVer_latest = Package.load_package(pkgFile, pkgName).version pkgVer_installed = @device[:installed_packages].select { |pkg| pkg[:name] == pkgName } [0][:version] + pkgHash_latest = Package.load_package(pkgFile, pkgName).get_binary_sha256(@device[:architecture]) pkgHash_installed = @device[:installed_packages].select { |pkg| pkg[:name] == pkgName } [0][:binary_sha256] - return @pkg.binary_sha256&.dig(ARCH.to_sym) != pkgHash_installed unless !pkgHash_installed || @pkg.binary_sha256&.dig(ARCH.to_sym) == 'null' - return @pkg.version != pkgVer_installed + return pkgHash_latest != pkgHash_installed unless !pkgHash_installed || pkgHash_latest == '' + return pkgVer_latest != pkgVer_installed end to_be_upgraded = [] @@ -739,18 +790,19 @@ def upgrade(*pkgs, build_from_source: false) end def download - url = @pkg.get_url - source = @pkg.is_source? + url = @pkg.get_url(@device[:architecture]) + source = @pkg.is_source?(@device[:architecture]) - filename = File.basename(url) - sha256sum = @pkg.get_sha256 + uri = URI.parse url + filename = File.basename(uri.path) + sha256sum = @pkg.get_sha256(@device[:architecture]) @extract_dir = @pkg.get_extract_dir - build_cachefile = File.join(CREW_CACHE_DIR, "#{@pkg.name}-#{@pkg.version}-build-#{ARCH}.tar.zst") + build_cachefile = File.join(CREW_CACHE_DIR, "#{@pkg.name}-#{@pkg.version}-build-#{@device[:architecture]}.tar.zst") return { source:, filename: } if CREW_CACHE_BUILD && File.file?(build_cachefile) if !url - abort "No precompiled binary or source is available for #{ARCH}.".lightred + abort "No precompiled binary or source is available for #{@device[:architecture]}.".lightred elsif @pkg.build_from_source puts 'Downloading source...' elsif !source @@ -761,12 +813,12 @@ def download puts 'No precompiled binary available for your platform, downloading source...' end - git = URI.parse(url).scheme.eql?('git') + git = uri.scheme.eql?('git') Dir.chdir CREW_BREW_DIR do - case filename + case File.basename(filename) # Sources that download with our internal downloader - when /\.zip$/, /\.tar.*$/, /\.deb$/, /\.AppImage$/ + when /\.zip$/i, /\.(tar(\.(gz|bz2|xz|lzma|lz|zst))?|tgz|tbz|tpxz|txz)$/i, /\.deb$/i, /\.AppImage$/i # Recall file from cache if requested if CREW_CACHE_ENABLED puts "Looking for #{@pkg.name} archive in cache".orange if @opt_verbose @@ -818,9 +870,9 @@ def download end return { source:, filename: } - when 'SKIP' + when /^SKIP$/i Dir.mkdir @extract_dir - when /\.git$/ # Source URLs which end with .git are git sources. + when /\.git$/i # Source URLs which end with .git are git sources. git = true else Dir.mkdir @extract_dir @@ -908,7 +960,7 @@ def unpack(meta) Dir.chdir CREW_BREW_DIR do FileUtils.mkdir_p @extract_dir, verbose: @fileutils_verbose - build_cachefile = File.join(CREW_CACHE_DIR, "#{@pkg.name}-#{@pkg.version}-build-#{ARCH}.tar.zst") + build_cachefile = File.join(CREW_CACHE_DIR, "#{@pkg.name}-#{@pkg.version}-build-#{@device[:architecture]}.tar.zst") if CREW_CACHE_BUILD && File.file?(build_cachefile) && File.file?("#{build_cachefile}.sha256") && ( system "cd #{CREW_CACHE_DIR} && sha256sum -c #{build_cachefile}.sha256" ) @pkg.cached_build = true puts "Extracting cached build directory from #{build_cachefile}".lightgreen @@ -1351,12 +1403,12 @@ def install_files(src, dst = File.join( CREW_PREFIX, src.delete_prefix('./usr/lo # Check for ACLs support. rsync_version = `rsync --version`.chomp if rsync_version.include?('ACLs') && !rsync_version.include?('no ACLs') - system 'rsync', "#{@verbose} -ahHAXW", '--remove-source-files', src, dst, exception: true + system 'rsync', "-ah#{@verbose}HAXW", '--remove-source-files', src, dst, exception: true else - system 'rsync', "#{@verbose} -ah#HXW", '--remove-source-files', src, dst, exception: true + system 'rsync', "-ah#{@verbose}HXW", '--remove-source-files', src, dst, exception: true end else - system "cd #{src}; tar -cf - ./* | (cd #{dst}; tar #{@verbose} -xp --keep-directory-symlink -f -)", exception: true + system "cd #{src}; tar -cf - ./* | (cd #{dst}; tar -x#{@verbose}p --keep-directory-symlink -f -)", exception: true end end else @@ -1479,8 +1531,8 @@ def resolve_dependencies # run preflight check for dependencies @dependencies.each do |depName| - set_package File.join(CREW_PACKAGES_PATH, "#{depName}.rb") - @pkg.preflight + dep_pkgPath = File.join(CREW_PACKAGES_PATH, "#{depName}.rb") + Package.load_package(dep_pkgPath, depName).preflight end return if @dependencies.empty? @@ -1552,7 +1604,7 @@ def install # If this fails, the install should fail before we create any # damage, and we should roughly be at maximal disk space usage at this # point anyways. - FileUtils.cp File.join(CREW_CONFIG_PATH, 'device.json'), File.join(CREW_CONFIG_PATH, 'device.json.tmp') + FileUtils.cp File.join(CREW_CONFIG_PATH, 'device.json'), "#{CREW_CONFIG_PATH}/device.json.tmp" # remove it just before the file copy if @pkg.in_upgrade @@ -1574,14 +1626,14 @@ def install end # add to installed packages - @device[:installed_packages].push(name: @pkg.name, version: @pkg.version, binary_sha256: @pkg.binary_sha256&.dig(ARCH.to_sym)) - File.open(File.join(CREW_CONFIG_PATH, 'device.json.tmp'), 'w') do |file| + @device[:installed_packages].push(name: @pkg.name, version: @pkg.version, binary_sha256: @pkg.get_binary_sha256(@device[:architecture])) + File.open("#{CREW_CONFIG_PATH}/device.json.tmp", 'w') do |file| output = JSON.parse @device.to_json file.write JSON.pretty_generate(output) end # Copy over original if the write to the tmp file succeeds. - FileUtils.cp File.join(CREW_CONFIG_PATH, 'device.json.tmp'), File.join(CREW_CONFIG_PATH, 'device.json') - FileUtils.rm File.join(CREW_CONFIG_PATH, 'device.json.tmp') + FileUtils.cp "#{CREW_CONFIG_PATH}/device.json.tmp", File.join(CREW_CONFIG_PATH, 'device.json') + FileUtils.rm "#{CREW_CONFIG_PATH}/device.json.tmp" end def resolve_dependencies_and_build @@ -1643,13 +1695,13 @@ def archive_package(crew_archive_dest) end if @pkg.no_zstd? || !crew_prefix_zstd_available puts 'Using xz to compress package. This may take some time.'.lightblue - pkg_name = "#{@pkg.name}-#{@pkg.version}-chromeos-#{ARCH}.tar.xz" + pkg_name = "#{@pkg.name}-#{@pkg.version}-chromeos-#{@device[:architecture]}.tar.xz" Dir.chdir CREW_DEST_DIR do - system "tar cJf #{crew_archive_dest}/#{pkg_name} * #{@verbose}" + system "tar c#{@verbose}Jf #{crew_archive_dest}/#{pkg_name} *" end else puts 'Using zstd to compress package. This may take some time.'.lightblue - pkg_name = "#{@pkg.name}-#{@pkg.version}-chromeos-#{ARCH}.tar.zst" + pkg_name = "#{@pkg.name}-#{@pkg.version}-chromeos-#{@device[:architecture]}.tar.zst" Dir.chdir CREW_DEST_DIR do # Using same zstd compression options as Arch, which privilege # decompression speed over compression speed. @@ -1657,7 +1709,7 @@ def archive_package(crew_archive_dest) # Use nice so that user can (possibly) do other things during compression. if crew_prefix_zstd_available puts 'Using standard zstd'.lightblue if @opt_verbose - system "tar c * #{@verbose} | nice -n 20 #{CREW_PREFIX}/bin/zstd -c -T0 --ultra -20 - > #{crew_archive_dest}/#{pkg_name}" + system "tar c#{@verbose} * | nice -n 20 #{CREW_PREFIX}/bin/zstd -c -T0 --ultra -20 - > #{crew_archive_dest}/#{pkg_name}" end end end @@ -1740,7 +1792,7 @@ def remove(pkgName) @device[:installed_packages].delete_if { |elem| elem[:name] == pkgName } # update the device manifest - File.write File.join(CREW_CONFIG_PATH, 'device.json'), JSON.pretty_generate(JSON.parse(@device.to_json)) + File.write "#{CREW_CONFIG_PATH}/device.json", JSON.pretty_generate(JSON.parse(@device.to_json)) search pkgName, true @pkg.remove unless @in_fixup @@ -1966,8 +2018,8 @@ def build_command(args) puts 'Compile not needed. Skipping build.'.lightred if @pkg.no_compile_needed? puts "Package #{@pkg.name} is not compatible with your device architecture (#{ARCH}). Skipping build.".lightred unless @pkg.compatible? puts 'Unable to build a fake package. Skipping build.'.lightred if @pkg.is_fake? - puts 'Unable to build without source. Skipping build.'.lightred unless @pkg.is_source? && @pkg.source_url.to_s.upcase != 'SKIP' - resolve_dependencies_and_build if !@pkg.is_fake? && @pkg.is_source? && @pkg.source_url.to_s.upcase != 'SKIP' && !@pkg.no_compile_needed? + puts 'Unable to build without source. Skipping build.'.lightred unless @pkg.is_source?(ARCH) && @pkg.source_url.to_s.upcase != 'SKIP' + resolve_dependencies_and_build if !@pkg.is_fake? && @pkg.is_source?(ARCH) && @pkg.source_url.to_s.upcase != 'SKIP' && !@pkg.no_compile_needed? end puts "Builds are located in #{CREW_LOCAL_BUILD_DIR}.".yellow end @@ -2064,7 +2116,7 @@ def postinstall_command(args) end end -def prop_command(_args) +def prop_command(_) prop end @@ -2137,8 +2189,12 @@ def sysinfo_command(_args) end end -def update_command(_args) - update +def update_command(args) + if args[''] + generate_compatible + else + update + end end def upgrade_command(args) = upgrade(*args[''], build_from_source: @opt_source) diff --git a/install.sh b/install.sh index fc2d80d55..be07052b7 100755 --- a/install.sh +++ b/install.sh @@ -176,7 +176,9 @@ fi # Create the device.json file. cd "${CREW_CONFIG_PATH}" echo_info "\nCreating device.json." -jq --arg key0 'installed_packages' '. | .[$key0]=[]' <<<'{}' > device.json +jq --arg key0 'architecture' --arg value0 "${ARCH}" \ + --arg key1 'installed_packages' \ + '. | .[$key0]=$value0 | .[$key1]=[]' <<<'{}' > device.json # Functions to maintain packages. diff --git a/lib/const.rb b/lib/const.rb index b21bbac4a..31ce792f2 100644 --- a/lib/const.rb +++ b/lib/const.rb @@ -1,7 +1,7 @@ # lib/const.rb # Defines common constants used in different parts of crew -CREW_VERSION = '1.43.3' +CREW_VERSION = '1.43.2' # kernel architecture KERN_ARCH = `uname -m`.chomp diff --git a/lib/package.rb b/lib/package.rb index 3b687a55c..56bac4ec6 100644 --- a/lib/package.rb +++ b/lib/package.rb @@ -30,10 +30,9 @@ class Package attr_accessor :name, :cached_build, :in_build, :build_from_source, :in_upgrade end - def self.load_package(pkgFile) + def self.load_package(pkgFile, pkgName = File.basename(pkgFile, '.rb')) # self.load_package: load a package under 'Package' class scope # - pkgName = File.basename(pkgFile, '.rb') className = pkgName.capitalize # read and eval package script under 'Package' class @@ -80,7 +79,7 @@ class Package @checked_list.merge!({ pkgName => pkgTags }) pkgObj = load_package("#{CREW_PACKAGES_PATH}/#{pkgName}.rb") - is_source = pkgObj.is_source? or pkgObj.build_from_source + is_source = pkgObj.is_source?(ARCH.to_sym) or pkgObj.build_from_source deps = pkgObj.dependencies # append buildessential to deps if building from source is needed/specified @@ -146,7 +145,14 @@ class Package end end - def self.compatible? = @compatibility.casecmp?('all') || @compatibility.include?(ARCH) + def self.compatible? + if @compatibility + return @compatibility.casecmp?('all') || @compatibility.include?(ARCH) + else + warn "#{name}: Missing `compatibility` field.".lightred + return false + end + end def self.depends_on(dependency, ver_range = nil) @dependencies ||= {} @@ -191,30 +197,38 @@ class Package @dependencies.store(depName, [dep_tags, ver_check]) end - def self.get_url - if is_binary? - return "https://gitlab.com/api/v4/projects/26210301/packages/generic/#{name}/#{version}_#{ARCH}/#{name}-#{version}-chromeos-#{ARCH}.#{binary_compression}" - elsif @source_url.is_a?(Hash) - return @source_url[ARCH.to_sym] + def self.get_url(architecture) + if !@build_from_source && @binary_sha256 && @binary_sha256.key?(architecture) + return get_binary_url(architecture) + elsif @source_url.respond_to?(:has_key?) + return @source_url.key?(architecture) ? @source_url[architecture] : nil else return @source_url end end - def self.get_sha256 - if is_binary? - return @binary_sha256[ARCH.to_sym] - elsif @source_sha256.is_a?(Hash) - return @source_sha256[ARCH.to_sym] + def self.get_binary_url(architecture) + architecture = 'armv7l' if architecture == 'aarch64' + return "https://gitlab.com/api/v4/projects/26210301/packages/generic/#{name}/#{version}_#{architecture}/#{name}-#{version}-chromeos-#{architecture}.#{binary_compression}" + end + + def self.get_source_url(architecture) = @source_url.key?(architecture) ? @source_url[architecture] : nil + + def self.get_sha256(architecture) + if !@build_from_source && @binary_sha256 && @binary_sha256.key?(architecture) + return @binary_sha256[architecture] + elsif @source_sha256.respond_to?(:has_key?) + return @source_sha256.key?(architecture) ? @source_sha256[architecture] : nil else return @source_sha256 end end + def self.get_binary_sha256(architecture) = @binary_sha256&.key?(architecture) ? @binary_sha256[architecture] : '' def self.get_extract_dir = "#{name}.#{Time.now.utc.strftime('%Y%m%d%H%M%S')}.dir" - def self.is_binary? = !@build_from_source && @binary_sha256&.key?(ARCH.to_sym) - def self.is_source? = !(is_binary? || is_fake?) + def self.is_binary?(architecture) = !@build_from_source && @binary_sha256 && @binary_sha256.key?(architecture) + def self.is_source?(architecture) = !(is_binary?(architecture) || is_fake?) def self.system(*args, **opt_args) @crew_env_options_hash = if no_env_options? diff --git a/packages/jdk8.rb b/packages/jdk8.rb index 7cf7a9cf8..6ba47d915 100644 --- a/packages/jdk8.rb +++ b/packages/jdk8.rb @@ -58,7 +58,7 @@ class Jdk8 < Package EOT end - return if File.exist?( URI( get_url ).path ) + return if File.exist?( URI( get_source_url(ARCH.to_sym) ).path ) # check if we should prompt user to the archive page or download page based on #{version} # download page only contains latest version while archive page only contains older versions @@ -98,6 +98,6 @@ class Jdk8 < Package def self.postinstall # remove jdk archive after installed - FileUtils.rm_f URI( get_url ).path + FileUtils.rm_f URI( get_source_url(ARCH.to_sym) ).path end end diff --git a/tests/cycle_test b/tests/cycle_test index 6e5bd876a..8f662c15e 100755 --- a/tests/cycle_test +++ b/tests/cycle_test @@ -16,7 +16,8 @@ puts "Running dependency cycle tests...\n".yellow # Loads all packages Dir['../packages/*.rb'].each do |filename| - pkg = Package.load_package(filename) + name = File.basename(filename, '.rb') + pkg = Package.load_package(filename, name) @all_pkgs[name] = pkg end