From 8b19dcd05addfb3d2ae0041098ab4595462e83d5 Mon Sep 17 00:00:00 2001 From: Maximilian Downey Twiss Date: Thu, 11 Apr 2024 04:00:04 +1000 Subject: [PATCH] Add lib/package_utils.rb to provide a number of package convenience functions. (#9617) --- bin/crew | 42 +++++++------ commands/files.rb | 5 +- commands/list.rb | 9 +-- commands/remove.rb | 3 +- lib/const.rb | 2 +- lib/fixup.rb | 3 + lib/package.rb | 41 ------------- lib/package_utils.rb | 33 ++++++++++ packages/jdk8.rb | 4 +- tests/lib/package_utils.rb | 123 +++++++++++++++++++++++++++++++++++++ 10 files changed, 193 insertions(+), 72 deletions(-) create mode 100644 lib/package_utils.rb create mode 100644 tests/lib/package_utils.rb diff --git a/bin/crew b/bin/crew index 21c6ecdb5..022215482 100755 --- a/bin/crew +++ b/bin/crew @@ -18,6 +18,7 @@ require_relative '../lib/docopt' require_relative '../lib/downloader' require_relative '../lib/gnome' require_relative '../lib/package' +require_relative '../lib/package_utils' require_relative '../lib/util' # Disallow sudo @@ -134,9 +135,9 @@ def print_package(pkg_path, extra = false) end def print_current_package(extra = false) - status = if @device[:installed_packages].any? { |elem| elem[:name] == @pkg.name } + status = if PackageUtils.installed?(@pkg.name) :installed - elsif !@pkg.compatible? + elsif !PackageUtils.compatible?(@pkg) :incompatible else :available @@ -188,7 +189,7 @@ def generate_compatible puts "Error with #{pkg_name}.rb: #{e}".red unless e.to_s.include?('uninitialized constant') end puts "Checking #{pkg_name} for compatibility.".orange if @opt_verbose - if @pkg.compatible? + if PackageUtils.compatible?(@pkg) # add to compatible packages puts "Adding #{pkg_name} #{@pkg.version} to compatible packages.".lightgreen if @opt_verbose @device[:compatible_packages].push(name: @pkg.name) @@ -313,8 +314,8 @@ def update next end different_version = (package[:version] != @pkg.version) - has_sha = !(@pkg.get_binary_sha256(@device[:architecture]).to_s.empty? || package[:binary_sha256].to_s.empty?) - different_sha = has_sha && package[:binary_sha256] != @pkg.get_binary_sha256(@device[:architecture]) + has_sha = !(PackageUtils.get_sha256(@pkg).to_s.empty? || package[:sha256].to_s.empty?) + different_sha = has_sha && package[:sha256] != PackageUtils.get_sha256(@pkg) can_be_updated += 1 if different_version || different_sha @@ -346,17 +347,18 @@ def upgrade(*pkgs, build_from_source: false) end pkgs.each do - unless @device[:installed_packages].any? { |package| package[:name] == pkg_name } + unless PackageUtils.installed?(pkg_name) puts 'Package '.lightred + pkg_name.orange + ' is not installed. 😔 You may try this: '.lightred + "crew install #{pkg_name}".lightblue return false end end + pkg_ver_latest = Package.load_package(pkg_file, pkg_name).version pkg_ver_installed = @device[:installed_packages].select { |pkg| pkg[:name] == pkg_name } [0][:version] - pkg_hash_latest = Package.load_package(pkg_file, pkg_name).get_binary_sha256(@device[:architecture]) - pkg_hash_installed = @device[:installed_packages].select { |pkg| pkg[:name] == pkg_name } [0][:binary_sha256] + pkg_hash_latest = PackageUtils.get_sha256(Package.load_package(pkg_file, pkg_name)) + pkg_hash_installed = @device[:installed_packages].select { |pkg| pkg[:name] == pkg_name } [0][:sha256] - return pkg_hash_latest != pkg_hash_installed unless !pkg_hash_installed || pkg_hash_latest == '' + return pkg_hash_latest != pkg_hash_installed unless !pkg_hash_installed || pkg_hash_latest.to_s.empty? || Package.load_package(pkg_file, pkg_name).is_fake return pkg_ver_latest != pkg_ver_installed end @@ -423,13 +425,13 @@ def upgrade(*pkgs, build_from_source: false) end def download - url = @pkg.get_url(@device[:architecture]) + url = PackageUtils.get_url(@pkg, build_from_source: @opt_source || @pkg.build_from_source) source = @pkg.source?(@device[:architecture]) uri = URI.parse url filename = File.basename(uri.path) - sha256sum = @pkg.get_sha256(@device[:architecture]) - @extract_dir = @pkg.get_extract_dir + sha256sum = PackageUtils.get_sha256(@pkg, build_from_source: @opt_source || @pkg.build_from_source) + @extract_dir = "#{@pkg.name}.#{Time.now.utc.strftime('%Y%m%d%H%M%S')}.dir" build_cachefile = File.join(CREW_CACHE_DIR, "#{@pkg.name}-#{@pkg.version}-build-#{@device[:architecture]}.tar.zst") return { source:, filename: } if CREW_CACHE_BUILD && File.file?(build_cachefile) @@ -1164,7 +1166,7 @@ def resolve_dependencies @dependencies.map!(&:keys).flatten! # abort if we have incompatible dependencies - abort "Some dependencies are not compatible with your device architecture (#{ARCH}). Unable to continue.".lightred if @dependencies.any? { |dep| !Package.load_package("#{CREW_PACKAGES_PATH}/#{dep}.rb").compatible? } + abort "Some dependencies are not compatible with your device architecture (#{ARCH}). Unable to continue.".lightred if @dependencies.any? { |dep| !PackageUtils.compatible?(Package.load_package("#{CREW_PACKAGES_PATH}/#{dep}.rb")) } # leave only not installed packages in dependencies @dependencies.reject! { |dep_name| @device[:installed_packages].any? { |pkg| pkg[:name] == dep_name } } @@ -1220,7 +1222,7 @@ def resolve_dependencies end def install - if !@pkg.in_upgrade && @device[:installed_packages].any? { |pkg| pkg[:name] == @pkg.name } + if !@pkg.in_upgrade && PackageUtils.installed?(@pkg.name) puts "Package #{@pkg.name} already installed, skipping...".lightgreen return end @@ -1270,7 +1272,7 @@ def install end # add to installed packages - @device[:installed_packages].push(name: @pkg.name, version: @pkg.version, binary_sha256: @pkg.get_binary_sha256(@device[:architecture])) + @device[:installed_packages].push(name: @pkg.name, version: @pkg.version, sha256: PackageUtils.get_sha256(@pkg, build_from_source: @opt_source)) File.open("#{CREW_CONFIG_PATH}/device.json.tmp", 'w') do |file| output = JSON.parse @device.to_json file.write JSON.pretty_generate(output) @@ -1363,7 +1365,7 @@ def archive_package(crew_archive_dest) if @opt_force FileUtils.cp "#{CREW_PACKAGES_PATH}/#{@pkg_name}.rb", "#{CREW_LOCAL_REPO_ROOT}/packages/" puts "The package file for #{@pkg_name} used has been copied to #{CREW_LOCAL_REPO_ROOT}/packages/".lightblue - if @device[:installed_packages].any? { |pkg| pkg[:name] == @pkg.name } + if PackageUtils.installed?(@pkg.name) puts "#{@pkg_name} will now be upgraded...".lightgreen @pkg.in_upgrade = true @pkg.build_from_source = false @@ -1659,11 +1661,11 @@ def build_command(args) # Process preflight block to see if package should be built pre_flight - if !@pkg.is_fake? && @pkg.compatible? && @pkg.source?(ARCH) && ( @pkg.no_source_build? || @pkg.source_url.to_s.upcase != 'SKIP' ) && !@pkg.no_compile_needed? + if !@pkg.is_fake? && PackageUtils.compatible?(@pkg) && @pkg.source?(ARCH) && ( @pkg.no_source_build? || @pkg.source_url.to_s.upcase != 'SKIP' ) && !@pkg.no_compile_needed? resolve_dependencies_and_build else puts 'Unable to build a fake package. Skipping build.'.lightred if @pkg.is_fake? - puts "Package #{@pkg.name} is not compatible with your device architecture (#{ARCH}). Skipping build.".lightred unless @pkg.compatible? + puts "Package #{@pkg.name} is not compatible with your device architecture (#{ARCH}). Skipping build.".lightred unless PackageUtils.compatible?(@pkg) puts 'Unable to build without source. Skipping build.'.lightred unless @pkg.source?(ARCH) && @pkg.source_url.to_s.upcase != 'SKIP' puts 'Compile not needed. Skipping build.'.lightred if @pkg.no_compile_needed? end @@ -1747,7 +1749,7 @@ def install_command(args) @pkg.build_from_source = true if @opt_source || @opt_recursive || CREW_BUILD_FROM_SOURCE next unless @pkg_name - if @pkg.compatible? + if PackageUtils.compatible?(@pkg) resolve_dependencies_and_install else puts "Package #{@pkg.name} is not compatible with your device architecture (#{ARCH}). Skipping install.".lightred @@ -1784,7 +1786,7 @@ def reinstall_command(args) @pkg.build_from_source = true if @opt_source || @opt_recursive || CREW_BUILD_FROM_SOURCE next unless @pkg_name - if @pkg.compatible? + if PackageUtils.compatible?(@pkg) @pkg.in_upgrade = true resolve_dependencies_and_install @pkg.in_upgrade = false diff --git a/commands/files.rb b/commands/files.rb index 1ef21c054..dfed0b741 100644 --- a/commands/files.rb +++ b/commands/files.rb @@ -1,12 +1,11 @@ -require 'json' require_relative '../lib/const' require_relative '../lib/convert_size' +require_relative '../lib/package_utils' class Command def self.files(pkg) # Check if the package is even installed first, as this is the most likely reason we cannot find a filelist. - device_json = JSON.load_file(File.join(CREW_CONFIG_PATH, 'device.json')) - if device_json['installed_packages'].none? { |entry| entry['name'] == pkg.name } + unless PackageUtils.installed?(pkg.name) puts "Package #{pkg.name} is not installed.".lightred return end diff --git a/commands/list.rb b/commands/list.rb index 25c03264e..27a505cbb 100644 --- a/commands/list.rb +++ b/commands/list.rb @@ -3,6 +3,7 @@ require 'json' require_relative '../lib/color' require_relative '../lib/const' require_relative '../lib/package' +require_relative '../lib/package_utils' class Command def self.list(available, installed, compatible, incompatible, verbose) @@ -17,7 +18,7 @@ class Command pkg_name = File.basename(filename, '.rb') next if installed_packages.key?(pkg_name) pkg = Package.load_package(filename) - puts pkg_name if pkg.compatible? + puts pkg_name if PackageUtils.compatible?(pkg) end elsif installed if verbose @@ -36,14 +37,14 @@ class Command Dir["#{CREW_PACKAGES_PATH}/*.rb"].each do |filename| pkg_name = File.basename(filename, '.rb') pkg = Package.load_package(filename) - puts pkg_name.lightgreen if pkg.compatible? && installed_packages.key?(pkg_name) - puts pkg_name if pkg.compatible? + puts pkg_name.lightgreen if PackageUtils.compatible?(pkg) && installed_packages.key?(pkg_name) + puts pkg_name if PackageUtils.compatible?(pkg) end elsif incompatible Dir["#{CREW_PACKAGES_PATH}/*.rb"].each do |filename| pkg_name = File.basename(filename, '.rb') pkg = Package.load_package(filename) - puts pkg_name.lightred unless pkg.compatible? + puts pkg_name.lightred unless PackageUtils.compatible?(pkg) end end end diff --git a/commands/remove.rb b/commands/remove.rb index af5fb5298..5dfdd8c53 100644 --- a/commands/remove.rb +++ b/commands/remove.rb @@ -1,13 +1,14 @@ require 'fileutils' require 'json' require_relative '../lib/const' +require_relative '../lib/package_utils' class Command def self.remove(pkg, verbose) device_json = JSON.load_file(File.join(CREW_CONFIG_PATH, 'device.json')) # Make sure the package is actually installed before we attempt to remove it. - if device_json['installed_packages'].none? { |entry| entry['name'] == pkg.name } + unless PackageUtils.installed?(pkg.name) puts "Package #{pkg.name} isn't installed.".lightred return end diff --git a/lib/const.rb b/lib/const.rb index 31116e3ba..bed94565e 100644 --- a/lib/const.rb +++ b/lib/const.rb @@ -1,7 +1,7 @@ # lib/const.rb # Defines common constants used in different parts of crew -CREW_VERSION = '1.46.6' +CREW_VERSION = '1.46.7' # kernel architecture KERN_ARCH = `uname -m`.chomp diff --git a/lib/fixup.rb b/lib/fixup.rb index 39a033d46..6f95f83e6 100644 --- a/lib/fixup.rb +++ b/lib/fixup.rb @@ -16,6 +16,9 @@ Dir.chdir CREW_LIB_PATH do system 'git sparse-checkout reapply' end +# Rename the binary_sha256 variable to sha256 in the device.json file +system(" sed -i 's/binary_sha256/sha256/g' #{File.join(CREW_CONFIG_PATH, 'device.json')}") + # Check for renamed and deprecated packages, and handle them. pkg_update_arr = [ diff --git a/lib/package.rb b/lib/package.rb index 928dfacb5..ef36d6a85 100644 --- a/lib/package.rb +++ b/lib/package.rb @@ -150,15 +150,6 @@ class Package end end - def self.compatible? - if @compatibility - return @compatibility.casecmp?('all') || @compatibility.include?(ARCH) - else - warn "#{name}: Missing `compatibility` field.".lightred - return false - end - end - def self.depends_on(dependency, ver_range = nil) @dependencies ||= {} ver_check = nil @@ -202,38 +193,6 @@ class Package @dependencies.store(dep_name, [dep_tags, ver_check]) end - def self.get_url(architecture) - if !@build_from_source && @binary_sha256 && @binary_sha256.key?(architecture) - return get_binary_url(architecture) - elsif @source_url.respond_to?(:has_key?) - return @source_url.key?(architecture) ? @source_url[architecture] : nil - else - return @source_url - end - end - - def self.get_binary_url(architecture) - architecture = 'armv7l' if architecture == 'aarch64' - binary_compress_ext = binary_compression.nil? ? 'tar.zst' : binary_compression - puts "binary_compress_ext is #{binary_compress_ext} and binary_compression is #{binary_compression}" if @opt_verbose - return "https://gitlab.com/api/v4/projects/26210301/packages/generic/#{name}/#{version}_#{architecture}/#{name}-#{version}-chromeos-#{architecture}.#{binary_compress_ext}" - end - - def self.get_source_url(architecture) = @source_url.key?(architecture) ? @source_url[architecture] : nil - - def self.get_sha256(architecture) - if !@build_from_source && @binary_sha256 && @binary_sha256.key?(architecture) - return @binary_sha256[architecture] - elsif @source_sha256.respond_to?(:has_key?) - return @source_sha256.key?(architecture) ? @source_sha256[architecture] : nil - else - return @source_sha256 - end - end - - def self.get_binary_sha256(architecture) = @binary_sha256&.key?(architecture) ? @binary_sha256[architecture] : '' - def self.get_extract_dir = "#{name}.#{Time.now.utc.strftime('%Y%m%d%H%M%S')}.dir" - def self.binary?(architecture) = !@build_from_source && @binary_sha256 && @binary_sha256.key?(architecture) def self.source?(architecture) = !(binary?(architecture) || is_fake?) diff --git a/lib/package_utils.rb b/lib/package_utils.rb new file mode 100644 index 000000000..1858f5943 --- /dev/null +++ b/lib/package_utils.rb @@ -0,0 +1,33 @@ +require 'json' +require_relative 'const' + +class PackageUtils + def self.installed?(pkg_name) + device_json = JSON.load_file(File.join(CREW_CONFIG_PATH, 'device.json')) + return device_json['installed_packages'].any? { |elem| elem['name'] == pkg_name } + end + + def self.compatible?(pkg) + return pkg.compatibility.casecmp?('all') || pkg.compatibility.include?(ARCH) + end + + def self.get_url(pkg, build_from_source: false) + if !build_from_source && pkg.binary_sha256&.key?(ARCH.to_sym) + return "https://gitlab.com/api/v4/projects/26210301/packages/generic/#{pkg.name}/#{pkg.version}_#{ARCH}/#{pkg.name}-#{pkg.version}-chromeos-#{ARCH}.#{pkg.binary_compression}" + elsif pkg.source_url.is_a?(Hash) && pkg.source_url&.key?(ARCH.to_sym) + return pkg.source_url[ARCH.to_sym] + else + return pkg.source_url + end + end + + def self.get_sha256(pkg, build_from_source: false) + if !build_from_source && pkg.binary_sha256&.key?(ARCH.to_sym) + return pkg.binary_sha256[ARCH.to_sym] + elsif pkg.source_sha256.is_a?(Hash) && pkg.source_sha256&.key?(ARCH.to_sym) + return pkg.source_sha256[ARCH.to_sym] + else + return pkg.source_sha256 + end + end +end diff --git a/packages/jdk8.rb b/packages/jdk8.rb index 6ba47d915..8b3c3ba8d 100644 --- a/packages/jdk8.rb +++ b/packages/jdk8.rb @@ -58,7 +58,7 @@ class Jdk8 < Package EOT end - return if File.exist?( URI( get_source_url(ARCH.to_sym) ).path ) + return if File.exist?( URI( source_url.key(ARCH.to_sym) ).path ) # check if we should prompt user to the archive page or download page based on #{version} # download page only contains latest version while archive page only contains older versions @@ -98,6 +98,6 @@ class Jdk8 < Package def self.postinstall # remove jdk archive after installed - FileUtils.rm_f URI( get_source_url(ARCH.to_sym) ).path + FileUtils.rm_f URI( source_url.key(ARCH.to_sym) ).path end end diff --git a/tests/lib/package_utils.rb b/tests/lib/package_utils.rb new file mode 100644 index 000000000..d83f308ed --- /dev/null +++ b/tests/lib/package_utils.rb @@ -0,0 +1,123 @@ +require 'minitest/autorun' +require_relative '../../lib/const' +require_relative '../../lib/package' +require_relative '../../lib/package_utils' + +class PackageUtilsTest < Minitest::Test + def test_installed + # Ruby is guaranteed to be installed as it is an essential package. + assert(PackageUtils.installed?('ruby')) + end + + def test_not_installed + # Create a nonsense package name which will not be installed. + refute(PackageUtils.installed?('99notinstalled')) + end + + def test_compatible_all + # Packages with 'all' compatibility are compatible on all architectures. + pkg = Class.new(Package) + pkg.instance_eval do + compatibility 'all' + end + assert(PackageUtils.compatible?(pkg)) + end + + def test_compatible_multiple_values + # Packages with a compatibility value that includes the current architecture are compatible with the current architecture. + pkg = Class.new(Package) + pkg.instance_eval do + compatibility "ia64 #{ARCH}" + end + assert(PackageUtils.compatible?(pkg)) + end + + def test_compatible_single_value + # Packages with a compatibility value that is the current architecture are compatible with the current architecture. + pkg = Class.new(Package) + pkg.instance_eval do + compatibility ARCH + end + assert(PackageUtils.compatible?(pkg)) + end + + def test_not_compatible_arch + # No packages are compatible with the Alternate Instruction Set architecture + pkg = Class.new(Package) + pkg.instance_eval do + compatibility 'ais' + end + refute(PackageUtils.compatible?(pkg)) + end + + def test_get_binary_url + pkg = Class.new(Package) + pkg.name = 'test_package' + pkg.instance_eval do + version '1.0' + binary_compression 'tar.zst' + binary_sha256({ ARCH.to_sym => '0000000000000000000000000000000000000000000000000000000000000000' }) + end + assert_equal("https://gitlab.com/api/v4/projects/26210301/packages/generic/test_package/1.0_#{ARCH}/test_package-1.0-chromeos-#{ARCH}.tar.zst", PackageUtils.get_url(pkg)) + end + + def test_get_source_url_hash + pkg = Class.new(Package) + pkg.instance_eval do + source_url({ ARCH.to_sym => 'https://example.com/example_package.tar.bz3' }) + end + assert_equal('https://example.com/example_package.tar.bz3', PackageUtils.get_url(pkg)) + end + + def test_get_source_url + pkg = Class.new(Package) + pkg.instance_eval do + source_url 'https://example.com/other_example_package.tar.zst' + end + assert_equal('https://example.com/other_example_package.tar.zst', PackageUtils.get_url(pkg)) + end + + def test_get_url_build_from_source + pkg = Class.new(Package) + pkg.instance_eval do + version '1.0' + source_url 'https://example.com/another_example_package.tar.xz' + binary_compression 'tar.zst' + binary_sha256({ ARCH.to_sym => '0000000000000000000000000000000000000000000000000000000000000000' }) + end + assert_equal('https://example.com/another_example_package.tar.xz', PackageUtils.get_url(pkg, build_from_source: true)) + end + + def test_get_binary_sha256 + pkg = Class.new(Package) + pkg.instance_eval do + binary_sha256({ ARCH.to_sym => '1111111111111111111111111111111111111111111111111111111111111111' }) + end + assert_equal('1111111111111111111111111111111111111111111111111111111111111111', PackageUtils.get_sha256(pkg)) + end + + def test_get_source_sha256_hash + pkg = Class.new(Package) + pkg.instance_eval do + source_sha256({ ARCH.to_sym => '2222222222222222222222222222222222222222222222222222222222222222' }) + end + assert_equal('2222222222222222222222222222222222222222222222222222222222222222', PackageUtils.get_sha256(pkg)) + end + + def test_get_source_sha256 + pkg = Class.new(Package) + pkg.instance_eval do + source_sha256 '3333333333333333333333333333333333333333333333333333333333333333' + end + assert_equal('3333333333333333333333333333333333333333333333333333333333333333', PackageUtils.get_sha256(pkg)) + end + + def test_get_sha256_build_from_source + pkg = Class.new(Package) + pkg.instance_eval do + source_sha256 '4444444444444444444444444444444444444444444444444444444444444444' + binary_sha256({ ARCH.to_sym => '0000000000000000000000000000000000000000000000000000000000000000' }) + end + assert_equal('4444444444444444444444444444444444444444444444444444444444444444', PackageUtils.get_sha256(pkg, build_from_source: true)) + end +end