From 3290443f5cd367eed8b820f64ec7b299ed78d2be Mon Sep 17 00:00:00 2001 From: satmandu Date: Tue, 4 May 2021 16:47:05 -0400 Subject: [PATCH] Lzma Cleanup & replacement with newer xzutils, Add pixz support (#5726) * replace lzma with current xz-utils * add po4a build dep to xzutils * Add xzutils, pixz * is_fake doesn't need source_url * add pixz tpxz format support to crew * Add xzutils to pixz deps * Add pixz if not installed. * change pixz logic * pixz changes * xzutils is a runtime dep for pixz * grammar fix * remove pixz warning --- bin/crew | 37 +++++++++++++++++++++++++++------ lib/const.rb | 2 +- packages/balena_etcher.rb | 2 +- packages/imagemagick6.rb | 2 +- packages/imagemagick7.rb | 2 +- packages/libsolv.rb | 2 +- packages/lzma.rb | 25 +++-------------------- packages/pixz.rb | 43 +++++++++++++++++++++++++++++++++++++++ packages/xorg_server.rb | 2 +- packages/xwayland.rb | 2 +- packages/xzutils.rb | 35 +++++++++++++++++-------------- 11 files changed, 104 insertions(+), 50 deletions(-) create mode 100644 packages/pixz.rb diff --git a/bin/crew b/bin/crew index 2b12b526d3..552b8878eb 100755 --- a/bin/crew +++ b/bin/crew @@ -234,9 +234,13 @@ def generate_compatible Dir[CREW_PACKAGES_PATH + '*.rb'].each do |filename| pkgName = File.basename filename, '.rb' set_package pkgName, filename + puts "Checking #{pkgName} for compatibility.".orange if @opt_verbose if @pkg.compatibility.include? 'all' or @pkg.compatibility.include? ARCH #add to compatible packages + puts "Adding #{pkgName} to compatible packages.".lightgreen if @opt_verbose @device[:compatible_packages].push(name: @pkg.name) + else + puts "#{pkgName} is not a compatible package.".lightred if @opt_verbose end end File.open(CREW_CONFIG_PATH + 'device.json', 'w') do |file| @@ -488,7 +492,7 @@ def update #update compatible packages puts 'Generating compatible packages...' generate_compatible - + puts 'Generating compatible packages done.'.orange if @opt_verbose #check for outdated installed packages puts 'Checking for package updates...' @@ -595,7 +599,7 @@ def download case File.basename(filename) # Sources that download with curl - when /\.zip$/i, /\.(tar(\.(gz|bz2|xz|lz))?|tgz|tbz|txz)$/i, /\.deb$/i, /\.AppImage$/i + when /\.zip$/i, /\.(tar(\.(gz|bz2|xz|lz))?|tgz|tbz|tpxz|txz)$/i, /\.deb$/i, /\.AppImage$/i # Recall file from cache if requested if CREW_CACHE_ENABLED puts "Looking for archive in cache".orange if @opt_verbose @@ -725,6 +729,12 @@ def unpack (meta) Dir.chdir @extract_dir do system "../#{meta[:filename]} --appimage-extract", exception: true end + when /\.tpxz$/i + unless File.exist?("#{CREW_PREFIX}/bin/pixz") + abort 'Pixz is needed for this install. Please install it with \'crew install pixz\''.lightred + end + puts "Unpacking 'tpxz' archive using 'tar', this may take a while..." + system "tar -Ipixz -x#{@verbose}f #{meta[:filename]} -C #{@extract_dir}", exception: true end if meta[:source] == true # Check the number of directories in the archive @@ -809,6 +819,12 @@ def prepare_package (destdir) # than install-info. # https://www.debian.org/doc/debian-policy/ch-docs.html#info-documents FileUtils.rm "#{CREW_DEST_PREFIX}/share/info/dir" if File.exist?("#{CREW_DEST_PREFIX}/share/info/dir") + + # Remove all perl module files which will conflict + if @pkg.name.include? 'perl_' + puts "Removing .packlist and perllocal.pod files to avoid conflicts with other perl packages.".orange + system "find #{CREW_DEST_DIR} -type f \\( -name '.packlist' -o -name perllocal.pod \\) -delete" + end # compress manual files compress_doc "#{CREW_DEST_PREFIX}/man" @@ -871,7 +887,7 @@ def strip_dir (dir) # Strip binaries but not compressed archives puts "Stripping binaries..." - extensions = [ 'bz2', 'gz', 'lha', 'lz', 'lzh', 'rar', 'tar', 'tbz', 'tgz', 'txz', 'xz', 'Z', 'zip' ] + extensions = [ 'bz2', 'gz', 'lha', 'lz', 'lzh', 'rar', 'tar', 'tbz', 'tgz', 'tpxz', 'txz', 'xz', 'Z', 'zip' ] inames = extensions.join(' ! -iname *\.') strip_find_files "find . -type f ! -iname *\.#{inames} -perm /111 -print | sed -e '/lib.*\.a$/d' -e '/lib.*\.so/d'" end @@ -1122,9 +1138,18 @@ def build_package (pwd) end def archive_package (pwd) - pkg_name = "#{@pkg.name}-#{@pkg.version}-chromeos-#{@device[:architecture]}.tar.xz" - Dir.chdir CREW_DEST_DIR do - system "tar c#{@verbose}Jf #{pwd}/#{pkg_name} *" + unless ENV['CREW_USE_PIXZ'] == '1' + pkg_name = "#{@pkg.name}-#{@pkg.version}-chromeos-#{@device[:architecture]}.tar.xz" + Dir.chdir CREW_DEST_DIR do + system "tar c#{@verbose}Jf #{pwd}/#{pkg_name} *" + end + else + puts "Using pixz to compress archive." + pkg_name = "#{@pkg.name}-#{@pkg.version}-chromeos-#{@device[:architecture]}.tpxz" + Dir.chdir CREW_DEST_DIR do + # Use smaller blocks with "-f0.25" to make random access faster. + system "tar c#{@verbose} * | pixz -f0.25 -9 > #{pwd}/#{pkg_name}" + end end system "sha256sum #{pwd}/#{pkg_name} > #{pwd}/#{pkg_name}.sha256" end diff --git a/lib/const.rb b/lib/const.rb index 4fda2925f0..4717c51b1e 100644 --- a/lib/const.rb +++ b/lib/const.rb @@ -1,6 +1,6 @@ # Defines common constants used in different parts of crew -CREW_VERSION = '1.9.2' +CREW_VERSION = '1.9.3' ARCH_ACTUAL = `uname -m`.strip # This helps with virtualized builds on aarch64 machines diff --git a/packages/balena_etcher.rb b/packages/balena_etcher.rb index 35e3baa58c..14f19c7c4f 100644 --- a/packages/balena_etcher.rb +++ b/packages/balena_etcher.rb @@ -19,7 +19,7 @@ class Balena_etcher < Package depends_on 'gtk2' depends_on 'freetype' depends_on 'cairo' - depends_on 'lzma' + depends_on 'xzutils' depends_on 'libnotify' depends_on 'nspr' depends_on 'libgconf' diff --git a/packages/imagemagick6.rb b/packages/imagemagick6.rb index 5e5d1e20fc..eb68c456aa 100644 --- a/packages/imagemagick6.rb +++ b/packages/imagemagick6.rb @@ -27,7 +27,7 @@ class Imagemagick6 < Package depends_on 'graphviz' depends_on 'jbigkit' depends_on 'jemalloc' - depends_on 'lzma' + depends_on 'xzutils' depends_on 'libheif' depends_on 'librsvg' depends_on 'libwebp' diff --git a/packages/imagemagick7.rb b/packages/imagemagick7.rb index c28a1b1fba..6d4fa5fc5c 100644 --- a/packages/imagemagick7.rb +++ b/packages/imagemagick7.rb @@ -30,7 +30,7 @@ class Imagemagick7 < Package depends_on 'graphviz' depends_on 'jbigkit' depends_on 'jemalloc' - depends_on 'lzma' + depends_on 'xzutils' depends_on 'libheif' depends_on 'librsvg' depends_on 'libwebp' diff --git a/packages/libsolv.rb b/packages/libsolv.rb index b9868da676..d2be691520 100644 --- a/packages/libsolv.rb +++ b/packages/libsolv.rb @@ -22,7 +22,7 @@ class Libsolv < Package x86_64: '1c9062a1b1cc87345941eedb01d7aacd9f942c7924d81db8c52d77cde19a32ed', }) - depends_on 'lzma' + depends_on 'xzutils' depends_on 'swig' depends_on 'zstd' diff --git a/packages/lzma.rb b/packages/lzma.rb index 67e948735b..903ce8efb0 100644 --- a/packages/lzma.rb +++ b/packages/lzma.rb @@ -3,31 +3,12 @@ require 'package' class Lzma < Package description 'LZMA Utils are legacy data compression software with high compression ratio.' homepage 'https://tukaani.org/lzma/' - version '4.32.7-1' + version '4.98' license 'public-domain' compatibility 'all' - source_url 'https://tukaani.org/lzma/lzma-4.32.7.tar.bz2' - source_sha256 '618e54513993b3a153fa1c150fccdf25788c72b36e84ab4db71911083531cf6a' - binary_url ({ - aarch64: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/lzma/4.32.7-1_armv7l/lzma-4.32.7-1-chromeos-armv7l.tar.xz', - armv7l: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/lzma/4.32.7-1_armv7l/lzma-4.32.7-1-chromeos-armv7l.tar.xz', - i686: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/lzma/4.32.7-1_i686/lzma-4.32.7-1-chromeos-i686.tar.xz', - x86_64: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/lzma/4.32.7-1_x86_64/lzma-4.32.7-1-chromeos-x86_64.tar.xz', - }) - binary_sha256 ({ - aarch64: 'cb62ba6e36b24cf50d0490e3ed9c35c35a76ca394f98e6c572bf3e49ab48f205', - armv7l: 'cb62ba6e36b24cf50d0490e3ed9c35c35a76ca394f98e6c572bf3e49ab48f205', - i686: '73b723ede8d112ff824f7ab62bb5108f711d0a0cad6daf9472d2303b634e1516', - x86_64: 'c8a08a86689baf36796baa5c86c1507b10b0b5cdaca733737be6c59782b07b1a', - }) + is_fake - def self.build - system "./configure --prefix=#{CREW_PREFIX} --libdir=#{CREW_LIB_PREFIX}" - system "make" - end + depends_on 'xzutils' - def self.install - system "make", "DESTDIR=#{CREW_DEST_DIR}", "install" - end end diff --git a/packages/pixz.rb b/packages/pixz.rb new file mode 100644 index 0000000000..5c69706fec --- /dev/null +++ b/packages/pixz.rb @@ -0,0 +1,43 @@ +# Adapted from Arch Linux pixz PKGBUILD at: +# https://github.com/archlinux/svntogit-community/raw/packages/pixz/trunk/PKGBUILD + +require 'package' + +class Pixz < Package + description 'Parallel, indexed xz compressor' + homepage 'https://github.com/vasi/pixz' + version '1.0.7-0829' + license 'BSD' + compatibility 'all' + source_url 'https://github.com/vasi/pixz.git' + git_hashtag '0829c7315c804a4e40abd63a9d624194dc1e4f0a' + + binary_url({ + aarch64: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/pixz/1.0.7-0829_armv7l/pixz-1.0.7-0829-chromeos-armv7l.tar.xz', + armv7l: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/pixz/1.0.7-0829_armv7l/pixz-1.0.7-0829-chromeos-armv7l.tar.xz', + i686: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/pixz/1.0.7-0829_i686/pixz-1.0.7-0829-chromeos-i686.tar.xz', + x86_64: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/pixz/1.0.7-0829_x86_64/pixz-1.0.7-0829-chromeos-x86_64.tar.xz' + }) + binary_sha256({ + aarch64: 'b3775ba946ecd95e87b68c8bd2d0c93c78027677eb4de15fc9df5d2e4d80f7c2', + armv7l: 'b3775ba946ecd95e87b68c8bd2d0c93c78027677eb4de15fc9df5d2e4d80f7c2', + i686: '8fff1c81a1b0f693a5e6fa592fba32410875e53e754c8b70e9bf5f14f8654627', + x86_64: '2112ae37128533e88bd492d0aae688028fe7ea8b8149e50c16866aa69f90034e' + }) + + depends_on 'libarchive' + depends_on 'asciidoc' => ':build' + depends_on 'xzutils' + + def self.build + system '[ -x configure ] || NOCONFIGURE=1 ./autogen.sh' + system "#{CREW_ENV_OPTIONS} \ + manpage=true \ + ./configure #{CREW_OPTIONS}" + system 'make' + end + + def self.install + system "make DESTDIR=#{CREW_DEST_DIR} install" + end +end diff --git a/packages/xorg_server.rb b/packages/xorg_server.rb index 187b1132b9..6586b90e64 100644 --- a/packages/xorg_server.rb +++ b/packages/xorg_server.rb @@ -39,7 +39,7 @@ class Xorg_server < Package depends_on 'font_util' depends_on 'libbsd' depends_on 'dbus' - depends_on 'lzma' => :build + depends_on 'xzutils' => :build depends_on 'xkbcomp' depends_on 'glproto' depends_on 'xcb_util_renderutil' => :build diff --git a/packages/xwayland.rb b/packages/xwayland.rb index c73deb5f18..d8611ce9c0 100644 --- a/packages/xwayland.rb +++ b/packages/xwayland.rb @@ -36,7 +36,7 @@ class Xwayland < Package depends_on 'libxkbfile' depends_on 'libxtrans' depends_on 'libxdmcp' - depends_on 'lzma' => :build + depends_on 'xzutils' => :build depends_on 'mesa' depends_on 'pixman' depends_on 'wayland' diff --git a/packages/xzutils.rb b/packages/xzutils.rb index 6371921e2b..51ac58e872 100644 --- a/packages/xzutils.rb +++ b/packages/xzutils.rb @@ -3,28 +3,33 @@ require 'package' class Xzutils < Package description 'XZ Utils is free general-purpose data compression software with a high compression ratio.' homepage 'http://tukaani.org/xz/' - version '5.2.5' + version '5.2.5-e7da' license 'public-domain, LGPL-2.1+ and GPL-2+' compatibility 'all' - source_url 'https://tukaani.org/xz/xz-5.2.5.tar.gz' - source_sha256 'f6f4910fd033078738bd82bfba4f49219d03b17eb0794eb91efbae419f4aba10' + source_url 'https://github.com/xz-mirror/xz.git' + git_hashtag 'e7da44d5151e21f153925781ad29334ae0786101' - binary_url ({ - aarch64: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/xzutils/5.2.5_armv7l/xzutils-5.2.5-chromeos-armv7l.tar.xz', - armv7l: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/xzutils/5.2.5_armv7l/xzutils-5.2.5-chromeos-armv7l.tar.xz', - i686: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/xzutils/5.2.5_i686/xzutils-5.2.5-chromeos-i686.tar.xz', - x86_64: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/xzutils/5.2.5_x86_64/xzutils-5.2.5-chromeos-x86_64.tar.xz', + binary_url({ + aarch64: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/xzutils/5.2.5-e7da_armv7l/xzutils-5.2.5-e7da-chromeos-armv7l.tar.xz', + armv7l: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/xzutils/5.2.5-e7da_armv7l/xzutils-5.2.5-e7da-chromeos-armv7l.tar.xz', + i686: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/xzutils/5.2.5-e7da_i686/xzutils-5.2.5-e7da-chromeos-i686.tar.xz', + x86_64: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/xzutils/5.2.5-e7da_x86_64/xzutils-5.2.5-e7da-chromeos-x86_64.tar.xz' }) - binary_sha256 ({ - aarch64: 'bbb38dfbc5f7845dde5d4e6c1aa46be40d4d94f5db26b0aa20f1603a14c6e61d', - armv7l: 'bbb38dfbc5f7845dde5d4e6c1aa46be40d4d94f5db26b0aa20f1603a14c6e61d', - i686: '34db46c8dc58db4c212ba83e36eba17c8852eab14964a6a1d7ab75aaf23be91c', - x86_64: '05f0e86514ed8fefa240e0fc2d40cb2b95414d628f670b19a230a3867e992a24', + binary_sha256({ + aarch64: '3b87b8150061f2f894f9ee69fceb6ab0b010fa9bdf63f79280c683eae28d588f', + armv7l: '3b87b8150061f2f894f9ee69fceb6ab0b010fa9bdf63f79280c683eae28d588f', + i686: 'a7da9276492788d722a0c853a81d8a5aff164eb16f49882514a1870d3f1397c7', + x86_64: 'a3080d777b7f220b35e9c368925e3099796de08c292239c2c5ca117b9beeb859' }) def self.build - system './configure', "--prefix=#{CREW_PREFIX}", "--libdir=#{CREW_LIB_PREFIX}", - '--disable-docs', '--enable-shared', '--disable-static', '--with-pic' + system '[ -x configure ] || NOCONFIGURE=1 ./autogen.sh --no-po4a' + system "#{CREW_ENV_OPTIONS} \ + ./configure \ + #{CREW_OPTIONS} \ + --disable-docs \ + --enable-shared \ + --with-pic" system 'make' end