From 56e3f518e797789aa1cc82128fdf79c09108b835 Mon Sep 17 00:00:00 2001 From: Maximilian Downey Twiss Date: Sat, 13 Sep 2025 23:54:09 +1000 Subject: [PATCH] Split writing deps in `tools/getrealdeps.rb` into `write_deps` function and add tests (#12758) * Move dependency writing logic to write_deps function in tools/getrealdeps.rb * Add tests for write_deps in tools/getrealdeps.rb --- tests/tools/getrealdeps.rb | 276 +++++++++++++++++++++++++++++++++++++ tools/getrealdeps.rb | 220 +++++++++++++++-------------- 2 files changed, 390 insertions(+), 106 deletions(-) create mode 100644 tests/tools/getrealdeps.rb diff --git a/tests/tools/getrealdeps.rb b/tests/tools/getrealdeps.rb new file mode 100644 index 000000000..195971daf --- /dev/null +++ b/tests/tools/getrealdeps.rb @@ -0,0 +1,276 @@ +require 'minitest/autorun' +require_relative '../../lib/const' +require_relative '../../lib/package' +require_relative '../../lib/package_utils' +require_relative '../../lib/buildsystems/pip' +require_relative '../../tools/getrealdeps' + +def test_wrapper(input_file, expected_pkg_file, deps, name: 'example', pkg_class: Package) + # Create the Package (or superclass) object, assigning it the relevant values. + pkg = Class.new(pkg_class) + pkg.name = name + + # Create the temporary package file, write the content to it, and rewind the stream. + pkg_file = Tempfile.create + pkg_file.puts input_file + pkg_file.rewind + + # Write the dependencies to the temporary package file using the created object. + write_deps(pkg_file.path, deps, pkg) + + # Close the temporary package file. + pkg_file.close + + # Read the actual package file into a variable so we can delete it after. + actual_pkg_file = File.read(pkg_file) + + # Delete the temporary package file. + File.unlink(pkg_file.path) + + # Remove the file copied to CREW_LOCAL_REPO_ROOT/packages, as to not leave any residue. + File.unlink("#{CREW_LOCAL_REPO_ROOT}/packages/#{pkg}.rb") + + # Test that the expected package file and the actual package file are the same. + assert_equal(expected_pkg_file, actual_pkg_file) +end + +class GetRealDepsTest < Minitest::Test + def test_add_single_dependency_to_empty + deps = ['libcanberra'] + input_file = <<~EOF + class Example < Package + binary_sha256({}) + end + EOF + expected_pkg_file = <<~EOF + class Example < Package + binary_sha256({}) + + depends_on 'libcanberra' # R + end + EOF + + test_wrapper(input_file, expected_pkg_file, deps) + end + + def test_add_single_dependency_in_order + deps = ['banner'] + input_file = <<~EOF + class Example < Package + binary_sha256({}) + + depends_on 'a2png' + depends_on 'lzlib' + end + EOF + expected_pkg_file = <<~EOF + class Example < Package + binary_sha256({}) + + depends_on 'a2png' + depends_on 'banner' # R + depends_on 'lzlib' + end + EOF + + test_wrapper(input_file, expected_pkg_file, deps) + end + + # TODO: The expected output in this case could be improved. + def test_add_multiple_dependencies_in_order + deps = %w[libcanberra banner] + input_file = <<~EOF + class Example < Package + binary_sha256({}) + + depends_on 'lzlib' + depends_on 'a2png' + depends_on 'libmaxminddb' + end + EOF + expected_pkg_file = <<~EOF + class Example < Package + binary_sha256({}) + + depends_on 'a2png' + depends_on 'banner' # R + depends_on 'libcanberra' # R depends_on 'libmaxminddb' + depends_on 'lzlib' + end + EOF + + test_wrapper(input_file, expected_pkg_file, deps) + end + + def test_add_special_dependency_to_empty + deps = [] + input_file = <<~EOF + class Example < Pip + binary_sha256({}) + end + EOF + expected_pkg_file = <<~EOF + class Example < Pip + binary_sha256({}) + + depends_on 'python3' # R + end + EOF + pkg_class = Pip + + test_wrapper(input_file, expected_pkg_file, deps, pkg_class: pkg_class) + end + + def test_simple_dependency_exception + deps = ['gcc_build'] + input_file = <<~EOF + class Libssp < Package + binary_sha256({}) + end + EOF + expected_pkg_file = <<~EOF + class Libssp < Package + binary_sha256({}) + end + EOF + name = 'libssp' + + test_wrapper(input_file, expected_pkg_file, deps, name: name) + end + + def test_regex_dependency_exception + deps = %w[llvm9_cheesecake asciinema] + input_file = <<~EOF + class Llvm21_build < Package + binary_sha256({}) + + depends_on 'glm' + end + EOF + expected_pkg_file = <<~EOF + class Llvm21_build < Package + binary_sha256({}) + + depends_on 'asciinema' # R + depends_on 'glm' + end + EOF + name = 'llvm21_build' + + test_wrapper(input_file, expected_pkg_file, deps, name: name) + end + + def test_add_single_duplicate_dependency + deps = ['libnftnl'] + input_file = <<~EOF + class Example < Package + binary_sha256({}) + + depends_on 'libnftnl' + end + EOF + expected_pkg_file = <<~EOF + class Example < Package + binary_sha256({}) + + depends_on 'libnftnl' + end + EOF + + test_wrapper(input_file, expected_pkg_file, deps) + end + + def test_add_runtime_duplicate_dependency + deps = %w[abcde libpng] + input_file = <<~EOF + class Example < Package + binary_sha256({}) + + depends_on 'glm' + depends_on 'libpng' # R + end + EOF + expected_pkg_file = <<~EOF + class Example < Package + binary_sha256({}) + + depends_on 'abcde' # R + depends_on 'glm' + depends_on 'libpng' # R + end + EOF + + test_wrapper(input_file, expected_pkg_file, deps) + end + + # TODO: The expected output in this case could be improved. + def test_add_special_duplicate_dependency + deps = ['python3'] + input_file = <<~EOF + class Example < Package + binary_sha256({}) + + depends_on 'libnftnl' + depends_on 'python3' # R + end + EOF + expected_pkg_file = <<~EOF + class Example < Package + binary_sha256({}) + + depends_on 'libnftnl' + depends_on 'python3' # R + depends_on 'python3' # R + end + EOF + pkg_class = Pip + + test_wrapper(input_file, expected_pkg_file, deps, pkg_class: pkg_class) + end + + def test_remove_runtime_dependency + deps = ['libspng'] + input_file = <<~EOF + class Example < Package + binary_sha256({}) + + depends_on 'qt5_x11extras' # R + depends_on 'libmatroska' + end + EOF + expected_pkg_file = <<~EOF + class Example < Package + binary_sha256({}) + + depends_on 'libmatroska' + depends_on 'libspng' # R + end + EOF + + test_wrapper(input_file, expected_pkg_file, deps) + end + + def test_remove_privileged_dependency + deps = ['libspng'] + input_file = <<~EOF + class Example < Package + binary_sha256({}) + + depends_on 'haveged' # R + depends_on 'libmms' + depends_on 'ruby' # R + end + EOF + expected_pkg_file = <<~EOF + class Example < Package + binary_sha256({}) + + depends_on 'libmms' + depends_on 'libspng' # R + depends_on 'ruby' # R + end + EOF + + test_wrapper(input_file, expected_pkg_file, deps) + end +end diff --git a/tools/getrealdeps.rb b/tools/getrealdeps.rb index a47a56857..c2e84123c 100755 --- a/tools/getrealdeps.rb +++ b/tools/getrealdeps.rb @@ -18,15 +18,13 @@ else $LOAD_PATH.unshift File.expand_path(File.join(crew_local_repo_root, 'lib'), __dir__) end -@rubocop_config = CREW_LOCAL_REPO_ROOT.to_s.empty? ? "#{CREW_LIB_PATH}/.rubocop.yml" : File.join(CREW_LOCAL_REPO_ROOT, '.rubocop.yml') - if ARGV.include?('--use-crew-dest-dir') ARGV.delete('--use-crew-dest-dir') @opt_use_crew_dest_dir = true end -# Exit quickly if an invalid package name is given. -if ARGV[0].nil? || ARGV[0].empty? || ARGV[0].include?('#') +# If we're running as a script, exit quickly if an invalid package name is given. +if __FILE__ == $PROGRAM_NAME && (ARGV[0].nil? || ARGV[0].empty? || ARGV[0].include?('#')) puts 'Getrealdeps checks for the runtime dependencies of a package.' puts 'The runtime dependencies are added if the package file is missing them.' puts 'Usage: getrealdeps.rb [--use_crew_dest_dir] ' @@ -44,6 +42,116 @@ def whatprovidesfxn(pkgdepslcl, pkg) filelcl.gsub(/.filelist.*/, '').gsub(%r{.*/}, '').split("\n").uniq.join("\n").gsub(':', '') end +# Write the missing dependencies to the package file. +def write_deps(pkg_file, pkgdeps, pkg) + # Look for missing runtime dependencies, ignoring build and optional deps. + missingpkgdeps = pkgdeps.reject { |i| File.read(pkg_file).include?("depends_on '#{i}'") unless File.read(pkg_file).include?("depends_on '#{i}' => :build") || File.read(pkg_file).include?("# depends_on '#{i}' # R (optional)") } + + # Add special deps for perl, pip, python, and ruby gem packages. + case pkg.superclass.to_s + when 'PERL' + missingpkgdeps << 'perl' + when 'Pip', 'Python' + missingpkgdeps << 'python3' + when 'RUBY' + missingpkgdeps << 'ruby' + end + + # These deps are sometimes architecture dependent or should not be + # removed for other reasons. + privileged_deps = %w[glibc glibc_lib gcc_lib perl python3 ruby] + + # Special cases where dependencies should not be automatically added: + + dependency_exceptions = Set[ + { name_regex: 'llvm.*_build', exclusion_regex: 'llvm.*_*', comments: 'created from the llvm build package.' }, + { name_regex: '(llvm.*_dev|llvm.*_lib|libclc|openmp)', exclusion_regex: 'llvm.*_build', comments: 'should only be a build dep.' }, + { name_regex: 'llvm.*_lib', exclusion_regex: 'llvm_lib', comments: 'should only be a build dep.' }, + { name_regex: 'gcc_build', exclusion_regex: 'gcc.*_*', comments: 'created from the gcc_build package.' }, + { name_regex: '(gcc_dev|gcc_lib|libssp)', exclusion_regex: 'gcc_build', comments: 'should only be a build dep.' }, + { name_regex: 'gcc_lib', exclusion_regex: 'gcc_lib', comments: 'should only be a build dep.' }, + { name_regex: 'python3', exclusion_regex: '(tcl|tk)', comments: 'optional for i686, which does not have gui libraries.' } + ] + + dependency_exceptions_pkgs = dependency_exceptions.map { |h| h[:name_regex] } + + dependency_exceptions_pkgs.each do |exception| + working_exception_pkg = dependency_exceptions.find { |i| i[:name_regex] == exception } + name_regex = working_exception_pkg[:name_regex] + exclusion_regex = working_exception_pkg[:exclusion_regex] + exclusion_comments = working_exception_pkg[:comments] + next unless /#{name_regex}/.match(pkg.name) + puts "#{pkg}: #{exclusion_regex} - #{exclusion_comments}..".orange if pkgdeps.select { |d| /#{exclusion_regex}/.match(d) }.length.positive? + missingpkgdeps.delete_if { |d| /#{exclusion_regex}/.match(d) } + pkgdeps.delete_if { |d| /#{exclusion_regex}/.match(d) } + end + + missingpkgdeps.delete_if { |d| File.read(pkg_file).include?("# depends_on '#{d}' # R (optional)") } + pkgdeps.delete_if { |d| File.read(pkg_file).include?("# depends_on '#{d}' # R (optional)") } + + puts "\nPackage #{pkg} has runtime library dependencies on these packages:".lightblue + pkgdeps.each do |i| + puts " depends_on '#{i}' # R".lightgreen + end + + # Get existing package deps entries so we can add to and sort as + # necessary. + pkgdepsblock = [] + pkgdepsblock += File.foreach(pkg_file).grep(/ depends_on '| # depends_on '/) + + unless missingpkgdeps.empty? + puts "\nPackage file #{pkg}.rb is missing these runtime library dependencies:".orange + puts " depends_on '#{missingpkgdeps.join("' # R\n depends_on '")}' # R".orange + + pkgdepsblock += missingpkgdeps.map { |add_dep| " depends_on '#{add_dep}' # R" } + end + pkgdepsblock.uniq! + pkgdepsblock = pkgdepsblock.sort_by { |dep| dep.split('depends_on ')[1] } + + puts "\n Adding to or replacing deps block in package..." + # First remove all dependencies. + system "sed -i '/ depends_on /d' #{pkg_file}" + system "sed -i '/^ # depends_on /d' #{pkg_file}" + # Now add back our sorted dependencies. + gawk_cmd = "gawk -i inplace -v dep=\"#{pkgdepsblock.join('QQQQQ')}\" 'FNR==NR{ if (/})/) p=NR; next} 1; FNR==p{ print \"\\n\" dep }' #{pkg_file} #{pkg_file}" + system(gawk_cmd) + # The first added line has two dependencies without a newline + # separating them. + system "sed -i 's/RQQQQQ/R\\n/' #{pkg_file}" + system "sed -i 's/QQQQQ//g' #{pkg_file}" + + # Check for and delete old runtime dependencies. + # Its unsafe to do this with other dependencies, because the packager might know something we don't. + lines_to_delete = {} + File.readlines(pkg_file).each_with_index do |line, line_number| + # Find all the explicitly marked runtime dependencies. + dep = line.match(/ depends_on '(.*)' # R/) + # Basically just a nil check, but this way we avoid matching twice. + next unless dep + # Skip unless the runtime dependency in the package does not match the runtime dependencies we've found. + next unless pkgdeps.none?(dep[1]) + # Skip if we're dealing with privileged deps. + next if privileged_deps.include?(dep[1]) + # Record the line content as the key and the line number (incremented by one because the index starts at 0) as the value. + lines_to_delete[line] = line_number + 1 + end + + # Find the location of the rubocop configuration. + rubocop_config = CREW_LOCAL_REPO_ROOT.to_s.empty? ? "#{CREW_LIB_PATH}/.rubocop.yml" : File.join(CREW_LOCAL_REPO_ROOT, '.rubocop.yml') + + # Clean with rubocop. + system "rubocop -c #{rubocop_config} -A #{pkg_file}" + (FileUtils.cp pkg_file, "#{CREW_LOCAL_REPO_ROOT}/packages/#{pkg}.rb" if lines_to_delete.empty?) unless CREW_LOCAL_REPO_ROOT.to_s.empty? + # Leave if there aren't any old runtime dependencies. + return if lines_to_delete.empty? + puts "\nPackage file #{pkg}.rb has these outdated runtime library dependencies:".lightpurple + puts lines_to_delete.keys + system("gawk -i inplace 'NR != #{lines_to_delete.values.join(' && NR != ')}' #{pkg_file}") + # Clean with rubocop. + system "rubocop -c #{rubocop_config} -A #{pkg_file}" + FileUtils.cp pkg_file, "#{CREW_LOCAL_REPO_ROOT}/packages/#{pkg}.rb" unless CREW_LOCAL_REPO_ROOT.to_s.empty? +end + def main(pkg) puts "Checking for the runtime dependencies of #{pkg}...".lightblue pkg_file = File.join(CREW_PACKAGES_PATH, "#{pkg}.rb") @@ -126,108 +234,8 @@ def main(pkg) # Leave early if we didn't find any dependencies. return if pkgdeps.empty? - # Look for missing runtime dependencies, ignoring build and optional deps. - missingpkgdeps = pkgdeps.reject { |i| File.read(pkg_file).include?("depends_on '#{i}'") unless File.read(pkg_file).include?("depends_on '#{i}' => :build") || File.read(pkg_file).include?("# depends_on '#{i}' # R (optional)") } - - # Add special deps for perl, pip, python, and ruby gem packages. - case @pkg.superclass.to_s - when 'PERL' - missingpkgdeps << 'perl' - when 'Pip', 'Python' - missingpkgdeps << 'python3' - when 'RUBY' - missingpkgdeps << 'ruby' - end - - # These deps are sometimes architecture dependent or should not be - # removed for other reasons. - privileged_deps = %w[glibc glibc_lib gcc_lib perl python3 ruby] - - # Special cases where dependencies should not be automatically added: - - dependency_exceptions = Set[ - { name_regex: 'llvm.*_build', exclusion_regex: 'llvm.*_*', comments: 'created from the llvm build package.' }, - { name_regex: '(llvm.*_dev|llvm.*_lib|libclc|openmp)', exclusion_regex: 'llvm.*_build', comments: 'should only be a build dep.' }, - { name_regex: 'llvm.*_lib', exclusion_regex: 'llvm_lib', comments: 'should only be a build dep.' }, - { name_regex: 'gcc_build', exclusion_regex: 'gcc.*_*', comments: 'created from the gcc_build package.' }, - { name_regex: '(gcc_dev|gcc_lib|libssp)', exclusion_regex: 'gcc_build', comments: 'should only be a build dep.' }, - { name_regex: 'gcc_lib', exclusion_regex: 'gcc_lib', comments: 'should only be a build dep.' }, - { name_regex: 'python3', exclusion_regex: '(tcl|tk)', comments: 'optional for i686, which does not have gui libraries.' } - ] - - dependency_exceptions_pkgs = dependency_exceptions.map { |h| h[:name_regex] } - - dependency_exceptions_pkgs.each do |exception| - working_exception_pkg = dependency_exceptions.find { |i| i[:name_regex] == exception } - name_regex = working_exception_pkg[:name_regex] - exclusion_regex = working_exception_pkg[:exclusion_regex] - exclusion_comments = working_exception_pkg[:comments] - next unless /#{name_regex}/.match(pkg) - puts "#{pkg}: #{exclusion_regex} - #{exclusion_comments}..".orange if pkgdeps.select { |d| /#{exclusion_regex}/.match(d) }.length.positive? - missingpkgdeps.delete_if { |d| /#{exclusion_regex}/.match(d) } - pkgdeps.delete_if { |d| /#{exclusion_regex}/.match(d) } - end - - missingpkgdeps.delete_if { |d| File.read(pkg_file).include?("# depends_on '#{d}' # R (optional)") } - pkgdeps.delete_if { |d| File.read(pkg_file).include?("# depends_on '#{d}' # R (optional)") } - - puts "\nPackage #{pkg} has runtime library dependencies on these packages:".lightblue - pkgdeps.each do |i| - puts " depends_on '#{i}' # R".lightgreen - end - - # Get existing package deps entries so we can add to and sort as - # necessary. - pkgdepsblock = [] - pkgdepsblock += File.foreach(pkg_file).grep(/ depends_on '| # depends_on '/) - - unless missingpkgdeps.empty? - puts "\nPackage file #{pkg}.rb is missing these runtime library dependencies:".orange - puts " depends_on '#{missingpkgdeps.join("' # R\n depends_on '")}' # R".orange - - pkgdepsblock += missingpkgdeps.map { |add_dep| " depends_on '#{add_dep}' # R" } - end - pkgdepsblock.uniq! - pkgdepsblock = pkgdepsblock.sort_by { |dep| dep.split('depends_on ')[1] } - - puts "\n Adding to or replacing deps block in package..." - # First remove all dependencies. - system "sed -i '/ depends_on /d' #{pkg_file}" - system "sed -i '/^ # depends_on /d' #{pkg_file}" - # Now add back our sorted dependencies. - gawk_cmd = "gawk -i inplace -v dep=\"#{pkgdepsblock.join('QQQQQ')}\" 'FNR==NR{ if (/})/) p=NR; next} 1; FNR==p{ print \"\\n\" dep }' #{pkg_file} #{pkg_file}" - system(gawk_cmd) - # The first added line has two dependencies without a newline - # separating them. - system "sed -i 's/RQQQQQ/R\\n/' #{pkg_file}" - system "sed -i 's/QQQQQ//g' #{pkg_file}" - - # Check for and delete old runtime dependencies. - # Its unsafe to do this with other dependencies, because the packager might know something we don't. - lines_to_delete = {} - File.readlines(pkg_file).each_with_index do |line, line_number| - # Find all the explicitly marked runtime dependencies. - dep = line.match(/ depends_on '(.*)' # R/) - # Basically just a nil check, but this way we avoid matching twice. - next unless dep - # Skip unless the runtime dependency in the package does not match the runtime dependencies we've found. - next unless pkgdeps.none?(dep[1]) - # Skip if we're dealing with privileged deps. - next if privileged_deps.include?(dep[1]) - # Record the line content as the key and the line number (incremented by one because the index starts at 0) as the value. - lines_to_delete[line] = line_number + 1 - end - # Clean with rubocop. - system "rubocop -c #{@rubocop_config} -A #{pkg_file}" - (FileUtils.cp pkg_file, "#{CREW_LOCAL_REPO_ROOT}/packages/#{pkg}.rb" if lines_to_delete.empty?) unless CREW_LOCAL_REPO_ROOT.to_s.empty? - # Leave if there aren't any old runtime dependencies. - return if lines_to_delete.empty? - puts "\nPackage file #{pkg}.rb has these outdated runtime library dependencies:".lightpurple - puts lines_to_delete.keys - system("gawk -i inplace 'NR != #{lines_to_delete.values.join(' && NR != ')}' #{pkg_file}") - # Clean with rubocop. - system "rubocop -c #{@rubocop_config} -A #{pkg_file}" - FileUtils.cp pkg_file, "#{CREW_LOCAL_REPO_ROOT}/packages/#{pkg}.rb" unless CREW_LOCAL_REPO_ROOT.to_s.empty? + # Write the changed dependencies to the package file. + write_deps(pkg_file, pkgdeps, @pkg) end ARGV.each do |package|