github/chromebrew
1
1
Fork 0
mirror of https://github.com/chromebrew/chromebrew.git synced 2026-01-09 15:37:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

Split writing deps in tools/getrealdeps.rb into write_deps function and add tests (#12758)

Browse Source 
* Move dependency writing logic to write_deps function in tools/getrealdeps.rb

* Add tests for write_deps in tools/getrealdeps.rb
This commit is contained in:
Maximilian Downey Twiss
2025-09-13 23:54:09 +10:00
committed by GitHub
parent 6490785261
commit 56e3f518e7
2 changed files with 390 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

276
tests/tools/getrealdeps.rb Normal file
View File

@@ -0,0 +1,276 @@
require 'minitest/autorun'
require_relative '../../lib/const'
require_relative '../../lib/package'
require_relative '../../lib/package_utils'
require_relative '../../lib/buildsystems/pip'
require_relative '../../tools/getrealdeps'
def test_wrapper(input_file, expected_pkg_file, deps, name: 'example', pkg_class: Package)
# Create the Package (or superclass) object, assigning it the relevant values.
pkg = Class.new(pkg_class)
pkg.name = name
# Create the temporary package file, write the content to it, and rewind the stream.
pkg_file = Tempfile.create
pkg_file.puts input_file
pkg_file.rewind
# Write the dependencies to the temporary package file using the created object.
write_deps(pkg_file.path, deps, pkg)
# Close the temporary package file.
pkg_file.close
# Read the actual package file into a variable so we can delete it after.
actual_pkg_file = File.read(pkg_file)
# Delete the temporary package file.
File.unlink(pkg_file.path)
# Remove the file copied to CREW_LOCAL_REPO_ROOT/packages, as to not leave any residue.
File.unlink("#{CREW_LOCAL_REPO_ROOT}/packages/#{pkg}.rb")
# Test that the expected package file and the actual package file are the same.
assert_equal(expected_pkg_file, actual_pkg_file)
end
class GetRealDepsTest < Minitest::Test
def test_add_single_dependency_to_empty
deps = ['libcanberra']
input_file = <<~EOF
class Example < Package
binary_sha256({})
end
EOF
expected_pkg_file = <<~EOF
class Example < Package
binary_sha256({})
depends_on 'libcanberra' # R
end
EOF
test_wrapper(input_file, expected_pkg_file, deps)
end
def test_add_single_dependency_in_order
deps = ['banner']
input_file = <<~EOF
class Example < Package
binary_sha256({})
depends_on 'a2png'
depends_on 'lzlib'
end
EOF
expected_pkg_file = <<~EOF
class Example < Package
binary_sha256({})
depends_on 'a2png'
depends_on 'banner' # R
depends_on 'lzlib'
end
EOF
test_wrapper(input_file, expected_pkg_file, deps)
end
# TODO: The expected output in this case could be improved.
def test_add_multiple_dependencies_in_order
deps = %w[libcanberra banner]
input_file = <<~EOF
class Example < Package
binary_sha256({})
depends_on 'lzlib'
depends_on 'a2png'
depends_on 'libmaxminddb'
end
EOF
expected_pkg_file = <<~EOF
class Example < Package
binary_sha256({})
depends_on 'a2png'
depends_on 'banner' # R
depends_on 'libcanberra' # R depends_on 'libmaxminddb'
depends_on 'lzlib'
end
EOF
test_wrapper(input_file, expected_pkg_file, deps)
end
def test_add_special_dependency_to_empty
deps = []
input_file = <<~EOF
class Example < Pip
binary_sha256({})
end
EOF
expected_pkg_file = <<~EOF
class Example < Pip
binary_sha256({})
depends_on 'python3' # R
end
EOF
pkg_class = Pip
test_wrapper(input_file, expected_pkg_file, deps, pkg_class: pkg_class)
end
def test_simple_dependency_exception
deps = ['gcc_build']
input_file = <<~EOF
class Libssp < Package
binary_sha256({})
end
EOF
expected_pkg_file = <<~EOF
class Libssp < Package
binary_sha256({})
end
EOF
name = 'libssp'
test_wrapper(input_file, expected_pkg_file, deps, name: name)
end
def test_regex_dependency_exception
deps = %w[llvm9_cheesecake asciinema]
input_file = <<~EOF
class Llvm21_build < Package
binary_sha256({})
depends_on 'glm'
end
EOF
expected_pkg_file = <<~EOF
class Llvm21_build < Package
binary_sha256({})
depends_on 'asciinema' # R
depends_on 'glm'
end
EOF
name = 'llvm21_build'
test_wrapper(input_file, expected_pkg_file, deps, name: name)
end
def test_add_single_duplicate_dependency
deps = ['libnftnl']
input_file = <<~EOF
class Example < Package
binary_sha256({})
depends_on 'libnftnl'
end
EOF
expected_pkg_file = <<~EOF
class Example < Package
binary_sha256({})
depends_on 'libnftnl'
end
EOF
test_wrapper(input_file, expected_pkg_file, deps)
end
def test_add_runtime_duplicate_dependency
deps = %w[abcde libpng]
input_file = <<~EOF
class Example < Package
binary_sha256({})
depends_on 'glm'
depends_on 'libpng' # R
end
EOF
expected_pkg_file = <<~EOF
class Example < Package
binary_sha256({})
depends_on 'abcde' # R
depends_on 'glm'
depends_on 'libpng' # R
end
EOF
test_wrapper(input_file, expected_pkg_file, deps)
end
# TODO: The expected output in this case could be improved.
def test_add_special_duplicate_dependency
deps = ['python3']
input_file = <<~EOF
class Example < Package
binary_sha256({})
depends_on 'libnftnl'
depends_on 'python3' # R
end
EOF
expected_pkg_file = <<~EOF
class Example < Package
binary_sha256({})
depends_on 'libnftnl'
depends_on 'python3' # R
depends_on 'python3' # R
end
EOF
pkg_class = Pip
test_wrapper(input_file, expected_pkg_file, deps, pkg_class: pkg_class)
end
def test_remove_runtime_dependency
deps = ['libspng']
input_file = <<~EOF
class Example < Package
binary_sha256({})
depends_on 'qt5_x11extras' # R
depends_on 'libmatroska'
end
EOF
expected_pkg_file = <<~EOF
class Example < Package
binary_sha256({})
depends_on 'libmatroska'
depends_on 'libspng' # R
end
EOF
test_wrapper(input_file, expected_pkg_file, deps)
end
def test_remove_privileged_dependency
deps = ['libspng']
input_file = <<~EOF
class Example < Package
binary_sha256({})
depends_on 'haveged' # R
depends_on 'libmms'
depends_on 'ruby' # R
end
EOF
expected_pkg_file = <<~EOF
class Example < Package
binary_sha256({})
depends_on 'libmms'
depends_on 'libspng' # R
depends_on 'ruby' # R
end
EOF
test_wrapper(input_file, expected_pkg_file, deps)
end
end

220
tools/getrealdeps.rb
View File

@@ -18,15 +18,13 @@ else
$LOAD_PATH.unshift File.expand_path(File.join(crew_local_repo_root, 'lib'), __dir__)
end
@rubocop_config = CREW_LOCAL_REPO_ROOT.to_s.empty? ? "#{CREW_LIB_PATH}/.rubocop.yml" : File.join(CREW_LOCAL_REPO_ROOT, '.rubocop.yml')
if ARGV.include?('--use-crew-dest-dir')
ARGV.delete('--use-crew-dest-dir')
@opt_use_crew_dest_dir = true
end
# Exit quickly if an invalid package name is given.
if ARGV[0].nil? || ARGV[0].empty? || ARGV[0].include?('#')
# If we're running as a script, exit quickly if an invalid package name is given.
if __FILE__ == $PROGRAM_NAME && (ARGV[0].nil? || ARGV[0].empty? || ARGV[0].include?('#'))
puts 'Getrealdeps checks for the runtime dependencies of a package.'
puts 'The runtime dependencies are added if the package file is missing them.'
puts 'Usage: getrealdeps.rb [--use_crew_dest_dir] <packagename>'
@@ -44,6 +42,116 @@ def whatprovidesfxn(pkgdepslcl, pkg)
filelcl.gsub(/.filelist.*/, '').gsub(%r{.*/}, '').split("\n").uniq.join("\n").gsub(':', '')
end
# Write the missing dependencies to the package file.
def write_deps(pkg_file, pkgdeps, pkg)
# Look for missing runtime dependencies, ignoring build and optional deps.
missingpkgdeps = pkgdeps.reject { |i| File.read(pkg_file).include?("depends_on '#{i}'") unless File.read(pkg_file).include?("depends_on '#{i}' => :build") || File.read(pkg_file).include?("# depends_on '#{i}' # R (optional)") }
# Add special deps for perl, pip, python, and ruby gem packages.
case pkg.superclass.to_s
when 'PERL'
missingpkgdeps << 'perl'
when 'Pip', 'Python'
missingpkgdeps << 'python3'
when 'RUBY'
missingpkgdeps << 'ruby'
end
# These deps are sometimes architecture dependent or should not be
# removed for other reasons.
privileged_deps = %w[glibc glibc_lib gcc_lib perl python3 ruby]
# Special cases where dependencies should not be automatically added:
dependency_exceptions = Set[
{ name_regex: 'llvm.*_build', exclusion_regex: 'llvm.*_*', comments: 'created from the llvm build package.' },
{ name_regex: '(llvm.*_dev|llvm.*_lib|libclc|openmp)', exclusion_regex: 'llvm.*_build', comments: 'should only be a build dep.' },
{ name_regex: 'llvm.*_lib', exclusion_regex: 'llvm_lib', comments: 'should only be a build dep.' },
{ name_regex: 'gcc_build', exclusion_regex: 'gcc.*_*', comments: 'created from the gcc_build package.' },
{ name_regex: '(gcc_dev|gcc_lib|libssp)', exclusion_regex: 'gcc_build', comments: 'should only be a build dep.' },
{ name_regex: 'gcc_lib', exclusion_regex: 'gcc_lib', comments: 'should only be a build dep.' },
{ name_regex: 'python3', exclusion_regex: '(tcl|tk)', comments: 'optional for i686, which does not have gui libraries.' }
]
dependency_exceptions_pkgs = dependency_exceptions.map { |h| h[:name_regex] }
dependency_exceptions_pkgs.each do |exception|
working_exception_pkg = dependency_exceptions.find { |i| i[:name_regex] == exception }
name_regex = working_exception_pkg[:name_regex]
exclusion_regex = working_exception_pkg[:exclusion_regex]
exclusion_comments = working_exception_pkg[:comments]
next unless /#{name_regex}/.match(pkg.name)
puts "#{pkg}: #{exclusion_regex} - #{exclusion_comments}..".orange if pkgdeps.select { |d| /#{exclusion_regex}/.match(d) }.length.positive?
missingpkgdeps.delete_if { |d| /#{exclusion_regex}/.match(d) }
pkgdeps.delete_if { |d| /#{exclusion_regex}/.match(d) }
end
missingpkgdeps.delete_if { |d| File.read(pkg_file).include?("# depends_on '#{d}' # R (optional)") }
pkgdeps.delete_if { |d| File.read(pkg_file).include?("# depends_on '#{d}' # R (optional)") }
puts "\nPackage #{pkg} has runtime library dependencies on these packages:".lightblue
pkgdeps.each do |i|
puts " depends_on '#{i}' # R".lightgreen
end
# Get existing package deps entries so we can add to and sort as
# necessary.
pkgdepsblock = []
pkgdepsblock += File.foreach(pkg_file).grep(/ depends_on '| # depends_on '/)
unless missingpkgdeps.empty?
puts "\nPackage file #{pkg}.rb is missing these runtime library dependencies:".orange
puts " depends_on '#{missingpkgdeps.join("' # R\n depends_on '")}' # R".orange
pkgdepsblock += missingpkgdeps.map { |add_dep| " depends_on '#{add_dep}' # R" }
end
pkgdepsblock.uniq!
pkgdepsblock = pkgdepsblock.sort_by { |dep| dep.split('depends_on ')[1] }
puts "\n Adding to or replacing deps block in package..."
# First remove all dependencies.
system "sed -i '/ depends_on /d' #{pkg_file}"
system "sed -i '/^ # depends_on /d' #{pkg_file}"
# Now add back our sorted dependencies.
gawk_cmd = "gawk -i inplace -v dep=\"#{pkgdepsblock.join('QQQQQ')}\" 'FNR==NR{ if (/})/) p=NR; next} 1; FNR==p{ print \"\\n\" dep }' #{pkg_file} #{pkg_file}"
system(gawk_cmd)
# The first added line has two dependencies without a newline
# separating them.
system "sed -i 's/RQQQQQ/R\\n/' #{pkg_file}"
system "sed -i 's/QQQQQ//g' #{pkg_file}"
# Check for and delete old runtime dependencies.
# Its unsafe to do this with other dependencies, because the packager might know something we don't.
lines_to_delete = {}
File.readlines(pkg_file).each_with_index do |line, line_number|
# Find all the explicitly marked runtime dependencies.
dep = line.match(/ depends_on '(.*)' # R/)
# Basically just a nil check, but this way we avoid matching twice.
next unless dep
# Skip unless the runtime dependency in the package does not match the runtime dependencies we've found.
next unless pkgdeps.none?(dep[1])
# Skip if we're dealing with privileged deps.
next if privileged_deps.include?(dep[1])
# Record the line content as the key and the line number (incremented by one because the index starts at 0) as the value.
lines_to_delete[line] = line_number + 1
end
# Find the location of the rubocop configuration.
rubocop_config = CREW_LOCAL_REPO_ROOT.to_s.empty? ? "#{CREW_LIB_PATH}/.rubocop.yml" : File.join(CREW_LOCAL_REPO_ROOT, '.rubocop.yml')
# Clean with rubocop.
system "rubocop -c #{rubocop_config} -A #{pkg_file}"
(FileUtils.cp pkg_file, "#{CREW_LOCAL_REPO_ROOT}/packages/#{pkg}.rb" if lines_to_delete.empty?) unless CREW_LOCAL_REPO_ROOT.to_s.empty?
# Leave if there aren't any old runtime dependencies.
return if lines_to_delete.empty?
puts "\nPackage file #{pkg}.rb has these outdated runtime library dependencies:".lightpurple
puts lines_to_delete.keys
system("gawk -i inplace 'NR != #{lines_to_delete.values.join(' && NR != ')}' #{pkg_file}")
# Clean with rubocop.
system "rubocop -c #{rubocop_config} -A #{pkg_file}"
FileUtils.cp pkg_file, "#{CREW_LOCAL_REPO_ROOT}/packages/#{pkg}.rb" unless CREW_LOCAL_REPO_ROOT.to_s.empty?
end
def main(pkg)
puts "Checking for the runtime dependencies of #{pkg}...".lightblue
pkg_file = File.join(CREW_PACKAGES_PATH, "#{pkg}.rb")
@@ -126,108 +234,8 @@ def main(pkg)
# Leave early if we didn't find any dependencies.
return if pkgdeps.empty?
# Look for missing runtime dependencies, ignoring build and optional deps.
missingpkgdeps = pkgdeps.reject { |i| File.read(pkg_file).include?("depends_on '#{i}'") unless File.read(pkg_file).include?("depends_on '#{i}' => :build") || File.read(pkg_file).include?("# depends_on '#{i}' # R (optional)") }
# Add special deps for perl, pip, python, and ruby gem packages.
case @pkg.superclass.to_s
when 'PERL'
missingpkgdeps << 'perl'
when 'Pip', 'Python'
missingpkgdeps << 'python3'
when 'RUBY'
missingpkgdeps << 'ruby'
end
# These deps are sometimes architecture dependent or should not be
# removed for other reasons.
privileged_deps = %w[glibc glibc_lib gcc_lib perl python3 ruby]
# Special cases where dependencies should not be automatically added:
dependency_exceptions = Set[
{ name_regex: 'llvm.*_build', exclusion_regex: 'llvm.*_*', comments: 'created from the llvm build package.' },
{ name_regex: '(llvm.*_dev|llvm.*_lib|libclc|openmp)', exclusion_regex: 'llvm.*_build', comments: 'should only be a build dep.' },
{ name_regex: 'llvm.*_lib', exclusion_regex: 'llvm_lib', comments: 'should only be a build dep.' },
{ name_regex: 'gcc_build', exclusion_regex: 'gcc.*_*', comments: 'created from the gcc_build package.' },
{ name_regex: '(gcc_dev|gcc_lib|libssp)', exclusion_regex: 'gcc_build', comments: 'should only be a build dep.' },
{ name_regex: 'gcc_lib', exclusion_regex: 'gcc_lib', comments: 'should only be a build dep.' },
{ name_regex: 'python3', exclusion_regex: '(tcl|tk)', comments: 'optional for i686, which does not have gui libraries.' }
]
dependency_exceptions_pkgs = dependency_exceptions.map { |h| h[:name_regex] }
dependency_exceptions_pkgs.each do |exception|
working_exception_pkg = dependency_exceptions.find { |i| i[:name_regex] == exception }
name_regex = working_exception_pkg[:name_regex]
exclusion_regex = working_exception_pkg[:exclusion_regex]
exclusion_comments = working_exception_pkg[:comments]
next unless /#{name_regex}/.match(pkg)
puts "#{pkg}: #{exclusion_regex} - #{exclusion_comments}..".orange if pkgdeps.select { |d| /#{exclusion_regex}/.match(d) }.length.positive?
missingpkgdeps.delete_if { |d| /#{exclusion_regex}/.match(d) }
pkgdeps.delete_if { |d| /#{exclusion_regex}/.match(d) }
end
missingpkgdeps.delete_if { |d| File.read(pkg_file).include?("# depends_on '#{d}' # R (optional)") }
pkgdeps.delete_if { |d| File.read(pkg_file).include?("# depends_on '#{d}' # R (optional)") }
puts "\nPackage #{pkg} has runtime library dependencies on these packages:".lightblue
pkgdeps.each do |i|
puts " depends_on '#{i}' # R".lightgreen
end
# Get existing package deps entries so we can add to and sort as
# necessary.
pkgdepsblock = []
pkgdepsblock += File.foreach(pkg_file).grep(/ depends_on '| # depends_on '/)
unless missingpkgdeps.empty?
puts "\nPackage file #{pkg}.rb is missing these runtime library dependencies:".orange
puts " depends_on '#{missingpkgdeps.join("' # R\n depends_on '")}' # R".orange
pkgdepsblock += missingpkgdeps.map { |add_dep| " depends_on '#{add_dep}' # R" }
end
pkgdepsblock.uniq!
pkgdepsblock = pkgdepsblock.sort_by { |dep| dep.split('depends_on ')[1] }
puts "\n Adding to or replacing deps block in package..."
# First remove all dependencies.
system "sed -i '/ depends_on /d' #{pkg_file}"
system "sed -i '/^ # depends_on /d' #{pkg_file}"
# Now add back our sorted dependencies.
gawk_cmd = "gawk -i inplace -v dep=\"#{pkgdepsblock.join('QQQQQ')}\" 'FNR==NR{ if (/})/) p=NR; next} 1; FNR==p{ print \"\\n\" dep }' #{pkg_file} #{pkg_file}"
system(gawk_cmd)
# The first added line has two dependencies without a newline
# separating them.
system "sed -i 's/RQQQQQ/R\\n/' #{pkg_file}"
system "sed -i 's/QQQQQ//g' #{pkg_file}"
# Check for and delete old runtime dependencies.
# Its unsafe to do this with other dependencies, because the packager might know something we don't.
lines_to_delete = {}
File.readlines(pkg_file).each_with_index do |line, line_number|
# Find all the explicitly marked runtime dependencies.
dep = line.match(/ depends_on '(.*)' # R/)
# Basically just a nil check, but this way we avoid matching twice.
next unless dep
# Skip unless the runtime dependency in the package does not match the runtime dependencies we've found.
next unless pkgdeps.none?(dep[1])
# Skip if we're dealing with privileged deps.
next if privileged_deps.include?(dep[1])
# Record the line content as the key and the line number (incremented by one because the index starts at 0) as the value.
lines_to_delete[line] = line_number + 1
end
# Clean with rubocop.
system "rubocop -c #{@rubocop_config} -A #{pkg_file}"
(FileUtils.cp pkg_file, "#{CREW_LOCAL_REPO_ROOT}/packages/#{pkg}.rb" if lines_to_delete.empty?) unless CREW_LOCAL_REPO_ROOT.to_s.empty?
# Leave if there aren't any old runtime dependencies.
return if lines_to_delete.empty?
puts "\nPackage file #{pkg}.rb has these outdated runtime library dependencies:".lightpurple
puts lines_to_delete.keys
system("gawk -i inplace 'NR != #{lines_to_delete.values.join(' && NR != ')}' #{pkg_file}")
# Clean with rubocop.
system "rubocop -c #{@rubocop_config} -A #{pkg_file}"
FileUtils.cp pkg_file, "#{CREW_LOCAL_REPO_ROOT}/packages/#{pkg}.rb" unless CREW_LOCAL_REPO_ROOT.to_s.empty?
# Write the changed dependencies to the package file.
write_deps(pkg_file, pkgdeps, @pkg)
end
ARGV.each do |package|