Refactor upload (#11821)

* Add crew lockfile implementation.

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

* Refactor crew upload, and add usage of lockfiles.

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

* Adjust crew and add test.

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

* Add more lightred.

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

---------

Signed-off-by: Satadru Pramanik <satadru@gmail.com>

bin/crew

@@ -32,6 +32,7 @@ require_relative '../commands/search'
 require_relative '../commands/sysinfo'
 require_relative '../commands/whatprovides'
 require_relative '../lib/const'
+require_relative '../lib/crew_lockfile'
 require_gem 'pry-byebug' if CREW_DEBUG
 # Debugging starts at the following breakpoint when enabled via crew
 # being started with a --debug flag.
@@ -203,8 +204,14 @@ def cache_build
       FileUtils.mv build_cachefile, "#{build_cachefile}.bak", force: true if File.file?(build_cachefile)
       FileUtils.mv "#{build_cachefile}.sha256", "#{build_cachefile}.sha256.bak", force: true if File.file?("#{build_cachefile}.sha256")
       Dir.chdir(CREW_BREW_DIR) do
+        @build_cachefile_lockfile = CrewLockfile.new "#{build_cachefile}.lock"
+        begin
+          @build_cachefile_lockfile.lock
           system "tar c#{@verbose} #{pkg_build_dirname} \
           | nice -n 20 zstd -c  --ultra --fast -f -o #{build_cachefile} -"
+        ensure
+          @build_cachefile_lockfile.unlock
+        end
       end
     end
     system "sha256sum #{build_cachefile} > #{build_cachefile}.sha256"
@@ -546,6 +553,13 @@ def download
         end
         puts "Git cachefile is #{cachefile}".orange if CREW_VERBOSE
         if File.file?(cachefile) && File.file?("#{cachefile}.sha256")
+          while File.file?("#{cachefile}.lock")
+            @cache_wait_timer = 0
+            puts "Waited #{@cache_wait_timer}s for #{cachefile} generation..."
+            sleep 1
+            @cache_wait_timer += 1
+            abort "Cachefile not available after #{@cache_wait_timer} seconds." if @cache_wait_timer > 300
+          end
           if Dir.chdir CREW_CACHE_DIR do
                system "sha256sum -c #{cachefile}.sha256"
              end
@@ -586,10 +600,16 @@ def download
         Dir.chdir @extract_dir do
           # Do not use --exclude-vcs to exclude .git
           # because some builds will use that information.
+          @git_cachefile_lockfile = CrewLockfile.new "#{cachefile}.lock"
+          begin
+            @git_cachefile_lockfile.lock
             system "tar c#{@verbose} \
               $(find -mindepth 1 -maxdepth 1 -printf '%P\n') | \
               nice -n 20 zstd -c -T0 --ultra -20 - >  \
               #{cachefile}"
+          ensure
+            @git_cachefile_lockfile.unlock
+          end
         end
         system 'sha256sum', cachefile, out: "#{cachefile}.sha256"
         puts 'Git repo cached.'.lightgreen
@@ -614,6 +634,13 @@ def unpack(meta)
       @extract_dir = `tar -Izstd --exclude='./*/*' -tf #{build_cachefile} | cut -d '/' -f 1 | LC_ALL=C sort -u`.chomp
     else
       @pkg.cached_build = false
+      while File.file?("#{meta[:filename]}.lock")
+        @archive_wait_timer = 0
+        puts "Waited #{@archive_wait_timer}s for #{meta[:filename]} to be available..."
+        sleep 1
+        @archive_wait_timer += 1
+        abort "#{meta[:filename]} not available after #{@archive_wait_timer} seconds." if @archive_wait_timer > 300
+      end
       case File.basename meta[:filename]
       when /\.zip$/i
         puts "Unpacking archive using 'unzip', this may take a while..."
@@ -1357,17 +1384,6 @@ def build_package(crew_archive_dest)
   meta = download
   target_dir = unpack meta
 
-  # Source ~/.bashrc before every build, and set PATH.
-  # From https://stackoverflow.com/a/12303443
-  env = `bash -c ". ~/.bashrc && env"`
-  env.split("\n").each do |line|
-    key, value = line.split('=', 2)
-    if key == 'PATH'
-      ENV[key] = value
-    else
-      ENV[key] ||= value unless value.nil? || value.empty?
-    end
-  end
   # Build from source and place binaries in CREW_DEST_DIR.
   build_and_preconfigure target_dir
 
@@ -1403,7 +1419,13 @@ def archive_package(crew_archive_dest)
     puts 'Using xz to compress package. This may take some time.'.lightblue
     pkg_name = "#{@pkg.name}-#{@pkg.version}-chromeos-#{@device[:architecture]}.tar.xz"
     Dir.chdir CREW_DEST_DIR do
+      @pkg_name_lockfile = CrewLockfile.new "#{crew_archive_dest}/#{pkg_name}.lock"
+      begin
+        @pkg_name_lockfile.lock
         system "tar c#{@verbose}Jf #{crew_archive_dest}/#{pkg_name} *"
+      ensure
+        @pkg_name_lockfile.unlock
+      end
     end
   else
     puts 'Using zstd to compress package. This may take some time.'.lightblue
@@ -1413,7 +1435,13 @@ def archive_package(crew_archive_dest)
       # decompression speed over compression speed.
       # See https://lists.archlinux.org/pipermail/arch-dev-public/2019-March/029542.html
       # Use nice so that user can (possibly) do other things during compression.
+      @pkg_name_lockfile = CrewLockfile.new "#{crew_archive_dest}/#{pkg_name}.lock"
+      begin
+        @pkg_name_lockfile.lock
         system "tar c#{@verbose} * | nice -n 20 zstd -c -T0 --ultra -20 - > #{crew_archive_dest}/#{pkg_name}"
+      ensure
+        @pkg_name_lockfile.unlock
+      end
     end
   end
   system "sha256sum #{crew_archive_dest}/#{pkg_name} > #{crew_archive_dest}/#{pkg_name}.sha256"
@@ -1436,6 +1464,41 @@ def archive_package(crew_archive_dest)
 end
 
 def upload(pkg_name = nil, pkg_version = nil, gitlab_token = nil, gitlab_token_username = nil, binary_compression = nil)
+  # Architecture independent:
+  # 1. Abort early if package manifests exist but are empty, as this
+  #    likely indicates a failed build.
+  # 2. Check for binary_compression value in file.
+  # 2a. If missing, determine binary_compression value:
+  #     Always set binary_compression to '.gem' for Ruby gems, otherwise
+  #     assume a default of 'tar.zst'
+  # 2b. Add missing binary_compression value to file.
+  #
+  # 3. Set sha256 hash variables for all architectures to nil.
+  #
+  # Per Architecture:
+  # 3a. Pull sha256 values for binary from package file, if there.
+  # 3b. Get gitlab URL for uploaded binary.
+  # 3c. Check to see if an existing upload exists.
+  # 3d. Check for local binary.
+  # 3e. If a local binary doesn't exist, but a remote binary exists,
+  #     download it.
+  # 3f. Skip architecture if a local or remote binary doesn't exist.
+  # 3g. If a local binary exists, check to make sure it isn't currently
+  #     being written to.
+  # 3h. Figure out sha256 for local binary. (local_sha256)
+  # 3i. If an existing upload exists, check its hash. (remote_sha256)
+  # 3j. If the hashes are different, then ask if the new binary
+  #     should be uploaded. (Default to no.)
+  # 3k. If the hashes are the same, don't bother uploading.
+  # 3l. If an existing upload does not exist, decide to upload.
+  # 4.  Upload.
+  # 5. Generate new or replacement binary_sha256 block and add to
+  #    package file.
+  # 6. If run architecture specfic manifests for package are missing,
+  #    attempt to install the package so the manifest files for the
+  #    currently running architecture are saved locally. (This is used
+  #    by build workflows to make sure updated manifests get
+  #    uploaded.)
   abort "\nPackage to be uploaded was not specified.\n".lightred if pkg_name.nil?
   abort "\nGITLAB_TOKEN environment variable not set.\n".lightred if gitlab_token.nil?
   abort "\nGITLAB_TOKEN_USERNAME environment variable not set.\n".lightred if gitlab_token_username.nil?
@@ -1444,37 +1507,21 @@ def upload(pkg_name = nil, pkg_version = nil, gitlab_token = nil, gitlab_token_u
   packages.strip!
 
   [packages].each do |package|
-    binary_compression_not_in_file = binary_compression.nil?
     pkg_file = "#{CREW_LOCAL_REPO_ROOT}/packages/#{package}.rb"
-    binary_sha256_hash = { armv7l: nil, i686: nil, x86_64: nil }
+    # 1. Abort early if package manifests exist but are empty, as this
-    # The following is used to figure out where a non-standard
+    #    likely indicates a failed build.
-    # binary_sha256 section might be, such as in a gcc_lib file.
+    abort "#{package} has an empty manifest. Something probably went wrong with the build.".lightred if File.empty?("#{CREW_LOCAL_REPO_ROOT}/manifest/#{ARCH}/#{@pkg.name.chr}/#{@pkg.name}.filelist")
-    starting_binary_sha256_hash = { armv7l: nil, i686: nil, x86_64: nil }
-    binary_compression = 'gem' if @pkg.superclass.to_s == 'RUBY'
-    %w[x86_64 i686 armv7l].each do |arch|
-      # Load existing binary_sha256 hash from the package file.
-      binary_sha256_hash[arch.to_sym] = @pkg.binary_sha256[arch.to_sym] if @pkg.binary_sha256&.key?(arch.to_sym)
-      starting_binary_sha256_hash[arch.to_sym] = @pkg.binary_sha256[arch.to_sym] if @pkg.binary_sha256&.key?(arch.to_sym)
-      release_dir = "#{CREW_LOCAL_REPO_ROOT}/release/#{arch}"
-      new_tarfile = if binary_compression.nil?
-                      Dir["#{release_dir}/#{package}-#{pkg_version}-chromeos-#{arch}.{tar.xz,tar.zst}"].max_by { |f| File.mtime(f) }
-                    else
-                      "#{release_dir}/#{package}-#{pkg_version}-chromeos-#{arch}.#{binary_compression}"
-                    end
-      if new_tarfile.nil? || !File.file?(new_tarfile)
-        puts "#{release_dir}/#{package}-#{pkg_version}-chromeos-#{arch}.#{binary_compression.nil? ? '(tar.xz|tar.zst)' : binary_compression} not found.\n".lightred
-        next arch
-      end
-
-      # If the filelist exists and is empty, the build was probably supposed to have files.
-      abort "#{package} has an empty manifest. Something probably went wrong with the build.".lightred if File.empty?("#{CREW_LOCAL_REPO_ROOT}/manifest/#{arch}/#{@pkg.name.chr}/#{@pkg.name}.filelist")
 
+    # 2. Check for binary_compression value in file.
+    binary_compression_not_in_file = binary_compression.nil?
     if binary_compression_not_in_file
-        ext = File.extname(new_tarfile)
+      # 2a. If missing, determine binary_compression value:
-        binary_compression = @pkg.superclass.to_s == 'RUBY' ? 'gem' : "tar#{ext}"
+      #     Always set binary_compression to '.gem' for Ruby gems, otherwise
+      #     assume a default of 'tar.zst'
+      binary_compression = @pkg.superclass.to_s == 'RUBY' ? 'gem' : 'tar.zst'
       binary_compression_line = "  binary_compression '#{binary_compression}'"
+      # 2b. Add missing binary_compression value to file.
       puts "Setting binary compression in #{pkg_file} to '#{binary_compression}'..."
-        # Add binary compression setting, and add the line if it doesn't exist.
       file = File.read(pkg_file)
       bc_re = /^\ \ binary_compression.*/
       source_re = /^\ \ source_sha256.*/
@@ -1495,48 +1542,119 @@ def upload(pkg_name = nil, pkg_version = nil, gitlab_token = nil, gitlab_token_u
         puts "Unable to tell where to add \"#{binary_compression_line}\" to #{pkg_file}. Please add it and manually.".lightblue
       end
     end
-      puts "Package: #{package}, Arch: #{arch}".yellow
+
+    # 3. Set sha256 hash variables for all architectures to nil.
+    binary_sha256_hash = { armv7l: nil, i686: nil, x86_64: nil }
+    # The following is used to figure out where a non-standard
+    # binary_sha256 section might be, such as in a gcc_lib file.
+    starting_binary_sha256_hash = { armv7l: nil, i686: nil, x86_64: nil }
+
+    %w[x86_64 i686 armv7l].each do |arch|
+      upload_binary = false
+      local_binary = false
+      puts "Processing package: #{package}, Arch: #{arch}".yellow
       puts
-      puts "\e[1A\e[KGenerating sha256sum ...\r".orange
+      # 3a. Pull sha256 values for binary from package file, if there.
-      new_sha256 = Digest::SHA256.hexdigest(File.read(new_tarfile))
+      binary_sha256_hash[arch.to_sym] = @pkg.binary_sha256[arch.to_sym] if @pkg.binary_sha256&.key?(arch.to_sym)
-      puts "Uploading #{new_tarfile} ...".orange if CREW_VERBOSE
+      starting_binary_sha256_hash[arch.to_sym] = @pkg.binary_sha256[arch.to_sym] if @pkg.binary_sha256&.key?(arch.to_sym)
-      noname = new_tarfile.split("#{package}-").last
+      release_dir = "#{CREW_LOCAL_REPO_ROOT}/release/#{arch}"
+      local_tarfile = "#{release_dir}/#{package}-#{pkg_version}-chromeos-#{arch}.#{binary_compression}"
+
+      # 3b. Get gitlab URL for uploaded binary.
+      noname = local_tarfile.split("#{package}-").last
       new_version = noname.split('-chromeos').first
-      new_url = "#{CREW_GITLAB_PKG_REPO}/generic/#{package}/#{new_version}_#{arch}/#{new_tarfile}".gsub("#{release_dir}/", '')
+      new_url = "#{CREW_GITLAB_PKG_REPO}/generic/#{package}/#{new_version}_#{arch}/#{local_tarfile}".gsub("#{release_dir}/", '')
-      token_label = gitlab_token.split('-').first == 'glpat' ? 'PRIVATE-TOKEN' : 'DEPLOY-TOKEN'
 
+      # 3c. Check to see if an existing upload exists.
       puts "\e[1A\e[KChecking for existing upload ...\r".orange
-      if `curl -fsI #{new_url}`.lines.first.split[1] == '200'
+      remote_binary = `curl -fsI #{new_url}`.lines.first.split[1] == '200'
-        puts "\n#{File.basename(new_tarfile)} has already been uploaded.\nPlease change the #{package} package version from #{new_version} and try again.\n".lightred
+      puts "\e[1A\e[KUpload exists.\n".green
-        if @opt_force || Package.agree_default_no('Do you want to overwrite the existing upload instead')
+
-          puts "\e[1A\e[KOverwriting existing upload...\r".orange
+      # 3d. Check for local binary.
-          crewlog "#{arch} = #{new_sha256}"
+      if local_tarfile.nil? || !File.file?(local_tarfile)
-          binary_sha256_hash[arch.to_sym] = new_sha256
+        puts "\e[1A\e[K#{local_tarfile} not found.\n\r".lightred
+        puts "\e[1A\e[KSkipping upload for #{arch}.\n\r".lightred
+        local_binary = false
+        next arch unless remote_binary
+        # 3e. If a local binary doesn't exist, but a remote binary exists,
+        #     download it.
+        system "curl -Ls #{new_url} > #{local_tarfile}"
       else
-          puts "\e[1A\e[KWill NOT overwite the existing upload. Determining sha256 of already uploaded file...\r".orange
+        local_binary = true
-          upstream_sha256 = `curl -Ls #{new_url} | sha256sum`.chomp.split.first
+        puts "\e[1A\e[KLocal package binary exists.\n".green
-          crewlog "#{arch} = #{upstream_sha256}"
-          binary_sha256_hash[arch.to_sym] = upstream_sha256
-          next arch
-        end
-      else
-        crewlog "#{arch} = #{new_sha256}"
-        binary_sha256_hash[arch.to_sym] = new_sha256
       end
 
-      puts "curl -# --header \"#{token_label}: #{gitlab_token}\" --upload-file \"#{new_tarfile}\" \"#{new_url}\" | cat" if CREW_VERBOSE
+      # 3f. Skip architecture if a local or remote binary doesn't exist.
+      next arch unless local_binary || remote_binary
+
+      puts "\e[1A\e[KGenerating sha256sums ...\r".orange
+      # 3g. If a local binary exists, check to make sure it isn't currently
+      #     being written to.
+      while File.file?("#{local_tarfile}.lock")
+        @pkg_binary_wait_timer = 0
+        puts "Waited #{@pkg_binary_wait_timer}s for #{local_tarfile} to be available..."
+        sleep 1
+        @pkg_binary_wait_timer += 1
+        abort "#{local_tarfile} not available after #{@pkg_binary_wait_timer} seconds." if @pkg_binary_wait_timer > 300
+      end
+      # At this point we either have a local build, or a downloaded
+      # binary from a prior build.
+      # 3h. Figure out sha256 for local binary. (local_sha256)
+      local_sha256 = Digest::SHA256.hexdigest(File.read(local_tarfile))
+
+      # 3i. If an existing upload exists, check its hash. (remote_sha256)
+      if remote_binary
+        remote_sha256 = if local_binary
+                          `curl -Ls #{new_url} | sha256sum`.chomp.split.first
+                        else
+                          local_sha256
+                        end
+      end
+
+      if remote_binary
+        if local_binary
+          if remote_sha256 == local_sha256
+            # 3k. If the hashes are the same, don't bother uploading.
+            puts "\e[1A\e[KThis build of #{File.basename(local_tarfile)} has already been uploaded.\n".lightred
+            crewlog "#{arch} = #{local_sha256}"
+            binary_sha256_hash[arch.to_sym] = local_sha256
+          elsif @opt_force || Package.agree_default_no('Do you want to overwrite the existing upload')
+            # 3j. If the hashes are different, then ask if the new binary
+            #     should be uploaded. (Default to no.)
+            puts "\e[1A\e[KOverwriting existing upload...\r".orange
+            crewlog "#{arch} = #{local_sha256}"
+            binary_sha256_hash[arch.to_sym] = local_sha256
+            upload_binary = true
+          else
+            puts "\e[1A\e[K\nAnother build of #{File.basename(local_tarfile)} with sha256 #{remote_sha256} has already been uploaded.\n".lightred
+            crewlog "#{arch} = #{remote_sha256}"
+            binary_sha256_hash[arch.to_sym] = remote_sha256
+          end
+        end
+      else
+        # 3l. If an existing upload does not exist, decide to upload.
+        upload_binary = true if local_binary
+        crewlog "#{arch} = #{local_sha256}"
+        binary_sha256_hash[arch.to_sym] = local_sha256
+      end
+
+      next unless upload_binary
+      # 4.  Upload.
+      puts "Uploading #{local_tarfile} ...".orange if CREW_VERBOSE
+      token_label = gitlab_token.split('-').first == 'glpat' ? 'PRIVATE-TOKEN' : 'DEPLOY-TOKEN'
+      puts "curl -# --header \"#{token_label}: #{gitlab_token}\" --upload-file \"#{local_tarfile}\" \"#{new_url}\" | cat" if CREW_VERBOSE
       puts "\e[1A\e[KUploading...\r".orange
-      output = `curl -# --header "#{token_label}: #{gitlab_token}" --upload-file "#{new_tarfile}" "#{new_url}" | cat`.chomp
+      output = `curl -# --header "#{token_label}: #{gitlab_token}" --upload-file "#{local_tarfile}" "#{new_url}" | cat`.chomp
       puts "\e[1A\e[KChecking upload...\r".orange
       if output.include?('201 Created')
         puts "curl -Ls #{new_url} | sha256sum" if CREW_VERBOSE
-        upstream_sha256 = `curl -Ls #{new_url} | sha256sum`.chomp.split.first
+        remote_sha256 = `curl -Ls #{new_url} | sha256sum`.chomp.split.first
-        if upstream_sha256 == new_sha256
+        if remote_sha256 == local_sha256
           puts "#{output}\n".lightgreen
         else
           if CREW_VERBOSE
-            puts "expected sha256 hash=#{new_sha256}"
+            puts "expected sha256 hash=#{local_sha256}"
-            puts "upstream sha256 hash=#{upstream_sha256}"
+            puts "remote sha256 hash=#{remote_sha256}"
           end
           puts "#{output}. Checksum mismatch. Skipping binary_sha256 update in #{pkg_file}...".lightred
           next
@@ -1548,7 +1666,8 @@ def upload(pkg_name = nil, pkg_version = nil, gitlab_token = nil, gitlab_token_u
       end
     end
 
-    # Generate new or replacement binary_sha256 block.
+    # 5. Generate new or replacement binary_sha256 block and add to
+    # package file.
     puts "\e[1A\e[KGenerating binary_sha256 block for package file...\r".orange
     binary_sha256_block = ''
     binary_sha256_block << "\n  binary_sha256({\n"
@@ -1601,6 +1720,12 @@ def upload(pkg_name = nil, pkg_version = nil, gitlab_token = nil, gitlab_token_u
         end
       end
     end
+    # 6. If run architecture specfic manifests for package are missing,
+    #    attempt to install the package so the manifest files for the
+    #    currently running architecture are saved locally. (This is used
+    #    by build workflows to make sure updated manifests get
+    #    uploaded.)
+    system "crew install #{@pkg.name}", exception: false unless File.exist?("#{CREW_LOCAL_REPO_ROOT}/manifest/#{ARCH}/#{@pkg.name.chr}/#{@pkg.name}.filelist")
     puts "\e[1A\e[K🎉 Uploads complete for #{package}. 🎉\r\n".lightgreen
   end
 end

lib/const.rb

@@ -3,7 +3,7 @@
 require 'etc'
 
 OLD_CREW_VERSION ||= defined?(CREW_VERSION) ? CREW_VERSION : '1.0'
-CREW_VERSION ||= '1.58.8' unless defined?(CREW_VERSION) && CREW_VERSION == OLD_CREW_VERSION
+CREW_VERSION ||= '1.58.9' unless defined?(CREW_VERSION) && CREW_VERSION == OLD_CREW_VERSION
 
 # Kernel architecture.
 KERN_ARCH ||= Etc.uname[:machine]

lib/crew_lockfile.rb

@@ -0,0 +1,569 @@
+# Based upon the fork of https://github.com/ahoward/lockfile at
+# https://github.com/mikisvaz/lockfile/tree/master and the PRs at
+# https://github.com/ahoward/lockfile/pulls
+# Usage:
+#    require_relative '../lib/crew_lockfile'
+#    lockfile = CrewLockfile.new 'file.lock'
+#    begin
+#      lockfile.lock
+#      p 42
+#    ensure
+#      lockfile.unlock
+#    end
+
+unless defined?($__crew_lockfile__) || defined?(CrewLockfile)
+
+  require 'socket'
+  require 'timeout'
+  require 'fileutils'
+  require_relative '../lib/color'
+
+  class CrewLockfile
+    VERSION = '2.1.9'
+    def self.version = CrewLockfile::VERSION
+    def version = CrewLockfile::VERSION
+
+    def self.description
+      'a ruby library for creating perfect and NFS safe lockfiles'
+    end
+
+    class LockError < StandardError; end
+    class StolenLockError < LockError; end
+    class StackingLockError < LockError; end
+    class StatLockError < LockError; end
+    class MaxTriesLockError < LockError; end
+    class TimeoutLockError < LockError; end
+    class NFSLockError < LockError; end
+    class UnLockError < LockError; end
+
+    class SleepCycle < Array
+      attr_reader :min, :max, :range, :inc
+
+      def initialize(min, max, inc)
+        super(min)
+        super(max)
+        super(inc)
+        @min = Float(min)
+        @max = Float(max)
+        @inc = Float(inc)
+        @range = @max - @min
+        raise RangeError, "max(#{@max}) <= min(#{@min})" if @max <= @min
+        raise RangeError, "inc(#{@inc}) > range(#{@range})" if @inc > @range
+        raise RangeError, "inc(#{@inc}) <= 0" if @inc <= 0
+        raise RangeError, "range(#{@range}) <= 0" if @range <= 0
+        s = @min
+        push(s) and s += @inc while s <= @max
+        self[-1] = @max if self[-1] < @max
+        reset
+      end
+
+      def next
+        ret = self[@idx]
+        @idx = ((@idx + 1) % size)
+        ret
+      end
+
+      def reset
+        @idx = 0
+      end
+    end
+
+    HOSTNAME = Socket.gethostname
+
+    DEFAULT_RETRIES          = nil    # maximum number of attempts
+    DEFAULT_TIMEOUT          = nil    # the longest we will try
+    DEFAULT_MAX_AGE          = 3600   # lockfiles older than this are stale
+    DEFAULT_SLEEP_INC        = 2      # sleep cycle is this much longer each time
+    DEFAULT_MIN_SLEEP        = 2      # shortest sleep time
+    DEFAULT_MAX_SLEEP        = 32     # longest sleep time
+    DEFAULT_SUSPEND          = 1800   # iff we steal a lock wait this long before we go on
+    DEFAULT_REFRESH          = 8      # how often we touch/validate the lock
+    DEFAULT_DONT_CLEAN       = false  # iff we leave lock files lying around
+    DEFAULT_POLL_RETRIES     = 16     # this many polls makes one 'try'
+    DEFAULT_POLL_MAX_SLEEP   = 0.08   # the longest we'll sleep between polls
+    DEFAULT_DONT_SWEEP       = false  # if we cleanup after other process on our host
+    DEFAULT_DONT_USE_LOCK_ID = false  # if we dump lock info into lockfile
+
+    DEFAULT_DEBUG = ENV['LOCKFILE_DEBUG'] || false
+
+    class << CrewLockfile
+      attr_accessor :retries, :max_age, :sleep_inc, :min_sleep, :max_sleep, :suspend, :timeout, :refresh, :debug, :dont_clean, :poll_retries, :poll_max_sleep, :dont_sweep, :dont_use_lock_id
+
+      def init
+        @retries          = DEFAULT_RETRIES
+        @max_age          = DEFAULT_MAX_AGE
+        @sleep_inc        = DEFAULT_SLEEP_INC
+        @min_sleep        = DEFAULT_MIN_SLEEP
+        @max_sleep        = DEFAULT_MAX_SLEEP
+        @suspend          = DEFAULT_SUSPEND
+        @timeout          = DEFAULT_TIMEOUT
+        @refresh          = DEFAULT_REFRESH
+        @dont_clean       = DEFAULT_DONT_CLEAN
+        @poll_retries     = DEFAULT_POLL_RETRIES
+        @poll_max_sleep   = DEFAULT_POLL_MAX_SLEEP
+        @dont_sweep       = DEFAULT_DONT_SWEEP
+        @dont_use_lock_id = DEFAULT_DONT_USE_LOCK_ID
+
+        @debug = DEFAULT_DEBUG
+
+        $stdout.sync = true if @debug
+        $stderr.sync = true if @debug
+      end
+    end
+
+    CrewLockfile.init
+
+    attr_reader :klass, :path, :opts, :locked, :thief, :refresher, :dirname, :basename, :clean, :retries, :max_age, :sleep_inc, :min_sleep, :max_sleep, :suspend, :refresh, :timeout, :dont_clean, :poll_retries, :poll_max_sleep, :dont_sweep, :dont_use_lock_id
+
+    attr_accessor :debug
+
+    alias thief? thief
+    alias locked? locked
+    alias debug? debug
+
+    def self.create(path, *options, &)
+      opts = {
+        'retries' => 0,
+        'min_sleep' => 0,
+        'max_sleep' => 1,
+        'sleep_inc' => 1,
+        'max_age' => nil,
+        'suspend' => 0,
+        'refresh' => nil,
+        'timeout' => nil,
+        'poll_retries' => 0,
+        'dont_clean' => true,
+        'dont_sweep' => false,
+        'dont_use_lock_id' => true
+      }
+      begin
+        new(path, opts).lock
+      rescue LockError
+        raise Errno::EEXIST, path
+      end
+      File.open(path, *options, &)
+    end
+
+    def self.finalizer_proc(file)
+      pid = Process.pid
+      lambda do |_id|
+        File.unlink file if Process.pid == pid
+      rescue StandardError
+        nil
+      end
+    end
+
+    def initialize(path, opts = {}, &block)
+      @klass = self.class
+      @path  = path
+      @opts  = opts
+
+      @retries          = getopt 'retries', @klass.retries
+      @max_age          = getopt 'max_age', @klass.max_age
+      @sleep_inc        = getopt 'sleep_inc', @klass.sleep_inc
+      @min_sleep        = getopt 'min_sleep', @klass.min_sleep
+      @max_sleep        = getopt 'max_sleep', @klass.max_sleep
+      @suspend          = getopt 'suspend', @klass.suspend
+      @timeout          = getopt 'timeout', @klass.timeout
+      @refresh          = getopt 'refresh', @klass.refresh
+      @dont_clean       = getopt 'dont_clean', @klass.dont_clean
+      @poll_retries     = getopt 'poll_retries', @klass.poll_retries
+      @poll_max_sleep   = getopt 'poll_max_sleep', @klass.poll_max_sleep
+      @dont_sweep       = getopt 'dont_sweep', @klass.dont_sweep
+      @dont_use_lock_id = getopt 'dont_use_lock_id', @klass.dont_use_lock_id
+      @debug            = getopt 'debug', @klass.debug
+
+      @semaphore = Mutex.new
+
+      @sleep_cycle = SleepCycle.new @min_sleep, @max_sleep, @sleep_inc
+
+      @clean    = @dont_clean ? nil : CrewLockfile.finalizer_proc(@path)
+
+      @dirname  = File.dirname @path
+      @basename = File.basename @path
+      @thief    = false
+      @locked   = false
+      @refresher = nil
+
+      lock(&block) if block
+    end
+
+    ##
+    # Executes the given block after acquiring the lock and
+    # ensures that the lock is relinquished afterwards.
+    #
+    def synchronize
+      raise ArgumentError, 'block must be given' unless block_given?
+      begin
+        lock
+        yield
+      ensure
+        unlock
+      end
+    end
+
+    def lock
+      raise StackingLockError, "<#{@path}> is locked!".lightred if @locked
+
+      sweep unless @dont_sweep
+
+      ret = nil
+
+      attempt do
+        @sleep_cycle.reset
+        create_tmplock do |f|
+          Timeout.timeout(@timeout) do
+            tmp_path = f.path
+            tmp_stat = f.lstat
+            n_retries = 0
+            trace { "attempting to lock <#{@path}>..." }
+            begin
+              i = 0
+              begin
+                trace { "polling attempt <#{i}>..." }
+                begin
+                  File.link tmp_path, @path
+                rescue Errno::ENOENT
+                  try_again!
+                end
+                lock_stat = File.lstat @path
+                raise StatLockError, 'stats do not agree'.lightred unless
+                  (tmp_stat.rdev == lock_stat.rdev) && (tmp_stat.ino == lock_stat.ino)
+                trace { "acquired lock <#{@path}>" }
+                @locked = true
+              rescue StandardError
+                i += 1
+                unless i >= @poll_retries
+                  t = [rand(@poll_max_sleep), @poll_max_sleep].min
+                  trace { "poll sleep <#{t}>..." }
+                  sleep t
+                  retry
+                end
+                raise
+              end
+            rescue StandardError
+              n_retries += 1
+              trace { "n_retries <#{n_retries}>" }
+              case validlock?
+              when true
+                raise MaxTriesLockError, "surpassed retries <#{@retries}>" if
+                  @retries && (n_retries >= @retries)
+                trace { 'found valid lock' }
+                sleeptime = @sleep_cycle.next
+                trace { "sleep <#{sleeptime}>..." }
+                sleep sleeptime
+              when false
+                trace { 'found invalid lock and removing' }
+                begin
+                  File.unlink @path
+                  @thief = true
+                  warn "<#{@path}> stolen by <#{Process.pid}> at <#{timestamp}>"
+                  trace { 'i am a thief!' }
+                rescue Errno::ENOENT => e
+                  warn(errmsg(e))
+                end
+                trace { "suspending <#{@suspend}>" }
+                sleep @suspend
+              when nil
+                raise MaxTriesLockError, "surpassed retries <#{@retries}>" if
+                  @retries && (n_retries >= @retries)
+              end
+              retry
+            end
+          end
+        rescue Timeout::Error
+          raise TimeoutLockError, "surpassed timeout <#{@timeout}>"
+        end
+
+        if block_given?
+          stolen = false
+          @refresher = (@refresh ? new_refresher : nil)
+          begin
+            begin
+              ret = yield @path
+            rescue StolenLockError
+              stolen = true
+              raise
+            end
+          ensure
+            begin
+              if @refresher&.status
+                begin
+                  @semaphore.synchronize do
+                    @refresher.kill
+                  end
+                rescue StandardError
+                  @refresher.kill
+                end
+              end
+              @refresher = nil
+            ensure
+              unlock unless stolen
+            end
+          end
+        else
+          @refresher = (@refresh ? new_refresher : nil)
+          ObjectSpace.define_finalizer self, @clean if @clean
+          ret = self
+        end
+      rescue Errno::ESTALE, Errno::EIO => e
+        raise(NFSLockError, errmsg(e))
+      end
+
+      return ret
+    end
+
+    def sweep
+      glob = File.join(@dirname, '.*lck')
+      paths = Dir[glob]
+      paths.each do |path|
+        basename = File.basename path
+        pat = /^\s*\.([^_]+)_([^_]+)/o
+        if pat.match(basename)
+          host = ::Regexp.last_match(1)
+          pid = ::Regexp.last_match(2)
+        else
+          next
+        end
+        host.gsub!(/^\.+|\.+$/, '')
+        quad = host.split '.'
+        host = quad.first
+        pat = /^\s*#{host}/i
+        if pat.match(HOSTNAME) && /^\s*\d+\s*$/.match(pid)
+          if alive?(pid)
+            trace { "process <#{pid}> on <#{host}> is still alive" }
+            trace { "ignoring <#{path}>" }
+          else
+            trace { "process <#{pid}> on <#{host}> is no longer alive" }
+            trace { "sweeping <#{path}>" }
+            FileUtils.rm_f path
+          end
+        else
+          trace { "ignoring <#{path}> generated by <#{host}>" }
+        end
+      rescue StandardError
+        next
+      end
+    rescue StandardError => e
+      warn(errmsg(e))
+    end
+
+    def alive?(pid)
+      pid = Integer(pid.to_s)
+      begin
+        Process.kill 0, pid
+        true
+      rescue Errno::ESRCH
+        false
+      end
+    end
+
+    def unlock
+      raise UnLockError, "<#{@path}> is not locked!" unless @locked
+
+      if @refresher&.status
+        begin
+          @semaphore.synchronize do
+            @refresher.kill
+          end
+        rescue StandardError
+          @refresher.kill
+        end
+      end
+
+      @refresher = nil
+
+      begin
+        File.unlink @path
+      rescue Errno::ENOENT
+        raise StolenLockError, @path
+      ensure
+        @thief = false
+        @locked = false
+        ObjectSpace.undefine_finalizer self if @clean
+      end
+    end
+
+    def new_refresher
+      Thread.new(Thread.current, @path, @refresh, @dont_use_lock_id) do |thread, path, refresh, dont_use_lock_id|
+        loop do
+          touch path
+          trace { "touched <#{path}> @ <#{Time.now.to_f}>" }
+          unless dont_use_lock_id
+            txt = nil
+            @semaphore.synchronize do
+              txt = File.read(path)
+            end
+            loaded = load_lock_id(txt)
+            trace { "loaded <\n#{loaded.inspect}\n>" }
+            raise unless loaded == @lock_id
+          end
+          sleep refresh
+        # rescue Exception => e
+        rescue StandardError => e
+          trace { errmsg e }
+          thread.raise StolenLockError
+          Thread.exit
+        end
+      end
+    end
+
+    def validlock?
+      if @max_age
+        begin
+          uncache @path
+        rescue StandardError
+          nil
+        end
+        begin
+          return((Time.now - File.stat(@path).mtime) < @max_age)
+        rescue Errno::ENOENT
+          return false
+        end
+      else
+        exist = File.exist?(@path)
+        return(exist ? true : nil)
+      end
+    end
+
+    def uncache(file)
+      refresh = nil
+      begin
+        is_a_file = file.is_a?(File)
+        path = (is_a_file ? file.path : file.to_s)
+        stat = (is_a_file ? file.stat : File.stat(file.to_s))
+        refresh = tmpnam(File.dirname(path))
+        File.link path, refresh
+        File.chmod stat.mode, path
+        File.utime stat.atime, stat.mtime, path
+      ensure
+        begin
+          File.unlink refresh if refresh
+        rescue Errno::ENOENT => e
+          warn(errmsg(e))
+        end
+      end
+    end
+
+    def create_tmplock
+      tmplock = tmpnam @dirname
+      begin
+        create(tmplock) do |f|
+          unless dont_use_lock_id
+            @lock_id = gen_lock_id
+            dumped = dump_lock_id
+            trace { "lock_id <\n#{@lock_id.inspect}\n>" }
+            f.write dumped
+            f.flush
+          end
+          yield f
+        end
+      ensure
+        if tmplock
+          begin
+            File.unlink tmplock
+          rescue Errno::ENOENT => e
+            warn(errmsg(e))
+          end
+        end
+      end
+    end
+
+    def gen_lock_id
+      { 'host' => HOSTNAME.to_s,
+        'pid' => Process.pid.to_s,
+        'ppid' => Process.ppid.to_s,
+        'time' => timestamp }
+    end
+
+    def timestamp
+      time = Time.now
+      usec = time.usec.to_s
+      usec << '0' while usec.size < 6
+      "#{time.strftime('%Y-%m-%d %H:%M:%S')}.#{usec}"
+    end
+
+    def dump_lock_id(lock_id = @lock_id)
+      @host, @pid, @ppid, @time = lock_id.values_at('host', 'pid', 'ppid', 'time')
+      format("host: %s\npid: %s\nppid: %s\ntime: %s\n", @host, @pid, @ppid, @time)
+    end
+
+    def load_lock_id(buf)
+      lock_id = {}
+      kv = /([^:]+):(.*)/o
+      buf.each_line do |line|
+        m = kv.match line
+        k = m[1]
+        v = m[2]
+        next unless m && k && v
+        lock_id[k.strip] = v.strip
+      end
+      lock_id
+    end
+
+    def tmpnam(dir, seed = File.basename($PROGRAM_NAME))
+      pid = Process.pid
+      time = Time.now
+      sec = time.to_i
+      usec = time.usec
+      format('%s%s.%s_%d_%s_%d_%d_%d.lck', dir, File::SEPARATOR, HOSTNAME, pid, seed, sec, usec, rand(sec))
+    end
+
+    def create(path)
+      umask = nil
+      f = nil
+      begin
+        umask = File.umask 0o22
+        f = File.open path, File::WRONLY | File::CREAT | File::EXCL, 0o644
+      ensure
+        File.umask umask if umask
+      end
+      return(block_given? ? begin; yield f; ensure; f.close; end : f)
+    end
+
+    def touch(path)
+      FileUtils.touch path
+    end
+
+    def getopt(key, default = nil)
+      [key, key.to_s, key.to_s.intern].each do |k|
+        return @opts[k] if @opts.key?(k)
+      end
+      return default
+    end
+
+    def to_str
+      @path
+    end
+    alias to_s to_str
+
+    def trace(tracemsg = nil)
+      warn(tracemsg || yield) if @debug
+    end
+
+    def errmsg(err)
+      format("%s (%s)\n%s\n", err.class, err.message, err.backtrace.join("\n")).lightred
+    end
+
+    def attempt
+      ret = nil
+      loop { break unless catch('attempt') { ret = yield } == 'try_again' }
+      ret
+    end
+
+    def try_again!
+      throw 'attempt', 'try_again'
+    end
+    alias again! try_again!
+
+    def give_up!
+      throw 'attempt', 'give_up'
+    end
+  end
+
+  def CrewLockfile(path, *options, &) # rubocop: disable Naming/MethodName
+    CrewLockfile.new(path, *options, &)
+  end
+
+  $__crew_lockfile__ = __FILE__
+end

tests/test_crew_lockfile.rb

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+require_relative '../lib/crew_lockfile'
+puts 'Before lock:'
+system 'ls test.lock'
+lockfile = CrewLockfile.new 'test.lock'
+lockfile.debug = true
+begin
+  lockfile.lock
+  puts 'During lock:'
+  system 'ls test.lock'
+  p 42
+ensure
+  lockfile.unlock
+end
+puts 'After lock:'
+system 'ls test.lock'