Merge pull request #3116 from alfredxing/incremental

lib/jekyll.rb

@@ -21,6 +21,7 @@ require 'time'
 require 'English'
 require 'pathname'
 require 'logger'
+require 'set'
 
 # 3rd party
 require 'safe_yaml/load'
@@ -48,6 +49,7 @@ module Jekyll
   autoload :Layout,              'jekyll/layout'
   autoload :LayoutReader,        'jekyll/layout_reader'
   autoload :LogAdapter,          'jekyll/log_adapter'
+  autoload :Metadata,            'jekyll/metadata'
   autoload :Page,                'jekyll/page'
   autoload :PluginManager,       'jekyll/plugin_manager'
   autoload :Post,                'jekyll/post'

lib/jekyll/cleaner.rb

@@ -13,6 +13,7 @@ module Jekyll
       # Cleans up the site's destination directory
       def cleanup!
         FileUtils.rm_rf(obsolete_files)
+        FileUtils.rm_rf(metadata_file) if @site.full_rebuild?
       end
 
       private
@@ -24,6 +25,13 @@ module Jekyll
         (existing_files - new_files - new_dirs + replaced_files).to_a
       end
 
+      # Private: The metadata file storing dependency tree and build history
+      #
+      # Returns an Array with the metdata file as the only item
+      def metadata_file
+        [site.metadata.metadata_file]
+      end
+
       # Private: The list of existing files, apart from those included in keep_files and hidden files.
       #
       # Returns a Set with the file paths

lib/jekyll/command.rb

@@ -58,6 +58,7 @@ module Jekyll
         c.option 'unpublished', '--unpublished', 'Render posts that were marked as unpublished'
         c.option 'quiet',   '-q', '--quiet', 'Silence output.'
         c.option 'verbose', '-V', '--verbose', 'Print verbose output.'
+        c.option 'full_rebuild', '-f', '--full-rebuild', 'Disable incremental rebuild.'
       end
 
     end

lib/jekyll/commands/build.rb

@@ -50,8 +50,10 @@ module Jekyll
         def build(site, options)
           source      = options['source']
           destination = options['destination']
+          full_build  = options['full_rebuild']
           Jekyll.logger.info "Source:", source
           Jekyll.logger.info "Destination:", destination
+          Jekyll.logger.info "Incremental build:", (full_build ? "disabled" : "enabled")
           Jekyll.logger.info "Generating..."
           process_site(site)
           Jekyll.logger.info "", "done."

lib/jekyll/commands/clean.rb

@@ -0,0 +1,42 @@
+module Jekyll
+  module Commands
+    class Clean < Command
+      class << self
+
+        def init_with_program(prog)
+          prog.command(:clean) do |c|
+            c.syntax 'clean [subcommand]'
+            c.description 'Clean the site (removes site output and metadata file) without building.'
+
+            c.action do |args, _|
+              Jekyll::Commands::Clean.process({})
+            end
+          end
+        end
+
+        def process(options)
+          options = configuration_from_options(options)
+          destination = options['destination']
+          metadata_file = File.join(options['source'], '.jekyll-metadata')
+
+          if File.directory? destination
+            Jekyll.logger.info "Cleaning #{destination}..."
+            FileUtils.rm_rf(destination)
+            Jekyll.logger.info "", "done."
+          else
+            Jekyll.logger.info "Nothing to do for #{destination}."
+          end
+
+          if File.file? metadata_file
+            Jekyll.logger.info "Removing #{metadata_file}..."
+            FileUtils.rm_rf(metadata_file)
+            Jekyll.logger.info "", "done."
+          else
+            Jekyll.logger.info "Nothing to do for #{metadata_file}."
+          end
+        end
+
+      end
+    end
+  end
+end

lib/jekyll/configuration.rb

@@ -22,6 +22,7 @@ module Jekyll
       'encoding'      => 'utf-8',
       'markdown_ext'  => 'markdown,mkdown,mkdn,mkd,md',
       'textile_ext'   => 'textile',
+      'full_rebuild'  => false,
 
       # Filtering Content
       'show_drafts'   => nil,

lib/jekyll/convertible.rb

@@ -168,6 +168,15 @@ module Jekyll
       true
     end
 
+    # Determine whether to regenerate the file based on metadata.
+    #
+    # Returns true if file needs to be regenerated
+    def regenerate?
+      asset_file? ||
+        data['regenerate'] ||
+        site.metadata.regenerate?(site.in_source_dir(relative_path))
+    end
+
     # Determine whether the file should be placed into layouts.
     #
     # Returns false if the document is an asset file.
@@ -207,6 +216,12 @@ module Jekyll
                                          info,
                                          File.join(site.config['layouts'], layout.name))
 
+        # Add layout to dependency tree
+        site.metadata.add_dependency(
+          site.in_source_dir(path),
+          site.in_source_dir(layout.path)
+        )
+
         if layout = layouts[layout.data["layout"]]
           if used.include?(layout)
             layout = nil # avoid recursive chain

lib/jekyll/document.rb

@@ -105,6 +105,13 @@ module Jekyll
       !(coffeescript_file? || yaml_file?)
     end
 
+    # Determine whether the document should be regenerated based on metadata.
+    #
+    # Returns true if the document needs to be regenerated.
+    def regenerate?
+      data['regenerate'] || site.metadata.regenerate?(path, write?)
+    end
+
     # Determine whether the file should be placed into layouts.
     #
     # Returns false if the document is either an asset file or a yaml file,

lib/jekyll/layout.rb

@@ -8,6 +8,9 @@ module Jekyll
     # Gets the name of this layout.
     attr_reader :name
 
+    # Gets the path to this layout.
+    attr_reader :path
+
     # Gets/Sets the extension of this layout.
     attr_accessor :ext
 
@@ -26,6 +29,7 @@ module Jekyll
       @site = site
       @base = base
       @name = name
+      @path = site.in_source_dir(base, name)
 
       self.data = {}
 

lib/jekyll/metadata.rb

@@ -0,0 +1,121 @@
+module Jekyll
+  class Metadata
+    attr_reader :site, :metadata, :cache
+
+    def initialize(site)
+      @site = site
+
+      # Read metadata from file
+      read_metadata
+
+      # Initialize cache to an empty hash
+      @cache = {}
+    end
+
+    # Add a path to the metadata
+    #
+    # Returns true, also on failure.
+    def add(path)
+      return true unless File.exist?(path)
+
+      metadata[path] = {
+        "mtime" => File.mtime(path),
+        "deps" => []
+      }
+      cache[path] = true
+    end
+
+    # Force a path to regenerate
+    #
+    # Returns true.
+    def force(path)
+      cache[path] = true
+    end
+
+    # Clear the metadata and cache
+    #
+    # Returns nothing
+    def clear
+      @metadata = {}
+      @cache = {}
+    end
+
+    # Checks if a path should be regenerated
+    #
+    # Returns a boolean.
+    def regenerate?(path, add = true)
+      return true if disabled?
+
+      # Check for path in cache
+      if cache.has_key? path
+        return cache[path]
+      end
+
+      # Check path that exists in metadata
+      data = metadata[path]
+      if data
+        data["deps"].each do |dependency|
+          if regenerate?(dependency)
+            return cache[dependency] = cache[path] = true
+          end
+        end
+        if data["mtime"].eql? File.mtime(path)
+          return cache[path] = false
+        else
+          return !add || add(path)
+        end
+      end
+
+      # Path does not exist in metadata, add it
+      return !add || add(path)
+    end
+
+    # Add a dependency of a path
+    #
+    # Returns nothing.
+    def add_dependency(path, dependency)
+      return if (metadata[path].nil? || @disabled)
+
+      metadata[path]["deps"] << dependency unless metadata[path]["deps"].include? dependency
+      regenerate? dependency
+    end
+
+    # Write the metadata to disk
+    #
+    # Returns nothing.
+    def write
+      File.open(metadata_file, 'w') do |f|
+        f.write(metadata.to_yaml)
+      end
+    end
+
+    # Produce the absolute path of the metadata file
+    #
+    # Returns the String path of the file.
+    def metadata_file
+      site.in_source_dir('.jekyll-metadata')
+    end
+
+    # Check if metadata has been disabled
+    #
+    # Returns a Boolean (true for disabled, false for enabled).
+    def disabled?
+      @disabled = site.full_rebuild? if @disabled.nil?
+      @disabled
+    end
+
+    private
+
+    # Read metadata from the metadata file, if no file is found,
+    # initialize with an empty hash
+    #
+    # Returns the read metadata.
+    def read_metadata
+      @metadata = if !disabled? && File.file?(metadata_file)
+        SafeYAML.load(File.read(metadata_file))
+      else
+        {}
+      end
+    end
+  end
+end

lib/jekyll/renderer.rb

@@ -138,6 +138,12 @@ module Jekyll
           File.join(site.config['layouts'], layout.name)
         )
 
+        # Add layout to dependency tree
+        site.metadata.add_dependency(
+          site.in_source_dir(document.path),
+          site.in_source_dir(layout.path)
+        ) if document.write?
+
         if layout = site.layouts[layout.data["layout"]]
           if used.include?(layout)
             layout = nil # avoid recursive chain

lib/jekyll/site.rb

@@ -11,6 +11,7 @@ module Jekyll
                   :gems, :plugin_manager
 
     attr_accessor :converters, :generators
+    attr_reader   :metadata
 
     # Public: Initialize a new Site.
     #
@@ -27,6 +28,9 @@ module Jekyll
       @source              = File.expand_path(config['source']).freeze
       @dest                = File.expand_path(config['destination']).freeze
 
+      # Build metadata
+      @metadata = Metadata.new(self)
+
       self.plugin_manager = Jekyll::PluginManager.new(self)
       self.plugins        = plugin_manager.plugins_path
 
@@ -289,13 +293,13 @@ module Jekyll
 
       collections.each do |label, collection|
         collection.docs.each do |document|
-          document.output = Jekyll::Renderer.new(self, document).run
+          document.output = Jekyll::Renderer.new(self, document).run if document.regenerate?
         end
       end
 
       payload = site_payload
       [posts, pages].flatten.each do |page_or_post|
-        page_or_post.render(layouts, payload)
+        page_or_post.render(layouts, payload) if page_or_post.regenerate?
       end
     rescue Errno::ENOENT => e
       # ignore missing layout dir
@@ -312,7 +316,10 @@ module Jekyll
     #
     # Returns nothing.
     def write
-      each_site_file { |item| item.write(dest) }
+      each_site_file { |item|
+        item.write(dest) if item.regenerate?
+      }
+      metadata.write unless full_rebuild?
     end
 
     # Construct a Hash of Posts indexed by the specified Post attribute.
@@ -483,6 +490,13 @@ module Jekyll
       @frontmatter_defaults ||= FrontmatterDefaults.new(self)
     end
 
+    # Whether to perform a full rebuild without metadata
+    #
+    # Returns a Boolean: true for a full rebuild, false for normal build
+    def full_rebuild?(override = {})
+      override['full_rebuild'] || config['full_rebuild']
+    end
+
     private
 
     def has_relative_page?

lib/jekyll/static_file.rb

@@ -67,6 +67,8 @@ module Jekyll
       true
     end
 
+    alias_method :regenerate?, :write?
+
     # Write the static file to the destination directory (if modified).
     #
     # dest - The String path to the destination dir.

lib/jekyll/tags/include.rb

@@ -105,13 +105,22 @@ eos
       end
 
       def render(context)
+        site = context.registers[:site]
         dir = resolved_includes_dir(context)
 
         file = render_variable(context) || @file
         validate_file_name(file)
 
         path = File.join(dir, file)
-        validate_path(path, dir, context.registers[:site].safe)
+        validate_path(path, dir, site.safe)
+
+        # Add include to dependency tree
+        if context.registers[:page] and context.registers[:page].has_key? "path"
+          site.metadata.add_dependency(
+            site.in_source_dir(context.registers[:page]["path"]),
+            path
+          )
+        end
 
         begin
           partial = Liquid::Template.parse(source(path, context))