Cache all the things!

docs/_docs/upgrading/3-to-4.md

@@ -19,9 +19,32 @@ If you're using Ruby >= 2.3.0, go ahead and fetch the latest version of Jekyll:
 gem update jekyll
 ```
 
----
 
-  *Insert sections here*
+### Template rendering
+
+We've slightly altered the way Jekyll parses and renders your various templates to improve
+the overall build times. Jekyll now parses a template once, caches it internally and then
+renders the parsed template multiple times as required by your pages and documents.
+
+The downside to this is that some of the community-authored plugins may not work as they
+previously used to.
+
+#### For Plugin-authors
+
+* If your plugin depends on the following code: `site.liquid_renderer.file(path).parse(content)`,
+note that the return value (`template`, an instance of *`Liquid::Template`*), from that line will
+always be the **same object** for a given `path`. <br/>
+The *`template`* instance is then rendered as previously, with respect to the `payload` passed to it.
+You'll therefore have to ensure that *`payload`* is not memoized or cached in your plugin instance.
+
+* If its a requirement that `template` you get from the above step *be different* at all times,
+you can invoke *`Liquid::Template`* directly:
+
+
+  ```diff
+  - template = site.liquid_renderer.file(path).parse(content)
+  + template = Liquid::Template.parse(content)
+  ```
 
 ---
 

lib/jekyll.rb

@@ -54,6 +54,7 @@ module Jekyll
   autoload :FrontmatterDefaults, "jekyll/frontmatter_defaults"
   autoload :Hooks,               "jekyll/hooks"
   autoload :Layout,              "jekyll/layout"
+  autoload :Cache,               "jekyll/cache"
   autoload :CollectionReader,    "jekyll/readers/collection_reader"
   autoload :DataReader,          "jekyll/readers/data_reader"
   autoload :LayoutReader,        "jekyll/readers/layout_reader"

lib/jekyll/cache.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require "pstore"
+
+module Jekyll
+  class Cache
+    extend Forwardable
+
+    # Get an existing named cache, or create a new one if none exists
+    #
+    # name - name of the cache
+    #
+    # Returns nothing.
+    # rubocop:disable Style/ClassVars
+    def initialize(name)
+      @@base_dir ||= File.expand_path(".jekyll-cache/Jekyll/Cache")
+      @@caches ||= {}
+      @cache = @@caches[name] ||= {}
+      @name = name
+      FileUtils.mkdir_p(path_to)
+    end
+
+    def self.clear
+      delete_cache_files
+      @@caches.each_value(&:clear)
+    end
+    # rubocop:enable Style/ClassVars
+
+    def clear
+      delete_cache_files
+      @cache.clear
+    end
+
+    def [](key)
+      return @cache[key] if @cache.key?(key)
+      path = path_to(hash(key))
+      if File.file?(path) && File.readable?(path)
+        @cache[key] = load(path)
+      else
+        raise
+      end
+    end
+
+    def getset(key)
+      return @cache[key] if @cache.key?(key)
+      path = path_to(hash(key))
+      if File.file?(path) && File.readable?(path)
+        value = load(path)
+      else
+        value = yield
+        dump(path, value)
+      end
+      @cache[key] = value
+    end
+
+    def []=(key, value)
+      @cache[key] = value
+      path = path_to(hash(key))
+      dump(path, value)
+    end
+
+    def delete(key)
+      @cache.delete(key)
+      path = path_to(hash(key))
+      File.delete(path)
+    end
+
+    def key?(key)
+      return true if @cache.key?(key)
+      path = path_to(hash(key))
+      File.file?(path) && File.readable?(path)
+    end
+
+    # rubocop:disable Style/ClassVars
+    def self.clear_if_config_changed(config)
+      config = config.inspect
+      cache = Jekyll::Cache.new "Jekyll::Cache"
+      unless cache.key?("config") && cache["config"] == config
+        delete_cache_files
+        @@caches = {}
+        cache = Jekyll::Cache.new "Jekyll::Cache"
+        cache["config"] = config
+      end
+    end
+    # rubocop:enable Style/ClassVars
+
+    private
+
+    def path_to(hash = nil)
+      @base_dir ||= File.join(@@base_dir, @name)
+      return @base_dir if hash.nil?
+      File.join(@base_dir, hash[0..1], hash[2..-1]).freeze
+    end
+
+    def hash(key)
+      Digest::SHA2.hexdigest(key).freeze
+    end
+
+    def delete_cache_files
+      FileUtils.rm_rf(path_to)
+    end
+
+    # rubocop:disable Security/MarshalLoad
+    def load(path)
+      cached_file = File.open(path, "rb")
+      value = Marshal.load(cached_file)
+      cached_file.close
+      value
+    end
+    # rubocop:enable Security/MarshalLoad
+
+    def dump(path, value)
+      dir, _file = File.split(path)
+      FileUtils.mkdir_p(dir)
+      cached_file = File.open(path, "wb")
+      Marshal.dump(value, cached_file)
+      cached_file.close
+    end
+
+    def self.delete_cache_files
+      FileUtils.rm_rf(@@base_dir)
+    end
+    private_class_method :delete_cache_files
+  end
+end

lib/jekyll/converters/markdown.rb

@@ -19,6 +19,8 @@ module Jekyll
           raise Errors::FatalException, "Bailing out; invalid Markdown processor."
         end
 
+        @cache = Jekyll::Cache.new("Jekyll::Converters::Markdown")
+
         @setup = true
       end
 
@@ -70,7 +72,9 @@ module Jekyll
 
       def convert(content)
         setup
-        @parser.convert(content)
+        @cache.getset(content) do
+          @parser.convert(content)
+        end
       end
 
       private

lib/jekyll/liquid_renderer.rb

@@ -53,7 +53,7 @@ module Jekyll
     private
 
     def filename_regex
-      @filename_regex ||= %r!\A(#{source_dir}/|#{theme_dir}/|\W*)(.*)!i
+      @filename_regex ||= %r!\A(#{source_dir}/|#{theme_dir}/|/*)(.*)!i
     end
 
     def new_profile_hash

lib/jekyll/liquid_renderer/file.rb

@@ -10,7 +10,9 @@ module Jekyll
 
       def parse(content)
         measure_time do
-          @template = Liquid::Template.parse(content, :line_numbers => true)
+          @template = Jekyll::Cache.new("Jekyll::LiquidRenderer::File").getset(content) do
+            Liquid::Template.parse(content, :line_numbers => true)
+          end
         end
 
         self
@@ -24,6 +26,7 @@ module Jekyll
         end
       end
 
+      # This method simply 'rethrows any error' before attempting to render the template.
       def render!(*args)
         measure_time do
           measure_bytes do

lib/jekyll/site.rb

@@ -100,6 +100,7 @@ module Jekyll
 
       raise ArgumentError, "limit_posts must be a non-negative number" if limit_posts.negative?
 
+      Jekyll::Cache.clear_if_config_changed config
       Jekyll::Hooks.trigger :site, :after_reset, self
     end
 

lib/jekyll/tags/highlight.rb

@@ -31,22 +31,25 @@ module Jekyll
       def render(context)
         prefix = context["highlighter_prefix"] || ""
         suffix = context["highlighter_suffix"] || ""
-        code = super.to_s.gsub(%r!\A(\n|\r)+|(\n|\r)+\z!, "")
+        key = super.to_s
+        cache.getset(key) do
+          code = key.gsub(%r!\A(\n|\r)+|(\n|\r)+\z!, "")
 
-        is_safe = !!context.registers[:site].safe
+          is_safe = !!context.registers[:site].safe
 
-        output =
-          case context.registers[:site].highlighter
-          when "pygments"
-            render_pygments(code, is_safe)
-          when "rouge"
-            render_rouge(code)
-          else
-            render_codehighlighter(code)
-          end
+          output =
+            case context.registers[:site].highlighter
+            when "pygments"
+              render_pygments(code, is_safe)
+            when "rouge"
+              render_rouge(code)
+            else
+              render_codehighlighter(code)
+            end
 
-        rendered_output = add_code_tag(output)
-        prefix + rendered_output + suffix
+          rendered_output = add_code_tag(output)
+          prefix + rendered_output + suffix
+        end
       end
 
       def sanitized_opts(opts, is_safe)
@@ -67,6 +70,10 @@ module Jekyll
 
       OPTIONS_REGEX = %r!(?:\w="[^"]*"|\w=\w|\w)+!
 
+      def cache
+        Jekyll::Cache.new("Jekyll::Tags::HighlightBlock")
+      end
+
       def parse_options(input)
         options = {}
         return options if input.empty?

lib/jekyll/tags/include.rb

@@ -90,13 +90,7 @@ module Jekyll
 
       # Render the variable if required
       def render_variable(context)
-        if @file =~ VARIABLE_SYNTAX
-          partial = context.registers[:site]
-            .liquid_renderer
-            .file("(variable)")
-            .parse(@file)
-          partial.render!(context)
-        end
+        Liquid::Template.parse(@file).render(context) if @file =~ VARIABLE_SYNTAX
       end
 
       def tag_includes_dirs(context)

lib/jekyll/tags/link.rb

@@ -17,9 +17,7 @@ module Jekyll
 
       def render(context)
         site = context.registers[:site]
-
-        liquid = site.liquid_renderer.file("(jekyll:link)")
-        relative_path = liquid.parse(@relative_path).render(context)
+        relative_path = Liquid::Template.parse(@relative_path).render(context)
 
         site.each_site_file do |item|
           return item.url if item.relative_path == relative_path

test/test_site.rb

@@ -76,6 +76,9 @@ class TestSite < JekyllUnitTest
       allow(File).to receive(:directory?).with(theme_dir("_sass")).and_return(true)
       allow(File).to receive(:directory?).with(theme_dir("_layouts")).and_return(true)
       allow(File).to receive(:directory?).with(theme_dir("_includes")).and_return(false)
+      allow(File).to receive(:directory?).with(
+        File.expand_path(".jekyll-cache/Jekyll/Cache/Jekyll::Cache")
+      ).and_return(true)
       site = fixture_site("theme" => "test-theme")
       assert_equal [source_dir("_includes")], site.includes_load_paths
     end