Debug Jekyll issue - 7328

Merge pull request 7228

Gemfile

@@ -31,7 +31,7 @@ group :test do
   gem "nokogiri", RUBY_VERSION >= "2.2" ? "~> 1.7" : "~> 1.7.0"
   gem "rspec"
   gem "rspec-mocks"
-  gem "rubocop", "~> 0.55.0"
+  gem "rubocop", "~> 0.56.0"
   gem "test-dependency-theme", :path => File.expand_path("test/fixtures/test-dependency-theme", __dir__)
   gem "test-theme", :path => File.expand_path("test/fixtures/test-theme", __dir__)
 

History.markdown

@@ -1,3 +1,25 @@
+## 3.8.4 / 2018-09-18
+
+### Bug Fixes
+
+  * security: fix `include` bypass of `EntryFilter#filter` symlink check (#7228)
+
+## 3.8.3 / 2018-06-05
+
+### Bug Fixes
+
+  * Fix --unpublished not affecting collection documents (#7027)
+
+## 3.8.2 / 2018-05-18
+
+### Development Fixes
+
+  * Update rubocop version (#7016)
+
+### Bug Fixes
+
+  * Add whitespace control to LIQUID_TAG_REGEX (#7015)
+
 ## 3.8.1 / 2018-05-01
 
 ### Bug Fixes

docs/_config.yml

@@ -1,5 +1,5 @@
 ---
-version: 3.8.1
+version: 3.8.4
 name: Jekyll • Simple, blog-aware, static sites
 description: Transform your plain text into static websites and blogs
 url: https://jekyllrb.com

docs/_docs/history.md

@@ -4,6 +4,38 @@ permalink: "/docs/history/"
 note: This file is autogenerated. Edit /History.markdown instead.
 ---
 
+## 3.8.4 / 2018-09-18
+{: #v3-8-4}
+
+### Bug Fixes
+{: #bug-fixes-v3-8-4}
+
+- security: fix `include` bypass of `EntryFilter#filter` symlink check ([#7228]({{ site.repository }}/issues/7228))
+
+
+## 3.8.3 / 2018-06-05
+{: #v3-8-3}
+
+### Bug Fixes
+{: #bug-fixes-v3-8-3}
+
+- Fix --unpublished not affecting collection documents ([#7027]({{ site.repository }}/issues/7027))
+
+
+## 3.8.2 / 2018-05-18
+{: #v3-8-2}
+
+### Development Fixes
+{: #development-fixes-v3-8-2}
+
+- Update rubocop version ([#7016]({{ site.repository }}/issues/7016))
+
+### Bug Fixes
+{: #bug-fixes-v3-8-2}
+
+- Add whitespace control to LIQUID_TAG_REGEX ([#7015]({{ site.repository }}/issues/7015))
+
+
 ## 3.8.1 / 2018-05-01
 {: #v3-8-1}
 

docs/_posts/2018-05-18-jekyll-3-8-2-released.markdown

@@ -0,0 +1,19 @@
+---
+title: 'Jekyll 3.8.2 Released'
+date: 2018-05-19 10:30:00 -0500
+author: pathawks
+version: 3.8.2
+categories: [release]
+---
+
+Hello Jekyllers!!
+
+Today we are releasing `v3.8.2`, which fixes the way Jekyll generates excerpts
+for posts when the first paragraph of the post contains Liquid tags that take
+advantage of [Liquid's whitespace control feature][Liquid whitespace].
+
+Big thanks to @kylebarbour, who first reported this issue and also very quickly
+submitted a fix. Also thanks to @nickskalkin for making sure that we are using
+the latest version of Rubocop to lint our code.
+
+[Liquid whitespace]: https://shopify.github.io/liquid/basics/whitespace/

docs/_posts/2018-06-04-jekyll-3-8-3-released.markdown

@@ -0,0 +1,13 @@
+---
+title: 'Jekyll 3.8.3 Released'
+date: 2018-06-05 09:00:00 -0500
+author: pathawks
+version: 3.8.3
+categories: [release]
+---
+
+This release fixes a regression in 3.8 where collections with `published: false`
+do not show when using the `--unpublished` flag.
+
+Thanks to @philipbelesky for reporting and fixing this issue; collections with
+`published: false` now behave the same way as Posts.

docs/latest_version.txt

@@ -1 +1 @@
-3.8.1
+3.8.4

features/collections.feature

@@ -263,6 +263,104 @@ Feature: Collections
     And the "_site/puppies/snowy.html" file should not exist
     And the "_site/puppies/hardy.html" file should not exist
 
+  Scenario: Access rendered collection with future date and unpublished flag via Liquid
+    Given I have a _puppies directory
+    And I have the following documents under the puppies collection:
+      | title  | date       | content             | published |
+      | Rover  | 2007-12-31 | content for Rover.  | true      |
+      | Figor  | 2007-12-31 | content for Figor.  | false     |
+      | Snowy  | 2199-12-31 | content for Snowy.  | true      |
+      | Hardy  | 2199-12-31 | content for Hardy.  | false     |
+    And I have a "_config.yml" file with content:
+    """
+    collections:
+      puppies:
+        output: true
+    """
+    And I have a "index.md" page that contains "{% for puppy in site.puppies %}<div>{{ puppy.title }}</div>{% endfor %}"
+    When I run jekyll build
+    Then I should get a zero exit status
+    And the _site directory should exist
+    And I should see "<div>Rover</div>" in "_site/index.html"
+    But I should see "<div>Snowy</div>" in "_site/index.html"
+    And I should not see "<div>Figor</div>" in "_site/index.html"
+    And I should not see "<div>Hardy</div>" in "_site/index.html"
+    And the "_site/puppies/rover.html" file should exist
+    And the "_site/puppies/snowy.html" file should not exist
+    And the "_site/puppies/figor.html" file should not exist
+    And the "_site/puppies/hardy.html" file should not exist
+    When I run jekyll build --unpublished
+    Then I should get a zero exit status
+    And the _site directory should exist
+    And I should see "<div>Rover</div>" in "_site/index.html"
+    And I should see "<div>Snowy</div>" in "_site/index.html"
+    And I should see "<div>Figor</div>" in "_site/index.html"
+    But I should see "<div>Hardy</div>" in "_site/index.html"
+    And the "_site/puppies/rover.html" file should exist
+    And the "_site/puppies/snowy.html" file should not exist
+    And the "_site/puppies/figor.html" file should exist
+    And the "_site/puppies/hardy.html" file should not exist
+    When I run jekyll build --unpublished --future
+    Then I should get a zero exit status
+    And the _site directory should exist
+    And I should see "<div>Rover</div>" in "_site/index.html"
+    And I should see "<div>Snowy</div>" in "_site/index.html"
+    And I should see "<div>Figor</div>" in "_site/index.html"
+    But I should see "<div>Hardy</div>" in "_site/index.html"
+    And the "_site/puppies/rover.html" file should exist
+    And the "_site/puppies/snowy.html" file should exist
+    And the "_site/puppies/figor.html" file should exist
+    And the "_site/puppies/hardy.html" file should exist
+
+  Scenario: Access unrendered collection with future date and unpublished flag via Liquid
+    Given I have a _puppies directory
+    And I have the following documents under the puppies collection:
+      | title  | date       | content             | published |
+      | Rover  | 2007-12-31 | content for Rover.  | true      |
+      | Figor  | 2007-12-31 | content for Figor.  | false     |
+      | Snowy  | 2199-12-31 | content for Snowy.  | true      |
+      | Hardy  | 2199-12-31 | content for Hardy.  | false     |
+    And I have a "_config.yml" file with content:
+    """
+    collections:
+      puppies:
+        output: false
+    """
+    And I have a "index.md" page that contains "{% for puppy in site.puppies %}<div>{{ puppy.title }}</div>{% endfor %}"
+    When I run jekyll build
+    Then I should get a zero exit status
+    And the _site directory should exist
+    And I should see "<div>Rover</div>" in "_site/index.html"
+    But I should see "<div>Snowy</div>" in "_site/index.html"
+    And I should not see "<div>Figor</div>" in "_site/index.html"
+    And I should not see "<div>Hardy</div>" in "_site/index.html"
+    And the "_site/puppies/rover.html" file should not exist
+    And the "_site/puppies/snowy.html" file should not exist
+    And the "_site/puppies/figor.html" file should not exist
+    And the "_site/puppies/hardy.html" file should not exist
+    When I run jekyll build --unpublished
+    Then I should get a zero exit status
+    And the _site directory should exist
+    And I should see "<div>Rover</div>" in "_site/index.html"
+    And I should see "<div>Snowy</div>" in "_site/index.html"
+    And I should see "<div>Figor</div>" in "_site/index.html"
+    But I should see "<div>Hardy</div>" in "_site/index.html"
+    And the "_site/puppies/rover.html" file should not exist
+    And the "_site/puppies/snowy.html" file should not exist
+    And the "_site/puppies/figor.html" file should not exist
+    And the "_site/puppies/hardy.html" file should not exist
+    When I run jekyll build --unpublished --future
+    Then I should get a zero exit status
+    And the _site directory should exist
+    And I should see "<div>Rover</div>" in "_site/index.html"
+    And I should see "<div>Snowy</div>" in "_site/index.html"
+    And I should see "<div>Figor</div>" in "_site/index.html"
+    But I should see "<div>Hardy</div>" in "_site/index.html"
+    And the "_site/puppies/rover.html" file should not exist
+    And the "_site/puppies/snowy.html" file should not exist
+    And the "_site/puppies/figor.html" file should not exist
+    And the "_site/puppies/hardy.html" file should not exist
+
   Scenario: All the documents
     Given I have an "index.html" page that contains "All documents: {% for doc in site.documents %}{{ doc.relative_path }} {% endfor %}"
     And I have fixture collections

lib/jekyll/collection.rb

@@ -212,7 +212,9 @@ module Jekyll
     def read_document(full_path)
       doc = Document.new(full_path, :site => site, :collection => self)
       doc.read
-      docs << doc unless doc.data["published"] == false
+      if site.unpublished || doc.published?
+        docs << doc
+      end
     end
 
     private

lib/jekyll/entry_filter.rb

@@ -31,9 +31,12 @@ module Jekyll
 
     def filter(entries)
       entries.reject do |e|
-        unless included?(e)
-          special?(e) || backup?(e) || excluded?(e) || symlink?(e)
-        end
+        # Reject this entry if it is a symlink.
+        next true if symlink?(e)
+        # Do not reject this entry if it is included.
+        next false if included?(e)
+        # Reject this entry if it is special, a backup file, or excluded.
+        special?(e) || backup?(e) || excluded?(e)
       end
     end
 

lib/jekyll/excerpt.rb

@@ -128,7 +128,7 @@ module Jekyll
     #
     # Returns excerpt String
 
-    LIQUID_TAG_REGEX = %r!{%\s*(\w+).+\s*%}!m
+    LIQUID_TAG_REGEX = %r!{%-?\s*(\w+).+\s*-?%}!m
     MKDWN_LINK_REF_REGEX = %r!^ {0,3}\[[^\]]+\]:.+$!
 
     def extract_excerpt(doc_content)
@@ -141,7 +141,7 @@ module Jekyll
         head =~ LIQUID_TAG_REGEX
         tag_name = Regexp.last_match(1)
 
-        if liquid_block?(tag_name) && head.match(%r!{%\s*end#{tag_name}\s*%}!).nil?
+        if liquid_block?(tag_name) && head.match(%r!{%-?\s*end#{tag_name}\s*-?%}!).nil?
           print_build_warning
           head << "\n{% end#{tag_name} %}"
         end
@@ -158,6 +158,11 @@ module Jekyll
 
     def liquid_block?(tag_name)
       Liquid::Template.tags[tag_name].superclass == Liquid::Block
+    rescue NoMethodError
+      Jekyll.logger.error "Error:",
+        "A Liquid tag in the excerpt of #{doc.relative_path} couldn't be " \
+        "parsed."
+      raise
     end
 
     def print_build_warning

lib/jekyll/liquid_renderer.rb

@@ -21,7 +21,12 @@ module Jekyll
     end
 
     def file(filename)
+      Jekyll.logger.info "FILENAME:", filename
       filename.match(filename_regex)
+
+      Jekyll.logger.info "CAPTURES:", Regexp.last_match.captures
+      puts
+
       filename =
         if Regexp.last_match(1) == theme_dir("")
           ::File.join(::File.basename(Regexp.last_match(1)), Regexp.last_match(2))

lib/jekyll/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Jekyll
-  VERSION = "3.8.1".freeze
+  VERSION = "3.8.4".freeze
 end

test/source/_posts/2018-05-15-closed-liquid-block-excerpt-whitespace-control.md

@@ -0,0 +1,9 @@
+---
+title:  LIQUID_TAG_REGEX excerpt whitespace control test
+layout: post
+---
+
+{%- for post in site.posts -%}
+    You are in a maze of twisty little passages, all alike.
+    There's lots more to say about this, but that's enough for now.
+{%- endfor -%}

test/source/_posts/2018-05-15-excerpt-whitespace-control-variable.md

@@ -0,0 +1,7 @@
+---
+title:  LIQUID_TAG_REGEX excerpt whitespace control test
+layout: post
+---
+
+{%- assign xyzzy = 'You are in a maze of twisty little passages, all alike.' %}
+{{- xyzzy -}}

test/source/_posts/2018-05-15-open-liquid-block-excerpt-whitespace-control.md

@@ -0,0 +1,10 @@
+---
+title:  LIQUID_TAG_REGEX excerpt whitespace control test
+layout: post
+---
+
+{%- for post in site.posts -%}
+    You are in a maze of twisty little passages, all alike.
+
+    There's lots more to say about this, but that's enough for now.
+{%- endfor -%}

test/source/symlink-test/symlinked-file-outside-source

@@ -0,0 +1 @@
+/etc/passwd

test/test_entry_filter.rb

@@ -5,7 +5,7 @@ require "helper"
 class TestEntryFilter < JekyllUnitTest
   context "Filtering entries" do
     setup do
-      @site = Site.new(site_configuration)
+      @site = fixture_site
     end
 
     should "filter entries" do
@@ -87,7 +87,7 @@ class TestEntryFilter < JekyllUnitTest
       # no support for symlinks on Windows
       skip_if_windows "Jekyll does not currently support symlinks on Windows."
 
-      site = Site.new(site_configuration("safe" => true))
+      site = fixture_site("safe" => true)
       site.reader.read_directories("symlink-test")
 
       assert_equal %w(main.scss symlinked-file).length, site.pages.length
@@ -99,11 +99,22 @@ class TestEntryFilter < JekyllUnitTest
       # no support for symlinks on Windows
       skip_if_windows "Jekyll does not currently support symlinks on Windows."
 
-      site = Site.new(site_configuration)
+      @site.reader.read_directories("symlink-test")
+      refute_equal [], @site.pages
+      refute_equal [], @site.static_files
+    end
 
+    should "include only safe symlinks in safe mode even when included" do
+      # no support for symlinks on Windows
+      skip_if_windows "Jekyll does not currently support symlinks on Windows."
+
+      site = fixture_site("safe" => true, "include" => ["symlinked-file-outside-source"])
       site.reader.read_directories("symlink-test")
-      refute_equal [], site.pages
-      refute_equal [], site.static_files
+
+      # rubocop:disable Performance/FixedSize
+      assert_equal %w(main.scss symlinked-file).length, site.pages.length
+      refute_includes site.static_files.map(&:name), "symlinked-file-outside-source"
+      # rubocop:enable Performance/FixedSize
     end
   end
 

test/test_excerpt.rb

@@ -212,4 +212,56 @@ class TestExcerpt < JekyllUnitTest
       assert_equal true, @excerpt.is_a?(Jekyll::Excerpt)
     end
   end
+
+  context "An excerpt with non-closed but valid Liquid block tag with whitespace control" do
+    setup do
+      clear_dest
+      @site = fixture_site
+      @post = setup_post("2018-05-15-open-liquid-block-excerpt-whitespace-control.md")
+      @excerpt = @post.data["excerpt"]
+
+      assert_includes @post.content, "{%- for"
+      refute_includes @post.content.split("\n\n")[0], "{%- endfor -%}"
+    end
+
+    should "be appended to as necessary and generated" do
+      assert_includes @excerpt.content, "{% endfor %}"
+      refute_includes @excerpt.content, "{% endfor %}\n\n{% endfor %}"
+      assert_equal true, @excerpt.is_a?(Jekyll::Excerpt)
+    end
+  end
+
+  context "An excerpt with valid closed Liquid block tag with whitespace control" do
+    setup do
+      clear_dest
+      @site = fixture_site
+      @post = setup_post("2018-05-15-closed-liquid-block-excerpt-whitespace-control.md")
+      @excerpt = @post.data["excerpt"]
+
+      assert_includes @post.content, "{%- for"
+      assert_includes @post.content.split("\n\n")[0], "{%- endfor -%}"
+    end
+
+    should "not be appended to but generated as is" do
+      assert_includes @excerpt.content, "{%- endfor -%}"
+      refute_includes @excerpt.content, "{% endfor %}\n\n{% endfor %}"
+      assert_equal true, @excerpt.is_a?(Jekyll::Excerpt)
+    end
+  end
+
+  context "An excerpt with valid Liquid variable with whitespace control" do
+    setup do
+      clear_dest
+      @site = fixture_site
+      @post = setup_post("2018-05-15-excerpt-whitespace-control-variable.md")
+      @excerpt = @post.data["excerpt"]
+
+      assert_includes @post.content, "{%- assign"
+    end
+
+    should "not be appended to but generated as is" do
+      assert_includes @excerpt.content, "{{- xyzzy -}}"
+      assert_equal true, @excerpt.is_a?(Jekyll::Excerpt)
+    end
+  end
 end

test/test_generated_site.rb

@@ -16,7 +16,7 @@ class TestGeneratedSite < JekyllUnitTest
     end
 
     should "ensure post count is as expected" do
-      assert_equal 54, @site.posts.size
+      assert_equal 57, @site.posts.size
     end
 
     should "insert site.posts into the index" do

test/test_layout_reader.rb

@@ -31,5 +31,51 @@ class TestLayoutReader < JekyllUnitTest
         assert_equal LayoutReader.new(@site).layout_directory, source_dir("blah/_layouts")
       end
     end
+
+    context "when a layout is a symlink" do
+      setup do
+        FileUtils.ln_sf("/etc/passwd", source_dir("_layouts", "symlink.html"))
+        @site = fixture_site(
+          "safe"    => true,
+          "include" => ["symlink.html"]
+        )
+      end
+
+      teardown do
+        FileUtils.rm(source_dir("_layouts", "symlink.html"))
+      end
+
+      should "only read the layouts which are in the site" do
+        skip_if_windows "Jekyll does not currently support symlinks on Windows."
+
+        layouts = LayoutReader.new(@site).read
+
+        refute layouts.key?("symlink"), "Should not read the symlinked layout"
+      end
+    end
+
+    context "with a theme" do
+      setup do
+        FileUtils.ln_sf("/etc/passwd", theme_dir("_layouts", "theme-symlink.html"))
+        @site = fixture_site(
+          "include" => ["theme-symlink.html"],
+          "theme"   => "test-theme",
+          "safe"    => true
+        )
+      end
+
+      teardown do
+        FileUtils.rm(theme_dir("_layouts", "theme-symlink.html"))
+      end
+
+      should "not read a symlink'd theme" do
+        skip_if_windows "Jekyll does not currently support symlinks on Windows."
+
+        layouts = LayoutReader.new(@site).read
+
+        refute layouts.key?("theme-symlink"), \
+               "Should not read symlinked layout from theme"
+      end
+    end
   end
 end