More writing.

Move QUERY into process method

.gitignore

@@ -1,6 +1,8 @@
+Gemfile.lock
 test/dest
 *.gem
 pkg/
 *.swp
 *~
 _site/
+.bundle/

Gemfile

@@ -0,0 +1,2 @@
+source :rubygems
+gemspec

History.txt

@@ -1,3 +1,42 @@
+== HEAD
+  * Major Enhancements
+    * Add command line importer functionality (#253)
+  * Minor Enhancements
+    * Switch to Albino gem
+    * Bundler support
+    * Use English library to avoid hoops (#292)
+    * Add Posterous importer (#254)
+    * Fixes for Wordpress importer (#274, #252, #271)
+    * Better error message for invalid post date (#291)
+    * Print formatted fatal exceptions to stdout on build failure
+  * Bug Fixes
+    * Secure additional path exploits
+
+== 0.10.0 / 2010-12-16
+  * Bug Fixes
+    * Add --no-server option.
+
+== 0.9.0 / 2010-12-15
+  * Minor Enhancements
+    * Use OptionParser's [no-] functionality for better boolean parsing.
+    * Add Drupal migrator (#245)
+    * Complain about YAML and Liquid errors (#249)
+    * Remove orphaned files during regeneration (#247)
+    * Add Marley migrator (#28)
+
+== 0.8.0 / 2010-11-22
+  * Minor Enhancements
+    * Add wordpress.com importer (#207)
+    * Add --limit-posts cli option (#212)
+    * Add uri_escape filter (#234)
+    * Add --base-url cli option (#235)
+    * Improve MT migrator (#238)
+    * Add kramdown support (#239)
+  * Bug Fixes
+    * Fixed filename basename generation (#208)
+    * Set mode to UTF8 on Sequel connections (#237)
+    * Prevent _includes dir from being a symlink
+
 == 0.7.0 / 2010-08-24
   * Minor Enhancements
     * Add support for rdiscount extensions (#173)
@@ -50,7 +89,7 @@
     * Empty tags causes error in read_posts (#84)
     * Fix pagination to adhere to read/render/write paradigm
   * Test Enhancement
-    * cucumber features no longer use site.ports.first where a better
+    * cucumber features no longer use site.posts.first where a better
       alternative is available
 
 == 0.5.6 / 2010-01-08

README.textile

@@ -27,7 +27,6 @@ h2. Runtime Dependencies
 * Classifier: Generating related posts (Ruby)
 * Maruku: Default markdown engine (Ruby)
 * Directory Watcher: Auto-regeneration of sites (Ruby)
-* Open4: Talking to pygments for syntax highlighting (Ruby)
 * Pygments: Syntax highlighting (Python)
 
 h2. Developer Dependencies
@@ -35,7 +34,6 @@ h2. Developer Dependencies
 * Shoulda: Test framework (Ruby)
 * RR: Mocking (Ruby)
 * RedGreen: Nicer test output (Ruby)
-* RDiscount: Discount Markdown Processor (Ruby)
 
 h2. License
 

bin/jekyll

@@ -9,7 +9,8 @@ Basic Command Line Usage:
   jekyll                                                   # . -> ./_site
   jekyll <path to write generated site>                    # . -> <path>
   jekyll <path to source> <path to write generated site>   # <path> -> <path>
-
+  jekyll import <importer name> <options>                  # imports posts using named import script
+  
   Configuration is read from '<source>/_config.yml' but can be overriden
   using the following options:
 
@@ -18,21 +19,43 @@ HELP
 require 'optparse'
 require 'jekyll'
 
+
 exec = {}
 options = {}
 opts = OptionParser.new do |opts|
   opts.banner = help
 
-  opts.on("--safe", "Safe mode (default unsafe)") do
-    options['safe'] = true
+  opts.on("--file [PATH]", "File to import from") do |import_file|
+    options['file'] = import_file
+  end
+  
+  opts.on("--dbname [TEXT]", "DB to import from") do |import_dbname|
+    options['dbname'] = import_dbname
+  end
+  
+  opts.on("--user [TEXT]", "Username to use when importing") do |import_user|
+    options['user'] = import_user
+  end
+  
+  opts.on("--pass [TEXT]", "Password to use when importing") do |import_pass|
+    options['pass'] = import_pass
+  end
+  
+  opts.on("--host [HOST ADDRESS]", "Host to import from") do |import_host|
+    options['host'] = import_host
+  end
+  
+  opts.on("--site [SITE NAME]", "Site to import from") do |import_site|
+    options['site'] = import_site
+  end
+  
+
+  opts.on("--[no-]safe", "Safe mode (default unsafe)") do |safe|
+    options['safe'] = safe
   end
 
-  opts.on("--auto", "Auto-regenerate") do
-    options['auto'] = true
-  end
-
-  opts.on("--no-auto", "No auto-regenerate") do
-    options['auto'] = false
+  opts.on("--[no-]auto", "Auto-regenerate") do |auto|
+    options['auto'] = auto
   end
 
   opts.on("--server [PORT]", "Start web server (default port 4000)") do |port|
@@ -40,28 +63,36 @@ opts = OptionParser.new do |opts|
     options['server_port'] = port unless port.nil?
   end
 
-  opts.on("--lsi", "Use LSI for better related posts") do
-    options['lsi'] = true
+  opts.on("--no-server", "Do not start a web server") do |part|
+    options['server'] = false
   end
 
-  opts.on("--pygments", "Use pygments to highlight code") do
-    options['pygments'] = true
+  opts.on("--base-url [BASE_URL]", "Serve website from a given base URL (default '/'") do |baseurl|
+      options['baseurl'] = baseurl
+  end
+
+  opts.on("--[no-]lsi", "Use LSI for better related posts") do |lsi|
+    options['lsi'] = lsi
+  end
+
+  opts.on("--[no-]pygments", "Use pygments to highlight code") do |pygments|
+    options['pygments'] = pygments
   end
 
   opts.on("--rdiscount", "Use rdiscount gem for Markdown") do
     options['markdown'] = 'rdiscount'
   end
 
+  opts.on("--kramdown", "Use kramdown gem for Markdown") do
+    options['markdown'] = 'kramdown'
+  end
+
   opts.on("--time [TIME]", "Time to generate the site for") do |time|
     options['time'] = Time.parse(time)
   end
 
-  opts.on("--future", "Render future dated posts") do
-    options['future'] = true
-  end
-
-  opts.on("--no-future", "Do not render future dated posts") do
-    options['future'] = false
+  opts.on("--[no-]future", "Render future dated posts") do |future|
+    options['future'] = future
   end
 
   opts.on("--permalink [TYPE]", "Use 'date' (default) for YYYY/MM/DD") do |style|
@@ -78,6 +109,16 @@ opts = OptionParser.new do |opts|
     end
   end
 
+  opts.on("--limit_posts [MAX_POSTS]", "Limit the number of posts to publish") do |limit_posts|
+    begin
+      options['limit_posts'] = limit_posts.to_i
+      raise ArgumentError if options['limit_posts'] < 1
+    rescue
+      puts 'you must specify a number of posts by page bigger than 0'
+      exit 0
+    end
+  end
+
   opts.on("--url [URL]", "Set custom site.url") do |url|
     options['url'] = url
   end
@@ -91,6 +132,59 @@ end
 # Read command line options into `options` hash
 opts.parse!
 
+
+# Check for import stuff
+if ARGV.size > 0
+  if ARGV[0] == 'import'
+    migrator = ARGV[1]
+
+    if migrator.nil?
+      puts "Invalid options. Run `jekyll --help` for assistance."
+      exit(1)
+    else
+      migrator = migrator.downcase
+    end
+    
+    cmd_options = []
+    ['file', 'dbname', 'user', 'pass', 'host', 'site'].each do |p|
+      cmd_options << "\"#{options[p]}\"" unless options[p].nil?
+    end
+    
+    # It's import time
+    puts "Importing..."
+    
+    # Ideally, this shouldn't be necessary. Maybe parse the actual
+    # src files for the migrator name?
+    migrators = {
+      :posterous => 'Posterous',
+      :wordpressdotcom => 'WordpressDotCom',
+      :wordpress => 'Wordpress',
+      :csv => 'CSV',
+      :drupal => 'Drupal',
+      :mephisto => 'Mephisto',
+      :mt => 'MT',
+      :textpattern => 'TextPattern',
+      :typo => 'Typo'
+    }
+    
+    app_root = File.join(File.dirname(__FILE__), '..')
+    
+    require "#{app_root}/lib/jekyll/migrators/#{migrator}"
+    
+    if Jekyll.const_defined?(migrators[migrator.to_sym])
+      migrator_class = Jekyll.const_get(migrators[migrator.to_sym])
+      migrator_class.process(*cmd_options)
+    else
+      puts "Invalid migrator. Run `jekyll --help` for assistance."
+      exit(1)
+    end
+    
+    exit(0)
+  end
+end
+
+
+
 # Get source and destintation from command line
 case ARGV.size
   when 0
@@ -148,7 +242,11 @@ else
   puts "Building site: #{source} -> #{destination}"
   begin
     site.process
-  rescue Jekyll::FatalException
+  rescue Jekyll::FatalException => e
+    puts
+    puts "ERROR: YOUR SITE COULD NOT BE BUILT:"
+    puts "------------------------------------"
+    puts e.message
     exit(1)
   end
   puts "Successfully generated site: #{source} -> #{destination}"
@@ -166,9 +264,9 @@ if options['server']
 
   s = HTTPServer.new(
     :Port            => options['server_port'],
-    :DocumentRoot    => destination,
     :MimeTypes       => mime_types
   )
+  s.mount(options['baseurl'], HTTPServlet::FileHandler, destination)
   t = Thread.new {
     s.start
   }

doc/.gitignore

@@ -0,0 +1 @@
+output

doc/.gitscribe

@@ -0,0 +1,7 @@
+---
+publish:  true
+edition:  0.1
+language: en
+version:  1.0
+author:   Your Name
+cover:    image/cover.jpg

doc/README.asciidoc

@@ -0,0 +1,9 @@
+This Book
+=========
+
+This book is written using using the git-scribe toolchain, which can be found at:
+
+http://github.com/schacon/git-scribe
+
+Instructions on how to install the tool and use it for things like editing this book,
+submitting errata and providing translations can be found at that site.

doc/book/book.asc

@@ -0,0 +1,10 @@
+Jekyll
+======
+:Author:    Tom Preston-Werner
+:Email:     <tom@mojombo.com>
+
+include::ch00-preface.asc[]
+
+include::ch01-quick-start.asc[]
+
+include::ch02-directory-layout.asc[]

doc/book/ch00-preface.asc

@@ -0,0 +1,41 @@
+== Preface
+
+Jekyll was born out the desire to create a blog engine that would make it
+possible to write posts in my local text editor, version those posts with Git,
+and keep up with my desire to tweak the styles and layout of my site.
+
+In other words, I wanted something that fit into my existing software
+development workflow and toolchain. Jekyll handles not only this case, but a
+wide variety of other situations that call for static site generation based on
+converted content and layout templates.
+
+At its core, Jekyll is a text transformation engine. The concept behind the
+system is this: you give it text written in your favorite markup language, be
+that Markdown, Textile, or just plain HTML, and it churns that through a
+layout or series of layout files. Throughout that process you can tweak how
+you want the site URLs to look, what data gets displayed on the layout and
+much more.
+
+If you're looking for a simple, yet powerful solution to your blogging or
+static site needs, Jekyll may be just what you've been looking for.
+
+
+=== What this book covers
+
+_Chapter 1, Quick Start_ covers installation, introduces the Jekyll command
+line interface, and runs through a quick example demonstrating the site
+generator, post generator and how to convert your Jekyll site into a static
+site.
+
+_Chapter 2, Directory Layout_ covers the various files and directories that
+comprise a Jekyll site.
+
+_Chapter 3, Tags and Filters_
+
+_Chapter X, Deploying your Jekyll Site_
+
+_Chapter X, Customizing Jekyll with Plugins_
+
+_Chapter X, Migrating to Jekyll from your Existing Blog_
+
+_Chapter X, Configuration Reference_

doc/book/ch01-quick-start.asc

@@ -0,0 +1,153 @@
+== Chapter 1: Quick Start
+
+This chapter is designed to get you up and running with Jekyll as quickly as
+possible.
+
+
+=== Installation
+
+The best way to install Jekyll is via RubyGems:
+
+----
+gem install jekyll
+----
+
+This is all you need in order to get started with a basic Jekyll site. Some
+options require additional packages to be installed.
+
+If you encounter errors during gem installation, you may need to install the
+header files for compiling extension modules for ruby 1.8:
+
+.Debian
+----
+sudo apt-get install ruby1.8-dev
+----
+
+.Red Hat / CentOS / Fedora systems
+----
+sudo yum install ruby-devel
+----
+
+.NearlyFreeSpeech
+----
+RB_USER_INSTALL=true gem install jekyll
+----
+
+If you encounter errors like +Failed to build gem native extension+ on Windows
+you may need to install http://wiki.github.com/oneclick/rubyinstaller/development-kit[RubyInstaller
+DevKit].
+
+==== LaTeX to PNG
+
+Maruku comes with optional support for LaTeX to PNG rendering via blahtex
+(Version 0.6) which must be in your $PATH along with @dvips@.
+
+(NOTE: "remi's fork of Maruku":http://github.com/remi/maruku/tree/master does
+not assume a fixed location for @dvips@ if you need that fixed)
+
+==== RDiscount
+
+If you prefer to use
+http://github.com/rtomayko/rdiscount/tree/master[RDiscount] instead of
+http://maruku.rubyforge.org/[Maruku] for markdown, just make sure it's
+installed:
+
+----
+sudo gem install rdiscount
+----
+
+And run Jekyll with the following option:
+
+----
+jekyll --rdiscount
+----
+
+Or, in your @_config.yml@ file put the following so you don't have to specify the flag:
+
+----
+markdown: rdiscount
+----
+
+==== Pygments
+
+If you want syntax highlighting via the @{% highlight %}@ tag in your posts,
+you'll need to install http://pygments.org/[Pygments].
+
+.On OSX with Homebrew
+----
+brew install pip && pip install pygments
+----
+
+.On OSX with MacPorts
+----
+sudo port install python25 py25-pygments
+----
+
+.Bare OS X Leopard
+----
+sudo easy_install Pygments
+----
+
+.Archlinux
+----
+sudo pacman -S python-pygments
+----
+
+.Archlinux python2 for Pygments
+----
+$ sudo pacman -S python2-pygments
+----
+
+NOTE: python2 pygments version creates a `pygmentize2` executable, while
+Jekyll tries to find `pygmentize`. Either create a symlink `# ln -s
+/usr/bin/pygmentize2 /usr/bin/pygmentize` or use the python3 version.
+
+.Ubuntu and Debian
+----
+sudo apt-get install python-pygments
+----
+
+.Gentoo
+----
+$ sudo emerge -av dev-python/pygments
+----
+
+
+=== Creating your First Site
+
+Jekyll comes with a handy generator that will create a barebones skeleton site
+to help you get up and running in no time. Simply create an empty directory to
+contain your site, navigate to it, and run the generator command:
+
+----
+$ mkdir mysite
+$ cd mysite
+$ jekyll gen
+----
+
+Make sure the directory is empty or Jekyll will refuse to run. If everything
+was successful, you'll be left with a complete, valid Jekyll site that's ready
+to be converted into a static site.
+
+To perform the conversion, make sure you're in the root of your Jekyll site
+directory and run:
+
+----
+$ jekyll --server
+----
+
+If all goes well, you should get a few lines with information about config
+file detection, source and destination paths, and a success message.
+
+The `--server` command line option fires up a simple web server that will
+serve the static site we just generated so that we can easily preview what it
+will look like once we deploy it to a production environment.
+
+Open up your favorite web browser and navigate to:
+
+----
+http://localhost:4000
+----
+
+Congratulations! You have now successfully created and converted your first
+Jekyll site!

doc/book/ch02-directory-layout.asc

@@ -0,0 +1,90 @@
+== Chapter 2: Directory Layout
+
+If you followed the Quick Start in the last chapter, you have a Jekyll site on
+your local machine. Let's take a closer look at it and see what makes it tick.
+The file layout should look something like this:
+
+----
+.
+|-- _config.yml
+|-- _layouts
+|   |-- default.html
+|   `-- post.html
+|-- _posts
+|   |-- 2007-10-29-why-every-programmer-should-play-nethack.textile
+|   `-- 2009-04-26-barcamp-boston-4-roundup.textile
+|-- _site
+|-- images
+|   `-- logo.png
+`-- index.html
+----
+
+Notice that some of the files and directories begin with an underscore. These
+have special meaning to Jekyll. The underscore ensures that they will not
+interfere with the rest of your site's normal content. It also means that if
+any of your normal files start with an underscore, they will cause problems,
+so try to avoid this.
+
+=== _config.yml
+
+This file stores configuration data. A majority of these options can be
+specified from the command line executable but it's easier to throw them in
+here so you don't have to type them out every time. Detailed explanations of
+configuration directives can be found in Chapter X.
+
+=== _layouts
+
+Files in this directory represent templates that can be used to wrap converted
+pages. Layouts are defined on a page-by-page basis in the YAML front matter.
+The liquid tag +{{ content }}+ specifies where the content will be placed
+during the conversion process.
+
+=== _posts
+
+If you're using Jekyll as a blog engine, this is where you'll place your blog
+posts. A post's filename contains several pieces of data, so you must be very
+careful about how these files are named. The filename format is:
++YEAR-MONTH-DATE-SLUG.MARKUP+. The YEAR must be four numbers and the MONTH and
+DATE must be two numbers each. The SLUG is what will appear in the URL. The
+MARKUP tells Jekyll the format of the post. The date and slug will be used
+along with any permalink options you specify (See Chapter X) to construct the
+final URL of the post.
+
+=== _site
+
+This is where the generated site will be placed (by default) once Jekyll is
+done transforming it. If you're using version control, you'll want to add this
+directory to the list of files to be ignored.
+
+=== Normal Files with YAML Front Matter
+
+All files outside of the special underscore directories and that do not
+themselves begin with an underscore will be scanned by Jekyll and subjected to
+conversion if they contain any YAML front matter.
+
+=== Everything Else
+
+Any files and directories that do not fall into one of the above categories
+will be copied to the static site as-is without modification. In this example,
++images/logo.png+ will be copied to the same location in the generated site.
+
+
+
+
+h2. Running Jekyll
+
+Usually this is done through the @jekyll@ executable, which is installed with
+the gem. In order to get a server up and running with your Jekyll site, run:
+
+@jekyll --server@ 
+
+and then browse to http://0.0.0.0:4000. There's plenty of [[configuration
+options|Configuration]] available to you as well.
+
+On Debian or Ubuntu, you may need to add @/var/lib/gems/1.8/bin/@ to your path.
+
+h2. Deployment
+
+Since Jekyll simply generates a folder filled with HTML files, it can be
+served using practically any available web server out there. Please check the
+[[Deployment]] page for more information regarding specific scenarios.

features/create_sites.feature

@@ -89,6 +89,6 @@ Feature: Create sites
     And I have an "_includes/about.textile" file that contains "Generated by {% include jekyll.textile %}"
     And I have an "_includes/jekyll.textile" file that contains "Jekyll"
     And I have an "index.html" page that contains "Basic Site with include tag: {% include about.textile %}"
-    When I run jekyll
+    When I debug jekyll
     Then the _site directory should exist
-    And I should see "Basic Site with include tag: Generated by Jekyll" in "_site/index.html"
+    And I should see "Basic Site with include tag: Generated by Jekyll" in "_site/index.html"

features/site_configuration.feature

@@ -48,6 +48,13 @@ Feature: Site configuration
     Then the _site directory should exist
     And I should see "<a href="http://google.com">Google</a>" in "_site/index.html"
 
+  Scenario: Use Kramdown for markup
+    Given I have an "index.markdown" page that contains "[Google](http://google.com)"
+    And I have a configuration file with "markdown" set to "kramdown"
+    When I run jekyll
+    Then the _site directory should exist
+    And I should see "<a href="http://google.com">Google</a>" in "_site/index.html"
+
   Scenario: Use Maruku for markup
     Given I have an "index.markdown" page that contains "[Google](http://google.com)"
     And I have a configuration file with "markdown" set to "maruku"
@@ -101,3 +108,19 @@ Feature: Site configuration
     And I should see "Page Layout: 2 on 2010-01-01" in "_site/index.html"
     And I should see "Post Layout: <p>content for entry1.</p>" in "_site/2007/12/31/entry1.html"
     And I should see "Post Layout: <p>content for entry2.</p>" in "_site/2020/01/31/entry2.html"
+
+  Scenario: Limit the number of posts generated by most recent date
+    Given I have a _posts directory
+    And I have a configuration file with:
+      | key         | value       |
+      | limit_posts | 2           |
+    And I have the following posts:
+      | title   | date      | content          |
+      | Apples  | 3/27/2009 | An article about apples |
+      | Oranges | 4/1/2009  | An article about oranges |
+      | Bananas | 4/5/2009  | An article about bananas |
+    When I run jekyll
+    Then the _site directory should exist
+    And the "_site/2009/04/05/bananas.html" file should exist
+    And the "_site/2009/04/01/oranges.html" file should exist
+    And the "_site/2009/03/27/apples.html" file should not exist

features/support/env.rb

@@ -14,3 +14,6 @@ def run_jekyll(opts = {})
   command << " >> /dev/null 2>&1" if opts[:debug].nil?
   system command
 end
+
+# work around "invalid option: --format" cucumber bug (see #296)
+Test::Unit.run = true

jekyll.gemspec

@@ -4,8 +4,8 @@ Gem::Specification.new do |s|
   s.rubygems_version = '1.3.5'
 
   s.name              = 'jekyll'
-  s.version           = '0.7.0'
-  s.date              = '2010-08-24'
+  s.version           = '0.10.0'
+  s.date              = '2010-12-16'
   s.rubyforge_project = 'jekyll'
 
   s.summary     = "A simple, blog aware, static site generator."
@@ -23,16 +23,19 @@ Gem::Specification.new do |s|
   s.rdoc_options = ["--charset=UTF-8"]
   s.extra_rdoc_files = %w[README.textile LICENSE]
 
-  s.add_runtime_dependency('liquid', [">= 1.9.0"])
-  s.add_runtime_dependency('classifier', [">= 1.3.1"])
-  s.add_runtime_dependency('directory_watcher', [">= 1.1.1"])
-  s.add_runtime_dependency('maruku', [">= 0.5.9"])
+  s.add_runtime_dependency('liquid', ">= 1.9.0")
+  s.add_runtime_dependency('classifier', ">= 1.3.1")
+  s.add_runtime_dependency('directory_watcher', ">= 1.1.1")
+  s.add_runtime_dependency('maruku', ">= 0.5.9")
+  s.add_runtime_dependency('kramdown', ">= 0.13.2")
+  s.add_runtime_dependency('albino', ">= 1.3.2")
 
-  s.add_development_dependency('redgreen', [">= 4.2.1"])
-  s.add_development_dependency('shoulda', [">= 4.2.1"])
-  s.add_development_dependency('rr', [">= 4.2.1"])
-  s.add_development_dependency('cucumber', [">= 4.2.1"])
-  s.add_development_dependency('RedCloth', [">= 4.2.1"])
+  s.add_development_dependency('redgreen', ">= 1.2.2")
+  s.add_development_dependency('shoulda', ">= 2.11.3")
+  s.add_development_dependency('rr', ">= 1.0.2")
+  s.add_development_dependency('cucumber', ">= 0.10.0")
+  s.add_development_dependency('RedCloth', ">= 4.2.1")
+  s.add_development_dependency('rdiscount', ">= 1.6.5")
 
   # = MANIFEST =
   s.files = %w[
@@ -67,10 +70,13 @@ Gem::Specification.new do |s|
     lib/jekyll/generators/pagination.rb
     lib/jekyll/layout.rb
     lib/jekyll/migrators/csv.rb
+    lib/jekyll/migrators/drupal.rb
+    lib/jekyll/migrators/marley.rb
     lib/jekyll/migrators/mephisto.rb
     lib/jekyll/migrators/mt.rb
     lib/jekyll/migrators/textpattern.rb
     lib/jekyll/migrators/typo.rb
+    lib/jekyll/migrators/wordpress.com.rb
     lib/jekyll/migrators/wordpress.rb
     lib/jekyll/page.rb
     lib/jekyll/plugin.rb
@@ -80,6 +86,7 @@ Gem::Specification.new do |s|
     lib/jekyll/tags/highlight.rb
     lib/jekyll/tags/include.rb
     test/helper.rb
+    test/source/.htaccess
     test/source/_includes/sig.markdown
     test/source/_layouts/default.html
     test/source/_layouts/simple.html
@@ -110,6 +117,7 @@ Gem::Specification.new do |s|
     test/source/category/_posts/2008-9-23-categories.textile
     test/source/contacts.html
     test/source/css/screen.css
+    test/source/deal.with.dots.html
     test/source/foo/_posts/bar/2008-12-12-topical-post.textile
     test/source/index.html
     test/source/sitemap.xml
@@ -120,6 +128,7 @@ Gem::Specification.new do |s|
     test/test_core_ext.rb
     test/test_filters.rb
     test/test_generated_site.rb
+    test/test_kramdown.rb
     test/test_page.rb
     test/test_pager.rb
     test/test_post.rb

lib/jekyll.rb

@@ -19,10 +19,12 @@ require 'rubygems'
 require 'fileutils'
 require 'time'
 require 'yaml'
+require 'English'
 
 # 3rd party
 require 'liquid'
 require 'maruku'
+require 'albino'
 
 # internal requires
 require 'jekyll/core_ext'
@@ -32,7 +34,6 @@ require 'jekyll/layout'
 require 'jekyll/page'
 require 'jekyll/post'
 require 'jekyll/filters'
-require 'jekyll/albino'
 require 'jekyll/static_file'
 require 'jekyll/errors'
 
@@ -45,7 +46,7 @@ require_all 'jekyll/generators'
 require_all 'jekyll/tags'
 
 module Jekyll
-  VERSION = '0.7.0'
+  VERSION = '0.10.0'
 
   # Default options. Overriden by values in _config.yml or command-line opts.
   # (Strings rather symbols used for compatability with YAML).
@@ -74,6 +75,22 @@ module Jekyll
     },
     'rdiscount'    => {
       'extensions' => []
+    },
+    'kramdown'        => {
+      'auto_ids'      => true,
+      'footnote_nr'   => 1,
+      'entity_output' => 'as_char',
+      'toc_levels'    => '1..6',
+      'use_coderay'   => false,
+
+      'coderay' => {
+        'coderay_wrap'              => 'div',
+        'coderay_line_numbers'      => 'inline',
+        'coderay_line_number_start' => 1,
+        'coderay_tab_width'         => 4,
+        'coderay_bold_every'        => 10,
+        'coderay_css'               => 'style'
+      }
     }
   }
 

lib/jekyll/albino.rb

@@ -1,120 +0,0 @@
-##
-# Wrapper for the Pygments command line tool, pygmentize.
-#
-# Pygments: http://pygments.org/
-#
-# Assumes pygmentize is in the path.  If not, set its location
-# with Albino.bin = '/path/to/pygmentize'
-#
-# Use like so:
-#
-#   @syntaxer = Albino.new('/some/file.rb', :ruby)
-#   puts @syntaxer.colorize
-#
-# This'll print out an HTMLized, Ruby-highlighted version
-# of '/some/file.rb'.
-#
-# To use another formatter, pass it as the third argument:
-#
-#   @syntaxer = Albino.new('/some/file.rb', :ruby, :bbcode)
-#   puts @syntaxer.colorize
-#
-# You can also use the #colorize class method:
-#
-#   puts Albino.colorize('/some/file.rb', :ruby)
-#
-# Another also: you get a #to_s, for somewhat nicer use in Rails views.
-#
-#   ... helper file ...
-#   def highlight(text)
-#     Albino.new(text, :ruby)
-#   end
-#
-#   ... view file ...
-#   <%= highlight text %>
-#
-# The default lexer is 'text'.  You need to specify a lexer yourself;
-# because we are using STDIN there is no auto-detect.
-#
-# To see all lexers and formatters available, run `pygmentize -L`.
-#
-# Chris Wanstrath // chris@ozmm.org
-#         GitHub // http://github.com
-#
-
-class Albino
-  @@bin = Rails.development? ? 'pygmentize' : '/usr/bin/pygmentize' rescue 'pygmentize'
-
-  def self.bin=(path)
-    @@bin = path
-  end
-
-  def self.colorize(*args)
-    new(*args).colorize
-  end
-
-  def initialize(target, lexer = :text, format = :html)
-    @target  = target
-    @options = { :l => lexer, :f => format, :O => 'encoding=utf-8' }
-  end
-
-  def execute(command)
-    output = ''
-    IO.popen(command, mode='r+') do |p|
-      p.write @target
-      p.close_write
-      output = p.read.strip
-    end
-    output
-  end
-
-  def colorize(options = {})
-    html = execute(@@bin + convert_options(options))
-    # Work around an RDiscount bug: http://gist.github.com/97682
-    html.to_s.sub(%r{</pre></div>\Z}, "</pre>\n</div>")
-  end
-  alias_method :to_s, :colorize
-
-  def convert_options(options = {})
-    @options.merge(options).inject('') do |string, (flag, value)|
-      string + " -#{flag} #{value}"
-    end
-  end
-end
-
-if $0 == __FILE__
-  require 'rubygems'
-  require 'test/spec'
-  require 'mocha'
-  begin require 'redgreen'; rescue LoadError; end
-
-  context "Albino" do
-    setup do
-      @syntaxer = Albino.new(__FILE__, :ruby)
-    end
-
-    specify "defaults to text" do
-      syntaxer = Albino.new(__FILE__)
-      syntaxer.expects(:execute).with('pygmentize -f html -l text').returns(true)
-      syntaxer.colorize
-    end
-
-    specify "accepts options" do
-      @syntaxer.expects(:execute).with('pygmentize -f html -l ruby').returns(true)
-      @syntaxer.colorize
-    end
-
-    specify "works with strings" do
-      syntaxer = Albino.new('class New; end', :ruby)
-      assert_match %r(highlight), syntaxer.colorize
-    end
-
-    specify "aliases to_s" do
-      assert_equal @syntaxer.colorize, @syntaxer.to_s
-    end
-
-    specify "class method colorize" do
-      assert_equal @syntaxer.colorize, Albino.colorize(__FILE__, :ruby)
-    end
-  end
-end

lib/jekyll/converters/markdown.rb

@@ -10,6 +10,14 @@ module Jekyll
       return if @setup
       # Set the Markdown interpreter (and Maruku self.config, if necessary)
       case @config['markdown']
+        when 'kramdown'
+          begin
+            require 'kramdown'
+          rescue LoadError
+            STDERR.puts 'You are missing a library required for Markdown. Please run:'
+            STDERR.puts '  $ [sudo] gem install kramdown'
+            raise FatalException.new("Missing dependency: kramdown")
+          end
         when 'rdiscount'
           begin
             require 'rdiscount'
@@ -52,7 +60,7 @@ module Jekyll
           end
         else
           STDERR.puts "Invalid Markdown processor: #{@config['markdown']}"
-          STDERR.puts "  Valid options are [ maruku | rdiscount ]"
+          STDERR.puts "  Valid options are [ maruku | rdiscount | kramdown ]"
           raise FatalException.new("Invalid Markdown process: #{@config['markdown']}")
       end
       @setup = true
@@ -69,6 +77,31 @@ module Jekyll
     def convert(content)
       setup
       case @config['markdown']
+        when 'kramdown'
+          # Check for use of coderay
+          if @config['kramdown']['use_coderay']
+            Kramdown::Document.new(content, {
+              :auto_ids      => @config['kramdown']['auto_ids'],
+              :footnote_nr   => @config['kramdown']['footnote_nr'],
+              :entity_output => @config['kramdown']['entity_output'],
+              :toc_levels    => @config['kramdown']['toc_levels'],
+
+              :coderay_wrap               => @config['kramdown']['coderay']['coderay_wrap'],
+              :coderay_line_numbers       => @config['kramdown']['coderay']['coderay_line_numbers'],
+              :coderay_line_number_start  => @config['kramdown']['coderay']['coderay_line_number_start'],
+              :coderay_tab_width          => @config['kramdown']['coderay']['coderay_tab_width'],
+              :coderay_bold_every         => @config['kramdown']['coderay']['coderay_bold_every'],
+              :coderay_css                => @config['kramdown']['coderay']['coderay_css']
+            }).to_html
+          else
+            # not using coderay
+            Kramdown::Document.new(content, {
+              :auto_ids      => @config['kramdown']['auto_ids'],
+              :footnote_nr   => @config['kramdown']['footnote_nr'],
+              :entity_output => @config['kramdown']['entity_output'],
+              :toc_levels    => @config['kramdown']['toc_levels']
+            }).to_html
+          end
         when 'rdiscount'
           RDiscount.new(content, *@rdiscount_extensions).to_html
         when 'maruku'

lib/jekyll/convertible.rb

@@ -10,23 +10,28 @@
 #   self.output=
 module Jekyll
   module Convertible
-    # Return the contents as a string
+    # Returns the contents as a String.
     def to_s
       self.content || ''
     end
 
-    # Read the YAML frontmatter
-    #   +base+ is the String path to the dir containing the file
-    #   +name+ is the String filename of the file
+    # Read the YAML frontmatter.
     #
-    # Returns nothing
+    # base - The String path to the dir containing the file.
+    # name - The String filename of the file.
+    #
+    # Returns nothing.
     def read_yaml(base, name)
       self.content = File.read(File.join(base, name))
 
       if self.content =~ /^(---\s*\n.*?\n?)^(---\s*$\n?)/m
-        self.content = self.content[($1.size + $2.size)..-1]
+        self.content = $POSTMATCH
 
-        self.data = YAML.load($1)
+        begin
+          self.data = YAML.load($1)
+        rescue => e
+          puts "YAML Exception: #{e.message}"
+        end
       end
 
       self.data ||= {}
@@ -34,36 +39,46 @@ module Jekyll
 
     # Transform the contents based on the content type.
     #
-    # Returns nothing
+    # Returns nothing.
     def transform
       self.content = converter.convert(self.content)
     end
 
-    # Determine the extension depending on content_type
+    # Determine the extension depending on content_type.
     #
-    # Returns the extensions for the output file
+    # Returns the String extension for the output file.
+    #   e.g. ".html" for an HTML output file.
     def output_ext
       converter.output_ext(self.ext)
     end
 
     # Determine which converter to use based on this convertible's
-    # extension
+    # extension.
+    #
+    # Returns the Converter instance.
     def converter
       @converter ||= self.site.converters.find { |c| c.matches(self.ext) }
     end
 
-    # Add any necessary layouts to this convertible document
-    #   +layouts+ is a Hash of {"name" => "layout"}
-    #   +site_payload+ is the site payload hash
+    # Add any necessary layouts to this convertible document.
     #
-    # Returns nothing
+    # payload - The site payload Hash.
+    # layouts - A Hash of {"name" => "layout"}.
+    #
+    # Returns nothing.
     def do_layout(payload, layouts)
       info = { :filters => [Jekyll::Filters], :registers => { :site => self.site } }
 
       # render and transform content (this becomes the final content of the object)
       payload["pygments_prefix"] = converter.pygments_prefix
       payload["pygments_suffix"] = converter.pygments_suffix
-      self.content = Liquid::Template.parse(self.content).render(payload, info)
+
+      begin
+        self.content = Liquid::Template.parse(self.content).render(payload, info)
+      rescue => e
+        puts "Liquid Exception: #{e.message} in #{self.data["layout"]}"
+      end
+
       self.transform
 
       # output keeps track of what will finally be written
@@ -73,7 +88,12 @@ module Jekyll
       layout = layouts[self.data["layout"]]
       while layout
         payload = payload.deep_merge({"content" => self.output, "page" => layout.data})
-        self.output = Liquid::Template.parse(layout.content).render(payload, info)
+
+        begin
+          self.output = Liquid::Template.parse(layout.content).render(payload, info)
+        rescue => e
+          puts "Liquid Exception: #{e.message} in #{self.data["layout"]}"
+        end
 
         layout = layouts[layout.data["layout"]]
       end

lib/jekyll/filters.rb

@@ -1,3 +1,5 @@
+require 'uri'
+
 module Jekyll
 
   module Filters
@@ -25,6 +27,10 @@ module Jekyll
       CGI::escape(input)
     end
 
+    def uri_escape(input)
+      URI.escape(input)
+    end
+
     def number_of_words(input)
       input.split.length
     end

lib/jekyll/migrators/drupal.rb

@@ -0,0 +1,86 @@
+require 'rubygems'
+require 'sequel'
+require 'fileutils'
+require 'yaml'
+
+# NOTE: This converter requires Sequel and the MySQL gems.
+# The MySQL gem can be difficult to install on OS X. Once you have MySQL
+# installed, running the following commands should work:
+# $ sudo gem install sequel
+# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config
+
+module Jekyll
+  module Drupal
+
+    # Reads a MySQL database via Sequel and creates a post file for each
+    # post in wp_posts that has post_status = 'publish'.
+    # This restriction is made because 'draft' posts are not guaranteed to
+    # have valid dates.
+    QUERY = "SELECT node.nid, node.title, node_revisions.body, node.created, node.status FROM node, node_revisions WHERE (node.type = 'blog' OR node.type = 'story') AND node.vid = node_revisions.vid"
+
+    def self.process(dbname, user, pass, host = 'localhost')
+      db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8')
+
+      FileUtils.mkdir_p "_posts"
+      FileUtils.mkdir_p "_drafts"
+
+      # Create the refresh layout
+      # Change the refresh url if you customized your permalink config
+      File.open("_layouts/refresh.html", "w") do |f|
+        f.puts <<EOF
+<!DOCTYPE html>
+<html>
+<head>
+<meta http-equiv="content-type" content="text/html; charset=utf-8" />
+<meta http-equiv="refresh" content="0;url={{ page.refresh_to_post_id }}.html" />
+</head>
+</html>
+EOF
+      end
+
+      db[QUERY].each do |post|
+        # Get required fields and construct Jekyll compatible name
+        node_id = post[:nid]
+        title = post[:title]
+        content = post[:body]
+        created = post[:created]
+        time = Time.at(created)
+        is_published = post[:status] == 1
+        dir = is_published ? "_posts" : "_drafts"
+        slug = title.strip.downcase.gsub(/(&|&amp;)/, ' and ').gsub(/[\s\.\/\\]/, '-').gsub(/[^\w-]/, '').gsub(/[-_]{2,}/, '-').gsub(/^[-_]/, '').gsub(/[-_]$/, '')
+        name = time.strftime("%Y-%m-%d-") + slug + '.md'
+
+        # Get the relevant fields as a hash, delete empty fields and convert
+        # to YAML for the header
+        data = {
+           'layout' => 'post',
+           'title' => title.to_s,
+           'created' => created,
+         }.delete_if { |k,v| v.nil? || v == ''}.to_yaml
+
+        # Write out the data and content to file
+        File.open("#{dir}/#{name}", "w") do |f|
+          f.puts data
+          f.puts "---"
+          f.puts content
+        end
+
+        # Make a file to redirect from the old Drupal URL
+        if is_published
+          FileUtils.mkdir_p "node/#{node_id}"
+          File.open("node/#{node_id}/index.md", "w") do |f|
+            f.puts "---"
+            f.puts "layout: refresh"
+            f.puts "refresh_to_post_id: /#{time.strftime("%Y/%m/%d/") + slug}"
+            f.puts "---"
+          end
+        end
+      end
+
+      # TODO: Make dirs & files for nodes of type 'page'
+        # Make refresh pages for these as well
+
+      # TODO: Make refresh dirs & files according to entries in url_alias table
+    end
+  end
+end

lib/jekyll/migrators/marley.rb

@@ -0,0 +1,53 @@
+require 'yaml'
+require 'fileutils'
+
+module Jekyll
+  module Marley
+
+    def self.regexp
+      { :id    => /^\d{0,4}-{0,1}(.*)$/,
+        :title => /^#\s*(.*)\s+$/,
+        :title_with_date => /^#\s*(.*)\s+\(([0-9\/]+)\)$/,
+        :published_on => /.*\s+\(([0-9\/]+)\)$/,
+        :perex => /^([^\#\n]+\n)$/,
+        :meta  => /^\{\{\n(.*)\}\}\n$/mi # Multiline Regexp 
+      }
+    end
+
+    def self.process(marley_data_dir)
+      raise ArgumentError, "marley dir #{marley_data_dir} not found" unless File.directory?(marley_data_dir)
+
+      FileUtils.mkdir_p "_posts"
+
+      posts = 0
+      Dir["#{marley_data_dir}/**/*.txt"].each do |f|
+        next unless File.exists?(f)
+
+        #copied over from marley's app/lib/post.rb
+        file_content  = File.read(f)
+        meta_content  = file_content.slice!( self.regexp[:meta] )
+        body          = file_content.sub( self.regexp[:title], '').sub( self.regexp[:perex], '').strip
+   
+        title = file_content.scan( self.regexp[:title] ).first.to_s.strip
+        prerex = file_content.scan( self.regexp[:perex] ).first.to_s.strip
+        published_on = DateTime.parse( post[:published_on] ) rescue File.mtime( File.dirname(f) )
+        meta          = ( meta_content ) ? YAML::load( meta_content.scan( self.regexp[:meta]).to_s ) : {}
+        meta['title'] = title
+        meta['layout'] = 'post'
+
+        formatted_date = published_on.strftime('%Y-%m-%d')
+        post_name =  File.dirname(f).split(%r{/}).last.gsub(/\A\d+-/, '')
+
+        name = "#{formatted_date}-#{post_name}" 
+        File.open("_posts/#{name}.markdown", "w") do |f|
+          f.puts meta.to_yaml
+          f.puts "---\n"
+          f.puts "\n#{prerex}\n\n" if prerex
+          f.puts body
+        end
+        posts += 1
+      end
+      "Created #{posts} posts!"
+    end
+  end
+end

lib/jekyll/migrators/mephisto.rb

@@ -40,7 +40,7 @@ module Jekyll
     QUERY = "SELECT id, permalink, body, published_at, title FROM contents WHERE user_id = 1 AND type = 'Article' AND published_at IS NOT NULL ORDER BY published_at"
 
     def self.process(dbname, user, pass, host = 'localhost')
-      db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host)
+      db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8')
 
       FileUtils.mkdir_p "_posts"
 

lib/jekyll/migrators/mt.rb

@@ -5,6 +5,7 @@
 require 'rubygems'
 require 'sequel'
 require 'fileutils'
+require 'yaml'
 
 # NOTE: This converter requires Sequel and the MySQL gems.
 # The MySQL gem can be difficult to install on OS X. Once you have MySQL
@@ -17,19 +18,20 @@ module Jekyll
     # This query will pull blog posts from all entries across all blogs. If
     # you've got unpublished, deleted or otherwise hidden posts please sift
     # through the created posts to make sure nothing is accidently published.
-    QUERY = "SELECT entry_id, entry_basename, entry_text, entry_text_more, entry_created_on, entry_title FROM mt_entry"
+    QUERY = "SELECT entry_id, entry_basename, entry_text, entry_text_more, entry_authored_on, entry_title, entry_convert_breaks FROM mt_entry"
 
     def self.process(dbname, user, pass, host = 'localhost')
-      db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host)
+      db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8')
 
       FileUtils.mkdir_p "_posts"
 
       db[QUERY].each do |post|
         title = post[:entry_title]
-        slug = post[:entry_basename]
-        date = post[:entry_created_on]
+        slug = post[:entry_basename].gsub(/_/, '-')
+        date = post[:entry_authored_on]
         content = post[:entry_text]
         more_content = post[:entry_text_more]
+        entry_convert_breaks = post[:entry_convert_breaks]
 
         # Be sure to include the body and extended body.
         if more_content != nil
@@ -39,12 +41,13 @@ module Jekyll
         # Ideally, this script would determine the post format (markdown, html
         # , etc) and create files with proper extensions. At this point it
         # just assumes that markdown will be acceptable.
-        name = [date.year, date.month, date.day, slug].join('-') + ".markdown"
+        name = [date.year, date.month, date.day, slug].join('-') + '.' + self.suffix(entry_convert_breaks)
 
         data = {
            'layout' => 'post',
            'title' => title.to_s,
            'mt_id' => post[:entry_id],
+           'date' => date
          }.delete_if { |k,v| v.nil? || v == ''}.to_yaml
 
         File.open("_posts/#{name}", "w") do |f|
@@ -53,7 +56,22 @@ module Jekyll
           f.puts content
         end
       end
+    end
 
+    def self.suffix(entry_type)
+      if entry_type.nil? || entry_type.include?("markdown")
+          # The markdown plugin I have saves this as "markdown_with_smarty_pants", so I just look for "markdown".
+          "markdown"
+        elsif entry_type.include?("textile")
+          # This is saved as "textile_2" on my installation of MT 5.1.
+          "textile"
+        elsif entry_type == "0" || entry_type.include?("richtext")
+          # richtext looks to me like it's saved as HTML, so I include it here.
+          "html"
+        else
+          # Other values might need custom work.
+          entry_type
+      end
     end
   end
 end

lib/jekyll/migrators/posterous.rb

@@ -0,0 +1,73 @@
+require 'rubygems'
+require 'jekyll'
+require 'fileutils'
+require 'net/http'
+require 'uri'
+require "json"
+
+# ruby -r './lib/jekyll/migrators/posterous.rb' -e 'Jekyll::Posterous.process(email, pass, blog)'
+
+module Jekyll
+  module Posterous
+    
+    def self.fetch(uri_str, limit = 10)
+      # You should choose better exception.
+      raise ArgumentError, 'Stuck in a redirect loop. Please double check your email and password' if limit == 0
+      
+      response = nil
+      Net::HTTP.start('posterous.com') {|http|
+        req = Net::HTTP::Get.new(uri_str)
+        req.basic_auth @email, @pass
+        response = http.request(req)
+      }
+
+      case response
+        when Net::HTTPSuccess     then response
+        when Net::HTTPRedirection then fetch(response['location'], limit - 1)
+        else response.error!
+      end
+    end
+        
+        
+    def self.process(email, pass, blog = 'primary')
+      @email, @pass = email, pass
+      @api_token = JSON.parse(self.fetch("/api/2/auth/token").body)['api_token']
+      FileUtils.mkdir_p "_posts"
+      
+      posts = JSON.parse(self.fetch("/api/v2/users/me/sites/#{blog}/posts?api_token=#{@api_token}").body)
+      page = 1
+      
+      while posts.any?
+      
+        posts.each do |post|
+          title = post["title"]
+          slug = title.gsub(/[^[:alnum:]]+/, '-').downcase  
+          date = Date.parse(post["display_date"])
+          content = post["body_html"]
+          published = !post["is_private"]
+          name = "%02d-%02d-%02d-%s.html" % [date.year, date.month, date.day, slug]
+        
+          # Get the relevant fields as a hash, delete empty fields and convert
+          # to YAML for the header
+          data = {
+             'layout' => 'post',
+             'title' => title.to_s,
+             'published' => published
+           }.delete_if { |k,v| v.nil? || v == ''}.to_yaml
+
+          # Write out the data and content to file
+          File.open("_posts/#{name}", "w") do |f|
+            f.puts data
+            f.puts "---"
+            f.puts content
+          end
+        
+        end
+        
+        page += 1
+        posts = JSON.parse(self.fetch("/api/v2/users/me/sites/#{blog}/posts?api_token=#{@api_token}&page=#{page}").body)        
+      end
+      
+    end
+  end
+end

lib/jekyll/migrators/textpattern.rb

@@ -17,7 +17,7 @@ module Jekyll
     QUERY = "select Title, url_title, Posted, Body, Keywords from textpattern where Status = '4' or Status = '5'"
 
     def self.process(dbname, user, pass, host = 'localhost')
-      db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host)
+      db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8')
 
       FileUtils.mkdir_p "_posts"
 

lib/jekyll/migrators/typo.rb

@@ -22,7 +22,7 @@ module Jekyll
 
     def self.process dbname, user, pass, host='localhost'
       FileUtils.mkdir_p '_posts'
-      db = Sequel.mysql dbname, :user => user, :password => pass, :host => host
+      db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8')
       db[SQL].each do |post|
         next unless post[:state] =~ /Published/
 

lib/jekyll/migrators/wordpress.rb

@@ -12,18 +12,18 @@ require 'yaml'
 module Jekyll
   module WordPress
 
-    # Reads a MySQL database via Sequel and creates a post file for each
-    # post in wp_posts that has post_status = 'publish'.
-    # This restriction is made because 'draft' posts are not guaranteed to
-    # have valid dates.
-    QUERY = "select post_title, post_name, post_date, post_content, post_excerpt, ID, guid from wp_posts where post_status = 'publish' and post_type = 'post'"
-
-    def self.process(dbname, user, pass, host = 'localhost')
-      db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host)
+    def self.process(dbname, user, pass, host = 'localhost', table_prefix = 'wp_')
+      db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8')
 
       FileUtils.mkdir_p "_posts"
+      
+      # Reads a MySQL database via Sequel and creates a post file for each
+      # post in wp_posts that has post_status = 'publish'.
+      # This restriction is made because 'draft' posts are not guaranteed to
+      # have valid dates.
+      query = "select post_title, post_name, post_date, post_content, post_excerpt, ID, guid from #{table_prefix}posts where post_status = 'publish' and post_type = 'post'"
 
-      db[QUERY].each do |post|
+      db[query].each do |post|
         # Get required fields and construct Jekyll compatible name
         title = post[:post_title]
         slug = post[:post_name]
@@ -39,7 +39,8 @@ module Jekyll
            'title' => title.to_s,
            'excerpt' => post[:post_excerpt].to_s,
            'wordpress_id' => post[:ID],
-           'wordpress_url' => post[:guid]
+           'wordpress_url' => post[:guid],
+           'date' => date
          }.delete_if { |k,v| v.nil? || v == ''}.to_yaml
 
         # Write out the data and content to file

lib/jekyll/migrators/wordpressdotcom.rb

@@ -0,0 +1,45 @@
+# coding: utf-8
+
+require 'rubygems'
+require 'hpricot'
+require 'fileutils'
+require 'yaml'
+
+module Jekyll
+  
+  # This importer takes a wordpress.xml file,
+  # which can be exported from your 
+  # wordpress.com blog (/wp-admin/export.php)
+  module WordpressDotCom
+    def self.process(filename = "wordpress.xml")
+      FileUtils.mkdir_p "_posts"
+      posts = 0
+
+			doc = Hpricot::XML(File.read(filename))
+			
+			(doc/:channel/:item).each do |item|
+				title = item.at(:title).inner_text.strip
+				permalink_title = item.at('wp:post_name').inner_text
+				date = Time.parse(item.at(:pubDate).inner_text)
+				tags = (item/:category).map{|c| c.inner_text}.reject{|c| c == 'Uncategorized'}.uniq
+				name = "#{date.strftime('%Y-%m-%d')}-#{permalink_title}.html"
+			  header = {
+			    'layout' => 'post', 
+			    'title'  => title, 
+			    'tags'   => tags
+			  }
+			  
+				File.open("_posts/#{name}", "w") do |f|
+				  f.puts header.to_yaml
+				  f.puts '---'
+          f.puts item.at('content:encoded').inner_text
+        end
+
+				posts += 1
+			end
+
+			puts "Imported #{posts} posts"
+    end
+  end
+  
+end

lib/jekyll/page.rb

@@ -55,14 +55,23 @@ module Jekyll
     #
     # Returns <String>
     def url
-      return permalink if permalink
+      return @url if @url
 
-      @url ||= {
-        "basename"   => self.basename,
-        "output_ext" => self.output_ext,
-      }.inject(template) { |result, token|
-        result.gsub(/:#{token.first}/, token.last)
-      }.gsub(/\/\//, "/")
+      url = if permalink
+        permalink
+      else
+        {
+          "basename"   => self.basename,
+          "output_ext" => self.output_ext,
+        }.inject(template) { |result, token|
+          result.gsub(/:#{token.first}/, token.last)
+        }.gsub(/\/\//, "/")
+      end
+
+      # sanitize url
+      @url = url.split('/').reject{ |part| part =~ /^\.+$/ }.join('/')
+      @url += "/" if url =~ /\/$/
+      @url
     end
 
     # Extract information from the page filename
@@ -71,7 +80,7 @@ module Jekyll
     # Returns nothing
     def process(name)
       self.ext = File.extname(name)
-      self.basename = name.split('.')[0..-2].first
+      self.basename = name[0 .. -self.ext.length-1]
     end
 
     # Add any necessary layouts to this post
@@ -93,24 +102,25 @@ module Jekyll
         "url"        => File.join(@dir, self.url),
         "content"    => self.content })
     end
+    
+    # Obtain destination path.
+    #   +dest+ is the String path to the destination dir
+    #
+    # Returns destination file path.
+    def destination(dest)
+      # The url needs to be unescaped in order to preserve the correct filename
+      path = File.join(dest, @dir, CGI.unescape(self.url))
+      path = File.join(path, "index.html") if self.url =~ /\/$/
+      path
+    end
 
     # Write the generated page file to the destination directory.
-    #   +dest_prefix+ is the String path to the destination dir
-    #   +dest_suffix+ is a suffix path to the destination dir
+    #   +dest+ is the String path to the destination dir
     #
     # Returns nothing
-    def write(dest_prefix, dest_suffix = nil)
-      dest = File.join(dest_prefix, @dir)
-      dest = File.join(dest, dest_suffix) if dest_suffix
-      FileUtils.mkdir_p(dest)
-
-      # The url needs to be unescaped in order to preserve the correct filename
-      path = File.join(dest, CGI.unescape(self.url))
-      if self.url =~ /\/$/
-        FileUtils.mkdir_p(path)
-        path = File.join(path, "index.html")
-      end
-
+    def write(dest)
+      path = destination(dest)
+      FileUtils.mkdir_p(File.dirname(path))
       File.open(path, 'w') do |f|
         f.write(self.output)
       end

lib/jekyll/post.rb

@@ -78,6 +78,8 @@ module Jekyll
       self.date = Time.parse(date)
       self.slug = slug
       self.ext = ext
+    rescue ArgumentError
+      raise FatalException.new("Post #{name} does not have a valid date.")
     end
 
     # The generated directory into which the post will be placed
@@ -117,20 +119,29 @@ module Jekyll
     #
     # Returns <String>
     def url
-      return permalink if permalink
+      return @url if @url
 
-      @url ||= {
-        "year"       => date.strftime("%Y"),
-        "month"      => date.strftime("%m"),
-        "day"        => date.strftime("%d"),
-        "title"      => CGI.escape(slug),
-        "i_day"      => date.strftime("%d").to_i.to_s,
-        "i_month"    => date.strftime("%m").to_i.to_s,
-        "categories" => categories.join('/'),
-        "output_ext" => self.output_ext
-      }.inject(template) { |result, token|
-        result.gsub(/:#{Regexp.escape token.first}/, token.last)
-      }.gsub(/\/\//, "/")
+      url = if permalink
+        permalink
+      else
+        {
+          "year"       => date.strftime("%Y"),
+          "month"      => date.strftime("%m"),
+          "day"        => date.strftime("%d"),
+          "title"      => CGI.escape(slug),
+          "i_day"      => date.strftime("%d").to_i.to_s,
+          "i_month"    => date.strftime("%m").to_i.to_s,
+          "categories" => categories.join('/'),
+          "output_ext" => self.output_ext
+        }.inject(template) { |result, token|
+          result.gsub(/:#{Regexp.escape token.first}/, token.last)
+        }.gsub(/\/\//, "/")
+      end
+
+      # sanitize url
+      @url = url.split('/').reject{ |part| part =~ /^\.+$/ }.join('/')
+      @url += "/" if url =~ /\/$/
+      @url
     end
 
     # The UID for this post (useful in feeds)
@@ -177,22 +188,25 @@ module Jekyll
 
       do_layout(payload, layouts)
     end
+    
+    # Obtain destination path.
+    #   +dest+ is the String path to the destination dir
+    #
+    # Returns destination file path.
+    def destination(dest)
+      # The url needs to be unescaped in order to preserve the correct filename
+      path = File.join(dest, CGI.unescape(self.url))
+      path = File.join(path, "index.html") if template[/\.html$/].nil?
+      path
+    end
 
     # Write the generated post file to the destination directory.
     #   +dest+ is the String path to the destination dir
     #
     # Returns nothing
     def write(dest)
-      FileUtils.mkdir_p(File.join(dest, dir))
-
-      # The url needs to be unescaped in order to preserve the correct filename
-      path = File.join(dest, CGI.unescape(self.url))
-
-      if template[/\.html$/].nil?
-        FileUtils.mkdir_p(path)
-        path = File.join(path, "index.html")
-      end
-
+      path = destination(dest)
+      FileUtils.mkdir_p(File.dirname(path))
       File.open(path, 'w') do |f|
         f.write(self.output)
       end

lib/jekyll/site.rb

@@ -1,9 +1,12 @@
+require 'set'
+
 module Jekyll
 
   class Site
     attr_accessor :config, :layouts, :posts, :pages, :static_files,
                   :categories, :exclude, :source, :dest, :lsi, :pygments,
-                  :permalink_style, :tags, :time, :future, :safe, :plugins
+                  :permalink_style, :tags, :time, :future, :safe, :plugins, :limit_posts
+
     attr_accessor :converters, :generators
 
     # Initialize the site
@@ -22,6 +25,7 @@ module Jekyll
       self.permalink_style = config['permalink'].to_sym
       self.exclude         = config['exclude'] || []
       self.future          = config['future']
+      self.limit_posts     = config['limit_posts'] || nil
 
       self.reset
       self.setup
@@ -39,6 +43,8 @@ module Jekyll
       self.static_files    = []
       self.categories      = Hash.new { |hash, key| hash[key] = [] }
       self.tags            = Hash.new { |hash, key| hash[key] = [] }
+
+      raise ArgumentError, "Limit posts must be nil or >= 1" if !self.limit_posts.nil? && self.limit_posts < 1
     end
 
     def setup
@@ -75,6 +81,7 @@ module Jekyll
       self.read
       self.generate
       self.render
+      self.cleanup
       self.write
     end
 
@@ -122,6 +129,9 @@ module Jekyll
       end
 
       self.posts.sort!
+
+      # limit the posts if :limit_posts option is set
+      self.posts = self.posts[-limit_posts, limit_posts] if limit_posts
     end
 
     def generate
@@ -144,6 +154,38 @@ module Jekyll
     rescue Errno::ENOENT => e
       # ignore missing layout dir
     end
+    
+    # Remove orphaned files and empty directories in destination
+    #
+    # Returns nothing
+    def cleanup
+      # all files and directories in destination, including hidden ones
+      dest_files = Set.new
+      Dir.glob(File.join(self.dest, "**", "*"), File::FNM_DOTMATCH) do |file|
+        dest_files << file unless file =~ /\/\.{1,2}$/
+      end
+
+      # files to be written
+      files = Set.new
+      self.posts.each do |post|
+        files << post.destination(self.dest)
+      end
+      self.pages.each do |page|
+        files << page.destination(self.dest)
+      end
+      self.static_files.each do |sf|
+        files << sf.destination(self.dest)
+      end
+      
+      # adding files' parent directories
+      dirs = Set.new
+      files.each { |file| dirs << File.dirname(file) }
+      files.merge(dirs)
+      
+      obsolete_files = dest_files - files
+      
+      FileUtils.rm_rf(obsolete_files.to_a)
+    end
 
     # Write static files, pages and posts
     #
@@ -168,7 +210,7 @@ module Jekyll
     # Returns nothing
     def read_directories(dir = '')
       base = File.join(self.source, dir)
-      entries = filter_entries(Dir.entries(base))
+      entries = Dir.chdir(base){ filter_entries(Dir['*']) }
 
       self.read_posts(dir)
 
@@ -226,7 +268,10 @@ module Jekyll
     def filter_entries(entries)
       entries = entries.reject do |e|
         unless ['.htaccess'].include?(e)
-          ['.', '_', '#'].include?(e[0..0]) || e[-1..-1] == '~' || self.exclude.include?(e)
+          ['.', '_', '#'].include?(e[0..0]) ||
+          e[-1..-1] == '~' ||
+          self.exclude.include?(e) ||
+          File.symlink?(e)
         end
       end
     end

lib/jekyll/static_file.rb

@@ -52,12 +52,11 @@ module Jekyll
     # Returns false if the file was not modified since last time (no-op).
     def write(dest)
       dest_path = destination(dest)
-      dest_dir = File.join(dest, @dir)
 
       return false if File.exist? dest_path and !modified?
       @@mtimes[path] = mtime
 
-      FileUtils.mkdir_p(dest_dir)
+      FileUtils.mkdir_p(File.dirname(dest_path))
       FileUtils.cp(path, dest_path)
 
       true

lib/jekyll/tags/include.rb

@@ -7,11 +7,17 @@ module Jekyll
     end
 
     def render(context)
+      includes_dir = File.join(context.registers[:site].source, '_includes')
+
+      if File.symlink?(includes_dir)
+        return "Includes directory '#{includes_dir}' cannot be a symlink"
+      end
+
       if @file !~ /^[a-zA-Z0-9_\/\.-]+$/ || @file =~ /\.\// || @file =~ /\/\./
         return "Include file '#{@file}' contains invalid characters or sequences"
       end
 
-      Dir.chdir(File.join(context.registers[:site].source, '_includes')) do
+      Dir.chdir(includes_dir) do
         choices = Dir['**/*'].reject { |x| File.symlink?(x) }
         if choices.include?(@file)
           source = File.read(@file)

test/helper.rb

@@ -5,8 +5,10 @@ require File.join(File.dirname(__FILE__), *%w[.. lib jekyll])
 
 require 'RedCloth'
 require 'rdiscount'
+require 'kramdown'
 
 require 'test/unit'
+require 'redgreen'
 require 'shoulda'
 require 'rr'
 

test/source/.htaccess

@@ -0,0 +1,8 @@
+---
+layout: nil
+---
+ErrorDocument 404 /404.html
+ErrorDocument 500 /500.html
+{% for post in site.posts %}
+  # {{ post.url }}
+{% endfor %}

test/source/deal.with.dots.html

@@ -0,0 +1,7 @@
+---
+title: Deal with dots
+permalink: /deal.with.dots/
+---
+
+Let's test if jekyll deals properly with dots.
+

test/suite.rb

@@ -1,11 +1,9 @@
-require 'rubygems'
-gem 'test-unit'
 require 'test/unit'
 
 # for some reason these tests fail when run via TextMate
 # but succeed when run on the command line.
 
-tests = Dir[File.expand_path("#{File.dirname(__FILE__)}/test_*.rb")]
+tests = Dir["#{File.dirname(__FILE__)}/test_*.rb"]
 tests.each do |file|
   require file
 end

test/test_filters.rb

@@ -45,5 +45,9 @@ class TestFilters < Test::Unit::TestCase
     should "escape special characters" do
       assert_equal "hey%21", @filter.cgi_escape("hey!")
     end
+
+    should "escape space as %20" do
+      assert_equal "my%20things", @filter.uri_escape("my things")
+    end
   end
 end

test/test_generated_site.rb

@@ -41,4 +41,32 @@ class TestGeneratedSite < Test::Unit::TestCase
       assert File.exists?(dest_dir('/contacts.html'))
     end
   end
+
+  context "generating limited posts" do
+    setup do
+      clear_dest
+      stub(Jekyll).configuration do
+        Jekyll::DEFAULTS.merge({'source' => source_dir, 'destination' => dest_dir, 'limit_posts' => 5})
+      end
+
+      @site = Site.new(Jekyll.configuration)
+      @site.process
+      @index = File.read(dest_dir('index.html'))
+    end
+
+    should "generate only the specified number of posts" do
+      assert_equal 5, @site.posts.size
+    end
+
+    should "ensure limit posts is 1 or more" do
+      assert_raise ArgumentError do
+        clear_dest
+        stub(Jekyll).configuration do
+          Jekyll::DEFAULTS.merge({'source' => source_dir, 'destination' => dest_dir, 'limit_posts' => 0})
+        end
+
+        @site = Site.new(Jekyll.configuration)
+      end
+    end
+  end
 end

test/test_kramdown.rb

@@ -0,0 +1,23 @@
+require File.dirname(__FILE__) + '/helper'
+
+class TestKramdown < Test::Unit::TestCase
+  context "kramdown" do
+    setup do
+      config = {
+        'markdown' => 'kramdown',
+        'kramdown' => {
+          'auto_ids'      => false,
+          'footnote_nr'   => 1,
+          'entity_output' => 'as_char',
+          'toc_levels'    => '1..6'
+        }
+      }
+      @markdown = MarkdownConverter.new config
+    end
+
+    # http://kramdown.rubyforge.org/converter/html.html#options
+    should "pass kramdown options" do
+      assert_equal "<h1>Some Header</h1>", @markdown.convert('# Some Header #').strip
+    end
+  end
+end

test/test_page.rb

@@ -16,13 +16,23 @@ class TestPage < Test::Unit::TestCase
       stub(Jekyll).configuration { Jekyll::DEFAULTS }
       @site = Site.new(Jekyll.configuration)
     end
-    
+
     context "processing pages" do
       should "create url based on filename" do
         @page = setup_page('contacts.html')
         assert_equal "/contacts.html", @page.url
       end
 
+      should "deal properly with extensions" do
+        @page = setup_page('deal.with.dots.html')
+        assert_equal ".html", @page.ext
+      end
+
+      should "deal properly with dots" do
+        @page = setup_page('deal.with.dots.html')
+        assert_equal "deal.with.dots", @page.basename
+      end
+
       context "with pretty url style" do
         setup do
           @site.permalink_style = :pretty
@@ -61,7 +71,7 @@ class TestPage < Test::Unit::TestCase
       setup do
         clear_dest
       end
-      
+
       should "write properly" do
         page = setup_page('contacts.html')
         do_render(page)
@@ -73,14 +83,14 @@ class TestPage < Test::Unit::TestCase
 
       should "write properly without html extension" do
         page = setup_page('contacts.html')
-        page.site.permalink_style = :pretty 
+        page.site.permalink_style = :pretty
         do_render(page)
         page.write(dest_dir)
 
         assert File.directory?(dest_dir)
         assert File.exists?(File.join(dest_dir, 'contacts', 'index.html'))
       end
-      
+
       should "write properly with extension different from html" do
         page = setup_page("sitemap.xml")
         page.site.permalink_style = :pretty
@@ -92,7 +102,16 @@ class TestPage < Test::Unit::TestCase
         assert File.directory?(dest_dir)
         assert File.exists?(File.join(dest_dir,'sitemap.xml'))
       end
+
+      should "write dotfiles properly" do
+        page = setup_page('.htaccess')
+        do_render(page)
+        page.write(dest_dir)
+
+        assert File.directory?(dest_dir)
+        assert File.exists?(File.join(dest_dir, '.htaccess'))
+      end
     end
 
-  end 
+  end
 end

test/test_post.rb

@@ -52,6 +52,12 @@ class TestPost < Test::Unit::TestCase
         assert_equal "/2008/09/09/foo-bar.html", @post.url
       end
 
+      should "raise a good error on invalid post date" do
+        assert_raise Jekyll::FatalException do
+          @post.process("2009-27-03-foo-bar.textile")
+        end
+      end
+
       should "CGI escape urls" do
         @post.categories = []
         @post.process("2009-03-12-hash-#1.markdown")

test/test_site.rb

@@ -132,6 +132,39 @@ class TestSite < Test::Unit::TestCase
       assert_equal includes, @site.filter_entries(excludes + includes)
     end
     
+    context 'with orphaned files in destination' do
+      setup do
+        clear_dest
+        @site.process
+        # generate some orphaned files:
+        # hidden file
+        File.open(dest_dir('.htpasswd'), 'w')
+        # single file
+        File.open(dest_dir('obsolete.html'), 'w')
+        # single file in sub directory
+        FileUtils.mkdir(dest_dir('qux'))
+        File.open(dest_dir('qux/obsolete.html'), 'w')
+        # empty directory
+        FileUtils.mkdir(dest_dir('quux'))
+      end
+      
+      teardown do
+        FileUtils.rm_f(dest_dir('.htpasswd'))
+        FileUtils.rm_f(dest_dir('obsolete.html'))
+        FileUtils.rm_rf(dest_dir('qux'))
+        FileUtils.rm_f(dest_dir('quux'))
+      end
+      
+      should 'remove orphaned files in destination' do
+        @site.process
+        assert !File.exist?(dest_dir('.htpasswd'))
+        assert !File.exist?(dest_dir('obsolete.html'))
+        assert !File.exist?(dest_dir('qux'))
+        assert !File.exist?(dest_dir('quux'))
+      end
+
+    end
+    
     context 'with an invalid markdown processor in the configuration' do
       should 'not throw an error at initialization time' do
         bad_processor = 'not a processor name'

test/test_tags.rb

@@ -1,5 +1,3 @@
-# coding: utf-8
-
 require File.dirname(__FILE__) + '/helper'
 
 class TestTags < Test::Unit::TestCase
@@ -114,5 +112,16 @@ CONTENT
         assert_match %r{<em>FINISH HIM</em>}, @result
       end
     end
+
+    context "using Kramdown" do
+      setup do
+        create_post(@content, 'markdown' => 'kramdown')
+      end
+
+      should "parse correctly" do
+        assert_match %r{<em>FIGHT!</em>}, @result
+        assert_match %r{<em>FINISH HIM</em>}, @result
+      end
+    end
   end
 end