More writing.

Move QUERY into process method

.gitignore

@@ -1,6 +1,8 @@
+Gemfile.lock
 test/dest
 *.gem
 pkg/
 *.swp
 *~
 _site/
+.bundle/

Gemfile

@@ -0,0 +1,2 @@
+source :rubygems
+gemspec

History.txt

@@ -1,3 +1,21 @@
+== HEAD
+  * Major Enhancements
+    * Add command line importer functionality (#253)
+  * Minor Enhancements
+    * Switch to Albino gem
+    * Bundler support
+    * Use English library to avoid hoops (#292)
+    * Add Posterous importer (#254)
+    * Fixes for Wordpress importer (#274, #252, #271)
+    * Better error message for invalid post date (#291)
+    * Print formatted fatal exceptions to stdout on build failure
+  * Bug Fixes
+    * Secure additional path exploits
+
+== 0.10.0 / 2010-12-16
+  * Bug Fixes
+    * Add --no-server option.
+
 == 0.9.0 / 2010-12-15
   * Minor Enhancements
     * Use OptionParser's [no-] functionality for better boolean parsing.
@@ -71,7 +89,7 @@
     * Empty tags causes error in read_posts (#84)
     * Fix pagination to adhere to read/render/write paradigm
   * Test Enhancement
-    * cucumber features no longer use site.ports.first where a better
+    * cucumber features no longer use site.posts.first where a better
       alternative is available
 
 == 0.5.6 / 2010-01-08

README.textile

@@ -27,7 +27,6 @@ h2. Runtime Dependencies
 * Classifier: Generating related posts (Ruby)
 * Maruku: Default markdown engine (Ruby)
 * Directory Watcher: Auto-regeneration of sites (Ruby)
-* Open4: Talking to pygments for syntax highlighting (Ruby)
 * Pygments: Syntax highlighting (Python)
 
 h2. Developer Dependencies

bin/jekyll

@@ -9,7 +9,8 @@ Basic Command Line Usage:
   jekyll                                                   # . -> ./_site
   jekyll <path to write generated site>                    # . -> <path>
   jekyll <path to source> <path to write generated site>   # <path> -> <path>
-
+  jekyll import <importer name> <options>                  # imports posts using named import script
+  
   Configuration is read from '<source>/_config.yml' but can be overriden
   using the following options:
 
@@ -18,11 +19,37 @@ HELP
 require 'optparse'
 require 'jekyll'
 
+
 exec = {}
 options = {}
 opts = OptionParser.new do |opts|
   opts.banner = help
 
+  opts.on("--file [PATH]", "File to import from") do |import_file|
+    options['file'] = import_file
+  end
+  
+  opts.on("--dbname [TEXT]", "DB to import from") do |import_dbname|
+    options['dbname'] = import_dbname
+  end
+  
+  opts.on("--user [TEXT]", "Username to use when importing") do |import_user|
+    options['user'] = import_user
+  end
+  
+  opts.on("--pass [TEXT]", "Password to use when importing") do |import_pass|
+    options['pass'] = import_pass
+  end
+  
+  opts.on("--host [HOST ADDRESS]", "Host to import from") do |import_host|
+    options['host'] = import_host
+  end
+  
+  opts.on("--site [SITE NAME]", "Site to import from") do |import_site|
+    options['site'] = import_site
+  end
+  
+
   opts.on("--[no-]safe", "Safe mode (default unsafe)") do |safe|
     options['safe'] = safe
   end
@@ -36,6 +63,10 @@ opts = OptionParser.new do |opts|
     options['server_port'] = port unless port.nil?
   end
 
+  opts.on("--no-server", "Do not start a web server") do |part|
+    options['server'] = false
+  end
+
   opts.on("--base-url [BASE_URL]", "Serve website from a given base URL (default '/'") do |baseurl|
       options['baseurl'] = baseurl
   end
@@ -101,6 +132,59 @@ end
 # Read command line options into `options` hash
 opts.parse!
 
+
+# Check for import stuff
+if ARGV.size > 0
+  if ARGV[0] == 'import'
+    migrator = ARGV[1]
+
+    if migrator.nil?
+      puts "Invalid options. Run `jekyll --help` for assistance."
+      exit(1)
+    else
+      migrator = migrator.downcase
+    end
+    
+    cmd_options = []
+    ['file', 'dbname', 'user', 'pass', 'host', 'site'].each do |p|
+      cmd_options << "\"#{options[p]}\"" unless options[p].nil?
+    end
+    
+    # It's import time
+    puts "Importing..."
+    
+    # Ideally, this shouldn't be necessary. Maybe parse the actual
+    # src files for the migrator name?
+    migrators = {
+      :posterous => 'Posterous',
+      :wordpressdotcom => 'WordpressDotCom',
+      :wordpress => 'Wordpress',
+      :csv => 'CSV',
+      :drupal => 'Drupal',
+      :mephisto => 'Mephisto',
+      :mt => 'MT',
+      :textpattern => 'TextPattern',
+      :typo => 'Typo'
+    }
+    
+    app_root = File.join(File.dirname(__FILE__), '..')
+    
+    require "#{app_root}/lib/jekyll/migrators/#{migrator}"
+    
+    if Jekyll.const_defined?(migrators[migrator.to_sym])
+      migrator_class = Jekyll.const_get(migrators[migrator.to_sym])
+      migrator_class.process(*cmd_options)
+    else
+      puts "Invalid migrator. Run `jekyll --help` for assistance."
+      exit(1)
+    end
+    
+    exit(0)
+  end
+end
+
+
+
 # Get source and destintation from command line
 case ARGV.size
   when 0
@@ -158,7 +242,11 @@ else
   puts "Building site: #{source} -> #{destination}"
   begin
     site.process
-  rescue Jekyll::FatalException
+  rescue Jekyll::FatalException => e
+    puts
+    puts "ERROR: YOUR SITE COULD NOT BE BUILT:"
+    puts "------------------------------------"
+    puts e.message
     exit(1)
   end
   puts "Successfully generated site: #{source} -> #{destination}"

doc/.gitignore

@@ -0,0 +1 @@
+output

doc/.gitscribe

@@ -0,0 +1,7 @@
+---
+publish:  true
+edition:  0.1
+language: en
+version:  1.0
+author:   Your Name
+cover:    image/cover.jpg

doc/README.asciidoc

@@ -0,0 +1,9 @@
+This Book
+=========
+
+This book is written using using the git-scribe toolchain, which can be found at:
+
+http://github.com/schacon/git-scribe
+
+Instructions on how to install the tool and use it for things like editing this book,
+submitting errata and providing translations can be found at that site.

doc/book/book.asc

@@ -0,0 +1,10 @@
+Jekyll
+======
+:Author:    Tom Preston-Werner
+:Email:     <tom@mojombo.com>
+
+include::ch00-preface.asc[]
+
+include::ch01-quick-start.asc[]
+
+include::ch02-directory-layout.asc[]

doc/book/ch00-preface.asc

@@ -0,0 +1,41 @@
+== Preface
+
+Jekyll was born out the desire to create a blog engine that would make it
+possible to write posts in my local text editor, version those posts with Git,
+and keep up with my desire to tweak the styles and layout of my site.
+
+In other words, I wanted something that fit into my existing software
+development workflow and toolchain. Jekyll handles not only this case, but a
+wide variety of other situations that call for static site generation based on
+converted content and layout templates.
+
+At its core, Jekyll is a text transformation engine. The concept behind the
+system is this: you give it text written in your favorite markup language, be
+that Markdown, Textile, or just plain HTML, and it churns that through a
+layout or series of layout files. Throughout that process you can tweak how
+you want the site URLs to look, what data gets displayed on the layout and
+much more.
+
+If you're looking for a simple, yet powerful solution to your blogging or
+static site needs, Jekyll may be just what you've been looking for.
+
+
+=== What this book covers
+
+_Chapter 1, Quick Start_ covers installation, introduces the Jekyll command
+line interface, and runs through a quick example demonstrating the site
+generator, post generator and how to convert your Jekyll site into a static
+site.
+
+_Chapter 2, Directory Layout_ covers the various files and directories that
+comprise a Jekyll site.
+
+_Chapter 3, Tags and Filters_
+
+_Chapter X, Deploying your Jekyll Site_
+
+_Chapter X, Customizing Jekyll with Plugins_
+
+_Chapter X, Migrating to Jekyll from your Existing Blog_
+
+_Chapter X, Configuration Reference_

doc/book/ch01-quick-start.asc

@@ -0,0 +1,153 @@
+== Chapter 1: Quick Start
+
+This chapter is designed to get you up and running with Jekyll as quickly as
+possible.
+
+
+=== Installation
+
+The best way to install Jekyll is via RubyGems:
+
+----
+gem install jekyll
+----
+
+This is all you need in order to get started with a basic Jekyll site. Some
+options require additional packages to be installed.
+
+If you encounter errors during gem installation, you may need to install the
+header files for compiling extension modules for ruby 1.8:
+
+.Debian
+----
+sudo apt-get install ruby1.8-dev
+----
+
+.Red Hat / CentOS / Fedora systems
+----
+sudo yum install ruby-devel
+----
+
+.NearlyFreeSpeech
+----
+RB_USER_INSTALL=true gem install jekyll
+----
+
+If you encounter errors like +Failed to build gem native extension+ on Windows
+you may need to install http://wiki.github.com/oneclick/rubyinstaller/development-kit[RubyInstaller
+DevKit].
+
+==== LaTeX to PNG
+
+Maruku comes with optional support for LaTeX to PNG rendering via blahtex
+(Version 0.6) which must be in your $PATH along with @dvips@.
+
+(NOTE: "remi's fork of Maruku":http://github.com/remi/maruku/tree/master does
+not assume a fixed location for @dvips@ if you need that fixed)
+
+==== RDiscount
+
+If you prefer to use
+http://github.com/rtomayko/rdiscount/tree/master[RDiscount] instead of
+http://maruku.rubyforge.org/[Maruku] for markdown, just make sure it's
+installed:
+
+----
+sudo gem install rdiscount
+----
+
+And run Jekyll with the following option:
+
+----
+jekyll --rdiscount
+----
+
+Or, in your @_config.yml@ file put the following so you don't have to specify the flag:
+
+----
+markdown: rdiscount
+----
+
+==== Pygments
+
+If you want syntax highlighting via the @{% highlight %}@ tag in your posts,
+you'll need to install http://pygments.org/[Pygments].
+
+.On OSX with Homebrew
+----
+brew install pip && pip install pygments
+----
+
+.On OSX with MacPorts
+----
+sudo port install python25 py25-pygments
+----
+
+.Bare OS X Leopard
+----
+sudo easy_install Pygments
+----
+
+.Archlinux
+----
+sudo pacman -S python-pygments
+----
+
+.Archlinux python2 for Pygments
+----
+$ sudo pacman -S python2-pygments
+----
+
+NOTE: python2 pygments version creates a `pygmentize2` executable, while
+Jekyll tries to find `pygmentize`. Either create a symlink `# ln -s
+/usr/bin/pygmentize2 /usr/bin/pygmentize` or use the python3 version.
+
+.Ubuntu and Debian
+----
+sudo apt-get install python-pygments
+----
+
+.Gentoo
+----
+$ sudo emerge -av dev-python/pygments
+----
+
+
+=== Creating your First Site
+
+Jekyll comes with a handy generator that will create a barebones skeleton site
+to help you get up and running in no time. Simply create an empty directory to
+contain your site, navigate to it, and run the generator command:
+
+----
+$ mkdir mysite
+$ cd mysite
+$ jekyll gen
+----
+
+Make sure the directory is empty or Jekyll will refuse to run. If everything
+was successful, you'll be left with a complete, valid Jekyll site that's ready
+to be converted into a static site.
+
+To perform the conversion, make sure you're in the root of your Jekyll site
+directory and run:
+
+----
+$ jekyll --server
+----
+
+If all goes well, you should get a few lines with information about config
+file detection, source and destination paths, and a success message.
+
+The `--server` command line option fires up a simple web server that will
+serve the static site we just generated so that we can easily preview what it
+will look like once we deploy it to a production environment.
+
+Open up your favorite web browser and navigate to:
+
+----
+http://localhost:4000
+----
+
+Congratulations! You have now successfully created and converted your first
+Jekyll site!

doc/book/ch02-directory-layout.asc

@@ -0,0 +1,90 @@
+== Chapter 2: Directory Layout
+
+If you followed the Quick Start in the last chapter, you have a Jekyll site on
+your local machine. Let's take a closer look at it and see what makes it tick.
+The file layout should look something like this:
+
+----
+.
+|-- _config.yml
+|-- _layouts
+|   |-- default.html
+|   `-- post.html
+|-- _posts
+|   |-- 2007-10-29-why-every-programmer-should-play-nethack.textile
+|   `-- 2009-04-26-barcamp-boston-4-roundup.textile
+|-- _site
+|-- images
+|   `-- logo.png
+`-- index.html
+----
+
+Notice that some of the files and directories begin with an underscore. These
+have special meaning to Jekyll. The underscore ensures that they will not
+interfere with the rest of your site's normal content. It also means that if
+any of your normal files start with an underscore, they will cause problems,
+so try to avoid this.
+
+=== _config.yml
+
+This file stores configuration data. A majority of these options can be
+specified from the command line executable but it's easier to throw them in
+here so you don't have to type them out every time. Detailed explanations of
+configuration directives can be found in Chapter X.
+
+=== _layouts
+
+Files in this directory represent templates that can be used to wrap converted
+pages. Layouts are defined on a page-by-page basis in the YAML front matter.
+The liquid tag +{{ content }}+ specifies where the content will be placed
+during the conversion process.
+
+=== _posts
+
+If you're using Jekyll as a blog engine, this is where you'll place your blog
+posts. A post's filename contains several pieces of data, so you must be very
+careful about how these files are named. The filename format is:
++YEAR-MONTH-DATE-SLUG.MARKUP+. The YEAR must be four numbers and the MONTH and
+DATE must be two numbers each. The SLUG is what will appear in the URL. The
+MARKUP tells Jekyll the format of the post. The date and slug will be used
+along with any permalink options you specify (See Chapter X) to construct the
+final URL of the post.
+
+=== _site
+
+This is where the generated site will be placed (by default) once Jekyll is
+done transforming it. If you're using version control, you'll want to add this
+directory to the list of files to be ignored.
+
+=== Normal Files with YAML Front Matter
+
+All files outside of the special underscore directories and that do not
+themselves begin with an underscore will be scanned by Jekyll and subjected to
+conversion if they contain any YAML front matter.
+
+=== Everything Else
+
+Any files and directories that do not fall into one of the above categories
+will be copied to the static site as-is without modification. In this example,
++images/logo.png+ will be copied to the same location in the generated site.
+
+
+
+
+h2. Running Jekyll
+
+Usually this is done through the @jekyll@ executable, which is installed with
+the gem. In order to get a server up and running with your Jekyll site, run:
+
+@jekyll --server@ 
+
+and then browse to http://0.0.0.0:4000. There's plenty of [[configuration
+options|Configuration]] available to you as well.
+
+On Debian or Ubuntu, you may need to add @/var/lib/gems/1.8/bin/@ to your path.
+
+h2. Deployment
+
+Since Jekyll simply generates a folder filled with HTML files, it can be
+served using practically any available web server out there. Please check the
+[[Deployment]] page for more information regarding specific scenarios.

features/support/env.rb

@@ -14,3 +14,6 @@ def run_jekyll(opts = {})
   command << " >> /dev/null 2>&1" if opts[:debug].nil?
   system command
 end
+
+# work around "invalid option: --format" cucumber bug (see #296)
+Test::Unit.run = true

jekyll.gemspec

@@ -4,8 +4,8 @@ Gem::Specification.new do |s|
   s.rubygems_version = '1.3.5'
 
   s.name              = 'jekyll'
-  s.version           = '0.9.0'
-  s.date              = '2010-12-15'
+  s.version           = '0.10.0'
+  s.date              = '2010-12-16'
   s.rubyforge_project = 'jekyll'
 
   s.summary     = "A simple, blog aware, static site generator."
@@ -23,17 +23,19 @@ Gem::Specification.new do |s|
   s.rdoc_options = ["--charset=UTF-8"]
   s.extra_rdoc_files = %w[README.textile LICENSE]
 
-  s.add_runtime_dependency('liquid', [">= 1.9.0"])
-  s.add_runtime_dependency('classifier', [">= 1.3.1"])
-  s.add_runtime_dependency('directory_watcher', [">= 1.1.1"])
-  s.add_runtime_dependency('maruku', [">= 0.5.9"])
+  s.add_runtime_dependency('liquid', ">= 1.9.0")
+  s.add_runtime_dependency('classifier', ">= 1.3.1")
+  s.add_runtime_dependency('directory_watcher', ">= 1.1.1")
+  s.add_runtime_dependency('maruku', ">= 0.5.9")
+  s.add_runtime_dependency('kramdown', ">= 0.13.2")
+  s.add_runtime_dependency('albino', ">= 1.3.2")
 
-  s.add_development_dependency('redgreen', [">= 4.2.1"])
-  s.add_development_dependency('shoulda', [">= 4.2.1"])
-  s.add_development_dependency('rr', [">= 4.2.1"])
-  s.add_development_dependency('cucumber', [">= 4.2.1"])
-  s.add_development_dependency('RedCloth', [">= 4.2.1"])
-  s.add_development_dependency('kramdown', [">= 0.12.0"])
+  s.add_development_dependency('redgreen', ">= 1.2.2")
+  s.add_development_dependency('shoulda', ">= 2.11.3")
+  s.add_development_dependency('rr', ">= 1.0.2")
+  s.add_development_dependency('cucumber', ">= 0.10.0")
+  s.add_development_dependency('RedCloth', ">= 4.2.1")
+  s.add_development_dependency('rdiscount', ">= 1.6.5")
 
   # = MANIFEST =
   s.files = %w[

lib/jekyll.rb

@@ -19,10 +19,12 @@ require 'rubygems'
 require 'fileutils'
 require 'time'
 require 'yaml'
+require 'English'
 
 # 3rd party
 require 'liquid'
 require 'maruku'
+require 'albino'
 
 # internal requires
 require 'jekyll/core_ext'
@@ -32,7 +34,6 @@ require 'jekyll/layout'
 require 'jekyll/page'
 require 'jekyll/post'
 require 'jekyll/filters'
-require 'jekyll/albino'
 require 'jekyll/static_file'
 require 'jekyll/errors'
 
@@ -45,7 +46,7 @@ require_all 'jekyll/generators'
 require_all 'jekyll/tags'
 
 module Jekyll
-  VERSION = '0.9.0'
+  VERSION = '0.10.0'
 
   # Default options. Overriden by values in _config.yml or command-line opts.
   # (Strings rather symbols used for compatability with YAML).

lib/jekyll/albino.rb

@@ -1,120 +0,0 @@
-##
-# Wrapper for the Pygments command line tool, pygmentize.
-#
-# Pygments: http://pygments.org/
-#
-# Assumes pygmentize is in the path.  If not, set its location
-# with Albino.bin = '/path/to/pygmentize'
-#
-# Use like so:
-#
-#   @syntaxer = Albino.new('/some/file.rb', :ruby)
-#   puts @syntaxer.colorize
-#
-# This'll print out an HTMLized, Ruby-highlighted version
-# of '/some/file.rb'.
-#
-# To use another formatter, pass it as the third argument:
-#
-#   @syntaxer = Albino.new('/some/file.rb', :ruby, :bbcode)
-#   puts @syntaxer.colorize
-#
-# You can also use the #colorize class method:
-#
-#   puts Albino.colorize('/some/file.rb', :ruby)
-#
-# Another also: you get a #to_s, for somewhat nicer use in Rails views.
-#
-#   ... helper file ...
-#   def highlight(text)
-#     Albino.new(text, :ruby)
-#   end
-#
-#   ... view file ...
-#   <%= highlight text %>
-#
-# The default lexer is 'text'.  You need to specify a lexer yourself;
-# because we are using STDIN there is no auto-detect.
-#
-# To see all lexers and formatters available, run `pygmentize -L`.
-#
-# Chris Wanstrath // chris@ozmm.org
-#         GitHub // http://github.com
-#
-
-class Albino
-  @@bin = Rails.development? ? 'pygmentize' : '/usr/bin/pygmentize' rescue 'pygmentize'
-
-  def self.bin=(path)
-    @@bin = path
-  end
-
-  def self.colorize(*args)
-    new(*args).colorize
-  end
-
-  def initialize(target, lexer = :text, format = :html)
-    @target  = target
-    @options = { :l => lexer, :f => format, :O => 'encoding=utf-8' }
-  end
-
-  def execute(command)
-    output = ''
-    IO.popen(command, mode='r+') do |p|
-      p.write @target
-      p.close_write
-      output = p.read.strip
-    end
-    output
-  end
-
-  def colorize(options = {})
-    html = execute(@@bin + convert_options(options))
-    # Work around an RDiscount bug: http://gist.github.com/97682
-    html.to_s.sub(%r{</pre></div>\Z}, "</pre>\n</div>")
-  end
-  alias_method :to_s, :colorize
-
-  def convert_options(options = {})
-    @options.merge(options).inject('') do |string, (flag, value)|
-      string + " -#{flag} #{value}"
-    end
-  end
-end
-
-if $0 == __FILE__
-  require 'rubygems'
-  require 'test/spec'
-  require 'mocha'
-  begin require 'redgreen'; rescue LoadError; end
-
-  context "Albino" do
-    setup do
-      @syntaxer = Albino.new(__FILE__, :ruby)
-    end
-
-    specify "defaults to text" do
-      syntaxer = Albino.new(__FILE__)
-      syntaxer.expects(:execute).with('pygmentize -f html -l text').returns(true)
-      syntaxer.colorize
-    end
-
-    specify "accepts options" do
-      @syntaxer.expects(:execute).with('pygmentize -f html -l ruby').returns(true)
-      @syntaxer.colorize
-    end
-
-    specify "works with strings" do
-      syntaxer = Albino.new('class New; end', :ruby)
-      assert_match %r(highlight), syntaxer.colorize
-    end
-
-    specify "aliases to_s" do
-      assert_equal @syntaxer.colorize, @syntaxer.to_s
-    end
-
-    specify "class method colorize" do
-      assert_equal @syntaxer.colorize, Albino.colorize(__FILE__, :ruby)
-    end
-  end
-end

lib/jekyll/convertible.rb

@@ -10,21 +10,22 @@
 #   self.output=
 module Jekyll
   module Convertible
-    # Return the contents as a string
+    # Returns the contents as a String.
     def to_s
       self.content || ''
     end
 
-    # Read the YAML frontmatter
-    #   +base+ is the String path to the dir containing the file
-    #   +name+ is the String filename of the file
+    # Read the YAML frontmatter.
     #
-    # Returns nothing
+    # base - The String path to the dir containing the file.
+    # name - The String filename of the file.
+    #
+    # Returns nothing.
     def read_yaml(base, name)
       self.content = File.read(File.join(base, name))
 
       if self.content =~ /^(---\s*\n.*?\n?)^(---\s*$\n?)/m
-        self.content = self.content[($1.size + $2.size)..-1]
+        self.content = $POSTMATCH
 
         begin
           self.data = YAML.load($1)
@@ -38,42 +39,46 @@ module Jekyll
 
     # Transform the contents based on the content type.
     #
-    # Returns nothing
+    # Returns nothing.
     def transform
       self.content = converter.convert(self.content)
     end
 
-    # Determine the extension depending on content_type
+    # Determine the extension depending on content_type.
     #
-    # Returns the extensions for the output file
+    # Returns the String extension for the output file.
+    #   e.g. ".html" for an HTML output file.
     def output_ext
       converter.output_ext(self.ext)
     end
 
     # Determine which converter to use based on this convertible's
-    # extension
+    # extension.
+    #
+    # Returns the Converter instance.
     def converter
       @converter ||= self.site.converters.find { |c| c.matches(self.ext) }
     end
 
-    # Add any necessary layouts to this convertible document
-    #   +layouts+ is a Hash of {"name" => "layout"}
-    #   +site_payload+ is the site payload hash
+    # Add any necessary layouts to this convertible document.
     #
-    # Returns nothing
+    # payload - The site payload Hash.
+    # layouts - A Hash of {"name" => "layout"}.
+    #
+    # Returns nothing.
     def do_layout(payload, layouts)
       info = { :filters => [Jekyll::Filters], :registers => { :site => self.site } }
 
       # render and transform content (this becomes the final content of the object)
       payload["pygments_prefix"] = converter.pygments_prefix
       payload["pygments_suffix"] = converter.pygments_suffix
-      
+
       begin
         self.content = Liquid::Template.parse(self.content).render(payload, info)
       rescue => e
         puts "Liquid Exception: #{e.message} in #{self.data["layout"]}"
       end
-      
+
       self.transform
 
       # output keeps track of what will finally be written

lib/jekyll/migrators/posterous.rb

@@ -0,0 +1,73 @@
+require 'rubygems'
+require 'jekyll'
+require 'fileutils'
+require 'net/http'
+require 'uri'
+require "json"
+
+# ruby -r './lib/jekyll/migrators/posterous.rb' -e 'Jekyll::Posterous.process(email, pass, blog)'
+
+module Jekyll
+  module Posterous
+    
+    def self.fetch(uri_str, limit = 10)
+      # You should choose better exception.
+      raise ArgumentError, 'Stuck in a redirect loop. Please double check your email and password' if limit == 0
+      
+      response = nil
+      Net::HTTP.start('posterous.com') {|http|
+        req = Net::HTTP::Get.new(uri_str)
+        req.basic_auth @email, @pass
+        response = http.request(req)
+      }
+
+      case response
+        when Net::HTTPSuccess     then response
+        when Net::HTTPRedirection then fetch(response['location'], limit - 1)
+        else response.error!
+      end
+    end
+        
+        
+    def self.process(email, pass, blog = 'primary')
+      @email, @pass = email, pass
+      @api_token = JSON.parse(self.fetch("/api/2/auth/token").body)['api_token']
+      FileUtils.mkdir_p "_posts"
+      
+      posts = JSON.parse(self.fetch("/api/v2/users/me/sites/#{blog}/posts?api_token=#{@api_token}").body)
+      page = 1
+      
+      while posts.any?
+      
+        posts.each do |post|
+          title = post["title"]
+          slug = title.gsub(/[^[:alnum:]]+/, '-').downcase  
+          date = Date.parse(post["display_date"])
+          content = post["body_html"]
+          published = !post["is_private"]
+          name = "%02d-%02d-%02d-%s.html" % [date.year, date.month, date.day, slug]
+        
+          # Get the relevant fields as a hash, delete empty fields and convert
+          # to YAML for the header
+          data = {
+             'layout' => 'post',
+             'title' => title.to_s,
+             'published' => published
+           }.delete_if { |k,v| v.nil? || v == ''}.to_yaml
+
+          # Write out the data and content to file
+          File.open("_posts/#{name}", "w") do |f|
+            f.puts data
+            f.puts "---"
+            f.puts content
+          end
+        
+        end
+        
+        page += 1
+        posts = JSON.parse(self.fetch("/api/v2/users/me/sites/#{blog}/posts?api_token=#{@api_token}&page=#{page}").body)        
+      end
+      
+    end
+  end
+end

lib/jekyll/migrators/wordpress.com.rb

@@ -1,38 +0,0 @@
-require 'rubygems'
-require 'hpricot'
-require 'fileutils'
-
-# This importer takes a wordpress.xml file,
-# which can be exported from your 
-# wordpress.com blog (/wp-admin/export.php)
-
-module Jekyll
-  module WordpressDotCom
-    def self.process(filename = "wordpress.xml")
-      FileUtils.mkdir_p "_posts"
-      posts = 0
-
-			doc = Hpricot::XML(File.read(filename))
-			
-			(doc/:channel/:item).each do |item|
-				title = item.at(:title).inner_text
-				name = "#{Date.parse((doc/:channel/:item).first.at(:pubDate).inner_text).to_s("%Y-%m-%d")}-#{title.downcase.gsub('[^a-z0-9]', '-')}.html"
-				
-				File.open("_posts/#{name}", "w") do |f|
-          f.puts <<-HEADER
----
-layout: post
-title: #{title}
----
- 
-HEADER
-          f.puts item.at('content:encoded').inner_text
-        end
-
-				posts += 1
-			end
-
-			"Imported #{posts} posts"
-    end
-  end
-end

lib/jekyll/migrators/wordpress.rb

@@ -12,18 +12,18 @@ require 'yaml'
 module Jekyll
   module WordPress
 
-    # Reads a MySQL database via Sequel and creates a post file for each
-    # post in wp_posts that has post_status = 'publish'.
-    # This restriction is made because 'draft' posts are not guaranteed to
-    # have valid dates.
-    QUERY = "select post_title, post_name, post_date, post_content, post_excerpt, ID, guid from wp_posts where post_status = 'publish' and post_type = 'post'"
-
-    def self.process(dbname, user, pass, host = 'localhost')
+    def self.process(dbname, user, pass, host = 'localhost', table_prefix = 'wp_')
       db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8')
 
       FileUtils.mkdir_p "_posts"
+      
+      # Reads a MySQL database via Sequel and creates a post file for each
+      # post in wp_posts that has post_status = 'publish'.
+      # This restriction is made because 'draft' posts are not guaranteed to
+      # have valid dates.
+      query = "select post_title, post_name, post_date, post_content, post_excerpt, ID, guid from #{table_prefix}posts where post_status = 'publish' and post_type = 'post'"
 
-      db[QUERY].each do |post|
+      db[query].each do |post|
         # Get required fields and construct Jekyll compatible name
         title = post[:post_title]
         slug = post[:post_name]
@@ -39,7 +39,8 @@ module Jekyll
            'title' => title.to_s,
            'excerpt' => post[:post_excerpt].to_s,
            'wordpress_id' => post[:ID],
-           'wordpress_url' => post[:guid]
+           'wordpress_url' => post[:guid],
+           'date' => date
          }.delete_if { |k,v| v.nil? || v == ''}.to_yaml
 
         # Write out the data and content to file

lib/jekyll/migrators/wordpressdotcom.rb

@@ -0,0 +1,45 @@
+# coding: utf-8
+
+require 'rubygems'
+require 'hpricot'
+require 'fileutils'
+require 'yaml'
+
+module Jekyll
+  
+  # This importer takes a wordpress.xml file,
+  # which can be exported from your 
+  # wordpress.com blog (/wp-admin/export.php)
+  module WordpressDotCom
+    def self.process(filename = "wordpress.xml")
+      FileUtils.mkdir_p "_posts"
+      posts = 0
+
+			doc = Hpricot::XML(File.read(filename))
+			
+			(doc/:channel/:item).each do |item|
+				title = item.at(:title).inner_text.strip
+				permalink_title = item.at('wp:post_name').inner_text
+				date = Time.parse(item.at(:pubDate).inner_text)
+				tags = (item/:category).map{|c| c.inner_text}.reject{|c| c == 'Uncategorized'}.uniq
+				name = "#{date.strftime('%Y-%m-%d')}-#{permalink_title}.html"
+			  header = {
+			    'layout' => 'post', 
+			    'title'  => title, 
+			    'tags'   => tags
+			  }
+			  
+				File.open("_posts/#{name}", "w") do |f|
+				  f.puts header.to_yaml
+				  f.puts '---'
+          f.puts item.at('content:encoded').inner_text
+        end
+
+				posts += 1
+			end
+
+			puts "Imported #{posts} posts"
+    end
+  end
+  
+end

lib/jekyll/page.rb

@@ -55,14 +55,23 @@ module Jekyll
     #
     # Returns <String>
     def url
-      return permalink if permalink
+      return @url if @url
 
-      @url ||= {
-        "basename"   => self.basename,
-        "output_ext" => self.output_ext,
-      }.inject(template) { |result, token|
-        result.gsub(/:#{token.first}/, token.last)
-      }.gsub(/\/\//, "/")
+      url = if permalink
+        permalink
+      else
+        {
+          "basename"   => self.basename,
+          "output_ext" => self.output_ext,
+        }.inject(template) { |result, token|
+          result.gsub(/:#{token.first}/, token.last)
+        }.gsub(/\/\//, "/")
+      end
+
+      # sanitize url
+      @url = url.split('/').reject{ |part| part =~ /^\.+$/ }.join('/')
+      @url += "/" if url =~ /\/$/
+      @url
     end
 
     # Extract information from the page filename

lib/jekyll/post.rb

@@ -78,6 +78,8 @@ module Jekyll
       self.date = Time.parse(date)
       self.slug = slug
       self.ext = ext
+    rescue ArgumentError
+      raise FatalException.new("Post #{name} does not have a valid date.")
     end
 
     # The generated directory into which the post will be placed
@@ -117,20 +119,29 @@ module Jekyll
     #
     # Returns <String>
     def url
-      return permalink if permalink
+      return @url if @url
 
-      @url ||= {
-        "year"       => date.strftime("%Y"),
-        "month"      => date.strftime("%m"),
-        "day"        => date.strftime("%d"),
-        "title"      => CGI.escape(slug),
-        "i_day"      => date.strftime("%d").to_i.to_s,
-        "i_month"    => date.strftime("%m").to_i.to_s,
-        "categories" => categories.join('/'),
-        "output_ext" => self.output_ext
-      }.inject(template) { |result, token|
-        result.gsub(/:#{Regexp.escape token.first}/, token.last)
-      }.gsub(/\/\//, "/")
+      url = if permalink
+        permalink
+      else
+        {
+          "year"       => date.strftime("%Y"),
+          "month"      => date.strftime("%m"),
+          "day"        => date.strftime("%d"),
+          "title"      => CGI.escape(slug),
+          "i_day"      => date.strftime("%d").to_i.to_s,
+          "i_month"    => date.strftime("%m").to_i.to_s,
+          "categories" => categories.join('/'),
+          "output_ext" => self.output_ext
+        }.inject(template) { |result, token|
+          result.gsub(/:#{Regexp.escape token.first}/, token.last)
+        }.gsub(/\/\//, "/")
+      end
+
+      # sanitize url
+      @url = url.split('/').reject{ |part| part =~ /^\.+$/ }.join('/')
+      @url += "/" if url =~ /\/$/
+      @url
     end
 
     # The UID for this post (useful in feeds)

lib/jekyll/site.rb

@@ -1,3 +1,5 @@
+require 'set'
+
 module Jekyll
 
   class Site
@@ -158,13 +160,13 @@ module Jekyll
     # Returns nothing
     def cleanup
       # all files and directories in destination, including hidden ones
-      dest_files = []
+      dest_files = Set.new
       Dir.glob(File.join(self.dest, "**", "*"), File::FNM_DOTMATCH) do |file|
         dest_files << file unless file =~ /\/\.{1,2}$/
       end
 
       # files to be written
-      files = []
+      files = Set.new
       self.posts.each do |post|
         files << post.destination(self.dest)
       end
@@ -176,11 +178,13 @@ module Jekyll
       end
       
       # adding files' parent directories
-      files.each { |file| files << File.dirname(file) unless files.include? File.dirname(file) }
+      dirs = Set.new
+      files.each { |file| dirs << File.dirname(file) }
+      files.merge(dirs)
       
       obsolete_files = dest_files - files
       
-      FileUtils.rm_rf(obsolete_files)
+      FileUtils.rm_rf(obsolete_files.to_a)
     end
 
     # Write static files, pages and posts
@@ -206,7 +210,7 @@ module Jekyll
     # Returns nothing
     def read_directories(dir = '')
       base = File.join(self.source, dir)
-      entries = filter_entries(Dir.entries(base))
+      entries = Dir.chdir(base){ filter_entries(Dir['*']) }
 
       self.read_posts(dir)
 
@@ -264,7 +268,10 @@ module Jekyll
     def filter_entries(entries)
       entries = entries.reject do |e|
         unless ['.htaccess'].include?(e)
-          ['.', '_', '#'].include?(e[0..0]) || e[-1..-1] == '~' || self.exclude.include?(e)
+          ['.', '_', '#'].include?(e[0..0]) ||
+          e[-1..-1] == '~' ||
+          self.exclude.include?(e) ||
+          File.symlink?(e)
         end
       end
     end

test/test_post.rb

@@ -52,6 +52,12 @@ class TestPost < Test::Unit::TestCase
         assert_equal "/2008/09/09/foo-bar.html", @post.url
       end
 
+      should "raise a good error on invalid post date" do
+        assert_raise Jekyll::FatalException do
+          @post.process("2009-27-03-foo-bar.textile")
+        end
+      end
+
       should "CGI escape urls" do
         @post.categories = []
         @post.process("2009-03-12-hash-#1.markdown")