From d1682f5b3f70f897f1e86834b0bf4520b2f6a6d9 Mon Sep 17 00:00:00 2001
From: Jeremy Ashkenas <jashkenas@gmail.com>
Date: Thu, 17 Dec 2009 08:23:07 -0500
Subject: [PATCH] multiline strings

---
 examples/code.cs                  | 7 ++++++-
 lib/coffee_script/command_line.rb | 8 ++++++++
 lib/coffee_script/lexer.rb        | 5 +++--
 3 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/examples/code.cs b/examples/code.cs
index 756ee24a..43e92231 100644
--- a/examples/code.cs
+++ b/examples/code.cs
@@ -121,4 +121,9 @@ wednesday: => eat_breakfast(); go_to_work(); eat_dinner(); .
 
 # Array slice literals.
 zero_to_nine: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
-three_to_six: zero_to_nine[3, 6]
\ No newline at end of file
+three_to_six: zero_to_nine[3, 6]
+
+# Multiline strings.
+story: "Lorem ipsum dolor sit amet, consectetuer adipiscing elit,
+sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna
+aliquam erat volutpat. Ut wisi enim ad."
\ No newline at end of file
diff --git a/lib/coffee_script/command_line.rb b/lib/coffee_script/command_line.rb
index ecf81da4..70e59004 100644
--- a/lib/coffee_script/command_line.rb
+++ b/lib/coffee_script/command_line.rb
@@ -30,6 +30,7 @@ Usage:
 
     def compile_javascript
       @sources.each do |source|
+        next tokens(source) if @options[:tokens]
         contents = CoffeeScript.compile(File.open(source))
         next puts(contents) if @options[:print]
         next lint(contents) if @options[:lint]
@@ -55,6 +56,10 @@ Usage:
       stdout.close and stderr.close
     end
 
+    def tokens(source)
+      puts Lexer.new.tokenize(File.read(source)).inspect
+    end
+
     # Write out JavaScript alongside CoffeeScript unless an output directory
     # is specified.
     def path_for(source)
@@ -76,6 +81,9 @@ Usage:
         opts.on('-l', '--lint', 'pipe the compiled javascript through JSLint') do |l|
           @options[:lint] = true
         end
+        opts.on('-t', '--tokens', 'print the tokens that the lexer produces') do |t|
+          @options[:tokens] = true
+        end
         opts.on_tail('-v', '--version', 'display coffee-script version') do
           puts "coffee-script version #{CoffeeScript::VERSION}"
           exit
diff --git a/lib/coffee_script/lexer.rb b/lib/coffee_script/lexer.rb
index b86d5e4e..5c5bd31d 100644
--- a/lib/coffee_script/lexer.rb
+++ b/lib/coffee_script/lexer.rb
@@ -11,7 +11,7 @@ class Lexer
 
   IDENTIFIER = /\A([a-zA-Z$_]\w*)/
   NUMBER     = /\A([0-9]+(\.[0-9]+)?)/
-  STRING     = /\A("(.*?)"|'(.*?)')/
+  STRING     = /\A("(.*?)"|'(.*?)')/m
   JS         = /\A(`(.*?)`)/
   OPERATOR   = /\A([+\*&|\/\-%=<>]+)/
   WHITESPACE = /\A([ \t\r]+)/
@@ -21,6 +21,7 @@ class Lexer
   REGEX      = /\A(\/(.*?)\/[imgy]{0,4})/
 
   JS_CLEANER = /(\A`|`\Z)/
+  MULTILINER = /[\r\n]/
 
   EXP_START  = ['{', '(', '[']
   EXP_END    = ['}', ')', ']']
@@ -71,7 +72,7 @@ class Lexer
 
   def string_token
     return false unless string = @chunk[STRING, 1]
-    @tokens << [:STRING, string]
+    @tokens << [:STRING, string.gsub(MULTILINER, "\\\n")]
     @i += string.length
   end