multiline strings

lib/coffee_script/command_line.rb

@@ -30,6 +30,7 @@ Usage:
 
     def compile_javascript
       @sources.each do |source|
+        next tokens(source) if @options[:tokens]
         contents = CoffeeScript.compile(File.open(source))
         next puts(contents) if @options[:print]
         next lint(contents) if @options[:lint]
@@ -55,6 +56,10 @@ Usage:
       stdout.close and stderr.close
     end
 
+    def tokens(source)
+      puts Lexer.new.tokenize(File.read(source)).inspect
+    end
+
     # Write out JavaScript alongside CoffeeScript unless an output directory
     # is specified.
     def path_for(source)
@@ -76,6 +81,9 @@ Usage:
         opts.on('-l', '--lint', 'pipe the compiled javascript through JSLint') do |l|
           @options[:lint] = true
         end
+        opts.on('-t', '--tokens', 'print the tokens that the lexer produces') do |t|
+          @options[:tokens] = true
+        end
         opts.on_tail('-v', '--version', 'display coffee-script version') do
           puts "coffee-script version #{CoffeeScript::VERSION}"
           exit

lib/coffee_script/lexer.rb

@@ -11,7 +11,7 @@ class Lexer
 
   IDENTIFIER = /\A([a-zA-Z$_]\w*)/
   NUMBER     = /\A([0-9]+(\.[0-9]+)?)/
-  STRING     = /\A("(.*?)"|'(.*?)')/
+  STRING     = /\A("(.*?)"|'(.*?)')/m
   JS         = /\A(`(.*?)`)/
   OPERATOR   = /\A([+\*&|\/\-%=<>]+)/
   WHITESPACE = /\A([ \t\r]+)/
@@ -21,6 +21,7 @@ class Lexer
   REGEX      = /\A(\/(.*?)\/[imgy]{0,4})/
 
   JS_CLEANER = /(\A`|`\Z)/
+  MULTILINER = /[\r\n]/
 
   EXP_START  = ['{', '(', '[']
   EXP_END    = ['}', ')', ']']
@@ -71,7 +72,7 @@ class Lexer
 
   def string_token
     return false unless string = @chunk[STRING, 1]
-    @tokens << [:STRING, string]
+    @tokens << [:STRING, string.gsub(MULTILINER, "\\\n")]
     @i += string.length
   end