From 20dae3758a911da81891c24945f79960dcb1ec23 Mon Sep 17 00:00:00 2001
From: satyr <murky.satyr@gmail.com>
Date: Thu, 23 Sep 2010 13:41:53 +0900
Subject: [PATCH 01/10] lexer: optimized regexes

---
 src/lexer.coffee | 107 +++++++++++++++++++++++------------------------
 1 file changed, 53 insertions(+), 54 deletions(-)

diff --git a/src/lexer.coffee b/src/lexer.coffee
index cd2215a0..6999591d 100644
--- a/src/lexer.coffee
+++ b/src/lexer.coffee
@@ -33,7 +33,7 @@ exports.Lexer = class Lexer
   # Before returning the token stream, run it through the [Rewriter](rewriter.html)
   # unless explicitly asked not to.
   tokenize: (code, options) ->
-    code     = code.replace /(\r|\s+$)/g, ''
+    code     = code.replace(/\r/g, '').replace /\s+$/, ''
     o        = options or {}
     @code    = code         # The remainder of the source code.
     @i       = 0            # Current character position we're parsing.
@@ -75,7 +75,7 @@ exports.Lexer = class Lexer
   # referenced as property names here, so you can still do `jQuery.is()` even
   # though `is` means `===` otherwise.
   identifierToken: ->
-    return false unless id = @match IDENTIFIER, 1
+    return false unless id = @match IDENTIFIER
     @i += id.length
     forcedIdentifier = @tagAccessor() or @match ASSIGNED, 1
     tag = 'IDENTIFIER'
@@ -104,7 +104,7 @@ exports.Lexer = class Lexer
   # Matches numbers, including decimals, hex, and exponential notation.
   # Be careful not to interfere with ranges-in-progress.
   numberToken: ->
-    return false unless number = @match NUMBER, 1
+    return false unless number = @match NUMBER
     return false if @tag() is '.' and starts number, '.'
     @i += number.length
     @token 'NUMBER', number
@@ -117,7 +117,7 @@ exports.Lexer = class Lexer
     return false unless string =
       @balancedToken(['"', '"'], ['#{', '}']) or
       @balancedToken ["'", "'"]
-    @interpolateString string.replace /\n/g, '\\\n'
+    @interpolateString string.replace MULTILINER, '\\\n'
     @line += count string, "\n"
     @i += string.length
     true
@@ -126,20 +126,22 @@ exports.Lexer = class Lexer
   # preserve whitespace, but ignore indentation to the left.
   heredocToken: ->
     return false unless match = @chunk.match HEREDOC
-    quote = match[1].substr 0, 1
-    doc = @sanitizeHeredoc match[2] or match[4] or '', {quote}
+    heredoc = match[0]
+    quote = heredoc.charAt 0
+    doc = @sanitizeHeredoc match[2], {quote}
     @interpolateString quote + doc + quote, heredoc: yes
-    @line += count match[1], "\n"
-    @i += match[1].length
+    @line += count heredoc, '\n'
+    @i += heredoc.length
     true
 
   # Matches and consumes comments.
   commentToken: ->
-    return false unless match = @chunk.match(COMMENT)
-    @line += count match[1], "\n"
-    @i += match[1].length
-    if match[2]
-      @token 'HERECOMMENT', @sanitizeHeredoc match[2],
+    return false unless match = @chunk.match COMMENT
+    [comment, here] = match
+    @line += count comment, '\n'
+    @i += comment.length
+    if here
+      @token 'HERECOMMENT', @sanitizeHeredoc here,
         herecomment: true, indent: Array(@indent + 1).join(' ')
       @token 'TERMINATOR', '\n'
     true
@@ -148,7 +150,7 @@ exports.Lexer = class Lexer
   jsToken: ->
     return false unless starts @chunk, '`'
     return false unless script = @balancedToken ['`', '`']
-    @token 'JS', script.replace JS_CLEANER, ''
+    @token 'JS', script.slice 1, -1
     @i += script.length
     true
 
@@ -161,18 +163,18 @@ exports.Lexer = class Lexer
     return false if first[1] is ' ' and @tag() not in ['CALL_START', '=']
     return false if     include NOT_REGEX, @tag()
     return false unless regex = @balancedToken ['/', '/']
-    return false unless end = @chunk.substr(regex.length).match REGEX_END
-    regex += flags = end[2] if end[2]
-    if regex.match REGEX_INTERPOLATION
-      str = regex.substring(1).split('/')[0]
-      str = str.replace REGEX_ESCAPE, (escaped) -> '\\' + escaped
-      @tokens = @tokens.concat [['(', '('], ['NEW', 'new'], ['IDENTIFIER', 'RegExp'], ['CALL_START', '(']]
+    return false unless end = @chunk[regex.length..].match REGEX_END
+    flags = end[0]
+    if REGEX_INTERPOLATION.test regex
+      str = regex.slice 1, -1
+      str = str.replace REGEX_ESCAPE, '\\$&'
+      @tokens.push ['(', '('], ['NEW', 'new'], ['IDENTIFIER', 'RegExp'], ['CALL_START', '(']
       @interpolateString "\"#{str}\"", escapeQuotes: yes
-      @tokens.splice @tokens.length, 0, [',', ','], ['STRING', "\"#{flags}\""] if flags
-      @tokens.splice @tokens.length, 0, [')', ')'], [')', ')']
+      @tokens.push [',', ','], ['STRING', "\"#{flags}\""] if flags
+      @tokens.push [')', ')'], [')', ')']
     else
-      @token 'REGEX', regex
-    @i += regex.length
+      @token 'REGEX', regex + flags
+    @i += regex.length + flags.length
     true
 
   # Matches a token in which which the passed delimiter pairs must be correctly
@@ -191,11 +193,11 @@ exports.Lexer = class Lexer
   # Keeps track of the level of indentation, because a single outdent token
   # can close multiple indents, so we need to know how far in we happen to be.
   lineToken: ->
-    return false unless indent = @match MULTI_DENT, 1
-    @line += count indent, "\n"
+    return false unless indent = @match MULTI_DENT
+    @line += count indent, '\n'
     @i    += indent.length
-    prev = @prev(2)
-    size = indent.match(LAST_DENTS).reverse()[0].match(LAST_DENT)[1].length
+    prev = @prev 2
+    size = indent.length - 1 - indent.lastIndexOf '\n'
     nextCharacter = @match NEXT_CHARACTER, 1
     noNewlines = nextCharacter is '.' or nextCharacter is ',' or @unfinished()
     if size - @indebt is @indent
@@ -235,13 +237,13 @@ exports.Lexer = class Lexer
         @outdebt = 0
         @token 'OUTDENT', dent
     @outdebt -= moveOut if dent
-    @token 'TERMINATOR', "\n" unless @tag() is 'TERMINATOR' or noNewlines
+    @token 'TERMINATOR', '\n' unless @tag() is 'TERMINATOR' or noNewlines
     true
 
   # Matches and consumes non-meaningful whitespace. Tag the previous token
   # as being "spaced", because there are some cases where it makes a difference.
   whitespaceToken: ->
-    return false unless space = @match WHITESPACE, 1
+    return false unless space = @match WHITESPACE
     prev = @prev()
     prev.spaced = true if prev
     @i += space.length
@@ -264,11 +266,11 @@ exports.Lexer = class Lexer
   # here. `;` and newlines are both treated as a `TERMINATOR`, we distinguish
   # parentheses that indicate a method call from regular parentheses, and so on.
   literalToken: ->
-    match = @chunk.match OPERATOR
-    value = match and match[1]
-    space = match and match[2]
-    @tagParameters() if value and value.match CODE
-    value or= @chunk.substr 0, 1
+    if match = @chunk.match OPERATOR
+      [value, space] = match
+      @tagParameters() if CODE.test value
+    else
+      value = @chunk.charAt 0
     @i += value.length
     spaced = (prev = @prev()) and prev.spaced
     tag = value
@@ -321,8 +323,8 @@ exports.Lexer = class Lexer
     indent = options.indent
     return doc if options.herecomment and not include doc, '\n'
     unless options.herecomment
-      while (match = HEREDOC_INDENT.exec(doc)) isnt null
-        attempt = if match[2]? then match[2] else match[3]
+      while (match = HEREDOC_INDENT.exec doc)
+        attempt = if match[1]? then match[1] else match[2]
         indent = attempt if not indent? or 0 < attempt.length < indent.length
     indent or= ''
     doc = doc.replace(new RegExp("^" + indent, 'gm'), '')
@@ -519,29 +521,26 @@ RESERVED = [
 JS_FORBIDDEN = JS_KEYWORDS.concat RESERVED
 
 # Token matching regexes.
-IDENTIFIER    = /^([a-zA-Z\$_](\w|\$)*)/
-NUMBER        = /^(((\b0(x|X)[0-9a-fA-F]+)|((\b[0-9]+(\.[0-9]+)?|\.[0-9]+)(e[+\-]?[0-9]+)?)))\b/i
-HEREDOC       = /^("{6}|'{6}|"{3}([\s\S]*?)\n?([ \t]*)"{3}|'{3}([\s\S]*?)\n?([ \t]*)'{3})/
-OPERATOR      = /^(-[\-=>]?|\+[+=]?|[*&|\/%=<>^:!?]+)([ \t]*)/
-WHITESPACE    = /^([ \t]+)/
-COMMENT       = /^(###([^#][\s\S]*?)(###[ \t]*\n|(###)?$)|(\s*#(?!##[^#])[^\n]*)+)/
-CODE          = /^((-|=)>)/
-MULTI_DENT    = /^((\n([ \t]*))+)(\.)?/
-LAST_DENTS    = /\n([ \t]*)/g
-LAST_DENT     = /\n([ \t]*)/
+IDENTIFIER = /^[a-zA-Z_$][\w$]*/
+NUMBER     = /^(?:0x[\da-f]+)|^(?:\d+(\.\d+)?|\.\d+)(?:e[+-]?\d+)?/i
+HEREDOC    = /^("""|''')([\s\S]*?)\n?[ \t]*\1/
+OPERATOR   = /^(?:-[-=>]?|\+[+=]?|[*&|\/%=<>^:!?]+)(?=([ \t]*))/
+WHITESPACE = /^[ \t]+/
+COMMENT    = /^###([^#][\s\S]*?)(?:###[ \t]*\n|(?:###)?$)|^(?:\s*#(?!##[^#])[^\n]*)+/
+CODE       = /^[-=]>/
+MULTI_DENT = /^(?:\n[ \t]*)+/
 
 # Regex-matching-regexes.
 REGEX_START         = /^\/([^\/])/
-REGEX_INTERPOLATION = /([^\\]#\{.*[^\\]\})/
-REGEX_END           = /^(([imgy]{1,4})\b|\W|$)/
-REGEX_ESCAPE        = /\\[^\$]/g
+REGEX_INTERPOLATION = /[^\\]#\{.*[^\\]\}/
+REGEX_END           = /^[imgy]{0,4}(?![a-zA-Z])/
+REGEX_ESCAPE        = /\\[^#]/g
 
 # Token cleaning regexes.
-JS_CLEANER      = /(^`|`$)/g
 MULTILINER      = /\n/g
-NO_NEWLINE      = /^([+\*&|\/\-%=<>!.\\][<>=&|]*|and|or|is|isnt|not|delete|typeof|instanceof)$/
-HEREDOC_INDENT  = /(\n+([ \t]*)|^([ \t]+))/g
-ASSIGNED        = /^\s*(([a-zA-Z\$_@]\w*|["'][^\r\n]+?["']|\d+)[ \t]*?[:=][^:=])/
+NO_NEWLINE      = /^(?:[-+*&|\/%=<>!.\\][<>=&|]*|and|or|is(?:nt)?|not|delete|typeof|instanceof)$/
+HEREDOC_INDENT  = /\n+([ \t]*)|^([ \t]+)/g
+ASSIGNED        = /^\s*((?:[a-zA-Z$_@]\w*|["'][^\n]+?["']|\d+)[ \t]*?[:=][^:=])/
 NEXT_CHARACTER  = /^\s*(\S)/
 
 # Compound assignment tokens.

From ed501ea37ec9bd062f446e3920d5c74859696e9a Mon Sep 17 00:00:00 2001
From: satyr <murky.satyr@gmail.com>
Date: Thu, 23 Sep 2010 14:11:31 +0900
Subject: [PATCH 02/10] lexer: improved consistency, preferring `charAt`,
 `slice` and single quotes

---
 src/lexer.coffee | 90 +++++++++++++++++++++++++-----------------------
 1 file changed, 46 insertions(+), 44 deletions(-)

diff --git a/src/lexer.coffee b/src/lexer.coffee
index 6999591d..af013438 100644
--- a/src/lexer.coffee
+++ b/src/lexer.coffee
@@ -105,7 +105,7 @@ exports.Lexer = class Lexer
   # Be careful not to interfere with ranges-in-progress.
   numberToken: ->
     return false unless number = @match NUMBER
-    return false if @tag() is '.' and starts number, '.'
+    return false if @tag() is '.' and number.charAt(0) is '.'
     @i += number.length
     @token 'NUMBER', number
     true
@@ -113,12 +113,12 @@ exports.Lexer = class Lexer
   # Matches strings, including multi-line strings. Ensures that quotation marks
   # are balanced within the string's contents, and within nested interpolations.
   stringToken: ->
-    return false unless starts(@chunk, '"') or starts(@chunk, "'")
+    return false unless @chunk.charAt(0) in ["'", '"']
     return false unless string =
       @balancedToken(['"', '"'], ['#{', '}']) or
       @balancedToken ["'", "'"]
     @interpolateString string.replace MULTILINER, '\\\n'
-    @line += count string, "\n"
+    @line += count string, '\n'
     @i += string.length
     true
 
@@ -148,7 +148,7 @@ exports.Lexer = class Lexer
 
   # Matches JavaScript interpolated directly into the source via backticks.
   jsToken: ->
-    return false unless starts @chunk, '`'
+    return false unless @chunk.charAt(0) is '`'
     return false unless script = @balancedToken ['`', '`']
     @token 'JS', script.slice 1, -1
     @i += script.length
@@ -251,13 +251,13 @@ exports.Lexer = class Lexer
 
   # Generate a newline token. Consecutive newlines get merged together.
   newlineToken: (newlines) ->
-    @token 'TERMINATOR', "\n" unless @tag() is 'TERMINATOR'
+    @token 'TERMINATOR', '\n' unless @tag() is 'TERMINATOR'
     true
 
   # Use a `\` at a line-ending to suppress the newline.
   # The slash is removed here once its job is done.
   suppressNewlines: ->
-    @tokens.pop() if @value() is "\\"
+    @tokens.pop() if @value() is '\\'
     true
 
   # We treat all other single characters as a token. Eg.: `( ) , . !`
@@ -327,10 +327,10 @@ exports.Lexer = class Lexer
         attempt = if match[1]? then match[1] else match[2]
         indent = attempt if not indent? or 0 < attempt.length < indent.length
     indent or= ''
-    doc = doc.replace(new RegExp("^" + indent, 'gm'), '')
+    doc = doc.replace(new RegExp('^' + indent, 'gm'), '')
     return doc if options.herecomment
-    doc = doc.replace(/^\n/, '')
-    doc.replace(MULTILINER, "\\n")
+    doc.replace(/^\n/, '')
+       .replace(MULTILINER, '\\n')
        .replace(new RegExp(options.quote, 'g'), "\\#{options.quote}")
 
   # A source of ambiguity in our grammar used to be parameter lists in function
@@ -372,8 +372,9 @@ exports.Lexer = class Lexer
     slash = delimited[0][0] is '/'
     levels = []
     i = 0
-    while i < str.length
-      if levels.length and starts str, '\\', i
+    slen = str.length
+    while i < slen
+      if levels.length and str.charAt(i) is '\\'
         i += 1
       else
         for pair in delimited
@@ -387,12 +388,12 @@ exports.Lexer = class Lexer
             levels.push(pair)
             i += open.length - 1
             break
-      break if not levels.length or slash and starts str, '\n', i
+      break if not levels.length or slash and str.charAt(i) is '\n'
       i += 1
     if levels.length
       return false if slash
       throw new Error "SyntaxError: Unterminated #{levels.pop()[0]} starting on line #{@line + 1}"
-    if not i then false else str.substring(0, i)
+    if not i then false else str[0...i]
 
   # Expand variables and expressions inside double-quoted strings using
   # [ECMA Harmony's interpolation syntax](http://wiki.ecmascript.org/doku.php?id=strawman:string_interpolation)
@@ -405,19 +406,20 @@ exports.Lexer = class Lexer
   # token stream.
   interpolateString: (str, options) ->
     options or= {}
-    if str.length < 3 or not starts str, '"'
+    if str.length < 3 or str.charAt(0) isnt '"'
       @token 'STRING', str
     else
       lexer   = new Lexer
       tokens  = []
-      quote   = str.substring 0, 1
+      quote   = str.charAt 0
       [i, pi] = [1, 1]
-      while i < str.length - 1
-        if starts str, '\\', i
+      end = str.length - 1
+      while i < end
+        if str.charAt(i) is '\\'
           i += 1
-        else if expr = @balancedString(str.substring(i), [['#{', '}']])
-          tokens.push ['STRING', quote + str.substring(pi, i) + quote] if pi < i
-          inner = expr.substring(2, expr.length - 1)
+        else if expr = @balancedString str[i..], [['#{', '}']]
+          tokens.push ['STRING', quote + str[pi...i] + quote] if pi < i
+          inner = expr.slice 2, -1
           if inner.length
             inner = inner.replace new RegExp('\\\\' + quote, 'g'), quote if options.heredoc
             nested = lexer.tokenize "(#{inner})", line: @line
@@ -429,7 +431,7 @@ exports.Lexer = class Lexer
           i += expr.length - 1
           pi = i + 1
         i += 1
-      tokens.push ['STRING', quote + str.substring(pi, i) + quote] if pi < i and pi < str.length - 1
+      tokens.push ['STRING', quote + str[pi...i] + quote] if i > pi < str.length - 1
       tokens.unshift ['STRING', '""'] unless tokens[0][0] is 'STRING'
       interpolated = tokens.length > 1
       @token '(', '(' if interpolated
@@ -438,7 +440,7 @@ exports.Lexer = class Lexer
         if tag is 'TOKENS'
           @tokens = @tokens.concat value
         else if tag is 'STRING' and options.escapeQuotes
-          escaped = value.substring(1, value.length - 1).replace(/"/g, '\\"')
+          escaped = value.slice(1, -1).replace(/"/g, '\\"')
           @token tag, "\"#{escaped}\""
         else
           @token tag, value
@@ -472,48 +474,48 @@ exports.Lexer = class Lexer
   # Attempt to match a string against the current chunk, returning the indexed
   # match if successful, and `false` otherwise.
   match: (regex, index) ->
-    return false unless m = @chunk.match regex
-    if m then m[index] else false
+    if m = @chunk.match regex then m[index or 0] else false
 
   # Are we in the midst of an unfinished expression?
   unfinished: ->
-    prev = @prev(2)
-    @value() and @value().match and @value().match(NO_NEWLINE) and
-      prev and (prev[0] isnt '.') and not @value().match(CODE) and
-      not @chunk.match ASSIGNED
+    prev  = @prev 2
+    value = @value()
+    value and NO_NEWLINE.test(value) and
+      prev and prev[0] isnt '.' and not CODE.test(value) and
+      not ASSIGNED.test(@chunk)
 
 # Constants
 # ---------
 
 # Keywords that CoffeeScript shares in common with JavaScript.
 JS_KEYWORDS = [
-  "if", "else",
-  "true", "false",
-  "new", "return",
-  "try", "catch", "finally", "throw",
-  "break", "continue",
-  "for", "in", "while",
-  "delete", "instanceof", "typeof",
-  "switch", "super", "extends", "class",
-  "this", "null", "debugger"
+  'if', 'else'
+  'true', 'false'
+  'new', 'return'
+  'try', 'catch', 'finally', 'throw'
+  'break', 'continue'
+  'for', 'in', 'while'
+  'delete', 'instanceof', 'typeof'
+  'switch', 'super', 'extends', 'class'
+  'this', 'null', 'debugger'
 ]
 
 # CoffeeScript-only keywords, which we're more relaxed about allowing. They can't
 # be used standalone, but you can reference them as an attached property.
-COFFEE_ALIASES =  ["and", "or", "is", "isnt", "not"]
+COFFEE_ALIASES =  ['and', 'or', 'is', 'isnt', 'not']
 COFFEE_KEYWORDS = COFFEE_ALIASES.concat [
-  "then", "unless", "until", "loop",
-  "yes", "no", "on", "off",
-  "of", "by", "where", "when"
+  'then', 'unless', 'until', 'loop'
+  'yes', 'no', 'on', 'off'
+  'of', 'by', 'where', 'when'
 ]
 
 # The list of keywords that are reserved by JavaScript, but not used, or are
 # used by CoffeeScript internally. We throw an error when these are encountered,
 # to avoid having a JavaScript error at runtime.
 RESERVED = [
-  "case", "default", "do", "function", "var", "void", "with",
-  "const", "let", "enum", "export", "import", "native",
-  "__hasProp", "__extends", "__slice"
+  'case', 'default', 'do', 'function', 'var', 'void', 'with'
+  'const', 'let', 'enum', 'export', 'import', 'native'
+  '__hasProp', '__extends', '__slice'
 ]
 
 # The superset of both JavaScript keywords and reserved words, none of which may

From f051d0880ef8ac7b939ff71dfc6834f444a607cf Mon Sep 17 00:00:00 2001
From: satyr <murky.satyr@gmail.com>
Date: Thu, 23 Sep 2010 14:14:18 +0900
Subject: [PATCH 03/10] lexer: improved logics

---
 lib/lexer.js     | 256 +++++++++++++++++++++--------------------------
 src/lexer.coffee |  52 +++++-----
 2 files changed, 142 insertions(+), 166 deletions(-)

diff --git a/lib/lexer.js b/lib/lexer.js
index 2f453939..bdd3d368 100644
--- a/lib/lexer.js
+++ b/lib/lexer.js
@@ -1,5 +1,5 @@
 (function() {
-  var ASSIGNED, CALLABLE, CODE, COFFEE_ALIASES, COFFEE_KEYWORDS, COMMENT, COMPARE, COMPOUND_ASSIGN, CONVERSIONS, HEREDOC, HEREDOC_INDENT, IDENTIFIER, JS_CLEANER, JS_FORBIDDEN, JS_KEYWORDS, LAST_DENT, LAST_DENTS, LINE_BREAK, LOGIC, Lexer, MATH, MULTILINER, MULTI_DENT, NEXT_CHARACTER, NOT_REGEX, NO_NEWLINE, NUMBER, OPERATOR, REGEX_END, REGEX_ESCAPE, REGEX_INTERPOLATION, REGEX_START, RESERVED, Rewriter, SHIFT, UNARY, WHITESPACE, _ref, compact, count, include, starts;
+  var ASSIGNED, CALLABLE, CODE, COFFEE_ALIASES, COFFEE_KEYWORDS, COMMENT, COMPARE, COMPOUND_ASSIGN, CONVERSIONS, HEREDOC, HEREDOC_INDENT, IDENTIFIER, JS_FORBIDDEN, JS_KEYWORDS, LINE_BREAK, LOGIC, Lexer, MATH, MULTILINER, MULTI_DENT, NEXT_CHARACTER, NOT_REGEX, NO_NEWLINE, NUMBER, OPERATOR, REGEX_END, REGEX_ESCAPE, REGEX_INTERPOLATION, REGEX_START, RESERVED, Rewriter, SHIFT, UNARY, WHITESPACE, _ref, compact, count, include, starts;
   var __slice = Array.prototype.slice;
   _ref = require('./rewriter');
   Rewriter = _ref.Rewriter;
@@ -12,7 +12,7 @@
     Lexer = function() {};
     Lexer.prototype.tokenize = function(code, options) {
       var o;
-      code = code.replace(/(\r|\s+$)/g, '');
+      code = code.replace(/\r/g, '').replace(/\s+$/, '');
       o = options || {};
       this.code = code;
       this.i = 0;
@@ -22,8 +22,7 @@
       this.outdebt = 0;
       this.indents = [];
       this.tokens = [];
-      while (this.i < this.code.length) {
-        this.chunk = this.code.slice(this.i);
+      while ((this.chunk = code.slice(this.i))) {
         this.extractNextToken();
       }
       this.closeIndentation();
@@ -33,56 +32,27 @@
       return (new Rewriter()).rewrite(this.tokens);
     };
     Lexer.prototype.extractNextToken = function() {
-      if (this.identifierToken()) {
-        return null;
-      }
-      if (this.commentToken()) {
-        return null;
-      }
-      if (this.whitespaceToken()) {
-        return null;
-      }
-      if (this.lineToken()) {
-        return null;
-      }
-      if (this.heredocToken()) {
-        return null;
-      }
-      if (this.stringToken()) {
-        return null;
-      }
-      if (this.numberToken()) {
-        return null;
-      }
-      if (this.regexToken()) {
-        return null;
-      }
-      if (this.jsToken()) {
-        return null;
-      }
-      return this.literalToken();
+      return this.identifierToken() || this.commentToken() || this.whitespaceToken() || this.lineToken() || this.heredocToken() || this.stringToken() || this.numberToken() || this.regexToken() || this.jsToken() || this.literalToken();
     };
     Lexer.prototype.identifierToken = function() {
       var close_index, forcedIdentifier, id, tag;
-      if (!(id = this.match(IDENTIFIER, 1))) {
+      if (!(id = this.match(IDENTIFIER))) {
         return false;
       }
       this.i += id.length;
       forcedIdentifier = this.tagAccessor() || this.match(ASSIGNED, 1);
       tag = 'IDENTIFIER';
-      if (include(JS_KEYWORDS, id) || (!forcedIdentifier && include(COFFEE_KEYWORDS, id))) {
+      if (include(JS_KEYWORDS, id) || !forcedIdentifier && include(COFFEE_KEYWORDS, id)) {
         tag = id.toUpperCase();
-      }
-      if (tag === 'WHEN' && include(LINE_BREAK, this.tag())) {
-        tag = 'LEADING_WHEN';
-      }
-      if (id === 'all' && this.tag() === 'FOR') {
+        if (tag === 'WHEN' && include(LINE_BREAK, this.tag())) {
+          tag = 'LEADING_WHEN';
+        }
+      } else if (id === 'all' && this.tag() === 'FOR') {
         tag = 'ALL';
       }
       if (include(UNARY, tag)) {
         tag = 'UNARY';
-      }
-      if (include(JS_FORBIDDEN, id)) {
+      } else if (include(JS_FORBIDDEN, id)) {
         if (forcedIdentifier) {
           tag = 'STRING';
           id = ("\"" + (id) + "\"");
@@ -101,11 +71,10 @@
         if (include(COFFEE_ALIASES, id)) {
           tag = (id = CONVERSIONS[id]);
         }
-        if (include(LOGIC, id)) {
-          tag = 'LOGIC';
-        }
         if (id === '!') {
           tag = 'UNARY';
+        } else if (include(LOGIC, id)) {
+          tag = 'LOGIC';
         }
       }
       this.token(tag, id);
@@ -116,10 +85,10 @@
     };
     Lexer.prototype.numberToken = function() {
       var number;
-      if (!(number = this.match(NUMBER, 1))) {
+      if (!(number = this.match(NUMBER))) {
         return false;
       }
-      if (this.tag() === '.' && starts(number, '.')) {
+      if (this.tag() === '.' && number.charAt(0) === '.') {
         return false;
       }
       this.i += number.length;
@@ -127,43 +96,47 @@
       return true;
     };
     Lexer.prototype.stringToken = function() {
-      var string;
-      if (!(starts(this.chunk, '"') || starts(this.chunk, "'"))) {
+      var _ref2, string;
+      if (!(("'" === (_ref2 = this.chunk.charAt(0)) || '"' === _ref2))) {
         return false;
       }
       if (!(string = this.balancedToken(['"', '"'], ['#{', '}']) || this.balancedToken(["'", "'"]))) {
         return false;
       }
-      this.interpolateString(string.replace(/\n/g, '\\\n'));
-      this.line += count(string, "\n");
+      this.interpolateString(string.replace(MULTILINER, '\\\n'));
+      this.line += count(string, '\n');
       this.i += string.length;
       return true;
     };
     Lexer.prototype.heredocToken = function() {
-      var doc, match, quote;
+      var doc, heredoc, match, quote;
       if (!(match = this.chunk.match(HEREDOC))) {
         return false;
       }
-      quote = match[1].substr(0, 1);
-      doc = this.sanitizeHeredoc(match[2] || match[4] || '', {
+      heredoc = match[0];
+      quote = heredoc.charAt(0);
+      doc = this.sanitizeHeredoc(match[2], {
         quote: quote
       });
       this.interpolateString(quote + doc + quote, {
         heredoc: true
       });
-      this.line += count(match[1], "\n");
-      this.i += match[1].length;
+      this.line += count(heredoc, '\n');
+      this.i += heredoc.length;
       return true;
     };
     Lexer.prototype.commentToken = function() {
-      var match;
+      var _ref2, comment, here, match;
       if (!(match = this.chunk.match(COMMENT))) {
         return false;
       }
-      this.line += count(match[1], "\n");
-      this.i += match[1].length;
-      if (match[2]) {
-        this.token('HERECOMMENT', this.sanitizeHeredoc(match[2], {
+      _ref2 = match;
+      comment = _ref2[0];
+      here = _ref2[1];
+      this.line += count(comment, '\n');
+      this.i += comment.length;
+      if (here) {
+        this.token('HERECOMMENT', this.sanitizeHeredoc(here, {
           herecomment: true,
           indent: Array(this.indent + 1).join(' ')
         }));
@@ -173,13 +146,13 @@
     };
     Lexer.prototype.jsToken = function() {
       var script;
-      if (!(starts(this.chunk, '`'))) {
+      if (this.chunk.charAt(0) !== '`') {
         return false;
       }
       if (!(script = this.balancedToken(['`', '`']))) {
         return false;
       }
-      this.token('JS', script.replace(JS_CLEANER, ''));
+      this.token('JS', script.slice(1, -1));
       this.i += script.length;
       return true;
     };
@@ -197,29 +170,25 @@
       if (!(regex = this.balancedToken(['/', '/']))) {
         return false;
       }
-      if (!(end = this.chunk.substr(regex.length).match(REGEX_END))) {
+      if (!(end = this.chunk.slice(regex.length).match(REGEX_END))) {
         return false;
       }
-      if (end[2]) {
-        regex += (flags = end[2]);
-      }
-      if (regex.match(REGEX_INTERPOLATION)) {
-        str = regex.substring(1).split('/')[0];
-        str = str.replace(REGEX_ESCAPE, function(escaped) {
-          return '\\' + escaped;
-        });
-        this.tokens = this.tokens.concat([['(', '('], ['NEW', 'new'], ['IDENTIFIER', 'RegExp'], ['CALL_START', '(']]);
+      flags = end[0];
+      if (REGEX_INTERPOLATION.test(regex)) {
+        str = regex.slice(1, -1);
+        str = str.replace(REGEX_ESCAPE, '\\$&');
+        this.tokens.push(['(', '('], ['NEW', 'new'], ['IDENTIFIER', 'RegExp'], ['CALL_START', '(']);
         this.interpolateString("\"" + (str) + "\"", {
           escapeQuotes: true
         });
         if (flags) {
-          this.tokens.splice(this.tokens.length, 0, [',', ','], ['STRING', ("\"" + (flags) + "\"")]);
+          this.tokens.push([',', ','], ['STRING', ("\"" + (flags) + "\"")]);
         }
-        this.tokens.splice(this.tokens.length, 0, [')', ')'], [')', ')']);
+        this.tokens.push([')', ')'], [')', ')']);
       } else {
-        this.token('REGEX', regex);
+        this.token('REGEX', regex + flags);
       }
-      this.i += regex.length;
+      this.i += regex.length + flags.length;
       return true;
     };
     Lexer.prototype.balancedToken = function() {
@@ -229,13 +198,13 @@
     };
     Lexer.prototype.lineToken = function() {
       var diff, indent, nextCharacter, noNewlines, prev, size;
-      if (!(indent = this.match(MULTI_DENT, 1))) {
+      if (!(indent = this.match(MULTI_DENT))) {
         return false;
       }
-      this.line += count(indent, "\n");
+      this.line += count(indent, '\n');
       this.i += indent.length;
       prev = this.prev(2);
-      size = indent.match(LAST_DENTS).reverse()[0].match(LAST_DENT)[1].length;
+      size = indent.length - 1 - indent.lastIndexOf('\n');
       nextCharacter = this.match(NEXT_CHARACTER, 1);
       noNewlines = nextCharacter === '.' || nextCharacter === ',' || this.unfinished();
       if (size - this.indebt === this.indent) {
@@ -283,13 +252,13 @@
         this.outdebt -= moveOut;
       }
       if (!(this.tag() === 'TERMINATOR' || noNewlines)) {
-        this.token('TERMINATOR', "\n");
+        this.token('TERMINATOR', '\n');
       }
       return true;
     };
     Lexer.prototype.whitespaceToken = function() {
       var prev, space;
-      if (!(space = this.match(WHITESPACE, 1))) {
+      if (!(space = this.match(WHITESPACE))) {
         return false;
       }
       prev = this.prev();
@@ -301,25 +270,28 @@
     };
     Lexer.prototype.newlineToken = function(newlines) {
       if (this.tag() !== 'TERMINATOR') {
-        this.token('TERMINATOR', "\n");
+        this.token('TERMINATOR', '\n');
       }
       return true;
     };
     Lexer.prototype.suppressNewlines = function() {
-      if (this.value() === "\\") {
+      if (this.value() === '\\') {
         this.tokens.pop();
       }
       return true;
     };
     Lexer.prototype.literalToken = function() {
       var _ref2, match, prev, space, spaced, tag, value;
-      match = this.chunk.match(OPERATOR);
-      value = match && match[1];
-      space = match && match[2];
-      if (value && value.match(CODE)) {
-        this.tagParameters();
+      if (match = this.chunk.match(OPERATOR)) {
+        _ref2 = match;
+        value = _ref2[0];
+        space = _ref2[1];
+        if (CODE.test(value)) {
+          this.tagParameters();
+        }
+      } else {
+        value = this.chunk.charAt(0);
       }
-      value || (value = this.chunk.substr(0, 1));
       this.i += value.length;
       spaced = (prev = this.prev()) && prev.spaced;
       tag = value;
@@ -354,11 +326,13 @@
           tag = 'CALL_START';
         } else if (value === '[') {
           tag = 'INDEX_START';
-          if (this.tag() === '?') {
-            this.tag(1, 'INDEX_SOAK');
-          }
-          if (this.tag() === '::') {
-            this.tag(1, 'INDEX_PROTO');
+          switch (this.tag()) {
+            case '?':
+              this.tag(1, 'INDEX_SOAK');
+              break;
+            case '::':
+              this.tag(1, 'INDEX_PROTO');
+              break;
           }
         }
       }
@@ -373,7 +347,7 @@
       accessor = (function() {
         if (prev[1] === '::') {
           return this.tag(1, 'PROTOTYPE_ACCESS');
-        } else if (prev[1] === '.' && !(this.value(2) === '.')) {
+        } else if (prev[1] === '.' && this.value(2) !== '.') {
           if (this.tag(2) === '?') {
             this.tag(1, 'SOAK_ACCESS');
             return this.tokens.splice(-2, 1);
@@ -393,20 +367,19 @@
         return doc;
       }
       if (!(options.herecomment)) {
-        while ((match = HEREDOC_INDENT.exec(doc)) !== null) {
-          attempt = (typeof (_ref2 = match[2]) !== "undefined" && _ref2 !== null) ? match[2] : match[3];
+        while ((match = HEREDOC_INDENT.exec(doc))) {
+          attempt = (typeof (_ref2 = match[1]) !== "undefined" && _ref2 !== null) ? match[1] : match[2];
           if (!(typeof indent !== "undefined" && indent !== null) || (0 < attempt.length) && (attempt.length < indent.length)) {
             indent = attempt;
           }
         }
       }
       indent || (indent = '');
-      doc = doc.replace(new RegExp("^" + indent, 'gm'), '');
+      doc = doc.replace(new RegExp('^' + indent, 'gm'), '');
       if (options.herecomment) {
         return doc;
       }
-      doc = doc.replace(/^\n/, '');
-      return doc.replace(MULTILINER, "\\n").replace(new RegExp(options.quote, 'g'), "\\" + (options.quote));
+      return doc.replace(/^\n/, '').replace(MULTILINER, '\\n').replace(new RegExp(options.quote, 'g'), "\\" + (options.quote));
     };
     Lexer.prototype.tagParameters = function() {
       var i, tok;
@@ -444,13 +417,14 @@
       throw new Error("SyntaxError: Reserved word \"" + (this.value()) + "\" on line " + (this.line + 1) + " can't be assigned");
     };
     Lexer.prototype.balancedString = function(str, delimited, options) {
-      var _i, _len, _ref2, _ref3, close, i, levels, open, pair, slash;
+      var _i, _len, _ref2, _ref3, close, i, levels, open, pair, slash, slen;
       options || (options = {});
       slash = delimited[0][0] === '/';
       levels = [];
       i = 0;
-      while (i < str.length) {
-        if (levels.length && starts(str, '\\', i)) {
+      slen = str.length;
+      while (i < slen) {
+        if (levels.length && str.charAt(i) === '\\') {
           i += 1;
         } else {
           _ref2 = delimited;
@@ -473,7 +447,7 @@
             }
           }
         }
-        if (!levels.length || slash && starts(str, '\n', i)) {
+        if (!levels.length || slash && str.charAt(i) === '\n') {
           break;
         }
         i += 1;
@@ -484,28 +458,29 @@
         }
         throw new Error("SyntaxError: Unterminated " + (levels.pop()[0]) + " starting on line " + (this.line + 1));
       }
-      return !i ? false : str.substring(0, i);
+      return !i ? false : str.slice(0, i);
     };
     Lexer.prototype.interpolateString = function(str, options) {
-      var _len, _ref2, _ref3, escaped, expr, i, idx, inner, interpolated, lexer, nested, pi, quote, tag, tok, token, tokens, value;
+      var _len, _ref2, _ref3, end, escaped, expr, i, idx, inner, interpolated, lexer, nested, pi, quote, tag, tok, token, tokens, value;
       options || (options = {});
-      if (str.length < 3 || !starts(str, '"')) {
+      if (str.length < 3 || str.charAt(0) !== '"') {
         return this.token('STRING', str);
       } else {
         lexer = new Lexer();
         tokens = [];
-        quote = str.substring(0, 1);
+        quote = str.charAt(0);
         _ref2 = [1, 1];
         i = _ref2[0];
         pi = _ref2[1];
-        while (i < str.length - 1) {
-          if (starts(str, '\\', i)) {
+        end = str.length - 1;
+        while (i < end) {
+          if (str.charAt(i) === '\\') {
             i += 1;
-          } else if (expr = this.balancedString(str.substring(i), [['#{', '}']])) {
+          } else if (expr = this.balancedString(str.slice(i), [['#{', '}']])) {
             if (pi < i) {
-              tokens.push(['STRING', quote + str.substring(pi, i) + quote]);
+              tokens.push(['STRING', quote + str.slice(pi, i) + quote]);
             }
-            inner = expr.substring(2, expr.length - 1);
+            inner = expr.slice(2, -1);
             if (inner.length) {
               if (options.heredoc) {
                 inner = inner.replace(new RegExp('\\\\' + quote, 'g'), quote);
@@ -530,8 +505,8 @@
           }
           i += 1;
         }
-        if (pi < i && pi < str.length - 1) {
-          tokens.push(['STRING', quote + str.substring(pi, i) + quote]);
+        if ((i > pi) && (pi < str.length - 1)) {
+          tokens.push(['STRING', quote + str.slice(pi, i) + quote]);
         }
         if (tokens[0][0] !== 'STRING') {
           tokens.unshift(['STRING', '""']);
@@ -549,7 +524,7 @@
           if (tag === 'TOKENS') {
             this.tokens = this.tokens.concat(value);
           } else if (tag === 'STRING' && options.escapeQuotes) {
-            escaped = value.substring(1, value.length - 1).replace(/"/g, '\\"');
+            escaped = value.slice(1, -1).replace(/"/g, '\\"');
             this.token(tag, "\"" + (escaped) + "\"");
           } else {
             this.token(tag, value);
@@ -592,42 +567,37 @@
     };
     Lexer.prototype.match = function(regex, index) {
       var m;
-      if (!(m = this.chunk.match(regex))) {
-        return false;
-      }
-      return m ? m[index] : false;
+      return (m = this.chunk.match(regex)) ? m[index || 0] : false;
     };
     Lexer.prototype.unfinished = function() {
-      var prev;
+      var prev, value;
       prev = this.prev(2);
-      return this.value() && this.value().match && this.value().match(NO_NEWLINE) && prev && (prev[0] !== '.') && !this.value().match(CODE) && !this.chunk.match(ASSIGNED);
+      value = this.value();
+      return value && NO_NEWLINE.test(value) && prev && prev[0] !== '.' && !CODE.test(value) && !ASSIGNED.test(this.chunk);
     };
     return Lexer;
   })();
-  JS_KEYWORDS = ["if", "else", "true", "false", "new", "return", "try", "catch", "finally", "throw", "break", "continue", "for", "in", "while", "delete", "instanceof", "typeof", "switch", "super", "extends", "class", "this", "null", "debugger"];
-  COFFEE_ALIASES = ["and", "or", "is", "isnt", "not"];
-  COFFEE_KEYWORDS = COFFEE_ALIASES.concat(["then", "unless", "until", "loop", "yes", "no", "on", "off", "of", "by", "where", "when"]);
-  RESERVED = ["case", "default", "do", "function", "var", "void", "with", "const", "let", "enum", "export", "import", "native", "__hasProp", "__extends", "__slice"];
+  JS_KEYWORDS = ['if', 'else', 'true', 'false', 'new', 'return', 'try', 'catch', 'finally', 'throw', 'break', 'continue', 'for', 'in', 'while', 'delete', 'instanceof', 'typeof', 'switch', 'super', 'extends', 'class', 'this', 'null', 'debugger'];
+  COFFEE_ALIASES = ['and', 'or', 'is', 'isnt', 'not'];
+  COFFEE_KEYWORDS = COFFEE_ALIASES.concat(['then', 'unless', 'until', 'loop', 'yes', 'no', 'on', 'off', 'of', 'by', 'where', 'when']);
+  RESERVED = ['case', 'default', 'do', 'function', 'var', 'void', 'with', 'const', 'let', 'enum', 'export', 'import', 'native', '__hasProp', '__extends', '__slice'];
   JS_FORBIDDEN = JS_KEYWORDS.concat(RESERVED);
-  IDENTIFIER = /^([a-zA-Z\$_](\w|\$)*)/;
-  NUMBER = /^(((\b0(x|X)[0-9a-fA-F]+)|((\b[0-9]+(\.[0-9]+)?|\.[0-9]+)(e[+\-]?[0-9]+)?)))\b/i;
-  HEREDOC = /^("{6}|'{6}|"{3}([\s\S]*?)\n?([ \t]*)"{3}|'{3}([\s\S]*?)\n?([ \t]*)'{3})/;
-  OPERATOR = /^(-[\-=>]?|\+[+=]?|[*&|\/%=<>^:!?]+)([ \t]*)/;
-  WHITESPACE = /^([ \t]+)/;
-  COMMENT = /^(###([^#][\s\S]*?)(###[ \t]*\n|(###)?$)|(\s*#(?!##[^#])[^\n]*)+)/;
-  CODE = /^((-|=)>)/;
-  MULTI_DENT = /^((\n([ \t]*))+)(\.)?/;
-  LAST_DENTS = /\n([ \t]*)/g;
-  LAST_DENT = /\n([ \t]*)/;
+  IDENTIFIER = /^[a-zA-Z_$][\w$]*/;
+  NUMBER = /^(?:0x[\da-f]+)|^(?:\d+(\.\d+)?|\.\d+)(?:e[+-]?\d+)?/i;
+  HEREDOC = /^("""|''')([\s\S]*?)\n?[ \t]*\1/;
+  OPERATOR = /^(?:-[-=>]?|\+[+=]?|[*&|\/%=<>^:!?]+)(?=([ \t]*))/;
+  WHITESPACE = /^[ \t]+/;
+  COMMENT = /^###([^#][\s\S]*?)(?:###[ \t]*\n|(?:###)?$)|^(?:\s*#(?!##[^#])[^\n]*)+/;
+  CODE = /^[-=]>/;
+  MULTI_DENT = /^(?:\n[ \t]*)+/;
   REGEX_START = /^\/([^\/])/;
-  REGEX_INTERPOLATION = /([^\\]#\{.*[^\\]\})/;
-  REGEX_END = /^(([imgy]{1,4})\b|\W|$)/;
-  REGEX_ESCAPE = /\\[^\$]/g;
-  JS_CLEANER = /(^`|`$)/g;
+  REGEX_INTERPOLATION = /[^\\]#\{.*[^\\]\}/;
+  REGEX_END = /^[imgy]{0,4}(?![a-zA-Z])/;
+  REGEX_ESCAPE = /\\[^#]/g;
   MULTILINER = /\n/g;
-  NO_NEWLINE = /^([+\*&|\/\-%=<>!.\\][<>=&|]*|and|or|is|isnt|not|delete|typeof|instanceof)$/;
-  HEREDOC_INDENT = /(\n+([ \t]*)|^([ \t]+))/g;
-  ASSIGNED = /^\s*(([a-zA-Z\$_@]\w*|["'][^\r\n]+?["']|\d+)[ \t]*?[:=][^:=])/;
+  NO_NEWLINE = /^(?:[-+*&|\/%=<>!.\\][<>=&|]*|and|or|is(?:nt)?|not|delete|typeof|instanceof)$/;
+  HEREDOC_INDENT = /\n+([ \t]*)|^([ \t]+)/g;
+  ASSIGNED = /^\s*((?:[a-zA-Z$_@]\w*|["'][^\n]+?["']|\d+)[ \t]*?[:=][^:=])/;
   NEXT_CHARACTER = /^\s*(\S)/;
   COMPOUND_ASSIGN = ['-=', '+=', '/=', '*=', '%=', '||=', '&&=', '?=', '<<=', '>>=', '>>>=', '&=', '^=', '|='];
   UNARY = ['UMINUS', 'UPLUS', '!', '!!', '~', 'TYPEOF', 'DELETE'];
diff --git a/src/lexer.coffee b/src/lexer.coffee
index af013438..a265e15e 100644
--- a/src/lexer.coffee
+++ b/src/lexer.coffee
@@ -43,8 +43,7 @@ exports.Lexer = class Lexer
     @outdebt = 0            # The under-outdentation at the current level.
     @indents = []           # The stack of all current indentation levels.
     @tokens  = []           # Stream of parsed tokens in the form ['TYPE', value, line]
-    while @i < @code.length
-      @chunk = @code[@i..]
+    while (@chunk = code[@i..])
       @extractNextToken()
     @closeIndentation()
     return @tokens if o.rewrite is off
@@ -54,16 +53,16 @@ exports.Lexer = class Lexer
   # short-circuiting if any of them succeed. Their order determines precedence:
   # `@literalToken` is the fallback catch-all.
   extractNextToken: ->
-    return if @identifierToken()
-    return if @commentToken()
-    return if @whitespaceToken()
-    return if @lineToken()
-    return if @heredocToken()
-    return if @stringToken()
-    return if @numberToken()
-    return if @regexToken()
-    return if @jsToken()
-    return    @literalToken()
+    @identifierToken() or
+    @commentToken()    or
+    @whitespaceToken() or
+    @lineToken()       or
+    @heredocToken()    or
+    @stringToken()     or
+    @numberToken()     or
+    @regexToken()      or
+    @jsToken()         or
+    @literalToken()
 
   # Tokenizers
   # ----------
@@ -79,11 +78,15 @@ exports.Lexer = class Lexer
     @i += id.length
     forcedIdentifier = @tagAccessor() or @match ASSIGNED, 1
     tag = 'IDENTIFIER'
-    tag = id.toUpperCase() if include(JS_KEYWORDS, id) or (not forcedIdentifier and include(COFFEE_KEYWORDS, id))
-    tag = 'LEADING_WHEN'   if tag is 'WHEN' and include LINE_BREAK, @tag()
-    tag = 'ALL'            if id is 'all' and @tag() is 'FOR'
-    tag = 'UNARY'          if include UNARY, tag
-    if include(JS_FORBIDDEN, id)
+    if include(JS_KEYWORDS, id) or
+       not forcedIdentifier and include(COFFEE_KEYWORDS, id)
+      tag = id.toUpperCase()
+      tag = 'LEADING_WHEN' if tag is 'WHEN' and include LINE_BREAK, @tag()
+    else if id is 'all' and @tag() is 'FOR'
+      tag = 'ALL'
+    if include UNARY, tag
+      tag = 'UNARY'
+    else if include JS_FORBIDDEN, id
       if forcedIdentifier
         tag = 'STRING'
         id  = "\"#{id}\""
@@ -95,8 +98,10 @@ exports.Lexer = class Lexer
         @identifierError id
     unless forcedIdentifier
       tag = id = CONVERSIONS[id] if include COFFEE_ALIASES, id
-      tag = 'LOGIC' if include LOGIC, id
-      tag = 'UNARY' if id is '!'
+      if id is '!'
+        tag = 'UNARY'
+      else if include LOGIC, id
+        tag = 'LOGIC'
     @token tag, id
     @token ']', ']' if close_index
     true
@@ -177,7 +182,7 @@ exports.Lexer = class Lexer
     @i += regex.length + flags.length
     true
 
-  # Matches a token in which which the passed delimiter pairs must be correctly
+  # Matches a token in which the passed delimiter pairs must be correctly
   # balanced (ie. strings, JS literals).
   balancedToken: (delimited...) ->
     @balancedString @chunk, delimited
@@ -292,8 +297,9 @@ exports.Lexer = class Lexer
         tag = 'CALL_START'
       else if value is '['
         tag = 'INDEX_START'
-        @tag 1, 'INDEX_SOAK'  if @tag() is '?'
-        @tag 1, 'INDEX_PROTO' if @tag() is '::'
+        switch @tag()
+          when '?'  then @tag 1, 'INDEX_SOAK'
+          when '::' then @tag 1, 'INDEX_PROTO'
     @token tag, value
     true
 
@@ -307,7 +313,7 @@ exports.Lexer = class Lexer
     return false if (not prev = @prev()) or (prev and prev.spaced)
     accessor = if prev[1] is '::'
       @tag 1, 'PROTOTYPE_ACCESS'
-    else if prev[1] is '.' and not (@value(2) is '.')
+    else if prev[1] is '.' and @value(2) isnt '.'
       if @tag(2) is '?'
         @tag(1, 'SOAK_ACCESS')
         @tokens.splice(-2, 1)

From d457423c24a9f9fef8ca3a9155449acea05b35af Mon Sep 17 00:00:00 2001
From: satyr <murky.satyr@gmail.com>
Date: Fri, 24 Sep 2010 22:38:28 +0900
Subject: [PATCH 04/10] made simple strings shortcut

---
 lib/lexer.js     | 32 ++++++++++++++++++++------------
 src/lexer.coffee | 19 ++++++++++++-------
 2 files changed, 32 insertions(+), 19 deletions(-)

diff --git a/lib/lexer.js b/lib/lexer.js
index bdd3d368..cea6eb80 100644
--- a/lib/lexer.js
+++ b/lib/lexer.js
@@ -1,5 +1,5 @@
 (function() {
-  var ASSIGNED, CALLABLE, CODE, COFFEE_ALIASES, COFFEE_KEYWORDS, COMMENT, COMPARE, COMPOUND_ASSIGN, CONVERSIONS, HEREDOC, HEREDOC_INDENT, IDENTIFIER, JS_FORBIDDEN, JS_KEYWORDS, LINE_BREAK, LOGIC, Lexer, MATH, MULTILINER, MULTI_DENT, NEXT_CHARACTER, NOT_REGEX, NO_NEWLINE, NUMBER, OPERATOR, REGEX_END, REGEX_ESCAPE, REGEX_INTERPOLATION, REGEX_START, RESERVED, Rewriter, SHIFT, UNARY, WHITESPACE, _ref, compact, count, include, starts;
+  var ASSIGNED, CALLABLE, CODE, COFFEE_ALIASES, COFFEE_KEYWORDS, COMMENT, COMPARE, COMPOUND_ASSIGN, CONVERSIONS, HEREDOC, HEREDOC_INDENT, IDENTIFIER, JSTOKEN, JS_FORBIDDEN, JS_KEYWORDS, LINE_BREAK, LOGIC, Lexer, MATH, MULTILINER, MULTI_DENT, NEXT_CHARACTER, NOT_REGEX, NO_NEWLINE, NUMBER, OPERATOR, REGEX_END, REGEX_ESCAPE, REGEX_INTERPOLATION, REGEX_START, RESERVED, Rewriter, SHIFT, SIMPLESTR, UNARY, WHITESPACE, _ref, compact, count, include, starts;
   var __slice = Array.prototype.slice;
   _ref = require('./rewriter');
   Rewriter = _ref.Rewriter;
@@ -96,14 +96,23 @@
       return true;
     };
     Lexer.prototype.stringToken = function() {
-      var _ref2, string;
-      if (!(("'" === (_ref2 = this.chunk.charAt(0)) || '"' === _ref2))) {
-        return false;
+      var string;
+      switch (this.chunk.charAt(0)) {
+        case "'":
+          if (!(string = this.match(SIMPLESTR))) {
+            return false;
+          }
+          this.token('STRING', string.replace(MULTILINER, '\\\n'));
+          break;
+        case '"':
+          if (!(string = this.balancedToken(['"', '"'], ['#{', '}']))) {
+            return false;
+          }
+          this.interpolateString(string.replace(MULTILINER, '\\\n'));
+          break;
+        default:
+          return false;
       }
-      if (!(string = this.balancedToken(['"', '"'], ['#{', '}']) || this.balancedToken(["'", "'"]))) {
-        return false;
-      }
-      this.interpolateString(string.replace(MULTILINER, '\\\n'));
       this.line += count(string, '\n');
       this.i += string.length;
       return true;
@@ -146,10 +155,7 @@
     };
     Lexer.prototype.jsToken = function() {
       var script;
-      if (this.chunk.charAt(0) !== '`') {
-        return false;
-      }
-      if (!(script = this.balancedToken(['`', '`']))) {
+      if (!(this.chunk.charAt(0) === '`' && (script = this.match(JSTOKEN)))) {
         return false;
       }
       this.token('JS', script.slice(1, -1));
@@ -590,6 +596,8 @@
   COMMENT = /^###([^#][\s\S]*?)(?:###[ \t]*\n|(?:###)?$)|^(?:\s*#(?!##[^#])[^\n]*)+/;
   CODE = /^[-=]>/;
   MULTI_DENT = /^(?:\n[ \t]*)+/;
+  SIMPLESTR = /^'[^\\']*(?:\\.[^\\']*)*'/;
+  JSTOKEN = /^`[^\\`]*(?:\\.[^\\`]*)*`/;
   REGEX_START = /^\/([^\/])/;
   REGEX_INTERPOLATION = /[^\\]#\{.*[^\\]\}/;
   REGEX_END = /^[imgy]{0,4}(?![a-zA-Z])/;
diff --git a/src/lexer.coffee b/src/lexer.coffee
index a265e15e..483d3bda 100644
--- a/src/lexer.coffee
+++ b/src/lexer.coffee
@@ -118,11 +118,15 @@ exports.Lexer = class Lexer
   # Matches strings, including multi-line strings. Ensures that quotation marks
   # are balanced within the string's contents, and within nested interpolations.
   stringToken: ->
-    return false unless @chunk.charAt(0) in ["'", '"']
-    return false unless string =
-      @balancedToken(['"', '"'], ['#{', '}']) or
-      @balancedToken ["'", "'"]
-    @interpolateString string.replace MULTILINER, '\\\n'
+    switch @chunk.charAt 0
+      when "'"
+        return false unless string = @match SIMPLESTR
+        @token 'STRING', string.replace MULTILINER, '\\\n'
+      when '"'
+        return false unless string = @balancedToken ['"', '"'], ['#{', '}']
+        @interpolateString string.replace MULTILINER, '\\\n'
+      else
+        return false
     @line += count string, '\n'
     @i += string.length
     true
@@ -153,8 +157,7 @@ exports.Lexer = class Lexer
 
   # Matches JavaScript interpolated directly into the source via backticks.
   jsToken: ->
-    return false unless @chunk.charAt(0) is '`'
-    return false unless script = @balancedToken ['`', '`']
+    return false unless @chunk.charAt(0) is '`' and script = @match JSTOKEN
     @token 'JS', script.slice 1, -1
     @i += script.length
     true
@@ -537,6 +540,8 @@ WHITESPACE = /^[ \t]+/
 COMMENT    = /^###([^#][\s\S]*?)(?:###[ \t]*\n|(?:###)?$)|^(?:\s*#(?!##[^#])[^\n]*)+/
 CODE       = /^[-=]>/
 MULTI_DENT = /^(?:\n[ \t]*)+/
+SIMPLESTR  = /^'[^\\']*(?:\\.[^\\']*)*'/
+JSTOKEN    = /^`[^\\`]*(?:\\.[^\\`]*)*`/
 
 # Regex-matching-regexes.
 REGEX_START         = /^\/([^\/])/

From 3fd7f9efdd7d735b670eeef95af1a1bc6837ba7b Mon Sep 17 00:00:00 2001
From: satyr <murky.satyr@gmail.com>
Date: Fri, 24 Sep 2010 23:06:49 +0900
Subject: [PATCH 05/10] added a test for JS literal

---
 test/test_literals.coffee | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/test/test_literals.coffee b/test/test_literals.coffee
index fd6d479d..b8b2bb72 100644
--- a/test/test_literals.coffee
+++ b/test/test_literals.coffee
@@ -239,3 +239,9 @@ b = identity identity identity
       a: 100
 
 ok b is 100
+
+
+# Inline JS
+ok '\\`' is `
+  "\\\`"
+`

From 9a3b736174fcd7afb996ee6df534be4a5aa0572f Mon Sep 17 00:00:00 2001
From: satyr <murky.satyr@gmail.com>
Date: Sat, 25 Sep 2010 16:00:07 +0900
Subject: [PATCH 06/10] lexer: fixed broken logics (due to f051d088) and a
 snakecased variable

---
 lib/lexer.js     | 18 ++++++++++--------
 src/lexer.coffee | 18 ++++++++++--------
 2 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/lib/lexer.js b/lib/lexer.js
index cea6eb80..22f1c388 100644
--- a/lib/lexer.js
+++ b/lib/lexer.js
@@ -35,29 +35,31 @@
       return this.identifierToken() || this.commentToken() || this.whitespaceToken() || this.lineToken() || this.heredocToken() || this.stringToken() || this.numberToken() || this.regexToken() || this.jsToken() || this.literalToken();
     };
     Lexer.prototype.identifierToken = function() {
-      var close_index, forcedIdentifier, id, tag;
+      var closeIndex, forcedIdentifier, id, tag;
       if (!(id = this.match(IDENTIFIER))) {
         return false;
       }
       this.i += id.length;
+      if (id === 'all' && this.tag() === 'FOR') {
+        this.token('ALL', id);
+        return true;
+      }
       forcedIdentifier = this.tagAccessor() || this.match(ASSIGNED, 1);
       tag = 'IDENTIFIER';
       if (include(JS_KEYWORDS, id) || !forcedIdentifier && include(COFFEE_KEYWORDS, id)) {
         tag = id.toUpperCase();
         if (tag === 'WHEN' && include(LINE_BREAK, this.tag())) {
           tag = 'LEADING_WHEN';
+        } else if (include(UNARY, tag)) {
+          tag = 'UNARY';
         }
-      } else if (id === 'all' && this.tag() === 'FOR') {
-        tag = 'ALL';
       }
-      if (include(UNARY, tag)) {
-        tag = 'UNARY';
-      } else if (include(JS_FORBIDDEN, id)) {
+      if (include(JS_FORBIDDEN, id)) {
         if (forcedIdentifier) {
           tag = 'STRING';
           id = ("\"" + (id) + "\"");
           if (forcedIdentifier === 'accessor') {
-            close_index = true;
+            closeIndex = true;
             if (this.tag() !== '@') {
               this.tokens.pop();
             }
@@ -78,7 +80,7 @@
         }
       }
       this.token(tag, id);
-      if (close_index) {
+      if (closeIndex) {
         this.token(']', ']');
       }
       return true;
diff --git a/src/lexer.coffee b/src/lexer.coffee
index 483d3bda..0cc35db4 100644
--- a/src/lexer.coffee
+++ b/src/lexer.coffee
@@ -76,22 +76,24 @@ exports.Lexer = class Lexer
   identifierToken: ->
     return false unless id = @match IDENTIFIER
     @i += id.length
+    if id is 'all' and @tag() is 'FOR'
+      @token 'ALL', id
+      return true
     forcedIdentifier = @tagAccessor() or @match ASSIGNED, 1
     tag = 'IDENTIFIER'
     if include(JS_KEYWORDS, id) or
        not forcedIdentifier and include(COFFEE_KEYWORDS, id)
       tag = id.toUpperCase()
-      tag = 'LEADING_WHEN' if tag is 'WHEN' and include LINE_BREAK, @tag()
-    else if id is 'all' and @tag() is 'FOR'
-      tag = 'ALL'
-    if include UNARY, tag
-      tag = 'UNARY'
-    else if include JS_FORBIDDEN, id
+      if tag is 'WHEN' and include LINE_BREAK, @tag()
+        tag = 'LEADING_WHEN'
+      else if include UNARY, tag
+        tag = 'UNARY'
+    if include JS_FORBIDDEN, id
       if forcedIdentifier
         tag = 'STRING'
         id  = "\"#{id}\""
         if forcedIdentifier is 'accessor'
-          close_index = true
+          closeIndex = on
           @tokens.pop() if @tag() isnt '@'
           @token 'INDEX_START', '['
       else if include(RESERVED, id)
@@ -103,7 +105,7 @@ exports.Lexer = class Lexer
       else if include LOGIC, id
         tag = 'LOGIC'
     @token tag, id
-    @token ']', ']' if close_index
+    @token ']', ']' if closeIndex
     true
 
   # Matches numbers, including decimals, hex, and exponential notation.

From c515aaac5a595fcc7567846ddaa607c2343aabe9 Mon Sep 17 00:00:00 2001
From: satyr <murky.satyr@gmail.com>
Date: Sat, 25 Sep 2010 23:37:33 +0900
Subject: [PATCH 07/10] lexer: fixed ASSIGNED

---
 lib/lexer.js     | 4 ++--
 src/lexer.coffee | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/lexer.js b/lib/lexer.js
index 22f1c388..896aa9f7 100644
--- a/lib/lexer.js
+++ b/lib/lexer.js
@@ -44,7 +44,7 @@
         this.token('ALL', id);
         return true;
       }
-      forcedIdentifier = this.tagAccessor() || this.match(ASSIGNED, 1);
+      forcedIdentifier = this.tagAccessor() || ASSIGNED.test(this.chunk);
       tag = 'IDENTIFIER';
       if (include(JS_KEYWORDS, id) || !forcedIdentifier && include(COFFEE_KEYWORDS, id)) {
         tag = id.toUpperCase();
@@ -607,7 +607,7 @@
   MULTILINER = /\n/g;
   NO_NEWLINE = /^(?:[-+*&|\/%=<>!.\\][<>=&|]*|and|or|is(?:nt)?|not|delete|typeof|instanceof)$/;
   HEREDOC_INDENT = /\n+([ \t]*)|^([ \t]+)/g;
-  ASSIGNED = /^\s*((?:[a-zA-Z$_@]\w*|["'][^\n]+?["']|\d+)[ \t]*?[:=][^:=])/;
+  ASSIGNED = /^\s*@?[$A-Za-z_][$\w]*[ \t]*?[:=][^:=>]/;
   NEXT_CHARACTER = /^\s*(\S)/;
   COMPOUND_ASSIGN = ['-=', '+=', '/=', '*=', '%=', '||=', '&&=', '?=', '<<=', '>>=', '>>>=', '&=', '^=', '|='];
   UNARY = ['UMINUS', 'UPLUS', '!', '!!', '~', 'TYPEOF', 'DELETE'];
diff --git a/src/lexer.coffee b/src/lexer.coffee
index 0cc35db4..ba827679 100644
--- a/src/lexer.coffee
+++ b/src/lexer.coffee
@@ -79,7 +79,7 @@ exports.Lexer = class Lexer
     if id is 'all' and @tag() is 'FOR'
       @token 'ALL', id
       return true
-    forcedIdentifier = @tagAccessor() or @match ASSIGNED, 1
+    forcedIdentifier = @tagAccessor() or ASSIGNED.test @chunk
     tag = 'IDENTIFIER'
     if include(JS_KEYWORDS, id) or
        not forcedIdentifier and include(COFFEE_KEYWORDS, id)
@@ -555,7 +555,7 @@ REGEX_ESCAPE        = /\\[^#]/g
 MULTILINER      = /\n/g
 NO_NEWLINE      = /^(?:[-+*&|\/%=<>!.\\][<>=&|]*|and|or|is(?:nt)?|not|delete|typeof|instanceof)$/
 HEREDOC_INDENT  = /\n+([ \t]*)|^([ \t]+)/g
-ASSIGNED        = /^\s*((?:[a-zA-Z$_@]\w*|["'][^\n]+?["']|\d+)[ \t]*?[:=][^:=])/
+ASSIGNED        = /^\s*@?[$A-Za-z_][$\w]*[ \t]*?[:=][^:=>]/
 NEXT_CHARACTER  = /^\s*(\S)/
 
 # Compound assignment tokens.

From 19f08888e8a8cc7e74b36edf30208f61deb7873b Mon Sep 17 00:00:00 2001
From: satyr <murky.satyr@gmail.com>
Date: Sun, 26 Sep 2010 03:25:53 +0900
Subject: [PATCH 08/10] lexer: more regexes fixes

---
 src/lexer.coffee          | 39 ++++++++++++++++++---------------------
 test/test_heredocs.coffee | 10 ++++++++++
 2 files changed, 28 insertions(+), 21 deletions(-)

diff --git a/src/lexer.coffee b/src/lexer.coffee
index ba827679..54b19559 100644
--- a/src/lexer.coffee
+++ b/src/lexer.coffee
@@ -139,7 +139,7 @@ exports.Lexer = class Lexer
     return false unless match = @chunk.match HEREDOC
     heredoc = match[0]
     quote = heredoc.charAt 0
-    doc = @sanitizeHeredoc match[2], {quote}
+    doc = @sanitizeHeredoc match[2], {quote, indent: null}
     @interpolateString quote + doc + quote, heredoc: yes
     @line += count heredoc, '\n'
     @i += heredoc.length
@@ -208,8 +208,8 @@ exports.Lexer = class Lexer
     @i    += indent.length
     prev = @prev 2
     size = indent.length - 1 - indent.lastIndexOf '\n'
-    nextCharacter = @match NEXT_CHARACTER, 1
-    noNewlines = nextCharacter is '.' or nextCharacter is ',' or @unfinished()
+    nextCharacter = NEXT_CHARACTER.exec(@chunk)[1]
+    noNewlines = (nextCharacter in ['.', ',']) or @unfinished()
     if size - @indebt is @indent
       return @suppressNewlines() if noNewlines
       return @newlineToken indent
@@ -331,18 +331,17 @@ exports.Lexer = class Lexer
   # Sanitize a heredoc or herecomment by escaping internal double quotes and
   # erasing all external indentation on the left-hand side.
   sanitizeHeredoc: (doc, options) ->
-    indent = options.indent
-    return doc if options.herecomment and not include doc, '\n'
-    unless options.herecomment
+    {indent, herecomment} = options
+    return doc if herecomment and not include doc, '\n'
+    unless herecomment
       while (match = HEREDOC_INDENT.exec doc)
-        attempt = if match[1]? then match[1] else match[2]
-        indent = attempt if not indent? or 0 < attempt.length < indent.length
-    indent or= ''
-    doc = doc.replace(new RegExp('^' + indent, 'gm'), '')
-    return doc if options.herecomment
+        attempt = match[1]
+        indent = attempt if indent is null or 0 < attempt.length < indent.length
+    doc = doc.replace /\n#{ indent }/g, '\n' if indent
+    return doc if herecomment
     doc.replace(/^\n/, '')
        .replace(MULTILINER, '\\n')
-       .replace(new RegExp(options.quote, 'g'), "\\#{options.quote}")
+       .replace(/#{ options.quote }/g, '\\$&')
 
   # A source of ambiguity in our grammar used to be parameter lists in function
   # definitions versus argument lists in function calls. Walk backwards, tagging
@@ -489,11 +488,9 @@ exports.Lexer = class Lexer
 
   # Are we in the midst of an unfinished expression?
   unfinished: ->
-    prev  = @prev 2
-    value = @value()
-    value and NO_NEWLINE.test(value) and
-      prev and prev[0] isnt '.' and not CODE.test(value) and
-      not ASSIGNED.test(@chunk)
+    (prev  = @prev 2 ) and prev[0] isnt '.' and
+    (value = @value()) and NO_NEWLINE.test(value) and not CODE.test(value) and
+    not ASSIGNED.test(@chunk)
 
 # Constants
 # ---------
@@ -535,11 +532,11 @@ JS_FORBIDDEN = JS_KEYWORDS.concat RESERVED
 
 # Token matching regexes.
 IDENTIFIER = /^[a-zA-Z_$][\w$]*/
-NUMBER     = /^(?:0x[\da-f]+)|^(?:\d+(\.\d+)?|\.\d+)(?:e[+-]?\d+)?/i
+NUMBER     = /^0x[\da-f]+|^(?:\d+(\.\d+)?|\.\d+)(?:e[+-]?\d+)?/i
 HEREDOC    = /^("""|''')([\s\S]*?)\n?[ \t]*\1/
 OPERATOR   = /^(?:-[-=>]?|\+[+=]?|[*&|\/%=<>^:!?]+)(?=([ \t]*))/
 WHITESPACE = /^[ \t]+/
-COMMENT    = /^###([^#][\s\S]*?)(?:###[ \t]*\n|(?:###)?$)|^(?:\s*#(?!##[^#])[^\n]*)+/
+COMMENT    = /^###([^#][\s\S]*?)(?:###[ \t]*\n|(?:###)?$)|^(?:\s*#(?!##[^#]).*)+/
 CODE       = /^[-=]>/
 MULTI_DENT = /^(?:\n[ \t]*)+/
 SIMPLESTR  = /^'[^\\']*(?:\\.[^\\']*)*'/
@@ -554,9 +551,9 @@ REGEX_ESCAPE        = /\\[^#]/g
 # Token cleaning regexes.
 MULTILINER      = /\n/g
 NO_NEWLINE      = /^(?:[-+*&|\/%=<>!.\\][<>=&|]*|and|or|is(?:nt)?|not|delete|typeof|instanceof)$/
-HEREDOC_INDENT  = /\n+([ \t]*)|^([ \t]+)/g
+HEREDOC_INDENT  = /\n+([ \t]*)/g
 ASSIGNED        = /^\s*@?[$A-Za-z_][$\w]*[ \t]*?[:=][^:=>]/
-NEXT_CHARACTER  = /^\s*(\S)/
+NEXT_CHARACTER  = /^\s*(\S?)/
 
 # Compound assignment tokens.
 COMPOUND_ASSIGN = ['-=', '+=', '/=', '*=', '%=', '||=', '&&=', '?=', '<<=', '>>=', '>>>=', '&=', '^=', '|=']
diff --git a/test/test_heredocs.coffee b/test/test_heredocs.coffee
index 8bf2ae98..7389fb09 100644
--- a/test/test_heredocs.coffee
+++ b/test/test_heredocs.coffee
@@ -88,3 +88,13 @@ a = """
 
     """
 ok a is "one\ntwo\n"
+
+
+equal ''' line 0
+  should not be relevant
+    to the indent level
+''', '
+ line 0\n
+should not be relevant\n
+  to the indent level
+'

From 7945178f3acdad60dc2357f39805bce8f8eb73e0 Mon Sep 17 00:00:00 2001
From: satyr <murky.satyr@gmail.com>
Date: Sun, 26 Sep 2010 07:01:24 +0900
Subject: [PATCH 09/10] lexer: unrolled @extractNextToken/@match

---
 src/lexer.coffee | 55 ++++++++++++++++++++++--------------------------
 1 file changed, 25 insertions(+), 30 deletions(-)

diff --git a/src/lexer.coffee b/src/lexer.coffee
index 54b19559..605a021f 100644
--- a/src/lexer.coffee
+++ b/src/lexer.coffee
@@ -43,27 +43,24 @@ exports.Lexer = class Lexer
     @outdebt = 0            # The under-outdentation at the current level.
     @indents = []           # The stack of all current indentation levels.
     @tokens  = []           # Stream of parsed tokens in the form ['TYPE', value, line]
+    # At every position, run through this list of attempted matches,
+    # short-circuiting if any of them succeed. Their order determines precedence:
+    # `@literalToken` is the fallback catch-all.
     while (@chunk = code[@i..])
-      @extractNextToken()
+      @identifierToken() or
+      @commentToken()    or
+      @whitespaceToken() or
+      @lineToken()       or
+      @heredocToken()    or
+      @stringToken()     or
+      @numberToken()     or
+      @regexToken()      or
+      @jsToken()         or
+      @literalToken()
     @closeIndentation()
     return @tokens if o.rewrite is off
     (new Rewriter).rewrite @tokens
 
-  # At every position, run through this list of attempted matches,
-  # short-circuiting if any of them succeed. Their order determines precedence:
-  # `@literalToken` is the fallback catch-all.
-  extractNextToken: ->
-    @identifierToken() or
-    @commentToken()    or
-    @whitespaceToken() or
-    @lineToken()       or
-    @heredocToken()    or
-    @stringToken()     or
-    @numberToken()     or
-    @regexToken()      or
-    @jsToken()         or
-    @literalToken()
-
   # Tokenizers
   # ----------
 
@@ -74,7 +71,8 @@ exports.Lexer = class Lexer
   # referenced as property names here, so you can still do `jQuery.is()` even
   # though `is` means `===` otherwise.
   identifierToken: ->
-    return false unless id = @match IDENTIFIER
+    return false unless match = IDENTIFIER.exec @chunk
+    id = match[0]
     @i += id.length
     if id is 'all' and @tag() is 'FOR'
       @token 'ALL', id
@@ -111,7 +109,8 @@ exports.Lexer = class Lexer
   # Matches numbers, including decimals, hex, and exponential notation.
   # Be careful not to interfere with ranges-in-progress.
   numberToken: ->
-    return false unless number = @match NUMBER
+    return false unless match = NUMBER.exec @chunk
+    number = match[0]
     return false if @tag() is '.' and number.charAt(0) is '.'
     @i += number.length
     @token 'NUMBER', number
@@ -122,8 +121,8 @@ exports.Lexer = class Lexer
   stringToken: ->
     switch @chunk.charAt 0
       when "'"
-        return false unless string = @match SIMPLESTR
-        @token 'STRING', string.replace MULTILINER, '\\\n'
+        return false unless match = SIMPLESTR.exec @chunk
+        @token 'STRING', (string = match[0]).replace MULTILINER, '\\\n'
       when '"'
         return false unless string = @balancedToken ['"', '"'], ['#{', '}']
         @interpolateString string.replace MULTILINER, '\\\n'
@@ -159,8 +158,8 @@ exports.Lexer = class Lexer
 
   # Matches JavaScript interpolated directly into the source via backticks.
   jsToken: ->
-    return false unless @chunk.charAt(0) is '`' and script = @match JSTOKEN
-    @token 'JS', script.slice 1, -1
+    return false unless @chunk.charAt(0) is '`' and match = JSTOKEN.exec @chunk
+    @token 'JS', (script = match[0]).slice 1, -1
     @i += script.length
     true
 
@@ -203,7 +202,8 @@ exports.Lexer = class Lexer
   # Keeps track of the level of indentation, because a single outdent token
   # can close multiple indents, so we need to know how far in we happen to be.
   lineToken: ->
-    return false unless indent = @match MULTI_DENT
+    return false unless match = MULTI_DENT.exec @chunk
+    indent = match[0]
     @line += count indent, '\n'
     @i    += indent.length
     prev = @prev 2
@@ -253,10 +253,10 @@ exports.Lexer = class Lexer
   # Matches and consumes non-meaningful whitespace. Tag the previous token
   # as being "spaced", because there are some cases where it makes a difference.
   whitespaceToken: ->
-    return false unless space = @match WHITESPACE
+    return false unless match = WHITESPACE.exec @chunk
     prev = @prev()
     prev.spaced = true if prev
-    @i += space.length
+    @i += match[0].length
     true
 
   # Generate a newline token. Consecutive newlines get merged together.
@@ -481,11 +481,6 @@ exports.Lexer = class Lexer
   prev: (index) ->
     @tokens[@tokens.length - (index or 1)]
 
-  # Attempt to match a string against the current chunk, returning the indexed
-  # match if successful, and `false` otherwise.
-  match: (regex, index) ->
-    if m = @chunk.match regex then m[index or 0] else false
-
   # Are we in the midst of an unfinished expression?
   unfinished: ->
     (prev  = @prev 2 ) and prev[0] isnt '.' and

From 3e0c35bd0fd403ac7b09d5130bd777b58ffaf0fd Mon Sep 17 00:00:00 2001
From: satyr <murky.satyr@gmail.com>
Date: Sun, 26 Sep 2010 07:06:14 +0900
Subject: [PATCH 10/10] lexer: enabled multiline interpolations

---
 lib/lexer.js              | 238 ++++++++++++++++++++------------------
 src/lexer.coffee          |  97 +++++++++-------
 test/test_heredocs.coffee |   8 ++
 3 files changed, 184 insertions(+), 159 deletions(-)

diff --git a/lib/lexer.js b/lib/lexer.js
index 896aa9f7..c8892e96 100644
--- a/lib/lexer.js
+++ b/lib/lexer.js
@@ -23,7 +23,7 @@
       this.indents = [];
       this.tokens = [];
       while ((this.chunk = code.slice(this.i))) {
-        this.extractNextToken();
+        this.identifierToken() || this.commentToken() || this.whitespaceToken() || this.lineToken() || this.heredocToken() || this.stringToken() || this.numberToken() || this.regexToken() || this.jsToken() || this.literalToken();
       }
       this.closeIndentation();
       if (o.rewrite === false) {
@@ -31,14 +31,12 @@
       }
       return (new Rewriter()).rewrite(this.tokens);
     };
-    Lexer.prototype.extractNextToken = function() {
-      return this.identifierToken() || this.commentToken() || this.whitespaceToken() || this.lineToken() || this.heredocToken() || this.stringToken() || this.numberToken() || this.regexToken() || this.jsToken() || this.literalToken();
-    };
     Lexer.prototype.identifierToken = function() {
-      var closeIndex, forcedIdentifier, id, tag;
-      if (!(id = this.match(IDENTIFIER))) {
+      var closeIndex, forcedIdentifier, id, match, tag;
+      if (!(match = IDENTIFIER.exec(this.chunk))) {
         return false;
       }
+      id = match[0];
       this.i += id.length;
       if (id === 'all' && this.tag() === 'FOR') {
         this.token('ALL', id);
@@ -86,10 +84,11 @@
       return true;
     };
     Lexer.prototype.numberToken = function() {
-      var number;
-      if (!(number = this.match(NUMBER))) {
+      var match, number;
+      if (!(match = NUMBER.exec(this.chunk))) {
         return false;
       }
+      number = match[0];
       if (this.tag() === '.' && number.charAt(0) === '.') {
         return false;
       }
@@ -98,19 +97,19 @@
       return true;
     };
     Lexer.prototype.stringToken = function() {
-      var string;
+      var match, string;
       switch (this.chunk.charAt(0)) {
         case "'":
-          if (!(string = this.match(SIMPLESTR))) {
+          if (!(match = SIMPLESTR.exec(this.chunk))) {
             return false;
           }
-          this.token('STRING', string.replace(MULTILINER, '\\\n'));
+          this.token('STRING', (string = match[0]).replace(MULTILINER, '\\\n'));
           break;
         case '"':
           if (!(string = this.balancedToken(['"', '"'], ['#{', '}']))) {
             return false;
           }
-          this.interpolateString(string.replace(MULTILINER, '\\\n'));
+          this.interpolateString(string);
           break;
         default:
           return false;
@@ -127,7 +126,8 @@
       heredoc = match[0];
       quote = heredoc.charAt(0);
       doc = this.sanitizeHeredoc(match[2], {
-        quote: quote
+        quote: quote,
+        indent: null
       });
       this.interpolateString(quote + doc + quote, {
         heredoc: true
@@ -156,11 +156,11 @@
       return true;
     };
     Lexer.prototype.jsToken = function() {
-      var script;
-      if (!(this.chunk.charAt(0) === '`' && (script = this.match(JSTOKEN)))) {
+      var match, script;
+      if (!(this.chunk.charAt(0) === '`' && (match = JSTOKEN.exec(this.chunk)))) {
         return false;
       }
-      this.token('JS', script.slice(1, -1));
+      this.token('JS', (script = match[0]).slice(1, -1));
       this.i += script.length;
       return true;
     };
@@ -205,16 +205,17 @@
       return this.balancedString(this.chunk, delimited);
     };
     Lexer.prototype.lineToken = function() {
-      var diff, indent, nextCharacter, noNewlines, prev, size;
-      if (!(indent = this.match(MULTI_DENT))) {
+      var diff, indent, match, nextCharacter, noNewlines, prev, size;
+      if (!(match = MULTI_DENT.exec(this.chunk))) {
         return false;
       }
+      indent = match[0];
       this.line += count(indent, '\n');
       this.i += indent.length;
       prev = this.prev(2);
       size = indent.length - 1 - indent.lastIndexOf('\n');
-      nextCharacter = this.match(NEXT_CHARACTER, 1);
-      noNewlines = nextCharacter === '.' || nextCharacter === ',' || this.unfinished();
+      nextCharacter = NEXT_CHARACTER.exec(this.chunk)[1];
+      noNewlines = (('.' === nextCharacter || ',' === nextCharacter)) || this.unfinished();
       if (size - this.indebt === this.indent) {
         if (noNewlines) {
           return this.suppressNewlines();
@@ -265,15 +266,15 @@
       return true;
     };
     Lexer.prototype.whitespaceToken = function() {
-      var prev, space;
-      if (!(space = this.match(WHITESPACE))) {
+      var match, prev;
+      if (!(match = WHITESPACE.exec(this.chunk))) {
         return false;
       }
       prev = this.prev();
       if (prev) {
         prev.spaced = true;
       }
-      this.i += space.length;
+      this.i += match[0].length;
       return true;
     };
     Lexer.prototype.newlineToken = function(newlines) {
@@ -369,25 +370,32 @@
       return accessor ? 'accessor' : false;
     };
     Lexer.prototype.sanitizeHeredoc = function(doc, options) {
-      var _ref2, attempt, indent, match;
-      indent = options.indent;
-      if (options.herecomment && !include(doc, '\n')) {
+      var _ref2, attempt, herecomment, indent, match;
+      _ref2 = options;
+      indent = _ref2.indent;
+      herecomment = _ref2.herecomment;
+      if (herecomment && !include(doc, '\n')) {
         return doc;
       }
-      if (!(options.herecomment)) {
+      if (!(herecomment)) {
         while ((match = HEREDOC_INDENT.exec(doc))) {
-          attempt = (typeof (_ref2 = match[1]) !== "undefined" && _ref2 !== null) ? match[1] : match[2];
-          if (!(typeof indent !== "undefined" && indent !== null) || (0 < attempt.length) && (attempt.length < indent.length)) {
+          attempt = match[1];
+          if (indent === null || (0 < attempt.length) && (attempt.length < indent.length)) {
             indent = attempt;
           }
         }
       }
-      indent || (indent = '');
-      doc = doc.replace(new RegExp('^' + indent, 'gm'), '');
-      if (options.herecomment) {
+      if (indent) {
+        doc = doc.replace(new RegExp("\\n" + (indent), "g"), '\n');
+      }
+      if (herecomment) {
         return doc;
       }
-      return doc.replace(/^\n/, '').replace(MULTILINER, '\\n').replace(new RegExp(options.quote, 'g'), "\\" + (options.quote));
+      doc = doc.replace(/^\n/, '').replace(new RegExp("" + (options.quote), "g"), '\\$&');
+      if (options.quote === "'") {
+        doc = this.oldline(doc, true);
+      }
+      return doc;
     };
     Lexer.prototype.tagParameters = function() {
       var i, tok;
@@ -469,83 +477,84 @@
       return !i ? false : str.slice(0, i);
     };
     Lexer.prototype.interpolateString = function(str, options) {
-      var _len, _ref2, _ref3, end, escaped, expr, i, idx, inner, interpolated, lexer, nested, pi, quote, tag, tok, token, tokens, value;
+      var _len, _ref2, _ref3, end, escaped, expr, i, idx, inner, interpolated, lexer, nested, pi, push, quote, s, tag, tok, token, tokens, value;
       options || (options = {});
-      if (str.length < 3 || str.charAt(0) !== '"') {
+      quote = str.charAt(0);
+      if (quote !== '"' || str.length < 3) {
         return this.token('STRING', str);
-      } else {
-        lexer = new Lexer();
-        tokens = [];
-        quote = str.charAt(0);
-        _ref2 = [1, 1];
-        i = _ref2[0];
-        pi = _ref2[1];
-        end = str.length - 1;
-        while (i < end) {
-          if (str.charAt(i) === '\\') {
-            i += 1;
-          } else if (expr = this.balancedString(str.slice(i), [['#{', '}']])) {
-            if (pi < i) {
-              tokens.push(['STRING', quote + str.slice(pi, i) + quote]);
-            }
-            inner = expr.slice(2, -1);
-            if (inner.length) {
-              if (options.heredoc) {
-                inner = inner.replace(new RegExp('\\\\' + quote, 'g'), quote);
-              }
-              nested = lexer.tokenize("(" + (inner) + ")", {
-                line: this.line
-              });
-              _ref2 = nested;
-              for (idx = 0, _len = _ref2.length; idx < _len; idx++) {
-                tok = _ref2[idx];
-                if (tok[0] === 'CALL_END') {
-                  (tok[0] = ')');
-                }
-              }
-              nested.pop();
-              tokens.push(['TOKENS', nested]);
-            } else {
-              tokens.push(['STRING', quote + quote]);
-            }
-            i += expr.length - 1;
-            pi = i + 1;
-          }
-          i += 1;
-        }
-        if ((i > pi) && (pi < str.length - 1)) {
-          tokens.push(['STRING', quote + str.slice(pi, i) + quote]);
-        }
-        if (tokens[0][0] !== 'STRING') {
-          tokens.unshift(['STRING', '""']);
-        }
-        interpolated = tokens.length > 1;
-        if (interpolated) {
-          this.token('(', '(');
-        }
-        _ref2 = tokens;
-        for (i = 0, _len = _ref2.length; i < _len; i++) {
-          token = _ref2[i];
-          _ref3 = token;
-          tag = _ref3[0];
-          value = _ref3[1];
-          if (tag === 'TOKENS') {
-            this.tokens = this.tokens.concat(value);
-          } else if (tag === 'STRING' && options.escapeQuotes) {
-            escaped = value.slice(1, -1).replace(/"/g, '\\"');
-            this.token(tag, "\"" + (escaped) + "\"");
-          } else {
-            this.token(tag, value);
-          }
-          if (i < tokens.length - 1) {
-            this.token('+', '+');
-          }
-        }
-        if (interpolated) {
-          this.token(')', ')');
-        }
-        return tokens;
       }
+      lexer = new Lexer();
+      tokens = [];
+      i = (pi = 1);
+      end = str.length - 1;
+      while (i < end) {
+        if (str.charAt(i) === '\\') {
+          i += 1;
+        } else if (expr = this.balancedString(str.slice(i), [['#{', '}']])) {
+          if (pi < i) {
+            s = quote + this.oldline(str.slice(pi, i), options.heredoc) + quote;
+            tokens.push(['STRING', s]);
+          }
+          inner = expr.slice(2, -1).replace(/^\s+/, '');
+          if (inner.length) {
+            if (options.heredoc) {
+              inner = inner.replace(RegExp('\\\\' + quote, 'g'), quote);
+            }
+            nested = lexer.tokenize("(" + (inner) + ")", {
+              line: this.line
+            });
+            _ref2 = nested;
+            for (idx = 0, _len = _ref2.length; idx < _len; idx++) {
+              tok = _ref2[idx];
+              if (tok[0] === 'CALL_END') {
+                (tok[0] = ')');
+              }
+            }
+            nested.pop();
+            tokens.push(['TOKENS', nested]);
+          } else {
+            tokens.push(['STRING', quote + quote]);
+          }
+          i += expr.length - 1;
+          pi = i + 1;
+        }
+        i += 1;
+      }
+      if ((i > pi) && (pi < str.length - 1)) {
+        s = str.slice(pi, i).replace(MULTILINER, options.heredoc ? '\\n' : '');
+        tokens.push(['STRING', quote + s + quote]);
+      }
+      if (tokens[0][0] !== 'STRING') {
+        tokens.unshift(['STRING', '""']);
+      }
+      interpolated = tokens.length > 1;
+      if (interpolated) {
+        this.token('(', '(');
+      }
+      _ref2 = tokens;
+      push = _ref2.push;
+      _ref2 = tokens;
+      for (i = 0, _len = _ref2.length; i < _len; i++) {
+        token = _ref2[i];
+        _ref3 = token;
+        tag = _ref3[0];
+        value = _ref3[1];
+        if (tag === 'TOKENS') {
+          push.apply(this.tokens, value);
+        } else if (tag === 'STRING' && options.escapeQuotes) {
+          escaped = value.slice(1, -1).replace(/"/g, '\\"');
+          this.token(tag, "\"" + (escaped) + "\"");
+        } else {
+          this.token(tag, value);
+        }
+        if (i < tokens.length - 1) {
+          this.token('+', '+');
+        }
+      }
+      if (interpolated) {
+        this.token(')', ')');
+      }
+      return tokens;
     };
     Lexer.prototype.token = function(tag, value) {
       return this.tokens.push([tag, value, this.line]);
@@ -579,9 +588,10 @@
     };
     Lexer.prototype.unfinished = function() {
       var prev, value;
-      prev = this.prev(2);
-      value = this.value();
-      return value && NO_NEWLINE.test(value) && prev && prev[0] !== '.' && !CODE.test(value) && !ASSIGNED.test(this.chunk);
+      return (prev = this.prev(2)) && prev[0] !== '.' && (value = this.value()) && NO_NEWLINE.test(value) && !CODE.test(value) && !ASSIGNED.test(this.chunk);
+    };
+    Lexer.prototype.oldline = function(str, heredoc) {
+      return str.replace(MULTILINER, heredoc ? '\\n' : '');
     };
     return Lexer;
   })();
@@ -591,11 +601,11 @@
   RESERVED = ['case', 'default', 'do', 'function', 'var', 'void', 'with', 'const', 'let', 'enum', 'export', 'import', 'native', '__hasProp', '__extends', '__slice'];
   JS_FORBIDDEN = JS_KEYWORDS.concat(RESERVED);
   IDENTIFIER = /^[a-zA-Z_$][\w$]*/;
-  NUMBER = /^(?:0x[\da-f]+)|^(?:\d+(\.\d+)?|\.\d+)(?:e[+-]?\d+)?/i;
+  NUMBER = /^0x[\da-f]+|^(?:\d+(\.\d+)?|\.\d+)(?:e[+-]?\d+)?/i;
   HEREDOC = /^("""|''')([\s\S]*?)\n?[ \t]*\1/;
   OPERATOR = /^(?:-[-=>]?|\+[+=]?|[*&|\/%=<>^:!?]+)(?=([ \t]*))/;
   WHITESPACE = /^[ \t]+/;
-  COMMENT = /^###([^#][\s\S]*?)(?:###[ \t]*\n|(?:###)?$)|^(?:\s*#(?!##[^#])[^\n]*)+/;
+  COMMENT = /^###([^#][\s\S]*?)(?:###[ \t]*\n|(?:###)?$)|^(?:\s*#(?!##[^#]).*)+/;
   CODE = /^[-=]>/;
   MULTI_DENT = /^(?:\n[ \t]*)+/;
   SIMPLESTR = /^'[^\\']*(?:\\.[^\\']*)*'/;
@@ -606,9 +616,9 @@
   REGEX_ESCAPE = /\\[^#]/g;
   MULTILINER = /\n/g;
   NO_NEWLINE = /^(?:[-+*&|\/%=<>!.\\][<>=&|]*|and|or|is(?:nt)?|not|delete|typeof|instanceof)$/;
-  HEREDOC_INDENT = /\n+([ \t]*)|^([ \t]+)/g;
+  HEREDOC_INDENT = /\n+([ \t]*)/g;
   ASSIGNED = /^\s*@?[$A-Za-z_][$\w]*[ \t]*?[:=][^:=>]/;
-  NEXT_CHARACTER = /^\s*(\S)/;
+  NEXT_CHARACTER = /^\s*(\S?)/;
   COMPOUND_ASSIGN = ['-=', '+=', '/=', '*=', '%=', '||=', '&&=', '?=', '<<=', '>>=', '>>>=', '&=', '^=', '|='];
   UNARY = ['UMINUS', 'UPLUS', '!', '!!', '~', 'TYPEOF', 'DELETE'];
   LOGIC = ['&', '|', '^', '&&', '||'];
diff --git a/src/lexer.coffee b/src/lexer.coffee
index 605a021f..a5d91c9f 100644
--- a/src/lexer.coffee
+++ b/src/lexer.coffee
@@ -125,7 +125,7 @@ exports.Lexer = class Lexer
         @token 'STRING', (string = match[0]).replace MULTILINER, '\\\n'
       when '"'
         return false unless string = @balancedToken ['"', '"'], ['#{', '}']
-        @interpolateString string.replace MULTILINER, '\\\n'
+        @interpolateString string
       else
         return false
     @line += count string, '\n'
@@ -339,9 +339,9 @@ exports.Lexer = class Lexer
         indent = attempt if indent is null or 0 < attempt.length < indent.length
     doc = doc.replace /\n#{ indent }/g, '\n' if indent
     return doc if herecomment
-    doc.replace(/^\n/, '')
-       .replace(MULTILINER, '\\n')
-       .replace(/#{ options.quote }/g, '\\$&')
+    doc = doc.replace(/^\n/, '').replace(/#{ options.quote }/g, '\\$&')
+    doc = @oldline doc, on if options.quote is "'"
+    doc
 
   # A source of ambiguity in our grammar used to be parameter lists in function
   # definitions versus argument lists in function calls. Walk backwards, tagging
@@ -406,7 +406,7 @@ exports.Lexer = class Lexer
     if not i then false else str[0...i]
 
   # Expand variables and expressions inside double-quoted strings using
-  # [ECMA Harmony's interpolation syntax](http://wiki.ecmascript.org/doku.php?id=strawman:string_interpolation)
+  # Ruby-like notation
   # for substitution of bare variables as well as arbitrary expressions.
   #
   #     "Hello #{name.capitalize()}."
@@ -415,48 +415,51 @@ exports.Lexer = class Lexer
   # new Lexer, tokenize the interpolated contents, and merge them into the
   # token stream.
   interpolateString: (str, options) ->
-    options or= {}
-    if str.length < 3 or str.charAt(0) isnt '"'
-      @token 'STRING', str
-    else
-      lexer   = new Lexer
-      tokens  = []
-      quote   = str.charAt 0
-      [i, pi] = [1, 1]
-      end = str.length - 1
-      while i < end
-        if str.charAt(i) is '\\'
-          i += 1
-        else if expr = @balancedString str[i..], [['#{', '}']]
-          tokens.push ['STRING', quote + str[pi...i] + quote] if pi < i
-          inner = expr.slice 2, -1
-          if inner.length
-            inner = inner.replace new RegExp('\\\\' + quote, 'g'), quote if options.heredoc
-            nested = lexer.tokenize "(#{inner})", line: @line
-            (tok[0] = ')') for tok, idx in nested when tok[0] is 'CALL_END'
-            nested.pop()
-            tokens.push ['TOKENS', nested]
-          else
-            tokens.push ['STRING', quote + quote]
-          i += expr.length - 1
-          pi = i + 1
+    {heredoc, escapeQuotes} = options or {}
+    quote = str.charAt 0
+    return @token 'STRING', str if quote isnt '"' or str.length < 3
+    lexer  = new Lexer
+    tokens = []
+    i = pi = 1
+    end = str.length - 1
+    while i < end
+      if str.charAt(i) is '\\'
         i += 1
-      tokens.push ['STRING', quote + str[pi...i] + quote] if i > pi < str.length - 1
-      tokens.unshift ['STRING', '""'] unless tokens[0][0] is 'STRING'
-      interpolated = tokens.length > 1
-      @token '(', '(' if interpolated
-      for token, i in tokens
-        [tag, value] = token
-        if tag is 'TOKENS'
-          @tokens = @tokens.concat value
-        else if tag is 'STRING' and options.escapeQuotes
-          escaped = value.slice(1, -1).replace(/"/g, '\\"')
-          @token tag, "\"#{escaped}\""
+      else if expr = @balancedString str[i..], [['#{', '}']]
+        if pi < i
+          s = quote + @oldline(str[pi...i], heredoc) + quote
+          tokens.push ['STRING', s]
+        inner = expr.slice(2, -1).replace /^[ \t]*\n/, ''
+        if inner.length
+          inner = inner.replace RegExp('\\\\' + quote, 'g'), quote if heredoc
+          nested = lexer.tokenize "(#{inner})", line: @line
+          (tok[0] = ')') for tok, idx in nested when tok[0] is 'CALL_END'
+          nested.pop()
+          tokens.push ['TOKENS', nested]
         else
-          @token tag, value
-        @token '+', '+' if i < tokens.length - 1
-      @token ')', ')' if interpolated
-      tokens
+          tokens.push ['STRING', quote + quote]
+        i += expr.length - 1
+        pi = i + 1
+      i += 1
+    if i > pi < str.length - 1
+      s = str[pi...i].replace MULTILINER, if heredoc then '\\n' else ''
+      tokens.push ['STRING', quote + s + quote]
+    tokens.unshift ['STRING', '""'] unless tokens[0][0] is 'STRING'
+    interpolated = tokens.length > 1
+    @token '(', '(' if interpolated
+    {push} = tokens
+    for token, i in tokens
+      [tag, value] = token
+      if tag is 'TOKENS'
+        push.apply @tokens, value
+      else if tag is 'STRING' and escapeQuotes
+        escaped = value.slice(1, -1).replace(/"/g, '\\"')
+        @token tag, "\"#{escaped}\""
+      else
+        @token tag, value
+      @token '+', '+' if i < tokens.length - 1
+    @token ')', ')' if interpolated
+    tokens
 
   # Helpers
   # -------
@@ -487,6 +490,10 @@ exports.Lexer = class Lexer
     (value = @value()) and NO_NEWLINE.test(value) and not CODE.test(value) and
     not ASSIGNED.test(@chunk)
 
+  # Converts newlines for string literals
+  oldline: (str, heredoc) ->
+    str.replace MULTILINER, if heredoc then '\\n' else ''
+
 # Constants
 # ---------
 
diff --git a/test/test_heredocs.coffee b/test/test_heredocs.coffee
index 7389fb09..5c727483 100644
--- a/test/test_heredocs.coffee
+++ b/test/test_heredocs.coffee
@@ -98,3 +98,11 @@ equal ''' line 0
 should not be relevant\n
   to the indent level
 '
+
+
+equal 'multiline nested interpolations work', """multiline #{
+  "nested #{(->
+    ok yes
+    "interpolations"
+  )()}"
+} work"""