fixed over-escaping in here documents and addressed Stan's comments

lib/lexer.js

@@ -103,8 +103,8 @@
           if (!(string = this.balancedString(this.chunk, [['"', '"'], ['#{', '}']]))) {
             return false;
           }
-          if (~string.indexOf('#{')) {
-            this.interpolateString(string);
+          if (0 < string.indexOf('#{', 1)) {
+            this.interpolateString(string.slice(1, -1));
           } else {
             this.token('STRING', this.escapeLines(string));
           }
@@ -127,12 +127,12 @@
         quote: quote,
         indent: null
       });
-      if (quote === '"' && ~doc.indexOf('#{')) {
-        this.interpolateString(quote + doc + quote, {
+      if (quote === '"' && (0 <= doc.indexOf('#{'))) {
+        this.interpolateString(doc, {
           heredoc: true
         });
       } else {
-        this.token('STRING', quote + this.escapeLines(doc, true) + quote);
+        this.token('STRING', this.makeString(doc, quote, true));
       }
       this.line += count(heredoc, '\n');
       this.i += heredoc.length;
@@ -186,7 +186,7 @@
       var _i, _len, _ref2, _ref3, _this, body, flags, heregex, re, tag, tokens, value;
       _ref2 = match, heregex = _ref2[0], body = _ref2[1], flags = _ref2[2];
       this.i += heregex.length;
-      if (!(~body.indexOf('#{'))) {
+      if (0 > body.indexOf('#{')) {
         re = body.replace(HEREGEX_OMIT, '').replace(/\//g, '\\/');
         this.token('REGEX', "/" + (re || '(?:)') + "/" + flags);
         return true;
@@ -194,7 +194,7 @@
       this.token('IDENTIFIER', 'RegExp');
       this.tokens.push(['CALL_START', '(']);
       tokens = [];
-      _ref2 = this.interpolateString('"' + body + '"', {
+      _ref2 = this.interpolateString(body, {
         regex: true
       });
       for (_i = 0, _len = _ref2.length; _i < _len; _i++) {
@@ -202,10 +202,11 @@
         if (tag === 'TOKENS') {
           tokens.push.apply(tokens, value);
         } else {
-          if (!(value = value.slice(1, -1).replace(HEREGEX_OMIT, ''))) {
+          if (!(value = value.replace(HEREGEX_OMIT, ''))) {
             continue;
           }
-          tokens.push(['STRING', '"' + value.replace(/[\\\"]/g, '\\$&') + '"']);
+          value = value.replace(/\\/g, '\\\\');
+          tokens.push(['STRING', this.makeString(value, '"', true)]);
         }
         tokens.push(['+', '+']);
       }
@@ -385,13 +386,13 @@
       return accessor ? 'accessor' : false;
     };
     Lexer.prototype.sanitizeHeredoc = function(doc, options) {
-      var _ref2, attempt, herecomment, indent, match, quote;
+      var _ref2, attempt, herecomment, indent, match;
       _ref2 = options, indent = _ref2.indent, herecomment = _ref2.herecomment;
-      if (herecomment && !include(doc, '\n')) {
+      if (herecomment && 0 > doc.indexOf('\n')) {
         return doc;
       }
       if (!(herecomment)) {
-        while ((match = HEREDOC_INDENT.exec(doc))) {
+        while (match = HEREDOC_INDENT.exec(doc)) {
           attempt = match[1];
           if (indent === null || (0 < (_ref2 = attempt.length)) && (_ref2 < indent.length)) {
             indent = attempt;
@@ -401,17 +402,8 @@
       if (indent) {
         doc = doc.replace(RegExp("\\n" + indent, "g"), '\n');
       }
-      if (herecomment) {
-        return doc;
-      }
-      quote = options.quote;
-      doc = doc.replace(/^\n/, '');
-      doc = doc.replace(/\\([\s\S])/g, function(m, c) {
-        return ('\n' === c || quote === c) ? c : m;
-      });
-      doc = doc.replace(RegExp("" + quote, "g"), '\\$&');
-      if (quote === "'") {
-        doc = this.escapeLines(doc, true);
+      if (!(herecomment)) {
+        doc = doc.replace(/^\n/, '');
       }
       return doc;
     };
@@ -487,15 +479,11 @@
       return !i ? false : str.slice(0, i);
     };
     Lexer.prototype.interpolateString = function(str, options) {
-      var _i, _len, _ref2, _this, char, expr, heredoc, i, inner, interpolated, lexer, nested, pi, regex, tag, tok, tokens, value;
-      if (str.length < 5) {
-        return this.token('STRING', this.escapeLines(str, heredoc));
-      }
+      var _len, _ref2, _this, char, expr, heredoc, i, inner, interpolated, nested, pi, regex, tag, tokens, value;
       _ref2 = options || (options = {}), heredoc = _ref2.heredoc, regex = _ref2.regex;
-      lexer = new Lexer;
       tokens = [];
-      pi = 1;
-      i = 0;
+      pi = 0;
+      i = -1;
       while (char = str.charAt(i += 1)) {
         if (char === '\\') {
           i += 1;
@@ -505,45 +493,39 @@
           continue;
         }
         if (pi < i) {
-          tokens.push(['STRING', '"' + str.slice(pi, i) + '"']);
+          tokens.push(['TO_BE_STRING', str.slice(pi, i)]);
         }
         inner = expr.slice(1, -1).replace(LEADING_SPACES, '').replace(TRAILING_SPACES, '');
         if (inner.length) {
-          if (heredoc) {
-            inner = inner.replace(/\\\"/g, '"');
-          }
-          nested = lexer.tokenize("(" + inner + "\n)", {
-            line: this.line
+          nested = new Lexer().tokenize(inner, {
+            line: this.line,
+            rewrite: false
           });
-          for (_i = 0, _len = nested.length; _i < _len; _i++) {
-            tok = nested[_i];
-            if (tok[0] === 'CALL_END') {
-              (tok[0] = ')');
-            }
-          }
           nested.pop();
-          if (nested.length < 5) {
-            nested.pop();
-            nested.shift();
+          if (nested.length > 1) {
+            nested.unshift(['(', '(']);
+            nested.push([')', ')']);
           }
           tokens.push(['TOKENS', nested]);
         }
         i += expr.length;
         pi = i + 1;
       }
-      if ((i > pi) && (pi < str.length - 1)) {
-        tokens.push(['STRING', '"' + str.slice(pi)]);
+      if ((i > pi) && (pi < str.length)) {
+        tokens.push(['TO_BE_STRING', str.slice(pi)]);
       }
       if (regex) {
         return tokens;
       }
-      interpolated = tokens.length > 1;
-      if ((((_ref2 = tokens[0]) != null) ? _ref2[0] !== 'STRING' : undefined)) {
-        tokens.unshift(['STRING', '""']);
+      if (!(tokens.length)) {
+        return this.token('STRING', '""');
       }
-      if (interpolated) {
+      if (interpolated = tokens.length > 1) {
         this.token('(', '(');
       }
+      if (tokens[0][0] !== 'TO_BE_STRING') {
+        this.tokens.push(['STRING', '""'], ['+', '+']);
+      }
       for (i = 0, _len = tokens.length; i < _len; i++) {
         _ref2 = tokens[i], tag = _ref2[0], value = _ref2[1];
         if (i) {
@@ -552,7 +534,7 @@
         if (tag === 'TOKENS') {
           (_this = this.tokens).push.apply(_this, value);
         } else {
-          this.token(tag, this.escapeLines(value, heredoc));
+          this.token('STRING', this.makeString(value, '"', heredoc));
         }
       }
       if (interpolated) {
@@ -584,6 +566,13 @@
     Lexer.prototype.escapeLines = function(str, heredoc) {
       return str.replace(MULTILINER, heredoc ? '\\n' : '');
     };
+    Lexer.prototype.makeString = function(body, quote, heredoc) {
+      body = body.replace(/\\([\s\S])/g, function($amp, $1) {
+        return ('\n' === $1 || quote === $1) ? $1 : $amp;
+      });
+      body = body.replace(RegExp("" + quote, "g"), '\\$&');
+      return quote + this.escapeLines(body, heredoc) + quote;
+    };
     return Lexer;
   })();
   JS_KEYWORDS = ['if', 'else', 'true', 'false', 'new', 'return', 'try', 'catch', 'finally', 'throw', 'break', 'continue', 'for', 'in', 'while', 'delete', 'instanceof', 'typeof', 'switch', 'super', 'extends', 'class', 'this', 'null', 'debugger'];
@@ -593,7 +582,7 @@
   JS_FORBIDDEN = JS_KEYWORDS.concat(RESERVED);
   IDENTIFIER = /^[a-zA-Z_$][\w$]*/;
   NUMBER = /^0x[\da-f]+|^(?:\d+(\.\d+)?|\.\d+)(?:e[+-]?\d+)?/i;
-  HEREDOC = /^("""|''')([\s\S]*?)\n?[ \t]*\1/;
+  HEREDOC = /^("""|''')([\s\S]*?)(?:\n[ \t]*)?\1/;
   OPERATOR = /^(?:-[-=>]?|\+[+=]?|[*&|\/%=<>^:!?]+)(?=([ \t]*))/;
   WHITESPACE = /^[ \t]+/;
   COMMENT = /^###([^#][\s\S]*?)(?:###[ \t]*\n|(?:###)?$)|^(?:\s*#(?!##[^#]).*)+/;