Merge pull request #3246 from xixixao/issue3229

Implements #3229 - Changed multiline string literals

lib/coffee-script/grammar.js

@@ -96,8 +96,7 @@
         return new Value($1);
       }), o('ObjAssignable : Expression', function() {
         return new Assign(LOC(1)(new Value($1)), $3, 'object');
-      }), o('ObjAssignable :\
-       INDENT Expression OUTDENT', function() {
+      }), o('ObjAssignable : INDENT Expression OUTDENT', function() {
         return new Assign(LOC(1)(new Value($1)), $4, 'object');
       }), o('Comment')
     ],
@@ -573,14 +572,11 @@
         } else {
           return new Op($2, $1, $3);
         }
-      }), o('SimpleAssignable COMPOUND_ASSIGN\
-       Expression', function() {
+      }), o('SimpleAssignable COMPOUND_ASSIGN Expression', function() {
         return new Assign($1, $3, $2);
-      }), o('SimpleAssignable COMPOUND_ASSIGN\
-       INDENT Expression OUTDENT', function() {
+      }), o('SimpleAssignable COMPOUND_ASSIGN INDENT Expression OUTDENT', function() {
         return new Assign($1, $4, $2);
-      }), o('SimpleAssignable COMPOUND_ASSIGN TERMINATOR\
-       Expression', function() {
+      }), o('SimpleAssignable COMPOUND_ASSIGN TERMINATOR Expression', function() {
         return new Assign($1, $4, $2);
       }), o('SimpleAssignable EXTENDS Expression', function() {
         return new Extends($1, $3);

lib/coffee-script/lexer.js

@@ -173,7 +173,7 @@
             return 0;
           }
           string = match[0];
-          this.token('STRING', string.replace(MULTILINER, '\\\n'), 0, string.length);
+          this.token('STRING', this.removeNewlines(string), 0, string.length);
           break;
         case '"':
           if (!(string = this.balancedString(this.chunk, '"'))) {
@@ -185,7 +185,7 @@
               lexedLength: string.length
             });
           } else {
-            this.token('STRING', this.escapeLines(string), 0, string.length);
+            this.token('STRING', this.removeNewlines(string), 0, string.length);
           }
           break;
         default:
@@ -762,8 +762,16 @@
       return LINE_CONTINUER.test(this.chunk) || ((_ref2 = this.tag()) === '\\' || _ref2 === '.' || _ref2 === '?.' || _ref2 === '?::' || _ref2 === 'UNARY' || _ref2 === 'MATH' || _ref2 === '+' || _ref2 === '-' || _ref2 === 'SHIFT' || _ref2 === 'RELATION' || _ref2 === 'COMPARE' || _ref2 === 'LOGIC' || _ref2 === 'THROW' || _ref2 === 'EXTENDS');
     };
 
+    Lexer.prototype.removeNewlines = function(str) {
+      return this.escapeLines(str.replace(/^(.)\s*\n\s*/, '$1').replace(/\s*\n\s*(.)$/, '$1'));
+    };
+
     Lexer.prototype.escapeLines = function(str, heredoc) {
-      return str.replace(MULTILINER, heredoc ? '\\n' : '');
+      if (heredoc) {
+        return str.replace(MULTILINER, '\\n');
+      } else {
+        return str.replace(/((^|[^\\])\\\\)\n/g, '$1 \\\n').replace(/\\\s*\n\s*/g, '').replace(/\s*\n\s*/g, ' ');
+      }
     };
 
     Lexer.prototype.makeString = function(body, quote, heredoc) {
@@ -771,7 +779,7 @@
         return quote + quote;
       }
       body = body.replace(/\\([\s\S])/g, function(match, contents) {
-        if (contents === '\n' || contents === quote) {
+        if (contents === quote || heredoc && contents === '\n') {
           return contents;
         } else {
           return match;
@@ -852,7 +860,7 @@
 
   MULTI_DENT = /^(?:\n[^\n\S]*)+/;
 
-  SIMPLESTR = /^'[^\\']*(?:\\.[^\\']*)*'/;
+  SIMPLESTR = /^'[^\\']*(?:\\[\s\S][^\\']*)*'/;
 
   JSTOKEN = /^`[^\\`]*(?:\\.[^\\`]*)*`/;
 

src/lexer.coffee

@@ -190,13 +190,13 @@ exports.Lexer = class Lexer
       when "'"
         return 0 unless match = SIMPLESTR.exec @chunk
         string = match[0]
-        @token 'STRING', string.replace(MULTILINER, '\\\n'), 0, string.length
+        @token 'STRING', @removeNewlines(string), 0, string.length
       when '"'
         return 0 unless string = @balancedString @chunk, '"'
         if 0 < string.indexOf '#{', 1
           @interpolateString string[1...-1], strOffset: 1, lexedLength: string.length
         else
-          @token 'STRING', @escapeLines(string), 0, string.length
+          @token 'STRING', @removeNewlines(string), 0, string.length
       else
         return 0
     if octalEsc = /^(?:\\.|[^\\])*\\(?:0[0-7]|[1-7])/.test string
@@ -684,15 +684,25 @@ exports.Lexer = class Lexer
     @tag() in ['\\', '.', '?.', '?::', 'UNARY', 'MATH', '+', '-', 'SHIFT', 'RELATION'
                'COMPARE', 'LOGIC', 'THROW', 'EXTENDS']
 
+  # Remove newlines from beginning and end of string literals.
+  # `str` includes quotes.
+  removeNewlines: (str) ->
+    @escapeLines str.replace(/^(.)\s*\n\s*/, '$1').replace(/\s*\n\s*(.)$/, '$1')
+
   # Converts newlines for string literals.
   escapeLines: (str, heredoc) ->
-    str.replace MULTILINER, if heredoc then '\\n' else ''
+    if heredoc
+      str.replace MULTILINER, '\\n'
+    else
+      str.replace(/((^|[^\\])\\\\)\n/g, '$1 \\\n') #escaped backslash
+         .replace(/\\\s*\n\s*/g, '') # backslash at EOL
+         .replace(/\s*\n\s*/g, ' ')
 
   # Constructs a string token by escaping quotes and newlines.
   makeString: (body, quote, heredoc) ->
     return quote + quote unless body
     body = body.replace /\\([\s\S])/g, (match, contents) ->
-      if contents in ['\n', quote] then contents else match
+      if contents is quote or heredoc and contents is '\n' then contents else match
     body = body.replace /// #{quote} ///g, '\\$&'
     quote + @escapeLines(body, heredoc) + quote
 
@@ -787,7 +797,7 @@ CODE       = /^[-=]>/
 
 MULTI_DENT = /^(?:\n[^\n\S]*)+/
 
-SIMPLESTR  = /^'[^\\']*(?:\\.[^\\']*)*'/
+SIMPLESTR  = /^'[^\\']*(?:\\[\s\S][^\\']*)*'/
 
 JSTOKEN    = /^`[^\\`]*(?:\\.[^\\`]*)*`/
 

test/interpolation.coffee

@@ -30,7 +30,7 @@ eq "#{6 / 2}",               '3'
 eq "#{6 / 2}#{6 / 2}",       '33' # parsed as division
 eq "#{6 + /2}#{6/ + 2}",     '6/2}#{6/2' # parsed as a regex
 eq "#{6/2}
-    #{6/2}",                 '3    3' # newline cannot be part of a regex, so it's division
+    #{6/2}",                 '3 3' # newline cannot be part of a regex, so it's division
 eq "#{/// "'/'"/" ///}",     '/"\'\\/\'"\\/"/' # heregex, stuffed with spicy characters
 eq "#{/\\'/}",               "/\\\\'/"
 

test/strings.coffee

@@ -18,6 +18,85 @@ eq "four five", 'four
 
  five'
 
+test "#3229, multiline strings", ->
+  # Separate lines by default by a single space in literal strings.
+  eq 'one
+      two', 'one two'
+  eq "one
+      two", 'one two'
+  eq '
+        a
+        b
+    ', 'a b'
+  eq "
+        a
+        b
+    ", 'a b'
+  eq 'one
+
+        two', 'one two'
+  eq "one
+
+        two", 'one two'
+  eq '
+    indentation
+      doesn\'t
+  matter', 'indentation doesn\'t matter'
+
+  # Use backslashes at the end of a line to specify whitespace between lines.
+  eq 'a \
+      b\
+      c  \
+      d', 'a bc  d'
+  eq "a \
+      b\
+      c  \
+      d", 'a bc  d'
+  eq 'ignore  \  
+      trailing whitespace', 'ignore  trailing whitespace'
+
+  # Backslash at the beginning of a literal string.
+  eq '\
+      ok', 'ok'
+  eq '  \
+      ok', '  ok'
+
+  # Same behavior in interpolated strings.
+  eq "interpolation #{1}
+      follows #{2}  \
+      too #{3}\
+      !", 'interpolation 1 follows 2  too 3!'
+  eq "a #{
+    'string ' + "inside
+                 interpolation"
+    }", "a string inside interpolation"
+
+  # Handle escaped backslashes correctly.
+  eq 'escaped backslash at EOL\\
+      next line', 'escaped backslash at EOL\\ next line'
+  eq '\\
+      next line', '\\ next line'
+  eq "#{1}\\
+      after interpolation", '1\\ after interpolation'
+  eq 'escaped backslash before slash\\  \
+      next line', 'escaped backslash before slash\\  next line'
+  eq 'triple backslash\\\
+      next line', 'triple backslash\\next line'
+
+  # Use backslashes at beginning of a line to specify whitespace between lines.
+  eq 'first line
+      \   backslash at BOL', 'first line \   backslash at BOL'
+  eq 'first line\
+      \   backslash at BOL', 'first line\   backslash at BOL'
+
+  # Edge case.
+  eq 'lone
+
+        \
+
+        backslash', 'lone backslash'
+
+
 #647
 eq "''Hello, World\\''", '''
 '\'Hello, World\\\''
@@ -91,11 +170,7 @@ ok a is "one\ntwo\n"
 eq ''' line 0
   should not be relevant
     to the indent level
-''', '
- line 0\n
-should not be relevant\n
-  to the indent level
-'
+''', ' line 0\nshould not be relevant\n  to the indent level'
 
 eq ''' '\\\' ''', " '\\' "
 eq """ "\\\" """, ' "\\" '