From efe8c68c759f5bf9034c0b7ae28c5140b53d7114 Mon Sep 17 00:00:00 2001 From: xixixao Date: Mon, 18 Nov 2013 04:32:15 +0000 Subject: [PATCH] Changed multiline string literals --- lib/coffee-script/grammar.js | 12 ++++------- lib/coffee-script/lexer.js | 20 ++++++++++++------ src/lexer.coffee | 20 ++++++++++++------ test/interpolation.coffee | 2 +- test/strings.coffee | 39 ++++++++++++++++++++++++++++++++---- 5 files changed, 68 insertions(+), 25 deletions(-) diff --git a/lib/coffee-script/grammar.js b/lib/coffee-script/grammar.js index 666042de..776cf859 100644 --- a/lib/coffee-script/grammar.js +++ b/lib/coffee-script/grammar.js @@ -96,8 +96,7 @@ return new Value($1); }), o('ObjAssignable : Expression', function() { return new Assign(LOC(1)(new Value($1)), $3, 'object'); - }), o('ObjAssignable :\ - INDENT Expression OUTDENT', function() { + }), o('ObjAssignable : INDENT Expression OUTDENT', function() { return new Assign(LOC(1)(new Value($1)), $4, 'object'); }), o('Comment') ], @@ -573,14 +572,11 @@ } else { return new Op($2, $1, $3); } - }), o('SimpleAssignable COMPOUND_ASSIGN\ - Expression', function() { + }), o('SimpleAssignable COMPOUND_ASSIGN Expression', function() { return new Assign($1, $3, $2); - }), o('SimpleAssignable COMPOUND_ASSIGN\ - INDENT Expression OUTDENT', function() { + }), o('SimpleAssignable COMPOUND_ASSIGN INDENT Expression OUTDENT', function() { return new Assign($1, $4, $2); - }), o('SimpleAssignable COMPOUND_ASSIGN TERMINATOR\ - Expression', function() { + }), o('SimpleAssignable COMPOUND_ASSIGN TERMINATOR Expression', function() { return new Assign($1, $4, $2); }), o('SimpleAssignable EXTENDS Expression', function() { return new Extends($1, $3); diff --git a/lib/coffee-script/lexer.js b/lib/coffee-script/lexer.js index 7c7d71e6..b3f80cb7 100644 --- a/lib/coffee-script/lexer.js +++ b/lib/coffee-script/lexer.js @@ -173,7 +173,7 @@ return 0; } string = match[0]; - this.token('STRING', string.replace(MULTILINER, '\\\n'), 0, string.length); + this.token('STRING', this.trimAndEscapeLines(string), 0, string.length); break; case '"': if (!(string = this.balancedString(this.chunk, '"'))) { @@ -185,7 +185,7 @@ lexedLength: string.length }); } else { - this.token('STRING', this.escapeLines(string), 0, string.length); + this.token('STRING', this.trimAndEscapeLines(string), 0, string.length); } break; default: @@ -762,16 +762,24 @@ return LINE_CONTINUER.test(this.chunk) || ((_ref2 = this.tag()) === '\\' || _ref2 === '.' || _ref2 === '?.' || _ref2 === '?::' || _ref2 === 'UNARY' || _ref2 === 'MATH' || _ref2 === '+' || _ref2 === '-' || _ref2 === 'SHIFT' || _ref2 === 'RELATION' || _ref2 === 'COMPARE' || _ref2 === 'LOGIC' || _ref2 === 'THROW' || _ref2 === 'EXTENDS'); }; + Lexer.prototype.trimAndEscapeLines = function(str) { + return this.escapeLines(str.replace(/^(.)\s*\n\s*/, '$1').replace(/\s*\n\s*(.)$/, '$1')); + }; + Lexer.prototype.escapeLines = function(str, heredoc) { - return str.replace(MULTILINER, heredoc ? '\\n' : ''); + if (heredoc) { + return str.replace(MULTILINER, '\\n'); + } else { + return str.replace(/\\\n\s*/g, '').replace(/\s*\n\s*/g, ' '); + } }; Lexer.prototype.makeString = function(body, quote, heredoc) { if (!body) { return quote + quote; } - body = body.replace(/\\([\s\S])/g, function(match, contents) { - if (contents === '\n' || contents === quote) { + body = body.replace(/\\([^])/g, function(match, contents) { + if (contents === quote || heredoc && contents === '\n') { return contents; } else { return match; @@ -852,7 +860,7 @@ MULTI_DENT = /^(?:\n[^\n\S]*)+/; - SIMPLESTR = /^'[^\\']*(?:\\.[^\\']*)*'/; + SIMPLESTR = /^'[^\\']*(?:\\[^][^\\']*)*'/; JSTOKEN = /^`[^\\`]*(?:\\.[^\\`]*)*`/; diff --git a/src/lexer.coffee b/src/lexer.coffee index d7d7e155..773e7696 100644 --- a/src/lexer.coffee +++ b/src/lexer.coffee @@ -190,13 +190,13 @@ exports.Lexer = class Lexer when "'" return 0 unless match = SIMPLESTR.exec @chunk string = match[0] - @token 'STRING', string.replace(MULTILINER, '\\\n'), 0, string.length + @token 'STRING', @trimAndEscapeLines(string), 0, string.length when '"' return 0 unless string = @balancedString @chunk, '"' if 0 < string.indexOf '#{', 1 @interpolateString string[1...-1], strOffset: 1, lexedLength: string.length else - @token 'STRING', @escapeLines(string), 0, string.length + @token 'STRING', @trimAndEscapeLines(string), 0, string.length else return 0 if octalEsc = /^(?:\\.|[^\\])*\\(?:0[0-7]|[1-7])/.test string @@ -684,15 +684,23 @@ exports.Lexer = class Lexer @tag() in ['\\', '.', '?.', '?::', 'UNARY', 'MATH', '+', '-', 'SHIFT', 'RELATION' 'COMPARE', 'LOGIC', 'THROW', 'EXTENDS'] + # Remove newlines from beginning and end of string literals. + # `str` includes quotes. + trimAndEscapeLines: (str) -> + @escapeLines str.replace(/^(.)\s*\n\s*/, '$1').replace(/\s*\n\s*(.)$/, '$1') + # Converts newlines for string literals. escapeLines: (str, heredoc) -> - str.replace MULTILINER, if heredoc then '\\n' else '' + if heredoc + str.replace MULTILINER, '\\n' + else + str.replace(/\\\n\s*/g, '').replace(/\s*\n\s*/g, ' ') # Constructs a string token by escaping quotes and newlines. makeString: (body, quote, heredoc) -> return quote + quote unless body - body = body.replace /\\([\s\S])/g, (match, contents) -> - if contents in ['\n', quote] then contents else match + body = body.replace /\\([^])/g, (match, contents) -> + if contents is quote or heredoc and contents is '\n' then contents else match body = body.replace /// #{quote} ///g, '\\$&' quote + @escapeLines(body, heredoc) + quote @@ -787,7 +795,7 @@ CODE = /^[-=]>/ MULTI_DENT = /^(?:\n[^\n\S]*)+/ -SIMPLESTR = /^'[^\\']*(?:\\.[^\\']*)*'/ +SIMPLESTR = /^'[^\\']*(?:\\[^][^\\']*)*'/ JSTOKEN = /^`[^\\`]*(?:\\.[^\\`]*)*`/ diff --git a/test/interpolation.coffee b/test/interpolation.coffee index 0cb98a9b..53e32f13 100644 --- a/test/interpolation.coffee +++ b/test/interpolation.coffee @@ -30,7 +30,7 @@ eq "#{6 / 2}", '3' eq "#{6 / 2}#{6 / 2}", '33' # parsed as division eq "#{6 + /2}#{6/ + 2}", '6/2}#{6/2' # parsed as a regex eq "#{6/2} - #{6/2}", '3 3' # newline cannot be part of a regex, so it's division + #{6/2}", '3 3' # newline cannot be part of a regex, so it's division eq "#{/// "'/'"/" ///}", '/"\'\\/\'"\\/"/' # heregex, stuffed with spicy characters eq "#{/\\'/}", "/\\\\'/" diff --git a/test/strings.coffee b/test/strings.coffee index c9f21856..6ba865eb 100644 --- a/test/strings.coffee +++ b/test/strings.coffee @@ -18,6 +18,38 @@ eq "four five", 'four five' +test "#3229, multine strings", -> + eq 'one + two', 'one two' + eq "one + two", 'one two' + eq 'a \ + b\ + c \ + d', 'a bc d' + eq "a \ + b\ + c \ + d", 'a bc d' + eq 'one + + two', 'one two' + eq "one + + two", 'one two' + eq ' + a + b + ', 'a b' + eq " + a + b + ", 'a b' + eq "interpolation #{1} + follows #{2} \ + too #{3}\ + !", 'interpolation 1 follows 2 too 3!' + #647 eq "''Hello, World\\''", ''' '\'Hello, World\\\'' @@ -91,10 +123,9 @@ ok a is "one\ntwo\n" eq ''' line 0 should not be relevant to the indent level -''', ' - line 0\n -should not be relevant\n - to the indent level +''', ' line 0\n\ + should not be relevant\n \ + to the indent level ' eq ''' '\\\' ''', " '\\' "