diff --git a/lib/lexer.js b/lib/lexer.js index 9bb42e68..87fd371c 100644 --- a/lib/lexer.js +++ b/lib/lexer.js @@ -1,5 +1,5 @@ (function() { - var ASSIGNED, CALLABLE, CODE, COFFEE_ALIASES, COFFEE_KEYWORDS, COMMENT, COMPARE, COMPOUND_ASSIGN, CONVERSIONS, HEREDOC, HEREDOC_INDENT, IDENTIFIER, JS_CLEANER, JS_FORBIDDEN, JS_KEYWORDS, LAST_DENT, LAST_DENTS, LINE_BREAK, LOGIC, Lexer, MATH, MULTILINER, MULTI_DENT, NEXT_CHARACTER, NOT_REGEX, NO_NEWLINE, NUMBER, OPERATOR, REGEX_END, REGEX_ESCAPE, REGEX_INTERPOLATION, REGEX_START, RESERVED, Rewriter, SHIFT, UNARY, WHITESPACE, _ref, compact, count, include, starts; + var ASSIGNED, CALLABLE, CODE, COFFEE_ALIASES, COFFEE_KEYWORDS, COMMENT, COMPARE, COMPOUND_ASSIGN, CONVERSIONS, HEREDOC, HEREDOC_INDENT, IDENTIFIER, JSTOKEN, JS_FORBIDDEN, JS_KEYWORDS, LINE_BREAK, LOGIC, Lexer, MATH, MULTILINER, MULTI_DENT, NEXT_CHARACTER, NOT_REGEX, NO_NEWLINE, NUMBER, OPERATOR, REGEX_END, REGEX_ESCAPE, REGEX_INTERPOLATION, REGEX_START, RESERVED, Rewriter, SHIFT, SIMPLESTR, UNARY, WHITESPACE, _ref, compact, count, include, starts; var __slice = Array.prototype.slice; _ref = require('./rewriter'); Rewriter = _ref.Rewriter; @@ -12,7 +12,7 @@ Lexer = function() {}; Lexer.prototype.tokenize = function(code, options) { var o; - code = code.replace(/(\r|\s+$)/g, ''); + code = code.replace(/\r/g, '').replace(/\s+$/, ''); o = options || {}; this.code = code; this.i = 0; @@ -22,9 +22,8 @@ this.outdebt = 0; this.indents = []; this.tokens = []; - while (this.i < this.code.length) { - this.chunk = this.code.slice(this.i); - this.extractNextToken(); + while ((this.chunk = code.slice(this.i))) { + this.identifierToken() || this.commentToken() || this.whitespaceToken() || this.lineToken() || this.heredocToken() || this.stringToken() || this.numberToken() || this.regexToken() || this.jsToken() || this.literalToken(); } this.closeIndentation(); if (o.rewrite === false) { @@ -32,62 +31,33 @@ } return (new Rewriter()).rewrite(this.tokens); }; - Lexer.prototype.extractNextToken = function() { - if (this.identifierToken()) { - return null; - } - if (this.commentToken()) { - return null; - } - if (this.whitespaceToken()) { - return null; - } - if (this.lineToken()) { - return null; - } - if (this.heredocToken()) { - return null; - } - if (this.stringToken()) { - return null; - } - if (this.numberToken()) { - return null; - } - if (this.regexToken()) { - return null; - } - if (this.jsToken()) { - return null; - } - return this.literalToken(); - }; Lexer.prototype.identifierToken = function() { - var close_index, forcedIdentifier, id, tag; - if (!(id = this.match(IDENTIFIER, 1))) { + var closeIndex, forcedIdentifier, id, match, tag; + if (!(match = IDENTIFIER.exec(this.chunk))) { return false; } + id = match[0]; this.i += id.length; - forcedIdentifier = this.tagAccessor() || this.match(ASSIGNED, 1); - tag = 'IDENTIFIER'; - if (include(JS_KEYWORDS, id) || (!forcedIdentifier && include(COFFEE_KEYWORDS, id))) { - tag = id.toUpperCase(); - } - if (tag === 'WHEN' && include(LINE_BREAK, this.tag())) { - tag = 'LEADING_WHEN'; - } if (id === 'all' && this.tag() === 'FOR') { - tag = 'ALL'; + this.token('ALL', id); + return true; } - if (include(UNARY, tag)) { - tag = 'UNARY'; + forcedIdentifier = this.tagAccessor() || ASSIGNED.test(this.chunk); + tag = 'IDENTIFIER'; + if (include(JS_KEYWORDS, id) || !forcedIdentifier && include(COFFEE_KEYWORDS, id)) { + tag = id.toUpperCase(); + if (tag === 'WHEN' && include(LINE_BREAK, this.tag())) { + tag = 'LEADING_WHEN'; + } else if (include(UNARY, tag)) { + tag = 'UNARY'; + } } if (include(JS_FORBIDDEN, id)) { if (forcedIdentifier) { tag = 'STRING'; id = ("\"" + (id) + "\""); if (forcedIdentifier === 'accessor') { - close_index = true; + closeIndex = true; if (this.tag() !== '@') { this.tokens.pop(); } @@ -101,25 +71,25 @@ if (include(COFFEE_ALIASES, id)) { tag = (id = CONVERSIONS[id]); } - if (include(LOGIC, id)) { - tag = 'LOGIC'; - } if (id === '!') { tag = 'UNARY'; + } else if (include(LOGIC, id)) { + tag = 'LOGIC'; } } this.token(tag, id); - if (close_index) { + if (closeIndex) { this.token(']', ']'); } return true; }; Lexer.prototype.numberToken = function() { - var number; - if (!(number = this.match(NUMBER, 1))) { + var match, number; + if (!(match = NUMBER.exec(this.chunk))) { return false; } - if (this.tag() === '.' && starts(number, '.')) { + number = match[0]; + if (this.tag() === '.' && number.charAt(0) === '.') { return false; } this.i += number.length; @@ -127,43 +97,57 @@ return true; }; Lexer.prototype.stringToken = function() { - var string; - if (!(starts(this.chunk, '"') || starts(this.chunk, "'"))) { - return false; + var match, string; + switch (this.chunk.charAt(0)) { + case "'": + if (!(match = SIMPLESTR.exec(this.chunk))) { + return false; + } + this.token('STRING', (string = match[0]).replace(MULTILINER, '\\\n')); + break; + case '"': + if (!(string = this.balancedToken(['"', '"'], ['#{', '}']))) { + return false; + } + this.interpolateString(string); + break; + default: + return false; } - if (!(string = this.balancedToken(['"', '"'], ['#{', '}']) || this.balancedToken(["'", "'"]))) { - return false; - } - this.interpolateString(string.replace(/\n/g, '\\\n')); - this.line += count(string, "\n"); + this.line += count(string, '\n'); this.i += string.length; return true; }; Lexer.prototype.heredocToken = function() { - var doc, match, quote; + var doc, heredoc, match, quote; if (!(match = this.chunk.match(HEREDOC))) { return false; } - quote = match[1].substr(0, 1); - doc = this.sanitizeHeredoc(match[2] || match[4] || '', { - quote: quote + heredoc = match[0]; + quote = heredoc.charAt(0); + doc = this.sanitizeHeredoc(match[2], { + quote: quote, + indent: null }); this.interpolateString(quote + doc + quote, { heredoc: true }); - this.line += count(match[1], "\n"); - this.i += match[1].length; + this.line += count(heredoc, '\n'); + this.i += heredoc.length; return true; }; Lexer.prototype.commentToken = function() { - var match; + var _ref2, comment, here, match; if (!(match = this.chunk.match(COMMENT))) { return false; } - this.line += count(match[1], "\n"); - this.i += match[1].length; - if (match[2]) { - this.token('HERECOMMENT', this.sanitizeHeredoc(match[2], { + _ref2 = match; + comment = _ref2[0]; + here = _ref2[1]; + this.line += count(comment, '\n'); + this.i += comment.length; + if (here) { + this.token('HERECOMMENT', this.sanitizeHeredoc(here, { herecomment: true, indent: Array(this.indent + 1).join(' ') })); @@ -172,14 +156,11 @@ return true; }; Lexer.prototype.jsToken = function() { - var script; - if (!(starts(this.chunk, '`'))) { + var match, script; + if (!(this.chunk.charAt(0) === '`' && (match = JSTOKEN.exec(this.chunk)))) { return false; } - if (!(script = this.balancedToken(['`', '`']))) { - return false; - } - this.token('JS', script.replace(JS_CLEANER, '')); + this.token('JS', (script = match[0]).slice(1, -1)); this.i += script.length; return true; }; @@ -197,29 +178,25 @@ if (!(regex = this.balancedToken(['/', '/']))) { return false; } - if (!(end = this.chunk.substr(regex.length).match(REGEX_END))) { + if (!(end = this.chunk.slice(regex.length).match(REGEX_END))) { return false; } - if (end[2]) { - regex += (flags = end[2]); - } - if (regex.match(REGEX_INTERPOLATION)) { - str = regex.substring(1).split('/')[0]; - str = str.replace(REGEX_ESCAPE, function(escaped) { - return '\\' + escaped; - }); - this.tokens = this.tokens.concat([['(', '('], ['NEW', 'new'], ['IDENTIFIER', 'RegExp'], ['CALL_START', '(']]); + flags = end[0]; + if (REGEX_INTERPOLATION.test(regex)) { + str = regex.slice(1, -1); + str = str.replace(REGEX_ESCAPE, '\\$&'); + this.tokens.push(['(', '('], ['NEW', 'new'], ['IDENTIFIER', 'RegExp'], ['CALL_START', '(']); this.interpolateString("\"" + (str) + "\"", { escapeQuotes: true }); if (flags) { - this.tokens.splice(this.tokens.length, 0, [',', ','], ['STRING', ("\"" + (flags) + "\"")]); + this.tokens.push([',', ','], ['STRING', ("\"" + (flags) + "\"")]); } - this.tokens.splice(this.tokens.length, 0, [')', ')'], [')', ')']); + this.tokens.push([')', ')'], [')', ')']); } else { - this.token('REGEX', regex); + this.token('REGEX', regex + flags); } - this.i += regex.length; + this.i += regex.length + flags.length; return true; }; Lexer.prototype.balancedToken = function() { @@ -228,16 +205,17 @@ return this.balancedString(this.chunk, delimited); }; Lexer.prototype.lineToken = function() { - var diff, indent, nextCharacter, noNewlines, prev, size; - if (!(indent = this.match(MULTI_DENT, 1))) { + var diff, indent, match, nextCharacter, noNewlines, prev, size; + if (!(match = MULTI_DENT.exec(this.chunk))) { return false; } - this.line += count(indent, "\n"); + indent = match[0]; + this.line += count(indent, '\n'); this.i += indent.length; prev = this.prev(2); - size = indent.match(LAST_DENTS).reverse()[0].match(LAST_DENT)[1].length; - nextCharacter = this.match(NEXT_CHARACTER, 1); - noNewlines = nextCharacter === '.' || nextCharacter === ',' || this.unfinished(); + size = indent.length - 1 - indent.lastIndexOf('\n'); + nextCharacter = NEXT_CHARACTER.exec(this.chunk)[1]; + noNewlines = (('.' === nextCharacter || ',' === nextCharacter)) || this.unfinished(); if (size - this.indebt === this.indent) { if (noNewlines) { return this.suppressNewlines(); @@ -283,43 +261,46 @@ this.outdebt -= moveOut; } if (!(this.tag() === 'TERMINATOR' || noNewlines)) { - this.token('TERMINATOR', "\n"); + this.token('TERMINATOR', '\n'); } return true; }; Lexer.prototype.whitespaceToken = function() { - var prev, space; - if (!(space = this.match(WHITESPACE, 1))) { + var match, prev; + if (!(match = WHITESPACE.exec(this.chunk))) { return false; } prev = this.prev(); if (prev) { prev.spaced = true; } - this.i += space.length; + this.i += match[0].length; return true; }; Lexer.prototype.newlineToken = function(newlines) { if (this.tag() !== 'TERMINATOR') { - this.token('TERMINATOR', "\n"); + this.token('TERMINATOR', '\n'); } return true; }; Lexer.prototype.suppressNewlines = function() { - if (this.value() === "\\") { + if (this.value() === '\\') { this.tokens.pop(); } return true; }; Lexer.prototype.literalToken = function() { var _ref2, match, prev, space, spaced, tag, value; - match = this.chunk.match(OPERATOR); - value = match && match[1]; - space = match && match[2]; - if (value && value.match(CODE)) { - this.tagParameters(); + if (match = this.chunk.match(OPERATOR)) { + _ref2 = match; + value = _ref2[0]; + space = _ref2[1]; + if (CODE.test(value)) { + this.tagParameters(); + } + } else { + value = this.chunk.charAt(0); } - value || (value = this.chunk.substr(0, 1)); this.i += value.length; spaced = (prev = this.prev()) && prev.spaced; tag = value; @@ -354,11 +335,13 @@ tag = 'CALL_START'; } else if (value === '[') { tag = 'INDEX_START'; - if (this.tag() === '?') { - this.tag(1, 'INDEX_SOAK'); - } - if (this.tag() === '::') { - this.tag(1, 'INDEX_PROTO'); + switch (this.tag()) { + case '?': + this.tag(1, 'INDEX_SOAK'); + break; + case '::': + this.tag(1, 'INDEX_PROTO'); + break; } } } @@ -373,7 +356,7 @@ accessor = (function() { if (prev[1] === '::') { return this.tag(1, 'PROTOTYPE_ACCESS'); - } else if (prev[1] === '.' && !(this.value(2) === '.')) { + } else if (prev[1] === '.' && this.value(2) !== '.') { if (this.tag(2) === '?') { this.tag(1, 'SOAK_ACCESS'); return this.tokens.splice(-2, 1); @@ -387,26 +370,32 @@ return accessor ? 'accessor' : false; }; Lexer.prototype.sanitizeHeredoc = function(doc, options) { - var _ref2, attempt, indent, match; - indent = options.indent; - if (options.herecomment && !include(doc, '\n')) { + var _ref2, attempt, herecomment, indent, match; + _ref2 = options; + indent = _ref2.indent; + herecomment = _ref2.herecomment; + if (herecomment && !include(doc, '\n')) { return doc; } - if (!(options.herecomment)) { - while ((match = HEREDOC_INDENT.exec(doc)) !== null) { - attempt = (typeof (_ref2 = match[2]) !== "undefined" && _ref2 !== null) ? match[2] : match[3]; - if (!(typeof indent !== "undefined" && indent !== null) || (0 < attempt.length) && (attempt.length < indent.length)) { + if (!(herecomment)) { + while ((match = HEREDOC_INDENT.exec(doc))) { + attempt = match[1]; + if (indent === null || (0 < attempt.length) && (attempt.length < indent.length)) { indent = attempt; } } } - indent || (indent = ''); - doc = doc.replace(new RegExp("^" + indent, 'gm'), ''); - if (options.herecomment) { + if (indent) { + doc = doc.replace(new RegExp("\\n" + (indent), "g"), '\n'); + } + if (herecomment) { return doc; } - doc = doc.replace(/^\n/, ''); - return doc.replace(MULTILINER, "\\n").replace(new RegExp(options.quote, 'g'), "\\" + (options.quote)); + doc = doc.replace(/^\n/, '').replace(new RegExp("" + (options.quote), "g"), '\\$&'); + if (options.quote === "'") { + doc = this.oldline(doc, true); + } + return doc; }; Lexer.prototype.tagParameters = function() { var i, tok; @@ -444,13 +433,14 @@ throw new Error("SyntaxError: Reserved word \"" + (this.value()) + "\" on line " + (this.line + 1) + " can't be assigned"); }; Lexer.prototype.balancedString = function(str, delimited, options) { - var _i, _len, _ref2, _ref3, close, i, levels, open, pair, slash; + var _i, _len, _ref2, _ref3, close, i, levels, open, pair, slash, slen; options || (options = {}); slash = delimited[0][0] === '/'; levels = []; i = 0; - while (i < str.length) { - if (levels.length && starts(str, '\\', i)) { + slen = str.length; + while (i < slen) { + if (levels.length && str.charAt(i) === '\\') { i += 1; } else { _ref2 = delimited; @@ -473,7 +463,7 @@ } } } - if (!levels.length || slash && starts(str, '\n', i)) { + if (!levels.length || slash && str.charAt(i) === '\n') { break; } i += 1; @@ -484,85 +474,87 @@ } throw new Error("SyntaxError: Unterminated " + (levels.pop()[0]) + " starting on line " + (this.line + 1)); } - return !i ? false : str.substring(0, i); + return !i ? false : str.slice(0, i); }; Lexer.prototype.interpolateString = function(str, options) { - var _len, _ref2, _ref3, escaped, expr, i, idx, inner, interpolated, lexer, nested, pi, quote, tag, tok, token, tokens, value; + var _len, _ref2, _ref3, end, escaped, expr, i, idx, inner, interpolated, lexer, nested, pi, push, quote, s, tag, tok, token, tokens, value; options || (options = {}); - if (str.length < 3 || !starts(str, '"')) { + quote = str.charAt(0); + if (quote !== '"' || str.length < 3) { return this.token('STRING', str); - } else { - lexer = new Lexer(); - tokens = []; - quote = str.substring(0, 1); - _ref2 = [1, 1]; - i = _ref2[0]; - pi = _ref2[1]; - while (i < str.length - 1) { - if (starts(str, '\\', i)) { - i += 1; - } else if (expr = this.balancedString(str.substring(i), [['#{', '}']])) { - if (pi < i) { - tokens.push(['STRING', quote + str.substring(pi, i) + quote]); - } - inner = expr.substring(2, expr.length - 1); - if (inner.length) { - if (options.heredoc) { - inner = inner.replace(new RegExp('\\\\' + quote, 'g'), quote); - } - nested = lexer.tokenize("(" + (inner) + ")", { - line: this.line - }); - _ref2 = nested; - for (idx = 0, _len = _ref2.length; idx < _len; idx++) { - tok = _ref2[idx]; - if (tok[0] === 'CALL_END') { - (tok[0] = ')'); - } - } - nested.pop(); - tokens.push(['TOKENS', nested]); - } else { - tokens.push(['STRING', quote + quote]); - } - i += expr.length - 1; - pi = i + 1; - } - i += 1; - } - if (pi < i && pi < str.length - 1) { - tokens.push(['STRING', quote + str.substring(pi, i) + quote]); - } - if (tokens[0][0] !== 'STRING') { - tokens.unshift(['STRING', '""']); - } - interpolated = tokens.length > 1; - if (interpolated) { - this.token('(', '('); - } - _ref2 = tokens; - for (i = 0, _len = _ref2.length; i < _len; i++) { - token = _ref2[i]; - _ref3 = token; - tag = _ref3[0]; - value = _ref3[1]; - if (tag === 'TOKENS') { - this.tokens = this.tokens.concat(value); - } else if (tag === 'STRING' && options.escapeQuotes) { - escaped = value.substring(1, value.length - 1).replace(/"/g, '\\"'); - this.token(tag, "\"" + (escaped) + "\""); - } else { - this.token(tag, value); - } - if (i < tokens.length - 1) { - this.token('+', '+'); - } - } - if (interpolated) { - this.token(')', ')'); - } - return tokens; } + lexer = new Lexer(); + tokens = []; + i = (pi = 1); + end = str.length - 1; + while (i < end) { + if (str.charAt(i) === '\\') { + i += 1; + } else if (expr = this.balancedString(str.slice(i), [['#{', '}']])) { + if (pi < i) { + s = quote + this.oldline(str.slice(pi, i), options.heredoc) + quote; + tokens.push(['STRING', s]); + } + inner = expr.slice(2, -1).replace(/^\s+/, ''); + if (inner.length) { + if (options.heredoc) { + inner = inner.replace(RegExp('\\\\' + quote, 'g'), quote); + } + nested = lexer.tokenize("(" + (inner) + ")", { + line: this.line + }); + _ref2 = nested; + for (idx = 0, _len = _ref2.length; idx < _len; idx++) { + tok = _ref2[idx]; + if (tok[0] === 'CALL_END') { + (tok[0] = ')'); + } + } + nested.pop(); + tokens.push(['TOKENS', nested]); + } else { + tokens.push(['STRING', quote + quote]); + } + i += expr.length - 1; + pi = i + 1; + } + i += 1; + } + if ((i > pi) && (pi < str.length - 1)) { + s = str.slice(pi, i).replace(MULTILINER, options.heredoc ? '\\n' : ''); + tokens.push(['STRING', quote + s + quote]); + } + if (tokens[0][0] !== 'STRING') { + tokens.unshift(['STRING', '""']); + } + interpolated = tokens.length > 1; + if (interpolated) { + this.token('(', '('); + } + _ref2 = tokens; + push = _ref2.push; + _ref2 = tokens; + for (i = 0, _len = _ref2.length; i < _len; i++) { + token = _ref2[i]; + _ref3 = token; + tag = _ref3[0]; + value = _ref3[1]; + if (tag === 'TOKENS') { + push.apply(this.tokens, value); + } else if (tag === 'STRING' && options.escapeQuotes) { + escaped = value.slice(1, -1).replace(/"/g, '\\"'); + this.token(tag, "\"" + (escaped) + "\""); + } else { + this.token(tag, value); + } + if (i < tokens.length - 1) { + this.token('+', '+'); + } + } + if (interpolated) { + this.token(')', ')'); + } + return tokens; }; Lexer.prototype.token = function(tag, value) { return this.tokens.push([tag, value, this.line]); @@ -592,43 +584,41 @@ }; Lexer.prototype.match = function(regex, index) { var m; - if (!(m = this.chunk.match(regex))) { - return false; - } - return m ? m[index] : false; + return (m = this.chunk.match(regex)) ? m[index || 0] : false; }; Lexer.prototype.unfinished = function() { - var prev; - prev = this.prev(2); - return this.value() && this.value().match && this.value().match(NO_NEWLINE) && prev && (prev[0] !== '.') && !this.value().match(CODE) && !this.chunk.match(ASSIGNED); + var prev, value; + return (prev = this.prev(2)) && prev[0] !== '.' && (value = this.value()) && NO_NEWLINE.test(value) && !CODE.test(value) && !ASSIGNED.test(this.chunk); + }; + Lexer.prototype.oldline = function(str, heredoc) { + return str.replace(MULTILINER, heredoc ? '\\n' : ''); }; return Lexer; })(); - JS_KEYWORDS = ["if", "else", "true", "false", "new", "return", "try", "catch", "finally", "throw", "break", "continue", "for", "in", "while", "delete", "instanceof", "typeof", "switch", "super", "extends", "class", "this", "null", "debugger"]; - COFFEE_ALIASES = ["and", "or", "is", "isnt", "not"]; - COFFEE_KEYWORDS = COFFEE_ALIASES.concat(["then", "unless", "until", "loop", "yes", "no", "on", "off", "of", "by", "where", "when"]); - RESERVED = ["case", "default", "do", "function", "var", "void", "with", "const", "let", "enum", "export", "import", "native", "__hasProp", "__extends", "__slice"]; + JS_KEYWORDS = ['if', 'else', 'true', 'false', 'new', 'return', 'try', 'catch', 'finally', 'throw', 'break', 'continue', 'for', 'in', 'while', 'delete', 'instanceof', 'typeof', 'switch', 'super', 'extends', 'class', 'this', 'null', 'debugger']; + COFFEE_ALIASES = ['and', 'or', 'is', 'isnt', 'not']; + COFFEE_KEYWORDS = COFFEE_ALIASES.concat(['then', 'unless', 'until', 'loop', 'yes', 'no', 'on', 'off', 'of', 'by', 'where', 'when']); + RESERVED = ['case', 'default', 'do', 'function', 'var', 'void', 'with', 'const', 'let', 'enum', 'export', 'import', 'native', '__hasProp', '__extends', '__slice']; JS_FORBIDDEN = JS_KEYWORDS.concat(RESERVED); - IDENTIFIER = /^([a-zA-Z\$_](\w|\$)*)/; - NUMBER = /^(((\b0(x|X)[0-9a-fA-F]+)|((\b[0-9]+(\.[0-9]+)?|\.[0-9]+)(e[+\-]?[0-9]+)?)))\b/i; - HEREDOC = /^("{6}|'{6}|"{3}([\s\S]*?)\n?([ \t]*)"{3}|'{3}([\s\S]*?)\n?([ \t]*)'{3})/; - OPERATOR = /^(-[\-=>]?|\+[+=]?|[*&|\/%=<>^:!?]+)([ \t]*)/; - WHITESPACE = /^([ \t]+)/; - COMMENT = /^(###([^#][\s\S]*?)(###[ \t]*\n|(###)?$)|(\s*#(?!##[^#])[^\n]*)+)/; - CODE = /^((-|=)>)/; - MULTI_DENT = /^((\n([ \t]*))+)(\.)?/; - LAST_DENTS = /\n([ \t]*)/g; - LAST_DENT = /\n([ \t]*)/; + IDENTIFIER = /^[a-zA-Z_$][\w$]*/; + NUMBER = /^0x[\da-f]+|^(?:\d+(\.\d+)?|\.\d+)(?:e[+-]?\d+)?/i; + HEREDOC = /^("""|''')([\s\S]*?)\n?[ \t]*\1/; + OPERATOR = /^(?:-[-=>]?|\+[+=]?|[*&|\/%=<>^:!?]+)(?=([ \t]*))/; + WHITESPACE = /^[ \t]+/; + COMMENT = /^###([^#][\s\S]*?)(?:###[ \t]*\n|(?:###)?$)|^(?:\s*#(?!##[^#]).*)+/; + CODE = /^[-=]>/; + MULTI_DENT = /^(?:\n[ \t]*)+/; + SIMPLESTR = /^'[^\\']*(?:\\.[^\\']*)*'/; + JSTOKEN = /^`[^\\`]*(?:\\.[^\\`]*)*`/; REGEX_START = /^\/([^\/])/; - REGEX_INTERPOLATION = /([^\\]#\{.*[^\\]\})/; - REGEX_END = /^(([imgy]{1,4})\b|\W|$)/; - REGEX_ESCAPE = /\\[^\$]/g; - JS_CLEANER = /(^`|`$)/g; + REGEX_INTERPOLATION = /[^\\]#\{.*[^\\]\}/; + REGEX_END = /^[imgy]{0,4}(?![a-zA-Z])/; + REGEX_ESCAPE = /\\[^#]/g; MULTILINER = /\n/g; - NO_NEWLINE = /^([+\*&|\/\-%=<>!.\\][<>=&|]*|and|or|is|isnt|not|delete|typeof|instanceof)$/; - HEREDOC_INDENT = /(\n+([ \t]*)|^([ \t]+))/g; - ASSIGNED = /^\s*(([a-zA-Z\$_@]\w*|["'][^\r\n]+?["']|\d+)[ \t]*?[:=][^:=])/; - NEXT_CHARACTER = /^\s*(\S)/; + NO_NEWLINE = /^(?:[-+*&|\/%=<>!.\\][<>=&|]*|and|or|is(?:nt)?|not|delete|typeof|instanceof)$/; + HEREDOC_INDENT = /\n+([ \t]*)/g; + ASSIGNED = /^\s*@?[$A-Za-z_][$\w]*[ \t]*?[:=][^:=>]/; + NEXT_CHARACTER = /^\s*(\S?)/; COMPOUND_ASSIGN = ['-=', '+=', '/=', '*=', '%=', '||=', '&&=', '?=', '<<=', '>>=', '>>>=', '&=', '^=', '|=']; UNARY = ['UMINUS', 'UPLUS', '!', '!!', '~', 'TYPEOF', 'DELETE']; LOGIC = ['&', '|', '^', '&&', '||']; diff --git a/src/lexer.coffee b/src/lexer.coffee index 02f7a433..237b4999 100644 --- a/src/lexer.coffee +++ b/src/lexer.coffee @@ -33,7 +33,7 @@ exports.Lexer = class Lexer # Before returning the token stream, run it through the [Rewriter](rewriter.html) # unless explicitly asked not to. tokenize: (code, options) -> - code = code.replace /(\r|\s+$)/g, '' + code = code.replace(/\r/g, '').replace /\s+$/, '' o = options or {} @code = code # The remainder of the source code. @i = 0 # Current character position we're parsing. @@ -43,28 +43,24 @@ exports.Lexer = class Lexer @outdebt = 0 # The under-outdentation at the current level. @indents = [] # The stack of all current indentation levels. @tokens = [] # Stream of parsed tokens in the form ['TYPE', value, line] - while @i < @code.length - @chunk = @code[@i..] - @extractNextToken() + # At every position, run through this list of attempted matches, + # short-circuiting if any of them succeed. Their order determines precedence: + # `@literalToken` is the fallback catch-all. + while (@chunk = code[@i..]) + @identifierToken() or + @commentToken() or + @whitespaceToken() or + @lineToken() or + @heredocToken() or + @stringToken() or + @numberToken() or + @regexToken() or + @jsToken() or + @literalToken() @closeIndentation() return @tokens if o.rewrite is off (new Rewriter).rewrite @tokens - # At every position, run through this list of attempted matches, - # short-circuiting if any of them succeed. Their order determines precedence: - # `@literalToken` is the fallback catch-all. - extractNextToken: -> - return if @identifierToken() - return if @commentToken() - return if @whitespaceToken() - return if @lineToken() - return if @heredocToken() - return if @stringToken() - return if @numberToken() - return if @regexToken() - return if @jsToken() - return @literalToken() - # Tokenizers # ---------- @@ -75,37 +71,47 @@ exports.Lexer = class Lexer # referenced as property names here, so you can still do `jQuery.is()` even # though `is` means `===` otherwise. identifierToken: -> - return false unless id = @match IDENTIFIER, 1 + return false unless match = IDENTIFIER.exec @chunk + id = match[0] @i += id.length - forcedIdentifier = @tagAccessor() or @match ASSIGNED, 1 + if id is 'all' and @tag() is 'FOR' + @token 'ALL', id + return true + forcedIdentifier = @tagAccessor() or ASSIGNED.test @chunk tag = 'IDENTIFIER' - tag = id.toUpperCase() if include(JS_KEYWORDS, id) or (not forcedIdentifier and include(COFFEE_KEYWORDS, id)) - tag = 'LEADING_WHEN' if tag is 'WHEN' and include LINE_BREAK, @tag() - tag = 'ALL' if id is 'all' and @tag() is 'FOR' - tag = 'UNARY' if include UNARY, tag - if include(JS_FORBIDDEN, id) + if include(JS_KEYWORDS, id) or + not forcedIdentifier and include(COFFEE_KEYWORDS, id) + tag = id.toUpperCase() + if tag is 'WHEN' and include LINE_BREAK, @tag() + tag = 'LEADING_WHEN' + else if include UNARY, tag + tag = 'UNARY' + if include JS_FORBIDDEN, id if forcedIdentifier tag = 'STRING' id = "\"#{id}\"" if forcedIdentifier is 'accessor' - close_index = true + closeIndex = on @tokens.pop() if @tag() isnt '@' @token 'INDEX_START', '[' else if include(RESERVED, id) @identifierError id unless forcedIdentifier tag = id = CONVERSIONS[id] if include COFFEE_ALIASES, id - tag = 'LOGIC' if include LOGIC, id - tag = 'UNARY' if id is '!' + if id is '!' + tag = 'UNARY' + else if include LOGIC, id + tag = 'LOGIC' @token tag, id - @token ']', ']' if close_index + @token ']', ']' if closeIndex true # Matches numbers, including decimals, hex, and exponential notation. # Be careful not to interfere with ranges-in-progress. numberToken: -> - return false unless number = @match NUMBER, 1 - return false if @tag() is '.' and starts number, '.' + return false unless match = NUMBER.exec @chunk + number = match[0] + return false if @tag() is '.' and number.charAt(0) is '.' @i += number.length @token 'NUMBER', number true @@ -113,12 +119,16 @@ exports.Lexer = class Lexer # Matches strings, including multi-line strings. Ensures that quotation marks # are balanced within the string's contents, and within nested interpolations. stringToken: -> - return false unless starts(@chunk, '"') or starts(@chunk, "'") - return false unless string = - @balancedToken(['"', '"'], ['#{', '}']) or - @balancedToken ["'", "'"] - @interpolateString string.replace /\n/g, '\\\n' - @line += count string, "\n" + switch @chunk.charAt 0 + when "'" + return false unless match = SIMPLESTR.exec @chunk + @token 'STRING', (string = match[0]).replace MULTILINER, '\\\n' + when '"' + return false unless string = @balancedToken ['"', '"'], ['#{', '}'] + @interpolateString string + else + return false + @line += count string, '\n' @i += string.length true @@ -126,29 +136,30 @@ exports.Lexer = class Lexer # preserve whitespace, but ignore indentation to the left. heredocToken: -> return false unless match = @chunk.match HEREDOC - quote = match[1].substr 0, 1 - doc = @sanitizeHeredoc match[2] or match[4] or '', {quote} + heredoc = match[0] + quote = heredoc.charAt 0 + doc = @sanitizeHeredoc match[2], {quote, indent: null} @interpolateString quote + doc + quote, heredoc: yes - @line += count match[1], "\n" - @i += match[1].length + @line += count heredoc, '\n' + @i += heredoc.length true # Matches and consumes comments. commentToken: -> - return false unless match = @chunk.match(COMMENT) - @line += count match[1], "\n" - @i += match[1].length - if match[2] - @token 'HERECOMMENT', @sanitizeHeredoc match[2], + return false unless match = @chunk.match COMMENT + [comment, here] = match + @line += count comment, '\n' + @i += comment.length + if here + @token 'HERECOMMENT', @sanitizeHeredoc here, herecomment: true, indent: Array(@indent + 1).join(' ') @token 'TERMINATOR', '\n' true # Matches JavaScript interpolated directly into the source via backticks. jsToken: -> - return false unless starts @chunk, '`' - return false unless script = @balancedToken ['`', '`'] - @token 'JS', script.replace JS_CLEANER, '' + return false unless @chunk.charAt(0) is '`' and match = JSTOKEN.exec @chunk + @token 'JS', (script = match[0]).slice 1, -1 @i += script.length true @@ -161,21 +172,21 @@ exports.Lexer = class Lexer return false if first[1] is ' ' and @tag() not in ['CALL_START', '='] return false if include NOT_REGEX, @tag() return false unless regex = @balancedToken ['/', '/'] - return false unless end = @chunk.substr(regex.length).match REGEX_END - regex += flags = end[2] if end[2] - if regex.match REGEX_INTERPOLATION - str = regex.substring(1).split('/')[0] - str = str.replace REGEX_ESCAPE, (escaped) -> '\\' + escaped - @tokens = @tokens.concat [['(', '('], ['NEW', 'new'], ['IDENTIFIER', 'RegExp'], ['CALL_START', '(']] + return false unless end = @chunk[regex.length..].match REGEX_END + flags = end[0] + if REGEX_INTERPOLATION.test regex + str = regex.slice 1, -1 + str = str.replace REGEX_ESCAPE, '\\$&' + @tokens.push ['(', '('], ['NEW', 'new'], ['IDENTIFIER', 'RegExp'], ['CALL_START', '('] @interpolateString "\"#{str}\"", escapeQuotes: yes - @tokens.splice @tokens.length, 0, [',', ','], ['STRING', "\"#{flags}\""] if flags - @tokens.splice @tokens.length, 0, [')', ')'], [')', ')'] + @tokens.push [',', ','], ['STRING', "\"#{flags}\""] if flags + @tokens.push [')', ')'], [')', ')'] else - @token 'REGEX', regex - @i += regex.length + @token 'REGEX', regex + flags + @i += regex.length + flags.length true - # Matches a token in which which the passed delimiter pairs must be correctly + # Matches a token in which the passed delimiter pairs must be correctly # balanced (ie. strings, JS literals). balancedToken: (delimited...) -> @balancedString @chunk, delimited @@ -191,13 +202,14 @@ exports.Lexer = class Lexer # Keeps track of the level of indentation, because a single outdent token # can close multiple indents, so we need to know how far in we happen to be. lineToken: -> - return false unless indent = @match MULTI_DENT, 1 - @line += count indent, "\n" + return false unless match = MULTI_DENT.exec @chunk + indent = match[0] + @line += count indent, '\n' @i += indent.length - prev = @prev(2) - size = indent.match(LAST_DENTS).reverse()[0].match(LAST_DENT)[1].length - nextCharacter = @match NEXT_CHARACTER, 1 - noNewlines = nextCharacter is '.' or nextCharacter is ',' or @unfinished() + prev = @prev 2 + size = indent.length - 1 - indent.lastIndexOf '\n' + nextCharacter = NEXT_CHARACTER.exec(@chunk)[1] + noNewlines = (nextCharacter in ['.', ',']) or @unfinished() if size - @indebt is @indent return @suppressNewlines() if noNewlines return @newlineToken indent @@ -235,27 +247,27 @@ exports.Lexer = class Lexer @outdebt = 0 @token 'OUTDENT', dent @outdebt -= moveOut if dent - @token 'TERMINATOR', "\n" unless @tag() is 'TERMINATOR' or noNewlines + @token 'TERMINATOR', '\n' unless @tag() is 'TERMINATOR' or noNewlines true # Matches and consumes non-meaningful whitespace. Tag the previous token # as being "spaced", because there are some cases where it makes a difference. whitespaceToken: -> - return false unless space = @match WHITESPACE, 1 + return false unless match = WHITESPACE.exec @chunk prev = @prev() prev.spaced = true if prev - @i += space.length + @i += match[0].length true # Generate a newline token. Consecutive newlines get merged together. newlineToken: (newlines) -> - @token 'TERMINATOR', "\n" unless @tag() is 'TERMINATOR' + @token 'TERMINATOR', '\n' unless @tag() is 'TERMINATOR' true # Use a `\` at a line-ending to suppress the newline. # The slash is removed here once its job is done. suppressNewlines: -> - @tokens.pop() if @value() is "\\" + @tokens.pop() if @value() is '\\' true # We treat all other single characters as a token. Eg.: `( ) , . !` @@ -264,11 +276,11 @@ exports.Lexer = class Lexer # here. `;` and newlines are both treated as a `TERMINATOR`, we distinguish # parentheses that indicate a method call from regular parentheses, and so on. literalToken: -> - match = @chunk.match OPERATOR - value = match and match[1] - space = match and match[2] - @tagParameters() if value and value.match CODE - value or= @chunk.substr 0, 1 + if match = @chunk.match OPERATOR + [value, space] = match + @tagParameters() if CODE.test value + else + value = @chunk.charAt 0 @i += value.length spaced = (prev = @prev()) and prev.spaced tag = value @@ -290,8 +302,9 @@ exports.Lexer = class Lexer tag = 'CALL_START' else if value is '[' tag = 'INDEX_START' - @tag 1, 'INDEX_SOAK' if @tag() is '?' - @tag 1, 'INDEX_PROTO' if @tag() is '::' + switch @tag() + when '?' then @tag 1, 'INDEX_SOAK' + when '::' then @tag 1, 'INDEX_PROTO' @token tag, value true @@ -305,7 +318,7 @@ exports.Lexer = class Lexer return false if (not prev = @prev()) or (prev and prev.spaced) accessor = if prev[1] is '::' @tag 1, 'PROTOTYPE_ACCESS' - else if prev[1] is '.' and not (@value(2) is '.') + else if prev[1] is '.' and @value(2) isnt '.' if @tag(2) is '?' @tag(1, 'SOAK_ACCESS') @tokens.splice(-2, 1) @@ -318,18 +331,17 @@ exports.Lexer = class Lexer # Sanitize a heredoc or herecomment by escaping internal double quotes and # erasing all external indentation on the left-hand side. sanitizeHeredoc: (doc, options) -> - indent = options.indent - return doc if options.herecomment and not include doc, '\n' - unless options.herecomment - while (match = HEREDOC_INDENT.exec(doc)) isnt null - attempt = if match[2]? then match[2] else match[3] - indent = attempt if not indent? or 0 < attempt.length < indent.length - indent or= '' - doc = doc.replace(new RegExp("^" + indent, 'gm'), '') - return doc if options.herecomment - doc = doc.replace(/^\n/, '') - doc.replace(MULTILINER, "\\n") - .replace(new RegExp(options.quote, 'g'), "\\#{options.quote}") + {indent, herecomment} = options + return doc if herecomment and not include doc, '\n' + unless herecomment + while (match = HEREDOC_INDENT.exec doc) + attempt = match[1] + indent = attempt if indent is null or 0 < attempt.length < indent.length + doc = doc.replace /\n#{ indent }/g, '\n' if indent + return doc if herecomment + doc = doc.replace(/^\n/, '').replace(/#{ options.quote }/g, '\\$&') + doc = @oldline doc, on if options.quote is "'" + doc # A source of ambiguity in our grammar used to be parameter lists in function # definitions versus argument lists in function calls. Walk backwards, tagging @@ -370,8 +382,9 @@ exports.Lexer = class Lexer slash = delimited[0][0] is '/' levels = [] i = 0 - while i < str.length - if levels.length and starts str, '\\', i + slen = str.length + while i < slen + if levels.length and str.charAt(i) is '\\' i += 1 else for pair in delimited @@ -385,15 +398,15 @@ exports.Lexer = class Lexer levels.push(pair) i += open.length - 1 break - break if not levels.length or slash and starts str, '\n', i + break if not levels.length or slash and str.charAt(i) is '\n' i += 1 if levels.length return false if slash throw new Error "SyntaxError: Unterminated #{levels.pop()[0]} starting on line #{@line + 1}" - if not i then false else str.substring(0, i) + if not i then false else str[0...i] # Expand variables and expressions inside double-quoted strings using - # [ECMA Harmony's interpolation syntax](http://wiki.ecmascript.org/doku.php?id=strawman:string_interpolation) + # Ruby-like notation # for substitution of bare variables as well as arbitrary expressions. # # "Hello #{name.capitalize()}." @@ -402,47 +415,51 @@ exports.Lexer = class Lexer # new Lexer, tokenize the interpolated contents, and merge them into the # token stream. interpolateString: (str, options) -> - options or= {} - if str.length < 3 or not starts str, '"' - @token 'STRING', str - else - lexer = new Lexer - tokens = [] - quote = str.substring 0, 1 - [i, pi] = [1, 1] - while i < str.length - 1 - if starts str, '\\', i - i += 1 - else if expr = @balancedString(str.substring(i), [['#{', '}']]) - tokens.push ['STRING', quote + str.substring(pi, i) + quote] if pi < i - inner = expr.substring(2, expr.length - 1) - if inner.length - inner = inner.replace new RegExp('\\\\' + quote, 'g'), quote if options.heredoc - nested = lexer.tokenize "(#{inner})", line: @line - (tok[0] = ')') for tok, idx in nested when tok[0] is 'CALL_END' - nested.pop() - tokens.push ['TOKENS', nested] - else - tokens.push ['STRING', quote + quote] - i += expr.length - 1 - pi = i + 1 + {heredoc, escapeQuotes} = options or {} + quote = str.charAt 0 + return @token 'STRING', str if quote isnt '"' or str.length < 3 + lexer = new Lexer + tokens = [] + i = pi = 1 + end = str.length - 1 + while i < end + if str.charAt(i) is '\\' i += 1 - tokens.push ['STRING', quote + str.substring(pi, i) + quote] if pi < i and pi < str.length - 1 - tokens.unshift ['STRING', '""'] unless tokens[0][0] is 'STRING' - interpolated = tokens.length > 1 - @token '(', '(' if interpolated - for token, i in tokens - [tag, value] = token - if tag is 'TOKENS' - @tokens = @tokens.concat value - else if tag is 'STRING' and options.escapeQuotes - escaped = value.substring(1, value.length - 1).replace(/"/g, '\\"') - @token tag, "\"#{escaped}\"" + else if expr = @balancedString str[i..], [['#{', '}']] + if pi < i + s = quote + @oldline(str[pi...i], heredoc) + quote + tokens.push ['STRING', s] + inner = expr.slice(2, -1).replace /^[ \t]*\n/, '' + if inner.length + inner = inner.replace RegExp('\\\\' + quote, 'g'), quote if heredoc + nested = lexer.tokenize "(#{inner})", line: @line + (tok[0] = ')') for tok, idx in nested when tok[0] is 'CALL_END' + nested.pop() + tokens.push ['TOKENS', nested] else - @token tag, value - @token '+', '+' if i < tokens.length - 1 - @token ')', ')' if interpolated - tokens + tokens.push ['STRING', quote + quote] + i += expr.length - 1 + pi = i + 1 + i += 1 + if i > pi < str.length - 1 + s = str[pi...i].replace MULTILINER, if heredoc then '\\n' else '' + tokens.push ['STRING', quote + s + quote] + tokens.unshift ['STRING', '""'] unless tokens[0][0] is 'STRING' + interpolated = tokens.length > 1 + @token '(', '(' if interpolated + {push} = tokens + for token, i in tokens + [tag, value] = token + if tag is 'TOKENS' + push.apply @tokens, value + else if tag is 'STRING' and escapeQuotes + escaped = value.slice(1, -1).replace(/"/g, '\\"') + @token tag, "\"#{escaped}\"" + else + @token tag, value + @token '+', '+' if i < tokens.length - 1 + @token ')', ')' if interpolated + tokens # Helpers # ------- @@ -467,51 +484,48 @@ exports.Lexer = class Lexer prev: (index) -> @tokens[@tokens.length - (index or 1)] - # Attempt to match a string against the current chunk, returning the indexed - # match if successful, and `false` otherwise. - match: (regex, index) -> - return false unless m = @chunk.match regex - if m then m[index] else false - # Are we in the midst of an unfinished expression? unfinished: -> - prev = @prev(2) - @value() and @value().match and @value().match(NO_NEWLINE) and - prev and (prev[0] isnt '.') and not @value().match(CODE) and - not @chunk.match ASSIGNED + (prev = @prev 2 ) and prev[0] isnt '.' and + (value = @value()) and NO_NEWLINE.test(value) and not CODE.test(value) and + not ASSIGNED.test(@chunk) + + # Converts newlines for string literals + oldline: (str, heredoc) -> + str.replace MULTILINER, if heredoc then '\\n' else '' # Constants # --------- # Keywords that CoffeeScript shares in common with JavaScript. JS_KEYWORDS = [ - "if", "else", - "true", "false", - "new", "return", - "try", "catch", "finally", "throw", - "break", "continue", - "for", "in", "while", - "delete", "instanceof", "typeof", - "switch", "super", "extends", "class", - "this", "null", "debugger" + 'if', 'else' + 'true', 'false' + 'new', 'return' + 'try', 'catch', 'finally', 'throw' + 'break', 'continue' + 'for', 'in', 'while' + 'delete', 'instanceof', 'typeof' + 'switch', 'super', 'extends', 'class' + 'this', 'null', 'debugger' ] # CoffeeScript-only keywords, which we're more relaxed about allowing. They can't # be used standalone, but you can reference them as an attached property. -COFFEE_ALIASES = ["and", "or", "is", "isnt", "not"] +COFFEE_ALIASES = ['and', 'or', 'is', 'isnt', 'not'] COFFEE_KEYWORDS = COFFEE_ALIASES.concat [ - "then", "unless", "until", "loop", - "yes", "no", "on", "off", - "of", "by", "where", "when" + 'then', 'unless', 'until', 'loop' + 'yes', 'no', 'on', 'off' + 'of', 'by', 'where', 'when' ] # The list of keywords that are reserved by JavaScript, but not used, or are # used by CoffeeScript internally. We throw an error when these are encountered, # to avoid having a JavaScript error at runtime. RESERVED = [ - "case", "default", "do", "function", "var", "void", "with", - "const", "let", "enum", "export", "import", "native", - "__hasProp", "__extends", "__slice" + 'case', 'default', 'do', 'function', 'var', 'void', 'with' + 'const', 'let', 'enum', 'export', 'import', 'native' + '__hasProp', '__extends', '__slice' ] # The superset of both JavaScript keywords and reserved words, none of which may @@ -519,30 +533,29 @@ RESERVED = [ JS_FORBIDDEN = JS_KEYWORDS.concat RESERVED # Token matching regexes. -IDENTIFIER = /^([a-zA-Z\$_](\w|\$)*)/ -NUMBER = /^(((\b0(x|X)[0-9a-fA-F]+)|((\b[0-9]+(\.[0-9]+)?|\.[0-9]+)(e[+\-]?[0-9]+)?)))\b/i -HEREDOC = /^("{6}|'{6}|"{3}([\s\S]*?)\n?([ \t]*)"{3}|'{3}([\s\S]*?)\n?([ \t]*)'{3})/ -OPERATOR = /^(-[\-=>]?|\+[+=]?|[*&|\/%=<>^:!?]+)([ \t]*)/ -WHITESPACE = /^([ \t]+)/ -COMMENT = /^(###([^#][\s\S]*?)(###[ \t]*\n|(###)?$)|(\s*#(?!##[^#])[^\n]*)+)/ -CODE = /^((-|=)>)/ -MULTI_DENT = /^((\n([ \t]*))+)(\.)?/ -LAST_DENTS = /\n([ \t]*)/g -LAST_DENT = /\n([ \t]*)/ +IDENTIFIER = /^[a-zA-Z_$][\w$]*/ +NUMBER = /^0x[\da-f]+|^(?:\d+(\.\d+)?|\.\d+)(?:e[+-]?\d+)?/i +HEREDOC = /^("""|''')([\s\S]*?)\n?[ \t]*\1/ +OPERATOR = /^(?:-[-=>]?|\+[+=]?|[*&|\/%=<>^:!?]+)(?=([ \t]*))/ +WHITESPACE = /^[ \t]+/ +COMMENT = /^###([^#][\s\S]*?)(?:###[ \t]*\n|(?:###)?$)|^(?:\s*#(?!##[^#]).*)+/ +CODE = /^[-=]>/ +MULTI_DENT = /^(?:\n[ \t]*)+/ +SIMPLESTR = /^'[^\\']*(?:\\.[^\\']*)*'/ +JSTOKEN = /^`[^\\`]*(?:\\.[^\\`]*)*`/ # Regex-matching-regexes. REGEX_START = /^\/([^\/])/ -REGEX_INTERPOLATION = /([^\\]#\{.*[^\\]\})/ -REGEX_END = /^(([imgy]{1,4})\b|\W|$)/ -REGEX_ESCAPE = /\\[^\$]/g +REGEX_INTERPOLATION = /[^\\]#\{.*[^\\]\}/ +REGEX_END = /^[imgy]{0,4}(?![a-zA-Z])/ +REGEX_ESCAPE = /\\[^#]/g # Token cleaning regexes. -JS_CLEANER = /(^`|`$)/g MULTILINER = /\n/g -NO_NEWLINE = /^([+\*&|\/\-%=<>!.\\][<>=&|]*|and|or|is|isnt|not|delete|typeof|instanceof)$/ -HEREDOC_INDENT = /(\n+([ \t]*)|^([ \t]+))/g -ASSIGNED = /^\s*(([a-zA-Z\$_@]\w*|["'][^\r\n]+?["']|\d+)[ \t]*?[:=][^:=])/ -NEXT_CHARACTER = /^\s*(\S)/ +NO_NEWLINE = /^(?:[-+*&|\/%=<>!.\\][<>=&|]*|and|or|is(?:nt)?|not|delete|typeof|instanceof)$/ +HEREDOC_INDENT = /\n+([ \t]*)/g +ASSIGNED = /^\s*@?[$A-Za-z_][$\w]*[ \t]*?[:=][^:=>]/ +NEXT_CHARACTER = /^\s*(\S?)/ # Compound assignment tokens. COMPOUND_ASSIGN = ['-=', '+=', '/=', '*=', '%=', '||=', '&&=', '?=', '<<=', '>>=', '>>>=', '&=', '^=', '|='] diff --git a/test/test_heredocs.coffee b/test/test_heredocs.coffee index 8bf2ae98..5c727483 100644 --- a/test/test_heredocs.coffee +++ b/test/test_heredocs.coffee @@ -88,3 +88,21 @@ a = """ """ ok a is "one\ntwo\n" + + +equal ''' line 0 + should not be relevant + to the indent level +''', ' + line 0\n +should not be relevant\n + to the indent level +' + + +equal 'multiline nested interpolations work', """multiline #{ + "nested #{(-> + ok yes + "interpolations" + )()}" +} work""" diff --git a/test/test_literals.coffee b/test/test_literals.coffee index fd6d479d..b8b2bb72 100644 --- a/test/test_literals.coffee +++ b/test/test_literals.coffee @@ -239,3 +239,9 @@ b = identity identity identity a: 100 ok b is 100 + + +# Inline JS +ok '\\`' is ` + "\\\`" +`