From 3df82a757d5cb4ebcf26515853d75fb002f8915a Mon Sep 17 00:00:00 2001 From: satyr Date: Sat, 25 Sep 2010 09:15:47 +0900 Subject: [PATCH 01/10] helpers: added test --- test/test_helpers.coffee | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 test/test_helpers.coffee diff --git a/test/test_helpers.coffee b/test/test_helpers.coffee new file mode 100644 index 00000000..1169a4b6 --- /dev/null +++ b/test/test_helpers.coffee @@ -0,0 +1,40 @@ +{ indexOf, include, starts, ends, compact, count, merge, extend, flatten, del +} = require('../lib/helpers').helpers + +array = [0..4] + +ok indexOf(array, 0) is 0 +ok indexOf(array, 2) is 2 +ok indexOf(array, 4) is 4 +ok indexOf(array, 6) is -1 + +ok include array, 0 +ok include array, 2 +ok include array, 4 +ok not include array, 6 + +string = array.join '' + +ok starts string, '012' +ok starts string, '34', 3 +ok not starts string, '42' +ok not starts string, '42', 6 + +ok ends string, '234' +ok ends string, '01', 3 +ok not ends string, '42' +ok not ends string, '42', 6 + +object = {} +merged = merge object, array + +ok merged isnt object +ok merged[3] is 3 + +ok object is extend object, array +ok object[3] is 3 + +ok "#{ flatten [0, [1, 2], 3, [4]] }" is "#{ array }" + +ok 1 is del object, 1 +ok 1 not of object From 9005682cf1ac1ffa9169c1bc8d478f97e8e49ecd Mon Sep 17 00:00:00 2001 From: satyr Date: Sat, 25 Sep 2010 09:18:47 +0900 Subject: [PATCH 02/10] helpers: refactored and fixed comments --- lib/helpers.js | 100 ++++++++++++++++----------------------------- src/helpers.coffee | 65 +++++++++++++---------------- 2 files changed, 65 insertions(+), 100 deletions(-) diff --git a/lib/helpers.js b/lib/helpers.js index 0360fb91..fe8c7f1b 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -1,11 +1,10 @@ (function() { - var compact, count, del, ends, extend, flatten, helpers, include, indexOf, merge, starts; + var extend, helpers, indexOf; helpers = (exports.helpers = {}); - helpers.indexOf = (indexOf = function(array, item, from) { + indexOf = (helpers.indexOf = Array.indexOf || (Array.prototype.indexOf ? function(array, item, from) { + return array.indexOf(item, from); + } : function(array, item, from) { var _len, _ref, index, other; - if (array.indexOf) { - return array.indexOf(item, from); - } _ref = array; for (index = 0, _len = _ref.length; index < _len; index++) { other = _ref[index]; @@ -14,19 +13,19 @@ } } return -1; - }); - helpers.include = (include = function(list, value) { - return indexOf(list, value) >= 0; - }); - helpers.starts = (starts = function(string, literal, start) { - return string.substring(start, (start || 0) + literal.length) === literal; - }); - helpers.ends = (ends = function(string, literal, back) { - var start; - start = string.length - literal.length - ((typeof back !== "undefined" && back !== null) ? back : 0); - return string.substring(start, start + literal.length) === literal; - }); - helpers.compact = (compact = function(array) { + })); + helpers.include = function(list, value) { + return 0 <= indexOf(list, value); + }; + helpers.starts = function(string, literal, start) { + return literal === string.substr(start, literal.length); + }; + helpers.ends = function(string, literal, back) { + var ll; + ll = literal.length; + return literal === string.substr(string.length - ll - (back || 0), ll); + }; + helpers.compact = function(array) { var _i, _len, _ref, _result, item; _result = []; _ref = array; for (_i = 0, _len = _ref.length; _i < _len; _i++) { @@ -36,61 +35,34 @@ } } return _result; - }); - helpers.count = (count = function(string, letter) { + }; + helpers.count = function(string, letter) { var num, pos; - num = 0; - pos = indexOf(string, letter); - while (pos !== -1) { - num += 1; - pos = indexOf(string, letter, pos + 1); + num = (pos = 0); + while (0 < (pos = 1 + string.indexOf(letter, pos))) { + num++; } return num; - }); - helpers.merge = (merge = function(options, overrides) { - var _ref, fresh, key, val; - fresh = {}; - _ref = options; + }; + helpers.merge = function(options, overrides) { + return extend(extend({}, options), overrides); + }; + extend = (helpers.extend = function(object, properties) { + var _ref, key, val; + _ref = properties; for (key in _ref) { val = _ref[key]; - (fresh[key] = val); + (object[key] = val); } - if (overrides) { - _ref = overrides; - for (key in _ref) { - val = _ref[key]; - (fresh[key] = val); - } - } - return fresh; + return object; }); - helpers.extend = (extend = function(object, properties) { - var _ref, _result, key, val; - _result = []; _ref = properties; - for (key in _ref) { - val = _ref[key]; - _result.push(object[key] = val); - } - return _result; - }); - helpers.flatten = (flatten = function(array) { - var _i, _len, _ref, item, memo; - memo = []; - _ref = array; - for (_i = 0, _len = _ref.length; _i < _len; _i++) { - item = _ref[_i]; - if (item instanceof Array) { - memo = memo.concat(item); - } else { - memo.push(item); - } - } - return memo; - }); - helpers.del = (del = function(obj, key) { + helpers.flatten = function(array) { + return array.concat.apply([], array); + }; + helpers.del = function(obj, key) { var val; val = obj[key]; delete obj[key]; return val; - }); + }; }).call(this); diff --git a/src/helpers.coffee b/src/helpers.coffee index 151521a1..f1247174 100644 --- a/src/helpers.coffee +++ b/src/helpers.coffee @@ -4,64 +4,57 @@ helpers = exports.helpers = {} -# Cross-browser indexOf, so that IE can join the party. -helpers.indexOf = indexOf = (array, item, from) -> - return array.indexOf item, from if array.indexOf - for other, index in array - if other is item and (not from or (from <= index)) - return index - -1 +# Cross-engine indexOf, so that JScript can join the party. +indexOf = helpers.indexOf = Array.indexOf or + if Array::indexOf + (array, item, from) -> array.indexOf item, from + else + (array, item, from) -> + for other, index in array + if other is item and (not from or from <= index) + return index + -1 # Does a list include a value? -helpers.include = include = (list, value) -> - indexOf(list, value) >= 0 +helpers.include = (list, value) -> 0 <= indexOf list, value # Peek at the beginning of a given string to see if it matches a sequence. -helpers.starts = starts = (string, literal, start) -> - string.substring(start, (start or 0) + literal.length) is literal +helpers.starts = (string, literal, start) -> + literal is string.substr start, literal.length # Peek at the end of a given string to see if it matches a sequence. -helpers.ends = ends = (string, literal, back) -> - start = string.length - literal.length - (back ? 0) - string.substring(start, start + literal.length) is literal +helpers.ends = (string, literal, back) -> + ll = literal.length + literal is string.substr string.length - ll - (back or 0), ll # Trim out all falsy values from an array. -helpers.compact = compact = (array) -> item for item in array when item +helpers.compact = (array) -> item for item in array when item # Count the number of occurences of a character in a string. -helpers.count = count = (string, letter) -> - num = 0 - pos = indexOf string, letter - while pos isnt -1 - num += 1 - pos = indexOf string, letter, pos + 1 +helpers.count = (string, letter) -> + num = pos = 0 + num++ while 0 < pos = 1 + string.indexOf letter, pos num # Merge objects, returning a fresh copy with attributes from both sides. # Used every time `BaseNode#compile` is called, to allow properties in the # options hash to propagate down the tree without polluting other branches. -helpers.merge = merge = (options, overrides) -> - fresh = {} - (fresh[key] = val) for all key, val of options - (fresh[key] = val) for all key, val of overrides if overrides - fresh +helpers.merge = (options, overrides) -> + extend (extend {}, options), overrides # Extend a source object with the properties of another object (shallow copy). # We use this to simulate Node's deprecated `process.mixin` -helpers.extend = extend = (object, properties) -> +extend = helpers.extend = (object, properties) -> (object[key] = val) for all key, val of properties + object -# Return a completely flattened version of an array. Handy for getting a -# list of `children` from the nodes. -helpers.flatten = flatten = (array) -> - memo = [] - for item in array - if item instanceof Array then memo = memo.concat(item) else memo.push(item) - memo +# Return a flattened version of an array (nonrecursive). +# Handy for getting a list of `children` from the nodes. +helpers.flatten = (array) -> array.concat.apply [], array # Delete a key from an object, returning the value. Useful when a node is # looking for a particular method in an options hash. -helpers.del = del = (obj, key) -> - val = obj[key] +helpers.del = (obj, key) -> + val = obj[key] delete obj[key] val From e0ed2542525824081a6713556924658c61d04553 Mon Sep 17 00:00:00 2001 From: satyr Date: Sat, 25 Sep 2010 09:29:44 +0900 Subject: [PATCH 03/10] helpers: now directly exported --- Cakefile | 2 +- bin/cake | 6 +++--- bin/coffee | 6 +++--- lib/cake.js | 2 +- lib/command.js | 3 +-- lib/helpers.js | 23 +++++++++++------------ lib/lexer.js | 2 +- lib/nodes.js | 2 +- lib/repl.js | 2 +- lib/rewriter.js | 2 +- lib/scope.js | 2 +- src/cake.coffee | 2 +- src/command.coffee | 2 +- src/helpers.coffee | 22 ++++++++++------------ src/lexer.coffee | 2 +- src/nodes.coffee | 2 +- src/repl.coffee | 2 +- src/rewriter.coffee | 2 +- src/scope.coffee | 2 +- test/test_helpers.coffee | 2 +- 20 files changed, 43 insertions(+), 47 deletions(-) diff --git a/Cakefile b/Cakefile index 10f1ef69..5e5a7073 100644 --- a/Cakefile +++ b/Cakefile @@ -1,5 +1,5 @@ fs = require 'fs' -{helpers} = require './lib/helpers' +helpers = require './lib/helpers' CoffeeScript = require './lib/coffee-script' {spawn, exec} = require 'child_process' path = require 'path' diff --git a/bin/cake b/bin/cake index 40ac7380..aa7475a6 100755 --- a/bin/cake +++ b/bin/cake @@ -1,8 +1,8 @@ #!/usr/bin/env node var path = require('path'); -var fs = require('fs'); -var lib = path.join(path.dirname(fs.realpathSync(__filename)), '../lib'); +var fs = require('fs'); +var lib = path.join(path.dirname(fs.realpathSync(__filename)), '../lib'); -require(lib + '/helpers').helpers.extend(global, require('sys')); +require(lib + '/helpers').extend(global, require('sys')); require(lib + '/cake').run(); diff --git a/bin/coffee b/bin/coffee index 4d2dd38f..c6dc359a 100755 --- a/bin/coffee +++ b/bin/coffee @@ -1,8 +1,8 @@ #!/usr/bin/env node var path = require('path'); -var fs = require('fs'); -var lib = path.join(path.dirname(fs.realpathSync(__filename)), '../lib'); +var fs = require('fs'); +var lib = path.join(path.dirname(fs.realpathSync(__filename)), '../lib'); -require(lib + '/helpers').helpers.extend(global, require('sys')); +require(lib + '/helpers').extend(global, require('sys')); require(lib + '/command').run(); diff --git a/lib/cake.js b/lib/cake.js index 43930c32..85e6f35b 100755 --- a/lib/cake.js +++ b/lib/cake.js @@ -2,7 +2,7 @@ var CoffeeScript, fs, helpers, missingTask, oparse, options, optparse, path, printTasks, switches, tasks; fs = require('fs'); path = require('path'); - helpers = require('./helpers').helpers; + helpers = require('./helpers'); optparse = require('./optparse'); CoffeeScript = require('./coffee-script'); tasks = {}; diff --git a/lib/command.js b/lib/command.js index ed44defa..8a6940ff 100644 --- a/lib/command.js +++ b/lib/command.js @@ -4,8 +4,7 @@ path = require('path'); optparse = require('./optparse'); CoffeeScript = require('./coffee-script'); - _ref = require('./helpers'); - helpers = _ref.helpers; + helpers = require('./helpers'); _ref = require('child_process'); spawn = _ref.spawn; exec = _ref.exec; diff --git a/lib/helpers.js b/lib/helpers.js index fe8c7f1b..49829b2f 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -1,7 +1,6 @@ (function() { - var extend, helpers, indexOf; - helpers = (exports.helpers = {}); - indexOf = (helpers.indexOf = Array.indexOf || (Array.prototype.indexOf ? function(array, item, from) { + var extend, indexOf; + indexOf = (exports.indexOf = Array.indexOf || (Array.prototype.indexOf ? function(array, item, from) { return array.indexOf(item, from); } : function(array, item, from) { var _len, _ref, index, other; @@ -14,18 +13,18 @@ } return -1; })); - helpers.include = function(list, value) { + exports.include = function(list, value) { return 0 <= indexOf(list, value); }; - helpers.starts = function(string, literal, start) { + exports.starts = function(string, literal, start) { return literal === string.substr(start, literal.length); }; - helpers.ends = function(string, literal, back) { + exports.ends = function(string, literal, back) { var ll; ll = literal.length; return literal === string.substr(string.length - ll - (back || 0), ll); }; - helpers.compact = function(array) { + exports.compact = function(array) { var _i, _len, _ref, _result, item; _result = []; _ref = array; for (_i = 0, _len = _ref.length; _i < _len; _i++) { @@ -36,7 +35,7 @@ } return _result; }; - helpers.count = function(string, letter) { + exports.count = function(string, letter) { var num, pos; num = (pos = 0); while (0 < (pos = 1 + string.indexOf(letter, pos))) { @@ -44,10 +43,10 @@ } return num; }; - helpers.merge = function(options, overrides) { + exports.merge = function(options, overrides) { return extend(extend({}, options), overrides); }; - extend = (helpers.extend = function(object, properties) { + extend = (exports.extend = function(object, properties) { var _ref, key, val; _ref = properties; for (key in _ref) { @@ -56,10 +55,10 @@ } return object; }); - helpers.flatten = function(array) { + exports.flatten = function(array) { return array.concat.apply([], array); }; - helpers.del = function(obj, key) { + exports.del = function(obj, key) { var val; val = obj[key]; delete obj[key]; diff --git a/lib/lexer.js b/lib/lexer.js index 2f453939..9bb42e68 100644 --- a/lib/lexer.js +++ b/lib/lexer.js @@ -3,7 +3,7 @@ var __slice = Array.prototype.slice; _ref = require('./rewriter'); Rewriter = _ref.Rewriter; - _ref = require('./helpers').helpers; + _ref = require('./helpers'); include = _ref.include; count = _ref.count; starts = _ref.starts; diff --git a/lib/nodes.js b/lib/nodes.js index 7d604ee5..61acf5a7 100644 --- a/lib/nodes.js +++ b/lib/nodes.js @@ -10,7 +10,7 @@ }; _ref = require('./scope'); Scope = _ref.Scope; - _ref = require('./helpers').helpers; + _ref = require('./helpers'); compact = _ref.compact; flatten = _ref.flatten; merge = _ref.merge; diff --git a/lib/repl.js b/lib/repl.js index 42bc3717..6e3ce536 100644 --- a/lib/repl.js +++ b/lib/repl.js @@ -1,7 +1,7 @@ (function() { var CoffeeScript, helpers, readline, repl, run, stdio; CoffeeScript = require('./coffee-script'); - helpers = require('./helpers').helpers; + helpers = require('./helpers'); readline = require('readline'); stdio = process.openStdin(); helpers.extend(global, { diff --git a/lib/rewriter.js b/lib/rewriter.js index f41a0e9a..98cafa76 100644 --- a/lib/rewriter.js +++ b/lib/rewriter.js @@ -1,7 +1,7 @@ (function() { var BALANCED_PAIRS, EXPRESSION_CLOSE, EXPRESSION_END, EXPRESSION_START, IMPLICIT_BLOCK, IMPLICIT_CALL, IMPLICIT_END, IMPLICIT_FUNC, INVERSES, LINEBREAKS, Rewriter, SINGLE_CLOSERS, SINGLE_LINERS, _i, _len, _ref, _result, include, pair; var __hasProp = Object.prototype.hasOwnProperty; - _ref = require('./helpers').helpers; + _ref = require('./helpers'); include = _ref.include; exports.Rewriter = (function() { Rewriter = function() {}; diff --git a/lib/scope.js b/lib/scope.js index c5915ae7..a3de4da7 100644 --- a/lib/scope.js +++ b/lib/scope.js @@ -1,7 +1,7 @@ (function() { var Scope, _ref, extend; var __hasProp = Object.prototype.hasOwnProperty; - _ref = require('./helpers').helpers; + _ref = require('./helpers'); extend = _ref.extend; exports.Scope = (function() { Scope = function(parent, expressions, method) { diff --git a/src/cake.coffee b/src/cake.coffee index 8f2c7f6b..fd093658 100644 --- a/src/cake.coffee +++ b/src/cake.coffee @@ -9,7 +9,7 @@ # External dependencies. fs = require 'fs' path = require 'path' -helpers = require('./helpers').helpers +helpers = require './helpers' optparse = require './optparse' CoffeeScript = require './coffee-script' diff --git a/src/command.coffee b/src/command.coffee index 223218b8..3465e48c 100644 --- a/src/command.coffee +++ b/src/command.coffee @@ -9,7 +9,7 @@ fs = require 'fs' path = require 'path' optparse = require './optparse' CoffeeScript = require './coffee-script' -{helpers} = require './helpers' +helpers = require './helpers' {spawn, exec} = require 'child_process' {EventEmitter} = require 'events' diff --git a/src/helpers.coffee b/src/helpers.coffee index f1247174..cd506d13 100644 --- a/src/helpers.coffee +++ b/src/helpers.coffee @@ -2,10 +2,8 @@ # the **Lexer**, **Rewriter**, and the **Nodes**. Merge objects, flatten # arrays, count characters, that sort of thing. -helpers = exports.helpers = {} - # Cross-engine indexOf, so that JScript can join the party. -indexOf = helpers.indexOf = Array.indexOf or +indexOf = exports.indexOf = Array.indexOf or if Array::indexOf (array, item, from) -> array.indexOf item, from else @@ -16,22 +14,22 @@ indexOf = helpers.indexOf = Array.indexOf or -1 # Does a list include a value? -helpers.include = (list, value) -> 0 <= indexOf list, value +exports.include = (list, value) -> 0 <= indexOf list, value # Peek at the beginning of a given string to see if it matches a sequence. -helpers.starts = (string, literal, start) -> +exports.starts = (string, literal, start) -> literal is string.substr start, literal.length # Peek at the end of a given string to see if it matches a sequence. -helpers.ends = (string, literal, back) -> +exports.ends = (string, literal, back) -> ll = literal.length literal is string.substr string.length - ll - (back or 0), ll # Trim out all falsy values from an array. -helpers.compact = (array) -> item for item in array when item +exports.compact = (array) -> item for item in array when item # Count the number of occurences of a character in a string. -helpers.count = (string, letter) -> +exports.count = (string, letter) -> num = pos = 0 num++ while 0 < pos = 1 + string.indexOf letter, pos num @@ -39,22 +37,22 @@ helpers.count = (string, letter) -> # Merge objects, returning a fresh copy with attributes from both sides. # Used every time `BaseNode#compile` is called, to allow properties in the # options hash to propagate down the tree without polluting other branches. -helpers.merge = (options, overrides) -> +exports.merge = (options, overrides) -> extend (extend {}, options), overrides # Extend a source object with the properties of another object (shallow copy). # We use this to simulate Node's deprecated `process.mixin` -extend = helpers.extend = (object, properties) -> +extend = exports.extend = (object, properties) -> (object[key] = val) for all key, val of properties object # Return a flattened version of an array (nonrecursive). # Handy for getting a list of `children` from the nodes. -helpers.flatten = (array) -> array.concat.apply [], array +exports.flatten = (array) -> array.concat.apply [], array # Delete a key from an object, returning the value. Useful when a node is # looking for a particular method in an options hash. -helpers.del = (obj, key) -> +exports.del = (obj, key) -> val = obj[key] delete obj[key] val diff --git a/src/lexer.coffee b/src/lexer.coffee index cd2215a0..02f7a433 100644 --- a/src/lexer.coffee +++ b/src/lexer.coffee @@ -10,7 +10,7 @@ {Rewriter} = require './rewriter' # Import the helpers we need. -{include, count, starts, compact} = require('./helpers').helpers +{include, count, starts, compact} = require './helpers' # The Lexer Class # --------------- diff --git a/src/nodes.coffee b/src/nodes.coffee index 941a446c..0031bb1d 100644 --- a/src/nodes.coffee +++ b/src/nodes.coffee @@ -6,7 +6,7 @@ {Scope} = require './scope' # Import the helpers we plan to use. -{compact, flatten, merge, del, include, indexOf, starts, ends} = require('./helpers').helpers +{compact, flatten, merge, del, include, indexOf, starts, ends} = require './helpers' #### BaseNode diff --git a/src/repl.coffee b/src/repl.coffee index f869da24..91dad9bb 100644 --- a/src/repl.coffee +++ b/src/repl.coffee @@ -6,7 +6,7 @@ # Require the **coffee-script** module to get access to the compiler. CoffeeScript = require './coffee-script' -helpers = require('./helpers').helpers +helpers = require './helpers' readline = require 'readline' # Start by opening up **stdio**. diff --git a/src/rewriter.coffee b/src/rewriter.coffee index b82f14d1..93e08807 100644 --- a/src/rewriter.coffee +++ b/src/rewriter.coffee @@ -6,7 +6,7 @@ # parentheses, balance incorrect nestings, and generally clean things up. # Import the helpers we need. -{include} = require('./helpers').helpers +{include} = require './helpers' # The **Rewriter** class is used by the [Lexer](lexer.html), directly against # its internal array of tokens. diff --git a/src/scope.coffee b/src/scope.coffee index 31f0c8b8..b4256911 100644 --- a/src/scope.coffee +++ b/src/scope.coffee @@ -6,7 +6,7 @@ # with the outside. # Import the helpers we plan to use. -{extend} = require('./helpers').helpers +{extend} = require './helpers' exports.Scope = class Scope diff --git a/test/test_helpers.coffee b/test/test_helpers.coffee index 1169a4b6..3f204c33 100644 --- a/test/test_helpers.coffee +++ b/test/test_helpers.coffee @@ -1,5 +1,5 @@ { indexOf, include, starts, ends, compact, count, merge, extend, flatten, del -} = require('../lib/helpers').helpers +} = require '../lib/helpers' array = [0..4] From c515aaac5a595fcc7567846ddaa607c2343aabe9 Mon Sep 17 00:00:00 2001 From: satyr Date: Sat, 25 Sep 2010 23:37:33 +0900 Subject: [PATCH 04/10] lexer: fixed ASSIGNED --- lib/lexer.js | 4 ++-- src/lexer.coffee | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/lexer.js b/lib/lexer.js index 22f1c388..896aa9f7 100644 --- a/lib/lexer.js +++ b/lib/lexer.js @@ -44,7 +44,7 @@ this.token('ALL', id); return true; } - forcedIdentifier = this.tagAccessor() || this.match(ASSIGNED, 1); + forcedIdentifier = this.tagAccessor() || ASSIGNED.test(this.chunk); tag = 'IDENTIFIER'; if (include(JS_KEYWORDS, id) || !forcedIdentifier && include(COFFEE_KEYWORDS, id)) { tag = id.toUpperCase(); @@ -607,7 +607,7 @@ MULTILINER = /\n/g; NO_NEWLINE = /^(?:[-+*&|\/%=<>!.\\][<>=&|]*|and|or|is(?:nt)?|not|delete|typeof|instanceof)$/; HEREDOC_INDENT = /\n+([ \t]*)|^([ \t]+)/g; - ASSIGNED = /^\s*((?:[a-zA-Z$_@]\w*|["'][^\n]+?["']|\d+)[ \t]*?[:=][^:=])/; + ASSIGNED = /^\s*@?[$A-Za-z_][$\w]*[ \t]*?[:=][^:=>]/; NEXT_CHARACTER = /^\s*(\S)/; COMPOUND_ASSIGN = ['-=', '+=', '/=', '*=', '%=', '||=', '&&=', '?=', '<<=', '>>=', '>>>=', '&=', '^=', '|=']; UNARY = ['UMINUS', 'UPLUS', '!', '!!', '~', 'TYPEOF', 'DELETE']; diff --git a/src/lexer.coffee b/src/lexer.coffee index 0cc35db4..ba827679 100644 --- a/src/lexer.coffee +++ b/src/lexer.coffee @@ -79,7 +79,7 @@ exports.Lexer = class Lexer if id is 'all' and @tag() is 'FOR' @token 'ALL', id return true - forcedIdentifier = @tagAccessor() or @match ASSIGNED, 1 + forcedIdentifier = @tagAccessor() or ASSIGNED.test @chunk tag = 'IDENTIFIER' if include(JS_KEYWORDS, id) or not forcedIdentifier and include(COFFEE_KEYWORDS, id) @@ -555,7 +555,7 @@ REGEX_ESCAPE = /\\[^#]/g MULTILINER = /\n/g NO_NEWLINE = /^(?:[-+*&|\/%=<>!.\\][<>=&|]*|and|or|is(?:nt)?|not|delete|typeof|instanceof)$/ HEREDOC_INDENT = /\n+([ \t]*)|^([ \t]+)/g -ASSIGNED = /^\s*((?:[a-zA-Z$_@]\w*|["'][^\n]+?["']|\d+)[ \t]*?[:=][^:=])/ +ASSIGNED = /^\s*@?[$A-Za-z_][$\w]*[ \t]*?[:=][^:=>]/ NEXT_CHARACTER = /^\s*(\S)/ # Compound assignment tokens. From 19f08888e8a8cc7e74b36edf30208f61deb7873b Mon Sep 17 00:00:00 2001 From: satyr Date: Sun, 26 Sep 2010 03:25:53 +0900 Subject: [PATCH 05/10] lexer: more regexes fixes --- src/lexer.coffee | 39 ++++++++++++++++++--------------------- test/test_heredocs.coffee | 10 ++++++++++ 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/src/lexer.coffee b/src/lexer.coffee index ba827679..54b19559 100644 --- a/src/lexer.coffee +++ b/src/lexer.coffee @@ -139,7 +139,7 @@ exports.Lexer = class Lexer return false unless match = @chunk.match HEREDOC heredoc = match[0] quote = heredoc.charAt 0 - doc = @sanitizeHeredoc match[2], {quote} + doc = @sanitizeHeredoc match[2], {quote, indent: null} @interpolateString quote + doc + quote, heredoc: yes @line += count heredoc, '\n' @i += heredoc.length @@ -208,8 +208,8 @@ exports.Lexer = class Lexer @i += indent.length prev = @prev 2 size = indent.length - 1 - indent.lastIndexOf '\n' - nextCharacter = @match NEXT_CHARACTER, 1 - noNewlines = nextCharacter is '.' or nextCharacter is ',' or @unfinished() + nextCharacter = NEXT_CHARACTER.exec(@chunk)[1] + noNewlines = (nextCharacter in ['.', ',']) or @unfinished() if size - @indebt is @indent return @suppressNewlines() if noNewlines return @newlineToken indent @@ -331,18 +331,17 @@ exports.Lexer = class Lexer # Sanitize a heredoc or herecomment by escaping internal double quotes and # erasing all external indentation on the left-hand side. sanitizeHeredoc: (doc, options) -> - indent = options.indent - return doc if options.herecomment and not include doc, '\n' - unless options.herecomment + {indent, herecomment} = options + return doc if herecomment and not include doc, '\n' + unless herecomment while (match = HEREDOC_INDENT.exec doc) - attempt = if match[1]? then match[1] else match[2] - indent = attempt if not indent? or 0 < attempt.length < indent.length - indent or= '' - doc = doc.replace(new RegExp('^' + indent, 'gm'), '') - return doc if options.herecomment + attempt = match[1] + indent = attempt if indent is null or 0 < attempt.length < indent.length + doc = doc.replace /\n#{ indent }/g, '\n' if indent + return doc if herecomment doc.replace(/^\n/, '') .replace(MULTILINER, '\\n') - .replace(new RegExp(options.quote, 'g'), "\\#{options.quote}") + .replace(/#{ options.quote }/g, '\\$&') # A source of ambiguity in our grammar used to be parameter lists in function # definitions versus argument lists in function calls. Walk backwards, tagging @@ -489,11 +488,9 @@ exports.Lexer = class Lexer # Are we in the midst of an unfinished expression? unfinished: -> - prev = @prev 2 - value = @value() - value and NO_NEWLINE.test(value) and - prev and prev[0] isnt '.' and not CODE.test(value) and - not ASSIGNED.test(@chunk) + (prev = @prev 2 ) and prev[0] isnt '.' and + (value = @value()) and NO_NEWLINE.test(value) and not CODE.test(value) and + not ASSIGNED.test(@chunk) # Constants # --------- @@ -535,11 +532,11 @@ JS_FORBIDDEN = JS_KEYWORDS.concat RESERVED # Token matching regexes. IDENTIFIER = /^[a-zA-Z_$][\w$]*/ -NUMBER = /^(?:0x[\da-f]+)|^(?:\d+(\.\d+)?|\.\d+)(?:e[+-]?\d+)?/i +NUMBER = /^0x[\da-f]+|^(?:\d+(\.\d+)?|\.\d+)(?:e[+-]?\d+)?/i HEREDOC = /^("""|''')([\s\S]*?)\n?[ \t]*\1/ OPERATOR = /^(?:-[-=>]?|\+[+=]?|[*&|\/%=<>^:!?]+)(?=([ \t]*))/ WHITESPACE = /^[ \t]+/ -COMMENT = /^###([^#][\s\S]*?)(?:###[ \t]*\n|(?:###)?$)|^(?:\s*#(?!##[^#])[^\n]*)+/ +COMMENT = /^###([^#][\s\S]*?)(?:###[ \t]*\n|(?:###)?$)|^(?:\s*#(?!##[^#]).*)+/ CODE = /^[-=]>/ MULTI_DENT = /^(?:\n[ \t]*)+/ SIMPLESTR = /^'[^\\']*(?:\\.[^\\']*)*'/ @@ -554,9 +551,9 @@ REGEX_ESCAPE = /\\[^#]/g # Token cleaning regexes. MULTILINER = /\n/g NO_NEWLINE = /^(?:[-+*&|\/%=<>!.\\][<>=&|]*|and|or|is(?:nt)?|not|delete|typeof|instanceof)$/ -HEREDOC_INDENT = /\n+([ \t]*)|^([ \t]+)/g +HEREDOC_INDENT = /\n+([ \t]*)/g ASSIGNED = /^\s*@?[$A-Za-z_][$\w]*[ \t]*?[:=][^:=>]/ -NEXT_CHARACTER = /^\s*(\S)/ +NEXT_CHARACTER = /^\s*(\S?)/ # Compound assignment tokens. COMPOUND_ASSIGN = ['-=', '+=', '/=', '*=', '%=', '||=', '&&=', '?=', '<<=', '>>=', '>>>=', '&=', '^=', '|='] diff --git a/test/test_heredocs.coffee b/test/test_heredocs.coffee index 8bf2ae98..7389fb09 100644 --- a/test/test_heredocs.coffee +++ b/test/test_heredocs.coffee @@ -88,3 +88,13 @@ a = """ """ ok a is "one\ntwo\n" + + +equal ''' line 0 + should not be relevant + to the indent level +''', ' + line 0\n +should not be relevant\n + to the indent level +' From 7945178f3acdad60dc2357f39805bce8f8eb73e0 Mon Sep 17 00:00:00 2001 From: satyr Date: Sun, 26 Sep 2010 07:01:24 +0900 Subject: [PATCH 06/10] lexer: unrolled @extractNextToken/@match --- src/lexer.coffee | 55 ++++++++++++++++++++++-------------------------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/src/lexer.coffee b/src/lexer.coffee index 54b19559..605a021f 100644 --- a/src/lexer.coffee +++ b/src/lexer.coffee @@ -43,27 +43,24 @@ exports.Lexer = class Lexer @outdebt = 0 # The under-outdentation at the current level. @indents = [] # The stack of all current indentation levels. @tokens = [] # Stream of parsed tokens in the form ['TYPE', value, line] + # At every position, run through this list of attempted matches, + # short-circuiting if any of them succeed. Their order determines precedence: + # `@literalToken` is the fallback catch-all. while (@chunk = code[@i..]) - @extractNextToken() + @identifierToken() or + @commentToken() or + @whitespaceToken() or + @lineToken() or + @heredocToken() or + @stringToken() or + @numberToken() or + @regexToken() or + @jsToken() or + @literalToken() @closeIndentation() return @tokens if o.rewrite is off (new Rewriter).rewrite @tokens - # At every position, run through this list of attempted matches, - # short-circuiting if any of them succeed. Their order determines precedence: - # `@literalToken` is the fallback catch-all. - extractNextToken: -> - @identifierToken() or - @commentToken() or - @whitespaceToken() or - @lineToken() or - @heredocToken() or - @stringToken() or - @numberToken() or - @regexToken() or - @jsToken() or - @literalToken() - # Tokenizers # ---------- @@ -74,7 +71,8 @@ exports.Lexer = class Lexer # referenced as property names here, so you can still do `jQuery.is()` even # though `is` means `===` otherwise. identifierToken: -> - return false unless id = @match IDENTIFIER + return false unless match = IDENTIFIER.exec @chunk + id = match[0] @i += id.length if id is 'all' and @tag() is 'FOR' @token 'ALL', id @@ -111,7 +109,8 @@ exports.Lexer = class Lexer # Matches numbers, including decimals, hex, and exponential notation. # Be careful not to interfere with ranges-in-progress. numberToken: -> - return false unless number = @match NUMBER + return false unless match = NUMBER.exec @chunk + number = match[0] return false if @tag() is '.' and number.charAt(0) is '.' @i += number.length @token 'NUMBER', number @@ -122,8 +121,8 @@ exports.Lexer = class Lexer stringToken: -> switch @chunk.charAt 0 when "'" - return false unless string = @match SIMPLESTR - @token 'STRING', string.replace MULTILINER, '\\\n' + return false unless match = SIMPLESTR.exec @chunk + @token 'STRING', (string = match[0]).replace MULTILINER, '\\\n' when '"' return false unless string = @balancedToken ['"', '"'], ['#{', '}'] @interpolateString string.replace MULTILINER, '\\\n' @@ -159,8 +158,8 @@ exports.Lexer = class Lexer # Matches JavaScript interpolated directly into the source via backticks. jsToken: -> - return false unless @chunk.charAt(0) is '`' and script = @match JSTOKEN - @token 'JS', script.slice 1, -1 + return false unless @chunk.charAt(0) is '`' and match = JSTOKEN.exec @chunk + @token 'JS', (script = match[0]).slice 1, -1 @i += script.length true @@ -203,7 +202,8 @@ exports.Lexer = class Lexer # Keeps track of the level of indentation, because a single outdent token # can close multiple indents, so we need to know how far in we happen to be. lineToken: -> - return false unless indent = @match MULTI_DENT + return false unless match = MULTI_DENT.exec @chunk + indent = match[0] @line += count indent, '\n' @i += indent.length prev = @prev 2 @@ -253,10 +253,10 @@ exports.Lexer = class Lexer # Matches and consumes non-meaningful whitespace. Tag the previous token # as being "spaced", because there are some cases where it makes a difference. whitespaceToken: -> - return false unless space = @match WHITESPACE + return false unless match = WHITESPACE.exec @chunk prev = @prev() prev.spaced = true if prev - @i += space.length + @i += match[0].length true # Generate a newline token. Consecutive newlines get merged together. @@ -481,11 +481,6 @@ exports.Lexer = class Lexer prev: (index) -> @tokens[@tokens.length - (index or 1)] - # Attempt to match a string against the current chunk, returning the indexed - # match if successful, and `false` otherwise. - match: (regex, index) -> - if m = @chunk.match regex then m[index or 0] else false - # Are we in the midst of an unfinished expression? unfinished: -> (prev = @prev 2 ) and prev[0] isnt '.' and From 3e0c35bd0fd403ac7b09d5130bd777b58ffaf0fd Mon Sep 17 00:00:00 2001 From: satyr Date: Sun, 26 Sep 2010 07:06:14 +0900 Subject: [PATCH 07/10] lexer: enabled multiline interpolations --- lib/lexer.js | 238 ++++++++++++++++++++------------------ src/lexer.coffee | 97 +++++++++------- test/test_heredocs.coffee | 8 ++ 3 files changed, 184 insertions(+), 159 deletions(-) diff --git a/lib/lexer.js b/lib/lexer.js index 896aa9f7..c8892e96 100644 --- a/lib/lexer.js +++ b/lib/lexer.js @@ -23,7 +23,7 @@ this.indents = []; this.tokens = []; while ((this.chunk = code.slice(this.i))) { - this.extractNextToken(); + this.identifierToken() || this.commentToken() || this.whitespaceToken() || this.lineToken() || this.heredocToken() || this.stringToken() || this.numberToken() || this.regexToken() || this.jsToken() || this.literalToken(); } this.closeIndentation(); if (o.rewrite === false) { @@ -31,14 +31,12 @@ } return (new Rewriter()).rewrite(this.tokens); }; - Lexer.prototype.extractNextToken = function() { - return this.identifierToken() || this.commentToken() || this.whitespaceToken() || this.lineToken() || this.heredocToken() || this.stringToken() || this.numberToken() || this.regexToken() || this.jsToken() || this.literalToken(); - }; Lexer.prototype.identifierToken = function() { - var closeIndex, forcedIdentifier, id, tag; - if (!(id = this.match(IDENTIFIER))) { + var closeIndex, forcedIdentifier, id, match, tag; + if (!(match = IDENTIFIER.exec(this.chunk))) { return false; } + id = match[0]; this.i += id.length; if (id === 'all' && this.tag() === 'FOR') { this.token('ALL', id); @@ -86,10 +84,11 @@ return true; }; Lexer.prototype.numberToken = function() { - var number; - if (!(number = this.match(NUMBER))) { + var match, number; + if (!(match = NUMBER.exec(this.chunk))) { return false; } + number = match[0]; if (this.tag() === '.' && number.charAt(0) === '.') { return false; } @@ -98,19 +97,19 @@ return true; }; Lexer.prototype.stringToken = function() { - var string; + var match, string; switch (this.chunk.charAt(0)) { case "'": - if (!(string = this.match(SIMPLESTR))) { + if (!(match = SIMPLESTR.exec(this.chunk))) { return false; } - this.token('STRING', string.replace(MULTILINER, '\\\n')); + this.token('STRING', (string = match[0]).replace(MULTILINER, '\\\n')); break; case '"': if (!(string = this.balancedToken(['"', '"'], ['#{', '}']))) { return false; } - this.interpolateString(string.replace(MULTILINER, '\\\n')); + this.interpolateString(string); break; default: return false; @@ -127,7 +126,8 @@ heredoc = match[0]; quote = heredoc.charAt(0); doc = this.sanitizeHeredoc(match[2], { - quote: quote + quote: quote, + indent: null }); this.interpolateString(quote + doc + quote, { heredoc: true @@ -156,11 +156,11 @@ return true; }; Lexer.prototype.jsToken = function() { - var script; - if (!(this.chunk.charAt(0) === '`' && (script = this.match(JSTOKEN)))) { + var match, script; + if (!(this.chunk.charAt(0) === '`' && (match = JSTOKEN.exec(this.chunk)))) { return false; } - this.token('JS', script.slice(1, -1)); + this.token('JS', (script = match[0]).slice(1, -1)); this.i += script.length; return true; }; @@ -205,16 +205,17 @@ return this.balancedString(this.chunk, delimited); }; Lexer.prototype.lineToken = function() { - var diff, indent, nextCharacter, noNewlines, prev, size; - if (!(indent = this.match(MULTI_DENT))) { + var diff, indent, match, nextCharacter, noNewlines, prev, size; + if (!(match = MULTI_DENT.exec(this.chunk))) { return false; } + indent = match[0]; this.line += count(indent, '\n'); this.i += indent.length; prev = this.prev(2); size = indent.length - 1 - indent.lastIndexOf('\n'); - nextCharacter = this.match(NEXT_CHARACTER, 1); - noNewlines = nextCharacter === '.' || nextCharacter === ',' || this.unfinished(); + nextCharacter = NEXT_CHARACTER.exec(this.chunk)[1]; + noNewlines = (('.' === nextCharacter || ',' === nextCharacter)) || this.unfinished(); if (size - this.indebt === this.indent) { if (noNewlines) { return this.suppressNewlines(); @@ -265,15 +266,15 @@ return true; }; Lexer.prototype.whitespaceToken = function() { - var prev, space; - if (!(space = this.match(WHITESPACE))) { + var match, prev; + if (!(match = WHITESPACE.exec(this.chunk))) { return false; } prev = this.prev(); if (prev) { prev.spaced = true; } - this.i += space.length; + this.i += match[0].length; return true; }; Lexer.prototype.newlineToken = function(newlines) { @@ -369,25 +370,32 @@ return accessor ? 'accessor' : false; }; Lexer.prototype.sanitizeHeredoc = function(doc, options) { - var _ref2, attempt, indent, match; - indent = options.indent; - if (options.herecomment && !include(doc, '\n')) { + var _ref2, attempt, herecomment, indent, match; + _ref2 = options; + indent = _ref2.indent; + herecomment = _ref2.herecomment; + if (herecomment && !include(doc, '\n')) { return doc; } - if (!(options.herecomment)) { + if (!(herecomment)) { while ((match = HEREDOC_INDENT.exec(doc))) { - attempt = (typeof (_ref2 = match[1]) !== "undefined" && _ref2 !== null) ? match[1] : match[2]; - if (!(typeof indent !== "undefined" && indent !== null) || (0 < attempt.length) && (attempt.length < indent.length)) { + attempt = match[1]; + if (indent === null || (0 < attempt.length) && (attempt.length < indent.length)) { indent = attempt; } } } - indent || (indent = ''); - doc = doc.replace(new RegExp('^' + indent, 'gm'), ''); - if (options.herecomment) { + if (indent) { + doc = doc.replace(new RegExp("\\n" + (indent), "g"), '\n'); + } + if (herecomment) { return doc; } - return doc.replace(/^\n/, '').replace(MULTILINER, '\\n').replace(new RegExp(options.quote, 'g'), "\\" + (options.quote)); + doc = doc.replace(/^\n/, '').replace(new RegExp("" + (options.quote), "g"), '\\$&'); + if (options.quote === "'") { + doc = this.oldline(doc, true); + } + return doc; }; Lexer.prototype.tagParameters = function() { var i, tok; @@ -469,83 +477,84 @@ return !i ? false : str.slice(0, i); }; Lexer.prototype.interpolateString = function(str, options) { - var _len, _ref2, _ref3, end, escaped, expr, i, idx, inner, interpolated, lexer, nested, pi, quote, tag, tok, token, tokens, value; + var _len, _ref2, _ref3, end, escaped, expr, i, idx, inner, interpolated, lexer, nested, pi, push, quote, s, tag, tok, token, tokens, value; options || (options = {}); - if (str.length < 3 || str.charAt(0) !== '"') { + quote = str.charAt(0); + if (quote !== '"' || str.length < 3) { return this.token('STRING', str); - } else { - lexer = new Lexer(); - tokens = []; - quote = str.charAt(0); - _ref2 = [1, 1]; - i = _ref2[0]; - pi = _ref2[1]; - end = str.length - 1; - while (i < end) { - if (str.charAt(i) === '\\') { - i += 1; - } else if (expr = this.balancedString(str.slice(i), [['#{', '}']])) { - if (pi < i) { - tokens.push(['STRING', quote + str.slice(pi, i) + quote]); - } - inner = expr.slice(2, -1); - if (inner.length) { - if (options.heredoc) { - inner = inner.replace(new RegExp('\\\\' + quote, 'g'), quote); - } - nested = lexer.tokenize("(" + (inner) + ")", { - line: this.line - }); - _ref2 = nested; - for (idx = 0, _len = _ref2.length; idx < _len; idx++) { - tok = _ref2[idx]; - if (tok[0] === 'CALL_END') { - (tok[0] = ')'); - } - } - nested.pop(); - tokens.push(['TOKENS', nested]); - } else { - tokens.push(['STRING', quote + quote]); - } - i += expr.length - 1; - pi = i + 1; - } - i += 1; - } - if ((i > pi) && (pi < str.length - 1)) { - tokens.push(['STRING', quote + str.slice(pi, i) + quote]); - } - if (tokens[0][0] !== 'STRING') { - tokens.unshift(['STRING', '""']); - } - interpolated = tokens.length > 1; - if (interpolated) { - this.token('(', '('); - } - _ref2 = tokens; - for (i = 0, _len = _ref2.length; i < _len; i++) { - token = _ref2[i]; - _ref3 = token; - tag = _ref3[0]; - value = _ref3[1]; - if (tag === 'TOKENS') { - this.tokens = this.tokens.concat(value); - } else if (tag === 'STRING' && options.escapeQuotes) { - escaped = value.slice(1, -1).replace(/"/g, '\\"'); - this.token(tag, "\"" + (escaped) + "\""); - } else { - this.token(tag, value); - } - if (i < tokens.length - 1) { - this.token('+', '+'); - } - } - if (interpolated) { - this.token(')', ')'); - } - return tokens; } + lexer = new Lexer(); + tokens = []; + i = (pi = 1); + end = str.length - 1; + while (i < end) { + if (str.charAt(i) === '\\') { + i += 1; + } else if (expr = this.balancedString(str.slice(i), [['#{', '}']])) { + if (pi < i) { + s = quote + this.oldline(str.slice(pi, i), options.heredoc) + quote; + tokens.push(['STRING', s]); + } + inner = expr.slice(2, -1).replace(/^\s+/, ''); + if (inner.length) { + if (options.heredoc) { + inner = inner.replace(RegExp('\\\\' + quote, 'g'), quote); + } + nested = lexer.tokenize("(" + (inner) + ")", { + line: this.line + }); + _ref2 = nested; + for (idx = 0, _len = _ref2.length; idx < _len; idx++) { + tok = _ref2[idx]; + if (tok[0] === 'CALL_END') { + (tok[0] = ')'); + } + } + nested.pop(); + tokens.push(['TOKENS', nested]); + } else { + tokens.push(['STRING', quote + quote]); + } + i += expr.length - 1; + pi = i + 1; + } + i += 1; + } + if ((i > pi) && (pi < str.length - 1)) { + s = str.slice(pi, i).replace(MULTILINER, options.heredoc ? '\\n' : ''); + tokens.push(['STRING', quote + s + quote]); + } + if (tokens[0][0] !== 'STRING') { + tokens.unshift(['STRING', '""']); + } + interpolated = tokens.length > 1; + if (interpolated) { + this.token('(', '('); + } + _ref2 = tokens; + push = _ref2.push; + _ref2 = tokens; + for (i = 0, _len = _ref2.length; i < _len; i++) { + token = _ref2[i]; + _ref3 = token; + tag = _ref3[0]; + value = _ref3[1]; + if (tag === 'TOKENS') { + push.apply(this.tokens, value); + } else if (tag === 'STRING' && options.escapeQuotes) { + escaped = value.slice(1, -1).replace(/"/g, '\\"'); + this.token(tag, "\"" + (escaped) + "\""); + } else { + this.token(tag, value); + } + if (i < tokens.length - 1) { + this.token('+', '+'); + } + } + if (interpolated) { + this.token(')', ')'); + } + return tokens; }; Lexer.prototype.token = function(tag, value) { return this.tokens.push([tag, value, this.line]); @@ -579,9 +588,10 @@ }; Lexer.prototype.unfinished = function() { var prev, value; - prev = this.prev(2); - value = this.value(); - return value && NO_NEWLINE.test(value) && prev && prev[0] !== '.' && !CODE.test(value) && !ASSIGNED.test(this.chunk); + return (prev = this.prev(2)) && prev[0] !== '.' && (value = this.value()) && NO_NEWLINE.test(value) && !CODE.test(value) && !ASSIGNED.test(this.chunk); + }; + Lexer.prototype.oldline = function(str, heredoc) { + return str.replace(MULTILINER, heredoc ? '\\n' : ''); }; return Lexer; })(); @@ -591,11 +601,11 @@ RESERVED = ['case', 'default', 'do', 'function', 'var', 'void', 'with', 'const', 'let', 'enum', 'export', 'import', 'native', '__hasProp', '__extends', '__slice']; JS_FORBIDDEN = JS_KEYWORDS.concat(RESERVED); IDENTIFIER = /^[a-zA-Z_$][\w$]*/; - NUMBER = /^(?:0x[\da-f]+)|^(?:\d+(\.\d+)?|\.\d+)(?:e[+-]?\d+)?/i; + NUMBER = /^0x[\da-f]+|^(?:\d+(\.\d+)?|\.\d+)(?:e[+-]?\d+)?/i; HEREDOC = /^("""|''')([\s\S]*?)\n?[ \t]*\1/; OPERATOR = /^(?:-[-=>]?|\+[+=]?|[*&|\/%=<>^:!?]+)(?=([ \t]*))/; WHITESPACE = /^[ \t]+/; - COMMENT = /^###([^#][\s\S]*?)(?:###[ \t]*\n|(?:###)?$)|^(?:\s*#(?!##[^#])[^\n]*)+/; + COMMENT = /^###([^#][\s\S]*?)(?:###[ \t]*\n|(?:###)?$)|^(?:\s*#(?!##[^#]).*)+/; CODE = /^[-=]>/; MULTI_DENT = /^(?:\n[ \t]*)+/; SIMPLESTR = /^'[^\\']*(?:\\.[^\\']*)*'/; @@ -606,9 +616,9 @@ REGEX_ESCAPE = /\\[^#]/g; MULTILINER = /\n/g; NO_NEWLINE = /^(?:[-+*&|\/%=<>!.\\][<>=&|]*|and|or|is(?:nt)?|not|delete|typeof|instanceof)$/; - HEREDOC_INDENT = /\n+([ \t]*)|^([ \t]+)/g; + HEREDOC_INDENT = /\n+([ \t]*)/g; ASSIGNED = /^\s*@?[$A-Za-z_][$\w]*[ \t]*?[:=][^:=>]/; - NEXT_CHARACTER = /^\s*(\S)/; + NEXT_CHARACTER = /^\s*(\S?)/; COMPOUND_ASSIGN = ['-=', '+=', '/=', '*=', '%=', '||=', '&&=', '?=', '<<=', '>>=', '>>>=', '&=', '^=', '|=']; UNARY = ['UMINUS', 'UPLUS', '!', '!!', '~', 'TYPEOF', 'DELETE']; LOGIC = ['&', '|', '^', '&&', '||']; diff --git a/src/lexer.coffee b/src/lexer.coffee index 605a021f..a5d91c9f 100644 --- a/src/lexer.coffee +++ b/src/lexer.coffee @@ -125,7 +125,7 @@ exports.Lexer = class Lexer @token 'STRING', (string = match[0]).replace MULTILINER, '\\\n' when '"' return false unless string = @balancedToken ['"', '"'], ['#{', '}'] - @interpolateString string.replace MULTILINER, '\\\n' + @interpolateString string else return false @line += count string, '\n' @@ -339,9 +339,9 @@ exports.Lexer = class Lexer indent = attempt if indent is null or 0 < attempt.length < indent.length doc = doc.replace /\n#{ indent }/g, '\n' if indent return doc if herecomment - doc.replace(/^\n/, '') - .replace(MULTILINER, '\\n') - .replace(/#{ options.quote }/g, '\\$&') + doc = doc.replace(/^\n/, '').replace(/#{ options.quote }/g, '\\$&') + doc = @oldline doc, on if options.quote is "'" + doc # A source of ambiguity in our grammar used to be parameter lists in function # definitions versus argument lists in function calls. Walk backwards, tagging @@ -406,7 +406,7 @@ exports.Lexer = class Lexer if not i then false else str[0...i] # Expand variables and expressions inside double-quoted strings using - # [ECMA Harmony's interpolation syntax](http://wiki.ecmascript.org/doku.php?id=strawman:string_interpolation) + # Ruby-like notation # for substitution of bare variables as well as arbitrary expressions. # # "Hello #{name.capitalize()}." @@ -415,48 +415,51 @@ exports.Lexer = class Lexer # new Lexer, tokenize the interpolated contents, and merge them into the # token stream. interpolateString: (str, options) -> - options or= {} - if str.length < 3 or str.charAt(0) isnt '"' - @token 'STRING', str - else - lexer = new Lexer - tokens = [] - quote = str.charAt 0 - [i, pi] = [1, 1] - end = str.length - 1 - while i < end - if str.charAt(i) is '\\' - i += 1 - else if expr = @balancedString str[i..], [['#{', '}']] - tokens.push ['STRING', quote + str[pi...i] + quote] if pi < i - inner = expr.slice 2, -1 - if inner.length - inner = inner.replace new RegExp('\\\\' + quote, 'g'), quote if options.heredoc - nested = lexer.tokenize "(#{inner})", line: @line - (tok[0] = ')') for tok, idx in nested when tok[0] is 'CALL_END' - nested.pop() - tokens.push ['TOKENS', nested] - else - tokens.push ['STRING', quote + quote] - i += expr.length - 1 - pi = i + 1 + {heredoc, escapeQuotes} = options or {} + quote = str.charAt 0 + return @token 'STRING', str if quote isnt '"' or str.length < 3 + lexer = new Lexer + tokens = [] + i = pi = 1 + end = str.length - 1 + while i < end + if str.charAt(i) is '\\' i += 1 - tokens.push ['STRING', quote + str[pi...i] + quote] if i > pi < str.length - 1 - tokens.unshift ['STRING', '""'] unless tokens[0][0] is 'STRING' - interpolated = tokens.length > 1 - @token '(', '(' if interpolated - for token, i in tokens - [tag, value] = token - if tag is 'TOKENS' - @tokens = @tokens.concat value - else if tag is 'STRING' and options.escapeQuotes - escaped = value.slice(1, -1).replace(/"/g, '\\"') - @token tag, "\"#{escaped}\"" + else if expr = @balancedString str[i..], [['#{', '}']] + if pi < i + s = quote + @oldline(str[pi...i], heredoc) + quote + tokens.push ['STRING', s] + inner = expr.slice(2, -1).replace /^[ \t]*\n/, '' + if inner.length + inner = inner.replace RegExp('\\\\' + quote, 'g'), quote if heredoc + nested = lexer.tokenize "(#{inner})", line: @line + (tok[0] = ')') for tok, idx in nested when tok[0] is 'CALL_END' + nested.pop() + tokens.push ['TOKENS', nested] else - @token tag, value - @token '+', '+' if i < tokens.length - 1 - @token ')', ')' if interpolated - tokens + tokens.push ['STRING', quote + quote] + i += expr.length - 1 + pi = i + 1 + i += 1 + if i > pi < str.length - 1 + s = str[pi...i].replace MULTILINER, if heredoc then '\\n' else '' + tokens.push ['STRING', quote + s + quote] + tokens.unshift ['STRING', '""'] unless tokens[0][0] is 'STRING' + interpolated = tokens.length > 1 + @token '(', '(' if interpolated + {push} = tokens + for token, i in tokens + [tag, value] = token + if tag is 'TOKENS' + push.apply @tokens, value + else if tag is 'STRING' and escapeQuotes + escaped = value.slice(1, -1).replace(/"/g, '\\"') + @token tag, "\"#{escaped}\"" + else + @token tag, value + @token '+', '+' if i < tokens.length - 1 + @token ')', ')' if interpolated + tokens # Helpers # ------- @@ -487,6 +490,10 @@ exports.Lexer = class Lexer (value = @value()) and NO_NEWLINE.test(value) and not CODE.test(value) and not ASSIGNED.test(@chunk) + # Converts newlines for string literals + oldline: (str, heredoc) -> + str.replace MULTILINER, if heredoc then '\\n' else '' + # Constants # --------- diff --git a/test/test_heredocs.coffee b/test/test_heredocs.coffee index 7389fb09..5c727483 100644 --- a/test/test_heredocs.coffee +++ b/test/test_heredocs.coffee @@ -98,3 +98,11 @@ equal ''' line 0 should not be relevant\n to the indent level ' + + +equal 'multiline nested interpolations work', """multiline #{ + "nested #{(-> + ok yes + "interpolations" + )()}" +} work""" From 45bd0854b698616e79b4766012e1d0551cc88547 Mon Sep 17 00:00:00 2001 From: Jeremy Ashkenas Date: Sun, 26 Sep 2010 10:28:48 -0400 Subject: [PATCH 08/10] Merging in satyr's helpers-refactor --- lib/helpers.js | 10 +++++----- src/helpers.coffee | 23 ++++++++++++++--------- test/test_helpers.coffee | 10 ++++++++-- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/lib/helpers.js b/lib/helpers.js index 49829b2f..797d9988 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -14,15 +14,15 @@ return -1; })); exports.include = function(list, value) { - return 0 <= indexOf(list, value); + return indexOf(list, value) >= 0; }; exports.starts = function(string, literal, start) { return literal === string.substr(start, literal.length); }; exports.ends = function(string, literal, back) { - var ll; - ll = literal.length; - return literal === string.substr(string.length - ll - (back || 0), ll); + var len; + len = literal.length; + return literal === string.substr(string.length - len - (back || 0), len); }; exports.compact = function(array) { var _i, _len, _ref, _result, item; @@ -51,7 +51,7 @@ _ref = properties; for (key in _ref) { val = _ref[key]; - (object[key] = val); + object[key] = val; } return object; }); diff --git a/src/helpers.coffee b/src/helpers.coffee index cd506d13..90786df8 100644 --- a/src/helpers.coffee +++ b/src/helpers.coffee @@ -2,7 +2,8 @@ # the **Lexer**, **Rewriter**, and the **Nodes**. Merge objects, flatten # arrays, count characters, that sort of thing. -# Cross-engine indexOf, so that JScript can join the party. +# Cross-engine `indexOf`, so that JScript can join the party. Use SpiderMonkey's +# functional-style `indexOf`, if it's available. indexOf = exports.indexOf = Array.indexOf or if Array::indexOf (array, item, from) -> array.indexOf item, from @@ -14,7 +15,8 @@ indexOf = exports.indexOf = Array.indexOf or -1 # Does a list include a value? -exports.include = (list, value) -> 0 <= indexOf list, value +exports.include = (list, value) -> + indexOf(list, value) >= 0 # Peek at the beginning of a given string to see if it matches a sequence. exports.starts = (string, literal, start) -> @@ -22,11 +24,12 @@ exports.starts = (string, literal, start) -> # Peek at the end of a given string to see if it matches a sequence. exports.ends = (string, literal, back) -> - ll = literal.length - literal is string.substr string.length - ll - (back or 0), ll + len = literal.length + literal is string.substr string.length - len - (back or 0), len # Trim out all falsy values from an array. -exports.compact = (array) -> item for item in array when item +exports.compact = (array) -> + item for item in array when item # Count the number of occurences of a character in a string. exports.count = (string, letter) -> @@ -41,14 +44,16 @@ exports.merge = (options, overrides) -> extend (extend {}, options), overrides # Extend a source object with the properties of another object (shallow copy). -# We use this to simulate Node's deprecated `process.mixin` +# We use this to simulate Node's deprecated `process.mixin`. extend = exports.extend = (object, properties) -> - (object[key] = val) for all key, val of properties + for all key, val of properties + object[key] = val object -# Return a flattened version of an array (nonrecursive). +# Return a flattened version of an array (shallow and nonrecursive). # Handy for getting a list of `children` from the nodes. -exports.flatten = (array) -> array.concat.apply [], array +exports.flatten = (array) -> + array.concat.apply [], array # Delete a key from an object, returning the value. Useful when a node is # looking for a particular method in an options hash. diff --git a/test/test_helpers.coffee b/test/test_helpers.coffee index 3f204c33..036fcb6c 100644 --- a/test/test_helpers.coffee +++ b/test/test_helpers.coffee @@ -1,6 +1,6 @@ -{ indexOf, include, starts, ends, compact, count, merge, extend, flatten, del -} = require '../lib/helpers' +{indexOf, include, starts, ends, compact, count, merge, extend, flatten, del} = require '../lib/helpers' +# Test `indexOf` array = [0..4] ok indexOf(array, 0) is 0 @@ -8,11 +8,13 @@ ok indexOf(array, 2) is 2 ok indexOf(array, 4) is 4 ok indexOf(array, 6) is -1 +# Test `include` ok include array, 0 ok include array, 2 ok include array, 4 ok not include array, 6 +# Test `starts` string = array.join '' ok starts string, '012' @@ -20,20 +22,24 @@ ok starts string, '34', 3 ok not starts string, '42' ok not starts string, '42', 6 +# Test `ends` ok ends string, '234' ok ends string, '01', 3 ok not ends string, '42' ok not ends string, '42', 6 +# Test `merge` object = {} merged = merge object, array ok merged isnt object ok merged[3] is 3 +# Test `extend` ok object is extend object, array ok object[3] is 3 +# Test `flatten` ok "#{ flatten [0, [1, 2], 3, [4]] }" is "#{ array }" ok 1 is del object, 1 From 008d2ba0d35efcbd369e2dd35833bb41bb8bf585 Mon Sep 17 00:00:00 2001 From: Jeremy Ashkenas Date: Sun, 26 Sep 2010 10:38:28 -0400 Subject: [PATCH 09/10] adding a cake:bench task --- Cakefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Cakefile b/Cakefile index 5e5a7073..da53da08 100644 --- a/Cakefile +++ b/Cakefile @@ -87,6 +87,9 @@ task 'doc:underscore', 'rebuild the Underscore.coffee documentation page', -> exec 'docco examples/underscore.coffee && cp -rf docs documentation && rm -r docs', (err) -> throw err if err +task 'bench', 'quick benchmark of compilation time (of everything in src)', -> + exec 'time bin/coffee -p src/ > /dev/null', (err, stdout, stderr) -> + print stderr task 'loc', 'count the lines of source code in the CoffeeScript compiler', -> sources = ['src/coffee-script.coffee', 'src/grammar.coffee', 'src/helpers.coffee', 'src/lexer.coffee', 'src/nodes.coffee', 'src/rewriter.coffee', 'src/scope.coffee'] From ecb23d15c478aeeb0bb358ff43c794431fe0bc84 Mon Sep 17 00:00:00 2001 From: Jeremy Ashkenas Date: Sun, 26 Sep 2010 10:57:03 -0400 Subject: [PATCH 10/10] Merging in satyr's rewrite-lexer.coffee --- lib/lexer.js | 24 +++++++++++------------- src/lexer.coffee | 11 +++++------ 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/lib/lexer.js b/lib/lexer.js index 87fd371c..cb45c2cf 100644 --- a/lib/lexer.js +++ b/lib/lexer.js @@ -393,7 +393,7 @@ } doc = doc.replace(/^\n/, '').replace(new RegExp("" + (options.quote), "g"), '\\$&'); if (options.quote === "'") { - doc = this.oldline(doc, true); + doc = this.escapeLines(doc, true); } return doc; }; @@ -477,8 +477,10 @@ return !i ? false : str.slice(0, i); }; Lexer.prototype.interpolateString = function(str, options) { - var _len, _ref2, _ref3, end, escaped, expr, i, idx, inner, interpolated, lexer, nested, pi, push, quote, s, tag, tok, token, tokens, value; - options || (options = {}); + var _len, _ref2, _ref3, end, escapeQuotes, escaped, expr, heredoc, i, idx, inner, interpolated, lexer, nested, pi, push, quote, s, tag, tok, token, tokens, value; + _ref2 = options || {}; + heredoc = _ref2.heredoc; + escapeQuotes = _ref2.escapeQuotes; quote = str.charAt(0); if (quote !== '"' || str.length < 3) { return this.token('STRING', str); @@ -492,12 +494,12 @@ i += 1; } else if (expr = this.balancedString(str.slice(i), [['#{', '}']])) { if (pi < i) { - s = quote + this.oldline(str.slice(pi, i), options.heredoc) + quote; + s = quote + this.escapeLines(str.slice(pi, i), heredoc) + quote; tokens.push(['STRING', s]); } - inner = expr.slice(2, -1).replace(/^\s+/, ''); + inner = expr.slice(2, -1).replace(/^[ \t]*\n/, ''); if (inner.length) { - if (options.heredoc) { + if (heredoc) { inner = inner.replace(RegExp('\\\\' + quote, 'g'), quote); } nested = lexer.tokenize("(" + (inner) + ")", { @@ -521,7 +523,7 @@ i += 1; } if ((i > pi) && (pi < str.length - 1)) { - s = str.slice(pi, i).replace(MULTILINER, options.heredoc ? '\\n' : ''); + s = str.slice(pi, i).replace(MULTILINER, heredoc ? '\\n' : ''); tokens.push(['STRING', quote + s + quote]); } if (tokens[0][0] !== 'STRING') { @@ -541,7 +543,7 @@ value = _ref3[1]; if (tag === 'TOKENS') { push.apply(this.tokens, value); - } else if (tag === 'STRING' && options.escapeQuotes) { + } else if (tag === 'STRING' && escapeQuotes) { escaped = value.slice(1, -1).replace(/"/g, '\\"'); this.token(tag, "\"" + (escaped) + "\""); } else { @@ -582,15 +584,11 @@ Lexer.prototype.prev = function(index) { return this.tokens[this.tokens.length - (index || 1)]; }; - Lexer.prototype.match = function(regex, index) { - var m; - return (m = this.chunk.match(regex)) ? m[index || 0] : false; - }; Lexer.prototype.unfinished = function() { var prev, value; return (prev = this.prev(2)) && prev[0] !== '.' && (value = this.value()) && NO_NEWLINE.test(value) && !CODE.test(value) && !ASSIGNED.test(this.chunk); }; - Lexer.prototype.oldline = function(str, heredoc) { + Lexer.prototype.escapeLines = function(str, heredoc) { return str.replace(MULTILINER, heredoc ? '\\n' : ''); }; return Lexer; diff --git a/src/lexer.coffee b/src/lexer.coffee index 237b4999..20952752 100644 --- a/src/lexer.coffee +++ b/src/lexer.coffee @@ -340,7 +340,7 @@ exports.Lexer = class Lexer doc = doc.replace /\n#{ indent }/g, '\n' if indent return doc if herecomment doc = doc.replace(/^\n/, '').replace(/#{ options.quote }/g, '\\$&') - doc = @oldline doc, on if options.quote is "'" + doc = @escapeLines doc, yes if options.quote is "'" doc # A source of ambiguity in our grammar used to be parameter lists in function @@ -406,8 +406,7 @@ exports.Lexer = class Lexer if not i then false else str[0...i] # Expand variables and expressions inside double-quoted strings using - # Ruby-like notation - # for substitution of bare variables as well as arbitrary expressions. + # Ruby-like notation for substitution of arbitrary expressions. # # "Hello #{name.capitalize()}." # @@ -427,7 +426,7 @@ exports.Lexer = class Lexer i += 1 else if expr = @balancedString str[i..], [['#{', '}']] if pi < i - s = quote + @oldline(str[pi...i], heredoc) + quote + s = quote + @escapeLines(str[pi...i], heredoc) + quote tokens.push ['STRING', s] inner = expr.slice(2, -1).replace /^[ \t]*\n/, '' if inner.length @@ -490,8 +489,8 @@ exports.Lexer = class Lexer (value = @value()) and NO_NEWLINE.test(value) and not CODE.test(value) and not ASSIGNED.test(@chunk) - # Converts newlines for string literals - oldline: (str, heredoc) -> + # Converts newlines for string literals. + escapeLines: (str, heredoc) -> str.replace MULTILINER, if heredoc then '\\n' else '' # Constants