github/coffeescript
1
1
Fork 0
mirror of https://github.com/jashkenas/coffeescript.git synced 2026-02-17 11:01:25 -05:00
Code Issues Packages Projects Releases Wiki Activity

first little piece of the rewriter

Browse Source
This commit is contained in:
Jeremy Ashkenas
2010-01-30 17:02:05 -05:00
parent e755188878
commit 84feab3492
6 changed files with 263 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
lib/coffee_script/lexer.js
View File

@@ -1,6 +1,7 @@
(function(){
var ASSIGNMENT, CALLABLE, CODE, COMMENT, COMMENT_CLEANER, HEREDOC, HEREDOC_INDENT, IDENTIFIER, JS, JS_CLEANER, KEYWORDS, LAST_DENT, LAST_DENTS, MULTILINER, MULTI_DENT, NOT_REGEX, NO_NEWLINE, NUMBER, OPERATOR, REGEX, STRING, STRING_NEWLINES, WHITESPACE, lex, sys;
var ASSIGNMENT, CALLABLE, CODE, COMMENT, COMMENT_CLEANER, HEREDOC, HEREDOC_INDENT, IDENTIFIER, JS, JS_CLEANER, KEYWORDS, LAST_DENT, LAST_DENTS, MULTILINER, MULTI_DENT, NOT_REGEX, NO_NEWLINE, NUMBER, OPERATOR, REGEX, Rewriter, STRING, STRING_NEWLINES, WHITESPACE, lex, sys;
sys = require('sys');
Rewriter = require('./rewriter').Rewriter;
// The lexer reads a stream of CoffeeScript and divvys it up into tagged
// tokens. A minor bit of the ambiguity in the grammar has been avoided by
// pushing some extra smarts into the Lexer.
@@ -56,8 +57,7 @@
}
// sys.puts "original stream: " + this.tokens if process.ENV['VERBOSE']
this.close_indentation();
// (new Rewriter()).rewrite(this.tokens)
return this.tokens;
return (new Rewriter()).rewrite(this.tokens);
};
// At every position, run through this list of attempted matches,
// short-circuiting if any of them succeed.
@@ -116,7 +116,8 @@
}
}
this.token(tag, id);
return this.i += id.length;
this.i += id.length;
return true;
};
// Matches numbers, including decimals, hex, and exponential notation.
lex.prototype.number_token = function number_token() {
@@ -125,7 +126,8 @@
return false;
}
this.token('NUMBER', number);
return this.i += number.length;
this.i += number.length;
return true;
};
// Matches strings, including multi-line strings.
lex.prototype.string_token = function string_token() {
@@ -136,7 +138,8 @@
escaped = string.replace(STRING_NEWLINES, " \\\n");
this.token('STRING', escaped);
this.line += this.count(string, "\n");
return this.i += string.length;
this.i += string.length;
return true;
};
// Matches heredocs, adjusting indentation to the correct level.
lex.prototype.heredoc_token = function heredoc_token() {
@@ -149,7 +152,8 @@
doc = doc.replace(new RegExp("^" + indent, 'g'), '').replace(MULTILINER, "\\n").replace('"', '\\"');
this.token('STRING', '"' + doc + '"');
this.line += this.count(match[1], "\n");
return this.i += match[1].length;
this.i += match[1].length;
return true;
};
// Matches interpolated JavaScript.
lex.prototype.js_token = function js_token() {
@@ -158,7 +162,8 @@
return false;
}
this.token('JS', script.replace(JS_CLEANER, ''));
return this.i += script.length;
this.i += script.length;
return true;
};
// Matches regular expression literals.
lex.prototype.regex_token = function regex_token() {
@@ -170,7 +175,8 @@
return false;
}
this.token('REGEX', regex);
return this.i += regex.length;
this.i += regex.length;
return true;
};
// Matches and conumes comments.
lex.prototype.comment_token = function comment_token() {
@@ -181,7 +187,8 @@
this.line += comment.match(MULTILINER).length;
this.token('COMMENT', comment.replace(COMMENT_CLEANER, '').split(MULTILINER));
this.token("\n", "\n");
return this.i += comment.length;
this.i += comment.length;
return true;
};
// Record tokens for indentation differing from the previous line.
lex.prototype.indent_token = function indent_token() {
@@ -207,7 +214,8 @@
} else {
this.outdent_token(this.indent - size);
}
return this.indent = size;
this.indent = size;
return true;
};
// Record an oudent token or tokens, if we're moving back inwards past
// multiple recorded indents.
@@ -218,7 +226,8 @@
this.token('OUTDENT', last_indent);
move_out -= last_indent;
}
return this.token("\n", "\n");
this.token("\n", "\n");
return true;
};
// Matches and consumes non-meaningful whitespace.
lex.prototype.whitespace_token = function whitespace_token() {
@@ -227,7 +236,8 @@
return false;
}
this.value().spaced = true;
return this.i += space.length;
this.i += space.length;
return true;
};
// Multiple newlines get merged together.
// Use a trailing \ to escape newlines.
@@ -265,7 +275,8 @@
}
}
this.token(tag, value);
return this.i += value.length;
this.i += value.length;
return true;
};
// Helpers =============================================================
// Add a token to the results, taking note of the line number.